Merge pull request #1679 from evandbrown/gce-service-accounts

Use golang/oauth2, no longer require client_secrets.json, and use

builder/googlecompute/account.go

@@ -13,16 +13,6 @@ type accountFile struct {
 	ClientId     string `json:"client_id"`
 }
 
-// clientSecretsFile represents the structure of the client secrets JSON file.
-type clientSecretsFile struct {
-	Web struct {
-		AuthURI     string `json:"auth_uri"`
-		ClientEmail string `json:"client_email"`
-		ClientId    string `json:"client_id"`
-		TokenURI    string `json:"token_uri"`
-	}
-}
-
 func loadJSON(result interface{}, path string) error {
 	f, err := os.Open(path)
 	if err != nil {

builder/googlecompute/builder.go

@@ -35,7 +35,7 @@ func (b *Builder) Prepare(raws ...interface{}) ([]string, error) {
 // representing a GCE machine image.
 func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packer.Artifact, error) {
 	driver, err := NewDriverGCE(
-		ui, b.config.ProjectId, &b.config.account, &b.config.clientSecrets)
+		ui, b.config.ProjectId, &b.config.account)
 	if err != nil {
 		return nil, err
 	}

builder/googlecompute/config.go

@@ -17,7 +17,6 @@ type Config struct {
 	common.PackerConfig `mapstructure:",squash"`
 
 	AccountFile       string `mapstructure:"account_file"`
-	ClientSecretsFile string `mapstructure:"client_secrets_file"`
 	ProjectId         string `mapstructure:"project_id"`
 
 	BucketName           string            `mapstructure:"bucket_name"`
@@ -38,7 +37,6 @@ type Config struct {
 	Zone                 string            `mapstructure:"zone"`
 
 	account         accountFile
-	clientSecrets   clientSecretsFile
 	instanceName    string
 	privateKeyBytes []byte
 	sshTimeout      time.Duration
@@ -106,7 +104,6 @@ func NewConfig(raws ...interface{}) (*Config, []string, error) {
 	// Process Templates
 	templates := map[string]*string{
 		"account_file":        &c.AccountFile,
-		"client_secrets_file": &c.ClientSecretsFile,
 
 		"bucket_name":             &c.BucketName,
 		"image_name":              &c.ImageName,
@@ -138,16 +135,6 @@ func NewConfig(raws ...interface{}) (*Config, []string, error) {
 			errs, errors.New("a bucket_name must be specified"))
 	}
 
-	if c.AccountFile == "" {
-		errs = packer.MultiErrorAppend(
-			errs, errors.New("an account_file must be specified"))
-	}
-
-	if c.ClientSecretsFile == "" {
-		errs = packer.MultiErrorAppend(
-			errs, errors.New("a client_secrets_file must be specified"))
-	}
-
 	if c.ProjectId == "" {
 		errs = packer.MultiErrorAppend(
 			errs, errors.New("a project_id must be specified"))
@@ -185,13 +172,6 @@ func NewConfig(raws ...interface{}) (*Config, []string, error) {
 		}
 	}
 
-	if c.ClientSecretsFile != "" {
-		if err := loadJSON(&c.clientSecrets, c.ClientSecretsFile); err != nil {
-			errs = packer.MultiErrorAppend(
-				errs, fmt.Errorf("Failed parsing client secrets file: %s", err))
-		}
-	}
-
 	// Check for any errors.
 	if errs != nil && len(errs.Errors) > 0 {
 		return nil, nil, errs

builder/googlecompute/config_test.go

@@ -9,7 +9,6 @@ func testConfig(t *testing.T) map[string]interface{} {
 	return map[string]interface{}{
 		"account_file":        testAccountFile(t),
 		"bucket_name":         "foo",
-		"client_secrets_file": testClientSecretsFile(t),
 		"project_id":          "hashicorp",
 		"source_image":        "foo",
 		"zone":                "us-east-1a",
@@ -69,22 +68,6 @@ func TestConfigPrepare(t *testing.T) {
 			false,
 		},
 
-		{
-			"client_secrets_file",
-			nil,
-			true,
-		},
-		{
-			"client_secrets_file",
-			testClientSecretsFile(t),
-			false,
-		},
-		{
-			"client_secrets_file",
-			"/tmp/i/should/not/exist",
-			true,
-		},
-
 		{
 			"private_key_file",
 			"/tmp/i/should/not/exist",
@@ -180,22 +163,6 @@ func testAccountFile(t *testing.T) string {
 	return tf.Name()
 }
 
-func testClientSecretsFile(t *testing.T) string {
-	tf, err := ioutil.TempFile("", "packer")
-	if err != nil {
-		t.Fatalf("err: %s", err)
-	}
-	defer tf.Close()
-
-	if _, err := tf.Write([]byte(testClientSecretsContent)); err != nil {
-		t.Fatalf("err: %s", err)
-	}
-
-	return tf.Name()
-}
-
 // This is just some dummy data that doesn't actually work (it was revoked
 // a long time ago).
 const testAccountContent = `{}`
-
-const testClientSecretsContent = `{"web":{"auth_uri":"https://accounts.google.com/o/oauth2/auth","token_uri":"https://accounts.google.com/o/oauth2/token","client_email":"774313886706-eorlsj0r4eqkh5e7nvea5fuf59ifr873@developer.gserviceaccount.com","client_x509_cert_url":"https://www.googleapis.com/robot/v1/metadata/x509/774313886706-eorlsj0r4eqkh5e7nvea5fuf59ifr873@developer.gserviceaccount.com","client_id":"774313886706-eorlsj0r4eqkh5e7nvea5fuf59ifr873.apps.googleusercontent.com","auth_provider_x509_cert_url":"https://www.googleapis.com/oauth2/v1/certs"}}`

builder/googlecompute/driver_gce.go

@@ -6,9 +6,9 @@ import (
 	"net/http"
 	"time"
 
-	"code.google.com/p/goauth2/oauth"
-	"code.google.com/p/goauth2/oauth/jwt"
 	"code.google.com/p/google-api-go-client/compute/v1"
+	"github.com/golang/oauth2"
+	"github.com/golang/oauth2/google"
 	"github.com/mitchellh/packer/packer"
 )
 
@@ -20,39 +20,35 @@ type driverGCE struct {
 	ui        packer.Ui
 }
 
-const DriverScopes string = "https://www.googleapis.com/auth/compute " +
-	"https://www.googleapis.com/auth/devstorage.full_control"
+var DriverScopes = []string{"https://www.googleapis.com/auth/compute", "https://www.googleapis.com/auth/devstorage.full_control"}
 
-func NewDriverGCE(ui packer.Ui, p string, a *accountFile, c *clientSecretsFile) (Driver, error) {
-	// Get the token for use in our requests
-	log.Printf("[INFO] Requesting Google token...")
+func NewDriverGCE(ui packer.Ui, p string, a *accountFile) (Driver, error) {
+	var f *oauth2.Flow
+	var err error
+
+	// Auth with AccountFile first if provided
+	if a.PrivateKey != "" {
+		log.Printf("[INFO] Requesting Google token via AccountFile...")
 		log.Printf("[INFO]   -- Email: %s", a.ClientEmail)
 		log.Printf("[INFO]   -- Scopes: %s", DriverScopes)
 		log.Printf("[INFO]   -- Private Key Length: %d", len(a.PrivateKey))
-	log.Printf("[INFO]   -- Token URL: %s", c.Web.TokenURI)
-	jwtTok := jwt.NewToken(
-		a.ClientEmail,
-		DriverScopes,
-		[]byte(a.PrivateKey))
-	jwtTok.ClaimSet.Aud = c.Web.TokenURI
-	token, err := jwtTok.Assert(new(http.Client))
-	if err != nil {
-		return nil, fmt.Errorf("Error retrieving auth token: %s", err)
+
+		f, err = oauth2.New(
+			oauth2.JWTClient(a.ClientEmail, []byte(a.PrivateKey)),
+			oauth2.Scope(DriverScopes...),
+			google.JWTEndpoint())
+	} else {
+		log.Printf("[INFO] Requesting Google token via GCE Service Role...")
+
+		f, err = oauth2.New(google.ComputeEngineAccount(""))
 	}
 
-	// Instantiate the transport to communicate to Google
-	transport := &oauth.Transport{
-		Config: &oauth.Config{
-			ClientId: a.ClientId,
-			Scope:    DriverScopes,
-			TokenURL: c.Web.TokenURI,
-			AuthURL:  c.Web.AuthURI,
-		},
-		Token: token,
+	if err != nil {
+		return nil, err
 	}
 
-	log.Printf("[INFO] Instantiating GCE client...")
-	service, err := compute.New(transport.Client())
+	log.Printf("[INFO] Instantiating GCE client using...")
+	service, err := compute.New(&http.Client{Transport: f.NewTransport()})
 	if err != nil {
 		return nil, err
 	}

test/README.md

@@ -41,8 +41,7 @@ Set the following self-explanatory environmental variables:
 Set the following environmental variables:
 
 * `GC_BUCKET_NAME`
-* `GC_CLIENT_SECRETS_FILE`
-* `GC_PRIVATE_KEY_FILE`
+* `GC_ACCOUNT_FILE`
 * `GC_PROJECT_ID`
 
 ### Running

test/builder_googlecompute.bats

@@ -8,8 +8,7 @@ fixtures builder-googlecompute
 
 # Required parameters
 : ${GC_BUCKET_NAME:?}
-: ${GC_CLIENT_SECRETS_FILE:?}
-: ${GC_PRIVATE_KEY_FILE:?}
+: ${GC_ACCOUNT_FILE:?}
 : ${GC_PROJECT_ID:?}
 command -v gcutil >/dev/null 2>&1 || {
     echo "'gcutil' must be installed" >&2
@@ -17,8 +16,7 @@ command -v gcutil >/dev/null 2>&1 || {
 }
 
 USER_VARS="-var bucket_name=${GC_BUCKET_NAME}"
-USER_VARS="${USER_VARS} -var client_secrets_file=${GC_CLIENT_SECRETS_FILE}"
-USER_VARS="${USER_VARS} -var private_key_file=${GC_PRIVATE_KEY_FILE}"
+USER_VARS="${USER_VARS} -var account_file=${GC_ACCOUNT_FILE}"
 USER_VARS="${USER_VARS} -var project_id=${GC_PROJECT_ID}"
 
 # This tests if GCE has an image that contains the given parameter.
@@ -30,7 +28,7 @@ gc_has_image() {
 teardown() {
     gcutil --format=names --project=${GC_PROJECT_ID} listimages \
         | grep packerbats \
-        | xargs -n1 gcutil --project=${GC_PROJECT_ID} --force deleteimage
+        | xargs -n1 gcutil --project=${GC_PROJECT_ID} deleteimage --force
 }
 
 @test "googlecompute: build minimal.json" {

test/fixtures/builder-googlecompute/minimal.json

 {
     "variables": {
         "bucket_name": null,
-        "client_secrets_file": null,
-        "private_key_file": null,
+        "account_file": null,
         "project_id": null
     },
 
     "builders": [{
         "type": "googlecompute",
         "bucket_name": "{{user `bucket_name`}}",
-        "client_secrets_file": "{{user `client_secrets_file`}}",
-        "private_key_file": "{{user `private_key_file`}}",
+        "account_file": "{{user `account_file`}}",
         "project_id": "{{user `project_id`}}",
 
         "image_name": "packerbats-minimal-{{timestamp}}",
-        "source_image": "debian-7-wheezy-v20131120",
+        "source_image": "debian-7-wheezy-v20141108",
         "zone": "us-central1-a"
     }]
 }

website/source/docs/builders/googlecompute.markdown

@@ -9,19 +9,49 @@ description: |-
 
 Type: `googlecompute`
 
-The `googlecompute` Packer builder is able to create
-[images](https://developers.google.com/compute/docs/images)
-for use with [Google Compute Engine](https://cloud.google.com/products/compute-engine)
-(GCE) based on existing images. Google Compute Engine doesn't allow the creation
-of images from scratch.
+The `googlecompute` Packer builder is able to create [images](https://developers.google.com/compute/docs/images) for use with
+[Google Compute Engine](https://cloud.google.com/products/compute-engine)(GCE) based on existing images. Google
+Compute Engine doesn't allow the creation of images from scratch.
 
 ## Authentication
 
-Authenticating with Google Cloud services requires two separate JSON
-files: one which we call the _account file_ and the _client secrets file_.
+Authenticating with Google Cloud services requires at most one JSON file, 
+called the _account file_. The _account file_ is **not** required if you are running
+the `googlecompute` Packer builder from a GCE instance with a properly-configured
+[Compute Engine Service Account](https://cloud.google.com/compute/docs/authentication.
 
-Both of these files are downloaded directly from the
-[Google Developers Console](https://console.developers.google.com). To make
+### Running With a Compute Engine Service Account
+If you run the `googlecompute` Packer builder from a GCE instance, you can configure that
+instance to use a [Compute Engine Service Account](https://cloud.google.com/compute/docs/authentication). This will allow Packer to authenticate
+to Google Cloud without having to bake in a separate credential/authentication file. 
+
+To create a GCE instance that uses a service account, provide the required scopes when
+launching the intance.
+
+For `gcloud`, do this via the `--scopes` parameter:
+
+```sh
+gcloud compute --project YOUR_PROJECT instances create "INSTANCE-NAME" ... \
+               --scopes "https://www.googleapis.com/auth/compute" \
+                        "https://www.googleapis.com/auth/devstorage.full_control" \
+               ...
+```
+
+For the [Google Developers Console](https://console.developers.google.com):
+
+1. Choose "Show advanced options"
+2. Tick "Enable Compute Engine service account"
+3. Choose "Read Write" for Compute
+4. Chose "Full" for "Storage"
+
+**The service account will be used automatically by Packer as long as there is
+no _account file_ specified in the Packer configuration file.**
+
+### Running Without a Compute Engine Service Account
+
+The [Google Developers Console](https://console.developers.google.com) allows you to
+create and download a credential file that will let you use the `googlecompute` Packer
+builder anywhere. To make
 the process more straightforwarded, it is documented here.
 
 1. Log into the [Google Developers Console](https://console.developers.google.com)
@@ -29,27 +59,22 @@ the process more straightforwarded, it is documented here.
 
 2. Under the "APIs & Auth" section, click "Credentials."
 
-3. Click the "Download JSON" button under the "Compute Engine and App Engine"
-   account in the OAuth section. The file should start with "client\_secrets".
-   This is your _client secrets file_.
+3. Click the "Create new Client ID" button, select "Service account", and click "Create Client ID"
 
-4. Create a new OAuth client ID and select "Service Account" as the type
-   of account. Once created, a JSON file should be downloaded. This is your
+4. Click "Generate new JSON key" for the Service Account you just created. A JSON file will be downloaded automatically. This is your
    _account file_.
 
 ## Basic Example
 
 Below is a fully functioning example. It doesn't do anything useful,
 since no provisioners are defined, but it will effectively repackage an
-existing GCE image. The client secrets file and private key file are the
-files obtained in the previous section.
+existing GCE image. The account file is obtained in the previous section.
 
 ```javascript
 {
   "type": "googlecompute",
   "bucket_name": "my-project-packer-images",
   "account_file": "account.json",
-  "client_secrets_file": "client_secret.json",
   "project_id": "my-project",
   "source_image": "debian-7-wheezy-v20140718",
   "zone": "us-central1-a"
@@ -63,17 +88,8 @@ each category, the available options are alphabetized and described.
 
 ### Required:
 
-* `account_file` (string) - The JSON file containing your account credentials.
-     Instructions for how to retrieve these are above.
-
 * `bucket_name` (string) - The Google Cloud Storage bucket to store the
-  images that are created. The bucket must already exist in your project.
-
-* `client_secrets_file` (string) - The client secrets JSON file that
-  was set up in the section above.
-
-* `private_key_file` (string) - The client private key file that was
-  generated in the section above.
+  images that are created. The bucket must already exist in your project
 
 * `project_id` (string) - The project ID that will be used to launch instances
   and store images.
@@ -86,6 +102,10 @@ each category, the available options are alphabetized and described.
 
 ### Optional:
 
+* `account_file` (string) - The JSON file containing your account credentials.
+     Not required if you run Packer on a GCE instance with a service account.
+     Instructions for creating file or using service accounts are above.
+
 * `disk_size` (integer) - The size of the disk in GB.
   This defaults to 10, which is 10GB.