Commit d1ac3ff0 authored by Gilad Ben-Yossef's avatar Gilad Ben-Yossef Committed by Mike Snitzer

dm verity: switch to using asynchronous hash crypto API

Use of the synchronous digest API limits dm-verity to using pure
CPU based algorithm providers and rules out the use of off CPU
algorithm providers which are normally asynchronous by nature,
potentially freeing CPU cycles.

This can reduce performance per Watt in situations such as during
boot time when a lot of concurrent file accesses are made to the
protected volume.
Signed-off-by: default avatarGilad Ben-Yossef <gilad@benyossef.com>
CC: Eric Biggers <ebiggers3@gmail.com>
CC: Ondrej Mosnáček <omosnacek+linux-crypto@gmail.com>
Tested-by: default avatarMilan Broz <gmazyland@gmail.com>
Signed-off-by: default avatarMike Snitzer <snitzer@redhat.com>
parent a1b89132
...@@ -188,7 +188,7 @@ static int fec_decode_bufs(struct dm_verity *v, struct dm_verity_fec_io *fio, ...@@ -188,7 +188,7 @@ static int fec_decode_bufs(struct dm_verity *v, struct dm_verity_fec_io *fio,
static int fec_is_erasure(struct dm_verity *v, struct dm_verity_io *io, static int fec_is_erasure(struct dm_verity *v, struct dm_verity_io *io,
u8 *want_digest, u8 *data) u8 *want_digest, u8 *data)
{ {
if (unlikely(verity_hash(v, verity_io_hash_desc(v, io), if (unlikely(verity_hash(v, verity_io_hash_req(v, io),
data, 1 << v->data_dev_block_bits, data, 1 << v->data_dev_block_bits,
verity_io_real_digest(v, io)))) verity_io_real_digest(v, io))))
return 0; return 0;
...@@ -397,7 +397,7 @@ static int fec_decode_rsb(struct dm_verity *v, struct dm_verity_io *io, ...@@ -397,7 +397,7 @@ static int fec_decode_rsb(struct dm_verity *v, struct dm_verity_io *io,
} }
/* Always re-validate the corrected block against the expected hash */ /* Always re-validate the corrected block against the expected hash */
r = verity_hash(v, verity_io_hash_desc(v, io), fio->output, r = verity_hash(v, verity_io_hash_req(v, io), fio->output,
1 << v->data_dev_block_bits, 1 << v->data_dev_block_bits,
verity_io_real_digest(v, io)); verity_io_real_digest(v, io));
if (unlikely(r < 0)) if (unlikely(r < 0))
......
...@@ -93,81 +93,123 @@ static sector_t verity_position_at_level(struct dm_verity *v, sector_t block, ...@@ -93,81 +93,123 @@ static sector_t verity_position_at_level(struct dm_verity *v, sector_t block,
} }
/* /*
* Wrapper for crypto_shash_init, which handles verity salting. * Callback function for asynchrnous crypto API completion notification
*/ */
static int verity_hash_init(struct dm_verity *v, struct shash_desc *desc) static void verity_op_done(struct crypto_async_request *base, int err)
{ {
int r; struct verity_result *res = (struct verity_result *)base->data;
desc->tfm = v->tfm; if (err == -EINPROGRESS)
desc->flags = CRYPTO_TFM_REQ_MAY_SLEEP; return;
r = crypto_shash_init(desc); res->err = err;
complete(&res->completion);
}
if (unlikely(r < 0)) { /*
DMERR("crypto_shash_init failed: %d", r); * Wait for async crypto API callback
return r; */
} static inline int verity_complete_op(struct verity_result *res, int ret)
{
switch (ret) {
case 0:
break;
if (likely(v->version >= 1)) { case -EINPROGRESS:
r = crypto_shash_update(desc, v->salt, v->salt_size); case -EBUSY:
ret = wait_for_completion_interruptible(&res->completion);
if (!ret)
ret = res->err;
reinit_completion(&res->completion);
break;
if (unlikely(r < 0)) { default:
DMERR("crypto_shash_update failed: %d", r); DMERR("verity_wait_hash: crypto op submission failed: %d", ret);
return r;
}
} }
return 0; if (unlikely(ret < 0))
DMERR("verity_wait_hash: crypto op failed: %d", ret);
return ret;
} }
static int verity_hash_update(struct dm_verity *v, struct shash_desc *desc, static int verity_hash_update(struct dm_verity *v, struct ahash_request *req,
const u8 *data, size_t len) const u8 *data, size_t len,
struct verity_result *res)
{ {
int r = crypto_shash_update(desc, data, len); struct scatterlist sg;
if (unlikely(r < 0)) sg_init_one(&sg, data, len);
DMERR("crypto_shash_update failed: %d", r); ahash_request_set_crypt(req, &sg, NULL, len);
return verity_complete_op(res, crypto_ahash_update(req));
}
/*
* Wrapper for crypto_ahash_init, which handles verity salting.
*/
static int verity_hash_init(struct dm_verity *v, struct ahash_request *req,
struct verity_result *res)
{
int r;
ahash_request_set_tfm(req, v->tfm);
ahash_request_set_callback(req, CRYPTO_TFM_REQ_MAY_SLEEP |
CRYPTO_TFM_REQ_MAY_BACKLOG,
verity_op_done, (void *)res);
init_completion(&res->completion);
r = verity_complete_op(res, crypto_ahash_init(req));
if (unlikely(r < 0)) {
DMERR("crypto_ahash_init failed: %d", r);
return r;
}
if (likely(v->version >= 1))
r = verity_hash_update(v, req, v->salt, v->salt_size, res);
return r; return r;
} }
static int verity_hash_final(struct dm_verity *v, struct shash_desc *desc, static int verity_hash_final(struct dm_verity *v, struct ahash_request *req,
u8 *digest) u8 *digest, struct verity_result *res)
{ {
int r; int r;
if (unlikely(!v->version)) { if (unlikely(!v->version)) {
r = crypto_shash_update(desc, v->salt, v->salt_size); r = verity_hash_update(v, req, v->salt, v->salt_size, res);
if (r < 0) { if (r < 0) {
DMERR("crypto_shash_update failed: %d", r); DMERR("verity_hash_final failed updating salt: %d", r);
return r; goto out;
} }
} }
r = crypto_shash_final(desc, digest); ahash_request_set_crypt(req, NULL, digest, 0);
r = verity_complete_op(res, crypto_ahash_final(req));
if (unlikely(r < 0)) out:
DMERR("crypto_shash_final failed: %d", r);
return r; return r;
} }
int verity_hash(struct dm_verity *v, struct shash_desc *desc, int verity_hash(struct dm_verity *v, struct ahash_request *req,
const u8 *data, size_t len, u8 *digest) const u8 *data, size_t len, u8 *digest)
{ {
int r; int r;
struct verity_result res;
r = verity_hash_init(v, desc); r = verity_hash_init(v, req, &res);
if (unlikely(r < 0)) if (unlikely(r < 0))
return r; goto out;
r = verity_hash_update(v, desc, data, len); r = verity_hash_update(v, req, data, len, &res);
if (unlikely(r < 0)) if (unlikely(r < 0))
return r; goto out;
return verity_hash_final(v, desc, digest); r = verity_hash_final(v, req, digest, &res);
out:
return r;
} }
static void verity_hash_at_level(struct dm_verity *v, sector_t block, int level, static void verity_hash_at_level(struct dm_verity *v, sector_t block, int level,
...@@ -275,7 +317,7 @@ static int verity_verify_level(struct dm_verity *v, struct dm_verity_io *io, ...@@ -275,7 +317,7 @@ static int verity_verify_level(struct dm_verity *v, struct dm_verity_io *io,
goto release_ret_r; goto release_ret_r;
} }
r = verity_hash(v, verity_io_hash_desc(v, io), r = verity_hash(v, verity_io_hash_req(v, io),
data, 1 << v->hash_dev_block_bits, data, 1 << v->hash_dev_block_bits,
verity_io_real_digest(v, io)); verity_io_real_digest(v, io));
if (unlikely(r < 0)) if (unlikely(r < 0))
...@@ -343,6 +385,49 @@ int verity_hash_for_block(struct dm_verity *v, struct dm_verity_io *io, ...@@ -343,6 +385,49 @@ int verity_hash_for_block(struct dm_verity *v, struct dm_verity_io *io,
return r; return r;
} }
/*
* Calculates the digest for the given bio
*/
int verity_for_io_block(struct dm_verity *v, struct dm_verity_io *io,
struct bvec_iter *iter, struct verity_result *res)
{
unsigned int todo = 1 << v->data_dev_block_bits;
struct bio *bio = dm_bio_from_per_bio_data(io, v->ti->per_io_data_size);
struct scatterlist sg;
struct ahash_request *req = verity_io_hash_req(v, io);
do {
int r;
unsigned int len;
struct bio_vec bv = bio_iter_iovec(bio, *iter);
sg_init_table(&sg, 1);
len = bv.bv_len;
if (likely(len >= todo))
len = todo;
/*
* Operating on a single page at a time looks suboptimal
* until you consider the typical block size is 4,096B.
* Going through this loops twice should be very rare.
*/
sg_set_page(&sg, bv.bv_page, len, bv.bv_offset);
ahash_request_set_crypt(req, &sg, NULL, len);
r = verity_complete_op(res, crypto_ahash_update(req));
if (unlikely(r < 0)) {
DMERR("verity_for_io_block crypto op failed: %d", r);
return r;
}
bio_advance_iter(bio, iter, len);
todo -= len;
} while (todo);
return 0;
}
/* /*
* Calls function process for 1 << v->data_dev_block_bits bytes in the bio_vec * Calls function process for 1 << v->data_dev_block_bits bytes in the bio_vec
* starting from iter. * starting from iter.
...@@ -381,12 +466,6 @@ int verity_for_bv_block(struct dm_verity *v, struct dm_verity_io *io, ...@@ -381,12 +466,6 @@ int verity_for_bv_block(struct dm_verity *v, struct dm_verity_io *io,
return 0; return 0;
} }
static int verity_bv_hash_update(struct dm_verity *v, struct dm_verity_io *io,
u8 *data, size_t len)
{
return verity_hash_update(v, verity_io_hash_desc(v, io), data, len);
}
static int verity_bv_zero(struct dm_verity *v, struct dm_verity_io *io, static int verity_bv_zero(struct dm_verity *v, struct dm_verity_io *io,
u8 *data, size_t len) u8 *data, size_t len)
{ {
...@@ -403,10 +482,11 @@ static int verity_verify_io(struct dm_verity_io *io) ...@@ -403,10 +482,11 @@ static int verity_verify_io(struct dm_verity_io *io)
struct dm_verity *v = io->v; struct dm_verity *v = io->v;
struct bvec_iter start; struct bvec_iter start;
unsigned b; unsigned b;
struct verity_result res;
for (b = 0; b < io->n_blocks; b++) { for (b = 0; b < io->n_blocks; b++) {
int r; int r;
struct shash_desc *desc = verity_io_hash_desc(v, io); struct ahash_request *req = verity_io_hash_req(v, io);
r = verity_hash_for_block(v, io, io->block + b, r = verity_hash_for_block(v, io, io->block + b,
verity_io_want_digest(v, io), verity_io_want_digest(v, io),
...@@ -427,16 +507,17 @@ static int verity_verify_io(struct dm_verity_io *io) ...@@ -427,16 +507,17 @@ static int verity_verify_io(struct dm_verity_io *io)
continue; continue;
} }
r = verity_hash_init(v, desc); r = verity_hash_init(v, req, &res);
if (unlikely(r < 0)) if (unlikely(r < 0))
return r; return r;
start = io->iter; start = io->iter;
r = verity_for_bv_block(v, io, &io->iter, verity_bv_hash_update); r = verity_for_io_block(v, io, &io->iter, &res);
if (unlikely(r < 0)) if (unlikely(r < 0))
return r; return r;
r = verity_hash_final(v, desc, verity_io_real_digest(v, io)); r = verity_hash_final(v, req, verity_io_real_digest(v, io),
&res);
if (unlikely(r < 0)) if (unlikely(r < 0))
return r; return r;
...@@ -705,7 +786,7 @@ static void verity_dtr(struct dm_target *ti) ...@@ -705,7 +786,7 @@ static void verity_dtr(struct dm_target *ti)
kfree(v->zero_digest); kfree(v->zero_digest);
if (v->tfm) if (v->tfm)
crypto_free_shash(v->tfm); crypto_free_ahash(v->tfm);
kfree(v->alg_name); kfree(v->alg_name);
...@@ -723,7 +804,7 @@ static void verity_dtr(struct dm_target *ti) ...@@ -723,7 +804,7 @@ static void verity_dtr(struct dm_target *ti)
static int verity_alloc_zero_digest(struct dm_verity *v) static int verity_alloc_zero_digest(struct dm_verity *v)
{ {
int r = -ENOMEM; int r = -ENOMEM;
struct shash_desc *desc; struct ahash_request *req;
u8 *zero_data; u8 *zero_data;
v->zero_digest = kmalloc(v->digest_size, GFP_KERNEL); v->zero_digest = kmalloc(v->digest_size, GFP_KERNEL);
...@@ -731,9 +812,9 @@ static int verity_alloc_zero_digest(struct dm_verity *v) ...@@ -731,9 +812,9 @@ static int verity_alloc_zero_digest(struct dm_verity *v)
if (!v->zero_digest) if (!v->zero_digest)
return r; return r;
desc = kmalloc(v->shash_descsize, GFP_KERNEL); req = kmalloc(v->ahash_reqsize, GFP_KERNEL);
if (!desc) if (!req)
return r; /* verity_dtr will free zero_digest */ return r; /* verity_dtr will free zero_digest */
zero_data = kzalloc(1 << v->data_dev_block_bits, GFP_KERNEL); zero_data = kzalloc(1 << v->data_dev_block_bits, GFP_KERNEL);
...@@ -741,11 +822,11 @@ static int verity_alloc_zero_digest(struct dm_verity *v) ...@@ -741,11 +822,11 @@ static int verity_alloc_zero_digest(struct dm_verity *v)
if (!zero_data) if (!zero_data)
goto out; goto out;
r = verity_hash(v, desc, zero_data, 1 << v->data_dev_block_bits, r = verity_hash(v, req, zero_data, 1 << v->data_dev_block_bits,
v->zero_digest); v->zero_digest);
out: out:
kfree(desc); kfree(req);
kfree(zero_data); kfree(zero_data);
return r; return r;
...@@ -923,21 +1004,21 @@ static int verity_ctr(struct dm_target *ti, unsigned argc, char **argv) ...@@ -923,21 +1004,21 @@ static int verity_ctr(struct dm_target *ti, unsigned argc, char **argv)
goto bad; goto bad;
} }
v->tfm = crypto_alloc_shash(v->alg_name, 0, 0); v->tfm = crypto_alloc_ahash(v->alg_name, 0, 0);
if (IS_ERR(v->tfm)) { if (IS_ERR(v->tfm)) {
ti->error = "Cannot initialize hash function"; ti->error = "Cannot initialize hash function";
r = PTR_ERR(v->tfm); r = PTR_ERR(v->tfm);
v->tfm = NULL; v->tfm = NULL;
goto bad; goto bad;
} }
v->digest_size = crypto_shash_digestsize(v->tfm); v->digest_size = crypto_ahash_digestsize(v->tfm);
if ((1 << v->hash_dev_block_bits) < v->digest_size * 2) { if ((1 << v->hash_dev_block_bits) < v->digest_size * 2) {
ti->error = "Digest size too big"; ti->error = "Digest size too big";
r = -EINVAL; r = -EINVAL;
goto bad; goto bad;
} }
v->shash_descsize = v->ahash_reqsize = sizeof(struct ahash_request) +
sizeof(struct shash_desc) + crypto_shash_descsize(v->tfm); crypto_ahash_reqsize(v->tfm);
v->root_digest = kmalloc(v->digest_size, GFP_KERNEL); v->root_digest = kmalloc(v->digest_size, GFP_KERNEL);
if (!v->root_digest) { if (!v->root_digest) {
...@@ -1037,7 +1118,7 @@ static int verity_ctr(struct dm_target *ti, unsigned argc, char **argv) ...@@ -1037,7 +1118,7 @@ static int verity_ctr(struct dm_target *ti, unsigned argc, char **argv)
} }
ti->per_io_data_size = sizeof(struct dm_verity_io) + ti->per_io_data_size = sizeof(struct dm_verity_io) +
v->shash_descsize + v->digest_size * 2; v->ahash_reqsize + v->digest_size * 2;
r = verity_fec_ctr(v); r = verity_fec_ctr(v);
if (r) if (r)
......
...@@ -37,7 +37,7 @@ struct dm_verity { ...@@ -37,7 +37,7 @@ struct dm_verity {
struct dm_target *ti; struct dm_target *ti;
struct dm_bufio_client *bufio; struct dm_bufio_client *bufio;
char *alg_name; char *alg_name;
struct crypto_shash *tfm; struct crypto_ahash *tfm;
u8 *root_digest; /* digest of the root block */ u8 *root_digest; /* digest of the root block */
u8 *salt; /* salt: its size is salt_size */ u8 *salt; /* salt: its size is salt_size */
u8 *zero_digest; /* digest for a zero block */ u8 *zero_digest; /* digest for a zero block */
...@@ -52,7 +52,7 @@ struct dm_verity { ...@@ -52,7 +52,7 @@ struct dm_verity {
unsigned char levels; /* the number of tree levels */ unsigned char levels; /* the number of tree levels */
unsigned char version; unsigned char version;
unsigned digest_size; /* digest size for the current hash algorithm */ unsigned digest_size; /* digest size for the current hash algorithm */
unsigned shash_descsize;/* the size of temporary space for crypto */ unsigned int ahash_reqsize;/* the size of temporary space for crypto */
int hash_failed; /* set to 1 if hash of any block failed */ int hash_failed; /* set to 1 if hash of any block failed */
enum verity_mode mode; /* mode for handling verification errors */ enum verity_mode mode; /* mode for handling verification errors */
unsigned corrupted_errs;/* Number of errors for corrupted blocks */ unsigned corrupted_errs;/* Number of errors for corrupted blocks */
...@@ -81,31 +81,36 @@ struct dm_verity_io { ...@@ -81,31 +81,36 @@ struct dm_verity_io {
/* /*
* Three variably-size fields follow this struct: * Three variably-size fields follow this struct:
* *
* u8 hash_desc[v->shash_descsize]; * u8 hash_req[v->ahash_reqsize];
* u8 real_digest[v->digest_size]; * u8 real_digest[v->digest_size];
* u8 want_digest[v->digest_size]; * u8 want_digest[v->digest_size];
* *
* To access them use: verity_io_hash_desc(), verity_io_real_digest() * To access them use: verity_io_hash_req(), verity_io_real_digest()
* and verity_io_want_digest(). * and verity_io_want_digest().
*/ */
}; };
static inline struct shash_desc *verity_io_hash_desc(struct dm_verity *v, struct verity_result {
struct completion completion;
int err;
};
static inline struct ahash_request *verity_io_hash_req(struct dm_verity *v,
struct dm_verity_io *io) struct dm_verity_io *io)
{ {
return (struct shash_desc *)(io + 1); return (struct ahash_request *)(io + 1);
} }
static inline u8 *verity_io_real_digest(struct dm_verity *v, static inline u8 *verity_io_real_digest(struct dm_verity *v,
struct dm_verity_io *io) struct dm_verity_io *io)
{ {
return (u8 *)(io + 1) + v->shash_descsize; return (u8 *)(io + 1) + v->ahash_reqsize;
} }
static inline u8 *verity_io_want_digest(struct dm_verity *v, static inline u8 *verity_io_want_digest(struct dm_verity *v,
struct dm_verity_io *io) struct dm_verity_io *io)
{ {
return (u8 *)(io + 1) + v->shash_descsize + v->digest_size; return (u8 *)(io + 1) + v->ahash_reqsize + v->digest_size;
} }
static inline u8 *verity_io_digest_end(struct dm_verity *v, static inline u8 *verity_io_digest_end(struct dm_verity *v,
...@@ -120,7 +125,7 @@ extern int verity_for_bv_block(struct dm_verity *v, struct dm_verity_io *io, ...@@ -120,7 +125,7 @@ extern int verity_for_bv_block(struct dm_verity *v, struct dm_verity_io *io,
struct dm_verity_io *io, struct dm_verity_io *io,
u8 *data, size_t len)); u8 *data, size_t len));
extern int verity_hash(struct dm_verity *v, struct shash_desc *desc, extern int verity_hash(struct dm_verity *v, struct ahash_request *req,
const u8 *data, size_t len, u8 *digest); const u8 *data, size_t len, u8 *digest);
extern int verity_hash_for_block(struct dm_verity *v, struct dm_verity_io *io, extern int verity_hash_for_block(struct dm_verity *v, struct dm_verity_io *io,
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment