Commit 9d12ba86 authored by Rob Rice's avatar Rob Rice Committed by Herbert Xu

crypto: brcm - Add Broadcom SPU driver

Add Broadcom Secure Processing Unit (SPU) crypto driver for SPU
hardware crypto offload. The driver supports ablkcipher, ahash,
and aead symmetric crypto operations.
Signed-off-by: default avatarSteve Lin <steven.lin1@broadcom.com>
Signed-off-by: default avatarRob Rice <rob.rice@broadcom.com>
Signed-off-by: default avatarHerbert Xu <herbert@gondor.apana.org.au>
parent 206dc4fc
...@@ -587,4 +587,19 @@ source "drivers/crypto/chelsio/Kconfig" ...@@ -587,4 +587,19 @@ source "drivers/crypto/chelsio/Kconfig"
source "drivers/crypto/virtio/Kconfig" source "drivers/crypto/virtio/Kconfig"
config CRYPTO_DEV_BCM_SPU
tristate "Broadcom symmetric crypto/hash acceleration support"
depends on ARCH_BCM_IPROC
depends on BCM_PDC_MBOX
default m
select CRYPTO_DES
select CRYPTO_MD5
select CRYPTO_SHA1
select CRYPTO_SHA256
select CRYPTO_SHA512
help
This driver provides support for Broadcom crypto acceleration using the
Secure Processing Unit (SPU). The SPU driver registers ablkcipher,
ahash, and aead algorithms with the kernel cryptographic API.
endif # CRYPTO_HW endif # CRYPTO_HW
...@@ -35,3 +35,4 @@ obj-$(CONFIG_CRYPTO_DEV_TALITOS) += talitos.o ...@@ -35,3 +35,4 @@ obj-$(CONFIG_CRYPTO_DEV_TALITOS) += talitos.o
obj-$(CONFIG_CRYPTO_DEV_UX500) += ux500/ obj-$(CONFIG_CRYPTO_DEV_UX500) += ux500/
obj-$(CONFIG_CRYPTO_DEV_VIRTIO) += virtio/ obj-$(CONFIG_CRYPTO_DEV_VIRTIO) += virtio/
obj-$(CONFIG_CRYPTO_DEV_VMX) += vmx/ obj-$(CONFIG_CRYPTO_DEV_VMX) += vmx/
obj-$(CONFIG_CRYPTO_DEV_BCM_SPU) += bcm/
# File: drivers/crypto/bcm/Makefile
#
# Makefile for crypto acceleration files for Broadcom SPU driver
#
# Uncomment to enable debug tracing in the SPU driver.
# CFLAGS_util.o := -DDEBUG
# CFLAGS_cipher.o := -DDEBUG
# CFLAGS_spu.o := -DDEBUG
# CFLAGS_spu2.o := -DDEBUG
obj-$(CONFIG_CRYPTO_DEV_BCM_SPU) := bcm_crypto_spu.o
bcm_crypto_spu-objs := util.o spu.o spu2.o cipher.o
ccflags-y += -I. -DBCMDRIVER
This source diff could not be displayed because it is too large. You can view the blob instead.
/*
* Copyright 2016 Broadcom
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License, version 2, as
* published by the Free Software Foundation (the "GPL").
*
* This program is distributed in the hope that it will be useful, but
* WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* General Public License version 2 (GPLv2) for more details.
*
* You should have received a copy of the GNU General Public License
* version 2 (GPLv2) along with this source code.
*/
#ifndef _CIPHER_H
#define _CIPHER_H
#include <linux/atomic.h>
#include <linux/mailbox/brcm-message.h>
#include <linux/mailbox_client.h>
#include <crypto/aes.h>
#include <crypto/internal/hash.h>
#include <crypto/aead.h>
#include <crypto/sha.h>
#include <crypto/sha3.h>
#include "spu.h"
#include "spum.h"
#include "spu2.h"
/* Driver supports up to MAX_SPUS SPU blocks */
#define MAX_SPUS 16
#define ARC4_MIN_KEY_SIZE 1
#define ARC4_MAX_KEY_SIZE 256
#define ARC4_BLOCK_SIZE 1
#define ARC4_STATE_SIZE 4
#define CCM_AES_IV_SIZE 16
#define GCM_AES_IV_SIZE 12
#define GCM_ESP_IV_SIZE 8
#define CCM_ESP_IV_SIZE 8
#define RFC4543_ICV_SIZE 16
#define MAX_KEY_SIZE ARC4_MAX_KEY_SIZE
#define MAX_IV_SIZE AES_BLOCK_SIZE
#define MAX_DIGEST_SIZE SHA3_512_DIGEST_SIZE
#define MAX_ASSOC_SIZE 512
/* size of salt value for AES-GCM-ESP and AES-CCM-ESP */
#define GCM_ESP_SALT_SIZE 4
#define CCM_ESP_SALT_SIZE 3
#define MAX_SALT_SIZE GCM_ESP_SALT_SIZE
#define GCM_ESP_SALT_OFFSET 0
#define CCM_ESP_SALT_OFFSET 1
#define GCM_ESP_DIGESTSIZE 16
#define MAX_HASH_BLOCK_SIZE SHA512_BLOCK_SIZE
/*
* Maximum number of bytes from a non-final hash request that can be deferred
* until more data is available. With new crypto API framework, this
* can be no more than one block of data.
*/
#define HASH_CARRY_MAX MAX_HASH_BLOCK_SIZE
/* Force at least 4-byte alignment of all SPU message fields */
#define SPU_MSG_ALIGN 4
/* Number of times to resend mailbox message if mb queue is full */
#define SPU_MB_RETRY_MAX 1000
/* op_counts[] indexes */
enum op_type {
SPU_OP_CIPHER,
SPU_OP_HASH,
SPU_OP_HMAC,
SPU_OP_AEAD,
SPU_OP_NUM
};
enum spu_spu_type {
SPU_TYPE_SPUM,
SPU_TYPE_SPU2,
};
/*
* SPUM_NS2 and SPUM_NSP are the SPU-M block on Northstar 2 and Northstar Plus,
* respectively.
*/
enum spu_spu_subtype {
SPU_SUBTYPE_SPUM_NS2,
SPU_SUBTYPE_SPUM_NSP,
SPU_SUBTYPE_SPU2_V1,
SPU_SUBTYPE_SPU2_V2
};
struct spu_type_subtype {
enum spu_spu_type type;
enum spu_spu_subtype subtype;
};
struct cipher_op {
enum spu_cipher_alg alg;
enum spu_cipher_mode mode;
};
struct auth_op {
enum hash_alg alg;
enum hash_mode mode;
};
struct iproc_alg_s {
u32 type;
union {
struct crypto_alg crypto;
struct ahash_alg hash;
struct aead_alg aead;
} alg;
struct cipher_op cipher_info;
struct auth_op auth_info;
bool auth_first;
bool registered;
};
/*
* Buffers for a SPU request/reply message pair. All part of one structure to
* allow a single alloc per request.
*/
struct spu_msg_buf {
/* Request message fragments */
/*
* SPU request message header. For SPU-M, holds MH, EMH, SCTX, BDESC,
* and BD header. For SPU2, holds FMD, OMD.
*/
u8 bcm_spu_req_hdr[ALIGN(SPU2_HEADER_ALLOC_LEN, SPU_MSG_ALIGN)];
/* IV or counter. Size to include salt. Also used for XTS tweek. */
u8 iv_ctr[ALIGN(2 * AES_BLOCK_SIZE, SPU_MSG_ALIGN)];
/* Hash digest. request and response. */
u8 digest[ALIGN(MAX_DIGEST_SIZE, SPU_MSG_ALIGN)];
/* SPU request message padding */
u8 spu_req_pad[ALIGN(SPU_PAD_LEN_MAX, SPU_MSG_ALIGN)];
/* SPU-M request message STATUS field */
u8 tx_stat[ALIGN(SPU_TX_STATUS_LEN, SPU_MSG_ALIGN)];
/* Response message fragments */
/* SPU response message header */
u8 spu_resp_hdr[ALIGN(SPU2_HEADER_ALLOC_LEN, SPU_MSG_ALIGN)];
/* SPU response message STATUS field padding */
u8 rx_stat_pad[ALIGN(SPU_STAT_PAD_MAX, SPU_MSG_ALIGN)];
/* SPU response message STATUS field */
u8 rx_stat[ALIGN(SPU_RX_STATUS_LEN, SPU_MSG_ALIGN)];
union {
/* Buffers only used for ablkcipher */
struct {
/*
* Field used for either SUPDT when RC4 is used
* -OR- tweak value when XTS/AES is used
*/
u8 supdt_tweak[ALIGN(SPU_SUPDT_LEN, SPU_MSG_ALIGN)];
} c;
/* Buffers only used for aead */
struct {
/* SPU response pad for GCM data */
u8 gcmpad[ALIGN(AES_BLOCK_SIZE, SPU_MSG_ALIGN)];
/* SPU request msg padding for GCM AAD */
u8 req_aad_pad[ALIGN(SPU_PAD_LEN_MAX, SPU_MSG_ALIGN)];
/* SPU response data to be discarded */
u8 resp_aad[ALIGN(MAX_ASSOC_SIZE + MAX_IV_SIZE,
SPU_MSG_ALIGN)];
} a;
};
};
struct iproc_ctx_s {
u8 enckey[MAX_KEY_SIZE + ARC4_STATE_SIZE];
unsigned int enckeylen;
u8 authkey[MAX_KEY_SIZE + ARC4_STATE_SIZE];
unsigned int authkeylen;
u8 salt[MAX_SALT_SIZE];
unsigned int salt_len;
unsigned int salt_offset;
u8 iv[MAX_IV_SIZE];
unsigned int digestsize;
struct iproc_alg_s *alg;
bool is_esp;
struct cipher_op cipher;
enum spu_cipher_type cipher_type;
struct auth_op auth;
bool auth_first;
/*
* The maximum length in bytes of the payload in a SPU message for this
* context. For SPU-M, the payload is the combination of AAD and data.
* For SPU2, the payload is just data. A value of SPU_MAX_PAYLOAD_INF
* indicates that there is no limit to the length of the SPU message
* payload.
*/
unsigned int max_payload;
struct crypto_aead *fallback_cipher;
/* auth_type is determined during processing of request */
u8 ipad[MAX_HASH_BLOCK_SIZE];
u8 opad[MAX_HASH_BLOCK_SIZE];
/*
* Buffer to hold SPU message header template. Template is created at
* setkey time for ablkcipher requests, since most of the fields in the
* header are known at that time. At request time, just fill in a few
* missing pieces related to length of data in the request and IVs, etc.
*/
u8 bcm_spu_req_hdr[ALIGN(SPU2_HEADER_ALLOC_LEN, SPU_MSG_ALIGN)];
/* Length of SPU request header */
u16 spu_req_hdr_len;
/* Expected length of SPU response header */
u16 spu_resp_hdr_len;
/*
* shash descriptor - needed to perform incremental hashing in
* in software, when hw doesn't support it.
*/
struct shash_desc *shash;
bool is_rfc4543; /* RFC 4543 style of GMAC */
};
/* state from iproc_reqctx_s necessary for hash state export/import */
struct spu_hash_export_s {
unsigned int total_todo;
unsigned int total_sent;
u8 hash_carry[HASH_CARRY_MAX];
unsigned int hash_carry_len;
u8 incr_hash[MAX_DIGEST_SIZE];
bool is_sw_hmac;
};
struct iproc_reqctx_s {
/* general context */
struct crypto_async_request *parent;
/* only valid after enqueue() */
struct iproc_ctx_s *ctx;
u8 chan_idx; /* Mailbox channel to be used to submit this request */
/* total todo, rx'd, and sent for this request */
unsigned int total_todo;
unsigned int total_received; /* only valid for ablkcipher */
unsigned int total_sent;
/*
* num bytes sent to hw from the src sg in this request. This can differ
* from total_sent for incremental hashing. total_sent includes previous
* init() and update() data. src_sent does not.
*/
unsigned int src_sent;
/*
* For AEAD requests, start of associated data. This will typically
* point to the beginning of the src scatterlist from the request,
* since assoc data is at the beginning of the src scatterlist rather
* than in its own sg.
*/
struct scatterlist *assoc;
/*
* scatterlist entry and offset to start of data for next chunk. Crypto
* API src scatterlist for AEAD starts with AAD, if present. For first
* chunk, src_sg is sg entry at beginning of input data (after AAD).
* src_skip begins at the offset in that sg entry where data begins.
*/
struct scatterlist *src_sg;
int src_nents; /* Number of src entries with data */
u32 src_skip; /* bytes of current sg entry already used */
/*
* Same for destination. For AEAD, if there is AAD, output data must
* be written at offset following AAD.
*/
struct scatterlist *dst_sg;
int dst_nents; /* Number of dst entries with data */
u32 dst_skip; /* bytes of current sg entry already written */
/* Mailbox message used to send this request to PDC driver */
struct brcm_message mb_mssg;
bool bd_suppress; /* suppress BD field in SPU response? */
/* cipher context */
bool is_encrypt;
/*
* CBC mode: IV. CTR mode: counter. Else empty. Used as a DMA
* buffer for AEAD requests. So allocate as DMAable memory. If IV
* concatenated with salt, includes the salt.
*/
u8 *iv_ctr;
/* Length of IV or counter, in bytes */
unsigned int iv_ctr_len;
/*
* Hash requests can be of any size, whether initial, update, or final.
* A non-final request must be submitted to the SPU as an integral
* number of blocks. This may leave data at the end of the request
* that is not a full block. Since the request is non-final, it cannot
* be padded. So, we write the remainder to this hash_carry buffer and
* hold it until the next request arrives. The carry data is then
* submitted at the beginning of the data in the next SPU msg.
* hash_carry_len is the number of bytes currently in hash_carry. These
* fields are only used for ahash requests.
*/
u8 hash_carry[HASH_CARRY_MAX];
unsigned int hash_carry_len;
unsigned int is_final; /* is this the final for the hash op? */
/*
* Digest from incremental hash is saved here to include in next hash
* operation. Cannot be stored in req->result for truncated hashes,
* since result may be sized for final digest. Cannot be saved in
* msg_buf because that gets deleted between incremental hash ops
* and is not saved as part of export().
*/
u8 incr_hash[MAX_DIGEST_SIZE];
/* hmac context */
bool is_sw_hmac;
/* aead context */
struct crypto_tfm *old_tfm;
crypto_completion_t old_complete;
void *old_data;
gfp_t gfp;
/* Buffers used to build SPU request and response messages */
struct spu_msg_buf msg_buf;
};
/*
* Structure encapsulates a set of function pointers specific to the type of
* SPU hardware running. These functions handling creation and parsing of
* SPU request messages and SPU response messages. Includes hardware-specific
* values read from device tree.
*/
struct spu_hw {
void (*spu_dump_msg_hdr)(u8 *buf, unsigned int buf_len);
u32 (*spu_ctx_max_payload)(enum spu_cipher_alg cipher_alg,
enum spu_cipher_mode cipher_mode,
unsigned int blocksize);
u32 (*spu_payload_length)(u8 *spu_hdr);
u16 (*spu_response_hdr_len)(u16 auth_key_len, u16 enc_key_len,
bool is_hash);
u16 (*spu_hash_pad_len)(enum hash_alg hash_alg,
enum hash_mode hash_mode, u32 chunksize,
u16 hash_block_size);
u32 (*spu_gcm_ccm_pad_len)(enum spu_cipher_mode cipher_mode,
unsigned int data_size);
u32 (*spu_assoc_resp_len)(enum spu_cipher_mode cipher_mode,
unsigned int assoc_len,
unsigned int iv_len, bool is_encrypt);
u8 (*spu_aead_ivlen)(enum spu_cipher_mode cipher_mode,
u16 iv_len);
enum hash_type (*spu_hash_type)(u32 src_sent);
u32 (*spu_digest_size)(u32 digest_size, enum hash_alg alg,
enum hash_type);
u32 (*spu_create_request)(u8 *spu_hdr,
struct spu_request_opts *req_opts,
struct spu_cipher_parms *cipher_parms,
struct spu_hash_parms *hash_parms,
struct spu_aead_parms *aead_parms,
unsigned int data_size);
u16 (*spu_cipher_req_init)(u8 *spu_hdr,
struct spu_cipher_parms *cipher_parms);
void (*spu_cipher_req_finish)(u8 *spu_hdr,
u16 spu_req_hdr_len,
unsigned int is_inbound,
struct spu_cipher_parms *cipher_parms,
bool update_key,
unsigned int data_size);
void (*spu_request_pad)(u8 *pad_start, u32 gcm_padding,
u32 hash_pad_len, enum hash_alg auth_alg,
enum hash_mode auth_mode,
unsigned int total_sent, u32 status_padding);
u8 (*spu_xts_tweak_in_payload)(void);
u8 (*spu_tx_status_len)(void);
u8 (*spu_rx_status_len)(void);
int (*spu_status_process)(u8 *statp);
void (*spu_ccm_update_iv)(unsigned int digestsize,
struct spu_cipher_parms *cipher_parms,
unsigned int assoclen, unsigned int chunksize,
bool is_encrypt, bool is_esp);
u32 (*spu_wordalign_padlen)(u32 data_size);
/* The base virtual address of the SPU hw registers */
void __iomem *reg_vbase[MAX_SPUS];
/* Version of the SPU hardware */
enum spu_spu_type spu_type;
/* Sub-version of the SPU hardware */
enum spu_spu_subtype spu_subtype;
/* The number of SPUs on this platform */
u32 num_spu;
};
struct device_private {
struct platform_device *pdev[MAX_SPUS];
struct spu_hw spu;
atomic_t session_count; /* number of streams active */
atomic_t stream_count; /* monotonic counter for streamID's */
/* Length of BCM header. Set to 0 when hw does not expect BCM HEADER. */
u8 bcm_hdr_len;
/* The index of the channel to use for the next crypto request */
atomic_t next_chan;
struct dentry *debugfs_dir;
struct dentry *debugfs_stats;
/* Number of request bytes processed and result bytes returned */
atomic64_t bytes_in;
atomic64_t bytes_out;
/* Number of operations of each type */
atomic_t op_counts[SPU_OP_NUM];
atomic_t cipher_cnt[CIPHER_ALG_LAST][CIPHER_MODE_LAST];
atomic_t hash_cnt[HASH_ALG_LAST];
atomic_t hmac_cnt[HASH_ALG_LAST];
atomic_t aead_cnt[AEAD_TYPE_LAST];
/* Number of calls to setkey() for each operation type */
atomic_t setkey_cnt[SPU_OP_NUM];
/* Number of times request was resubmitted because mb was full */
atomic_t mb_no_spc;
/* Number of mailbox send failures */
atomic_t mb_send_fail;
/* Number of ICV check failures for AEAD messages */
atomic_t bad_icv;
struct mbox_client mcl[MAX_SPUS];
/* Array of mailbox channel pointers, one for each channel */
struct mbox_chan *mbox[MAX_SPUS];
/* Driver initialized */
bool inited;
};
extern struct device_private iproc_priv;
#endif
/*
* Copyright 2016 Broadcom
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License, version 2, as
* published by the Free Software Foundation (the "GPL").
*
* This program is distributed in the hope that it will be useful, but
* WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* General Public License version 2 (GPLv2) for more details.
*
* You should have received a copy of the GNU General Public License
* version 2 (GPLv2) along with this source code.
*/
#include <linux/kernel.h>
#include <linux/string.h>
#include "util.h"
#include "spu.h"
#include "spum.h"
#include "cipher.h"
/* This array is based on the hash algo type supported in spu.h */
char *tag_to_hash_idx[] = { "none", "md5", "sha1", "sha224", "sha256" };
char *hash_alg_name[] = { "None", "md5", "sha1", "sha224", "sha256", "aes",
"sha384", "sha512", "sha3_224", "sha3_256", "sha3_384", "sha3_512" };
char *aead_alg_name[] = { "ccm(aes)", "gcm(aes)", "authenc" };
/* Assumes SPU-M messages are in big endian */
void spum_dump_msg_hdr(u8 *buf, unsigned int buf_len)
{
u8 *ptr = buf;
struct SPUHEADER *spuh = (struct SPUHEADER *)buf;
unsigned int hash_key_len = 0;
unsigned int hash_state_len = 0;
unsigned int cipher_key_len = 0;
unsigned int iv_len;
u32 pflags;
u32 cflags;
u32 ecf;
u32 cipher_alg;
u32 cipher_mode;
u32 cipher_type;
u32 hash_alg;
u32 hash_mode;
u32 hash_type;
u32 sctx_size; /* SCTX length in words */
u32 sctx_pl_len; /* SCTX payload length in bytes */
packet_log("\n");
packet_log("SPU Message header %p len: %u\n", buf, buf_len);
/* ========== Decode MH ========== */
packet_log(" MH 0x%08x\n", be32_to_cpu(*((u32 *)ptr)));
if (spuh->mh.flags & MH_SCTX_PRES)
packet_log(" SCTX present\n");
if (spuh->mh.flags & MH_BDESC_PRES)
packet_log(" BDESC present\n");
if (spuh->mh.flags & MH_MFM_PRES)
packet_log(" MFM present\n");
if (spuh->mh.flags & MH_BD_PRES)
packet_log(" BD present\n");
if (spuh->mh.flags & MH_HASH_PRES)
packet_log(" HASH present\n");
if (spuh->mh.flags & MH_SUPDT_PRES)
packet_log(" SUPDT present\n");
packet_log(" Opcode 0x%02x\n", spuh->mh.op_code);
ptr += sizeof(spuh->mh) + sizeof(spuh->emh); /* skip emh. unused */
/* ========== Decode SCTX ========== */
if (spuh->mh.flags & MH_SCTX_PRES) {
pflags = be32_to_cpu(spuh->sa.proto_flags);
packet_log(" SCTX[0] 0x%08x\n", pflags);
sctx_size = pflags & SCTX_SIZE;
packet_log(" Size %u words\n", sctx_size);
cflags = be32_to_cpu(spuh->sa.cipher_flags);
packet_log(" SCTX[1] 0x%08x\n", cflags);
packet_log(" Inbound:%lu (1:decrypt/vrfy 0:encrypt/auth)\n",
(cflags & CIPHER_INBOUND) >> CIPHER_INBOUND_SHIFT);
packet_log(" Order:%lu (1:AuthFirst 0:EncFirst)\n",
(cflags & CIPHER_ORDER) >> CIPHER_ORDER_SHIFT);
packet_log(" ICV_IS_512:%lx\n",
(cflags & ICV_IS_512) >> ICV_IS_512_SHIFT);
cipher_alg = (cflags & CIPHER_ALG) >> CIPHER_ALG_SHIFT;
cipher_mode = (cflags & CIPHER_MODE) >> CIPHER_MODE_SHIFT;
cipher_type = (cflags & CIPHER_TYPE) >> CIPHER_TYPE_SHIFT;
packet_log(" Crypto Alg:%u Mode:%u Type:%u\n",
cipher_alg, cipher_mode, cipher_type);
hash_alg = (cflags & HASH_ALG) >> HASH_ALG_SHIFT;
hash_mode = (cflags & HASH_MODE) >> HASH_MODE_SHIFT;
hash_type = (cflags & HASH_TYPE) >> HASH_TYPE_SHIFT;
packet_log(" Hash Alg:%x Mode:%x Type:%x\n",
hash_alg, hash_mode, hash_type);
packet_log(" UPDT_Offset:%u\n", cflags & UPDT_OFST);
ecf = be32_to_cpu(spuh->sa.ecf);
packet_log(" SCTX[2] 0x%08x\n", ecf);
packet_log(" WriteICV:%lu CheckICV:%lu ICV_SIZE:%u ",
(ecf & INSERT_ICV) >> INSERT_ICV_SHIFT,
(ecf & CHECK_ICV) >> CHECK_ICV_SHIFT,
(ecf & ICV_SIZE) >> ICV_SIZE_SHIFT);
packet_log("BD_SUPPRESS:%lu\n",
(ecf & BD_SUPPRESS) >> BD_SUPPRESS_SHIFT);
packet_log(" SCTX_IV:%lu ExplicitIV:%lu GenIV:%lu ",
(ecf & SCTX_IV) >> SCTX_IV_SHIFT,
(ecf & EXPLICIT_IV) >> EXPLICIT_IV_SHIFT,
(ecf & GEN_IV) >> GEN_IV_SHIFT);
packet_log("IV_OV_OFST:%lu EXP_IV_SIZE:%u\n",
(ecf & IV_OFFSET) >> IV_OFFSET_SHIFT,
ecf & EXP_IV_SIZE);
ptr += sizeof(struct SCTX);
if (hash_alg && hash_mode) {
char *name = "NONE";
switch (hash_alg) {
case HASH_ALG_MD5:
hash_key_len = 16;
name = "MD5";
break;
case HASH_ALG_SHA1:
hash_key_len = 20;
name = "SHA1";
break;
case HASH_ALG_SHA224:
hash_key_len = 28;
name = "SHA224";
break;
case HASH_ALG_SHA256:
hash_key_len = 32;
name = "SHA256";
break;
case HASH_ALG_SHA384:
hash_key_len = 48;
name = "SHA384";
break;
case HASH_ALG_SHA512:
hash_key_len = 64;
name = "SHA512";
break;
case HASH_ALG_AES:
hash_key_len = 0;
name = "AES";
break;
case HASH_ALG_NONE:
break;
}
packet_log(" Auth Key Type:%s Length:%u Bytes\n",
name, hash_key_len);
packet_dump(" KEY: ", ptr, hash_key_len);
ptr += hash_key_len;
} else if ((hash_alg == HASH_ALG_AES) &&
(hash_mode == HASH_MODE_XCBC)) {
char *name = "NONE";
switch (cipher_type) {
case CIPHER_TYPE_AES128:
hash_key_len = 16;
name = "AES128-XCBC";
break;
case CIPHER_TYPE_AES192:
hash_key_len = 24;
name = "AES192-XCBC";
break;
case CIPHER_TYPE_AES256:
hash_key_len = 32;
name = "AES256-XCBC";
break;
}
packet_log(" Auth Key Type:%s Length:%u Bytes\n",
name, hash_key_len);
packet_dump(" KEY: ", ptr, hash_key_len);
ptr += hash_key_len;
}
if (hash_alg && (hash_mode == HASH_MODE_NONE) &&
(hash_type == HASH_TYPE_UPDT)) {
char *name = "NONE";
switch (hash_alg) {
case HASH_ALG_MD5:
hash_state_len = 16;
name = "MD5";
break;
case HASH_ALG_SHA1:
hash_state_len = 20;
name = "SHA1";
break;
case HASH_ALG_SHA224:
hash_state_len = 32;
name = "SHA224";
break;
case HASH_ALG_SHA256:
hash_state_len = 32;
name = "SHA256";
break;
case HASH_ALG_SHA384:
hash_state_len = 48;
name = "SHA384";
break;
case HASH_ALG_SHA512:
hash_state_len = 64;
name = "SHA512";
break;
case HASH_ALG_AES:
hash_state_len = 0;
name = "AES";
break;
case HASH_ALG_NONE:
break;
}
packet_log(" Auth State Type:%s Length:%u Bytes\n",
name, hash_state_len);
packet_dump(" State: ", ptr, hash_state_len);
ptr += hash_state_len;
}
if (cipher_alg) {
char *name = "NONE";
switch (cipher_alg) {
case CIPHER_ALG_DES:
cipher_key_len = 8;
name = "DES";
break;
case CIPHER_ALG_3DES:
cipher_key_len = 24;
name = "3DES";
break;
case CIPHER_ALG_RC4:
cipher_key_len = 260;
name = "ARC4";
break;
case CIPHER_ALG_AES:
switch (cipher_type) {
case CIPHER_TYPE_AES128:
cipher_key_len = 16;
name = "AES128";
break;
case CIPHER_TYPE_AES192:
cipher_key_len = 24;
name = "AES192";
break;
case CIPHER_TYPE_AES256:
cipher_key_len = 32;
name = "AES256";
break;
}
break;
case CIPHER_ALG_NONE:
break;
}
packet_log(" Cipher Key Type:%s Length:%u Bytes\n",
name, cipher_key_len);
/* XTS has two keys */
if (cipher_mode == CIPHER_MODE_XTS) {
packet_dump(" KEY2: ", ptr, cipher_key_len);
ptr += cipher_key_len;
packet_dump(" KEY1: ", ptr, cipher_key_len);
ptr += cipher_key_len;
cipher_key_len *= 2;
} else {
packet_dump(" KEY: ", ptr, cipher_key_len);
ptr += cipher_key_len;
}
if (ecf & SCTX_IV) {
sctx_pl_len = sctx_size * sizeof(u32) -
sizeof(struct SCTX);
iv_len = sctx_pl_len -
(hash_key_len + hash_state_len +
cipher_key_len);
packet_log(" IV Length:%u Bytes\n", iv_len);
packet_dump(" IV: ", ptr, iv_len);
ptr += iv_len;
}
}
}
/* ========== Decode BDESC ========== */
if (spuh->mh.flags & MH_BDESC_PRES) {
#ifdef DEBUG
struct BDESC_HEADER *bdesc = (struct BDESC_HEADER *)ptr;
#endif
packet_log(" BDESC[0] 0x%08x\n", be32_to_cpu(*((u32 *)ptr)));
packet_log(" OffsetMAC:%u LengthMAC:%u\n",
be16_to_cpu(bdesc->offset_mac),
be16_to_cpu(bdesc->length_mac));
ptr += sizeof(u32);
packet_log(" BDESC[1] 0x%08x\n", be32_to_cpu(*((u32 *)ptr)));
packet_log(" OffsetCrypto:%u LengthCrypto:%u\n",
be16_to_cpu(bdesc->offset_crypto),
be16_to_cpu(bdesc->length_crypto));
ptr += sizeof(u32);
packet_log(" BDESC[2] 0x%08x\n", be32_to_cpu(*((u32 *)ptr)));
packet_log(" OffsetICV:%u OffsetIV:%u\n",
be16_to_cpu(bdesc->offset_icv),
be16_to_cpu(bdesc->offset_iv));
ptr += sizeof(u32);
}
/* ========== Decode BD ========== */
if (spuh->mh.flags & MH_BD_PRES) {
#ifdef DEBUG
struct BD_HEADER *bd = (struct BD_HEADER *)ptr;
#endif
packet_log(" BD[0] 0x%08x\n", be32_to_cpu(*((u32 *)ptr)));
packet_log(" Size:%ubytes PrevLength:%u\n",
be16_to_cpu(bd->size), be16_to_cpu(bd->prev_length));
ptr += 4;
}
/* Double check sanity */
if (buf + buf_len != ptr) {
packet_log(" Packet parsed incorrectly. ");
packet_log("buf:%p buf_len:%u buf+buf_len:%p ptr:%p\n",
buf, buf_len, buf + buf_len, ptr);
}
packet_log("\n");
}
/**
* spum_ns2_ctx_max_payload() - Determine the max length of the payload for a
* SPU message for a given cipher and hash alg context.
* @cipher_alg: The cipher algorithm
* @cipher_mode: The cipher mode
* @blocksize: The size of a block of data for this algo
*
* The max payload must be a multiple of the blocksize so that if a request is
* too large to fit in a single SPU message, the request can be broken into
* max_payload sized chunks. Each chunk must be a multiple of blocksize.
*
* Return: Max payload length in bytes
*/
u32 spum_ns2_ctx_max_payload(enum spu_cipher_alg cipher_alg,
enum spu_cipher_mode cipher_mode,
unsigned int blocksize)
{
u32 max_payload = SPUM_NS2_MAX_PAYLOAD;
u32 excess;
/* In XTS on SPU-M, we'll need to insert tweak before input data */
if (cipher_mode == CIPHER_MODE_XTS)
max_payload -= SPU_XTS_TWEAK_SIZE;
excess = max_payload % blocksize;
return max_payload - excess;
}
/**
* spum_nsp_ctx_max_payload() - Determine the max length of the payload for a
* SPU message for a given cipher and hash alg context.
* @cipher_alg: The cipher algorithm
* @cipher_mode: The cipher mode
* @blocksize: The size of a block of data for this algo
*
* The max payload must be a multiple of the blocksize so that if a request is
* too large to fit in a single SPU message, the request can be broken into
* max_payload sized chunks. Each chunk must be a multiple of blocksize.
*
* Return: Max payload length in bytes
*/
u32 spum_nsp_ctx_max_payload(enum spu_cipher_alg cipher_alg,
enum spu_cipher_mode cipher_mode,
unsigned int blocksize)
{
u32 max_payload = SPUM_NSP_MAX_PAYLOAD;
u32 excess;
/* In XTS on SPU-M, we'll need to insert tweak before input data */
if (cipher_mode == CIPHER_MODE_XTS)
max_payload -= SPU_XTS_TWEAK_SIZE;
excess = max_payload % blocksize;
return max_payload - excess;
}
/** spum_payload_length() - Given a SPU-M message header, extract the payload
* length.
* @spu_hdr: Start of SPU header
*
* Assumes just MH, EMH, BD (no SCTX, BDESC. Works for response frames.
*
* Return: payload length in bytes
*/
u32 spum_payload_length(u8 *spu_hdr)
{
struct BD_HEADER *bd;
u32 pl_len;
/* Find BD header. skip MH, EMH */
bd = (struct BD_HEADER *)(spu_hdr + 8);
pl_len = be16_to_cpu(bd->size);
return pl_len;
}
/**
* spum_response_hdr_len() - Given the length of the hash key and encryption
* key, determine the expected length of a SPU response header.
* @auth_key_len: authentication key length (bytes)
* @enc_key_len: encryption key length (bytes)
* @is_hash: true if response message is for a hash operation
*
* Return: length of SPU response header (bytes)
*/
u16 spum_response_hdr_len(u16 auth_key_len, u16 enc_key_len, bool is_hash)
{
if (is_hash)
return SPU_HASH_RESP_HDR_LEN;
else
return SPU_RESP_HDR_LEN;
}
/**
* spum_hash_pad_len() - Calculate the length of hash padding required to extend
* data to a full block size.
* @hash_alg: hash algorithm
* @hash_mode: hash mode
* @chunksize: length of data, in bytes
* @hash_block_size: size of a block of data for hash algorithm
*
* Reserve space for 1 byte (0x80) start of pad and the total length as u64
*
* Return: length of hash pad in bytes
*/
u16 spum_hash_pad_len(enum hash_alg hash_alg, enum hash_mode hash_mode,
u32 chunksize, u16 hash_block_size)
{
unsigned int length_len;
unsigned int used_space_last_block;
int hash_pad_len;
/* AES-XCBC hash requires just padding to next block boundary */
if ((hash_alg == HASH_ALG_AES) && (hash_mode == HASH_MODE_XCBC)) {
used_space_last_block = chunksize % hash_block_size;
hash_pad_len = hash_block_size - used_space_last_block;
if (hash_pad_len >= hash_block_size)
hash_pad_len -= hash_block_size;
return hash_pad_len;
}
used_space_last_block = chunksize % hash_block_size + 1;
if ((hash_alg == HASH_ALG_SHA384) || (hash_alg == HASH_ALG_SHA512))
length_len = 2 * sizeof(u64);
else
length_len = sizeof(u64);
used_space_last_block += length_len;
hash_pad_len = hash_block_size - used_space_last_block;
if (hash_pad_len < 0)
hash_pad_len += hash_block_size;
hash_pad_len += 1 + length_len;
return hash_pad_len;
}
/**
* spum_gcm_ccm_pad_len() - Determine the required length of GCM or CCM padding.
* @cipher_mode: Algo type
* @data_size: Length of plaintext (bytes)
*
* @Return: Length of padding, in bytes
*/
u32 spum_gcm_ccm_pad_len(enum spu_cipher_mode cipher_mode,
unsigned int data_size)
{
u32 pad_len = 0;
u32 m1 = SPU_GCM_CCM_ALIGN - 1;
if ((cipher_mode == CIPHER_MODE_GCM) ||
(cipher_mode == CIPHER_MODE_CCM))
pad_len = ((data_size + m1) & ~m1) - data_size;
return pad_len;
}
/**
* spum_assoc_resp_len() - Determine the size of the receive buffer required to
* catch associated data.
* @cipher_mode: cipher mode
* @assoc_len: length of associated data (bytes)
* @iv_len: length of IV (bytes)
* @is_encrypt: true if encrypting. false if decrypting.
*
* Return: length of associated data in response message (bytes)
*/
u32 spum_assoc_resp_len(enum spu_cipher_mode cipher_mode,
unsigned int assoc_len, unsigned int iv_len,
bool is_encrypt)
{
u32 buflen = 0;
u32 pad;
if (assoc_len)
buflen = assoc_len;
if (cipher_mode == CIPHER_MODE_GCM) {
/* AAD needs to be padded in responses too */
pad = spum_gcm_ccm_pad_len(cipher_mode, buflen);
buflen += pad;
}
if (cipher_mode == CIPHER_MODE_CCM) {
/*
* AAD needs to be padded in responses too
* for CCM, len + 2 needs to be 128-bit aligned.
*/
pad = spum_gcm_ccm_pad_len(cipher_mode, buflen + 2);
buflen += pad;
}
return buflen;
}
/**
* spu_aead_ivlen() - Calculate the length of the AEAD IV to be included
* in a SPU request after the AAD and before the payload.
* @cipher_mode: cipher mode
* @iv_ctr_len: initialization vector length in bytes
*
* In Linux ~4.2 and later, the assoc_data sg includes the IV. So no need
* to include the IV as a separate field in the SPU request msg.
*
* Return: Length of AEAD IV in bytes
*/
u8 spum_aead_ivlen(enum spu_cipher_mode cipher_mode, u16 iv_len)
{
return 0;
}
/**
* spum_hash_type() - Determine the type of hash operation.
* @src_sent: The number of bytes in the current request that have already
* been sent to the SPU to be hashed.
*
* We do not use HASH_TYPE_FULL for requests that fit in a single SPU message.
* Using FULL causes failures (such as when the string to be hashed is empty).
* For similar reasons, we never use HASH_TYPE_FIN. Instead, submit messages
* as INIT or UPDT and do the hash padding in sw.
*/
enum hash_type spum_hash_type(u32 src_sent)
{
return src_sent ? HASH_TYPE_UPDT : HASH_TYPE_INIT;
}
/**
* spum_digest_size() - Determine the size of a hash digest to expect the SPU to
* return.
* alg_digest_size: Number of bytes in the final digest for the given algo
* alg: The hash algorithm
* htype: Type of hash operation (init, update, full, etc)
*
* When doing incremental hashing for an algorithm with a truncated hash
* (e.g., SHA224), the SPU returns the full digest so that it can be fed back as
* a partial result for the next chunk.
*/
u32 spum_digest_size(u32 alg_digest_size, enum hash_alg alg,
enum hash_type htype)
{
u32 digestsize = alg_digest_size;
/* SPU returns complete digest when doing incremental hash and truncated
* hash algo.
*/
if ((htype == HASH_TYPE_INIT) || (htype == HASH_TYPE_UPDT)) {
if (alg == HASH_ALG_SHA224)
digestsize = SHA256_DIGEST_SIZE;
else if (alg == HASH_ALG_SHA384)
digestsize = SHA512_DIGEST_SIZE;
}
return digestsize;
}
/**
* spum_create_request() - Build a SPU request message header, up to and
* including the BD header. Construct the message starting at spu_hdr. Caller
* should allocate this buffer in DMA-able memory at least SPU_HEADER_ALLOC_LEN
* bytes long.
* @spu_hdr: Start of buffer where SPU request header is to be written
* @req_opts: SPU request message options
* @cipher_parms: Parameters related to cipher algorithm
* @hash_parms: Parameters related to hash algorithm
* @aead_parms: Parameters related to AEAD operation
* @data_size: Length of data to be encrypted or authenticated. If AEAD, does
* not include length of AAD.
* Return: the length of the SPU header in bytes. 0 if an error occurs.
*/
u32 spum_create_request(u8 *spu_hdr,
struct spu_request_opts *req_opts,
struct spu_cipher_parms *cipher_parms,
struct spu_hash_parms *hash_parms,
struct spu_aead_parms *aead_parms,
unsigned int data_size)
{
struct SPUHEADER *spuh;
struct BDESC_HEADER *bdesc;
struct BD_HEADER *bd;
u8 *ptr;
u32 protocol_bits = 0;
u32 cipher_bits = 0;
u32 ecf_bits = 0;
u8 sctx_words = 0;
unsigned int buf_len = 0;
/* size of the cipher payload */
unsigned int cipher_len = hash_parms->prebuf_len + data_size +
hash_parms->pad_len;
/* offset of prebuf or data from end of BD header */
unsigned int cipher_offset = aead_parms->assoc_size +
aead_parms->iv_len + aead_parms->aad_pad_len;
/* total size of the DB data (without STAT word padding) */
unsigned int real_db_size = spu_real_db_size(aead_parms->assoc_size,
aead_parms->iv_len,
hash_parms->prebuf_len,
data_size,
aead_parms->aad_pad_len,
aead_parms->data_pad_len,
hash_parms->pad_len);
unsigned int auth_offset = 0;
unsigned int offset_iv = 0;
/* size/offset of the auth payload */
unsigned int auth_len;
auth_len = real_db_size;
if (req_opts->is_aead && req_opts->is_inbound)
cipher_len -= hash_parms->digestsize;
if (req_opts->is_aead && req_opts->is_inbound)
auth_len -= hash_parms->digestsize;
if ((hash_parms->alg == HASH_ALG_AES) &&
(hash_parms->mode == HASH_MODE_XCBC)) {
auth_len -= hash_parms->pad_len;
cipher_len -= hash_parms->pad_len;
}
flow_log("%s()\n", __func__);
flow_log(" in:%u authFirst:%u\n",
req_opts->is_inbound, req_opts->auth_first);
flow_log(" %s. cipher alg:%u mode:%u type %u\n",
spu_alg_name(cipher_parms->alg, cipher_parms->mode),
cipher_parms->alg, cipher_parms->mode, cipher_parms->type);
flow_log(" key: %d\n", cipher_parms->key_len);
flow_dump(" key: ", cipher_parms->key_buf, cipher_parms->key_len);
flow_log(" iv: %d\n", cipher_parms->iv_len);
flow_dump(" iv: ", cipher_parms->iv_buf, cipher_parms->iv_len);
flow_log(" auth alg:%u mode:%u type %u\n",
hash_parms->alg, hash_parms->mode, hash_parms->type);
flow_log(" digestsize: %u\n", hash_parms->digestsize);
flow_log(" authkey: %d\n", hash_parms->key_len);
flow_dump(" authkey: ", hash_parms->key_buf, hash_parms->key_len);
flow_log(" assoc_size:%u\n", aead_parms->assoc_size);
flow_log(" prebuf_len:%u\n", hash_parms->prebuf_len);
flow_log(" data_size:%u\n", data_size);
flow_log(" hash_pad_len:%u\n", hash_parms->pad_len);
flow_log(" real_db_size:%u\n", real_db_size);
flow_log(" auth_offset:%u auth_len:%u cipher_offset:%u cipher_len:%u\n",
auth_offset, auth_len, cipher_offset, cipher_len);
flow_log(" aead_iv: %u\n", aead_parms->iv_len);
/* starting out: zero the header (plus some) */
ptr = spu_hdr;
memset(ptr, 0, sizeof(struct SPUHEADER));
/* format master header word */
/* Do not set the next bit even though the datasheet says to */
spuh = (struct SPUHEADER *)ptr;
ptr += sizeof(struct SPUHEADER);
buf_len += sizeof(struct SPUHEADER);
spuh->mh.op_code = SPU_CRYPTO_OPERATION_GENERIC;
spuh->mh.flags |= (MH_SCTX_PRES | MH_BDESC_PRES | MH_BD_PRES);
/* Format sctx word 0 (protocol_bits) */
sctx_words = 3; /* size in words */
/* Format sctx word 1 (cipher_bits) */
if (req_opts->is_inbound)
cipher_bits |= CIPHER_INBOUND;
if (req_opts->auth_first)
cipher_bits |= CIPHER_ORDER;
/* Set the crypto parameters in the cipher.flags */
cipher_bits |= cipher_parms->alg << CIPHER_ALG_SHIFT;
cipher_bits |= cipher_parms->mode << CIPHER_MODE_SHIFT;
cipher_bits |= cipher_parms->type << CIPHER_TYPE_SHIFT;
/* Set the auth parameters in the cipher.flags */
cipher_bits |= hash_parms->alg << HASH_ALG_SHIFT;
cipher_bits |= hash_parms->mode << HASH_MODE_SHIFT;
cipher_bits |= hash_parms->type << HASH_TYPE_SHIFT;
/*
* Format sctx extensions if required, and update main fields if
* required)
*/
if (hash_parms->alg) {
/* Write the authentication key material if present */
if (hash_parms->key_len) {
memcpy(ptr, hash_parms->key_buf, hash_parms->key_len);
ptr += hash_parms->key_len;
buf_len += hash_parms->key_len;
sctx_words += hash_parms->key_len / 4;
}
if ((cipher_parms->mode == CIPHER_MODE_GCM) ||
(cipher_parms->mode == CIPHER_MODE_CCM))
/* unpadded length */
offset_iv = aead_parms->assoc_size;
/* if GCM/CCM we need to write ICV into the payload */
if (!req_opts->is_inbound) {
if ((cipher_parms->mode == CIPHER_MODE_GCM) ||
(cipher_parms->mode == CIPHER_MODE_CCM))
ecf_bits |= 1 << INSERT_ICV_SHIFT;
} else {
ecf_bits |= CHECK_ICV;
}
/* Inform the SPU of the ICV size (in words) */
if (hash_parms->digestsize == 64)
cipher_bits |= ICV_IS_512;
else
ecf_bits |=
(hash_parms->digestsize / 4) << ICV_SIZE_SHIFT;
}
if (req_opts->bd_suppress)
ecf_bits |= BD_SUPPRESS;
/* copy the encryption keys in the SAD entry */
if (cipher_parms->alg) {
if (cipher_parms->key_len) {
memcpy(ptr, cipher_parms->key_buf,
cipher_parms->key_len);
ptr += cipher_parms->key_len;
buf_len += cipher_parms->key_len;
sctx_words += cipher_parms->key_len / 4;
}
/*
* if encrypting then set IV size, use SCTX IV unless no IV
* given here
*/
if (cipher_parms->iv_buf && cipher_parms->iv_len) {
/* Use SCTX IV */
ecf_bits |= SCTX_IV;
/* cipher iv provided so put it in here */
memcpy(ptr, cipher_parms->iv_buf, cipher_parms->iv_len);
ptr += cipher_parms->iv_len;
buf_len += cipher_parms->iv_len;
sctx_words += cipher_parms->iv_len / 4;
}
}
/*
* RFC4543 (GMAC/ESP) requires data to be sent as part of AAD
* so we need to override the BDESC parameters.
*/
if (req_opts->is_rfc4543) {
if (req_opts->is_inbound)
data_size -= hash_parms->digestsize;
offset_iv = aead_parms->assoc_size + data_size;
cipher_len = 0;
cipher_offset = offset_iv;
auth_len = cipher_offset + aead_parms->data_pad_len;
}
/* write in the total sctx length now that we know it */
protocol_bits |= sctx_words;
/* Endian adjust the SCTX */
spuh->sa.proto_flags = cpu_to_be32(protocol_bits);
spuh->sa.cipher_flags = cpu_to_be32(cipher_bits);
spuh->sa.ecf = cpu_to_be32(ecf_bits);
/* === create the BDESC section === */
bdesc = (struct BDESC_HEADER *)ptr;
bdesc->offset_mac = cpu_to_be16(auth_offset);
bdesc->length_mac = cpu_to_be16(auth_len);
bdesc->offset_crypto = cpu_to_be16(cipher_offset);
bdesc->length_crypto = cpu_to_be16(cipher_len);
/*
* CCM in SPU-M requires that ICV not be in same 32-bit word as data or
* padding. So account for padding as necessary.
*/
if (cipher_parms->mode == CIPHER_MODE_CCM)
auth_len += spum_wordalign_padlen(auth_len);
bdesc->offset_icv = cpu_to_be16(auth_len);
bdesc->offset_iv = cpu_to_be16(offset_iv);
ptr += sizeof(struct BDESC_HEADER);
buf_len += sizeof(struct BDESC_HEADER);
/* === no MFM section === */
/* === create the BD section === */
/* add the BD header */
bd = (struct BD_HEADER *)ptr;
bd->size = cpu_to_be16(real_db_size);
bd->prev_length = 0;
ptr += sizeof(struct BD_HEADER);
buf_len += sizeof(struct BD_HEADER);
packet_dump(" SPU request header: ", spu_hdr, buf_len);
return buf_len;
}
/**
* spum_cipher_req_init() - Build a SPU request message header, up to and
* including the BD header.
* @spu_hdr: Start of SPU request header (MH)
* @cipher_parms: Parameters that describe the cipher request
*
* Construct the message starting at spu_hdr. Caller should allocate this buffer
* in DMA-able memory at least SPU_HEADER_ALLOC_LEN bytes long.
*
* Return: the length of the SPU header in bytes. 0 if an error occurs.
*/
u16 spum_cipher_req_init(u8 *spu_hdr, struct spu_cipher_parms *cipher_parms)
{
struct SPUHEADER *spuh;
u32 protocol_bits = 0;
u32 cipher_bits = 0;
u32 ecf_bits = 0;
u8 sctx_words = 0;
u8 *ptr = spu_hdr;
flow_log("%s()\n", __func__);
flow_log(" cipher alg:%u mode:%u type %u\n", cipher_parms->alg,
cipher_parms->mode, cipher_parms->type);
flow_log(" cipher_iv_len: %u\n", cipher_parms->iv_len);
flow_log(" key: %d\n", cipher_parms->key_len);
flow_dump(" key: ", cipher_parms->key_buf, cipher_parms->key_len);
/* starting out: zero the header (plus some) */
memset(spu_hdr, 0, sizeof(struct SPUHEADER));
ptr += sizeof(struct SPUHEADER);
/* format master header word */
/* Do not set the next bit even though the datasheet says to */
spuh = (struct SPUHEADER *)spu_hdr;
spuh->mh.op_code = SPU_CRYPTO_OPERATION_GENERIC;
spuh->mh.flags |= (MH_SCTX_PRES | MH_BDESC_PRES | MH_BD_PRES);
/* Format sctx word 0 (protocol_bits) */
sctx_words = 3; /* size in words */
/* copy the encryption keys in the SAD entry */
if (cipher_parms->alg) {
if (cipher_parms->key_len) {
ptr += cipher_parms->key_len;
sctx_words += cipher_parms->key_len / 4;
}
/*
* if encrypting then set IV size, use SCTX IV unless no IV
* given here
*/
if (cipher_parms->iv_len) {
/* Use SCTX IV */
ecf_bits |= SCTX_IV;
ptr += cipher_parms->iv_len;
sctx_words += cipher_parms->iv_len / 4;
}
}
/* Set the crypto parameters in the cipher.flags */
cipher_bits |= cipher_parms->alg << CIPHER_ALG_SHIFT;
cipher_bits |= cipher_parms->mode << CIPHER_MODE_SHIFT;
cipher_bits |= cipher_parms->type << CIPHER_TYPE_SHIFT;
/* copy the encryption keys in the SAD entry */
if (cipher_parms->alg && cipher_parms->key_len)
memcpy(spuh + 1, cipher_parms->key_buf, cipher_parms->key_len);
/* write in the total sctx length now that we know it */
protocol_bits |= sctx_words;
/* Endian adjust the SCTX */
spuh->sa.proto_flags = cpu_to_be32(protocol_bits);
/* Endian adjust the SCTX */
spuh->sa.cipher_flags = cpu_to_be32(cipher_bits);
spuh->sa.ecf = cpu_to_be32(ecf_bits);
packet_dump(" SPU request header: ", spu_hdr,
sizeof(struct SPUHEADER));
return sizeof(struct SPUHEADER) + cipher_parms->key_len +
cipher_parms->iv_len + sizeof(struct BDESC_HEADER) +
sizeof(struct BD_HEADER);
}
/**
* spum_cipher_req_finish() - Finish building a SPU request message header for a
* block cipher request. Assumes much of the header was already filled in at
* setkey() time in spu_cipher_req_init().
* @spu_hdr: Start of the request message header (MH field)
* @spu_req_hdr_len: Length in bytes of the SPU request header
* @isInbound: 0 encrypt, 1 decrypt
* @cipher_parms: Parameters describing cipher operation to be performed
* @update_key: If true, rewrite the cipher key in SCTX
* @data_size: Length of the data in the BD field
*
* Assumes much of the header was already filled in at setkey() time in
* spum_cipher_req_init().
* spum_cipher_req_init() fills in the encryption key. For RC4, when submitting
* a request for a non-first chunk, we use the 260-byte SUPDT field from the
* previous response as the key. update_key is true for this case. Unused in all
* other cases.
*/
void spum_cipher_req_finish(u8 *spu_hdr,
u16 spu_req_hdr_len,
unsigned int is_inbound,
struct spu_cipher_parms *cipher_parms,
bool update_key,
unsigned int data_size)
{
struct SPUHEADER *spuh;
struct BDESC_HEADER *bdesc;
struct BD_HEADER *bd;
u8 *bdesc_ptr = spu_hdr + spu_req_hdr_len -
(sizeof(struct BD_HEADER) + sizeof(struct BDESC_HEADER));
u32 cipher_bits;
flow_log("%s()\n", __func__);
flow_log(" in: %u\n", is_inbound);
flow_log(" cipher alg: %u, cipher_type: %u\n", cipher_parms->alg,
cipher_parms->type);
if (update_key) {
flow_log(" cipher key len: %u\n", cipher_parms->key_len);
flow_dump(" key: ", cipher_parms->key_buf,
cipher_parms->key_len);
}
/*
* In XTS mode, API puts "i" parameter (block tweak) in IV. For
* SPU-M, should be in start of the BD; tx_sg_create() copies it there.
* IV in SPU msg for SPU-M should be 0, since that's the "j" parameter
* (block ctr within larger data unit) - given we can send entire disk
* block (<= 4KB) in 1 SPU msg, don't need to use this parameter.
*/
if (cipher_parms->mode == CIPHER_MODE_XTS)
memset(cipher_parms->iv_buf, 0, cipher_parms->iv_len);
flow_log(" iv len: %d\n", cipher_parms->iv_len);
flow_dump(" iv: ", cipher_parms->iv_buf, cipher_parms->iv_len);
flow_log(" data_size: %u\n", data_size);
/* format master header word */
/* Do not set the next bit even though the datasheet says to */
spuh = (struct SPUHEADER *)spu_hdr;
/* cipher_bits was initialized at setkey time */
cipher_bits = be32_to_cpu(spuh->sa.cipher_flags);
/* Format sctx word 1 (cipher_bits) */
if (is_inbound)
cipher_bits |= CIPHER_INBOUND;
else
cipher_bits &= ~CIPHER_INBOUND;
/* update encryption key for RC4 on non-first chunk */
if (update_key) {
spuh->sa.cipher_flags |=
cipher_parms->type << CIPHER_TYPE_SHIFT;
memcpy(spuh + 1, cipher_parms->key_buf, cipher_parms->key_len);
}
if (cipher_parms->alg && cipher_parms->iv_buf && cipher_parms->iv_len)
/* cipher iv provided so put it in here */
memcpy(bdesc_ptr - cipher_parms->iv_len, cipher_parms->iv_buf,
cipher_parms->iv_len);
spuh->sa.cipher_flags = cpu_to_be32(cipher_bits);
/* === create the BDESC section === */
bdesc = (struct BDESC_HEADER *)bdesc_ptr;
bdesc->offset_mac = 0;
bdesc->length_mac = 0;
bdesc->offset_crypto = 0;
/* XTS mode, data_size needs to include tweak parameter */
if (cipher_parms->mode == CIPHER_MODE_XTS)
bdesc->length_crypto = cpu_to_be16(data_size +
SPU_XTS_TWEAK_SIZE);
else
bdesc->length_crypto = cpu_to_be16(data_size);
bdesc->offset_icv = 0;
bdesc->offset_iv = 0;
/* === no MFM section === */
/* === create the BD section === */
/* add the BD header */
bd = (struct BD_HEADER *)(bdesc_ptr + sizeof(struct BDESC_HEADER));
bd->size = cpu_to_be16(data_size);
/* XTS mode, data_size needs to include tweak parameter */
if (cipher_parms->mode == CIPHER_MODE_XTS)
bd->size = cpu_to_be16(data_size + SPU_XTS_TWEAK_SIZE);
else
bd->size = cpu_to_be16(data_size);
bd->prev_length = 0;
packet_dump(" SPU request header: ", spu_hdr, spu_req_hdr_len);
}
/**
* spum_request_pad() - Create pad bytes at the end of the data.
* @pad_start: Start of buffer where pad bytes are to be written
* @gcm_ccm_padding: length of GCM/CCM padding, in bytes
* @hash_pad_len: Number of bytes of padding extend data to full block
* @auth_alg: authentication algorithm
* @auth_mode: authentication mode
* @total_sent: length inserted at end of hash pad
* @status_padding: Number of bytes of padding to align STATUS word
*
* There may be three forms of pad:
* 1. GCM/CCM pad - for GCM/CCM mode ciphers, pad to 16-byte alignment
* 2. hash pad - pad to a block length, with 0x80 data terminator and
* size at the end
* 3. STAT pad - to ensure the STAT field is 4-byte aligned
*/
void spum_request_pad(u8 *pad_start,
u32 gcm_ccm_padding,
u32 hash_pad_len,
enum hash_alg auth_alg,
enum hash_mode auth_mode,
unsigned int total_sent, u32 status_padding)
{
u8 *ptr = pad_start;
/* fix data alignent for GCM/CCM */
if (gcm_ccm_padding > 0) {
flow_log(" GCM: padding to 16 byte alignment: %u bytes\n",
gcm_ccm_padding);
memset(ptr, 0, gcm_ccm_padding);
ptr += gcm_ccm_padding;
}
if (hash_pad_len > 0) {
/* clear the padding section */
memset(ptr, 0, hash_pad_len);
if ((auth_alg == HASH_ALG_AES) &&
(auth_mode == HASH_MODE_XCBC)) {
/* AES/XCBC just requires padding to be 0s */
ptr += hash_pad_len;
} else {
/* terminate the data */
*ptr = 0x80;
ptr += (hash_pad_len - sizeof(u64));
/* add the size at the end as required per alg */
if (auth_alg == HASH_ALG_MD5)
*(u64 *)ptr = cpu_to_le64((u64)total_sent * 8);
else /* SHA1, SHA2-224, SHA2-256 */
*(u64 *)ptr = cpu_to_be64((u64)total_sent * 8);
ptr += sizeof(u64);
}
}
/* pad to a 4byte alignment for STAT */
if (status_padding > 0) {
flow_log(" STAT: padding to 4 byte alignment: %u bytes\n",
status_padding);
memset(ptr, 0, status_padding);
ptr += status_padding;
}
}
/**
* spum_xts_tweak_in_payload() - Indicate that SPUM DOES place the XTS tweak
* field in the packet payload (rather than using IV)
*
* Return: 1
*/
u8 spum_xts_tweak_in_payload(void)
{
return 1;
}
/**
* spum_tx_status_len() - Return the length of the STATUS field in a SPU
* response message.
*
* Return: Length of STATUS field in bytes.
*/
u8 spum_tx_status_len(void)
{
return SPU_TX_STATUS_LEN;
}
/**
* spum_rx_status_len() - Return the length of the STATUS field in a SPU
* response message.
*
* Return: Length of STATUS field in bytes.
*/
u8 spum_rx_status_len(void)
{
return SPU_RX_STATUS_LEN;
}
/**
* spum_status_process() - Process the status from a SPU response message.
* @statp: start of STATUS word
* Return:
* 0 - if status is good and response should be processed
* !0 - status indicates an error and response is invalid
*/
int spum_status_process(u8 *statp)
{
u32 status;
status = __be32_to_cpu(*(__be32 *)statp);
flow_log("SPU response STATUS %#08x\n", status);
if (status & SPU_STATUS_ERROR_FLAG) {
pr_err("%s() Warning: Error result from SPU: %#08x\n",
__func__, status);
if (status & SPU_STATUS_INVALID_ICV)
return SPU_INVALID_ICV;
return -EBADMSG;
}
return 0;
}
/**
* spum_ccm_update_iv() - Update the IV as per the requirements for CCM mode.
*
* @digestsize: Digest size of this request
* @cipher_parms: (pointer to) cipher parmaeters, includes IV buf & IV len
* @assoclen: Length of AAD data
* @chunksize: length of input data to be sent in this req
* @is_encrypt: true if this is an output/encrypt operation
* @is_esp: true if this is an ESP / RFC4309 operation
*
*/
void spum_ccm_update_iv(unsigned int digestsize,
struct spu_cipher_parms *cipher_parms,
unsigned int assoclen,
unsigned int chunksize,
bool is_encrypt,
bool is_esp)
{
u8 L; /* L from CCM algorithm, length of plaintext data */
u8 mprime; /* M' from CCM algo, (M - 2) / 2, where M=authsize */
u8 adata;
if (cipher_parms->iv_len != CCM_AES_IV_SIZE) {
pr_err("%s(): Invalid IV len %d for CCM mode, should be %d\n",
__func__, cipher_parms->iv_len, CCM_AES_IV_SIZE);
return;
}
/*
* IV needs to be formatted as follows:
*
* | Byte 0 | Bytes 1 - N | Bytes (N+1) - 15 |
* | 7 | 6 | 5 | 4 | 3 | 2 | 1 | 0 | Bits 7 - 0 | Bits 7 - 0 |
* | 0 |Ad?|(M - 2) / 2| L - 1 | Nonce | Plaintext Length |
*
* Ad? = 1 if AAD present, 0 if not present
* M = size of auth field, 8, 12, or 16 bytes (SPU-M) -or-
* 4, 6, 8, 10, 12, 14, 16 bytes (SPU2)
* L = Size of Plaintext Length field; Nonce size = 15 - L
*
* It appears that the crypto API already expects the L-1 portion
* to be set in the first byte of the IV, which implicitly determines
* the nonce size, and also fills in the nonce. But the other bits
* in byte 0 as well as the plaintext length need to be filled in.
*
* In rfc4309/esp mode, L is not already in the supplied IV and
* we need to fill it in, as well as move the IV data to be after
* the salt
*/
if (is_esp) {
L = CCM_ESP_L_VALUE; /* RFC4309 has fixed L */
} else {
/* L' = plaintext length - 1 so Plaintext length is L' + 1 */
L = ((cipher_parms->iv_buf[0] & CCM_B0_L_PRIME) >>
CCM_B0_L_PRIME_SHIFT) + 1;
}
mprime = (digestsize - 2) >> 1; /* M' = (M - 2) / 2 */
adata = (assoclen > 0); /* adata = 1 if any associated data */
cipher_parms->iv_buf[0] = (adata << CCM_B0_ADATA_SHIFT) |
(mprime << CCM_B0_M_PRIME_SHIFT) |
((L - 1) << CCM_B0_L_PRIME_SHIFT);
/* Nonce is already filled in by crypto API, and is 15 - L bytes */
/* Don't include digest in plaintext size when decrypting */
if (!is_encrypt)
chunksize -= digestsize;
/* Fill in length of plaintext, formatted to be L bytes long */
format_value_ccm(chunksize, &cipher_parms->iv_buf[15 - L + 1], L);
}
/**
* spum_wordalign_padlen() - Given the length of a data field, determine the
* padding required to align the data following this field on a 4-byte boundary.
* @data_size: length of data field in bytes
*
* Return: length of status field padding, in bytes
*/
u32 spum_wordalign_padlen(u32 data_size)
{
return ((data_size + 3) & ~3) - data_size;
}
/*
* Copyright 2016 Broadcom
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License, version 2, as
* published by the Free Software Foundation (the "GPL").
*
* This program is distributed in the hope that it will be useful, but
* WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* General Public License version 2 (GPLv2) for more details.
*
* You should have received a copy of the GNU General Public License
* version 2 (GPLv2) along with this source code.
*/
/*
* This file contains the definition of SPU messages. There are currently two
* SPU message formats: SPU-M and SPU2. The hardware uses different values to
* identify the same things in SPU-M vs SPU2. So this file defines values that
* are hardware independent. Software can use these values for any version of
* SPU hardware. These values are used in APIs in spu.c. Functions internal to
* spu.c and spu2.c convert these to hardware-specific values.
*/
#ifndef _SPU_H
#define _SPU_H
#include <linux/types.h>
#include <linux/scatterlist.h>
#include <crypto/sha.h>
enum spu_cipher_alg {
CIPHER_ALG_NONE = 0x0,
CIPHER_ALG_RC4 = 0x1,
CIPHER_ALG_DES = 0x2,
CIPHER_ALG_3DES = 0x3,
CIPHER_ALG_AES = 0x4,
CIPHER_ALG_LAST = 0x5
};
enum spu_cipher_mode {
CIPHER_MODE_NONE = 0x0,
CIPHER_MODE_ECB = 0x0,
CIPHER_MODE_CBC = 0x1,
CIPHER_MODE_OFB = 0x2,
CIPHER_MODE_CFB = 0x3,
CIPHER_MODE_CTR = 0x4,
CIPHER_MODE_CCM = 0x5,
CIPHER_MODE_GCM = 0x6,
CIPHER_MODE_XTS = 0x7,
CIPHER_MODE_LAST = 0x8
};
enum spu_cipher_type {
CIPHER_TYPE_NONE = 0x0,
CIPHER_TYPE_DES = 0x0,
CIPHER_TYPE_3DES = 0x0,
CIPHER_TYPE_INIT = 0x0, /* used for ARC4 */
CIPHER_TYPE_AES128 = 0x0,
CIPHER_TYPE_AES192 = 0x1,
CIPHER_TYPE_UPDT = 0x1, /* used for ARC4 */
CIPHER_TYPE_AES256 = 0x2,
};
enum hash_alg {
HASH_ALG_NONE = 0x0,
HASH_ALG_MD5 = 0x1,
HASH_ALG_SHA1 = 0x2,
HASH_ALG_SHA224 = 0x3,
HASH_ALG_SHA256 = 0x4,
HASH_ALG_AES = 0x5,
HASH_ALG_SHA384 = 0x6,
HASH_ALG_SHA512 = 0x7,
/* Keep SHA3 algorithms at the end always */
HASH_ALG_SHA3_224 = 0x8,
HASH_ALG_SHA3_256 = 0x9,
HASH_ALG_SHA3_384 = 0xa,
HASH_ALG_SHA3_512 = 0xb,
HASH_ALG_LAST
};
enum hash_mode {
HASH_MODE_NONE = 0x0,
HASH_MODE_HASH = 0x0,
HASH_MODE_XCBC = 0x0,
HASH_MODE_CMAC = 0x1,
HASH_MODE_CTXT = 0x1,
HASH_MODE_HMAC = 0x2,
HASH_MODE_RABIN = 0x4,
HASH_MODE_FHMAC = 0x6,
HASH_MODE_CCM = 0x5,
HASH_MODE_GCM = 0x6,
};
enum hash_type {
HASH_TYPE_NONE = 0x0,
HASH_TYPE_FULL = 0x0,
HASH_TYPE_INIT = 0x1,
HASH_TYPE_UPDT = 0x2,
HASH_TYPE_FIN = 0x3,
HASH_TYPE_AES128 = 0x0,
HASH_TYPE_AES192 = 0x1,
HASH_TYPE_AES256 = 0x2
};
enum aead_type {
AES_CCM,
AES_GCM,
AUTHENC,
AEAD_TYPE_LAST
};
extern char *hash_alg_name[HASH_ALG_LAST];
extern char *aead_alg_name[AEAD_TYPE_LAST];
struct spu_request_opts {
bool is_inbound;
bool auth_first;
bool is_aead;
bool is_esp;
bool bd_suppress;
bool is_rfc4543;
};
struct spu_cipher_parms {
enum spu_cipher_alg alg;
enum spu_cipher_mode mode;
enum spu_cipher_type type;
u8 *key_buf;
u16 key_len;
/* iv_buf and iv_len include salt, if applicable */
u8 *iv_buf;
u16 iv_len;
};
struct spu_hash_parms {
enum hash_alg alg;
enum hash_mode mode;
enum hash_type type;
u8 digestsize;
u8 *key_buf;
u16 key_len;
u16 prebuf_len;
/* length of hash pad. signed, needs to handle roll-overs */
int pad_len;
};
struct spu_aead_parms {
u32 assoc_size;
u16 iv_len; /* length of IV field between assoc data and data */
u8 aad_pad_len; /* For AES GCM/CCM, length of padding after AAD */
u8 data_pad_len;/* For AES GCM/CCM, length of padding after data */
bool return_iv; /* True if SPU should return an IV */
u32 ret_iv_len; /* Length in bytes of returned IV */
u32 ret_iv_off; /* Offset into full IV if partial IV returned */
};
/************** SPU sizes ***************/
#define SPU_RX_STATUS_LEN 4
/* Max length of padding for 4-byte alignment of STATUS field */
#define SPU_STAT_PAD_MAX 4
/* Max length of pad fragment. 4 is for 4-byte alignment of STATUS field */
#define SPU_PAD_LEN_MAX (SPU_GCM_CCM_ALIGN + MAX_HASH_BLOCK_SIZE + \
SPU_STAT_PAD_MAX)
/* GCM and CCM require 16-byte alignment */
#define SPU_GCM_CCM_ALIGN 16
/* Length up SUPDT field in SPU response message for RC4 */
#define SPU_SUPDT_LEN 260
/* SPU status error codes. These used as common error codes across all
* SPU variants.
*/
#define SPU_INVALID_ICV 1
/* Indicates no limit to the length of the payload in a SPU message */
#define SPU_MAX_PAYLOAD_INF 0xFFFFFFFF
/* Size of XTS tweak ("i" parameter), in bytes */
#define SPU_XTS_TWEAK_SIZE 16
/* CCM B_0 field definitions, common for SPU-M and SPU2 */
#define CCM_B0_ADATA 0x40
#define CCM_B0_ADATA_SHIFT 6
#define CCM_B0_M_PRIME 0x38
#define CCM_B0_M_PRIME_SHIFT 3
#define CCM_B0_L_PRIME 0x07
#define CCM_B0_L_PRIME_SHIFT 0
#define CCM_ESP_L_VALUE 4
/**
* spu_req_incl_icv() - Return true if SPU request message should include the
* ICV as a separate buffer.
* @cipher_mode: the cipher mode being requested
* @is_encrypt: true if encrypting. false if decrypting.
*
* Return: true if ICV to be included as separate buffer
*/
static __always_inline bool spu_req_incl_icv(enum spu_cipher_mode cipher_mode,
bool is_encrypt)
{
if ((cipher_mode == CIPHER_MODE_GCM) && !is_encrypt)
return true;
if ((cipher_mode == CIPHER_MODE_CCM) && !is_encrypt)
return true;
return false;
}
static __always_inline u32 spu_real_db_size(u32 assoc_size,
u32 aead_iv_buf_len,
u32 prebuf_len,
u32 data_size,
u32 aad_pad_len,
u32 gcm_pad_len,
u32 hash_pad_len)
{
return assoc_size + aead_iv_buf_len + prebuf_len + data_size +
aad_pad_len + gcm_pad_len + hash_pad_len;
}
/************** SPU Functions Prototypes **************/
void spum_dump_msg_hdr(u8 *buf, unsigned int buf_len);
u32 spum_ns2_ctx_max_payload(enum spu_cipher_alg cipher_alg,
enum spu_cipher_mode cipher_mode,
unsigned int blocksize);
u32 spum_nsp_ctx_max_payload(enum spu_cipher_alg cipher_alg,
enum spu_cipher_mode cipher_mode,
unsigned int blocksize);
u32 spum_payload_length(u8 *spu_hdr);
u16 spum_response_hdr_len(u16 auth_key_len, u16 enc_key_len, bool is_hash);
u16 spum_hash_pad_len(enum hash_alg hash_alg, enum hash_mode hash_mode,
u32 chunksize, u16 hash_block_size);
u32 spum_gcm_ccm_pad_len(enum spu_cipher_mode cipher_mode,
unsigned int data_size);
u32 spum_assoc_resp_len(enum spu_cipher_mode cipher_mode,
unsigned int assoc_len, unsigned int iv_len,
bool is_encrypt);
u8 spum_aead_ivlen(enum spu_cipher_mode cipher_mode, u16 iv_len);
bool spu_req_incl_icv(enum spu_cipher_mode cipher_mode, bool is_encrypt);
enum hash_type spum_hash_type(u32 src_sent);
u32 spum_digest_size(u32 alg_digest_size, enum hash_alg alg,
enum hash_type htype);
u32 spum_create_request(u8 *spu_hdr,
struct spu_request_opts *req_opts,
struct spu_cipher_parms *cipher_parms,
struct spu_hash_parms *hash_parms,
struct spu_aead_parms *aead_parms,
unsigned int data_size);
u16 spum_cipher_req_init(u8 *spu_hdr, struct spu_cipher_parms *cipher_parms);
void spum_cipher_req_finish(u8 *spu_hdr,
u16 spu_req_hdr_len,
unsigned int is_inbound,
struct spu_cipher_parms *cipher_parms,
bool update_key,
unsigned int data_size);
void spum_request_pad(u8 *pad_start,
u32 gcm_padding,
u32 hash_pad_len,
enum hash_alg auth_alg,
enum hash_mode auth_mode,
unsigned int total_sent, u32 status_padding);
u8 spum_xts_tweak_in_payload(void);
u8 spum_tx_status_len(void);
u8 spum_rx_status_len(void);
int spum_status_process(u8 *statp);
void spum_ccm_update_iv(unsigned int digestsize,
struct spu_cipher_parms *cipher_parms,
unsigned int assoclen,
unsigned int chunksize,
bool is_encrypt,
bool is_esp);
u32 spum_wordalign_padlen(u32 data_size);
#endif
/*
* Copyright 2016 Broadcom
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License, version 2, as
* published by the Free Software Foundation (the "GPL").
*
* This program is distributed in the hope that it will be useful, but
* WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* General Public License version 2 (GPLv2) for more details.
*
* You should have received a copy of the GNU General Public License
* version 2 (GPLv2) along with this source code.
*/
/*
* This file works with the SPU2 version of the SPU. SPU2 has different message
* formats than the previous version of the SPU. All SPU message format
* differences should be hidden in the spux.c,h files.
*/
#include <linux/kernel.h>
#include <linux/string.h>
#include "util.h"
#include "spu.h"
#include "spu2.h"
#define SPU2_TX_STATUS_LEN 0 /* SPU2 has no STATUS in input packet */
/*
* Controlled by pkt_stat_cnt field in CRYPTO_SS_SPU0_CORE_SPU2_CONTROL0
* register. Defaults to 2.
*/
#define SPU2_RX_STATUS_LEN 2
enum spu2_proto_sel {
SPU2_PROTO_RESV = 0,
SPU2_MACSEC_SECTAG8_ECB = 1,
SPU2_MACSEC_SECTAG8_SCB = 2,
SPU2_MACSEC_SECTAG16 = 3,
SPU2_MACSEC_SECTAG16_8_XPN = 4,
SPU2_IPSEC = 5,
SPU2_IPSEC_ESN = 6,
SPU2_TLS_CIPHER = 7,
SPU2_TLS_AEAD = 8,
SPU2_DTLS_CIPHER = 9,
SPU2_DTLS_AEAD = 10
};
char *spu2_cipher_type_names[] = { "None", "AES128", "AES192", "AES256",
"DES", "3DES"
};
char *spu2_cipher_mode_names[] = { "ECB", "CBC", "CTR", "CFB", "OFB", "XTS",
"CCM", "GCM"
};
char *spu2_hash_type_names[] = { "None", "AES128", "AES192", "AES256",
"Reserved", "Reserved", "MD5", "SHA1", "SHA224", "SHA256", "SHA384",
"SHA512", "SHA512/224", "SHA512/256", "SHA3-224", "SHA3-256",
"SHA3-384", "SHA3-512"
};
char *spu2_hash_mode_names[] = { "CMAC", "CBC-MAC", "XCBC-MAC", "HMAC",
"Rabin", "CCM", "GCM", "Reserved"
};
static char *spu2_ciph_type_name(enum spu2_cipher_type cipher_type)
{
if (cipher_type >= SPU2_CIPHER_TYPE_LAST)
return "Reserved";
return spu2_cipher_type_names[cipher_type];
}
static char *spu2_ciph_mode_name(enum spu2_cipher_mode cipher_mode)
{
if (cipher_mode >= SPU2_CIPHER_MODE_LAST)
return "Reserved";
return spu2_cipher_mode_names[cipher_mode];
}
static char *spu2_hash_type_name(enum spu2_hash_type hash_type)
{
if (hash_type >= SPU2_HASH_TYPE_LAST)
return "Reserved";
return spu2_hash_type_names[hash_type];
}
static char *spu2_hash_mode_name(enum spu2_hash_mode hash_mode)
{
if (hash_mode >= SPU2_HASH_MODE_LAST)
return "Reserved";
return spu2_hash_mode_names[hash_mode];
}
/*
* Convert from a software cipher mode value to the corresponding value
* for SPU2.
*/
static int spu2_cipher_mode_xlate(enum spu_cipher_mode cipher_mode,
enum spu2_cipher_mode *spu2_mode)
{
switch (cipher_mode) {
case CIPHER_MODE_ECB:
*spu2_mode = SPU2_CIPHER_MODE_ECB;
break;
case CIPHER_MODE_CBC:
*spu2_mode = SPU2_CIPHER_MODE_CBC;
break;
case CIPHER_MODE_OFB:
*spu2_mode = SPU2_CIPHER_MODE_OFB;
break;
case CIPHER_MODE_CFB:
*spu2_mode = SPU2_CIPHER_MODE_CFB;
break;
case CIPHER_MODE_CTR:
*spu2_mode = SPU2_CIPHER_MODE_CTR;
break;
case CIPHER_MODE_CCM:
*spu2_mode = SPU2_CIPHER_MODE_CCM;
break;
case CIPHER_MODE_GCM:
*spu2_mode = SPU2_CIPHER_MODE_GCM;
break;
case CIPHER_MODE_XTS:
*spu2_mode = SPU2_CIPHER_MODE_XTS;
break;
default:
return -EINVAL;
}
return 0;
}
/**
* spu2_cipher_xlate() - Convert a cipher {alg/mode/type} triple to a SPU2
* cipher type and mode.
* @cipher_alg: [in] cipher algorithm value from software enumeration
* @cipher_mode: [in] cipher mode value from software enumeration
* @cipher_type: [in] cipher type value from software enumeration
* @spu2_type: [out] cipher type value used by spu2 hardware
* @spu2_mode: [out] cipher mode value used by spu2 hardware
*
* Return: 0 if successful
*/
static int spu2_cipher_xlate(enum spu_cipher_alg cipher_alg,
enum spu_cipher_mode cipher_mode,
enum spu_cipher_type cipher_type,
enum spu2_cipher_type *spu2_type,
enum spu2_cipher_mode *spu2_mode)
{
int err;
err = spu2_cipher_mode_xlate(cipher_mode, spu2_mode);
if (err) {
flow_log("Invalid cipher mode %d\n", cipher_mode);
return err;
}
switch (cipher_alg) {
case CIPHER_ALG_NONE:
*spu2_type = SPU2_CIPHER_TYPE_NONE;
break;
case CIPHER_ALG_RC4:
/* SPU2 does not support RC4 */
err = -EINVAL;
*spu2_type = SPU2_CIPHER_TYPE_NONE;
break;
case CIPHER_ALG_DES:
*spu2_type = SPU2_CIPHER_TYPE_DES;
break;
case CIPHER_ALG_3DES:
*spu2_type = SPU2_CIPHER_TYPE_3DES;
break;
case CIPHER_ALG_AES:
switch (cipher_type) {
case CIPHER_TYPE_AES128:
*spu2_type = SPU2_CIPHER_TYPE_AES128;
break;
case CIPHER_TYPE_AES192:
*spu2_type = SPU2_CIPHER_TYPE_AES192;
break;
case CIPHER_TYPE_AES256:
*spu2_type = SPU2_CIPHER_TYPE_AES256;
break;
default:
err = -EINVAL;
}
break;
case CIPHER_ALG_LAST:
default:
err = -EINVAL;
break;
}
if (err)
flow_log("Invalid cipher alg %d or type %d\n",
cipher_alg, cipher_type);
return err;
}
/*
* Convert from a software hash mode value to the corresponding value
* for SPU2. Note that HASH_MODE_NONE and HASH_MODE_XCBC have the same value.
*/
static int spu2_hash_mode_xlate(enum hash_mode hash_mode,
enum spu2_hash_mode *spu2_mode)
{
switch (hash_mode) {
case HASH_MODE_XCBC:
*spu2_mode = SPU2_HASH_MODE_XCBC_MAC;
break;
case HASH_MODE_CMAC:
*spu2_mode = SPU2_HASH_MODE_CMAC;
break;
case HASH_MODE_HMAC:
*spu2_mode = SPU2_HASH_MODE_HMAC;
break;
case HASH_MODE_CCM:
*spu2_mode = SPU2_HASH_MODE_CCM;
break;
case HASH_MODE_GCM:
*spu2_mode = SPU2_HASH_MODE_GCM;
break;
default:
return -EINVAL;
}
return 0;
}
/**
* spu2_hash_xlate() - Convert a hash {alg/mode/type} triple to a SPU2 hash type
* and mode.
* @hash_alg: [in] hash algorithm value from software enumeration
* @hash_mode: [in] hash mode value from software enumeration
* @hash_type: [in] hash type value from software enumeration
* @ciph_type: [in] cipher type value from software enumeration
* @spu2_type: [out] hash type value used by SPU2 hardware
* @spu2_mode: [out] hash mode value used by SPU2 hardware
*
* Return: 0 if successful
*/
static int
spu2_hash_xlate(enum hash_alg hash_alg, enum hash_mode hash_mode,
enum hash_type hash_type, enum spu_cipher_type ciph_type,
enum spu2_hash_type *spu2_type, enum spu2_hash_mode *spu2_mode)
{
int err;
err = spu2_hash_mode_xlate(hash_mode, spu2_mode);
if (err) {
flow_log("Invalid hash mode %d\n", hash_mode);
return err;
}
switch (hash_alg) {
case HASH_ALG_NONE:
*spu2_type = SPU2_HASH_TYPE_NONE;
break;
case HASH_ALG_MD5:
*spu2_type = SPU2_HASH_TYPE_MD5;
break;
case HASH_ALG_SHA1:
*spu2_type = SPU2_HASH_TYPE_SHA1;
break;
case HASH_ALG_SHA224:
*spu2_type = SPU2_HASH_TYPE_SHA224;
break;
case HASH_ALG_SHA256:
*spu2_type = SPU2_HASH_TYPE_SHA256;
break;
case HASH_ALG_SHA384:
*spu2_type = SPU2_HASH_TYPE_SHA384;
break;
case HASH_ALG_SHA512:
*spu2_type = SPU2_HASH_TYPE_SHA512;
break;
case HASH_ALG_AES:
switch (ciph_type) {
case CIPHER_TYPE_AES128:
*spu2_type = SPU2_HASH_TYPE_AES128;
break;
case CIPHER_TYPE_AES192:
*spu2_type = SPU2_HASH_TYPE_AES192;
break;
case CIPHER_TYPE_AES256:
*spu2_type = SPU2_HASH_TYPE_AES256;
break;
default:
err = -EINVAL;
}
break;
case HASH_ALG_SHA3_224:
*spu2_type = SPU2_HASH_TYPE_SHA3_224;
break;
case HASH_ALG_SHA3_256:
*spu2_type = SPU2_HASH_TYPE_SHA3_256;
break;
case HASH_ALG_SHA3_384:
*spu2_type = SPU2_HASH_TYPE_SHA3_384;
break;
case HASH_ALG_SHA3_512:
*spu2_type = SPU2_HASH_TYPE_SHA3_512;
case HASH_ALG_LAST:
default:
err = -EINVAL;
break;
}
if (err)
flow_log("Invalid hash alg %d or type %d\n",
hash_alg, hash_type);
return err;
}
/* Dump FMD ctrl0. The ctrl0 input is in host byte order */
static void spu2_dump_fmd_ctrl0(u64 ctrl0)
{
enum spu2_cipher_type ciph_type;
enum spu2_cipher_mode ciph_mode;
enum spu2_hash_type hash_type;
enum spu2_hash_mode hash_mode;
char *ciph_name;
char *ciph_mode_name;
char *hash_name;
char *hash_mode_name;
u8 cfb;
u8 proto;
packet_log(" FMD CTRL0 %#16llx\n", ctrl0);
if (ctrl0 & SPU2_CIPH_ENCRYPT_EN)
packet_log(" encrypt\n");
else
packet_log(" decrypt\n");
ciph_type = (ctrl0 & SPU2_CIPH_TYPE) >> SPU2_CIPH_TYPE_SHIFT;
ciph_name = spu2_ciph_type_name(ciph_type);
packet_log(" Cipher type: %s\n", ciph_name);
if (ciph_type != SPU2_CIPHER_TYPE_NONE) {
ciph_mode = (ctrl0 & SPU2_CIPH_MODE) >> SPU2_CIPH_MODE_SHIFT;
ciph_mode_name = spu2_ciph_mode_name(ciph_mode);
packet_log(" Cipher mode: %s\n", ciph_mode_name);
}
cfb = (ctrl0 & SPU2_CFB_MASK) >> SPU2_CFB_MASK_SHIFT;
packet_log(" CFB %#x\n", cfb);
proto = (ctrl0 & SPU2_PROTO_SEL) >> SPU2_PROTO_SEL_SHIFT;
packet_log(" protocol %#x\n", proto);
if (ctrl0 & SPU2_HASH_FIRST)
packet_log(" hash first\n");
else
packet_log(" cipher first\n");
if (ctrl0 & SPU2_CHK_TAG)
packet_log(" check tag\n");
hash_type = (ctrl0 & SPU2_HASH_TYPE) >> SPU2_HASH_TYPE_SHIFT;
hash_name = spu2_hash_type_name(hash_type);
packet_log(" Hash type: %s\n", hash_name);
if (hash_type != SPU2_HASH_TYPE_NONE) {
hash_mode = (ctrl0 & SPU2_HASH_MODE) >> SPU2_HASH_MODE_SHIFT;
hash_mode_name = spu2_hash_mode_name(hash_mode);
packet_log(" Hash mode: %s\n", hash_mode_name);
}
if (ctrl0 & SPU2_CIPH_PAD_EN) {
packet_log(" Cipher pad: %#2llx\n",
(ctrl0 & SPU2_CIPH_PAD) >> SPU2_CIPH_PAD_SHIFT);
}
}
/* Dump FMD ctrl1. The ctrl1 input is in host byte order */
static void spu2_dump_fmd_ctrl1(u64 ctrl1)
{
u8 hash_key_len;
u8 ciph_key_len;
u8 ret_iv_len;
u8 iv_offset;
u8 iv_len;
u8 hash_tag_len;
u8 ret_md;
packet_log(" FMD CTRL1 %#16llx\n", ctrl1);
if (ctrl1 & SPU2_TAG_LOC)
packet_log(" Tag after payload\n");
packet_log(" Msg includes ");
if (ctrl1 & SPU2_HAS_FR_DATA)
packet_log("FD ");
if (ctrl1 & SPU2_HAS_AAD1)
packet_log("AAD1 ");
if (ctrl1 & SPU2_HAS_NAAD)
packet_log("NAAD ");
if (ctrl1 & SPU2_HAS_AAD2)
packet_log("AAD2 ");
if (ctrl1 & SPU2_HAS_ESN)
packet_log("ESN ");
packet_log("\n");
hash_key_len = (ctrl1 & SPU2_HASH_KEY_LEN) >> SPU2_HASH_KEY_LEN_SHIFT;
packet_log(" Hash key len %u\n", hash_key_len);
ciph_key_len = (ctrl1 & SPU2_CIPH_KEY_LEN) >> SPU2_CIPH_KEY_LEN_SHIFT;
packet_log(" Cipher key len %u\n", ciph_key_len);
if (ctrl1 & SPU2_GENIV)
packet_log(" Generate IV\n");
if (ctrl1 & SPU2_HASH_IV)
packet_log(" IV included in hash\n");
if (ctrl1 & SPU2_RET_IV)
packet_log(" Return IV in output before payload\n");
ret_iv_len = (ctrl1 & SPU2_RET_IV_LEN) >> SPU2_RET_IV_LEN_SHIFT;
packet_log(" Length of returned IV %u bytes\n",
ret_iv_len ? ret_iv_len : 16);
iv_offset = (ctrl1 & SPU2_IV_OFFSET) >> SPU2_IV_OFFSET_SHIFT;
packet_log(" IV offset %u\n", iv_offset);
iv_len = (ctrl1 & SPU2_IV_LEN) >> SPU2_IV_LEN_SHIFT;
packet_log(" Input IV len %u bytes\n", iv_len);
hash_tag_len = (ctrl1 & SPU2_HASH_TAG_LEN) >> SPU2_HASH_TAG_LEN_SHIFT;
packet_log(" Hash tag length %u bytes\n", hash_tag_len);
packet_log(" Return ");
ret_md = (ctrl1 & SPU2_RETURN_MD) >> SPU2_RETURN_MD_SHIFT;
if (ret_md)
packet_log("FMD ");
if (ret_md == SPU2_RET_FMD_OMD)
packet_log("OMD ");
else if (ret_md == SPU2_RET_FMD_OMD_IV)
packet_log("OMD IV ");
if (ctrl1 & SPU2_RETURN_FD)
packet_log("FD ");
if (ctrl1 & SPU2_RETURN_AAD1)
packet_log("AAD1 ");
if (ctrl1 & SPU2_RETURN_NAAD)
packet_log("NAAD ");
if (ctrl1 & SPU2_RETURN_AAD2)
packet_log("AAD2 ");
if (ctrl1 & SPU2_RETURN_PAY)
packet_log("Payload");
packet_log("\n");
}
/* Dump FMD ctrl2. The ctrl2 input is in host byte order */
static void spu2_dump_fmd_ctrl2(u64 ctrl2)
{
packet_log(" FMD CTRL2 %#16llx\n", ctrl2);
packet_log(" AAD1 offset %llu length %llu bytes\n",
ctrl2 & SPU2_AAD1_OFFSET,
(ctrl2 & SPU2_AAD1_LEN) >> SPU2_AAD1_LEN_SHIFT);
packet_log(" AAD2 offset %llu\n",
(ctrl2 & SPU2_AAD2_OFFSET) >> SPU2_AAD2_OFFSET_SHIFT);
packet_log(" Payload offset %llu\n",
(ctrl2 & SPU2_PL_OFFSET) >> SPU2_PL_OFFSET_SHIFT);
}
/* Dump FMD ctrl3. The ctrl3 input is in host byte order */
static void spu2_dump_fmd_ctrl3(u64 ctrl3)
{
packet_log(" FMD CTRL3 %#16llx\n", ctrl3);
packet_log(" Payload length %llu bytes\n", ctrl3 & SPU2_PL_LEN);
packet_log(" TLS length %llu bytes\n",
(ctrl3 & SPU2_TLS_LEN) >> SPU2_TLS_LEN_SHIFT);
}
static void spu2_dump_fmd(struct SPU2_FMD *fmd)
{
spu2_dump_fmd_ctrl0(le64_to_cpu(fmd->ctrl0));
spu2_dump_fmd_ctrl1(le64_to_cpu(fmd->ctrl1));
spu2_dump_fmd_ctrl2(le64_to_cpu(fmd->ctrl2));
spu2_dump_fmd_ctrl3(le64_to_cpu(fmd->ctrl3));
}
static void spu2_dump_omd(u8 *omd, u16 hash_key_len, u16 ciph_key_len,
u16 hash_iv_len, u16 ciph_iv_len)
{
u8 *ptr = omd;
packet_log(" OMD:\n");
if (hash_key_len) {
packet_log(" Hash Key Length %u bytes\n", hash_key_len);
packet_dump(" KEY: ", ptr, hash_key_len);
ptr += hash_key_len;
}
if (ciph_key_len) {
packet_log(" Cipher Key Length %u bytes\n", ciph_key_len);
packet_dump(" KEY: ", ptr, ciph_key_len);
ptr += ciph_key_len;
}
if (hash_iv_len) {
packet_log(" Hash IV Length %u bytes\n", hash_iv_len);
packet_dump(" hash IV: ", ptr, hash_iv_len);
ptr += ciph_key_len;
}
if (ciph_iv_len) {
packet_log(" Cipher IV Length %u bytes\n", ciph_iv_len);
packet_dump(" cipher IV: ", ptr, ciph_iv_len);
}
}
/* Dump a SPU2 header for debug */
void spu2_dump_msg_hdr(u8 *buf, unsigned int buf_len)
{
struct SPU2_FMD *fmd = (struct SPU2_FMD *)buf;
u8 *omd;
u64 ctrl1;
u16 hash_key_len;
u16 ciph_key_len;
u16 hash_iv_len;
u16 ciph_iv_len;
u16 omd_len;
packet_log("\n");
packet_log("SPU2 message header %p len: %u\n", buf, buf_len);
spu2_dump_fmd(fmd);
omd = (u8 *)(fmd + 1);
ctrl1 = le64_to_cpu(fmd->ctrl1);
hash_key_len = (ctrl1 & SPU2_HASH_KEY_LEN) >> SPU2_HASH_KEY_LEN_SHIFT;
ciph_key_len = (ctrl1 & SPU2_CIPH_KEY_LEN) >> SPU2_CIPH_KEY_LEN_SHIFT;
hash_iv_len = 0;
ciph_iv_len = (ctrl1 & SPU2_IV_LEN) >> SPU2_IV_LEN_SHIFT;
spu2_dump_omd(omd, hash_key_len, ciph_key_len, hash_iv_len,
ciph_iv_len);
/* Double check sanity */
omd_len = hash_key_len + ciph_key_len + hash_iv_len + ciph_iv_len;
if (FMD_SIZE + omd_len != buf_len) {
packet_log
(" Packet parsed incorrectly. buf_len %u, sum of MD %zu\n",
buf_len, FMD_SIZE + omd_len);
}
packet_log("\n");
}
/**
* spu2_fmd_init() - At setkey time, initialize the fixed meta data for
* subsequent ablkcipher requests for this context.
* @spu2_cipher_type: Cipher algorithm
* @spu2_mode: Cipher mode
* @cipher_key_len: Length of cipher key, in bytes
* @cipher_iv_len: Length of cipher initialization vector, in bytes
*
* Return: 0 (success)
*/
static int spu2_fmd_init(struct SPU2_FMD *fmd,
enum spu2_cipher_type spu2_type,
enum spu2_cipher_mode spu2_mode,
u32 cipher_key_len, u32 cipher_iv_len)
{
u64 ctrl0;
u64 ctrl1;
u64 ctrl2;
u64 ctrl3;
u32 aad1_offset;
u32 aad2_offset;
u16 aad1_len = 0;
u64 payload_offset;
ctrl0 = (spu2_type << SPU2_CIPH_TYPE_SHIFT) |
(spu2_mode << SPU2_CIPH_MODE_SHIFT);
ctrl1 = (cipher_key_len << SPU2_CIPH_KEY_LEN_SHIFT) |
((u64)cipher_iv_len << SPU2_IV_LEN_SHIFT) |
((u64)SPU2_RET_FMD_ONLY << SPU2_RETURN_MD_SHIFT) | SPU2_RETURN_PAY;
/*
* AAD1 offset is from start of FD. FD length is always 0 for this
* driver. So AAD1_offset is always 0.
*/
aad1_offset = 0;
aad2_offset = aad1_offset;
payload_offset = 0;
ctrl2 = aad1_offset |
(aad1_len << SPU2_AAD1_LEN_SHIFT) |
(aad2_offset << SPU2_AAD2_OFFSET_SHIFT) |
(payload_offset << SPU2_PL_OFFSET_SHIFT);
ctrl3 = 0;
fmd->ctrl0 = cpu_to_le64(ctrl0);
fmd->ctrl1 = cpu_to_le64(ctrl1);
fmd->ctrl2 = cpu_to_le64(ctrl2);
fmd->ctrl3 = cpu_to_le64(ctrl3);
return 0;
}
/**
* spu2_fmd_ctrl0_write() - Write ctrl0 field in fixed metadata (FMD) field of
* SPU request packet.
* @fmd: Start of FMD field to be written
* @is_inbound: true if decrypting. false if encrypting.
* @authFirst: true if alg authenticates before encrypting
* @protocol: protocol selector
* @cipher_type: cipher algorithm
* @cipher_mode: cipher mode
* @auth_type: authentication type
* @auth_mode: authentication mode
*/
static void spu2_fmd_ctrl0_write(struct SPU2_FMD *fmd,
bool is_inbound, bool auth_first,
enum spu2_proto_sel protocol,
enum spu2_cipher_type cipher_type,
enum spu2_cipher_mode cipher_mode,
enum spu2_hash_type auth_type,
enum spu2_hash_mode auth_mode)
{
u64 ctrl0 = 0;
if ((cipher_type != SPU2_CIPHER_TYPE_NONE) && !is_inbound)
ctrl0 |= SPU2_CIPH_ENCRYPT_EN;
ctrl0 |= ((u64)cipher_type << SPU2_CIPH_TYPE_SHIFT) |
((u64)cipher_mode << SPU2_CIPH_MODE_SHIFT);
if (protocol)
ctrl0 |= (u64)protocol << SPU2_PROTO_SEL_SHIFT;
if (auth_first)
ctrl0 |= SPU2_HASH_FIRST;
if (is_inbound && (auth_type != SPU2_HASH_TYPE_NONE))
ctrl0 |= SPU2_CHK_TAG;
ctrl0 |= (((u64)auth_type << SPU2_HASH_TYPE_SHIFT) |
((u64)auth_mode << SPU2_HASH_MODE_SHIFT));
fmd->ctrl0 = cpu_to_le64(ctrl0);
}
/**
* spu2_fmd_ctrl1_write() - Write ctrl1 field in fixed metadata (FMD) field of
* SPU request packet.
* @fmd: Start of FMD field to be written
* @assoc_size: Length of additional associated data, in bytes
* @auth_key_len: Length of authentication key, in bytes
* @cipher_key_len: Length of cipher key, in bytes
* @gen_iv: If true, hw generates IV and returns in response
* @hash_iv: IV participates in hash. Used for IPSEC and TLS.
* @return_iv: Return IV in output packet before payload
* @ret_iv_len: Length of IV returned from SPU, in bytes
* @ret_iv_offset: Offset into full IV of start of returned IV
* @cipher_iv_len: Length of input cipher IV, in bytes
* @digest_size: Length of digest (aka, hash tag or ICV), in bytes
* @return_payload: Return payload in SPU response
* @return_md : return metadata in SPU response
*
* Packet can have AAD2 w/o AAD1. For algorithms currently supported,
* associated data goes in AAD2.
*/
static void spu2_fmd_ctrl1_write(struct SPU2_FMD *fmd, bool is_inbound,
u64 assoc_size,
u64 auth_key_len, u64 cipher_key_len,
bool gen_iv, bool hash_iv, bool return_iv,
u64 ret_iv_len, u64 ret_iv_offset,
u64 cipher_iv_len, u64 digest_size,
bool return_payload, bool return_md)
{
u64 ctrl1 = 0;
if (is_inbound && digest_size)
ctrl1 |= SPU2_TAG_LOC;
if (assoc_size) {
ctrl1 |= SPU2_HAS_AAD2;
ctrl1 |= SPU2_RETURN_AAD2; /* need aad2 for gcm aes esp */
}
if (auth_key_len)
ctrl1 |= ((auth_key_len << SPU2_HASH_KEY_LEN_SHIFT) &
SPU2_HASH_KEY_LEN);
if (cipher_key_len)
ctrl1 |= ((cipher_key_len << SPU2_CIPH_KEY_LEN_SHIFT) &
SPU2_CIPH_KEY_LEN);
if (gen_iv)
ctrl1 |= SPU2_GENIV;
if (hash_iv)
ctrl1 |= SPU2_HASH_IV;
if (return_iv) {
ctrl1 |= SPU2_RET_IV;
ctrl1 |= ret_iv_len << SPU2_RET_IV_LEN_SHIFT;
ctrl1 |= ret_iv_offset << SPU2_IV_OFFSET_SHIFT;
}
ctrl1 |= ((cipher_iv_len << SPU2_IV_LEN_SHIFT) & SPU2_IV_LEN);
if (digest_size)
ctrl1 |= ((digest_size << SPU2_HASH_TAG_LEN_SHIFT) &
SPU2_HASH_TAG_LEN);
/* Let's ask for the output pkt to include FMD, but don't need to
* get keys and IVs back in OMD.
*/
if (return_md)
ctrl1 |= ((u64)SPU2_RET_FMD_ONLY << SPU2_RETURN_MD_SHIFT);
else
ctrl1 |= ((u64)SPU2_RET_NO_MD << SPU2_RETURN_MD_SHIFT);
/* Crypto API does not get assoc data back. So no need for AAD2. */
if (return_payload)
ctrl1 |= SPU2_RETURN_PAY;
fmd->ctrl1 = cpu_to_le64(ctrl1);
}
/**
* spu2_fmd_ctrl2_write() - Set the ctrl2 field in the fixed metadata field of
* SPU2 header.
* @fmd: Start of FMD field to be written
* @cipher_offset: Number of bytes from Start of Packet (end of FD field) where
* data to be encrypted or decrypted begins
* @auth_key_len: Length of authentication key, in bytes
* @auth_iv_len: Length of authentication initialization vector, in bytes
* @cipher_key_len: Length of cipher key, in bytes
* @cipher_iv_len: Length of cipher IV, in bytes
*/
static void spu2_fmd_ctrl2_write(struct SPU2_FMD *fmd, u64 cipher_offset,
u64 auth_key_len, u64 auth_iv_len,
u64 cipher_key_len, u64 cipher_iv_len)
{
u64 ctrl2;
u64 aad1_offset;
u64 aad2_offset;
u16 aad1_len = 0;
u64 payload_offset;
/* AAD1 offset is from start of FD. FD length always 0. */
aad1_offset = 0;
aad2_offset = aad1_offset;
payload_offset = cipher_offset;
ctrl2 = aad1_offset |
(aad1_len << SPU2_AAD1_LEN_SHIFT) |
(aad2_offset << SPU2_AAD2_OFFSET_SHIFT) |
(payload_offset << SPU2_PL_OFFSET_SHIFT);
fmd->ctrl2 = cpu_to_le64(ctrl2);
}
/**
* spu2_fmd_ctrl3_write() - Set the ctrl3 field in FMD
* @fmd: Fixed meta data. First field in SPU2 msg header.
* @payload_len: Length of payload, in bytes
*/
static void spu2_fmd_ctrl3_write(struct SPU2_FMD *fmd, u64 payload_len)
{
u64 ctrl3;
ctrl3 = payload_len & SPU2_PL_LEN;
fmd->ctrl3 = cpu_to_le64(ctrl3);
}
/**
* spu2_ctx_max_payload() - Determine the maximum length of the payload for a
* SPU message for a given cipher and hash alg context.
* @cipher_alg: The cipher algorithm
* @cipher_mode: The cipher mode
* @blocksize: The size of a block of data for this algo
*
* For SPU2, the hardware generally ignores the PayloadLen field in ctrl3 of
* FMD and just keeps computing until it receives a DMA descriptor with the EOF
* flag set. So we consider the max payload to be infinite. AES CCM is an
* exception.
*
* Return: Max payload length in bytes
*/
u32 spu2_ctx_max_payload(enum spu_cipher_alg cipher_alg,
enum spu_cipher_mode cipher_mode,
unsigned int blocksize)
{
if ((cipher_alg == CIPHER_ALG_AES) &&
(cipher_mode == CIPHER_MODE_CCM)) {
u32 excess = SPU2_MAX_PAYLOAD % blocksize;
return SPU2_MAX_PAYLOAD - excess;
} else {
return SPU_MAX_PAYLOAD_INF;
}
}
/**
* spu_payload_length() - Given a SPU2 message header, extract the payload
* length.
* @spu_hdr: Start of SPU message header (FMD)
*
* Return: payload length, in bytes
*/
u32 spu2_payload_length(u8 *spu_hdr)
{
struct SPU2_FMD *fmd = (struct SPU2_FMD *)spu_hdr;
u32 pl_len;
u64 ctrl3;
ctrl3 = le64_to_cpu(fmd->ctrl3);
pl_len = ctrl3 & SPU2_PL_LEN;
return pl_len;
}
/**
* spu_response_hdr_len() - Determine the expected length of a SPU response
* header.
* @auth_key_len: Length of authentication key, in bytes
* @enc_key_len: Length of encryption key, in bytes
*
* For SPU2, includes just FMD. OMD is never requested.
*
* Return: Length of FMD, in bytes
*/
u16 spu2_response_hdr_len(u16 auth_key_len, u16 enc_key_len, bool is_hash)
{
return FMD_SIZE;
}
/**
* spu_hash_pad_len() - Calculate the length of hash padding required to extend
* data to a full block size.
* @hash_alg: hash algorithm
* @hash_mode: hash mode
* @chunksize: length of data, in bytes
* @hash_block_size: size of a hash block, in bytes
*
* SPU2 hardware does all hash padding
*
* Return: length of hash pad in bytes
*/
u16 spu2_hash_pad_len(enum hash_alg hash_alg, enum hash_mode hash_mode,
u32 chunksize, u16 hash_block_size)
{
return 0;
}
/**
* spu2_gcm_ccm_padlen() - Determine the length of GCM/CCM padding for either
* the AAD field or the data.
*
* Return: 0. Unlike SPU-M, SPU2 hardware does any GCM/CCM padding required.
*/
u32 spu2_gcm_ccm_pad_len(enum spu_cipher_mode cipher_mode,
unsigned int data_size)
{
return 0;
}
/**
* spu_assoc_resp_len() - Determine the size of the AAD2 buffer needed to catch
* associated data in a SPU2 output packet.
* @cipher_mode: cipher mode
* @assoc_len: length of additional associated data, in bytes
* @iv_len: length of initialization vector, in bytes
* @is_encrypt: true if encrypting. false if decrypt.
*
* Return: Length of buffer to catch associated data in response
*/
u32 spu2_assoc_resp_len(enum spu_cipher_mode cipher_mode,
unsigned int assoc_len, unsigned int iv_len,
bool is_encrypt)
{
u32 resp_len = assoc_len;
if (is_encrypt)
/* gcm aes esp has to write 8-byte IV in response */
resp_len += iv_len;
return resp_len;
}
/*
* spu_aead_ivlen() - Calculate the length of the AEAD IV to be included
* in a SPU request after the AAD and before the payload.
* @cipher_mode: cipher mode
* @iv_ctr_len: initialization vector length in bytes
*
* For SPU2, AEAD IV is included in OMD and does not need to be repeated
* prior to the payload.
*
* Return: Length of AEAD IV in bytes
*/
u8 spu2_aead_ivlen(enum spu_cipher_mode cipher_mode, u16 iv_len)
{
return 0;
}
/**
* spu2_hash_type() - Determine the type of hash operation.
* @src_sent: The number of bytes in the current request that have already
* been sent to the SPU to be hashed.
*
* SPU2 always does a FULL hash operation
*/
enum hash_type spu2_hash_type(u32 src_sent)
{
return HASH_TYPE_FULL;
}
/**
* spu2_digest_size() - Determine the size of a hash digest to expect the SPU to
* return.
* alg_digest_size: Number of bytes in the final digest for the given algo
* alg: The hash algorithm
* htype: Type of hash operation (init, update, full, etc)
*
*/
u32 spu2_digest_size(u32 alg_digest_size, enum hash_alg alg,
enum hash_type htype)
{
return alg_digest_size;
}
/**
* spu_create_request() - Build a SPU2 request message header, includint FMD and
* OMD.
* @spu_hdr: Start of buffer where SPU request header is to be written
* @req_opts: SPU request message options
* @cipher_parms: Parameters related to cipher algorithm
* @hash_parms: Parameters related to hash algorithm
* @aead_parms: Parameters related to AEAD operation
* @data_size: Length of data to be encrypted or authenticated. If AEAD, does
* not include length of AAD.
*
* Construct the message starting at spu_hdr. Caller should allocate this buffer
* in DMA-able memory at least SPU_HEADER_ALLOC_LEN bytes long.
*
* Return: the length of the SPU header in bytes. 0 if an error occurs.
*/
u32 spu2_create_request(u8 *spu_hdr,
struct spu_request_opts *req_opts,
struct spu_cipher_parms *cipher_parms,
struct spu_hash_parms *hash_parms,
struct spu_aead_parms *aead_parms,
unsigned int data_size)
{
struct SPU2_FMD *fmd;
u8 *ptr;
unsigned int buf_len;
int err;
enum spu2_cipher_type spu2_ciph_type = SPU2_CIPHER_TYPE_NONE;
enum spu2_cipher_mode spu2_ciph_mode;
enum spu2_hash_type spu2_auth_type = SPU2_HASH_TYPE_NONE;
enum spu2_hash_mode spu2_auth_mode;
bool return_md = true;
enum spu2_proto_sel proto = SPU2_PROTO_RESV;
/* size of the payload */
unsigned int payload_len =
hash_parms->prebuf_len + data_size + hash_parms->pad_len -
((req_opts->is_aead && req_opts->is_inbound) ?
hash_parms->digestsize : 0);
/* offset of prebuf or data from start of AAD2 */
unsigned int cipher_offset = aead_parms->assoc_size +
aead_parms->aad_pad_len + aead_parms->iv_len;
#ifdef DEBUG
/* total size of the data following OMD (without STAT word padding) */
unsigned int real_db_size = spu_real_db_size(aead_parms->assoc_size,
aead_parms->iv_len,
hash_parms->prebuf_len,
data_size,
aead_parms->aad_pad_len,
aead_parms->data_pad_len,
hash_parms->pad_len);
#endif
unsigned int assoc_size = aead_parms->assoc_size;
if (req_opts->is_aead &&
(cipher_parms->alg == CIPHER_ALG_AES) &&
(cipher_parms->mode == CIPHER_MODE_GCM))
/*
* On SPU 2, aes gcm cipher first on encrypt, auth first on
* decrypt
*/
req_opts->auth_first = req_opts->is_inbound;
/* and do opposite for ccm (auth 1st on encrypt) */
if (req_opts->is_aead &&
(cipher_parms->alg == CIPHER_ALG_AES) &&
(cipher_parms->mode == CIPHER_MODE_CCM))
req_opts->auth_first = !req_opts->is_inbound;
flow_log("%s()\n", __func__);
flow_log(" in:%u authFirst:%u\n",
req_opts->is_inbound, req_opts->auth_first);
flow_log(" cipher alg:%u mode:%u type %u\n", cipher_parms->alg,
cipher_parms->mode, cipher_parms->type);
flow_log(" is_esp: %s\n", req_opts->is_esp ? "yes" : "no");
flow_log(" key: %d\n", cipher_parms->key_len);
flow_dump(" key: ", cipher_parms->key_buf, cipher_parms->key_len);
flow_log(" iv: %d\n", cipher_parms->iv_len);
flow_dump(" iv: ", cipher_parms->iv_buf, cipher_parms->iv_len);
flow_log(" auth alg:%u mode:%u type %u\n",
hash_parms->alg, hash_parms->mode, hash_parms->type);
flow_log(" digestsize: %u\n", hash_parms->digestsize);
flow_log(" authkey: %d\n", hash_parms->key_len);
flow_dump(" authkey: ", hash_parms->key_buf, hash_parms->key_len);
flow_log(" assoc_size:%u\n", assoc_size);
flow_log(" prebuf_len:%u\n", hash_parms->prebuf_len);
flow_log(" data_size:%u\n", data_size);
flow_log(" hash_pad_len:%u\n", hash_parms->pad_len);
flow_log(" real_db_size:%u\n", real_db_size);
flow_log(" cipher_offset:%u payload_len:%u\n",
cipher_offset, payload_len);
flow_log(" aead_iv: %u\n", aead_parms->iv_len);
/* Convert to spu2 values for cipher alg, hash alg */
err = spu2_cipher_xlate(cipher_parms->alg, cipher_parms->mode,
cipher_parms->type,
&spu2_ciph_type, &spu2_ciph_mode);
/* If we are doing GCM hashing only - either via rfc4543 transform
* or because we happen to do GCM with AAD only and no payload - we
* need to configure hardware to use hash key rather than cipher key
* and put data into payload. This is because unlike SPU-M, running
* GCM cipher with 0 size payload is not permitted.
*/
if ((req_opts->is_rfc4543) ||
((spu2_ciph_mode == SPU2_CIPHER_MODE_GCM) &&
(payload_len == 0))) {
/* Use hashing (only) and set up hash key */
spu2_ciph_type = SPU2_CIPHER_TYPE_NONE;
hash_parms->key_len = cipher_parms->key_len;
memcpy(hash_parms->key_buf, cipher_parms->key_buf,
cipher_parms->key_len);
cipher_parms->key_len = 0;
if (req_opts->is_rfc4543)
payload_len += assoc_size;
else
payload_len = assoc_size;
cipher_offset = 0;
assoc_size = 0;
}
if (err)
return 0;
flow_log("spu2 cipher type %s, cipher mode %s\n",
spu2_ciph_type_name(spu2_ciph_type),
spu2_ciph_mode_name(spu2_ciph_mode));
err = spu2_hash_xlate(hash_parms->alg, hash_parms->mode,
hash_parms->type,
cipher_parms->type,
&spu2_auth_type, &spu2_auth_mode);
if (err)
return 0;
flow_log("spu2 hash type %s, hash mode %s\n",
spu2_hash_type_name(spu2_auth_type),
spu2_hash_mode_name(spu2_auth_mode));
fmd = (struct SPU2_FMD *)spu_hdr;
spu2_fmd_ctrl0_write(fmd, req_opts->is_inbound, req_opts->auth_first,
proto, spu2_ciph_type, spu2_ciph_mode,
spu2_auth_type, spu2_auth_mode);
spu2_fmd_ctrl1_write(fmd, req_opts->is_inbound, assoc_size,
hash_parms->key_len, cipher_parms->key_len,
false, false,
aead_parms->return_iv, aead_parms->ret_iv_len,
aead_parms->ret_iv_off,
cipher_parms->iv_len, hash_parms->digestsize,
!req_opts->bd_suppress, return_md);
spu2_fmd_ctrl2_write(fmd, cipher_offset, hash_parms->key_len, 0,
cipher_parms->key_len, cipher_parms->iv_len);
spu2_fmd_ctrl3_write(fmd, payload_len);
ptr = (u8 *)(fmd + 1);
buf_len = sizeof(struct SPU2_FMD);
/* Write OMD */
if (hash_parms->key_len) {
memcpy(ptr, hash_parms->key_buf, hash_parms->key_len);
ptr += hash_parms->key_len;
buf_len += hash_parms->key_len;
}
if (cipher_parms->key_len) {
memcpy(ptr, cipher_parms->key_buf, cipher_parms->key_len);
ptr += cipher_parms->key_len;
buf_len += cipher_parms->key_len;
}
if (cipher_parms->iv_len) {
memcpy(ptr, cipher_parms->iv_buf, cipher_parms->iv_len);
ptr += cipher_parms->iv_len;
buf_len += cipher_parms->iv_len;
}
packet_dump(" SPU request header: ", spu_hdr, buf_len);
return buf_len;
}
/**
* spu_cipher_req_init() - Build an ablkcipher SPU2 request message header,
* including FMD and OMD.
* @spu_hdr: Location of start of SPU request (FMD field)
* @cipher_parms: Parameters describing cipher request
*
* Called at setkey time to initialize a msg header that can be reused for all
* subsequent ablkcipher requests. Construct the message starting at spu_hdr.
* Caller should allocate this buffer in DMA-able memory at least
* SPU_HEADER_ALLOC_LEN bytes long.
*
* Return: the total length of the SPU header (FMD and OMD) in bytes. 0 if an
* error occurs.
*/
u16 spu2_cipher_req_init(u8 *spu_hdr, struct spu_cipher_parms *cipher_parms)
{
struct SPU2_FMD *fmd;
u8 *omd;
enum spu2_cipher_type spu2_type = SPU2_CIPHER_TYPE_NONE;
enum spu2_cipher_mode spu2_mode;
int err;
flow_log("%s()\n", __func__);
flow_log(" cipher alg:%u mode:%u type %u\n", cipher_parms->alg,
cipher_parms->mode, cipher_parms->type);
flow_log(" cipher_iv_len: %u\n", cipher_parms->iv_len);
flow_log(" key: %d\n", cipher_parms->key_len);
flow_dump(" key: ", cipher_parms->key_buf, cipher_parms->key_len);
/* Convert to spu2 values */
err = spu2_cipher_xlate(cipher_parms->alg, cipher_parms->mode,
cipher_parms->type, &spu2_type, &spu2_mode);
if (err)
return 0;
flow_log("spu2 cipher type %s, cipher mode %s\n",
spu2_ciph_type_name(spu2_type),
spu2_ciph_mode_name(spu2_mode));
/* Construct the FMD header */
fmd = (struct SPU2_FMD *)spu_hdr;
err = spu2_fmd_init(fmd, spu2_type, spu2_mode, cipher_parms->key_len,
cipher_parms->iv_len);
if (err)
return 0;
/* Write cipher key to OMD */
omd = (u8 *)(fmd + 1);
if (cipher_parms->key_buf && cipher_parms->key_len)
memcpy(omd, cipher_parms->key_buf, cipher_parms->key_len);
packet_dump(" SPU request header: ", spu_hdr,
FMD_SIZE + cipher_parms->key_len + cipher_parms->iv_len);
return FMD_SIZE + cipher_parms->key_len + cipher_parms->iv_len;
}
/**
* spu_cipher_req_finish() - Finish building a SPU request message header for a
* block cipher request.
* @spu_hdr: Start of the request message header (MH field)
* @spu_req_hdr_len: Length in bytes of the SPU request header
* @isInbound: 0 encrypt, 1 decrypt
* @cipher_parms: Parameters describing cipher operation to be performed
* @update_key: If true, rewrite the cipher key in SCTX
* @data_size: Length of the data in the BD field
*
* Assumes much of the header was already filled in at setkey() time in
* spu_cipher_req_init().
* spu_cipher_req_init() fills in the encryption key. For RC4, when submitting a
* request for a non-first chunk, we use the 260-byte SUPDT field from the
* previous response as the key. update_key is true for this case. Unused in all
* other cases.
*/
void spu2_cipher_req_finish(u8 *spu_hdr,
u16 spu_req_hdr_len,
unsigned int is_inbound,
struct spu_cipher_parms *cipher_parms,
bool update_key,
unsigned int data_size)
{
struct SPU2_FMD *fmd;
u8 *omd; /* start of optional metadata */
u64 ctrl0;
u64 ctrl3;
flow_log("%s()\n", __func__);
flow_log(" in: %u\n", is_inbound);
flow_log(" cipher alg: %u, cipher_type: %u\n", cipher_parms->alg,
cipher_parms->type);
if (update_key) {
flow_log(" cipher key len: %u\n", cipher_parms->key_len);
flow_dump(" key: ", cipher_parms->key_buf,
cipher_parms->key_len);
}
flow_log(" iv len: %d\n", cipher_parms->iv_len);
flow_dump(" iv: ", cipher_parms->iv_buf, cipher_parms->iv_len);
flow_log(" data_size: %u\n", data_size);
fmd = (struct SPU2_FMD *)spu_hdr;
omd = (u8 *)(fmd + 1);
/*
* FMD ctrl0 was initialized at setkey time. update it to indicate
* whether we are encrypting or decrypting.
*/
ctrl0 = le64_to_cpu(fmd->ctrl0);
if (is_inbound)
ctrl0 &= ~SPU2_CIPH_ENCRYPT_EN; /* decrypt */
else
ctrl0 |= SPU2_CIPH_ENCRYPT_EN; /* encrypt */
fmd->ctrl0 = cpu_to_le64(ctrl0);
if (cipher_parms->alg && cipher_parms->iv_buf && cipher_parms->iv_len) {
/* cipher iv provided so put it in here */
memcpy(omd + cipher_parms->key_len, cipher_parms->iv_buf,
cipher_parms->iv_len);
}
ctrl3 = le64_to_cpu(fmd->ctrl3);
data_size &= SPU2_PL_LEN;
ctrl3 |= data_size;
fmd->ctrl3 = cpu_to_le64(ctrl3);
packet_dump(" SPU request header: ", spu_hdr, spu_req_hdr_len);
}
/**
* spu_request_pad() - Create pad bytes at the end of the data.
* @pad_start: Start of buffer where pad bytes are to be written
* @gcm_padding: Length of GCM padding, in bytes
* @hash_pad_len: Number of bytes of padding extend data to full block
* @auth_alg: Authentication algorithm
* @auth_mode: Authentication mode
* @total_sent: Length inserted at end of hash pad
* @status_padding: Number of bytes of padding to align STATUS word
*
* There may be three forms of pad:
* 1. GCM pad - for GCM mode ciphers, pad to 16-byte alignment
* 2. hash pad - pad to a block length, with 0x80 data terminator and
* size at the end
* 3. STAT pad - to ensure the STAT field is 4-byte aligned
*/
void spu2_request_pad(u8 *pad_start, u32 gcm_padding, u32 hash_pad_len,
enum hash_alg auth_alg, enum hash_mode auth_mode,
unsigned int total_sent, u32 status_padding)
{
u8 *ptr = pad_start;
/* fix data alignent for GCM */
if (gcm_padding > 0) {
flow_log(" GCM: padding to 16 byte alignment: %u bytes\n",
gcm_padding);
memset(ptr, 0, gcm_padding);
ptr += gcm_padding;
}
if (hash_pad_len > 0) {
/* clear the padding section */
memset(ptr, 0, hash_pad_len);
/* terminate the data */
*ptr = 0x80;
ptr += (hash_pad_len - sizeof(u64));
/* add the size at the end as required per alg */
if (auth_alg == HASH_ALG_MD5)
*(u64 *)ptr = cpu_to_le64((u64)total_sent * 8);
else /* SHA1, SHA2-224, SHA2-256 */
*(u64 *)ptr = cpu_to_be64((u64)total_sent * 8);
ptr += sizeof(u64);
}
/* pad to a 4byte alignment for STAT */
if (status_padding > 0) {
flow_log(" STAT: padding to 4 byte alignment: %u bytes\n",
status_padding);
memset(ptr, 0, status_padding);
ptr += status_padding;
}
}
/**
* spu2_xts_tweak_in_payload() - Indicate that SPU2 does NOT place the XTS
* tweak field in the packet payload (it uses IV instead)
*
* Return: 0
*/
u8 spu2_xts_tweak_in_payload(void)
{
return 0;
}
/**
* spu2_tx_status_len() - Return the length of the STATUS field in a SPU
* response message.
*
* Return: Length of STATUS field in bytes.
*/
u8 spu2_tx_status_len(void)
{
return SPU2_TX_STATUS_LEN;
}
/**
* spu2_rx_status_len() - Return the length of the STATUS field in a SPU
* response message.
*
* Return: Length of STATUS field in bytes.
*/
u8 spu2_rx_status_len(void)
{
return SPU2_RX_STATUS_LEN;
}
/**
* spu_status_process() - Process the status from a SPU response message.
* @statp: start of STATUS word
*
* Return: 0 - if status is good and response should be processed
* !0 - status indicates an error and response is invalid
*/
int spu2_status_process(u8 *statp)
{
/* SPU2 status is 2 bytes by default - SPU_RX_STATUS_LEN */
u16 status = le16_to_cpu(*(__le16 *)statp);
if (status == 0)
return 0;
flow_log("rx status is %#x\n", status);
if (status == SPU2_INVALID_ICV)
return SPU_INVALID_ICV;
return -EBADMSG;
}
/**
* spu2_ccm_update_iv() - Update the IV as per the requirements for CCM mode.
*
* @digestsize: Digest size of this request
* @cipher_parms: (pointer to) cipher parmaeters, includes IV buf & IV len
* @assoclen: Length of AAD data
* @chunksize: length of input data to be sent in this req
* @is_encrypt: true if this is an output/encrypt operation
* @is_esp: true if this is an ESP / RFC4309 operation
*
*/
void spu2_ccm_update_iv(unsigned int digestsize,
struct spu_cipher_parms *cipher_parms,
unsigned int assoclen, unsigned int chunksize,
bool is_encrypt, bool is_esp)
{
int L; /* size of length field, in bytes */
/*
* In RFC4309 mode, L is fixed at 4 bytes; otherwise, IV from
* testmgr contains (L-1) in bottom 3 bits of first byte,
* per RFC 3610.
*/
if (is_esp)
L = CCM_ESP_L_VALUE;
else
L = ((cipher_parms->iv_buf[0] & CCM_B0_L_PRIME) >>
CCM_B0_L_PRIME_SHIFT) + 1;
/* SPU2 doesn't want these length bytes nor the first byte... */
cipher_parms->iv_len -= (1 + L);
memmove(cipher_parms->iv_buf, &cipher_parms->iv_buf[1],
cipher_parms->iv_len);
}
/**
* spu2_wordalign_padlen() - SPU2 does not require padding.
* @data_size: length of data field in bytes
*
* Return: length of status field padding, in bytes (always 0 on SPU2)
*/
u32 spu2_wordalign_padlen(u32 data_size)
{
return 0;
}
/*
* Copyright 2016 Broadcom
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License, version 2, as
* published by the Free Software Foundation (the "GPL").
*
* This program is distributed in the hope that it will be useful, but
* WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* General Public License version 2 (GPLv2) for more details.
*
* You should have received a copy of the GNU General Public License
* version 2 (GPLv2) along with this source code.
*/
/*
* This file contains SPU message definitions specific to SPU2.
*/
#ifndef _SPU2_H
#define _SPU2_H
enum spu2_cipher_type {
SPU2_CIPHER_TYPE_NONE = 0x0,
SPU2_CIPHER_TYPE_AES128 = 0x1,
SPU2_CIPHER_TYPE_AES192 = 0x2,
SPU2_CIPHER_TYPE_AES256 = 0x3,
SPU2_CIPHER_TYPE_DES = 0x4,
SPU2_CIPHER_TYPE_3DES = 0x5,
SPU2_CIPHER_TYPE_LAST
};
enum spu2_cipher_mode {
SPU2_CIPHER_MODE_ECB = 0x0,
SPU2_CIPHER_MODE_CBC = 0x1,
SPU2_CIPHER_MODE_CTR = 0x2,
SPU2_CIPHER_MODE_CFB = 0x3,
SPU2_CIPHER_MODE_OFB = 0x4,
SPU2_CIPHER_MODE_XTS = 0x5,
SPU2_CIPHER_MODE_CCM = 0x6,
SPU2_CIPHER_MODE_GCM = 0x7,
SPU2_CIPHER_MODE_LAST
};
enum spu2_hash_type {
SPU2_HASH_TYPE_NONE = 0x0,
SPU2_HASH_TYPE_AES128 = 0x1,
SPU2_HASH_TYPE_AES192 = 0x2,
SPU2_HASH_TYPE_AES256 = 0x3,
SPU2_HASH_TYPE_MD5 = 0x6,
SPU2_HASH_TYPE_SHA1 = 0x7,
SPU2_HASH_TYPE_SHA224 = 0x8,
SPU2_HASH_TYPE_SHA256 = 0x9,
SPU2_HASH_TYPE_SHA384 = 0xa,
SPU2_HASH_TYPE_SHA512 = 0xb,
SPU2_HASH_TYPE_SHA512_224 = 0xc,
SPU2_HASH_TYPE_SHA512_256 = 0xd,
SPU2_HASH_TYPE_SHA3_224 = 0xe,
SPU2_HASH_TYPE_SHA3_256 = 0xf,
SPU2_HASH_TYPE_SHA3_384 = 0x10,
SPU2_HASH_TYPE_SHA3_512 = 0x11,
SPU2_HASH_TYPE_LAST
};
enum spu2_hash_mode {
SPU2_HASH_MODE_CMAC = 0x0,
SPU2_HASH_MODE_CBC_MAC = 0x1,
SPU2_HASH_MODE_XCBC_MAC = 0x2,
SPU2_HASH_MODE_HMAC = 0x3,
SPU2_HASH_MODE_RABIN = 0x4,
SPU2_HASH_MODE_CCM = 0x5,
SPU2_HASH_MODE_GCM = 0x6,
SPU2_HASH_MODE_RESERVED = 0x7,
SPU2_HASH_MODE_LAST
};
enum spu2_ret_md_opts {
SPU2_RET_NO_MD = 0, /* return no metadata */
SPU2_RET_FMD_OMD = 1, /* return both FMD and OMD */
SPU2_RET_FMD_ONLY = 2, /* return only FMD */
SPU2_RET_FMD_OMD_IV = 3, /* return FMD and OMD with just IVs */
};
/* Fixed Metadata format */
struct SPU2_FMD {
u64 ctrl0;
u64 ctrl1;
u64 ctrl2;
u64 ctrl3;
};
#define FMD_SIZE sizeof(struct SPU2_FMD)
/* Fixed part of request message header length in bytes. Just FMD. */
#define SPU2_REQ_FIXED_LEN FMD_SIZE
#define SPU2_HEADER_ALLOC_LEN (SPU_REQ_FIXED_LEN + \
2 * MAX_KEY_SIZE + 2 * MAX_IV_SIZE)
/* FMD ctrl0 field masks */
#define SPU2_CIPH_ENCRYPT_EN 0x1 /* 0: decrypt, 1: encrypt */
#define SPU2_CIPH_TYPE 0xF0 /* one of spu2_cipher_type */
#define SPU2_CIPH_TYPE_SHIFT 4
#define SPU2_CIPH_MODE 0xF00 /* one of spu2_cipher_mode */
#define SPU2_CIPH_MODE_SHIFT 8
#define SPU2_CFB_MASK 0x7000 /* cipher feedback mask */
#define SPU2_CFB_MASK_SHIFT 12
#define SPU2_PROTO_SEL 0xF00000 /* MACsec, IPsec, TLS... */
#define SPU2_PROTO_SEL_SHIFT 20
#define SPU2_HASH_FIRST 0x1000000 /* 1: hash input is input pkt
* data
*/
#define SPU2_CHK_TAG 0x2000000 /* 1: check digest provided */
#define SPU2_HASH_TYPE 0x1F0000000 /* one of spu2_hash_type */
#define SPU2_HASH_TYPE_SHIFT 28
#define SPU2_HASH_MODE 0xF000000000 /* one of spu2_hash_mode */
#define SPU2_HASH_MODE_SHIFT 36
#define SPU2_CIPH_PAD_EN 0x100000000000 /* 1: Add pad to end of payload for
* enc
*/
#define SPU2_CIPH_PAD 0xFF000000000000 /* cipher pad value */
#define SPU2_CIPH_PAD_SHIFT 48
/* FMD ctrl1 field masks */
#define SPU2_TAG_LOC 0x1 /* 1: end of payload, 0: undef */
#define SPU2_HAS_FR_DATA 0x2 /* 1: msg has frame data */
#define SPU2_HAS_AAD1 0x4 /* 1: msg has AAD1 field */
#define SPU2_HAS_NAAD 0x8 /* 1: msg has NAAD field */
#define SPU2_HAS_AAD2 0x10 /* 1: msg has AAD2 field */
#define SPU2_HAS_ESN 0x20 /* 1: msg has ESN field */
#define SPU2_HASH_KEY_LEN 0xFF00 /* len of hash key in bytes.
* HMAC only.
*/
#define SPU2_HASH_KEY_LEN_SHIFT 8
#define SPU2_CIPH_KEY_LEN 0xFF00000 /* len of cipher key in bytes */
#define SPU2_CIPH_KEY_LEN_SHIFT 20
#define SPU2_GENIV 0x10000000 /* 1: hw generates IV */
#define SPU2_HASH_IV 0x20000000 /* 1: IV incl in hash */
#define SPU2_RET_IV 0x40000000 /* 1: return IV in output msg
* b4 payload
*/
#define SPU2_RET_IV_LEN 0xF00000000 /* length in bytes of IV returned.
* 0 = 16 bytes
*/
#define SPU2_RET_IV_LEN_SHIFT 32
#define SPU2_IV_OFFSET 0xF000000000 /* gen IV offset */
#define SPU2_IV_OFFSET_SHIFT 36
#define SPU2_IV_LEN 0x1F0000000000 /* length of input IV in bytes */
#define SPU2_IV_LEN_SHIFT 40
#define SPU2_HASH_TAG_LEN 0x7F000000000000 /* hash tag length in bytes */
#define SPU2_HASH_TAG_LEN_SHIFT 48
#define SPU2_RETURN_MD 0x300000000000000 /* return metadata */
#define SPU2_RETURN_MD_SHIFT 56
#define SPU2_RETURN_FD 0x400000000000000
#define SPU2_RETURN_AAD1 0x800000000000000
#define SPU2_RETURN_NAAD 0x1000000000000000
#define SPU2_RETURN_AAD2 0x2000000000000000
#define SPU2_RETURN_PAY 0x4000000000000000 /* return payload */
/* FMD ctrl2 field masks */
#define SPU2_AAD1_OFFSET 0xFFF /* byte offset of AAD1 field */
#define SPU2_AAD1_LEN 0xFF000 /* length of AAD1 in bytes */
#define SPU2_AAD1_LEN_SHIFT 12
#define SPU2_AAD2_OFFSET 0xFFF00000 /* byte offset of AAD2 field */
#define SPU2_AAD2_OFFSET_SHIFT 20
#define SPU2_PL_OFFSET 0xFFFFFFFF00000000 /* payload offset from AAD2 */
#define SPU2_PL_OFFSET_SHIFT 32
/* FMD ctrl3 field masks */
#define SPU2_PL_LEN 0xFFFFFFFF /* payload length in bytes */
#define SPU2_TLS_LEN 0xFFFF00000000 /* TLS encrypt: cipher len
* TLS decrypt: compressed len
*/
#define SPU2_TLS_LEN_SHIFT 32
/*
* Max value that can be represented in the Payload Length field of the
* ctrl3 word of FMD.
*/
#define SPU2_MAX_PAYLOAD SPU2_PL_LEN
/* Error values returned in STATUS field of response messages */
#define SPU2_INVALID_ICV 1
void spu2_dump_msg_hdr(u8 *buf, unsigned int buf_len);
u32 spu2_ctx_max_payload(enum spu_cipher_alg cipher_alg,
enum spu_cipher_mode cipher_mode,
unsigned int blocksize);
u32 spu2_payload_length(u8 *spu_hdr);
u16 spu2_response_hdr_len(u16 auth_key_len, u16 enc_key_len, bool is_hash);
u16 spu2_hash_pad_len(enum hash_alg hash_alg, enum hash_mode hash_mode,
u32 chunksize, u16 hash_block_size);
u32 spu2_gcm_ccm_pad_len(enum spu_cipher_mode cipher_mode,
unsigned int data_size);
u32 spu2_assoc_resp_len(enum spu_cipher_mode cipher_mode,
unsigned int assoc_len, unsigned int iv_len,
bool is_encrypt);
u8 spu2_aead_ivlen(enum spu_cipher_mode cipher_mode,
u16 iv_len);
enum hash_type spu2_hash_type(u32 src_sent);
u32 spu2_digest_size(u32 alg_digest_size, enum hash_alg alg,
enum hash_type htype);
u32 spu2_create_request(u8 *spu_hdr,
struct spu_request_opts *req_opts,
struct spu_cipher_parms *cipher_parms,
struct spu_hash_parms *hash_parms,
struct spu_aead_parms *aead_parms,
unsigned int data_size);
u16 spu2_cipher_req_init(u8 *spu_hdr, struct spu_cipher_parms *cipher_parms);
void spu2_cipher_req_finish(u8 *spu_hdr,
u16 spu_req_hdr_len,
unsigned int is_inbound,
struct spu_cipher_parms *cipher_parms,
bool update_key,
unsigned int data_size);
void spu2_request_pad(u8 *pad_start, u32 gcm_padding, u32 hash_pad_len,
enum hash_alg auth_alg, enum hash_mode auth_mode,
unsigned int total_sent, u32 status_padding);
u8 spu2_xts_tweak_in_payload(void);
u8 spu2_tx_status_len(void);
u8 spu2_rx_status_len(void);
int spu2_status_process(u8 *statp);
void spu2_ccm_update_iv(unsigned int digestsize,
struct spu_cipher_parms *cipher_parms,
unsigned int assoclen, unsigned int chunksize,
bool is_encrypt, bool is_esp);
u32 spu2_wordalign_padlen(u32 data_size);
#endif
/*
* Copyright 2016 Broadcom
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License, version 2, as
* published by the Free Software Foundation (the "GPL").
*
* This program is distributed in the hope that it will be useful, but
* WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* General Public License version 2 (GPLv2) for more details.
*
* You should have received a copy of the GNU General Public License
* version 2 (GPLv2) along with this source code.
*/
/*
* This file contains SPU message definitions specific to SPU-M.
*/
#ifndef _SPUM_H_
#define _SPUM_H_
#define SPU_CRYPTO_OPERATION_GENERIC 0x1
/* Length of STATUS field in tx and rx packets */
#define SPU_TX_STATUS_LEN 4
/* SPU-M error codes */
#define SPU_STATUS_MASK 0x0000FF00
#define SPU_STATUS_SUCCESS 0x00000000
#define SPU_STATUS_INVALID_ICV 0x00000100
#define SPU_STATUS_ERROR_FLAG 0x00020000
/* Request message. MH + EMH + BDESC + BD header */
#define SPU_REQ_FIXED_LEN 24
/*
* Max length of a SPU message header. Used to allocate a buffer where
* the SPU message header is constructed. Can be used for either a SPU-M
* header or a SPU2 header.
* For SPU-M, sum of the following:
* MH - 4 bytes
* EMH - 4
* SCTX - 3 +
* max auth key len - 64
* max cipher key len - 264 (RC4)
* max IV len - 16
* BDESC - 12
* BD header - 4
* Total: 371
*
* For SPU2, FMD_SIZE (32) plus lengths of hash and cipher keys,
* hash and cipher IVs. If SPU2 does not support RC4, then
*/
#define SPU_HEADER_ALLOC_LEN (SPU_REQ_FIXED_LEN + MAX_KEY_SIZE + \
MAX_KEY_SIZE + MAX_IV_SIZE)
/*
* Response message header length. Normally MH, EMH, BD header, but when
* BD_SUPPRESS is used for hash requests, there is no BD header.
*/
#define SPU_RESP_HDR_LEN 12
#define SPU_HASH_RESP_HDR_LEN 8
/*
* Max value that can be represented in the Payload Length field of the BD
* header. This is a 16-bit field.
*/
#define SPUM_NS2_MAX_PAYLOAD (BIT(16) - 1)
/*
* NSP SPU is limited to ~9KB because of FA2 FIFO size limitations;
* Set MAX_PAYLOAD to 8k to allow for addition of header, digest, etc.
* and stay within limitation.
*/
#define SPUM_NSP_MAX_PAYLOAD 8192
/* Buffer Descriptor Header [BDESC]. SPU in big-endian mode. */
struct BDESC_HEADER {
u16 offset_mac; /* word 0 [31-16] */
u16 length_mac; /* word 0 [15-0] */
u16 offset_crypto; /* word 1 [31-16] */
u16 length_crypto; /* word 1 [15-0] */
u16 offset_icv; /* word 2 [31-16] */
u16 offset_iv; /* word 2 [15-0] */
};
/* Buffer Data Header [BD]. SPU in big-endian mode. */
struct BD_HEADER {
u16 size;
u16 prev_length;
};
/* Command Context Header. SPU-M in big endian mode. */
struct MHEADER {
u8 flags; /* [31:24] */
u8 op_code; /* [23:16] */
u16 reserved; /* [15:0] */
};
/* MH header flags bits */
#define MH_SUPDT_PRES BIT(0)
#define MH_HASH_PRES BIT(2)
#define MH_BD_PRES BIT(3)
#define MH_MFM_PRES BIT(4)
#define MH_BDESC_PRES BIT(5)
#define MH_SCTX_PRES BIT(7)
/* SCTX word 0 bit offsets and fields masks */
#define SCTX_SIZE 0x000000FF
/* SCTX word 1 bit shifts and field masks */
#define UPDT_OFST 0x000000FF /* offset of SCTX updateable fld */
#define HASH_TYPE 0x00000300 /* hash alg operation type */
#define HASH_TYPE_SHIFT 8
#define HASH_MODE 0x00001C00 /* one of spu2_hash_mode */
#define HASH_MODE_SHIFT 10
#define HASH_ALG 0x0000E000 /* hash algorithm */
#define HASH_ALG_SHIFT 13
#define CIPHER_TYPE 0x00030000 /* encryption operation type */
#define CIPHER_TYPE_SHIFT 16
#define CIPHER_MODE 0x001C0000 /* encryption mode */
#define CIPHER_MODE_SHIFT 18
#define CIPHER_ALG 0x00E00000 /* encryption algo */
#define CIPHER_ALG_SHIFT 21
#define ICV_IS_512 BIT(27)
#define ICV_IS_512_SHIFT 27
#define CIPHER_ORDER BIT(30)
#define CIPHER_ORDER_SHIFT 30
#define CIPHER_INBOUND BIT(31)
#define CIPHER_INBOUND_SHIFT 31
/* SCTX word 2 bit shifts and field masks */
#define EXP_IV_SIZE 0x7
#define IV_OFFSET BIT(3)
#define IV_OFFSET_SHIFT 3
#define GEN_IV BIT(5)
#define GEN_IV_SHIFT 5
#define EXPLICIT_IV BIT(6)
#define EXPLICIT_IV_SHIFT 6
#define SCTX_IV BIT(7)
#define SCTX_IV_SHIFT 7
#define ICV_SIZE 0x0F00
#define ICV_SIZE_SHIFT 8
#define CHECK_ICV BIT(12)
#define CHECK_ICV_SHIFT 12
#define INSERT_ICV BIT(13)
#define INSERT_ICV_SHIFT 13
#define BD_SUPPRESS BIT(19)
#define BD_SUPPRESS_SHIFT 19
/* Generic Mode Security Context Structure [SCTX] */
struct SCTX {
/* word 0: protocol flags */
u32 proto_flags;
/* word 1: cipher flags */
u32 cipher_flags;
/* word 2: Extended cipher flags */
u32 ecf;
};
struct SPUHEADER {
struct MHEADER mh;
u32 emh;
struct SCTX sa;
};
#endif /* _SPUM_H_ */
/*
* Copyright 2016 Broadcom
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License, version 2, as
* published by the Free Software Foundation (the "GPL").
*
* This program is distributed in the hope that it will be useful, but
* WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* General Public License version 2 (GPLv2) for more details.
*
* You should have received a copy of the GNU General Public License
* version 2 (GPLv2) along with this source code.
*/
#include <linux/debugfs.h>
#include "cipher.h"
#include "util.h"
/* offset of SPU_OFIFO_CTRL register */
#define SPU_OFIFO_CTRL 0x40
#define SPU_FIFO_WATERMARK 0x1FF
/**
* spu_sg_at_offset() - Find the scatterlist entry at a given distance from the
* start of a scatterlist.
* @sg: [in] Start of a scatterlist
* @skip: [in] Distance from the start of the scatterlist, in bytes
* @sge: [out] Scatterlist entry at skip bytes from start
* @sge_offset: [out] Number of bytes from start of sge buffer to get to
* requested distance.
*
* Return: 0 if entry found at requested distance
* < 0 otherwise
*/
int spu_sg_at_offset(struct scatterlist *sg, unsigned int skip,
struct scatterlist **sge, unsigned int *sge_offset)
{
/* byte index from start of sg to the end of the previous entry */
unsigned int index = 0;
/* byte index from start of sg to the end of the current entry */
unsigned int next_index;
next_index = sg->length;
while (next_index <= skip) {
sg = sg_next(sg);
index = next_index;
if (!sg)
return -EINVAL;
next_index += sg->length;
}
*sge_offset = skip - index;
*sge = sg;
return 0;
}
/* Copy len bytes of sg data, starting at offset skip, to a dest buffer */
void sg_copy_part_to_buf(struct scatterlist *src, u8 *dest,
unsigned int len, unsigned int skip)
{
size_t copied;
unsigned int nents = sg_nents(src);
copied = sg_pcopy_to_buffer(src, nents, dest, len, skip);
if (copied != len) {
flow_log("%s copied %u bytes of %u requested. ",
__func__, (u32)copied, len);
flow_log("sg with %u entries and skip %u\n", nents, skip);
}
}
/*
* Copy data into a scatterlist starting at a specified offset in the
* scatterlist. Specifically, copy len bytes of data in the buffer src
* into the scatterlist dest, starting skip bytes into the scatterlist.
*/
void sg_copy_part_from_buf(struct scatterlist *dest, u8 *src,
unsigned int len, unsigned int skip)
{
size_t copied;
unsigned int nents = sg_nents(dest);
copied = sg_pcopy_from_buffer(dest, nents, src, len, skip);
if (copied != len) {
flow_log("%s copied %u bytes of %u requested. ",
__func__, (u32)copied, len);
flow_log("sg with %u entries and skip %u\n", nents, skip);
}
}
/**
* spu_sg_count() - Determine number of elements in scatterlist to provide a
* specified number of bytes.
* @sg_list: scatterlist to examine
* @skip: index of starting point
* @nbytes: consider elements of scatterlist until reaching this number of
* bytes
*
* Return: the number of sg entries contributing to nbytes of data
*/
int spu_sg_count(struct scatterlist *sg_list, unsigned int skip, int nbytes)
{
struct scatterlist *sg;
int sg_nents = 0;
unsigned int offset;
if (!sg_list)
return 0;
if (spu_sg_at_offset(sg_list, skip, &sg, &offset) < 0)
return 0;
while (sg && (nbytes > 0)) {
sg_nents++;
nbytes -= (sg->length - offset);
offset = 0;
sg = sg_next(sg);
}
return sg_nents;
}
/**
* spu_msg_sg_add() - Copy scatterlist entries from one sg to another, up to a
* given length.
* @to_sg: scatterlist to copy to
* @from_sg: scatterlist to copy from
* @from_skip: number of bytes to skip in from_sg. Non-zero when previous
* request included part of the buffer in entry in from_sg.
* Assumes from_skip < from_sg->length.
* @from_nents number of entries in from_sg
* @length number of bytes to copy. may reach this limit before exhausting
* from_sg.
*
* Copies the entries themselves, not the data in the entries. Assumes to_sg has
* enough entries. Does not limit the size of an individual buffer in to_sg.
*
* to_sg, from_sg, skip are all updated to end of copy
*
* Return: Number of bytes copied
*/
u32 spu_msg_sg_add(struct scatterlist **to_sg,
struct scatterlist **from_sg, u32 *from_skip,
u8 from_nents, u32 length)
{
struct scatterlist *sg; /* an entry in from_sg */
struct scatterlist *to = *to_sg;
struct scatterlist *from = *from_sg;
u32 skip = *from_skip;
u32 offset;
int i;
u32 entry_len = 0;
u32 frag_len = 0; /* length of entry added to to_sg */
u32 copied = 0; /* number of bytes copied so far */
if (length == 0)
return 0;
for_each_sg(from, sg, from_nents, i) {
/* number of bytes in this from entry not yet used */
entry_len = sg->length - skip;
frag_len = min(entry_len, length - copied);
offset = sg->offset + skip;
if (frag_len)
sg_set_page(to++, sg_page(sg), frag_len, offset);
copied += frag_len;
if (copied == entry_len) {
/* used up all of from entry */
skip = 0; /* start at beginning of next entry */
}
if (copied == length)
break;
}
*to_sg = to;
*from_sg = sg;
if (frag_len < entry_len)
*from_skip = skip + frag_len;
else
*from_skip = 0;
return copied;
}
void add_to_ctr(u8 *ctr_pos, unsigned int increment)
{
__be64 *high_be = (__be64 *)ctr_pos;
__be64 *low_be = high_be + 1;
u64 orig_low = __be64_to_cpu(*low_be);
u64 new_low = orig_low + (u64)increment;
*low_be = __cpu_to_be64(new_low);
if (new_low < orig_low)
/* there was a carry from the low 8 bytes */
*high_be = __cpu_to_be64(__be64_to_cpu(*high_be) + 1);
}
struct sdesc {
struct shash_desc shash;
char ctx[];
};
/* do a synchronous decrypt operation */
int do_decrypt(char *alg_name,
void *key_ptr, unsigned int key_len,
void *iv_ptr, void *src_ptr, void *dst_ptr,
unsigned int block_len)
{
struct scatterlist sg_in[1], sg_out[1];
struct crypto_blkcipher *tfm =
crypto_alloc_blkcipher(alg_name, 0, CRYPTO_ALG_ASYNC);
struct blkcipher_desc desc = {.tfm = tfm, .flags = 0 };
int ret = 0;
void *iv;
int ivsize;
flow_log("%s() name:%s block_len:%u\n", __func__, alg_name, block_len);
if (IS_ERR(tfm))
return PTR_ERR(tfm);
crypto_blkcipher_setkey((void *)tfm, key_ptr, key_len);
sg_init_table(sg_in, 1);
sg_set_buf(sg_in, src_ptr, block_len);
sg_init_table(sg_out, 1);
sg_set_buf(sg_out, dst_ptr, block_len);
iv = crypto_blkcipher_crt(tfm)->iv;
ivsize = crypto_blkcipher_ivsize(tfm);
memcpy(iv, iv_ptr, ivsize);
ret = crypto_blkcipher_decrypt(&desc, sg_out, sg_in, block_len);
crypto_free_blkcipher(tfm);
if (ret < 0)
pr_err("aes_decrypt failed %d\n", ret);
return ret;
}
/**
* do_shash() - Do a synchronous hash operation in software
* @name: The name of the hash algorithm
* @result: Buffer where digest is to be written
* @data1: First part of data to hash. May be NULL.
* @data1_len: Length of data1, in bytes
* @data2: Second part of data to hash. May be NULL.
* @data2_len: Length of data2, in bytes
* @key: Key (if keyed hash)
* @key_len: Length of key, in bytes (or 0 if non-keyed hash)
*
* Note that the crypto API will not select this driver's own transform because
* this driver only registers asynchronous algos.
*
* Return: 0 if hash successfully stored in result
* < 0 otherwise
*/
int do_shash(unsigned char *name, unsigned char *result,
const u8 *data1, unsigned int data1_len,
const u8 *data2, unsigned int data2_len,
const u8 *key, unsigned int key_len)
{
int rc;
unsigned int size;
struct crypto_shash *hash;
struct sdesc *sdesc;
hash = crypto_alloc_shash(name, 0, 0);
if (IS_ERR(hash)) {
rc = PTR_ERR(hash);
pr_err("%s: Crypto %s allocation error %d", __func__, name, rc);
return rc;
}
size = sizeof(struct shash_desc) + crypto_shash_descsize(hash);
sdesc = kmalloc(size, GFP_KERNEL);
if (!sdesc) {
rc = -ENOMEM;
pr_err("%s: Memory allocation failure", __func__);
goto do_shash_err;
}
sdesc->shash.tfm = hash;
sdesc->shash.flags = 0x0;
if (key_len > 0) {
rc = crypto_shash_setkey(hash, key, key_len);
if (rc) {
pr_err("%s: Could not setkey %s shash", __func__, name);
goto do_shash_err;
}
}
rc = crypto_shash_init(&sdesc->shash);
if (rc) {
pr_err("%s: Could not init %s shash", __func__, name);
goto do_shash_err;
}
rc = crypto_shash_update(&sdesc->shash, data1, data1_len);
if (rc) {
pr_err("%s: Could not update1", __func__);
goto do_shash_err;
}
if (data2 && data2_len) {
rc = crypto_shash_update(&sdesc->shash, data2, data2_len);
if (rc) {
pr_err("%s: Could not update2", __func__);
goto do_shash_err;
}
}
rc = crypto_shash_final(&sdesc->shash, result);
if (rc)
pr_err("%s: Could not genereate %s hash", __func__, name);
do_shash_err:
crypto_free_shash(hash);
kfree(sdesc);
return rc;
}
/* Dump len bytes of a scatterlist starting at skip bytes into the sg */
void __dump_sg(struct scatterlist *sg, unsigned int skip, unsigned int len)
{
u8 dbuf[16];
unsigned int idx = skip;
unsigned int num_out = 0; /* number of bytes dumped so far */
unsigned int count;
if (packet_debug_logging) {
while (num_out < len) {
count = (len - num_out > 16) ? 16 : len - num_out;
sg_copy_part_to_buf(sg, dbuf, count, idx);
num_out += count;
print_hex_dump(KERN_ALERT, " sg: ", DUMP_PREFIX_NONE,
4, 1, dbuf, count, false);
idx += 16;
}
}
if (debug_logging_sleep)
msleep(debug_logging_sleep);
}
/* Returns the name for a given cipher alg/mode */
char *spu_alg_name(enum spu_cipher_alg alg, enum spu_cipher_mode mode)
{
switch (alg) {
case CIPHER_ALG_RC4:
return "rc4";
case CIPHER_ALG_AES:
switch (mode) {
case CIPHER_MODE_CBC:
return "cbc(aes)";
case CIPHER_MODE_ECB:
return "ecb(aes)";
case CIPHER_MODE_OFB:
return "ofb(aes)";
case CIPHER_MODE_CFB:
return "cfb(aes)";
case CIPHER_MODE_CTR:
return "ctr(aes)";
case CIPHER_MODE_XTS:
return "xts(aes)";
case CIPHER_MODE_GCM:
return "gcm(aes)";
default:
return "aes";
}
break;
case CIPHER_ALG_DES:
switch (mode) {
case CIPHER_MODE_CBC:
return "cbc(des)";
case CIPHER_MODE_ECB:
return "ecb(des)";
case CIPHER_MODE_CTR:
return "ctr(des)";
default:
return "des";
}
break;
case CIPHER_ALG_3DES:
switch (mode) {
case CIPHER_MODE_CBC:
return "cbc(des3_ede)";
case CIPHER_MODE_ECB:
return "ecb(des3_ede)";
case CIPHER_MODE_CTR:
return "ctr(des3_ede)";
default:
return "3des";
}
break;
default:
return "other";
}
}
static ssize_t spu_debugfs_read(struct file *filp, char __user *ubuf,
size_t count, loff_t *offp)
{
struct device_private *ipriv;
char *buf;
ssize_t ret, out_offset, out_count;
int i;
u32 fifo_len;
u32 spu_ofifo_ctrl;
u32 alg;
u32 mode;
u32 op_cnt;
out_count = 2048;
buf = kmalloc(out_count, GFP_KERNEL);
if (!buf)
return -ENOMEM;
ipriv = filp->private_data;
out_offset = 0;
out_offset += snprintf(buf + out_offset, out_count - out_offset,
"Number of SPUs.........%u\n",
ipriv->spu.num_spu);
out_offset += snprintf(buf + out_offset, out_count - out_offset,
"Current sessions.......%u\n",
atomic_read(&ipriv->session_count));
out_offset += snprintf(buf + out_offset, out_count - out_offset,
"Session count..........%u\n",
atomic_read(&ipriv->stream_count));
out_offset += snprintf(buf + out_offset, out_count - out_offset,
"Cipher setkey..........%u\n",
atomic_read(&ipriv->setkey_cnt[SPU_OP_CIPHER]));
out_offset += snprintf(buf + out_offset, out_count - out_offset,
"Cipher Ops.............%u\n",
atomic_read(&ipriv->op_counts[SPU_OP_CIPHER]));
for (alg = 0; alg < CIPHER_ALG_LAST; alg++) {
for (mode = 0; mode < CIPHER_MODE_LAST; mode++) {
op_cnt = atomic_read(&ipriv->cipher_cnt[alg][mode]);
if (op_cnt) {
out_offset += snprintf(buf + out_offset,
out_count - out_offset,
" %-13s%11u\n",
spu_alg_name(alg, mode), op_cnt);
}
}
}
out_offset += snprintf(buf + out_offset, out_count - out_offset,
"Hash Ops...............%u\n",
atomic_read(&ipriv->op_counts[SPU_OP_HASH]));
for (alg = 0; alg < HASH_ALG_LAST; alg++) {
op_cnt = atomic_read(&ipriv->hash_cnt[alg]);
if (op_cnt) {
out_offset += snprintf(buf + out_offset,
out_count - out_offset,
" %-13s%11u\n",
hash_alg_name[alg], op_cnt);
}
}
out_offset += snprintf(buf + out_offset, out_count - out_offset,
"HMAC setkey............%u\n",
atomic_read(&ipriv->setkey_cnt[SPU_OP_HMAC]));
out_offset += snprintf(buf + out_offset, out_count - out_offset,
"HMAC Ops...............%u\n",
atomic_read(&ipriv->op_counts[SPU_OP_HMAC]));
for (alg = 0; alg < HASH_ALG_LAST; alg++) {
op_cnt = atomic_read(&ipriv->hmac_cnt[alg]);
if (op_cnt) {
out_offset += snprintf(buf + out_offset,
out_count - out_offset,
" %-13s%11u\n",
hash_alg_name[alg], op_cnt);
}
}
out_offset += snprintf(buf + out_offset, out_count - out_offset,
"AEAD setkey............%u\n",
atomic_read(&ipriv->setkey_cnt[SPU_OP_AEAD]));
out_offset += snprintf(buf + out_offset, out_count - out_offset,
"AEAD Ops...............%u\n",
atomic_read(&ipriv->op_counts[SPU_OP_AEAD]));
for (alg = 0; alg < AEAD_TYPE_LAST; alg++) {
op_cnt = atomic_read(&ipriv->aead_cnt[alg]);
if (op_cnt) {
out_offset += snprintf(buf + out_offset,
out_count - out_offset,
" %-13s%11u\n",
aead_alg_name[alg], op_cnt);
}
}
out_offset += snprintf(buf + out_offset, out_count - out_offset,
"Bytes of req data......%llu\n",
(u64)atomic64_read(&ipriv->bytes_out));
out_offset += snprintf(buf + out_offset, out_count - out_offset,
"Bytes of resp data.....%llu\n",
(u64)atomic64_read(&ipriv->bytes_in));
out_offset += snprintf(buf + out_offset, out_count - out_offset,
"Mailbox full...........%u\n",
atomic_read(&ipriv->mb_no_spc));
out_offset += snprintf(buf + out_offset, out_count - out_offset,
"Mailbox send failures..%u\n",
atomic_read(&ipriv->mb_send_fail));
out_offset += snprintf(buf + out_offset, out_count - out_offset,
"Check ICV errors.......%u\n",
atomic_read(&ipriv->bad_icv));
if (ipriv->spu.spu_type == SPU_TYPE_SPUM)
for (i = 0; i < ipriv->spu.num_spu; i++) {
spu_ofifo_ctrl = ioread32(ipriv->spu.reg_vbase[i] +
SPU_OFIFO_CTRL);
fifo_len = spu_ofifo_ctrl & SPU_FIFO_WATERMARK;
out_offset += snprintf(buf + out_offset,
out_count - out_offset,
"SPU %d output FIFO high water.....%u\n",
i, fifo_len);
}
if (out_offset > out_count)
out_offset = out_count;
ret = simple_read_from_buffer(ubuf, count, offp, buf, out_offset);
kfree(buf);
return ret;
}
static const struct file_operations spu_debugfs_stats = {
.owner = THIS_MODULE,
.open = simple_open,
.read = spu_debugfs_read,
};
/*
* Create the debug FS directories. If the top-level directory has not yet
* been created, create it now. Create a stats file in this directory for
* a SPU.
*/
void spu_setup_debugfs(void)
{
if (!debugfs_initialized())
return;
if (!iproc_priv.debugfs_dir)
iproc_priv.debugfs_dir = debugfs_create_dir(KBUILD_MODNAME,
NULL);
if (!iproc_priv.debugfs_stats)
/* Create file with permissions S_IRUSR */
debugfs_create_file("stats", 0400, iproc_priv.debugfs_dir,
&iproc_priv, &spu_debugfs_stats);
}
void spu_free_debugfs(void)
{
debugfs_remove_recursive(iproc_priv.debugfs_dir);
iproc_priv.debugfs_dir = NULL;
}
/**
* format_value_ccm() - Format a value into a buffer, using a specified number
* of bytes (i.e. maybe writing value X into a 4 byte
* buffer, or maybe into a 12 byte buffer), as per the
* SPU CCM spec.
*
* @val: value to write (up to max of unsigned int)
* @buf: (pointer to) buffer to write the value
* @len: number of bytes to use (0 to 255)
*
*/
void format_value_ccm(unsigned int val, u8 *buf, u8 len)
{
int i;
/* First clear full output buffer */
memset(buf, 0, len);
/* Then, starting from right side, fill in with data */
for (i = 0; i < len; i++) {
buf[len - i - 1] = (val >> (8 * i)) & 0xff;
if (i >= 3)
break; /* Only handle up to 32 bits of 'val' */
}
}
/*
* Copyright 2016 Broadcom
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License, version 2, as
* published by the Free Software Foundation (the "GPL").
*
* This program is distributed in the hope that it will be useful, but
* WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* General Public License version 2 (GPLv2) for more details.
*
* You should have received a copy of the GNU General Public License
* version 2 (GPLv2) along with this source code.
*/
#ifndef _UTIL_H
#define _UTIL_H
#include <linux/kernel.h>
#include <linux/delay.h>
#include "spu.h"
extern int flow_debug_logging;
extern int packet_debug_logging;
extern int debug_logging_sleep;
#ifdef DEBUG
#define flow_log(...) \
do { \
if (flow_debug_logging) { \
printk(__VA_ARGS__); \
if (debug_logging_sleep) \
msleep(debug_logging_sleep); \
} \
} while (0)
#define flow_dump(msg, var, var_len) \
do { \
if (flow_debug_logging) { \
print_hex_dump(KERN_ALERT, msg, DUMP_PREFIX_NONE, \
16, 1, var, var_len, false); \
if (debug_logging_sleep) \
msleep(debug_logging_sleep); \
} \
} while (0)
#define packet_log(...) \
do { \
if (packet_debug_logging) { \
printk(__VA_ARGS__); \
if (debug_logging_sleep) \
msleep(debug_logging_sleep); \
} \
} while (0)
#define packet_dump(msg, var, var_len) \
do { \
if (packet_debug_logging) { \
print_hex_dump(KERN_ALERT, msg, DUMP_PREFIX_NONE, \
16, 1, var, var_len, false); \
if (debug_logging_sleep) \
msleep(debug_logging_sleep); \
} \
} while (0)
void __dump_sg(struct scatterlist *sg, unsigned int skip, unsigned int len);
#define dump_sg(sg, skip, len) __dump_sg(sg, skip, len)
#else /* !DEBUG_ON */
#define flow_log(...) do {} while (0)
#define flow_dump(msg, var, var_len) do {} while (0)
#define packet_log(...) do {} while (0)
#define packet_dump(msg, var, var_len) do {} while (0)
#define dump_sg(sg, skip, len) do {} while (0)
#endif /* DEBUG_ON */
int spu_sg_at_offset(struct scatterlist *sg, unsigned int skip,
struct scatterlist **sge, unsigned int *sge_offset);
/* Copy sg data, from skip, length len, to dest */
void sg_copy_part_to_buf(struct scatterlist *src, u8 *dest,
unsigned int len, unsigned int skip);
/* Copy src into scatterlist from offset, length len */
void sg_copy_part_from_buf(struct scatterlist *dest, u8 *src,
unsigned int len, unsigned int skip);
int spu_sg_count(struct scatterlist *sg_list, unsigned int skip, int nbytes);
u32 spu_msg_sg_add(struct scatterlist **to_sg,
struct scatterlist **from_sg, u32 *skip,
u8 from_nents, u32 tot_len);
void add_to_ctr(u8 *ctr_pos, unsigned int increment);
/* do a synchronous decrypt operation */
int do_decrypt(char *alg_name,
void *key_ptr, unsigned int key_len,
void *iv_ptr, void *src_ptr, void *dst_ptr,
unsigned int block_len);
/* produce a message digest from data of length n bytes */
int do_shash(unsigned char *name, unsigned char *result,
const u8 *data1, unsigned int data1_len,
const u8 *data2, unsigned int data2_len,
const u8 *key, unsigned int key_len);
char *spu_alg_name(enum spu_cipher_alg alg, enum spu_cipher_mode mode);
void spu_setup_debugfs(void);
void spu_free_debugfs(void);
void format_value_ccm(unsigned int val, u8 *buf, u8 len);
#endif
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment