diff options
Diffstat (limited to 'drivers/crypto')
42 files changed, 698 insertions, 762 deletions
diff --git a/drivers/crypto/Kconfig b/drivers/crypto/Kconfig index 0af08081e305..4bdd761a8ca9 100644 --- a/drivers/crypto/Kconfig +++ b/drivers/crypto/Kconfig @@ -469,11 +469,9 @@ if CRYPTO_DEV_UX500 endif # if CRYPTO_DEV_UX500 config CRYPTO_DEV_ATMEL_AUTHENC - tristate "Support for Atmel IPSEC/SSL hw accelerator" + bool "Support for Atmel IPSEC/SSL hw accelerator" depends on ARCH_AT91 || COMPILE_TEST - select CRYPTO_AUTHENC - select CRYPTO_DEV_ATMEL_AES - select CRYPTO_DEV_ATMEL_SHA + depends on CRYPTO_DEV_ATMEL_AES help Some Atmel processors can combine the AES and SHA hw accelerators to enhance support of IPSEC/SSL. @@ -486,6 +484,8 @@ config CRYPTO_DEV_ATMEL_AES select CRYPTO_AES select CRYPTO_AEAD select CRYPTO_BLKCIPHER + select CRYPTO_AUTHENC if CRYPTO_DEV_ATMEL_AUTHENC + select CRYPTO_DEV_ATMEL_SHA if CRYPTO_DEV_ATMEL_AUTHENC help Some Atmel processors have AES hw accelerator. Select this if you want to use the Atmel module for @@ -700,7 +700,7 @@ source "drivers/crypto/stm32/Kconfig" config CRYPTO_DEV_SAFEXCEL tristate "Inside Secure's SafeXcel cryptographic engine driver" depends on OF - depends on (ARM64 && ARCH_MVEBU) || (COMPILE_TEST && 64BIT) + depends on (ARM64 && ARCH_MVEBU) || (COMPILE_TEST && 64BIT) && HAS_IOMEM select CRYPTO_AES select CRYPTO_AUTHENC select CRYPTO_BLKCIPHER diff --git a/drivers/crypto/amcc/crypto4xx_core.c b/drivers/crypto/amcc/crypto4xx_core.c index 53e2ba9b0c02..6351606a5d7f 100644 --- a/drivers/crypto/amcc/crypto4xx_core.c +++ b/drivers/crypto/amcc/crypto4xx_core.c @@ -366,12 +366,8 @@ static u32 crypto4xx_build_sdr(struct crypto4xx_device *dev) dma_alloc_coherent(dev->core_dev->device, PPC4XX_SD_BUFFER_SIZE * PPC4XX_NUM_SD, &dev->scatter_buffer_pa, GFP_ATOMIC); - if (!dev->scatter_buffer_va) { - dma_free_coherent(dev->core_dev->device, - sizeof(struct ce_sd) * PPC4XX_NUM_SD, - dev->sdr, dev->sdr_pa); + if (!dev->scatter_buffer_va) return -ENOMEM; - } for (i = 0; i < PPC4XX_NUM_SD; i++) { dev->sdr[i].ptr = dev->scatter_buffer_pa + diff --git a/drivers/crypto/atmel-aes.c b/drivers/crypto/atmel-aes.c index 2b7af44c7b85..35f1735b25dd 100644 --- a/drivers/crypto/atmel-aes.c +++ b/drivers/crypto/atmel-aes.c @@ -88,7 +88,6 @@ struct atmel_aes_caps { bool has_dualbuff; bool has_cfb64; - bool has_ctr32; bool has_gcm; bool has_xts; bool has_authenc; @@ -145,7 +144,7 @@ struct atmel_aes_xts_ctx { u32 key2[AES_KEYSIZE_256 / sizeof(u32)]; }; -#ifdef CONFIG_CRYPTO_DEV_ATMEL_AUTHENC +#if IS_ENABLED(CONFIG_CRYPTO_DEV_ATMEL_AUTHENC) struct atmel_aes_authenc_ctx { struct atmel_aes_base_ctx base; struct atmel_sha_authenc_ctx *auth; @@ -157,7 +156,7 @@ struct atmel_aes_reqctx { u32 lastc[AES_BLOCK_SIZE / sizeof(u32)]; }; -#ifdef CONFIG_CRYPTO_DEV_ATMEL_AUTHENC +#if IS_ENABLED(CONFIG_CRYPTO_DEV_ATMEL_AUTHENC) struct atmel_aes_authenc_reqctx { struct atmel_aes_reqctx base; @@ -486,13 +485,36 @@ static inline bool atmel_aes_is_encrypt(const struct atmel_aes_dev *dd) return (dd->flags & AES_FLAGS_ENCRYPT); } -#ifdef CONFIG_CRYPTO_DEV_ATMEL_AUTHENC +#if IS_ENABLED(CONFIG_CRYPTO_DEV_ATMEL_AUTHENC) static void atmel_aes_authenc_complete(struct atmel_aes_dev *dd, int err); #endif +static void atmel_aes_set_iv_as_last_ciphertext_block(struct atmel_aes_dev *dd) +{ + struct ablkcipher_request *req = ablkcipher_request_cast(dd->areq); + struct atmel_aes_reqctx *rctx = ablkcipher_request_ctx(req); + struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); + unsigned int ivsize = crypto_ablkcipher_ivsize(ablkcipher); + + if (req->nbytes < ivsize) + return; + + if (rctx->mode & AES_FLAGS_ENCRYPT) { + scatterwalk_map_and_copy(req->info, req->dst, + req->nbytes - ivsize, ivsize, 0); + } else { + if (req->src == req->dst) + memcpy(req->info, rctx->lastc, ivsize); + else + scatterwalk_map_and_copy(req->info, req->src, + req->nbytes - ivsize, + ivsize, 0); + } +} + static inline int atmel_aes_complete(struct atmel_aes_dev *dd, int err) { -#ifdef CONFIG_CRYPTO_DEV_ATMEL_AUTHENC +#if IS_ENABLED(CONFIG_CRYPTO_DEV_ATMEL_AUTHENC) if (dd->ctx->is_aead) atmel_aes_authenc_complete(dd, err); #endif @@ -500,26 +522,8 @@ static inline int atmel_aes_complete(struct atmel_aes_dev *dd, int err) clk_disable(dd->iclk); dd->flags &= ~AES_FLAGS_BUSY; - if (!dd->ctx->is_aead) { - struct ablkcipher_request *req = - ablkcipher_request_cast(dd->areq); - struct atmel_aes_reqctx *rctx = ablkcipher_request_ctx(req); - struct crypto_ablkcipher *ablkcipher = - crypto_ablkcipher_reqtfm(req); - int ivsize = crypto_ablkcipher_ivsize(ablkcipher); - - if (rctx->mode & AES_FLAGS_ENCRYPT) { - scatterwalk_map_and_copy(req->info, req->dst, - req->nbytes - ivsize, ivsize, 0); - } else { - if (req->src == req->dst) { - memcpy(req->info, rctx->lastc, ivsize); - } else { - scatterwalk_map_and_copy(req->info, req->src, - req->nbytes - ivsize, ivsize, 0); - } - } - } + if (!dd->ctx->is_aead) + atmel_aes_set_iv_as_last_ciphertext_block(dd); if (dd->is_async) dd->areq->complete(dd->areq, err); @@ -1008,8 +1012,9 @@ static int atmel_aes_ctr_transfer(struct atmel_aes_dev *dd) struct atmel_aes_ctr_ctx *ctx = atmel_aes_ctr_ctx_cast(dd->ctx); struct ablkcipher_request *req = ablkcipher_request_cast(dd->areq); struct scatterlist *src, *dst; - u32 ctr, blocks; size_t datalen; + u32 ctr; + u16 blocks, start, end; bool use_dma, fragmented = false; /* Check for transfer completion. */ @@ -1021,27 +1026,17 @@ static int atmel_aes_ctr_transfer(struct atmel_aes_dev *dd) datalen = req->nbytes - ctx->offset; blocks = DIV_ROUND_UP(datalen, AES_BLOCK_SIZE); ctr = be32_to_cpu(ctx->iv[3]); - if (dd->caps.has_ctr32) { - /* Check 32bit counter overflow. */ - u32 start = ctr; - u32 end = start + blocks - 1; - - if (end < start) { - ctr |= 0xffffffff; - datalen = AES_BLOCK_SIZE * -start; - fragmented = true; - } - } else { - /* Check 16bit counter overflow. */ - u16 start = ctr & 0xffff; - u16 end = start + (u16)blocks - 1; - - if (blocks >> 16 || end < start) { - ctr |= 0xffff; - datalen = AES_BLOCK_SIZE * (0x10000-start); - fragmented = true; - } + + /* Check 16bit counter overflow. */ + start = ctr & 0xffff; + end = start + blocks - 1; + + if (blocks >> 16 || end < start) { + ctr |= 0xffff; + datalen = AES_BLOCK_SIZE * (0x10000 - start); + fragmented = true; } + use_dma = (datalen >= ATMEL_AES_DMA_THRESHOLD); /* Jump to offset. */ @@ -1125,10 +1120,12 @@ static int atmel_aes_crypt(struct ablkcipher_request *req, unsigned long mode) rctx->mode = mode; if (!(mode & AES_FLAGS_ENCRYPT) && (req->src == req->dst)) { - int ivsize = crypto_ablkcipher_ivsize(ablkcipher); + unsigned int ivsize = crypto_ablkcipher_ivsize(ablkcipher); - scatterwalk_map_and_copy(rctx->lastc, req->src, - (req->nbytes - ivsize), ivsize, 0); + if (req->nbytes >= ivsize) + scatterwalk_map_and_copy(rctx->lastc, req->src, + req->nbytes - ivsize, + ivsize, 0); } return atmel_aes_handle_queue(dd, &req->base); @@ -1973,7 +1970,7 @@ static struct crypto_alg aes_xts_alg = { } }; -#ifdef CONFIG_CRYPTO_DEV_ATMEL_AUTHENC +#if IS_ENABLED(CONFIG_CRYPTO_DEV_ATMEL_AUTHENC) /* authenc aead functions */ static int atmel_aes_authenc_start(struct atmel_aes_dev *dd); @@ -2460,7 +2457,7 @@ static void atmel_aes_unregister_algs(struct atmel_aes_dev *dd) { int i; -#ifdef CONFIG_CRYPTO_DEV_ATMEL_AUTHENC +#if IS_ENABLED(CONFIG_CRYPTO_DEV_ATMEL_AUTHENC) if (dd->caps.has_authenc) for (i = 0; i < ARRAY_SIZE(aes_authenc_algs); i++) crypto_unregister_aead(&aes_authenc_algs[i]); @@ -2507,7 +2504,7 @@ static int atmel_aes_register_algs(struct atmel_aes_dev *dd) goto err_aes_xts_alg; } -#ifdef CONFIG_CRYPTO_DEV_ATMEL_AUTHENC +#if IS_ENABLED(CONFIG_CRYPTO_DEV_ATMEL_AUTHENC) if (dd->caps.has_authenc) { for (i = 0; i < ARRAY_SIZE(aes_authenc_algs); i++) { err = crypto_register_aead(&aes_authenc_algs[i]); @@ -2519,7 +2516,7 @@ static int atmel_aes_register_algs(struct atmel_aes_dev *dd) return 0; -#ifdef CONFIG_CRYPTO_DEV_ATMEL_AUTHENC +#if IS_ENABLED(CONFIG_CRYPTO_DEV_ATMEL_AUTHENC) /* i = ARRAY_SIZE(aes_authenc_algs); */ err_aes_authenc_alg: for (j = 0; j < i; j++) @@ -2543,7 +2540,6 @@ static void atmel_aes_get_cap(struct atmel_aes_dev *dd) { dd->caps.has_dualbuff = 0; dd->caps.has_cfb64 = 0; - dd->caps.has_ctr32 = 0; dd->caps.has_gcm = 0; dd->caps.has_xts = 0; dd->caps.has_authenc = 0; @@ -2554,7 +2550,6 @@ static void atmel_aes_get_cap(struct atmel_aes_dev *dd) case 0x500: dd->caps.has_dualbuff = 1; dd->caps.has_cfb64 = 1; - dd->caps.has_ctr32 = 1; dd->caps.has_gcm = 1; dd->caps.has_xts = 1; dd->caps.has_authenc = 1; @@ -2563,7 +2558,6 @@ static void atmel_aes_get_cap(struct atmel_aes_dev *dd) case 0x200: dd->caps.has_dualbuff = 1; dd->caps.has_cfb64 = 1; - dd->caps.has_ctr32 = 1; dd->caps.has_gcm = 1; dd->caps.max_burst_size = 4; break; @@ -2710,7 +2704,7 @@ static int atmel_aes_probe(struct platform_device *pdev) atmel_aes_get_cap(aes_dd); -#ifdef CONFIG_CRYPTO_DEV_ATMEL_AUTHENC +#if IS_ENABLED(CONFIG_CRYPTO_DEV_ATMEL_AUTHENC) if (aes_dd->caps.has_authenc && !atmel_sha_authenc_is_ready()) { err = -EPROBE_DEFER; goto iclk_unprepare; diff --git a/drivers/crypto/atmel-authenc.h b/drivers/crypto/atmel-authenc.h index cbd37a2edada..d6de810df44f 100644 --- a/drivers/crypto/atmel-authenc.h +++ b/drivers/crypto/atmel-authenc.h @@ -12,7 +12,7 @@ #ifndef __ATMEL_AUTHENC_H__ #define __ATMEL_AUTHENC_H__ -#ifdef CONFIG_CRYPTO_DEV_ATMEL_AUTHENC +#if IS_ENABLED(CONFIG_CRYPTO_DEV_ATMEL_AUTHENC) #include <crypto/authenc.h> #include <crypto/hash.h> diff --git a/drivers/crypto/atmel-sha.c b/drivers/crypto/atmel-sha.c index ab0cfe748931..47c291cb2aa9 100644 --- a/drivers/crypto/atmel-sha.c +++ b/drivers/crypto/atmel-sha.c @@ -1918,12 +1918,7 @@ static int atmel_sha_hmac_setkey(struct crypto_ahash *tfm, const u8 *key, { struct atmel_sha_hmac_ctx *hmac = crypto_ahash_ctx(tfm); - if (atmel_sha_hmac_key_set(&hmac->hkey, key, keylen)) { - crypto_ahash_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN); - return -EINVAL; - } - - return 0; + return atmel_sha_hmac_key_set(&hmac->hkey, key, keylen); } static int atmel_sha_hmac_init(struct ahash_request *req) @@ -2212,7 +2207,7 @@ static struct ahash_alg sha_hmac_algs[] = { }, }; -#ifdef CONFIG_CRYPTO_DEV_ATMEL_AUTHENC +#if IS_ENABLED(CONFIG_CRYPTO_DEV_ATMEL_AUTHENC) /* authenc functions */ static int atmel_sha_authenc_init2(struct atmel_sha_dev *dd); diff --git a/drivers/crypto/axis/artpec6_crypto.c b/drivers/crypto/axis/artpec6_crypto.c index 80fa04ef215f..5c69dcbc53c4 100644 --- a/drivers/crypto/axis/artpec6_crypto.c +++ b/drivers/crypto/axis/artpec6_crypto.c @@ -1251,7 +1251,7 @@ static int artpec6_crypto_aead_set_key(struct crypto_aead *tfm, const u8 *key, if (len != 16 && len != 24 && len != 32) { crypto_aead_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN); - return -1; + return -EINVAL; } ctx->key_length = len; diff --git a/drivers/crypto/caam/caamalg_desc.c b/drivers/crypto/caam/caamalg_desc.c index bb47ae5dca38..2cae1824bc06 100644 --- a/drivers/crypto/caam/caamalg_desc.c +++ b/drivers/crypto/caam/caamalg_desc.c @@ -1512,7 +1512,13 @@ EXPORT_SYMBOL(cnstr_shdsc_skcipher_decap); */ void cnstr_shdsc_xts_skcipher_encap(u32 * const desc, struct alginfo *cdata) { - __be64 sector_size = cpu_to_be64(512); + /* + * Set sector size to a big value, practically disabling + * sector size segmentation in xts implementation. We cannot + * take full advantage of this HW feature with existing + * crypto API / dm-crypt SW architecture. + */ + __be64 sector_size = cpu_to_be64(BIT(15)); u32 *key_jump_cmd; init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX); @@ -1563,7 +1569,13 @@ EXPORT_SYMBOL(cnstr_shdsc_xts_skcipher_encap); */ void cnstr_shdsc_xts_skcipher_decap(u32 * const desc, struct alginfo *cdata) { - __be64 sector_size = cpu_to_be64(512); + /* + * Set sector size to a big value, practically disabling + * sector size segmentation in xts implementation. We cannot + * take full advantage of this HW feature with existing + * crypto API / dm-crypt SW architecture. + */ + __be64 sector_size = cpu_to_be64(BIT(15)); u32 *key_jump_cmd; init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX); diff --git a/drivers/crypto/caam/caamalg_qi2.c b/drivers/crypto/caam/caamalg_qi2.c index b949944c8e55..039e69d91347 100644 --- a/drivers/crypto/caam/caamalg_qi2.c +++ b/drivers/crypto/caam/caamalg_qi2.c @@ -2395,7 +2395,7 @@ static struct caam_aead_alg driver_aeads[] = { .cra_name = "echainiv(authenc(hmac(sha256)," "cbc(des)))", .cra_driver_name = "echainiv-authenc-" - "hmac-sha256-cbc-desi-" + "hmac-sha256-cbc-des-" "caam-qi2", .cra_blocksize = DES_BLOCK_SIZE, }, diff --git a/drivers/crypto/cavium/cpt/cptvf_algs.c b/drivers/crypto/cavium/cpt/cptvf_algs.c index e9f4704494fb..10fd8f314c1c 100644 --- a/drivers/crypto/cavium/cpt/cptvf_algs.c +++ b/drivers/crypto/cavium/cpt/cptvf_algs.c @@ -201,6 +201,7 @@ static inline int cvm_enc_dec(struct ablkcipher_request *req, u32 enc) int status; memset(req_info, 0, sizeof(struct cpt_request_info)); + req_info->may_sleep = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) != 0; memset(fctx, 0, sizeof(struct fc_context)); create_input_list(req, enc, enc_iv_len); create_output_list(req, enc_iv_len); diff --git a/drivers/crypto/cavium/cpt/cptvf_reqmanager.c b/drivers/crypto/cavium/cpt/cptvf_reqmanager.c index 7a24019356b5..e343249c8d05 100644 --- a/drivers/crypto/cavium/cpt/cptvf_reqmanager.c +++ b/drivers/crypto/cavium/cpt/cptvf_reqmanager.c @@ -133,7 +133,7 @@ static inline int setup_sgio_list(struct cpt_vf *cptvf, /* Setup gather (input) components */ g_sz_bytes = ((req->incnt + 3) / 4) * sizeof(struct sglist_component); - info->gather_components = kzalloc(g_sz_bytes, GFP_KERNEL); + info->gather_components = kzalloc(g_sz_bytes, req->may_sleep ? GFP_KERNEL : GFP_ATOMIC); if (!info->gather_components) { ret = -ENOMEM; goto scatter_gather_clean; @@ -150,7 +150,7 @@ static inline int setup_sgio_list(struct cpt_vf *cptvf, /* Setup scatter (output) components */ s_sz_bytes = ((req->outcnt + 3) / 4) * sizeof(struct sglist_component); - info->scatter_components = kzalloc(s_sz_bytes, GFP_KERNEL); + info->scatter_components = kzalloc(s_sz_bytes, req->may_sleep ? GFP_KERNEL : GFP_ATOMIC); if (!info->scatter_components) { ret = -ENOMEM; goto scatter_gather_clean; @@ -167,7 +167,7 @@ static inline int setup_sgio_list(struct cpt_vf *cptvf, /* Create and initialize DPTR */ info->dlen = g_sz_bytes + s_sz_bytes + SG_LIST_HDR_SIZE; - info->in_buffer = kzalloc(info->dlen, GFP_KERNEL); + info->in_buffer = kzalloc(info->dlen, req->may_sleep ? GFP_KERNEL : GFP_ATOMIC); if (!info->in_buffer) { ret = -ENOMEM; goto scatter_gather_clean; @@ -195,7 +195,7 @@ static inline int setup_sgio_list(struct cpt_vf *cptvf, } /* Create and initialize RPTR */ - info->out_buffer = kzalloc(COMPLETION_CODE_SIZE, GFP_KERNEL); + info->out_buffer = kzalloc(COMPLETION_CODE_SIZE, req->may_sleep ? GFP_KERNEL : GFP_ATOMIC); if (!info->out_buffer) { ret = -ENOMEM; goto scatter_gather_clean; @@ -421,7 +421,7 @@ int process_request(struct cpt_vf *cptvf, struct cpt_request_info *req) struct cpt_vq_command vq_cmd; union cpt_inst_s cptinst; - info = kzalloc(sizeof(*info), GFP_KERNEL); + info = kzalloc(sizeof(*info), req->may_sleep ? GFP_KERNEL : GFP_ATOMIC); if (unlikely(!info)) { dev_err(&pdev->dev, "Unable to allocate memory for info_buffer\n"); return -ENOMEM; @@ -443,7 +443,7 @@ int process_request(struct cpt_vf *cptvf, struct cpt_request_info *req) * Get buffer for union cpt_res_s response * structure and its physical address */ - info->completion_addr = kzalloc(sizeof(union cpt_res_s), GFP_KERNEL); + info->completion_addr = kzalloc(sizeof(union cpt_res_s), req->may_sleep ? GFP_KERNEL : GFP_ATOMIC); if (unlikely(!info->completion_addr)) { dev_err(&pdev->dev, "Unable to allocate memory for completion_addr\n"); ret = -ENOMEM; diff --git a/drivers/crypto/cavium/cpt/request_manager.h b/drivers/crypto/cavium/cpt/request_manager.h index 3514b082eca7..1e8dd9ebcc17 100644 --- a/drivers/crypto/cavium/cpt/request_manager.h +++ b/drivers/crypto/cavium/cpt/request_manager.h @@ -62,6 +62,8 @@ struct cpt_request_info { union ctrl_info ctrl; /* User control information */ struct cptvf_request req; /* Request Information (Core specific) */ + bool may_sleep; + struct buf_ptr in[MAX_BUF_CNT]; struct buf_ptr out[MAX_BUF_CNT]; diff --git a/drivers/crypto/cavium/nitrox/nitrox_main.c b/drivers/crypto/cavium/nitrox/nitrox_main.c index fe825d0ef9ca..2d11795e2561 100644 --- a/drivers/crypto/cavium/nitrox/nitrox_main.c +++ b/drivers/crypto/cavium/nitrox/nitrox_main.c @@ -191,7 +191,7 @@ static void nitrox_remove_from_devlist(struct nitrox_device *ndev) struct nitrox_device *nitrox_get_first_device(void) { - struct nitrox_device *ndev = NULL; + struct nitrox_device *ndev; mutex_lock(&devlist_lock); list_for_each_entry(ndev, &ndevlist, list) { @@ -199,7 +199,7 @@ struct nitrox_device *nitrox_get_first_device(void) break; } mutex_unlock(&devlist_lock); - if (!ndev) + if (&ndev->list == &ndevlist) return NULL; refcount_inc(&ndev->refcnt); diff --git a/drivers/crypto/ccp/Kconfig b/drivers/crypto/ccp/Kconfig index 48f3edc1e3fb..f711d95277f1 100644 --- a/drivers/crypto/ccp/Kconfig +++ b/drivers/crypto/ccp/Kconfig @@ -10,10 +10,9 @@ config CRYPTO_DEV_CCP_DD config CRYPTO_DEV_SP_CCP bool "Cryptographic Coprocessor device" default y - depends on CRYPTO_DEV_CCP_DD + depends on CRYPTO_DEV_CCP_DD && DMADEVICES select HW_RANDOM select DMA_ENGINE - select DMADEVICES select CRYPTO_SHA1 select CRYPTO_SHA256 help diff --git a/drivers/crypto/ccp/ccp-dev-v3.c b/drivers/crypto/ccp/ccp-dev-v3.c index 2b7d47ed5c74..490b47572278 100644 --- a/drivers/crypto/ccp/ccp-dev-v3.c +++ b/drivers/crypto/ccp/ccp-dev-v3.c @@ -587,6 +587,7 @@ const struct ccp_vdata ccpv3_platform = { .setup = NULL, .perform = &ccp3_actions, .offset = 0, + .rsamax = CCP_RSA_MAX_WIDTH, }; const struct ccp_vdata ccpv3 = { diff --git a/drivers/crypto/ccp/ccp-dev.h b/drivers/crypto/ccp/ccp-dev.h index 5e624920fd99..ee9d2ca5e651 100644 --- a/drivers/crypto/ccp/ccp-dev.h +++ b/drivers/crypto/ccp/ccp-dev.h @@ -468,6 +468,7 @@ struct ccp_sg_workarea { unsigned int sg_used; struct scatterlist *dma_sg; + struct scatterlist *dma_sg_head; struct device *dma_dev; unsigned int dma_count; enum dma_data_direction dma_dir; diff --git a/drivers/crypto/ccp/ccp-dmaengine.c b/drivers/crypto/ccp/ccp-dmaengine.c index 7f22a45bbc11..03817df17728 100644 --- a/drivers/crypto/ccp/ccp-dmaengine.c +++ b/drivers/crypto/ccp/ccp-dmaengine.c @@ -337,6 +337,7 @@ static struct ccp_dma_desc *ccp_alloc_dma_desc(struct ccp_dma_chan *chan, desc->tx_desc.flags = flags; desc->tx_desc.tx_submit = ccp_tx_submit; desc->ccp = chan->ccp; + INIT_LIST_HEAD(&desc->entry); INIT_LIST_HEAD(&desc->pending); INIT_LIST_HEAD(&desc->active); desc->status = DMA_IN_PROGRESS; diff --git a/drivers/crypto/ccp/ccp-ops.c b/drivers/crypto/ccp/ccp-ops.c index e12506c93a52..95d06cfb7421 100644 --- a/drivers/crypto/ccp/ccp-ops.c +++ b/drivers/crypto/ccp/ccp-ops.c @@ -64,7 +64,7 @@ static u32 ccp_gen_jobid(struct ccp_device *ccp) static void ccp_sg_free(struct ccp_sg_workarea *wa) { if (wa->dma_count) - dma_unmap_sg(wa->dma_dev, wa->dma_sg, wa->nents, wa->dma_dir); + dma_unmap_sg(wa->dma_dev, wa->dma_sg_head, wa->nents, wa->dma_dir); wa->dma_count = 0; } @@ -93,6 +93,7 @@ static int ccp_init_sg_workarea(struct ccp_sg_workarea *wa, struct device *dev, return 0; wa->dma_sg = sg; + wa->dma_sg_head = sg; wa->dma_dev = dev; wa->dma_dir = dma_dir; wa->dma_count = dma_map_sg(dev, sg, wa->nents, dma_dir); @@ -105,14 +106,28 @@ static int ccp_init_sg_workarea(struct ccp_sg_workarea *wa, struct device *dev, static void ccp_update_sg_workarea(struct ccp_sg_workarea *wa, unsigned int len) { unsigned int nbytes = min_t(u64, len, wa->bytes_left); + unsigned int sg_combined_len = 0; if (!wa->sg) return; wa->sg_used += nbytes; wa->bytes_left -= nbytes; - if (wa->sg_used == wa->sg->length) { - wa->sg = sg_next(wa->sg); + if (wa->sg_used == sg_dma_len(wa->dma_sg)) { + /* Advance to the next DMA scatterlist entry */ + wa->dma_sg = sg_next(wa->dma_sg); + + /* In the case that the DMA mapped scatterlist has entries + * that have been merged, the non-DMA mapped scatterlist + * must be advanced multiple times for each merged entry. + * This ensures that the current non-DMA mapped entry + * corresponds to the current DMA mapped entry. + */ + do { + sg_combined_len += wa->sg->length; + wa->sg = sg_next(wa->sg); + } while (wa->sg_used > sg_combined_len); + wa->sg_used = 0; } } @@ -301,7 +316,7 @@ static unsigned int ccp_queue_buf(struct ccp_data *data, unsigned int from) /* Update the structures and generate the count */ buf_count = 0; while (sg_wa->bytes_left && (buf_count < dm_wa->length)) { - nbytes = min(sg_wa->sg->length - sg_wa->sg_used, + nbytes = min(sg_dma_len(sg_wa->dma_sg) - sg_wa->sg_used, dm_wa->length - buf_count); nbytes = min_t(u64, sg_wa->bytes_left, nbytes); @@ -333,11 +348,11 @@ static void ccp_prepare_data(struct ccp_data *src, struct ccp_data *dst, * and destination. The resulting len values will always be <= UINT_MAX * because the dma length is an unsigned int. */ - sg_src_len = sg_dma_len(src->sg_wa.sg) - src->sg_wa.sg_used; + sg_src_len = sg_dma_len(src->sg_wa.dma_sg) - src->sg_wa.sg_used; sg_src_len = min_t(u64, src->sg_wa.bytes_left, sg_src_len); if (dst) { - sg_dst_len = sg_dma_len(dst->sg_wa.sg) - dst->sg_wa.sg_used; + sg_dst_len = sg_dma_len(dst->sg_wa.dma_sg) - dst->sg_wa.sg_used; sg_dst_len = min_t(u64, src->sg_wa.bytes_left, sg_dst_len); op_len = min(sg_src_len, sg_dst_len); } else { @@ -367,7 +382,7 @@ static void ccp_prepare_data(struct ccp_data *src, struct ccp_data *dst, /* Enough data in the sg element, but we need to * adjust for any previously copied data */ - op->src.u.dma.address = sg_dma_address(src->sg_wa.sg); + op->src.u.dma.address = sg_dma_address(src->sg_wa.dma_sg); op->src.u.dma.offset = src->sg_wa.sg_used; op->src.u.dma.length = op_len & ~(block_size - 1); @@ -388,7 +403,7 @@ static void ccp_prepare_data(struct ccp_data *src, struct ccp_data *dst, /* Enough room in the sg element, but we need to * adjust for any previously used area */ - op->dst.u.dma.address = sg_dma_address(dst->sg_wa.sg); + op->dst.u.dma.address = sg_dma_address(dst->sg_wa.dma_sg); op->dst.u.dma.offset = dst->sg_wa.sg_used; op->dst.u.dma.length = op->src.u.dma.length; } @@ -2040,7 +2055,7 @@ static int ccp_run_passthru_cmd(struct ccp_cmd_queue *cmd_q, dst.sg_wa.sg_used = 0; for (i = 1; i <= src.sg_wa.dma_count; i++) { if (!dst.sg_wa.sg || - (dst.sg_wa.sg->length < src.sg_wa.sg->length)) { + (sg_dma_len(dst.sg_wa.sg) < sg_dma_len(src.sg_wa.sg))) { ret = -EINVAL; goto e_dst; } @@ -2066,8 +2081,8 @@ static int ccp_run_passthru_cmd(struct ccp_cmd_queue *cmd_q, goto e_dst; } - dst.sg_wa.sg_used += src.sg_wa.sg->length; - if (dst.sg_wa.sg_used == dst.sg_wa.sg->length) { + dst.sg_wa.sg_used += sg_dma_len(src.sg_wa.sg); + if (dst.sg_wa.sg_used == sg_dma_len(dst.sg_wa.sg)) { dst.sg_wa.sg = sg_next(dst.sg_wa.sg); dst.sg_wa.sg_used = 0; } diff --git a/drivers/crypto/ccree/cc_buffer_mgr.c b/drivers/crypto/ccree/cc_buffer_mgr.c index c81ad33f9115..b7a043178e0c 100644 --- a/drivers/crypto/ccree/cc_buffer_mgr.c +++ b/drivers/crypto/ccree/cc_buffer_mgr.c @@ -87,6 +87,8 @@ static unsigned int cc_get_sgl_nents(struct device *dev, { unsigned int nents = 0; + *lbytes = 0; + while (nbytes && sg_list) { nents++; /* get the number of bytes in the last entry */ @@ -95,6 +97,7 @@ static unsigned int cc_get_sgl_nents(struct device *dev, nbytes : sg_list->length; sg_list = sg_next(sg_list); } + dev_dbg(dev, "nents %d last bytes %d\n", nents, *lbytes); return nents; } @@ -311,37 +314,25 @@ static int cc_map_sg(struct device *dev, struct scatterlist *sg, unsigned int nbytes, int direction, u32 *nents, u32 max_sg_nents, u32 *lbytes, u32 *mapped_nents) { - if (sg_is_last(sg)) { - /* One entry only case -set to DLLI */ - if (dma_map_sg(dev, sg, 1, direction) != 1) { - dev_err(dev, "dma_map_sg() single buffer failed\n"); - return -ENOMEM; - } - dev_dbg(dev, "Mapped sg: dma_address=%pad page=%p addr=%pK offset=%u length=%u\n", - &sg_dma_address(sg), sg_page(sg), sg_virt(sg), - sg->offset, sg->length); - *lbytes = nbytes; - *nents = 1; - *mapped_nents = 1; - } else { /*sg_is_last*/ - *nents = cc_get_sgl_nents(dev, sg, nbytes, lbytes); - if (*nents > max_sg_nents) { - *nents = 0; - dev_err(dev, "Too many fragments. current %d max %d\n", - *nents, max_sg_nents); - return -ENOMEM; - } - /* In case of mmu the number of mapped nents might - * be changed from the original sgl nents - */ - *mapped_nents = dma_map_sg(dev, sg, *nents, direction); - if (*mapped_nents == 0) { - *nents = 0; - dev_err(dev, "dma_map_sg() sg buffer failed\n"); - return -ENOMEM; - } + int ret = 0; + + *nents = cc_get_sgl_nents(dev, sg, nbytes, lbytes); + if (*nents > max_sg_nents) { + *nents = 0; + dev_err(dev, "Too many fragments. current %d max %d\n", + *nents, max_sg_nents); + return -ENOMEM; + } + + ret = dma_map_sg(dev, sg, *nents, direction); + if (dma_mapping_error(dev, ret)) { + *nents = 0; + dev_err(dev, "dma_map_sg() sg buffer failed %d\n", ret); + return -ENOMEM; } + *mapped_nents = ret; + return 0; } @@ -576,11 +567,12 @@ void cc_unmap_aead_request(struct device *dev, struct aead_request *req) sg_virt(req->src), areq_ctx->src.nents, areq_ctx->assoc.nents, areq_ctx->assoclen, req->cryptlen); - dma_unmap_sg(dev, req->src, sg_nents(req->src), DMA_BIDIRECTIONAL); + dma_unmap_sg(dev, req->src, areq_ctx->src.mapped_nents, + DMA_BIDIRECTIONAL); if (req->src != req->dst) { dev_dbg(dev, "Unmapping dst sgl: req->dst=%pK\n", sg_virt(req->dst)); - dma_unmap_sg(dev, req->dst, sg_nents(req->dst), + dma_unmap_sg(dev, req->dst, areq_ctx->dst.mapped_nents, DMA_BIDIRECTIONAL); } if (drvdata->coherent && @@ -902,7 +894,7 @@ static int cc_aead_chain_data(struct cc_drvdata *drvdata, &src_last_bytes); sg_index = areq_ctx->src_sgl->length; //check where the data starts - while (sg_index <= size_to_skip) { + while (src_mapped_nents && (sg_index <= size_to_skip)) { src_mapped_nents--; offset -= areq_ctx->src_sgl->length; sgl = sg_next(areq_ctx->src_sgl); @@ -923,13 +915,17 @@ static int cc_aead_chain_data(struct cc_drvdata *drvdata, if (req->src != req->dst) { size_for_map = areq_ctx->assoclen + req->cryptlen; - size_for_map += (direct == DRV_CRYPTO_DIRECTION_ENCRYPT) ? - authsize : 0; + + if (direct == DRV_CRYPTO_DIRECTION_ENCRYPT) + size_for_map += authsize; + else + size_for_map -= authsize; + if (is_gcm4543) size_for_map += crypto_aead_ivsize(tfm); rc = cc_map_sg(dev, req->dst, size_for_map, DMA_BIDIRECTIONAL, - &areq_ctx->dst.nents, + &areq_ctx->dst.mapped_nents, LLI_MAX_NUM_OF_DATA_ENTRIES, &dst_last_bytes, &dst_mapped_nents); if (rc) @@ -942,7 +938,7 @@ static int cc_aead_chain_data(struct cc_drvdata *drvdata, offset = size_to_skip; //check where the data starts - while (sg_index <= size_to_skip) { + while (dst_mapped_nents && sg_index <= size_to_skip) { dst_mapped_nents--; offset -= areq_ctx->dst_sgl->length; sgl = sg_next(areq_ctx->dst_sgl); @@ -1138,13 +1134,15 @@ int cc_map_aead_request(struct cc_drvdata *drvdata, struct aead_request *req) } size_to_map = req->cryptlen + areq_ctx->assoclen; - if (areq_ctx->gen_ctx.op_type == DRV_CRYPTO_DIRECTION_ENCRYPT) + /* If we do in-place encryption, we also need the auth tag */ + if ((areq_ctx->gen_ctx.op_type == DRV_CRYPTO_DIRECTION_ENCRYPT) && + (req->src == req->dst)) { size_to_map += authsize; - + } if (is_gcm4543) size_to_map += crypto_aead_ivsize(tfm); rc = cc_map_sg(dev, req->src, size_to_map, DMA_BIDIRECTIONAL, - &areq_ctx->src.nents, + &areq_ctx->src.mapped_nents, (LLI_MAX_NUM_OF_ASSOC_DATA_ENTRIES + LLI_MAX_NUM_OF_DATA_ENTRIES), &dummy, &mapped_nents); diff --git a/drivers/crypto/ccree/cc_buffer_mgr.h b/drivers/crypto/ccree/cc_buffer_mgr.h index a726016bdbc1..846b262a9029 100644 --- a/drivers/crypto/ccree/cc_buffer_mgr.h +++ b/drivers/crypto/ccree/cc_buffer_mgr.h @@ -25,6 +25,7 @@ enum cc_sg_cpy_direct { struct cc_mlli { cc_sram_addr_t sram_addr; + unsigned int mapped_nents; unsigned int nents; //sg nents unsigned int mlli_nents; //mlli nents might be different than the above }; diff --git a/drivers/crypto/ccree/cc_cipher.c b/drivers/crypto/ccree/cc_cipher.c index 5b58226ea24d..5604e188d2d4 100644 --- a/drivers/crypto/ccree/cc_cipher.c +++ b/drivers/crypto/ccree/cc_cipher.c @@ -167,7 +167,6 @@ static int cc_cipher_init(struct crypto_tfm *tfm) skcipher_alg.base); struct device *dev = drvdata_to_dev(cc_alg->drvdata); unsigned int max_key_buf_size = cc_alg->skcipher_alg.max_keysize; - int rc = 0; dev_dbg(dev, "Initializing context @%p for %s\n", ctx_p, crypto_tfm_alg_name(tfm)); @@ -179,10 +178,19 @@ static int cc_cipher_init(struct crypto_tfm *tfm) ctx_p->flow_mode = cc_alg->flow_mode; ctx_p->drvdata = cc_alg->drvdata; + if (ctx_p->cipher_mode == DRV_CIPHER_ESSIV) { + /* Alloc hash tfm for essiv */ + ctx_p->shash_tfm = crypto_alloc_shash("sha256-generic", 0, 0); + if (IS_ERR(ctx_p->shash_tfm)) { + dev_err(dev, "Error allocating hash tfm for ESSIV.\n"); + return PTR_ERR(ctx_p->shash_tfm); + } + } + /* Allocate key buffer, cache line aligned */ ctx_p->user.key = kmalloc(max_key_buf_size, GFP_KERNEL); if (!ctx_p->user.key) - return -ENOMEM; + goto free_shash; dev_dbg(dev, "Allocated key buffer in context. key=@%p\n", ctx_p->user.key); @@ -194,21 +202,19 @@ static int cc_cipher_init(struct crypto_tfm *tfm) if (dma_mapping_error(dev, ctx_p->user.key_dma_addr)) { dev_err(dev, "Mapping Key %u B at va=%pK for DMA failed\n", max_key_buf_size, ctx_p->user.key); - return -ENOMEM; + goto free_key; } dev_dbg(dev, "Mapped key %u B at va=%pK to dma=%pad\n", max_key_buf_size, ctx_p->user.key, &ctx_p->user.key_dma_addr); - if (ctx_p->cipher_mode == DRV_CIPHER_ESSIV) { - /* Alloc hash tfm for essiv */ - ctx_p->shash_tfm = crypto_alloc_shash("sha256-generic", 0, 0); - if (IS_ERR(ctx_p->shash_tfm)) { - dev_err(dev, "Error allocating hash tfm for ESSIV.\n"); - return PTR_ERR(ctx_p->shash_tfm); - } - } + return 0; - return rc; +free_key: + kfree(ctx_p->user.key); +free_shash: + crypto_free_shash(ctx_p->shash_tfm); + + return -ENOMEM; } static void cc_cipher_exit(struct crypto_tfm *tfm) @@ -534,6 +540,7 @@ static void cc_setup_readiv_desc(struct crypto_tfm *tfm, } } + static void cc_setup_state_desc(struct crypto_tfm *tfm, struct cipher_req_ctx *req_ctx, unsigned int ivsize, unsigned int nbytes, @@ -545,8 +552,6 @@ static void cc_setup_state_desc(struct crypto_tfm *tfm, int cipher_mode = ctx_p->cipher_mode; int flow_mode = ctx_p->flow_mode; int direction = req_ctx->gen_ctx.op_type; - dma_addr_t key_dma_addr = ctx_p->user.key_dma_addr; - unsigned int key_len = ctx_p->keylen; dma_addr_t iv_dma_addr = req_ctx->gen_ctx.iv_dma_addr; unsigned int du_size = nbytes; @@ -582,6 +587,47 @@ static void cc_setup_state_desc(struct crypto_tfm *tfm, case DRV_CIPHER_XTS: case DRV_CIPHER_ESSIV: case DRV_CIPHER_BITLOCKER: + break; + default: + dev_err(dev, "Unsupported cipher mode (%d)\n", cipher_mode); + } +} + + +static void cc_setup_xex_state_desc(struct crypto_tfm *tfm, + struct cipher_req_ctx *req_ctx, + unsigned int ivsize, unsigned int nbytes, + struct cc_hw_desc desc[], + unsigned int *seq_size) +{ + struct cc_cipher_ctx *ctx_p = crypto_tfm_ctx(tfm); + struct device *dev = drvdata_to_dev(ctx_p->drvdata); + int cipher_mode = ctx_p->cipher_mode; + int flow_mode = ctx_p->flow_mode; + int direction = req_ctx->gen_ctx.op_type; + dma_addr_t key_dma_addr = ctx_p->user.key_dma_addr; + unsigned int key_len = ctx_p->keylen; + dma_addr_t iv_dma_addr = req_ctx->gen_ctx.iv_dma_addr; + unsigned int du_size = nbytes; + + struct cc_crypto_alg *cc_alg = + container_of(tfm->__crt_alg, struct cc_crypto_alg, + skcipher_alg.base); + + if (cc_alg->data_unit) + du_size = cc_alg->data_unit; + + switch (cipher_mode) { + case DRV_CIPHER_ECB: + break; + case DRV_CIPHER_CBC: + case DRV_CIPHER_CBC_CTS: + case DRV_CIPHER_CTR: + case DRV_CIPHER_OFB: + break; + case DRV_CIPHER_XTS: + case DRV_CIPHER_ESSIV: + case DRV_CIPHER_BITLOCKER: /* load XEX key */ hw_desc_init(&desc[*seq_size]); set_cipher_mode(&desc[*seq_size], cipher_mode); @@ -892,12 +938,14 @@ static int cc_cipher_process(struct skcipher_request *req, /* STAT_PHASE_2: Create sequence */ - /* Setup IV and XEX key used */ + /* Setup state (IV) */ cc_setup_state_desc(tfm, req_ctx, ivsize, nbytes, desc, &seq_len); /* Setup MLLI line, if needed */ cc_setup_mlli_desc(tfm, req_ctx, dst, src, nbytes, req, desc, &seq_len); /* Setup key */ cc_setup_key_desc(tfm, req_ctx, nbytes, desc, &seq_len); + /* Setup state (IV and XEX key) */ + cc_setup_xex_state_desc(tfm, req_ctx, ivsize, nbytes, desc, &seq_len); /* Data processing */ cc_setup_flow_desc(tfm, req_ctx, dst, src, nbytes, desc, &seq_len); /* Read next IV */ diff --git a/drivers/crypto/ccree/cc_driver.h b/drivers/crypto/ccree/cc_driver.h index b76181335c08..bfa485dce342 100644 --- a/drivers/crypto/ccree/cc_driver.h +++ b/drivers/crypto/ccree/cc_driver.h @@ -166,6 +166,7 @@ struct cc_drvdata { int std_bodies; bool sec_disabled; u32 comp_mask; + bool pm_on; }; struct cc_crypto_alg { diff --git a/drivers/crypto/ccree/cc_pm.c b/drivers/crypto/ccree/cc_pm.c index 2dad9c9543c6..759ef4d585ad 100644 --- a/drivers/crypto/ccree/cc_pm.c +++ b/drivers/crypto/ccree/cc_pm.c @@ -23,14 +23,8 @@ const struct dev_pm_ops ccree_pm = { int cc_pm_suspend(struct device *dev) { struct cc_drvdata *drvdata = dev_get_drvdata(dev); - int rc; dev_dbg(dev, "set HOST_POWER_DOWN_EN\n"); - rc = cc_suspend_req_queue(drvdata); - if (rc) { - dev_err(dev, "cc_suspend_req_queue (%x)\n", rc); - return rc; - } fini_cc_regs(drvdata); cc_iowrite(drvdata, CC_REG(HOST_POWER_DOWN_EN), POWER_DOWN_ENABLE); cc_clk_off(drvdata); @@ -59,13 +53,6 @@ int cc_pm_resume(struct device *dev) /* check if tee fips error occurred during power down */ cc_tee_handle_fips_error(drvdata); - rc = cc_resume_req_queue(drvdata); - if (rc) { - dev_err(dev, "cc_resume_req_queue (%x)\n", rc); - return rc; - } - - /* must be after the queue resuming as it uses the HW queue*/ cc_init_hash_sram(drvdata); cc_init_iv_sram(drvdata); @@ -77,12 +64,10 @@ int cc_pm_get(struct device *dev) int rc = 0; struct cc_drvdata *drvdata = dev_get_drvdata(dev); - if (cc_req_queue_suspended(drvdata)) + if (drvdata->pm_on) rc = pm_runtime_get_sync(dev); - else - pm_runtime_get_noresume(dev); - return rc; + return (rc == 1 ? 0 : rc); } int cc_pm_put_suspend(struct device *dev) @@ -90,14 +75,11 @@ int cc_pm_put_suspend(struct device *dev) int rc = 0; struct cc_drvdata *drvdata = dev_get_drvdata(dev); - if (!cc_req_queue_suspended(drvdata)) { + if (drvdata->pm_on) { pm_runtime_mark_last_busy(dev); rc = pm_runtime_put_autosuspend(dev); - } else { - /* Something wrong happens*/ - dev_err(dev, "request to suspend already suspended queue"); - rc = -EBUSY; } + return rc; } @@ -108,7 +90,7 @@ int cc_pm_init(struct cc_drvdata *drvdata) /* must be before the enabling to avoid resdundent suspending */ pm_runtime_set_autosuspend_delay(dev, CC_SUSPEND_TIMEOUT); pm_runtime_use_autosuspend(dev); - /* activate the PM module */ + /* set us as active - note we won't do PM ops until cc_pm_go()! */ return pm_runtime_set_active(dev); } @@ -116,9 +98,11 @@ int cc_pm_init(struct cc_drvdata *drvdata) void cc_pm_go(struct cc_drvdata *drvdata) { pm_runtime_enable(drvdata_to_dev(drvdata)); + drvdata->pm_on = true; } void cc_pm_fini(struct cc_drvdata *drvdata) { pm_runtime_disable(drvdata_to_dev(drvdata)); + drvdata->pm_on = false; } diff --git a/drivers/crypto/ccree/cc_request_mgr.c b/drivers/crypto/ccree/cc_request_mgr.c index 0bc6ccb0b899..31d6ccf9cb35 100644 --- a/drivers/crypto/ccree/cc_request_mgr.c +++ b/drivers/crypto/ccree/cc_request_mgr.c @@ -42,7 +42,6 @@ struct cc_req_mgr_handle { #else struct tasklet_struct comptask; #endif - bool is_runtime_suspended; }; struct cc_bl_item { @@ -436,6 +435,7 @@ static void cc_proc_backlog(struct cc_drvdata *drvdata) spin_lock(&mgr->bl_lock); list_del(&bli->list); --mgr->bl_len; + kfree(bli); } spin_unlock(&mgr->bl_lock); @@ -712,52 +712,3 @@ static void comp_handler(unsigned long devarg) cc_proc_backlog(drvdata); dev_dbg(dev, "Comp. handler done.\n"); } - -/* - * resume the queue configuration - no need to take the lock as this happens - * inside the spin lock protection - */ -#if defined(CONFIG_PM) -int cc_resume_req_queue(struct cc_drvdata *drvdata) -{ - struct cc_req_mgr_handle *request_mgr_handle = - drvdata->request_mgr_handle; - - spin_lock_bh(&request_mgr_handle->hw_lock); - request_mgr_handle->is_runtime_suspended = false; - spin_unlock_bh(&request_mgr_handle->hw_lock); - - return 0; -} - -/* - * suspend the queue configuration. Since it is used for the runtime suspend - * only verify that the queue can be suspended. - */ -int cc_suspend_req_queue(struct cc_drvdata *drvdata) -{ - struct cc_req_mgr_handle *request_mgr_handle = - drvdata->request_mgr_handle; - - /* lock the send_request */ - spin_lock_bh(&request_mgr_handle->hw_lock); - if (request_mgr_handle->req_queue_head != - request_mgr_handle->req_queue_tail) { - spin_unlock_bh(&request_mgr_handle->hw_lock); - return -EBUSY; - } - request_mgr_handle->is_runtime_suspended = true; - spin_unlock_bh(&request_mgr_handle->hw_lock); - - return 0; -} - -bool cc_req_queue_suspended(struct cc_drvdata *drvdata) -{ - struct cc_req_mgr_handle *request_mgr_handle = - drvdata->request_mgr_handle; - - return request_mgr_handle->is_runtime_suspended; -} - -#endif diff --git a/drivers/crypto/ccree/cc_request_mgr.h b/drivers/crypto/ccree/cc_request_mgr.h index f46cf766fe4d..ff7746aaaf35 100644 --- a/drivers/crypto/ccree/cc_request_mgr.h +++ b/drivers/crypto/ccree/cc_request_mgr.h @@ -40,12 +40,4 @@ void complete_request(struct cc_drvdata *drvdata); void cc_req_mgr_fini(struct cc_drvdata *drvdata); -#if defined(CONFIG_PM) -int cc_resume_req_queue(struct cc_drvdata *drvdata); - -int cc_suspend_req_queue(struct cc_drvdata *drvdata); - -bool cc_req_queue_suspended(struct cc_drvdata *drvdata); -#endif - #endif /*__REQUEST_MGR_H__*/ diff --git a/drivers/crypto/chelsio/chcr_algo.c b/drivers/crypto/chelsio/chcr_algo.c index 177f572b9589..6c3ca14ea3f6 100644 --- a/drivers/crypto/chelsio/chcr_algo.c +++ b/drivers/crypto/chelsio/chcr_algo.c @@ -2830,7 +2830,7 @@ static void fill_sec_cpl_for_aead(struct cpl_tx_sec_pdu *sec_cpl, unsigned int mac_mode = CHCR_SCMD_AUTH_MODE_CBCMAC; unsigned int c_id = a_ctx(tfm)->tx_chan_id; unsigned int ccm_xtra; - unsigned char tag_offset = 0, auth_offset = 0; + unsigned int tag_offset = 0, auth_offset = 0; unsigned int assoclen; if (get_aead_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309) @@ -3206,9 +3206,6 @@ static int chcr_gcm_setauthsize(struct crypto_aead *tfm, unsigned int authsize) aeadctx->mayverify = VERIFY_SW; break; default: - - crypto_tfm_set_flags((struct crypto_tfm *) tfm, - CRYPTO_TFM_RES_BAD_KEY_LEN); return -EINVAL; } return crypto_aead_setauthsize(aeadctx->sw_cipher, authsize); @@ -3233,8 +3230,6 @@ static int chcr_4106_4309_setauthsize(struct crypto_aead *tfm, aeadctx->mayverify = VERIFY_HW; break; default: - crypto_tfm_set_flags((struct crypto_tfm *)tfm, - CRYPTO_TFM_RES_BAD_KEY_LEN); return -EINVAL; } return crypto_aead_setauthsize(aeadctx->sw_cipher, authsize); @@ -3275,8 +3270,6 @@ static int chcr_ccm_setauthsize(struct crypto_aead *tfm, aeadctx->mayverify = VERIFY_HW; break; default: - crypto_tfm_set_flags((struct crypto_tfm *)tfm, - CRYPTO_TFM_RES_BAD_KEY_LEN); return -EINVAL; } return crypto_aead_setauthsize(aeadctx->sw_cipher, authsize); @@ -3301,8 +3294,7 @@ static int chcr_ccm_common_setkey(struct crypto_aead *aead, ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_256; mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_256; } else { - crypto_tfm_set_flags((struct crypto_tfm *)aead, - CRYPTO_TFM_RES_BAD_KEY_LEN); + crypto_aead_set_flags(aead, CRYPTO_TFM_RES_BAD_KEY_LEN); aeadctx->enckey_len = 0; return -EINVAL; } @@ -3340,8 +3332,7 @@ static int chcr_aead_rfc4309_setkey(struct crypto_aead *aead, const u8 *key, int error; if (keylen < 3) { - crypto_tfm_set_flags((struct crypto_tfm *)aead, - CRYPTO_TFM_RES_BAD_KEY_LEN); + crypto_aead_set_flags(aead, CRYPTO_TFM_RES_BAD_KEY_LEN); aeadctx->enckey_len = 0; return -EINVAL; } @@ -3391,8 +3382,7 @@ static int chcr_gcm_setkey(struct crypto_aead *aead, const u8 *key, } else if (keylen == AES_KEYSIZE_256) { ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_256; } else { - crypto_tfm_set_flags((struct crypto_tfm *)aead, - CRYPTO_TFM_RES_BAD_KEY_LEN); + crypto_aead_set_flags(aead, CRYPTO_TFM_RES_BAD_KEY_LEN); pr_err("GCM: Invalid key length %d\n", keylen); ret = -EINVAL; goto out; diff --git a/drivers/crypto/chelsio/chcr_core.c b/drivers/crypto/chelsio/chcr_core.c index 029a7354f541..5c16f368879b 100644 --- a/drivers/crypto/chelsio/chcr_core.c +++ b/drivers/crypto/chelsio/chcr_core.c @@ -125,8 +125,6 @@ static void chcr_dev_init(struct uld_ctx *u_ctx) atomic_set(&dev->inflight, 0); mutex_lock(&drv_data.drv_mutex); list_add_tail(&u_ctx->entry, &drv_data.inact_dev); - if (!drv_data.last_dev) - drv_data.last_dev = u_ctx; mutex_unlock(&drv_data.drv_mutex); } diff --git a/drivers/crypto/chelsio/chtls/chtls_cm.c b/drivers/crypto/chelsio/chtls/chtls_cm.c index 774d991d7cca..dffa2aa855fd 100644 --- a/drivers/crypto/chelsio/chtls/chtls_cm.c +++ b/drivers/crypto/chelsio/chtls/chtls_cm.c @@ -727,6 +727,14 @@ static int chtls_close_listsrv_rpl(struct chtls_dev *cdev, struct sk_buff *skb) return 0; } +static void chtls_purge_wr_queue(struct sock *sk) +{ + struct sk_buff *skb; + + while ((skb = dequeue_wr(sk)) != NULL) + kfree_skb(skb); +} + static void chtls_release_resources(struct sock *sk) { struct chtls_sock *csk = rcu_dereference_sk_user_data(sk); @@ -741,6 +749,11 @@ static void chtls_release_resources(struct sock *sk) kfree_skb(csk->txdata_skb_cache); csk->txdata_skb_cache = NULL; + if (csk->wr_credits != csk->wr_max_credits) { + chtls_purge_wr_queue(sk); + chtls_reset_wr_list(csk); + } + if (csk->l2t_entry) { cxgb4_l2t_release(csk->l2t_entry); csk->l2t_entry = NULL; @@ -1297,7 +1310,7 @@ static void make_established(struct sock *sk, u32 snd_isn, unsigned int opt) tp->write_seq = snd_isn; tp->snd_nxt = snd_isn; tp->snd_una = snd_isn; - inet_sk(sk)->inet_id = tp->write_seq ^ jiffies; + inet_sk(sk)->inet_id = prandom_u32(); assign_rxopt(sk, opt); if (tp->rcv_wnd > (RCV_BUFSIZ_M << 10)) @@ -1735,6 +1748,7 @@ static void chtls_peer_close(struct sock *sk, struct sk_buff *skb) else sk_wake_async(sk, SOCK_WAKE_WAITD, POLL_IN); } + kfree_skb(skb); } static void chtls_close_con_rpl(struct sock *sk, struct sk_buff *skb) @@ -2062,19 +2076,6 @@ rel_skb: return 0; } -static struct sk_buff *dequeue_wr(struct sock *sk) -{ - struct chtls_sock *csk = rcu_dereference_sk_user_data(sk); - struct sk_buff *skb = csk->wr_skb_head; - - if (likely(skb)) { - /* Don't bother clearing the tail */ - csk->wr_skb_head = WR_SKB_CB(skb)->next_wr; - WR_SKB_CB(skb)->next_wr = NULL; - } - return skb; -} - static void chtls_rx_ack(struct sock *sk, struct sk_buff *skb) { struct cpl_fw4_ack *hdr = cplhdr(skb) + RSS_HDR; diff --git a/drivers/crypto/chelsio/chtls/chtls_cm.h b/drivers/crypto/chelsio/chtls/chtls_cm.h index 129d7ac649a9..3fac0c74a41f 100644 --- a/drivers/crypto/chelsio/chtls/chtls_cm.h +++ b/drivers/crypto/chelsio/chtls/chtls_cm.h @@ -185,6 +185,12 @@ static inline void chtls_kfree_skb(struct sock *sk, struct sk_buff *skb) kfree_skb(skb); } +static inline void chtls_reset_wr_list(struct chtls_sock *csk) +{ + csk->wr_skb_head = NULL; + csk->wr_skb_tail = NULL; +} + static inline void enqueue_wr(struct chtls_sock *csk, struct sk_buff *skb) { WR_SKB_CB(skb)->next_wr = NULL; @@ -197,4 +203,19 @@ static inline void enqueue_wr(struct chtls_sock *csk, struct sk_buff *skb) WR_SKB_CB(csk->wr_skb_tail)->next_wr = skb; csk->wr_skb_tail = skb; } + +static inline struct sk_buff *dequeue_wr(struct sock *sk) +{ + struct chtls_sock *csk = rcu_dereference_sk_user_data(sk); + struct sk_buff *skb = NULL; + + skb = csk->wr_skb_head; + + if (likely(skb)) { + /* Don't bother clearing the tail */ + csk->wr_skb_head = WR_SKB_CB(skb)->next_wr; + WR_SKB_CB(skb)->next_wr = NULL; + } + return skb; +} #endif diff --git a/drivers/crypto/chelsio/chtls/chtls_hw.c b/drivers/crypto/chelsio/chtls/chtls_hw.c index f2424f4c5f78..0731bae155f2 100644 --- a/drivers/crypto/chelsio/chtls/chtls_hw.c +++ b/drivers/crypto/chelsio/chtls/chtls_hw.c @@ -358,6 +358,7 @@ int chtls_setkey(struct chtls_sock *csk, u32 keylen, u32 optname) kwr->sc_imm.cmd_more = cpu_to_be32(ULPTX_CMD_V(ULP_TX_SC_IMM)); kwr->sc_imm.len = cpu_to_be32(klen); + lock_sock(sk); /* key info */ kctx = (struct _key_ctx *)(kwr + 1); ret = chtls_key_info(csk, kctx, keylen, optname); @@ -396,8 +397,10 @@ int chtls_setkey(struct chtls_sock *csk, u32 keylen, u32 optname) csk->tlshws.txkey = keyid; } + release_sock(sk); return ret; out_notcb: + release_sock(sk); free_tls_keyid(sk); out_nokey: kfree_skb(skb); diff --git a/drivers/crypto/chelsio/chtls/chtls_io.c b/drivers/crypto/chelsio/chtls/chtls_io.c index 551bca6fef24..9b59cea09e03 100644 --- a/drivers/crypto/chelsio/chtls/chtls_io.c +++ b/drivers/crypto/chelsio/chtls/chtls_io.c @@ -682,7 +682,7 @@ int chtls_push_frames(struct chtls_sock *csk, int comp) make_tx_data_wr(sk, skb, immdlen, len, credits_needed, completion); tp->snd_nxt += len; - tp->lsndtime = tcp_time_stamp(tp); + tp->lsndtime = tcp_jiffies32; if (completion) ULP_SKB_CB(skb)->flags &= ~ULPCB_FLAG_NEED_HDR; } else { @@ -1701,7 +1701,7 @@ int chtls_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, return peekmsg(sk, msg, len, nonblock, flags); if (sk_can_busy_loop(sk) && - skb_queue_empty(&sk->sk_receive_queue) && + skb_queue_empty_lockless(&sk->sk_receive_queue) && sk->sk_state == TCP_ESTABLISHED) sk_busy_loop(sk, nonblock); diff --git a/drivers/crypto/geode-aes.c b/drivers/crypto/geode-aes.c index d81a1297cb9e..73a899e6f837 100644 --- a/drivers/crypto/geode-aes.c +++ b/drivers/crypto/geode-aes.c @@ -10,6 +10,7 @@ #include <linux/spinlock.h> #include <crypto/algapi.h> #include <crypto/aes.h> +#include <crypto/internal/skcipher.h> #include <linux/io.h> #include <linux/delay.h> @@ -23,12 +24,12 @@ static spinlock_t lock; /* Write a 128 bit field (either a writable key or IV) */ static inline void -_writefield(u32 offset, void *value) +_writefield(u32 offset, const void *value) { int i; for (i = 0; i < 4; i++) - iowrite32(((u32 *) value)[i], _iobase + offset + (i * 4)); + iowrite32(((const u32 *) value)[i], _iobase + offset + (i * 4)); } /* Read a 128 bit field (either a writable key or IV) */ @@ -42,12 +43,12 @@ _readfield(u32 offset, void *value) } static int -do_crypt(void *src, void *dst, int len, u32 flags) +do_crypt(const void *src, void *dst, u32 len, u32 flags) { u32 status; u32 counter = AES_OP_TIMEOUT; - iowrite32(virt_to_phys(src), _iobase + AES_SOURCEA_REG); + iowrite32(virt_to_phys((void *)src), _iobase + AES_SOURCEA_REG); iowrite32(virt_to_phys(dst), _iobase + AES_DSTA_REG); iowrite32(len, _iobase + AES_LENA_REG); @@ -64,16 +65,14 @@ do_crypt(void *src, void *dst, int len, u32 flags) return counter ? 0 : 1; } -static unsigned int -geode_aes_crypt(struct geode_aes_op *op) +static void +geode_aes_crypt(const struct geode_aes_tfm_ctx *tctx, const void *src, + void *dst, u32 len, u8 *iv, int mode, int dir) { u32 flags = 0; unsigned long iflags; int ret; - if (op->len == 0) - return 0; - /* If the source and destination is the same, then * we need to turn on the coherent flags, otherwise * we don't need to worry @@ -81,32 +80,28 @@ geode_aes_crypt(struct geode_aes_op *op) flags |= (AES_CTRL_DCA | AES_CTRL_SCA); - if (op->dir == AES_DIR_ENCRYPT) + if (dir == AES_DIR_ENCRYPT) flags |= AES_CTRL_ENCRYPT; /* Start the critical section */ spin_lock_irqsave(&lock, iflags); - if (op->mode == AES_MODE_CBC) { + if (mode == AES_MODE_CBC) { flags |= AES_CTRL_CBC; - _writefield(AES_WRITEIV0_REG, op->iv); + _writefield(AES_WRITEIV0_REG, iv); } - if (!(op->flags & AES_FLAGS_HIDDENKEY)) { - flags |= AES_CTRL_WRKEY; - _writefield(AES_WRITEKEY0_REG, op->key); - } + flags |= AES_CTRL_WRKEY; + _writefield(AES_WRITEKEY0_REG, tctx->key); - ret = do_crypt(op->src, op->dst, op->len, flags); + ret = do_crypt(src, dst, len, flags); BUG_ON(ret); - if (op->mode == AES_MODE_CBC) - _readfield(AES_WRITEIV0_REG, op->iv); + if (mode == AES_MODE_CBC) + _readfield(AES_WRITEIV0_REG, iv); spin_unlock_irqrestore(&lock, iflags); - - return op->len; } /* CRYPTO-API Functions */ @@ -114,13 +109,13 @@ geode_aes_crypt(struct geode_aes_op *op) static int geode_setkey_cip(struct crypto_tfm *tfm, const u8 *key, unsigned int len) { - struct geode_aes_op *op = crypto_tfm_ctx(tfm); + struct geode_aes_tfm_ctx *tctx = crypto_tfm_ctx(tfm); unsigned int ret; - op->keylen = len; + tctx->keylen = len; if (len == AES_KEYSIZE_128) { - memcpy(op->key, key, len); + memcpy(tctx->key, key, len); return 0; } @@ -133,135 +128,93 @@ static int geode_setkey_cip(struct crypto_tfm *tfm, const u8 *key, /* * The requested key size is not supported by HW, do a fallback */ - op->fallback.cip->base.crt_flags &= ~CRYPTO_TFM_REQ_MASK; - op->fallback.cip->base.crt_flags |= (tfm->crt_flags & CRYPTO_TFM_REQ_MASK); + tctx->fallback.cip->base.crt_flags &= ~CRYPTO_TFM_REQ_MASK; + tctx->fallback.cip->base.crt_flags |= + (tfm->crt_flags & CRYPTO_TFM_REQ_MASK); - ret = crypto_cipher_setkey(op->fallback.cip, key, len); + ret = crypto_cipher_setkey(tctx->fallback.cip, key, len); if (ret) { tfm->crt_flags &= ~CRYPTO_TFM_RES_MASK; - tfm->crt_flags |= (op->fallback.cip->base.crt_flags & CRYPTO_TFM_RES_MASK); + tfm->crt_flags |= (tctx->fallback.cip->base.crt_flags & + CRYPTO_TFM_RES_MASK); } return ret; } -static int geode_setkey_blk(struct crypto_tfm *tfm, const u8 *key, - unsigned int len) +static int geode_setkey_skcipher(struct crypto_skcipher *tfm, const u8 *key, + unsigned int len) { - struct geode_aes_op *op = crypto_tfm_ctx(tfm); + struct geode_aes_tfm_ctx *tctx = crypto_skcipher_ctx(tfm); unsigned int ret; - op->keylen = len; + tctx->keylen = len; if (len == AES_KEYSIZE_128) { - memcpy(op->key, key, len); + memcpy(tctx->key, key, len); return 0; } if (len != AES_KEYSIZE_192 && len != AES_KEYSIZE_256) { /* not supported at all */ - tfm->crt_flags |= CRYPTO_TFM_RES_BAD_KEY_LEN; + crypto_skcipher_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN); return -EINVAL; } /* * The requested key size is not supported by HW, do a fallback */ - op->fallback.blk->base.crt_flags &= ~CRYPTO_TFM_REQ_MASK; - op->fallback.blk->base.crt_flags |= (tfm->crt_flags & CRYPTO_TFM_REQ_MASK); - - ret = crypto_blkcipher_setkey(op->fallback.blk, key, len); - if (ret) { - tfm->crt_flags &= ~CRYPTO_TFM_RES_MASK; - tfm->crt_flags |= (op->fallback.blk->base.crt_flags & CRYPTO_TFM_RES_MASK); - } - return ret; -} - -static int fallback_blk_dec(struct blkcipher_desc *desc, - struct scatterlist *dst, struct scatterlist *src, - unsigned int nbytes) -{ - unsigned int ret; - struct crypto_blkcipher *tfm; - struct geode_aes_op *op = crypto_blkcipher_ctx(desc->tfm); - - tfm = desc->tfm; - desc->tfm = op->fallback.blk; - - ret = crypto_blkcipher_decrypt_iv(desc, dst, src, nbytes); - - desc->tfm = tfm; - return ret; -} -static int fallback_blk_enc(struct blkcipher_desc *desc, - struct scatterlist *dst, struct scatterlist *src, - unsigned int nbytes) -{ - unsigned int ret; - struct crypto_blkcipher *tfm; - struct geode_aes_op *op = crypto_blkcipher_ctx(desc->tfm); - - tfm = desc->tfm; - desc->tfm = op->fallback.blk; - - ret = crypto_blkcipher_encrypt_iv(desc, dst, src, nbytes); - - desc->tfm = tfm; + crypto_skcipher_clear_flags(tctx->fallback.skcipher, + CRYPTO_TFM_REQ_MASK); + crypto_skcipher_set_flags(tctx->fallback.skcipher, + crypto_skcipher_get_flags(tfm) & + CRYPTO_TFM_REQ_MASK); + ret = crypto_skcipher_setkey(tctx->fallback.skcipher, key, len); + crypto_skcipher_set_flags(tfm, + crypto_skcipher_get_flags(tctx->fallback.skcipher) & + CRYPTO_TFM_RES_MASK); return ret; } static void geode_encrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in) { - struct geode_aes_op *op = crypto_tfm_ctx(tfm); + const struct geode_aes_tfm_ctx *tctx = crypto_tfm_ctx(tfm); - if (unlikely(op->keylen != AES_KEYSIZE_128)) { - crypto_cipher_encrypt_one(op->fallback.cip, out, in); + if (unlikely(tctx->keylen != AES_KEYSIZE_128)) { + crypto_cipher_encrypt_one(tctx->fallback.cip, out, in); return; } - op->src = (void *) in; - op->dst = (void *) out; - op->mode = AES_MODE_ECB; - op->flags = 0; - op->len = AES_BLOCK_SIZE; - op->dir = AES_DIR_ENCRYPT; - - geode_aes_crypt(op); + geode_aes_crypt(tctx, in, out, AES_BLOCK_SIZE, NULL, + AES_MODE_ECB, AES_DIR_ENCRYPT); } static void geode_decrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in) { - struct geode_aes_op *op = crypto_tfm_ctx(tfm); + const struct geode_aes_tfm_ctx *tctx = crypto_tfm_ctx(tfm); - if (unlikely(op->keylen != AES_KEYSIZE_128)) { - crypto_cipher_decrypt_one(op->fallback.cip, out, in); + if (unlikely(tctx->keylen != AES_KEYSIZE_128)) { + crypto_cipher_decrypt_one(tctx->fallback.cip, out, in); return; } - op->src = (void *) in; - op->dst = (void *) out; - op->mode = AES_MODE_ECB; - op->flags = 0; - op->len = AES_BLOCK_SIZE; - op->dir = AES_DIR_DECRYPT; - - geode_aes_crypt(op); + geode_aes_crypt(tctx, in, out, AES_BLOCK_SIZE, NULL, + AES_MODE_ECB, AES_DIR_DECRYPT); } static int fallback_init_cip(struct crypto_tfm *tfm) { const char *name = crypto_tfm_alg_name(tfm); - struct geode_aes_op *op = crypto_tfm_ctx(tfm); + struct geode_aes_tfm_ctx *tctx = crypto_tfm_ctx(tfm); - op->fallback.cip = crypto_alloc_cipher(name, 0, - CRYPTO_ALG_NEED_FALLBACK); + tctx->fallback.cip = crypto_alloc_cipher(name, 0, + CRYPTO_ALG_NEED_FALLBACK); - if (IS_ERR(op->fallback.cip)) { + if (IS_ERR(tctx->fallback.cip)) { printk(KERN_ERR "Error allocating fallback algo %s\n", name); - return PTR_ERR(op->fallback.cip); + return PTR_ERR(tctx->fallback.cip); } return 0; @@ -269,10 +222,9 @@ static int fallback_init_cip(struct crypto_tfm *tfm) static void fallback_exit_cip(struct crypto_tfm *tfm) { - struct geode_aes_op *op = crypto_tfm_ctx(tfm); + struct geode_aes_tfm_ctx *tctx = crypto_tfm_ctx(tfm); - crypto_free_cipher(op->fallback.cip); - op->fallback.cip = NULL; + crypto_free_cipher(tctx->fallback.cip); } static struct crypto_alg geode_alg = { @@ -285,7 +237,7 @@ static struct crypto_alg geode_alg = { .cra_init = fallback_init_cip, .cra_exit = fallback_exit_cip, .cra_blocksize = AES_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct geode_aes_op), + .cra_ctxsize = sizeof(struct geode_aes_tfm_ctx), .cra_module = THIS_MODULE, .cra_u = { .cipher = { @@ -298,209 +250,126 @@ static struct crypto_alg geode_alg = { } }; -static int -geode_cbc_decrypt(struct blkcipher_desc *desc, - struct scatterlist *dst, struct scatterlist *src, - unsigned int nbytes) +static int geode_init_skcipher(struct crypto_skcipher *tfm) { - struct geode_aes_op *op = crypto_blkcipher_ctx(desc->tfm); - struct blkcipher_walk walk; - int err, ret; - - if (unlikely(op->keylen != AES_KEYSIZE_128)) - return fallback_blk_dec(desc, dst, src, nbytes); - - blkcipher_walk_init(&walk, dst, src, nbytes); - err = blkcipher_walk_virt(desc, &walk); - op->iv = walk.iv; - - while ((nbytes = walk.nbytes)) { - op->src = walk.src.virt.addr, - op->dst = walk.dst.virt.addr; - op->mode = AES_MODE_CBC; - op->len = nbytes - (nbytes % AES_BLOCK_SIZE); - op->dir = AES_DIR_DECRYPT; + const char *name = crypto_tfm_alg_name(&tfm->base); + struct geode_aes_tfm_ctx *tctx = crypto_skcipher_ctx(tfm); - ret = geode_aes_crypt(op); - - nbytes -= ret; - err = blkcipher_walk_done(desc, &walk, nbytes); + tctx->fallback.skcipher = + crypto_alloc_skcipher(name, 0, CRYPTO_ALG_NEED_FALLBACK | + CRYPTO_ALG_ASYNC); + if (IS_ERR(tctx->fallback.skcipher)) { + printk(KERN_ERR "Error allocating fallback algo %s\n", name); + return PTR_ERR(tctx->fallback.skcipher); } - return err; + crypto_skcipher_set_reqsize(tfm, sizeof(struct skcipher_request) + + crypto_skcipher_reqsize(tctx->fallback.skcipher)); + return 0; } -static int -geode_cbc_encrypt(struct blkcipher_desc *desc, - struct scatterlist *dst, struct scatterlist *src, - unsigned int nbytes) +static void geode_exit_skcipher(struct crypto_skcipher *tfm) { - struct geode_aes_op *op = crypto_blkcipher_ctx(desc->tfm); - struct blkcipher_walk walk; - int err, ret; - - if (unlikely(op->keylen != AES_KEYSIZE_128)) - return fallback_blk_enc(desc, dst, src, nbytes); - - blkcipher_walk_init(&walk, dst, src, nbytes); - err = blkcipher_walk_virt(desc, &walk); - op->iv = walk.iv; - - while ((nbytes = walk.nbytes)) { - op->src = walk.src.virt.addr, - op->dst = walk.dst.virt.addr; - op->mode = AES_MODE_CBC; - op->len = nbytes - (nbytes % AES_BLOCK_SIZE); - op->dir = AES_DIR_ENCRYPT; - - ret = geode_aes_crypt(op); - nbytes -= ret; - err = blkcipher_walk_done(desc, &walk, nbytes); - } + struct geode_aes_tfm_ctx *tctx = crypto_skcipher_ctx(tfm); - return err; + crypto_free_skcipher(tctx->fallback.skcipher); } -static int fallback_init_blk(struct crypto_tfm *tfm) +static int geode_skcipher_crypt(struct skcipher_request *req, int mode, int dir) { - const char *name = crypto_tfm_alg_name(tfm); - struct geode_aes_op *op = crypto_tfm_ctx(tfm); + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + const struct geode_aes_tfm_ctx *tctx = crypto_skcipher_ctx(tfm); + struct skcipher_walk walk; + unsigned int nbytes; + int err; + + if (unlikely(tctx->keylen != AES_KEYSIZE_128)) { + struct skcipher_request *subreq = skcipher_request_ctx(req); + + *subreq = *req; + skcipher_request_set_tfm(subreq, tctx->fallback.skcipher); + if (dir == AES_DIR_DECRYPT) + return crypto_skcipher_decrypt(subreq); + else + return crypto_skcipher_encrypt(subreq); + } - op->fallback.blk = crypto_alloc_blkcipher(name, 0, - CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK); + err = skcipher_walk_virt(&walk, req, false); - if (IS_ERR(op->fallback.blk)) { - printk(KERN_ERR "Error allocating fallback algo %s\n", name); - return PTR_ERR(op->fallback.blk); + while ((nbytes = walk.nbytes) != 0) { + geode_aes_crypt(tctx, walk.src.virt.addr, walk.dst.virt.addr, + round_down(nbytes, AES_BLOCK_SIZE), + walk.iv, mode, dir); + err = skcipher_walk_done(&walk, nbytes % AES_BLOCK_SIZE); } - return 0; + return err; } -static void fallback_exit_blk(struct crypto_tfm *tfm) +static int geode_cbc_encrypt(struct skcipher_request *req) { - struct geode_aes_op *op = crypto_tfm_ctx(tfm); - - crypto_free_blkcipher(op->fallback.blk); - op->fallback.blk = NULL; + return geode_skcipher_crypt(req, AES_MODE_CBC, AES_DIR_ENCRYPT); } -static struct crypto_alg geode_cbc_alg = { - .cra_name = "cbc(aes)", - .cra_driver_name = "cbc-aes-geode", - .cra_priority = 400, - .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER | - CRYPTO_ALG_KERN_DRIVER_ONLY | - CRYPTO_ALG_NEED_FALLBACK, - .cra_init = fallback_init_blk, - .cra_exit = fallback_exit_blk, - .cra_blocksize = AES_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct geode_aes_op), - .cra_alignmask = 15, - .cra_type = &crypto_blkcipher_type, - .cra_module = THIS_MODULE, - .cra_u = { - .blkcipher = { - .min_keysize = AES_MIN_KEY_SIZE, - .max_keysize = AES_MAX_KEY_SIZE, - .setkey = geode_setkey_blk, - .encrypt = geode_cbc_encrypt, - .decrypt = geode_cbc_decrypt, - .ivsize = AES_BLOCK_SIZE, - } - } -}; - -static int -geode_ecb_decrypt(struct blkcipher_desc *desc, - struct scatterlist *dst, struct scatterlist *src, - unsigned int nbytes) +static int geode_cbc_decrypt(struct skcipher_request *req) { - struct geode_aes_op *op = crypto_blkcipher_ctx(desc->tfm); - struct blkcipher_walk walk; - int err, ret; - - if (unlikely(op->keylen != AES_KEYSIZE_128)) - return fallback_blk_dec(desc, dst, src, nbytes); - - blkcipher_walk_init(&walk, dst, src, nbytes); - err = blkcipher_walk_virt(desc, &walk); - - while ((nbytes = walk.nbytes)) { - op->src = walk.src.virt.addr, - op->dst = walk.dst.virt.addr; - op->mode = AES_MODE_ECB; - op->len = nbytes - (nbytes % AES_BLOCK_SIZE); - op->dir = AES_DIR_DECRYPT; - - ret = geode_aes_crypt(op); - nbytes -= ret; - err = blkcipher_walk_done(desc, &walk, nbytes); - } - - return err; + return geode_skcipher_crypt(req, AES_MODE_CBC, AES_DIR_DECRYPT); } -static int -geode_ecb_encrypt(struct blkcipher_desc *desc, - struct scatterlist *dst, struct scatterlist *src, - unsigned int nbytes) +static int geode_ecb_encrypt(struct skcipher_request *req) { - struct geode_aes_op *op = crypto_blkcipher_ctx(desc->tfm); - struct blkcipher_walk walk; - int err, ret; - - if (unlikely(op->keylen != AES_KEYSIZE_128)) - return fallback_blk_enc(desc, dst, src, nbytes); - - blkcipher_walk_init(&walk, dst, src, nbytes); - err = blkcipher_walk_virt(desc, &walk); - - while ((nbytes = walk.nbytes)) { - op->src = walk.src.virt.addr, - op->dst = walk.dst.virt.addr; - op->mode = AES_MODE_ECB; - op->len = nbytes - (nbytes % AES_BLOCK_SIZE); - op->dir = AES_DIR_ENCRYPT; - - ret = geode_aes_crypt(op); - nbytes -= ret; - ret = blkcipher_walk_done(desc, &walk, nbytes); - } + return geode_skcipher_crypt(req, AES_MODE_ECB, AES_DIR_ENCRYPT); +} - return err; +static int geode_ecb_decrypt(struct skcipher_request *req) +{ + return geode_skcipher_crypt(req, AES_MODE_ECB, AES_DIR_DECRYPT); } -static struct crypto_alg geode_ecb_alg = { - .cra_name = "ecb(aes)", - .cra_driver_name = "ecb-aes-geode", - .cra_priority = 400, - .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER | - CRYPTO_ALG_KERN_DRIVER_ONLY | - CRYPTO_ALG_NEED_FALLBACK, - .cra_init = fallback_init_blk, - .cra_exit = fallback_exit_blk, - .cra_blocksize = AES_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct geode_aes_op), - .cra_alignmask = 15, - .cra_type = &crypto_blkcipher_type, - .cra_module = THIS_MODULE, - .cra_u = { - .blkcipher = { - .min_keysize = AES_MIN_KEY_SIZE, - .max_keysize = AES_MAX_KEY_SIZE, - .setkey = geode_setkey_blk, - .encrypt = geode_ecb_encrypt, - .decrypt = geode_ecb_decrypt, - } - } +static struct skcipher_alg geode_skcipher_algs[] = { + { + .base.cra_name = "cbc(aes)", + .base.cra_driver_name = "cbc-aes-geode", + .base.cra_priority = 400, + .base.cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_NEED_FALLBACK, + .base.cra_blocksize = AES_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct geode_aes_tfm_ctx), + .base.cra_alignmask = 15, + .base.cra_module = THIS_MODULE, + .init = geode_init_skcipher, + .exit = geode_exit_skcipher, + .setkey = geode_setkey_skcipher, + .encrypt = geode_cbc_encrypt, + .decrypt = geode_cbc_decrypt, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + }, { + .base.cra_name = "ecb(aes)", + .base.cra_driver_name = "ecb-aes-geode", + .base.cra_priority = 400, + .base.cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_NEED_FALLBACK, + .base.cra_blocksize = AES_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct geode_aes_tfm_ctx), + .base.cra_alignmask = 15, + .base.cra_module = THIS_MODULE, + .init = geode_init_skcipher, + .exit = geode_exit_skcipher, + .setkey = geode_setkey_skcipher, + .encrypt = geode_ecb_encrypt, + .decrypt = geode_ecb_decrypt, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + }, }; static void geode_aes_remove(struct pci_dev *dev) { crypto_unregister_alg(&geode_alg); - crypto_unregister_alg(&geode_ecb_alg); - crypto_unregister_alg(&geode_cbc_alg); + crypto_unregister_skciphers(geode_skcipher_algs, + ARRAY_SIZE(geode_skcipher_algs)); pci_iounmap(dev, _iobase); _iobase = NULL; @@ -538,20 +407,14 @@ static int geode_aes_probe(struct pci_dev *dev, const struct pci_device_id *id) if (ret) goto eiomap; - ret = crypto_register_alg(&geode_ecb_alg); + ret = crypto_register_skciphers(geode_skcipher_algs, + ARRAY_SIZE(geode_skcipher_algs)); if (ret) goto ealg; - ret = crypto_register_alg(&geode_cbc_alg); - if (ret) - goto eecb; - dev_notice(&dev->dev, "GEODE AES engine enabled.\n"); return 0; - eecb: - crypto_unregister_alg(&geode_ecb_alg); - ealg: crypto_unregister_alg(&geode_alg); diff --git a/drivers/crypto/geode-aes.h b/drivers/crypto/geode-aes.h index 5c6e131a8f9d..6d0a0cdc7647 100644 --- a/drivers/crypto/geode-aes.h +++ b/drivers/crypto/geode-aes.h @@ -46,21 +46,10 @@ #define AES_OP_TIMEOUT 0x50000 -struct geode_aes_op { - - void *src; - void *dst; - - u32 mode; - u32 dir; - u32 flags; - int len; - +struct geode_aes_tfm_ctx { u8 key[AES_KEYSIZE_128]; - u8 *iv; - union { - struct crypto_blkcipher *blk; + struct crypto_skcipher *skcipher; struct crypto_cipher *cip; } fallback; u32 keylen; diff --git a/drivers/crypto/hisilicon/sec/sec_algs.c b/drivers/crypto/hisilicon/sec/sec_algs.c index 8c789b8671fc..9a98efc95333 100644 --- a/drivers/crypto/hisilicon/sec/sec_algs.c +++ b/drivers/crypto/hisilicon/sec/sec_algs.c @@ -157,7 +157,8 @@ static int sec_alloc_and_fill_hw_sgl(struct sec_hw_sgl **sec_sgl, dma_addr_t *psec_sgl, struct scatterlist *sgl, int count, - struct sec_dev_info *info) + struct sec_dev_info *info, + gfp_t gfp) { struct sec_hw_sgl *sgl_current = NULL; struct sec_hw_sgl *sgl_next; @@ -172,7 +173,7 @@ static int sec_alloc_and_fill_hw_sgl(struct sec_hw_sgl **sec_sgl, sge_index = i % SEC_MAX_SGE_NUM; if (sge_index == 0) { sgl_next = dma_pool_zalloc(info->hw_sgl_pool, - GFP_KERNEL, &sgl_next_dma); + gfp, &sgl_next_dma); if (!sgl_next) { ret = -ENOMEM; goto err_free_hw_sgls; @@ -555,14 +556,14 @@ void sec_alg_callback(struct sec_bd_info *resp, void *shadow) } static int sec_alg_alloc_and_calc_split_sizes(int length, size_t **split_sizes, - int *steps) + int *steps, gfp_t gfp) { size_t *sizes; int i; /* Split into suitable sized blocks */ *steps = roundup(length, SEC_REQ_LIMIT) / SEC_REQ_LIMIT; - sizes = kcalloc(*steps, sizeof(*sizes), GFP_KERNEL); + sizes = kcalloc(*steps, sizeof(*sizes), gfp); if (!sizes) return -ENOMEM; @@ -578,7 +579,7 @@ static int sec_map_and_split_sg(struct scatterlist *sgl, size_t *split_sizes, int steps, struct scatterlist ***splits, int **splits_nents, int sgl_len_in, - struct device *dev) + struct device *dev, gfp_t gfp) { int ret, count; @@ -586,12 +587,12 @@ static int sec_map_and_split_sg(struct scatterlist *sgl, size_t *split_sizes, if (!count) return -EINVAL; - *splits = kcalloc(steps, sizeof(struct scatterlist *), GFP_KERNEL); + *splits = kcalloc(steps, sizeof(struct scatterlist *), gfp); if (!*splits) { ret = -ENOMEM; goto err_unmap_sg; } - *splits_nents = kcalloc(steps, sizeof(int), GFP_KERNEL); + *splits_nents = kcalloc(steps, sizeof(int), gfp); if (!*splits_nents) { ret = -ENOMEM; goto err_free_splits; @@ -599,7 +600,7 @@ static int sec_map_and_split_sg(struct scatterlist *sgl, size_t *split_sizes, /* output the scatter list before and after this */ ret = sg_split(sgl, count, 0, steps, split_sizes, - *splits, *splits_nents, GFP_KERNEL); + *splits, *splits_nents, gfp); if (ret) { ret = -ENOMEM; goto err_free_splits_nents; @@ -640,13 +641,13 @@ static struct sec_request_el int el_size, bool different_dest, struct scatterlist *sgl_in, int n_ents_in, struct scatterlist *sgl_out, int n_ents_out, - struct sec_dev_info *info) + struct sec_dev_info *info, gfp_t gfp) { struct sec_request_el *el; struct sec_bd_info *req; int ret; - el = kzalloc(sizeof(*el), GFP_KERNEL); + el = kzalloc(sizeof(*el), gfp); if (!el) return ERR_PTR(-ENOMEM); el->el_length = el_size; @@ -678,7 +679,7 @@ static struct sec_request_el el->sgl_in = sgl_in; ret = sec_alloc_and_fill_hw_sgl(&el->in, &el->dma_in, el->sgl_in, - n_ents_in, info); + n_ents_in, info, gfp); if (ret) goto err_free_el; @@ -689,7 +690,7 @@ static struct sec_request_el el->sgl_out = sgl_out; ret = sec_alloc_and_fill_hw_sgl(&el->out, &el->dma_out, el->sgl_out, - n_ents_out, info); + n_ents_out, info, gfp); if (ret) goto err_free_hw_sgl_in; @@ -730,6 +731,7 @@ static int sec_alg_skcipher_crypto(struct skcipher_request *skreq, int *splits_out_nents = NULL; struct sec_request_el *el, *temp; bool split = skreq->src != skreq->dst; + gfp_t gfp = skreq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL : GFP_ATOMIC; mutex_init(&sec_req->lock); sec_req->req_base = &skreq->base; @@ -738,13 +740,13 @@ static int sec_alg_skcipher_crypto(struct skcipher_request *skreq, sec_req->len_in = sg_nents(skreq->src); ret = sec_alg_alloc_and_calc_split_sizes(skreq->cryptlen, &split_sizes, - &steps); + &steps, gfp); if (ret) return ret; sec_req->num_elements = steps; ret = sec_map_and_split_sg(skreq->src, split_sizes, steps, &splits_in, &splits_in_nents, sec_req->len_in, - info->dev); + info->dev, gfp); if (ret) goto err_free_split_sizes; @@ -752,7 +754,7 @@ static int sec_alg_skcipher_crypto(struct skcipher_request *skreq, sec_req->len_out = sg_nents(skreq->dst); ret = sec_map_and_split_sg(skreq->dst, split_sizes, steps, &splits_out, &splits_out_nents, - sec_req->len_out, info->dev); + sec_req->len_out, info->dev, gfp); if (ret) goto err_unmap_in_sg; } @@ -785,7 +787,7 @@ static int sec_alg_skcipher_crypto(struct skcipher_request *skreq, splits_in[i], splits_in_nents[i], split ? splits_out[i] : NULL, split ? splits_out_nents[i] : 0, - info); + info, gfp); if (IS_ERR(el)) { ret = PTR_ERR(el); goto err_free_elements; diff --git a/drivers/crypto/mxs-dcp.c b/drivers/crypto/mxs-dcp.c index bdc4c42d3ac8..c06e81802813 100644 --- a/drivers/crypto/mxs-dcp.c +++ b/drivers/crypto/mxs-dcp.c @@ -20,6 +20,7 @@ #include <crypto/sha.h> #include <crypto/internal/hash.h> #include <crypto/internal/skcipher.h> +#include <crypto/scatterwalk.h> #define DCP_MAX_CHANS 4 #define DCP_BUF_SZ PAGE_SIZE @@ -621,49 +622,46 @@ static int dcp_sha_req_to_buf(struct crypto_async_request *arq) struct dcp_async_ctx *actx = crypto_ahash_ctx(tfm); struct dcp_sha_req_ctx *rctx = ahash_request_ctx(req); struct hash_alg_common *halg = crypto_hash_alg_common(tfm); - const int nents = sg_nents(req->src); uint8_t *in_buf = sdcp->coh->sha_in_buf; uint8_t *out_buf = sdcp->coh->sha_out_buf; - uint8_t *src_buf; - struct scatterlist *src; - unsigned int i, len, clen; + unsigned int i, len, clen, oft = 0; int ret; int fin = rctx->fini; if (fin) rctx->fini = 0; - for_each_sg(req->src, src, nents, i) { - src_buf = sg_virt(src); - len = sg_dma_len(src); - - do { - if (actx->fill + len > DCP_BUF_SZ) - clen = DCP_BUF_SZ - actx->fill; - else - clen = len; - - memcpy(in_buf + actx->fill, src_buf, clen); - len -= clen; - src_buf += clen; - actx->fill += clen; + src = req->src; + len = req->nbytes; - /* - * If we filled the buffer and still have some - * more data, submit the buffer. - */ - if (len && actx->fill == DCP_BUF_SZ) { - ret = mxs_dcp_run_sha(req); - if (ret) - return ret; - actx->fill = 0; - rctx->init = 0; - } - } while (len); + while (len) { + if (actx->fill + len > DCP_BUF_SZ) + clen = DCP_BUF_SZ - actx->fill; + else + clen = len; + + scatterwalk_map_and_copy(in_buf + actx->fill, src, oft, clen, + 0); + + len -= clen; + oft += clen; + actx->fill += clen; + + /* + * If we filled the buffer and still have some + * more data, submit the buffer. + */ + if (len && actx->fill == DCP_BUF_SZ) { + ret = mxs_dcp_run_sha(req); + if (ret) + return ret; + actx->fill = 0; + rctx->init = 0; + } } if (fin) { diff --git a/drivers/crypto/omap-sham.c b/drivers/crypto/omap-sham.c index e8e2907bd9f4..107cf3c579ce 100644 --- a/drivers/crypto/omap-sham.c +++ b/drivers/crypto/omap-sham.c @@ -165,8 +165,6 @@ struct omap_sham_hmac_ctx { }; struct omap_sham_ctx { - struct omap_sham_dev *dd; - unsigned long flags; /* fallback stuff */ @@ -918,27 +916,35 @@ static int omap_sham_update_dma_stop(struct omap_sham_dev *dd) return 0; } +struct omap_sham_dev *omap_sham_find_dev(struct omap_sham_reqctx *ctx) +{ + struct omap_sham_dev *dd; + + if (ctx->dd) + return ctx->dd; + + spin_lock_bh(&sham.lock); + dd = list_first_entry(&sham.dev_list, struct omap_sham_dev, list); + list_move_tail(&dd->list, &sham.dev_list); + ctx->dd = dd; + spin_unlock_bh(&sham.lock); + + return dd; +} + static int omap_sham_init(struct ahash_request *req) { struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); struct omap_sham_ctx *tctx = crypto_ahash_ctx(tfm); struct omap_sham_reqctx *ctx = ahash_request_ctx(req); - struct omap_sham_dev *dd = NULL, *tmp; + struct omap_sham_dev *dd; int bs = 0; - spin_lock_bh(&sham.lock); - if (!tctx->dd) { - list_for_each_entry(tmp, &sham.dev_list, list) { - dd = tmp; - break; - } - tctx->dd = dd; - } else { - dd = tctx->dd; - } - spin_unlock_bh(&sham.lock); + ctx->dd = NULL; - ctx->dd = dd; + dd = omap_sham_find_dev(ctx); + if (!dd) + return -ENODEV; ctx->flags = 0; @@ -1187,8 +1193,7 @@ err1: static int omap_sham_enqueue(struct ahash_request *req, unsigned int op) { struct omap_sham_reqctx *ctx = ahash_request_ctx(req); - struct omap_sham_ctx *tctx = crypto_tfm_ctx(req->base.tfm); - struct omap_sham_dev *dd = tctx->dd; + struct omap_sham_dev *dd = ctx->dd; ctx->op = op; @@ -1198,7 +1203,7 @@ static int omap_sham_enqueue(struct ahash_request *req, unsigned int op) static int omap_sham_update(struct ahash_request *req) { struct omap_sham_reqctx *ctx = ahash_request_ctx(req); - struct omap_sham_dev *dd = ctx->dd; + struct omap_sham_dev *dd = omap_sham_find_dev(ctx); if (!req->nbytes) return 0; @@ -1302,21 +1307,8 @@ static int omap_sham_setkey(struct crypto_ahash *tfm, const u8 *key, struct omap_sham_hmac_ctx *bctx = tctx->base; int bs = crypto_shash_blocksize(bctx->shash); int ds = crypto_shash_digestsize(bctx->shash); - struct omap_sham_dev *dd = NULL, *tmp; int err, i; - spin_lock_bh(&sham.lock); - if (!tctx->dd) { - list_for_each_entry(tmp, &sham.dev_list, list) { - dd = tmp; - break; - } - tctx->dd = dd; - } else { - dd = tctx->dd; - } - spin_unlock_bh(&sham.lock); - err = crypto_shash_setkey(tctx->fallback, key, keylen); if (err) return err; @@ -1334,7 +1326,7 @@ static int omap_sham_setkey(struct crypto_ahash *tfm, const u8 *key, memset(bctx->ipad + keylen, 0, bs - keylen); - if (!test_bit(FLAGS_AUTO_XOR, &dd->flags)) { + if (!test_bit(FLAGS_AUTO_XOR, &sham.flags)) { memcpy(bctx->opad, bctx->ipad, bs); for (i = 0; i < bs; i++) { @@ -2137,6 +2129,7 @@ static int omap_sham_probe(struct platform_device *pdev) } dd->flags |= dd->pdata->flags; + sham.flags |= dd->pdata->flags; pm_runtime_use_autosuspend(dev); pm_runtime_set_autosuspend_delay(dev, DEFAULT_AUTOSUSPEND_DELAY); @@ -2164,6 +2157,9 @@ static int omap_sham_probe(struct platform_device *pdev) spin_unlock(&sham.lock); for (i = 0; i < dd->pdata->algs_info_size; i++) { + if (dd->pdata->algs_info[i].registered) + break; + for (j = 0; j < dd->pdata->algs_info[i].size; j++) { struct ahash_alg *alg; @@ -2215,9 +2211,11 @@ static int omap_sham_remove(struct platform_device *pdev) list_del(&dd->list); spin_unlock(&sham.lock); for (i = dd->pdata->algs_info_size - 1; i >= 0; i--) - for (j = dd->pdata->algs_info[i].registered - 1; j >= 0; j--) + for (j = dd->pdata->algs_info[i].registered - 1; j >= 0; j--) { crypto_unregister_ahash( &dd->pdata->algs_info[i].algs_list[j]); + dd->pdata->algs_info[i].registered--; + } tasklet_kill(&dd->done_task); pm_runtime_disable(&pdev->dev); diff --git a/drivers/crypto/picoxcell_crypto.c b/drivers/crypto/picoxcell_crypto.c index b985cb85c9bc..104eddfd800f 100644 --- a/drivers/crypto/picoxcell_crypto.c +++ b/drivers/crypto/picoxcell_crypto.c @@ -1621,6 +1621,11 @@ static const struct of_device_id spacc_of_id_table[] = { MODULE_DEVICE_TABLE(of, spacc_of_id_table); #endif /* CONFIG_OF */ +static void spacc_tasklet_kill(void *data) +{ + tasklet_kill(data); +} + static int spacc_probe(struct platform_device *pdev) { int i, err, ret; @@ -1664,6 +1669,14 @@ static int spacc_probe(struct platform_device *pdev) return -ENXIO; } + tasklet_init(&engine->complete, spacc_spacc_complete, + (unsigned long)engine); + + ret = devm_add_action(&pdev->dev, spacc_tasklet_kill, + &engine->complete); + if (ret) + return ret; + if (devm_request_irq(&pdev->dev, irq->start, spacc_spacc_irq, 0, engine->name, engine)) { dev_err(engine->dev, "failed to request IRQ\n"); @@ -1721,8 +1734,6 @@ static int spacc_probe(struct platform_device *pdev) INIT_LIST_HEAD(&engine->completed); INIT_LIST_HEAD(&engine->in_progress); engine->in_flight = 0; - tasklet_init(&engine->complete, spacc_spacc_complete, - (unsigned long)engine); platform_set_drvdata(pdev, engine); diff --git a/drivers/crypto/qat/qat_common/qat_uclo.c b/drivers/crypto/qat/qat_common/qat_uclo.c index 6bd8f6a2a24f..aeb03081415c 100644 --- a/drivers/crypto/qat/qat_common/qat_uclo.c +++ b/drivers/crypto/qat/qat_common/qat_uclo.c @@ -332,13 +332,18 @@ static int qat_uclo_create_batch_init_list(struct icp_qat_fw_loader_handle } return 0; out_err: + /* Do not free the list head unless we allocated it. */ + tail_old = tail_old->next; + if (flag) { + kfree(*init_tab_base); + *init_tab_base = NULL; + } + while (tail_old) { mem_init = tail_old->next; kfree(tail_old); tail_old = mem_init; } - if (flag) - kfree(*init_tab_base); return -ENOMEM; } diff --git a/drivers/crypto/stm32/stm32_crc32.c b/drivers/crypto/stm32/stm32_crc32.c index 440c9f1bd006..5da7d4a8aad4 100644 --- a/drivers/crypto/stm32/stm32_crc32.c +++ b/drivers/crypto/stm32/stm32_crc32.c @@ -28,8 +28,10 @@ /* Registers values */ #define CRC_CR_RESET BIT(0) -#define CRC_CR_REVERSE (BIT(7) | BIT(6) | BIT(5)) -#define CRC_INIT_DEFAULT 0xFFFFFFFF +#define CRC_CR_REV_IN_WORD (BIT(6) | BIT(5)) +#define CRC_CR_REV_IN_BYTE BIT(5) +#define CRC_CR_REV_OUT BIT(7) +#define CRC32C_INIT_DEFAULT 0xFFFFFFFF #define CRC_AUTOSUSPEND_DELAY 50 @@ -38,8 +40,6 @@ struct stm32_crc { struct device *dev; void __iomem *regs; struct clk *clk; - u8 pending_data[sizeof(u32)]; - size_t nb_pending_bytes; }; struct stm32_crc_list { @@ -59,14 +59,13 @@ struct stm32_crc_ctx { struct stm32_crc_desc_ctx { u32 partial; /* crc32c: partial in first 4 bytes of that struct */ - struct stm32_crc *crc; }; static int stm32_crc32_cra_init(struct crypto_tfm *tfm) { struct stm32_crc_ctx *mctx = crypto_tfm_ctx(tfm); - mctx->key = CRC_INIT_DEFAULT; + mctx->key = 0; mctx->poly = CRC32_POLY_LE; return 0; } @@ -75,7 +74,7 @@ static int stm32_crc32c_cra_init(struct crypto_tfm *tfm) { struct stm32_crc_ctx *mctx = crypto_tfm_ctx(tfm); - mctx->key = CRC_INIT_DEFAULT; + mctx->key = CRC32C_INIT_DEFAULT; mctx->poly = CRC32C_POLY_LE; return 0; } @@ -94,32 +93,42 @@ static int stm32_crc_setkey(struct crypto_shash *tfm, const u8 *key, return 0; } +static struct stm32_crc *stm32_crc_get_next_crc(void) +{ + struct stm32_crc *crc; + + spin_lock_bh(&crc_list.lock); + crc = list_first_entry(&crc_list.dev_list, struct stm32_crc, list); + if (crc) + list_move_tail(&crc->list, &crc_list.dev_list); + spin_unlock_bh(&crc_list.lock); + + return crc; +} + static int stm32_crc_init(struct shash_desc *desc) { struct stm32_crc_desc_ctx *ctx = shash_desc_ctx(desc); struct stm32_crc_ctx *mctx = crypto_shash_ctx(desc->tfm); struct stm32_crc *crc; - spin_lock_bh(&crc_list.lock); - list_for_each_entry(crc, &crc_list.dev_list, list) { - ctx->crc = crc; - break; - } - spin_unlock_bh(&crc_list.lock); + crc = stm32_crc_get_next_crc(); + if (!crc) + return -ENODEV; - pm_runtime_get_sync(ctx->crc->dev); + pm_runtime_get_sync(crc->dev); /* Reset, set key, poly and configure in bit reverse mode */ - writel_relaxed(bitrev32(mctx->key), ctx->crc->regs + CRC_INIT); - writel_relaxed(bitrev32(mctx->poly), ctx->crc->regs + CRC_POL); - writel_relaxed(CRC_CR_RESET | CRC_CR_REVERSE, ctx->crc->regs + CRC_CR); + writel_relaxed(bitrev32(mctx->key), crc->regs + CRC_INIT); + writel_relaxed(bitrev32(mctx->poly), crc->regs + CRC_POL); + writel_relaxed(CRC_CR_RESET | CRC_CR_REV_IN_WORD | CRC_CR_REV_OUT, + crc->regs + CRC_CR); /* Store partial result */ - ctx->partial = readl_relaxed(ctx->crc->regs + CRC_DR); - ctx->crc->nb_pending_bytes = 0; + ctx->partial = readl_relaxed(crc->regs + CRC_DR); - pm_runtime_mark_last_busy(ctx->crc->dev); - pm_runtime_put_autosuspend(ctx->crc->dev); + pm_runtime_mark_last_busy(crc->dev); + pm_runtime_put_autosuspend(crc->dev); return 0; } @@ -128,31 +137,49 @@ static int stm32_crc_update(struct shash_desc *desc, const u8 *d8, unsigned int length) { struct stm32_crc_desc_ctx *ctx = shash_desc_ctx(desc); - struct stm32_crc *crc = ctx->crc; - u32 *d32; - unsigned int i; + struct stm32_crc_ctx *mctx = crypto_shash_ctx(desc->tfm); + struct stm32_crc *crc; + + crc = stm32_crc_get_next_crc(); + if (!crc) + return -ENODEV; pm_runtime_get_sync(crc->dev); - if (unlikely(crc->nb_pending_bytes)) { - while (crc->nb_pending_bytes != sizeof(u32) && length) { - /* Fill in pending data */ - crc->pending_data[crc->nb_pending_bytes++] = *(d8++); + /* + * Restore previously calculated CRC for this context as init value + * Restore polynomial configuration + * Configure in register for word input data, + * Configure out register in reversed bit mode data. + */ + writel_relaxed(bitrev32(ctx->partial), crc->regs + CRC_INIT); + writel_relaxed(bitrev32(mctx->poly), crc->regs + CRC_POL); + writel_relaxed(CRC_CR_RESET | CRC_CR_REV_IN_WORD | CRC_CR_REV_OUT, + crc->regs + CRC_CR); + + if (d8 != PTR_ALIGN(d8, sizeof(u32))) { + /* Configure for byte data */ + writel_relaxed(CRC_CR_REV_IN_BYTE | CRC_CR_REV_OUT, + crc->regs + CRC_CR); + while (d8 != PTR_ALIGN(d8, sizeof(u32)) && length) { + writeb_relaxed(*d8++, crc->regs + CRC_DR); length--; } - - if (crc->nb_pending_bytes == sizeof(u32)) { - /* Process completed pending data */ - writel_relaxed(*(u32 *)crc->pending_data, - crc->regs + CRC_DR); - crc->nb_pending_bytes = 0; - } + /* Configure for word data */ + writel_relaxed(CRC_CR_REV_IN_WORD | CRC_CR_REV_OUT, + crc->regs + CRC_CR); } - d32 = (u32 *)d8; - for (i = 0; i < length >> 2; i++) - /* Process 32 bits data */ - writel_relaxed(*(d32++), crc->regs + CRC_DR); + for (; length >= sizeof(u32); d8 += sizeof(u32), length -= sizeof(u32)) + writel_relaxed(*((u32 *)d8), crc->regs + CRC_DR); + + if (length) { + /* Configure for byte data */ + writel_relaxed(CRC_CR_REV_IN_BYTE | CRC_CR_REV_OUT, + crc->regs + CRC_CR); + while (length--) + writeb_relaxed(*d8++, crc->regs + CRC_DR); + } /* Store partial result */ ctx->partial = readl_relaxed(crc->regs + CRC_DR); @@ -160,22 +187,6 @@ static int stm32_crc_update(struct shash_desc *desc, const u8 *d8, pm_runtime_mark_last_busy(crc->dev); pm_runtime_put_autosuspend(crc->dev); - /* Check for pending data (non 32 bits) */ - length &= 3; - if (likely(!length)) - return 0; - - if ((crc->nb_pending_bytes + length) >= sizeof(u32)) { - /* Shall not happen */ - dev_err(crc->dev, "Pending data overflow\n"); - return -EINVAL; - } - - d8 = (const u8 *)d32; - for (i = 0; i < length; i++) - /* Store pending data */ - crc->pending_data[crc->nb_pending_bytes++] = *(d8++); - return 0; } @@ -204,6 +215,8 @@ static int stm32_crc_digest(struct shash_desc *desc, const u8 *data, return stm32_crc_init(desc) ?: stm32_crc_finup(desc, data, length, out); } +static unsigned int refcnt; +static DEFINE_MUTEX(refcnt_lock); static struct shash_alg algs[] = { /* CRC-32 */ { @@ -296,12 +309,18 @@ static int stm32_crc_probe(struct platform_device *pdev) list_add(&crc->list, &crc_list.dev_list); spin_unlock(&crc_list.lock); - ret = crypto_register_shashes(algs, ARRAY_SIZE(algs)); - if (ret) { - dev_err(dev, "Failed to register\n"); - clk_disable_unprepare(crc->clk); - return ret; + mutex_lock(&refcnt_lock); + if (!refcnt) { + ret = crypto_register_shashes(algs, ARRAY_SIZE(algs)); + if (ret) { + mutex_unlock(&refcnt_lock); + dev_err(dev, "Failed to register\n"); + clk_disable_unprepare(crc->clk); + return ret; + } } + refcnt++; + mutex_unlock(&refcnt_lock); dev_info(dev, "Initialized\n"); @@ -322,7 +341,10 @@ static int stm32_crc_remove(struct platform_device *pdev) list_del(&crc->list); spin_unlock(&crc_list.lock); - crypto_unregister_shashes(algs, ARRAY_SIZE(algs)); + mutex_lock(&refcnt_lock); + if (!--refcnt) + crypto_unregister_shashes(algs, ARRAY_SIZE(algs)); + mutex_unlock(&refcnt_lock); pm_runtime_disable(crc->dev); pm_runtime_put_noidle(crc->dev); diff --git a/drivers/crypto/sunxi-ss/sun4i-ss-cipher.c b/drivers/crypto/sunxi-ss/sun4i-ss-cipher.c index 7b0c42882830..e74ba066d002 100644 --- a/drivers/crypto/sunxi-ss/sun4i-ss-cipher.c +++ b/drivers/crypto/sunxi-ss/sun4i-ss-cipher.c @@ -12,7 +12,7 @@ */ #include "sun4i-ss.h" -static int sun4i_ss_opti_poll(struct skcipher_request *areq) +static int noinline_for_stack sun4i_ss_opti_poll(struct skcipher_request *areq) { struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq); struct sun4i_tfm_ctx *op = crypto_skcipher_ctx(tfm); @@ -72,7 +72,8 @@ static int sun4i_ss_opti_poll(struct skcipher_request *areq) oi = 0; oo = 0; do { - todo = min3(rx_cnt, ileft, (mi.length - oi) / 4); + todo = min(rx_cnt, ileft); + todo = min_t(size_t, todo, (mi.length - oi) / 4); if (todo) { ileft -= todo; writesl(ss->base + SS_RXFIFO, mi.addr + oi, todo); @@ -87,7 +88,8 @@ static int sun4i_ss_opti_poll(struct skcipher_request *areq) rx_cnt = SS_RXFIFO_SPACES(spaces); tx_cnt = SS_TXFIFO_SPACES(spaces); - todo = min3(tx_cnt, oleft, (mo.length - oo) / 4); + todo = min(tx_cnt, oleft); + todo = min_t(size_t, todo, (mo.length - oo) / 4); if (todo) { oleft -= todo; readsl(ss->base + SS_TXFIFO, mo.addr + oo, todo); @@ -114,6 +116,29 @@ release_ss: return err; } + +static int noinline_for_stack sun4i_ss_cipher_poll_fallback(struct skcipher_request *areq) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq); + struct sun4i_tfm_ctx *op = crypto_skcipher_ctx(tfm); + struct sun4i_cipher_req_ctx *ctx = skcipher_request_ctx(areq); + SYNC_SKCIPHER_REQUEST_ON_STACK(subreq, op->fallback_tfm); + int err; + + skcipher_request_set_sync_tfm(subreq, op->fallback_tfm); + skcipher_request_set_callback(subreq, areq->base.flags, NULL, + NULL); + skcipher_request_set_crypt(subreq, areq->src, areq->dst, + areq->cryptlen, areq->iv); + if (ctx->mode & SS_DECRYPTION) + err = crypto_skcipher_decrypt(subreq); + else + err = crypto_skcipher_encrypt(subreq); + skcipher_request_zero(subreq); + + return err; +} + /* Generic function that support SG with size not multiple of 4 */ static int sun4i_ss_cipher_poll(struct skcipher_request *areq) { @@ -140,8 +165,6 @@ static int sun4i_ss_cipher_poll(struct skcipher_request *areq) unsigned int todo; struct sg_mapping_iter mi, mo; unsigned int oi, oo; /* offset for in and out */ - char buf[4 * SS_RX_MAX];/* buffer for linearize SG src */ - char bufo[4 * SS_TX_MAX]; /* buffer for linearize SG dst */ unsigned int ob = 0; /* offset in buf */ unsigned int obo = 0; /* offset in bufo*/ unsigned int obl = 0; /* length of data in bufo */ @@ -178,20 +201,8 @@ static int sun4i_ss_cipher_poll(struct skcipher_request *areq) if (no_chunk == 1 && !need_fallback) return sun4i_ss_opti_poll(areq); - if (need_fallback) { - SYNC_SKCIPHER_REQUEST_ON_STACK(subreq, op->fallback_tfm); - skcipher_request_set_sync_tfm(subreq, op->fallback_tfm); - skcipher_request_set_callback(subreq, areq->base.flags, NULL, - NULL); - skcipher_request_set_crypt(subreq, areq->src, areq->dst, - areq->cryptlen, areq->iv); - if (ctx->mode & SS_DECRYPTION) - err = crypto_skcipher_decrypt(subreq); - else - err = crypto_skcipher_encrypt(subreq); - skcipher_request_zero(subreq); - return err; - } + if (need_fallback) + return sun4i_ss_cipher_poll_fallback(areq); spin_lock_irqsave(&ss->slock, flags); @@ -224,11 +235,14 @@ static int sun4i_ss_cipher_poll(struct skcipher_request *areq) while (oleft) { if (ileft) { + char buf[4 * SS_RX_MAX];/* buffer for linearize SG src */ + /* * todo is the number of consecutive 4byte word that we * can read from current SG */ - todo = min3(rx_cnt, ileft / 4, (mi.length - oi) / 4); + todo = min(rx_cnt, ileft / 4); + todo = min_t(size_t, todo, (mi.length - oi) / 4); if (todo && !ob) { writesl(ss->base + SS_RXFIFO, mi.addr + oi, todo); @@ -242,8 +256,8 @@ static int sun4i_ss_cipher_poll(struct skcipher_request *areq) * we need to be able to write all buf in one * pass, so it is why we min() with rx_cnt */ - todo = min3(rx_cnt * 4 - ob, ileft, - mi.length - oi); + todo = min(rx_cnt * 4 - ob, ileft); + todo = min_t(size_t, todo, mi.length - oi); memcpy(buf + ob, mi.addr + oi, todo); ileft -= todo; oi += todo; @@ -263,7 +277,8 @@ static int sun4i_ss_cipher_poll(struct skcipher_request *areq) spaces = readl(ss->base + SS_FCSR); rx_cnt = SS_RXFIFO_SPACES(spaces); tx_cnt = SS_TXFIFO_SPACES(spaces); - dev_dbg(ss->dev, "%x %u/%u %u/%u cnt=%u %u/%u %u/%u cnt=%u %u\n", + dev_dbg(ss->dev, + "%x %u/%zu %u/%u cnt=%u %u/%zu %u/%u cnt=%u %u\n", mode, oi, mi.length, ileft, areq->cryptlen, rx_cnt, oo, mo.length, oleft, areq->cryptlen, tx_cnt, ob); @@ -271,7 +286,8 @@ static int sun4i_ss_cipher_poll(struct skcipher_request *areq) if (!tx_cnt) continue; /* todo in 4bytes word */ - todo = min3(tx_cnt, oleft / 4, (mo.length - oo) / 4); + todo = min(tx_cnt, oleft / 4); + todo = min_t(size_t, todo, (mo.length - oo) / 4); if (todo) { readsl(ss->base + SS_TXFIFO, mo.addr + oo, todo); oleft -= todo * 4; @@ -281,6 +297,8 @@ static int sun4i_ss_cipher_poll(struct skcipher_request *areq) oo = 0; } } else { + char bufo[4 * SS_TX_MAX]; /* buffer for linearize SG dst */ + /* * read obl bytes in bufo, we read at maximum for * emptying the device @@ -295,7 +313,8 @@ static int sun4i_ss_cipher_poll(struct skcipher_request *areq) * no more than remaining buffer * no need to test against oleft */ - todo = min(mo.length - oo, obl - obo); + todo = min_t(size_t, + mo.length - oo, obl - obo); memcpy(mo.addr + oo, bufo + obo, todo); oleft -= todo; obo += todo; diff --git a/drivers/crypto/sunxi-ss/sun4i-ss-hash.c b/drivers/crypto/sunxi-ss/sun4i-ss-hash.c index d2b6d89aad28..a200eadc691b 100644 --- a/drivers/crypto/sunxi-ss/sun4i-ss-hash.c +++ b/drivers/crypto/sunxi-ss/sun4i-ss-hash.c @@ -175,7 +175,7 @@ static int sun4i_hash(struct ahash_request *areq) */ unsigned int i = 0, end, fill, min_fill, nwait, nbw = 0, j = 0, todo; unsigned int in_i = 0; - u32 spaces, rx_cnt = SS_RX_DEFAULT, bf[32] = {0}, wb = 0, v, ivmode = 0; + u32 spaces, rx_cnt = SS_RX_DEFAULT, bf[32] = {0}, v, ivmode = 0; struct sun4i_req_ctx *op = ahash_request_ctx(areq); struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); struct sun4i_tfm_ctx *tfmctx = crypto_ahash_ctx(tfm); @@ -184,6 +184,7 @@ static int sun4i_hash(struct ahash_request *areq) struct sg_mapping_iter mi; int in_r, err = 0; size_t copied = 0; + __le32 wb = 0; dev_dbg(ss->dev, "%s %s bc=%llu len=%u mode=%x wl=%u h0=%0x", __func__, crypto_tfm_alg_name(areq->base.tfm), @@ -272,8 +273,8 @@ static int sun4i_hash(struct ahash_request *areq) */ while (op->len < 64 && i < end) { /* how many bytes we can read from current SG */ - in_r = min3(mi.length - in_i, end - i, - 64 - op->len); + in_r = min(end - i, 64 - op->len); + in_r = min_t(size_t, mi.length - in_i, in_r); memcpy(op->buf + op->len, mi.addr + in_i, in_r); op->len += in_r; i += in_r; @@ -293,8 +294,8 @@ static int sun4i_hash(struct ahash_request *areq) } if (mi.length - in_i > 3 && i < end) { /* how many bytes we can read from current SG */ - in_r = min3(mi.length - in_i, areq->nbytes - i, - ((mi.length - in_i) / 4) * 4); + in_r = min_t(size_t, mi.length - in_i, areq->nbytes - i); + in_r = min_t(size_t, ((mi.length - in_i) / 4) * 4, in_r); /* how many bytes we can write in the device*/ todo = min3((u32)(end - i) / 4, rx_cnt, (u32)in_r / 4); writesl(ss->base + SS_RXFIFO, mi.addr + in_i, todo); @@ -320,8 +321,8 @@ static int sun4i_hash(struct ahash_request *areq) if ((areq->nbytes - i) < 64) { while (i < areq->nbytes && in_i < mi.length && op->len < 64) { /* how many bytes we can read from current SG */ - in_r = min3(mi.length - in_i, areq->nbytes - i, - 64 - op->len); + in_r = min(areq->nbytes - i, 64 - op->len); + in_r = min_t(size_t, mi.length - in_i, in_r); memcpy(op->buf + op->len, mi.addr + in_i, in_r); op->len += in_r; i += in_r; @@ -395,7 +396,7 @@ hash_final: nbw = op->len - 4 * nwait; if (nbw) { - wb = *(u32 *)(op->buf + nwait * 4); + wb = cpu_to_le32(*(u32 *)(op->buf + nwait * 4)); wb &= GENMASK((nbw * 8) - 1, 0); op->byte_count += nbw; @@ -404,7 +405,7 @@ hash_final: /* write the remaining bytes of the nbw buffer */ wb |= ((1 << 7) << (nbw * 8)); - bf[j++] = wb; + bf[j++] = le32_to_cpu(wb); /* * number of space to pad to obtain 64o minus 8(size) minus 4 (final 1) @@ -423,13 +424,13 @@ hash_final: /* write the length of data */ if (op->mode == SS_OP_SHA1) { - __be64 bits = cpu_to_be64(op->byte_count << 3); - bf[j++] = lower_32_bits(bits); - bf[j++] = upper_32_bits(bits); + __be64 *bits = (__be64 *)&bf[j]; + *bits = cpu_to_be64(op->byte_count << 3); + j += 2; } else { - __le64 bits = op->byte_count << 3; - bf[j++] = lower_32_bits(bits); - bf[j++] = upper_32_bits(bits); + __le64 *bits = (__le64 *)&bf[j]; + *bits = cpu_to_le64(op->byte_count << 3); + j += 2; } writesl(ss->base + SS_RXFIFO, bf, j); @@ -471,7 +472,7 @@ hash_final: } } else { for (i = 0; i < 4; i++) { - v = readl(ss->base + SS_MD0 + i * 4); + v = cpu_to_le32(readl(ss->base + SS_MD0 + i * 4)); memcpy(areq->result + i * 4, &v, 4); } } diff --git a/drivers/crypto/virtio/virtio_crypto_algs.c b/drivers/crypto/virtio/virtio_crypto_algs.c index 10f266d462d6..7fbf82c0cce3 100644 --- a/drivers/crypto/virtio/virtio_crypto_algs.c +++ b/drivers/crypto/virtio/virtio_crypto_algs.c @@ -105,8 +105,6 @@ virtio_crypto_alg_validate_key(int key_len, uint32_t *alg) *alg = VIRTIO_CRYPTO_CIPHER_AES_CBC; break; default: - pr_err("virtio_crypto: Unsupported key length: %d\n", - key_len); return -EINVAL; } return 0; @@ -357,13 +355,18 @@ __virtio_crypto_ablkcipher_do_req(struct virtio_crypto_sym_request *vc_sym_req, int err; unsigned long flags; struct scatterlist outhdr, iv_sg, status_sg, **sgs; - int i; u64 dst_len; unsigned int num_out = 0, num_in = 0; int sg_total; uint8_t *iv; + struct scatterlist *sg; src_nents = sg_nents_for_len(req->src, req->nbytes); + if (src_nents < 0) { + pr_err("Invalid number of src SG.\n"); + return src_nents; + } + dst_nents = sg_nents(req->dst); pr_debug("virtio_crypto: Number of sgs (src_nents: %d, dst_nents: %d)\n", @@ -439,17 +442,22 @@ __virtio_crypto_ablkcipher_do_req(struct virtio_crypto_sym_request *vc_sym_req, goto free; } memcpy(iv, req->info, ivsize); + if (!vc_sym_req->encrypt) + scatterwalk_map_and_copy(req->info, req->src, + req->nbytes - AES_BLOCK_SIZE, + AES_BLOCK_SIZE, 0); + sg_init_one(&iv_sg, iv, ivsize); sgs[num_out++] = &iv_sg; vc_sym_req->iv = iv; /* Source data */ - for (i = 0; i < src_nents; i++) - sgs[num_out++] = &req->src[i]; + for (sg = req->src; src_nents; sg = sg_next(sg), src_nents--) + sgs[num_out++] = sg; /* Destination data */ - for (i = 0; i < dst_nents; i++) - sgs[num_out + num_in++] = &req->dst[i]; + for (sg = req->dst; sg; sg = sg_next(sg)) + sgs[num_out + num_in++] = sg; /* Status */ sg_init_one(&status_sg, &vc_req->status, sizeof(vc_req->status)); @@ -486,6 +494,11 @@ static int virtio_crypto_ablkcipher_encrypt(struct ablkcipher_request *req) /* Use the first data virtqueue as default */ struct data_queue *data_vq = &vcrypto->data_vq[0]; + if (!req->nbytes) + return 0; + if (req->nbytes % AES_BLOCK_SIZE) + return -EINVAL; + vc_req->dataq = data_vq; vc_req->alg_cb = virtio_crypto_dataq_sym_callback; vc_sym_req->ablkcipher_ctx = ctx; @@ -506,6 +519,11 @@ static int virtio_crypto_ablkcipher_decrypt(struct ablkcipher_request *req) /* Use the first data virtqueue as default */ struct data_queue *data_vq = &vcrypto->data_vq[0]; + if (!req->nbytes) + return 0; + if (req->nbytes % AES_BLOCK_SIZE) + return -EINVAL; + vc_req->dataq = data_vq; vc_req->alg_cb = virtio_crypto_dataq_sym_callback; vc_sym_req->ablkcipher_ctx = ctx; @@ -565,10 +583,15 @@ static void virtio_crypto_ablkcipher_finalize_req( struct ablkcipher_request *req, int err) { - crypto_finalize_ablkcipher_request(vc_sym_req->base.dataq->engine, - req, err); + if (vc_sym_req->encrypt) + scatterwalk_map_and_copy(req->info, req->dst, + req->nbytes - AES_BLOCK_SIZE, + AES_BLOCK_SIZE, 0); kzfree(vc_sym_req->iv); virtcrypto_clear_request(&vc_sym_req->base); + + crypto_finalize_ablkcipher_request(vc_sym_req->base.dataq->engine, + req, err); } static struct virtio_crypto_algo virtio_crypto_algs[] = { { diff --git a/drivers/crypto/vmx/Makefile b/drivers/crypto/vmx/Makefile index cab32cfec9c4..709670d2b553 100644 --- a/drivers/crypto/vmx/Makefile +++ b/drivers/crypto/vmx/Makefile @@ -3,13 +3,13 @@ obj-$(CONFIG_CRYPTO_DEV_VMX_ENCRYPT) += vmx-crypto.o vmx-crypto-objs := vmx.o aesp8-ppc.o ghashp8-ppc.o aes.o aes_cbc.o aes_ctr.o aes_xts.o ghash.o ifeq ($(CONFIG_CPU_LITTLE_ENDIAN),y) -TARGET := linux-ppc64le +override flavour := linux-ppc64le else -TARGET := linux-ppc64 +override flavour := linux-ppc64 endif quiet_cmd_perl = PERL $@ - cmd_perl = $(PERL) $(<) $(TARGET) > $(@) + cmd_perl = $(PERL) $(<) $(flavour) > $(@) targets += aesp8-ppc.S ghashp8-ppc.S |