#include <linux/dma-mapping.h>
#include <linux/scatterlist.h>
#include <linux/highmem.h>
-#include <linux/interrupt.h>
#include <linux/crypto.h>
#include <linux/hw_random.h>
#include <linux/ktime.h>
#define HIFN_D_DST_RSIZE 80*4
#define HIFN_D_RES_RSIZE 24*4
-#define HIFN_QUEUE_LENGTH HIFN_D_CMD_RSIZE-5
+#define HIFN_D_DST_DALIGN 4
+
+#define HIFN_QUEUE_LENGTH HIFN_D_CMD_RSIZE-1
#define AES_MIN_KEY_SIZE 16
#define AES_MAX_KEY_SIZE 32
char *offtbl = NULL;
int i;
- for (i = 0; i < sizeof(pci2id)/sizeof(pci2id[0]); i++) {
+ for (i = 0; i < ARRAY_SIZE(pci2id); i++) {
if (pci2id[i].pci_vendor == dev->pdev->vendor &&
pci2id[i].pci_prod == dev->pdev->device) {
offtbl = pci2id[i].card_id;
return cmd_len;
}
-static int hifn_setup_src_desc(struct hifn_device *dev, struct page *page,
- unsigned int offset, unsigned int size)
-{
- struct hifn_dma *dma = (struct hifn_dma *)dev->desc_virt;
- int idx;
- dma_addr_t addr;
-
- addr = pci_map_page(dev->pdev, page, offset, size, PCI_DMA_TODEVICE);
-
- idx = dma->srci;
-
- dma->srcr[idx].p = __cpu_to_le32(addr);
- dma->srcr[idx].l = __cpu_to_le32(size | HIFN_D_VALID |
- HIFN_D_MASKDONEIRQ | HIFN_D_NOINVALID | HIFN_D_LAST);
-
- if (++idx == HIFN_D_SRC_RSIZE) {
- dma->srcr[idx].l = __cpu_to_le32(HIFN_D_VALID |
- HIFN_D_JUMP |
- HIFN_D_MASKDONEIRQ | HIFN_D_LAST);
- idx = 0;
- }
-
- dma->srci = idx;
- dma->srcu++;
-
- if (!(dev->flags & HIFN_FLAG_SRC_BUSY)) {
- hifn_write_1(dev, HIFN_1_DMA_CSR, HIFN_DMACSR_S_CTRL_ENA);
- dev->flags |= HIFN_FLAG_SRC_BUSY;
- }
-
- return size;
-}
-
-static void hifn_setup_res_desc(struct hifn_device *dev)
-{
- struct hifn_dma *dma = (struct hifn_dma *)dev->desc_virt;
-
- dma->resr[dma->resi].l = __cpu_to_le32(HIFN_USED_RESULT |
- HIFN_D_VALID | HIFN_D_LAST);
- /*
- * dma->resr[dma->resi].l = __cpu_to_le32(HIFN_MAX_RESULT | HIFN_D_VALID |
- * HIFN_D_LAST | HIFN_D_NOINVALID);
- */
-
- if (++dma->resi == HIFN_D_RES_RSIZE) {
- dma->resr[HIFN_D_RES_RSIZE].l = __cpu_to_le32(HIFN_D_VALID |
- HIFN_D_JUMP | HIFN_D_MASKDONEIRQ | HIFN_D_LAST);
- dma->resi = 0;
- }
-
- dma->resu++;
-
- if (!(dev->flags & HIFN_FLAG_RES_BUSY)) {
- hifn_write_1(dev, HIFN_1_DMA_CSR, HIFN_DMACSR_R_CTRL_ENA);
- dev->flags |= HIFN_FLAG_RES_BUSY;
- }
-}
-
-static void hifn_setup_dst_desc(struct hifn_device *dev, struct page *page,
- unsigned offset, unsigned size)
-{
- struct hifn_dma *dma = (struct hifn_dma *)dev->desc_virt;
- int idx;
- dma_addr_t addr;
-
- addr = pci_map_page(dev->pdev, page, offset, size, PCI_DMA_FROMDEVICE);
-
- idx = dma->dsti;
- dma->dstr[idx].p = __cpu_to_le32(addr);
- dma->dstr[idx].l = __cpu_to_le32(size | HIFN_D_VALID |
- HIFN_D_MASKDONEIRQ | HIFN_D_NOINVALID | HIFN_D_LAST);
-
- if (++idx == HIFN_D_DST_RSIZE) {
- dma->dstr[idx].l = __cpu_to_le32(HIFN_D_VALID |
- HIFN_D_JUMP | HIFN_D_MASKDONEIRQ |
- HIFN_D_LAST | HIFN_D_NOINVALID);
- idx = 0;
- }
- dma->dsti = idx;
- dma->dstu++;
-
- if (!(dev->flags & HIFN_FLAG_DST_BUSY)) {
- hifn_write_1(dev, HIFN_1_DMA_CSR, HIFN_DMACSR_D_CTRL_ENA);
- dev->flags |= HIFN_FLAG_DST_BUSY;
- }
-}
-
-static int hifn_setup_dma(struct hifn_device *dev, struct page *spage, unsigned int soff,
- struct page *dpage, unsigned int doff, unsigned int nbytes, void *priv,
- struct hifn_context *ctx)
+static int hifn_setup_cmd_desc(struct hifn_device *dev,
+ struct hifn_context *ctx, void *priv, unsigned int nbytes)
{
struct hifn_dma *dma = (struct hifn_dma *)dev->desc_virt;
int cmd_len, sa_idx;
u8 *buf, *buf_pos;
u16 mask;
- dprintk("%s: spage: %p, soffset: %u, dpage: %p, doffset: %u, nbytes: %u, priv: %p, ctx: %p.\n",
- dev->name, spage, soff, dpage, doff, nbytes, priv, ctx);
-
- sa_idx = dma->resi;
-
- hifn_setup_src_desc(dev, spage, soff, nbytes);
-
+ sa_idx = dma->cmdi;
buf_pos = buf = dma->command_bufs[dma->cmdi];
mask = 0;
hifn_write_1(dev, HIFN_1_DMA_CSR, HIFN_DMACSR_C_CTRL_ENA);
dev->flags |= HIFN_FLAG_CMD_BUSY;
}
-
- hifn_setup_dst_desc(dev, dpage, doff, nbytes);
- hifn_setup_res_desc(dev);
-
return 0;
err_out:
return -EINVAL;
}
+static int hifn_setup_src_desc(struct hifn_device *dev, struct page *page,
+ unsigned int offset, unsigned int size)
+{
+ struct hifn_dma *dma = (struct hifn_dma *)dev->desc_virt;
+ int idx;
+ dma_addr_t addr;
+
+ addr = pci_map_page(dev->pdev, page, offset, size, PCI_DMA_TODEVICE);
+
+ idx = dma->srci;
+
+ dma->srcr[idx].p = __cpu_to_le32(addr);
+ dma->srcr[idx].l = __cpu_to_le32(size | HIFN_D_VALID |
+ HIFN_D_MASKDONEIRQ | HIFN_D_LAST);
+
+ if (++idx == HIFN_D_SRC_RSIZE) {
+ dma->srcr[idx].l = __cpu_to_le32(HIFN_D_VALID |
+ HIFN_D_JUMP |
+ HIFN_D_MASKDONEIRQ | HIFN_D_LAST);
+ idx = 0;
+ }
+
+ dma->srci = idx;
+ dma->srcu++;
+
+ if (!(dev->flags & HIFN_FLAG_SRC_BUSY)) {
+ hifn_write_1(dev, HIFN_1_DMA_CSR, HIFN_DMACSR_S_CTRL_ENA);
+ dev->flags |= HIFN_FLAG_SRC_BUSY;
+ }
+
+ return size;
+}
+
+static void hifn_setup_res_desc(struct hifn_device *dev)
+{
+ struct hifn_dma *dma = (struct hifn_dma *)dev->desc_virt;
+
+ dma->resr[dma->resi].l = __cpu_to_le32(HIFN_USED_RESULT |
+ HIFN_D_VALID | HIFN_D_LAST);
+ /*
+ * dma->resr[dma->resi].l = __cpu_to_le32(HIFN_MAX_RESULT | HIFN_D_VALID |
+ * HIFN_D_LAST);
+ */
+
+ if (++dma->resi == HIFN_D_RES_RSIZE) {
+ dma->resr[HIFN_D_RES_RSIZE].l = __cpu_to_le32(HIFN_D_VALID |
+ HIFN_D_JUMP | HIFN_D_MASKDONEIRQ | HIFN_D_LAST);
+ dma->resi = 0;
+ }
+
+ dma->resu++;
+
+ if (!(dev->flags & HIFN_FLAG_RES_BUSY)) {
+ hifn_write_1(dev, HIFN_1_DMA_CSR, HIFN_DMACSR_R_CTRL_ENA);
+ dev->flags |= HIFN_FLAG_RES_BUSY;
+ }
+}
+
+static void hifn_setup_dst_desc(struct hifn_device *dev, struct page *page,
+ unsigned offset, unsigned size)
+{
+ struct hifn_dma *dma = (struct hifn_dma *)dev->desc_virt;
+ int idx;
+ dma_addr_t addr;
+
+ addr = pci_map_page(dev->pdev, page, offset, size, PCI_DMA_FROMDEVICE);
+
+ idx = dma->dsti;
+ dma->dstr[idx].p = __cpu_to_le32(addr);
+ dma->dstr[idx].l = __cpu_to_le32(size | HIFN_D_VALID |
+ HIFN_D_MASKDONEIRQ | HIFN_D_LAST);
+
+ if (++idx == HIFN_D_DST_RSIZE) {
+ dma->dstr[idx].l = __cpu_to_le32(HIFN_D_VALID |
+ HIFN_D_JUMP | HIFN_D_MASKDONEIRQ |
+ HIFN_D_LAST);
+ idx = 0;
+ }
+ dma->dsti = idx;
+ dma->dstu++;
+
+ if (!(dev->flags & HIFN_FLAG_DST_BUSY)) {
+ hifn_write_1(dev, HIFN_1_DMA_CSR, HIFN_DMACSR_D_CTRL_ENA);
+ dev->flags |= HIFN_FLAG_DST_BUSY;
+ }
+}
+
+static int hifn_setup_dma(struct hifn_device *dev, struct page *spage, unsigned int soff,
+ struct page *dpage, unsigned int doff, unsigned int nbytes, void *priv,
+ struct hifn_context *ctx)
+{
+ dprintk("%s: spage: %p, soffset: %u, dpage: %p, doffset: %u, nbytes: %u, priv: %p, ctx: %p.\n",
+ dev->name, spage, soff, dpage, doff, nbytes, priv, ctx);
+
+ hifn_setup_src_desc(dev, spage, soff, nbytes);
+ hifn_setup_cmd_desc(dev, ctx, priv, nbytes);
+ hifn_setup_dst_desc(dev, dpage, doff, nbytes);
+ hifn_setup_res_desc(dev);
+ return 0;
+}
+
static int ablkcipher_walk_init(struct ablkcipher_walk *w,
int num, gfp_t gfp_flags)
{
return -EINVAL;
while (size) {
- copy = min(drest, src->length);
+ copy = min(drest, min(size, src->length));
saddr = kmap_atomic(sg_page(src), KM_SOFTIRQ1);
memcpy(daddr, saddr + src->offset, copy);
static int ablkcipher_walk(struct ablkcipher_request *req,
struct ablkcipher_walk *w)
{
- unsigned blocksize =
- crypto_ablkcipher_blocksize(crypto_ablkcipher_reqtfm(req));
- unsigned alignmask =
- crypto_ablkcipher_alignmask(crypto_ablkcipher_reqtfm(req));
struct scatterlist *src, *dst, *t;
void *daddr;
unsigned int nbytes = req->nbytes, offset, copy, diff;
dst = &req->dst[idx];
dprintk("\n%s: slen: %u, dlen: %u, soff: %u, doff: %u, offset: %u, "
- "blocksize: %u, nbytes: %u.\n",
+ "nbytes: %u.\n",
__func__, src->length, dst->length, src->offset,
- dst->offset, offset, blocksize, nbytes);
-
- if (src->length & (blocksize - 1) ||
- src->offset & (alignmask - 1) ||
- dst->length & (blocksize - 1) ||
- dst->offset & (alignmask - 1) ||
- offset) {
- unsigned slen = src->length - offset;
+ dst->offset, offset, nbytes);
+
+ if (!IS_ALIGNED(dst->offset, HIFN_D_DST_DALIGN) ||
+ !IS_ALIGNED(dst->length, HIFN_D_DST_DALIGN) ||
+ offset) {
+ unsigned slen = min(src->length - offset, nbytes);
unsigned dlen = PAGE_SIZE;
t = &w->cache[idx];
idx += err;
- copy = slen & ~(blocksize - 1);
- diff = slen & (blocksize - 1);
+ copy = slen & ~(HIFN_D_DST_DALIGN - 1);
+ diff = slen & (HIFN_D_DST_DALIGN - 1);
if (dlen < nbytes) {
/*
* to put there additional blocksized chunk,
* so we mark that page as containing only
* blocksize aligned chunks:
- * t->length = (slen & ~(blocksize - 1));
+ * t->length = (slen & ~(HIFN_D_DST_DALIGN - 1));
* and increase number of bytes to be processed
* in next chunk:
* nbytes += diff;
kunmap_atomic(daddr, KM_SOFTIRQ0);
} else {
- nbytes -= src->length;
+ nbytes -= min(src->length, nbytes);
idx++;
}
struct hifn_context *ctx = crypto_tfm_ctx(req->base.tfm);
struct hifn_device *dev = ctx->dev;
struct page *spage, *dpage;
- unsigned long soff, doff, flags;
+ unsigned long soff, doff, dlen, flags;
unsigned int nbytes = req->nbytes, idx = 0, len;
int err = -EINVAL, sg_num;
struct scatterlist *src, *dst, *t;
- unsigned blocksize =
- crypto_ablkcipher_blocksize(crypto_ablkcipher_reqtfm(req));
- unsigned alignmask =
- crypto_ablkcipher_alignmask(crypto_ablkcipher_reqtfm(req));
if (ctx->iv && !ctx->ivsize && ctx->mode != ACRYPTO_MODE_ECB)
goto err_out_exit;
ctx->walk.flags = 0;
while (nbytes) {
- src = &req->src[idx];
dst = &req->dst[idx];
+ dlen = min(dst->length, nbytes);
- if (src->length & (blocksize - 1) ||
- src->offset & (alignmask - 1) ||
- dst->length & (blocksize - 1) ||
- dst->offset & (alignmask - 1)) {
+ if (!IS_ALIGNED(dst->offset, HIFN_D_DST_DALIGN) ||
+ !IS_ALIGNED(dlen, HIFN_D_DST_DALIGN))
ctx->walk.flags |= ASYNC_FLAGS_MISALIGNED;
- }
- nbytes -= src->length;
+ nbytes -= dlen;
idx++;
}
idx = 0;
sg_num = ablkcipher_walk(req, &ctx->walk);
-
+ if (sg_num < 0) {
+ err = sg_num;
+ goto err_out_exit;
+ }
atomic_set(&ctx->sg_num, sg_num);
spin_lock_irqsave(&dev->lock, flags);
if (err)
goto err_out;
- nbytes -= len;
+ nbytes -= min(len, nbytes);
}
dev->active = HIFN_DEFAULT_ACTIVE_NUM;
return -EINVAL;
while (size) {
-
- copy = min(dst->length, srest);
+ copy = min(srest, min(dst->length, size));
daddr = kmap_atomic(sg_page(dst), KM_IRQ0);
memcpy(daddr + dst->offset + offset, saddr, copy);
sg_page(dst), dst->length, nbytes);
if (!t->length) {
- nbytes -= dst->length;
+ nbytes -= min(dst->length, nbytes);
idx++;
continue;
}
* 3DES ECB, CBC, CFB and OFB modes.
*/
{
- .name = "cfb(des3_ede)", .drv_name = "hifn-3des", .bsize = 8,
+ .name = "cfb(des3_ede)", .drv_name = "cfb-3des", .bsize = 8,
.ablkcipher = {
.min_keysize = HIFN_3DES_KEY_LENGTH,
.max_keysize = HIFN_3DES_KEY_LENGTH,
},
},
{
- .name = "ofb(des3_ede)", .drv_name = "hifn-3des", .bsize = 8,
+ .name = "ofb(des3_ede)", .drv_name = "ofb-3des", .bsize = 8,
.ablkcipher = {
.min_keysize = HIFN_3DES_KEY_LENGTH,
.max_keysize = HIFN_3DES_KEY_LENGTH,
},
},
{
- .name = "cbc(des3_ede)", .drv_name = "hifn-3des", .bsize = 8,
+ .name = "cbc(des3_ede)", .drv_name = "cbc-3des", .bsize = 8,
.ablkcipher = {
+ .ivsize = HIFN_IV_LENGTH,
.min_keysize = HIFN_3DES_KEY_LENGTH,
.max_keysize = HIFN_3DES_KEY_LENGTH,
.setkey = hifn_setkey,
},
},
{
- .name = "ecb(des3_ede)", .drv_name = "hifn-3des", .bsize = 8,
+ .name = "ecb(des3_ede)", .drv_name = "ecb-3des", .bsize = 8,
.ablkcipher = {
.min_keysize = HIFN_3DES_KEY_LENGTH,
.max_keysize = HIFN_3DES_KEY_LENGTH,
* DES ECB, CBC, CFB and OFB modes.
*/
{
- .name = "cfb(des)", .drv_name = "hifn-des", .bsize = 8,
+ .name = "cfb(des)", .drv_name = "cfb-des", .bsize = 8,
.ablkcipher = {
.min_keysize = HIFN_DES_KEY_LENGTH,
.max_keysize = HIFN_DES_KEY_LENGTH,
},
},
{
- .name = "ofb(des)", .drv_name = "hifn-des", .bsize = 8,
+ .name = "ofb(des)", .drv_name = "ofb-des", .bsize = 8,
.ablkcipher = {
.min_keysize = HIFN_DES_KEY_LENGTH,
.max_keysize = HIFN_DES_KEY_LENGTH,
},
},
{
- .name = "cbc(des)", .drv_name = "hifn-des", .bsize = 8,
+ .name = "cbc(des)", .drv_name = "cbc-des", .bsize = 8,
.ablkcipher = {
+ .ivsize = HIFN_IV_LENGTH,
.min_keysize = HIFN_DES_KEY_LENGTH,
.max_keysize = HIFN_DES_KEY_LENGTH,
.setkey = hifn_setkey,
},
},
{
- .name = "ecb(des)", .drv_name = "hifn-des", .bsize = 8,
+ .name = "ecb(des)", .drv_name = "ecb-des", .bsize = 8,
.ablkcipher = {
.min_keysize = HIFN_DES_KEY_LENGTH,
.max_keysize = HIFN_DES_KEY_LENGTH,
* AES ECB, CBC, CFB and OFB modes.
*/
{
- .name = "ecb(aes)", .drv_name = "hifn-aes", .bsize = 16,
+ .name = "ecb(aes)", .drv_name = "ecb-aes", .bsize = 16,
.ablkcipher = {
.min_keysize = AES_MIN_KEY_SIZE,
.max_keysize = AES_MAX_KEY_SIZE,
},
},
{
- .name = "cbc(aes)", .drv_name = "hifn-aes", .bsize = 16,
+ .name = "cbc(aes)", .drv_name = "cbc-aes", .bsize = 16,
.ablkcipher = {
+ .ivsize = HIFN_AES_IV_LENGTH,
.min_keysize = AES_MIN_KEY_SIZE,
.max_keysize = AES_MAX_KEY_SIZE,
.setkey = hifn_setkey,
},
},
{
- .name = "cfb(aes)", .drv_name = "hifn-aes", .bsize = 16,
+ .name = "cfb(aes)", .drv_name = "cfb-aes", .bsize = 16,
.ablkcipher = {
.min_keysize = AES_MIN_KEY_SIZE,
.max_keysize = AES_MAX_KEY_SIZE,
},
},
{
- .name = "ofb(aes)", .drv_name = "hifn-aes", .bsize = 16,
+ .name = "ofb(aes)", .drv_name = "ofb-aes", .bsize = 16,
.ablkcipher = {
.min_keysize = AES_MIN_KEY_SIZE,
.max_keysize = AES_MAX_KEY_SIZE,
return -ENOMEM;
snprintf(alg->alg.cra_name, CRYPTO_MAX_ALG_NAME, "%s", t->name);
- snprintf(alg->alg.cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s", t->drv_name);
+ snprintf(alg->alg.cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s-%s",
+ t->drv_name, dev->name);
alg->alg.cra_priority = 300;
alg->alg.cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC;
alg->alg.cra_blocksize = t->bsize;
alg->alg.cra_ctxsize = sizeof(struct hifn_context);
- alg->alg.cra_alignmask = 15;
- if (t->bsize == 8)
- alg->alg.cra_alignmask = 3;
+ alg->alg.cra_alignmask = 0;
alg->alg.cra_type = &crypto_ablkcipher_type;
alg->alg.cra_module = THIS_MODULE;
alg->alg.cra_u.ablkcipher = t->ablkcipher;