--- 1/crypto/cipher.c 2004-11-23 16:43:49.000000000 +0100 +++ 2/crypto/cipher.c 2004-11-30 23:34:30.000000000 +0100 @@ -39,91 +39,49 @@ } -/* - * Generic encrypt/decrypt wrapper for ciphers, handles operations across - * multiple page boundaries by using temporary blocks. In user context, - * the kernel is given a chance to schedule us once per block. - */ -static int crypt(struct crypto_tfm *tfm, - struct scatterlist *dst, - struct scatterlist *src, - unsigned int nbytes, cryptfn_t crfn, - procfn_t prfn, int enc, void *info) -{ - struct scatter_walk walk_in, walk_out; - const unsigned int bsize = crypto_tfm_alg_blocksize(tfm); - u8 tmp_src[bsize]; - u8 tmp_dst[bsize]; - - if (!nbytes) - return 0; - - if (nbytes % bsize) { - tfm->crt_flags |= CRYPTO_TFM_RES_BAD_BLOCK_LEN; - return -EINVAL; - } - - scatterwalk_start(&walk_in, src); - scatterwalk_start(&walk_out, dst); - - for(;;) { - u8 *src_p, *dst_p; - int in_place; - - scatterwalk_map(&walk_in, 0); - scatterwalk_map(&walk_out, 1); - src_p = scatterwalk_whichbuf(&walk_in, bsize, tmp_src); - dst_p = scatterwalk_whichbuf(&walk_out, bsize, tmp_dst); - in_place = scatterwalk_samebuf(&walk_in, &walk_out, - src_p, dst_p); - - nbytes -= bsize; - - scatterwalk_copychunks(src_p, &walk_in, bsize, 0); - - prfn(tfm, dst_p, src_p, crfn, enc, info, in_place); - - scatterwalk_done(&walk_in, 0, nbytes); - - scatterwalk_copychunks(dst_p, &walk_out, bsize, 1); - scatterwalk_done(&walk_out, 1, nbytes); - - if (!nbytes) - return 0; - - crypto_yield(tfm); - } -} - -static void cbc_process(struct crypto_tfm *tfm, u8 *dst, u8 *src, - cryptfn_t fn, int enc, void *info, int in_place) -{ - u8 *iv = info; +struct cbc_process_priv { + struct crypto_tfm *tfm; + int enc; + cryptfn_t *crfn; + u8 *curIV; + u8 *nextIV; +}; + +static void cbc_process_gw(void *_priv, int nsg, void **buf) +{ + struct cbc_process_priv *priv = (struct cbc_process_priv *)_priv; + u8 *iv = priv->curIV; + struct crypto_tfm *tfm = priv->tfm; + int bsize = crypto_tfm_alg_blocksize(tfm); + u8 *dst = buf[0]; + u8 *src = buf[1]; /* Null encryption */ if (!iv) return; - - if (enc) { + + if (priv->enc) { tfm->crt_u.cipher.cit_xor_block(iv, src); - fn(crypto_tfm_ctx(tfm), dst, iv); - memcpy(iv, dst, crypto_tfm_alg_blocksize(tfm)); + priv->crfn(crypto_tfm_ctx(tfm), dst, iv); + memcpy(iv, dst, bsize); } else { - u8 stack[in_place ? crypto_tfm_alg_blocksize(tfm) : 0]; - u8 *buf = in_place ? stack : dst; - - fn(crypto_tfm_ctx(tfm), buf, src); - tfm->crt_u.cipher.cit_xor_block(buf, iv); - memcpy(iv, src, crypto_tfm_alg_blocksize(tfm)); - if (buf != dst) - memcpy(dst, buf, crypto_tfm_alg_blocksize(tfm)); + memcpy(priv->nextIV,src,bsize); + priv->crfn(crypto_tfm_ctx(tfm), dst, src); + tfm->crt_u.cipher.cit_xor_block(dst, iv); + priv->curIV = priv->nextIV; + priv->nextIV = iv; } } -static void ecb_process(struct crypto_tfm *tfm, u8 *dst, u8 *src, - cryptfn_t fn, int enc, void *info, int in_place) +struct ecb_process_priv { + struct crypto_tfm *tfm; + cryptfn_t *crfn; +}; + +static void ecb_process_gw(void *_priv, int nsg, void **buf) { - fn(crypto_tfm_ctx(tfm), dst, src); + struct ecb_process_priv *priv = (struct ecb_process_priv *)_priv; + priv->crfn(crypto_tfm_ctx(priv->tfm), buf[0], buf[1]); } static int setkey(struct crypto_tfm *tfm, const u8 *key, unsigned int keylen) @@ -142,9 +100,12 @@ struct scatterlist *dst, struct scatterlist *src, unsigned int nbytes) { - return crypt(tfm, dst, src, nbytes, - tfm->__crt_alg->cra_cipher.cia_encrypt, - ecb_process, 1, NULL); + struct ecb_process_priv priv = { + .tfm = tfm, + .crfn = tfm->__crt_alg->cra_cipher.cia_encrypt, + }; + int bsize = crypto_tfm_alg_blocksize(tfm); + return scatterwalk_walker_generic(ecb_process_gw, &priv, nbytes/bsize, 2, dst, bsize, 1, src, bsize, 0); } static int ecb_decrypt(struct crypto_tfm *tfm, @@ -152,29 +113,48 @@ struct scatterlist *src, unsigned int nbytes) { - return crypt(tfm, dst, src, nbytes, - tfm->__crt_alg->cra_cipher.cia_decrypt, - ecb_process, 1, NULL); + struct ecb_process_priv priv = { + .tfm = tfm, + .crfn = tfm->__crt_alg->cra_cipher.cia_decrypt, + }; + int bsize = crypto_tfm_alg_blocksize(tfm); + return scatterwalk_walker_generic(ecb_process_gw, &priv, nbytes/bsize, 2, dst, bsize, 1, src, bsize, 0); } -static int cbc_encrypt(struct crypto_tfm *tfm, - struct scatterlist *dst, - struct scatterlist *src, - unsigned int nbytes) +static int cbc_encrypt_iv(struct crypto_tfm *tfm, + struct scatterlist *dst, + struct scatterlist *src, + unsigned int nbytes, u8 *iv) { - return crypt(tfm, dst, src, nbytes, - tfm->__crt_alg->cra_cipher.cia_encrypt, - cbc_process, 1, tfm->crt_cipher.cit_iv); + int bsize = crypto_tfm_alg_blocksize(tfm); + struct cbc_process_priv priv = { + .tfm = tfm, + .enc = 1, + .curIV = iv, + .crfn = tfm->__crt_alg->cra_cipher.cia_encrypt, + }; + return scatterwalk_walker_generic(cbc_process_gw, &priv, nbytes/bsize, 2, dst, bsize, 1, src, bsize, 0); } -static int cbc_encrypt_iv(struct crypto_tfm *tfm, +static int cbc_decrypt_iv(struct crypto_tfm *tfm, struct scatterlist *dst, struct scatterlist *src, unsigned int nbytes, u8 *iv) { - return crypt(tfm, dst, src, nbytes, - tfm->__crt_alg->cra_cipher.cia_encrypt, - cbc_process, 1, iv); + int bsize = crypto_tfm_alg_blocksize(tfm); + u8 IVscratch[bsize]; + int r; + struct cbc_process_priv priv = { + .tfm = tfm, + .enc = 0, + .curIV = iv, + .nextIV = IVscratch, + .crfn = tfm->__crt_alg->cra_cipher.cia_decrypt, + }; + r = scatterwalk_walker_generic(cbc_process_gw, &priv, nbytes/bsize, 2, dst, bsize, 1, src, bsize, 0); + if(priv.curIV != iv) + memcpy(iv,priv.curIV,bsize); + return r; } static int cbc_decrypt(struct crypto_tfm *tfm, @@ -182,19 +162,15 @@ struct scatterlist *src, unsigned int nbytes) { - return crypt(tfm, dst, src, nbytes, - tfm->__crt_alg->cra_cipher.cia_decrypt, - cbc_process, 0, tfm->crt_cipher.cit_iv); + return cbc_decrypt_iv(tfm, dst, src, nbytes, tfm->crt_cipher.cit_iv); } -static int cbc_decrypt_iv(struct crypto_tfm *tfm, - struct scatterlist *dst, - struct scatterlist *src, - unsigned int nbytes, u8 *iv) +static int cbc_encrypt(struct crypto_tfm *tfm, + struct scatterlist *dst, + struct scatterlist *src, + unsigned int nbytes) { - return crypt(tfm, dst, src, nbytes, - tfm->__crt_alg->cra_cipher.cia_decrypt, - cbc_process, 0, iv); + return cbc_encrypt_iv(tfm, dst, src, nbytes, tfm->crt_cipher.cit_iv); } static int nocrypt(struct crypto_tfm *tfm, --- 1/crypto/api.c 2004-12-13 11:44:10.000000000 +0100 +++ 2/crypto/api.c 2004-12-11 17:04:45.000000000 +0100 @@ -131,7 +131,7 @@ struct crypto_tfm *tfm = NULL; struct crypto_alg *alg; int tfm_size; - + alg = crypto_alg_mod_lookup(name); if (alg == NULL) goto out; --- 1/crypto/scatterwalk.h 2004-11-23 16:44:04.000000000 +0100 +++ 2/crypto/scatterwalk.h 2004-11-30 23:31:05.000000000 +0100 @@ -16,6 +16,7 @@ #define _CRYPTO_SCATTERWALK_H #include #include +#include struct scatter_walk { struct scatterlist *sg; @@ -26,6 +27,15 @@ unsigned int offset; }; +#define scatterwalk_needscratch(walk, nbytes) \ + ((nbytes) <= (walk)->len_this_page && \ + (((unsigned long)(walk)->data) & (PAGE_CACHE_SIZE - 1)) + (nbytes) <= \ + PAGE_CACHE_SIZE) \ + + +#define scatterwalk_whichbuf(walk,nbytes,scratch) \ + (scatterwalk_needscratch(walk,nbytes)?(walk)->data:(scratch)) + /* Define sg_next is an inline routine now in case we want to change scatterlist to a linked list later. */ static inline struct scatterlist *sg_next(struct scatterlist *sg) @@ -42,10 +52,6 @@ walk_in->data == src_p && walk_out->data == dst_p; } -void *scatterwalk_whichbuf(struct scatter_walk *walk, unsigned int nbytes, void *scratch); -void scatterwalk_start(struct scatter_walk *walk, struct scatterlist *sg); -int scatterwalk_copychunks(void *buf, struct scatter_walk *walk, size_t nbytes, int out); -void scatterwalk_map(struct scatter_walk *walk, int out); -void scatterwalk_done(struct scatter_walk *walk, int out, int more); +int scatterwalk_walker_generic(void (function)(void *priv, int length, void **buflist), void *priv, int steps, int nsl, ...); #endif /* _CRYPTO_SCATTERWALK_H */ --- 1/crypto/scatterwalk.c 2004-11-23 16:43:57.000000000 +0100 +++ 2/crypto/scatterwalk.c 2004-11-30 23:31:06.000000000 +0100 @@ -28,17 +28,7 @@ KM_SOFTIRQ1, }; -void *scatterwalk_whichbuf(struct scatter_walk *walk, unsigned int nbytes, void *scratch) -{ - if (nbytes <= walk->len_this_page && - (((unsigned long)walk->data) & (PAGE_CACHE_SIZE - 1)) + nbytes <= - PAGE_CACHE_SIZE) - return walk->data; - else - return scratch; -} - -static void memcpy_dir(void *buf, void *sgdata, size_t nbytes, int out) +static inline void memcpy_dir(void *buf, void *sgdata, size_t nbytes, int out) { if (out) memcpy(sgdata, buf, nbytes); @@ -46,7 +36,7 @@ memcpy(buf, sgdata, nbytes); } -void scatterwalk_start(struct scatter_walk *walk, struct scatterlist *sg) +static inline void scatterwalk_start(struct scatter_walk *walk, struct scatterlist *sg) { unsigned int rest_of_page; @@ -60,10 +50,8 @@ walk->offset = sg->offset; } -void scatterwalk_map(struct scatter_walk *walk, int out) -{ - walk->data = crypto_kmap(walk->page, out) + walk->offset; -} +#define scatterwalk_map(walk,out) \ + (walk)->data = crypto_kmap((walk)->page, (out)) + (walk)->offset; static void scatterwalk_pagedone(struct scatter_walk *walk, int out, unsigned int more) @@ -89,36 +77,98 @@ } } -void scatterwalk_done(struct scatter_walk *walk, int out, int more) -{ - crypto_kunmap(walk->data, out); - if (walk->len_this_page == 0 || !more) - scatterwalk_pagedone(walk, out, more); -} +#define scatterwalk_copychunks(xbuf, walk, xnbytes, out) do { \ + int nbytes = (xnbytes); \ + char *buf = (xbuf); \ + if (buf != (walk)->data) { \ + while (nbytes > (walk)->len_this_page) { \ + memcpy_dir(buf, (walk)->data, (walk)->len_this_page, (out)); \ + buf += (walk)->len_this_page; \ + nbytes -= (walk)->len_this_page; \ + \ + crypto_kunmap((walk)->data, (out)); \ + scatterwalk_pagedone((walk), (out), 1); \ + scatterwalk_map((walk), (out)); \ + } \ + memcpy_dir(buf, (walk)->data, nbytes, (out)); \ + } \ + \ + (walk)->offset += nbytes; \ + (walk)->len_this_page -= nbytes; \ + (walk)->len_this_segment -= nbytes; \ +} while(0); + +int scatterwalk_walker_generic(void (function)(void *priv, int length, void **buflist), void *priv, int steps, int nsl, ...) +{ + va_list ap; + int i; + int r = -EINVAL; + struct sg_proc { + struct scatterlist *sg; + struct scatter_walk sw; + int stepsize; + int ioflag; + char *buf; + }; + struct sg_proc sg_proc_info[nsl+1]; + struct sg_proc *csg; + + void *dispatch_list[nsl]; + void **cbuf; + + va_start(ap,nsl); + for(i=0; isg; csg++, cbuf++) { + /* If a scratch is needed, do a lazy kmallocation */ + *cbuf = scatterwalk_needscratch(&csg->sw,csg->stepsize)? + csg->sw.data:( + csg->buf == NULL? + ({ + csg->buf = kmalloc(csg->stepsize,GFP_KERNEL); + if(csg->buf == NULL) { + r = -ENOMEM; + goto out_buffers; + } + csg->buf; + }):csg->buf); -/* - * Do not call this unless the total length of all of the fragments - * has been verified as multiple of the block size. - */ -int scatterwalk_copychunks(void *buf, struct scatter_walk *walk, - size_t nbytes, int out) -{ - if (buf != walk->data) { - while (nbytes > walk->len_this_page) { - memcpy_dir(buf, walk->data, walk->len_this_page, out); - buf += walk->len_this_page; - nbytes -= walk->len_this_page; - - crypto_kunmap(walk->data, out); - scatterwalk_pagedone(walk, out, 1); - scatterwalk_map(walk, out); + if(csg->ioflag == 0) + scatterwalk_copychunks(*cbuf,&csg->sw,csg->stepsize,0); } - memcpy_dir(buf, walk->data, nbytes, out); - } + function(priv, nsl, dispatch_list); - walk->offset += nbytes; - walk->len_this_page -= nbytes; - walk->len_this_segment -= nbytes; - return 0; + for(csg = sg_proc_info, cbuf = dispatch_list; csg->sg; csg++, cbuf++) { + if(csg->ioflag == 1) + scatterwalk_copychunks(*cbuf,&csg->sw,csg->stepsize,1); + + if (csg->sw.len_this_page == 0 && steps) { + crypto_kunmap(csg->sw.data, csg->ioflag); + scatterwalk_pagedone(&csg->sw, csg->ioflag, steps); + scatterwalk_map(&csg->sw,csg->ioflag); + } else { + csg->sw.data += csg->stepsize; + } + } + } + r = 0; +out_buffers: + for(csg = sg_proc_info; csg->sg; csg++) { + crypto_kunmap(csg->sw.data, csg->ioflag); + scatterwalk_pagedone(&csg->sw, csg->ioflag, 0); + if(csg->buf) + kfree(csg->buf); + } + return r; }