blowfish_glue.c 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482
  1. /*
  2. * Glue Code for assembler optimized version of Blowfish
  3. *
  4. * Copyright (c) 2011 Jussi Kivilinna <jussi.kivilinna@mbnet.fi>
  5. *
  6. * CBC & ECB parts based on code (crypto/cbc.c,ecb.c) by:
  7. * Copyright (c) 2006 Herbert Xu <herbert@gondor.apana.org.au>
  8. * CTR part based on code (crypto/ctr.c) by:
  9. * (C) Copyright IBM Corp. 2007 - Joy Latten <latten@us.ibm.com>
  10. *
  11. * This program is free software; you can redistribute it and/or modify
  12. * it under the terms of the GNU General Public License as published by
  13. * the Free Software Foundation; either version 2 of the License, or
  14. * (at your option) any later version.
  15. *
  16. * This program is distributed in the hope that it will be useful,
  17. * but WITHOUT ANY WARRANTY; without even the implied warranty of
  18. * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  19. * GNU General Public License for more details.
  20. *
  21. * You should have received a copy of the GNU General Public License
  22. * along with this program; if not, write to the Free Software
  23. * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
  24. * USA
  25. *
  26. */
  27. #include <asm/processor.h>
  28. #include <crypto/blowfish.h>
  29. #include <linux/crypto.h>
  30. #include <linux/init.h>
  31. #include <linux/module.h>
  32. #include <linux/types.h>
  33. #include <crypto/algapi.h>
  34. /* regular block cipher functions */
  35. asmlinkage void __blowfish_enc_blk(struct bf_ctx *ctx, u8 *dst, const u8 *src,
  36. bool xor);
  37. asmlinkage void blowfish_dec_blk(struct bf_ctx *ctx, u8 *dst, const u8 *src);
  38. /* 4-way parallel cipher functions */
  39. asmlinkage void __blowfish_enc_blk_4way(struct bf_ctx *ctx, u8 *dst,
  40. const u8 *src, bool xor);
  41. asmlinkage void blowfish_dec_blk_4way(struct bf_ctx *ctx, u8 *dst,
  42. const u8 *src);
  43. static inline void blowfish_enc_blk(struct bf_ctx *ctx, u8 *dst, const u8 *src)
  44. {
  45. __blowfish_enc_blk(ctx, dst, src, false);
  46. }
  47. static inline void blowfish_enc_blk_xor(struct bf_ctx *ctx, u8 *dst,
  48. const u8 *src)
  49. {
  50. __blowfish_enc_blk(ctx, dst, src, true);
  51. }
  52. static inline void blowfish_enc_blk_4way(struct bf_ctx *ctx, u8 *dst,
  53. const u8 *src)
  54. {
  55. __blowfish_enc_blk_4way(ctx, dst, src, false);
  56. }
  57. static inline void blowfish_enc_blk_xor_4way(struct bf_ctx *ctx, u8 *dst,
  58. const u8 *src)
  59. {
  60. __blowfish_enc_blk_4way(ctx, dst, src, true);
  61. }
  62. static void blowfish_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
  63. {
  64. blowfish_enc_blk(crypto_tfm_ctx(tfm), dst, src);
  65. }
  66. static void blowfish_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
  67. {
  68. blowfish_dec_blk(crypto_tfm_ctx(tfm), dst, src);
  69. }
  70. static int ecb_crypt(struct blkcipher_desc *desc, struct blkcipher_walk *walk,
  71. void (*fn)(struct bf_ctx *, u8 *, const u8 *),
  72. void (*fn_4way)(struct bf_ctx *, u8 *, const u8 *))
  73. {
  74. struct bf_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
  75. unsigned int bsize = BF_BLOCK_SIZE;
  76. unsigned int nbytes;
  77. int err;
  78. err = blkcipher_walk_virt(desc, walk);
  79. while ((nbytes = walk->nbytes)) {
  80. u8 *wsrc = walk->src.virt.addr;
  81. u8 *wdst = walk->dst.virt.addr;
  82. /* Process four block batch */
  83. if (nbytes >= bsize * 4) {
  84. do {
  85. fn_4way(ctx, wdst, wsrc);
  86. wsrc += bsize * 4;
  87. wdst += bsize * 4;
  88. nbytes -= bsize * 4;
  89. } while (nbytes >= bsize * 4);
  90. if (nbytes < bsize)
  91. goto done;
  92. }
  93. /* Handle leftovers */
  94. do {
  95. fn(ctx, wdst, wsrc);
  96. wsrc += bsize;
  97. wdst += bsize;
  98. nbytes -= bsize;
  99. } while (nbytes >= bsize);
  100. done:
  101. err = blkcipher_walk_done(desc, walk, nbytes);
  102. }
  103. return err;
  104. }
  105. static int ecb_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
  106. struct scatterlist *src, unsigned int nbytes)
  107. {
  108. struct blkcipher_walk walk;
  109. blkcipher_walk_init(&walk, dst, src, nbytes);
  110. return ecb_crypt(desc, &walk, blowfish_enc_blk, blowfish_enc_blk_4way);
  111. }
  112. static int ecb_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
  113. struct scatterlist *src, unsigned int nbytes)
  114. {
  115. struct blkcipher_walk walk;
  116. blkcipher_walk_init(&walk, dst, src, nbytes);
  117. return ecb_crypt(desc, &walk, blowfish_dec_blk, blowfish_dec_blk_4way);
  118. }
  119. static unsigned int __cbc_encrypt(struct blkcipher_desc *desc,
  120. struct blkcipher_walk *walk)
  121. {
  122. struct bf_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
  123. unsigned int bsize = BF_BLOCK_SIZE;
  124. unsigned int nbytes = walk->nbytes;
  125. u64 *src = (u64 *)walk->src.virt.addr;
  126. u64 *dst = (u64 *)walk->dst.virt.addr;
  127. u64 *iv = (u64 *)walk->iv;
  128. do {
  129. *dst = *src ^ *iv;
  130. blowfish_enc_blk(ctx, (u8 *)dst, (u8 *)dst);
  131. iv = dst;
  132. src += 1;
  133. dst += 1;
  134. nbytes -= bsize;
  135. } while (nbytes >= bsize);
  136. *(u64 *)walk->iv = *iv;
  137. return nbytes;
  138. }
  139. static int cbc_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
  140. struct scatterlist *src, unsigned int nbytes)
  141. {
  142. struct blkcipher_walk walk;
  143. int err;
  144. blkcipher_walk_init(&walk, dst, src, nbytes);
  145. err = blkcipher_walk_virt(desc, &walk);
  146. while ((nbytes = walk.nbytes)) {
  147. nbytes = __cbc_encrypt(desc, &walk);
  148. err = blkcipher_walk_done(desc, &walk, nbytes);
  149. }
  150. return err;
  151. }
  152. static unsigned int __cbc_decrypt(struct blkcipher_desc *desc,
  153. struct blkcipher_walk *walk)
  154. {
  155. struct bf_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
  156. unsigned int bsize = BF_BLOCK_SIZE;
  157. unsigned int nbytes = walk->nbytes;
  158. u64 *src = (u64 *)walk->src.virt.addr;
  159. u64 *dst = (u64 *)walk->dst.virt.addr;
  160. u64 ivs[4 - 1];
  161. u64 last_iv;
  162. /* Start of the last block. */
  163. src += nbytes / bsize - 1;
  164. dst += nbytes / bsize - 1;
  165. last_iv = *src;
  166. /* Process four block batch */
  167. if (nbytes >= bsize * 4) {
  168. do {
  169. nbytes -= bsize * 4 - bsize;
  170. src -= 4 - 1;
  171. dst -= 4 - 1;
  172. ivs[0] = src[0];
  173. ivs[1] = src[1];
  174. ivs[2] = src[2];
  175. blowfish_dec_blk_4way(ctx, (u8 *)dst, (u8 *)src);
  176. dst[1] ^= ivs[0];
  177. dst[2] ^= ivs[1];
  178. dst[3] ^= ivs[2];
  179. nbytes -= bsize;
  180. if (nbytes < bsize)
  181. goto done;
  182. *dst ^= *(src - 1);
  183. src -= 1;
  184. dst -= 1;
  185. } while (nbytes >= bsize * 4);
  186. }
  187. /* Handle leftovers */
  188. for (;;) {
  189. blowfish_dec_blk(ctx, (u8 *)dst, (u8 *)src);
  190. nbytes -= bsize;
  191. if (nbytes < bsize)
  192. break;
  193. *dst ^= *(src - 1);
  194. src -= 1;
  195. dst -= 1;
  196. }
  197. done:
  198. *dst ^= *(u64 *)walk->iv;
  199. *(u64 *)walk->iv = last_iv;
  200. return nbytes;
  201. }
  202. static int cbc_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
  203. struct scatterlist *src, unsigned int nbytes)
  204. {
  205. struct blkcipher_walk walk;
  206. int err;
  207. blkcipher_walk_init(&walk, dst, src, nbytes);
  208. err = blkcipher_walk_virt(desc, &walk);
  209. while ((nbytes = walk.nbytes)) {
  210. nbytes = __cbc_decrypt(desc, &walk);
  211. err = blkcipher_walk_done(desc, &walk, nbytes);
  212. }
  213. return err;
  214. }
  215. static void ctr_crypt_final(struct bf_ctx *ctx, struct blkcipher_walk *walk)
  216. {
  217. u8 *ctrblk = walk->iv;
  218. u8 keystream[BF_BLOCK_SIZE];
  219. u8 *src = walk->src.virt.addr;
  220. u8 *dst = walk->dst.virt.addr;
  221. unsigned int nbytes = walk->nbytes;
  222. blowfish_enc_blk(ctx, keystream, ctrblk);
  223. crypto_xor(keystream, src, nbytes);
  224. memcpy(dst, keystream, nbytes);
  225. crypto_inc(ctrblk, BF_BLOCK_SIZE);
  226. }
  227. static unsigned int __ctr_crypt(struct blkcipher_desc *desc,
  228. struct blkcipher_walk *walk)
  229. {
  230. struct bf_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
  231. unsigned int bsize = BF_BLOCK_SIZE;
  232. unsigned int nbytes = walk->nbytes;
  233. u64 *src = (u64 *)walk->src.virt.addr;
  234. u64 *dst = (u64 *)walk->dst.virt.addr;
  235. u64 ctrblk = be64_to_cpu(*(__be64 *)walk->iv);
  236. __be64 ctrblocks[4];
  237. /* Process four block batch */
  238. if (nbytes >= bsize * 4) {
  239. do {
  240. if (dst != src) {
  241. dst[0] = src[0];
  242. dst[1] = src[1];
  243. dst[2] = src[2];
  244. dst[3] = src[3];
  245. }
  246. /* create ctrblks for parallel encrypt */
  247. ctrblocks[0] = cpu_to_be64(ctrblk++);
  248. ctrblocks[1] = cpu_to_be64(ctrblk++);
  249. ctrblocks[2] = cpu_to_be64(ctrblk++);
  250. ctrblocks[3] = cpu_to_be64(ctrblk++);
  251. blowfish_enc_blk_xor_4way(ctx, (u8 *)dst,
  252. (u8 *)ctrblocks);
  253. src += 4;
  254. dst += 4;
  255. } while ((nbytes -= bsize * 4) >= bsize * 4);
  256. if (nbytes < bsize)
  257. goto done;
  258. }
  259. /* Handle leftovers */
  260. do {
  261. if (dst != src)
  262. *dst = *src;
  263. ctrblocks[0] = cpu_to_be64(ctrblk++);
  264. blowfish_enc_blk_xor(ctx, (u8 *)dst, (u8 *)ctrblocks);
  265. src += 1;
  266. dst += 1;
  267. } while ((nbytes -= bsize) >= bsize);
  268. done:
  269. *(__be64 *)walk->iv = cpu_to_be64(ctrblk);
  270. return nbytes;
  271. }
  272. static int ctr_crypt(struct blkcipher_desc *desc, struct scatterlist *dst,
  273. struct scatterlist *src, unsigned int nbytes)
  274. {
  275. struct blkcipher_walk walk;
  276. int err;
  277. blkcipher_walk_init(&walk, dst, src, nbytes);
  278. err = blkcipher_walk_virt_block(desc, &walk, BF_BLOCK_SIZE);
  279. while ((nbytes = walk.nbytes) >= BF_BLOCK_SIZE) {
  280. nbytes = __ctr_crypt(desc, &walk);
  281. err = blkcipher_walk_done(desc, &walk, nbytes);
  282. }
  283. if (walk.nbytes) {
  284. ctr_crypt_final(crypto_blkcipher_ctx(desc->tfm), &walk);
  285. err = blkcipher_walk_done(desc, &walk, 0);
  286. }
  287. return err;
  288. }
  289. static struct crypto_alg bf_algs[4] = { {
  290. .cra_name = "blowfish",
  291. .cra_driver_name = "blowfish-asm",
  292. .cra_priority = 200,
  293. .cra_flags = CRYPTO_ALG_TYPE_CIPHER,
  294. .cra_blocksize = BF_BLOCK_SIZE,
  295. .cra_ctxsize = sizeof(struct bf_ctx),
  296. .cra_alignmask = 0,
  297. .cra_module = THIS_MODULE,
  298. .cra_u = {
  299. .cipher = {
  300. .cia_min_keysize = BF_MIN_KEY_SIZE,
  301. .cia_max_keysize = BF_MAX_KEY_SIZE,
  302. .cia_setkey = blowfish_setkey,
  303. .cia_encrypt = blowfish_encrypt,
  304. .cia_decrypt = blowfish_decrypt,
  305. }
  306. }
  307. }, {
  308. .cra_name = "ecb(blowfish)",
  309. .cra_driver_name = "ecb-blowfish-asm",
  310. .cra_priority = 300,
  311. .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
  312. .cra_blocksize = BF_BLOCK_SIZE,
  313. .cra_ctxsize = sizeof(struct bf_ctx),
  314. .cra_alignmask = 0,
  315. .cra_type = &crypto_blkcipher_type,
  316. .cra_module = THIS_MODULE,
  317. .cra_u = {
  318. .blkcipher = {
  319. .min_keysize = BF_MIN_KEY_SIZE,
  320. .max_keysize = BF_MAX_KEY_SIZE,
  321. .setkey = blowfish_setkey,
  322. .encrypt = ecb_encrypt,
  323. .decrypt = ecb_decrypt,
  324. },
  325. },
  326. }, {
  327. .cra_name = "cbc(blowfish)",
  328. .cra_driver_name = "cbc-blowfish-asm",
  329. .cra_priority = 300,
  330. .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
  331. .cra_blocksize = BF_BLOCK_SIZE,
  332. .cra_ctxsize = sizeof(struct bf_ctx),
  333. .cra_alignmask = 0,
  334. .cra_type = &crypto_blkcipher_type,
  335. .cra_module = THIS_MODULE,
  336. .cra_u = {
  337. .blkcipher = {
  338. .min_keysize = BF_MIN_KEY_SIZE,
  339. .max_keysize = BF_MAX_KEY_SIZE,
  340. .ivsize = BF_BLOCK_SIZE,
  341. .setkey = blowfish_setkey,
  342. .encrypt = cbc_encrypt,
  343. .decrypt = cbc_decrypt,
  344. },
  345. },
  346. }, {
  347. .cra_name = "ctr(blowfish)",
  348. .cra_driver_name = "ctr-blowfish-asm",
  349. .cra_priority = 300,
  350. .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
  351. .cra_blocksize = 1,
  352. .cra_ctxsize = sizeof(struct bf_ctx),
  353. .cra_alignmask = 0,
  354. .cra_type = &crypto_blkcipher_type,
  355. .cra_module = THIS_MODULE,
  356. .cra_u = {
  357. .blkcipher = {
  358. .min_keysize = BF_MIN_KEY_SIZE,
  359. .max_keysize = BF_MAX_KEY_SIZE,
  360. .ivsize = BF_BLOCK_SIZE,
  361. .setkey = blowfish_setkey,
  362. .encrypt = ctr_crypt,
  363. .decrypt = ctr_crypt,
  364. },
  365. },
  366. } };
  367. static bool is_blacklisted_cpu(void)
  368. {
  369. if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL)
  370. return false;
  371. if (boot_cpu_data.x86 == 0x0f) {
  372. /*
  373. * On Pentium 4, blowfish-x86_64 is slower than generic C
  374. * implementation because use of 64bit rotates (which are really
  375. * slow on P4). Therefore blacklist P4s.
  376. */
  377. return true;
  378. }
  379. return false;
  380. }
  381. static int force;
  382. module_param(force, int, 0);
  383. MODULE_PARM_DESC(force, "Force module load, ignore CPU blacklist");
  384. static int __init init(void)
  385. {
  386. if (!force && is_blacklisted_cpu()) {
  387. printk(KERN_INFO
  388. "blowfish-x86_64: performance on this CPU "
  389. "would be suboptimal: disabling "
  390. "blowfish-x86_64.\n");
  391. return -ENODEV;
  392. }
  393. return crypto_register_algs(bf_algs, ARRAY_SIZE(bf_algs));
  394. }
  395. static void __exit fini(void)
  396. {
  397. crypto_unregister_algs(bf_algs, ARRAY_SIZE(bf_algs));
  398. }
  399. module_init(init);
  400. module_exit(fini);
  401. MODULE_LICENSE("GPL");
  402. MODULE_DESCRIPTION("Blowfish Cipher Algorithm, asm optimized");
  403. MODULE_ALIAS_CRYPTO("blowfish");
  404. MODULE_ALIAS_CRYPTO("blowfish-asm");