serpent-avx-x86_64-asm_64.S 23 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782
  1. /*
  2. * Serpent Cipher 8-way parallel algorithm (x86_64/AVX)
  3. *
  4. * Copyright (C) 2012 Johannes Goetzfried
  5. * <Johannes.Goetzfried@informatik.stud.uni-erlangen.de>
  6. *
  7. * Copyright © 2011-2013 Jussi Kivilinna <jussi.kivilinna@iki.fi>
  8. *
  9. * This program is free software; you can redistribute it and/or modify
  10. * it under the terms of the GNU General Public License as published by
  11. * the Free Software Foundation; either version 2 of the License, or
  12. * (at your option) any later version.
  13. *
  14. * This program is distributed in the hope that it will be useful,
  15. * but WITHOUT ANY WARRANTY; without even the implied warranty of
  16. * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  17. * GNU General Public License for more details.
  18. *
  19. * You should have received a copy of the GNU General Public License
  20. * along with this program; if not, write to the Free Software
  21. * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
  22. * USA
  23. *
  24. */
  25. #include <linux/linkage.h>
  26. #include "glue_helper-asm-avx.S"
  27. .file "serpent-avx-x86_64-asm_64.S"
  28. .data
  29. .align 16
  30. .Lbswap128_mask:
  31. .byte 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0
  32. .Lxts_gf128mul_and_shl1_mask:
  33. .byte 0x87, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0
  34. .text
  35. #define CTX %rdi
  36. /**********************************************************************
  37. 8-way AVX serpent
  38. **********************************************************************/
  39. #define RA1 %xmm0
  40. #define RB1 %xmm1
  41. #define RC1 %xmm2
  42. #define RD1 %xmm3
  43. #define RE1 %xmm4
  44. #define tp %xmm5
  45. #define RA2 %xmm6
  46. #define RB2 %xmm7
  47. #define RC2 %xmm8
  48. #define RD2 %xmm9
  49. #define RE2 %xmm10
  50. #define RNOT %xmm11
  51. #define RK0 %xmm12
  52. #define RK1 %xmm13
  53. #define RK2 %xmm14
  54. #define RK3 %xmm15
  55. #define S0_1(x0, x1, x2, x3, x4) \
  56. vpor x0, x3, tp; \
  57. vpxor x3, x0, x0; \
  58. vpxor x2, x3, x4; \
  59. vpxor RNOT, x4, x4; \
  60. vpxor x1, tp, x3; \
  61. vpand x0, x1, x1; \
  62. vpxor x4, x1, x1; \
  63. vpxor x0, x2, x2;
  64. #define S0_2(x0, x1, x2, x3, x4) \
  65. vpxor x3, x0, x0; \
  66. vpor x0, x4, x4; \
  67. vpxor x2, x0, x0; \
  68. vpand x1, x2, x2; \
  69. vpxor x2, x3, x3; \
  70. vpxor RNOT, x1, x1; \
  71. vpxor x4, x2, x2; \
  72. vpxor x2, x1, x1;
  73. #define S1_1(x0, x1, x2, x3, x4) \
  74. vpxor x0, x1, tp; \
  75. vpxor x3, x0, x0; \
  76. vpxor RNOT, x3, x3; \
  77. vpand tp, x1, x4; \
  78. vpor tp, x0, x0; \
  79. vpxor x2, x3, x3; \
  80. vpxor x3, x0, x0; \
  81. vpxor x3, tp, x1;
  82. #define S1_2(x0, x1, x2, x3, x4) \
  83. vpxor x4, x3, x3; \
  84. vpor x4, x1, x1; \
  85. vpxor x2, x4, x4; \
  86. vpand x0, x2, x2; \
  87. vpxor x1, x2, x2; \
  88. vpor x0, x1, x1; \
  89. vpxor RNOT, x0, x0; \
  90. vpxor x2, x0, x0; \
  91. vpxor x1, x4, x4;
  92. #define S2_1(x0, x1, x2, x3, x4) \
  93. vpxor RNOT, x3, x3; \
  94. vpxor x0, x1, x1; \
  95. vpand x2, x0, tp; \
  96. vpxor x3, tp, tp; \
  97. vpor x0, x3, x3; \
  98. vpxor x1, x2, x2; \
  99. vpxor x1, x3, x3; \
  100. vpand tp, x1, x1;
  101. #define S2_2(x0, x1, x2, x3, x4) \
  102. vpxor x2, tp, tp; \
  103. vpand x3, x2, x2; \
  104. vpor x1, x3, x3; \
  105. vpxor RNOT, tp, tp; \
  106. vpxor tp, x3, x3; \
  107. vpxor tp, x0, x4; \
  108. vpxor x2, tp, x0; \
  109. vpor x2, x1, x1;
  110. #define S3_1(x0, x1, x2, x3, x4) \
  111. vpxor x3, x1, tp; \
  112. vpor x0, x3, x3; \
  113. vpand x0, x1, x4; \
  114. vpxor x2, x0, x0; \
  115. vpxor tp, x2, x2; \
  116. vpand x3, tp, x1; \
  117. vpxor x3, x2, x2; \
  118. vpor x4, x0, x0; \
  119. vpxor x3, x4, x4;
  120. #define S3_2(x0, x1, x2, x3, x4) \
  121. vpxor x0, x1, x1; \
  122. vpand x3, x0, x0; \
  123. vpand x4, x3, x3; \
  124. vpxor x2, x3, x3; \
  125. vpor x1, x4, x4; \
  126. vpand x1, x2, x2; \
  127. vpxor x3, x4, x4; \
  128. vpxor x3, x0, x0; \
  129. vpxor x2, x3, x3;
  130. #define S4_1(x0, x1, x2, x3, x4) \
  131. vpand x0, x3, tp; \
  132. vpxor x3, x0, x0; \
  133. vpxor x2, tp, tp; \
  134. vpor x3, x2, x2; \
  135. vpxor x1, x0, x0; \
  136. vpxor tp, x3, x4; \
  137. vpor x0, x2, x2; \
  138. vpxor x1, x2, x2;
  139. #define S4_2(x0, x1, x2, x3, x4) \
  140. vpand x0, x1, x1; \
  141. vpxor x4, x1, x1; \
  142. vpand x2, x4, x4; \
  143. vpxor tp, x2, x2; \
  144. vpxor x0, x4, x4; \
  145. vpor x1, tp, x3; \
  146. vpxor RNOT, x1, x1; \
  147. vpxor x0, x3, x3;
  148. #define S5_1(x0, x1, x2, x3, x4) \
  149. vpor x0, x1, tp; \
  150. vpxor tp, x2, x2; \
  151. vpxor RNOT, x3, x3; \
  152. vpxor x0, x1, x4; \
  153. vpxor x2, x0, x0; \
  154. vpand x4, tp, x1; \
  155. vpor x3, x4, x4; \
  156. vpxor x0, x4, x4;
  157. #define S5_2(x0, x1, x2, x3, x4) \
  158. vpand x3, x0, x0; \
  159. vpxor x3, x1, x1; \
  160. vpxor x2, x3, x3; \
  161. vpxor x1, x0, x0; \
  162. vpand x4, x2, x2; \
  163. vpxor x2, x1, x1; \
  164. vpand x0, x2, x2; \
  165. vpxor x2, x3, x3;
  166. #define S6_1(x0, x1, x2, x3, x4) \
  167. vpxor x0, x3, x3; \
  168. vpxor x2, x1, tp; \
  169. vpxor x0, x2, x2; \
  170. vpand x3, x0, x0; \
  171. vpor x3, tp, tp; \
  172. vpxor RNOT, x1, x4; \
  173. vpxor tp, x0, x0; \
  174. vpxor x2, tp, x1;
  175. #define S6_2(x0, x1, x2, x3, x4) \
  176. vpxor x4, x3, x3; \
  177. vpxor x0, x4, x4; \
  178. vpand x0, x2, x2; \
  179. vpxor x1, x4, x4; \
  180. vpxor x3, x2, x2; \
  181. vpand x1, x3, x3; \
  182. vpxor x0, x3, x3; \
  183. vpxor x2, x1, x1;
  184. #define S7_1(x0, x1, x2, x3, x4) \
  185. vpxor RNOT, x1, tp; \
  186. vpxor RNOT, x0, x0; \
  187. vpand x2, tp, x1; \
  188. vpxor x3, x1, x1; \
  189. vpor tp, x3, x3; \
  190. vpxor x2, tp, x4; \
  191. vpxor x3, x2, x2; \
  192. vpxor x0, x3, x3; \
  193. vpor x1, x0, x0;
  194. #define S7_2(x0, x1, x2, x3, x4) \
  195. vpand x0, x2, x2; \
  196. vpxor x4, x0, x0; \
  197. vpxor x3, x4, x4; \
  198. vpand x0, x3, x3; \
  199. vpxor x1, x4, x4; \
  200. vpxor x4, x2, x2; \
  201. vpxor x1, x3, x3; \
  202. vpor x0, x4, x4; \
  203. vpxor x1, x4, x4;
  204. #define SI0_1(x0, x1, x2, x3, x4) \
  205. vpxor x0, x1, x1; \
  206. vpor x1, x3, tp; \
  207. vpxor x1, x3, x4; \
  208. vpxor RNOT, x0, x0; \
  209. vpxor tp, x2, x2; \
  210. vpxor x0, tp, x3; \
  211. vpand x1, x0, x0; \
  212. vpxor x2, x0, x0;
  213. #define SI0_2(x0, x1, x2, x3, x4) \
  214. vpand x3, x2, x2; \
  215. vpxor x4, x3, x3; \
  216. vpxor x3, x2, x2; \
  217. vpxor x3, x1, x1; \
  218. vpand x0, x3, x3; \
  219. vpxor x0, x1, x1; \
  220. vpxor x2, x0, x0; \
  221. vpxor x3, x4, x4;
  222. #define SI1_1(x0, x1, x2, x3, x4) \
  223. vpxor x3, x1, x1; \
  224. vpxor x2, x0, tp; \
  225. vpxor RNOT, x2, x2; \
  226. vpor x1, x0, x4; \
  227. vpxor x3, x4, x4; \
  228. vpand x1, x3, x3; \
  229. vpxor x2, x1, x1; \
  230. vpand x4, x2, x2;
  231. #define SI1_2(x0, x1, x2, x3, x4) \
  232. vpxor x1, x4, x4; \
  233. vpor x3, x1, x1; \
  234. vpxor tp, x3, x3; \
  235. vpxor tp, x2, x2; \
  236. vpor x4, tp, x0; \
  237. vpxor x4, x2, x2; \
  238. vpxor x0, x1, x1; \
  239. vpxor x1, x4, x4;
  240. #define SI2_1(x0, x1, x2, x3, x4) \
  241. vpxor x1, x2, x2; \
  242. vpxor RNOT, x3, tp; \
  243. vpor x2, tp, tp; \
  244. vpxor x3, x2, x2; \
  245. vpxor x0, x3, x4; \
  246. vpxor x1, tp, x3; \
  247. vpor x2, x1, x1; \
  248. vpxor x0, x2, x2;
  249. #define SI2_2(x0, x1, x2, x3, x4) \
  250. vpxor x4, x1, x1; \
  251. vpor x3, x4, x4; \
  252. vpxor x3, x2, x2; \
  253. vpxor x2, x4, x4; \
  254. vpand x1, x2, x2; \
  255. vpxor x3, x2, x2; \
  256. vpxor x4, x3, x3; \
  257. vpxor x0, x4, x4;
  258. #define SI3_1(x0, x1, x2, x3, x4) \
  259. vpxor x1, x2, x2; \
  260. vpand x2, x1, tp; \
  261. vpxor x0, tp, tp; \
  262. vpor x1, x0, x0; \
  263. vpxor x3, x1, x4; \
  264. vpxor x3, x0, x0; \
  265. vpor tp, x3, x3; \
  266. vpxor x2, tp, x1;
  267. #define SI3_2(x0, x1, x2, x3, x4) \
  268. vpxor x3, x1, x1; \
  269. vpxor x2, x0, x0; \
  270. vpxor x3, x2, x2; \
  271. vpand x1, x3, x3; \
  272. vpxor x0, x1, x1; \
  273. vpand x2, x0, x0; \
  274. vpxor x3, x4, x4; \
  275. vpxor x0, x3, x3; \
  276. vpxor x1, x0, x0;
  277. #define SI4_1(x0, x1, x2, x3, x4) \
  278. vpxor x3, x2, x2; \
  279. vpand x1, x0, tp; \
  280. vpxor x2, tp, tp; \
  281. vpor x3, x2, x2; \
  282. vpxor RNOT, x0, x4; \
  283. vpxor tp, x1, x1; \
  284. vpxor x2, tp, x0; \
  285. vpand x4, x2, x2;
  286. #define SI4_2(x0, x1, x2, x3, x4) \
  287. vpxor x0, x2, x2; \
  288. vpor x4, x0, x0; \
  289. vpxor x3, x0, x0; \
  290. vpand x2, x3, x3; \
  291. vpxor x3, x4, x4; \
  292. vpxor x1, x3, x3; \
  293. vpand x0, x1, x1; \
  294. vpxor x1, x4, x4; \
  295. vpxor x3, x0, x0;
  296. #define SI5_1(x0, x1, x2, x3, x4) \
  297. vpor x2, x1, tp; \
  298. vpxor x1, x2, x2; \
  299. vpxor x3, tp, tp; \
  300. vpand x1, x3, x3; \
  301. vpxor x3, x2, x2; \
  302. vpor x0, x3, x3; \
  303. vpxor RNOT, x0, x0; \
  304. vpxor x2, x3, x3; \
  305. vpor x0, x2, x2;
  306. #define SI5_2(x0, x1, x2, x3, x4) \
  307. vpxor tp, x1, x4; \
  308. vpxor x4, x2, x2; \
  309. vpand x0, x4, x4; \
  310. vpxor tp, x0, x0; \
  311. vpxor x3, tp, x1; \
  312. vpand x2, x0, x0; \
  313. vpxor x3, x2, x2; \
  314. vpxor x2, x0, x0; \
  315. vpxor x4, x2, x2; \
  316. vpxor x3, x4, x4;
  317. #define SI6_1(x0, x1, x2, x3, x4) \
  318. vpxor x2, x0, x0; \
  319. vpand x3, x0, tp; \
  320. vpxor x3, x2, x2; \
  321. vpxor x2, tp, tp; \
  322. vpxor x1, x3, x3; \
  323. vpor x0, x2, x2; \
  324. vpxor x3, x2, x2; \
  325. vpand tp, x3, x3;
  326. #define SI6_2(x0, x1, x2, x3, x4) \
  327. vpxor RNOT, tp, tp; \
  328. vpxor x1, x3, x3; \
  329. vpand x2, x1, x1; \
  330. vpxor tp, x0, x4; \
  331. vpxor x4, x3, x3; \
  332. vpxor x2, x4, x4; \
  333. vpxor x1, tp, x0; \
  334. vpxor x0, x2, x2;
  335. #define SI7_1(x0, x1, x2, x3, x4) \
  336. vpand x0, x3, tp; \
  337. vpxor x2, x0, x0; \
  338. vpor x3, x2, x2; \
  339. vpxor x1, x3, x4; \
  340. vpxor RNOT, x0, x0; \
  341. vpor tp, x1, x1; \
  342. vpxor x0, x4, x4; \
  343. vpand x2, x0, x0; \
  344. vpxor x1, x0, x0;
  345. #define SI7_2(x0, x1, x2, x3, x4) \
  346. vpand x2, x1, x1; \
  347. vpxor x2, tp, x3; \
  348. vpxor x3, x4, x4; \
  349. vpand x3, x2, x2; \
  350. vpor x0, x3, x3; \
  351. vpxor x4, x1, x1; \
  352. vpxor x4, x3, x3; \
  353. vpand x0, x4, x4; \
  354. vpxor x2, x4, x4;
  355. #define get_key(i, j, t) \
  356. vbroadcastss (4*(i)+(j))*4(CTX), t;
  357. #define K2(x0, x1, x2, x3, x4, i) \
  358. get_key(i, 0, RK0); \
  359. get_key(i, 1, RK1); \
  360. get_key(i, 2, RK2); \
  361. get_key(i, 3, RK3); \
  362. vpxor RK0, x0 ## 1, x0 ## 1; \
  363. vpxor RK1, x1 ## 1, x1 ## 1; \
  364. vpxor RK2, x2 ## 1, x2 ## 1; \
  365. vpxor RK3, x3 ## 1, x3 ## 1; \
  366. vpxor RK0, x0 ## 2, x0 ## 2; \
  367. vpxor RK1, x1 ## 2, x1 ## 2; \
  368. vpxor RK2, x2 ## 2, x2 ## 2; \
  369. vpxor RK3, x3 ## 2, x3 ## 2;
  370. #define LK2(x0, x1, x2, x3, x4, i) \
  371. vpslld $13, x0 ## 1, x4 ## 1; \
  372. vpsrld $(32 - 13), x0 ## 1, x0 ## 1; \
  373. vpor x4 ## 1, x0 ## 1, x0 ## 1; \
  374. vpxor x0 ## 1, x1 ## 1, x1 ## 1; \
  375. vpslld $3, x2 ## 1, x4 ## 1; \
  376. vpsrld $(32 - 3), x2 ## 1, x2 ## 1; \
  377. vpor x4 ## 1, x2 ## 1, x2 ## 1; \
  378. vpxor x2 ## 1, x1 ## 1, x1 ## 1; \
  379. vpslld $13, x0 ## 2, x4 ## 2; \
  380. vpsrld $(32 - 13), x0 ## 2, x0 ## 2; \
  381. vpor x4 ## 2, x0 ## 2, x0 ## 2; \
  382. vpxor x0 ## 2, x1 ## 2, x1 ## 2; \
  383. vpslld $3, x2 ## 2, x4 ## 2; \
  384. vpsrld $(32 - 3), x2 ## 2, x2 ## 2; \
  385. vpor x4 ## 2, x2 ## 2, x2 ## 2; \
  386. vpxor x2 ## 2, x1 ## 2, x1 ## 2; \
  387. vpslld $1, x1 ## 1, x4 ## 1; \
  388. vpsrld $(32 - 1), x1 ## 1, x1 ## 1; \
  389. vpor x4 ## 1, x1 ## 1, x1 ## 1; \
  390. vpslld $3, x0 ## 1, x4 ## 1; \
  391. vpxor x2 ## 1, x3 ## 1, x3 ## 1; \
  392. vpxor x4 ## 1, x3 ## 1, x3 ## 1; \
  393. get_key(i, 1, RK1); \
  394. vpslld $1, x1 ## 2, x4 ## 2; \
  395. vpsrld $(32 - 1), x1 ## 2, x1 ## 2; \
  396. vpor x4 ## 2, x1 ## 2, x1 ## 2; \
  397. vpslld $3, x0 ## 2, x4 ## 2; \
  398. vpxor x2 ## 2, x3 ## 2, x3 ## 2; \
  399. vpxor x4 ## 2, x3 ## 2, x3 ## 2; \
  400. get_key(i, 3, RK3); \
  401. vpslld $7, x3 ## 1, x4 ## 1; \
  402. vpsrld $(32 - 7), x3 ## 1, x3 ## 1; \
  403. vpor x4 ## 1, x3 ## 1, x3 ## 1; \
  404. vpslld $7, x1 ## 1, x4 ## 1; \
  405. vpxor x1 ## 1, x0 ## 1, x0 ## 1; \
  406. vpxor x3 ## 1, x0 ## 1, x0 ## 1; \
  407. vpxor x3 ## 1, x2 ## 1, x2 ## 1; \
  408. vpxor x4 ## 1, x2 ## 1, x2 ## 1; \
  409. get_key(i, 0, RK0); \
  410. vpslld $7, x3 ## 2, x4 ## 2; \
  411. vpsrld $(32 - 7), x3 ## 2, x3 ## 2; \
  412. vpor x4 ## 2, x3 ## 2, x3 ## 2; \
  413. vpslld $7, x1 ## 2, x4 ## 2; \
  414. vpxor x1 ## 2, x0 ## 2, x0 ## 2; \
  415. vpxor x3 ## 2, x0 ## 2, x0 ## 2; \
  416. vpxor x3 ## 2, x2 ## 2, x2 ## 2; \
  417. vpxor x4 ## 2, x2 ## 2, x2 ## 2; \
  418. get_key(i, 2, RK2); \
  419. vpxor RK1, x1 ## 1, x1 ## 1; \
  420. vpxor RK3, x3 ## 1, x3 ## 1; \
  421. vpslld $5, x0 ## 1, x4 ## 1; \
  422. vpsrld $(32 - 5), x0 ## 1, x0 ## 1; \
  423. vpor x4 ## 1, x0 ## 1, x0 ## 1; \
  424. vpslld $22, x2 ## 1, x4 ## 1; \
  425. vpsrld $(32 - 22), x2 ## 1, x2 ## 1; \
  426. vpor x4 ## 1, x2 ## 1, x2 ## 1; \
  427. vpxor RK0, x0 ## 1, x0 ## 1; \
  428. vpxor RK2, x2 ## 1, x2 ## 1; \
  429. vpxor RK1, x1 ## 2, x1 ## 2; \
  430. vpxor RK3, x3 ## 2, x3 ## 2; \
  431. vpslld $5, x0 ## 2, x4 ## 2; \
  432. vpsrld $(32 - 5), x0 ## 2, x0 ## 2; \
  433. vpor x4 ## 2, x0 ## 2, x0 ## 2; \
  434. vpslld $22, x2 ## 2, x4 ## 2; \
  435. vpsrld $(32 - 22), x2 ## 2, x2 ## 2; \
  436. vpor x4 ## 2, x2 ## 2, x2 ## 2; \
  437. vpxor RK0, x0 ## 2, x0 ## 2; \
  438. vpxor RK2, x2 ## 2, x2 ## 2;
  439. #define KL2(x0, x1, x2, x3, x4, i) \
  440. vpxor RK0, x0 ## 1, x0 ## 1; \
  441. vpxor RK2, x2 ## 1, x2 ## 1; \
  442. vpsrld $5, x0 ## 1, x4 ## 1; \
  443. vpslld $(32 - 5), x0 ## 1, x0 ## 1; \
  444. vpor x4 ## 1, x0 ## 1, x0 ## 1; \
  445. vpxor RK3, x3 ## 1, x3 ## 1; \
  446. vpxor RK1, x1 ## 1, x1 ## 1; \
  447. vpsrld $22, x2 ## 1, x4 ## 1; \
  448. vpslld $(32 - 22), x2 ## 1, x2 ## 1; \
  449. vpor x4 ## 1, x2 ## 1, x2 ## 1; \
  450. vpxor x3 ## 1, x2 ## 1, x2 ## 1; \
  451. vpxor RK0, x0 ## 2, x0 ## 2; \
  452. vpxor RK2, x2 ## 2, x2 ## 2; \
  453. vpsrld $5, x0 ## 2, x4 ## 2; \
  454. vpslld $(32 - 5), x0 ## 2, x0 ## 2; \
  455. vpor x4 ## 2, x0 ## 2, x0 ## 2; \
  456. vpxor RK3, x3 ## 2, x3 ## 2; \
  457. vpxor RK1, x1 ## 2, x1 ## 2; \
  458. vpsrld $22, x2 ## 2, x4 ## 2; \
  459. vpslld $(32 - 22), x2 ## 2, x2 ## 2; \
  460. vpor x4 ## 2, x2 ## 2, x2 ## 2; \
  461. vpxor x3 ## 2, x2 ## 2, x2 ## 2; \
  462. vpxor x3 ## 1, x0 ## 1, x0 ## 1; \
  463. vpslld $7, x1 ## 1, x4 ## 1; \
  464. vpxor x1 ## 1, x0 ## 1, x0 ## 1; \
  465. vpxor x4 ## 1, x2 ## 1, x2 ## 1; \
  466. vpsrld $1, x1 ## 1, x4 ## 1; \
  467. vpslld $(32 - 1), x1 ## 1, x1 ## 1; \
  468. vpor x4 ## 1, x1 ## 1, x1 ## 1; \
  469. vpxor x3 ## 2, x0 ## 2, x0 ## 2; \
  470. vpslld $7, x1 ## 2, x4 ## 2; \
  471. vpxor x1 ## 2, x0 ## 2, x0 ## 2; \
  472. vpxor x4 ## 2, x2 ## 2, x2 ## 2; \
  473. vpsrld $1, x1 ## 2, x4 ## 2; \
  474. vpslld $(32 - 1), x1 ## 2, x1 ## 2; \
  475. vpor x4 ## 2, x1 ## 2, x1 ## 2; \
  476. vpsrld $7, x3 ## 1, x4 ## 1; \
  477. vpslld $(32 - 7), x3 ## 1, x3 ## 1; \
  478. vpor x4 ## 1, x3 ## 1, x3 ## 1; \
  479. vpxor x0 ## 1, x1 ## 1, x1 ## 1; \
  480. vpslld $3, x0 ## 1, x4 ## 1; \
  481. vpxor x4 ## 1, x3 ## 1, x3 ## 1; \
  482. vpsrld $7, x3 ## 2, x4 ## 2; \
  483. vpslld $(32 - 7), x3 ## 2, x3 ## 2; \
  484. vpor x4 ## 2, x3 ## 2, x3 ## 2; \
  485. vpxor x0 ## 2, x1 ## 2, x1 ## 2; \
  486. vpslld $3, x0 ## 2, x4 ## 2; \
  487. vpxor x4 ## 2, x3 ## 2, x3 ## 2; \
  488. vpsrld $13, x0 ## 1, x4 ## 1; \
  489. vpslld $(32 - 13), x0 ## 1, x0 ## 1; \
  490. vpor x4 ## 1, x0 ## 1, x0 ## 1; \
  491. vpxor x2 ## 1, x1 ## 1, x1 ## 1; \
  492. vpxor x2 ## 1, x3 ## 1, x3 ## 1; \
  493. vpsrld $3, x2 ## 1, x4 ## 1; \
  494. vpslld $(32 - 3), x2 ## 1, x2 ## 1; \
  495. vpor x4 ## 1, x2 ## 1, x2 ## 1; \
  496. vpsrld $13, x0 ## 2, x4 ## 2; \
  497. vpslld $(32 - 13), x0 ## 2, x0 ## 2; \
  498. vpor x4 ## 2, x0 ## 2, x0 ## 2; \
  499. vpxor x2 ## 2, x1 ## 2, x1 ## 2; \
  500. vpxor x2 ## 2, x3 ## 2, x3 ## 2; \
  501. vpsrld $3, x2 ## 2, x4 ## 2; \
  502. vpslld $(32 - 3), x2 ## 2, x2 ## 2; \
  503. vpor x4 ## 2, x2 ## 2, x2 ## 2;
  504. #define S(SBOX, x0, x1, x2, x3, x4) \
  505. SBOX ## _1(x0 ## 1, x1 ## 1, x2 ## 1, x3 ## 1, x4 ## 1); \
  506. SBOX ## _2(x0 ## 1, x1 ## 1, x2 ## 1, x3 ## 1, x4 ## 1); \
  507. SBOX ## _1(x0 ## 2, x1 ## 2, x2 ## 2, x3 ## 2, x4 ## 2); \
  508. SBOX ## _2(x0 ## 2, x1 ## 2, x2 ## 2, x3 ## 2, x4 ## 2);
  509. #define SP(SBOX, x0, x1, x2, x3, x4, i) \
  510. get_key(i, 0, RK0); \
  511. SBOX ## _1(x0 ## 1, x1 ## 1, x2 ## 1, x3 ## 1, x4 ## 1); \
  512. get_key(i, 2, RK2); \
  513. SBOX ## _2(x0 ## 1, x1 ## 1, x2 ## 1, x3 ## 1, x4 ## 1); \
  514. get_key(i, 3, RK3); \
  515. SBOX ## _1(x0 ## 2, x1 ## 2, x2 ## 2, x3 ## 2, x4 ## 2); \
  516. get_key(i, 1, RK1); \
  517. SBOX ## _2(x0 ## 2, x1 ## 2, x2 ## 2, x3 ## 2, x4 ## 2); \
  518. #define transpose_4x4(x0, x1, x2, x3, t0, t1, t2) \
  519. vpunpckldq x1, x0, t0; \
  520. vpunpckhdq x1, x0, t2; \
  521. vpunpckldq x3, x2, t1; \
  522. vpunpckhdq x3, x2, x3; \
  523. \
  524. vpunpcklqdq t1, t0, x0; \
  525. vpunpckhqdq t1, t0, x1; \
  526. vpunpcklqdq x3, t2, x2; \
  527. vpunpckhqdq x3, t2, x3;
  528. #define read_blocks(x0, x1, x2, x3, t0, t1, t2) \
  529. transpose_4x4(x0, x1, x2, x3, t0, t1, t2)
  530. #define write_blocks(x0, x1, x2, x3, t0, t1, t2) \
  531. transpose_4x4(x0, x1, x2, x3, t0, t1, t2)
  532. .align 8
  533. __serpent_enc_blk8_avx:
  534. /* input:
  535. * %rdi: ctx, CTX
  536. * RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2: blocks
  537. * output:
  538. * RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2: encrypted blocks
  539. */
  540. vpcmpeqd RNOT, RNOT, RNOT;
  541. read_blocks(RA1, RB1, RC1, RD1, RK0, RK1, RK2);
  542. read_blocks(RA2, RB2, RC2, RD2, RK0, RK1, RK2);
  543. K2(RA, RB, RC, RD, RE, 0);
  544. S(S0, RA, RB, RC, RD, RE); LK2(RC, RB, RD, RA, RE, 1);
  545. S(S1, RC, RB, RD, RA, RE); LK2(RE, RD, RA, RC, RB, 2);
  546. S(S2, RE, RD, RA, RC, RB); LK2(RB, RD, RE, RC, RA, 3);
  547. S(S3, RB, RD, RE, RC, RA); LK2(RC, RA, RD, RB, RE, 4);
  548. S(S4, RC, RA, RD, RB, RE); LK2(RA, RD, RB, RE, RC, 5);
  549. S(S5, RA, RD, RB, RE, RC); LK2(RC, RA, RD, RE, RB, 6);
  550. S(S6, RC, RA, RD, RE, RB); LK2(RD, RB, RA, RE, RC, 7);
  551. S(S7, RD, RB, RA, RE, RC); LK2(RC, RA, RE, RD, RB, 8);
  552. S(S0, RC, RA, RE, RD, RB); LK2(RE, RA, RD, RC, RB, 9);
  553. S(S1, RE, RA, RD, RC, RB); LK2(RB, RD, RC, RE, RA, 10);
  554. S(S2, RB, RD, RC, RE, RA); LK2(RA, RD, RB, RE, RC, 11);
  555. S(S3, RA, RD, RB, RE, RC); LK2(RE, RC, RD, RA, RB, 12);
  556. S(S4, RE, RC, RD, RA, RB); LK2(RC, RD, RA, RB, RE, 13);
  557. S(S5, RC, RD, RA, RB, RE); LK2(RE, RC, RD, RB, RA, 14);
  558. S(S6, RE, RC, RD, RB, RA); LK2(RD, RA, RC, RB, RE, 15);
  559. S(S7, RD, RA, RC, RB, RE); LK2(RE, RC, RB, RD, RA, 16);
  560. S(S0, RE, RC, RB, RD, RA); LK2(RB, RC, RD, RE, RA, 17);
  561. S(S1, RB, RC, RD, RE, RA); LK2(RA, RD, RE, RB, RC, 18);
  562. S(S2, RA, RD, RE, RB, RC); LK2(RC, RD, RA, RB, RE, 19);
  563. S(S3, RC, RD, RA, RB, RE); LK2(RB, RE, RD, RC, RA, 20);
  564. S(S4, RB, RE, RD, RC, RA); LK2(RE, RD, RC, RA, RB, 21);
  565. S(S5, RE, RD, RC, RA, RB); LK2(RB, RE, RD, RA, RC, 22);
  566. S(S6, RB, RE, RD, RA, RC); LK2(RD, RC, RE, RA, RB, 23);
  567. S(S7, RD, RC, RE, RA, RB); LK2(RB, RE, RA, RD, RC, 24);
  568. S(S0, RB, RE, RA, RD, RC); LK2(RA, RE, RD, RB, RC, 25);
  569. S(S1, RA, RE, RD, RB, RC); LK2(RC, RD, RB, RA, RE, 26);
  570. S(S2, RC, RD, RB, RA, RE); LK2(RE, RD, RC, RA, RB, 27);
  571. S(S3, RE, RD, RC, RA, RB); LK2(RA, RB, RD, RE, RC, 28);
  572. S(S4, RA, RB, RD, RE, RC); LK2(RB, RD, RE, RC, RA, 29);
  573. S(S5, RB, RD, RE, RC, RA); LK2(RA, RB, RD, RC, RE, 30);
  574. S(S6, RA, RB, RD, RC, RE); LK2(RD, RE, RB, RC, RA, 31);
  575. S(S7, RD, RE, RB, RC, RA); K2(RA, RB, RC, RD, RE, 32);
  576. write_blocks(RA1, RB1, RC1, RD1, RK0, RK1, RK2);
  577. write_blocks(RA2, RB2, RC2, RD2, RK0, RK1, RK2);
  578. ret;
  579. ENDPROC(__serpent_enc_blk8_avx)
  580. .align 8
  581. __serpent_dec_blk8_avx:
  582. /* input:
  583. * %rdi: ctx, CTX
  584. * RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2: encrypted blocks
  585. * output:
  586. * RC1, RD1, RB1, RE1, RC2, RD2, RB2, RE2: decrypted blocks
  587. */
  588. vpcmpeqd RNOT, RNOT, RNOT;
  589. read_blocks(RA1, RB1, RC1, RD1, RK0, RK1, RK2);
  590. read_blocks(RA2, RB2, RC2, RD2, RK0, RK1, RK2);
  591. K2(RA, RB, RC, RD, RE, 32);
  592. SP(SI7, RA, RB, RC, RD, RE, 31); KL2(RB, RD, RA, RE, RC, 31);
  593. SP(SI6, RB, RD, RA, RE, RC, 30); KL2(RA, RC, RE, RB, RD, 30);
  594. SP(SI5, RA, RC, RE, RB, RD, 29); KL2(RC, RD, RA, RE, RB, 29);
  595. SP(SI4, RC, RD, RA, RE, RB, 28); KL2(RC, RA, RB, RE, RD, 28);
  596. SP(SI3, RC, RA, RB, RE, RD, 27); KL2(RB, RC, RD, RE, RA, 27);
  597. SP(SI2, RB, RC, RD, RE, RA, 26); KL2(RC, RA, RE, RD, RB, 26);
  598. SP(SI1, RC, RA, RE, RD, RB, 25); KL2(RB, RA, RE, RD, RC, 25);
  599. SP(SI0, RB, RA, RE, RD, RC, 24); KL2(RE, RC, RA, RB, RD, 24);
  600. SP(SI7, RE, RC, RA, RB, RD, 23); KL2(RC, RB, RE, RD, RA, 23);
  601. SP(SI6, RC, RB, RE, RD, RA, 22); KL2(RE, RA, RD, RC, RB, 22);
  602. SP(SI5, RE, RA, RD, RC, RB, 21); KL2(RA, RB, RE, RD, RC, 21);
  603. SP(SI4, RA, RB, RE, RD, RC, 20); KL2(RA, RE, RC, RD, RB, 20);
  604. SP(SI3, RA, RE, RC, RD, RB, 19); KL2(RC, RA, RB, RD, RE, 19);
  605. SP(SI2, RC, RA, RB, RD, RE, 18); KL2(RA, RE, RD, RB, RC, 18);
  606. SP(SI1, RA, RE, RD, RB, RC, 17); KL2(RC, RE, RD, RB, RA, 17);
  607. SP(SI0, RC, RE, RD, RB, RA, 16); KL2(RD, RA, RE, RC, RB, 16);
  608. SP(SI7, RD, RA, RE, RC, RB, 15); KL2(RA, RC, RD, RB, RE, 15);
  609. SP(SI6, RA, RC, RD, RB, RE, 14); KL2(RD, RE, RB, RA, RC, 14);
  610. SP(SI5, RD, RE, RB, RA, RC, 13); KL2(RE, RC, RD, RB, RA, 13);
  611. SP(SI4, RE, RC, RD, RB, RA, 12); KL2(RE, RD, RA, RB, RC, 12);
  612. SP(SI3, RE, RD, RA, RB, RC, 11); KL2(RA, RE, RC, RB, RD, 11);
  613. SP(SI2, RA, RE, RC, RB, RD, 10); KL2(RE, RD, RB, RC, RA, 10);
  614. SP(SI1, RE, RD, RB, RC, RA, 9); KL2(RA, RD, RB, RC, RE, 9);
  615. SP(SI0, RA, RD, RB, RC, RE, 8); KL2(RB, RE, RD, RA, RC, 8);
  616. SP(SI7, RB, RE, RD, RA, RC, 7); KL2(RE, RA, RB, RC, RD, 7);
  617. SP(SI6, RE, RA, RB, RC, RD, 6); KL2(RB, RD, RC, RE, RA, 6);
  618. SP(SI5, RB, RD, RC, RE, RA, 5); KL2(RD, RA, RB, RC, RE, 5);
  619. SP(SI4, RD, RA, RB, RC, RE, 4); KL2(RD, RB, RE, RC, RA, 4);
  620. SP(SI3, RD, RB, RE, RC, RA, 3); KL2(RE, RD, RA, RC, RB, 3);
  621. SP(SI2, RE, RD, RA, RC, RB, 2); KL2(RD, RB, RC, RA, RE, 2);
  622. SP(SI1, RD, RB, RC, RA, RE, 1); KL2(RE, RB, RC, RA, RD, 1);
  623. S(SI0, RE, RB, RC, RA, RD); K2(RC, RD, RB, RE, RA, 0);
  624. write_blocks(RC1, RD1, RB1, RE1, RK0, RK1, RK2);
  625. write_blocks(RC2, RD2, RB2, RE2, RK0, RK1, RK2);
  626. ret;
  627. ENDPROC(__serpent_dec_blk8_avx)
  628. ENTRY(serpent_ecb_enc_8way_avx)
  629. /* input:
  630. * %rdi: ctx, CTX
  631. * %rsi: dst
  632. * %rdx: src
  633. */
  634. load_8way(%rdx, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2);
  635. call __serpent_enc_blk8_avx;
  636. store_8way(%rsi, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2);
  637. ret;
  638. ENDPROC(serpent_ecb_enc_8way_avx)
  639. ENTRY(serpent_ecb_dec_8way_avx)
  640. /* input:
  641. * %rdi: ctx, CTX
  642. * %rsi: dst
  643. * %rdx: src
  644. */
  645. load_8way(%rdx, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2);
  646. call __serpent_dec_blk8_avx;
  647. store_8way(%rsi, RC1, RD1, RB1, RE1, RC2, RD2, RB2, RE2);
  648. ret;
  649. ENDPROC(serpent_ecb_dec_8way_avx)
  650. ENTRY(serpent_cbc_dec_8way_avx)
  651. /* input:
  652. * %rdi: ctx, CTX
  653. * %rsi: dst
  654. * %rdx: src
  655. */
  656. load_8way(%rdx, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2);
  657. call __serpent_dec_blk8_avx;
  658. store_cbc_8way(%rdx, %rsi, RC1, RD1, RB1, RE1, RC2, RD2, RB2, RE2);
  659. ret;
  660. ENDPROC(serpent_cbc_dec_8way_avx)
  661. ENTRY(serpent_ctr_8way_avx)
  662. /* input:
  663. * %rdi: ctx, CTX
  664. * %rsi: dst
  665. * %rdx: src
  666. * %rcx: iv (little endian, 128bit)
  667. */
  668. load_ctr_8way(%rcx, .Lbswap128_mask, RA1, RB1, RC1, RD1, RA2, RB2, RC2,
  669. RD2, RK0, RK1, RK2);
  670. call __serpent_enc_blk8_avx;
  671. store_ctr_8way(%rdx, %rsi, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2);
  672. ret;
  673. ENDPROC(serpent_ctr_8way_avx)
  674. ENTRY(serpent_xts_enc_8way_avx)
  675. /* input:
  676. * %rdi: ctx, CTX
  677. * %rsi: dst
  678. * %rdx: src
  679. * %rcx: iv (t ⊕ αⁿ ∈ GF(2¹²⁸))
  680. */
  681. /* regs <= src, dst <= IVs, regs <= regs xor IVs */
  682. load_xts_8way(%rcx, %rdx, %rsi, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2,
  683. RK0, RK1, RK2, .Lxts_gf128mul_and_shl1_mask);
  684. call __serpent_enc_blk8_avx;
  685. /* dst <= regs xor IVs(in dst) */
  686. store_xts_8way(%rsi, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2);
  687. ret;
  688. ENDPROC(serpent_xts_enc_8way_avx)
  689. ENTRY(serpent_xts_dec_8way_avx)
  690. /* input:
  691. * %rdi: ctx, CTX
  692. * %rsi: dst
  693. * %rdx: src
  694. * %rcx: iv (t ⊕ αⁿ ∈ GF(2¹²⁸))
  695. */
  696. /* regs <= src, dst <= IVs, regs <= regs xor IVs */
  697. load_xts_8way(%rcx, %rdx, %rsi, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2,
  698. RK0, RK1, RK2, .Lxts_gf128mul_and_shl1_mask);
  699. call __serpent_dec_blk8_avx;
  700. /* dst <= regs xor IVs(in dst) */
  701. store_xts_8way(%rsi, RC1, RD1, RB1, RE1, RC2, RD2, RB2, RE2);
  702. ret;
  703. ENDPROC(serpent_xts_dec_8way_avx)