crc32c-pcl-intel-asm_64.S 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464
  1. /*
  2. * Implement fast CRC32C with PCLMULQDQ instructions. (x86_64)
  3. *
  4. * The white papers on CRC32C calculations with PCLMULQDQ instruction can be
  5. * downloaded from:
  6. * http://www.intel.com/content/dam/www/public/us/en/documents/white-papers/crc-iscsi-polynomial-crc32-instruction-paper.pdf
  7. * http://www.intel.com/content/dam/www/public/us/en/documents/white-papers/fast-crc-computation-paper.pdf
  8. *
  9. * Copyright (C) 2012 Intel Corporation.
  10. *
  11. * Authors:
  12. * Wajdi Feghali <wajdi.k.feghali@intel.com>
  13. * James Guilford <james.guilford@intel.com>
  14. * David Cote <david.m.cote@intel.com>
  15. * Tim Chen <tim.c.chen@linux.intel.com>
  16. *
  17. * This software is available to you under a choice of one of two
  18. * licenses. You may choose to be licensed under the terms of the GNU
  19. * General Public License (GPL) Version 2, available from the file
  20. * COPYING in the main directory of this source tree, or the
  21. * OpenIB.org BSD license below:
  22. *
  23. * Redistribution and use in source and binary forms, with or
  24. * without modification, are permitted provided that the following
  25. * conditions are met:
  26. *
  27. * - Redistributions of source code must retain the above
  28. * copyright notice, this list of conditions and the following
  29. * disclaimer.
  30. *
  31. * - Redistributions in binary form must reproduce the above
  32. * copyright notice, this list of conditions and the following
  33. * disclaimer in the documentation and/or other materials
  34. * provided with the distribution.
  35. *
  36. * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  37. * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  38. * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  39. * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  40. * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  41. * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  42. * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  43. * SOFTWARE.
  44. */
  45. #include <asm/inst.h>
  46. #include <linux/linkage.h>
  47. #include <asm/nospec-branch.h>
  48. ## ISCSI CRC 32 Implementation with crc32 and pclmulqdq Instruction
  49. .macro LABEL prefix n
  50. \prefix\n\():
  51. .endm
  52. .macro JMPTBL_ENTRY i
  53. .word crc_\i - crc_array
  54. .endm
  55. .macro JNC_LESS_THAN j
  56. jnc less_than_\j
  57. .endm
  58. # Define threshold where buffers are considered "small" and routed to more
  59. # efficient "by-1" code. This "by-1" code only handles up to 255 bytes, so
  60. # SMALL_SIZE can be no larger than 255.
  61. #define SMALL_SIZE 200
  62. .if (SMALL_SIZE > 255)
  63. .error "SMALL_ SIZE must be < 256"
  64. .endif
  65. # unsigned int crc_pcl(u8 *buffer, int len, unsigned int crc_init);
  66. .text
  67. ENTRY(crc_pcl)
  68. #define bufp %rdi
  69. #define bufp_dw %edi
  70. #define bufp_w %di
  71. #define bufp_b %dil
  72. #define bufptmp %rcx
  73. #define block_0 %rcx
  74. #define block_1 %rdx
  75. #define block_2 %r11
  76. #define len %rsi
  77. #define len_dw %esi
  78. #define len_w %si
  79. #define len_b %sil
  80. #define crc_init_arg %rdx
  81. #define tmp %rbx
  82. #define crc_init %r8
  83. #define crc_init_dw %r8d
  84. #define crc1 %r9
  85. #define crc2 %r10
  86. pushq %rbx
  87. pushq %rdi
  88. pushq %rsi
  89. ## Move crc_init for Linux to a different
  90. mov crc_init_arg, crc_init
  91. ################################################################
  92. ## 1) ALIGN:
  93. ################################################################
  94. mov bufp, bufptmp # rdi = *buf
  95. neg bufp
  96. and $7, bufp # calculate the unalignment amount of
  97. # the address
  98. je proc_block # Skip if aligned
  99. ## If len is less than 8 and we're unaligned, we need to jump
  100. ## to special code to avoid reading beyond the end of the buffer
  101. cmp $8, len
  102. jae do_align
  103. # less_than_8 expects length in upper 3 bits of len_dw
  104. # less_than_8_post_shl1 expects length = carryflag * 8 + len_dw[31:30]
  105. shl $32-3+1, len_dw
  106. jmp less_than_8_post_shl1
  107. do_align:
  108. #### Calculate CRC of unaligned bytes of the buffer (if any)
  109. movq (bufptmp), tmp # load a quadward from the buffer
  110. add bufp, bufptmp # align buffer pointer for quadword
  111. # processing
  112. sub bufp, len # update buffer length
  113. align_loop:
  114. crc32b %bl, crc_init_dw # compute crc32 of 1-byte
  115. shr $8, tmp # get next byte
  116. dec bufp
  117. jne align_loop
  118. proc_block:
  119. ################################################################
  120. ## 2) PROCESS BLOCKS:
  121. ################################################################
  122. ## compute num of bytes to be processed
  123. movq len, tmp # save num bytes in tmp
  124. cmpq $128*24, len
  125. jae full_block
  126. continue_block:
  127. cmpq $SMALL_SIZE, len
  128. jb small
  129. ## len < 128*24
  130. movq $2731, %rax # 2731 = ceil(2^16 / 24)
  131. mul len_dw
  132. shrq $16, %rax
  133. ## eax contains floor(bytes / 24) = num 24-byte chunks to do
  134. ## process rax 24-byte chunks (128 >= rax >= 0)
  135. ## compute end address of each block
  136. ## block 0 (base addr + RAX * 8)
  137. ## block 1 (base addr + RAX * 16)
  138. ## block 2 (base addr + RAX * 24)
  139. lea (bufptmp, %rax, 8), block_0
  140. lea (block_0, %rax, 8), block_1
  141. lea (block_1, %rax, 8), block_2
  142. xor crc1, crc1
  143. xor crc2, crc2
  144. ## branch into array
  145. lea jump_table(%rip), bufp
  146. movzxw (bufp, %rax, 2), len
  147. offset=crc_array-jump_table
  148. lea offset(bufp, len, 1), bufp
  149. JMP_NOSPEC bufp
  150. ################################################################
  151. ## 2a) PROCESS FULL BLOCKS:
  152. ################################################################
  153. full_block:
  154. movl $128,%eax
  155. lea 128*8*2(block_0), block_1
  156. lea 128*8*3(block_0), block_2
  157. add $128*8*1, block_0
  158. xor crc1,crc1
  159. xor crc2,crc2
  160. # Fall thruogh into top of crc array (crc_128)
  161. ################################################################
  162. ## 3) CRC Array:
  163. ################################################################
  164. crc_array:
  165. i=128
  166. .rept 128-1
  167. .altmacro
  168. LABEL crc_ %i
  169. .noaltmacro
  170. crc32q -i*8(block_0), crc_init
  171. crc32q -i*8(block_1), crc1
  172. crc32q -i*8(block_2), crc2
  173. i=(i-1)
  174. .endr
  175. .altmacro
  176. LABEL crc_ %i
  177. .noaltmacro
  178. crc32q -i*8(block_0), crc_init
  179. crc32q -i*8(block_1), crc1
  180. # SKIP crc32 -i*8(block_2), crc2 ; Don't do this one yet
  181. mov block_2, block_0
  182. ################################################################
  183. ## 4) Combine three results:
  184. ################################################################
  185. lea (K_table-8)(%rip), bufp # first entry is for idx 1
  186. shlq $3, %rax # rax *= 8
  187. pmovzxdq (bufp,%rax), %xmm0 # 2 consts: K1:K2
  188. leal (%eax,%eax,2), %eax # rax *= 3 (total *24)
  189. subq %rax, tmp # tmp -= rax*24
  190. movq crc_init, %xmm1 # CRC for block 1
  191. PCLMULQDQ 0x00,%xmm0,%xmm1 # Multiply by K2
  192. movq crc1, %xmm2 # CRC for block 2
  193. PCLMULQDQ 0x10, %xmm0, %xmm2 # Multiply by K1
  194. pxor %xmm2,%xmm1
  195. movq %xmm1, %rax
  196. xor -i*8(block_2), %rax
  197. mov crc2, crc_init
  198. crc32 %rax, crc_init
  199. ################################################################
  200. ## 5) Check for end:
  201. ################################################################
  202. LABEL crc_ 0
  203. mov tmp, len
  204. cmp $128*24, tmp
  205. jae full_block
  206. cmp $24, tmp
  207. jae continue_block
  208. less_than_24:
  209. shl $32-4, len_dw # less_than_16 expects length
  210. # in upper 4 bits of len_dw
  211. jnc less_than_16
  212. crc32q (bufptmp), crc_init
  213. crc32q 8(bufptmp), crc_init
  214. jz do_return
  215. add $16, bufptmp
  216. # len is less than 8 if we got here
  217. # less_than_8 expects length in upper 3 bits of len_dw
  218. # less_than_8_post_shl1 expects length = carryflag * 8 + len_dw[31:30]
  219. shl $2, len_dw
  220. jmp less_than_8_post_shl1
  221. #######################################################################
  222. ## 6) LESS THAN 256-bytes REMAIN AT THIS POINT (8-bits of len are full)
  223. #######################################################################
  224. small:
  225. shl $32-8, len_dw # Prepare len_dw for less_than_256
  226. j=256
  227. .rept 5 # j = {256, 128, 64, 32, 16}
  228. .altmacro
  229. LABEL less_than_ %j # less_than_j: Length should be in
  230. # upper lg(j) bits of len_dw
  231. j=(j/2)
  232. shl $1, len_dw # Get next MSB
  233. JNC_LESS_THAN %j
  234. .noaltmacro
  235. i=0
  236. .rept (j/8)
  237. crc32q i(bufptmp), crc_init # Compute crc32 of 8-byte data
  238. i=i+8
  239. .endr
  240. jz do_return # Return if remaining length is zero
  241. add $j, bufptmp # Advance buf
  242. .endr
  243. less_than_8: # Length should be stored in
  244. # upper 3 bits of len_dw
  245. shl $1, len_dw
  246. less_than_8_post_shl1:
  247. jnc less_than_4
  248. crc32l (bufptmp), crc_init_dw # CRC of 4 bytes
  249. jz do_return # return if remaining data is zero
  250. add $4, bufptmp
  251. less_than_4: # Length should be stored in
  252. # upper 2 bits of len_dw
  253. shl $1, len_dw
  254. jnc less_than_2
  255. crc32w (bufptmp), crc_init_dw # CRC of 2 bytes
  256. jz do_return # return if remaining data is zero
  257. add $2, bufptmp
  258. less_than_2: # Length should be stored in the MSB
  259. # of len_dw
  260. shl $1, len_dw
  261. jnc less_than_1
  262. crc32b (bufptmp), crc_init_dw # CRC of 1 byte
  263. less_than_1: # Length should be zero
  264. do_return:
  265. movq crc_init, %rax
  266. popq %rsi
  267. popq %rdi
  268. popq %rbx
  269. ret
  270. ################################################################
  271. ## jump table Table is 129 entries x 2 bytes each
  272. ################################################################
  273. .align 4
  274. jump_table:
  275. i=0
  276. .rept 129
  277. .altmacro
  278. JMPTBL_ENTRY %i
  279. .noaltmacro
  280. i=i+1
  281. .endr
  282. ENDPROC(crc_pcl)
  283. ################################################################
  284. ## PCLMULQDQ tables
  285. ## Table is 128 entries x 2 words (8 bytes) each
  286. ################################################################
  287. .section .rodata, "a", %progbits
  288. .align 8
  289. K_table:
  290. .long 0x493c7d27, 0x00000001
  291. .long 0xba4fc28e, 0x493c7d27
  292. .long 0xddc0152b, 0xf20c0dfe
  293. .long 0x9e4addf8, 0xba4fc28e
  294. .long 0x39d3b296, 0x3da6d0cb
  295. .long 0x0715ce53, 0xddc0152b
  296. .long 0x47db8317, 0x1c291d04
  297. .long 0x0d3b6092, 0x9e4addf8
  298. .long 0xc96cfdc0, 0x740eef02
  299. .long 0x878a92a7, 0x39d3b296
  300. .long 0xdaece73e, 0x083a6eec
  301. .long 0xab7aff2a, 0x0715ce53
  302. .long 0x2162d385, 0xc49f4f67
  303. .long 0x83348832, 0x47db8317
  304. .long 0x299847d5, 0x2ad91c30
  305. .long 0xb9e02b86, 0x0d3b6092
  306. .long 0x18b33a4e, 0x6992cea2
  307. .long 0xb6dd949b, 0xc96cfdc0
  308. .long 0x78d9ccb7, 0x7e908048
  309. .long 0xbac2fd7b, 0x878a92a7
  310. .long 0xa60ce07b, 0x1b3d8f29
  311. .long 0xce7f39f4, 0xdaece73e
  312. .long 0x61d82e56, 0xf1d0f55e
  313. .long 0xd270f1a2, 0xab7aff2a
  314. .long 0xc619809d, 0xa87ab8a8
  315. .long 0x2b3cac5d, 0x2162d385
  316. .long 0x65863b64, 0x8462d800
  317. .long 0x1b03397f, 0x83348832
  318. .long 0xebb883bd, 0x71d111a8
  319. .long 0xb3e32c28, 0x299847d5
  320. .long 0x064f7f26, 0xffd852c6
  321. .long 0xdd7e3b0c, 0xb9e02b86
  322. .long 0xf285651c, 0xdcb17aa4
  323. .long 0x10746f3c, 0x18b33a4e
  324. .long 0xc7a68855, 0xf37c5aee
  325. .long 0x271d9844, 0xb6dd949b
  326. .long 0x8e766a0c, 0x6051d5a2
  327. .long 0x93a5f730, 0x78d9ccb7
  328. .long 0x6cb08e5c, 0x18b0d4ff
  329. .long 0x6b749fb2, 0xbac2fd7b
  330. .long 0x1393e203, 0x21f3d99c
  331. .long 0xcec3662e, 0xa60ce07b
  332. .long 0x96c515bb, 0x8f158014
  333. .long 0xe6fc4e6a, 0xce7f39f4
  334. .long 0x8227bb8a, 0xa00457f7
  335. .long 0xb0cd4768, 0x61d82e56
  336. .long 0x39c7ff35, 0x8d6d2c43
  337. .long 0xd7a4825c, 0xd270f1a2
  338. .long 0x0ab3844b, 0x00ac29cf
  339. .long 0x0167d312, 0xc619809d
  340. .long 0xf6076544, 0xe9adf796
  341. .long 0x26f6a60a, 0x2b3cac5d
  342. .long 0xa741c1bf, 0x96638b34
  343. .long 0x98d8d9cb, 0x65863b64
  344. .long 0x49c3cc9c, 0xe0e9f351
  345. .long 0x68bce87a, 0x1b03397f
  346. .long 0x57a3d037, 0x9af01f2d
  347. .long 0x6956fc3b, 0xebb883bd
  348. .long 0x42d98888, 0x2cff42cf
  349. .long 0x3771e98f, 0xb3e32c28
  350. .long 0xb42ae3d9, 0x88f25a3a
  351. .long 0x2178513a, 0x064f7f26
  352. .long 0xe0ac139e, 0x4e36f0b0
  353. .long 0x170076fa, 0xdd7e3b0c
  354. .long 0x444dd413, 0xbd6f81f8
  355. .long 0x6f345e45, 0xf285651c
  356. .long 0x41d17b64, 0x91c9bd4b
  357. .long 0xff0dba97, 0x10746f3c
  358. .long 0xa2b73df1, 0x885f087b
  359. .long 0xf872e54c, 0xc7a68855
  360. .long 0x1e41e9fc, 0x4c144932
  361. .long 0x86d8e4d2, 0x271d9844
  362. .long 0x651bd98b, 0x52148f02
  363. .long 0x5bb8f1bc, 0x8e766a0c
  364. .long 0xa90fd27a, 0xa3c6f37a
  365. .long 0xb3af077a, 0x93a5f730
  366. .long 0x4984d782, 0xd7c0557f
  367. .long 0xca6ef3ac, 0x6cb08e5c
  368. .long 0x234e0b26, 0x63ded06a
  369. .long 0xdd66cbbb, 0x6b749fb2
  370. .long 0x4597456a, 0x4d56973c
  371. .long 0xe9e28eb4, 0x1393e203
  372. .long 0x7b3ff57a, 0x9669c9df
  373. .long 0xc9c8b782, 0xcec3662e
  374. .long 0x3f70cc6f, 0xe417f38a
  375. .long 0x93e106a4, 0x96c515bb
  376. .long 0x62ec6c6d, 0x4b9e0f71
  377. .long 0xd813b325, 0xe6fc4e6a
  378. .long 0x0df04680, 0xd104b8fc
  379. .long 0x2342001e, 0x8227bb8a
  380. .long 0x0a2a8d7e, 0x5b397730
  381. .long 0x6d9a4957, 0xb0cd4768
  382. .long 0xe8b6368b, 0xe78eb416
  383. .long 0xd2c3ed1a, 0x39c7ff35
  384. .long 0x995a5724, 0x61ff0e01
  385. .long 0x9ef68d35, 0xd7a4825c
  386. .long 0x0c139b31, 0x8d96551c
  387. .long 0xf2271e60, 0x0ab3844b
  388. .long 0x0b0bf8ca, 0x0bf80dd2
  389. .long 0x2664fd8b, 0x0167d312
  390. .long 0xed64812d, 0x8821abed
  391. .long 0x02ee03b2, 0xf6076544
  392. .long 0x8604ae0f, 0x6a45d2b2
  393. .long 0x363bd6b3, 0x26f6a60a
  394. .long 0x135c83fd, 0xd8d26619
  395. .long 0x5fabe670, 0xa741c1bf
  396. .long 0x35ec3279, 0xde87806c
  397. .long 0x00bcf5f6, 0x98d8d9cb
  398. .long 0x8ae00689, 0x14338754
  399. .long 0x17f27698, 0x49c3cc9c
  400. .long 0x58ca5f00, 0x5bd2011f
  401. .long 0xaa7c7ad5, 0x68bce87a
  402. .long 0xb5cfca28, 0xdd07448e
  403. .long 0xded288f8, 0x57a3d037
  404. .long 0x59f229bc, 0xdde8f5b9
  405. .long 0x6d390dec, 0x6956fc3b
  406. .long 0x37170390, 0xa3e3e02c
  407. .long 0x6353c1cc, 0x42d98888
  408. .long 0xc4584f5c, 0xd73c7bea
  409. .long 0xf48642e9, 0x3771e98f
  410. .long 0x531377e2, 0x80ff0093
  411. .long 0xdd35bc8d, 0xb42ae3d9
  412. .long 0xb25b29f2, 0x8fe4c34d
  413. .long 0x9a5ede41, 0x2178513a
  414. .long 0xa563905d, 0xdf99fc11
  415. .long 0x45cddf4e, 0xe0ac139e
  416. .long 0xacfa3103, 0x6c23e841
  417. .long 0xa51b6135, 0x170076fa