memcpy-sh4.S 15 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799
  1. /*
  2. * "memcpy" implementation of SuperH
  3. *
  4. * Copyright (C) 1999 Niibe Yutaka
  5. * Copyright (c) 2002 STMicroelectronics Ltd
  6. * Modified from memcpy.S and micro-optimised for SH4
  7. * Stuart Menefy (stuart.menefy@st.com)
  8. *
  9. */
  10. #include <linux/linkage.h>
  11. /*
  12. * void *memcpy(void *dst, const void *src, size_t n);
  13. *
  14. * It is assumed that there is no overlap between src and dst.
  15. * If there is an overlap, then the results are undefined.
  16. */
  17. !
  18. ! GHIJ KLMN OPQR --> ...G HIJK LMNO PQR.
  19. !
  20. ! Size is 16 or greater, and may have trailing bytes
  21. .balign 32
  22. .Lcase1:
  23. ! Read a long word and write a long word at once
  24. ! At the start of each iteration, r7 contains last long load
  25. add #-1,r5 ! 79 EX
  26. mov r4,r2 ! 5 MT (0 cycles latency)
  27. mov.l @(r0,r5),r7 ! 21 LS (2 cycles latency)
  28. add #-4,r5 ! 50 EX
  29. add #7,r2 ! 79 EX
  30. !
  31. #ifdef CONFIG_CPU_LITTLE_ENDIAN
  32. ! 6 cycles, 4 bytes per iteration
  33. 3: mov.l @(r0,r5),r1 ! 21 LS (latency=2) ! NMLK
  34. mov r7, r3 ! 5 MT (latency=0) ! RQPO
  35. cmp/hi r2,r0 ! 57 MT
  36. shll16 r3 ! 103 EX
  37. mov r1,r6 ! 5 MT (latency=0)
  38. shll8 r3 ! 102 EX ! Oxxx
  39. shlr8 r6 ! 106 EX ! xNML
  40. mov r1, r7 ! 5 MT (latency=0)
  41. or r6,r3 ! 82 EX ! ONML
  42. bt/s 3b ! 109 BR
  43. mov.l r3,@-r0 ! 30 LS
  44. #else
  45. 3: mov.l @(r0,r5),r1 ! 21 LS (latency=2) ! KLMN
  46. mov r7,r3 ! 5 MT (latency=0) ! OPQR
  47. cmp/hi r2,r0 ! 57 MT
  48. shlr16 r3 ! 107 EX
  49. shlr8 r3 ! 106 EX ! xxxO
  50. mov r1,r6 ! 5 MT (latency=0)
  51. shll8 r6 ! 102 EX ! LMNx
  52. mov r1,r7 ! 5 MT (latency=0)
  53. or r6,r3 ! 82 EX ! LMNO
  54. bt/s 3b ! 109 BR
  55. mov.l r3,@-r0 ! 30 LS
  56. #endif
  57. ! Finally, copy a byte at once, if necessary
  58. add #4,r5 ! 50 EX
  59. cmp/eq r4,r0 ! 54 MT
  60. add #-6,r2 ! 50 EX
  61. bt 9f ! 109 BR
  62. 8: cmp/hi r2,r0 ! 57 MT
  63. mov.b @(r0,r5),r1 ! 20 LS (latency=2)
  64. bt/s 8b ! 109 BR
  65. mov.b r1,@-r0 ! 29 LS
  66. 9: rts
  67. nop
  68. !
  69. ! GHIJ KLMN OPQR --> .GHI JKLM NOPQ R...
  70. !
  71. ! Size is 16 or greater, and may have trailing bytes
  72. .balign 32
  73. .Lcase3:
  74. ! Read a long word and write a long word at once
  75. ! At the start of each iteration, r7 contains last long load
  76. add #-3,r5 ! 79 EX
  77. mov r4,r2 ! 5 MT (0 cycles latency)
  78. mov.l @(r0,r5),r7 ! 21 LS (2 cycles latency)
  79. add #-4,r5 ! 50 EX
  80. add #7,r2 ! 79 EX
  81. !
  82. #ifdef CONFIG_CPU_LITTLE_ENDIAN
  83. ! 6 cycles, 4 bytes per iteration
  84. 3: mov.l @(r0,r5),r1 ! 21 LS (latency=2) ! NMLK
  85. mov r7, r3 ! 5 MT (latency=0) ! RQPO
  86. cmp/hi r2,r0 ! 57 MT
  87. shll8 r3 ! 102 EX ! QPOx
  88. mov r1,r6 ! 5 MT (latency=0)
  89. shlr16 r6 ! 107 EX
  90. shlr8 r6 ! 106 EX ! xxxN
  91. mov r1, r7 ! 5 MT (latency=0)
  92. or r6,r3 ! 82 EX ! QPON
  93. bt/s 3b ! 109 BR
  94. mov.l r3,@-r0 ! 30 LS
  95. #else
  96. 3: mov r7,r3 ! OPQR
  97. shlr8 r3 ! xOPQ
  98. mov.l @(r0,r5),r7 ! KLMN
  99. mov r7,r6
  100. shll16 r6
  101. shll8 r6 ! Nxxx
  102. or r6,r3 ! NOPQ
  103. cmp/hi r2,r0
  104. bt/s 3b
  105. mov.l r3,@-r0
  106. #endif
  107. ! Finally, copy a byte at once, if necessary
  108. add #6,r5 ! 50 EX
  109. cmp/eq r4,r0 ! 54 MT
  110. add #-6,r2 ! 50 EX
  111. bt 9f ! 109 BR
  112. 8: cmp/hi r2,r0 ! 57 MT
  113. mov.b @(r0,r5),r1 ! 20 LS (latency=2)
  114. bt/s 8b ! 109 BR
  115. mov.b r1,@-r0 ! 29 LS
  116. 9: rts
  117. nop
  118. ENTRY(memcpy)
  119. ! Calculate the invariants which will be used in the remainder
  120. ! of the code:
  121. !
  122. ! r4 --> [ ... ] DST [ ... ] SRC
  123. ! [ ... ] [ ... ]
  124. ! : :
  125. ! r0 --> [ ... ] r0+r5 --> [ ... ]
  126. !
  127. !
  128. ! Short circuit the common case of src, dst and len being 32 bit aligned
  129. ! and test for zero length move
  130. mov r6, r0 ! 5 MT (0 cycle latency)
  131. or r4, r0 ! 82 EX
  132. or r5, r0 ! 82 EX
  133. tst r6, r6 ! 86 MT
  134. bt/s 99f ! 111 BR (zero len)
  135. tst #3, r0 ! 87 MT
  136. mov r4, r0 ! 5 MT (0 cycle latency)
  137. add r6, r0 ! 49 EX
  138. mov #16, r1 ! 6 EX
  139. bt/s .Lcase00 ! 111 BR (aligned)
  140. sub r4, r5 ! 75 EX
  141. ! Arguments are not nicely long word aligned or zero len.
  142. ! Check for small copies, and if so do a simple byte at a time copy.
  143. !
  144. ! Deciding on an exact value of 'small' is not easy, as the point at which
  145. ! using the optimised routines become worthwhile varies (these are the
  146. ! cycle counts for differnet sizes using byte-at-a-time vs. optimised):
  147. ! size byte-at-time long word byte
  148. ! 16 42 39-40 46-50 50-55
  149. ! 24 58 43-44 54-58 62-67
  150. ! 36 82 49-50 66-70 80-85
  151. ! However the penalty for getting it 'wrong' is much higher for long word
  152. ! aligned data (and this is more common), so use a value of 16.
  153. cmp/gt r6,r1 ! 56 MT
  154. add #-1,r5 ! 50 EX
  155. bf/s 6f ! 108 BR (not small)
  156. mov r5, r3 ! 5 MT (latency=0)
  157. shlr r6 ! 104 EX
  158. mov.b @(r0,r5),r1 ! 20 LS (latency=2)
  159. bf/s 4f ! 111 BR
  160. add #-1,r3 ! 50 EX
  161. tst r6, r6 ! 86 MT
  162. bt/s 98f ! 110 BR
  163. mov.b r1,@-r0 ! 29 LS
  164. ! 4 cycles, 2 bytes per iteration
  165. 3: mov.b @(r0,r5),r1 ! 20 LS (latency=2)
  166. 4: mov.b @(r0,r3),r2 ! 20 LS (latency=2)
  167. dt r6 ! 67 EX
  168. mov.b r1,@-r0 ! 29 LS
  169. bf/s 3b ! 111 BR
  170. mov.b r2,@-r0 ! 29 LS
  171. 98:
  172. rts
  173. nop
  174. 99: rts
  175. mov r4, r0
  176. ! Size is not small, so its worthwhile looking for optimisations.
  177. ! First align destination to a long word boundary.
  178. !
  179. ! r5 = normal value -1
  180. 6: tst #3, r0 ! 87 MT
  181. mov #3, r3 ! 6 EX
  182. bt/s 2f ! 111 BR
  183. and r0,r3 ! 78 EX
  184. ! 3 cycles, 1 byte per iteration
  185. 1: dt r3 ! 67 EX
  186. mov.b @(r0,r5),r1 ! 19 LS (latency=2)
  187. add #-1, r6 ! 79 EX
  188. bf/s 1b ! 109 BR
  189. mov.b r1,@-r0 ! 28 LS
  190. 2: add #1, r5 ! 79 EX
  191. ! Now select the appropriate bulk transfer code based on relative
  192. ! alignment of src and dst.
  193. mov r0, r3 ! 5 MT (latency=0)
  194. mov r5, r0 ! 5 MT (latency=0)
  195. tst #1, r0 ! 87 MT
  196. bf/s 1f ! 111 BR
  197. mov #64, r7 ! 6 EX
  198. ! bit 0 clear
  199. cmp/ge r7, r6 ! 55 MT
  200. bt/s 2f ! 111 BR
  201. tst #2, r0 ! 87 MT
  202. ! small
  203. bt/s .Lcase0
  204. mov r3, r0
  205. bra .Lcase2
  206. nop
  207. ! big
  208. 2: bt/s .Lcase0b
  209. mov r3, r0
  210. bra .Lcase2b
  211. nop
  212. ! bit 0 set
  213. 1: tst #2, r0 ! 87 MT
  214. bt/s .Lcase1
  215. mov r3, r0
  216. bra .Lcase3
  217. nop
  218. !
  219. ! GHIJ KLMN OPQR --> GHIJ KLMN OPQR
  220. !
  221. ! src, dst and size are all long word aligned
  222. ! size is non-zero
  223. .balign 32
  224. .Lcase00:
  225. mov #64, r1 ! 6 EX
  226. mov r5, r3 ! 5 MT (latency=0)
  227. cmp/gt r6, r1 ! 56 MT
  228. add #-4, r5 ! 50 EX
  229. bf .Lcase00b ! 108 BR (big loop)
  230. shlr2 r6 ! 105 EX
  231. shlr r6 ! 104 EX
  232. mov.l @(r0, r5), r1 ! 21 LS (latency=2)
  233. bf/s 4f ! 111 BR
  234. add #-8, r3 ! 50 EX
  235. tst r6, r6 ! 86 MT
  236. bt/s 5f ! 110 BR
  237. mov.l r1,@-r0 ! 30 LS
  238. ! 4 cycles, 2 long words per iteration
  239. 3: mov.l @(r0, r5), r1 ! 21 LS (latency=2)
  240. 4: mov.l @(r0, r3), r2 ! 21 LS (latency=2)
  241. dt r6 ! 67 EX
  242. mov.l r1, @-r0 ! 30 LS
  243. bf/s 3b ! 109 BR
  244. mov.l r2, @-r0 ! 30 LS
  245. 5: rts
  246. nop
  247. ! Size is 16 or greater and less than 64, but may have trailing bytes
  248. .balign 32
  249. .Lcase0:
  250. add #-4, r5 ! 50 EX
  251. mov r4, r7 ! 5 MT (latency=0)
  252. mov.l @(r0, r5), r1 ! 21 LS (latency=2)
  253. mov #4, r2 ! 6 EX
  254. add #11, r7 ! 50 EX
  255. tst r2, r6 ! 86 MT
  256. mov r5, r3 ! 5 MT (latency=0)
  257. bt/s 4f ! 111 BR
  258. add #-4, r3 ! 50 EX
  259. mov.l r1,@-r0 ! 30 LS
  260. ! 4 cycles, 2 long words per iteration
  261. 3: mov.l @(r0, r5), r1 ! 21 LS (latency=2)
  262. 4: mov.l @(r0, r3), r2 ! 21 LS (latency=2)
  263. cmp/hi r7, r0
  264. mov.l r1, @-r0 ! 30 LS
  265. bt/s 3b ! 109 BR
  266. mov.l r2, @-r0 ! 30 LS
  267. ! Copy the final 0-3 bytes
  268. add #3,r5 ! 50 EX
  269. cmp/eq r0, r4 ! 54 MT
  270. add #-10, r7 ! 50 EX
  271. bt 9f ! 110 BR
  272. ! 3 cycles, 1 byte per iteration
  273. 1: mov.b @(r0,r5),r1 ! 19 LS
  274. cmp/hi r7,r0 ! 57 MT
  275. bt/s 1b ! 111 BR
  276. mov.b r1,@-r0 ! 28 LS
  277. 9: rts
  278. nop
  279. ! Size is at least 64 bytes, so will be going round the big loop at least once.
  280. !
  281. ! r2 = rounded up r4
  282. ! r3 = rounded down r0
  283. .balign 32
  284. .Lcase0b:
  285. add #-4, r5 ! 50 EX
  286. .Lcase00b:
  287. mov r0, r3 ! 5 MT (latency=0)
  288. mov #(~0x1f), r1 ! 6 EX
  289. and r1, r3 ! 78 EX
  290. mov r4, r2 ! 5 MT (latency=0)
  291. cmp/eq r3, r0 ! 54 MT
  292. add #0x1f, r2 ! 50 EX
  293. bt/s 1f ! 110 BR
  294. and r1, r2 ! 78 EX
  295. ! copy initial words until cache line aligned
  296. mov.l @(r0, r5), r1 ! 21 LS (latency=2)
  297. tst #4, r0 ! 87 MT
  298. mov r5, r6 ! 5 MT (latency=0)
  299. add #-4, r6 ! 50 EX
  300. bt/s 4f ! 111 BR
  301. add #8, r3 ! 50 EX
  302. tst #0x18, r0 ! 87 MT
  303. bt/s 1f ! 109 BR
  304. mov.l r1,@-r0 ! 30 LS
  305. ! 4 cycles, 2 long words per iteration
  306. 3: mov.l @(r0, r5), r1 ! 21 LS (latency=2)
  307. 4: mov.l @(r0, r6), r7 ! 21 LS (latency=2)
  308. cmp/eq r3, r0 ! 54 MT
  309. mov.l r1, @-r0 ! 30 LS
  310. bf/s 3b ! 109 BR
  311. mov.l r7, @-r0 ! 30 LS
  312. ! Copy the cache line aligned blocks
  313. !
  314. ! In use: r0, r2, r4, r5
  315. ! Scratch: r1, r3, r6, r7
  316. !
  317. ! We could do this with the four scratch registers, but if src
  318. ! and dest hit the same cache line, this will thrash, so make
  319. ! use of additional registers.
  320. !
  321. ! We also need r0 as a temporary (for movca), so 'undo' the invariant:
  322. ! r5: src (was r0+r5)
  323. ! r1: dest (was r0)
  324. ! this can be reversed at the end, so we don't need to save any extra
  325. ! state.
  326. !
  327. 1: mov.l r8, @-r15 ! 30 LS
  328. add r0, r5 ! 49 EX
  329. mov.l r9, @-r15 ! 30 LS
  330. mov r0, r1 ! 5 MT (latency=0)
  331. mov.l r10, @-r15 ! 30 LS
  332. add #-0x1c, r5 ! 50 EX
  333. mov.l r11, @-r15 ! 30 LS
  334. ! 16 cycles, 32 bytes per iteration
  335. 2: mov.l @(0x00,r5),r0 ! 18 LS (latency=2)
  336. add #-0x20, r1 ! 50 EX
  337. mov.l @(0x04,r5),r3 ! 18 LS (latency=2)
  338. mov.l @(0x08,r5),r6 ! 18 LS (latency=2)
  339. mov.l @(0x0c,r5),r7 ! 18 LS (latency=2)
  340. mov.l @(0x10,r5),r8 ! 18 LS (latency=2)
  341. mov.l @(0x14,r5),r9 ! 18 LS (latency=2)
  342. mov.l @(0x18,r5),r10 ! 18 LS (latency=2)
  343. mov.l @(0x1c,r5),r11 ! 18 LS (latency=2)
  344. movca.l r0,@r1 ! 40 LS (latency=3-7)
  345. mov.l r3,@(0x04,r1) ! 33 LS
  346. mov.l r6,@(0x08,r1) ! 33 LS
  347. mov.l r7,@(0x0c,r1) ! 33 LS
  348. mov.l r8,@(0x10,r1) ! 33 LS
  349. add #-0x20, r5 ! 50 EX
  350. mov.l r9,@(0x14,r1) ! 33 LS
  351. cmp/eq r2,r1 ! 54 MT
  352. mov.l r10,@(0x18,r1) ! 33 LS
  353. bf/s 2b ! 109 BR
  354. mov.l r11,@(0x1c,r1) ! 33 LS
  355. mov r1, r0 ! 5 MT (latency=0)
  356. mov.l @r15+, r11 ! 15 LS
  357. sub r1, r5 ! 75 EX
  358. mov.l @r15+, r10 ! 15 LS
  359. cmp/eq r4, r0 ! 54 MT
  360. bf/s 1f ! 109 BR
  361. mov.l @r15+, r9 ! 15 LS
  362. rts
  363. 1: mov.l @r15+, r8 ! 15 LS
  364. sub r4, r1 ! 75 EX (len remaining)
  365. ! number of trailing bytes is non-zero
  366. !
  367. ! invariants restored (r5 already decremented by 4)
  368. ! also r1=num bytes remaining
  369. mov #4, r2 ! 6 EX
  370. mov r4, r7 ! 5 MT (latency=0)
  371. add #0x1c, r5 ! 50 EX (back to -4)
  372. cmp/hs r2, r1 ! 58 MT
  373. bf/s 5f ! 108 BR
  374. add #11, r7 ! 50 EX
  375. mov.l @(r0, r5), r6 ! 21 LS (latency=2)
  376. tst r2, r1 ! 86 MT
  377. mov r5, r3 ! 5 MT (latency=0)
  378. bt/s 4f ! 111 BR
  379. add #-4, r3 ! 50 EX
  380. cmp/hs r2, r1 ! 58 MT
  381. bt/s 5f ! 111 BR
  382. mov.l r6,@-r0 ! 30 LS
  383. ! 4 cycles, 2 long words per iteration
  384. 3: mov.l @(r0, r5), r6 ! 21 LS (latency=2)
  385. 4: mov.l @(r0, r3), r2 ! 21 LS (latency=2)
  386. cmp/hi r7, r0
  387. mov.l r6, @-r0 ! 30 LS
  388. bt/s 3b ! 109 BR
  389. mov.l r2, @-r0 ! 30 LS
  390. ! Copy the final 0-3 bytes
  391. 5: cmp/eq r0, r4 ! 54 MT
  392. add #-10, r7 ! 50 EX
  393. bt 9f ! 110 BR
  394. add #3,r5 ! 50 EX
  395. ! 3 cycles, 1 byte per iteration
  396. 1: mov.b @(r0,r5),r1 ! 19 LS
  397. cmp/hi r7,r0 ! 57 MT
  398. bt/s 1b ! 111 BR
  399. mov.b r1,@-r0 ! 28 LS
  400. 9: rts
  401. nop
  402. !
  403. ! GHIJ KLMN OPQR --> ..GH IJKL MNOP QR..
  404. !
  405. .balign 32
  406. .Lcase2:
  407. ! Size is 16 or greater and less then 64, but may have trailing bytes
  408. 2: mov r5, r6 ! 5 MT (latency=0)
  409. add #-2,r5 ! 50 EX
  410. mov r4,r2 ! 5 MT (latency=0)
  411. add #-4,r6 ! 50 EX
  412. add #7,r2 ! 50 EX
  413. 3: mov.w @(r0,r5),r1 ! 20 LS (latency=2)
  414. mov.w @(r0,r6),r3 ! 20 LS (latency=2)
  415. cmp/hi r2,r0 ! 57 MT
  416. mov.w r1,@-r0 ! 29 LS
  417. bt/s 3b ! 111 BR
  418. mov.w r3,@-r0 ! 29 LS
  419. bra 10f
  420. nop
  421. .balign 32
  422. .Lcase2b:
  423. ! Size is at least 64 bytes, so will be going round the big loop at least once.
  424. !
  425. ! r2 = rounded up r4
  426. ! r3 = rounded down r0
  427. mov r0, r3 ! 5 MT (latency=0)
  428. mov #(~0x1f), r1 ! 6 EX
  429. and r1, r3 ! 78 EX
  430. mov r4, r2 ! 5 MT (latency=0)
  431. cmp/eq r3, r0 ! 54 MT
  432. add #0x1f, r2 ! 50 EX
  433. add #-2, r5 ! 50 EX
  434. bt/s 1f ! 110 BR
  435. and r1, r2 ! 78 EX
  436. ! Copy a short word one at a time until we are cache line aligned
  437. ! Normal values: r0, r2, r3, r4
  438. ! Unused: r1, r6, r7
  439. ! Mod: r5 (=r5-2)
  440. !
  441. add #2, r3 ! 50 EX
  442. 2: mov.w @(r0,r5),r1 ! 20 LS (latency=2)
  443. cmp/eq r3,r0 ! 54 MT
  444. bf/s 2b ! 111 BR
  445. mov.w r1,@-r0 ! 29 LS
  446. ! Copy the cache line aligned blocks
  447. !
  448. ! In use: r0, r2, r4, r5 (=r5-2)
  449. ! Scratch: r1, r3, r6, r7
  450. !
  451. ! We could do this with the four scratch registers, but if src
  452. ! and dest hit the same cache line, this will thrash, so make
  453. ! use of additional registers.
  454. !
  455. ! We also need r0 as a temporary (for movca), so 'undo' the invariant:
  456. ! r5: src (was r0+r5)
  457. ! r1: dest (was r0)
  458. ! this can be reversed at the end, so we don't need to save any extra
  459. ! state.
  460. !
  461. 1: mov.l r8, @-r15 ! 30 LS
  462. add r0, r5 ! 49 EX
  463. mov.l r9, @-r15 ! 30 LS
  464. mov r0, r1 ! 5 MT (latency=0)
  465. mov.l r10, @-r15 ! 30 LS
  466. add #-0x1e, r5 ! 50 EX
  467. mov.l r11, @-r15 ! 30 LS
  468. mov.l r12, @-r15 ! 30 LS
  469. ! 17 cycles, 32 bytes per iteration
  470. #ifdef CONFIG_CPU_LITTLE_ENDIAN
  471. 2: mov.w @r5+, r0 ! 14 LS (latency=2) ..JI
  472. add #-0x20, r1 ! 50 EX
  473. mov.l @r5+, r3 ! 15 LS (latency=2) NMLK
  474. mov.l @r5+, r6 ! 15 LS (latency=2) RQPO
  475. shll16 r0 ! 103 EX JI..
  476. mov.l @r5+, r7 ! 15 LS (latency=2)
  477. xtrct r3, r0 ! 48 EX LKJI
  478. mov.l @r5+, r8 ! 15 LS (latency=2)
  479. xtrct r6, r3 ! 48 EX PONM
  480. mov.l @r5+, r9 ! 15 LS (latency=2)
  481. xtrct r7, r6 ! 48 EX
  482. mov.l @r5+, r10 ! 15 LS (latency=2)
  483. xtrct r8, r7 ! 48 EX
  484. mov.l @r5+, r11 ! 15 LS (latency=2)
  485. xtrct r9, r8 ! 48 EX
  486. mov.w @r5+, r12 ! 15 LS (latency=2)
  487. xtrct r10, r9 ! 48 EX
  488. movca.l r0,@r1 ! 40 LS (latency=3-7)
  489. xtrct r11, r10 ! 48 EX
  490. mov.l r3, @(0x04,r1) ! 33 LS
  491. xtrct r12, r11 ! 48 EX
  492. mov.l r6, @(0x08,r1) ! 33 LS
  493. mov.l r7, @(0x0c,r1) ! 33 LS
  494. mov.l r8, @(0x10,r1) ! 33 LS
  495. add #-0x40, r5 ! 50 EX
  496. mov.l r9, @(0x14,r1) ! 33 LS
  497. cmp/eq r2,r1 ! 54 MT
  498. mov.l r10, @(0x18,r1) ! 33 LS
  499. bf/s 2b ! 109 BR
  500. mov.l r11, @(0x1c,r1) ! 33 LS
  501. #else
  502. 2: mov.w @(0x1e,r5), r0 ! 17 LS (latency=2)
  503. add #-2, r5 ! 50 EX
  504. mov.l @(0x1c,r5), r3 ! 18 LS (latency=2)
  505. add #-4, r1 ! 50 EX
  506. mov.l @(0x18,r5), r6 ! 18 LS (latency=2)
  507. shll16 r0 ! 103 EX
  508. mov.l @(0x14,r5), r7 ! 18 LS (latency=2)
  509. xtrct r3, r0 ! 48 EX
  510. mov.l @(0x10,r5), r8 ! 18 LS (latency=2)
  511. xtrct r6, r3 ! 48 EX
  512. mov.l @(0x0c,r5), r9 ! 18 LS (latency=2)
  513. xtrct r7, r6 ! 48 EX
  514. mov.l @(0x08,r5), r10 ! 18 LS (latency=2)
  515. xtrct r8, r7 ! 48 EX
  516. mov.l @(0x04,r5), r11 ! 18 LS (latency=2)
  517. xtrct r9, r8 ! 48 EX
  518. mov.l @(0x00,r5), r12 ! 18 LS (latency=2)
  519. xtrct r10, r9 ! 48 EX
  520. movca.l r0,@r1 ! 40 LS (latency=3-7)
  521. add #-0x1c, r1 ! 50 EX
  522. mov.l r3, @(0x18,r1) ! 33 LS
  523. xtrct r11, r10 ! 48 EX
  524. mov.l r6, @(0x14,r1) ! 33 LS
  525. xtrct r12, r11 ! 48 EX
  526. mov.l r7, @(0x10,r1) ! 33 LS
  527. mov.l r8, @(0x0c,r1) ! 33 LS
  528. add #-0x1e, r5 ! 50 EX
  529. mov.l r9, @(0x08,r1) ! 33 LS
  530. cmp/eq r2,r1 ! 54 MT
  531. mov.l r10, @(0x04,r1) ! 33 LS
  532. bf/s 2b ! 109 BR
  533. mov.l r11, @(0x00,r1) ! 33 LS
  534. #endif
  535. mov.l @r15+, r12
  536. mov r1, r0 ! 5 MT (latency=0)
  537. mov.l @r15+, r11 ! 15 LS
  538. sub r1, r5 ! 75 EX
  539. mov.l @r15+, r10 ! 15 LS
  540. cmp/eq r4, r0 ! 54 MT
  541. bf/s 1f ! 109 BR
  542. mov.l @r15+, r9 ! 15 LS
  543. rts
  544. 1: mov.l @r15+, r8 ! 15 LS
  545. add #0x1e, r5 ! 50 EX
  546. ! Finish off a short word at a time
  547. ! r5 must be invariant - 2
  548. 10: mov r4,r2 ! 5 MT (latency=0)
  549. add #1,r2 ! 50 EX
  550. cmp/hi r2, r0 ! 57 MT
  551. bf/s 1f ! 109 BR
  552. add #2, r2 ! 50 EX
  553. 3: mov.w @(r0,r5),r1 ! 20 LS
  554. cmp/hi r2,r0 ! 57 MT
  555. bt/s 3b ! 109 BR
  556. mov.w r1,@-r0 ! 29 LS
  557. 1:
  558. !
  559. ! Finally, copy the last byte if necessary
  560. cmp/eq r4,r0 ! 54 MT
  561. bt/s 9b
  562. add #1,r5
  563. mov.b @(r0,r5),r1
  564. rts
  565. mov.b r1,@-r0