entry.S 17 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752
  1. /*
  2. * Low-level exception handling code
  3. *
  4. * Copyright (C) 2012 ARM Ltd.
  5. * Authors: Catalin Marinas <catalin.marinas@arm.com>
  6. * Will Deacon <will.deacon@arm.com>
  7. *
  8. * This program is free software; you can redistribute it and/or modify
  9. * it under the terms of the GNU General Public License version 2 as
  10. * published by the Free Software Foundation.
  11. *
  12. * This program is distributed in the hope that it will be useful,
  13. * but WITHOUT ANY WARRANTY; without even the implied warranty of
  14. * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  15. * GNU General Public License for more details.
  16. *
  17. * You should have received a copy of the GNU General Public License
  18. * along with this program. If not, see <http://www.gnu.org/licenses/>.
  19. */
  20. #include <linux/init.h>
  21. #include <linux/linkage.h>
  22. #include <asm/alternative.h>
  23. #include <asm/assembler.h>
  24. #include <asm/asm-offsets.h>
  25. #include <asm/cpufeature.h>
  26. #include <asm/errno.h>
  27. #include <asm/esr.h>
  28. #include <asm/memory.h>
  29. #include <asm/thread_info.h>
  30. #include <asm/asm-uaccess.h>
  31. #include <asm/unistd.h>
  32. /*
  33. * Context tracking subsystem. Used to instrument transitions
  34. * between user and kernel mode.
  35. */
  36. .macro ct_user_exit, syscall = 0
  37. #ifdef CONFIG_CONTEXT_TRACKING
  38. bl context_tracking_user_exit
  39. .if \syscall == 1
  40. /*
  41. * Save/restore needed during syscalls. Restore syscall arguments from
  42. * the values already saved on stack during kernel_entry.
  43. */
  44. ldp x0, x1, [sp]
  45. ldp x2, x3, [sp, #S_X2]
  46. ldp x4, x5, [sp, #S_X4]
  47. ldp x6, x7, [sp, #S_X6]
  48. .endif
  49. #endif
  50. .endm
  51. .macro ct_user_enter
  52. #ifdef CONFIG_CONTEXT_TRACKING
  53. bl context_tracking_user_enter
  54. #endif
  55. .endm
  56. /*
  57. * Bad Abort numbers
  58. *-----------------
  59. */
  60. #define BAD_SYNC 0
  61. #define BAD_IRQ 1
  62. #define BAD_FIQ 2
  63. #define BAD_ERROR 3
  64. .macro kernel_entry, el, regsize = 64
  65. sub sp, sp, #S_FRAME_SIZE
  66. .if \regsize == 32
  67. mov w0, w0 // zero upper 32 bits of x0
  68. .endif
  69. stp x0, x1, [sp, #16 * 0]
  70. stp x2, x3, [sp, #16 * 1]
  71. stp x4, x5, [sp, #16 * 2]
  72. stp x6, x7, [sp, #16 * 3]
  73. stp x8, x9, [sp, #16 * 4]
  74. stp x10, x11, [sp, #16 * 5]
  75. stp x12, x13, [sp, #16 * 6]
  76. stp x14, x15, [sp, #16 * 7]
  77. stp x16, x17, [sp, #16 * 8]
  78. stp x18, x19, [sp, #16 * 9]
  79. stp x20, x21, [sp, #16 * 10]
  80. stp x22, x23, [sp, #16 * 11]
  81. stp x24, x25, [sp, #16 * 12]
  82. stp x26, x27, [sp, #16 * 13]
  83. stp x28, x29, [sp, #16 * 14]
  84. .if \el == 0
  85. mrs x21, sp_el0
  86. get_thread_info tsk // Ensure MDSCR_EL1.SS is clear,
  87. ldr x19, [tsk, #TI_FLAGS] // since we can unmask debug
  88. disable_step_tsk x19, x20 // exceptions when scheduling.
  89. .else
  90. add x21, sp, #S_FRAME_SIZE
  91. get_thread_info tsk
  92. /* Save the task's original addr_limit and set USER_DS (TASK_SIZE_64) */
  93. ldr x20, [tsk, #TI_ADDR_LIMIT]
  94. str x20, [sp, #S_ORIG_ADDR_LIMIT]
  95. mov x20, #TASK_SIZE_64
  96. str x20, [tsk, #TI_ADDR_LIMIT]
  97. .endif /* \el == 0 */
  98. mrs x22, elr_el1
  99. mrs x23, spsr_el1
  100. stp lr, x21, [sp, #S_LR]
  101. stp x22, x23, [sp, #S_PC]
  102. /*
  103. * Set syscallno to -1 by default (overridden later if real syscall).
  104. */
  105. .if \el == 0
  106. mvn x21, xzr
  107. str x21, [sp, #S_SYSCALLNO]
  108. .endif
  109. /*
  110. * Registers that may be useful after this macro is invoked:
  111. *
  112. * x21 - aborted SP
  113. * x22 - aborted PC
  114. * x23 - aborted PSTATE
  115. */
  116. .endm
  117. .macro kernel_exit, el
  118. .if \el != 0
  119. /* Restore the task's original addr_limit. */
  120. ldr x20, [sp, #S_ORIG_ADDR_LIMIT]
  121. str x20, [tsk, #TI_ADDR_LIMIT]
  122. .endif
  123. ldp x21, x22, [sp, #S_PC] // load ELR, SPSR
  124. .if \el == 0
  125. ct_user_enter
  126. ldr x23, [sp, #S_SP] // load return stack pointer
  127. msr sp_el0, x23
  128. #ifdef CONFIG_ARM64_ERRATUM_845719
  129. alternative_if_not ARM64_WORKAROUND_845719
  130. nop
  131. nop
  132. #ifdef CONFIG_PID_IN_CONTEXTIDR
  133. nop
  134. #endif
  135. alternative_else
  136. tbz x22, #4, 1f
  137. #ifdef CONFIG_PID_IN_CONTEXTIDR
  138. mrs x29, contextidr_el1
  139. msr contextidr_el1, x29
  140. #else
  141. msr contextidr_el1, xzr
  142. #endif
  143. 1:
  144. alternative_endif
  145. #endif
  146. .endif
  147. msr elr_el1, x21 // set up the return data
  148. msr spsr_el1, x22
  149. ldp x0, x1, [sp, #16 * 0]
  150. ldp x2, x3, [sp, #16 * 1]
  151. ldp x4, x5, [sp, #16 * 2]
  152. ldp x6, x7, [sp, #16 * 3]
  153. ldp x8, x9, [sp, #16 * 4]
  154. ldp x10, x11, [sp, #16 * 5]
  155. ldp x12, x13, [sp, #16 * 6]
  156. ldp x14, x15, [sp, #16 * 7]
  157. ldp x16, x17, [sp, #16 * 8]
  158. ldp x18, x19, [sp, #16 * 9]
  159. ldp x20, x21, [sp, #16 * 10]
  160. ldp x22, x23, [sp, #16 * 11]
  161. ldp x24, x25, [sp, #16 * 12]
  162. ldp x26, x27, [sp, #16 * 13]
  163. ldp x28, x29, [sp, #16 * 14]
  164. ldr lr, [sp, #S_LR]
  165. add sp, sp, #S_FRAME_SIZE // restore sp
  166. eret // return to kernel
  167. .endm
  168. .macro get_thread_info, rd
  169. mov \rd, sp
  170. and \rd, \rd, #~(THREAD_SIZE - 1) // top of stack
  171. .endm
  172. /*
  173. * These are the registers used in the syscall handler, and allow us to
  174. * have in theory up to 7 arguments to a function - x0 to x6.
  175. *
  176. * x7 is reserved for the system call number in 32-bit mode.
  177. */
  178. sc_nr .req x25 // number of system calls
  179. scno .req x26 // syscall number
  180. stbl .req x27 // syscall table pointer
  181. tsk .req x28 // current thread_info
  182. /*
  183. * Interrupt handling.
  184. */
  185. .macro irq_handler
  186. adrp x1, handle_arch_irq
  187. ldr x1, [x1, #:lo12:handle_arch_irq]
  188. mov x0, sp
  189. blr x1
  190. .endm
  191. .text
  192. /*
  193. * Exception vectors.
  194. */
  195. .align 11
  196. ENTRY(vectors)
  197. ventry el1_sync_invalid // Synchronous EL1t
  198. ventry el1_irq_invalid // IRQ EL1t
  199. ventry el1_fiq_invalid // FIQ EL1t
  200. ventry el1_error_invalid // Error EL1t
  201. ventry el1_sync // Synchronous EL1h
  202. ventry el1_irq // IRQ EL1h
  203. ventry el1_fiq_invalid // FIQ EL1h
  204. ventry el1_error_invalid // Error EL1h
  205. ventry el0_sync // Synchronous 64-bit EL0
  206. ventry el0_irq // IRQ 64-bit EL0
  207. ventry el0_fiq_invalid // FIQ 64-bit EL0
  208. ventry el0_error_invalid // Error 64-bit EL0
  209. #ifdef CONFIG_COMPAT
  210. ventry el0_sync_compat // Synchronous 32-bit EL0
  211. ventry el0_irq_compat // IRQ 32-bit EL0
  212. ventry el0_fiq_invalid_compat // FIQ 32-bit EL0
  213. ventry el0_error_invalid_compat // Error 32-bit EL0
  214. #else
  215. ventry el0_sync_invalid // Synchronous 32-bit EL0
  216. ventry el0_irq_invalid // IRQ 32-bit EL0
  217. ventry el0_fiq_invalid // FIQ 32-bit EL0
  218. ventry el0_error_invalid // Error 32-bit EL0
  219. #endif
  220. END(vectors)
  221. /*
  222. * Invalid mode handlers
  223. */
  224. .macro inv_entry, el, reason, regsize = 64
  225. kernel_entry \el, \regsize
  226. mov x0, sp
  227. mov x1, #\reason
  228. mrs x2, esr_el1
  229. b bad_mode
  230. .endm
  231. el0_sync_invalid:
  232. inv_entry 0, BAD_SYNC
  233. ENDPROC(el0_sync_invalid)
  234. el0_irq_invalid:
  235. inv_entry 0, BAD_IRQ
  236. ENDPROC(el0_irq_invalid)
  237. el0_fiq_invalid:
  238. inv_entry 0, BAD_FIQ
  239. ENDPROC(el0_fiq_invalid)
  240. el0_error_invalid:
  241. inv_entry 0, BAD_ERROR
  242. ENDPROC(el0_error_invalid)
  243. #ifdef CONFIG_COMPAT
  244. el0_fiq_invalid_compat:
  245. inv_entry 0, BAD_FIQ, 32
  246. ENDPROC(el0_fiq_invalid_compat)
  247. el0_error_invalid_compat:
  248. inv_entry 0, BAD_ERROR, 32
  249. ENDPROC(el0_error_invalid_compat)
  250. #endif
  251. el1_sync_invalid:
  252. inv_entry 1, BAD_SYNC
  253. ENDPROC(el1_sync_invalid)
  254. el1_irq_invalid:
  255. inv_entry 1, BAD_IRQ
  256. ENDPROC(el1_irq_invalid)
  257. el1_fiq_invalid:
  258. inv_entry 1, BAD_FIQ
  259. ENDPROC(el1_fiq_invalid)
  260. el1_error_invalid:
  261. inv_entry 1, BAD_ERROR
  262. ENDPROC(el1_error_invalid)
  263. /*
  264. * EL1 mode handlers.
  265. */
  266. .align 6
  267. el1_sync:
  268. kernel_entry 1
  269. mrs x1, esr_el1 // read the syndrome register
  270. lsr x24, x1, #ESR_ELx_EC_SHIFT // exception class
  271. cmp x24, #ESR_ELx_EC_DABT_CUR // data abort in EL1
  272. b.eq el1_da
  273. cmp x24, #ESR_ELx_EC_SYS64 // configurable trap
  274. b.eq el1_undef
  275. cmp x24, #ESR_ELx_EC_SP_ALIGN // stack alignment exception
  276. b.eq el1_sp_pc
  277. cmp x24, #ESR_ELx_EC_PC_ALIGN // pc alignment exception
  278. b.eq el1_sp_pc
  279. cmp x24, #ESR_ELx_EC_UNKNOWN // unknown exception in EL1
  280. b.eq el1_undef
  281. cmp x24, #ESR_ELx_EC_BREAKPT_CUR // debug exception in EL1
  282. b.ge el1_dbg
  283. b el1_inv
  284. el1_da:
  285. /*
  286. * Data abort handling
  287. */
  288. mrs x3, far_el1
  289. enable_dbg
  290. // re-enable interrupts if they were enabled in the aborted context
  291. tbnz x23, #7, 1f // PSR_I_BIT
  292. enable_irq
  293. 1:
  294. clear_address_tag x0, x3
  295. mov x2, sp // struct pt_regs
  296. bl do_mem_abort
  297. // disable interrupts before pulling preserved data off the stack
  298. disable_irq
  299. kernel_exit 1
  300. el1_sp_pc:
  301. /*
  302. * Stack or PC alignment exception handling
  303. */
  304. mrs x0, far_el1
  305. enable_dbg
  306. mov x2, sp
  307. b do_sp_pc_abort
  308. el1_undef:
  309. /*
  310. * Undefined instruction
  311. */
  312. enable_dbg
  313. mov x0, sp
  314. b do_undefinstr
  315. el1_dbg:
  316. /*
  317. * Debug exception handling
  318. */
  319. cmp x24, #ESR_ELx_EC_BRK64 // if BRK64
  320. cinc x24, x24, eq // set bit '0'
  321. tbz x24, #0, el1_inv // EL1 only
  322. mrs x0, far_el1
  323. mov x2, sp // struct pt_regs
  324. bl do_debug_exception
  325. kernel_exit 1
  326. el1_inv:
  327. // TODO: add support for undefined instructions in kernel mode
  328. enable_dbg
  329. mov x0, sp
  330. mov x2, x1
  331. mov x1, #BAD_SYNC
  332. b bad_mode
  333. ENDPROC(el1_sync)
  334. .align 6
  335. el1_irq:
  336. kernel_entry 1
  337. enable_dbg
  338. #ifdef CONFIG_TRACE_IRQFLAGS
  339. bl trace_hardirqs_off
  340. #endif
  341. irq_handler
  342. #ifdef CONFIG_PREEMPT
  343. get_thread_info tsk
  344. ldr w24, [tsk, #TI_PREEMPT] // get preempt count
  345. cbnz w24, 1f // preempt count != 0
  346. ldr x0, [tsk, #TI_FLAGS] // get flags
  347. tbz x0, #TIF_NEED_RESCHED, 1f // needs rescheduling?
  348. bl el1_preempt
  349. 1:
  350. #endif
  351. #ifdef CONFIG_TRACE_IRQFLAGS
  352. bl trace_hardirqs_on
  353. #endif
  354. kernel_exit 1
  355. ENDPROC(el1_irq)
  356. #ifdef CONFIG_PREEMPT
  357. el1_preempt:
  358. mov x24, lr
  359. 1: bl preempt_schedule_irq // irq en/disable is done inside
  360. ldr x0, [tsk, #TI_FLAGS] // get new tasks TI_FLAGS
  361. tbnz x0, #TIF_NEED_RESCHED, 1b // needs rescheduling?
  362. ret x24
  363. #endif
  364. /*
  365. * EL0 mode handlers.
  366. */
  367. .align 6
  368. el0_sync:
  369. kernel_entry 0
  370. mrs x25, esr_el1 // read the syndrome register
  371. lsr x24, x25, #ESR_ELx_EC_SHIFT // exception class
  372. cmp x24, #ESR_ELx_EC_SVC64 // SVC in 64-bit state
  373. b.eq el0_svc
  374. cmp x24, #ESR_ELx_EC_DABT_LOW // data abort in EL0
  375. b.eq el0_da
  376. cmp x24, #ESR_ELx_EC_IABT_LOW // instruction abort in EL0
  377. b.eq el0_ia
  378. cmp x24, #ESR_ELx_EC_FP_ASIMD // FP/ASIMD access
  379. b.eq el0_fpsimd_acc
  380. cmp x24, #ESR_ELx_EC_FP_EXC64 // FP/ASIMD exception
  381. b.eq el0_fpsimd_exc
  382. cmp x24, #ESR_ELx_EC_SYS64 // configurable trap
  383. b.eq el0_undef
  384. cmp x24, #ESR_ELx_EC_SP_ALIGN // stack alignment exception
  385. b.eq el0_sp_pc
  386. cmp x24, #ESR_ELx_EC_PC_ALIGN // pc alignment exception
  387. b.eq el0_sp_pc
  388. cmp x24, #ESR_ELx_EC_UNKNOWN // unknown exception in EL0
  389. b.eq el0_undef
  390. cmp x24, #ESR_ELx_EC_BREAKPT_LOW // debug exception in EL0
  391. b.ge el0_dbg
  392. b el0_inv
  393. #ifdef CONFIG_COMPAT
  394. .align 6
  395. el0_sync_compat:
  396. kernel_entry 0, 32
  397. mrs x25, esr_el1 // read the syndrome register
  398. lsr x24, x25, #ESR_ELx_EC_SHIFT // exception class
  399. cmp x24, #ESR_ELx_EC_SVC32 // SVC in 32-bit state
  400. b.eq el0_svc_compat
  401. cmp x24, #ESR_ELx_EC_DABT_LOW // data abort in EL0
  402. b.eq el0_da
  403. cmp x24, #ESR_ELx_EC_IABT_LOW // instruction abort in EL0
  404. b.eq el0_ia
  405. cmp x24, #ESR_ELx_EC_FP_ASIMD // FP/ASIMD access
  406. b.eq el0_fpsimd_acc
  407. cmp x24, #ESR_ELx_EC_FP_EXC32 // FP/ASIMD exception
  408. b.eq el0_fpsimd_exc
  409. cmp x24, #ESR_ELx_EC_PC_ALIGN // pc alignment exception
  410. b.eq el0_sp_pc
  411. cmp x24, #ESR_ELx_EC_UNKNOWN // unknown exception in EL0
  412. b.eq el0_undef
  413. cmp x24, #ESR_ELx_EC_CP15_32 // CP15 MRC/MCR trap
  414. b.eq el0_undef
  415. cmp x24, #ESR_ELx_EC_CP15_64 // CP15 MRRC/MCRR trap
  416. b.eq el0_undef
  417. cmp x24, #ESR_ELx_EC_CP14_MR // CP14 MRC/MCR trap
  418. b.eq el0_undef
  419. cmp x24, #ESR_ELx_EC_CP14_LS // CP14 LDC/STC trap
  420. b.eq el0_undef
  421. cmp x24, #ESR_ELx_EC_CP14_64 // CP14 MRRC/MCRR trap
  422. b.eq el0_undef
  423. cmp x24, #ESR_ELx_EC_BREAKPT_LOW // debug exception in EL0
  424. b.ge el0_dbg
  425. b el0_inv
  426. el0_svc_compat:
  427. /*
  428. * AArch32 syscall handling
  429. */
  430. adrp stbl, compat_sys_call_table // load compat syscall table pointer
  431. uxtw scno, w7 // syscall number in w7 (r7)
  432. mov sc_nr, #__NR_compat_syscalls
  433. b el0_svc_naked
  434. .align 6
  435. el0_irq_compat:
  436. kernel_entry 0, 32
  437. b el0_irq_naked
  438. #endif
  439. el0_da:
  440. /*
  441. * Data abort handling
  442. */
  443. mrs x26, far_el1
  444. // enable interrupts before calling the main handler
  445. enable_dbg_and_irq
  446. ct_user_exit
  447. clear_address_tag x0, x26
  448. mov x1, x25
  449. mov x2, sp
  450. bl do_mem_abort
  451. b ret_to_user
  452. el0_ia:
  453. /*
  454. * Instruction abort handling
  455. */
  456. mrs x26, far_el1
  457. // enable interrupts before calling the main handler
  458. enable_dbg_and_irq
  459. ct_user_exit
  460. mov x0, x26
  461. orr x1, x25, #1 << 24 // use reserved ISS bit for instruction aborts
  462. mov x2, sp
  463. bl do_mem_abort
  464. b ret_to_user
  465. el0_fpsimd_acc:
  466. /*
  467. * Floating Point or Advanced SIMD access
  468. */
  469. enable_dbg
  470. ct_user_exit
  471. mov x0, x25
  472. mov x1, sp
  473. bl do_fpsimd_acc
  474. b ret_to_user
  475. el0_fpsimd_exc:
  476. /*
  477. * Floating Point or Advanced SIMD exception
  478. */
  479. enable_dbg
  480. ct_user_exit
  481. mov x0, x25
  482. mov x1, sp
  483. bl do_fpsimd_exc
  484. b ret_to_user
  485. el0_sp_pc:
  486. /*
  487. * Stack or PC alignment exception handling
  488. */
  489. mrs x26, far_el1
  490. // enable interrupts before calling the main handler
  491. enable_dbg_and_irq
  492. ct_user_exit
  493. mov x0, x26
  494. mov x1, x25
  495. mov x2, sp
  496. bl do_sp_pc_abort
  497. b ret_to_user
  498. el0_undef:
  499. /*
  500. * Undefined instruction
  501. */
  502. // enable interrupts before calling the main handler
  503. enable_dbg_and_irq
  504. ct_user_exit
  505. mov x0, sp
  506. bl do_undefinstr
  507. b ret_to_user
  508. el0_dbg:
  509. /*
  510. * Debug exception handling
  511. */
  512. tbnz x24, #0, el0_inv // EL0 only
  513. mrs x0, far_el1
  514. mov x1, x25
  515. mov x2, sp
  516. bl do_debug_exception
  517. enable_dbg
  518. ct_user_exit
  519. b ret_to_user
  520. el0_inv:
  521. enable_dbg
  522. ct_user_exit
  523. mov x0, sp
  524. mov x1, #BAD_SYNC
  525. mov x2, x25
  526. bl bad_el0_sync
  527. b ret_to_user
  528. ENDPROC(el0_sync)
  529. .align 6
  530. el0_irq:
  531. kernel_entry 0
  532. el0_irq_naked:
  533. enable_dbg
  534. #ifdef CONFIG_TRACE_IRQFLAGS
  535. bl trace_hardirqs_off
  536. #endif
  537. ct_user_exit
  538. irq_handler
  539. #ifdef CONFIG_TRACE_IRQFLAGS
  540. bl trace_hardirqs_on
  541. #endif
  542. b ret_to_user
  543. ENDPROC(el0_irq)
  544. /*
  545. * Register switch for AArch64. The callee-saved registers need to be saved
  546. * and restored. On entry:
  547. * x0 = previous task_struct (must be preserved across the switch)
  548. * x1 = next task_struct
  549. * Previous and next are guaranteed not to be the same.
  550. *
  551. */
  552. ENTRY(cpu_switch_to)
  553. mov x10, #THREAD_CPU_CONTEXT
  554. add x8, x0, x10
  555. mov x9, sp
  556. stp x19, x20, [x8], #16 // store callee-saved registers
  557. stp x21, x22, [x8], #16
  558. stp x23, x24, [x8], #16
  559. stp x25, x26, [x8], #16
  560. stp x27, x28, [x8], #16
  561. stp x29, x9, [x8], #16
  562. str lr, [x8]
  563. add x8, x1, x10
  564. ldp x19, x20, [x8], #16 // restore callee-saved registers
  565. ldp x21, x22, [x8], #16
  566. ldp x23, x24, [x8], #16
  567. ldp x25, x26, [x8], #16
  568. ldp x27, x28, [x8], #16
  569. ldp x29, x9, [x8], #16
  570. ldr lr, [x8]
  571. mov sp, x9
  572. ret
  573. ENDPROC(cpu_switch_to)
  574. /*
  575. * This is the fast syscall return path. We do as little as possible here,
  576. * and this includes saving x0 back into the kernel stack.
  577. */
  578. ret_fast_syscall:
  579. disable_irq // disable interrupts
  580. str x0, [sp, #S_X0] // returned x0
  581. ldr x1, [tsk, #TI_FLAGS] // re-check for syscall tracing
  582. and x2, x1, #_TIF_SYSCALL_WORK
  583. cbnz x2, ret_fast_syscall_trace
  584. and x2, x1, #_TIF_WORK_MASK
  585. cbnz x2, work_pending
  586. enable_step_tsk x1, x2
  587. kernel_exit 0
  588. ret_fast_syscall_trace:
  589. enable_irq // enable interrupts
  590. b __sys_trace_return_skipped // we already saved x0
  591. /*
  592. * Ok, we need to do extra processing, enter the slow path.
  593. */
  594. work_pending:
  595. tbnz x1, #TIF_NEED_RESCHED, work_resched
  596. /* TIF_SIGPENDING, TIF_NOTIFY_RESUME or TIF_FOREIGN_FPSTATE case */
  597. ldr x2, [sp, #S_PSTATE]
  598. mov x0, sp // 'regs'
  599. tst x2, #PSR_MODE_MASK // user mode regs?
  600. b.ne no_work_pending // returning to kernel
  601. enable_irq // enable interrupts for do_notify_resume()
  602. bl do_notify_resume
  603. b ret_to_user
  604. work_resched:
  605. #ifdef CONFIG_TRACE_IRQFLAGS
  606. bl trace_hardirqs_off // the IRQs are off here, inform the tracing code
  607. #endif
  608. bl schedule
  609. /*
  610. * "slow" syscall return path.
  611. */
  612. ret_to_user:
  613. disable_irq // disable interrupts
  614. ldr x1, [tsk, #TI_FLAGS]
  615. and x2, x1, #_TIF_WORK_MASK
  616. cbnz x2, work_pending
  617. enable_step_tsk x1, x2
  618. no_work_pending:
  619. kernel_exit 0
  620. ENDPROC(ret_to_user)
  621. /*
  622. * This is how we return from a fork.
  623. */
  624. ENTRY(ret_from_fork)
  625. bl schedule_tail
  626. cbz x19, 1f // not a kernel thread
  627. mov x0, x20
  628. blr x19
  629. 1: get_thread_info tsk
  630. b ret_to_user
  631. ENDPROC(ret_from_fork)
  632. /*
  633. * SVC handler.
  634. */
  635. .align 6
  636. el0_svc:
  637. adrp stbl, sys_call_table // load syscall table pointer
  638. uxtw scno, w8 // syscall number in w8
  639. mov sc_nr, #__NR_syscalls
  640. el0_svc_naked: // compat entry point
  641. stp x0, scno, [sp, #S_ORIG_X0] // save the original x0 and syscall number
  642. enable_dbg_and_irq
  643. ct_user_exit 1
  644. ldr x16, [tsk, #TI_FLAGS] // check for syscall hooks
  645. tst x16, #_TIF_SYSCALL_WORK
  646. b.ne __sys_trace
  647. cmp scno, sc_nr // check upper syscall limit
  648. b.hs ni_sys
  649. ldr x16, [stbl, scno, lsl #3] // address in the syscall table
  650. blr x16 // call sys_* routine
  651. b ret_fast_syscall
  652. ni_sys:
  653. mov x0, sp
  654. bl do_ni_syscall
  655. b ret_fast_syscall
  656. ENDPROC(el0_svc)
  657. /*
  658. * This is the really slow path. We're going to be doing context
  659. * switches, and waiting for our parent to respond.
  660. */
  661. __sys_trace:
  662. mov w0, #-1 // set default errno for
  663. cmp scno, x0 // user-issued syscall(-1)
  664. b.ne 1f
  665. mov x0, #-ENOSYS
  666. str x0, [sp, #S_X0]
  667. 1: mov x0, sp
  668. bl syscall_trace_enter
  669. cmp w0, #-1 // skip the syscall?
  670. b.eq __sys_trace_return_skipped
  671. uxtw scno, w0 // syscall number (possibly new)
  672. mov x1, sp // pointer to regs
  673. cmp scno, sc_nr // check upper syscall limit
  674. b.hs __ni_sys_trace
  675. ldp x0, x1, [sp] // restore the syscall args
  676. ldp x2, x3, [sp, #S_X2]
  677. ldp x4, x5, [sp, #S_X4]
  678. ldp x6, x7, [sp, #S_X6]
  679. ldr x16, [stbl, scno, lsl #3] // address in the syscall table
  680. blr x16 // call sys_* routine
  681. __sys_trace_return:
  682. str x0, [sp, #S_X0] // save returned x0
  683. __sys_trace_return_skipped:
  684. mov x0, sp
  685. bl syscall_trace_exit
  686. b ret_to_user
  687. __ni_sys_trace:
  688. mov x0, sp
  689. bl do_ni_syscall
  690. b __sys_trace_return
  691. /*
  692. * Special system call wrappers.
  693. */
  694. ENTRY(sys_rt_sigreturn_wrapper)
  695. mov x0, sp
  696. b sys_rt_sigreturn
  697. ENDPROC(sys_rt_sigreturn_wrapper)