mirrors/Atmosphere
1
0
Fork 0
mirror of https://github.com/Atmosphere-NX/Atmosphere.git synced 2025-05-18 17:14:28 -04:00
Code Issues Projects Releases Packages Wiki Activity

libmesosphere: move .s to kernel (prevent manifesting in kernelldr)

Browse source
This commit is contained in:
Michael Scire 2020-02-13 22:05:20 -08:00
parent 8c93eb5712
commit 364b04b68a
21 changed files with 100 additions and 53 deletions
Download patch file Download diff file Expand all files Collapse all files

610
libraries/libmesosphere/source/arch/arm64/kern_exception_handlers_asm.s
View file

@ -1,610 +0,0 @@
/*
* Copyright (c) 2018-2020 Atmosphère-NX
*
* This program is free software; you can redistribute it and/or modify it
* under the terms and conditions of the GNU General Public License,
* version 2, as published by the Free Software Foundation.
*
* This program is distributed in the hope it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
/* ams::kern::arm64::EL1IrqExceptionHandler() */
.section .text._ZN3ams4kern5arm6422EL1IrqExceptionHandlerEv, "ax", %progbits
.global _ZN3ams4kern5arm6422EL1IrqExceptionHandlerEv
.type _ZN3ams4kern5arm6422EL1IrqExceptionHandlerEv, %function
_ZN3ams4kern5arm6422EL1IrqExceptionHandlerEv:
/* Save registers that need saving. */
sub sp, sp, #(8 * 24)
stp x0, x1, [sp, #(8 * 0)]
stp x2, x3, [sp, #(8 * 2)]
stp x4, x5, [sp, #(8 * 4)]
stp x6, x7, [sp, #(8 * 6)]
stp x8, x9, [sp, #(8 * 8)]
stp x10, x11, [sp, #(8 * 10)]
stp x12, x13, [sp, #(8 * 12)]
stp x14, x15, [sp, #(8 * 14)]
stp x16, x17, [sp, #(8 * 16)]
stp x18, x19, [sp, #(8 * 18)]
stp x20, x21, [sp, #(8 * 20)]
stp x22, x30, [sp, #(8 * 22)]
mrs x19, sp_el0
mrs x20, elr_el1
mrs x21, spsr_el1
mov w21, w21
/* Invoke KInterruptManager::HandleInterrupt(bool user_mode). */
mrs x18, tpidr_el1
mov x0, #0
bl _ZN3ams4kern5arm6417KInterruptManager15HandleInterruptEb
/* Restore registers that we saved. */
msr sp_el0, x19
msr elr_el1, x20
msr spsr_el1, x21
ldp x0, x1, [sp, #(8 * 0)]
ldp x2, x3, [sp, #(8 * 2)]
ldp x4, x5, [sp, #(8 * 4)]
ldp x6, x7, [sp, #(8 * 6)]
ldp x8, x9, [sp, #(8 * 8)]
ldp x10, x11, [sp, #(8 * 10)]
ldp x12, x13, [sp, #(8 * 12)]
ldp x14, x15, [sp, #(8 * 14)]
ldp x16, x17, [sp, #(8 * 16)]
ldp x18, x19, [sp, #(8 * 18)]
ldp x20, x21, [sp, #(8 * 20)]
ldp x22, x30, [sp, #(8 * 22)]
add sp, sp, #(8 * 24)
/* Return from the exception. */
eret
/* ams::kern::arm64::EL0IrqExceptionHandler() */
.section .text._ZN3ams4kern5arm6422EL0IrqExceptionHandlerEv, "ax", %progbits
.global _ZN3ams4kern5arm6422EL0IrqExceptionHandlerEv
.type _ZN3ams4kern5arm6422EL0IrqExceptionHandlerEv, %function
_ZN3ams4kern5arm6422EL0IrqExceptionHandlerEv:
/* Save registers that need saving. */
sub sp, sp, #(8 * 36)
stp x0, x1, [sp, #(8 * 0)]
stp x2, x3, [sp, #(8 * 2)]
stp x4, x5, [sp, #(8 * 4)]
stp x6, x7, [sp, #(8 * 6)]
stp x8, x9, [sp, #(8 * 8)]
stp x10, x11, [sp, #(8 * 10)]
stp x12, x13, [sp, #(8 * 12)]
stp x14, x15, [sp, #(8 * 14)]
stp x16, x17, [sp, #(8 * 16)]
stp x18, x19, [sp, #(8 * 18)]
stp x20, x21, [sp, #(8 * 20)]
stp x22, x23, [sp, #(8 * 22)]
stp x24, x25, [sp, #(8 * 24)]
stp x26, x27, [sp, #(8 * 26)]
stp x28, x29, [sp, #(8 * 28)]
mrs x20, sp_el0
mrs x21, elr_el1
mrs x22, spsr_el1
mrs x23, tpidr_el0
mov w22, w22
stp x30, x20, [sp, #(8 * 30)]
stp x21, x22, [sp, #(8 * 32)]
str x23, [sp, #(8 * 34)]
/* Invoke KInterruptManager::HandleInterrupt(bool user_mode). */
mrs x18, tpidr_el1
mov x0, #1
bl _ZN3ams4kern5arm6417KInterruptManager15HandleInterruptEb
/* Restore state from the context. */
ldp x30, x20, [sp, #(8 * 30)]
ldp x21, x22, [sp, #(8 * 32)]
ldr x23, [sp, #(8 * 34)]
msr sp_el0, x20
msr elr_el1, x21
msr spsr_el1, x22
msr tpidr_el0, x23
ldp x0, x1, [sp, #(8 * 0)]
ldp x2, x3, [sp, #(8 * 2)]
ldp x4, x5, [sp, #(8 * 4)]
ldp x6, x7, [sp, #(8 * 6)]
ldp x8, x9, [sp, #(8 * 8)]
ldp x10, x11, [sp, #(8 * 10)]
ldp x12, x13, [sp, #(8 * 12)]
ldp x14, x15, [sp, #(8 * 14)]
ldp x16, x17, [sp, #(8 * 16)]
ldp x18, x19, [sp, #(8 * 18)]
ldp x20, x21, [sp, #(8 * 20)]
ldp x22, x23, [sp, #(8 * 22)]
ldp x24, x25, [sp, #(8 * 24)]
ldp x26, x27, [sp, #(8 * 26)]
ldp x28, x29, [sp, #(8 * 28)]
add sp, sp, #0x120
/* Return from the exception. */
eret
/* ams::kern::arm64::EL0SynchronousExceptionHandler() */
.section .text._ZN3ams4kern5arm6430EL0SynchronousExceptionHandlerEv, "ax", %progbits
.global _ZN3ams4kern5arm6430EL0SynchronousExceptionHandlerEv
.type _ZN3ams4kern5arm6430EL0SynchronousExceptionHandlerEv, %function
_ZN3ams4kern5arm6430EL0SynchronousExceptionHandlerEv:
/* Save x16 and x17, so that we can use them as scratch. */
stp x16, x17, [sp, #-16]!
/* Get and parse the exception syndrome register. */
mrs x16, esr_el1
lsr x17, x16, #0x1a
/* Is this an aarch32 SVC? */
cmp x17, #0x11
b.eq 2f
/* Is this an aarch64 SVC? */
cmp x17, #0x15
b.eq 3f
/* Is this an FPU error? */
cmp x17, #0x7
b.eq 4f
/* Is this an instruction abort? */
cmp x17, #0x21
b.eq 5f
/* Is this a data abort? */
cmp x17, #0x25
b.eq 5f
1: /* The exception is not a data abort or instruction abort caused by a TLB conflict. */
/* It is also not an SVC or an FPU exception. Handle it generically! */
/* Restore x16 and x17. */
ldp x16, x17, [sp], 16
/* Create a KExceptionContext to pass to HandleException. */
sub sp, sp, #0x120
stp x0, x1, [sp, #(8 * 0)]
stp x2, x3, [sp, #(8 * 2)]
stp x4, x5, [sp, #(8 * 4)]
stp x6, x7, [sp, #(8 * 6)]
stp x8, x9, [sp, #(8 * 8)]
stp x10, x11, [sp, #(8 * 10)]
stp x12, x13, [sp, #(8 * 12)]
stp x14, x15, [sp, #(8 * 14)]
stp x16, x17, [sp, #(8 * 16)]
stp x18, x19, [sp, #(8 * 18)]
stp x20, x21, [sp, #(8 * 20)]
stp x22, x23, [sp, #(8 * 22)]
stp x24, x25, [sp, #(8 * 24)]
stp x26, x27, [sp, #(8 * 26)]
stp x28, x29, [sp, #(8 * 28)]
mrs x20, sp_el0
mrs x21, elr_el1
mrs x22, spsr_el1
mrs x23, tpidr_el0
mov w22, w22
stp x30, x20, [sp, #(8 * 30)]
stp x21, x22, [sp, #(8 * 32)]
str x23, [sp, #(8 * 34)]
/* Call ams::kern::arm64::HandleException(ams::kern::arm64::KExceptionContext *) */
mrs x18, tpidr_el1
mov x0, sp
bl _ZN3ams4kern5arm6415HandleExceptionEPNS1_17KExceptionContextE
/* Restore state from the context. */
ldp x30, x20, [sp, #(8 * 30)]
ldp x21, x22, [sp, #(8 * 32)]
ldr x23, [sp, #(8 * 34)]
msr sp_el0, x20
msr elr_el1, x21
msr spsr_el1, x22
msr tpidr_el0, x23
ldp x0, x1, [sp, #(8 * 0)]
ldp x2, x3, [sp, #(8 * 2)]
ldp x4, x5, [sp, #(8 * 4)]
ldp x6, x7, [sp, #(8 * 6)]
ldp x8, x9, [sp, #(8 * 8)]
ldp x10, x11, [sp, #(8 * 10)]
ldp x12, x13, [sp, #(8 * 12)]
ldp x14, x15, [sp, #(8 * 14)]
ldp x16, x17, [sp, #(8 * 16)]
ldp x18, x19, [sp, #(8 * 18)]
ldp x20, x21, [sp, #(8 * 20)]
ldp x22, x23, [sp, #(8 * 22)]
ldp x24, x25, [sp, #(8 * 24)]
ldp x26, x27, [sp, #(8 * 26)]
ldp x28, x29, [sp, #(8 * 28)]
add sp, sp, #0x120
/* Return from the exception. */
eret
2: /* SVC from aarch32. */
ldp x16, x17, [sp], 16
b _ZN3ams4kern5arm6412SvcHandler32Ev
3: /* SVC from aarch64. */
ldp x16, x17, [sp], 16
b _ZN3ams4kern5arm6412SvcHandler64Ev
4: /* FPU exception. */
ldp x16, x17, [sp], 16
b _ZN3ams4kern5arm6425FpuAccessExceptionHandlerEv
5: /* Check if there's a TLB conflict that caused the abort. */
and x17, x16, #0x3F
cmp x17, #0x30
b.ne 1b
/* Get the ASID in x17. */
mrs x17, ttbr0_el1
and x17, x17, #(0xFFFF << 48)
/* Check if FAR is valid by examining the FnV bit. */
tbnz x16, #10, 6f
/* FAR is valid, so we can invalidate the address it holds. */
mrs x16, far_el1
lsr x16, x16, #12
orr x17, x16, x17
tlbi vaae1, x17
b 7f
6: /* There's a TLB conflict and FAR isn't valid. */
/* Invalidate the entire TLB. */
tlbi vmalle1
7: /* Return from a TLB conflict. */
/* Ensure instruction consistency. */
dsb ish
isb
/* Restore x16 and x17. */
ldp x16, x17, [sp], 16
/* Return from the exception. */
eret
/* ams::kern::arm64::EL1SynchronousExceptionHandler() */
.section .text._ZN3ams4kern5arm6430EL1SynchronousExceptionHandlerEv, "ax", %progbits
.global _ZN3ams4kern5arm6430EL1SynchronousExceptionHandlerEv
.type _ZN3ams4kern5arm6430EL1SynchronousExceptionHandlerEv, %function
_ZN3ams4kern5arm6430EL1SynchronousExceptionHandlerEv:
/* Nintendo uses the "unused" virtual timer compare value as a scratch register. */
msr cntv_cval_el0, x0
/* Get and parse the exception syndrome register. */
mrs x0, esr_el1
lsr x0, x0, #0x1a
/* Is this an instruction abort? */
cmp x0, #0x21
b.eq 5f
/* Is this a data abort? */
cmp x0, #0x25
b.eq 5f
1: /* The exception is not a data abort or instruction abort caused by a TLB conflict. */
/* Load the CoreLocalContext into x0. */
mrs x0, tpidr_el1
cbz x0, 2f
/* Load the exception stack top from the context. */
ldr x0, [x0, #0x28]
/* Setup the stack for a generic exception handle */
sub x0, x0, #0x20
str x1, [x0, #16]
mov x1, sp
str x1, [x0]
mov sp, x0
ldr x1, [x0, #16]
mrs x0, cntv_cval_el0
str x0, [sp, #8]
/* Check again if this is a data abort from EL1. */
mrs x0, esr_el1
lsr x1, x0, #0x1a
cmp x1, #0x25
b.ne 3f
/* Data abort. Check if it was from trying to access userspace memory. */
mrs x1, elr_el1
adr x0, _ZN3ams4kern5arm6438UserspaceMemoryAccessFunctionAreaBeginEv
cmp x1, x0
b.lo 3f
adr x0, _ZN3ams4kern5arm6436UserspaceMemoryAccessFunctionAreaEndEv
cmp x1, x0
b.hs 3f
/* We aborted trying to access userspace memory. */
/* All functions that access user memory return a boolean for whether they succeeded. */
/* With that in mind, we can simply restore the stack pointer and return false directly. */
ldr x0, [sp]
mov sp, x0
/* Return false. */
mov x0, #0x0
msr elr_el1, x30
eret
2: /* The CoreLocalContext is nullptr. */
/* Setup the stack for a generic exception handle. */
/* NOTE: Nintendo does not restore X0 here, and thus saves nullptr. */
/* This is probably not their intention, so we'll fix it. */
/* NOTE: Nintendo also does not really save SP correctly, and so we */
/* will also fix that. */
mov x0, sp
sub x0, x0, #0x20
str x1, [x0, #16]
mov x1, sp
str x1, [x0]
mov sp, x0
mrs x0, cntv_cval_el0
str x0, [sp, #8]
3: /* The exception wasn't an triggered by copying memory from userspace. */
ldr x0, [sp, #8]
ldr x1, [sp, #16]
/* Create a KExceptionContext to pass to HandleException. */
sub sp, sp, #0x120
stp x0, x1, [sp, #(8 * 0)]
stp x2, x3, [sp, #(8 * 2)]
stp x4, x5, [sp, #(8 * 4)]
stp x6, x7, [sp, #(8 * 6)]
stp x8, x9, [sp, #(8 * 8)]
stp x10, x11, [sp, #(8 * 10)]
stp x12, x13, [sp, #(8 * 12)]
stp x14, x15, [sp, #(8 * 14)]
stp x16, x17, [sp, #(8 * 16)]
stp x18, x19, [sp, #(8 * 18)]
stp x20, x21, [sp, #(8 * 20)]
stp x22, x23, [sp, #(8 * 22)]
stp x24, x25, [sp, #(8 * 24)]
stp x26, x27, [sp, #(8 * 26)]
stp x28, x29, [sp, #(8 * 28)]
mrs x20, sp_el0
mrs x21, elr_el1
mrs x22, spsr_el1
mrs x23, tpidr_el0
mov w22, w22
stp x30, x20, [sp, #(8 * 30)]
stp x21, x22, [sp, #(8 * 32)]
str x23, [sp, #(8 * 34)]
/* Call ams::kern::arm64::HandleException(ams::kern::arm64::KExceptionContext *) */
mrs x18, tpidr_el1
mov x0, sp
bl _ZN3ams4kern5arm6415HandleExceptionEPNS1_17KExceptionContextE
4: /* HandleException should never return. The best we can do is infinite loop. */
b 4b
5: /* Check if there's a TLB conflict that caused the abort. */
/* NOTE: There is a Nintendo bug in this code that we correct. */
/* Nintendo compares the low 6 bits of x0 without restoring the value. */
/* They intend to check the DFSC/IFSC bits of esr_el1, but because they */
/* shifted esr earlier, the check is invalid and always fails. */
mrs x0, esr_el1
and x0, x0, #0x3F
cmp x0, #0x30
b.ne 1b
/* Check if FAR is valid by examining the FnV bit. */
/* NOTE: Nintendo again has a bug here, the same as above. */
/* They do not refresh the value of x0, and again compare with */
/* the relevant bit already masked out of x0. */
mrs x0, esr_el1
tbnz x0, #10, 6f
/* FAR is valid, so we can invalidate the address it holds. */
mrs x0, far_el1
lsr x0, x0, #12
tlbi vaae1, x0
b 7f
6: /* There's a TLB conflict and FAR isn't valid. */
/* Invalidate the entire TLB. */
tlbi vmalle1
7: /* Return from a TLB conflict. */
/* Ensure instruction consistency. */
dsb ish
isb
/* Restore x0 from scratch. */
mrs x0, cntv_cval_el0
/* Return from the exception. */
eret
/* ams::kern::arm64::FpuAccessExceptionHandler() */
.section .text._ZN3ams4kern5arm6425FpuAccessExceptionHandlerEv, "ax", %progbits
.global _ZN3ams4kern5arm6425FpuAccessExceptionHandlerEv
.type _ZN3ams4kern5arm6425FpuAccessExceptionHandlerEv, %function
_ZN3ams4kern5arm6425FpuAccessExceptionHandlerEv:
/* Save registers that need saving. */
sub sp, sp, #(8 * 24)
stp x0, x1, [sp, #(8 * 0)]
stp x2, x3, [sp, #(8 * 2)]
stp x4, x5, [sp, #(8 * 4)]
stp x6, x7, [sp, #(8 * 6)]
stp x8, x9, [sp, #(8 * 8)]
stp x10, x11, [sp, #(8 * 10)]
stp x12, x13, [sp, #(8 * 12)]
stp x14, x15, [sp, #(8 * 14)]
stp x16, x17, [sp, #(8 * 16)]
stp x18, x19, [sp, #(8 * 18)]
stp x20, x21, [sp, #(8 * 20)]
stp x22, x30, [sp, #(8 * 22)]
mrs x18, tpidr_el1
mrs x19, sp_el0
mrs x20, elr_el1
mrs x21, spsr_el1
/* Invoke the FPU context switch handler. */
bl _ZN3ams4kern5arm6423FpuContextSwitchHandlerEv
/* Restore registers that we saved. */
msr sp_el0, x19
msr elr_el1, x20
msr spsr_el1, x21
ldp x0, x1, [sp, #(8 * 0)]
ldp x2, x3, [sp, #(8 * 2)]
ldp x4, x5, [sp, #(8 * 4)]
ldp x6, x7, [sp, #(8 * 6)]
ldp x8, x9, [sp, #(8 * 8)]
ldp x10, x11, [sp, #(8 * 10)]
ldp x12, x13, [sp, #(8 * 12)]
ldp x14, x15, [sp, #(8 * 14)]
ldp x16, x17, [sp, #(8 * 16)]
ldp x18, x19, [sp, #(8 * 18)]
ldp x20, x21, [sp, #(8 * 20)]
ldp x22, x30, [sp, #(8 * 22)]
add sp, sp, #(8 * 24)
/* Return from the exception. */
eret
/* ams::kern::arm64::EL1SystemErrorHandler() */
.section .text._ZN3ams4kern5arm6421EL1SystemErrorHandlerEv, "ax", %progbits
.global _ZN3ams4kern5arm6421EL1SystemErrorHandlerEv
.type _ZN3ams4kern5arm6421EL1SystemErrorHandlerEv, %function
_ZN3ams4kern5arm6421EL1SystemErrorHandlerEv:
/* Nintendo uses the "unused" virtual timer compare value as a scratch register. */
msr cntv_cval_el0, x0
/* Load the exception stack top from the context. */
ldr x0, [x0, #0x28]
/* Setup the stack for a generic exception handle */
sub x0, x0, #0x20
str x1, [x0, #16]
mov x1, sp
str x1, [x0]
mov sp, x0
ldr x1, [x0, #16]
mrs x0, cntv_cval_el0
str x0, [sp, #8]
/* Create a KExceptionContext to pass to HandleException. */
sub sp, sp, #0x120
stp x0, x1, [sp, #(8 * 0)]
stp x2, x3, [sp, #(8 * 2)]
stp x4, x5, [sp, #(8 * 4)]
stp x6, x7, [sp, #(8 * 6)]
stp x8, x9, [sp, #(8 * 8)]
stp x10, x11, [sp, #(8 * 10)]
stp x12, x13, [sp, #(8 * 12)]
stp x14, x15, [sp, #(8 * 14)]
stp x16, x17, [sp, #(8 * 16)]
stp x18, x19, [sp, #(8 * 18)]
stp x20, x21, [sp, #(8 * 20)]
stp x22, x23, [sp, #(8 * 22)]
stp x24, x25, [sp, #(8 * 24)]
stp x26, x27, [sp, #(8 * 26)]
stp x28, x29, [sp, #(8 * 28)]
mrs x20, sp_el0
mrs x21, elr_el1
mrs x22, spsr_el1
mrs x23, tpidr_el0
mov w22, w22
stp x30, x20, [sp, #(8 * 30)]
stp x21, x22, [sp, #(8 * 32)]
str x23, [sp, #(8 * 34)]
/* Invoke ams::kern::arm64::HandleException(ams::kern::arm64::KExceptionContext *). */
mrs x18, tpidr_el1
mov x0, sp
bl _ZN3ams4kern5arm6415HandleExceptionEPNS1_17KExceptionContextE
1: /* HandleException should never return. The best we can do is infinite loop. */
b 1b
/* Return from the exception. */
eret
/* ams::kern::arm64::EL0SystemErrorHandler() */
.section .text._ZN3ams4kern5arm6421EL0SystemErrorHandlerEv, "ax", %progbits
.global _ZN3ams4kern5arm6421EL0SystemErrorHandlerEv
.type _ZN3ams4kern5arm6421EL0SystemErrorHandlerEv, %function
_ZN3ams4kern5arm6421EL0SystemErrorHandlerEv:
/* Create a KExceptionContext to pass to HandleException. */
sub sp, sp, #0x120
stp x0, x1, [sp, #(8 * 0)]
stp x2, x3, [sp, #(8 * 2)]
stp x4, x5, [sp, #(8 * 4)]
stp x6, x7, [sp, #(8 * 6)]
stp x8, x9, [sp, #(8 * 8)]
stp x10, x11, [sp, #(8 * 10)]
stp x12, x13, [sp, #(8 * 12)]
stp x14, x15, [sp, #(8 * 14)]
stp x16, x17, [sp, #(8 * 16)]
stp x18, x19, [sp, #(8 * 18)]
stp x20, x21, [sp, #(8 * 20)]
stp x22, x23, [sp, #(8 * 22)]
stp x24, x25, [sp, #(8 * 24)]
stp x26, x27, [sp, #(8 * 26)]
stp x28, x29, [sp, #(8 * 28)]
mrs x20, sp_el0
mrs x21, elr_el1
mrs x22, spsr_el1
mrs x23, tpidr_el0
mov w22, w22
stp x30, x20, [sp, #(8 * 30)]
stp x21, x22, [sp, #(8 * 32)]
str x23, [sp, #(8 * 34)]
/* Invoke ams::kern::arm64::HandleException(ams::kern::arm64::KExceptionContext *). */
mrs x18, tpidr_el1
mov x0, sp
bl _ZN3ams4kern5arm6415HandleExceptionEPNS1_17KExceptionContextE
/* Restore state from the context. */
ldp x30, x20, [sp, #(8 * 30)]
ldp x21, x22, [sp, #(8 * 32)]
ldr x23, [sp, #(8 * 34)]
msr sp_el0, x20
msr elr_el1, x21
msr spsr_el1, x22
msr tpidr_el0, x23
ldp x0, x1, [sp, #(8 * 0)]
ldp x2, x3, [sp, #(8 * 2)]
ldp x4, x5, [sp, #(8 * 4)]
ldp x6, x7, [sp, #(8 * 6)]
ldp x8, x9, [sp, #(8 * 8)]
ldp x10, x11, [sp, #(8 * 10)]
ldp x12, x13, [sp, #(8 * 12)]
ldp x14, x15, [sp, #(8 * 14)]
ldp x16, x17, [sp, #(8 * 16)]
ldp x18, x19, [sp, #(8 * 18)]
ldp x20, x21, [sp, #(8 * 20)]
ldp x22, x23, [sp, #(8 * 22)]
ldp x24, x25, [sp, #(8 * 24)]
ldp x26, x27, [sp, #(8 * 26)]
ldp x28, x29, [sp, #(8 * 28)]
add sp, sp, #0x120
/* Return from the exception. */
eret

1
libraries/libmesosphere/source/arch/arm64/kern_k_page_table.cpp
View file

@ -181,7 +181,6 @@ namespace ams::kern::arm64 {
Result KPageTable::MapContiguous(KProcessAddress virt_addr, KPhysicalAddress phys_addr, size_t num_pages, PageTableEntry entry_template, PageLinkedList *page_list, bool reuse_ll) {
MESOSPHERE_ASSERT(this->IsLockedByCurrentThread());
MESOSPHERE_LOG("KPageTable::MapContiguous(%016lx, %016lx, %zu)\n", GetInteger(virt_addr), GetInteger(phys_addr), num_pages);
/* Cache initial addresses for use on cleanup. */
const KProcessAddress orig_virt_addr = virt_addr;

278
libraries/libmesosphere/source/arch/arm64/kern_k_scheduler_asm.s
View file

@ -1,278 +0,0 @@
/*
* Copyright (c) 2018-2020 Atmosphère-NX
*
* This program is free software; you can redistribute it and/or modify it
* under the terms and conditions of the GNU General Public License,
* version 2, as published by the Free Software Foundation.
*
* This program is distributed in the hope it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define SAVE_THREAD_CONTEXT(ctx, tmp0, tmp1, done_label) \
/* Save the callee save registers + SP and cpacr. */ \
mov tmp0, sp; \
mrs tmp1, cpacr_el1; \
stp x19, x20, [ctx, #(8 * 0)]; \
stp x21, x22, [ctx, #(8 * 2)]; \
stp x23, x24, [ctx, #(8 * 4)]; \
stp x25, x26, [ctx, #(8 * 6)]; \
stp x27, x28, [ctx, #(8 * 8)]; \
stp x29, x30, [ctx, #(8 * 10)]; \
\
stp tmp0, tmp1, [ctx, #0x60]; \
\
/* Check whether the FPU is enabled. */ \
/* If it isn't, skip saving FPU state. */ \
and tmp1, tmp1, #0x300000; \
cbz tmp1, done_label; \
\
/* Save fpcr and fpsr. */ \
mrs tmp0, fpcr; \
mrs tmp1, fpsr; \
stp tmp0, tmp1, [ctx, #0x70]; \
\
/* Save the FPU registers. */ \
stp q0, q1, [ctx, #(16 * 0 + 0x80)]; \
stp q2, q3, [ctx, #(16 * 2 + 0x80)]; \
stp q4, q5, [ctx, #(16 * 4 + 0x80)]; \
stp q6, q7, [ctx, #(16 * 6 + 0x80)]; \
stp q8, q9, [ctx, #(16 * 8 + 0x80)]; \
stp q10, q11, [ctx, #(16 * 10 + 0x80)]; \
stp q12, q13, [ctx, #(16 * 12 + 0x80)]; \
stp q14, q15, [ctx, #(16 * 14 + 0x80)]; \
stp q16, q17, [ctx, #(16 * 16 + 0x80)]; \
stp q18, q19, [ctx, #(16 * 18 + 0x80)]; \
stp q20, q21, [ctx, #(16 * 20 + 0x80)]; \
stp q22, q23, [ctx, #(16 * 22 + 0x80)]; \
stp q24, q25, [ctx, #(16 * 24 + 0x80)]; \
stp q26, q27, [ctx, #(16 * 26 + 0x80)]; \
stp q28, q29, [ctx, #(16 * 28 + 0x80)]; \
stp q30, q31, [ctx, #(16 * 30 + 0x80)];
#define RESTORE_THREAD_CONTEXT(ctx, tmp0, tmp1, done_label) \
/* Restore the callee save registers + SP and cpacr. */ \
ldp tmp0, tmp1, [ctx, #0x60]; \
mov sp, tmp0; \
ldp x19, x20, [ctx, #(8 * 0)]; \
ldp x21, x22, [ctx, #(8 * 2)]; \
ldp x23, x24, [ctx, #(8 * 4)]; \
ldp x25, x26, [ctx, #(8 * 6)]; \
ldp x27, x28, [ctx, #(8 * 8)]; \
ldp x29, x30, [ctx, #(8 * 10)]; \
\
msr cpacr_el1, tmp1; \
isb; \
\
/* Check whether the FPU is enabled. */ \
/* If it isn't, skip saving FPU state. */ \
and tmp1, tmp1, #0x300000; \
cbz tmp1, done_label; \
\
/* Save fpcr and fpsr. */ \
ldp tmp0, tmp1, [ctx, #0x70]; \
msr fpcr, tmp0; \
msr fpsr, tmp1; \
\
/* Save the FPU registers. */ \
ldp q0, q1, [ctx, #(16 * 0 + 0x80)]; \
ldp q2, q3, [ctx, #(16 * 2 + 0x80)]; \
ldp q4, q5, [ctx, #(16 * 4 + 0x80)]; \
ldp q6, q7, [ctx, #(16 * 6 + 0x80)]; \
ldp q8, q9, [ctx, #(16 * 8 + 0x80)]; \
ldp q10, q11, [ctx, #(16 * 10 + 0x80)]; \
ldp q12, q13, [ctx, #(16 * 12 + 0x80)]; \
ldp q14, q15, [ctx, #(16 * 14 + 0x80)]; \
ldp q16, q17, [ctx, #(16 * 16 + 0x80)]; \
ldp q18, q19, [ctx, #(16 * 18 + 0x80)]; \
ldp q20, q21, [ctx, #(16 * 20 + 0x80)]; \
ldp q22, q23, [ctx, #(16 * 22 + 0x80)]; \
ldp q24, q25, [ctx, #(16 * 24 + 0x80)]; \
ldp q26, q27, [ctx, #(16 * 26 + 0x80)]; \
ldp q28, q29, [ctx, #(16 * 28 + 0x80)]; \
ldp q30, q31, [ctx, #(16 * 30 + 0x80)];
/* ams::kern::KScheduler::ScheduleImpl() */
.section .text._ZN3ams4kern10KScheduler12ScheduleImplEv, "ax", %progbits
.global _ZN3ams4kern10KScheduler12ScheduleImplEv
.type _ZN3ams4kern10KScheduler12ScheduleImplEv, %function
/* Ensure ScheduleImpl is aligned to 0x40 bytes. */
.balign 0x40
_ZN3ams4kern10KScheduler12ScheduleImplEv:
/* Right now, x0 contains (this). We want x1 to point to the scheduling state, */
/* Current KScheduler layout has state at +0x0. */
mov x1, x0
/* First thing we want to do is check whether the interrupt task thread is runnable. */
ldrb w3, [x1, #1]
cbz w3, 0f
/* If it is, we want to call KScheduler::SetInterruptTaskThreadRunnable() to note it runnable. */
stp x0, x1, [sp, #-16]!
stp x30, xzr, [sp, #-16]!
bl _ZN3ams4kern10KScheduler30SetInterruptTaskThreadRunnableEv
ldp x30, xzr, [sp], 16
ldp x0, x1, [sp], 16
/* Clear the interrupt task thread as runnable. */
strb wzr, [x1, #1]
0: /* Interrupt task thread runnable checked. */
/* Now we want to check if there's any scheduling to do. */
/* First, clear the need's scheduling bool (and dmb ish after, as it's an atomic). */
/* TODO: Should this be a stlrb? Nintendo does not do one. */
strb wzr, [x1]
dmb ish
/* Check if the highest priority thread is the same as the current thread. */
ldr x7, [x1, 16]
ldr x2, [x18]
cmp x7, x2
b.ne 1f
/* If they're the same, then we can just return as there's nothing to do. */
ret
1: /* The highest priority thread is not the same as the current thread. */
/* Get a reference to the current thread's stack parameters. */
add x2, sp, #0x1000
and x2, x2, #~(0x1000-1)
/* Check if the thread has terminated. We can do this by checking the DPC flags for DpcFlag_Terminated. */
ldurb w3, [x2, #-0x20]
tbnz w3, #1, 3f
/* The current thread hasn't terminated, so we want to save its context. */
ldur x2, [x2, #-0x10]
SAVE_THREAD_CONTEXT(x2, x4, x5, 2f)
2: /* We're done saving this thread's context, so we need to unlock it. */
/* We can just do an atomic write to the relevant KThreadContext member. */
add x2, x2, #0x280
stlrb wzr, [x2]
3: /* The current thread's context has been entirely taken care of. */
/* Now we want to loop until we successfully switch the thread context. */
/* Start by saving all the values we care about in callee-save registers. */
mov x19, x0 /* this */
mov x20, x1 /* this->state */
mov x21, x7 /* highest priority thread */
/* Set our stack to the idle thread stack. */
ldr x3, [x20, #0x18]
mov sp, x3
b 5f
4: /* We failed to successfully do the context switch, and need to retry. */
/* Clear the exclusive monitor. */
clrex
/* Clear the need's scheduling bool (and dmb ish after, as it's an atomic). */
/* TODO: Should this be a stlrb? Nintendo does not do one. */
strb wzr, [x20]
dmb ish
/* Refresh the highest priority thread. */
ldr x21, [x20, 16]
5: /* We're starting to try to do the context switch. */
/* Check if the highest priority thread if null. */
/* If it is, we want to branch to a special idle thread loop. */
cbz x21, 11f
/* Get the highest priority thread's context, and save it. */
/* ams::kern::KThread::GetContextForSchedulerLoop() */
mov x0, x21
bl _ZN3ams4kern7KThread26GetContextForSchedulerLoopEv
mov x22, x0
/* Prepare to try to acquire the context lock. */
add x1, x22, #0x280
mov w2, #1
6: /* We want to try to lock the highest priority thread's context. */
/* Check if the lock is already held. */
ldaxrb w3, [x1]
cbnz w3, 7f
/* If it's not, try to take it. */
stxrb w3, w2, [x1]
cbnz w3, 6b
/* We hold the lock, so we can now switch the thread. */
b 8f
7: /* The highest priority thread's context is already locked. */
/* Check if we need scheduling. If we don't, we can retry directly. */
ldarb w3, [x20]
cbz w3, 6b
/* If we do, another core is interfering, and we must start from the top. */
b 4b
8: /* It's time to switch the thread. */
/* Switch to the highest priority thread. */
mov x0, x19
mov x1, x21
/* Call ams::kern::KScheduler::SwitchThread(ams::kern::KThread *) */
bl _ZN3ams4kern10KScheduler12SwitchThreadEPNS0_7KThreadE
/* Check if we need scheduling. If we don't, then we can't complete the switch and should retry. */
ldarb w1, [x20]
cbnz w1, 10f
/* Restore the thread context. */
mov x0, x22
RESTORE_THREAD_CONTEXT(x0, x1, x2, 9f)
9: /* We're done restoring the thread context, and can return safely. */
ret
10: /* Our switch failed. */
/* We should unlock the thread context, and then retry. */
add x1, x22, #0x280
stlrb wzr, [x1]
b 4b
11: /* The next thread is nullptr! */
/* Switch to nullptr. This will actually switch to the idle thread. */
mov x0, x19
mov x1, #0
/* Call ams::kern::KScheduler::SwitchThread(ams::kern::KThread *) */
bl _ZN3ams4kern10KScheduler12SwitchThreadEPNS0_7KThreadE
12: /* We've switched to the idle thread, so we want to loop until we schedule a non-idle thread. */
/* Check if we need scheduling. */
ldarb w3, [x20]
cbnz w3, 13f
/* If we don't, wait for an interrupt and check again. */
wfi
msr daifclr, #2
msr daifset, #2
b 12b
13: /* We need scheduling again! */
/* Check whether the interrupt task thread needs to be set runnable. */
ldrb w3, [x20, #1]
cbz w3, 4b
/* It does, so do so. We're using the idle thread stack so no register state preserve needed. */
bl _ZN3ams4kern10KScheduler30SetInterruptTaskThreadRunnableEv
/* Clear the interrupt task thread as runnable. */
strb wzr, [x20, #1]
/* Retry the scheduling loop. */
b 4b

143
libraries/libmesosphere/source/arch/arm64/kern_k_thread_context_asm.s
View file

@ -1,143 +0,0 @@
/*
* Copyright (c) 2018-2020 Atmosphère-NX
*
* This program is free software; you can redistribute it and/or modify it
* under the terms and conditions of the GNU General Public License,
* version 2, as published by the Free Software Foundation.
*
* This program is distributed in the hope it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
/* ams::kern::arm64::UserModeThreadStarter() */
.section .text._ZN3ams4kern5arm6421UserModeThreadStarterEv, "ax", %progbits
.global _ZN3ams4kern5arm6421UserModeThreadStarterEv
.type _ZN3ams4kern5arm6421UserModeThreadStarterEv, %function
_ZN3ams4kern5arm6421UserModeThreadStarterEv:
/* NOTE: Stack layout on entry looks like following: */
/* SP */
/* | */
/* v */
/* | KExceptionContext (size 0x120) | KThread::StackParameters (size 0x30) | */
/* Clear the disable count for this thread's stack parameters. */
str wzr, [sp, #(0x120 + 0x18)]
/* Call ams::kern::arm64::OnThreadStart() */
bl _ZN3ams4kern5arm6413OnThreadStartEv
/* Restore thread state from the KExceptionContext on stack */
ldp x30, x19, [sp, #(8 * 30)] /* x30 = lr, x19 = sp */
ldp x20, x21, [sp, #(8 * 30 + 16)] /* x20 = pc, x21 = psr */
ldr x22, [sp, #(8 * 30 + 32)] /* x22 = tpidr */
msr sp_el0, x19
msr elr_el1, x20
msr spsr_el1, x21
msr tpidr_el1, x22
ldp x0, x1, [sp, #(8 * 0)]
ldp x2, x3, [sp, #(8 * 2)]
ldp x4, x5, [sp, #(8 * 4)]
ldp x6, x7, [sp, #(8 * 6)]
ldp x8, x9, [sp, #(8 * 8)]
ldp x10, x11, [sp, #(8 * 10)]
ldp x12, x13, [sp, #(8 * 12)]
ldp x14, x15, [sp, #(8 * 14)]
ldp x16, x17, [sp, #(8 * 16)]
ldp x18, x19, [sp, #(8 * 18)]
ldp x20, x21, [sp, #(8 * 20)]
ldp x22, x23, [sp, #(8 * 22)]
ldp x24, x25, [sp, #(8 * 24)]
ldp x26, x27, [sp, #(8 * 26)]
ldp x28, x29, [sp, #(8 * 28)]
/* Increment stack pointer above the KExceptionContext */
add sp, sp, #0x120
/* Return to EL0 */
eret
/* ams::kern::arm64::SupervisorModeThreadStarter() */
.section .text._ZN3ams4kern5arm6427SupervisorModeThreadStarterEv, "ax", %progbits
.global _ZN3ams4kern5arm6427SupervisorModeThreadStarterEv
.type _ZN3ams4kern5arm6427SupervisorModeThreadStarterEv, %function
_ZN3ams4kern5arm6427SupervisorModeThreadStarterEv:
/* NOTE: Stack layout on entry looks like following: */
/* SP */
/* | */
/* v */
/* | u64 argument | u64 entrypoint | KThread::StackParameters (size 0x30) | */
/* Load the argument and entrypoint. */
ldp x0, x1, [sp], #0x10
/* Clear the disable count for this thread's stack parameters. */
str wzr, [sp, #(0x18)]
/* Mask I bit in DAIF */
msr daifclr, #2
br x1
/* This should never execute, but Nintendo includes an ERET here. */
eret
/* ams::kern::arm64::KThreadContext::RestoreFpuRegisters64(const KThreadContext &) */
.section .text._ZN3ams4kern5arm6414KThreadContext21RestoreFpuRegisters64ERKS2_, "ax", %progbits
.global _ZN3ams4kern5arm6414KThreadContext21RestoreFpuRegisters64ERKS2_
.type _ZN3ams4kern5arm6414KThreadContext21RestoreFpuRegisters64ERKS2_, %function
_ZN3ams4kern5arm6414KThreadContext21RestoreFpuRegisters64ERKS2_:
/* Load and restore FPCR and FPSR from the context. */
ldr x1, [x0, #0x70]
msr fpcr, x1
ldr x1, [x0, #0x78]
msr fpsr, x1
/* Restore the FPU registers. */
ldp q0, q1, [sp, #(16 * 0 + 0x80)]
ldp q2, q3, [sp, #(16 * 2 + 0x80)]
ldp q4, q5, [sp, #(16 * 4 + 0x80)]
ldp q6, q7, [sp, #(16 * 6 + 0x80)]
ldp q8, q9, [sp, #(16 * 8 + 0x80)]
ldp q10, q11, [sp, #(16 * 10 + 0x80)]
ldp q12, q13, [sp, #(16 * 12 + 0x80)]
ldp q14, q15, [sp, #(16 * 14 + 0x80)]
ldp q16, q17, [sp, #(16 * 16 + 0x80)]
ldp q18, q19, [sp, #(16 * 18 + 0x80)]
ldp q20, q21, [sp, #(16 * 20 + 0x80)]
ldp q22, q23, [sp, #(16 * 22 + 0x80)]
ldp q24, q25, [sp, #(16 * 24 + 0x80)]
ldp q26, q27, [sp, #(16 * 26 + 0x80)]
ldp q28, q29, [sp, #(16 * 28 + 0x80)]
ldp q30, q31, [sp, #(16 * 30 + 0x80)]
ret
/* ams::kern::arm64::KThreadContext::RestoreFpuRegisters32(const KThreadContext &) */
.section .text._ZN3ams4kern5arm6414KThreadContext21RestoreFpuRegisters32ERKS2_, "ax", %progbits
.global _ZN3ams4kern5arm6414KThreadContext21RestoreFpuRegisters32ERKS2_
.type _ZN3ams4kern5arm6414KThreadContext21RestoreFpuRegisters32ERKS2_, %function
_ZN3ams4kern5arm6414KThreadContext21RestoreFpuRegisters32ERKS2_:
/* Load and restore FPCR and FPSR from the context. */
ldr x1, [x0, #0x70]
msr fpcr, x1
ldr x1, [x0, #0x78]
msr fpsr, x1
/* Restore the FPU registers. */
ldp q0, q1, [sp, #(16 * 0 + 0x80)]
ldp q2, q3, [sp, #(16 * 2 + 0x80)]
ldp q4, q5, [sp, #(16 * 4 + 0x80)]
ldp q6, q7, [sp, #(16 * 6 + 0x80)]
ldp q8, q9, [sp, #(16 * 8 + 0x80)]
ldp q10, q11, [sp, #(16 * 10 + 0x80)]
ldp q12, q13, [sp, #(16 * 12 + 0x80)]
ldp q14, q15, [sp, #(16 * 14 + 0x80)]
ret

35
libraries/libmesosphere/source/arch/arm64/kern_userspace_memory_access_asm.s
View file

@ -1,35 +0,0 @@
/*
* Copyright (c) 2018-2020 Atmosphère-NX
*
* This program is free software; you can redistribute it and/or modify it
* under the terms and conditions of the GNU General Public License,
* version 2, as published by the Free Software Foundation.
*
* This program is distributed in the hope it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
/* ams::kern::arm64::UserspaceMemoryAccessFunctionAreaBegin() */
.section .text._ZN3ams4kern5arm6438UserspaceMemoryAccessFunctionAreaBeginEv, "ax", %progbits
.global _ZN3ams4kern5arm6438UserspaceMemoryAccessFunctionAreaBeginEv
.type _ZN3ams4kern5arm6438UserspaceMemoryAccessFunctionAreaBeginEv, %function
_ZN3ams4kern5arm6438UserspaceMemoryAccessFunctionAreaBeginEv:
/* NOTE: This is not a real function, and only exists as a label for safety. */
/* ================ All Userspace Memory Functions after this line. ================ */
/* TODO */
/* ================ All Userspace Memory Functions before this line. ================ */
/* ams::kern::arm64::UserspaceMemoryAccessFunctionAreaEnd() */
.section .text._ZN3ams4kern5arm6436UserspaceMemoryAccessFunctionAreaEndEv, "ax", %progbits
.global _ZN3ams4kern5arm6436UserspaceMemoryAccessFunctionAreaEndEv
.type _ZN3ams4kern5arm6436UserspaceMemoryAccessFunctionAreaEndEv, %function
_ZN3ams4kern5arm6436UserspaceMemoryAccessFunctionAreaEndEv:
/* NOTE: This is not a real function, and only exists as a label for safety. */

5
libraries/libmesosphere/source/board/nintendo/switch/kern_debug_log_impl.cpp
View file

@ -28,6 +28,8 @@ namespace ams::kern {
UartRegister_LSR = 5,
UartRegister_IRSA_CSR = 8,
UartRegister_DLL = 0,
UartRegister_DLH = 1,
};
@ -71,8 +73,9 @@ namespace ams::kern {
/* Disable UART interrupts. */
WriteUartRegister(UartRegister_IER, 0x00);
/* Configure the FIFOO to be enabled and clear receive. */
/* Configure the FIFO to be enabled and clear receive. */
WriteUartRegister(UartRegister_FCR, 0x03);
WriteUartRegister(UartRegister_IRSA_CSR, 0x02);
ReadUartRegister(UartRegister_FCR);
return true;

2
libraries/libmesosphere/source/board/nintendo/switch/kern_k_system_control.cpp
View file

@ -260,7 +260,7 @@ namespace ams::kern {
void KSystemControl::StopSystem() {
if (g_call_smc_on_panic) {
/* Display a panic screen via secure monitor. */
smc::Panic(0xF00);
/* TODO: Enable in release: smc::Panic(0xF00); */
}
while (true) { /* ... */ }
}

6
libraries/libmesosphere/source/board/nintendo/switch/kern_secure_monitor.cpp
View file

@ -116,7 +116,7 @@ namespace ams::kern::smc {
void GetConfig(u64 *out, size_t num_qwords, ConfigItem config_item) {
SecureMonitorArguments args = { FunctionId_GetConfig, static_cast<u32>(config_item) };
CallPrivilegedSecureMonitorFunctionForInit(args);
MESOSPHERE_ABORT_UNLESS((static_cast<SmcResult>(args.x[0]) == SmcResult::Success));
MESOSPHERE_INIT_ABORT_UNLESS((static_cast<SmcResult>(args.x[0]) == SmcResult::Success));
for (size_t i = 0; i < num_qwords && i < 7; i++) {
out[i] = args.x[1 + i];
}
@ -125,10 +125,10 @@ namespace ams::kern::smc {
void GenerateRandomBytes(void *dst, size_t size) {
/* Call SmcGenerateRandomBytes() */
SecureMonitorArguments args = { FunctionId_GenerateRandomBytes, size };
MESOSPHERE_ABORT_UNLESS(size <= sizeof(args) - sizeof(args.x[0]));
MESOSPHERE_INIT_ABORT_UNLESS(size <= sizeof(args) - sizeof(args.x[0]));
CallPrivilegedSecureMonitorFunctionForInit(args);
MESOSPHERE_ABORT_UNLESS((static_cast<SmcResult>(args.x[0]) == SmcResult::Success));
MESOSPHERE_INIT_ABORT_UNLESS((static_cast<SmcResult>(args.x[0]) == SmcResult::Success));
/* Copy output. */
std::memcpy(dst, &args.x[1], size);

4
libraries/libmesosphere/source/kern_k_scheduler.cpp
View file

@ -17,6 +17,10 @@
namespace ams::kern {
bool KScheduler::s_scheduler_update_needed;
KScheduler::LockType KScheduler::s_scheduler_lock;
KSchedulerPriorityQueue KScheduler::s_priority_queue;
namespace {
class KSchedulerInterruptTask : public KInterruptTask {

12
libraries/libmesosphere/source/kern_kernel.cpp
View file

@ -17,18 +17,6 @@
namespace ams::kern {
/* Declare kernel data members in kernel TU. */
Kernel::State Kernel::s_state = Kernel::State::Invalid;
KThread Kernel::s_main_threads[cpu::NumCores];
KThread Kernel::s_idle_threads[cpu::NumCores];
KResourceLimit Kernel::s_system_resource_limit;
KMemoryManager Kernel::s_memory_manager;
KPageTableManager Kernel::s_page_table_manager;
KMemoryBlockSlabManager Kernel::s_app_memory_block_manager;
KMemoryBlockSlabManager Kernel::s_sys_memory_block_manager;
KBlockInfoManager Kernel::s_block_info_manager;
KSupervisorPageTable Kernel::s_supervisor_page_table;
namespace {
template<typename T>