mirrors/Atmosphere
1
0
Fork 0
mirror of https://github.com/Atmosphere-NX/Atmosphere.git synced 2025-05-31 23:08:22 -04:00
Code Issues Projects Releases Packages Wiki Activity

kern: implement new thread context/fpu semantics

Browse source
This commit is contained in:
Michael Scire 2022-03-22 19:59:39 -07:00 • committed by SciresM
parent 79afa3b64c
commit 6e17317d5d
21 changed files with 1291 additions and 668 deletions
Download patch file Download diff file Expand all files Collapse all files

12
libraries/libmesosphere/source/arch/arm64/kern_exception_handlers.cpp
View file

@ -214,16 +214,16 @@ namespace ams::kern::arch::arm64 {
context->psr = 0x10;
}
/* Set exception SVC permissions. */
cur_process.CopyEnterExceptionSvcPermissionsTo(GetCurrentThread().GetStackParametersForExceptionSvcPermission());
/* Process that we're entering a usermode exception on the current thread. */
GetCurrentThread().OnEnterUsermodeException();
return;
}
}
/* If we should, clear the thread's state as single-step. */
#if defined(MESOSPHERE_ENABLE_HARDWARE_SINGLE_STEP)
if (AMS_UNLIKELY(GetCurrentThread().IsSingleStep())) {
GetCurrentThread().ClearSingleStep();
if (AMS_UNLIKELY(GetCurrentThread().IsHardwareSingleStep())) {
GetCurrentThread().ClearHardwareSingleStep();
cpu::MonitorDebugSystemControlRegisterAccessor().SetSoftwareStep(false).Store();
cpu::InstructionMemoryBarrier();
}
@ -388,8 +388,8 @@ namespace ams::kern::arch::arm64 {
/* Try to leave the user exception. */
if (cur_process.LeaveUserException()) {
/* We left user exception. Alter our SVC permissions accordingly. */
cur_process.CopyLeaveExceptionSvcPermissionsTo(cur_thread->GetStackParametersForExceptionSvcPermission());
/* Process that we're leaving a usermode exception on the current thread. */
GetCurrentThread().OnLeaveUsermodeException();
/* Copy the user context to the thread context. */
if (is_aarch64) {

20
libraries/libmesosphere/source/arch/arm64/kern_k_debug.cpp
View file

@ -202,16 +202,13 @@ namespace ams::kern::arch::arm64 {
/* Get the FPU registers, if required. */
if ((context_flags & ams::svc::ThreadContextFlag_Fpu) != 0) {
static_assert(util::size(ams::svc::ThreadContext{}.v) == KThreadContext::NumFpuRegisters);
const u128 *f = t_ctx->GetFpuRegisters();
const auto &caller_save = thread->GetCallerSaveFpuRegisters();
const auto &callee_save = t_ctx->GetCalleeSaveFpuRegisters();
if (this->Is64Bit()) {
for (size_t i = 0; i < KThreadContext::NumFpuRegisters; ++i) {
out->v[i] = f[i];
}
KThreadContext::GetFpuRegisters(out->v, caller_save.fpu64, callee_save.fpu64);
} else {
for (size_t i = 0; i < KThreadContext::NumFpuRegisters / 2; ++i) {
out->v[i] = f[i];
}
KThreadContext::GetFpuRegisters(out->v, caller_save.fpu32, callee_save.fpu32);
for (size_t i = KThreadContext::NumFpuRegisters / 2; i < KThreadContext::NumFpuRegisters; ++i) {
out->v[i] = 0;
}
@ -240,7 +237,14 @@ namespace ams::kern::arch::arm64 {
/* Set the FPU registers, if required. */
if ((context_flags & ams::svc::ThreadContextFlag_Fpu) != 0) {
static_assert(util::size(ams::svc::ThreadContext{}.v) == KThreadContext::NumFpuRegisters);
t_ctx->SetFpuRegisters(ctx.v, this->Is64Bit());
auto &caller_save = thread->GetCallerSaveFpuRegisters();
auto &callee_save = t_ctx->GetCalleeSaveFpuRegisters();
if (this->Is64Bit()) {
KThreadContext::SetFpuRegisters(caller_save.fpu64, callee_save.fpu64, ctx.v);
} else {
KThreadContext::SetFpuRegisters(caller_save.fpu32, callee_save.fpu32, ctx.v);
}
}
R_SUCCEED();

36
libraries/libmesosphere/source/arch/arm64/kern_k_interrupt_manager.cpp
View file

@ -178,19 +178,37 @@ namespace ams::kern::arch::arm64 {
/* If we need scheduling, */
if (needs_scheduling) {
/* If the user disable count is set, we may need to pin the current thread. */
if (user_mode && GetCurrentThread().GetUserDisableCount() != 0 && GetCurrentProcess().GetPinnedThread(GetCurrentCoreId()) == nullptr) {
KScopedSchedulerLock sl;
if (user_mode) {
/* If the interrupt occurred in the middle of a userland cache maintenance operation, ensure memory consistency before rescheduling. */
if (GetCurrentThread().IsInUserCacheMaintenanceOperation()) {
cpu::DataSynchronizationBarrier();
}
/* Pin the current thread. */
GetCurrentProcess().PinCurrentThread();
/* If the user disable count is set, we may need to pin the current thread. */
if (GetCurrentThread().GetUserDisableCount() != 0 && GetCurrentProcess().GetPinnedThread(GetCurrentCoreId()) == nullptr) {
KScopedSchedulerLock sl;
/* Set the interrupt flag for the thread. */
GetCurrentThread().SetInterruptFlag();
/* Pin the current thread. */
GetCurrentProcess().PinCurrentThread();
/* Request interrupt scheduling. */
Kernel::GetScheduler().RequestScheduleOnInterrupt();
/* Set the interrupt flag for the thread. */
GetCurrentThread().SetInterruptFlag();
/* Request interrupt scheduling. */
Kernel::GetScheduler().RequestScheduleOnInterrupt();
} else {
/* Request interrupt scheduling. */
Kernel::GetScheduler().RequestScheduleOnInterrupt();
}
} else {
/* If the interrupt occurred in the middle of a cache maintenance operation, ensure memory consistency before rescheduling. */
if (GetCurrentThread().IsInCacheMaintenanceOperation()) {
cpu::DataSynchronizationBarrier();
} else if (GetCurrentThread().IsInTlbMaintenanceOperation()) {
/* Otherwise, if we're in the middle of a tlb maintenance operation, ensure inner shareable memory consistency before rescheduling. */
cpu::DataSynchronizationBarrierInnerShareable();
}
/* Request interrupt scheduling. */
Kernel::GetScheduler().RequestScheduleOnInterrupt();
}

95
libraries/libmesosphere/source/arch/arm64/kern_k_thread_context.cpp
View file

@ -52,11 +52,11 @@ namespace ams::kern::arch::arm64 {
}
uintptr_t SetupStackForUserModeThreadStarter(KVirtualAddress pc, KVirtualAddress k_sp, KVirtualAddress u_sp, uintptr_t arg, const bool is_64_bit) {
/* NOTE: Stack layout on entry looks like following: */
/* SP */
/* | */
/* v */
/* | KExceptionContext (size 0x120) | KThread::StackParameters (size 0x30) | */
/* NOTE: Stack layout on entry looks like following: */
/* SP */
/* | */
/* v */
/* | KExceptionContext (size 0x120) | KThread::StackParameters (size 0x130) | */
KExceptionContext *ctx = GetPointer<KExceptionContext>(k_sp) - 1;
/* Clear context. */
@ -92,12 +92,12 @@ namespace ams::kern::arch::arm64 {
}
uintptr_t SetupStackForSupervisorModeThreadStarter(KVirtualAddress pc, KVirtualAddress sp, uintptr_t arg) {
/* NOTE: Stack layout on entry looks like following: */
/* SP */
/* | */
/* v */
/* | u64 argument | u64 entrypoint | KThread::StackParameters (size 0x30) | */
static_assert(sizeof(KThread::StackParameters) == 0x30);
/* NOTE: Stack layout on entry looks like following: */
/* SP */
/* | */
/* v */
/* | u64 argument | u64 entrypoint | KThread::StackParameters (size 0x130) | */
static_assert(sizeof(KThread::StackParameters) == 0x130);
u64 *stack = GetPointer<u64>(sp);
*(--stack) = GetInteger(pc);
@ -142,9 +142,8 @@ namespace ams::kern::arch::arm64 {
/* Clear FPU state. */
m_fpcr = 0;
m_fpsr = 0;
m_cpacr = 0;
for (size_t i = 0; i < util::size(m_fpu_registers); i++) {
m_fpu_registers[i] = 0;
for (size_t i = 0; i < util::size(m_callee_saved_fpu.fpu64.v); ++i) {
m_callee_saved_fpu.fpu64.v[i] = 0;
}
/* Lock the context, if we're a main thread. */
@ -153,37 +152,18 @@ namespace ams::kern::arch::arm64 {
R_SUCCEED();
}
Result KThreadContext::Finalize() {
/* This doesn't actually do anything. */
R_SUCCEED();
}
void KThreadContext::SetArguments(uintptr_t arg0, uintptr_t arg1) {
u64 *stack = reinterpret_cast<u64 *>(m_sp);
stack[0] = arg0;
stack[1] = arg1;
}
void KThreadContext::FpuContextSwitchHandler(KThread *thread) {
MESOSPHERE_ASSERT(!KInterruptManager::AreInterruptsEnabled());
MESOSPHERE_ASSERT(!IsFpuEnabled());
/* Enable the FPU. */
EnableFpu();
/* Restore the FPU registers. */
KProcess *process = thread->GetOwnerProcess();
MESOSPHERE_ASSERT(process != nullptr);
if (process->Is64Bit()) {
RestoreFpuRegisters64(thread->GetContext());
} else {
RestoreFpuRegisters32(thread->GetContext());
}
}
void KThreadContext::CloneFpuStatus() {
u64 pcr, psr;
cpu::InstructionMemoryBarrier();
KScopedInterruptDisable di;
if (IsFpuEnabled()) {
__asm__ __volatile__("mrs %[pcr], fpcr" : [pcr]"=r"(pcr) :: "memory");
__asm__ __volatile__("mrs %[psr], fpsr" : [psr]"=r"(psr) :: "memory");
@ -196,18 +176,6 @@ namespace ams::kern::arch::arm64 {
this->SetFpsr(psr);
}
void KThreadContext::SetFpuRegisters(const u128 *v, bool is_64_bit) {
if (is_64_bit) {
for (size_t i = 0; i < KThreadContext::NumFpuRegisters; ++i) {
m_fpu_registers[i] = v[i];
}
} else {
for (size_t i = 0; i < KThreadContext::NumFpuRegisters / 2; ++i) {
m_fpu_registers[i] = v[i];
}
}
}
void GetUserContext(ams::svc::ThreadContext *out, const KThread *thread) {
MESOSPHERE_ASSERT(KScheduler::IsSchedulerLockedByCurrentThread());
MESOSPHERE_ASSERT(thread->IsSuspended());
@ -244,9 +212,17 @@ namespace ams::kern::arch::arm64 {
/* Copy fpu registers. */
static_assert(util::size(ams::svc::ThreadContext{}.v) == KThreadContext::NumFpuRegisters);
const u128 *f = t_ctx->GetFpuRegisters();
for (size_t i = 0; i < KThreadContext::NumFpuRegisters; ++i) {
out->v[i] = f[i];
static_assert(KThreadContext::NumCallerSavedFpuRegisters == KThreadContext::NumCalleeSavedFpuRegisters * 3);
static_assert(KThreadContext::NumFpuRegisters == KThreadContext::NumCallerSavedFpuRegisters + KThreadContext::NumCalleeSavedFpuRegisters);
const auto &caller_save_fpu = thread->GetCallerSaveFpuRegisters().fpu64;
const auto &callee_save_fpu = t_ctx->GetCalleeSaveFpuRegisters().fpu64;
if (!thread->IsCallingSvc() || thread->IsInUsermodeExceptionHandler()) {
KThreadContext::GetFpuRegisters(out->v, caller_save_fpu, callee_save_fpu);
} else {
for (size_t i = 0; i < KThreadContext::NumCalleeSavedFpuRegisters; ++i) {
out->v[(KThreadContext::NumCallerSavedFpuRegisters / 3) + i] = caller_save_fpu.v[i];
}
}
} else {
/* Set special registers. */
@ -271,12 +247,17 @@ namespace ams::kern::arch::arm64 {
/* Copy fpu registers. */
static_assert(util::size(ams::svc::ThreadContext{}.v) == KThreadContext::NumFpuRegisters);
const u128 *f = t_ctx->GetFpuRegisters();
for (size_t i = 0; i < KThreadContext::NumFpuRegisters / 2; ++i) {
out->v[i] = f[i];
}
for (size_t i = KThreadContext::NumFpuRegisters / 2; i < KThreadContext::NumFpuRegisters; ++i) {
out->v[i] = 0;
static_assert(KThreadContext::NumCallerSavedFpuRegisters == KThreadContext::NumCalleeSavedFpuRegisters * 3);
static_assert(KThreadContext::NumFpuRegisters == KThreadContext::NumCallerSavedFpuRegisters + KThreadContext::NumCalleeSavedFpuRegisters);
const auto &caller_save_fpu = thread->GetCallerSaveFpuRegisters().fpu32;
const auto &callee_save_fpu = t_ctx->GetCalleeSaveFpuRegisters().fpu32;
if (!thread->IsCallingSvc() || thread->IsInUsermodeExceptionHandler()) {
KThreadContext::GetFpuRegisters(out->v, caller_save_fpu, callee_save_fpu);
} else {
for (size_t i = 0; i < KThreadContext::NumCalleeSavedFpuRegisters / 2; ++i) {
out->v[((KThreadContext::NumCallerSavedFpuRegisters / 3) / 2) + i] = caller_save_fpu.v[i];
}
}
}

18
libraries/libmesosphere/source/arch/arm64/svc/kern_svc_exception_asm.s
View file

@ -13,7 +13,7 @@
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include <mesosphere/kern_select_assembly_offsets.h>
#include <mesosphere/kern_select_assembly_macros.h>
/* ams::kern::svc::CallReturnFromException64(Result result) */
.section .text._ZN3ams4kern3svc25CallReturnFromException64Ev, "ax", %progbits
@ -82,8 +82,20 @@ _ZN3ams4kern3svc14RestoreContextEm:
b 0b
1: /* We're done with DPC, and should return from the svc. */
/* Clear our in-SVC note. */
strb wzr, [sp, #(EXCEPTION_CONTEXT_SIZE + THREAD_STACK_PARAMETERS_IS_CALLING_SVC)]
/* Get our exception flags. */
ldrb w9, [sp, #(EXCEPTION_CONTEXT_SIZE + THREAD_STACK_PARAMETERS_EXCEPTION_FLAGS)]
/* Clear in-svc and needs-fpu-restore flags. */
and w10, w9, #(~(THREAD_EXCEPTION_FLAG_IS_FPU_CONTEXT_RESTORE_NEEDED))
and w10, w10, #(~(THREAD_EXCEPTION_FLAG_IS_CALLING_SVC))
strb w10, [sp, #(EXCEPTION_CONTEXT_SIZE + THREAD_STACK_PARAMETERS_EXCEPTION_FLAGS)]
/* If we don't need to restore the fpu, skip restoring it. */
tbz w9, #(THREAD_EXCEPTION_FLAG_BIT_INDEX_IS_FPU_CONTEXT_RESTORE_NEEDED), 3f
/* Enable and restore the fpu. */
ENABLE_AND_RESTORE_FPU(x10, x8, x9, w8, w9, 2, 3)
/* Restore registers. */
ldp x30, x8, [sp, #(EXCEPTION_CONTEXT_X30_SP)]

229
libraries/libmesosphere/source/arch/arm64/svc/kern_svc_handlers_asm.s
View file

@ -14,7 +14,7 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include <mesosphere/kern_build_config.hpp>
#include <mesosphere/kern_select_assembly_offsets.h>
#include <mesosphere/kern_select_assembly_macros.h>
/* ams::kern::arch::arm64::SvcHandler64() */
.section .text._ZN3ams4kern4arch5arm6412SvcHandler64Ev, "ax", %progbits
@ -81,9 +81,10 @@ _ZN3ams4kern4arch5arm6412SvcHandler64Ev:
cbz x11, 3f
/* Note that we're calling the SVC. */
mov w10, #1
strb w10, [sp, #(EXCEPTION_CONTEXT_SIZE + THREAD_STACK_PARAMETERS_IS_CALLING_SVC)]
strb w8, [sp, #(EXCEPTION_CONTEXT_SIZE + THREAD_STACK_PARAMETERS_CURRENT_SVC_ID)]
ldrb w9, [sp, #(EXCEPTION_CONTEXT_SIZE + THREAD_STACK_PARAMETERS_EXCEPTION_FLAGS)]
orr w9, w9, #(THREAD_EXCEPTION_FLAG_IS_CALLING_SVC)
strb w9, [sp, #(EXCEPTION_CONTEXT_SIZE + THREAD_STACK_PARAMETERS_EXCEPTION_FLAGS)]
strb w8, [sp, #(EXCEPTION_CONTEXT_SIZE + THREAD_STACK_PARAMETERS_CURRENT_SVC_ID)]
/* If we should, trace the svc entry. */
#if defined(MESOSPHERE_BUILD_FOR_TRACING)
@ -111,7 +112,7 @@ _ZN3ams4kern4arch5arm6412SvcHandler64Ev:
2: /* We completed the SVC, and we should handle DPC. */
/* Check the dpc flags. */
ldrb w8, [sp, #(EXCEPTION_CONTEXT_SIZE + THREAD_STACK_PARAMETERS_DPC_FLAGS)]
cbz w8, 4f
cbz w8, 5f
/* We have DPC to do! */
/* Save registers and call ams::kern::KDpcManager::HandleDpc(). */
@ -150,7 +151,18 @@ _ZN3ams4kern4arch5arm6412SvcHandler64Ev:
mov x0, sp
bl _ZN3ams4kern4arch5arm6415HandleExceptionEPNS2_17KExceptionContextE
/* Restore registers. */
/* If we don't need to restore the fpu, skip restoring it. */
ldrb w9, [sp, #(EXCEPTION_CONTEXT_SIZE + THREAD_STACK_PARAMETERS_EXCEPTION_FLAGS)]
tbz w9, #(THREAD_EXCEPTION_FLAG_BIT_INDEX_IS_FPU_CONTEXT_RESTORE_NEEDED), 4f
/* Clear the needs-fpu-restore flag. */
and w9, w9, #(~THREAD_EXCEPTION_FLAG_IS_FPU_CONTEXT_RESTORE_NEEDED)
strb w9, [sp, #(EXCEPTION_CONTEXT_SIZE + THREAD_STACK_PARAMETERS_EXCEPTION_FLAGS)]
/* Enable and restore the fpu. */
ENABLE_AND_RESTORE_FPU64(x10, x8, x9, w8, w9)
4: /* Restore registers. */
ldp x30, x8, [sp, #(EXCEPTION_CONTEXT_X30_SP)]
ldp x9, x10, [sp, #(EXCEPTION_CONTEXT_PC_PSR)]
ldr x11, [sp, #(EXCEPTION_CONTEXT_TPIDR)]
@ -184,9 +196,7 @@ _ZN3ams4kern4arch5arm6412SvcHandler64Ev:
add sp, sp, #(EXCEPTION_CONTEXT_SIZE)
eret
4: /* Return from SVC. */
/* Clear our in-SVC note. */
strb wzr, [sp, #(EXCEPTION_CONTEXT_SIZE + THREAD_STACK_PARAMETERS_IS_CALLING_SVC)]
5: /* Return from SVC. */
/* If we should, trace the svc exit. */
#if defined(MESOSPHERE_BUILD_FOR_TRACING)
@ -204,7 +214,60 @@ _ZN3ams4kern4arch5arm6412SvcHandler64Ev:
add sp, sp, #0x40
#endif
/* Restore registers. */
/* Get our exception flags. */
ldrb w9, [sp, #(EXCEPTION_CONTEXT_SIZE + THREAD_STACK_PARAMETERS_EXCEPTION_FLAGS)]
/* Clear in-svc and needs-fpu-restore flags. */
and w8, w9, #(~(THREAD_EXCEPTION_FLAG_IS_FPU_CONTEXT_RESTORE_NEEDED))
and w8, w8, #(~(THREAD_EXCEPTION_FLAG_IS_CALLING_SVC))
strb w8, [sp, #(EXCEPTION_CONTEXT_SIZE + THREAD_STACK_PARAMETERS_EXCEPTION_FLAGS)]
/* If we don't need to restore the fpu, skip restoring it. */
tbz w9, #(THREAD_EXCEPTION_FLAG_BIT_INDEX_IS_FPU_CONTEXT_RESTORE_NEEDED), 7f
/* If we need to restore the fpu, check if we need to do a full restore. */
tbnz w9, #(THREAD_EXCEPTION_FLAG_BIT_INDEX_IS_IN_USERMODE_EXCEPTION_HANDLER), 6f
/* Enable the fpu. */
ENABLE_FPU(x8)
/* Get the thread context and restore fpsr/fpcr. */
GET_THREAD_CONTEXT_AND_RESTORE_FPCR_FPSR(x10, x8, x9, w8, w9)
/* Restore callee-saved registers to 64-bit fpu. */
RESTORE_FPU64_CALLEE_SAVE_REGISTERS(x10)
/* Clear caller-saved registers to 64-bit fpu. */
movi v0.2d, #0
movi v1.2d, #0
movi v2.2d, #0
movi v3.2d, #0
movi v4.2d, #0
movi v5.2d, #0
movi v6.2d, #0
movi v7.2d, #0
movi v16.2d, #0
movi v17.2d, #0
movi v18.2d, #0
movi v19.2d, #0
movi v20.2d, #0
movi v21.2d, #0
movi v22.2d, #0
movi v23.2d, #0
movi v24.2d, #0
movi v25.2d, #0
movi v26.2d, #0
movi v27.2d, #0
movi v28.2d, #0
movi v29.2d, #0
movi v30.2d, #0
movi v31.2d, #0
b 7f
6: /* We need to do a full fpu restore. */
ENABLE_AND_RESTORE_FPU64(x10, x8, x9, w8, w9)
7: /* Restore registers. */
ldp x30, x8, [sp, #(EXCEPTION_CONTEXT_X30_SP)]
ldp x9, x10, [sp, #(EXCEPTION_CONTEXT_PC_PSR)]
ldr x11, [sp, #(EXCEPTION_CONTEXT_TPIDR)]
@ -273,38 +336,40 @@ _ZN3ams4kern4arch5arm6412SvcHandler32Ev:
stp x14, xzr, [sp, #(EXCEPTION_CONTEXT_X14_X15)]
/* Check if the SVC index is out of range. */
mrs x16, esr_el1
and x16, x16, #0xFF
cmp x16, #(AMS_KERN_NUM_SUPERVISOR_CALLS)
mrs x8, esr_el1
and x8, x8, #0xFF
cmp x8, #(AMS_KERN_NUM_SUPERVISOR_CALLS)
b.ge 3f
/* Check the specific SVC permission bit for allowal. */
mov x20, sp
add x20, x20, x16, lsr#3
ldrb w20, [x20, #(EXCEPTION_CONTEXT_SIZE + THREAD_STACK_PARAMETERS_SVC_PERMISSION)]
and x17, x16, #0x7
lsr x17, x20, x17
tst x17, #1
mov x9, sp
add x9, x9, x8, lsr#3
ldrb w9, [x9, #(EXCEPTION_CONTEXT_SIZE + THREAD_STACK_PARAMETERS_SVC_PERMISSION)]
and x10, x8, #0x7
lsr x10, x9, x10
tst x10, #1
b.eq 3f
/* Check if our disable count allows us to call SVCs. */
mrs x15, tpidrro_el0
ldrh w15, [x15, #(THREAD_LOCAL_REGION_DISABLE_COUNT)]
cbz w15, 1f
mrs x10, tpidrro_el0
ldrh w10, [x10, #(THREAD_LOCAL_REGION_DISABLE_COUNT)]
cbz w10, 1f
/* It might not, so check the stack params to see if we must not allow the SVC. */
ldrb w15, [sp, #(EXCEPTION_CONTEXT_SIZE + THREAD_STACK_PARAMETERS_IS_PINNED)]
cbz w15, 3f
ldrb w10, [sp, #(EXCEPTION_CONTEXT_SIZE + THREAD_STACK_PARAMETERS_IS_PINNED)]
cbz w10, 3f
1: /* We can call the SVC. */
adr x15, _ZN3ams4kern3svc16SvcTable64From32E
ldr x19, [x15, x16, lsl#3]
cbz x19, 3f
adr x10, _ZN3ams4kern3svc16SvcTable64From32E
ldr x11, [x10, x8, lsl#3]
cbz x11, 3f
/* Note that we're calling the SVC. */
mov w15, #1
strb w15, [sp, #(EXCEPTION_CONTEXT_SIZE + THREAD_STACK_PARAMETERS_IS_CALLING_SVC)]
strb w16, [sp, #(EXCEPTION_CONTEXT_SIZE + THREAD_STACK_PARAMETERS_CURRENT_SVC_ID)]
ldrb w9, [sp, #(EXCEPTION_CONTEXT_SIZE + THREAD_STACK_PARAMETERS_EXCEPTION_FLAGS)]
orr w9, w9, #(THREAD_EXCEPTION_FLAG_IS_CALLING_SVC)
strb w9, [sp, #(EXCEPTION_CONTEXT_SIZE + THREAD_STACK_PARAMETERS_EXCEPTION_FLAGS)]
strb w8, [sp, #(EXCEPTION_CONTEXT_SIZE + THREAD_STACK_PARAMETERS_CURRENT_SVC_ID)]
/* If we should, trace the svc entry. */
#if defined(MESOSPHERE_BUILD_FOR_TRACING)
@ -313,26 +378,26 @@ _ZN3ams4kern4arch5arm6412SvcHandler32Ev:
stp x2, x3, [sp, #(8 * 2)]
stp x4, x5, [sp, #(8 * 4)]
stp x6, x7, [sp, #(8 * 6)]
str x19, [sp, #(8 * 8)]
str x11, [sp, #(8 * 8)]
mov x0, sp
bl _ZN3ams4kern3svc13TraceSvcEntryEPKm
ldp x0, x1, [sp, #(8 * 0)]
ldp x2, x3, [sp, #(8 * 2)]
ldp x4, x5, [sp, #(8 * 4)]
ldp x6, x7, [sp, #(8 * 6)]
ldr x19, [sp, #(8 * 8)]
ldr x11, [sp, #(8 * 8)]
add sp, sp, #0x50
#endif
/* Invoke the SVC handler. */
msr daifclr, #2
blr x19
blr x11
msr daifset, #2
2: /* We completed the SVC, and we should handle DPC. */
/* Check the dpc flags. */
ldrb w16, [sp, #(EXCEPTION_CONTEXT_SIZE + THREAD_STACK_PARAMETERS_DPC_FLAGS)]
cbz w16, 4f
ldrb w8, [sp, #(EXCEPTION_CONTEXT_SIZE + THREAD_STACK_PARAMETERS_DPC_FLAGS)]
cbz w8, 5f
/* We have DPC to do! */
/* Save registers and call ams::kern::KDpcManager::HandleDpc(). */
@ -368,18 +433,29 @@ _ZN3ams4kern4arch5arm6412SvcHandler32Ev:
mov x0, sp
bl _ZN3ams4kern4arch5arm6415HandleExceptionEPNS2_17KExceptionContextE
/* Restore registers. */
ldp x17, x20, [sp, #(EXCEPTION_CONTEXT_PC_PSR)]
ldr x19, [sp, #(EXCEPTION_CONTEXT_TPIDR)]
/* If we don't need to restore the fpu, skip restoring it. */
ldrb w9, [sp, #(EXCEPTION_CONTEXT_SIZE + THREAD_STACK_PARAMETERS_EXCEPTION_FLAGS)]
tbz w9, #(THREAD_EXCEPTION_FLAG_BIT_INDEX_IS_FPU_CONTEXT_RESTORE_NEEDED), 4f
/* Clear the needs-fpu-restore flag. */
and w9, w9, #(~THREAD_EXCEPTION_FLAG_IS_FPU_CONTEXT_RESTORE_NEEDED)
strb w9, [sp, #(EXCEPTION_CONTEXT_SIZE + THREAD_STACK_PARAMETERS_EXCEPTION_FLAGS)]
/* Enable and restore the fpu. */
ENABLE_AND_RESTORE_FPU32(x10, x8, x9, w8, w9)
4: /* Restore registers. */
ldp x9, x10, [sp, #(EXCEPTION_CONTEXT_PC_PSR)]
ldr x11, [sp, #(EXCEPTION_CONTEXT_TPIDR)]
#if defined(MESOSPHERE_ENABLE_HARDWARE_SINGLE_STEP)
/* Since we're returning from an SVC, make sure SPSR.SS is cleared so that if we're single-stepping we break instantly on the instruction after the SVC. */
bic x20, x20, #(1 << 21)
bic x10, x10, #(1 << 21)
#endif
msr elr_el1, x17
msr spsr_el1, x20
msr tpidr_el0, x19
msr elr_el1, x9
msr spsr_el1, x10
msr tpidr_el0, x11
ldp x0, x1, [sp, #(EXCEPTION_CONTEXT_X0_X1)]
ldp x2, x3, [sp, #(EXCEPTION_CONTEXT_X2_X3)]
ldp x4, x5, [sp, #(EXCEPTION_CONTEXT_X4_X5)]
@ -393,9 +469,7 @@ _ZN3ams4kern4arch5arm6412SvcHandler32Ev:
add sp, sp, #(EXCEPTION_CONTEXT_SIZE)
eret
4: /* Return from SVC. */
/* Clear our in-SVC note. */
strb wzr, [sp, #(EXCEPTION_CONTEXT_SIZE + THREAD_STACK_PARAMETERS_IS_CALLING_SVC)]
5: /* Return from SVC. */
/* If we should, trace the svc exit. */
#if defined(MESOSPHERE_BUILD_FOR_TRACING)
@ -413,22 +487,63 @@ _ZN3ams4kern4arch5arm6412SvcHandler32Ev:
add sp, sp, #0x40
#endif
/* Restore registers. */
/* Get our exception flags. */
ldrb w9, [sp, #(EXCEPTION_CONTEXT_SIZE + THREAD_STACK_PARAMETERS_EXCEPTION_FLAGS)]
/* Clear in-svc and needs-fpu-restore flags. */
and w8, w9, #(~(THREAD_EXCEPTION_FLAG_IS_FPU_CONTEXT_RESTORE_NEEDED))
and w8, w8, #(~(THREAD_EXCEPTION_FLAG_IS_CALLING_SVC))
strb w8, [sp, #(EXCEPTION_CONTEXT_SIZE + THREAD_STACK_PARAMETERS_EXCEPTION_FLAGS)]
/* If we don't need to restore the fpu, skip restoring it. */
tbz w9, #(THREAD_EXCEPTION_FLAG_BIT_INDEX_IS_FPU_CONTEXT_RESTORE_NEEDED), 7f
/* If we need to restore the fpu, check if we need to do a full restore. */
tbnz w9, #(THREAD_EXCEPTION_FLAG_BIT_INDEX_IS_IN_USERMODE_EXCEPTION_HANDLER), 6f
/* Enable the fpu. */
ENABLE_FPU(x8)
/* Get the thread context and restore fpsr/fpcr. */
GET_THREAD_CONTEXT_AND_RESTORE_FPCR_FPSR(x10, x8, x9, w8, w9)
/* Restore callee-saved registers to 32-bit fpu. */
RESTORE_FPU32_CALLEE_SAVE_REGISTERS(x10)
/* Clear caller-saved registers to 32-bit fpu. */
movi v0.2d, #0
movi v1.2d, #0
movi v2.2d, #0
movi v3.2d, #0
movi v8.2d, #0
movi v9.2d, #0
movi v10.2d, #0
movi v11.2d, #0
movi v12.2d, #0
movi v13.2d, #0
movi v14.2d, #0
movi v15.2d, #0
b 7f
6: /* We need to do a full fpu restore. */
ENABLE_AND_RESTORE_FPU32(x10, x8, x9, w8, w9)
7: /* Restore registers. */
ldp x9, x10, [sp, #(EXCEPTION_CONTEXT_PC_PSR)]
ldr x11, [sp, #(EXCEPTION_CONTEXT_TPIDR)]
#if defined(MESOSPHERE_ENABLE_HARDWARE_SINGLE_STEP)
/* Since we're returning from an SVC, make sure SPSR.SS is cleared so that if we're single-stepping we break instantly on the instruction after the SVC. */
bic x10, x10, #(1 << 21)
#endif
msr elr_el1, x9
msr spsr_el1, x10
msr tpidr_el0, x11
ldp x8, x9, [sp, #(EXCEPTION_CONTEXT_X8_X9)]
ldp x10, x11, [sp, #(EXCEPTION_CONTEXT_X10_X11)]
ldp x12, x13, [sp, #(EXCEPTION_CONTEXT_X12_X13)]
ldp x14, xzr, [sp, #(EXCEPTION_CONTEXT_X14_X15)]
ldp x17, x20, [sp, #(EXCEPTION_CONTEXT_PC_PSR)]
ldr x19, [sp, #(EXCEPTION_CONTEXT_TPIDR)]
#if defined(MESOSPHERE_ENABLE_HARDWARE_SINGLE_STEP)
/* Since we're returning from an SVC, make sure SPSR.SS is cleared so that if we're single-stepping we break instantly on the instruction after the SVC. */
bic x20, x20, #(1 << 21)
#endif
msr elr_el1, x17
msr spsr_el1, x20
msr tpidr_el0, x19
/* Return. */
add sp, sp, #(EXCEPTION_CONTEXT_SIZE)

8
libraries/libmesosphere/source/kern_k_debug_base.cpp
View file

@ -447,7 +447,7 @@ namespace ams::kern {
{
auto end = target->GetThreadList().end();
for (auto it = target->GetThreadList().begin(); it != end; ++it) {
it->ClearSingleStep();
it->ClearHardwareSingleStep();
}
}
#endif
@ -595,13 +595,13 @@ namespace ams::kern {
{
if ((context_flags & ams::svc::ThreadContextFlag_SetSingleStep) != 0) {
/* Set single step. */
thread->SetSingleStep();
thread->SetHardwareSingleStep();
/* If no other thread flags are present, we're done. */
R_SUCCEED_IF((context_flags & ~ams::svc::ThreadContextFlag_SetSingleStep) == 0);
} else if ((context_flags & ams::svc::ThreadContextFlag_ClearSingleStep) != 0) {
/* Clear single step. */
thread->ClearSingleStep();
thread->ClearHardwareSingleStep();
/* If no other thread flags are present, we're done. */
R_SUCCEED_IF((context_flags & ~ams::svc::ThreadContextFlag_ClearSingleStep) == 0);
@ -1022,7 +1022,7 @@ namespace ams::kern {
for (auto it = process->GetThreadList().begin(); it != end; ++it) {
#if defined(MESOSPHERE_ENABLE_HARDWARE_SINGLE_STEP)
/* Clear the thread's single-step state. */
it->ClearSingleStep();
it->ClearHardwareSingleStep();
#endif
if (resume) {

2
libraries/libmesosphere/source/kern_k_scheduler.cpp
View file

@ -255,7 +255,7 @@ namespace ams::kern {
/* in EL0...which implies a return-from-exception has occurred since we set the bit. Thus, forcing */
/* an ISB is unnecessary, and we can modify the register safely and be confident it will affect the next */
/* userland instruction executed. */
cpu::MonitorDebugSystemControlRegisterAccessor().SetSoftwareStep(next_thread->IsSingleStep()).Store();
cpu::MonitorDebugSystemControlRegisterAccessor().SetSoftwareStep(next_thread->IsHardwareSingleStep()).Store();
#endif
/* Switch the current process, if we're switching processes. */

118
libraries/libmesosphere/source/kern_k_thread.cpp
View file

@ -68,6 +68,91 @@ namespace ams::kern {
}
ALWAYS_INLINE void KThread::SetPinnedSvcPermissions() {
/* Get our stack parameters. */
auto &sp = this->GetStackParameters();
/* Get our parent's svc permissions. */
MESOSPHERE_ASSERT(m_parent != nullptr);
const auto &svc_permissions = m_parent->GetSvcPermissions();
/* Get whether we have access to return from exception. */
const bool return_from_exception = sp.svc_access_flags[svc::SvcId_ReturnFromException];
/* Clear all permissions. */
sp.svc_access_flags.Reset();
/* Set SynchronizePreemptionState if allowed. */
if (svc_permissions[svc::SvcId_SynchronizePreemptionState]) {
sp.svc_access_flags[svc::SvcId_SynchronizePreemptionState] = true;
}
/* If we previously had ReturnFromException, potentially grant it and GetInfo. */
if (return_from_exception) {
/* Set ReturnFromException (guaranteed allowed, if we're here). */
sp.svc_access_flags[svc::SvcId_ReturnFromException] = true;
/* Set GetInfo if allowed. */
if (svc_permissions[svc::SvcId_GetInfo]) {
sp.svc_access_flags[svc::SvcId_GetInfo] = true;
}
}
}
ALWAYS_INLINE void KThread::SetUnpinnedSvcPermissions() {
/* Get our stack parameters. */
auto &sp = this->GetStackParameters();
/* Get our parent's svc permissions. */
MESOSPHERE_ASSERT(m_parent != nullptr);
const auto &svc_permissions = m_parent->GetSvcPermissions();
/* Get whether we have access to return from exception. */
const bool return_from_exception = sp.svc_access_flags[svc::SvcId_ReturnFromException];
/* Copy permissions. */
sp.svc_access_flags = svc_permissions;
/* Clear specific SVCs based on our state. */
sp.svc_access_flags[svc::SvcId_SynchronizePreemptionState] = false;
if (!return_from_exception) {
sp.svc_access_flags[svc::SvcId_ReturnFromException] = false;
}
}
ALWAYS_INLINE void KThread::SetUsermodeExceptionSvcPermissions() {
/* Get our stack parameters. */
auto &sp = this->GetStackParameters();
/* Get our parent's svc permissions. */
MESOSPHERE_ASSERT(m_parent != nullptr);
const auto &svc_permissions = m_parent->GetSvcPermissions();
/* Set ReturnFromException if allowed. */
if (svc_permissions[svc::SvcId_ReturnFromException]) {
sp.svc_access_flags[svc::SvcId_ReturnFromException] = true;
}
/* Set GetInfo if allowed. */
if (svc_permissions[svc::SvcId_GetInfo]) {
sp.svc_access_flags[svc::SvcId_GetInfo] = true;
}
}
ALWAYS_INLINE void KThread::ClearUsermodeExceptionSvcPermissions() {
/* Get our stack parameters. */
auto &sp = this->GetStackParameters();
/* Clear ReturnFromException. */
sp.svc_access_flags[svc::SvcId_ReturnFromException] = false;
/* If pinned, clear GetInfo. */
if (sp.is_pinned) {
sp.svc_access_flags[svc::SvcId_GetInfo] = false;
}
}
Result KThread::Initialize(KThreadFunction func, uintptr_t arg, void *kern_stack_top, KProcessAddress user_stack_top, s32 prio, s32 virt_core, KProcess *owner, ThreadType type) {
/* Assert parameters are valid. */
MESOSPHERE_ASSERT_THIS();
@ -209,18 +294,23 @@ namespace ams::kern {
const bool is_64_bit = m_parent ? m_parent->Is64Bit() : IsDefault64Bit;
const bool is_user = (type == ThreadType_User);
const bool is_main = (type == ThreadType_Main);
m_thread_context.Initialize(reinterpret_cast<uintptr_t>(func), reinterpret_cast<uintptr_t>(this->GetStackTop()), GetInteger(user_stack_top), arg, is_user, is_64_bit, is_main);
this->GetContext().Initialize(reinterpret_cast<uintptr_t>(func), reinterpret_cast<uintptr_t>(this->GetStackTop()), GetInteger(user_stack_top), arg, is_user, is_64_bit, is_main);
/* Setup the stack parameters. */
StackParameters &sp = this->GetStackParameters();
if (m_parent != nullptr) {
m_parent->CopySvcPermissionsTo(sp);
this->SetUnpinnedSvcPermissions();
this->ClearUsermodeExceptionSvcPermissions();
}
sp.context = std::addressof(m_thread_context);
sp.cur_thread = this;
sp.disable_count = 1;
sp.caller_save_fpu_registers = std::addressof(m_caller_save_fpu_registers);
sp.cur_thread = this;
sp.disable_count = 1;
this->SetInExceptionHandler();
if (m_parent != nullptr && is_64_bit) {
this->SetFpu64Bit();
}
/* Set thread ID. */
m_thread_id = g_thread_id++;
@ -329,9 +419,6 @@ namespace ams::kern {
}
}
/* Finalize the thread context. */
m_thread_context.Finalize();
/* Cleanup the kernel stack. */
if (m_kernel_stack_top != nullptr) {
CleanupKernelStack(reinterpret_cast<uintptr_t>(m_kernel_stack_top));
@ -411,6 +498,16 @@ namespace ams::kern {
this->FinishTermination();
}
void KThread::OnEnterUsermodeException() {
this->SetUsermodeExceptionSvcPermissions();
this->SetInUsermodeExceptionHandler();
}
void KThread::OnLeaveUsermodeException() {
this->ClearUsermodeExceptionSvcPermissions();
this->ClearInUsermodeExceptionHandler();
}
void KThread::Pin() {
MESOSPHERE_ASSERT_THIS();
MESOSPHERE_ASSERT(KScheduler::IsSchedulerLockedByCurrentThread());
@ -458,8 +555,7 @@ namespace ams::kern {
}
/* Update our SVC access permissions. */
MESOSPHERE_ASSERT(m_parent != nullptr);
m_parent->CopyPinnedSvcPermissionsTo(this->GetStackParameters());
this->SetPinnedSvcPermissions();
}
void KThread::Unpin() {
@ -507,7 +603,7 @@ namespace ams::kern {
/* Update our SVC access permissions. */
MESOSPHERE_ASSERT(m_parent != nullptr);
m_parent->CopyUnpinnedSvcPermissionsTo(this->GetStackParameters());
this->SetUnpinnedSvcPermissions();
}
/* Resume any threads that began waiting on us while we were pinned. */