diff --git a/libraries/libmesosphere/include/mesosphere/arch/arm64/kern_k_page_table.hpp b/libraries/libmesosphere/include/mesosphere/arch/arm64/kern_k_page_table.hpp
index 14aba9913..370211b2b 100644
--- a/libraries/libmesosphere/include/mesosphere/arch/arm64/kern_k_page_table.hpp
+++ b/libraries/libmesosphere/include/mesosphere/arch/arm64/kern_k_page_table.hpp
@@ -93,9 +93,13 @@ namespace ams::kern::arch::arm64 {
MESOSPHERE_ASSERT(alignment < L1BlockSize);
return KPageTable::GetBlockSize(static_cast<KPageTable::BlockType>(KPageTable::GetBlockType(alignment) + 1));
}
+ public:
+ /* TODO: How should this size be determined. Does the KProcess slab count need to go in a header as a define? */
+ static constexpr size_t NumTtbr0Entries = 81;
+ private:
+ static constinit inline const volatile u64 s_ttbr0_entries[NumTtbr0Entries] = {};
private:
KPageTableManager *m_manager;
- u64 m_ttbr;
u8 m_asid;
protected:
Result OperateImpl(PageLinkedList *page_list, KProcessAddress virt_addr, size_t num_pages, KPhysicalAddress phys_addr, bool is_pa_valid, const KPageProperties properties, OperationType operation, bool reuse_ll);
@@ -168,17 +172,28 @@ namespace ams::kern::arch::arm64 {
return entry;
}
public:
- constexpr explicit KPageTable(util::ConstantInitializeTag) : KPageTableBase(util::ConstantInitialize), m_manager(), m_ttbr(), m_asid() { /* ... */ }
+ constexpr explicit KPageTable(util::ConstantInitializeTag) : KPageTableBase(util::ConstantInitialize), m_manager(), m_asid() { /* ... */ }
explicit KPageTable() { /* ... */ }
static NOINLINE void Initialize(s32 core_id);
- ALWAYS_INLINE void Activate(u32 proc_id) {
- cpu::SwitchProcess(m_ttbr, proc_id);
+ static const volatile u64 &GetTtbr0Entry(size_t index) { return s_ttbr0_entries[index]; }
+
+ static ALWAYS_INLINE u64 GetKernelTtbr0() {
+ return s_ttbr0_entries[0];
+ }
+
+ static ALWAYS_INLINE void ActivateKernel() {
+ /* Activate, using asid 0 and process id = 0xFFFFFFFF */
+ cpu::SwitchProcess(GetKernelTtbr0(), 0xFFFFFFFF);
+ }
+
+ static ALWAYS_INLINE void ActivateProcess(size_t proc_idx, u32 proc_id) {
+ cpu::SwitchProcess(s_ttbr0_entries[proc_idx + 1], proc_id);
}
NOINLINE Result InitializeForKernel(void *table, KVirtualAddress start, KVirtualAddress end);
- NOINLINE Result InitializeForProcess(ams::svc::CreateProcessFlag flags, bool from_back, KMemoryManager::Pool pool, KProcessAddress code_address, size_t code_size, KSystemResource *system_resource, KResourceLimit *resource_limit);
+ NOINLINE Result InitializeForProcess(ams::svc::CreateProcessFlag flags, bool from_back, KMemoryManager::Pool pool, KProcessAddress code_address, size_t code_size, KSystemResource *system_resource, KResourceLimit *resource_limit, size_t process_index);
Result Finalize();
private:
Result MapL1Blocks(KProcessAddress virt_addr, KPhysicalAddress phys_addr, size_t num_pages, PageTableEntry entry_template, bool disable_head_merge, PageLinkedList *page_list, bool reuse_ll);
diff --git a/libraries/libmesosphere/include/mesosphere/arch/arm64/kern_k_process_page_table.hpp b/libraries/libmesosphere/include/mesosphere/arch/arm64/kern_k_process_page_table.hpp
index 640b9e5e5..3ad3c96a1 100644
--- a/libraries/libmesosphere/include/mesosphere/arch/arm64/kern_k_process_page_table.hpp
+++ b/libraries/libmesosphere/include/mesosphere/arch/arm64/kern_k_process_page_table.hpp
@@ -23,13 +23,13 @@ namespace ams::kern::arch::arm64 {
private:
KPageTable m_page_table;
public:
- void Activate(u64 id) {
+ void Activate(size_t process_index, u64 id) {
/* Activate the page table with the specified contextidr. */
- m_page_table.Activate(id);
+ m_page_table.ActivateProcess(process_index, id);
}
- Result Initialize(ams::svc::CreateProcessFlag flags, bool from_back, KMemoryManager::Pool pool, KProcessAddress code_address, size_t code_size, KSystemResource *system_resource, KResourceLimit *resource_limit) {
- R_RETURN(m_page_table.InitializeForProcess(flags, from_back, pool, code_address, code_size, system_resource, resource_limit));
+ Result Initialize(ams::svc::CreateProcessFlag flags, bool from_back, KMemoryManager::Pool pool, KProcessAddress code_address, size_t code_size, KSystemResource *system_resource, KResourceLimit *resource_limit, size_t process_index) {
+ R_RETURN(m_page_table.InitializeForProcess(flags, from_back, pool, code_address, code_size, system_resource, resource_limit, process_index));
}
void Finalize() { m_page_table.Finalize(); }
diff --git a/libraries/libmesosphere/include/mesosphere/arch/arm64/kern_k_supervisor_page_table.hpp b/libraries/libmesosphere/include/mesosphere/arch/arm64/kern_k_supervisor_page_table.hpp
index 73d886d84..0c0602289 100644
--- a/libraries/libmesosphere/include/mesosphere/arch/arm64/kern_k_supervisor_page_table.hpp
+++ b/libraries/libmesosphere/include/mesosphere/arch/arm64/kern_k_supervisor_page_table.hpp
@@ -29,8 +29,7 @@ namespace ams::kern::arch::arm64 {
NOINLINE void Initialize(s32 core_id);
void Activate() {
- /* Activate, using process id = 0xFFFFFFFF */
- m_page_table.Activate(0xFFFFFFFF);
+ m_page_table.ActivateKernel();
}
void ActivateForInit() {
diff --git a/libraries/libmesosphere/include/mesosphere/kern_k_process.hpp b/libraries/libmesosphere/include/mesosphere/kern_k_process.hpp
index 5a37e3a6b..c4ab7f2a5 100644
--- a/libraries/libmesosphere/include/mesosphere/kern_k_process.hpp
+++ b/libraries/libmesosphere/include/mesosphere/kern_k_process.hpp
@@ -374,7 +374,7 @@ namespace ams::kern {
/* Update the current page table. */
if (next_process) {
- next_process->GetPageTable().Activate(next_process->GetProcessId());
+ next_process->GetPageTable().Activate(next_process->GetSlabIndex(), next_process->GetProcessId());
} else {
Kernel::GetKernelPageTable().Activate();
}
diff --git a/libraries/libmesosphere/source/arch/arm64/kern_k_page_table.cpp b/libraries/libmesosphere/source/arch/arm64/kern_k_page_table.cpp
index 55e4fc52b..c1b0c76b4 100644
--- a/libraries/libmesosphere/source/arch/arm64/kern_k_page_table.cpp
+++ b/libraries/libmesosphere/source/arch/arm64/kern_k_page_table.cpp
@@ -85,77 +85,6 @@ namespace ams::kern::arch::arm64 {
return (static_cast<u64>(asid) << 48) | (static_cast<u64>(GetInteger(table)));
}
- class KPageTableAsidManager {
- private:
- using WordType = u32;
- static constexpr u8 ReservedAsids[] = { 0 };
- static constexpr size_t NumReservedAsids = util::size(ReservedAsids);
- static constexpr size_t BitsPerWord = BITSIZEOF(WordType);
- static constexpr size_t AsidCount = 0x100;
- static constexpr size_t NumWords = AsidCount / BitsPerWord;
- static constexpr WordType FullWord = ~WordType(0u);
- private:
- WordType m_state[NumWords];
- KLightLock m_lock;
- u8 m_hint;
- private:
- constexpr bool TestImpl(u8 asid) const {
- return m_state[asid / BitsPerWord] & (1u << (asid % BitsPerWord));
- }
- constexpr void ReserveImpl(u8 asid) {
- MESOSPHERE_ASSERT(!this->TestImpl(asid));
- m_state[asid / BitsPerWord] |= (1u << (asid % BitsPerWord));
- }
-
- constexpr void ReleaseImpl(u8 asid) {
- MESOSPHERE_ASSERT(this->TestImpl(asid));
- m_state[asid / BitsPerWord] &= ~(1u << (asid % BitsPerWord));
- }
-
- constexpr u8 FindAvailable() const {
- for (size_t i = 0; i < util::size(m_state); i++) {
- if (m_state[i] == FullWord) {
- continue;
- }
- const WordType clear_bit = (m_state[i] + 1) ^ (m_state[i]);
- return BitsPerWord * i + BitsPerWord - 1 - ClearLeadingZero(clear_bit);
- }
- if (m_state[util::size(m_state)-1] == FullWord) {
- MESOSPHERE_PANIC("Unable to reserve ASID");
- }
- __builtin_unreachable();
- }
-
- static constexpr ALWAYS_INLINE WordType ClearLeadingZero(WordType value) {
- return __builtin_clzll(value) - (BITSIZEOF(unsigned long long) - BITSIZEOF(WordType));
- }
- public:
- constexpr KPageTableAsidManager() : m_state(), m_lock(), m_hint() {
- for (size_t i = 0; i < NumReservedAsids; i++) {
- this->ReserveImpl(ReservedAsids[i]);
- }
- }
-
- u8 Reserve() {
- KScopedLightLock lk(m_lock);
-
- if (this->TestImpl(m_hint)) {
- m_hint = this->FindAvailable();
- }
-
- this->ReserveImpl(m_hint);
-
- return m_hint++;
- }
-
- void Release(u8 asid) {
- KScopedLightLock lk(m_lock);
- this->ReleaseImpl(asid);
- }
- };
-
- KPageTableAsidManager g_asid_manager;
-
}
ALWAYS_INLINE void KPageTable::NoteUpdated() const {
@@ -184,6 +113,7 @@ namespace ams::kern::arch::arm64 {
this->OnKernelTableSinglePageUpdated(virt_addr);
}
+
void KPageTable::Initialize(s32 core_id) {
/* Nothing actually needed here. */
MESOSPHERE_UNUSED(core_id);
@@ -194,38 +124,29 @@ namespace ams::kern::arch::arm64 {
m_asid = 0;
m_manager = Kernel::GetSystemSystemResource().GetPageTableManagerPointer();
- /* Allocate a page for ttbr. */
- /* NOTE: It is a postcondition of page table manager allocation that the page is all-zero. */
- const u64 asid_tag = (static_cast<u64>(m_asid) << 48ul);
- const KVirtualAddress page = m_manager->Allocate();
- MESOSPHERE_ASSERT(page != Null<KVirtualAddress>);
- m_ttbr = GetInteger(KPageTableBase::GetLinearMappedPhysicalAddress(page)) | asid_tag;
-
/* Initialize the base page table. */
MESOSPHERE_R_ABORT_UNLESS(KPageTableBase::InitializeForKernel(true, table, start, end));
R_SUCCEED();
}
- Result KPageTable::InitializeForProcess(ams::svc::CreateProcessFlag flags, bool from_back, KMemoryManager::Pool pool, KProcessAddress code_address, size_t code_size, KSystemResource *system_resource, KResourceLimit *resource_limit) {
- /* Get an ASID */
- m_asid = g_asid_manager.Reserve();
- ON_RESULT_FAILURE { g_asid_manager.Release(m_asid); };
+ Result KPageTable::InitializeForProcess(ams::svc::CreateProcessFlag flags, bool from_back, KMemoryManager::Pool pool, KProcessAddress code_address, size_t code_size, KSystemResource *system_resource, KResourceLimit *resource_limit, size_t process_index) {
+ /* Determine our ASID */
+ m_asid = process_index + 1;
+ MESOSPHERE_ABORT_UNLESS(0 < m_asid && m_asid < util::size(s_ttbr0_entries));
/* Set our manager. */
m_manager = system_resource->GetPageTableManagerPointer();
- /* Allocate a new table, and set our ttbr value. */
- const KVirtualAddress new_table = m_manager->Allocate();
- R_UNLESS(new_table != Null<KVirtualAddress>, svc::ResultOutOfResource());
- m_ttbr = EncodeTtbr(GetPageTablePhysicalAddress(new_table), m_asid);
- ON_RESULT_FAILURE_2 { m_manager->Free(new_table); };
+ /* Get the virtual address of our L1 table. */
+ const KPhysicalAddress ttbr0_phys = KPhysicalAddress(s_ttbr0_entries[m_asid] & UINT64_C(0xFFFFFFFFFFFE));
+ const KVirtualAddress ttbr0_virt = KMemoryLayout::GetLinearVirtualAddress(ttbr0_phys);
/* Initialize our base table. */
const size_t as_width = GetAddressSpaceWidth(flags);
const KProcessAddress as_start = 0;
const KProcessAddress as_end = (1ul << as_width);
- R_TRY(KPageTableBase::InitializeForProcess(flags, from_back, pool, GetVoidPointer(new_table), as_start, as_end, code_address, code_size, system_resource, resource_limit));
+ R_TRY(KPageTableBase::InitializeForProcess(flags, from_back, pool, GetVoidPointer(ttbr0_virt), as_start, as_end, code_address, code_size, system_resource, resource_limit));
/* Note that we've updated the table (since we created it). */
this->NoteUpdated();
@@ -329,20 +250,16 @@ namespace ams::kern::arch::arm64 {
}
}
- /* Free the L1 table. */
+ /* Clear the L1 table. */
{
const KVirtualAddress l1_table = reinterpret_cast<uintptr_t>(impl.Finalize());
ClearPageTable(l1_table);
- this->GetPageTableManager().Free(l1_table);
}
/* Perform inherited finalization. */
KPageTableBase::Finalize();
}
- /* Release our asid. */
- g_asid_manager.Release(m_asid);
-
R_SUCCEED();
}
diff --git a/libraries/libmesosphere/source/board/nintendo/nx/kern_k_sleep_manager.cpp b/libraries/libmesosphere/source/board/nintendo/nx/kern_k_sleep_manager.cpp
index bb782d390..5e5983c86 100644
--- a/libraries/libmesosphere/source/board/nintendo/nx/kern_k_sleep_manager.cpp
+++ b/libraries/libmesosphere/source/board/nintendo/nx/kern_k_sleep_manager.cpp
@@ -31,7 +31,6 @@ namespace ams::kern::board::nintendo::nx {
/* Struct representing registers saved on wake/sleep. */
class SavedSystemRegisters {
private:
- u64 ttbr0_el1;
u64 elr_el1;
u64 sp_el0;
u64 spsr_el1;
@@ -90,7 +89,6 @@ namespace ams::kern::board::nintendo::nx {
void SavedSystemRegisters::Save() {
/* Save system registers. */
- this->ttbr0_el1 = cpu::GetTtbr0El1();
this->tpidr_el0 = cpu::GetTpidrEl0();
this->elr_el1 = cpu::GetElrEl1();
this->sp_el0 = cpu::GetSpEl0();
@@ -405,7 +403,7 @@ namespace ams::kern::board::nintendo::nx {
cpu::EnsureInstructionConsistency();
/* Restore system registers. */
- cpu::SetTtbr0El1 (this->ttbr0_el1);
+ cpu::SetTtbr0El1 (KPageTable::GetKernelTtbr0());
cpu::SetTpidrEl0 (this->tpidr_el0);
cpu::SetElrEl1 (this->elr_el1);
cpu::SetSpEl0 (this->sp_el0);
diff --git a/libraries/libmesosphere/source/kern_k_process.cpp b/libraries/libmesosphere/source/kern_k_process.cpp
index 789bdbc09..96d82f975 100644
--- a/libraries/libmesosphere/source/kern_k_process.cpp
+++ b/libraries/libmesosphere/source/kern_k_process.cpp
@@ -299,7 +299,7 @@ namespace ams::kern {
/* Setup page table. */
{
const bool from_back = (params.flags & ams::svc::CreateProcessFlag_EnableAslr) == 0;
- R_TRY(m_page_table.Initialize(static_cast<ams::svc::CreateProcessFlag>(params.flags), from_back, pool, params.code_address, params.code_num_pages * PageSize, m_system_resource, res_limit));
+ R_TRY(m_page_table.Initialize(static_cast<ams::svc::CreateProcessFlag>(params.flags), from_back, pool, params.code_address, params.code_num_pages * PageSize, m_system_resource, res_limit, this->GetSlabIndex()));
}
ON_RESULT_FAILURE_2 { m_page_table.Finalize(); };
@@ -378,7 +378,7 @@ namespace ams::kern {
/* Setup page table. */
{
const bool from_back = (params.flags & ams::svc::CreateProcessFlag_EnableAslr) == 0;
- R_TRY(m_page_table.Initialize(static_cast<ams::svc::CreateProcessFlag>(params.flags), from_back, pool, params.code_address, code_size, m_system_resource, res_limit));
+ R_TRY(m_page_table.Initialize(static_cast<ams::svc::CreateProcessFlag>(params.flags), from_back, pool, params.code_address, code_size, m_system_resource, res_limit, this->GetSlabIndex()));
}
ON_RESULT_FAILURE_2 { m_page_table.Finalize(); };
diff --git a/mesosphere/kernel/kernel.ld b/mesosphere/kernel/kernel.ld
index 779488a35..bcd658572 100644
--- a/mesosphere/kernel/kernel.ld
+++ b/mesosphere/kernel/kernel.ld
@@ -125,6 +125,8 @@ SECTIONS
.gnu.version_r : { *(.gnu.version_r) } :rodata
.note.gnu.build-id : { *(.note.gnu.build-id) } :rodata
+ __rodata_end = .;
+
/* =========== DATA section =========== */
. = ALIGN(0x1000);
__data_start = . ;
diff --git a/mesosphere/kernel/source/arch/arm64/init/kern_init_core.cpp b/mesosphere/kernel/source/arch/arm64/init/kern_init_core.cpp
index faf34576d..10a000c6d 100644
--- a/mesosphere/kernel/source/arch/arm64/init/kern_init_core.cpp
+++ b/mesosphere/kernel/source/arch/arm64/init/kern_init_core.cpp
@@ -15,6 +15,9 @@
*/
#include <mesosphere.hpp>
+extern "C" void __rodata_start();
+extern "C" void __rodata_end();
+
extern "C" void __bin_start__();
extern "C" void __bin_end__();
@@ -220,6 +223,31 @@ namespace ams::kern::init {
};
static_assert(kern::arch::arm64::init::IsInitialPageAllocator<KInitialPageAllocatorForFinalizeIdentityMapping>);
+ void SetupAllTtbr0Entries(KInitialPageTable &init_pt, KInitialPageAllocator &allocator) {
+ /* Validate that the ttbr0 array is in rodata. */
+ const uintptr_t rodata_start = reinterpret_cast<uintptr_t>(__rodata_start);
+ const uintptr_t rodata_end = reinterpret_cast<uintptr_t>(__rodata_end);
+ MESOSPHERE_INIT_ABORT_UNLESS(rodata_start < rodata_end);
+ MESOSPHERE_INIT_ABORT_UNLESS(rodata_start <= reinterpret_cast<uintptr_t>(std::addressof(KPageTable::GetTtbr0Entry(0))));
+ MESOSPHERE_INIT_ABORT_UNLESS(reinterpret_cast<uintptr_t>(std::addressof(KPageTable::GetTtbr0Entry(KPageTable::NumTtbr0Entries))) < rodata_end);
+
+ /* Allocate pages for all ttbr0 entries. */
+ for (size_t i = 0; i < KPageTable::NumTtbr0Entries; ++i) {
+ /* Allocate a page. */
+ KPhysicalAddress page = allocator.Allocate(PageSize);
+ MESOSPHERE_INIT_ABORT_UNLESS(page != Null<KPhysicalAddress>);
+
+ /* Check that the page is allowed to be a ttbr0 entry. */
+ MESOSPHERE_INIT_ABORT_UNLESS((GetInteger(page) & UINT64_C(0xFFFF000000000001)) == 0);
+
+ /* Get the physical address of the ttbr0 entry. */
+ const auto ttbr0_phys_ptr = init_pt.GetPhysicalAddress(KVirtualAddress(std::addressof(KPageTable::GetTtbr0Entry(i))));
+
+ /* Set the entry to the newly allocated page. */
+ *reinterpret_cast<volatile u64 *>(GetInteger(ttbr0_phys_ptr)) = (static_cast<u64>(i) << 48) | GetInteger(page);
+ }
+ }
+
void FinalizeIdentityMapping(KInitialPageTable &init_pt, KInitialPageAllocator &allocator, u64 phys_to_virt_offset) {
/* Create an allocator for identity mapping finalization. */
KInitialPageAllocatorForFinalizeIdentityMapping finalize_allocator(allocator, phys_to_virt_offset);
@@ -591,6 +619,9 @@ namespace ams::kern::init {
/* Create page table object for use during remaining initialization. */
KInitialPageTable init_pt;
+ /* Setup all ttbr0 pages. */
+ SetupAllTtbr0Entries(init_pt, g_initial_page_allocator);
+
/* Unmap the identity mapping. */
FinalizeIdentityMapping(init_pt, g_initial_page_allocator, g_phase2_linear_region_phys_to_virt_diff);