mirrors/Atmosphere
1
0
Fork 0
mirror of https://github.com/Atmosphere-NX/Atmosphere.git synced 2025-06-03 08:08:39 -04:00
Code Issues Projects Releases Packages Wiki Activity

kern: refactor init (kill identity map, merge cpu on logic)

Browse source
This commit is contained in:
Michael Scire 2023-02-21 10:38:48 -07:00
parent 42e6c1fd59
commit 709e1969bb
20 changed files with 431 additions and 387 deletions
Download patch file Download diff file Expand all files Collapse all files

4
libraries/libmesosphere/include/mesosphere/arch/arm64/init/kern_k_init_arguments.hpp
View file

@ -29,8 +29,6 @@ namespace ams::kern::init {
u64 sp;
u64 entrypoint;
u64 argument;
u64 setup_function;
u64 exception_stack;
};
static_assert(alignof(KInitArguments) == util::CeilingPowerOfTwo(INIT_ARGUMENTS_SIZE));
static_assert(sizeof(KInitArguments) == std::max(INIT_ARGUMENTS_SIZE, util::CeilingPowerOfTwo(INIT_ARGUMENTS_SIZE)));
@ -45,7 +43,5 @@ namespace ams::kern::init {
static_assert(AMS_OFFSETOF(KInitArguments, sp) == INIT_ARGUMENTS_SP);
static_assert(AMS_OFFSETOF(KInitArguments, entrypoint) == INIT_ARGUMENTS_ENTRYPOINT);
static_assert(AMS_OFFSETOF(KInitArguments, argument) == INIT_ARGUMENTS_ARGUMENT);
static_assert(AMS_OFFSETOF(KInitArguments, setup_function) == INIT_ARGUMENTS_SETUP_FUNCTION);
static_assert(AMS_OFFSETOF(KInitArguments, exception_stack) == INIT_ARGUMENTS_EXCEPTION_STACK);
}

157
libraries/libmesosphere/include/mesosphere/arch/arm64/init/kern_k_init_page_table.hpp
View file

@ -23,17 +23,6 @@
namespace ams::kern::arch::arm64::init {
inline void ClearPhysicalMemory(KPhysicalAddress address, size_t size) {
MESOSPHERE_INIT_ABORT_UNLESS(util::IsAligned(size, sizeof(u64)));
/* This Physical Address -> void * conversion is valid, because this is init page table code. */
/* The MMU is necessarily not yet turned on, if we are creating an initial page table. */
volatile u64 *ptr = reinterpret_cast<volatile u64 *>(GetInteger(address));
for (size_t i = 0; i < size / sizeof(u64); ++i) {
ptr[i] = 0;
}
}
/* NOTE: Nintendo uses virtual functions, rather than a concept + template. */
template<typename T>
concept IsInitialPageAllocator = requires (T &t, KPhysicalAddress phys_addr, size_t size) {
@ -41,25 +30,23 @@ namespace ams::kern::arch::arm64::init {
{ t.Free(phys_addr, size) } -> std::same_as<void>;
};
template<IsInitialPageAllocator _PageAllocator>
class KInitialPageTableTemplate {
public:
using PageAllocator = _PageAllocator;
class KInitialPageTable {
private:
KPhysicalAddress m_l1_tables[2];
u32 m_num_entries[2];
public:
KInitialPageTableTemplate(KVirtualAddress start_address, KVirtualAddress end_address, PageAllocator &allocator) {
template<IsInitialPageAllocator PageAllocator>
KInitialPageTable(KVirtualAddress start_address, KVirtualAddress end_address, PageAllocator &allocator) {
/* Set tables. */
m_l1_tables[0] = AllocateNewPageTable(allocator);
m_l1_tables[1] = AllocateNewPageTable(allocator);
m_l1_tables[0] = AllocateNewPageTable(allocator, 0);
m_l1_tables[1] = AllocateNewPageTable(allocator, 0);
/* Set counts. */
m_num_entries[0] = MaxPageTableEntries;
m_num_entries[1] = ((end_address / L1BlockSize) & (MaxPageTableEntries - 1)) - ((start_address / L1BlockSize) & (MaxPageTableEntries - 1)) + 1;
}
KInitialPageTableTemplate() {
KInitialPageTable() {
/* Set tables. */
m_l1_tables[0] = util::AlignDown(cpu::GetTtbr0El1(), PageSize);
m_l1_tables[1] = util::AlignDown(cpu::GetTtbr1El1(), PageSize);
@ -82,30 +69,35 @@ namespace ams::kern::arch::arm64::init {
return GetInteger(m_l1_tables[1]);
}
private:
constexpr ALWAYS_INLINE L1PageTableEntry *GetL1Entry(KVirtualAddress address) const {
constexpr ALWAYS_INLINE L1PageTableEntry *GetL1Entry(KVirtualAddress address, u64 phys_to_virt_offset = 0) const {
const size_t index = (GetInteger(address) >> (BITSIZEOF(address) - 1)) & 1;
L1PageTableEntry *l1_table = reinterpret_cast<L1PageTableEntry *>(GetInteger(m_l1_tables[index]));
L1PageTableEntry *l1_table = reinterpret_cast<L1PageTableEntry *>(GetInteger(m_l1_tables[index]) + phys_to_virt_offset);
return l1_table + ((GetInteger(address) / L1BlockSize) & (m_num_entries[index] - 1));
}
static constexpr ALWAYS_INLINE L2PageTableEntry *GetL2Entry(const L1PageTableEntry *entry, KVirtualAddress address) {
L2PageTableEntry *l2_table = reinterpret_cast<L2PageTableEntry *>(GetInteger(entry->GetTable()));
static constexpr ALWAYS_INLINE L2PageTableEntry *GetL2Entry(const L1PageTableEntry *entry, KVirtualAddress address, u64 phys_to_virt_offset = 0) {
L2PageTableEntry *l2_table = reinterpret_cast<L2PageTableEntry *>(GetInteger(entry->GetTable()) + phys_to_virt_offset);
return l2_table + ((GetInteger(address) / L2BlockSize) & (MaxPageTableEntries - 1));
}
static constexpr ALWAYS_INLINE L3PageTableEntry *GetL3Entry(const L2PageTableEntry *entry, KVirtualAddress address) {
L3PageTableEntry *l3_table = reinterpret_cast<L3PageTableEntry *>(GetInteger(entry->GetTable()));
static constexpr ALWAYS_INLINE L3PageTableEntry *GetL3Entry(const L2PageTableEntry *entry, KVirtualAddress address, u64 phys_to_virt_offset = 0) {
L3PageTableEntry *l3_table = reinterpret_cast<L3PageTableEntry *>(GetInteger(entry->GetTable()) + phys_to_virt_offset);
return l3_table + ((GetInteger(address) / L3BlockSize) & (MaxPageTableEntries - 1));
}
static ALWAYS_INLINE KPhysicalAddress AllocateNewPageTable(PageAllocator &allocator) {
template<IsInitialPageAllocator PageAllocator>
static ALWAYS_INLINE KPhysicalAddress AllocateNewPageTable(PageAllocator &allocator, u64 phys_to_virt_offset) {
auto address = allocator.Allocate(PageSize);
ClearNewPageTable(address);
ClearNewPageTable(address, phys_to_virt_offset);
return address;
}
static ALWAYS_INLINE void ClearNewPageTable(KPhysicalAddress address) {
ClearPhysicalMemory(address, PageSize);
static ALWAYS_INLINE void ClearNewPageTable(KPhysicalAddress address, u64 phys_to_virt_offset) {
/* Convert to a deferenceable address, and clear. */
volatile u64 *ptr = reinterpret_cast<volatile u64 *>(GetInteger(address) + phys_to_virt_offset);
for (size_t i = 0; i < PageSize / sizeof(u64); ++i) {
ptr[i] = 0;
}
}
public:
static consteval size_t GetMaximumOverheadSize(size_t size) {
@ -327,7 +319,8 @@ namespace ams::kern::arch::arm64::init {
}
}
public:
void NOINLINE Map(KVirtualAddress virt_addr, size_t size, KPhysicalAddress phys_addr, const PageTableEntry &attr, PageAllocator &allocator) {
template<IsInitialPageAllocator PageAllocator>
void NOINLINE Map(KVirtualAddress virt_addr, size_t size, KPhysicalAddress phys_addr, const PageTableEntry &attr, PageAllocator &allocator, u64 phys_to_virt_offset) {
/* Ensure that addresses and sizes are page aligned. */
MESOSPHERE_INIT_ABORT_UNLESS(util::IsAligned(GetInteger(virt_addr), PageSize));
MESOSPHERE_INIT_ABORT_UNLESS(util::IsAligned(GetInteger(phys_addr), PageSize));
@ -335,7 +328,7 @@ namespace ams::kern::arch::arm64::init {
/* Iteratively map pages until the requested region is mapped. */
while (size > 0) {
L1PageTableEntry *l1_entry = this->GetL1Entry(virt_addr);
L1PageTableEntry *l1_entry = this->GetL1Entry(virt_addr, phys_to_virt_offset);
/* Can we make an L1 block? */
if (util::IsAligned(GetInteger(virt_addr), L1BlockSize) && util::IsAligned(GetInteger(phys_addr), L1BlockSize) && size >= L1BlockSize) {
@ -349,12 +342,12 @@ namespace ams::kern::arch::arm64::init {
/* If we don't already have an L2 table, we need to make a new one. */
if (!l1_entry->IsTable()) {
KPhysicalAddress new_table = AllocateNewPageTable(allocator);
KPhysicalAddress new_table = AllocateNewPageTable(allocator, phys_to_virt_offset);
cpu::DataSynchronizationBarrierInnerShareable();
*l1_entry = L1PageTableEntry(PageTableEntry::TableTag{}, new_table, attr.IsPrivilegedExecuteNever());
}
L2PageTableEntry *l2_entry = GetL2Entry(l1_entry, virt_addr);
L2PageTableEntry *l2_entry = GetL2Entry(l1_entry, virt_addr, phys_to_virt_offset);
/* Can we make a contiguous L2 block? */
if (util::IsAligned(GetInteger(virt_addr), L2ContiguousBlockSize) && util::IsAligned(GetInteger(phys_addr), L2ContiguousBlockSize) && size >= L2ContiguousBlockSize) {
@ -380,12 +373,12 @@ namespace ams::kern::arch::arm64::init {
/* If we don't already have an L3 table, we need to make a new one. */
if (!l2_entry->IsTable()) {
KPhysicalAddress new_table = AllocateNewPageTable(allocator);
KPhysicalAddress new_table = AllocateNewPageTable(allocator, phys_to_virt_offset);
cpu::DataSynchronizationBarrierInnerShareable();
*l2_entry = L2PageTableEntry(PageTableEntry::TableTag{}, new_table, attr.IsPrivilegedExecuteNever());
}
L3PageTableEntry *l3_entry = GetL3Entry(l2_entry, virt_addr);
L3PageTableEntry *l3_entry = GetL3Entry(l2_entry, virt_addr, phys_to_virt_offset);
/* Can we make a contiguous L3 block? */
if (util::IsAligned(GetInteger(virt_addr), L3ContiguousBlockSize) && util::IsAligned(GetInteger(phys_addr), L3ContiguousBlockSize) && size >= L3ContiguousBlockSize) {
@ -410,6 +403,98 @@ namespace ams::kern::arch::arm64::init {
cpu::DataSynchronizationBarrierInnerShareable();
}
void UnmapTtbr0Entries(u64 phys_to_virt_offset) {
/* Ensure data consistency before we unmap. */
cpu::DataSynchronizationBarrierInnerShareable();
/* Define helper, as we only want to clear non-nGnRE pages. */
constexpr auto ShouldUnmap = [](const PageTableEntry *entry) ALWAYS_INLINE_LAMBDA -> bool {
return entry->GetPageAttribute() != PageTableEntry::PageAttribute_Device_nGnRE;
};
/* Iterate all L1 entries. */
L1PageTableEntry * const l1_table = reinterpret_cast<L1PageTableEntry *>(GetInteger(m_l1_tables[0]) + phys_to_virt_offset);
for (size_t l1_index = 0; l1_index < m_num_entries[0]; l1_index++) {
/* Get L1 entry. */
L1PageTableEntry * const l1_entry = l1_table + l1_index;
if (l1_entry->IsBlock()) {
/* Unmap the L1 entry, if we should. */
if (ShouldUnmap(l1_entry)) {
*static_cast<PageTableEntry *>(l1_entry) = InvalidPageTableEntry;
}
} else if (l1_entry->IsTable()) {
/* Get the L2 table. */
L2PageTableEntry * const l2_table = reinterpret_cast<L2PageTableEntry *>(GetInteger(l1_entry->GetTable()) + phys_to_virt_offset);
/* Unmap all L2 entries, as relevant. */
size_t remaining_l2_entries = 0;
for (size_t l2_index = 0; l2_index < MaxPageTableEntries; ++l2_index) {
/* Get L2 entry. */
L2PageTableEntry * const l2_entry = l2_table + l2_index;
if (l2_entry->IsBlock()) {
const size_t num_to_clear = (l2_entry->IsContiguous() ? L2ContiguousBlockSize : L2BlockSize) / L2BlockSize;
if (ShouldUnmap(l2_entry)) {
for (size_t i = 0; i < num_to_clear; ++i) {
static_cast<PageTableEntry *>(l2_entry)[i] = InvalidPageTableEntry;
}
} else {
remaining_l2_entries += num_to_clear;
}
l2_index = l2_index + num_to_clear - 1;
} else if (l2_entry->IsTable()) {
/* Get the L3 table. */
L3PageTableEntry * const l3_table = reinterpret_cast<L3PageTableEntry *>(GetInteger(l2_entry->GetTable()) + phys_to_virt_offset);
/* Unmap all L3 entries, as relevant. */
size_t remaining_l3_entries = 0;
for (size_t l3_index = 0; l3_index < MaxPageTableEntries; ++l3_index) {
/* Get L3 entry. */
if (L3PageTableEntry * const l3_entry = l3_table + l3_index; l3_entry->IsBlock()) {
const size_t num_to_clear = (l3_entry->IsContiguous() ? L3ContiguousBlockSize : L3BlockSize) / L3BlockSize;
if (ShouldUnmap(l3_entry)) {
for (size_t i = 0; i < num_to_clear; ++i) {
static_cast<PageTableEntry *>(l3_entry)[i] = InvalidPageTableEntry;
}
} else {
remaining_l3_entries += num_to_clear;
}
l3_index = l3_index + num_to_clear - 1;
}
}
/* If we unmapped all L3 entries, clear the L2 entry. */
if (remaining_l3_entries == 0) {
*static_cast<PageTableEntry *>(l2_entry) = InvalidPageTableEntry;
/* Invalidate the entire tlb. */
cpu::DataSynchronizationBarrierInnerShareable();
cpu::InvalidateEntireTlb();
} else {
remaining_l2_entries++;
}
}
}
/* If we unmapped all L2 entries, clear the L1 entry. */
if (remaining_l2_entries == 0) {
*static_cast<PageTableEntry *>(l1_entry) = InvalidPageTableEntry;
/* Invalidate the entire tlb. */
cpu::DataSynchronizationBarrierInnerShareable();
cpu::InvalidateEntireTlb();
}
}
}
/* Invalidate the entire tlb. */
cpu::DataSynchronizationBarrierInnerShareable();
cpu::InvalidateEntireTlb();
}
KPhysicalAddress GetPhysicalAddress(KVirtualAddress virt_addr) const {
/* Get the L1 entry. */
const L1PageTableEntry *l1_entry = this->GetL1Entry(virt_addr);
@ -861,6 +946,4 @@ namespace ams::kern::arch::arm64::init {
};
static_assert(IsInitialPageAllocator<KInitialPageAllocator>);
using KInitialPageTable = KInitialPageTableTemplate<KInitialPageAllocator>;
}

4
libraries/libmesosphere/include/mesosphere/arch/arm64/kern_assembly_offsets.h
View file

@ -246,7 +246,7 @@
#define THREAD_LOCAL_REGION_SIZE 0x200
/* ams::kern::init::KInitArguments, https://github.com/Atmosphere-NX/Atmosphere/blob/master/libraries/libmesosphere/include/mesosphere/arch/arm64/init/kern_k_init_arguments.hpp */
#define INIT_ARGUMENTS_SIZE 0x60
#define INIT_ARGUMENTS_SIZE 0x50
#define INIT_ARGUMENTS_TTBR0 0x00
#define INIT_ARGUMENTS_TTBR1 0x08
#define INIT_ARGUMENTS_TCR 0x10
@ -257,8 +257,6 @@
#define INIT_ARGUMENTS_SP 0x38
#define INIT_ARGUMENTS_ENTRYPOINT 0x40
#define INIT_ARGUMENTS_ARGUMENT 0x48
#define INIT_ARGUMENTS_SETUP_FUNCTION 0x50
#define INIT_ARGUMENTS_EXCEPTION_STACK 0x58
/* ams::kern::KScheduler (::SchedulingState), https://github.com/Atmosphere-NX/Atmosphere/blob/master/libraries/libmesosphere/include/mesosphere/kern_k_scheduler.hpp */
/* NOTE: Due to constraints on ldarb relative offsets, KSCHEDULER_NEEDS_SCHEDULING cannot trivially be changed, and will require assembly edits. */

8
libraries/libmesosphere/include/mesosphere/arch/arm64/kern_cpu.hpp
View file

@ -189,6 +189,14 @@ namespace ams::kern::arch::arm64::cpu {
return (par >> (BITSIZEOF(par) - BITSIZEOF(u8))) == 0xFF;
}
ALWAYS_INLINE void StoreDataCacheForInitArguments(const void *addr, size_t size) {
const uintptr_t start = util::AlignDown(reinterpret_cast<uintptr_t>(addr), DataCacheLineSize);
for (size_t stored = 0; stored < size; stored += cpu::DataCacheLineSize) {
__asm__ __volatile__("dc cvac, %[cur]" :: [cur]"r"(start + stored) : "memory");
}
DataSynchronizationBarrier();
}
/* Synchronization helpers. */
NOINLINE void SynchronizeAllCores();
void SynchronizeCores(u64 core_mask);

9
libraries/libmesosphere/include/mesosphere/arch/arm64/kern_k_page_table_entry.hpp
View file

@ -161,9 +161,12 @@ namespace ams::kern::arch::arm64 {
constexpr ALWAYS_INLINE bool IsPrivilegedExecuteNever() const { return this->GetBits(53, 1) != 0; }
constexpr ALWAYS_INLINE bool IsContiguous() const { return this->GetBits(52, 1) != 0; }
constexpr ALWAYS_INLINE bool IsGlobal() const { return this->GetBits(11, 1) == 0; }
constexpr ALWAYS_INLINE AccessFlag GetAccessFlag() const { return static_cast<AccessFlag>(this->GetBits(10, 1)); }
constexpr ALWAYS_INLINE Shareable GetShareable() const { return static_cast<Shareable>(this->GetBits(8, 2)); }
constexpr ALWAYS_INLINE PageAttribute GetPageAttribute() const { return static_cast<PageAttribute>(this->GetBits(2, 3)); }
constexpr ALWAYS_INLINE AccessFlag GetAccessFlag() const { return static_cast<AccessFlag>(this->SelectBits(10, 1)); }
constexpr ALWAYS_INLINE Shareable GetShareable() const { return static_cast<Shareable>(this->SelectBits(8, 2)); }
constexpr ALWAYS_INLINE PageAttribute GetPageAttribute() const { return static_cast<PageAttribute>(this->SelectBits(2, 3)); }
constexpr ALWAYS_INLINE int GetAccessFlagInteger() const { return static_cast<int>(this->GetBits(10, 1)); }
constexpr ALWAYS_INLINE int GetShareableInteger() const { return static_cast<int>(this->GetBits(8, 2)); }
constexpr ALWAYS_INLINE int GetPageAttributeInteger() const { return static_cast<int>(this->GetBits(2, 3)); }
constexpr ALWAYS_INLINE bool IsReadOnly() const { return this->GetBits(7, 1) != 0; }
constexpr ALWAYS_INLINE bool IsUserAccessible() const { return this->GetBits(6, 1) != 0; }
constexpr ALWAYS_INLINE bool IsNonSecure() const { return this->GetBits(5, 1) != 0; }

5
libraries/libmesosphere/include/mesosphere/board/nintendo/nx/kern_k_system_control.hpp
View file

@ -25,12 +25,15 @@ namespace ams::kern::board::nintendo::nx {
static constexpr size_t SecureAppletMemorySize = 4_MB;
public:
class Init : public KSystemControlBase::Init {
private:
friend class KSystemControlBase::Init;
private:
static void CpuOnImpl(u64 core_id, uintptr_t entrypoint, uintptr_t arg);
public:
/* Initialization. */
static size_t GetRealMemorySize();
static size_t GetIntendedMemorySize();
static bool ShouldIncreaseThreadResourceLimit();
static void CpuOn(u64 core_id, uintptr_t entrypoint, uintptr_t arg);
static size_t GetApplicationPoolSize();
static size_t GetAppletPoolSize();
static size_t GetMinimumNonSecureSystemPoolSize();

2
libraries/libmesosphere/include/mesosphere/init/kern_init_arguments_select.hpp
View file

@ -26,6 +26,6 @@ namespace ams::kern::init {
static_assert(util::IsPowerOfTwo(alignof(KInitArguments)) && util::IsPowerOfTwo(sizeof(KInitArguments)));
KPhysicalAddress GetInitArgumentsAddress(s32 core_id);
KInitArguments *GetInitArguments(s32 core_id);
}

6
libraries/libmesosphere/include/mesosphere/kern_k_memory_layout.hpp
View file

@ -183,10 +183,10 @@ namespace ams::kern {
return std::make_tuple(total_size, kernel_size);
}
static void InitializeLinearMemoryAddresses(KPhysicalAddress aligned_linear_phys_start, KVirtualAddress linear_virtual_start) {
static void InitializeLinearMemoryAddresses(u64 phys_to_virt_diff) {
/* Set static differences. */
s_linear_phys_to_virt_diff = GetInteger(linear_virtual_start) - GetInteger(aligned_linear_phys_start);
s_linear_virt_to_phys_diff = GetInteger(aligned_linear_phys_start) - GetInteger(linear_virtual_start);
s_linear_phys_to_virt_diff = phys_to_virt_diff;
s_linear_virt_to_phys_diff = -phys_to_virt_diff;
}
static void InitializeLinearMemoryRegionTrees();

10
libraries/libmesosphere/include/mesosphere/kern_k_system_control_base.hpp
View file

@ -21,6 +21,12 @@ namespace ams::kern {
struct InitialProcessBinaryLayout;
namespace init {
struct KInitArguments;
}
}
namespace ams::kern {
@ -40,6 +46,8 @@ namespace ams::kern {
static constinit inline KSpinLock s_random_lock;
public:
class Init {
private:
static void CpuOnImpl(u64 core_id, uintptr_t entrypoint, uintptr_t arg);
public:
/* Initialization. */
static size_t GetRealMemorySize();
@ -47,7 +55,7 @@ namespace ams::kern {
static KPhysicalAddress GetKernelPhysicalBaseAddress(KPhysicalAddress base_address);
static void GetInitialProcessBinaryLayout(InitialProcessBinaryLayout *out);
static bool ShouldIncreaseThreadResourceLimit();
static void CpuOn(u64 core_id, uintptr_t entrypoint, uintptr_t arg);
static void TurnOnCpu(u64 core_id, const ams::kern::init::KInitArguments *args);
static size_t GetApplicationPoolSize();
static size_t GetAppletPoolSize();
static size_t GetMinimumNonSecureSystemPoolSize();

18
libraries/libmesosphere/source/arch/arm64/kern_k_page_table_impl.cpp
View file

@ -353,12 +353,12 @@ namespace ams::kern::arch::arm64 {
l1_entry->IsPrivilegedExecuteNever(),
l1_entry->IsContiguous(),
!l1_entry->IsGlobal(),
static_cast<int>(l1_entry->GetAccessFlag()),
static_cast<unsigned int>(l1_entry->GetShareable()),
static_cast<int>(l1_entry->GetAccessFlagInteger()),
static_cast<unsigned int>(l1_entry->GetShareableInteger()),
l1_entry->IsReadOnly(),
l1_entry->IsUserAccessible(),
l1_entry->IsNonSecure(),
static_cast<int>(l1_entry->GetPageAttribute()),
static_cast<int>(l1_entry->GetPageAttributeInteger()),
l1_entry->IsHeadMergeDisabled(),
l1_entry->IsHeadAndBodyMergeDisabled(),
l1_entry->IsTailMergeDisabled());
@ -398,12 +398,12 @@ namespace ams::kern::arch::arm64 {
l2_entry->IsPrivilegedExecuteNever(),
l2_entry->IsContiguous(),
!l2_entry->IsGlobal(),
static_cast<int>(l2_entry->GetAccessFlag()),
static_cast<unsigned int>(l2_entry->GetShareable()),
static_cast<int>(l2_entry->GetAccessFlagInteger()),
static_cast<unsigned int>(l2_entry->GetShareableInteger()),
l2_entry->IsReadOnly(),
l2_entry->IsUserAccessible(),
l2_entry->IsNonSecure(),
static_cast<int>(l2_entry->GetPageAttribute()),
static_cast<int>(l2_entry->GetPageAttributeInteger()),
l2_entry->IsHeadMergeDisabled(),
l2_entry->IsHeadAndBodyMergeDisabled(),
l2_entry->IsTailMergeDisabled());
@ -443,12 +443,12 @@ namespace ams::kern::arch::arm64 {
l3_entry->IsPrivilegedExecuteNever(),
l3_entry->IsContiguous(),
!l3_entry->IsGlobal(),
static_cast<int>(l3_entry->GetAccessFlag()),
static_cast<unsigned int>(l3_entry->GetShareable()),
static_cast<int>(l3_entry->GetAccessFlagInteger()),
static_cast<unsigned int>(l3_entry->GetShareableInteger()),
l3_entry->IsReadOnly(),
l3_entry->IsUserAccessible(),
l3_entry->IsNonSecure(),
static_cast<int>(l3_entry->GetPageAttribute()),
static_cast<int>(l3_entry->GetPageAttributeInteger()),
l3_entry->IsHeadMergeDisabled(),
l3_entry->IsHeadAndBodyMergeDisabled(),
l3_entry->IsTailMergeDisabled());

53
libraries/libmesosphere/source/board/nintendo/nx/kern_k_sleep_manager.cpp
View file

@ -18,6 +18,12 @@
#include "kern_secure_monitor.hpp"
#include "kern_lps_driver.hpp"
namespace ams::kern::init {
void StartOtherCore(const ams::kern::init::KInitArguments *init_args);
}
namespace ams::kern::board::nintendo::nx {
namespace {
@ -67,15 +73,10 @@ namespace ams::kern::board::nintendo::nx {
constinit KLightLock g_request_lock;
constinit KLightLock g_cv_lock;
constinit KLightConditionVariable g_cv{util::ConstantInitialize};
constinit KPhysicalAddress g_sleep_buffer_phys_addrs[cpu::NumCores];
alignas(1_KB) constinit u64 g_sleep_buffers[cpu::NumCores][1_KB / sizeof(u64)];
constinit ams::kern::init::KInitArguments g_sleep_init_arguments[cpu::NumCores];
constinit SavedSystemRegisters g_sleep_system_registers[cpu::NumCores] = {};
void PowerOnCpu(int core_id, KPhysicalAddress entry_phys_addr, u64 context_id) {
/* Request the secure monitor power on the core. */
::ams::kern::arch::arm64::smc::CpuOn<smc::SmcId_Supervisor, true>(cpu::MultiprocessorAffinityRegisterAccessor().GetCpuOnArgument() | core_id, GetInteger(entry_phys_addr), context_id);
}
void WaitOtherCpuPowerOff() {
constexpr u64 PmcPhysicalAddress = 0x7000E400;
constexpr u32 PWRGATE_STATUS_CE123_MASK = ((1u << 3) - 1) << 9;
@ -473,18 +474,20 @@ namespace ams::kern::board::nintendo::nx {
}
}
void KSleepManager::ProcessRequests(uintptr_t buffer) {
void KSleepManager::ProcessRequests(uintptr_t sleep_buffer) {
const auto target_fw = GetTargetFirmware();
const s32 core_id = GetCurrentCoreId();
KPhysicalAddress resume_entry_phys_addr = Null<KPhysicalAddress>;
ams::kern::init::KInitArguments * const init_args = g_sleep_init_arguments + core_id;
KPhysicalAddress start_core_phys_addr = Null<KPhysicalAddress>;
KPhysicalAddress init_args_phys_addr = Null<KPhysicalAddress>;
/* Get the physical addresses we'll need. */
{
MESOSPHERE_ABORT_UNLESS(Kernel::GetKernelPageTable().GetPhysicalAddress(std::addressof(g_sleep_buffer_phys_addrs[core_id]), KProcessAddress(buffer)));
MESOSPHERE_ABORT_UNLESS(Kernel::GetKernelPageTable().GetPhysicalAddress(std::addressof(resume_entry_phys_addr), KProcessAddress(&::ams::kern::board::nintendo::nx::KSleepManager::ResumeEntry)));
MESOSPHERE_ABORT_UNLESS(Kernel::GetKernelPageTable().GetPhysicalAddress(std::addressof(start_core_phys_addr), KProcessAddress(&::ams::kern::init::StartOtherCore)));
MESOSPHERE_ABORT_UNLESS(Kernel::GetKernelPageTable().GetPhysicalAddress(std::addressof(init_args_phys_addr), KProcessAddress(init_args)));
}
const KPhysicalAddress sleep_buffer_phys_addr = g_sleep_buffer_phys_addrs[core_id];
const u64 target_core_mask = (1ul << core_id);
const bool use_legacy_lps_driver = target_fw < TargetFirmware_2_0_0;
@ -547,15 +550,29 @@ namespace ams::kern::board::nintendo::nx {
/* Save the interrupt manager's state. */
Kernel::GetInterruptManager().Save(core_id);
/* Setup the initial arguments. */
{
init_args->ttbr0 = cpu::GetTtbr0El1();
init_args->ttbr1 = cpu::GetTtbr1El1();
init_args->tcr = cpu::GetTcrEl1();
init_args->mair = cpu::GetMairEl1();
init_args->cpuactlr = cpu::GetCpuActlrEl1();
init_args->cpuectlr = cpu::GetCpuEctlrEl1();
init_args->sctlr = cpu::GetSctlrEl1();
init_args->sp = 0;
init_args->entrypoint = reinterpret_cast<uintptr_t>(::ams::kern::board::nintendo::nx::KSleepManager::ResumeEntry);
init_args->argument = sleep_buffer;
}
/* Ensure that all cores get to this point before continuing. */
cpu::SynchronizeAllCores();
/* Log that the core is going to sleep. */
MESOSPHERE_LOG("Core[%d]: Going to sleep, buffer = %010lx\n", core_id, GetInteger(sleep_buffer_phys_addr));
MESOSPHERE_LOG("Core[%d]: Going to sleep, buffer = %010lx\n", core_id, sleep_buffer);
/* If we're on a core other than zero, we can just invoke the sleep handler. */
if (core_id != 0) {
CpuSleepHandler(GetInteger(sleep_buffer_phys_addr), GetInteger(resume_entry_phys_addr));
CpuSleepHandler(sleep_buffer, GetInteger(start_core_phys_addr), GetInteger(init_args_phys_addr));
} else {
/* Wait for all other cores to be powered off. */
WaitOtherCpuPowerOff();
@ -574,9 +591,9 @@ namespace ams::kern::board::nintendo::nx {
/* Invoke the sleep handler. */
if (!use_legacy_lps_driver) {
/* When not using the legacy driver, invoke directly. */
CpuSleepHandler(GetInteger(sleep_buffer_phys_addr), GetInteger(resume_entry_phys_addr));
CpuSleepHandler(sleep_buffer, GetInteger(start_core_phys_addr), GetInteger(init_args_phys_addr));
} else {
lps::InvokeCpuSleepHandler(GetInteger(sleep_buffer_phys_addr), GetInteger(resume_entry_phys_addr));
lps::InvokeCpuSleepHandler(sleep_buffer, GetInteger(start_core_phys_addr), GetInteger(init_args_phys_addr));
}
/* Restore the debug log state. */
@ -586,8 +603,10 @@ namespace ams::kern::board::nintendo::nx {
MESOSPHERE_LOG("Exiting SC7\n");
/* Wake up the other cores. */
cpu::MultiprocessorAffinityRegisterAccessor mpidr;
const auto arg = mpidr.GetCpuOnArgument();
for (s32 i = 1; i < static_cast<s32>(cpu::NumCores); ++i) {
PowerOnCpu(i, resume_entry_phys_addr, GetInteger(g_sleep_buffer_phys_addrs[i]));
KSystemControl::Init::TurnOnCpu(arg | i, g_sleep_init_arguments + i);
}
}

4
libraries/libmesosphere/source/board/nintendo/nx/kern_k_sleep_manager.hpp
View file

@ -22,14 +22,12 @@ namespace ams::kern::board::nintendo::nx {
private:
static void ResumeEntry(uintptr_t arg);
static void InvalidateDataCacheForResumeEntry(uintptr_t level);
static void ProcessRequests(uintptr_t buffer);
public:
static void Initialize();
static void SleepSystem();
public:
static void CpuSleepHandler(uintptr_t arg, uintptr_t entry);
static void CpuSleepHandler(uintptr_t arg, uintptr_t entry, uintptr_t entry_args);
};

218
libraries/libmesosphere/source/board/nintendo/nx/kern_k_sleep_manager_asm.s
View file

@ -22,14 +22,14 @@
mov reg, #(((val) >> 0x00) & 0xFFFF); \
movk reg, #(((val) >> 0x10) & 0xFFFF), lsl#16
/* ams::kern::board::nintendo::nx::KSleepManager::CpuSleepHandler(uintptr_t arg, uintptr_t entry) */
.section .sleep._ZN3ams4kern5board8nintendo2nx13KSleepManager15CpuSleepHandlerEmm, "ax", %progbits
.global _ZN3ams4kern5board8nintendo2nx13KSleepManager15CpuSleepHandlerEmm
.type _ZN3ams4kern5board8nintendo2nx13KSleepManager15CpuSleepHandlerEmm, %function
_ZN3ams4kern5board8nintendo2nx13KSleepManager15CpuSleepHandlerEmm:
/* ams::kern::board::nintendo::nx::KSleepManager::CpuSleepHandler(uintptr_t arg, uintptr_t entry, uintptr_t entry_arg) */
.section .sleep._ZN3ams4kern5board8nintendo2nx13KSleepManager15CpuSleepHandlerEmmm, "ax", %progbits
.global _ZN3ams4kern5board8nintendo2nx13KSleepManager15CpuSleepHandlerEmmm
.type _ZN3ams4kern5board8nintendo2nx13KSleepManager15CpuSleepHandlerEmmm, %function
_ZN3ams4kern5board8nintendo2nx13KSleepManager15CpuSleepHandlerEmmm:
/* Save arguments. */
mov x16, x0
mov x17, x1
mov x16, x1
mov x17, x2
/* Enable access to FPU registers. */
mrs x1, cpacr_el1
@ -74,28 +74,8 @@ _ZN3ams4kern5board8nintendo2nx13KSleepManager15CpuSleepHandlerEmm:
stp q28, q29, [x0], #0x20
stp q30, q31, [x0], #0x20
/* Save cpuactlr/cpuectlr. */
mrs x1, cpuectlr_el1
mrs x2, cpuactlr_el1
stp x1, x2, [x0], #0x10
/* Save ttbr0/ttbr1. */
mrs x1, ttbr0_el1
mrs x2, ttbr1_el1
stp x1, x2, [x0], #0x10
/* Save tcr/mair. */
mrs x1, tcr_el1
mrs x2, mair_el1
stp x1, x2, [x0], #0x10
/* Save sctlr/tpidr. */
mrs x1, sctlr_el1
mrs x2, tpidr_el1
stp x1, x2, [x0], #0x10
/* Save the virtual resumption entrypoint and cntv_cval_el0. */
adr x1, 77f
/* Save tpidr/cntv_cval_el0. */
mrs x1, tpidr_el1
mrs x2, cntv_cval_el0
stp x1, x2, [x0], #0x10
@ -114,8 +94,8 @@ _ZN3ams4kern5board8nintendo2nx13KSleepManager15CpuSleepHandlerEmm:
1: /* Suspend. */
LOAD_IMMEDIATE_32(x0, 0xC4000001)
LOAD_IMMEDIATE_32(x1, 0x0201001B)
mov x2, x17
mov x3, x16
mov x2, x16
mov x3, x17
smc #1
0: b 0b
@ -124,65 +104,6 @@ _ZN3ams4kern5board8nintendo2nx13KSleepManager15CpuSleepHandlerEmm:
.global _ZN3ams4kern5board8nintendo2nx13KSleepManager11ResumeEntryEm
.type _ZN3ams4kern5board8nintendo2nx13KSleepManager11ResumeEntryEm, %function
_ZN3ams4kern5board8nintendo2nx13KSleepManager11ResumeEntryEm:
/* Mask interrupts. */
msr daifset, #0xF
/* Save the argument. */
mov x21, x0
/* Check that we're at the correct exception level. */
mrs x0, currentel
/* Check if we're EL1. */
cmp x0, #0x4
b.eq 3f
/* Check if we're EL2. */
cmp x0, #0x8
b.eq 2f
1: /* We're running at EL3. */
b 1b
2: /* We're running at EL2. */
b 2b
3: /* We're running at EL1. */
/* Invalidate the L1 cache. */
mov x0, #0
bl _ZN3ams4kern5board8nintendo2nx13KSleepManager33InvalidateDataCacheForResumeEntryEm
/* Get the current core id. */
mrs x0, mpidr_el1
and x0, x0, #0xFF
/* If we're on core0, we want to invalidate the L2 cache. */
cbnz x0, 4f
mov x0, #1
bl _ZN3ams4kern5board8nintendo2nx13KSleepManager33InvalidateDataCacheForResumeEntryEm
4: /* Invalidate the L1 cache. */
mov x0, #0
bl _ZN3ams4kern5board8nintendo2nx13KSleepManager33InvalidateDataCacheForResumeEntryEm
/* Invalidate the instruction cache. */
ic ialluis
dsb sy
isb
/* Invalidate the entire tlb. */
tlbi vmalle1is
dsb sy
isb
/* Switch to sp 1. */
msr spsel, #1
/* Prepare to restore the saved context. */
mov x0, x21
/* Enable access to FPU registers. */
mrs x1, cpacr_el1
orr x1, x1, #0x100000
@ -226,121 +147,12 @@ _ZN3ams4kern5board8nintendo2nx13KSleepManager11ResumeEntryEm:
ldp q28, q29, [x0], #0x20
ldp q30, q31, [x0], #0x20
/* Restore cpuactlr/cpuectlr. */
/* Restore tpidr/cntv_cval_el0. */
ldp x1, x2, [x0], #0x10
mrs x3, cpuectlr_el1
cmp x1, x3
5: b.ne 5b
mrs x3, cpuactlr_el1
cmp x2, x3
6: b.ne 6b
/* Restore ttbr0/ttbr1. */
ldp x1, x2, [x0], #0x10
msr ttbr0_el1, x1
msr ttbr1_el1, x2
/* Restore tcr/mair. */
ldp x1, x2, [x0], #0x10
msr tcr_el1, x1
msr mair_el1, x2
/* Get sctlr, tpidr, the entrypoint, and cntv_cval_el0. */
ldp x1, x2, [x0], #0x10
ldp x3, x4, [x0], #0x10
/* Set the global context back into x18/tpidr. */
msr tpidr_el1, x2
msr cntv_cval_el0, x4
dsb sy
isb
/* Restore sctlr with the wxn bit cleared. */
bic x2, x1, #0x80000
msr sctlr_el1, x2
dsb sy
isb
/* Jump to the entrypoint. */
br x3
77: /* Virtual resumption entrypoint. */
/* Restore sctlr. */
msr sctlr_el1, x1
msr tpidr_el1, x1
msr cntv_cval_el0, x2
dsb sy
isb
ret
/* ams::kern::board::nintendo::nx::KSleepManager::InvalidateDataCacheForResumeEntry(uintptr_t level) */
.section .sleep._ZN3ams4kern5board8nintendo2nx13KSleepManager33InvalidateDataCacheForResumeEntryEm, "ax", %progbits
.global _ZN3ams4kern5board8nintendo2nx13KSleepManager33InvalidateDataCacheForResumeEntryEm
.type _ZN3ams4kern5board8nintendo2nx13KSleepManager33InvalidateDataCacheForResumeEntryEm, %function
_ZN3ams4kern5board8nintendo2nx13KSleepManager33InvalidateDataCacheForResumeEntryEm:
/* cpu::DataSynchronizationBarrier(); */
dsb sy
/* const u64 level_sel_value = level << 1; */
lsl x8, x0, #1
/* cpu::SetCsselrEl1(level_sel_value); */
msr csselr_el1, x8
/* cpu::InstructionMemoryBarrier(); */
isb
/* CacheSizeIdAccessor ccsidr_el1; */
mrs x13, ccsidr_el1
/* const int num_ways = ccsidr_el1.GetAssociativity(); */
ubfx w10, w13, #3, #0xA
/* const int line_size = ccsidr_el1.GetLineSize(); */
and w11, w13, #7
/* const int num_sets = ccsidr_el1.GetNumberOfSets(); */
ubfx w13, w13, #0xD, #0xF
/* int way = 0; */
mov w9, wzr
/* const u64 set_shift = static_cast<u64>(line_size + 4); */
add w11, w11, #4
/* const u64 way_shift = static_cast<u64>(__builtin_clz(num_ways)); */
clz w12, w10
0: /* do { */
/* int set = 0; */
mov w14, wzr
/* const u64 way_value = (static_cast<u64>(way) << way_shift); */
lsl w15, w9, w12
1: /* do { */
/* const u64 isw_value = (static_cast<u64>(set) << set_shift) | way_value | level_sel_value; */
lsl w16, w14, w11
orr w16, w16, w15
sxtw x16, w16
orr x16, x16, x8
/* __asm__ __volatile__("dc isw, %0" :: "r"(isw_value) : "memory"); */
dc isw, x16
/* while (set <= num_sets); */
cmp w13, w14
add w14, w14, #1
b.ne 1b
/* while (way <= num_ways); */
cmp w9, w10
add w9, w9, #1
b.ne 0b
/* cpu::EnsureInstructionConsistency(); */
dsb sy
isb
/* Return. */
ret

2
libraries/libmesosphere/source/board/nintendo/nx/kern_k_system_control.cpp
View file

@ -382,7 +382,7 @@ namespace ams::kern::board::nintendo::nx {
return static_cast<u8>((value >> 32) & 0xFF);
}
void KSystemControl::Init::CpuOn(u64 core_id, uintptr_t entrypoint, uintptr_t arg) {
void KSystemControl::Init::CpuOnImpl(u64 core_id, uintptr_t entrypoint, uintptr_t arg) {
MESOSPHERE_INIT_ABORT_UNLESS((::ams::kern::arch::arm64::smc::CpuOn<smc::SmcId_Supervisor, false>(core_id, entrypoint, arg)) == 0);
}

4
libraries/libmesosphere/source/board/nintendo/nx/kern_lps_driver.cpp
View file

@ -395,7 +395,7 @@ namespace ams::kern::board::nintendo::nx::lps {
R_SUCCEED();
}
void InvokeCpuSleepHandler(uintptr_t arg, uintptr_t entry) {
void InvokeCpuSleepHandler(uintptr_t arg, uintptr_t entry, uintptr_t entry_arg) {
/* Verify that we're allowed to perform suspension. */
MESOSPHERE_ABORT_UNLESS(g_lps_init_done);
MESOSPHERE_ABORT_UNLESS(GetCurrentCoreId() == 0);
@ -416,7 +416,7 @@ namespace ams::kern::board::nintendo::nx::lps {
Read(g_pmc_address + APBDEV_PMC_SCRATCH0);
/* Invoke the sleep hander. */
KSleepManager::CpuSleepHandler(arg, entry);
KSleepManager::CpuSleepHandler(arg, entry, entry_arg);
/* Disable deep power down. */
Write(g_pmc_address + APBDEV_PMC_DPD_ENABLE, 0);

2
libraries/libmesosphere/source/board/nintendo/nx/kern_lps_driver.hpp
View file

@ -22,7 +22,7 @@ namespace ams::kern::board::nintendo::nx {
void Initialize();
Result EnableSuspend(bool enable);
void InvokeCpuSleepHandler(uintptr_t arg, uintptr_t entry);
void InvokeCpuSleepHandler(uintptr_t arg, uintptr_t entry, uintptr_t entry_arg);
void ResumeBpmpFirmware();
}

25
libraries/libmesosphere/source/kern_k_system_control_base.cpp
View file

@ -20,6 +20,13 @@
namespace ams::kern {
namespace init {
/* TODO: Is this function name architecture specific? */
void StartOtherCore(const ams::kern::init::KInitArguments *init_args);
}
/* Initialization. */
size_t KSystemControlBase::Init::GetRealMemorySize() {
return ams::kern::MainMemorySize;
@ -68,7 +75,7 @@ namespace ams::kern {
return 0;
}
void KSystemControlBase::Init::CpuOn(u64 core_id, uintptr_t entrypoint, uintptr_t arg) {
void KSystemControlBase::Init::CpuOnImpl(u64 core_id, uintptr_t entrypoint, uintptr_t arg) {
#if defined(ATMOSPHERE_ARCH_ARM64)
MESOSPHERE_INIT_ABORT_UNLESS((::ams::kern::arch::arm64::smc::CpuOn<0, false>(core_id, entrypoint, arg)) == 0);
#else
@ -76,6 +83,22 @@ namespace ams::kern {
#endif
}
void KSystemControlBase::Init::TurnOnCpu(u64 core_id, const ams::kern::init::KInitArguments *args) {
/* Get entrypoint. */
KPhysicalAddress entrypoint = Null<KPhysicalAddress>;
while (!cpu::GetPhysicalAddressReadable(std::addressof(entrypoint), reinterpret_cast<uintptr_t>(::ams::kern::init::StartOtherCore), true)) { /* ... */ }
/* Get arguments. */
KPhysicalAddress args_addr = Null<KPhysicalAddress>;
while (!cpu::GetPhysicalAddressReadable(std::addressof(args_addr), reinterpret_cast<uintptr_t>(args), true)) { /* ... */ }
/* Ensure cache is correct for the initial arguments. */
cpu::StoreDataCacheForInitArguments(args, sizeof(*args));
/* Turn on the cpu. */
KSystemControl::Init::CpuOnImpl(core_id, GetInteger(entrypoint), GetInteger(args_addr));
}
/* Randomness for Initialization. */
void KSystemControlBase::Init::GenerateRandom(u64 *dst, size_t count) {
if (AMS_UNLIKELY(!s_initialized_random_generator)) {