mirrors/Atmosphere
1
0
Fork 0
mirror of https://github.com/Atmosphere-NX/Atmosphere.git synced 2025-05-31 14:58:22 -04:00
Code Issues Projects Releases Packages Wiki Activity

kern: update for new hw maintenance semantics

Browse source
This commit is contained in:
Michael Scire 2022-03-22 21:33:43 -07:00 • committed by SciresM
parent 6e17317d5d
commit 9d89835ff8
19 changed files with 386 additions and 244 deletions
Download patch file Download diff file Expand all files Collapse all files

22
libraries/libmesosphere/include/mesosphere/arch/arm64/init/kern_k_init_page_table.hpp
View file

@ -279,20 +279,21 @@ namespace ams::kern::arch::arm64::init {
/* Invalidate the entire tlb. */
cpu::DataSynchronizationBarrierInnerShareable();
cpu::InvalidateEntireTlbInnerShareable();
cpu::InvalidateEntireTlb();
/* Copy data, if we should. */
const u64 negative_block_size_for_mask = static_cast<u64>(-static_cast<s64>(block_size));
const u64 offset_mask = negative_block_size_for_mask & ((1ul << 48) - 1);
const KVirtualAddress copy_src_addr = KVirtualAddress(src_saved.GetRawAttributesUnsafeForSwap() & offset_mask);
const KVirtualAddress copy_dst_addr = KVirtualAddress(dst_saved.GetRawAttributesUnsafeForSwap() & offset_mask);
if (block_size && do_copy) {
if (do_copy) {
u8 tmp[0x100];
for (size_t ofs = 0; ofs < block_size; ofs += sizeof(tmp)) {
std::memcpy(tmp, GetVoidPointer(copy_src_addr + ofs), sizeof(tmp));
std::memcpy(GetVoidPointer(copy_src_addr + ofs), GetVoidPointer(copy_dst_addr + ofs), sizeof(tmp));
std::memcpy(GetVoidPointer(copy_dst_addr + ofs), tmp, sizeof(tmp));
}
cpu::DataSynchronizationBarrierInnerShareable();
}
/* Swap the mappings. */
@ -339,7 +340,6 @@ namespace ams::kern::arch::arm64::init {
/* Can we make an L1 block? */
if (util::IsAligned(GetInteger(virt_addr), L1BlockSize) && util::IsAligned(GetInteger(phys_addr), L1BlockSize) && size >= L1BlockSize) {
*l1_entry = L1PageTableEntry(PageTableEntry::BlockTag{}, phys_addr, attr, PageTableEntry::SoftwareReservedBit_None, false);
cpu::DataSynchronizationBarrierInnerShareable();
virt_addr += L1BlockSize;
phys_addr += L1BlockSize;
@ -350,8 +350,8 @@ namespace ams::kern::arch::arm64::init {
/* If we don't already have an L2 table, we need to make a new one. */
if (!l1_entry->IsTable()) {
KPhysicalAddress new_table = AllocateNewPageTable(allocator);
*l1_entry = L1PageTableEntry(PageTableEntry::TableTag{}, new_table, attr.IsPrivilegedExecuteNever());
cpu::DataSynchronizationBarrierInnerShareable();
*l1_entry = L1PageTableEntry(PageTableEntry::TableTag{}, new_table, attr.IsPrivilegedExecuteNever());
}
L2PageTableEntry *l2_entry = GetL2Entry(l1_entry, virt_addr);
@ -365,14 +365,12 @@ namespace ams::kern::arch::arm64::init {
phys_addr += L2BlockSize;
size -= L2BlockSize;
}
cpu::DataSynchronizationBarrierInnerShareable();
continue;
}
/* Can we make an L2 block? */
if (util::IsAligned(GetInteger(virt_addr), L2BlockSize) && util::IsAligned(GetInteger(phys_addr), L2BlockSize) && size >= L2BlockSize) {
*l2_entry = L2PageTableEntry(PageTableEntry::BlockTag{}, phys_addr, attr, PageTableEntry::SoftwareReservedBit_None, false);
cpu::DataSynchronizationBarrierInnerShareable();
virt_addr += L2BlockSize;
phys_addr += L2BlockSize;
@ -383,8 +381,8 @@ namespace ams::kern::arch::arm64::init {
/* If we don't already have an L3 table, we need to make a new one. */
if (!l2_entry->IsTable()) {
KPhysicalAddress new_table = AllocateNewPageTable(allocator);
*l2_entry = L2PageTableEntry(PageTableEntry::TableTag{}, new_table, attr.IsPrivilegedExecuteNever());
cpu::DataSynchronizationBarrierInnerShareable();
*l2_entry = L2PageTableEntry(PageTableEntry::TableTag{}, new_table, attr.IsPrivilegedExecuteNever());
}
L3PageTableEntry *l3_entry = GetL3Entry(l2_entry, virt_addr);
@ -398,17 +396,18 @@ namespace ams::kern::arch::arm64::init {
phys_addr += L3BlockSize;
size -= L3BlockSize;
}
cpu::DataSynchronizationBarrierInnerShareable();
continue;
}
/* Make an L3 block. */
*l3_entry = L3PageTableEntry(PageTableEntry::BlockTag{}, phys_addr, attr, PageTableEntry::SoftwareReservedBit_None, false);
cpu::DataSynchronizationBarrierInnerShareable();
virt_addr += L3BlockSize;
phys_addr += L3BlockSize;
size -= L3BlockSize;
}
/* Ensure data consistency after our mapping is added. */
cpu::DataSynchronizationBarrierInnerShareable();
}
KPhysicalAddress GetPhysicalAddress(KVirtualAddress virt_addr) const {
@ -556,9 +555,6 @@ namespace ams::kern::arch::arm64::init {
}
void Reprotect(KVirtualAddress virt_addr, size_t size, const PageTableEntry &attr_before, const PageTableEntry &attr_after) {
/* Ensure data consistency before we begin reprotection. */
cpu::DataSynchronizationBarrierInnerShareable();
/* Ensure that addresses and sizes are page aligned. */
MESOSPHERE_INIT_ABORT_UNLESS(util::IsAligned(GetInteger(virt_addr), PageSize));
MESOSPHERE_INIT_ABORT_UNLESS(util::IsAligned(size, PageSize));
@ -699,7 +695,7 @@ namespace ams::kern::arch::arm64::init {
this->PhysicallyRandomize(virt_addr, size, L2BlockSize, do_copy);
this->PhysicallyRandomize(virt_addr, size, L3ContiguousBlockSize, do_copy);
this->PhysicallyRandomize(virt_addr, size, L3BlockSize, do_copy);
cpu::StoreEntireCacheForInit();
cpu::StoreCacheForInit(GetVoidPointer(virt_addr), size);
}
};

34
libraries/libmesosphere/include/mesosphere/arch/arm64/kern_cpu.hpp
View file

@ -48,6 +48,10 @@ namespace ams::kern::arch::arm64::cpu {
__asm__ __volatile__("dsb ish" ::: "memory");
}
ALWAYS_INLINE void DataSynchronizationBarrierInnerShareableStore() {
__asm__ __volatile__("dsb ishst" ::: "memory");
}
ALWAYS_INLINE void DataMemoryBarrier() {
__asm__ __volatile__("dmb sy" ::: "memory");
}
@ -56,16 +60,20 @@ namespace ams::kern::arch::arm64::cpu {
__asm__ __volatile__("dmb ish" ::: "memory");
}
ALWAYS_INLINE void DataMemoryBarrierInnerShareableStore() {
__asm__ __volatile__("dmb ishst" ::: "memory");
}
ALWAYS_INLINE void InstructionMemoryBarrier() {
__asm__ __volatile__("isb" ::: "memory");
}
ALWAYS_INLINE void EnsureInstructionConsistencyInnerShareable() {
ALWAYS_INLINE void EnsureInstructionConsistency() {
DataSynchronizationBarrierInnerShareable();
InstructionMemoryBarrier();
}
ALWAYS_INLINE void EnsureInstructionConsistency() {
ALWAYS_INLINE void EnsureInstructionConsistencyFullSystem() {
DataSynchronizationBarrier();
InstructionMemoryBarrier();
}
@ -182,28 +190,23 @@ namespace ams::kern::arch::arm64::cpu {
NOINLINE void SynchronizeAllCores();
/* Cache management helpers. */
void StoreEntireCacheForInit();
void FlushEntireCacheForInit();
void StoreCacheForInit(void *addr, size_t size);
void FlushEntireDataCache();
Result InvalidateDataCache(void *addr, size_t size);
Result StoreDataCache(const void *addr, size_t size);
Result FlushDataCache(const void *addr, size_t size);
Result InvalidateInstructionCache(void *addr, size_t size);
void InvalidateEntireInstructionCache();
void ClearPageToZeroImpl(void *);
ALWAYS_INLINE void ClearPageToZero(void * const page) {
MESOSPHERE_ASSERT(util::IsAligned(reinterpret_cast<uintptr_t>(page), PageSize));
MESOSPHERE_ASSERT(page != nullptr);
uintptr_t cur = reinterpret_cast<uintptr_t>(__builtin_assume_aligned(page, PageSize));
const uintptr_t last = cur + PageSize - DataCacheLineSize;
for (/* ... */; cur <= last; cur += DataCacheLineSize) {
__asm__ __volatile__("dc zva, %[cur]" :: [cur]"r"(cur) : "memory");
}
ClearPageToZeroImpl(page);
}
ALWAYS_INLINE void InvalidateTlbByAsid(u32 asid) {
@ -223,20 +226,15 @@ namespace ams::kern::arch::arm64::cpu {
EnsureInstructionConsistency();
}
ALWAYS_INLINE void InvalidateEntireTlbInnerShareable() {
__asm__ __volatile__("tlbi vmalle1is" ::: "memory");
EnsureInstructionConsistencyInnerShareable();
}
ALWAYS_INLINE void InvalidateEntireTlbDataOnly() {
__asm__ __volatile__("tlbi vmalle1is" ::: "memory");
DataSynchronizationBarrier();
DataSynchronizationBarrierInnerShareable();
}
ALWAYS_INLINE void InvalidateTlbByVaDataOnly(KProcessAddress virt_addr) {
const u64 value = ((GetInteger(virt_addr) >> 12) & 0xFFFFFFFFFFFul);
__asm__ __volatile__("tlbi vaae1is, %[value]" :: [value]"r"(value) : "memory");
DataSynchronizationBarrier();
DataSynchronizationBarrierInnerShareable();
}
ALWAYS_INLINE uintptr_t GetCurrentThreadPointerValue() {

2
libraries/libmesosphere/include/mesosphere/arch/arm64/kern_k_interrupt_manager.hpp
View file

@ -96,8 +96,6 @@ namespace ams::kern::arch::arm64 {
}
static void HandleInterrupt(bool user_mode);
/* Implement more KInterruptManager functionality. */
private:
Result BindGlobal(KInterruptHandler *handler, s32 irq, s32 core_id, s32 priority, bool manual_clear, bool level);
Result BindLocal(KInterruptHandler *handler, s32 irq, s32 priority, bool manual_clear);

24
libraries/libmesosphere/include/mesosphere/arch/arm64/kern_k_page_table.hpp
View file

@ -174,7 +174,6 @@ namespace ams::kern::arch::arm64 {
static NOINLINE void Initialize(s32 core_id);
ALWAYS_INLINE void Activate(u32 proc_id) {
cpu::DataSynchronizationBarrier();
cpu::SwitchProcess(m_ttbr, proc_id);
}
@ -219,12 +218,13 @@ namespace ams::kern::arch::arm64 {
Result ChangePermissions(KProcessAddress virt_addr, size_t num_pages, PageTableEntry entry_template, DisableMergeAttribute disable_merge_attr, bool refresh_mapping, PageLinkedList *page_list, bool reuse_ll);
static ALWAYS_INLINE void PteDataSynchronizationBarrier() {
cpu::DataSynchronizationBarrierInnerShareable();
static ALWAYS_INLINE void PteDataMemoryBarrier() {
cpu::DataMemoryBarrierInnerShareableStore();
}
static ALWAYS_INLINE void ClearPageTable(KVirtualAddress table) {
cpu::ClearPageToZero(GetVoidPointer(table));
cpu::DataSynchronizationBarrierInnerShareable();
}
ALWAYS_INLINE void OnTableUpdated() const {
@ -239,22 +239,8 @@ namespace ams::kern::arch::arm64 {
cpu::InvalidateTlbByVaDataOnly(virt_addr);
}
ALWAYS_INLINE void NoteUpdated() const {
cpu::DataSynchronizationBarrier();
if (this->IsKernel()) {
this->OnKernelTableUpdated();
} else {
this->OnTableUpdated();
}
}
ALWAYS_INLINE void NoteSingleKernelPageUpdated(KProcessAddress virt_addr) const {
MESOSPHERE_ASSERT(this->IsKernel());
cpu::DataSynchronizationBarrier();
this->OnKernelTableSinglePageUpdated(virt_addr);
}
void NoteUpdated() const;
void NoteSingleKernelPageUpdated(KProcessAddress virt_addr) const;
KVirtualAddress AllocatePageTable(PageLinkedList *page_list, bool reuse_ll) const {
KVirtualAddress table = this->GetPageTableManager().Allocate();

1
libraries/libmesosphere/include/mesosphere/arch/arm64/kern_userspace_memory_access.hpp
View file

@ -46,7 +46,6 @@ namespace ams::kern::arch::arm64 {
static bool StoreDataCache(uintptr_t start, uintptr_t end);
static bool FlushDataCache(uintptr_t start, uintptr_t end);
static bool InvalidateDataCache(uintptr_t start, uintptr_t end);
static bool InvalidateInstructionCache(uintptr_t start, uintptr_t end);
static bool ReadIoMemory32Bit(void *dst, const void *src, size_t size);
static bool ReadIoMemory16Bit(void *dst, const void *src, size_t size);

2
libraries/libmesosphere/include/mesosphere/kern_k_scheduler_impls.hpp
View file

@ -35,7 +35,7 @@ namespace ams::kern {
ALWAYS_INLINE void KScheduler::RescheduleOtherCores(u64 cores_needing_scheduling) {
if (const u64 core_mask = cores_needing_scheduling & ~(1ul << m_core_id); core_mask != 0) {
cpu::DataSynchronizationBarrier();
cpu::DataSynchronizationBarrierInnerShareable();
Kernel::GetInterruptManager().SendInterProcessorInterrupt(KInterruptName_Scheduler, core_mask);
}
}