kern: switch->nx, implement sleep manager init

libraries/libmesosphere/source/arch/arm64/kern_k_page_table_impl.cpp

@@ -20,11 +20,55 @@ namespace ams::kern::arch::arm64 {
     void KPageTableImpl::InitializeForKernel(void *tb, KVirtualAddress start, KVirtualAddress end) {
         this->table       = static_cast<L1PageTableEntry *>(tb);
         this->is_kernel   = true;
-        this->num_entries = util::AlignUp(end - start, AddressSpaceSize) / AddressSpaceSize;
+        this->num_entries = util::AlignUp(end - start, L1BlockSize) / L1BlockSize;
     }
 
     L1PageTableEntry *KPageTableImpl::Finalize() {
         return this->table;
     }
 
+    bool KPageTableImpl::GetPhysicalAddress(KPhysicalAddress *out, KProcessAddress address) const {
+        /* Validate that we can read the actual entry. */
+        const size_t l0_index = GetL0Index(address);
+        const size_t l1_index = GetL1Index(address);
+        if (this->is_kernel) {
+            /* Kernel entries must be accessed via TTBR1. */
+            if ((l0_index != MaxPageTableEntries - 1) || (l1_index < MaxPageTableEntries - this->num_entries)) {
+                return false;
+            }
+        } else {
+            /* User entries must be accessed with TTBR0. */
+            if ((l0_index != 0) || l1_index >= this->num_entries) {
+                return false;
+            }
+        }
+
+        /* Try to get from l1 table. */
+        const L1PageTableEntry *l1_entry = this->GetL1Entry(address);
+        if (l1_entry->IsBlock()) {
+            *out = l1_entry->GetBlock() + GetL1Offset(address);
+            return true;
+        } else if (!l1_entry->IsTable()) {
+            return false;
+        }
+
+        /* Try to get from l2 table. */
+        const L2PageTableEntry *l2_entry = this->GetL2Entry(l1_entry, address);
+        if (l2_entry->IsBlock()) {
+            *out = l2_entry->GetBlock() + GetL2Offset(address);
+            return true;
+        } else if (!l2_entry->IsTable()) {
+            return false;
+        }
+
+        /* Try to get from l3 table. */
+        const L3PageTableEntry *l3_entry = this->GetL3Entry(l2_entry, address);
+        if (l3_entry->IsBlock()) {
+            *out = l3_entry->GetBlock() + GetL3Offset(address);
+            return true;
+        }
+
+        return false;
+    }
+
 }

libraries/libmesosphere/source/board/nintendo/nx/kern_k_sleep_manager.cpp

@@ -0,0 +1,158 @@
+/*
+ * Copyright (c) 2018-2020 Atmosphère-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#include <mesosphere.hpp>
+#include "kern_k_sleep_manager.hpp"
+#include "kern_secure_monitor.hpp"
+
+namespace ams::kern::board::nintendo::nx {
+
+    namespace {
+
+        /* Struct representing registers saved on wake/sleep. */
+        class SavedSystemRegisters {
+            private:
+                u64 ttbr0_el1;
+                u64 tcr_el1;
+                u64 elr_el1;
+                u64 sp_el0;
+                u64 spsr_el1;
+                u64 daif;
+                u64 cpacr_el1;
+                u64 vbar_el1;
+                u64 csselr_el1;
+                u64 cntp_ctl_el0;
+                u64 cntp_cval_el0;
+                u64 cntkctl_el1;
+                u64 tpidr_el0;
+                u64 tpidrro_el0;
+                u64 mdscr_el1;
+                u64 contextidr_el1;
+                u64 dbgwcrN_el1[16];
+                u64 dbgwvrN_el1[16];
+                u64 dbgbcrN_el1[16];
+                u64 dbgbvrN_el1[16];
+                u64 pmccfiltr_el0;
+                u64 pmccntr_el0;
+                u64 pmcntenset_el0;
+                u64 pmcr_el0;
+                u64 pmevcntrN_el0[31];
+                u64 pmevtyperN_el0[31];
+                u64 pmcntenset_el1;
+                u64 pmovsset_el0;
+                u64 pmselr_el0;
+                u64 pmuserenr_el0;
+            public:
+                void Save();
+                void Restore() const;
+        };
+
+        constexpr s32 SleepManagerThreadPriority = 2;
+
+        /* Globals for sleep/wake. */
+        u64 g_sleep_target_cores;
+        KLightLock g_request_lock;
+        KLightLock g_cv_lock;
+        KLightConditionVariable g_cv;
+        KPhysicalAddress g_sleep_buffer_phys_addrs[cpu::NumCores];
+        alignas(16) u64 g_sleep_buffers[cpu::NumCores][1_KB / sizeof(u64)];
+        SavedSystemRegisters g_sleep_system_registers[cpu::NumCores] = {};
+
+    }
+
+    void KSleepManager::Initialize() {
+        /* Create a sleep manager thread for each core. */
+        for (size_t core_id = 0; core_id < cpu::NumCores; core_id++) {
+            /* Reserve a thread from the system limit. */
+            MESOSPHERE_ABORT_UNLESS(Kernel::GetSystemResourceLimit().Reserve(ams::svc::LimitableResource_ThreadCountMax, 1));
+
+            /* Create a new thread. */
+            KThread *new_thread = KThread::Create();
+            MESOSPHERE_ABORT_UNLESS(new_thread != nullptr);
+
+            /* Launch the new thread. */
+            MESOSPHERE_R_ABORT_UNLESS(KThread::InitializeKernelThread(new_thread, KSleepManager::ProcessRequests, reinterpret_cast<uintptr_t>(g_sleep_buffers[core_id]), SleepManagerThreadPriority, static_cast<s32>(core_id)));
+
+            /* Register the new thread. */
+            KThread::Register(new_thread);
+
+            /* Run the thread. */
+            new_thread->Run();
+        }
+    }
+
+    void KSleepManager::SleepSystem() {
+        /* Ensure device mappings are not modified during sleep. */
+        MESOSPHERE_TODO("KDevicePageTable::Lock();");
+        ON_SCOPE_EXIT { MESOSPHERE_TODO("KDevicePageTable::Unlock();"); };
+
+        /* Request that the system sleep. */
+        {
+            KScopedLightLock lk(g_request_lock);
+
+            /* Signal the manager to sleep on all cores. */
+            {
+                KScopedLightLock lk(g_cv_lock);
+                MESOSPHERE_ABORT_UNLESS(g_sleep_target_cores == 0);
+
+                g_sleep_target_cores = (1ul << (cpu::NumCores - 1));
+                g_cv.Broadcast();
+
+                while (g_sleep_target_cores != 0) {
+                    g_cv.Wait(std::addressof(g_cv_lock));
+                }
+            }
+        }
+    }
+
+    void KSleepManager::ProcessRequests(uintptr_t buffer) {
+        const s32 core_id = GetCurrentCoreId();
+        KPhysicalAddress resume_entry_phys_addr = Null<KPhysicalAddress>;
+
+        /* Get the physical addresses we'll need. */
+        {
+            MESOSPHERE_ABORT_UNLESS(Kernel::GetKernelPageTable().GetPhysicalAddress(std::addressof(g_sleep_buffer_phys_addrs[core_id]), KProcessAddress(buffer)));
+            MESOSPHERE_ABORT_UNLESS(Kernel::GetKernelPageTable().GetPhysicalAddress(std::addressof(resume_entry_phys_addr), KProcessAddress(&::ams::kern::board::nintendo::nx::KSleepManager::ResumeEntry)));
+
+        }
+        const KPhysicalAddress sleep_buffer_phys_addr = g_sleep_buffer_phys_addrs[core_id];
+        const u64 target_core_mask = (1ul << core_id);
+
+        /* Loop, processing sleep when requested. */
+        while (true) {
+            /* Wait for a request. */
+            {
+                KScopedLightLock lk(g_cv_lock);
+                while (!(g_sleep_target_cores & target_core_mask)) {
+                    g_cv.Wait(std::addressof(g_cv_lock));
+                }
+            }
+
+            MESOSPHERE_TODO("Implement Sleep/Wake");
+            (void)(g_sleep_system_registers[core_id]);
+            (void)(sleep_buffer_phys_addr);
+
+            /* Signal request completed. */
+            {
+                KScopedLightLock lk(g_cv_lock);
+                g_sleep_target_cores &= ~target_core_mask;
+                if (g_sleep_target_cores == 0) {
+                    g_cv.Broadcast();
+                }
+            }
+        }
+    }
+
+}

libraries/libmesosphere/source/board/nintendo/nx/kern_k_sleep_manager.hpp

@@ -0,0 +1,33 @@
+/*
+ * Copyright (c) 2018-2020 Atmosphère-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#pragma once
+#include <mesosphere.hpp>
+
+namespace ams::kern::board::nintendo::nx {
+
+    class KSleepManager {
+        private:
+            static void CpuSleepHandler(uintptr_t arg, uintptr_t entry);
+            static void ResumeEntry(uintptr_t arg);
+
+            static void ProcessRequests(uintptr_t buffer);
+        public:
+            static void Initialize();
+            static void SleepSystem();
+    };
+
+
+}

libraries/libmesosphere/source/board/nintendo/nx/kern_k_sleep_manager_asm.s

@@ -0,0 +1,23 @@
+/*
+ * Copyright (c) 2018-2020 Atmosphère-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+/* ams::kern::board::nintendo::nx::KSleepManager::ResumeEntry(uintptr_t arg) */
+.section    .text._ZN3ams4kern5board8nintendo2nx13KSleepManager11ResumeEntryEm, "ax", %progbits
+.global     _ZN3ams4kern5board8nintendo2nx13KSleepManager11ResumeEntryEm
+.type       _ZN3ams4kern5board8nintendo2nx13KSleepManager11ResumeEntryEm, %function
+_ZN3ams4kern5board8nintendo2nx13KSleepManager11ResumeEntryEm:
+    /* TODO: Implement a real function here. */
+    brk 1000

libraries/libmesosphere/source/board/nintendo/nx/kern_k_system_control.cpp

@@ -15,14 +15,20 @@
  */
 #include <mesosphere.hpp>
 #include "kern_secure_monitor.hpp"
+#include "kern_k_sleep_manager.hpp"
 
-namespace ams::kern {
+namespace ams::kern::board::nintendo::nx {
 
     namespace {
 
         /* Global variables for panic. */
         bool g_call_smc_on_panic;
 
+        /* Global variables for secure memory. */
+        constexpr size_t SecureAppletReservedMemorySize = 4_MB;
+        KVirtualAddress g_secure_applet_memory_address;
+
+
         /* Global variables for randomness. */
         /* Nintendo uses std::mt19937_t for randomness. */
         /* To save space (and because mt19337_t isn't secure anyway), */
@@ -195,7 +201,7 @@ namespace ams::kern {
     }
 
     /* System Initialization. */
-    void KSystemControl::Initialize() {
+    void KSystemControl::InitializePhase1() {
         /* Set IsDebugMode. */
         {
             KTargetSystem::SetIsDebugMode(GetConfigBool(smc::ConfigItem::IsDebugMode));
@@ -250,6 +256,21 @@ namespace ams::kern {
         }
     }
 
+    void KSystemControl::InitializePhase2() {
+        /* Initialize the sleep manager. */
+        KSleepManager::Initialize();
+
+        /* Reserve secure applet memory. */
+        {
+            MESOSPHERE_ABORT_UNLESS(g_secure_applet_memory_address == Null<KVirtualAddress>);
+            MESOSPHERE_ABORT_UNLESS(Kernel::GetSystemResourceLimit().Reserve(ams::svc::LimitableResource_PhysicalMemoryMax, SecureAppletReservedMemorySize));
+
+            constexpr auto SecureAppletAllocateOption = KMemoryManager::EncodeOption(KMemoryManager::Pool_System, KMemoryManager::Direction_FromFront);
+            g_secure_applet_memory_address = Kernel::GetMemoryManager().AllocateContinuous(SecureAppletReservedMemorySize / PageSize, 1, SecureAppletAllocateOption);
+            MESOSPHERE_ABORT_UNLESS(g_secure_applet_memory_address != Null<KVirtualAddress>);
+        }
+    }
+
     u32 KSystemControl::GetInitialProcessBinaryPool() {
         return KMemoryManager::Pool_Application;
     }
@@ -274,6 +295,11 @@ namespace ams::kern {
         return GenerateUniformRange(min, max, GenerateRandomU64FromGenerator);
     }
 
+    void KSystemControl::SleepSystem() {
+        MESOSPHERE_LOG("SleepSystem() was called\n");
+        KSleepManager::SleepSystem();
+    }
+
     void KSystemControl::StopSystem() {
         if (g_call_smc_on_panic) {
             /* Display a panic screen via secure monitor. */

libraries/libmesosphere/source/board/nintendo/nx/kern_secure_monitor.cpp

@@ -16,7 +16,7 @@
 #include <mesosphere.hpp>
 #include "kern_secure_monitor.hpp"
 
-namespace ams::kern::smc {
+namespace ams::kern::board::nintendo::nx::smc {
 
     namespace {
 

libraries/libmesosphere/source/board/nintendo/nx/kern_secure_monitor.hpp

@@ -16,7 +16,7 @@
 #pragma once
 #include <mesosphere.hpp>
 
-namespace ams::kern::smc {
+namespace ams::kern::board::nintendo::nx::smc {
 
     /* Types. */
     enum MemorySize {

libraries/libmesosphere/source/kern_debug_log_impl.board.nintendo_nx.cpp

@@ -14,7 +14,7 @@
  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  */
 #include <mesosphere.hpp>
-#include "../../../kern_debug_log_impl.hpp"
+#include "kern_debug_log_impl.hpp"
 
 namespace ams::kern {
 

libraries/libmesosphere/source/kern_main.cpp

@@ -46,8 +46,8 @@ namespace ams::kern {
         });
 
         if (core_id == 0) {
-            /* Initialize KSystemControl. */
-            KSystemControl::Initialize();
+            /* Initialize the carveout and the system resource limit. */
+            KSystemControl::InitializePhase1();
 
             /* Initialize the memory manager and the KPageBuffer slabheap. */
             {
@@ -107,9 +107,11 @@ namespace ams::kern {
 
         /* Perform more core-0 specific initialization. */
         if (core_id == 0) {
+            /* Initialize the exit worker manager, so that threads and processes may exit cleanly. */
             Kernel::GetWorkerTaskManager(KWorkerTaskManager::WorkerType_Exit).Initialize(KWorkerTaskManager::WorkerType_Exit, KWorkerTaskManager::ExitWorkerPriority);
 
-            MESOSPHERE_TODO("KSystemControl::InitializeSleepManagerAndAppletSecureMemory();");
+            /* Setup so that we may sleep later, and reserve memory for secure applets. */
+            KSystemControl::InitializePhase2();
 
             MESOSPHERE_TODO("KDeviceAddressSpace::Initialize();");