diff --git a/libraries/libmesosphere/include/mesosphere.hpp b/libraries/libmesosphere/include/mesosphere.hpp
index 279eb2a0a..66be8d6d8 100644
--- a/libraries/libmesosphere/include/mesosphere.hpp
+++ b/libraries/libmesosphere/include/mesosphere.hpp
@@ -27,6 +27,9 @@
#include <mesosphere/kern_initial_process.hpp>
#include <mesosphere/kern_k_exception_context.hpp>
+/* Tracing functionality. */
+#include <mesosphere/kern_k_trace.hpp>
+
/* Core pre-initialization includes. */
#include <mesosphere/kern_select_cpu.hpp>
#include <mesosphere/kern_select_system_control.hpp>
diff --git a/libraries/libmesosphere/include/mesosphere/kern_common.hpp b/libraries/libmesosphere/include/mesosphere/kern_common.hpp
index e27cfc96e..b1123b397 100644
--- a/libraries/libmesosphere/include/mesosphere/kern_common.hpp
+++ b/libraries/libmesosphere/include/mesosphere/kern_common.hpp
@@ -37,4 +37,6 @@ namespace ams::kern {
#define MESOSPHERE_ENABLE_DEBUG_PRINT
#endif
+#define MESOSPHERE_BUILD_FOR_TRACING
+
#include <mesosphere/svc/kern_svc_results.hpp>
diff --git a/libraries/libmesosphere/include/mesosphere/kern_k_memory_layout.hpp b/libraries/libmesosphere/include/mesosphere/kern_k_memory_layout.hpp
index d70485b62..6f0d6c51f 100644
--- a/libraries/libmesosphere/include/mesosphere/kern_k_memory_layout.hpp
+++ b/libraries/libmesosphere/include/mesosphere/kern_k_memory_layout.hpp
@@ -100,8 +100,6 @@ namespace ams::kern {
KMemoryRegionType_DramSystemNonSecurePool = 0xDA6 | KMemoryRegionAttr_NoUserMap | KMemoryRegionAttr_LinearMapped,
KMemoryRegionType_DramSystemPool = 0x13A6 | KMemoryRegionAttr_NoUserMap | KMemoryRegionAttr_LinearMapped | KMemoryRegionAttr_CarveoutProtected,
-
-
KMemoryRegionType_DramKernel = 0xE | KMemoryRegionAttr_NoUserMap | KMemoryRegionAttr_CarveoutProtected,
KMemoryRegionType_DramKernelCode = 0xCE | KMemoryRegionAttr_NoUserMap | KMemoryRegionAttr_CarveoutProtected,
KMemoryRegionType_DramKernelSlab = 0x14E | KMemoryRegionAttr_NoUserMap | KMemoryRegionAttr_CarveoutProtected,
@@ -526,6 +524,10 @@ namespace ams::kern {
return *GetVirtualMemoryRegionTree().FindFirstRegionByType(KMemoryRegionType_KernelTemp);
}
+ static NOINLINE KMemoryRegion &GetKernelTraceBufferRegion() {
+ return *GetVirtualLinearMemoryRegionTree().FindFirstRegionByType(KMemoryRegionType_VirtualKernelTraceBuffer);
+ }
+
static NOINLINE KMemoryRegion &GetVirtualLinearRegion(KVirtualAddress address) {
return *GetVirtualLinearMemoryRegionTree().FindContainingRegion(GetInteger(address));
}
@@ -730,6 +732,10 @@ namespace ams::kern {
return GetVirtualLinearExtents(GetLinearRegionPhysicalExtents());
}
+ static NOINLINE auto GetMainMemoryPhysicalExtents() {
+ return GetPhysicalMemoryRegionTree().GetDerivedRegionExtents(KMemoryRegionType_Dram);
+ }
+
static NOINLINE auto GetCarveoutRegionExtents() {
return GetPhysicalMemoryRegionTree().GetDerivedRegionExtents(KMemoryRegionAttr_CarveoutProtected);
}
@@ -777,6 +783,10 @@ namespace ams::kern {
static NOINLINE auto GetKernelApplicationPoolRegionPhysicalExtents() {
return GetPhysicalMemoryRegionTree().GetDerivedRegionExtents(KMemoryRegionType_DramApplicationPool);
}
+
+ static NOINLINE auto GetKernelTraceBufferRegionPhysicalExtents() {
+ return GetPhysicalMemoryRegionTree().GetDerivedRegionExtents(KMemoryRegionType_KernelTraceBuffer);
+ }
};
diff --git a/libraries/libmesosphere/include/mesosphere/kern_k_trace.hpp b/libraries/libmesosphere/include/mesosphere/kern_k_trace.hpp
new file mode 100644
index 000000000..ae4bd65b3
--- /dev/null
+++ b/libraries/libmesosphere/include/mesosphere/kern_k_trace.hpp
@@ -0,0 +1,57 @@
+/*
+ * Copyright (c) 2018-2020 Atmosphère-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+#pragma once
+#include <mesosphere/kern_common.hpp>
+#include <mesosphere/kern_k_spin_lock.hpp>
+
+namespace ams::kern {
+
+ #if defined(MESOSPHERE_BUILD_FOR_TRACING)
+ constexpr inline bool IsKTraceEnabled = true;
+ #else
+ constexpr inline bool IsKTraceEnabled = false;
+ #endif
+
+ constexpr inline size_t KTraceBufferSize = IsKTraceEnabled ? 16_MB : 0;
+
+ static_assert(IsKTraceEnabled || !IsKTraceEnabled);
+
+ class KTrace {
+ private:
+ static bool s_is_active;
+ public:
+ static void Initialize(KVirtualAddress address, size_t size);
+ static void Start();
+ static void Stop();
+
+ static ALWAYS_INLINE bool IsActive() { return s_is_active; }
+ };
+
+}
+
+#define MESOSPHERE_KTRACE_RESUME() \
+ ({ \
+ if constexpr (::ams::kern::IsKTraceEnabled) { \
+ ::ams::kern::KTrace::Start(); \
+ } \
+ })
+
+#define MESOSPHERE_KTRACE_PAUSE() \
+ ({ \
+ if constexpr (::ams::kern::IsKTraceEnabled) { \
+ ::ams::kern::KTrace::Stop(); \
+ } \
+ })
diff --git a/libraries/libmesosphere/source/board/nintendo/nx/kern_k_system_control.cpp b/libraries/libmesosphere/source/board/nintendo/nx/kern_k_system_control.cpp
index 3c73bc298..bf98bdd1d 100644
--- a/libraries/libmesosphere/source/board/nintendo/nx/kern_k_system_control.cpp
+++ b/libraries/libmesosphere/source/board/nintendo/nx/kern_k_system_control.cpp
@@ -338,39 +338,52 @@ namespace ams::kern::board::nintendo::nx {
}
size_t KSystemControl::Init::GetApplicationPoolSize() {
- switch (GetMemoryArrangeForInit()) {
- case smc::MemoryArrangement_4GB:
- default:
- return 3285_MB;
- case smc::MemoryArrangement_4GBForAppletDev:
- return 2048_MB;
- case smc::MemoryArrangement_4GBForSystemDev:
- return 3285_MB;
- case smc::MemoryArrangement_6GB:
- return 4916_MB;
- case smc::MemoryArrangement_6GBForAppletDev:
- return 3285_MB;
- case smc::MemoryArrangement_8GB:
- return 4916_MB;
- }
+ /* Get the base pool size. */
+ const size_t base_pool_size = [] ALWAYS_INLINE_LAMBDA () -> size_t {
+ switch (GetMemoryArrangeForInit()) {
+ case smc::MemoryArrangement_4GB:
+ default:
+ return 3285_MB;
+ case smc::MemoryArrangement_4GBForAppletDev:
+ return 2048_MB;
+ case smc::MemoryArrangement_4GBForSystemDev:
+ return 3285_MB;
+ case smc::MemoryArrangement_6GB:
+ return 4916_MB;
+ case smc::MemoryArrangement_6GBForAppletDev:
+ return 3285_MB;
+ case smc::MemoryArrangement_8GB:
+ return 4916_MB;
+ }
+ }();
+
+ /* Return (possibly) adjusted size. */
+ return base_pool_size;
}
size_t KSystemControl::Init::GetAppletPoolSize() {
- switch (GetMemoryArrangeForInit()) {
- case smc::MemoryArrangement_4GB:
- default:
- return 507_MB;
- case smc::MemoryArrangement_4GBForAppletDev:
- return 1554_MB;
- case smc::MemoryArrangement_4GBForSystemDev:
- return 448_MB;
- case smc::MemoryArrangement_6GB:
- return 562_MB;
- case smc::MemoryArrangement_6GBForAppletDev:
- return 2193_MB;
- case smc::MemoryArrangement_8GB:
- return 2193_MB;
- }
+ /* Get the base pool size. */
+ const size_t base_pool_size = [] ALWAYS_INLINE_LAMBDA () -> size_t {
+ switch (GetMemoryArrangeForInit()) {
+ case smc::MemoryArrangement_4GB:
+ default:
+ return 507_MB;
+ case smc::MemoryArrangement_4GBForAppletDev:
+ return 1554_MB;
+ case smc::MemoryArrangement_4GBForSystemDev:
+ return 448_MB;
+ case smc::MemoryArrangement_6GB:
+ return 562_MB;
+ case smc::MemoryArrangement_6GBForAppletDev:
+ return 2193_MB;
+ case smc::MemoryArrangement_8GB:
+ return 2193_MB;
+ }
+ }();
+
+ /* Return (possibly) adjusted size. */
+ constexpr size_t ExtraSystemMemoryForAtmosphere = 33_MB;
+ return base_pool_size - ExtraSystemMemoryForAtmosphere - KTraceBufferSize;
}
size_t KSystemControl::Init::GetMinimumNonSecureSystemPoolSize() {
@@ -461,6 +474,12 @@ namespace ams::kern::board::nintendo::nx {
g_secure_applet_memory_address = Kernel::GetMemoryManager().AllocateContinuous(SecureAppletMemorySize / PageSize, 1, SecureAppletAllocateOption);
MESOSPHERE_ABORT_UNLESS(g_secure_applet_memory_address != Null<KVirtualAddress>);
}
+
+ /* Initialize KTrace. */
+ if constexpr (IsKTraceEnabled) {
+ const auto &ktrace = KMemoryLayout::GetKernelTraceBufferRegion();
+ KTrace::Initialize(ktrace.GetAddress(), ktrace.GetSize());
+ }
}
u32 KSystemControl::GetInitialProcessBinaryPool() {
diff --git a/libraries/libmesosphere/source/kern_k_memory_layout.cpp b/libraries/libmesosphere/source/kern_k_memory_layout.cpp
index 6ea5a2c43..3f281d059 100644
--- a/libraries/libmesosphere/source/kern_k_memory_layout.cpp
+++ b/libraries/libmesosphere/source/kern_k_memory_layout.cpp
@@ -227,7 +227,9 @@ namespace ams::kern {
void SetupPoolPartitionMemoryRegions() {
/* Start by identifying the extents of the DRAM memory region. */
- const auto dram_extents = KMemoryLayout::GetPhysicalMemoryRegionTree().GetDerivedRegionExtents(KMemoryRegionType_Dram);
+ const auto dram_extents = KMemoryLayout::GetMainMemoryPhysicalExtents();
+
+ const uintptr_t pool_end = dram_extents.GetEndAddress() - KTraceBufferSize;
/* Get Application and Applet pool sizes. */
const size_t application_pool_size = KSystemControl::Init::GetApplicationPoolSize();
@@ -242,7 +244,7 @@ namespace ams::kern {
const uintptr_t pool_partitions_start = KMemoryLayout::GetPhysicalMemoryRegionTree().FindFirstRegionByTypeAttr(KMemoryRegionType_DramPoolPartition)->GetAddress();
/* Decide on starting addresses for our pools. */
- const uintptr_t application_pool_start = dram_extents.GetEndAddress() - application_pool_size;
+ const uintptr_t application_pool_start = pool_end - application_pool_size;
const uintptr_t applet_pool_start = application_pool_start - applet_pool_size;
const uintptr_t unsafe_system_pool_start = std::min(kernel_dram_start + CarveoutSizeMax, util::AlignDown(applet_pool_start - unsafe_system_pool_min_size, CarveoutAlignment));
const size_t unsafe_system_pool_size = applet_pool_start - unsafe_system_pool_start;
diff --git a/libraries/libmesosphere/source/kern_k_trace.cpp b/libraries/libmesosphere/source/kern_k_trace.cpp
new file mode 100644
index 000000000..15ab7f75a
--- /dev/null
+++ b/libraries/libmesosphere/source/kern_k_trace.cpp
@@ -0,0 +1,104 @@
+/*
+ * Copyright (c) 2018-2020 Atmosphère-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+#include <mesosphere.hpp>
+
+namespace ams::kern {
+
+ /* Static initializations. */
+ constinit bool KTrace::s_is_active = false;
+
+ namespace {
+
+ constinit KSpinLock g_ktrace_lock;
+ constinit KVirtualAddress g_ktrace_buffer_address = Null<KVirtualAddress>;
+ constinit size_t g_ktrace_buffer_size = 0;
+
+ struct KTraceHeader {
+ u32 magic;
+ u32 offset;
+ u32 index;
+ u32 count;
+
+ static constexpr u32 Magic = util::FourCC<'K','T','R','0'>::Code;
+ };
+ static_assert(util::is_pod<KTraceHeader>::value);
+
+ struct KTraceRecord {
+ u8 core_id;
+ u8 type;
+ u16 process_id;
+ u32 thread_id;
+ u64 tick;
+ u64 data[6];
+ };
+ static_assert(util::is_pod<KTraceRecord>::value);
+ static_assert(sizeof(KTraceRecord) == 0x40);
+
+ }
+
+ void KTrace::Initialize(KVirtualAddress address, size_t size) {
+ /* Only perform tracing when on development hardware. */
+ if (KTargetSystem::IsDebugMode()) {
+ const size_t offset = util::AlignUp(sizeof(KTraceHeader), sizeof(KTraceRecord));
+ if (offset < size) {
+ /* Clear the trace buffer. */
+ std::memset(GetVoidPointer(address), 0, size);
+
+ /* Initialize the KTrace header. */
+ KTraceHeader *header = GetPointer<KTraceHeader>(address);
+ header->magic = KTraceHeader::Magic;
+ header->offset = offset;
+ header->index = 0;
+ header->count = (size - offset) / sizeof(KTraceRecord);
+
+ /* Set the global data. */
+ g_ktrace_buffer_address = address;
+ g_ktrace_buffer_size = size;
+ }
+ }
+ }
+
+ void KTrace::Start() {
+ if (g_ktrace_buffer_address != Null<KVirtualAddress>) {
+ /* Get exclusive access to the trace buffer. */
+ KScopedInterruptDisable di;
+ KScopedSpinLock lk(g_ktrace_lock);
+
+ /* Reset the header. */
+ KTraceHeader *header = GetPointer<KTraceHeader>(g_ktrace_buffer_address);
+ header->index = 0;
+
+ /* Reset the records. */
+ KTraceRecord *records = GetPointer<KTraceRecord>(g_ktrace_buffer_address + header->offset);
+ std::memset(records, 0, sizeof(*records) * header->count);
+
+ /* Note that we're active. */
+ s_is_active = true;
+ }
+ }
+
+ void KTrace::Stop() {
+ if (g_ktrace_buffer_address != Null<KVirtualAddress>) {
+ /* Get exclusive access to the trace buffer. */
+ KScopedInterruptDisable di;
+ KScopedSpinLock lk(g_ktrace_lock);
+
+ /* Note that we're paused. */
+ s_is_active = false;
+ }
+ }
+
+}
diff --git a/libraries/libmesosphere/source/kern_kernel.cpp b/libraries/libmesosphere/source/kern_kernel.cpp
index f9aa6f747..94171c726 100644
--- a/libraries/libmesosphere/source/kern_kernel.cpp
+++ b/libraries/libmesosphere/source/kern_kernel.cpp
@@ -157,6 +157,10 @@ namespace ams::kern {
PrintMemoryRegion(" SystemUnsafe", KMemoryLayout::GetKernelSystemNonSecurePoolRegionPhysicalExtents());
PrintMemoryRegion(" Applet", KMemoryLayout::GetKernelAppletPoolRegionPhysicalExtents());
PrintMemoryRegion(" Application", KMemoryLayout::GetKernelApplicationPoolRegionPhysicalExtents());
+ if constexpr (IsKTraceEnabled) {
+ MESOSPHERE_LOG(" Debug\n");
+ PrintMemoryRegion(" Trace Buffer", KMemoryLayout::GetKernelTraceBufferRegionPhysicalExtents());
+ }
MESOSPHERE_LOG("\n");
}
diff --git a/libraries/libmesosphere/source/svc/kern_svc_kernel_debug.cpp b/libraries/libmesosphere/source/svc/kern_svc_kernel_debug.cpp
index b8ba28c6f..177f4a50d 100644
--- a/libraries/libmesosphere/source/svc/kern_svc_kernel_debug.cpp
+++ b/libraries/libmesosphere/source/svc/kern_svc_kernel_debug.cpp
@@ -39,12 +39,12 @@ namespace ams::kern::svc {
switch (kern_trace_state) {
case ams::svc::KernelTraceState_Enabled:
{
- /* TODO: MESOSPHERE_KTRACE_RESUME(); */
+ MESOSPHERE_KTRACE_RESUME();
}
break;
case ams::svc::KernelTraceState_Disabled:
{
- /* TODO: MESOSPHERE_KTRACE_PAUSE(); */
+ MESOSPHERE_KTRACE_PAUSE();
}
break;
default:
diff --git a/mesosphere/kernel/source/arch/arm64/init/kern_init_core.cpp b/mesosphere/kernel/source/arch/arm64/init/kern_init_core.cpp
index 995dad19d..e578a591a 100644
--- a/mesosphere/kernel/source/arch/arm64/init/kern_init_core.cpp
+++ b/mesosphere/kernel/source/arch/arm64/init/kern_init_core.cpp
@@ -183,6 +183,11 @@ namespace ams::kern::init {
MESOSPHERE_INIT_ABORT_UNLESS(KMemoryLayout::GetVirtualMemoryRegionTree().Insert(GetInteger(misc_unk_debug_virt_addr), MiscUnknownDebugRegionSize, KMemoryRegionType_KernelMiscUnknownDebug));
ttbr1_table.Map(misc_unk_debug_virt_addr, MiscUnknownDebugRegionSize, misc_unk_debug_phys_addr, KernelRoDataAttribute, g_initial_page_allocator);
}
+ constexpr size_t MiscUnknownDebugRegionSize = PageSize;
+ constexpr size_t MiscUnknownDebugRegionAlign = PageSize;
+ const KVirtualAddress misc_unk_debug_virt_addr = KMemoryLayout::GetVirtualMemoryRegionTree().GetRandomAlignedRegionWithGuard(MiscUnknownDebugRegionSize, MiscUnknownDebugRegionAlign, KMemoryRegionType_KernelMisc, PageSize);
+ MESOSPHERE_INIT_ABORT_UNLESS(KMemoryLayout::GetVirtualMemoryRegionTree().Insert(GetInteger(misc_unk_debug_virt_addr), MiscUnknownDebugRegionSize, KMemoryRegionType_KernelMiscUnknownDebug));
+ ttbr1_table.Map(misc_unk_debug_virt_addr, MiscUnknownDebugRegionSize, misc_unk_debug_phys_addr, KernelRoDataAttribute, g_initial_page_allocator);
/* Setup board-specific device physical regions. */
SetupDevicePhysicalMemoryRegions();
@@ -241,6 +246,14 @@ namespace ams::kern::init {
/* Insert a physical region for the kernel page table heap region */
MESOSPHERE_INIT_ABORT_UNLESS(KMemoryLayout::GetPhysicalMemoryRegionTree().Insert(GetInteger(slab_end_phys_addr), page_table_heap_size, KMemoryRegionType_DramKernelPtHeap));
+ /* Insert a physical region for the kernel trace buffer. */
+ static_assert(!IsKTraceEnabled || KTraceBufferSize > 0);
+ if constexpr (IsKTraceEnabled) {
+ const auto dram_extents = KMemoryLayout::GetMainMemoryPhysicalExtents();
+ const KPhysicalAddress ktrace_buffer_phys_addr = dram_extents.GetEndAddress() - KTraceBufferSize;
+ MESOSPHERE_INIT_ABORT_UNLESS(KMemoryLayout::GetPhysicalMemoryRegionTree().Insert(GetInteger(ktrace_buffer_phys_addr), KTraceBufferSize, KMemoryRegionType_KernelTraceBuffer));
+ }
+
/* All DRAM regions that we haven't tagged by this point will be mapped under the linear mapping. Tag them. */
for (auto ®ion : KMemoryLayout::GetPhysicalMemoryRegionTree()) {
if (region.GetType() == KMemoryRegionType_Dram) {
@@ -258,17 +271,37 @@ namespace ams::kern::init {
const uintptr_t linear_region_phys_to_virt_diff = GetInteger(linear_region_start) - GetInteger(aligned_linear_phys_start);
/* Map and create regions for all the linearly-mapped data. */
- for (auto ®ion : KMemoryLayout::GetPhysicalMemoryRegionTree()) {
- if (!region.HasTypeAttribute(KMemoryRegionAttr_LinearMapped)) {
- continue;
+ {
+ uintptr_t cur_phys_addr = 0;
+ uintptr_t cur_size = 0;
+ for (auto ®ion : KMemoryLayout::GetPhysicalMemoryRegionTree()) {
+ if (!region.HasTypeAttribute(KMemoryRegionAttr_LinearMapped)) {
+ continue;
+ }
+
+ if (cur_phys_addr == 0) {
+ cur_phys_addr = region.GetAddress();
+ cur_size = region.GetSize();
+ } else if (cur_phys_addr + cur_size == region.GetAddress()) {
+ cur_size += region.GetSize();
+ } else {
+ const uintptr_t cur_virt_addr = cur_phys_addr + linear_region_phys_to_virt_diff;
+ ttbr1_table.Map(cur_virt_addr, cur_size, cur_phys_addr, KernelRwDataAttribute, g_initial_page_allocator);
+ cur_phys_addr = region.GetAddress();
+ cur_size = region.GetSize();
+ }
+
+ const uintptr_t region_virt_addr = region.GetAddress() + linear_region_phys_to_virt_diff;
+ MESOSPHERE_INIT_ABORT_UNLESS(KMemoryLayout::GetVirtualMemoryRegionTree().Insert(region_virt_addr, region.GetSize(), GetTypeForVirtualLinearMapping(region.GetType())));
+ region.SetPairAddress(region_virt_addr);
+ KMemoryLayout::GetVirtualMemoryRegionTree().FindContainingRegion(region_virt_addr)->SetPairAddress(region.GetAddress());
}
- const uintptr_t region_virt_addr = region.GetAddress() + linear_region_phys_to_virt_diff;
- ttbr1_table.Map(region_virt_addr, region.GetSize(), region.GetAddress(), KernelRwDataAttribute, g_initial_page_allocator);
-
- MESOSPHERE_INIT_ABORT_UNLESS(KMemoryLayout::GetVirtualMemoryRegionTree().Insert(region_virt_addr, region.GetSize(), GetTypeForVirtualLinearMapping(region.GetType())));
- region.SetPairAddress(region_virt_addr);
- KMemoryLayout::GetVirtualMemoryRegionTree().FindContainingRegion(region_virt_addr)->SetPairAddress(region.GetAddress());
+ /* Map the last block, which we may have skipped. */
+ if (cur_phys_addr != 0) {
+ const uintptr_t cur_virt_addr = cur_phys_addr + linear_region_phys_to_virt_diff;
+ ttbr1_table.Map(cur_virt_addr, cur_size, cur_phys_addr, KernelRwDataAttribute, g_initial_page_allocator);
+ }
}
/* Create regions for and map all core-specific stacks. */