mirrors/Atmosphere
1
0
Fork 0
mirror of https://github.com/Atmosphere-NX/Atmosphere.git synced 2025-05-25 12:14:24 -04:00
Code Issues Projects Releases Packages Wiki Activity

kern: support dynamic resource expansion for system heaps/events/sessions.

Browse source
This commit is contained in:
Michael Scire 2021-09-17 22:01:58 -07:00 • committed by SciresM
parent 2b91956051
commit 2c4bd44d7e
37 changed files with 856 additions and 328 deletions
Download patch file Download diff file Expand all files Collapse all files

31
libraries/libmesosphere/include/mesosphere/arch/arm64/kern_k_slab_heap_impl.hpp
View file

@ -16,6 +16,7 @@
#pragma once
#include <vapours.hpp>
#include <mesosphere/kern_select_cpu.hpp>
#include <mesosphere/kern_select_interrupt_manager.hpp>
namespace ams::kern::arch::arm64 {
@ -24,6 +25,32 @@ namespace ams::kern::arch::arm64 {
{ t.next } -> std::convertible_to<T *>;
};
ALWAYS_INLINE bool IsSlabAtomicValid() {
/* Without careful consideration, slab heaps atomics are vulnerable to */
/* the ABA problem, when doing compare and swap of node pointers. */
/* We resolve this by using the ARM exclusive monitor; we bundle the */
/* load and store of the relevant values into a single exclusive monitor */
/* hold, preventing the ABA problem. */
/* However, our assembly must do both a load and a store under a single */
/* hold, at different memory addresses. Considering the case where the */
/* addresses are distinct but resolve to the same cache set (by chance), */
/* we can note that under a 1-way associative (direct-mapped) cache */
/* we would have as a guarantee that the second access would evict the */
/* cache line from the first access, invalidating our exclusive monitor */
/* hold. Thus, we require that the cache is not 1-way associative, for */
/* our implementation to be correct. */
{
/* Disable interrupts. */
KScopedInterruptDisable di;
/* Select L1 cache. */
cpu::SetCsselrEl1(0);
/* Check that the L1 cache is not direct-mapped. */
return cpu::CacheSizeIdRegisterAccessor().GetAssociativity() != 0;
}
}
template<typename T> requires SlabHeapNode<T>
ALWAYS_INLINE T *AllocateFromSlabAtomic(T **head) {
u32 tmp;
@ -36,10 +63,7 @@ namespace ams::kern::arch::arm64 {
" ldr %[next], [%[node]]\n"
" stlxr %w[tmp], %[next], [%[head]]\n"
" cbnz %w[tmp], 1b\n"
" b 3f\n"
"2:\n"
" clrex\n"
"3:\n"
: [tmp]"=&r"(tmp), [node]"=&r"(node), [next]"=&r"(next), [head]"+&r"(head)
:
: "cc", "memory"
@ -59,7 +83,6 @@ namespace ams::kern::arch::arm64 {
" str %[next], [%[node]]\n"
" stlxr %w[tmp], %[node], [%[head]]\n"
" cbnz %w[tmp], 1b\n"
"2:\n"
: [tmp]"=&r"(tmp), [node]"+&r"(node), [next]"=&r"(next), [head]"+&r"(head)
:
: "cc", "memory"

60
libraries/libmesosphere/include/mesosphere/kern_k_dynamic_resource_manager.hpp Normal file
View file

@ -0,0 +1,60 @@
/*
* Copyright (c) 2018-2020 Atmosphère-NX
*
* This program is free software; you can redistribute it and/or modify it
* under the terms and conditions of the GNU General Public License,
* version 2, as published by the Free Software Foundation.
*
* This program is distributed in the hope it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#pragma once
#include <mesosphere/kern_common.hpp>
#include <mesosphere/kern_k_dynamic_slab_heap.hpp>
namespace ams::kern {
template<typename T, bool ClearNode = false>
class KDynamicResourceManager {
NON_COPYABLE(KDynamicResourceManager);
NON_MOVEABLE(KDynamicResourceManager);
public:
using DynamicSlabType = KDynamicSlabHeap<T, ClearNode>;
private:
KDynamicPageManager *m_page_allocator{};
DynamicSlabType *m_slab_heap{};
public:
constexpr KDynamicResourceManager() = default;
constexpr ALWAYS_INLINE KVirtualAddress GetAddress() const { return m_slab_heap->GetAddress(); }
constexpr ALWAYS_INLINE size_t GetSize() const { return m_slab_heap->GetSize(); }
constexpr ALWAYS_INLINE size_t GetUsed() const { return m_slab_heap->GetUsed(); }
constexpr ALWAYS_INLINE size_t GetPeak() const { return m_slab_heap->GetPeak(); }
constexpr ALWAYS_INLINE size_t GetCount() const { return m_slab_heap->GetCount(); }
ALWAYS_INLINE void Initialize(KDynamicPageManager *page_allocator, DynamicSlabType *slab_heap) {
m_page_allocator = page_allocator;
m_slab_heap = slab_heap;
}
T *Allocate() const {
return m_slab_heap->Allocate(m_page_allocator);
}
void Free(T *t) const {
m_slab_heap->Free(t);
}
};
class KBlockInfoManager : public KDynamicResourceManager<KBlockInfo>{};
class KMemoryBlockSlabManager : public KDynamicResourceManager<KMemoryBlock>{};
using KBlockInfoSlabHeap = typename KBlockInfoManager::DynamicSlabType;
using KMemoryBlockSlabHeap = typename KMemoryBlockSlabManager::DynamicSlabType;
}

91
libraries/libmesosphere/include/mesosphere/kern_k_dynamic_slab_heap.hpp
View file

@ -23,95 +23,71 @@
namespace ams::kern {
template<typename T, bool ClearNode = false>
class KDynamicSlabHeap {
class KDynamicSlabHeap : protected impl::KSlabHeapImpl {
NON_COPYABLE(KDynamicSlabHeap);
NON_MOVEABLE(KDynamicSlabHeap);
private:
using Impl = impl::KSlabHeapImpl;
using PageBuffer = KDynamicPageManager::PageBuffer;
private:
Impl m_impl;
KDynamicPageManager *m_page_allocator;
std::atomic<size_t> m_used;
std::atomic<size_t> m_peak;
std::atomic<size_t> m_count;
KVirtualAddress m_address;
size_t m_size;
private:
ALWAYS_INLINE Impl *GetImpl() {
return std::addressof(m_impl);
}
ALWAYS_INLINE const Impl *GetImpl() const {
return std::addressof(m_impl);
}
std::atomic<size_t> m_used{};
std::atomic<size_t> m_peak{};
std::atomic<size_t> m_count{};
KVirtualAddress m_address{};
size_t m_size{};
public:
constexpr KDynamicSlabHeap() : m_impl(), m_page_allocator(), m_used(), m_peak(), m_count(), m_address(), m_size() { /* ... */ }
constexpr KDynamicSlabHeap() = default;
constexpr KVirtualAddress GetAddress() const { return m_address; }
constexpr size_t GetSize() const { return m_size; }
constexpr size_t GetUsed() const { return m_used.load(); }
constexpr size_t GetPeak() const { return m_peak.load(); }
constexpr size_t GetCount() const { return m_count.load(); }
constexpr ALWAYS_INLINE KVirtualAddress GetAddress() const { return m_address; }
constexpr ALWAYS_INLINE size_t GetSize() const { return m_size; }
constexpr ALWAYS_INLINE size_t GetUsed() const { return m_used.load(); }
constexpr ALWAYS_INLINE size_t GetPeak() const { return m_peak.load(); }
constexpr ALWAYS_INLINE size_t GetCount() const { return m_count.load(); }
constexpr bool IsInRange(KVirtualAddress addr) const {
constexpr ALWAYS_INLINE bool IsInRange(KVirtualAddress addr) const {
return this->GetAddress() <= addr && addr <= this->GetAddress() + this->GetSize() - 1;
}
void Initialize(KVirtualAddress memory, size_t sz) {
/* Set tracking fields. */
m_address = memory;
m_count = sz / sizeof(T);
m_size = m_count * sizeof(T);
/* Free blocks to memory. */
u8 *cur = GetPointer<u8>(m_address + m_size);
for (size_t i = 0; i < sz / sizeof(T); i++) {
cur -= sizeof(T);
this->GetImpl()->Free(cur);
}
}
void Initialize(KDynamicPageManager *page_allocator) {
m_page_allocator = page_allocator;
m_address = m_page_allocator->GetAddress();
m_size = m_page_allocator->GetSize();
}
void Initialize(KDynamicPageManager *page_allocator, size_t num_objects) {
ALWAYS_INLINE void Initialize(KDynamicPageManager *page_allocator, size_t num_objects) {
MESOSPHERE_ASSERT(page_allocator != nullptr);
/* Initialize members. */
this->Initialize(page_allocator);
m_address = page_allocator->GetAddress();
m_size = page_allocator->GetSize();
/* Initialize the base allocator. */
KSlabHeapImpl::Initialize();
/* Allocate until we have the correct number of objects. */
while (m_count.load() < num_objects) {
auto *allocated = reinterpret_cast<T *>(m_page_allocator->Allocate());
auto *allocated = reinterpret_cast<T *>(page_allocator->Allocate());
MESOSPHERE_ABORT_UNLESS(allocated != nullptr);
for (size_t i = 0; i < sizeof(PageBuffer) / sizeof(T); i++) {
this->GetImpl()->Free(allocated + i);
KSlabHeapImpl::Free(allocated + i);
}
m_count.fetch_add(sizeof(PageBuffer) / sizeof(T));
}
}
T *Allocate() {
T *allocated = reinterpret_cast<T *>(this->GetImpl()->Allocate());
ALWAYS_INLINE T *Allocate(KDynamicPageManager *page_allocator) {
T *allocated = static_cast<T *>(KSlabHeapImpl::Allocate());
/* If we successfully allocated and we should clear the node, do so. */
if constexpr (ClearNode) {
if (AMS_LIKELY(allocated != nullptr)) {
reinterpret_cast<Impl::Node *>(allocated)->next = nullptr;
reinterpret_cast<KSlabHeapImpl::Node *>(allocated)->next = nullptr;
}
}
/* If we fail to allocate, try to get a new page from our next allocator. */
if (AMS_UNLIKELY(allocated == nullptr)) {
if (m_page_allocator != nullptr) {
allocated = reinterpret_cast<T *>(m_page_allocator->Allocate());
if (AMS_UNLIKELY(allocated == nullptr) ) {
if (page_allocator != nullptr) {
allocated = reinterpret_cast<T *>(page_allocator->Allocate());
if (allocated != nullptr) {
/* If we succeeded in getting a page, free the rest to our slab. */
for (size_t i = 1; i < sizeof(PageBuffer) / sizeof(T); i++) {
this->GetImpl()->Free(allocated + i);
KSlabHeapImpl::Free(allocated + i);
}
m_count.fetch_add(sizeof(PageBuffer) / sizeof(T));
}
@ -135,13 +111,10 @@ namespace ams::kern {
return allocated;
}
void Free(T *t) {
this->GetImpl()->Free(t);
ALWAYS_INLINE void Free(T *t) {
KSlabHeapImpl::Free(t);
m_used.fetch_sub(1);
}
};
class KBlockInfoManager : public KDynamicSlabHeap<KBlockInfo>{};
class KMemoryBlockSlabManager : public KDynamicSlabHeap<KMemoryBlock>{};
}

2
libraries/libmesosphere/include/mesosphere/kern_k_event.hpp
View file

@ -21,7 +21,7 @@
namespace ams::kern {
class KEvent final : public KAutoObjectWithSlabHeapAndContainer<KEvent, KAutoObjectWithList> {
class KEvent final : public KAutoObjectWithSlabHeapAndContainer<KEvent, KAutoObjectWithList, true> {
MESOSPHERE_AUTOOBJECT_TRAITS(KEvent, KAutoObject);
private:
KReadableEvent m_readable_event;

2
libraries/libmesosphere/include/mesosphere/kern_k_light_session.hpp
View file

@ -25,7 +25,7 @@ namespace ams::kern {
class KClientPort;
class KProcess;
class KLightSession final : public KAutoObjectWithSlabHeapAndContainer<KLightSession, KAutoObjectWithList> {
class KLightSession final : public KAutoObjectWithSlabHeapAndContainer<KLightSession, KAutoObjectWithList, true> {
MESOSPHERE_AUTOOBJECT_TRAITS(KLightSession, KAutoObject);
private:
enum class State : u8 {

1
libraries/libmesosphere/include/mesosphere/kern_k_memory_layout.hpp
View file

@ -144,6 +144,7 @@ namespace ams::kern {
static NOINLINE const KMemoryRegion &GetPageTableHeapRegion() { return Dereference(GetVirtualMemoryRegionTree().FindByType(KMemoryRegionType_VirtualDramKernelPtHeap)); }
static NOINLINE const KMemoryRegion &GetKernelStackRegion() { return Dereference(GetVirtualMemoryRegionTree().FindByType(KMemoryRegionType_KernelStack)); }
static NOINLINE const KMemoryRegion &GetTempRegion() { return Dereference(GetVirtualMemoryRegionTree().FindByType(KMemoryRegionType_KernelTemp)); }
static NOINLINE const KMemoryRegion &GetSlabRegion() { return Dereference(GetVirtualMemoryRegionTree().FindByType(KMemoryRegionType_KernelSlab)); }
static NOINLINE const KMemoryRegion &GetKernelTraceBufferRegion() { return Dereference(GetVirtualLinearMemoryRegionTree().FindByType(KMemoryRegionType_VirtualDramKernelTraceBuffer)); }

83
libraries/libmesosphere/include/mesosphere/kern_k_page_table_manager.hpp
View file

@ -15,58 +15,27 @@
*/
#pragma once
#include <mesosphere/kern_common.hpp>
#include <mesosphere/kern_slab_helpers.hpp>
#include <mesosphere/kern_k_dynamic_slab_heap.hpp>
#include <mesosphere/kern_k_page_table_slab_heap.hpp>
#include <mesosphere/kern_k_dynamic_resource_manager.hpp>
namespace ams::kern {
namespace impl {
class PageTablePage {
private:
u8 m_buffer[PageSize];
public:
ALWAYS_INLINE PageTablePage() { /* Do not initialize anything. */ }
};
static_assert(sizeof(PageTablePage) == PageSize);
}
class KPageTableManager : public KDynamicSlabHeap<impl::PageTablePage, true> {
class KPageTableManager : public KDynamicResourceManager<impl::PageTablePage, true> {
public:
using RefCount = u16;
static constexpr size_t PageTableSize = sizeof(impl::PageTablePage);
static_assert(PageTableSize == PageSize);
using RefCount = KPageTableSlabHeap::RefCount;
static constexpr size_t PageTableSize = KPageTableSlabHeap::PageTableSize;
private:
using BaseHeap = KDynamicSlabHeap<impl::PageTablePage, true>;
using BaseHeap = KDynamicResourceManager<impl::PageTablePage, true>;
private:
RefCount *m_ref_counts;
KPageTableSlabHeap *m_pt_heap{};
public:
static constexpr size_t CalculateReferenceCountSize(size_t size) {
return (size / PageSize) * sizeof(RefCount);
}
public:
constexpr KPageTableManager() : BaseHeap(), m_ref_counts() { /* ... */ }
private:
void Initialize(RefCount *rc) {
m_ref_counts = rc;
for (size_t i = 0; i < this->GetSize() / PageSize; i++) {
m_ref_counts[i] = 0;
}
}
constexpr KPageTableManager() = default;
constexpr RefCount *GetRefCountPointer(KVirtualAddress addr) const {
return std::addressof(m_ref_counts[(addr - this->GetAddress()) / PageSize]);
}
public:
void Initialize(KDynamicPageManager *page_allocator, RefCount *rc) {
BaseHeap::Initialize(page_allocator);
this->Initialize(rc);
}
ALWAYS_INLINE void Initialize(KDynamicPageManager *page_allocator, KPageTableSlabHeap *pt_heap) {
m_pt_heap = pt_heap;
void Initialize(KDynamicPageManager *page_allocator, size_t object_count, RefCount *rc) {
BaseHeap::Initialize(page_allocator, object_count);
this->Initialize(rc);
static_assert(std::derived_from<KPageTableSlabHeap, DynamicSlabType>);
BaseHeap::Initialize(page_allocator, pt_heap);
}
KVirtualAddress Allocate() {
@ -74,33 +43,23 @@ namespace ams::kern {
}
void Free(KVirtualAddress addr) {
/* Free the page. */
BaseHeap::Free(GetPointer<impl::PageTablePage>(addr));
return BaseHeap::Free(GetPointer<impl::PageTablePage>(addr));
}
RefCount GetRefCount(KVirtualAddress addr) const {
MESOSPHERE_ASSERT(this->IsInRange(addr));
return *this->GetRefCountPointer(addr);
ALWAYS_INLINE RefCount GetRefCount(KVirtualAddress addr) const {
return m_pt_heap->GetRefCount(addr);
}
void Open(KVirtualAddress addr, int count) {
MESOSPHERE_ASSERT(this->IsInRange(addr));
*this->GetRefCountPointer(addr) += count;
MESOSPHERE_ABORT_UNLESS(this->GetRefCount(addr) > 0);
ALWAYS_INLINE void Open(KVirtualAddress addr, int count) {
return m_pt_heap->Open(addr, count);
}
bool Close(KVirtualAddress addr, int count) {
MESOSPHERE_ASSERT(this->IsInRange(addr));
MESOSPHERE_ABORT_UNLESS(this->GetRefCount(addr) >= count);
*this->GetRefCountPointer(addr) -= count;
return this->GetRefCount(addr) == 0;
ALWAYS_INLINE bool Close(KVirtualAddress addr, int count) {
return m_pt_heap->Close(addr, count);
}
constexpr bool IsInPageTableHeap(KVirtualAddress addr) const {
return this->IsInRange(addr);
constexpr ALWAYS_INLINE bool IsInPageTableHeap(KVirtualAddress addr) const {
return m_pt_heap->IsInRange(addr);
}
};

93
libraries/libmesosphere/include/mesosphere/kern_k_page_table_slab_heap.hpp Normal file
View file

@ -0,0 +1,93 @@
/*
* Copyright (c) 2018-2020 Atmosphère-NX
*
* This program is free software; you can redistribute it and/or modify it
* under the terms and conditions of the GNU General Public License,
* version 2, as published by the Free Software Foundation.
*
* This program is distributed in the hope it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#pragma once
#include <mesosphere/kern_common.hpp>
#include <mesosphere/kern_slab_helpers.hpp>
#include <mesosphere/kern_k_dynamic_slab_heap.hpp>
namespace ams::kern {
namespace impl {
class PageTablePage {
private:
u8 m_buffer[PageSize];
public:
ALWAYS_INLINE PageTablePage() { /* Do not initialize anything. */ }
};
static_assert(sizeof(PageTablePage) == PageSize);
}
class KPageTableSlabHeap : public KDynamicSlabHeap<impl::PageTablePage, true> {
public:
using RefCount = u16;
static constexpr size_t PageTableSize = sizeof(impl::PageTablePage);
static_assert(PageTableSize == PageSize);
private:
using BaseHeap = KDynamicSlabHeap<impl::PageTablePage, true>;
private:
RefCount *m_ref_counts{};
public:
static constexpr ALWAYS_INLINE size_t CalculateReferenceCountSize(size_t size) {
return (size / PageSize) * sizeof(RefCount);
}
public:
constexpr KPageTableSlabHeap() = default;
private:
ALWAYS_INLINE void Initialize(RefCount *rc) {
m_ref_counts = rc;
for (size_t i = 0; i < this->GetSize() / PageSize; i++) {
m_ref_counts[i] = 0;
}
}
constexpr ALWAYS_INLINE RefCount *GetRefCountPointer(KVirtualAddress addr) const {
return m_ref_counts + ((addr - this->GetAddress()) / PageSize);
}
public:
ALWAYS_INLINE void Initialize(KDynamicPageManager *page_allocator, size_t object_count, RefCount *rc) {
BaseHeap::Initialize(page_allocator, object_count);
this->Initialize(rc);
}
ALWAYS_INLINE RefCount GetRefCount(KVirtualAddress addr) const {
MESOSPHERE_ASSERT(this->IsInRange(addr));
return *this->GetRefCountPointer(addr);
}
ALWAYS_INLINE void Open(KVirtualAddress addr, int count) {
MESOSPHERE_ASSERT(this->IsInRange(addr));
*this->GetRefCountPointer(addr) += count;
MESOSPHERE_ABORT_UNLESS(this->GetRefCount(addr) > 0);
}
ALWAYS_INLINE bool Close(KVirtualAddress addr, int count) {
MESOSPHERE_ASSERT(this->IsInRange(addr));
MESOSPHERE_ABORT_UNLESS(this->GetRefCount(addr) >= count);
*this->GetRefCountPointer(addr) -= count;
return this->GetRefCount(addr) == 0;
}
constexpr ALWAYS_INLINE bool IsInPageTableHeap(KVirtualAddress addr) const {
return this->IsInRange(addr);
}
};
}

5
libraries/libmesosphere/include/mesosphere/kern_k_process.hpp
View file

@ -29,7 +29,7 @@
#include <mesosphere/kern_k_address_arbiter.hpp>
#include <mesosphere/kern_k_capabilities.hpp>
#include <mesosphere/kern_k_wait_object.hpp>
#include <mesosphere/kern_k_dynamic_slab_heap.hpp>
#include <mesosphere/kern_k_dynamic_resource_manager.hpp>
#include <mesosphere/kern_k_page_table_manager.hpp>
namespace ams::kern {
@ -121,6 +121,9 @@ namespace ams::kern {
KMemoryBlockSlabManager m_memory_block_slab_manager{};
KBlockInfoManager m_block_info_manager{};
KPageTableManager m_page_table_manager{};
KMemoryBlockSlabHeap m_memory_block_heap{};
KBlockInfoSlabHeap m_block_info_heap{};
KPageTableSlabHeap m_page_table_heap{};
private:
Result Initialize(const ams::svc::CreateProcessParameter &params);

2
libraries/libmesosphere/include/mesosphere/kern_k_resource_limit.hpp
View file

@ -47,6 +47,8 @@ namespace ams::kern {
Result SetLimitValue(ams::svc::LimitableResource which, s64 value);
void Add(ams::svc::LimitableResource which, s64 value);
bool Reserve(ams::svc::LimitableResource which, s64 value);
bool Reserve(ams::svc::LimitableResource which, s64 value, s64 timeout);
void Release(ams::svc::LimitableResource which, s64 value);

2
libraries/libmesosphere/include/mesosphere/kern_k_session.hpp
View file

@ -25,7 +25,7 @@ namespace ams::kern {
class KClientPort;
class KProcess;
class KSession final : public KAutoObjectWithSlabHeapAndContainer<KSession, KAutoObjectWithList> {
class KSession final : public KAutoObjectWithSlabHeapAndContainer<KSession, KAutoObjectWithList, true> {
MESOSPHERE_AUTOOBJECT_TRAITS(KSession, KAutoObject);
private:
enum class State : u8 {

10
libraries/libmesosphere/include/mesosphere/kern_k_session_request.hpp
View file

@ -24,7 +24,7 @@
namespace ams::kern {
class KSessionRequest final : public KSlabAllocated<KSessionRequest>, public KAutoObject, public util::IntrusiveListBaseNode<KSessionRequest> {
class KSessionRequest final : public KSlabAllocated<KSessionRequest, true>, public KAutoObject, public util::IntrusiveListBaseNode<KSessionRequest> {
MESOSPHERE_AUTOOBJECT_TRAITS(KSessionRequest, KAutoObject);
public:
class SessionMappings {
@ -140,6 +140,14 @@ namespace ams::kern {
return req;
}
static KSessionRequest *CreateFromUnusedSlabMemory() {
KSessionRequest *req = KSessionRequest::AllocateFromUnusedSlabMemory();
if (req != nullptr) {
KAutoObject::Create(req);
}
return req;
}
virtual void Destroy() override {
this->Finalize();
KSessionRequest::Free(this);

2
libraries/libmesosphere/include/mesosphere/kern_k_shared_memory.hpp
View file

@ -35,7 +35,7 @@ namespace ams::kern {
bool m_is_initialized;
public:
explicit KSharedMemory()
: m_page_group(std::addressof(Kernel::GetBlockInfoManager())), m_resource_limit(nullptr), m_owner_process_id(std::numeric_limits<u64>::max()),
: m_page_group(std::addressof(Kernel::GetSystemBlockInfoManager())), m_resource_limit(nullptr), m_owner_process_id(std::numeric_limits<u64>::max()),
m_owner_perm(ams::svc::MemoryPermission_None), m_remote_perm(ams::svc::MemoryPermission_None), m_is_initialized(false)
{
/* ... */

173
libraries/libmesosphere/include/mesosphere/kern_k_slab_heap.hpp
View file

@ -16,11 +16,13 @@
#pragma once
#include <mesosphere/kern_common.hpp>
#include <mesosphere/kern_k_typed_address.hpp>
#include <mesosphere/kern_k_memory_layout.hpp>
#if defined(ATMOSPHERE_ARCH_ARM64)
#include <mesosphere/arch/arm64/kern_k_slab_heap_impl.hpp>
namespace ams::kern {
using ams::kern::arch::arm64::IsSlabAtomicValid;
using ams::kern::arch::arm64::AllocateFromSlabAtomic;
using ams::kern::arch::arm64::FreeToSlabAtomic;
}
@ -44,78 +46,73 @@ namespace ams::kern {
Node *next;
};
private:
Node * m_head;
size_t m_obj_size;
Node *m_head{nullptr};
public:
constexpr KSlabHeapImpl() : m_head(nullptr), m_obj_size(0) { MESOSPHERE_ASSERT_THIS(); }
constexpr KSlabHeapImpl() = default;
void Initialize(size_t size) {
MESOSPHERE_INIT_ABORT_UNLESS(m_head == nullptr);
m_obj_size = size;
void Initialize() {
MESOSPHERE_ABORT_UNLESS(m_head == nullptr);
MESOSPHERE_ABORT_UNLESS(IsSlabAtomicValid());
}
Node *GetHead() const {
ALWAYS_INLINE Node *GetHead() const {
return m_head;
}
size_t GetObjectSize() const {
return m_obj_size;
}
void *Allocate() {
MESOSPHERE_ASSERT_THIS();
ALWAYS_INLINE void *Allocate() {
return AllocateFromSlabAtomic(std::addressof(m_head));
}
void Free(void *obj) {
MESOSPHERE_ASSERT_THIS();
Node *node = reinterpret_cast<Node *>(obj);
return FreeToSlabAtomic(std::addressof(m_head), node);
ALWAYS_INLINE void Free(void *obj) {
return FreeToSlabAtomic(std::addressof(m_head), static_cast<Node *>(obj));
}
};
}
class KSlabHeapBase {
template<bool SupportDynamicExpansion>
class KSlabHeapBase : protected impl::KSlabHeapImpl {
NON_COPYABLE(KSlabHeapBase);
NON_MOVEABLE(KSlabHeapBase);
private:
using Impl = impl::KSlabHeapImpl;
size_t m_obj_size{};
uintptr_t m_peak{};
uintptr_t m_start{};
uintptr_t m_end{};
private:
Impl m_impl;
uintptr_t m_peak;
uintptr_t m_start;
uintptr_t m_end;
private:
ALWAYS_INLINE Impl *GetImpl() {
return std::addressof(m_impl);
}
ALWAYS_INLINE const Impl *GetImpl() const {
return std::addressof(m_impl);
ALWAYS_INLINE void UpdatePeakImpl(uintptr_t obj) {
static_assert(std::atomic_ref<uintptr_t>::is_always_lock_free);
std::atomic_ref<uintptr_t> peak_ref(m_peak);
const uintptr_t alloc_peak = obj + this->GetObjectSize();
uintptr_t cur_peak = m_peak;
do {
if (alloc_peak <= cur_peak) {
break;
}
} while (!peak_ref.compare_exchange_strong(cur_peak, alloc_peak));
}
public:
constexpr KSlabHeapBase() : m_impl(), m_peak(0), m_start(0), m_end(0) { MESOSPHERE_ASSERT_THIS(); }
constexpr KSlabHeapBase() = default;
ALWAYS_INLINE bool Contains(uintptr_t address) const {
return m_start <= address && address < m_end;
}
void InitializeImpl(size_t obj_size, void *memory, size_t memory_size) {
MESOSPHERE_ASSERT_THIS();
void Initialize(size_t obj_size, void *memory, size_t memory_size) {
/* Ensure we don't initialize a slab using null memory. */
MESOSPHERE_ABORT_UNLESS(memory != nullptr);
/* Set our object size. */
m_obj_size = obj_size;
/* Initialize the base allocator. */
this->GetImpl()->Initialize(obj_size);
KSlabHeapImpl::Initialize();
/* Set our tracking variables. */
const size_t num_obj = (memory_size / obj_size);
m_start = reinterpret_cast<uintptr_t>(memory);
m_end = m_start + num_obj * obj_size;
m_end = m_start + num_obj * obj_size;
m_peak = m_start;
/* Free the objects. */
@ -123,75 +120,91 @@ namespace ams::kern {
for (size_t i = 0; i < num_obj; i++) {
cur -= obj_size;
this->GetImpl()->Free(cur);
KSlabHeapImpl::Free(cur);
}
}
size_t GetSlabHeapSize() const {
ALWAYS_INLINE size_t GetSlabHeapSize() const {
return (m_end - m_start) / this->GetObjectSize();
}
size_t GetObjectSize() const {
return this->GetImpl()->GetObjectSize();
ALWAYS_INLINE size_t GetObjectSize() const {
return m_obj_size;
}
void *AllocateImpl() {
MESOSPHERE_ASSERT_THIS();
void *obj = this->GetImpl()->Allocate();
ALWAYS_INLINE void *Allocate() {
void *obj = KSlabHeapImpl::Allocate();
/* Track the allocated peak. */
#if defined(MESOSPHERE_BUILD_FOR_DEBUGGING)
if (AMS_LIKELY(obj != nullptr)) {
static_assert(std::atomic_ref<uintptr_t>::is_always_lock_free);
std::atomic_ref<uintptr_t> peak_ref(m_peak);
const uintptr_t alloc_peak = reinterpret_cast<uintptr_t>(obj) + this->GetObjectSize();
uintptr_t cur_peak = m_peak;
do {
if (alloc_peak <= cur_peak) {
break;
if constexpr (SupportDynamicExpansion) {
if (this->Contains(reinterpret_cast<uintptr_t>(obj))) {
this->UpdatePeakImpl(reinterpret_cast<uintptr_t>(obj));
} else {
this->UpdatePeakImpl(reinterpret_cast<uintptr_t>(m_end) - this->GetObjectSize());
}
} while (!peak_ref.compare_exchange_strong(cur_peak, alloc_peak));
} else {
this->UpdatePeakImpl(reinterpret_cast<uintptr_t>(obj));
}
}
#endif
return obj;
}
void FreeImpl(void *obj) {
MESOSPHERE_ASSERT_THIS();
ALWAYS_INLINE void Free(void *obj) {
/* Don't allow freeing an object that wasn't allocated from this heap. */
MESOSPHERE_ABORT_UNLESS(this->Contains(reinterpret_cast<uintptr_t>(obj)));
const bool contained = this->Contains(reinterpret_cast<uintptr_t>(obj));
if constexpr (SupportDynamicExpansion) {
const bool is_slab = KMemoryLayout::GetSlabRegion().Contains(reinterpret_cast<uintptr_t>(obj));
MESOSPHERE_ABORT_UNLESS(contained || is_slab);
} else {
MESOSPHERE_ABORT_UNLESS(contained);
}
this->GetImpl()->Free(obj);
KSlabHeapImpl::Free(obj);
}
size_t GetObjectIndexImpl(const void *obj) const {
ALWAYS_INLINE size_t GetObjectIndex(const void *obj) const {
if constexpr (SupportDynamicExpansion) {
if (!this->Contains(reinterpret_cast<uintptr_t>(obj))) {
return std::numeric_limits<size_t>::max();
}
}
return (reinterpret_cast<uintptr_t>(obj) - m_start) / this->GetObjectSize();
}
size_t GetPeakIndex() const {
return this->GetObjectIndexImpl(reinterpret_cast<const void *>(m_peak));
ALWAYS_INLINE size_t GetPeakIndex() const {
return this->GetObjectIndex(reinterpret_cast<const void *>(m_peak));
}
uintptr_t GetSlabHeapAddress() const {
ALWAYS_INLINE uintptr_t GetSlabHeapAddress() const {
return m_start;
}
size_t GetNumRemaining() const {
ALWAYS_INLINE size_t GetNumRemaining() const {
size_t remaining = 0;
/* Only calculate the number of remaining objects under debug configuration. */
#if defined(MESOSPHERE_BUILD_FOR_DEBUGGING)
while (true) {
auto *cur = this->GetImpl()->GetHead();
auto *cur = this->GetHead();
remaining = 0;
while (this->Contains(reinterpret_cast<uintptr_t>(cur))) {
++remaining;
cur = cur->next;
if constexpr (SupportDynamicExpansion) {
const auto &slab_region = KMemoryLayout::GetSlabRegion();
while (this->Contains(reinterpret_cast<uintptr_t>(cur)) || slab_region.Contains(reinterpret_cast<uintptr_t>(cur))) {
++remaining;
cur = cur->next;
}
} else {
while (this->Contains(reinterpret_cast<uintptr_t>(cur))) {
++remaining;
cur = cur->next;
}
}
if (cur == nullptr) {
@ -204,29 +217,31 @@ namespace ams::kern {
}
};
template<typename T>
class KSlabHeap : public KSlabHeapBase {
template<typename T, bool SupportDynamicExpansion>
class KSlabHeap : public KSlabHeapBase<SupportDynamicExpansion> {
private:
using BaseHeap = KSlabHeapBase<SupportDynamicExpansion>;
public:
constexpr KSlabHeap() : KSlabHeapBase() { /* ... */ }
constexpr KSlabHeap() = default;
void Initialize(void *memory, size_t memory_size) {
this->InitializeImpl(sizeof(T), memory, memory_size);
BaseHeap::Initialize(sizeof(T), memory, memory_size);
}
T *Allocate() {
T *obj = reinterpret_cast<T *>(this->AllocateImpl());
ALWAYS_INLINE T *Allocate() {
T *obj = static_cast<T *>(BaseHeap::Allocate());
if (AMS_LIKELY(obj != nullptr)) {
std::construct_at(obj);
}
return obj;
}
void Free(T *obj) {
this->FreeImpl(obj);
ALWAYS_INLINE void Free(T *obj) {
BaseHeap::Free(obj);
}
size_t GetObjectIndex(const T *obj) const {
return this->GetObjectIndexImpl(obj);
ALWAYS_INLINE size_t GetObjectIndex(const T *obj) const {
return BaseHeap::GetObjectIndex(obj);
}
};

15
libraries/libmesosphere/include/mesosphere/kern_k_target_system.hpp
View file

@ -23,12 +23,13 @@ namespace ams::kern {
private:
friend class KSystemControl;
private:
static inline bool s_is_debug_mode;
static inline bool s_enable_debug_logging;
static inline bool s_enable_user_exception_handlers;
static inline bool s_enable_debug_memory_fill;
static inline bool s_enable_user_pmu_access;
static inline bool s_enable_kernel_debugging;
static inline constinit bool s_is_debug_mode;
static inline constinit bool s_enable_debug_logging;
static inline constinit bool s_enable_user_exception_handlers;
static inline constinit bool s_enable_debug_memory_fill;
static inline constinit bool s_enable_user_pmu_access;
static inline constinit bool s_enable_kernel_debugging;
static inline constinit bool s_enable_dynamic_resource_limits;
private:
static ALWAYS_INLINE void SetIsDebugMode(bool en) { s_is_debug_mode = en; }
static ALWAYS_INLINE void EnableDebugLogging(bool en) { s_enable_debug_logging = en; }
@ -36,6 +37,7 @@ namespace ams::kern {
static ALWAYS_INLINE void EnableDebugMemoryFill(bool en) { s_enable_debug_memory_fill = en; }
static ALWAYS_INLINE void EnableUserPmuAccess(bool en) { s_enable_user_pmu_access = en; }
static ALWAYS_INLINE void EnableKernelDebugging(bool en) { s_enable_kernel_debugging = en; }
static ALWAYS_INLINE void EnableDynamicResourceLimits(bool en) { s_enable_dynamic_resource_limits = en; }
public:
static ALWAYS_INLINE bool IsDebugMode() { return s_is_debug_mode; }
static ALWAYS_INLINE bool IsDebugLoggingEnabled() { return s_enable_debug_logging; }
@ -43,6 +45,7 @@ namespace ams::kern {
static ALWAYS_INLINE bool IsDebugMemoryFillEnabled() { return s_enable_debug_memory_fill; }
static ALWAYS_INLINE bool IsUserPmuAccessEnabled() { return s_enable_user_pmu_access; }
static ALWAYS_INLINE bool IsKernelDebuggingEnabled() { return s_enable_kernel_debugging; }
static ALWAYS_INLINE bool IsDynamicResourceLimitsEnabled() { return s_enable_dynamic_resource_limits; }
};
}

27
libraries/libmesosphere/include/mesosphere/kern_k_unused_slab_memory.hpp Normal file
View file

@ -0,0 +1,27 @@
/*
* Copyright (c) 2018-2020 Atmosphère-NX
*
* This program is free software; you can redistribute it and/or modify it
* under the terms and conditions of the GNU General Public License,
* version 2, as published by the Free Software Foundation.
*
* This program is distributed in the hope it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#pragma once
#include <mesosphere/kern_common.hpp>
#include <mesosphere/kern_k_typed_address.hpp>
namespace ams::kern {
/* Utilities to allocate/free memory from the "unused" gaps between slab heaps. */
/* See KTargetSystem::IsDynamicResourceLimitsEnabled() usage for more context. */
KVirtualAddress AllocateUnusedSlabMemory(size_t size, size_t alignment);
void FreeUnusedSlabMemory(KVirtualAddress address, size_t size);
}

27
libraries/libmesosphere/include/mesosphere/kern_kernel.hpp
View file

@ -63,14 +63,21 @@ namespace ams::kern {
static constexpr size_t ApplicationMemoryBlockSlabHeapSize = 20000;
static constexpr size_t SystemMemoryBlockSlabHeapSize = 10000;
static constexpr size_t BlockInfoSlabHeapSize = 4000;
static constexpr size_t ReservedDynamicPageCount = 70;
private:
static State s_state;
static KResourceLimit s_system_resource_limit;
static KMemoryManager s_memory_manager;
static KPageTableManager s_page_table_manager;
static KPageTableSlabHeap s_page_table_heap;
static KMemoryBlockSlabHeap s_app_memory_block_heap;
static KMemoryBlockSlabHeap s_sys_memory_block_heap;
static KBlockInfoSlabHeap s_block_info_heap;
static KPageTableManager s_app_page_table_manager;
static KPageTableManager s_sys_page_table_manager;
static KMemoryBlockSlabManager s_app_memory_block_manager;
static KMemoryBlockSlabManager s_sys_memory_block_manager;
static KBlockInfoManager s_block_info_manager;
static KBlockInfoManager s_app_block_info_manager;
static KBlockInfoManager s_sys_block_info_manager;
static KSupervisorPageTable s_supervisor_page_table;
static KUnsafeMemory s_unsafe_memory;
static KWorkerTaskManager s_worker_task_managers[KWorkerTaskManager::WorkerType_Count];
@ -130,12 +137,20 @@ namespace ams::kern {
return s_sys_memory_block_manager;
}
static ALWAYS_INLINE KBlockInfoManager &GetBlockInfoManager() {
return s_block_info_manager;
static ALWAYS_INLINE KBlockInfoManager &GetApplicationBlockInfoManager() {
return s_app_block_info_manager;
}
static ALWAYS_INLINE KPageTableManager &GetPageTableManager() {
return s_page_table_manager;
static ALWAYS_INLINE KBlockInfoManager &GetSystemBlockInfoManager() {
return s_sys_block_info_manager;
}
static ALWAYS_INLINE KPageTableManager &GetApplicationPageTableManager() {
return s_app_page_table_manager;
}
static ALWAYS_INLINE KPageTableManager &GetSystemPageTableManager() {
return s_sys_page_table_manager;
}
static ALWAYS_INLINE KSupervisorPageTable &GetKernelPageTable() {

42
libraries/libmesosphere/include/mesosphere/kern_slab_helpers.hpp
View file

@ -18,15 +18,16 @@
#include <mesosphere/kern_k_auto_object.hpp>
#include <mesosphere/kern_k_slab_heap.hpp>
#include <mesosphere/kern_k_auto_object_container.hpp>
#include <mesosphere/kern_k_unused_slab_memory.hpp>
namespace ams::kern {
template<class Derived>
template<class Derived, bool SupportDynamicExpansion = false>
class KSlabAllocated {
private:
static inline KSlabHeap<Derived> s_slab_heap;
static constinit inline KSlabHeap<Derived, SupportDynamicExpansion> s_slab_heap;
public:
constexpr KSlabAllocated() { /* ... */ }
constexpr KSlabAllocated() = default;
size_t GetSlabIndex() const {
return s_slab_heap.GetIndex(static_cast<const Derived *>(this));
@ -36,14 +37,25 @@ namespace ams::kern {
s_slab_heap.Initialize(memory, memory_size);
}
static ALWAYS_INLINE Derived *Allocate() {
static Derived *Allocate() {
return s_slab_heap.Allocate();
}
static ALWAYS_INLINE void Free(Derived *obj) {
static void Free(Derived *obj) {
s_slab_heap.Free(obj);
}
template<bool Enable = SupportDynamicExpansion, typename = typename std::enable_if<Enable>::type>
static Derived *AllocateFromUnusedSlabMemory() {
static_assert(Enable == SupportDynamicExpansion);
Derived * const obj = GetPointer<Derived>(AllocateUnusedSlabMemory(sizeof(Derived), alignof(Derived)));
if (AMS_LIKELY(obj != nullptr)) {
std::construct_at(obj);
}
return obj;
}
static size_t GetObjectSize() { return s_slab_heap.GetObjectSize(); }
static size_t GetSlabHeapSize() { return s_slab_heap.GetSlabHeapSize(); }
static size_t GetPeakIndex() { return s_slab_heap.GetPeakIndex(); }
@ -52,12 +64,12 @@ namespace ams::kern {
static size_t GetNumRemaining() { return s_slab_heap.GetNumRemaining(); }
};
template<typename Derived, typename Base>
template<typename Derived, typename Base, bool SupportDynamicExpansion = false>
class KAutoObjectWithSlabHeapAndContainer : public Base {
static_assert(std::is_base_of<KAutoObjectWithList, Base>::value);
private:
static inline KSlabHeap<Derived> s_slab_heap;
static inline KAutoObjectWithListContainer s_container;
static constinit inline KSlabHeap<Derived, SupportDynamicExpansion> s_slab_heap;
static constinit inline KAutoObjectWithListContainer s_container;
private:
static ALWAYS_INLINE Derived *Allocate() {
return s_slab_heap.Allocate();
@ -73,7 +85,7 @@ namespace ams::kern {
ALWAYS_INLINE ~ListAccessor() { /* ... */ }
};
public:
constexpr KAutoObjectWithSlabHeapAndContainer() : Base() { /* ... */ }
constexpr KAutoObjectWithSlabHeapAndContainer() = default;
virtual void Destroy() override {
const bool is_initialized = this->IsInitialized();
@ -109,6 +121,18 @@ namespace ams::kern {
return obj;
}
template<bool Enable = SupportDynamicExpansion, typename = typename std::enable_if<Enable>::type>
static Derived *CreateFromUnusedSlabMemory() {
static_assert(Enable == SupportDynamicExpansion);
Derived * const obj = GetPointer<Derived>(AllocateUnusedSlabMemory(sizeof(Derived), alignof(Derived)));
if (AMS_LIKELY(obj != nullptr)) {
std::construct_at(obj);
KAutoObject::Create(obj);
}
return obj;
}
static void Register(Derived *obj) {
return s_container.Register(obj);
}