mirror of
https://github.com/Atmosphere-NX/Atmosphere.git
synced 2025-05-31 23:08:22 -04:00
kern/util: use custom atomics wrapper to substantially improve codegen
This commit is contained in:
parent
52332e8d75
commit
d74f364107
26 changed files with 688 additions and 260 deletions
|
@ -41,8 +41,48 @@ namespace ams::kern {
|
|||
virtual const char *GetTypeName() { return GetStaticTypeName(); } \
|
||||
private:
|
||||
|
||||
|
||||
class KAutoObject {
|
||||
public:
|
||||
class ReferenceCount {
|
||||
NON_COPYABLE(ReferenceCount);
|
||||
NON_MOVEABLE(ReferenceCount);
|
||||
private:
|
||||
using Storage = u32;
|
||||
private:
|
||||
util::Atomic<Storage> m_value;
|
||||
public:
|
||||
ALWAYS_INLINE explicit ReferenceCount() { /* ... */ }
|
||||
constexpr ALWAYS_INLINE explicit ReferenceCount(Storage v) : m_value(v) { /* ... */ }
|
||||
|
||||
ALWAYS_INLINE void operator=(Storage v) { m_value = v; }
|
||||
|
||||
ALWAYS_INLINE Storage GetValue() const { return m_value.Load(); }
|
||||
|
||||
ALWAYS_INLINE bool Open() {
|
||||
/* Atomically increment the reference count, only if it's positive. */
|
||||
u32 cur = m_value.Load<std::memory_order_relaxed>();
|
||||
do {
|
||||
if (AMS_UNLIKELY(cur == 0)) {
|
||||
MESOSPHERE_AUDIT(cur != 0);
|
||||
return false;
|
||||
}
|
||||
MESOSPHERE_ABORT_UNLESS(cur < cur + 1);
|
||||
} while (AMS_UNLIKELY(!m_value.CompareExchangeWeak<std::memory_order_relaxed>(cur, cur + 1)));
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
ALWAYS_INLINE bool Close() {
|
||||
/* Atomically decrement the reference count, not allowing it to become negative. */
|
||||
u32 cur = m_value.Load<std::memory_order_relaxed>();
|
||||
do {
|
||||
MESOSPHERE_ABORT_UNLESS(cur > 0);
|
||||
} while (AMS_UNLIKELY(!m_value.CompareExchangeWeak<std::memory_order_relaxed>(cur, cur - 1)));
|
||||
|
||||
/* Return whether the object was closed. */
|
||||
return cur - 1 == 0;
|
||||
}
|
||||
};
|
||||
protected:
|
||||
class TypeObj {
|
||||
private:
|
||||
|
@ -74,7 +114,7 @@ namespace ams::kern {
|
|||
MESOSPHERE_AUTOOBJECT_TRAITS(KAutoObject, KAutoObject);
|
||||
private:
|
||||
KAutoObject *m_next_closed_object;
|
||||
std::atomic<u32> m_ref_count;
|
||||
ReferenceCount m_ref_count;
|
||||
#if defined(MESOSPHERE_ENABLE_DEVIRTUALIZED_DYNAMIC_CAST)
|
||||
ClassTokenType m_class_token;
|
||||
#endif
|
||||
|
@ -98,7 +138,7 @@ namespace ams::kern {
|
|||
virtual KProcess *GetOwner() const { return nullptr; }
|
||||
|
||||
u32 GetReferenceCount() const {
|
||||
return m_ref_count.load();
|
||||
return m_ref_count.GetValue();
|
||||
}
|
||||
|
||||
ALWAYS_INLINE bool IsDerivedFrom(const TypeObj &rhs) const {
|
||||
|
@ -141,8 +181,19 @@ namespace ams::kern {
|
|||
}
|
||||
}
|
||||
|
||||
bool Open();
|
||||
void Close();
|
||||
NOINLINE bool Open() {
|
||||
MESOSPHERE_ASSERT_THIS();
|
||||
|
||||
return m_ref_count.Open();
|
||||
}
|
||||
|
||||
NOINLINE void Close() {
|
||||
MESOSPHERE_ASSERT_THIS();
|
||||
|
||||
if (m_ref_count.Close()) {
|
||||
this->ScheduleDestruction();
|
||||
}
|
||||
}
|
||||
private:
|
||||
/* NOTE: This has to be defined *after* KThread is defined. */
|
||||
/* Nintendo seems to handle this by defining Open/Close() in a cpp, but we'd like them to remain in headers. */
|
||||
|
@ -256,5 +307,52 @@ namespace ams::kern {
|
|||
constexpr ALWAYS_INLINE bool IsNotNull() const { return m_obj != nullptr; }
|
||||
};
|
||||
|
||||
template<typename T> requires std::derived_from<T, KAutoObject>
|
||||
class KSharedAutoObject {
|
||||
private:
|
||||
T *m_object;
|
||||
KAutoObject::ReferenceCount m_ref_count;
|
||||
public:
|
||||
explicit KSharedAutoObject() : m_object(nullptr) { /* ... */ }
|
||||
|
||||
void Attach(T *obj) {
|
||||
MESOSPHERE_ASSERT(m_object == nullptr);
|
||||
|
||||
/* Set our object. */
|
||||
m_object = obj;
|
||||
|
||||
/* Open reference to our object. */
|
||||
m_object->Open();
|
||||
|
||||
/* Set our reference count. */
|
||||
m_ref_count = 1;
|
||||
}
|
||||
|
||||
bool Open() {
|
||||
return m_ref_count.Open();
|
||||
}
|
||||
|
||||
void Close() {
|
||||
if (m_ref_count.Close()) {
|
||||
this->Detach();
|
||||
}
|
||||
}
|
||||
|
||||
ALWAYS_INLINE T *Get() const {
|
||||
return m_object;
|
||||
}
|
||||
private:
|
||||
void Detach() {
|
||||
/* Close our object, if we have one. */
|
||||
if (T * const object = m_object; AMS_LIKELY(object != nullptr)) {
|
||||
/* Set our object to a debug sentinel value, which will cause crash if accessed. */
|
||||
m_object = reinterpret_cast<T *>(1);
|
||||
|
||||
/* Close reference to our object. */
|
||||
object->Close();
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
}
|
||||
|
|
|
@ -28,12 +28,12 @@ namespace ams::kern {
|
|||
class KClientPort final : public KSynchronizationObject {
|
||||
MESOSPHERE_AUTOOBJECT_TRAITS(KClientPort, KSynchronizationObject);
|
||||
private:
|
||||
std::atomic<s32> m_num_sessions;
|
||||
std::atomic<s32> m_peak_sessions;
|
||||
util::Atomic<s32> m_num_sessions;
|
||||
util::Atomic<s32> m_peak_sessions;
|
||||
s32 m_max_sessions;
|
||||
KPort *m_parent;
|
||||
public:
|
||||
constexpr KClientPort() : m_num_sessions(), m_peak_sessions(), m_max_sessions(), m_parent() { /* ... */ }
|
||||
constexpr KClientPort() : m_num_sessions(0), m_peak_sessions(0), m_max_sessions(), m_parent() { /* ... */ }
|
||||
|
||||
void Initialize(KPort *parent, s32 max_sessions);
|
||||
void OnSessionFinalized();
|
||||
|
@ -41,8 +41,8 @@ namespace ams::kern {
|
|||
|
||||
constexpr const KPort *GetParent() const { return m_parent; }
|
||||
|
||||
ALWAYS_INLINE s32 GetNumSessions() const { return m_num_sessions; }
|
||||
ALWAYS_INLINE s32 GetPeakSessions() const { return m_peak_sessions; }
|
||||
ALWAYS_INLINE s32 GetNumSessions() const { return m_num_sessions.Load(); }
|
||||
ALWAYS_INLINE s32 GetPeakSessions() const { return m_peak_sessions.Load(); }
|
||||
ALWAYS_INLINE s32 GetMaxSessions() const { return m_max_sessions; }
|
||||
|
||||
bool IsLight() const;
|
||||
|
|
|
@ -25,26 +25,10 @@ namespace ams::kern {
|
|||
class KDebugBase : public KSynchronizationObject {
|
||||
protected:
|
||||
using DebugEventList = util::IntrusiveListBaseTraits<KEventInfo>::ListType;
|
||||
private:
|
||||
class ProcessHolder {
|
||||
private:
|
||||
friend class KDebugBase;
|
||||
private:
|
||||
KProcess *m_process;
|
||||
std::atomic<u32> m_ref_count;
|
||||
private:
|
||||
explicit ProcessHolder() : m_process(nullptr) { /* ... */ }
|
||||
|
||||
void Attach(KProcess *process);
|
||||
void Detach();
|
||||
|
||||
bool Open();
|
||||
void Close();
|
||||
};
|
||||
private:
|
||||
DebugEventList m_event_info_list;
|
||||
u32 m_continue_flags;
|
||||
ProcessHolder m_process_holder;
|
||||
KSharedAutoObject<KProcess> m_process_holder;
|
||||
KLightLock m_lock;
|
||||
KProcess::State m_old_process_state;
|
||||
bool m_is_attached;
|
||||
|
@ -89,7 +73,7 @@ namespace ams::kern {
|
|||
}
|
||||
|
||||
ALWAYS_INLINE KProcess *GetProcessUnsafe() const {
|
||||
return m_process_holder.m_process;
|
||||
return m_process_holder.Get();
|
||||
}
|
||||
private:
|
||||
void PushDebugEvent(ams::svc::DebugEvent event, uintptr_t param0 = 0, uintptr_t param1 = 0, uintptr_t param2 = 0, uintptr_t param3 = 0, uintptr_t param4 = 0);
|
||||
|
|
|
@ -29,9 +29,9 @@ namespace ams::kern {
|
|||
private:
|
||||
using PageBuffer = KDynamicPageManager::PageBuffer;
|
||||
private:
|
||||
std::atomic<size_t> m_used{};
|
||||
std::atomic<size_t> m_peak{};
|
||||
std::atomic<size_t> m_count{};
|
||||
util::Atomic<size_t> m_used{0};
|
||||
util::Atomic<size_t> m_peak{0};
|
||||
util::Atomic<size_t> m_count{0};
|
||||
KVirtualAddress m_address{};
|
||||
size_t m_size{};
|
||||
public:
|
||||
|
@ -39,9 +39,9 @@ namespace ams::kern {
|
|||
|
||||
constexpr ALWAYS_INLINE KVirtualAddress GetAddress() const { return m_address; }
|
||||
constexpr ALWAYS_INLINE size_t GetSize() const { return m_size; }
|
||||
constexpr ALWAYS_INLINE size_t GetUsed() const { return m_used.load(); }
|
||||
constexpr ALWAYS_INLINE size_t GetPeak() const { return m_peak.load(); }
|
||||
constexpr ALWAYS_INLINE size_t GetCount() const { return m_count.load(); }
|
||||
constexpr ALWAYS_INLINE size_t GetUsed() const { return m_used.Load(); }
|
||||
constexpr ALWAYS_INLINE size_t GetPeak() const { return m_peak.Load(); }
|
||||
constexpr ALWAYS_INLINE size_t GetCount() const { return m_count.Load(); }
|
||||
|
||||
constexpr ALWAYS_INLINE bool IsInRange(KVirtualAddress addr) const {
|
||||
return this->GetAddress() <= addr && addr <= this->GetAddress() + this->GetSize() - 1;
|
||||
|
@ -58,7 +58,7 @@ namespace ams::kern {
|
|||
KSlabHeapImpl::Initialize();
|
||||
|
||||
/* Allocate until we have the correct number of objects. */
|
||||
while (m_count.load() < num_objects) {
|
||||
while (m_count.Load() < num_objects) {
|
||||
auto *allocated = reinterpret_cast<T *>(page_allocator->Allocate());
|
||||
MESOSPHERE_ABORT_UNLESS(allocated != nullptr);
|
||||
|
||||
|
@ -66,7 +66,7 @@ namespace ams::kern {
|
|||
KSlabHeapImpl::Free(allocated + i);
|
||||
}
|
||||
|
||||
m_count.fetch_add(sizeof(PageBuffer) / sizeof(T));
|
||||
m_count.FetchAdd(sizeof(PageBuffer) / sizeof(T));
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -89,7 +89,7 @@ namespace ams::kern {
|
|||
for (size_t i = 1; i < sizeof(PageBuffer) / sizeof(T); i++) {
|
||||
KSlabHeapImpl::Free(allocated + i);
|
||||
}
|
||||
m_count.fetch_add(sizeof(PageBuffer) / sizeof(T));
|
||||
m_count.FetchAdd(sizeof(PageBuffer) / sizeof(T));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
@ -99,10 +99,10 @@ namespace ams::kern {
|
|||
std::construct_at(allocated);
|
||||
|
||||
/* Update our tracking. */
|
||||
size_t used = m_used.fetch_add(1) + 1;
|
||||
size_t peak = m_peak.load();
|
||||
size_t used = m_used.FetchAdd(1) + 1;
|
||||
size_t peak = m_peak.Load();
|
||||
while (peak < used) {
|
||||
if (m_peak.compare_exchange_weak(peak, used, std::memory_order_relaxed)) {
|
||||
if (m_peak.CompareExchangeWeak<std::memory_order_relaxed>(peak, used)) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
@ -113,7 +113,7 @@ namespace ams::kern {
|
|||
|
||||
ALWAYS_INLINE void Free(T *t) {
|
||||
KSlabHeapImpl::Free(t);
|
||||
m_used.fetch_sub(1);
|
||||
m_used.FetchSub(1);
|
||||
}
|
||||
};
|
||||
|
||||
|
|
|
@ -23,7 +23,7 @@ namespace ams::kern {
|
|||
|
||||
class KLightLock {
|
||||
private:
|
||||
std::atomic<uintptr_t> m_tag;
|
||||
util::Atomic<uintptr_t> m_tag;
|
||||
public:
|
||||
constexpr KLightLock() : m_tag(0) { /* ... */ }
|
||||
|
||||
|
@ -31,12 +31,11 @@ namespace ams::kern {
|
|||
MESOSPHERE_ASSERT_THIS();
|
||||
|
||||
const uintptr_t cur_thread = reinterpret_cast<uintptr_t>(GetCurrentThreadPointer());
|
||||
const uintptr_t cur_thread_tag = (cur_thread | 1);
|
||||
|
||||
while (true) {
|
||||
uintptr_t old_tag = m_tag.load(std::memory_order_relaxed);
|
||||
uintptr_t old_tag = m_tag.Load<std::memory_order_relaxed>();
|
||||
|
||||
while (!m_tag.compare_exchange_weak(old_tag, (old_tag == 0) ? cur_thread : old_tag | 1, std::memory_order_acquire)) {
|
||||
while (!m_tag.CompareExchangeWeak<std::memory_order_acquire>(old_tag, (old_tag == 0) ? cur_thread : (old_tag | 1))) {
|
||||
/* ... */
|
||||
}
|
||||
|
||||
|
@ -52,7 +51,7 @@ namespace ams::kern {
|
|||
const uintptr_t cur_thread = reinterpret_cast<uintptr_t>(GetCurrentThreadPointer());
|
||||
|
||||
uintptr_t expected = cur_thread;
|
||||
if (!m_tag.compare_exchange_strong(expected, 0, std::memory_order_release)) {
|
||||
if (!m_tag.CompareExchangeStrong<std::memory_order_release>(expected, 0)) {
|
||||
this->UnlockSlowPath(cur_thread);
|
||||
}
|
||||
}
|
||||
|
@ -60,8 +59,8 @@ namespace ams::kern {
|
|||
bool LockSlowPath(uintptr_t owner, uintptr_t cur_thread);
|
||||
void UnlockSlowPath(uintptr_t cur_thread);
|
||||
|
||||
ALWAYS_INLINE bool IsLocked() const { return m_tag.load() != 0; }
|
||||
ALWAYS_INLINE bool IsLockedByCurrentThread() const { return (m_tag.load() | 0x1ul) == (reinterpret_cast<uintptr_t>(GetCurrentThreadPointer()) | 0x1ul); }
|
||||
ALWAYS_INLINE bool IsLocked() const { return m_tag.Load() != 0; }
|
||||
ALWAYS_INLINE bool IsLockedByCurrentThread() const { return (m_tag.Load() | 0x1ul) == (reinterpret_cast<uintptr_t>(GetCurrentThreadPointer()) | 0x1ul); }
|
||||
};
|
||||
|
||||
using KScopedLightLock = KScopedLock<KLightLock>;
|
||||
|
|
|
@ -58,7 +58,7 @@ namespace ams::kern {
|
|||
using TLPIterator = TLPTree::iterator;
|
||||
private:
|
||||
KProcessPageTable m_page_table{};
|
||||
std::atomic<size_t> m_used_kernel_memory_size{};
|
||||
util::Atomic<size_t> m_used_kernel_memory_size{0};
|
||||
TLPTree m_fully_used_tlp_tree{};
|
||||
TLPTree m_partially_used_tlp_tree{};
|
||||
s32 m_ideal_core_id{};
|
||||
|
@ -77,7 +77,7 @@ namespace ams::kern {
|
|||
bool m_is_initialized{};
|
||||
bool m_is_application{};
|
||||
char m_name[13]{};
|
||||
std::atomic<u16> m_num_running_threads{};
|
||||
util::Atomic<u16> m_num_running_threads{0};
|
||||
u32 m_flags{};
|
||||
KMemoryManager::Pool m_memory_pool{};
|
||||
s64 m_schedule_count{};
|
||||
|
@ -109,14 +109,14 @@ namespace ams::kern {
|
|||
KThread *m_running_threads[cpu::NumCores]{};
|
||||
u64 m_running_thread_idle_counts[cpu::NumCores]{};
|
||||
KThread *m_pinned_threads[cpu::NumCores]{};
|
||||
std::atomic<s64> m_cpu_time{};
|
||||
std::atomic<s64> m_num_process_switches{};
|
||||
std::atomic<s64> m_num_thread_switches{};
|
||||
std::atomic<s64> m_num_fpu_switches{};
|
||||
std::atomic<s64> m_num_supervisor_calls{};
|
||||
std::atomic<s64> m_num_ipc_messages{};
|
||||
std::atomic<s64> m_num_ipc_replies{};
|
||||
std::atomic<s64> m_num_ipc_receives{};
|
||||
util::Atomic<s64> m_cpu_time{0};
|
||||
util::Atomic<s64> m_num_process_switches{0};
|
||||
util::Atomic<s64> m_num_thread_switches{0};
|
||||
util::Atomic<s64> m_num_fpu_switches{0};
|
||||
util::Atomic<s64> m_num_supervisor_calls{0};
|
||||
util::Atomic<s64> m_num_ipc_messages{0};
|
||||
util::Atomic<s64> m_num_ipc_replies{0};
|
||||
util::Atomic<s64> m_num_ipc_receives{0};
|
||||
KDynamicPageManager m_dynamic_page_manager{};
|
||||
KMemoryBlockSlabManager m_memory_block_slab_manager{};
|
||||
KBlockInfoManager m_block_info_manager{};
|
||||
|
@ -288,8 +288,8 @@ namespace ams::kern {
|
|||
|
||||
KThread *GetExceptionThread() const { return m_exception_thread; }
|
||||
|
||||
void AddCpuTime(s64 diff) { m_cpu_time += diff; }
|
||||
s64 GetCpuTime() { return m_cpu_time; }
|
||||
void AddCpuTime(s64 diff) { m_cpu_time.FetchAdd(diff); }
|
||||
s64 GetCpuTime() { return m_cpu_time.Load(); }
|
||||
|
||||
constexpr s64 GetScheduledCount() const { return m_schedule_count; }
|
||||
void IncrementScheduledCount() { ++m_schedule_count; }
|
||||
|
|
|
@ -39,14 +39,16 @@ namespace ams::kern {
|
|||
static_assert(ams::svc::HighestThreadPriority <= HighestCoreMigrationAllowedPriority);
|
||||
|
||||
struct SchedulingState {
|
||||
std::atomic<u8> needs_scheduling;
|
||||
bool interrupt_task_runnable;
|
||||
bool should_count_idle;
|
||||
u64 idle_count;
|
||||
KThread *highest_priority_thread;
|
||||
void *idle_thread_stack;
|
||||
KThread *prev_thread;
|
||||
KInterruptTaskManager *interrupt_task_manager;
|
||||
util::Atomic<u8> needs_scheduling{false};
|
||||
bool interrupt_task_runnable{false};
|
||||
bool should_count_idle{false};
|
||||
u64 idle_count{0};
|
||||
KThread *highest_priority_thread{nullptr};
|
||||
void *idle_thread_stack{nullptr};
|
||||
util::Atomic<KThread *> prev_thread{nullptr};
|
||||
KInterruptTaskManager *interrupt_task_manager{nullptr};
|
||||
|
||||
constexpr SchedulingState() = default;
|
||||
};
|
||||
private:
|
||||
friend class KScopedSchedulerLock;
|
||||
|
@ -58,7 +60,7 @@ namespace ams::kern {
|
|||
s32 m_core_id;
|
||||
s64 m_last_context_switch_time;
|
||||
KThread *m_idle_thread;
|
||||
std::atomic<KThread *> m_current_thread;
|
||||
util::Atomic<KThread *> m_current_thread;
|
||||
public:
|
||||
constexpr KScheduler()
|
||||
: m_state(), m_is_active(false), m_core_id(0), m_last_context_switch_time(0), m_idle_thread(nullptr), m_current_thread(nullptr)
|
||||
|
@ -98,11 +100,11 @@ namespace ams::kern {
|
|||
}
|
||||
|
||||
ALWAYS_INLINE KThread *GetPreviousThread() const {
|
||||
return m_state.prev_thread;
|
||||
return m_state.prev_thread.Load<std::memory_order_relaxed>();
|
||||
}
|
||||
|
||||
ALWAYS_INLINE KThread *GetSchedulerCurrentThread() const {
|
||||
return m_current_thread;
|
||||
return m_current_thread.Load();
|
||||
}
|
||||
|
||||
ALWAYS_INLINE s64 GetLastContextSwitchTime() const {
|
||||
|
@ -182,7 +184,7 @@ namespace ams::kern {
|
|||
|
||||
GetCurrentThread().EnableDispatch();
|
||||
|
||||
if (m_state.needs_scheduling.load()) {
|
||||
if (m_state.needs_scheduling.Load()) {
|
||||
/* Disable interrupts, and then check again if rescheduling is needed. */
|
||||
KScopedInterruptDisable intr_disable;
|
||||
|
||||
|
@ -192,7 +194,7 @@ namespace ams::kern {
|
|||
|
||||
ALWAYS_INLINE void RescheduleCurrentCoreImpl() {
|
||||
/* Check that scheduling is needed. */
|
||||
if (AMS_LIKELY(m_state.needs_scheduling.load())) {
|
||||
if (AMS_LIKELY(m_state.needs_scheduling.Load())) {
|
||||
GetCurrentThread().DisableDispatch();
|
||||
this->Schedule();
|
||||
GetCurrentThread().EnableDispatch();
|
||||
|
|
|
@ -35,7 +35,7 @@ namespace ams::kern {
|
|||
ServerClosed = 3,
|
||||
};
|
||||
private:
|
||||
std::atomic<std::underlying_type<State>::type> m_atomic_state;
|
||||
util::Atomic<std::underlying_type<State>::type> m_atomic_state;
|
||||
bool m_initialized;
|
||||
KServerSession m_server;
|
||||
KClientSession m_client;
|
||||
|
@ -48,7 +48,7 @@ namespace ams::kern {
|
|||
}
|
||||
|
||||
ALWAYS_INLINE State GetState() const {
|
||||
return static_cast<State>(m_atomic_state.load());
|
||||
return static_cast<State>(m_atomic_state.Load());
|
||||
}
|
||||
public:
|
||||
constexpr KSession()
|
||||
|
|
|
@ -76,21 +76,18 @@ namespace ams::kern {
|
|||
NON_MOVEABLE(KSlabHeapBase);
|
||||
private:
|
||||
size_t m_obj_size{};
|
||||
uintptr_t m_peak{};
|
||||
util::Atomic<uintptr_t> m_peak{0};
|
||||
uintptr_t m_start{};
|
||||
uintptr_t m_end{};
|
||||
private:
|
||||
ALWAYS_INLINE void UpdatePeakImpl(uintptr_t obj) {
|
||||
static_assert(std::atomic_ref<uintptr_t>::is_always_lock_free);
|
||||
std::atomic_ref<uintptr_t> peak_ref(m_peak);
|
||||
|
||||
const uintptr_t alloc_peak = obj + this->GetObjectSize();
|
||||
uintptr_t cur_peak = m_peak;
|
||||
uintptr_t cur_peak = m_peak.Load<std::memory_order_relaxed>();
|
||||
do {
|
||||
if (alloc_peak <= cur_peak) {
|
||||
break;
|
||||
}
|
||||
} while (!peak_ref.compare_exchange_strong(cur_peak, alloc_peak));
|
||||
} while (!m_peak.CompareExchangeStrong(cur_peak, alloc_peak));
|
||||
}
|
||||
public:
|
||||
constexpr KSlabHeapBase() = default;
|
||||
|
@ -112,8 +109,9 @@ namespace ams::kern {
|
|||
/* Set our tracking variables. */
|
||||
const size_t num_obj = (memory_size / obj_size);
|
||||
m_start = reinterpret_cast<uintptr_t>(memory);
|
||||
m_end = m_start + num_obj * obj_size;
|
||||
m_peak = m_start;
|
||||
m_end = m_start + num_obj * obj_size;
|
||||
|
||||
m_peak.Store<std::memory_order_relaxed>(m_start);
|
||||
|
||||
/* Free the objects. */
|
||||
u8 *cur = reinterpret_cast<u8 *>(m_end);
|
||||
|
@ -177,7 +175,7 @@ namespace ams::kern {
|
|||
}
|
||||
|
||||
ALWAYS_INLINE size_t GetPeakIndex() const {
|
||||
return this->GetObjectIndex(reinterpret_cast<const void *>(m_peak));
|
||||
return this->GetObjectIndex(reinterpret_cast<const void *>(m_peak.Load<std::memory_order_relaxed>()));
|
||||
}
|
||||
|
||||
ALWAYS_INLINE uintptr_t GetSlabHeapAddress() const {
|
||||
|
|
|
@ -37,7 +37,7 @@ namespace ams::kern {
|
|||
virtual void OnFinalizeSynchronizationObject() { MESOSPHERE_ASSERT_THIS(); }
|
||||
|
||||
void NotifyAvailable(Result result);
|
||||
void NotifyAvailable() {
|
||||
ALWAYS_INLINE void NotifyAvailable() {
|
||||
return this->NotifyAvailable(ResultSuccess());
|
||||
}
|
||||
public:
|
||||
|
|
|
@ -89,7 +89,7 @@ namespace ams::kern {
|
|||
KThreadContext *context;
|
||||
KThread *cur_thread;
|
||||
s16 disable_count;
|
||||
std::atomic<u8> dpc_flags;
|
||||
util::Atomic<u8> dpc_flags;
|
||||
u8 current_svc_id;
|
||||
bool is_calling_svc;
|
||||
bool is_in_exception_handler;
|
||||
|
@ -177,7 +177,7 @@ namespace ams::kern {
|
|||
static_assert(ams::util::HasRedBlackKeyType<ConditionVariableComparator>);
|
||||
static_assert(std::same_as<ams::util::RedBlackKeyType<ConditionVariableComparator, void>, ConditionVariableComparator::RedBlackKeyType>);
|
||||
private:
|
||||
static constinit inline std::atomic<u64> s_next_thread_id = 0;
|
||||
static constinit inline util::Atomic<u64> s_next_thread_id{0};
|
||||
private:
|
||||
util::IntrusiveListNode m_process_list_node{};
|
||||
util::IntrusiveRedBlackTreeNode m_condvar_arbiter_tree_node{};
|
||||
|
@ -192,7 +192,7 @@ namespace ams::kern {
|
|||
u64 m_virtual_affinity_mask{};
|
||||
KAffinityMask m_physical_affinity_mask{};
|
||||
u64 m_thread_id{};
|
||||
std::atomic<s64> m_cpu_time{};
|
||||
util::Atomic<s64> m_cpu_time{0};
|
||||
KProcessAddress m_address_key{};
|
||||
KProcess *m_parent{};
|
||||
void *m_kernel_stack_top{};
|
||||
|
@ -227,7 +227,7 @@ namespace ams::kern {
|
|||
s32 m_original_physical_ideal_core_id{};
|
||||
s32 m_num_core_migration_disables{};
|
||||
ThreadState m_thread_state{};
|
||||
std::atomic<u8> m_termination_requested{};
|
||||
util::Atomic<u8> m_termination_requested{false};
|
||||
bool m_wait_cancelled{};
|
||||
bool m_cancellable{};
|
||||
bool m_signaled{};
|
||||
|
@ -348,15 +348,15 @@ namespace ams::kern {
|
|||
#endif
|
||||
|
||||
ALWAYS_INLINE void RegisterDpc(DpcFlag flag) {
|
||||
this->GetStackParameters().dpc_flags.fetch_or(flag);
|
||||
this->GetStackParameters().dpc_flags.FetchOr(flag);
|
||||
}
|
||||
|
||||
ALWAYS_INLINE void ClearDpc(DpcFlag flag) {
|
||||
this->GetStackParameters().dpc_flags.fetch_and(~flag);
|
||||
this->GetStackParameters().dpc_flags.FetchAnd(~flag);
|
||||
}
|
||||
|
||||
ALWAYS_INLINE u8 GetDpc() const {
|
||||
return this->GetStackParameters().dpc_flags.load();
|
||||
return this->GetStackParameters().dpc_flags.Load();
|
||||
}
|
||||
|
||||
ALWAYS_INLINE bool HasDpc() const {
|
||||
|
@ -516,7 +516,7 @@ namespace ams::kern {
|
|||
m_closed_object = object;
|
||||
|
||||
/* Schedule destruction DPC. */
|
||||
if ((this->GetStackParameters().dpc_flags.load(std::memory_order_relaxed) & DpcFlag_PerformDestruction) == 0) {
|
||||
if ((this->GetStackParameters().dpc_flags.Load<std::memory_order_relaxed>() & DpcFlag_PerformDestruction) == 0) {
|
||||
this->RegisterDpc(DpcFlag_PerformDestruction);
|
||||
}
|
||||
}
|
||||
|
@ -544,12 +544,12 @@ namespace ams::kern {
|
|||
constexpr bool IsAttachedToDebugger() const { return m_debug_attached; }
|
||||
|
||||
void AddCpuTime(s32 core_id, s64 amount) {
|
||||
m_cpu_time += amount;
|
||||
m_cpu_time.FetchAdd(amount);
|
||||
/* TODO: Debug kernels track per-core tick counts. Should we? */
|
||||
MESOSPHERE_UNUSED(core_id);
|
||||
}
|
||||
|
||||
s64 GetCpuTime() const { return m_cpu_time.load(); }
|
||||
s64 GetCpuTime() const { return m_cpu_time.Load(); }
|
||||
|
||||
s64 GetCpuTime(s32 core_id) const {
|
||||
MESOSPHERE_ABORT_UNLESS(0 <= core_id && core_id < static_cast<s32>(cpu::NumCores));
|
||||
|
@ -591,7 +591,7 @@ namespace ams::kern {
|
|||
ALWAYS_INLINE void *GetKernelStackTop() const { return m_kernel_stack_top; }
|
||||
|
||||
ALWAYS_INLINE bool IsTerminationRequested() const {
|
||||
return m_termination_requested.load() || this->GetRawState() == ThreadState_Terminated;
|
||||
return m_termination_requested.Load() || this->GetRawState() == ThreadState_Terminated;
|
||||
}
|
||||
|
||||
size_t GetKernelStackUsage() const;
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue