mirrors/Atmosphere
1
0
Fork 0
mirror of https://github.com/Atmosphere-NX/Atmosphere.git synced 2025-05-27 21:24:11 -04:00
Code Issues Projects Releases Packages Wiki Activity

kern/util: use custom atomics wrapper to substantially improve codegen

Browse source
This commit is contained in:
Michael Scire 2021-10-19 15:24:15 -07:00
parent 52332e8d75
commit d74f364107
26 changed files with 688 additions and 260 deletions
Download patch file Download diff file Expand all files Collapse all files

20
libraries/libmesosphere/source/arch/arm64/kern_cpu.cpp
View file

@ -96,7 +96,7 @@ namespace ams::kern::arch::arm64::cpu {
KLightLock m_lock;
KLightLock m_cv_lock;
KLightConditionVariable m_cv;
std::atomic<u64> m_target_cores;
util::Atomic<u64> m_target_cores;
volatile Operation m_operation;
private:
static void ThreadFunction(uintptr_t _this) {
@ -109,7 +109,7 @@ namespace ams::kern::arch::arm64::cpu {
/* Wait for a request to come in. */
{
KScopedLightLock lk(m_cv_lock);
while ((m_target_cores.load() & (1ul << core_id)) == 0) {
while ((m_target_cores.Load() & (1ul << core_id)) == 0) {
m_cv.Wait(std::addressof(m_cv_lock));
}
}
@ -120,7 +120,7 @@ namespace ams::kern::arch::arm64::cpu {
/* Broadcast, if there's nothing pending. */
{
KScopedLightLock lk(m_cv_lock);
if (m_target_cores.load() == 0) {
if (m_target_cores.Load() == 0) {
m_cv.Broadcast();
}
}
@ -129,7 +129,7 @@ namespace ams::kern::arch::arm64::cpu {
void ProcessOperation();
public:
constexpr KCacheHelperInterruptHandler() : KInterruptHandler(), m_lock(), m_cv_lock(), m_cv(), m_target_cores(), m_operation(Operation::Idle) { /* ... */ }
constexpr KCacheHelperInterruptHandler() : KInterruptHandler(), m_lock(), m_cv_lock(), m_cv(), m_target_cores(0), m_operation(Operation::Idle) { /* ... */ }
void Initialize(s32 core_id) {
/* Reserve a thread from the system limit. */
@ -163,7 +163,7 @@ namespace ams::kern::arch::arm64::cpu {
if ((op == Operation::InstructionMemoryBarrier) || (Kernel::GetState() == Kernel::State::Initializing)) {
/* Check that there's no on-going operation. */
MESOSPHERE_ABORT_UNLESS(m_operation == Operation::Idle);
MESOSPHERE_ABORT_UNLESS(m_target_cores.load() == 0);
MESOSPHERE_ABORT_UNLESS(m_target_cores.Load() == 0);
/* Set operation. */
m_operation = op;
@ -171,13 +171,13 @@ namespace ams::kern::arch::arm64::cpu {
/* For certain operations, we want to send an interrupt. */
m_target_cores = other_cores_mask;
const u64 target_mask = m_target_cores.load();
const u64 target_mask = m_target_cores.Load();
DataSynchronizationBarrier();
Kernel::GetInterruptManager().SendInterProcessorInterrupt(KInterruptName_CacheOperation, target_mask);
this->ProcessOperation();
while (m_target_cores.load() != 0) {
while (m_target_cores.Load() != 0) {
cpu::Yield();
}
@ -189,7 +189,7 @@ namespace ams::kern::arch::arm64::cpu {
/* Check that there's no on-going operation. */
MESOSPHERE_ABORT_UNLESS(m_operation == Operation::Idle);
MESOSPHERE_ABORT_UNLESS(m_target_cores.load() == 0);
MESOSPHERE_ABORT_UNLESS(m_target_cores.Load() == 0);
/* Set operation. */
m_operation = op;
@ -199,7 +199,7 @@ namespace ams::kern::arch::arm64::cpu {
/* Use the condvar. */
m_cv.Broadcast();
while (m_target_cores.load() != 0) {
while (m_target_cores.Load() != 0) {
m_cv.Wait(std::addressof(m_cv_lock));
}
@ -287,7 +287,7 @@ namespace ams::kern::arch::arm64::cpu {
break;
}
m_target_cores &= ~(1ul << GetCurrentCoreId());
m_target_cores.FetchAnd(~(1ul << GetCurrentCoreId()));
}
ALWAYS_INLINE void SetEventLocally() {

31
libraries/libmesosphere/source/kern_k_auto_object.cpp
View file

@ -27,35 +27,4 @@ namespace ams::kern {
return obj;
}
NOINLINE bool KAutoObject::Open() {
MESOSPHERE_ASSERT_THIS();
/* Atomically increment the reference count, only if it's positive. */
u32 cur_ref_count = m_ref_count.load(std::memory_order_relaxed);
do {
if (AMS_UNLIKELY(cur_ref_count == 0)) {
MESOSPHERE_AUDIT(cur_ref_count != 0);
return false;
}
MESOSPHERE_ABORT_UNLESS(cur_ref_count < cur_ref_count + 1);
} while (!m_ref_count.compare_exchange_weak(cur_ref_count, cur_ref_count + 1, std::memory_order_relaxed));
return true;
}
NOINLINE void KAutoObject::Close() {
MESOSPHERE_ASSERT_THIS();
/* Atomically decrement the reference count, not allowing it to become negative. */
u32 cur_ref_count = m_ref_count.load(std::memory_order_relaxed);
do {
MESOSPHERE_ABORT_UNLESS(cur_ref_count > 0);
} while (!m_ref_count.compare_exchange_weak(cur_ref_count, cur_ref_count - 1, std::memory_order_relaxed));
/* If ref count hits zero, schedule the object for destruction. */
if (cur_ref_count - 1 == 0) {
this->ScheduleDestruction();
}
}
}

22
libraries/libmesosphere/source/kern_k_client_port.cpp
View file

@ -28,8 +28,7 @@ namespace ams::kern {
void KClientPort::OnSessionFinalized() {
KScopedSchedulerLock sl;
const auto prev = m_num_sessions--;
if (prev == m_max_sessions) {
if (m_num_sessions.FetchSub(1) == m_max_sessions) {
this->NotifyAvailable();
}
}
@ -56,7 +55,7 @@ namespace ams::kern {
bool KClientPort::IsSignaled() const {
MESOSPHERE_ASSERT_THIS();
return m_num_sessions < m_max_sessions;
return m_num_sessions.Load() < m_max_sessions;
}
Result KClientPort::CreateSession(KClientSession **out) {
@ -99,7 +98,6 @@ namespace ams::kern {
/* Check that we successfully created a session. */
R_UNLESS(session != nullptr, svc::ResultOutOfResource());
/* Update the session counts. */
auto count_guard = SCOPE_GUARD { session->Close(); };
{
@ -107,22 +105,22 @@ namespace ams::kern {
s32 new_sessions;
{
const auto max = m_max_sessions;
auto cur_sessions = m_num_sessions.load(std::memory_order_acquire);
auto cur_sessions = m_num_sessions.Load();
do {
R_UNLESS(cur_sessions < max, svc::ResultOutOfSessions());
new_sessions = cur_sessions + 1;
} while (!m_num_sessions.compare_exchange_weak(cur_sessions, new_sessions, std::memory_order_relaxed));
} while (!m_num_sessions.CompareExchangeWeak<std::memory_order_relaxed>(cur_sessions, new_sessions));
}
/* Atomically update the peak session tracking. */
{
auto peak = m_peak_sessions.load(std::memory_order_acquire);
auto peak = m_peak_sessions.Load();
do {
if (peak >= new_sessions) {
break;
}
} while (!m_peak_sessions.compare_exchange_weak(peak, new_sessions, std::memory_order_relaxed));
} while (!m_peak_sessions.CompareExchangeWeak<std::memory_order_relaxed>(peak, new_sessions));
}
}
count_guard.Cancel();
@ -183,22 +181,22 @@ namespace ams::kern {
s32 new_sessions;
{
const auto max = m_max_sessions;
auto cur_sessions = m_num_sessions.load(std::memory_order_acquire);
auto cur_sessions = m_num_sessions.Load();
do {
R_UNLESS(cur_sessions < max, svc::ResultOutOfSessions());
new_sessions = cur_sessions + 1;
} while (!m_num_sessions.compare_exchange_weak(cur_sessions, new_sessions, std::memory_order_relaxed));
} while (!m_num_sessions.CompareExchangeWeak<std::memory_order_relaxed>(cur_sessions, new_sessions));
}
/* Atomically update the peak session tracking. */
{
auto peak = m_peak_sessions.load(std::memory_order_acquire);
auto peak = m_peak_sessions.Load();
do {
if (peak >= new_sessions) {
break;
}
} while (!m_peak_sessions.compare_exchange_weak(peak, new_sessions, std::memory_order_relaxed));
} while (!m_peak_sessions.CompareExchangeWeak<std::memory_order_relaxed>(peak, new_sessions));
}
}
count_guard.Cancel();

25
libraries/libmesosphere/source/kern_k_debug_base.cpp
View file

@ -25,31 +25,6 @@ namespace ams::kern {
}
void KDebugBase::ProcessHolder::Attach(KProcess *process) {
MESOSPHERE_ASSERT(m_process == nullptr);
/* Set our process. */
m_process = process;
/* Open reference to our process. */
m_process->Open();
/* Set our reference count. */
m_ref_count = 1;
}
void KDebugBase::ProcessHolder::Detach() {
/* Close our process, if we have one. */
KProcess * const process = m_process;
if (AMS_LIKELY(process != nullptr)) {
/* Set our process to a debug sentinel value, which will cause crash if accessed. */
m_process = reinterpret_cast<KProcess *>(1);
/* Close reference to our process. */
process->Close();
}
}
void KDebugBase::Initialize() {
/* Clear the continue flags. */
m_continue_flags = 0;

47
libraries/libmesosphere/source/kern_k_debug_base_process_holder.cpp
View file

@ -1,47 +0,0 @@
/*
* Copyright (c) Atmosphère-NX
*
* This program is free software; you can redistribute it and/or modify it
* under the terms and conditions of the GNU General Public License,
* version 2, as published by the Free Software Foundation.
*
* This program is distributed in the hope it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include <mesosphere.hpp>
namespace ams::kern {
NOINLINE bool KDebugBase::ProcessHolder::Open() {
/* Atomically increment the reference count, only if it's positive. */
u32 cur_ref_count = m_ref_count.load(std::memory_order_relaxed);
do {
if (AMS_UNLIKELY(cur_ref_count == 0)) {
MESOSPHERE_AUDIT(cur_ref_count != 0);
return false;
}
MESOSPHERE_ABORT_UNLESS(cur_ref_count < cur_ref_count + 1);
} while (!m_ref_count.compare_exchange_weak(cur_ref_count, cur_ref_count + 1, std::memory_order_relaxed));
return true;
}
NOINLINE void KDebugBase::ProcessHolder::Close() {
/* Atomically decrement the reference count, not allowing it to become negative. */
u32 cur_ref_count = m_ref_count.load(std::memory_order_relaxed);
do {
MESOSPHERE_ABORT_UNLESS(cur_ref_count > 0);
} while (!m_ref_count.compare_exchange_weak(cur_ref_count, cur_ref_count - 1, std::memory_order_relaxed));
/* If ref count hits zero, schedule the object for destruction. */
if (cur_ref_count - 1 == 0) {
this->Detach();
}
}
}

11
libraries/libmesosphere/source/kern_k_light_lock.cpp
View file

@ -40,7 +40,7 @@ namespace ams::kern {
KScopedSchedulerLock sl;
/* Ensure we actually have locking to do. */
if (m_tag.load(std::memory_order_relaxed) != _owner) {
if (m_tag.Load<std::memory_order_relaxed>() != _owner) {
return false;
}
@ -68,16 +68,13 @@ namespace ams::kern {
KScopedSchedulerLock sl;
/* Get the next owner. */
s32 num_waiters = 0;
s32 num_waiters;
KThread *next_owner = owner_thread->RemoveWaiterByKey(std::addressof(num_waiters), reinterpret_cast<uintptr_t>(std::addressof(m_tag)));
/* Pass the lock to the next owner. */
uintptr_t next_tag = 0;
if (next_owner != nullptr) {
next_tag = reinterpret_cast<uintptr_t>(next_owner);
if (num_waiters > 1) {
next_tag |= 0x1;
}
next_tag = reinterpret_cast<uintptr_t>(next_owner) | static_cast<uintptr_t>(num_waiters > 1);
next_owner->EndWait(ResultSuccess());
@ -92,7 +89,7 @@ namespace ams::kern {
}
/* Write the new tag value. */
m_tag.store(next_tag);
m_tag.Store<std::memory_order_release>(next_tag);
}
}

16
libraries/libmesosphere/source/kern_k_process.cpp
View file

@ -25,8 +25,8 @@ namespace ams::kern {
constexpr u64 ProcessIdMin = InitialProcessIdMax + 1;
constexpr u64 ProcessIdMax = std::numeric_limits<u64>::max();
std::atomic<u64> g_initial_process_id = InitialProcessIdMin;
std::atomic<u64> g_process_id = ProcessIdMin;
constinit util::Atomic<u64> g_initial_process_id{InitialProcessIdMin};
constinit util::Atomic<u64> g_process_id{ProcessIdMin};
Result TerminateChildren(KProcess *process, const KThread *thread_to_not_terminate) {
/* Request that all children threads terminate. */
@ -299,7 +299,7 @@ namespace ams::kern {
R_TRY(m_capabilities.Initialize(caps, num_caps, std::addressof(m_page_table)));
/* Initialize the process id. */
m_process_id = g_initial_process_id++;
m_process_id = g_initial_process_id.FetchAdd(1);
MESOSPHERE_ABORT_UNLESS(InitialProcessIdMin <= m_process_id);
MESOSPHERE_ABORT_UNLESS(m_process_id <= InitialProcessIdMax);
@ -409,7 +409,7 @@ namespace ams::kern {
R_TRY(m_capabilities.Initialize(user_caps, num_caps, std::addressof(m_page_table)));
/* Initialize the process id. */
m_process_id = g_process_id++;
m_process_id = g_process_id.FetchAdd(1);
MESOSPHERE_ABORT_UNLESS(ProcessIdMin <= m_process_id);
MESOSPHERE_ABORT_UNLESS(m_process_id <= ProcessIdMax);
@ -789,15 +789,15 @@ namespace ams::kern {
}
void KProcess::IncrementRunningThreadCount() {
MESOSPHERE_ASSERT(m_num_running_threads.load() >= 0);
MESOSPHERE_ASSERT(m_num_running_threads.Load() >= 0);
m_num_running_threads.fetch_add(1);
m_num_running_threads.FetchAdd(1);
}
void KProcess::DecrementRunningThreadCount() {
MESOSPHERE_ASSERT(m_num_running_threads.load() > 0);
MESOSPHERE_ASSERT(m_num_running_threads.Load() > 0);
if (m_num_running_threads.fetch_sub(1) == 1) {
if (m_num_running_threads.FetchSub(1) == 1) {
this->Terminate();
}
}

10
libraries/libmesosphere/source/kern_k_scheduler.cpp
View file

@ -246,9 +246,9 @@ namespace ams::kern {
if (cur_process != nullptr) {
/* NOTE: Combining this into AMS_LIKELY(!... && ...) triggers an internal compiler error: Segmentation fault in GCC 9.2.0. */
if (AMS_LIKELY(!cur_thread->IsTerminationRequested()) && AMS_LIKELY(cur_thread->GetActiveCore() == m_core_id)) {
m_state.prev_thread = cur_thread;
m_state.prev_thread.Store<std::memory_order_relaxed>(cur_thread);
} else {
m_state.prev_thread = nullptr;
m_state.prev_thread.Store<std::memory_order_relaxed>(nullptr);
}
}
@ -270,13 +270,9 @@ namespace ams::kern {
void KScheduler::ClearPreviousThread(KThread *thread) {
MESOSPHERE_ASSERT(IsSchedulerLockedByCurrentThread());
for (size_t i = 0; i < cpu::NumCores; ++i) {
/* Get an atomic reference to the core scheduler's previous thread. */
std::atomic_ref<KThread *> prev_thread(Kernel::GetScheduler(static_cast<s32>(i)).m_state.prev_thread);
static_assert(std::atomic_ref<KThread *>::is_always_lock_free);
/* Atomically clear the previous thread if it's our target. */
KThread *compare = thread;
prev_thread.compare_exchange_strong(compare, nullptr);
Kernel::GetScheduler(static_cast<s32>(i)).m_state.prev_thread.CompareExchangeStrong(compare, nullptr);
}
}

6
libraries/libmesosphere/source/kern_k_thread.cpp
View file

@ -219,7 +219,7 @@ namespace ams::kern {
this->SetInExceptionHandler();
/* Set thread ID. */
m_thread_id = s_next_thread_id++;
m_thread_id = s_next_thread_id.FetchAdd(1);
/* We initialized! */
m_initialized = true;
@ -707,7 +707,7 @@ namespace ams::kern {
KScopedSchedulerLock sl;
/* Determine the priority value to use. */
const s32 target_priority = m_termination_requested.load() && priority >= TerminatingThreadPriority ? TerminatingThreadPriority : priority;
const s32 target_priority = m_termination_requested.Load() && priority >= TerminatingThreadPriority ? TerminatingThreadPriority : priority;
/* Change our base priority. */
if (this->GetStackParameters().is_pinned) {
@ -1183,7 +1183,7 @@ namespace ams::kern {
const bool first_request = [&] ALWAYS_INLINE_LAMBDA () -> bool {
/* Perform an atomic compare-and-swap from false to true. */
u8 expected = false;
return m_termination_requested.compare_exchange_strong(expected, true);
return m_termination_requested.CompareExchangeStrong(expected, true);
}();
/* If this is the first request, start termination procedure. */

21
libraries/libmesosphere/source/kern_panic.cpp
View file

@ -42,15 +42,15 @@ namespace ams::kern {
return arr;
}();
std::atomic<s32> g_next_ticket = 0;
std::atomic<s32> g_current_ticket = 0;
constinit util::Atomic<s32> g_next_ticket{0};
constinit util::Atomic<s32> g_current_ticket{0};
std::array<s32, cpu::NumCores> g_core_tickets = NegativeArray;
constinit std::array<s32, cpu::NumCores> g_core_tickets = NegativeArray;
s32 GetCoreTicket() {
const s32 core_id = GetCurrentCoreId();
if (g_core_tickets[core_id] == -1) {
g_core_tickets[core_id] = 2 * g_next_ticket.fetch_add(1);
g_core_tickets[core_id] = 2 * g_next_ticket.FetchAdd(1);
}
return g_core_tickets[core_id];
}
@ -58,24 +58,21 @@ namespace ams::kern {
void WaitCoreTicket() {
const s32 expected = GetCoreTicket();
const s32 desired = expected + 1;
s32 compare = g_current_ticket;
s32 compare = g_current_ticket.Load<std::memory_order_relaxed>();
do {
if (compare == desired) {
break;
}
compare = expected;
} while (!g_current_ticket.compare_exchange_weak(compare, desired));
} while (!g_current_ticket.CompareExchangeWeak(compare, desired));
}
void ReleaseCoreTicket() {
const s32 expected = GetCoreTicket() + 1;
const s32 desired = expected + 1;
s32 compare = g_current_ticket;
do {
if (compare != expected) {
break;
}
} while (!g_current_ticket.compare_exchange_weak(compare, desired));
s32 compare = expected;
g_current_ticket.CompareExchangeStrong(compare, desired);
}
ALWAYS_INLINE KExceptionContext *GetPanicExceptionContext(int core_id) {