Introduce adapting condition variable class

By spin waiting for a small period before falling back to an actual condition variable, some of the overheads inherent to futex's can be avoided. The used constants were tuned for optimal performance on 8G1 on Skyrim and PGLE.
This commit is contained in:
Billy Laws 2023-02-27 22:05:24 +00:00
parent 444e35e34f
commit b1e57bc7bc
2 changed files with 154 additions and 6 deletions

View File

@ -4,15 +4,19 @@
#include <chrono> #include <chrono>
#include <thread> #include <thread>
#include "spin_lock.h" #include "spin_lock.h"
#include "utils.h"
namespace skyline { namespace skyline {
static constexpr size_t LockAttemptsPerYield{256}; static constexpr size_t LockAttemptsPerYield{32};
static constexpr size_t LockAttemptsPerSleep{1024}; static constexpr size_t LockAttemptsPerSleep{1024};
static constexpr size_t SleepDurationUs{100}; static constexpr size_t SleepDurationUs{50};
template<typename Func> template<typename Func>
void FalloffLock(Func &&func) { void FalloffLock(Func &&func) {
for (size_t i{}; !func(); i++) { for (size_t i{1}; !func(i); i++) {
asm volatile("DMB ISHST;"
"YIELD;");
if (i % LockAttemptsPerYield == 0) if (i % LockAttemptsPerYield == 0)
std::this_thread::yield(); std::this_thread::yield();
if (i % LockAttemptsPerSleep == 0) if (i % LockAttemptsPerSleep == 0)
@ -21,20 +25,34 @@ namespace skyline {
} }
void __attribute__ ((noinline)) SpinLock::LockSlow() { void __attribute__ ((noinline)) SpinLock::LockSlow() {
FalloffLock([this] { FalloffLock([this] (size_t i) {
return try_lock(); return try_lock();
}); });
} }
void __attribute__ ((noinline)) SharedSpinLock::LockSlow() { void __attribute__ ((noinline)) SharedSpinLock::LockSlow() {
FalloffLock([this] { FalloffLock([this] (size_t i) {
return try_lock(); return try_lock();
}); });
} }
void __attribute__ ((noinline)) SharedSpinLock::LockSlowShared() { void __attribute__ ((noinline)) SharedSpinLock::LockSlowShared() {
FalloffLock([this] { FalloffLock([this] (size_t i) {
return try_lock_shared(); return try_lock_shared();
}); });
} }
static constexpr size_t AdaptiveWaitIters{1024}; //!< Number of wait iterations before waiting should fallback to a regular condition variable
void __attribute__ ((noinline)) AdaptiveSingleWaiterConditionVariable::SpinWait() {
FalloffLock([this] (size_t i) {
return !unsignalled.test_and_set() || i >= AdaptiveWaitIters;
});
}
void __attribute__ ((noinline)) AdaptiveSingleWaiterConditionVariable::SpinWait(i64 maxEndTimeNs) {
FalloffLock([maxEndTimeNs, this] (size_t i) {
return util::GetTimeNs() > maxEndTimeNs || !unsignalled.test_and_set() || i >= AdaptiveWaitIters;
});
}
} }

View File

@ -4,8 +4,11 @@
#pragma once #pragma once
#include <atomic> #include <atomic>
#include <condition_variable>
#include <thread> #include <thread>
#include <mutex>
#include "base.h" #include "base.h"
#include "utils.h"
namespace skyline { namespace skyline {
/** /**
@ -130,4 +133,131 @@ namespace skyline {
} }
} }
}; };
/**
* @brief A condition variable that spins for a bit before falling back to a regular condition variable, for cases where only a single thread can wait at the same time
*/
class AdaptiveSingleWaiterConditionVariable {
private:
/**
* @brief Spins either until the condition variable is signalled or the spin wait times out (to fall back to a regular condition variable)
*/
void SpinWait();
/**
* @brief Spins either until the condition variable is signalled or the spin wait times out (to fall back to a regular condition variable, or until the given time is reached)
* @param maxEndTimeNs The maximum time to spin for
*/
void SpinWait(i64 maxEndTimeNs);
std::condition_variable fallback; //<! Fallback condition variable for when the spin wait times out
std::mutex fallbackMutex; //!< Used to allow taking in arbitrary mutex types and to synchronise access to fallbackWaiter
std::atomic_flag unsignalled{true}; //!< Set to false when the condition variable is signalled
bool fallbackWaiter{}; //!< True if the waiter is waiting on the fallback condition variable
public:
/**
* @brief Signals the condition variable
*/
void notify() {
unsignalled.clear();
std::scoped_lock fallbackLock{fallbackMutex};
if (fallbackWaiter)
fallback.notify_one();
}
/**
* @brief Waits for the condition variable to be signalled
* @param lock The lock to unlock while waiting
* @param pred The predicate to check, if it returns true then the wait will end
*/
void wait(auto &lock, auto pred) {
// 'notify' calls should only wake the condition variable when called during waiting
unsignalled.test_and_set();
if (!pred()) {
// First spin wait for a bit, to hopefully avoid the costs of condition variables under heavy thrashing
lock.unlock();
SpinWait();
lock.lock();
} else {
return;
}
// The spin wait has either timed out or succeeded, check the predicate to confirm which is the case
if (pred())
return;
// If the spin wait timed out then fallback to a regular condition variable
while (!pred()) {
std::unique_lock fallbackLock{fallbackMutex};
// Store that we are currently waiting on the fallback condition variable, `notify()` can check this when `fallbackLock` ends up being unlocked by `fallback.wait()` to avoid redundantly performing futex wakes (on older bionic versions)
fallbackWaiter = true;
lock.unlock();
fallback.wait(fallbackLock);
// The predicate has been satisfied, we're done here
fallbackWaiter = false;
fallbackMutex.unlock();
lock.lock();
}
}
/**
* @brief Waits for the condition variable to be signalled or the given duration to elapse
* @param lock The lock to unlock while waiting
* @param duration The duration to wait for
* @param pred The predicate to check, if it returns true then the wait will end
* @return True if the predicate returned true, false if the duration elapsed
*/
bool wait_for(auto &lock, const auto &duration, auto pred) {
// 'notify' calls should only wake the condition variable when called during waiting
unsignalled.test_and_set();
auto endTimeNs{util::GetTimeNs() + std::chrono::nanoseconds(duration).count()};
if (!pred()) {
// First spin wait for a bit, to hopefully avoid the costs of condition variables under heavy thrashing
lock.unlock();
SpinWait(endTimeNs);
lock.lock();
} else {
return true;
}
// The spin wait has either timed out (due to wanting to fallback), timed out (due to the duration being exceeded) or succeeded, check the predicate and current time to confirm which is the case
if (pred())
return true;
else if (util::GetTimeNs() > endTimeNs)
return false;
// If the spin wait timed out (due to wanting to fallback), then fallback to a regular condition variable
// Calculate chrono-based end time only in the fallback path to avoid polluting the fast past
auto endTime{std::chrono::system_clock::now() + std::chrono::nanoseconds(endTimeNs - util::GetTimeNs())};
std::cv_status status{std::cv_status::no_timeout};
while (status == std::cv_status::no_timeout && !pred()) {
std::unique_lock fallbackLock{fallbackMutex};
// Store that we are currently waiting on the fallback condition variable, `notify()` can check this when `fallbackLock` ends up being unlocked by `fallback.wait()` to avoid redundantly performing futex wakes (on older bionic versions)
fallbackWaiter = true;
lock.unlock();
status = fallback.wait_until(fallbackLock, endTime);
fallbackWaiter = false;
fallbackLock.unlock();
lock.lock();
}
// The predicate has been satisfied, we're done here
return pred();
}
};
} }