Introduce adapting condition variable class

By spin waiting for a small period before falling back to an actual condition variable, some of the overheads inherent to futex's can be avoided. The used constants were tuned for optimal performance on 8G1 on Skyrim and PGLE.

app/src/main/cpp/skyline/common/spin_lock.cpp

@@ -4,15 +4,19 @@
 #include <chrono>
 #include <thread>
 #include "spin_lock.h"
+#include "utils.h"
 
 namespace skyline {
-    static constexpr size_t LockAttemptsPerYield{256};
+    static constexpr size_t LockAttemptsPerYield{32};
     static constexpr size_t LockAttemptsPerSleep{1024};
-    static constexpr size_t SleepDurationUs{100};
+    static constexpr size_t SleepDurationUs{50};
 
     template<typename Func>
     void FalloffLock(Func &&func) {
-        for (size_t i{}; !func(); i++) {
+        for (size_t i{1}; !func(i); i++) {
+            asm volatile("DMB ISHST;"
+                         "YIELD;");
+
             if (i % LockAttemptsPerYield == 0)
                 std::this_thread::yield();
             if (i % LockAttemptsPerSleep == 0)
@@ -21,20 +25,34 @@ namespace skyline {
     }
 
     void  __attribute__ ((noinline)) SpinLock::LockSlow() {
-        FalloffLock([this] {
+        FalloffLock([this] (size_t i) {
             return try_lock();
         });
     }
 
     void  __attribute__ ((noinline)) SharedSpinLock::LockSlow() {
-        FalloffLock([this] {
+        FalloffLock([this] (size_t i) {
             return try_lock();
         });
     }
 
     void  __attribute__ ((noinline)) SharedSpinLock::LockSlowShared() {
-        FalloffLock([this] {
+        FalloffLock([this] (size_t i) {
             return try_lock_shared();
         });
     }
+
+    static constexpr size_t AdaptiveWaitIters{1024}; //!< Number of wait iterations before waiting should fallback to a regular condition variable
+
+    void __attribute__ ((noinline)) AdaptiveSingleWaiterConditionVariable::SpinWait() {
+        FalloffLock([this] (size_t i) {
+            return !unsignalled.test_and_set() || i >= AdaptiveWaitIters;
+        });
+    }
+
+    void __attribute__ ((noinline)) AdaptiveSingleWaiterConditionVariable::SpinWait(i64 maxEndTimeNs) {
+        FalloffLock([maxEndTimeNs, this] (size_t i) {
+            return util::GetTimeNs() > maxEndTimeNs  || !unsignalled.test_and_set() || i >= AdaptiveWaitIters;
+        });
+    }
 }

app/src/main/cpp/skyline/common/spin_lock.h

@@ -4,8 +4,11 @@
 #pragma once
 
 #include <atomic>
+#include <condition_variable>
 #include <thread>
+#include <mutex>
 #include "base.h"
+#include "utils.h"
 
 namespace skyline {
     /**
@@ -130,4 +133,131 @@ namespace skyline {
             }
         }
     };
+
+    /**
+     * @brief A condition variable that spins for a bit before falling back to a regular condition variable, for cases where only a single thread can wait at the same time
+     */
+    class AdaptiveSingleWaiterConditionVariable {
+      private:
+        /**
+         * @brief Spins either until the condition variable is signalled or the spin wait times out (to fall back to a regular condition variable)
+         */
+        void SpinWait();
+
+        /**
+         * @brief Spins either until the condition variable is signalled or the spin wait times out (to fall back to a regular condition variable, or until the given time is reached)
+         * @param maxEndTimeNs The maximum time to spin for
+         */
+        void SpinWait(i64 maxEndTimeNs);
+
+        std::condition_variable fallback; //<! Fallback condition variable for when the spin wait times out
+        std::mutex fallbackMutex; //!< Used to allow taking in arbitrary mutex types and to synchronise access to fallbackWaiter
+        std::atomic_flag unsignalled{true}; //!< Set to false when the condition variable is signalled
+        bool fallbackWaiter{}; //!< True if the waiter is waiting on the fallback condition variable
+
+      public:
+        /**
+         * @brief Signals the condition variable
+         */
+        void notify() {
+            unsignalled.clear();
+
+            std::scoped_lock fallbackLock{fallbackMutex};
+            if (fallbackWaiter)
+                fallback.notify_one();
+        }
+
+        /**
+         * @brief Waits for the condition variable to be signalled
+         * @param lock The lock to unlock while waiting
+         * @param pred The predicate to check, if it returns true then the wait will end
+         */
+        void wait(auto &lock, auto pred) {
+            // 'notify' calls should only wake the condition variable when called during waiting
+            unsignalled.test_and_set();
+
+            if (!pred()) {
+                // First spin wait for a bit, to hopefully avoid the costs of condition variables under heavy thrashing
+                lock.unlock();
+                SpinWait();
+                lock.lock();
+            } else {
+                return;
+            }
+
+            // The spin wait has either timed out or succeeded, check the predicate to confirm which is the case
+            if (pred())
+                return;
+
+            // If the spin wait timed out then fallback to a regular condition variable
+            while (!pred()) {
+                std::unique_lock fallbackLock{fallbackMutex};
+
+                // Store that we are currently waiting on the fallback condition variable, `notify()` can check this when `fallbackLock` ends up being unlocked by `fallback.wait()` to avoid redundantly performing futex wakes (on older bionic versions)
+                fallbackWaiter = true;
+
+                lock.unlock();
+                fallback.wait(fallbackLock);
+
+                // The predicate has been satisfied, we're done here
+                fallbackWaiter = false;
+                fallbackMutex.unlock();
+
+                lock.lock();
+            }
+        }
+
+        /**
+         * @brief Waits for the condition variable to be signalled or the given duration to elapse
+         * @param lock The lock to unlock while waiting
+         * @param duration The duration to wait for
+         * @param pred The predicate to check, if it returns true then the wait will end
+         * @return True if the predicate returned true, false if the duration elapsed
+         */
+        bool wait_for(auto &lock, const auto &duration, auto pred) {
+            // 'notify' calls should only wake the condition variable when called during waiting
+            unsignalled.test_and_set();
+
+            auto endTimeNs{util::GetTimeNs() + std::chrono::nanoseconds(duration).count()};
+
+            if (!pred()) {
+                // First spin wait for a bit, to hopefully avoid the costs of condition variables under heavy thrashing
+                lock.unlock();
+                SpinWait(endTimeNs);
+                lock.lock();
+            } else {
+                return true;
+            }
+
+            // The spin wait has either timed out (due to wanting to fallback), timed out (due to the duration being exceeded) or succeeded, check the predicate and current time to confirm which is the case
+            if (pred())
+                return true;
+            else if (util::GetTimeNs() > endTimeNs)
+                return false;
+
+
+            // If the spin wait timed out (due to wanting to fallback), then fallback to a regular condition variable
+
+            // Calculate chrono-based end time only in the fallback path to avoid polluting the fast past
+            auto endTime{std::chrono::system_clock::now() + std::chrono::nanoseconds(endTimeNs - util::GetTimeNs())};
+            std::cv_status status{std::cv_status::no_timeout};
+            while (status == std::cv_status::no_timeout && !pred()) {
+                std::unique_lock fallbackLock{fallbackMutex};
+
+                // Store that we are currently waiting on the fallback condition variable, `notify()` can check this when `fallbackLock` ends up being unlocked by `fallback.wait()` to avoid redundantly performing futex wakes (on older bionic versions)
+                fallbackWaiter = true;
+
+                lock.unlock();
+                status = fallback.wait_until(fallbackLock, endTime);
+
+                fallbackWaiter = false;
+                fallbackLock.unlock();
+
+                lock.lock();
+            }
+
+            // The predicate has been satisfied, we're done here
+            return pred();
+        }
+    };
 }