Implement Conditional Variables

app/src/main/cpp/skyline/kernel/scheduler.cpp

@@ -22,8 +22,7 @@ namespace skyline::kernel {
         }
     }
 
-    Scheduler::CoreContext &Scheduler::LoadBalance() {
-        auto &thread{state.thread};
+    Scheduler::CoreContext &Scheduler::LoadBalance(const std::shared_ptr<type::KThread> &thread) {
         std::lock_guard migrationLock(thread->coreMigrationMutex);
         auto *currentCore{&cores.at(thread->coreId)};
 
@@ -66,21 +65,21 @@ namespace skyline::kernel {
 
                 thread->coreId = optimalCore->id;
 
-                state.logger->Debug("Load Balancing: C{} -> C{}", currentCore->id, optimalCore->id);
+                state.logger->Debug("Load Balancing T{}: C{} -> C{}", thread->id, currentCore->id, optimalCore->id);
             } else {
-                state.logger->Debug("Load Balancing: C{} (Late)", currentCore->id);
+                state.logger->Debug("Load Balancing T{}: C{} (Late)", thread->id, currentCore->id);
             }
 
             return *optimalCore;
         }
 
-        state.logger->Debug("Load Balancing: C{} (Early)", currentCore->id);
+        state.logger->Debug("Load Balancing T{}: C{} (Early)", thread->id, currentCore->id);
 
         return *currentCore;
     }
 
-    void Scheduler::InsertThread(const std::shared_ptr<type::KThread> &thread, bool loadBalance) {
-        auto &core{loadBalance ? LoadBalance() : cores.at(thread->coreId)};
+    void Scheduler::InsertThread(const std::shared_ptr<type::KThread> &thread) {
+        auto &core{cores.at(thread->coreId)};
 
         thread_local bool signalHandlerSetup{};
         if (!signalHandlerSetup) {
@@ -125,20 +124,20 @@ namespace skyline::kernel {
         thread->needsReorder = true; // We need to reorder the thread from back to align it with other threads of it's priority and ensure strict ordering amongst priorities
     }
 
-    void Scheduler::WaitSchedule() {
+    void Scheduler::WaitSchedule(bool loadBalance) {
         auto &thread{state.thread};
         auto *core{&cores.at(thread->coreId)};
 
         std::shared_lock lock(core->mutex);
-        if (thread->affinityMask.count() > 1) {
+        if (loadBalance && thread->affinityMask.count() > 1) {
             std::chrono::milliseconds loadBalanceThreshold{PreemptiveTimeslice * 2}; //!< The amount of time that needs to pass unscheduled for a thread to attempt load balancing
             while (!core->frontCondition.wait_for(lock, loadBalanceThreshold, [&]() { return !core->queue.empty() && core->queue.front() == thread; })) {
                 lock.unlock();
-                LoadBalance();
+                LoadBalance(state.thread);
                 if (thread->coreId == core->id) {
                     lock.lock();
                 } else {
-                    InsertThread(state.thread, false);
+                    InsertThread(state.thread);
                     core = &cores.at(thread->coreId);
                     lock = std::shared_lock(core->mutex);
                 }
@@ -158,6 +157,26 @@ namespace skyline::kernel {
         thread->timesliceStart = util::GetTimeTicks();
     }
 
+    bool Scheduler::TimedWaitSchedule(std::chrono::nanoseconds timeout) {
+        auto &thread{state.thread};
+        auto *core{&cores.at(thread->coreId)};
+
+        std::shared_lock lock(core->mutex);
+        if (core->frontCondition.wait_for(lock, timeout, [&]() { return !core->queue.empty() && core->queue.front() == thread; })) {
+            if (thread->priority == core->preemptionPriority) {
+                struct itimerspec spec{.it_value = {.tv_nsec = std::chrono::duration_cast<std::chrono::nanoseconds>(PreemptiveTimeslice).count()}};
+                timer_settime(*thread->preemptionTimer, 0, &spec, nullptr);
+                thread->isPreempted = true;
+            }
+
+            thread->timesliceStart = util::GetTimeTicks();
+
+            return true;
+        } else {
+            return false;
+        }
+    }
+
     void Scheduler::Rotate(bool cooperative) {
         auto &thread{state.thread};
         auto &core{cores.at(thread->coreId)};

app/src/main/cpp/skyline/kernel/scheduler.h

@@ -67,24 +67,30 @@ namespace skyline {
             static void SignalHandler(int signal, siginfo *info, ucontext *ctx, void **tls);
 
             /**
-             * @brief Checks all cores and migrates the calling thread to the core where the calling thread should be scheduled the earliest
+             * @brief Checks all cores and migrates the specified thread to the core where the calling thread should be scheduled the earliest
              * @return A reference to the CoreContext of the core which the calling thread is running on after load balancing
              * @note This doesn't insert the thread into the migrated process's queue after load balancing
              */
-            CoreContext& LoadBalance();
+            CoreContext& LoadBalance(const std::shared_ptr<type::KThread> &thread);
 
             /**
              * @brief Inserts the specified thread into the scheduler queue at the appropriate location based on it's priority
-             * @param loadBalance If to load balance or use the thread's current core (KThread::coreId)
-             * @note It isn't supported to load balance if the supplied thread isn't the calling thread, it'll lead to UB
              */
-            void InsertThread(const std::shared_ptr<type::KThread>& thread, bool loadBalance = true);
+            void InsertThread(const std::shared_ptr<type::KThread>& thread);
 
             /**
              * @brief Wait for the current thread to be scheduled on it's resident core
+             * @param loadBalance If the thread is appropriate for load balancing then if to load balance it occassionally or not
              * @note There is an assumption of the thread being on it's resident core queue, if it's not this'll never return
              */
-            void WaitSchedule();
+            void WaitSchedule(bool loadBalance = true);
+
+            /**
+             * @brief Wait for the current thread to be scheduled on it's resident core or for the timeout to expire
+             * @return If the thread has been scheduled (true) or if the timer expired before it could be (false)
+             * @note This will never load balance as it uses the timeout itself as a result this shouldn't be used as a replacement for regular waits
+             */
+            bool TimedWaitSchedule(std::chrono::nanoseconds timeout);
 
             /**
              * @brief Rotates the calling thread's resident core queue, if it is at the front of it

app/src/main/cpp/skyline/kernel/svc.cpp

@@ -398,7 +398,7 @@ namespace skyline::kernel::svc {
 
                 state.scheduler->RemoveThread();
                 thread->coreId = idealCore;
-                state.scheduler->InsertThread(state.thread, false);
+                state.scheduler->InsertThread(state.thread);
                 state.scheduler->WaitSchedule();
             }
 
@@ -520,6 +520,10 @@ namespace skyline::kernel::svc {
 
             state.logger->Debug("svcResetSignal: Resetting signal: 0x{:X}", handle);
             state.ctx->gpr.w0 = Result{};
+
+            // There is an implicit yield while resetting a signal
+            state.scheduler->Rotate();
+            state.scheduler->WaitSchedule();
         } catch (const std::out_of_range &) {
             state.logger->Warn("svcResetSignal: 'handle' invalid: 0x{:X}", handle);
             state.ctx->gpr.w0 = result::InvalidHandle;
@@ -550,6 +554,7 @@ namespace skyline::kernel::svc {
                 case type::KType::KThread:
                 case type::KType::KEvent:
                 case type::KType::KSession:
+                    objectTable.push_back(std::static_pointer_cast<type::KSyncObject>(object));
                     break;
 
                 default: {
@@ -557,38 +562,41 @@ namespace skyline::kernel::svc {
                     return;
                 }
             }
-
-            objectTable.push_back(std::static_pointer_cast<type::KSyncObject>(object));
         }
 
         u64 timeout{state.ctx->gpr.x3};
         state.logger->Debug("svcWaitSynchronization: Waiting on handles:\n{}Timeout: 0x{:X} ns", handleStr, timeout);
 
+        // The thread shouldn't be occupying the core while it's waiting on objects
+        state.scheduler->Rotate();
         auto start{util::GetTimeNs()};
-        while (true) {
-            if (state.thread->cancelSync) {
-                state.thread->cancelSync = false;
-                state.ctx->gpr.w0 = result::Cancelled;
-                break;
-            }
-
-            u32 index{};
-            for (const auto &object : objectTable) {
-                if (object->signalled) {
-                    state.logger->Debug("svcWaitSynchronization: Signalled handle: 0x{:X}", waitHandles[index]);
-                    state.ctx->gpr.w0 = Result{};
-                    state.ctx->gpr.w1 = index;
+        [&] () {
+            while (true) {
+                if (state.thread->cancelSync) {
+                    state.thread->cancelSync = false;
+                    state.ctx->gpr.w0 = result::Cancelled;
                     return;
                 }
-                index++;
-            }
 
-            if ((util::GetTimeNs() - start) >= timeout) {
-                state.logger->Debug("svcWaitSynchronization: Wait has timed out");
-                state.ctx->gpr.w0 = result::TimedOut;
-                return;
+                u32 index{};
+                for (const auto &object : objectTable) {
+                    if (object->signalled) {
+                        state.logger->Debug("svcWaitSynchronization: Signalled handle: 0x{:X}", waitHandles[index]);
+                        state.ctx->gpr.w0 = Result{};
+                        state.ctx->gpr.w1 = index;
+                        return;
+                    }
+                    index++;
+                }
+
+                if ((util::GetTimeNs() - start) >= timeout) {
+                    state.logger->Debug("svcWaitSynchronization: Wait has timed out");
+                    state.ctx->gpr.w0 = result::TimedOut;
+                    return;
+                }
             }
-        }
+        }();
+        state.scheduler->WaitSchedule();
     }
 
     void CancelSynchronization(const DeviceState &state) {
@@ -608,18 +616,15 @@ namespace skyline::kernel::svc {
             return;
         }
 
-        KHandle ownerHandle{state.ctx->gpr.w0};
-        KHandle requesterHandle{state.ctx->gpr.w2};
-        if (requesterHandle != state.thread->handle)
-            throw exception("svcWaitProcessWideKeyAtomic: Handle doesn't match current thread: 0x{:X} for thread 0x{:X}", requesterHandle, state.thread->handle);
-
         state.logger->Debug("svcArbitrateLock: Locking mutex at 0x{:X}", pointer);
 
-        auto result{state.process->MutexLock(pointer, ownerHandle)};
+        KHandle ownerHandle{state.ctx->gpr.w0};
+        KHandle requesterHandle{state.ctx->gpr.w2};
+        auto result{state.process->MutexLock(pointer, ownerHandle, requesterHandle)};
         if (result == Result{})
             state.logger->Debug("svcArbitrateLock: Locked mutex at 0x{:X}", pointer);
         else if (result == result::InvalidHandle)
-            state.logger->Warn("svcArbitrateLock: 'handle' invalid: 0x{:X}", ownerHandle);
+            state.logger->Warn("svcArbitrateLock: 'ownerHandle' invalid: 0x{:X}", ownerHandle);
         else if (result == result::InvalidCurrentMemory)
             state.logger->Debug("svcArbitrateLock: Owner handle did not match current owner for mutex or didn't have waiter flag at 0x{:X}", pointer);
 
@@ -635,10 +640,9 @@ namespace skyline::kernel::svc {
         }
 
         state.logger->Debug("svcArbitrateUnlock: Unlocking mutex at 0x{:X}", mutex);
-
         state.process->MutexUnlock(mutex);
-
         state.logger->Debug("svcArbitrateUnlock: Unlocked mutex at 0x{:X}", mutex);
+
         state.ctx->gpr.w0 = Result{};
     }
 
@@ -650,31 +654,26 @@ namespace skyline::kernel::svc {
             return;
         }
 
-        auto conditional{reinterpret_cast<void *>(state.ctx->gpr.x1)};
-        KHandle handle{state.ctx->gpr.w2};
-        if (handle != state.thread->handle)
-            throw exception("svcWaitProcessWideKeyAtomic: Handle doesn't match current thread: 0x{:X} for thread 0x{:X}", handle, state.thread->handle);
+        auto conditional{reinterpret_cast<u32 *>(state.ctx->gpr.x1)};
+        KHandle requesterHandle{state.ctx->gpr.w2};
 
-        state.process->MutexUnlock(mutex);
-
-        u64 timeout{state.ctx->gpr.x3};
+        i64 timeout{static_cast<i64>(state.ctx->gpr.x3)};
         state.logger->Debug("svcWaitProcessWideKeyAtomic: Mutex: 0x{:X}, Conditional-Variable: 0x{:X}, Timeout: {} ns", mutex, conditional, timeout);
 
-        if (state.process->ConditionalVariableWait(conditional, mutex, timeout)) {
-            state.logger->Debug("svcWaitProcessWideKeyAtomic: Waited for conditional variable and relocked mutex");
-            state.ctx->gpr.w0 = Result{};
-        } else {
+        auto result{state.process->ConditionalVariableWait(conditional, mutex, requesterHandle, timeout)};
+        if (result == Result{})
+            state.logger->Debug("svcWaitProcessWideKeyAtomic: Waited for conditional variable and reacquired mutex");
+        else if (result == result::TimedOut)
             state.logger->Debug("svcWaitProcessWideKeyAtomic: Wait has timed out");
-            state.ctx->gpr.w0 = result::TimedOut;
-        }
+        state.ctx->gpr.w0 = result;
     }
 
     void SignalProcessWideKey(const DeviceState &state) {
-        auto conditional{reinterpret_cast<void *>(state.ctx->gpr.x0)};
+        auto key{reinterpret_cast<u32 *>(state.ctx->gpr.x0)};
         KHandle count{state.ctx->gpr.w1};
 
-        state.logger->Debug("svcSignalProcessWideKey: Signalling Conditional-Variable at 0x{:X} for {}", conditional, count);
-        state.process->ConditionalVariableSignal(conditional, count);
+        state.logger->Debug("svcSignalProcessWideKey: Signalling Conditional-Variable at 0x{:X} for {}", key, count);
+        state.process->ConditionalVariableSignal(key, count);
         state.ctx->gpr.w0 = Result{};
     }
 

app/src/main/cpp/skyline/kernel/types/KProcess.cpp

@@ -109,13 +109,12 @@ namespace skyline::kernel::type {
         return std::nullopt;
     }
 
-    constexpr u32 HandleWaitMask{0x40000000}; //!< A mask of a bit which denotes if the handle has waiters or not
+    constexpr u32 HandleWaitersBit{0b01000000000000000000000000000000}; //!< A bit which denotes if a mutex psuedo-handle has waiters or not
 
-
-    Result KProcess::MutexLock(u32 *mutex, KHandle ownerHandle) {
+    Result KProcess::MutexLock(u32 *mutex, KHandle ownerHandle, KHandle tag) {
         std::shared_ptr<KThread> owner;
         try {
-            owner = GetHandle<KThread>(ownerHandle & ~HandleWaitMask);
+            owner = GetHandle<KThread>(ownerHandle);
         } catch (const std::out_of_range &) {
             return result::InvalidHandle;
         }
@@ -125,10 +124,10 @@ namespace skyline::kernel::type {
             std::lock_guard lock(owner->waiterMutex);
 
             u32 value{};
-            if (__atomic_compare_exchange_n(mutex, &value, (HandleWaitMask & state.thread->handle), false, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST))
+            if (__atomic_compare_exchange_n(mutex, &value, tag, false, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST))
                 // We try to do a CAS to get ownership of the mutex in the case that it's unoccupied
                 return {};
-            if (value != (ownerHandle | HandleWaitMask))
+            if (value != (ownerHandle | HandleWaitersBit))
                 // We ensure that the mutex's value is the handle with the waiter bit set
                 return result::InvalidCurrentMemory;
 
@@ -138,6 +137,7 @@ namespace skyline::kernel::type {
 
             std::atomic_store(&state.thread->waitThread, owner);
             state.thread->waitKey = mutex;
+            state.thread->waitTag = tag;
         }
 
         if (isHighestPriority) {
@@ -178,7 +178,7 @@ namespace skyline::kernel::type {
             } while (owner);
         }
 
-        state.scheduler->WaitSchedule();
+        state.scheduler->WaitSchedule(false);
 
         return {};
     }
@@ -195,12 +195,13 @@ namespace skyline::kernel::type {
 
             // Move all threads waiting on this key to the next owner's waiter list
             std::shared_ptr<KThread> nextWaiter{};
-            for (auto it{waiters.erase(nextOwnerIt)}; it != waiters.end(); it++) {
-                if ((*it)->waitKey == mutex) {
+            for (auto it{waiters.erase(nextOwnerIt)}, nextIt{std::next(it)}; it != waiters.end(); it = nextIt++) {
+                auto thread{*it};
+                if (thread->waitKey == mutex) {
                     nextOwner->waiters.splice(std::upper_bound(nextOwner->waiters.begin(), nextOwner->waiters.end(), (*it)->priority.load(), KThread::IsHigherPriority), waiters, it);
-                    std::atomic_store(&(*it)->waitThread, nextOwner);
+                    std::atomic_store(&thread->waitThread, nextOwner);
                     if (!nextWaiter)
-                        nextWaiter = *it;
+                        nextWaiter = thread;
                 }
             }
 
@@ -223,128 +224,73 @@ namespace skyline::kernel::type {
                     priority = std::min(ownerPriority, nextWaiter->priority.load());
                 } while (ownerPriority != priority && nextOwner->priority.compare_exchange_strong(ownerPriority, priority));
 
-                __atomic_store_n(mutex, nextOwner->handle | HandleWaitMask, __ATOMIC_SEQ_CST);
+                __atomic_store_n(mutex, nextOwner->waitTag | HandleWaitersBit, __ATOMIC_SEQ_CST);
             } else {
-                __atomic_store_n(mutex, nextOwner->handle, __ATOMIC_SEQ_CST);
+                __atomic_store_n(mutex, nextOwner->waitTag, __ATOMIC_SEQ_CST);
             }
 
             // Finally, schedule the next owner accordingly
-            state.scheduler->InsertThread(nextOwner, false);
+            state.scheduler->InsertThread(nextOwner);
         } else {
             __atomic_store_n(mutex, 0, __ATOMIC_SEQ_CST);
-            return;
         }
+
+        state.scheduler->Rotate();
+        state.scheduler->WaitSchedule();
     }
 
-    bool KProcess::ConditionalVariableWait(void *conditional, u32 *mutex, u64 timeout) {
-        /* FIXME
-        std::unique_lock lock(conditionalLock);
+    Result KProcess::ConditionalVariableWait(u32 *key, u32 *mutex, KHandle tag, i64 timeout) {
+        {
+            std::lock_guard lock(syncWaiterMutex);
+            auto queue{syncWaiters.equal_range(key)};
+            auto it{syncWaiters.insert(std::upper_bound(queue.first, queue.second, state.thread->priority.load(), [](const i8 priority, const SyncWaiters::value_type &it) { return it.second->priority > priority; }), {key, state.thread})};
 
-        auto &condWaiters{conditionals[reinterpret_cast<u64>(conditional)]};
-        std::shared_ptr<WaitStatus> status;
-        for (auto it{condWaiters.begin()};; it++) {
-            if (it != condWaiters.end() && (*it)->priority >= state.thread->priority)
-                continue;
+            // TODO: REMOVE THIS AFTER TESTING
+            auto prevIt{std::prev(it)}, nextIt{std::next(it)};
+            if ((prevIt != syncWaiters.begin() && prevIt->first == key && prevIt->second->priority > state.thread->priority.load()))
+                throw exception("Previous node incorrect");
+            if ((nextIt != syncWaiters.end() && nextIt->first == key && nextIt->second->priority < state.thread->priority.load()))
+                throw exception("Next node incorrect");
 
-            status = std::make_shared<WaitStatus>(state.thread->priority, state.thread->handle, mutex);
-            condWaiters.insert(it, status);
-            break;
+            __atomic_store_n(key, true, __ATOMIC_SEQ_CST); // We need to notify any userspace threads that there are waiters on this conditional variable by writing back a boolean flag denoting it
+
+            MutexUnlock(mutex);
+            state.scheduler->RemoveThread();
         }
 
-        lock.unlock();
-
-        bool timedOut{};
-        auto start{util::GetTimeNs()};
-        while (!status->flag)
-            if ((util::GetTimeNs() - start) >= timeout)
-                timedOut = true;
-
-        lock.lock();
-
-        if (!status->flag)
-            timedOut = false;
+        bool hasTimedOut{};
+        if (timeout > 0)
+            hasTimedOut = !state.scheduler->TimedWaitSchedule(std::chrono::nanoseconds(timeout));
         else
-            status->flag = false;
+            state.scheduler->WaitSchedule(false);
 
-        for (auto it{condWaiters.begin()}; it != condWaiters.end(); it++) {
-            if ((*it)->handle == state.thread->handle) {
-                condWaiters.erase(it);
-                break;
-            }
+        if (hasTimedOut) {
+            std::lock_guard lock(syncWaiterMutex);
+            auto queue{syncWaiters.equal_range(key)};
+            syncWaiters.erase(std::find(queue.first, queue.second, SyncWaiters::value_type{key, state.thread}));
+            return result::TimedOut;
         }
 
-        lock.unlock();
-
-        return !timedOut;
-         */
-        return false;
+        while (true) {
+            KHandle value{};
+            if (__atomic_compare_exchange_n(mutex, &value, tag, false, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST))
+                return {};
+            if (!(value & HandleWaitersBit))
+                if (!__atomic_compare_exchange_n(mutex, &value, value | HandleWaitersBit, false, __ATOMIC_SEQ_CST, __ATOMIC_RELAXED))
+                    continue;
+            if (MutexLock(mutex, value & ~HandleWaitersBit, tag) == Result{})
+                return {};
+        }
     }
 
-    void KProcess::ConditionalVariableSignal(void *conditional, u64 amount) {
-        /* FIXME
-        std::unique_lock condLock(conditionalLock);
-
-        auto &condWaiters{conditionals[reinterpret_cast<u64>(conditional)]};
-        u64 count{};
-
-        auto iter{condWaiters.begin()};
-        while (iter != condWaiters.end() && count < amount) {
-            auto &thread{*iter};
-            auto mtx{thread->mutex};
-            u32 mtxValue{__atomic_load_n(mtx, __ATOMIC_SEQ_CST)};
-
-            while (true) {
-                u32 mtxDesired{};
-
-                if (!mtxValue)
-                    mtxDesired = (constant::MtxOwnerMask & thread->handle);
-                else if ((mtxValue & constant::MtxOwnerMask) == state.thread->handle)
-                    mtxDesired = mtxValue | (constant::MtxOwnerMask & thread->handle);
-                else if (mtxValue & ~constant::MtxOwnerMask)
-                    mtxDesired = mtxValue | ~constant::MtxOwnerMask;
-                else
-                    break;
-
-                if (__atomic_compare_exchange_n(mtx, &mtxValue, mtxDesired, false, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST))
-                    break;
-            }
-            if (mtxValue && ((mtxValue & constant::MtxOwnerMask) != state.thread->handle)) {
-                std::unique_lock mtxLock(mutexLock);
-
-                auto &mtxWaiters{mutexes[reinterpret_cast<u64>(thread->mutex)]};
-                std::shared_ptr<WaitStatus> status;
-
-                for (auto it{mtxWaiters.begin()};; it++) {
-                    if (it != mtxWaiters.end() && (*it)->priority >= thread->priority)
-                        continue;
-                    status = std::make_shared<WaitStatus>(thread->priority, thread->handle);
-                    mtxWaiters.insert(it, status);
-                    break;
-                }
-
-                mtxLock.unlock();
-                while (!status->flag);
-                mtxLock.lock();
-                status->flag = false;
-
-                for (auto it{mtxWaiters.begin()}; it != mtxWaiters.end(); it++) {
-                    if ((*it)->handle == thread->handle) {
-                        mtxWaiters.erase(it);
-                        break;
-                    }
-                }
-
-                mtxLock.unlock();
-            }
-
-            thread->flag = true;
-            iter++;
-            count++;
-
-            condLock.unlock();
-            while (thread->flag);
-            condLock.lock();
-        }
-         */
+    void KProcess::ConditionalVariableSignal(u32 *key, u64 amount) {
+        std::unique_lock lock(syncWaiterMutex);
+        auto queue{syncWaiters.equal_range(key)};
+        auto it{queue.first};
+        if (queue.first != queue.second)
+            for (; it != queue.second && amount; it = syncWaiters.erase(it), amount--)
+                state.scheduler->InsertThread(it->second);
+        if (it == queue.second)
+            __atomic_store_n(key, 0, __ATOMIC_SEQ_CST); // We need to update the boolean flag denoting that there are no more threads waiting on this conditional variable
     }
 }

app/src/main/cpp/skyline/kernel/types/KProcess.h

@@ -30,6 +30,10 @@ namespace skyline {
             bool disableThreadCreation{}; //!< If to disable thread creation, we use this to prevent thread creation after all threads have been killed
             std::vector<std::shared_ptr<KThread>> threads;
 
+            using SyncWaiters = std::multimap<void*, std::shared_ptr<KThread>>;
+            std::mutex syncWaiterMutex; //!< Synchronizes all mutations to the map to prevent races
+            SyncWaiters syncWaiters; //!< All threads waiting on process-wide synchronization primitives (Atomic keys + Address Arbiter)
+
             /**
             * @brief The status of a single TLS page (A page is 4096 bytes on ARMv8)
             * Each TLS page has 8 slots, each 0x200 (512) bytes in size
@@ -101,10 +105,10 @@ namespace skyline {
             };
 
             /**
-            * @brief Creates a new handle to a KObject and adds it to the process handle_table
-            * @tparam objectClass The class of the kernel object to create
-            * @param args The arguments for the kernel object except handle, pid and state
-            */
+             * @brief Creates a new handle to a KObject and adds it to the process handle_table
+             * @tparam objectClass The class of the kernel object to create
+             * @param args The arguments for the kernel object except handle, pid and state
+             */
             template<typename objectClass, typename ...objectArgs>
             HandleOut<objectClass> NewHandle(objectArgs... args) {
                 std::unique_lock lock(handleMutex);
@@ -119,9 +123,9 @@ namespace skyline {
             }
 
             /**
-            * @brief Inserts an item into the process handle table
-            * @return The handle of the corresponding item in the handle table
-            */
+             * @brief Inserts an item into the process handle table
+             * @return The handle of the corresponding item in the handle table
+             */
             template<typename objectClass>
             KHandle InsertItem(std::shared_ptr<objectClass> &item) {
                 std::unique_lock lock(handleMutex);
@@ -177,10 +181,10 @@ namespace skyline {
             }
 
             /**
-            * @brief Retrieves a kernel memory object that owns the specified address
-            * @param address The address to look for
-            * @return A shared pointer to the corresponding KMemory object
-            */
+             * @brief Retrieves a kernel memory object that owns the specified address
+             * @param address The address to look for
+             * @return A shared pointer to the corresponding KMemory object
+             */
             std::optional<HandleOut<KMemory>> GetMemoryObject(u8 *ptr);
 
             /**
@@ -191,27 +195,26 @@ namespace skyline {
             }
 
             /**
-            * @brief Locks the mutex at the specified address
-            * @param ownerHandle The psuedo-handle of the current mutex owner
-            */
-            Result MutexLock(u32 *mutex, KHandle ownerHandle);
+             * @brief Locks the mutex at the specified address
+             * @param ownerHandle The psuedo-handle of the current mutex owner
+             * @param tag The handle of the thread which is requesting this lock
+             */
+            Result MutexLock(u32 *mutex, KHandle ownerHandle, KHandle tag);
 
             /**
-            * @brief Unlocks the mutex at the specified address
-            */
+             * @brief Unlocks the mutex at the specified address
+             */
             void MutexUnlock(u32 *mutex);
 
             /**
-            * @param timeout The amount of time to wait for the conditional variable
-            * @return If the conditional variable was successfully waited for or timed out
-            */
-            bool ConditionalVariableWait(void *conditional, u32 *mutex, u64 timeout);
+             * @brief Waits on the conditional variable at the specified address
+             */
+            Result ConditionalVariableWait(u32 *key, u32 *mutex, KHandle tag, i64 timeout);
 
             /**
-            * @brief Signals a number of conditional variable waiters
-            * @param amount The amount of waiters to signal
-            */
-            void ConditionalVariableSignal(void *conditional, u64 amount);
+             * @brief Signals the conditional variable at the specified address
+             */
+            void ConditionalVariableSignal(u32 *key, u64 amount);
 
             /**
              * @brief Resets the object to an unsignalled state

app/src/main/cpp/skyline/kernel/types/KThread.cpp

@@ -58,7 +58,6 @@ namespace skyline::kernel::type {
         signal::SetSignalHandler({SIGINT, SIGILL, SIGTRAP, SIGBUS, SIGFPE, SIGSEGV}, nce::NCE::SignalHandler);
 
         try {
-            state.scheduler->InsertThread(state.thread);
             state.scheduler->WaitSchedule();
 
             asm volatile(
@@ -165,7 +164,9 @@ namespace skyline::kernel::type {
         std::unique_lock lock(mutex);
         if (!running) {
             running = true;
-            state.logger->Debug("Starting thread #{}", id);
+            auto sharedThis{shared_from_this()};
+            state.scheduler->LoadBalance(sharedThis);
+            state.scheduler->InsertThread(sharedThis);
             if (self) {
                 pthread = pthread_self();
                 lock.unlock();

app/src/main/cpp/skyline/kernel/types/KThread.h

@@ -26,7 +26,7 @@ namespace skyline {
 
           public:
             std::mutex mutex; //!< Synchronizes all thread state changes
-            bool running{false};
+            bool running{false}; //!< If the host thread that corresponds to this thread is running, this doesn't reflect guest scheduling changes
             std::atomic<bool> cancelSync{false}; //!< This is to flag to a thread to cancel a synchronization call it currently is in
 
             KHandle handle;
@@ -51,7 +51,8 @@ namespace skyline {
             bool isPreempted{}; //!< If the preemption timer has been armed and will fire
             bool needsReorder{}; //!< If the thread needs to reorder itself during scheduler rotation
             std::mutex waiterMutex; //!< Synchronizes operations on mutation of the waiter members
-            u32* waitKey; //!< The key on which this thread is waiting on
+            u32* waitKey; //!< The key of the mutex which this thread is waiting on
+            KHandle waitTag; //!< The handle of the thread which requested the mutex lock
             std::shared_ptr<KThread> waitThread; //!< The thread which this thread is waiting on
             std::list<std::shared_ptr<type::KThread>> waiters; //!< A queue of threads waiting on this thread sorted by priority
 

app/src/main/cpp/skyline/os.cpp

@@ -39,6 +39,7 @@ namespace skyline::kernel {
         process->InitializeHeap();
         auto thread{process->CreateThread(entry)};
         if (thread) {
+            state.logger->Debug("Starting main HOS thread");
             thread->Start(true);
             process->Kill(true, true, true);
         }

app/src/main/cpp/skyline/services/am/controller/ICommonStateGetter.cpp

@@ -7,7 +7,7 @@
 
 namespace skyline::service::am {
     void ICommonStateGetter::QueueMessage(ICommonStateGetter::Message message) {
-        messageQueue.emplace(message);
+        messageQueue.emplace_back(message);
         messageEvent->Signal();
     }
 
@@ -29,7 +29,11 @@ namespace skyline::service::am {
             return result::NoMessages;
 
         response.Push(messageQueue.front());
-        messageQueue.pop();
+        messageQueue.pop_front();
+
+        if (messageQueue.empty())
+            messageEvent->ResetSignal();
+
         return {};
     }
 

app/src/main/cpp/skyline/services/am/controller/ICommonStateGetter.h

@@ -3,7 +3,6 @@
 
 #pragma once
 
-#include <queue>
 #include <kernel/types/KEvent.h>
 #include <services/serviceman.h>
 
@@ -33,7 +32,7 @@ namespace skyline::service::am {
         };
 
         std::shared_ptr<type::KEvent> messageEvent; //!< The event signalled when there is a message available
-        std::queue<Message> messageQueue; //!< A queue of all the messages that the program is yet to consume
+        std::deque<Message> messageQueue; //!< A queue of all the messages that the program is yet to consume
 
         enum class FocusState : u8 {
             InFocus = 1,    //!< The application is in foreground

app/src/main/cpp/skyline/services/hosbinder/GraphicBufferProducer.cpp

@@ -132,7 +132,6 @@ namespace skyline::service::hosbinder {
             }
             if (!nvBuffer)
                 throw exception("A QueueBuffer request has an invalid NVMap Handle ({}) and ID ({})", gbpBuffer.nvmapHandle, gbpBuffer.nvmapId);
-            nvmapLock.unlock();
         }
 
         gpu::texture::Format format;