From e8fa867f141fc296132b8bc1116a7a11eac28629 Mon Sep 17 00:00:00 2001
From: Robin Kertels <robin.kertels@gmail.com>
Date: Sat, 1 Oct 2022 01:05:41 +0200
Subject: [PATCH] VideoBackends:Vulkan: Only synchronize with submission thread
 when necessary

We only need to synchronize with the submission thread
when submitting on the GPU thread or when waiting for a command buffer.
---
 .../Vulkan/CommandBufferManager.cpp           | 33 ++++++++++++-------
 .../Vulkan/CommandBufferManager.h             |  4 +--
 2 files changed, 23 insertions(+), 14 deletions(-)

diff --git a/Source/Core/VideoBackends/Vulkan/CommandBufferManager.cpp b/Source/Core/VideoBackends/Vulkan/CommandBufferManager.cpp
index e7931cdcc6..98b3a82088 100644
--- a/Source/Core/VideoBackends/Vulkan/CommandBufferManager.cpp
+++ b/Source/Core/VideoBackends/Vulkan/CommandBufferManager.cpp
@@ -15,7 +15,7 @@
 namespace Vulkan
 {
 CommandBufferManager::CommandBufferManager(bool use_threaded_submission)
-    : m_submit_semaphore(1, 1), m_use_threaded_submission(use_threaded_submission)
+    : m_use_threaded_submission(use_threaded_submission)
 {
 }
 
@@ -24,6 +24,7 @@ CommandBufferManager::~CommandBufferManager()
   // If the worker thread is enabled, stop and block until it exits.
   if (m_use_threaded_submission)
   {
+    WaitForWorkerThreadIdle();
     m_submit_loop->Stop();
     m_submit_thread.join();
   }
@@ -194,6 +195,8 @@ bool CommandBufferManager::CreateSubmitThread()
         if (m_pending_submits.empty())
         {
           m_submit_loop->AllowSleep();
+          m_submit_worker_idle = true;
+          m_submit_worker_condvar.notify_all();
           return;
         }
 
@@ -203,6 +206,15 @@ bool CommandBufferManager::CreateSubmitThread()
 
       SubmitCommandBuffer(submit.command_buffer_index, submit.present_swap_chain,
                           submit.present_image_index);
+
+      {
+        std::lock_guard<std::mutex> guard(m_pending_submit_lock);
+        if (m_pending_submits.empty())
+        {
+          m_submit_worker_idle = true;
+          m_submit_worker_condvar.notify_all();
+        }
+      }
     });
   });
 
@@ -211,9 +223,11 @@ bool CommandBufferManager::CreateSubmitThread()
 
 void CommandBufferManager::WaitForWorkerThreadIdle()
 {
-  // Drain the semaphore, then allow another request in the future.
-  m_submit_semaphore.Wait();
-  m_submit_semaphore.Post();
+  if (!m_use_threaded_submission)
+    return;
+
+  std::unique_lock lock{m_pending_submit_lock};
+  m_submit_worker_condvar.wait(lock, [&] { return m_submit_worker_idle; });
 }
 
 void CommandBufferManager::WaitForFenceCounter(u64 fence_counter)
@@ -286,17 +300,13 @@ void CommandBufferManager::SubmitCommandBuffer(bool submit_on_worker_thread,
     }
   }
 
-  // Grab the semaphore before submitting command buffer either on-thread or off-thread.
-  // This prevents a race from occurring where a second command buffer is executed
-  // before the worker thread has woken and executed the first one yet.
-  m_submit_semaphore.Wait();
-
   // Submitting off-thread?
   if (m_use_threaded_submission && submit_on_worker_thread && !wait_for_completion)
   {
     // Push to the pending submit queue.
     {
       std::lock_guard<std::mutex> guard(m_pending_submit_lock);
+      m_submit_worker_idle = false;
       m_pending_submits.push_back({present_swap_chain, present_image_index, m_current_frame});
     }
 
@@ -305,6 +315,8 @@ void CommandBufferManager::SubmitCommandBuffer(bool submit_on_worker_thread,
   }
   else
   {
+    WaitForWorkerThreadIdle();
+
     // Pass through to normal submission path.
     SubmitCommandBuffer(m_current_frame, present_swap_chain, present_image_index);
     if (wait_for_completion)
@@ -394,9 +406,6 @@ void CommandBufferManager::SubmitCommandBuffer(u32 command_buffer_index,
 #endif
     }
   }
-
-  // Command buffer has been queued, so permit the next one.
-  m_submit_semaphore.Post();
 }
 
 void CommandBufferManager::BeginCommandBuffer()
diff --git a/Source/Core/VideoBackends/Vulkan/CommandBufferManager.h b/Source/Core/VideoBackends/Vulkan/CommandBufferManager.h
index 632e43534a..8493559f0f 100644
--- a/Source/Core/VideoBackends/Vulkan/CommandBufferManager.h
+++ b/Source/Core/VideoBackends/Vulkan/CommandBufferManager.h
@@ -123,8 +123,6 @@ private:
   u32 m_current_frame = 0;
 
   // Threaded command buffer execution
-  // Semaphore determines when a command buffer can be queued
-  Common::Semaphore m_submit_semaphore;
   std::thread m_submit_thread;
   std::unique_ptr<Common::BlockingLoop> m_submit_loop;
   struct PendingCommandBufferSubmit
@@ -136,6 +134,8 @@ private:
   VkSemaphore m_present_semaphore = VK_NULL_HANDLE;
   std::deque<PendingCommandBufferSubmit> m_pending_submits;
   std::mutex m_pending_submit_lock;
+  std::condition_variable m_submit_worker_condvar;
+  bool m_submit_worker_idle = true;
   Common::Flag m_last_present_failed;
   VkResult m_last_present_result = VK_SUCCESS;
   bool m_use_threaded_submission = false;