diff --git a/Source/Core/VideoBackends/Vulkan/CommandBufferManager.cpp b/Source/Core/VideoBackends/Vulkan/CommandBufferManager.cpp
index d8ef681b56..5c1df797c0 100644
--- a/Source/Core/VideoBackends/Vulkan/CommandBufferManager.cpp
+++ b/Source/Core/VideoBackends/Vulkan/CommandBufferManager.cpp
@@ -239,6 +239,8 @@ bool CommandBufferManager::CreateSubmitThread()
 
       SubmitCommandBuffer(submit.command_buffer_index, submit.present_swap_chain,
                           submit.present_image_index);
+      CmdBufferResources& resources = m_command_buffers[submit.command_buffer_index];
+      resources.waiting_for_submit.store(false, std::memory_order_release);
 
       {
         std::lock_guard<std::mutex> guard(m_pending_submit_lock);
@@ -284,11 +286,16 @@ void CommandBufferManager::WaitForFenceCounter(u64 fence_counter)
 
 void CommandBufferManager::WaitForCommandBufferCompletion(u32 index)
 {
-  // Ensure this command buffer has been submitted.
-  WaitForWorkerThreadIdle();
-
   CmdBufferResources& resources = m_command_buffers[index];
 
+  // Ensure this command buffer has been submitted.
+  if (resources.waiting_for_submit.load(std::memory_order_acquire))
+  {
+    WaitForWorkerThreadIdle();
+    ASSERT_MSG(VIDEO, !resources.waiting_for_submit.load(std::memory_order_relaxed),
+               "Submit thread is idle but command buffer is still waiting for submission!");
+  }
+
   // Wait for this command buffer to be completed.
   VkResult res =
       vkWaitForFences(g_vulkan_context->GetDevice(), 1, &resources.fence, VK_TRUE, UINT64_MAX);
@@ -339,6 +346,7 @@ void CommandBufferManager::SubmitCommandBuffer(bool submit_on_worker_thread,
   // Submitting off-thread?
   if (m_use_threaded_submission && submit_on_worker_thread && !wait_for_completion)
   {
+    resources.waiting_for_submit.store(true, std::memory_order_relaxed);
     // Push to the pending submit queue.
     {
       std::lock_guard<std::mutex> guard(m_pending_submit_lock);
diff --git a/Source/Core/VideoBackends/Vulkan/CommandBufferManager.h b/Source/Core/VideoBackends/Vulkan/CommandBufferManager.h
index 85e0ec8741..2f80132e35 100644
--- a/Source/Core/VideoBackends/Vulkan/CommandBufferManager.h
+++ b/Source/Core/VideoBackends/Vulkan/CommandBufferManager.h
@@ -118,6 +118,7 @@ private:
     u64 fence_counter = 0;
     bool init_command_buffer_used = false;
     bool semaphore_used = false;
+    std::atomic<bool> waiting_for_submit{false};
     u32 frame_index = 0;
 
     std::vector<std::function<void()>> cleanup_resources;