From efbb85da4322fe523db6d73b262f5531d5ac1e0e Mon Sep 17 00:00:00 2001 From: Stenzek Date: Wed, 17 Feb 2016 02:44:34 +1000 Subject: [PATCH] D3D12: Improve robustness of command allocator and fence tracking --- .../D3D12/D3DCommandListManager.cpp | 157 ++++++++++-------- .../D3D12/D3DCommandListManager.h | 9 +- 2 files changed, 96 insertions(+), 70 deletions(-) diff --git a/Source/Core/VideoBackends/D3D12/D3DCommandListManager.cpp b/Source/Core/VideoBackends/D3D12/D3DCommandListManager.cpp index fc4a98ae84..4fe6875f13 100644 --- a/Source/Core/VideoBackends/D3D12/D3DCommandListManager.cpp +++ b/Source/Core/VideoBackends/D3D12/D3DCommandListManager.cpp @@ -2,6 +2,7 @@ // Licensed under GPLv2+ // Refer to the license.txt file included. +#include #include #include @@ -69,6 +70,9 @@ D3DCommandListManager::D3DCommandListManager( } m_current_deferred_destruction_list = 0; + + std::fill(m_command_allocator_list_fences.begin(), m_command_allocator_list_fences.end(), 0); + std::fill(m_deferred_destruction_list_fences.begin(), m_deferred_destruction_list_fences.end(), 0); } void D3DCommandListManager::SetInitialCommandListState() @@ -114,37 +118,26 @@ void D3DCommandListManager::ExecuteQueuedWork(bool wait_for_gpu_completion) m_queued_command_list->QueueFenceGpuSignal(m_queue_fence, m_queue_fence_value); - ResetCommandListWithIdleCommandAllocator(); - - m_queued_command_list->ProcessQueuedItems(); + m_queued_command_list->ProcessQueuedItems(wait_for_gpu_completion); #else CheckHR(m_backing_command_list->Close()); - ID3D12CommandList* const commandListsToExecute[1] = { m_backing_command_list }; - m_command_queue->ExecuteCommandLists(1, commandListsToExecute); + ID3D12CommandList* const execute_list[1] = { m_backing_command_list }; + m_command_queue->ExecuteCommandLists(1, execute_list); - if (wait_for_gpu_completion) - { - CheckHR(m_command_queue->Signal(m_queue_fence, m_queue_fence_value)); - } - - if (m_current_command_allocator == 0) - { - PerformGpuRolloverChecks(); - } - - ResetCommandListWithIdleCommandAllocator(); + CheckHR(m_command_queue->Signal(m_queue_fence, m_queue_fence_value)); #endif + // Notify observers of the fence value for the current work to finish. for (auto it : m_queue_fence_callbacks) it.second(it.first, m_queue_fence_value); - SetInitialCommandListState(); - if (wait_for_gpu_completion) - { - WaitOnCPUForFence(m_queue_fence, m_queue_fence_value); - } + WaitForGPUCompletion(); + + // Re-open the command list, using the current allocator. + ResetCommandList(); + SetInitialCommandListState(); } void D3DCommandListManager::ExecuteQueuedWorkAndPresent(IDXGISwapChain* swap_chain, UINT sync_interval, UINT flags) @@ -154,60 +147,67 @@ void D3DCommandListManager::ExecuteQueuedWorkAndPresent(IDXGISwapChain* swap_cha #ifdef USE_D3D12_QUEUED_COMMAND_LISTS CheckHR(m_queued_command_list->Close()); m_queued_command_list->QueueExecute(); - m_queued_command_list->QueueFenceGpuSignal(m_queue_fence, m_queue_fence_value); m_queued_command_list->QueuePresent(swap_chain, sync_interval, flags); - m_queued_command_list->ProcessQueuedItems(true); - - if (m_current_command_allocator == 0) - { - PerformGpuRolloverChecks(); - } - - m_current_command_allocator = (m_current_command_allocator + 1) % m_command_allocator_lists[m_current_command_allocator_list].size(); - - ResetCommandListWithIdleCommandAllocator(); - - SetInitialCommandListState(); -#else - ExecuteQueuedWork(); - m_command_queue->Signal(m_queue_fence, m_queue_fence_value); - CheckHR(swap_chain->Present(sync_interval, flags)); -#endif - - for (auto it : m_queue_fence_callbacks) - it.second(it.first, m_queue_fence_value); -} - -void D3DCommandListManager::WaitForQueuedWorkToBeExecutedOnGPU() -{ - // Wait for GPU to finish all outstanding work. - m_queue_fence_value++; - -#ifdef USE_D3D12_QUEUED_COMMAND_LISTS - m_queued_command_list->QueueExecute(); m_queued_command_list->QueueFenceGpuSignal(m_queue_fence, m_queue_fence_value); - m_queued_command_list->ProcessQueuedItems(true); #else + CheckHR(m_backing_command_list->Close()); + + ID3D12CommandList* const execute_list[1] = { m_backing_command_list }; + m_command_queue->ExecuteCommandLists(1, execute_list); + + CheckHR(swap_chain->Present(sync_interval, flags)); CheckHR(m_command_queue->Signal(m_queue_fence, m_queue_fence_value)); #endif - WaitOnCPUForFence(m_queue_fence, m_queue_fence_value); + // Notify observers of the fence value for the current work to finish. + for (auto it : m_queue_fence_callbacks) + it.second(it.first, m_queue_fence_value); + + // Move to the next command allocator, this may mean switching allocator lists. + MoveToNextCommandAllocator(); + ResetCommandList(); + SetInitialCommandListState(); +} + +void D3DCommandListManager::WaitForGPUCompletion() +{ + // Wait for GPU to finish all outstanding work. + // This method assumes that no command lists are open. + m_queue_frame_fence_value++; + +#ifdef USE_D3D12_QUEUED_COMMAND_LISTS + m_queued_command_list->QueueFenceGpuSignal(m_queue_frame_fence, m_queue_frame_fence_value); + m_queued_command_list->ProcessQueuedItems(true); +#else + CheckHR(m_command_queue->Signal(m_queue_frame_fence, m_queue_frame_fence_value)); +#endif + + WaitOnCPUForFence(m_queue_frame_fence, m_queue_frame_fence_value); + + // GPU is up to date with us. Therefore, it has finished with any pending resources. + ImmediatelyDestroyAllResourcesScheduledForDestruction(); + + // Command allocators are also up-to-date, so reset these. + ResetAllCommandAllocators(); } void D3DCommandListManager::PerformGpuRolloverChecks() { - // Insert fence to measure GPU progress, ensure we aren't using in-use command allocators. - if (m_queue_frame_fence->GetCompletedValue() < m_queue_frame_fence_value) - { - WaitOnCPUForFence(m_queue_frame_fence, m_queue_frame_fence_value); - } + m_queue_frame_fence_value++; + +#ifdef USE_D3D12_QUEUED_COMMAND_LISTS + m_queued_command_list->QueueFenceGpuSignal(m_queue_frame_fence, m_queue_frame_fence_value); +#else + CheckHR(m_command_queue->Signal(m_queue_frame_fence, m_queue_frame_fence_value)); +#endif // We now know that the previous 'set' of command lists has completed on GPU, and it is safe to // release resources / start back at beginning of command allocator list. // Begin Deferred Resource Destruction UINT safe_to_delete_deferred_destruction_list = (m_current_deferred_destruction_list - 1) % m_deferred_destruction_lists.size(); + WaitOnCPUForFence(m_queue_frame_fence, m_deferred_destruction_list_fences[safe_to_delete_deferred_destruction_list]); for (UINT i = 0; i < m_deferred_destruction_lists[safe_to_delete_deferred_destruction_list].size(); i++) { @@ -216,30 +216,37 @@ void D3DCommandListManager::PerformGpuRolloverChecks() m_deferred_destruction_lists[safe_to_delete_deferred_destruction_list].clear(); + m_deferred_destruction_list_fences[m_current_deferred_destruction_list] = m_queue_frame_fence_value; m_current_deferred_destruction_list = (m_current_deferred_destruction_list + 1) % m_deferred_destruction_lists.size(); // End Deferred Resource Destruction // Begin Command Allocator Resets UINT safe_to_reset_command_allocator_list = (m_current_command_allocator_list - 1) % m_command_allocator_lists.size(); + WaitOnCPUForFence(m_queue_frame_fence, m_command_allocator_list_fences[safe_to_reset_command_allocator_list]); for (UINT i = 0; i < m_command_allocator_lists[safe_to_reset_command_allocator_list].size(); i++) { CheckHR(m_command_allocator_lists[safe_to_reset_command_allocator_list][i]->Reset()); } + m_command_allocator_list_fences[m_current_command_allocator_list] = m_queue_frame_fence_value; m_current_command_allocator_list = (m_current_command_allocator_list + 1) % m_command_allocator_lists.size(); + m_current_command_allocator = 0; // End Command Allocator Resets - - m_queue_frame_fence_value++; -#ifdef USE_D3D12_QUEUED_COMMAND_LISTS - m_queued_command_list->QueueFenceGpuSignal(m_queue_frame_fence, m_queue_frame_fence_value); -#else - CheckHR(m_command_queue->Signal(m_queue_frame_fence, m_queue_frame_fence_value)); -#endif } -void D3DCommandListManager::ResetCommandListWithIdleCommandAllocator() +void D3DCommandListManager::MoveToNextCommandAllocator() +{ + // Move to the next allocator in the current allocator list. + m_current_command_allocator = (m_current_command_allocator + 1) % m_command_allocator_lists[m_current_command_allocator_list].size(); + + // Did we wrap around? Move to the next set of allocators. + if (m_current_command_allocator == 0) + PerformGpuRolloverChecks(); +} + +void D3DCommandListManager::ResetCommandList() { #ifdef USE_D3D12_QUEUED_COMMAND_LISTS ID3D12QueuedCommandList* command_list = m_queued_command_list; @@ -268,6 +275,20 @@ void D3DCommandListManager::ImmediatelyDestroyAllResourcesScheduledForDestructio } } +void D3DCommandListManager::ResetAllCommandAllocators() +{ + for (auto& allocator_list : m_command_allocator_lists) + { + for (auto& allocator : allocator_list) + allocator->Reset(); + } + + // Move back to the start, using the first allocator of first list. + m_current_command_allocator = 0; + m_current_command_allocator_list = 0; + m_current_deferred_destruction_list = 0; +} + void D3DCommandListManager::ClearQueueAndWaitForCompletionOfInflightWork() { // Wait for GPU to finish all outstanding work. @@ -305,8 +326,10 @@ D3DCommandListManager::~D3DCommandListManager() void D3DCommandListManager::WaitOnCPUForFence(ID3D12Fence* fence, UINT64 fence_value) { - CheckHR(fence->SetEventOnCompletion(fence_value, m_wait_on_cpu_fence_event)); + if (fence->GetCompletedValue() >= fence_value) + return; + CheckHR(fence->SetEventOnCompletion(fence_value, m_wait_on_cpu_fence_event)); WaitForSingleObject(m_wait_on_cpu_fence_event, INFINITE); } diff --git a/Source/Core/VideoBackends/D3D12/D3DCommandListManager.h b/Source/Core/VideoBackends/D3D12/D3DCommandListManager.h index b9622df5fc..3f932dfc0e 100644 --- a/Source/Core/VideoBackends/D3D12/D3DCommandListManager.h +++ b/Source/Core/VideoBackends/D3D12/D3DCommandListManager.h @@ -38,11 +38,10 @@ public: void ExecuteQueuedWork(bool wait_for_gpu_completion = false); void ExecuteQueuedWorkAndPresent(IDXGISwapChain* swap_chain, UINT sync_interval, UINT flags); - void WaitForQueuedWorkToBeExecutedOnGPU(); - void ClearQueueAndWaitForCompletionOfInflightWork(); void DestroyResourceAfterCurrentCommandListExecuted(ID3D12Resource* resource); void ImmediatelyDestroyAllResourcesScheduledForDestruction(); + void ResetAllCommandAllocators(); void SetCommandListDirtyState(unsigned int command_list_state, bool dirty); bool GetCommandListDirtyState(COMMAND_LIST_STATE command_list_state) const; @@ -65,8 +64,10 @@ public: private: + void WaitForGPUCompletion(); void PerformGpuRolloverChecks(); - void ResetCommandListWithIdleCommandAllocator(); + void MoveToNextCommandAllocator(); + void ResetCommandList(); unsigned int m_command_list_dirty_state = UINT_MAX; D3D_PRIMITIVE_TOPOLOGY m_command_list_current_topology = D3D_PRIMITIVE_TOPOLOGY_UNDEFINED; @@ -85,6 +86,7 @@ private: UINT m_current_command_allocator; UINT m_current_command_allocator_list; std::array, 2> m_command_allocator_lists; + std::array m_command_allocator_list_fences; ID3D12GraphicsCommandList* m_backing_command_list; ID3D12QueuedCommandList* m_queued_command_list; @@ -93,6 +95,7 @@ private: UINT m_current_deferred_destruction_list; std::array, 2> m_deferred_destruction_lists; + std::array m_deferred_destruction_list_fences; }; } // namespace \ No newline at end of file