Fix invalid usage by waiting on CommandBuffer fence before resetting CommandBuffer

framework/decode/vulkan_object_info.h

@@ -314,6 +314,7 @@ struct DeviceInfo : public VulkanObjectInfo<VkDevice>
 
 struct QueueInfo : public VulkanObjectInfo<VkQueue>
 {
+    VkDevice                             parent{ VK_NULL_HANDLE };
     std::unordered_map<uint32_t, size_t> array_counts;
     uint32_t                             family_index;
     uint32_t                             queue_index;

framework/decode/vulkan_replay_consumer_base.cpp

@@ -3015,6 +3015,7 @@ void VulkanReplayConsumerBase::OverrideGetDeviceQueue(PFN_vkGetDeviceQueue
     // This is necessary for the virtual swapchain to determine which command buffer to use when
     // Bliting the images on the Presenting Queue.
     auto queue_info          = reinterpret_cast<QueueInfo*>(pQueue->GetConsumerData(0));
+    queue_info->parent       = device;
     queue_info->family_index = queueFamilyIndex;
     queue_info->queue_index  = queueIndex;
 }
@@ -3038,6 +3039,7 @@ void VulkanReplayConsumerBase::OverrideGetDeviceQueue2(PFN_vkGetDeviceQueue2
     // This is necessary for the virtual swapchain to determine which command buffer to use when
     // Bliting the images on the Presenting Queue.
     auto queue_info          = reinterpret_cast<QueueInfo*>(pQueue->GetConsumerData(0));
+    queue_info->parent       = device;
     queue_info->family_index = in_pQueueInfo->queueFamilyIndex;
     queue_info->queue_index  = in_pQueueInfo->queueIndex;
 }

framework/decode/vulkan_virtual_swapchain.cpp

@@ -134,6 +134,10 @@ void VulkanVirtualSwapchain::CleanSwapchainResourceData(const DeviceInfo*
                 {
                     device_table_->DestroySemaphore(device, semaphore, nullptr);
                 }
+                for (auto& fence : copy_cmd_data.second.fences)
+                {
+                    device_table_->DestroyFence(device, fence, nullptr);
+                }
             }
 
             swapchain_resources_.erase(swapchain);
@@ -362,6 +366,31 @@ VkResult VulkanVirtualSwapchain::CreateSwapchainResourceData(const DeviceInfo*
                         copy_cmd_data.semaphores[ii] = semaphore;
                     }
                 }
+                uint32_t fence_count = static_cast<uint32_t>(copy_cmd_data.fences.size());
+                if (fence_count < capture_image_count)
+                {
+                    copy_cmd_data.fences.resize(capture_image_count);
+
+                    for (uint32_t ii = fence_count; ii < capture_image_count; ++ii)
+                    {
+                        VkFenceCreateInfo fence_create_info = {
+                            VK_STRUCTURE_TYPE_FENCE_CREATE_INFO, // sType
+                            nullptr,                             // pNext
+                            VK_FENCE_CREATE_SIGNALED_BIT         // flags
+                        };
+
+                        VkFence fence = VK_NULL_HANDLE;
+                        result        = device_table_->CreateFence(device, &fence_create_info, nullptr, &fence);
+                        if (result != VK_SUCCESS)
+                        {
+                            GFXRECON_LOG_ERROR("Virtual swapchain failed creating internal copy fence for "
+                                               "swapchain (ID = %" PRIu64 ")",
+                                               swapchain_info->capture_id);
+                            return result;
+                        }
+                        copy_cmd_data.fences[ii] = fence;
+                    }
+                }
             }
         }
     }
@@ -431,7 +460,18 @@ VkResult VulkanVirtualSwapchain::CreateSwapchainResourceData(const DeviceInfo*
             begin_info.pInheritanceInfo         = nullptr;
 
             auto command_buffer = swapchain_resources->copy_cmd_data[copy_queue_family_index].command_buffers[0];
+            auto copy_fence     = swapchain_resources->copy_cmd_data[copy_queue_family_index].fences[0];
 
+            result = device_table_->WaitForFences(device, 1, &copy_fence, VK_TRUE, ~0UL);
+            if (result != VK_SUCCESS)
+            {
+                return result;
+            }
+            result = device_table_->ResetFences(device, 1, &copy_fence);
+            if (result != VK_SUCCESS)
+            {
+                return result;
+            }
             result = device_table_->ResetCommandBuffer(command_buffer, 0);
             if (result != VK_SUCCESS)
             {
@@ -500,7 +540,7 @@ VkResult VulkanVirtualSwapchain::CreateSwapchainResourceData(const DeviceInfo*
             submit_info.commandBufferCount = 1;
             submit_info.pCommandBuffers    = &command_buffer;
 
-            result = device_table_->QueueSubmit(initial_copy_queue, 1, &submit_info, VK_NULL_HANDLE);
+            result = device_table_->QueueSubmit(initial_copy_queue, 1, &submit_info, copy_fence);
             if (result != VK_SUCCESS)
             {
                 GFXRECON_LOG_ERROR(
@@ -673,6 +713,7 @@ VkResult VulkanVirtualSwapchain::QueuePresentKHR(VkResult
         return func(queue_info->handle, present_info);
     }
 
+    VkDevice device             = queue_info->parent;
     VkQueue  queue              = queue_info->handle;
     uint32_t queue_family_index = queue_info->family_index;
 
@@ -769,6 +810,7 @@ VkResult VulkanVirtualSwapchain::QueuePresentKHR(VkResult
         auto& copy_cmd_data  = swapchain_resources->copy_cmd_data[queue_family_index];
         auto  command_buffer = copy_cmd_data.command_buffers[capture_image_index];
         auto  copy_semaphore = copy_cmd_data.semaphores[capture_image_index];
+        auto  copy_fence     = copy_cmd_data.fences[capture_image_index];
 
         std::vector<VkSemaphore> wait_semaphores;
         std::vector<VkSemaphore> signal_semaphores;
@@ -789,6 +831,16 @@ VkResult VulkanVirtualSwapchain::QueuePresentKHR(VkResult
             present_wait_semaphores.emplace_back(copy_semaphore);
         }
 
+        result = device_table_->WaitForFences(device, 1, &copy_fence, VK_TRUE, ~0UL);
+        if (result != VK_SUCCESS)
+        {
+            return result;
+        }
+        result = device_table_->ResetFences(device, 1, &copy_fence);
+        if (result != VK_SUCCESS)
+        {
+            return result;
+        }
         result = device_table_->ResetCommandBuffer(command_buffer, 0);
         if (result != VK_SUCCESS)
         {
@@ -892,7 +944,7 @@ VkResult VulkanVirtualSwapchain::QueuePresentKHR(VkResult
         submit_info.commandBufferCount   = 1;
         submit_info.pCommandBuffers      = &command_buffer;
 
-        result = device_table_->QueueSubmit(queue, 1, &submit_info, VK_NULL_HANDLE);
+        result = device_table_->QueueSubmit(queue, 1, &submit_info, copy_fence);
 
         if (result != VK_SUCCESS)
         {

framework/decode/vulkan_virtual_swapchain.h

@@ -133,6 +133,7 @@ class VulkanVirtualSwapchain : public VulkanSwapchain
     {
         VkCommandPool                command_pool;
         std::vector<VkCommandBuffer> command_buffers;
+        std::vector<VkFence>         fences;
         std::vector<VkSemaphore>     semaphores;
     };