diff --git a/src/command.cpp b/src/command.cpp index 27643e832..89a4a0b87 100644 --- a/src/command.cpp +++ b/src/command.cpp @@ -20,11 +20,8 @@ namespace ncnn { -Command::Command(const VulkanDevice* _vkdev, uint32_t _queue_index) : vkdev(_vkdev), queue_index(_queue_index) +Command::Command(const VulkanDevice* _vkdev, uint32_t _queue_family_index) : vkdev(_vkdev), queue_family_index(_queue_family_index) { - // get queue - vkGetDeviceQueue(vkdev->vkdevice(), queue_index, 0, &queue); - create_command_pool(); create_command_buffer(); @@ -57,7 +54,7 @@ int Command::create_command_pool() commandPoolCreateInfo.sType = VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO; commandPoolCreateInfo.pNext = 0; commandPoolCreateInfo.flags = VK_COMMAND_POOL_CREATE_RESET_COMMAND_BUFFER_BIT; - commandPoolCreateInfo.queueFamilyIndex = queue_index; + commandPoolCreateInfo.queueFamilyIndex = queue_family_index; VkResult ret = vkCreateCommandPool(vkdev->vkdevice(), &commandPoolCreateInfo, 0, &command_pool); if (ret != VK_SUCCESS) @@ -122,46 +119,54 @@ int Command::end_command_buffer() return 0; } -int Command::queue_submit() +int Command::queue_submit_and_wait_fence() { + // acquire queue and reclaim on return + VkQueue queue = vkdev->acquire_queue(queue_family_index); + if (queue == 0) + { + fprintf(stderr, "out of compute queue\n"); + return -1; + } + // fprintf(stderr, "==================== submit\n"); - - VkSubmitInfo submitInfo; - submitInfo.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO; - submitInfo.pNext = 0; - submitInfo.waitSemaphoreCount = 0; - submitInfo.pWaitSemaphores = 0; - submitInfo.pWaitDstStageMask = 0; - submitInfo.commandBufferCount = 1; - submitInfo.pCommandBuffers = &command_buffer; - submitInfo.signalSemaphoreCount = 0; - submitInfo.pSignalSemaphores = 0; - - VkResult ret = vkQueueSubmit(queue, 1, &submitInfo, fence); - if (ret != VK_SUCCESS) { - fprintf(stderr, "vkQueueSubmit failed %d\n", ret); - return -1; + VkSubmitInfo submitInfo; + submitInfo.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO; + submitInfo.pNext = 0; + submitInfo.waitSemaphoreCount = 0; + submitInfo.pWaitSemaphores = 0; + submitInfo.pWaitDstStageMask = 0; + submitInfo.commandBufferCount = 1; + submitInfo.pCommandBuffers = &command_buffer; + submitInfo.signalSemaphoreCount = 0; + submitInfo.pSignalSemaphores = 0; + + VkResult ret = vkQueueSubmit(queue, 1, &submitInfo, fence); + if (ret != VK_SUCCESS) + { + fprintf(stderr, "vkQueueSubmit failed %d\n", ret); + vkdev->reclaim_queue(queue_family_index, queue); + return -1; + } } - return 0; -} - -int Command::wait_fence() -{ // fprintf(stderr, "==================== wait\n"); - - VkResult ret = vkWaitForFences(vkdev->vkdevice(), 1, &fence, VK_TRUE, UINT64_MAX); - if (ret != VK_SUCCESS) { - fprintf(stderr, "vkWaitForFences failed %d\n", ret); - return -1; + VkResult ret = vkWaitForFences(vkdev->vkdevice(), 1, &fence, VK_TRUE, UINT64_MAX); + if (ret != VK_SUCCESS) + { + fprintf(stderr, "vkWaitForFences failed %d\n", ret); + vkdev->reclaim_queue(queue_family_index, queue); + return -1; + } } + vkdev->reclaim_queue(queue_family_index, queue); return 0; } -VkCompute::VkCompute(const VulkanDevice* _vkdev) : Command(_vkdev, _vkdev->info.compute_queue_index) +VkCompute::VkCompute(const VulkanDevice* _vkdev) : Command(_vkdev, _vkdev->info.compute_queue_family_index) { if (vkdev->info.support_VK_KHR_push_descriptor) { @@ -500,13 +505,13 @@ void VkCompute::record_prepare_compute_barrier(const VkMat& m) m.data->state = 3; } -int VkCompute::submit() +int VkCompute::submit_and_wait() { if (vkdev->info.support_VK_KHR_push_descriptor) { end_command_buffer(); - return queue_submit(); + return queue_submit_and_wait_fence(); } begin_command_buffer(); @@ -555,12 +560,7 @@ int VkCompute::submit() delayed_records.clear(); - return queue_submit(); -} - -int VkCompute::wait() -{ - return wait_fence(); + return queue_submit_and_wait_fence(); } int VkCompute::reset() @@ -727,7 +727,7 @@ void VkCompute::transfer_transfer_barrier(VkBuffer buffer, size_t offset, size_t vkCmdPipelineBarrier(command_buffer, srcStageMask, dstStageMask, 0, 0, 0, 1, &bufferBarrier, 0, 0); } -VkTransfer::VkTransfer(const VulkanDevice* _vkdev) : Command(_vkdev, _vkdev->info.transfer_queue_index) +VkTransfer::VkTransfer(const VulkanDevice* _vkdev) : Command(_vkdev, _vkdev->info.transfer_queue_family_index) { buffer_offset_alignment = vkdev->info.buffer_offset_alignment; staging_data = 0; @@ -767,7 +767,7 @@ void VkTransfer::record_upload(const Mat& src, VkMat& dst) delayed_records.push_back(r); } -int VkTransfer::submit() +int VkTransfer::submit_and_wait() { if (delayed_records.empty()) return 0; @@ -813,15 +813,7 @@ int VkTransfer::submit() end_command_buffer(); - return queue_submit(); -} - -int VkTransfer::wait() -{ - if (delayed_records.empty()) - return 0; - - int ret = wait_fence(); + int ret = queue_submit_and_wait_fence(); // deallocate staging buffer staging_vkallocator->fastFree(staging_data); diff --git a/src/command.h b/src/command.h index d6cdcb0dc..30f14ad71 100644 --- a/src/command.h +++ b/src/command.h @@ -29,7 +29,7 @@ namespace ncnn { class Command { public: - Command(const VulkanDevice* vkdev, uint32_t queue_index); + Command(const VulkanDevice* vkdev, uint32_t queue_family_index); ~Command(); protected: @@ -39,14 +39,11 @@ protected: // record issue int begin_command_buffer(); int end_command_buffer(); - int queue_submit(); - int wait_fence(); + int queue_submit_and_wait_fence(); protected: const VulkanDevice* vkdev; - uint32_t queue_index; - - VkQueue queue; + uint32_t queue_family_index; VkCommandPool command_pool; VkCommandBuffer command_buffer; @@ -72,9 +69,7 @@ public: void record_pipeline(const Pipeline* pipeline, const std::vector& bindings, const std::vector& constants, const VkMat& m); - int submit(); - - int wait(); + int submit_and_wait(); int reset(); @@ -156,9 +151,7 @@ public: void record_upload(const Mat& src, VkMat& dst); - int submit(); - - int wait(); + int submit_and_wait(); public: VkAllocator* weight_vkallocator; diff --git a/src/gpu.cpp b/src/gpu.cpp index 5491e505d..58dcecb59 100644 --- a/src/gpu.cpp +++ b/src/gpu.cpp @@ -131,7 +131,8 @@ static uint32_t find_device_compute_queue(const std::vector queueFamilyProperties(queueFamilyPropertiesCount); vkGetPhysicalDeviceQueueFamilyProperties(physicalDevice, &queueFamilyPropertiesCount, queueFamilyProperties.data()); - gpu_info.compute_queue_index = find_device_compute_queue(queueFamilyProperties); - gpu_info.transfer_queue_index = find_device_transfer_queue(queueFamilyProperties); + gpu_info.compute_queue_family_index = find_device_compute_queue(queueFamilyProperties); + gpu_info.transfer_queue_family_index = find_device_transfer_queue(queueFamilyProperties); + + gpu_info.compute_queue_count = queueFamilyProperties[gpu_info.compute_queue_family_index].queueCount; + gpu_info.transfer_queue_count = queueFamilyProperties[gpu_info.transfer_queue_family_index].queueCount; // find memory type index VkPhysicalDeviceMemoryProperties physicalDeviceMemoryProperties; @@ -677,19 +683,19 @@ int create_gpu_instance() vkGetPhysicalDeviceFeatures2KHR(physicalDevice, &queryFeatures); - if (gpu_info.support_VK_KHR_8bit_storage) - { - gpu_info.support_int8_storage = query8BitStorageFeatures.storageBuffer8BitAccess && query8BitStorageFeatures.uniformAndStorageBuffer8BitAccess; - } - if (gpu_info.support_VK_KHR_16bit_storage) - { - gpu_info.support_fp16_storage = query16BitStorageFeatures.storageBuffer16BitAccess && query16BitStorageFeatures.uniformAndStorageBuffer16BitAccess; - } - if (gpu_info.support_VK_KHR_shader_float16_int8) - { - gpu_info.support_fp16_arithmetic = queryFloat16Int8Features.shaderFloat16; - gpu_info.support_int8_arithmetic = queryFloat16Int8Features.shaderInt8; - } +// if (gpu_info.support_VK_KHR_8bit_storage) +// { +// gpu_info.support_int8_storage = query8BitStorageFeatures.storageBuffer8BitAccess && query8BitStorageFeatures.uniformAndStorageBuffer8BitAccess; +// } +// if (gpu_info.support_VK_KHR_16bit_storage) +// { +// gpu_info.support_fp16_storage = query16BitStorageFeatures.storageBuffer16BitAccess && query16BitStorageFeatures.uniformAndStorageBuffer16BitAccess; +// } +// if (gpu_info.support_VK_KHR_shader_float16_int8) +// { +// gpu_info.support_fp16_arithmetic = queryFloat16Int8Features.shaderFloat16; +// gpu_info.support_int8_arithmetic = queryFloat16Int8Features.shaderInt8; +// } } else { @@ -698,8 +704,9 @@ int create_gpu_instance() // vkGetPhysicalDeviceFeatures(physicalDevice, &features); } - fprintf(stderr, "[%u %s] queueC=%u queueT=%u memU=%u memDL=%u memHV=%u\n", i, physicalDeviceProperties.deviceName, - gpu_info.compute_queue_index, gpu_info.transfer_queue_index, + fprintf(stderr, "[%u %s] queueC=%u[%u] queueT=%u[%u] memU=%u memDL=%u memHV=%u\n", i, physicalDeviceProperties.deviceName, + gpu_info.compute_queue_family_index, gpu_info.compute_queue_count, + gpu_info.transfer_queue_family_index, gpu_info.transfer_queue_count, gpu_info.unified_memory_index, gpu_info.device_local_memory_index, gpu_info.host_visible_memory_index); fprintf(stderr, "[%u %s] fp16s=%d fp16a=%d int8s=%d int8a=%d\n", i, physicalDeviceProperties.deviceName, @@ -762,8 +769,6 @@ static const int layer_shader_registry_entry_count = sizeof(layer_shader_registr VulkanDevice::VulkanDevice(int device_index) : info(g_gpu_infos[device_index]) { - const float queuePriorities[1] = { 1.f };// 0.f ~ 1.f - std::vector enabledExtensions; if (info.support_VK_KHR_8bit_storage) enabledExtensions.push_back("VK_KHR_8bit_storage"); @@ -827,25 +832,28 @@ VulkanDevice::VulkanDevice(int device_index) : info(g_gpu_infos[device_index]) enabledExtensionFeatures = &enabledFloat16Int8Features; } + std::vector compute_queue_priorities(info.compute_queue_count, 1.f);// 0.f ~ 1.f + std::vector transfer_queue_priorities(info.transfer_queue_count, 1.f);// 0.f ~ 1.f + VkDeviceQueueCreateInfo deviceQueueCreateInfos[2]; deviceQueueCreateInfos[0].sType = VK_STRUCTURE_TYPE_DEVICE_QUEUE_CREATE_INFO; deviceQueueCreateInfos[0].pNext = 0; deviceQueueCreateInfos[0].flags = 0; - deviceQueueCreateInfos[0].queueFamilyIndex = info.compute_queue_index; - deviceQueueCreateInfos[0].queueCount = 1; - deviceQueueCreateInfos[0].pQueuePriorities = queuePriorities; + deviceQueueCreateInfos[0].queueFamilyIndex = info.compute_queue_family_index; + deviceQueueCreateInfos[0].queueCount = info.compute_queue_count; + deviceQueueCreateInfos[0].pQueuePriorities = compute_queue_priorities.data(); deviceQueueCreateInfos[1].sType = VK_STRUCTURE_TYPE_DEVICE_QUEUE_CREATE_INFO; deviceQueueCreateInfos[1].pNext = 0; deviceQueueCreateInfos[1].flags = 0; - deviceQueueCreateInfos[1].queueFamilyIndex = info.transfer_queue_index; - deviceQueueCreateInfos[1].queueCount = 1; - deviceQueueCreateInfos[1].pQueuePriorities = queuePriorities; + deviceQueueCreateInfos[1].queueFamilyIndex = info.transfer_queue_family_index; + deviceQueueCreateInfos[1].queueCount = info.transfer_queue_count; + deviceQueueCreateInfos[1].pQueuePriorities = transfer_queue_priorities.data(); VkDeviceCreateInfo deviceCreateInfo; deviceCreateInfo.sType = VK_STRUCTURE_TYPE_DEVICE_CREATE_INFO; deviceCreateInfo.pNext = enabledExtensionFeatures; deviceCreateInfo.flags = 0; - if (info.compute_queue_index == info.transfer_queue_index) + if (info.compute_queue_family_index == info.transfer_queue_family_index) { deviceCreateInfo.queueCreateInfoCount = 1; } @@ -870,6 +878,20 @@ VulkanDevice::VulkanDevice(int device_index) : info(g_gpu_infos[device_index]) create_shader_module(); + compute_queues.resize(info.compute_queue_count); + for (uint32_t i = 0; i < info.compute_queue_count; i++) + { + vkGetDeviceQueue(device, info.compute_queue_family_index, i, &compute_queues[i]); + } + if (info.compute_queue_family_index != info.transfer_queue_family_index) + { + transfer_queues.resize(info.transfer_queue_count); + for (uint32_t i = 0; i < info.transfer_queue_count; i++) + { + vkGetDeviceQueue(device, info.transfer_queue_family_index, i, &transfer_queues[i]); + } + } + blob_buffer_allocator = new VkBlobBufferAllocator(this); staging_buffer_allocator = new VkStagingBufferAllocator(this); } @@ -916,6 +938,54 @@ VkShaderModule VulkanDevice::compile_shader_module(const uint32_t* spv_data, siz return shader_module; } +VkQueue VulkanDevice::acquire_queue(uint32_t queue_family_index) const +{ + if (queue_family_index != info.compute_queue_family_index && queue_family_index != info.transfer_queue_family_index) + { + fprintf(stderr, "invalid queue_family_index %u\n", queue_family_index); + return 0; + } + + MutexLockGuard lock(queue_lock); + + std::vector& queues = queue_family_index == info.compute_queue_family_index ? compute_queues : transfer_queues; + for (int i=0; i<(int)queues.size(); i++) + { + VkQueue queue = queues[i]; + if (queue) + { + queues[i] = 0; + return queue; + } + } + + // out of hardware queue + return 0; +} + +void VulkanDevice::reclaim_queue(uint32_t queue_family_index, VkQueue queue) const +{ + if (queue_family_index != info.compute_queue_family_index && queue_family_index != info.transfer_queue_family_index) + { + fprintf(stderr, "invalid queue_family_index %u\n", queue_family_index); + return; + } + + MutexLockGuard lock(queue_lock); + + std::vector& queues = queue_family_index == info.compute_queue_family_index ? compute_queues : transfer_queues; + for (int i=0; i<(int)queues.size(); i++) + { + if (!queues[i]) + { + queues[i] = queue; + return; + } + } + + fprintf(stderr, "FATAL ERROR! reclaim_queue get wild queue %u %p\n", queue_family_index, queue); +} + VkAllocator* VulkanDevice::allocator() const { return blob_buffer_allocator; diff --git a/src/gpu.h b/src/gpu.h index 8bbf13613..3969d5507 100644 --- a/src/gpu.h +++ b/src/gpu.h @@ -73,8 +73,11 @@ public: size_t buffer_offset_alignment; // runtime - uint32_t compute_queue_index; - uint32_t transfer_queue_index; + uint32_t compute_queue_family_index; + uint32_t transfer_queue_family_index; + + uint32_t compute_queue_count; + uint32_t transfer_queue_count; uint32_t unified_memory_index; uint32_t device_local_memory_index; @@ -116,6 +119,9 @@ public: VkShaderModule compile_shader_module(const uint32_t* spv_data, size_t spv_data_size) const; + VkQueue acquire_queue(uint32_t queue_family_index) const; + void reclaim_queue(uint32_t queue_family_index, VkQueue queue) const; + // create allocator on this device VkAllocator* allocator() const; VkAllocator* staging_allocator() const; @@ -146,6 +152,11 @@ private: VkDevice device; std::vector shader_modules; + // hardware queue + mutable std::vector compute_queues; + mutable std::vector transfer_queues; + mutable Mutex queue_lock; + // default locked allocator VkAllocator* blob_buffer_allocator; VkAllocator* staging_buffer_allocator; diff --git a/src/net.cpp b/src/net.cpp index 591eaaaf4..6f5ad7421 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -1057,9 +1057,7 @@ int Net::upload_model() } } - cmd.submit(); - - cmd.wait(); + cmd.submit_and_wait(); return 0; } @@ -1629,9 +1627,7 @@ int Net::forward_layer(int layer_index, std::vector& blob_mats, std::vector bottom_blob_unpacked_fp32.prepare_staging_buffer(); cmd.record_download(bottom_blob_unpacked_fp32); - cmd.submit(); - - cmd.wait(); + cmd.submit_and_wait(); cmd.reset(); @@ -1742,9 +1738,7 @@ int Net::forward_layer(int layer_index, std::vector& blob_mats, std::vector } { - cmd.submit(); - - cmd.wait(); + cmd.submit_and_wait(); cmd.reset(); } @@ -1960,9 +1954,7 @@ int Extractor::extract(int blob_index, Mat& feat) feat_gpu_unpacked_fp32.prepare_staging_buffer(); cmd.record_download(feat_gpu_unpacked_fp32); - cmd.submit(); - - cmd.wait(); + cmd.submit_and_wait(); Mat& feat_cpu = blob_mats[blob_index]; feat_cpu.create_like(feat_gpu_unpacked_fp32, opt.blob_allocator);