From 738fb6bb14374ba77c3db0ee27cf43d3f13a6b53 Mon Sep 17 00:00:00 2001 From: nihuini Date: Wed, 22 May 2019 11:26:28 +0800 Subject: [PATCH] print gpu per layer benchmark --- src/command.cpp | 104 ++++++++++++++++++++++++++++++++++++++++++++++++ src/command.h | 24 +++++++++++ src/net.cpp | 102 ++++++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 228 insertions(+), 2 deletions(-) diff --git a/src/command.cpp b/src/command.cpp index 89a4a0b87..cb2fd2f7f 100644 --- a/src/command.cpp +++ b/src/command.cpp @@ -168,6 +168,11 @@ int Command::queue_submit_and_wait_fence() VkCompute::VkCompute(const VulkanDevice* _vkdev) : Command(_vkdev, _vkdev->info.compute_queue_family_index) { +#if NCNN_BENCHMARK + query_count = 0; + query_pool = 0; +#endif // NCNN_BENCHMARK + if (vkdev->info.support_VK_KHR_push_descriptor) { begin_command_buffer(); @@ -184,6 +189,13 @@ VkCompute::~VkCompute() vkDestroyDescriptorPool(vkdev->vkdevice(), descriptor_pools[i], 0); } } + +#if NCNN_BENCHMARK + if (query_pool) + { + vkDestroyQueryPool(vkdev->vkdevice(), query_pool, 0); + } +#endif // NCNN_BENCHMARK } void VkCompute::record_upload(const VkMat& m) @@ -294,6 +306,19 @@ void VkCompute::record_pipeline(const Pipeline* pipeline, const std::vectorinfo.support_VK_KHR_push_descriptor) + return write_timestamp(query); + + record_type r; + r.type = 10; + r.write_timestamp.query = query; + delayed_records.push_back(r); +} +#endif // NCNN_BENCHMARK + void VkCompute::record_bind_pipeline(VkPipeline pipeline) { if (vkdev->info.support_VK_KHR_push_descriptor) @@ -516,6 +541,10 @@ int VkCompute::submit_and_wait() begin_command_buffer(); +#if NCNN_BENCHMARK + reset_query_pool(); +#endif // NCNN_BENCHMARK + // handle delayed records for (size_t i=0; iinfo.support_VK_KHR_push_descriptor) { begin_command_buffer(); + +#if NCNN_BENCHMARK + reset_query_pool(); +#endif // NCNN_BENCHMARK } return 0; } +#if NCNN_BENCHMARK +int VkCompute::create_query_pool(uint32_t _query_count) +{ + query_count = _query_count; + + VkQueryPoolCreateInfo queryPoolCreateInfo; + queryPoolCreateInfo.sType = VK_STRUCTURE_TYPE_QUERY_POOL_CREATE_INFO; + queryPoolCreateInfo.pNext = 0; + queryPoolCreateInfo.flags = 0; + queryPoolCreateInfo.queryType = VK_QUERY_TYPE_TIMESTAMP; + queryPoolCreateInfo.queryCount = query_count; + queryPoolCreateInfo.pipelineStatistics = 0; + + VkResult ret = vkCreateQueryPool(vkdev->vkdevice(), &queryPoolCreateInfo, 0, &query_pool); + if (ret != VK_SUCCESS) + { + fprintf(stderr, "vkCreateQueryPool failed %d\n", ret); + return -1; + } + + if (vkdev->info.support_VK_KHR_push_descriptor) + { + reset_query_pool(); + } + + return 0; +} + +int VkCompute::get_query_pool_results(uint32_t first_query, uint32_t query_count, std::vector& results) +{ + if (results.size() < first_query + query_count) + { + fprintf(stderr, "results not large enough\n"); + return -1; + } + + VkResult ret = vkGetQueryPoolResults(vkdev->vkdevice(), query_pool, first_query, query_count, + query_count * sizeof(uint64_t), results.data() + first_query, sizeof(uint64_t), VK_QUERY_RESULT_64_BIT | VK_QUERY_RESULT_WITH_AVAILABILITY_BIT); + if (ret != VK_SUCCESS && ret != VK_NOT_READY) + { + fprintf(stderr, "vkGetQueryPoolResults failed %d\n", ret); + return -1; + } + + return 0; +} +#endif // NCNN_BENCHMARK + void VkCompute::copy_buffer(VkBuffer src, size_t src_offset, VkBuffer dst, size_t dst_offset, size_t size) { // fprintf(stderr, "cmd copy %p to %p\n", src, dst); @@ -727,6 +813,24 @@ void VkCompute::transfer_transfer_barrier(VkBuffer buffer, size_t offset, size_t vkCmdPipelineBarrier(command_buffer, srcStageMask, dstStageMask, 0, 0, 0, 1, &bufferBarrier, 0, 0); } +#if NCNN_BENCHMARK +void VkCompute::reset_query_pool() +{ +// fprintf(stderr, "cmd reset_query_pool\n"); + + if (query_pool) + vkCmdResetQueryPool(command_buffer, query_pool, 0, query_count); +} + +void VkCompute::write_timestamp(uint32_t query) +{ +// fprintf(stderr, "cmd write_timestamp %u\n", query); + + if (query_pool) + vkCmdWriteTimestamp(command_buffer, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, query_pool, query); +} +#endif // NCNN_BENCHMARK + VkTransfer::VkTransfer(const VulkanDevice* _vkdev) : Command(_vkdev, _vkdev->info.transfer_queue_family_index) { buffer_offset_alignment = vkdev->info.buffer_offset_alignment; diff --git a/src/command.h b/src/command.h index 30f14ad71..6566cc8ca 100644 --- a/src/command.h +++ b/src/command.h @@ -69,10 +69,18 @@ public: void record_pipeline(const Pipeline* pipeline, const std::vector& bindings, const std::vector& constants, const VkMat& m); + void record_write_timestamp(uint32_t query); + int submit_and_wait(); int reset(); +#if NCNN_BENCHMARK + int create_query_pool(uint32_t query_count); + + int get_query_pool_results(uint32_t first_query, uint32_t query_count, std::vector& results); +#endif // NCNN_BENCHMARK + protected: // record pipeline things void record_bind_pipeline(VkPipeline pipeline); @@ -90,6 +98,10 @@ protected: void record_prepare_transfer_barrier(const VkMat& m); void record_prepare_compute_barrier(const VkMat& m); +#if NCNN_BENCHMARK + void reset_query_pool(); +#endif // NCNN_BENCHMARK + protected: // recording issue void copy_buffer(VkBuffer src, size_t src_offset, VkBuffer dst, size_t dst_offset, size_t size); @@ -103,6 +115,9 @@ protected: void compute_transfer_barrier(VkBuffer buffer, size_t offset, size_t size); void compute_compute_barrier(VkBuffer buffer, size_t offset, size_t size); void transfer_transfer_barrier(VkBuffer buffer, size_t offset, size_t size); +#if NCNN_BENCHMARK + void write_timestamp(uint32_t query); +#endif // NCNN_BENCHMARK protected: // delayed record @@ -121,6 +136,7 @@ protected: // 7=compute-transfer barrier // 8=compute-compute barrier // 9=transfer-transfer barrier + // 10=write timestamp int type; union @@ -135,12 +151,20 @@ protected: struct { VkBuffer buffer; size_t offset; size_t size; } compute_transfer_barrier; struct { VkBuffer buffer; size_t offset; size_t size; } compute_compute_barrier; struct { VkBuffer buffer; size_t offset; size_t size; } transfer_transfer_barrier; +#if NCNN_BENCHMARK + struct { uint32_t query; } write_timestamp; +#endif // NCNN_BENCHMARK }; std::vector regions; std::vector constants; }; std::vector delayed_records; + +#if NCNN_BENCHMARK + uint32_t query_count; + VkQueryPool query_pool; +#endif // NCNN_BENCHMARK }; class VkTransfer : public Command diff --git a/src/net.cpp b/src/net.cpp index 6f5ad7421..1f3c75c56 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -1447,7 +1447,13 @@ int Net::forward_layer(int layer_index, std::vector& blob_mats, std::vector if (opt.lightmode && layer->support_inplace) { VkMat& bottom_top_blob = bottom_blob; +#if NCNN_BENCHMARK + cmd.record_write_timestamp(layer_index * 2); int ret = layer->forward_inplace(bottom_top_blob, cmd, opt); + cmd.record_write_timestamp(layer_index * 2 + 1); +#else + int ret = layer->forward_inplace(bottom_top_blob, cmd, opt); +#endif // NCNN_BENCHMARK if (ret != 0) return ret; @@ -1457,7 +1463,13 @@ int Net::forward_layer(int layer_index, std::vector& blob_mats, std::vector else { VkMat top_blob; +#if NCNN_BENCHMARK + cmd.record_write_timestamp(layer_index * 2); int ret = layer->forward(bottom_blob, top_blob, cmd, opt); + cmd.record_write_timestamp(layer_index * 2 + 1); +#else + int ret = layer->forward(bottom_blob, top_blob, cmd, opt); +#endif // NCNN_BENCHMARK if (ret != 0) return ret; @@ -1538,7 +1550,13 @@ int Net::forward_layer(int layer_index, std::vector& blob_mats, std::vector if (opt.lightmode && layer->support_inplace) { std::vector& bottom_top_blobs = bottom_blobs; +#if NCNN_BENCHMARK + cmd.record_write_timestamp(layer_index * 2); int ret = layer->forward_inplace(bottom_top_blobs, cmd, opt); + cmd.record_write_timestamp(layer_index * 2 + 1); +#else + int ret = layer->forward_inplace(bottom_top_blobs, cmd, opt); +#endif // NCNN_BENCHMARK if (ret != 0) return ret; @@ -1553,7 +1571,13 @@ int Net::forward_layer(int layer_index, std::vector& blob_mats, std::vector else { std::vector top_blobs(layer->tops.size()); +#if NCNN_BENCHMARK + cmd.record_write_timestamp(layer_index * 2); int ret = layer->forward(bottom_blobs, top_blobs, cmd, opt); + cmd.record_write_timestamp(layer_index * 2 + 1); +#else + int ret = layer->forward(bottom_blobs, top_blobs, cmd, opt); +#endif // NCNN_BENCHMARK if (ret != 0) return ret; @@ -1629,6 +1653,21 @@ int Net::forward_layer(int layer_index, std::vector& blob_mats, std::vector cmd.submit_and_wait(); +#if NCNN_BENCHMARK + std::vector results(layer_index * 2); + cmd.get_query_pool_results(0, layer_index * 2, results); + for (int i=0; iinfo.timestamp_period / 1000; + fprintf(stderr, "%-24s %-30s %8.2lfus |\n", layers[i]->type.c_str(), layers[i]->name.c_str(), duration_us); + } +#endif // NCNN_BENCHMARK + cmd.reset(); Mat& bottom_blob_cpu = blob_mats[bottom_blob_index]; @@ -1656,8 +1695,14 @@ int Net::forward_layer(int layer_index, std::vector& blob_mats, std::vector if (opt.lightmode && layer->support_inplace) { Mat& bottom_top_blob = bottom_blob; - +#if NCNN_BENCHMARK + double start = get_current_time(); int ret = layer->forward_inplace(bottom_top_blob, opt); + double end = get_current_time(); + benchmark(layer, bottom_top_blob, bottom_top_blob, start, end); +#else + int ret = layer->forward_inplace(bottom_top_blob, opt); +#endif // NCNN_BENCHMARK if (ret != 0) return ret; @@ -1667,7 +1712,14 @@ int Net::forward_layer(int layer_index, std::vector& blob_mats, std::vector else { Mat top_blob; +#if NCNN_BENCHMARK + double start = get_current_time(); int ret = layer->forward(bottom_blob, top_blob, opt); + double end = get_current_time(); + benchmark(layer, bottom_blob, top_blob, start, end); +#else + int ret = layer->forward(bottom_blob, top_blob, opt); +#endif // NCNN_BENCHMARK if (ret != 0) return ret; @@ -1740,6 +1792,21 @@ int Net::forward_layer(int layer_index, std::vector& blob_mats, std::vector { cmd.submit_and_wait(); +#if NCNN_BENCHMARK + std::vector results(layer_index * 2); + cmd.get_query_pool_results(0, layer_index * 2, results); + for (int i=0; iinfo.timestamp_period / 1000; + fprintf(stderr, "%-24s %-30s %8.2lfus |\n", layers[i]->type.c_str(), layers[i]->name.c_str(), duration_us); + } +#endif // NCNN_BENCHMARK + cmd.reset(); } @@ -1777,8 +1844,14 @@ int Net::forward_layer(int layer_index, std::vector& blob_mats, std::vector if (opt.lightmode && layer->support_inplace) { std::vector& bottom_top_blobs = bottom_blobs; - +#if NCNN_BENCHMARK + double start = get_current_time(); int ret = layer->forward_inplace(bottom_top_blobs, opt); + double end = get_current_time(); + benchmark(layer, start, end); +#else + int ret = layer->forward_inplace(bottom_top_blobs, opt); +#endif // NCNN_BENCHMARK if (ret != 0) return ret; @@ -1793,7 +1866,14 @@ int Net::forward_layer(int layer_index, std::vector& blob_mats, std::vector else { std::vector top_blobs(layer->tops.size()); +#if NCNN_BENCHMARK + double start = get_current_time(); int ret = layer->forward(bottom_blobs, top_blobs, opt); + double end = get_current_time(); + benchmark(layer, start, end); +#else + int ret = layer->forward(bottom_blobs, top_blobs, opt); +#endif // NCNN_BENCHMARK if (ret != 0) return ret; @@ -1929,6 +2009,9 @@ int Extractor::extract(int blob_index, Mat& feat) if (opt.vulkan_compute) { ncnn::VkCompute cmd(net->vkdev); +#if NCNN_BENCHMARK + cmd.create_query_pool(net->layers.size() * 2); +#endif // NCNN_BENCHMARK VkMat feat_gpu; ret = extract(blob_index, feat_gpu, cmd); @@ -1956,6 +2039,21 @@ int Extractor::extract(int blob_index, Mat& feat) cmd.submit_and_wait(); +#if NCNN_BENCHMARK + std::vector results(net->layers.size() * 2); + cmd.get_query_pool_results(0, net->layers.size() * 2, results); + for (int i=0; ilayers.size(); i++) + { + uint64_t start = results[i*2]; + uint64_t end = results[i*2+1]; + if (start == 0 || end == 0) + continue; + + double duration_us = (end - start) * net->vkdev->info.timestamp_period / 1000; + fprintf(stderr, "%-24s %-30s %8.2lfus |\n", net->layers[i]->type.c_str(), net->layers[i]->name.c_str(), duration_us); + } +#endif // NCNN_BENCHMARK + Mat& feat_cpu = blob_mats[blob_index]; feat_cpu.create_like(feat_gpu_unpacked_fp32, opt.blob_allocator); feat_gpu_unpacked_fp32.download(feat_cpu);