From d753fe25898ab1f823749c67868c8a0e315139ff Mon Sep 17 00:00:00 2001 From: nihui Date: Sat, 23 Mar 2019 10:34:55 +0800 Subject: [PATCH] upload fp16 weight, enable fp16 storage and arithmetic --- src/CMakeLists.txt | 4 -- src/command.cpp | 10 ++++ src/net.cpp | 133 ++++++++++++++++++++++++++++++++++++++------- src/net.h | 2 + src/pipeline.cpp | 2 - 5 files changed, 125 insertions(+), 26 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index d20df30f5..148fd11e9 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -105,10 +105,6 @@ macro(ncnn_add_layer class) list(APPEND SHADER_SPV_HEX_FILES ${SHADER_SPV_HEX_FILE}) - if("${name}" STREQUAL "cast") - continue() - endif() - # fp16 storage set(SHADER_fp16s_SRC_NAME_WE "${SHADER_SRC_NAME_WE}_fp16s") diff --git a/src/command.cpp b/src/command.cpp index e183383a8..e6bdc2de9 100644 --- a/src/command.cpp +++ b/src/command.cpp @@ -737,6 +737,16 @@ VkTransfer::~VkTransfer() void VkTransfer::record_upload(const Mat& src, VkMat& dst) { + if (vkdev->info.support_fp16_storage && src.elemsize / src.packing == 4) + { + Mat src_fp16; + cast_float32_to_float16(src, src_fp16); + + record_upload(src_fp16, dst); + + return; + } + dst.create_like(src, weight_vkallocator, staging_vkallocator); // set weight blob as readonly diff --git a/src/net.cpp b/src/net.cpp index 2553d98bb..4461e3d8d 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -51,6 +51,8 @@ Net::Net() weight_vkallocator = 0; weight_staging_vkallocator = 0; + cast_float32_to_float16 = 0; + cast_float16_to_float32 = 0; packing_pack1 = 0; packing_pack4 = 0; #endif // NCNN_VULKAN @@ -63,6 +65,8 @@ Net::~Net() #if NCNN_VULKAN delete vkdev_local; + delete cast_float32_to_float16; + delete cast_float16_to_float32; delete packing_pack1; delete packing_pack4; #endif // NCNN_VULKAN @@ -1050,6 +1054,34 @@ int Net::create_pipeline() } } + { + cast_float32_to_float16 = ncnn::create_layer(ncnn::LayerType::Cast); + cast_float32_to_float16->vkdev = vkdev; + + ncnn::ParamDict pd; + pd.set(0, 1); + pd.set(1, 2); + pd.use_vulkan_compute = 1; + + cast_float32_to_float16->load_param(pd); + } + + { + cast_float16_to_float32 = ncnn::create_layer(ncnn::LayerType::Cast); + cast_float16_to_float32->vkdev = vkdev; + + ncnn::ParamDict pd; + pd.set(0, 2); + pd.set(1, 1); + pd.use_vulkan_compute = 1; + + cast_float16_to_float32->load_param(pd); + } + + cast_float32_to_float16->create_pipeline(); + + cast_float16_to_float32->create_pipeline(); + { packing_pack1 = ncnn::create_layer(ncnn::LayerType::Packing); packing_pack1->vkdev = vkdev; @@ -1090,6 +1122,12 @@ int Net::destroy_pipeline() } } + if (cast_float32_to_float16) + cast_float32_to_float16->destroy_pipeline(); + + if (cast_float16_to_float32) + cast_float16_to_float32->destroy_pipeline(); + if (packing_pack1) packing_pack1->destroy_pipeline(); @@ -1357,9 +1395,20 @@ int Net::forward_layer(int layer_index, std::vector& blob_mats, std::vector cmd.record_upload(bottom_blob_unpacked); + // cast to fp16 + VkMat bottom_blob_unpacked_fp16; + if (vkdev->info.support_fp16_storage) + { + cast_float32_to_float16->forward(bottom_blob_unpacked, bottom_blob_unpacked_fp16, cmd, opt); + } + else + { + bottom_blob_unpacked_fp16 = bottom_blob_unpacked; + } + // packing VkMat& bottom_blob = blob_mats_gpu[bottom_blob_index]; - packing_pack4->forward(bottom_blob_unpacked, bottom_blob, cmd, opt); + packing_pack4->forward(bottom_blob_unpacked_fp16, bottom_blob, cmd, opt); // fprintf(stderr, "upload %d %d %d %d %lu %d\n", bottom_blob.total() * bottom_blob.elemsize, bottom_blob.w, bottom_blob.h, bottom_blob.c, bottom_blob.elemsize, bottom_blob.packing); } @@ -1458,9 +1507,20 @@ int Net::forward_layer(int layer_index, std::vector& blob_mats, std::vector cmd.record_upload(bottom_blob_unpacked); + // cast to fp16 + VkMat bottom_blob_unpacked_fp16; + if (vkdev->info.support_fp16_storage) + { + cast_float32_to_float16->forward(bottom_blob_unpacked, bottom_blob_unpacked_fp16, cmd, opt); + } + else + { + bottom_blob_unpacked_fp16 = bottom_blob_unpacked; + } + // packing VkMat& bottom_blob = blob_mats_gpu[bottom_blob_index]; - packing_pack4->forward(bottom_blob_unpacked, bottom_blob, cmd, opt); + packing_pack4->forward(bottom_blob_unpacked_fp16, bottom_blob, cmd, opt); // fprintf(stderr, "upload %d %d %d %d %lu %d\n", bottom_blob.total() * bottom_blob.elemsize, bottom_blob.w, bottom_blob.h, bottom_blob.c, bottom_blob.elemsize, bottom_blob.packing); } @@ -1573,9 +1633,20 @@ int Net::forward_layer(int layer_index, std::vector& blob_mats, std::vector VkMat bottom_blob_unpacked; packing_pack1->forward(bottom_blob, bottom_blob_unpacked, cmd, opt); + // cast to fp32 + VkMat bottom_blob_unpacked_fp32; + if (vkdev->info.support_fp16_storage) + { + cast_float16_to_float32->forward(bottom_blob_unpacked, bottom_blob_unpacked_fp32, cmd, opt); + } + else + { + bottom_blob_unpacked_fp32 = bottom_blob_unpacked; + } + // download - bottom_blob_unpacked.prepare_staging_buffer(); - cmd.record_download(bottom_blob_unpacked); + bottom_blob_unpacked_fp32.prepare_staging_buffer(); + cmd.record_download(bottom_blob_unpacked_fp32); cmd.submit(); @@ -1584,10 +1655,10 @@ int Net::forward_layer(int layer_index, std::vector& blob_mats, std::vector cmd.reset(); Mat& bottom_blob_cpu = blob_mats[bottom_blob_index]; - bottom_blob_cpu.create_like(bottom_blob_unpacked, opt.blob_allocator); - bottom_blob_unpacked.download(bottom_blob_cpu); + bottom_blob_cpu.create_like(bottom_blob_unpacked_fp32, opt.blob_allocator); + bottom_blob_unpacked_fp32.download(bottom_blob_cpu); - bottom_blob_unpacked.discard_staging_buffer(); + bottom_blob_unpacked_fp32.discard_staging_buffer(); // fprintf(stderr, "download %d %d %d %d %lu %d\n", bottom_blob.total() * bottom_blob.elemsize, bottom_blob.w, bottom_blob.h, bottom_blob.c, bottom_blob.elemsize, bottom_blob.packing); } @@ -1634,8 +1705,8 @@ int Net::forward_layer(int layer_index, std::vector& blob_mats, std::vector else { // load bottom blobs - std::vector bottom_blobs_unpacked; - bottom_blobs_unpacked.resize(layer->bottoms.size()); + std::vector bottom_blobs_unpacked_fp32; + bottom_blobs_unpacked_fp32.resize(layer->bottoms.size()); for (size_t i=0; ibottoms.size(); i++) { int bottom_blob_index = layer->bottoms[i]; @@ -1654,11 +1725,22 @@ int Net::forward_layer(int layer_index, std::vector& blob_mats, std::vector const VkMat& bottom_blob = blob_mats_gpu[bottom_blob_index]; // unpacking - packing_pack1->forward(bottom_blob, bottom_blobs_unpacked[i], cmd, opt); + VkMat bottom_blob_unpacked; + packing_pack1->forward(bottom_blob, bottom_blob_unpacked, cmd, opt); + + // cast to fp32 + if (vkdev->info.support_fp16_storage) + { + cast_float16_to_float32->forward(bottom_blob_unpacked, bottom_blobs_unpacked_fp32[i], cmd, opt); + } + else + { + bottom_blobs_unpacked_fp32[i] = bottom_blob_unpacked; + } // download - bottom_blobs_unpacked[i].prepare_staging_buffer(); - cmd.record_download(bottom_blobs_unpacked[i]); + bottom_blobs_unpacked_fp32[i].prepare_staging_buffer(); + cmd.record_download(bottom_blobs_unpacked_fp32[i]); } } } @@ -1680,10 +1762,10 @@ int Net::forward_layer(int layer_index, std::vector& blob_mats, std::vector if (blob_mats[bottom_blob_index].dims == 0) { Mat& bottom_blob_cpu = blob_mats[bottom_blob_index]; - bottom_blob_cpu.create_like(bottom_blobs_unpacked[i], opt.blob_allocator); - bottom_blobs_unpacked[i].download(bottom_blob_cpu); + bottom_blob_cpu.create_like(bottom_blobs_unpacked_fp32[i], opt.blob_allocator); + bottom_blobs_unpacked_fp32[i].download(bottom_blob_cpu); - bottom_blobs_unpacked[i].discard_staging_buffer(); + bottom_blobs_unpacked_fp32[i].discard_staging_buffer(); // fprintf(stderr, "download %d %d %d %d %lu %d\n", bottom_blob.total() * bottom_blob.elemsize, bottom_blob.w, bottom_blob.h, bottom_blob.c, bottom_blob.elemsize, bottom_blob.packing); } @@ -1871,19 +1953,30 @@ int Extractor::extract(int blob_index, Mat& feat) VkMat feat_gpu_unpacked; net->packing_pack1->forward(feat_gpu, feat_gpu_unpacked, cmd, opt); + // cast to fp32 + VkMat feat_gpu_unpacked_fp32; + if (net->vkdev->info.support_fp16_storage) + { + net->cast_float16_to_float32->forward(feat_gpu_unpacked, feat_gpu_unpacked_fp32, cmd, opt); + } + else + { + feat_gpu_unpacked_fp32 = feat_gpu_unpacked; + } + // download - feat_gpu_unpacked.prepare_staging_buffer(); - cmd.record_download(feat_gpu_unpacked); + feat_gpu_unpacked_fp32.prepare_staging_buffer(); + cmd.record_download(feat_gpu_unpacked_fp32); cmd.submit(); cmd.wait(); Mat& feat_cpu = blob_mats[blob_index]; - feat_cpu.create_like(feat_gpu_unpacked, opt.blob_allocator); - feat_gpu_unpacked.download(feat_cpu); + feat_cpu.create_like(feat_gpu_unpacked_fp32, opt.blob_allocator); + feat_gpu_unpacked_fp32.download(feat_cpu); - feat_gpu_unpacked.discard_staging_buffer(); + feat_gpu_unpacked_fp32.discard_staging_buffer(); } } else diff --git a/src/net.h b/src/net.h index 6f53de9ed..fb29bb0c3 100644 --- a/src/net.h +++ b/src/net.h @@ -152,6 +152,8 @@ protected: VkAllocator* weight_vkallocator; VkAllocator* weight_staging_vkallocator; + ncnn::Layer* cast_float32_to_float16; + ncnn::Layer* cast_float16_to_float32; ncnn::Layer* packing_pack1; ncnn::Layer* packing_pack4; #endif // NCNN_VULKAN diff --git a/src/pipeline.cpp b/src/pipeline.cpp index 33d0c2197..76d13b0f9 100644 --- a/src/pipeline.cpp +++ b/src/pipeline.cpp @@ -259,7 +259,6 @@ int Pipeline::create_pipeline(const char* _name, const std::vectorinfo.support_fp16_arithmetic) { name += "_fp16a"; @@ -268,7 +267,6 @@ int Pipeline::create_pipeline(const char* _name, const std::vectorget_shader_module(name.c_str());