From a86c2f44c34309d43e0b62f028c59dc536b4cccf Mon Sep 17 00:00:00 2001 From: nihuini Date: Wed, 25 Dec 2019 14:34:34 +0800 Subject: [PATCH] vkimagemat, vkimageallocator, convenient construct from android hardware buffer --- src/allocator.cpp | 313 ++++++++++++++++++++++++++++++++++++++ src/allocator.h | 73 +++++++++ src/command.cpp | 16 +- src/command.h | 6 +- src/mat.cpp | 27 ++++ src/mat.h | 171 ++++++++++++++++++++- src/mat_pixel_android.cpp | 6 +- src/pipeline.cpp | 130 ++-------------- src/pipeline.h | 6 +- 9 files changed, 610 insertions(+), 138 deletions(-) diff --git a/src/allocator.cpp b/src/allocator.cpp index 0f838f53d..dba4ae08f 100644 --- a/src/allocator.cpp +++ b/src/allocator.cpp @@ -17,6 +17,11 @@ #include #include #include "gpu.h" +#include "pipeline.h" + +#if __ANDROID_API__ >= 26 +#include +#endif // __ANDROID_API__ >= 26 namespace ncnn { @@ -942,6 +947,314 @@ void VkWeightStagingBufferAllocator::fastFree(VkBufferMemory* ptr) delete ptr; } +VkImageAllocator::VkImageAllocator(const VulkanDevice* _vkdev) : VkAllocator(_vkdev) +{ + memory_type_index = (uint32_t)-1; +} + +VkImage VkImageAllocator::create_image(int width, int height, VkFormat format, VkImageUsageFlags usage) +{ + VkImageCreateInfo imageCreateInfo; + imageCreateInfo.sType = VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO, + imageCreateInfo.pNext = 0; + imageCreateInfo.flags = 0; + imageCreateInfo.imageType = VK_IMAGE_TYPE_2D; + imageCreateInfo.format = format; + imageCreateInfo.extent.width = width; + imageCreateInfo.extent.height = height; + imageCreateInfo.extent.depth = 1; + imageCreateInfo.mipLevels = 1; + imageCreateInfo.arrayLayers = 1; + imageCreateInfo.samples = VK_SAMPLE_COUNT_1_BIT; + imageCreateInfo.tiling = VK_IMAGE_TILING_OPTIMAL; + imageCreateInfo.usage = usage; + imageCreateInfo.sharingMode = VK_SHARING_MODE_EXCLUSIVE; + imageCreateInfo.queueFamilyIndexCount = 0; + imageCreateInfo.pQueueFamilyIndices = 0; + imageCreateInfo.initialLayout = VK_IMAGE_LAYOUT_UNDEFINED; + + VkImage image; + VkResult ret = vkCreateImage(vkdev->vkdevice(), &imageCreateInfo, 0, &image); + if (ret != VK_SUCCESS) + { + fprintf(stderr, "vkCreateImage failed %d\n", ret); + return 0; + } + + return image; +} + +VkImageView VkImageAllocator::create_imageview(VkImage image, VkFormat format) +{ + VkImageViewCreateInfo imageViewCreateInfo; + imageViewCreateInfo.sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO; + imageViewCreateInfo.pNext = 0; + imageViewCreateInfo.flags = 0; + imageViewCreateInfo.image = image; + imageViewCreateInfo.viewType = VK_IMAGE_VIEW_TYPE_2D; + imageViewCreateInfo.format = format; + imageViewCreateInfo.components.r = VK_COMPONENT_SWIZZLE_IDENTITY; + imageViewCreateInfo.components.g = VK_COMPONENT_SWIZZLE_IDENTITY; + imageViewCreateInfo.components.b = VK_COMPONENT_SWIZZLE_IDENTITY; + imageViewCreateInfo.components.a = VK_COMPONENT_SWIZZLE_IDENTITY; + imageViewCreateInfo.subresourceRange.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT; + imageViewCreateInfo.subresourceRange.baseMipLevel = 0; + imageViewCreateInfo.subresourceRange.levelCount = 1; + imageViewCreateInfo.subresourceRange.baseArrayLayer = 0; + imageViewCreateInfo.subresourceRange.layerCount = 1; + + VkImageView imageview; + VkResult ret = vkCreateImageView(vkdev->vkdevice(), &imageViewCreateInfo, 0, &imageview); + if (ret != VK_SUCCESS) + { + fprintf(stderr, "vkCreateImageView failed %d\n", ret); + return 0; + } + + return imageview; +} + +VkDeviceMemory VkImageAllocator::allocate_dedicated_memory(size_t size, VkImage image) +{ + VkMemoryAllocateInfo memoryAllocateInfo; + memoryAllocateInfo.sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO; + memoryAllocateInfo.pNext = 0; + memoryAllocateInfo.allocationSize = size; + memoryAllocateInfo.memoryTypeIndex = memory_type_index; + + VkMemoryDedicatedAllocateInfoKHR memoryDedicatedAllocateInfo; + memoryDedicatedAllocateInfo.sType = VK_STRUCTURE_TYPE_MEMORY_DEDICATED_ALLOCATE_INFO_KHR; + memoryDedicatedAllocateInfo.pNext = 0; + memoryDedicatedAllocateInfo.image = image; + memoryDedicatedAllocateInfo.buffer = 0; + memoryAllocateInfo.pNext = &memoryDedicatedAllocateInfo; + + VkDeviceMemory memory = 0; + VkResult ret = vkAllocateMemory(vkdev->vkdevice(), &memoryAllocateInfo, 0, &memory); + if (ret != VK_SUCCESS) + { + fprintf(stderr, "vkAllocateMemory failed %d\n", ret); + } + + return memory; +} + +VkSimpleImageAllocator::VkSimpleImageAllocator(const VulkanDevice* _vkdev) : VkImageAllocator(_vkdev) +{ +} + +VkSimpleImageAllocator::~VkSimpleImageAllocator() +{ +} + +VkImageMemory* VkSimpleImageAllocator::fastMalloc(int width, int height, VkFormat format) +{ + VkImageMemory* ptr = new VkImageMemory; + + ptr->image = create_image(width, height, format, VK_IMAGE_USAGE_STORAGE_BIT); + + VkMemoryRequirements memoryRequirements; + vkGetImageMemoryRequirements(vkdev->vkdevice(), ptr->image, &memoryRequirements); + + // setup memory type + if (memory_type_index == (uint32_t)-1) + { + if (vkdev->info.type == 1) + { + // integrated gpu, prefer unified memory + memory_type_index = vkdev->find_memory_index(memoryRequirements.memoryTypeBits, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT, VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT, 0); + } + else + { + // discrete gpu, device local + memory_type_index = vkdev->find_memory_index(memoryRequirements.memoryTypeBits, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT, 0, VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT); + } + } + + ptr->memory = allocate_memory(memoryRequirements.size); + + vkBindImageMemory(vkdev->vkdevice(), ptr->image, ptr->memory, 0); + + ptr->imageview = create_imageview(ptr->image, format); + + ptr->state = 1; + + return ptr; +} + +void VkSimpleImageAllocator::fastFree(VkImageMemory* ptr) +{ + vkDestroyImageView(vkdev->vkdevice(), ptr->imageview, 0); + vkDestroyImage(vkdev->vkdevice(), ptr->image, 0); + vkFreeMemory(vkdev->vkdevice(), ptr->memory, 0); + + delete ptr; +} + +#if __ANDROID_API__ >= 26 +VkAndroidHardwareBufferImageAllocator::VkAndroidHardwareBufferImageAllocator(const VulkanDevice* _vkdev, const ImportAndroidHardwareBufferPipeline* p) : VkImageAllocator(_vkdev), q(p) +{ +} + +VkAndroidHardwareBufferImageAllocator::~VkAndroidHardwareBufferImageAllocator() +{ +} + +VkImageMemory* VkAndroidHardwareBufferImageAllocator::fastMalloc(AHardwareBuffer* hb) +{ + VkResult ret; + + AHardwareBuffer_Desc bufferDesc; + AHardwareBuffer_describe(hb, &bufferDesc); + + VkAndroidHardwareBufferFormatPropertiesANDROID bufferFormatProperties; + bufferFormatProperties.sType = VK_STRUCTURE_TYPE_ANDROID_HARDWARE_BUFFER_FORMAT_PROPERTIES_ANDROID; + bufferFormatProperties.pNext = 0; + + VkAndroidHardwareBufferPropertiesANDROID bufferProperties; + bufferProperties.sType = VK_STRUCTURE_TYPE_ANDROID_HARDWARE_BUFFER_PROPERTIES_ANDROID; + bufferProperties.pNext = &bufferFormatProperties; + + ret = vkGetAndroidHardwareBufferPropertiesANDROID(vkdev->vkdevice(), hb, &bufferProperties); + if (ret != VK_SUCCESS) + { + fprintf(stderr, "vkGetAndroidHardwareBufferPropertiesANDROID failed %d\n", ret); + return 0; + } + + VkExternalFormatANDROID externalFormat; + externalFormat.sType = VK_STRUCTURE_TYPE_EXTERNAL_FORMAT_ANDROID; + externalFormat.pNext = 0; + externalFormat.externalFormat = bufferFormatProperties.externalFormat; + + VkExternalMemoryImageCreateInfo externalMemoryImageCreateInfo; + externalMemoryImageCreateInfo.sType = VK_STRUCTURE_TYPE_EXTERNAL_MEMORY_IMAGE_CREATE_INFO, + externalMemoryImageCreateInfo.pNext = &externalFormat, + externalMemoryImageCreateInfo.handleTypes = VK_EXTERNAL_MEMORY_HANDLE_TYPE_ANDROID_HARDWARE_BUFFER_BIT_ANDROID; + + VkImageCreateInfo imageCreateInfo; + imageCreateInfo.sType = VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO, + imageCreateInfo.pNext = &externalMemoryImageCreateInfo; + imageCreateInfo.flags = 0; + imageCreateInfo.imageType = VK_IMAGE_TYPE_2D; + imageCreateInfo.format = VK_FORMAT_UNDEFINED; + imageCreateInfo.extent.width = bufferDesc.width; + imageCreateInfo.extent.height = bufferDesc.height; + imageCreateInfo.extent.depth = 1; + imageCreateInfo.mipLevels = 1; + imageCreateInfo.arrayLayers = 1; + imageCreateInfo.samples = VK_SAMPLE_COUNT_1_BIT; + imageCreateInfo.tiling = VK_IMAGE_TILING_OPTIMAL; + imageCreateInfo.usage = VK_IMAGE_USAGE_SAMPLED_BIT; + imageCreateInfo.sharingMode = VK_SHARING_MODE_EXCLUSIVE; + imageCreateInfo.queueFamilyIndexCount = 0; + imageCreateInfo.pQueueFamilyIndices = 0; + imageCreateInfo.initialLayout = VK_IMAGE_LAYOUT_UNDEFINED; + + VkImage image = 0; + ret = vkCreateImage(vkdev->vkdevice(), &imageCreateInfo, 0, &image); + if (ret != VK_SUCCESS) + { + fprintf(stderr, "vkCreateImage failed %d\n", ret); + return 0; + } + + // setup memory type + if (memory_type_index == (uint32_t)-1) + { + memory_type_index = vkdev->find_memory_index(bufferProperties.memoryTypeBits, 0, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT, VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT); + } + + VkImportAndroidHardwareBufferInfoANDROID importAndroidHardwareBufferInfo; + importAndroidHardwareBufferInfo.sType = VK_STRUCTURE_TYPE_IMPORT_ANDROID_HARDWARE_BUFFER_INFO_ANDROID; + importAndroidHardwareBufferInfo.pNext = 0; + importAndroidHardwareBufferInfo.buffer = hb; + + VkMemoryDedicatedAllocateInfo memoryDedicatedAllocateInfo; + memoryDedicatedAllocateInfo.sType = VK_STRUCTURE_TYPE_MEMORY_DEDICATED_ALLOCATE_INFO; + memoryDedicatedAllocateInfo.pNext = &importAndroidHardwareBufferInfo; + memoryDedicatedAllocateInfo.image = image; + memoryDedicatedAllocateInfo.buffer = VK_NULL_HANDLE; + + VkMemoryAllocateInfo memoryAllocateInfo; + memoryAllocateInfo.sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO; + memoryAllocateInfo.pNext = &memoryDedicatedAllocateInfo; + memoryAllocateInfo.allocationSize = bufferProperties.allocationSize; + memoryAllocateInfo.memoryTypeIndex = memory_type_index; + + VkDeviceMemory memory = 0; + ret = vkAllocateMemory(vkdev->vkdevice(), &memoryAllocateInfo, 0, &memory); + if (ret != VK_SUCCESS) + { + fprintf(stderr, "vkAllocateMemory failed %d\n", ret); + return 0; + } + + VkBindImageMemoryInfo bindImageMemoryInfo; + bindImageMemoryInfo.sType = VK_STRUCTURE_TYPE_BIND_IMAGE_MEMORY_INFO; + bindImageMemoryInfo.pNext = 0; + bindImageMemoryInfo.image = image; + bindImageMemoryInfo.memory = memory; + bindImageMemoryInfo.memoryOffset = 0; + ret = vkdev->vkBindImageMemory2KHR(vkdev->vkdevice(), 1, &bindImageMemoryInfo); + if (ret != VK_SUCCESS) + { + fprintf(stderr, "vkBindImageMemory2KHR failed %d\n", ret); + vkDestroyImage(vkdev->vkdevice(), image, 0); + return 0; + } + + VkSamplerYcbcrConversionInfoKHR samplerYcbcrConversionInfo; + samplerYcbcrConversionInfo.sType = VK_STRUCTURE_TYPE_SAMPLER_YCBCR_CONVERSION_INFO_KHR; + samplerYcbcrConversionInfo.pNext = &externalFormat; + samplerYcbcrConversionInfo.conversion = q->samplerYcbcrConversion; + + VkImageViewCreateInfo imageViewCreateInfo; + imageViewCreateInfo.sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO; + imageViewCreateInfo.pNext = &samplerYcbcrConversionInfo; + imageViewCreateInfo.flags = 0; + imageViewCreateInfo.image = image; + imageViewCreateInfo.viewType = VK_IMAGE_VIEW_TYPE_2D; + imageViewCreateInfo.format = VK_FORMAT_UNDEFINED; + imageViewCreateInfo.components.r = VK_COMPONENT_SWIZZLE_IDENTITY; + imageViewCreateInfo.components.g = VK_COMPONENT_SWIZZLE_IDENTITY; + imageViewCreateInfo.components.b = VK_COMPONENT_SWIZZLE_IDENTITY; + imageViewCreateInfo.components.a = VK_COMPONENT_SWIZZLE_IDENTITY; + imageViewCreateInfo.subresourceRange.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT; + imageViewCreateInfo.subresourceRange.baseMipLevel = 0; + imageViewCreateInfo.subresourceRange.levelCount = 1; + imageViewCreateInfo.subresourceRange.baseArrayLayer = 0; + imageViewCreateInfo.subresourceRange.layerCount = 1; + + VkImageView imageview = 0; + ret = vkCreateImageView(vkdev->vkdevice(), &imageViewCreateInfo, 0, &imageview); + if (ret != VK_SUCCESS) + { + fprintf(stderr, "vkCreateImageView failed %d\n", ret); + vkDestroyImage(vkdev->vkdevice(), image, 0); + vkFreeMemory(vkdev->vkdevice(), memory, 0); + return 0; + } + + VkImageMemory* ptr = new VkImageMemory; + ptr->image = image; + ptr->memory = memory; + ptr->imageview = imageview; + ptr->state = 1; + + return ptr; +} + +void VkAndroidHardwareBufferImageAllocator::fastFree(VkImageMemory* ptr) +{ + vkDestroyImageView(vkdev->vkdevice(), ptr->imageview, 0); + vkDestroyImage(vkdev->vkdevice(), ptr->image, 0); + vkFreeMemory(vkdev->vkdevice(), ptr->memory, 0); + + delete ptr; +} +#endif // __ANDROID_API__ >= 26 + #endif // NCNN_VULKAN } // namespace ncnn diff --git a/src/allocator.h b/src/allocator.h index e4d4b16db..22ebffc63 100644 --- a/src/allocator.h +++ b/src/allocator.h @@ -32,6 +32,8 @@ #include "gpu.h" #endif // NCNN_VULKAN +struct AHardwareBuffer; + namespace ncnn { // the alignment of all the allocated buffers @@ -296,6 +298,77 @@ public: private: }; +class VkImageMemory +{ +public: + VkImage image; + VkImageView imageview; + + VkDeviceMemory memory; + + // buffer state, modified by command functions internally + // 0=null + // 1=created + // 2=transfer + // 3=compute + // 4=readonly + mutable int state; + + // initialize and modified by mat + int refcount; +}; + +class VkImageAllocator : public VkAllocator +{ +public: + VkImageAllocator(const VulkanDevice* _vkdev); + virtual ~VkImageAllocator() { clear(); } + virtual void clear() {} + virtual VkImageMemory* fastMalloc(int width, int height, VkFormat format) = 0; + virtual void fastFree(VkImageMemory* ptr) = 0; + +protected: + virtual VkBufferMemory* fastMalloc(size_t /*size*/) { return 0; } + virtual void fastFree(VkBufferMemory* /*ptr*/) {} + +protected: + VkImage create_image(int width, int height, VkFormat format, VkImageUsageFlags usage); + VkImageView create_imageview(VkImage image, VkFormat format); + VkDeviceMemory allocate_dedicated_memory(size_t size, VkImage image); +}; + +class VkSimpleImageAllocator : public VkImageAllocator +{ +public: + VkSimpleImageAllocator(const VulkanDevice* vkdev); + virtual ~VkSimpleImageAllocator(); + +public: + virtual VkImageMemory* fastMalloc(int width, int height, VkFormat format); + virtual void fastFree(VkImageMemory* ptr); +}; + +#if __ANDROID_API__ >= 26 +class ImportAndroidHardwareBufferPipeline; +class VkAndroidHardwareBufferImageAllocator : public VkImageAllocator +{ +public: + VkAndroidHardwareBufferImageAllocator(const VulkanDevice* vkdev, const ImportAndroidHardwareBufferPipeline* p); + virtual ~VkAndroidHardwareBufferImageAllocator(); + +public: + virtual VkImageMemory* fastMalloc(AHardwareBuffer* hb); + virtual void fastFree(VkImageMemory* ptr); + +protected: + virtual VkImageMemory* fastMalloc(int /*width*/, int /*height*/, VkFormat /*format*/) { return 0; } + virtual VkBufferMemory* fastMalloc(size_t /*size*/) { return 0; } + +private: + const ImportAndroidHardwareBufferPipeline* const q; +}; +#endif // __ANDROID_API__ >= 26 + #endif // NCNN_VULKAN } // namespace ncnn diff --git a/src/command.cpp b/src/command.cpp index 6ae394f2d..afae31d5a 100644 --- a/src/command.cpp +++ b/src/command.cpp @@ -324,13 +324,13 @@ void VkCompute::record_write_timestamp(uint32_t query) #endif // NCNN_BENCHMARK #if __ANDROID_API__ >= 26 -void VkCompute::record_import_android_hardware_buffer(const ImportAndroidHardwareBufferPipeline* pipeline, VkImage image, VkImageView imageView, const VkMat& m) +void VkCompute::record_import_android_hardware_buffer(const ImportAndroidHardwareBufferPipeline* pipeline, const VkImageMat& im, const VkMat& m) { - record_initial_image_compute_barrier(image); + record_initial_image_compute_barrier(im); record_bind_pipeline(pipeline->pipeline); - record_update_import_android_hardware_buffer_bindings(pipeline->pipeline_layout, pipeline->descriptorset_layout, pipeline->descriptor_update_template, pipeline->sampler, imageView, m); + record_update_import_android_hardware_buffer_bindings(pipeline->pipeline_layout, pipeline->descriptorset_layout, pipeline->descriptor_update_template, pipeline->sampler, im, m); uint32_t group_count_xyz[3]; group_count_xyz[0] = (m.w + 7) / 8; @@ -552,23 +552,23 @@ void VkCompute::record_prepare_compute_barrier(const VkMat& m) m.data->state = 3; } -void VkCompute::record_initial_image_compute_barrier(VkImage image) +void VkCompute::record_initial_image_compute_barrier(const VkImageMat& im) { if (vkdev->info.support_VK_KHR_push_descriptor) - return initial_image_compute_barrier(image); + return initial_image_compute_barrier(im.image()); record_type r; r.type = 11; - r.initial_image_compute_barrier.image = image; + r.initial_image_compute_barrier.image = im.image(); delayed_records.push_back(r); } #if __ANDROID_API__ >= 26 -void VkCompute::record_update_import_android_hardware_buffer_bindings(VkPipelineLayout pipeline_layout, VkDescriptorSetLayout descriptorset_layout, VkDescriptorUpdateTemplateKHR descriptor_update_template, VkSampler sampler, VkImageView imageView, const VkMat& m) +void VkCompute::record_update_import_android_hardware_buffer_bindings(VkPipelineLayout pipeline_layout, VkDescriptorSetLayout descriptorset_layout, VkDescriptorUpdateTemplateKHR descriptor_update_template, VkSampler sampler, const VkImageMat& im, const VkMat& m) { VkDescriptorImageInfo descriptorImageInfo; descriptorImageInfo.sampler = sampler; - descriptorImageInfo.imageView = imageView; + descriptorImageInfo.imageView = im.imageview(); descriptorImageInfo.imageLayout = VK_IMAGE_LAYOUT_GENERAL; VkDescriptorBufferInfo descriptorBufferInfo; diff --git a/src/command.h b/src/command.h index 345bd5e2a..cdcf6f3c8 100644 --- a/src/command.h +++ b/src/command.h @@ -72,7 +72,7 @@ public: void record_write_timestamp(uint32_t query); #if __ANDROID_API__ >= 26 - void record_import_android_hardware_buffer(const ImportAndroidHardwareBufferPipeline* pipeline, VkImage image, VkImageView imageView, const VkMat& m); + void record_import_android_hardware_buffer(const ImportAndroidHardwareBufferPipeline* pipeline, const VkImageMat& im, const VkMat& m); #endif // __ANDROID_API__ >= 26 int submit_and_wait(); @@ -102,10 +102,10 @@ protected: void record_prepare_transfer_barrier(const VkMat& m); void record_prepare_compute_barrier(const VkMat& m); - void record_initial_image_compute_barrier(VkImage image); + void record_initial_image_compute_barrier(const VkImageMat& im); #if __ANDROID_API__ >= 26 - void record_update_import_android_hardware_buffer_bindings(VkPipelineLayout pipeline_layout, VkDescriptorSetLayout descriptorset_layout, VkDescriptorUpdateTemplateKHR descriptor_update_template, VkSampler sampler, VkImageView imageView, const VkMat& m); + void record_update_import_android_hardware_buffer_bindings(VkPipelineLayout pipeline_layout, VkDescriptorSetLayout descriptorset_layout, VkDescriptorUpdateTemplateKHR descriptor_update_template, VkSampler sampler, const VkImageMat& im, const VkMat& m); #endif // __ANDROID_API__ >= 26 #if NCNN_BENCHMARK diff --git a/src/mat.cpp b/src/mat.cpp index 8c2f66581..c10802002 100644 --- a/src/mat.cpp +++ b/src/mat.cpp @@ -24,6 +24,10 @@ #include "layer_type.h" #include "layer.h" +#if __ANDROID_API__ >= 26 +#include +#endif // __ANDROID_API__ >= 26 + namespace ncnn { void Mat::substract_mean_normalize(const float* mean_vals, const float* norm_vals) @@ -222,6 +226,29 @@ Mat Mat::from_float16(const unsigned short* data, int size) return m; } +#if __ANDROID_API__ >= 26 +VkImageMat VkImageMat::from_android_hardware_buffer(AHardwareBuffer* hb, VkAndroidHardwareBufferImageAllocator* allocator) +{ + AHardwareBuffer_Desc bufferDesc; + AHardwareBuffer_describe(hb, &bufferDesc); + + VkImageMat m; + + m.allocator = allocator; + + m.width = bufferDesc.width; + m.height = bufferDesc.height; + m.format = VK_FORMAT_UNDEFINED; + + m.data = allocator->fastMalloc(hb); + + m.refcount = (int*)((unsigned char*)m.data + offsetof(VkImageMemory, refcount)); + *m.refcount = 1; + + return m; +} +#endif // __ANDROID_API__ >= 26 + void copy_make_border(const Mat& src, Mat& dst, int top, int bottom, int left, int right, int type, float v, const Option& opt) { ncnn::Layer* padding = ncnn::create_layer(ncnn::LayerType::Padding); diff --git a/src/mat.h b/src/mat.h index 0fe7cd68f..11bb23422 100644 --- a/src/mat.h +++ b/src/mat.h @@ -229,7 +229,7 @@ public: // the allocator Allocator* allocator; - // the dimensionality + // the dimension rank int dims; int w; @@ -362,7 +362,7 @@ public: VkAllocator* allocator; VkAllocator* staging_allocator; - // the dimensionality + // the dimension rank int dims; int w; @@ -372,6 +372,56 @@ public: size_t cstep; }; +class VkImageMat +{ +public: + // empty + VkImageMat(); + // image + VkImageMat(int width, int height, VkFormat format, VkImageAllocator* allocator); + // copy + VkImageMat(const VkImageMat& m); + // external image + VkImageMat(int width, int height, VkImageMemory* data, VkFormat format, VkImageAllocator* allocator); + // release + ~VkImageMat(); + // assign + VkImageMat& operator=(const VkImageMat& m); + // allocate image + void create(int width, int height, VkFormat format, VkImageAllocator* allocator); + + // refcount++ + void addref(); + // refcount-- + void release(); + + bool empty() const; + size_t total() const; + + // low-level reference + VkImage image() const; + VkImageView imageview() const; + +#if __ANDROID_API__ >= 26 + // convenient construct from android hardware buffer + static VkImageMat from_android_hardware_buffer(AHardwareBuffer* hb, VkAndroidHardwareBufferImageAllocator* allocator); +#endif // __ANDROID_API__ >= 26 + + // device image + VkImageMemory* data; + + // pointer to the reference counter + // when points to user-allocated data, the pointer is NULL + int* refcount; + + // the allocator + VkImageAllocator* allocator; + + int width; + int height; + VkFormat format; +}; + // type for vulkan specialization constant and push constant union vk_specialization_type { int i; float f; uint32_t u32; }; union vk_constant_type { int i; float f; }; @@ -1648,6 +1698,123 @@ inline size_t VkMat::staging_buffer_offset() const { return staging_data->offset; } + +inline VkImageMat::VkImageMat() + : data(0), refcount(0), allocator(0), width(0), height(0), format(VK_FORMAT_UNDEFINED) +{ +} + +inline VkImageMat::VkImageMat(int _width, int _height, VkFormat _format, VkImageAllocator* _allocator) + : data(0), refcount(0), allocator(0), width(0), height(0), format(VK_FORMAT_UNDEFINED) +{ + create(_width, _height, _format, _allocator); +} + +inline VkImageMat::VkImageMat(const VkImageMat& m) + : data(m.data), refcount(m.refcount), allocator(m.allocator), width(m.width), height(m.height), format(m.format) +{ + if (refcount) + NCNN_XADD(refcount, 1); +} + +inline VkImageMat::VkImageMat(int _width, int _height, VkImageMemory* _data, VkFormat _format, VkImageAllocator* _allocator) + : data(_data), refcount(0), allocator(_allocator), width(_width), height(_height), format(_format) +{ +} + +inline VkImageMat::~VkImageMat() +{ + release(); +} + +inline VkImageMat& VkImageMat::operator=(const VkImageMat& m) +{ + if (this == &m) + return *this; + + if (m.refcount) + NCNN_XADD(m.refcount, 1); + + release(); + + data = m.data; + refcount = m.refcount; + allocator = m.allocator; + + width = m.width; + height = m.height; + format = m.format; + + return *this; +} + +inline void VkImageMat::create(int _width, int _height, VkFormat _format, VkImageAllocator* _allocator) +{ + if (width == _width && height == _height && format == _format && allocator == _allocator) + return; + + release(); + + allocator = _allocator; + + width = _width; + height = _height; + format = _format; + + if (total() > 0) + { + data = allocator->fastMalloc(width, height, format); + + refcount = (int*)((unsigned char*)data + offsetof(VkImageMemory, refcount)); + *refcount = 1; + } +} + +inline void VkImageMat::addref() +{ + if (refcount) + NCNN_XADD(refcount, 1); +} + +inline void VkImageMat::release() +{ + if (refcount && NCNN_XADD(refcount, -1) == 1) + { + if (allocator && data) + { + allocator->fastFree(data); + } + } + + data = 0; + + width = 0; + height = 0; + format = VK_FORMAT_UNDEFINED; + + refcount = 0; +} + +inline bool VkImageMat::empty() const +{ + return data == 0 || total() == 0; +} + +inline size_t VkImageMat::total() const +{ + return width * height; +} + +inline VkImage VkImageMat::image() const +{ + return data->image; +} + +inline VkImageView VkImageMat::imageview() const +{ + return data->imageview; +} + #endif // NCNN_VULKAN } // namespace ncnn diff --git a/src/mat_pixel_android.cpp b/src/mat_pixel_android.cpp index fa4cec887..e20d77dba 100644 --- a/src/mat_pixel_android.cpp +++ b/src/mat_pixel_android.cpp @@ -15,13 +15,15 @@ #include "mat.h" #if NCNN_PIXEL -#if __ANDROID_API__ >= 9 +#if __ANDROID_API__ >= 9 #include #include +#endif // __ANDROID_API__ >= 9 namespace ncnn { +#if __ANDROID_API__ >= 9 Mat Mat::from_android_bitmap(JNIEnv* env, jobject bitmap, int type_to, Allocator* allocator) { AndroidBitmapInfo info; @@ -136,8 +138,8 @@ void Mat::to_android_bitmap(JNIEnv* env, jobject bitmap, int type_from) const AndroidBitmap_unlockPixels(env, bitmap); } +#endif // __ANDROID_API__ >= 9 } // namespace ncnn -#endif // __ANDROID_API__ >= 9 #endif // NCNN_PIXEL diff --git a/src/pipeline.cpp b/src/pipeline.cpp index c665eaa14..fa95504eb 100644 --- a/src/pipeline.cpp +++ b/src/pipeline.cpp @@ -496,19 +496,6 @@ int ImportAndroidHardwareBufferPipeline::create(AHardwareBuffer* hb, int _type_t out_elempack = 4; } - bufferFormatProperties.sType = VK_STRUCTURE_TYPE_ANDROID_HARDWARE_BUFFER_FORMAT_PROPERTIES_ANDROID; - bufferFormatProperties.pNext = 0; - - bufferProperties.sType = VK_STRUCTURE_TYPE_ANDROID_HARDWARE_BUFFER_PROPERTIES_ANDROID; - bufferProperties.pNext = &bufferFormatProperties; - - VkResult ret = vkGetAndroidHardwareBufferPropertiesANDROID(vkdev->vkdevice(), hb, &bufferProperties); - if (ret != VK_SUCCESS) - { - fprintf(stderr, "vkGetAndroidHardwareBufferPropertiesANDROID failed %d\n", ret); - return -1; - } - set_local_size_xyz(8, 8, 1); std::vector specializations(4); @@ -517,7 +504,7 @@ int ImportAndroidHardwareBufferPipeline::create(AHardwareBuffer* hb, int _type_t specializations[2].i = type_to; specializations[3].i = rotate_from; - create_sampler(); + create_sampler(hb); create_descriptorset_layout(); @@ -567,118 +554,25 @@ void ImportAndroidHardwareBufferPipeline::destroy() Pipeline::destroy(); } -int ImportAndroidHardwareBufferPipeline::create_image_memory_imageview(AHardwareBuffer* hb, VkImage* image, VkDeviceMemory* memory, VkImageView* imageView) const +int ImportAndroidHardwareBufferPipeline::create_sampler(AHardwareBuffer* hb) { - VkExternalFormatANDROID externalFormat; - externalFormat.sType = VK_STRUCTURE_TYPE_EXTERNAL_FORMAT_ANDROID; - externalFormat.pNext = 0; - externalFormat.externalFormat = bufferFormatProperties.externalFormat; + VkResult ret; - VkExternalMemoryImageCreateInfo externalMemoryImageCreateInfo; - externalMemoryImageCreateInfo.sType = VK_STRUCTURE_TYPE_EXTERNAL_MEMORY_IMAGE_CREATE_INFO, - externalMemoryImageCreateInfo.pNext = &externalFormat, - externalMemoryImageCreateInfo.handleTypes = VK_EXTERNAL_MEMORY_HANDLE_TYPE_ANDROID_HARDWARE_BUFFER_BIT_ANDROID; + VkAndroidHardwareBufferFormatPropertiesANDROID bufferFormatProperties; + bufferFormatProperties.sType = VK_STRUCTURE_TYPE_ANDROID_HARDWARE_BUFFER_FORMAT_PROPERTIES_ANDROID; + bufferFormatProperties.pNext = 0; - VkImageCreateInfo imageCreateInfo; - imageCreateInfo.sType = VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO, - imageCreateInfo.pNext = &externalMemoryImageCreateInfo; - imageCreateInfo.flags = 0; - imageCreateInfo.imageType = VK_IMAGE_TYPE_2D; - imageCreateInfo.format = VK_FORMAT_UNDEFINED; - imageCreateInfo.extent.width = w; - imageCreateInfo.extent.height = h; - imageCreateInfo.extent.depth = 1; - imageCreateInfo.mipLevels = 1; - imageCreateInfo.arrayLayers = 1; - imageCreateInfo.samples = VK_SAMPLE_COUNT_1_BIT; - imageCreateInfo.tiling = VK_IMAGE_TILING_OPTIMAL; - imageCreateInfo.usage = VK_IMAGE_USAGE_SAMPLED_BIT; - imageCreateInfo.sharingMode = VK_SHARING_MODE_EXCLUSIVE; - imageCreateInfo.queueFamilyIndexCount = 0; - imageCreateInfo.pQueueFamilyIndices = 0; - imageCreateInfo.initialLayout = VK_IMAGE_LAYOUT_UNDEFINED; + VkAndroidHardwareBufferPropertiesANDROID bufferProperties; + bufferProperties.sType = VK_STRUCTURE_TYPE_ANDROID_HARDWARE_BUFFER_PROPERTIES_ANDROID; + bufferProperties.pNext = &bufferFormatProperties; - *image = 0; - VkResult ret = vkCreateImage(vkdev->vkdevice(), &imageCreateInfo, 0, image); + ret = vkGetAndroidHardwareBufferPropertiesANDROID(vkdev->vkdevice(), hb, &bufferProperties); if (ret != VK_SUCCESS) { - fprintf(stderr, "vkCreateImage failed %d\n", ret); + fprintf(stderr, "vkGetAndroidHardwareBufferPropertiesANDROID failed %d\n", ret); return -1; } - VkImportAndroidHardwareBufferInfoANDROID importAndroidHardwareBufferInfo; - importAndroidHardwareBufferInfo.sType = VK_STRUCTURE_TYPE_IMPORT_ANDROID_HARDWARE_BUFFER_INFO_ANDROID; - importAndroidHardwareBufferInfo.pNext = 0; - importAndroidHardwareBufferInfo.buffer = hb; - - VkMemoryDedicatedAllocateInfo memoryDedicatedAllocateInfo; - memoryDedicatedAllocateInfo.sType = VK_STRUCTURE_TYPE_MEMORY_DEDICATED_ALLOCATE_INFO; - memoryDedicatedAllocateInfo.pNext = &importAndroidHardwareBufferInfo; - memoryDedicatedAllocateInfo.image = *image; - memoryDedicatedAllocateInfo.buffer = VK_NULL_HANDLE; - - VkMemoryAllocateInfo memoryAllocateInfo; - memoryAllocateInfo.sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO; - memoryAllocateInfo.pNext = &memoryDedicatedAllocateInfo; - memoryAllocateInfo.allocationSize = bufferProperties.allocationSize; - memoryAllocateInfo.memoryTypeIndex = vkdev->find_memory_index(bufferProperties.memoryTypeBits, 0, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT, VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT); - - *memory = 0; - ret = vkAllocateMemory(vkdev->vkdevice(), &memoryAllocateInfo, 0, memory); - if (ret != VK_SUCCESS) - { - fprintf(stderr, "vkAllocateMemory failed %d\n", ret); - return -1; - } - - VkBindImageMemoryInfo bindImageMemoryInfo; - bindImageMemoryInfo.sType = VK_STRUCTURE_TYPE_BIND_IMAGE_MEMORY_INFO; - bindImageMemoryInfo.pNext = 0; - bindImageMemoryInfo.image = *image; - bindImageMemoryInfo.memory = *memory; - bindImageMemoryInfo.memoryOffset = 0; - ret = vkdev->vkBindImageMemory2KHR(vkdev->vkdevice(), 1, &bindImageMemoryInfo); - if (ret != VK_SUCCESS) - { - fprintf(stderr, "vkBindImageMemory2KHR failed %d\n", ret); - return -1; - } - - VkSamplerYcbcrConversionInfoKHR samplerYcbcrConversionInfo; - samplerYcbcrConversionInfo.sType = VK_STRUCTURE_TYPE_SAMPLER_YCBCR_CONVERSION_INFO_KHR; - samplerYcbcrConversionInfo.pNext = &externalFormat; - samplerYcbcrConversionInfo.conversion = samplerYcbcrConversion; - - VkImageViewCreateInfo imageViewCreateInfo; - imageViewCreateInfo.sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO; - imageViewCreateInfo.pNext = &samplerYcbcrConversionInfo; - imageViewCreateInfo.flags = 0; - imageViewCreateInfo.image = *image; - imageViewCreateInfo.viewType = VK_IMAGE_VIEW_TYPE_2D; - imageViewCreateInfo.format = VK_FORMAT_UNDEFINED; - imageViewCreateInfo.components.r = VK_COMPONENT_SWIZZLE_IDENTITY; - imageViewCreateInfo.components.g = VK_COMPONENT_SWIZZLE_IDENTITY; - imageViewCreateInfo.components.b = VK_COMPONENT_SWIZZLE_IDENTITY; - imageViewCreateInfo.components.a = VK_COMPONENT_SWIZZLE_IDENTITY; - imageViewCreateInfo.subresourceRange.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT; - imageViewCreateInfo.subresourceRange.baseMipLevel = 0; - imageViewCreateInfo.subresourceRange.levelCount = 1; - imageViewCreateInfo.subresourceRange.baseArrayLayer = 0; - imageViewCreateInfo.subresourceRange.layerCount = 1; - - *imageView = 0; - ret = vkCreateImageView(vkdev->vkdevice(), &imageViewCreateInfo, 0, imageView); - if (ret != VK_SUCCESS) - { - fprintf(stderr, "vkCreateImageView failed %d\n", ret); - return -1; - } - - return 0; -} - -int ImportAndroidHardwareBufferPipeline::create_sampler() -{ VkExternalFormatANDROID externalFormat; externalFormat.sType = VK_STRUCTURE_TYPE_EXTERNAL_FORMAT_ANDROID; externalFormat.pNext = 0; @@ -696,7 +590,7 @@ int ImportAndroidHardwareBufferPipeline::create_sampler() samplerYcbcrConversionCreateInfo.chromaFilter = VK_FILTER_NEAREST; samplerYcbcrConversionCreateInfo.forceExplicitReconstruction = VK_FALSE; - VkResult ret = vkdev->vkCreateSamplerYcbcrConversionKHR(vkdev->vkdevice(), &samplerYcbcrConversionCreateInfo, 0, &samplerYcbcrConversion); + ret = vkdev->vkCreateSamplerYcbcrConversionKHR(vkdev->vkdevice(), &samplerYcbcrConversionCreateInfo, 0, &samplerYcbcrConversion); if (ret != VK_SUCCESS) { fprintf(stderr, "vkCreateSamplerYcbcrConversionKHR failed %d\n", ret); diff --git a/src/pipeline.h b/src/pipeline.h index 275161e50..5be2cfb82 100644 --- a/src/pipeline.h +++ b/src/pipeline.h @@ -80,12 +80,10 @@ public: int create(AHardwareBuffer* hb, int type_to, int rotate_from, const Option& opt); void destroy(); - int create_image_memory_imageview(AHardwareBuffer* hb, VkImage* image, VkDeviceMemory* memory, VkImageView* imageView) const; - friend class VkCompute; protected: - int create_sampler(); + int create_sampler(AHardwareBuffer* hb); int create_descriptorset_layout(); int create_descriptor_update_template(); @@ -99,8 +97,6 @@ public: int out_elempack; int type_to; int rotate_from; - VkAndroidHardwareBufferFormatPropertiesANDROID bufferFormatProperties; - VkAndroidHardwareBufferPropertiesANDROID bufferProperties; VkSamplerYcbcrConversionKHR samplerYcbcrConversion; VkSampler sampler;