From e1ec6ffeca75e09caac7bc99679d06373dc72145 Mon Sep 17 00:00:00 2001 From: wang-bin Date: Wed, 19 Aug 2015 12:08:40 +0800 Subject: [PATCH] cuda_gl: map device to host in interop to support video capture for 0- copy mode --- src/codec/video/SurfaceInteropCUDA.cpp | 65 +++++++++++++++++++------- src/codec/video/SurfaceInteropCUDA.h | 18 ++++--- src/codec/video/VideoDecoderCUDA.cpp | 30 ++---------- src/cuda/cuda_api.cpp | 8 ++-- src/cuda/cuda_api.h | 51 ++++++++++++-------- 5 files changed, 96 insertions(+), 76 deletions(-) diff --git a/src/codec/video/SurfaceInteropCUDA.cpp b/src/codec/video/SurfaceInteropCUDA.cpp index 9871abfc..2ef9be26 100644 --- a/src/codec/video/SurfaceInteropCUDA.cpp +++ b/src/codec/video/SurfaceInteropCUDA.cpp @@ -56,10 +56,48 @@ InteropResource::~InteropResource() //CUDA_ENSURE(cuCtxDestroy(ctx)); } -void SurfaceInteropCUDA::setSurface(int picIndex, CUVIDPROCPARAMS param, int height, int coded_height) +void* InteropResource::mapToHost(const VideoFormat &format, void *handle, int picIndex, const CUVIDPROCPARAMS ¶m, int width, int height, int coded_height) +{ + AutoCtxLock locker((cuda_api*)this, lock); + Q_UNUSED(locker); + CUdeviceptr devptr; + unsigned int pitch; + + CUDA_ENSURE(cuvidMapVideoFrame(dec, picIndex, &devptr, &pitch, const_cast(¶m)), NULL); + CUVIDAutoUnmapper unmapper(this, dec, devptr); + Q_UNUSED(unmapper); + uchar* host_data = NULL; + const size_t host_size = pitch*coded_height*3/2; + CUDA_ENSURE(cuMemAllocHost((void**)&host_data, host_size), NULL); + // copy to the memory not allocated by cuda is possible but much slower + CUDA_ENSURE(cuMemcpyDtoH(host_data, devptr, host_size), NULL); + + VideoFrame frame(width, height, VideoFormat::Format_NV12); + uchar *planes[] = { + host_data, + host_data + pitch * coded_height + }; + frame.setBits(planes); + int pitches[] = { (int)pitch, (int)pitch }; + frame.setBytesPerLine(pitches); + + VideoFrame *f = reinterpret_cast(handle); + frame.setTimestamp(f->timestamp()); + frame.setDisplayAspectRatio(f->displayAspectRatio()); + if (format == frame.format()) + *f = frame.clone(); + else + *f = frame.to(format); + + cuMemFreeHost(host_data); + return f; +} + +void SurfaceInteropCUDA::setSurface(int picIndex, CUVIDPROCPARAMS param, int width, int height, int coded_height) { m_index = picIndex; m_param = param; + w = width; h = height; H = coded_height; } @@ -76,10 +114,10 @@ void* SurfaceInteropCUDA::map(SurfaceType type, const VideoFormat &fmt, void *ha return 0; if (type == GLTextureSurface) { // FIXME: to strong ref may delay the delete and cuda resource maybe already destoryed after strong ref is finished - if (m_resource.toStrongRef()->map(m_index, m_param, *((GLuint*)handle), h, H, plane)) + if (m_resource.toStrongRef()->map(m_index, m_param, *((GLuint*)handle), w, h, H, plane)) return handle; } else if (type == HostMemorySurface) { - return NULL;//mapToHost(fmt, handle, plane); + return m_resource.toStrongRef()->mapToHost(fmt, handle, m_index, m_param, w, h, H); } return NULL; } @@ -171,28 +209,18 @@ GLInteropResource::GLInteropResource(CUdevice d, CUvideodecoder decoder, CUvideo : InteropResource(d, decoder, lk) {} -bool GLInteropResource::map(int picIndex, const CUVIDPROCPARAMS ¶m, GLuint tex, int h, int ch, int plane) +bool GLInteropResource::map(int picIndex, const CUVIDPROCPARAMS ¶m, GLuint tex, int w, int h, int ch, int plane) { AutoCtxLock locker((cuda_api*)this, lock); Q_UNUSED(locker); - if (!ensureResource(h, ch, tex, plane)) // TODO surface size instead of frame size because we copy the device data + if (!ensureResource(w, h, ch, tex, plane)) // TODO surface size instead of frame size because we copy the device data return false; //CUDA_ENSURE(cuCtxPushCurrent(ctx), false); CUdeviceptr devptr; unsigned int pitch; CUDA_ENSURE(cuvidMapVideoFrame(dec, picIndex, &devptr, &pitch, const_cast(¶m)), false); - class AutoUnmapper { - cuda_api *api; - CUvideodecoder dec; - CUdeviceptr devptr; - public: - AutoUnmapper(cuda_api *a, CUvideodecoder d, CUdeviceptr p) : api(a), dec(d), devptr(p) {} - ~AutoUnmapper() { - CUDA_WARN2(api->cuvidUnmapVideoFrame(dec, devptr)); - } - }; - AutoUnmapper unmapper(this, dec, devptr); + CUVIDAutoUnmapper unmapper(this, dec, devptr); Q_UNUSED(unmapper); // TODO: why can not use res[plane].stream? CUDA_ERROR_INVALID_HANDLE CUDA_ENSURE(cuGraphicsMapResources(1, &res[plane].cuRes, 0), false); @@ -267,11 +295,11 @@ bool GLInteropResource::unmap(GLuint tex) return true; } -bool GLInteropResource::ensureResource(int h, int ch, GLuint tex, int plane) +bool GLInteropResource::ensureResource(int w, int h, int ch, GLuint tex, int plane) { Q_ASSERT(plane < 2 && "plane number must be 0 or 1 for NV12"); TexRes &r = res[plane]; - if (r.texture == tex && r.h == h && r.H == ch && r.cuRes) + if (r.texture == tex && r.w == w && r.h == h && r.H == ch && r.cuRes) return true; if (!ctx) { // TODO: how to use pop/push decoder's context without the context in opengl context @@ -289,6 +317,7 @@ bool GLInteropResource::ensureResource(int h, int ch, GLuint tex, int plane) } CUDA_ENSURE(cuGraphicsGLRegisterImage(&r.cuRes, tex, GL_TEXTURE_2D, CU_GRAPHICS_REGISTER_FLAGS_NONE), false); r.texture = tex; + r.w = w; r.h = h; r.H = ch; return true; diff --git a/src/codec/video/SurfaceInteropCUDA.h b/src/codec/video/SurfaceInteropCUDA.h index 3313c451..fcfe2b51 100644 --- a/src/codec/video/SurfaceInteropCUDA.h +++ b/src/codec/video/SurfaceInteropCUDA.h @@ -58,8 +58,10 @@ public: * \param plane useless now * \return true if success */ - virtual bool map(int picIndex, const CUVIDPROCPARAMS& param, GLuint tex, int h, int ch, int plane) = 0; + virtual bool map(int picIndex, const CUVIDPROCPARAMS& param, GLuint tex, int w, int h, int ch, int plane) = 0; virtual bool unmap(GLuint tex) { Q_UNUSED(tex); return true;} + /// copy from gpu and convert to target format if necessary. used by VideoCapture + void* mapToHost(const VideoFormat &format, void *handle, int picIndex, const CUVIDPROCPARAMS ¶m, int width, int height, int coded_height); protected: CUdevice dev; CUcontext ctx; @@ -68,7 +70,7 @@ protected: typedef struct { GLuint texture; - int h, H; + int w, h, H; CUgraphicsResource cuRes; CUstream stream; // for async works } TexRes; @@ -88,10 +90,11 @@ public: * If use CUdeviceptr, we must cuMemAllocPitch() and cuMemcpyDtoD for each plane every time. But we can not know whether * the decoded picture the index pointed to is changed when rendering, so the displayed images may out of order. * CUdeviceptr is associated with context + * \param width frame width(visual width) without alignment, <= cuda decoded surface(picture) width * \param height frame height(visual height) without alignment, <= cuda decoded surface(picture) height * \param coded_height cuda decoded surface(picture) height */ - void setSurface(int picIndex, CUVIDPROCPARAMS param, int height, int coded_height); + void setSurface(int picIndex, CUVIDPROCPARAMS param, int width, int height, int coded_height); /// GLTextureSurface only supports rgb32 void* map(SurfaceType type, const VideoFormat& fmt, void* handle, int plane) Q_DECL_OVERRIDE; void unmap(void *handle) Q_DECL_OVERRIDE; @@ -101,17 +104,18 @@ private: CUVIDPROCPARAMS m_param; // decoder is deleted but interop is still alive QWeakPointer m_resource; - int h, H; + int w, h, H; }; #if QTAV_HAVE(CUDA_EGL) +// NOT IMPLEMENTED. CUDA kernel to convert nv12 to rgb is required class EGL; class EGLInteropResource Q_DECL_FINAL: public InteropResource { public: EGLInteropResource(CUdevice d, CUvideodecoder decoder, CUvideoctxlock declock); ~EGLInteropResource(); - //bool map(int picIndex, const CUVIDPROCPARAMS& param, GLuint tex, int h, int ch, int plane) Q_DECL_OVERRIDE { return false;} + //bool map(int picIndex, const CUVIDPROCPARAMS& param, GLuint tex, int w, int h, int ch, int plane) Q_DECL_OVERRIDE { return false;} private: void releaseEGL(); @@ -127,7 +131,7 @@ class GLInteropResource Q_DECL_FINAL: public InteropResource { public: GLInteropResource(CUdevice d, CUvideodecoder decoder, CUvideoctxlock lk); - bool map(int picIndex, const CUVIDPROCPARAMS& param, GLuint tex, int h, int ch, int plane) Q_DECL_OVERRIDE; + bool map(int picIndex, const CUVIDPROCPARAMS& param, GLuint tex, int w, int h, int ch, int plane) Q_DECL_OVERRIDE; bool unmap(GLuint tex) Q_DECL_OVERRIDE; private: /* @@ -135,7 +139,7 @@ private: * playing a new stream will recreate the decoder and interop * All we need to ensure is register when texture changed. But there's no way to check the texture change. */ - bool ensureResource(int h, int ch, GLuint tex, int plane); + bool ensureResource(int w, int h, int ch, GLuint tex, int plane); }; #endif //QTAV_HAVE(CUDA_GL) } //namespace cuda diff --git a/src/codec/video/VideoDecoderCUDA.cpp b/src/codec/video/VideoDecoderCUDA.cpp index 8ac3b4e2..67e4165f 100644 --- a/src/codec/video/VideoDecoderCUDA.cpp +++ b/src/codec/video/VideoDecoderCUDA.cpp @@ -151,16 +151,6 @@ static AVCodecID mapCodecToFFmpeg(cudaVideoCodec cudaCodec) return QTAV_CODEC_ID(NONE); } -#if NV_CONFIG(DLLAPI_CUDA) || defined(CUDA_LINK) -class AutoCtxLock -{ -private: - CUvideoctxlock m_lock; -public: - AutoCtxLock(cuda_api*, CUvideoctxlock lck) { m_lock=lck; cuvidCtxLock(m_lock, 0); } - ~AutoCtxLock() { cuvidCtxUnlock(m_lock, 0); } -}; -#endif //NV_CONFIG(DLLAPI_CUDA) || defined(CUDA_LINK) class VideoDecoderCUDAPrivate Q_DECL_FINAL: public VideoDecoderPrivate , protected cuda_api { @@ -741,20 +731,10 @@ bool VideoDecoderCUDAPrivate::processDecodedData(CUVIDPARSERDISPINFO *cuviddisp, Q_UNUSED(lock); //CUDA_ENSURE(cuCtxPushCurrent(cuctx), false); CUDA_ENSURE(cuvidMapVideoFrame(dec, cuviddisp->picture_index, &devptr, &pitch, &proc_params), false); - class AutoUnmapper { - cuda_api *api; - CUvideodecoder dec; - CUdeviceptr devptr; - public: - AutoUnmapper(cuda_api *a, CUvideodecoder d, CUdeviceptr p) : api(a), dec(d), devptr(p) {} - ~AutoUnmapper() { - CUDA_WARN2(api->cuvidUnmapVideoFrame(dec, devptr)); - } - }; - AutoUnmapper unmapper(this, dec, devptr); + CUVIDAutoUnmapper unmapper(this, dec, devptr); Q_UNUSED(unmapper); if (copy_mode != VideoDecoderCUDA::ZeroCopy) { - int size = pitch*ch*3/2; + const int size = pitch*ch*3/2; if (size > host_data_size && host_data) { cuMemFreeHost(host_data); host_data = 0; @@ -764,10 +744,6 @@ bool VideoDecoderCUDAPrivate::processDecodedData(CUVIDPARSERDISPINFO *cuviddisp, CUDA_ENSURE(cuMemAllocHost((void**)&host_data, size), false); host_data_size = size; } - if (!host_data) { - qWarning("No valid staging memory!"); - return false; - } // copy to the memory not allocated by cuda is possible but much slower CUDA_ENSURE(cuMemcpyDtoHAsync(host_data, devptr, size, stream), false); CUDA_WARN(cuStreamSynchronize(stream)); @@ -779,7 +755,7 @@ bool VideoDecoderCUDAPrivate::processDecodedData(CUVIDPARSERDISPINFO *cuviddisp, VideoFrame frame(codec_ctx->width, codec_ctx->height, VideoFormat::Format_NV12); if (copy_mode == VideoDecoderCUDA::ZeroCopy && interop_res) { cuda::SurfaceInteropCUDA *interop = new cuda::SurfaceInteropCUDA(interop_res); - interop->setSurface(cuviddisp->picture_index, proc_params, codec_ctx->height, ch); //TODO: both surface size(for copy 2d) and frame size(for map host) + interop->setSurface(cuviddisp->picture_index, proc_params, codec_ctx->width, codec_ctx->height, ch); //TODO: both surface size(for copy 2d) and frame size(for map host) frame.setMetaData(QStringLiteral("surface_interop"), QVariant::fromValue(VideoSurfaceInteropPtr(interop))); } else { uchar *planes[] = { diff --git a/src/cuda/cuda_api.cpp b/src/cuda/cuda_api.cpp index a66a61b1..fa4d77bc 100644 --- a/src/cuda/cuda_api.cpp +++ b/src/cuda/cuda_api.cpp @@ -106,7 +106,7 @@ public: tcuMemAllocHost* cuMemAllocHost; typedef CUresult CUDAAPI tcuMemFreeHost(void *p); tcuMemFreeHost* cuMemFreeHost; - typedef CUresult CUDAAPI tcuMemcpyDtoH(void *dstHost, CUdeviceptr srcDevice, unsigned int ByteCount ); + typedef CUresult CUDAAPI tcuMemcpyDtoH(void *dstHost, CUdeviceptr srcDevice, unsigned int ByteCount); tcuMemcpyDtoH* cuMemcpyDtoH; typedef CUresult CUDAAPI tcuMemcpyDtoHAsync(void *dstHost, CUdeviceptr srcDevice, unsigned int ByteCount, CUstream hStream); tcuMemcpyDtoHAsync* cuMemcpyDtoHAsync; @@ -144,7 +144,7 @@ public: typedef CUresult CUDAAPI tcuCtxSynchronize(); tcuCtxSynchronize* cuCtxSynchronize; - typedef CUresult CUDAAPI tcuGLCtxCreate(CUcontext *pCtx, unsigned int Flags, CUdevice device ); + typedef CUresult CUDAAPI tcuGLCtxCreate(CUcontext *pCtx, unsigned int Flags, CUdevice device); tcuGLCtxCreate* cuGLCtxCreate; typedef CUresult CUDAAPI tcuGraphicsGLRegisterImage(CUgraphicsResource *pCudaResource, GLuint image, GLenum target, unsigned int Flags); tcuGraphicsGLRegisterImage* cuGraphicsGLRegisterImage; @@ -305,7 +305,7 @@ CUresult cuda_api::cuMemFreeHost(void *p) return ctx->api.cuMemFreeHost(p); } -CUresult cuda_api::cuMemcpyDtoH(void *dstHost, CUdeviceptr srcDevice, unsigned int ByteCount ) +CUresult cuda_api::cuMemcpyDtoH(void *dstHost, CUdeviceptr srcDevice, unsigned int ByteCount) { if (!ctx->api.cuMemcpyDtoH) ctx->api.cuMemcpyDtoH = (context::api_t::tcuMemcpyDtoH*)ctx->cuda_dll.resolve("cuMemcpyDtoH"); @@ -452,7 +452,7 @@ CUresult cuda_api::cuGraphicsUnmapResources(unsigned int count, CUgraphicsResour //////////////////////////////////////////////////// /// D3D Interop //////////////////////////////////////////////////// -//CUresult cuda_api::cuD3D9CtxCreate( CUcontext *pCtx, CUdevice *pCudaDevice, unsigned int Flags, IDirect3DDevice9 *pD3DDevice ); +//CUresult cuda_api::cuD3D9CtxCreate( CUcontext *pCtx, CUdevice *pCudaDevice, unsigned int Flags, IDirect3DDevice9 *pD3DDevice); //////////////////////////////////////////////////// /// CUVID functions diff --git a/src/cuda/cuda_api.h b/src/cuda/cuda_api.h index 6e880dc5..baadad03 100644 --- a/src/cuda/cuda_api.h +++ b/src/cuda/cuda_api.h @@ -56,18 +56,18 @@ public: CUresult cuGetErrorString(CUresult error, const char **pStr); // since 6.0. fallback to a empty string if symbol not found CUresult cuInit(unsigned int Flags); CUresult cuCtxCreate(CUcontext *pctx, unsigned int flags, CUdevice dev); - CUresult cuCtxDestroy(CUcontext cuctx ); + CUresult cuCtxDestroy(CUcontext cuctx); CUresult cuCtxPushCurrent(CUcontext cuctx); CUresult cuCtxPopCurrent( CUcontext *pctx); CUresult cuCtxGetCurrent(CUcontext *pctx); CUresult cuCtxSynchronize(); - CUresult cuMemAllocHost(void **pp, unsigned int bytesize); + CUresult cuMemAllocHost(void **pp, unsigned int bytesize); //TODO: size_t in new version CUresult cuMemFreeHost(void *p); - CUresult cuMemcpyDtoH (void *dstHost, CUdeviceptr srcDevice, unsigned int ByteCount ); - CUresult cuMemcpyDtoHAsync(void *dstHost, CUdeviceptr srcDevice, unsigned int ByteCount, CUstream hStream); + CUresult cuMemcpyDtoH(void *dstHost, CUdeviceptr srcDevice, unsigned int ByteCount); //TODO: size_t in new version + CUresult cuMemcpyDtoHAsync(void *dstHost, CUdeviceptr srcDevice, unsigned int ByteCount, CUstream hStream); //TODO: size_t in new version CUresult cuMemcpy2D(const CUDA_MEMCPY2D *pCopy); CUresult cuMemcpy2DAsync(const CUDA_MEMCPY2D *pCopy, CUstream hStream); - CUresult cuStreamCreate(CUstream *phStream, unsigned int Flags); + CUresult cuStreamCreate(CUstream *phStream, unsigned int Flags); //TODO: size_t in new version CUresult cuStreamDestroy(CUstream hStream); CUresult cuStreamQuery(CUstream hStream); CUresult cuStreamSynchronize(CUstream hStream); @@ -77,7 +77,7 @@ public: CUresult cuDeviceComputeCapability(int *major, int *minor, CUdevice dev); CUresult cuDeviceGetAttribute(int *pi, CUdevice_attribute attrib, CUdevice dev); - CUresult cuGLCtxCreate(CUcontext *pCtx, unsigned int Flags, CUdevice device ); + CUresult cuGLCtxCreate(CUcontext *pCtx, unsigned int Flags, CUdevice device); CUresult cuGraphicsGLRegisterImage(CUgraphicsResource *pCudaResource, GLuint image, GLenum target, unsigned int Flags); CUresult cuGraphicsUnregisterResource(CUgraphicsResource resource); CUresult cuGraphicsMapResources(unsigned int count, CUgraphicsResource *resources, CUstream hStream); @@ -87,7 +87,7 @@ public: //////////////////////////////////////////////////// /// D3D Interop //////////////////////////////////////////////////// - //CUresult cuD3D9CtxCreate(CUcontext *pCtx, CUdevice *pCudaDevice, unsigned int Flags, IDirect3DDevice9 *pD3DDevice ); + //CUresult cuD3D9CtxCreate(CUcontext *pCtx, CUdevice *pCudaDevice, unsigned int Flags, IDirect3DDevice9 *pD3DDevice); //////////////////////////////////////////////////// /// CUVID functions @@ -108,22 +108,11 @@ public: // Decode a single picture (field or frame) CUresult cuvidDecodePicture(CUvideodecoder hDecoder, CUVIDPICPARAMS *pPicParams); - // Post-process and map a video frame for use in cuda + // Post-process and map a video frame for use in cuda. TODO: unsigned long for x64 CUresult cuvidMapVideoFrame(CUvideodecoder hDecoder, int nPicIdx, unsigned int *pDevPtr, unsigned int *pPitch, CUVIDPROCPARAMS *pVPP); // Unmap a previously mapped video frame CUresult cuvidUnmapVideoFrame(CUvideodecoder hDecoder, unsigned int DevPtr); - - class AutoCtxLock { - cuda_api *m_api; - CUvideoctxlock m_lock; - public: - AutoCtxLock(cuda_api *api, CUvideoctxlock lck) : m_api(api), m_lock(lck) { - m_api->cuvidCtxLock(m_lock, 0); - } - ~AutoCtxLock() { m_api->cuvidCtxUnlock(m_lock, 0); } - }; - #endif // !NV_CONFIG(DLLAPI_CUDA) && !defined(CUDA_LINK) // This function returns the best Graphics GPU based on performance int GetMaxGflopsGraphicsDeviceId(); @@ -132,7 +121,6 @@ private: context *ctx; }; - #define CUDA_CHECK(f, ...) \ do { \ CUresult cuR = f; \ @@ -155,4 +143,27 @@ private: } \ } while (0) + +// TODO: error check +class AutoCtxLock { + cuda_api *m_api; + CUvideoctxlock m_lock; +public: + AutoCtxLock(cuda_api *api, CUvideoctxlock lck) : m_api(api), m_lock(lck) { + m_api->cuvidCtxLock(m_lock, 0); + } + ~AutoCtxLock() { m_api->cuvidCtxUnlock(m_lock, 0); } +}; + +class CUVIDAutoUnmapper { + cuda_api *api; + CUvideodecoder dec; + CUdeviceptr devptr; +public: + CUVIDAutoUnmapper(cuda_api *a, CUvideodecoder d, CUdeviceptr p) : api(a), dec(d), devptr(p) {} + ~CUVIDAutoUnmapper() { + api->cuvidUnmapVideoFrame(dec, devptr); + } +}; + #endif // CUDA_API_H