d3d11: interop with desktop gl

not working for intel? DXOpenDeviceNV fails.
Unlike d3d9, d3d11 supports copying sub region of a texture, so using
the original pixel format like nv12 should be possible.
TODO: try ANGLE https://github.com/google/angle/blob/master/src/
libANGLE/renderer/gl/wgl/DisplayWGL.cpp

src/codec/video/VideoDecoderD3D11.cpp

@@ -111,7 +111,7 @@ public:
         dll = LoadLibrary(TEXT("d3d11.dll"));
         available = !!dll;
 #endif
-        if (d3d11::InteropResource::isSupported())
+        if (d3d11::InteropResource::isSupported(d3d11::InteropEGL))
             copy_mode = VideoDecoderFFmpegHW::ZeroCopy;
     }
     ~VideoDecoderD3D11Private() {
@@ -168,12 +168,16 @@ VideoFrame VideoDecoderD3D11::frame()
     }
     D3D11_VIDEO_DECODER_OUTPUT_VIEW_DESC view_desc;
     surface->GetDesc(&view_desc);
-
+    D3D11_TEXTURE2D_DESC tex_desc;
+    texture->GetDesc(&tex_desc);
     if (copyMode() == VideoDecoderFFmpegHW::ZeroCopy && d.interop_res) {
         d3d11::SurfaceInterop *interop = new d3d11::SurfaceInterop(d.interop_res);
         interop->setSurface(texture, view_desc.Texture2D.ArraySlice, d.width, d.height);
-        VideoFrame f(d.width, d.height, VideoFormat::Format_RGB32);
-        f.setBytesPerLine(d.width * 4); //used by gl to compute texture size
+        VideoFormat fmt(d.interop_res->format(tex_desc.Format));
+        VideoFrame f(d.width, d.height, fmt);
+        for (int i = 0; i < fmt.planeCount(); ++i) {
+            f.setBytesPerLine(fmt.bytesPerLine(d.width, i), i); //used by gl to compute texture size
+        }
         f.setMetaData(QStringLiteral("surface_interop"), QVariant::fromValue(VideoSurfaceInteropPtr(interop)));
         f.setTimestamp(d.frame->pkt_pts/1000.0);
         f.setDisplayAspectRatio(d.getDAR(d.frame));
@@ -198,8 +202,6 @@ VideoFrame VideoDecoderD3D11::frame()
     Q_UNUSED(sm);
     int pitch[3] = { (int)mapped.RowPitch, 0, 0}; //compute chroma later
     uint8_t *src[] = { (uint8_t*)mapped.pData, 0, 0}; //compute chroma later
-    D3D11_TEXTURE2D_DESC tex_desc;
-    texture->GetDesc(&tex_desc);
     const VideoFormat format = pixelFormatFromFourcc(d.format_fcc); //tex_desc
     return copyToFrame(format, tex_desc.Height, src, pitch, false);
 }

src/directx/SurfaceInteropD3D11.cpp

@@ -33,7 +33,7 @@
 #endif
 #endif //QTAV_HAVE(EGL_CAPI)
 #if defined(QT_OPENGL_DYNAMIC) || !defined(QT_OPENGL_ES_2)
-//#define QTAV_HAVE_D3D11_GL 1
+#define QTAV_HAVE_D3D11_GL 1
 #endif
 
 namespace QtAV {
@@ -60,6 +60,7 @@ bool InteropResource::isSupported(InteropType type)
 }
 
 extern InteropResource* CreateInteropEGL();
+extern InteropResource* CreateInteropGL();
 InteropResource* InteropResource::create(InteropType type)
 {
     if (type == InteropAuto || type == InteropEGL) {
@@ -94,12 +95,9 @@ void* SurfaceInterop::map(SurfaceType type, const VideoFormat &fmt, void *handle
 {
     if (!handle)
         return NULL;
-
     if (!m_surface)
         return 0;
     if (type == GLTextureSurface) {
-        if (!fmt.isRGB())
-            return NULL;
         if (m_resource->map(m_surface, m_index, *((GLuint*)handle), frame_width, frame_height, plane))
             return handle;
     } else if (type == HostMemorySurface) {

src/directx/SurfaceInteropD3D11.h

@@ -46,8 +46,9 @@ public:
      */
     static bool isSupported(InteropType type = InteropAuto);
 
-    void setDevice(ComPtr<ID3D11Device> dev);
     virtual ~InteropResource() {}
+    void setDevice(ComPtr<ID3D11Device> dev);
+    virtual VideoFormat::PixelFormat format(DXGI_FORMAT dxfmt) const = 0;
     /*!
      * \brief map
      * \param surface dxva decoded surface
@@ -62,9 +63,7 @@ public:
     virtual bool unmap(GLuint tex) { Q_UNUSED(tex); return true;}
 protected:
     ComPtr<ID3D11Device> d3ddev;
-    ComPtr<ID3D11Texture2D> d3dtex;
     int width, height; // video frame width and dx_surface width without alignment, not dxva decoded surface width
-private:
 };
 typedef QSharedPointer<InteropResource> InteropResourcePtr;
 

src/directx/SurfaceInteropD3D11EGL.cpp

@@ -53,6 +53,7 @@ public:
         , vp(0)
     {}
     ~EGLInteropResource();
+    VideoFormat::PixelFormat format(DXGI_FORMAT) const Q_DECL_OVERRIDE { return VideoFormat::Format_RGB32;}
     bool map(ComPtr<ID3D11Texture2D> surface, int index, GLuint tex, int w, int h, int) Q_DECL_OVERRIDE;
 
 private:
@@ -61,6 +62,7 @@ private:
 
     EGL* egl;
     dx::D3D11VP *vp;
+    ComPtr<ID3D11Texture2D> d3dtex;
 };
 
 InteropResource* CreateInteropEGL() { return new EGLInteropResource();}

src/directx/SurfaceInteropD3D11GL.cpp

@@ -0,0 +1,182 @@
+/******************************************************************************
+    QtAV:  Multimedia framework based on Qt and FFmpeg
+    Copyright (C) 2012-2016 Wang Bin <wbsecg1@gmail.com>
+
+*   This file is part of QtAV (from 2016)
+
+    This library is free software; you can redistribute it and/or
+    modify it under the terms of the GNU Lesser General Public
+    License as published by the Free Software Foundation; either
+    version 2.1 of the License, or (at your option) any later version.
+
+    This library is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+    Lesser General Public License for more details.
+
+    You should have received a copy of the GNU Lesser General Public
+    License along with this library; if not, write to the Free Software
+    Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+******************************************************************************/
+
+#include "SurfaceInteropD3D11.h"
+#define DX_LOG_COMPONENT "D3D11GL Interop"
+#include "utils/DirectXHelper.h"
+#include "opengl/OpenGLHelper.h"
+#include <QtCore/QVector>
+
+// TODO: does intel supports d3d11-gl interop?
+//dynamic gl or desktop gl
+namespace QtAV {
+namespace d3d11 {
+class GLInteropResource Q_DECL_FINAL: public InteropResource
+{
+public:
+    GLInteropResource();
+    ~GLInteropResource();
+    VideoFormat::PixelFormat format(DXGI_FORMAT dxfmt) const Q_DECL_OVERRIDE {
+        if (dxfmt == DXGI_FORMAT_NV12)
+            return VideoFormat::Format_NV12;
+        if (dxfmt == DXGI_FORMAT_P010)
+            return VideoFormat::Format_YUV420P10LE;
+        if (dxfmt == DXGI_FORMAT_P016)
+            return VideoFormat::Format_YUV420P16LE;
+        return VideoFormat::Format_Invalid;
+    }
+    bool map(ComPtr<ID3D11Texture2D> surface, int index, GLuint tex, int w, int h, int plane) Q_DECL_OVERRIDE;
+    bool unmap(GLuint tex) Q_DECL_OVERRIDE;
+private:
+    bool ensureResource(DXGI_FORMAT fmt, int w, int h, GLuint tex, int plane);
+
+    HANDLE interop_dev;
+    HANDLE interop_obj[2];
+    DXGI_FORMAT tex_format;
+    QVector<ComPtr<ID3D11Texture2D> > d3dtex;
+    int mapped;
+    GLuint gltex[2];
+};
+
+InteropResource* CreateInteropGL()
+{
+    return new GLInteropResource();
+}
+
+GLInteropResource::GLInteropResource()
+    : interop_dev(NULL)
+    , tex_format(DXGI_FORMAT_UNKNOWN)
+    , mapped(0)
+{
+    d3dtex.reserve(3);
+    d3dtex.resize(2);
+    memset(interop_obj, 0, sizeof(interop_obj));
+    memset(gltex, 0, sizeof(gltex));
+}
+
+GLInteropResource::~GLInteropResource()
+{
+    // FIXME: why unregister/close interop obj/dev here will crash(tested on intel driver)? must be in current opengl context?
+}
+
+bool GLInteropResource::map(ComPtr<ID3D11Texture2D> surface, int index, GLuint tex, int w, int h, int plane)
+{
+    gltex[plane] = tex;
+    D3D11_TEXTURE2D_DESC desc;
+    surface->GetDesc(&desc);
+    if (!ensureResource(desc.Format, w, h, tex, plane))
+        return false;
+    // open/close and register/unregster in every map/unmap to ensure called in current context and avoid crash (tested on intel driver)
+    // interop operations begin
+    WGL_ENSURE((interop_dev = gl().DXOpenDeviceNV(d3ddev.Get())) != NULL, false);
+    // call in ensureResource or in map?
+    WGL_ENSURE((interop_obj[plane] = gl().DXRegisterObjectNV(interop_dev, d3dtex[plane].Get(), tex, GL_TEXTURE_2D, WGL_ACCESS_READ_ONLY_NV)) != NULL, false);
+    // prepare dx resources for gl
+    D3D11_BOX box = {0};
+    box.right = w;
+    if (plane == 0) {
+        box.bottom = h;
+    } else {
+        box.top = desc.Height; // maybe > h
+        box.bottom = box.top + h/2;
+    }
+    ComPtr<ID3D11DeviceContext> ctx;
+    d3ddev->GetImmediateContext(&ctx);
+    ctx->CopySubresourceRegion(d3dtex[plane].Get(), 0, 0, 0, 0
+                               , surface.Get()
+                               , index
+                               , &box);
+    // lock dx resources
+    WGL_ENSURE(gl().DXLockObjectsNV(interop_dev, 1, &interop_obj[plane]), false);
+    WGL_ENSURE(gl().DXObjectAccessNV(interop_obj[plane], WGL_ACCESS_READ_ONLY_NV), false);
+    DYGL(glBindTexture(GL_TEXTURE_2D, tex));
+    return true;
+}
+
+bool GLInteropResource::unmap(GLuint tex)
+{
+    Q_UNUSED(tex);
+    int plane = 0;
+    if (gltex[plane] != tex)
+        plane = 1;
+    if (!interop_obj[plane])
+        return false;
+    DYGL(glBindTexture(GL_TEXTURE_2D, 0));
+    WGL_ENSURE(gl().DXUnlockObjectsNV(interop_dev, 1, &interop_obj[plane]), false);
+    WGL_WARN(gl().DXUnregisterObjectNV(interop_dev, interop_obj[plane]));
+    // interop operations end
+    WGL_WARN(gl().DXCloseDeviceNV(interop_dev));
+    interop_obj[plane] = NULL;
+    return true;
+}
+
+
+static const struct {
+    DXGI_FORMAT fmt;
+    DXGI_FORMAT plane_fmt[2];
+} plane_formats[] = {
+{ DXGI_FORMAT_R8G8B8A8_UNORM, {DXGI_FORMAT_R8G8B8A8_UNORM, DXGI_FORMAT_UNKNOWN}},
+{ DXGI_FORMAT_NV12, {DXGI_FORMAT_R8_UNORM, DXGI_FORMAT_R8G8_UNORM}},
+{ DXGI_FORMAT_P010, {DXGI_FORMAT_R16_UNORM, DXGI_FORMAT_R16G16_UNORM}},
+{ DXGI_FORMAT_P016, {DXGI_FORMAT_R16_UNORM, DXGI_FORMAT_R16G16_UNORM}},
+};
+
+static DXGI_FORMAT GetPlaneFormat(DXGI_FORMAT fmt, int plane)
+{
+    for (int i = 0; i < sizeof(plane_formats)/sizeof(plane_formats[0]); ++i) {
+        if (plane_formats[i].fmt == fmt)
+            return plane_formats[i].plane_fmt[plane];
+    }
+    return DXGI_FORMAT_UNKNOWN;
+}
+
+// IDirect3DDevice9 can not be used on WDDM OSes(>=vista)
+bool GLInteropResource::ensureResource(DXGI_FORMAT fmt, int w, int h, GLuint tex, int plane)
+{
+    Q_UNUSED(tex);
+    Q_ASSERT(gl().DXRegisterObjectNV && "WGL_NV_DX_interop is required");
+    if (fmt == tex_format && d3dtex[plane].Get() && width == w && height == h)
+        return true;
+    if (mapped == 2) {
+        d3dtex.clear();
+        d3dtex.resize(2);
+        mapped = 0;
+    }
+    CD3D11_TEXTURE2D_DESC desc(GetPlaneFormat(fmt, plane), w, h, 1, 1);
+    desc.BindFlags |= D3D11_BIND_RENDER_TARGET;
+    desc.MiscFlags = D3D11_RESOURCE_MISC_SHARED;
+    DX_ENSURE(d3ddev->CreateTexture2D(&desc, NULL, &d3dtex[plane]), false);
+#if 0
+    ComPtr<IDXGIResource> resource;
+    DX_ENSURE(d3dtex[plane].As(&resource), false);
+    HANDLE share_handle = NULL;
+    DX_ENSURE(resource->GetSharedHandle(&share_handle), false);
+    // required by d3d9 not d3d10&11: https://www.opengl.org/registry/specs/NV/DX_interop2.txt
+    WGL_WARN(gl().DXSetResourceShareHandleNV(d3dtex[plane].Get(), share_handle));
+#endif
+    width = w;
+    height = h;
+    tex_format = fmt;
+    mapped++;
+    return true;
+}
+}
+}

src/libQtAV.pro

@@ -273,6 +273,7 @@ config_d3d11va {
     SOURCES += directx/SurfaceInteropD3D11EGL.cpp
   }
   enable_desktopgl {
+    SOURCES += directx/SurfaceInteropD3D11GL.cpp
   }
   winrt: LIBS *= -ld3d11
 }