vo_opengl: fix d3d11 hardware decoding probing on Windows 7

Although D3D11 video decoding is unuspported on Windows 7, the
associated APIs almost work. Where they fail is texture creation, where
we try to create D3D11_BIND_DECODER surfaces. So specifically try to
detect this situation.

One issue is that once the hwdec interop is created, the damage is done,
and it can't use another backend (because currently only 1 hwdec backend
is supported). So that's where we prevent attempts to use it.

It still can fail when trying to use d3d11va-copy (since that doesn't
require an interop backend), but at that point we don't care anymore -
dxva2(-copy) is tried before that anyway.

video/out/opengl/angle_common.c

@@ -0,0 +1,13 @@
+#include "angle_common.h"
+
+// Test if Direct3D11 can be used by us. Basically, this prevents trying to use
+// D3D11 on Win7, and then failing somewhere in the process.
+bool d3d11_check_decoding(ID3D11Device *dev)
+{
+    HRESULT hr;
+    // We assume that NV12 is always supported, if hw decoding is supported at
+    // all.
+    UINT supported = 0;
+    hr = ID3D11Device_CheckFormatSupport(dev, DXGI_FORMAT_NV12, &supported);
+    return !FAILED(hr) && (supported & D3D11_BIND_DECODER);
+}

video/out/opengl/angle_common.h

@@ -0,0 +1,13 @@
+#ifndef MP_ANGLE_COMMON_H
+#define MP_ANGLE_COMMON_H
+
+#include <initguid.h>
+#include <assert.h>
+#include <windows.h>
+#include <d3d11.h>
+
+#include <stdbool.h>
+
+bool d3d11_check_decoding(ID3D11Device *dev);
+
+#endif

video/out/opengl/hwdec_d3d11egl.c

@@ -23,6 +23,7 @@
 #include <EGL/egl.h>
 #include <EGL/eglext.h>
 
+#include "angle_common.h"
 #include "angle_dynamic.h"
 
 #include "common/common.h"
@@ -173,6 +174,11 @@ static int create(struct gl_hwdec *hw)
         goto fail;
     ID3D11Device_AddRef(p->d3d11_device);
 
+    if (!d3d11_check_decoding(p->d3d11_device)) {
+        MP_VERBOSE(hw, "D3D11 video decoding not supported on this system.\n");
+        goto fail;
+    }
+
     ID3D10Multithread *multithread;
     hr = ID3D11Device_QueryInterface(p->d3d11_device, &IID_ID3D10Multithread,
                                      (void **)&multithread);

video/out/opengl/hwdec_d3d11eglrgb.c

@@ -23,6 +23,7 @@
 #include <EGL/egl.h>
 #include <EGL/eglext.h>
 
+#include "angle_common.h"
 #include "angle_dynamic.h"
 
 #include "common/common.h"
@@ -154,6 +155,11 @@ static int create(struct gl_hwdec *hw)
     ID3D10Multithread_SetMultithreadProtected(multithread, TRUE);
     ID3D10Multithread_Release(multithread);
 
+    if (!d3d11_check_decoding(p->d3d11_device)) {
+        MP_VERBOSE(hw, "D3D11 video decoding not supported on this system.\n");
+        goto fail;
+    }
+
     hr = ID3D11Device_QueryInterface(p->d3d11_device, &IID_ID3D11VideoDevice,
                                      (void **)&p->video_dev);
     if (FAILED(hr))

wscript_build.py

@@ -331,6 +331,7 @@ def build(ctx):
         ( "video/out/dither.c" ),
         ( "video/out/filter_kernels.c" ),
         ( "video/out/opengl/angle_dynamic.c",    "egl-angle" ),
+        ( "video/out/opengl/angle_common.c",     "egl-angle" ),
         ( "video/out/opengl/common.c",           "gl" ),
         ( "video/out/opengl/context.c",          "gl" ),
         ( "video/out/opengl/context_angle.c",    "egl-angle" ),