vo_opengl: add infrastructure for hardware decoding OpenGL interop

Most hardware decoding APIs provide some OpenGL interop. This allows
using vo_opengl, without having to read the video data back from GPU.

This requires adding a backend for each hardware decoding API. (Each
backend is an entry in gl_hwdec_vaglx[].) The backends expose video data
as a set of OpenGL textures.

Add infrastructure to support this. The next commit will add support for
VA-API.

DOCS/man/en/vo.rst

@@ -241,6 +241,11 @@ Available video output drivers are:
     Some features are available with OpenGL 3 capable graphics drivers only
     (or if the necessary extensions are available).
 
+    Hardware decoding over OpenGL-interop is supported to some degree. Note
+    that in this mode, some corner case might not be gracefully handled, and
+    colorspace conversion and chroma upsampling is generally in the hand of
+    the hardware decoder APIs.
+
     ``lscale=<filter>``
 
         ``bilinear``

video/decode/dec_video.h

@@ -47,6 +47,11 @@ int vd_control(struct sh_video *sh_video, int cmd, void *arg);
 struct mp_hwdec_info {
     struct mp_vdpau_ctx *vdpau_ctx;
     struct mp_vaapi_ctx *vaapi_ctx;
+    // Can be used to lazily load a requested API.
+    // api_name is e.g. "vdpau" (like the fields above, without "_ctx")
+    // Can be NULL, is idempotent, caller checks _ctx fields for success/access.
+    void (*load_api)(struct mp_hwdec_info *info, const char *api_name);
+    void *load_api_ctx;
 };
 
 #endif /* MPLAYER_DEC_VIDEO_H */

video/decode/lavc.h

@@ -86,6 +86,8 @@ bool hwdec_check_codec_support(const char *decoder,
                                const struct hwdec_profile_entry *table);
 int hwdec_get_max_refs(struct lavc_ctx *ctx);
 
+void hwdec_request_api(struct mp_hwdec_info *info, const char *api_name);
+
 // lavc_dr1.c
 int mp_codec_get_buffer(AVCodecContext *s, AVFrame *frame);
 void mp_codec_release_buffer(AVCodecContext *s, AVFrame *frame);

video/decode/vd_lavc.c

@@ -46,6 +46,7 @@
 #include "video/img_format.h"
 #include "video/mp_image_pool.h"
 #include "video/filter/vf.h"
+#include "video/decode/dec_video.h"
 #include "demux/stheader.h"
 #include "demux/demux_packet.h"
 #include "osdep/numcores.h"
@@ -196,6 +197,12 @@ int hwdec_get_max_refs(struct lavc_ctx *ctx)
     return ctx->avctx->codec_id == AV_CODEC_ID_H264 ? 16 : 2;
 }
 
+void hwdec_request_api(struct mp_hwdec_info *info, const char *api_name)
+{
+    if (info && info->load_api)
+        info->load_api(info, api_name);
+}
+
 static int hwdec_probe(struct vd_lavc_hwdec *hwdec, struct mp_hwdec_info *info,
                        const char *decoder, const char **hw_decoder)
 {

video/out/gl_common.c

@@ -1008,3 +1008,7 @@ void mp_log_source(struct mp_log *log, int lev, const char *src)
         src = next;
     }
 }
+
+const struct gl_hwdec_driver *mpgl_hwdec_drivers[] = {
+    NULL
+};

video/out/gl_common.h

@@ -164,6 +164,44 @@ void mpgl_set_backend_w32(MPGLContext *ctx);
 void mpgl_set_backend_x11(MPGLContext *ctx);
 void mpgl_set_backend_wayland(MPGLContext *ctx);
 
+struct mp_hwdec_info;
+
+struct gl_hwdec {
+    const struct gl_hwdec_driver *driver;
+    struct mp_log *log;
+    struct MPGLContext *mpgl;
+    struct mp_hwdec_info *info;
+    // For free use by hwdec driver
+    void *priv;
+    // hwdec backends must set this to an IMGFMT_ that has an equivalent
+    // internal representation in gl_video.c as the hardware texture.
+    // It's used to build the rendering chain, and also as screenshot format.
+    int converted_imgfmt;
+};
+
+struct gl_hwdec_driver {
+    // Same name as used by mp_hwdec_info->load_api()
+    const char *api_name;
+    // Test whether the given IMGFMT_ is supported.
+    bool (*query_format)(int imgfmt);
+    // Create the hwdec device. It must fill in hw->info, if applicable.
+    int (*create)(struct gl_hwdec *hw);
+    // Prepare for rendering video. (E.g. create textures.)
+    // Called on initialization, and every time the video size changes.
+    int (*reinit)(struct gl_hwdec *hw, int w, int h);
+    // Return textures that contain the given hw_image.
+    // Note that the caller keeps a reference to hw_image until unbind_image
+    // is called, so the callee doesn't need to do that.
+    int (*load_image)(struct gl_hwdec *hw, struct mp_image *hw_image,
+                      GLuint *out_textures);
+    // Undo load_image(). The user of load_image() calls this when the textures
+    // are not needed anymore.
+    void (*unload_image)(struct gl_hwdec *hw);
+    void (*destroy)(struct gl_hwdec *hw);
+};
+
+extern const struct gl_hwdec_driver *mpgl_hwdec_drivers[];
+
 void *mp_getdladdr(const char *s);
 
 void mpgl_load_functions(GL *gl, void *(*getProcAddress)(const GLubyte *),

video/out/gl_video.c

@@ -114,6 +114,7 @@ struct texplane {
 struct video_image {
     struct texplane planes[4];
     bool image_flipped;
+    struct mp_image *hwimage;   // if hw decoding is active
 };
 
 struct scaler {
@@ -202,6 +203,9 @@ struct gl_video {
     int last_dither_matrix_size;
     float *last_dither_matrix;
 
+    struct gl_hwdec *hwdec;
+    bool hwdec_active;
+
     void *scratch;
 };
 
@@ -1129,7 +1133,7 @@ static void reinit_rendering(struct gl_video *p)
 
     uninit_rendering(p);
 
-    if (!p->image.planes[0].gl_texture)
+    if (!p->image_format)
         return;
 
     for (int n = 0; n < 2; n++)
@@ -1196,17 +1200,41 @@ void gl_video_set_lut3d(struct gl_video *p, struct lut3d *lut3d)
     debug_check_gl(p, "after 3d lut creation");
 }
 
-static void set_image_textures(struct gl_video *p, struct video_image *vimg)
+static void set_image_textures(struct gl_video *p, struct video_image *vimg,
+                               GLuint imgtex[4])
+{
+    GL *gl = p->gl;
+    GLuint dummy[4];
+    if (!imgtex)
+        imgtex = dummy;
+
+    if (p->hwdec_active) {
+        assert(vimg->hwimage);
+        p->hwdec->driver->load_image(p->hwdec, vimg->hwimage, imgtex);
+    } else {
+        for (int n = 0; n < p->plane_count; n++)
+            imgtex[n] = vimg->planes[n].gl_texture;
+    }
+
+    for (int n = 0; n < 4; n++) {
+        gl->ActiveTexture(GL_TEXTURE0 + n);
+        gl->BindTexture(GL_TEXTURE_2D, imgtex[n]);
+    }
+    gl->ActiveTexture(GL_TEXTURE0);
+}
+
+static void unset_image_textures(struct gl_video *p)
 {
     GL *gl = p->gl;
 
-    for (int n = 0; n < p->plane_count; n++) {
-        struct texplane *plane = &vimg->planes[n];
-
+    for (int n = 0; n < 4; n++) {
         gl->ActiveTexture(GL_TEXTURE0 + n);
-        gl->BindTexture(GL_TEXTURE_2D, plane->gl_texture);
+        gl->BindTexture(GL_TEXTURE_2D, 0);
     }
     gl->ActiveTexture(GL_TEXTURE0);
+
+    if (p->hwdec_active)
+        p->hwdec->driver->unload_image(p->hwdec);
 }
 
 static void init_video(struct gl_video *p)
@@ -1243,21 +1271,27 @@ static void init_video(struct gl_video *p)
         plane->w = full_w >> p->image_desc.xs[n];
         plane->h = full_h >> p->image_desc.ys[n];
 
-        texture_size(p, plane->w, plane->h,
-                        &plane->tex_w, &plane->tex_h);
+        if (p->hwdec_active) {
+            // We expect hwdec backends to allocate exact size
+            plane->tex_w = plane->w;
+            plane->tex_h = plane->h;
+        } else {
+            texture_size(p, plane->w, plane->h,
+                            &plane->tex_w, &plane->tex_h);
+
+            gl->ActiveTexture(GL_TEXTURE0 + n);
+            gl->GenTextures(1, &plane->gl_texture);
+            gl->BindTexture(GL_TEXTURE_2D, plane->gl_texture);
+
+            gl->TexImage2D(GL_TEXTURE_2D, 0, plane->gl_internal_format,
+                           plane->tex_w, plane->tex_h, 0,
+                           plane->gl_format, plane->gl_type, NULL);
+
+            default_tex_params(gl, GL_TEXTURE_2D, GL_LINEAR);
+        }
 
         MP_VERBOSE(p, "Texture for plane %d: %dx%d\n",
                    n, plane->tex_w, plane->tex_h);
-
-        gl->ActiveTexture(GL_TEXTURE0 + n);
-        gl->GenTextures(1, &plane->gl_texture);
-        gl->BindTexture(GL_TEXTURE_2D, plane->gl_texture);
-
-        gl->TexImage2D(GL_TEXTURE_2D, 0, plane->gl_internal_format,
-                       plane->tex_w, plane->tex_h, 0,
-                       plane->gl_format, plane->gl_type, NULL);
-
-        default_tex_params(gl, GL_TEXTURE_2D, GL_LINEAR);
     }
     gl->ActiveTexture(GL_TEXTURE0);
 
@@ -1266,6 +1300,11 @@ static void init_video(struct gl_video *p)
 
     debug_check_gl(p, "after video texture creation");
 
+    if (p->hwdec_active) {
+        if (p->hwdec->driver->reinit(p->hwdec, p->image_w, p->image_h) < 0)
+            MP_ERR(p, "Initializing hardware ddecoding video texture failed.\n");
+    }
+
     reinit_rendering(p);
 }
 
@@ -1287,6 +1326,7 @@ static void uninit_video(struct gl_video *p)
         plane->buffer_ptr = NULL;
         plane->buffer_size = 0;
     }
+    mp_image_unrefp(&vimg->hwimage);
 
     fbotex_uninit(p, &p->indirect_fbo);
     fbotex_uninit(p, &p->scale_sep_fbo);
@@ -1373,14 +1413,15 @@ void gl_video_render_frame(struct gl_video *p)
     // Order of processing:
     //  [indirect -> [scale_sep ->]] final
 
-    set_image_textures(p, vimg);
+    GLuint imgtex[4] = {0};
+    set_image_textures(p, vimg, imgtex);
 
     struct fbotex chain = {
         .vp_w = p->image_w,
         .vp_h = p->image_h,
         .tex_w = p->texture_w,
         .tex_h = p->texture_h,
-        .texture = vimg->planes[0].gl_texture,
+        .texture = imgtex[0],
     };
 
     handle_pass(p, &chain, &p->indirect_fbo, p->indirect_program);
@@ -1441,6 +1482,8 @@ void gl_video_render_frame(struct gl_video *p)
 
     gl->UseProgram(0);
 
+    unset_image_textures(p);
+
     p->frames_rendered++;
 
     debug_check_gl(p, "after video rendering");
@@ -1552,16 +1595,21 @@ static bool get_image(struct gl_video *p, struct mp_image *mpi)
 void gl_video_upload_image(struct gl_video *p, struct mp_image *mpi)
 {
     GL *gl = p->gl;
-    int n;
-
-    assert(mpi->num_planes == p->plane_count);
 
     struct video_image *vimg = &p->image;
 
+    if (p->hwdec_active) {
+        mp_image_setrefp(&vimg->hwimage, mpi);
+        p->have_image = true;
+        return;
+    }
+
+    assert(mpi->num_planes == p->plane_count);
+
     mp_image_t mpi2 = *mpi;
     bool pbo = false;
     if (!vimg->planes[0].buffer_ptr && get_image(p, &mpi2)) {
-        for (n = 0; n < p->plane_count; n++) {
+        for (int n = 0; n < p->plane_count; n++) {
             int line_bytes = mpi->plane_w[n] * p->image_desc.bytes[n];
             memcpy_pic(mpi2.planes[n], mpi->planes[n], line_bytes, mpi->plane_h[n],
                        mpi2.stride[n], mpi->stride[n]);
@@ -1570,7 +1618,7 @@ void gl_video_upload_image(struct gl_video *p, struct mp_image *mpi)
         pbo = true;
     }
     vimg->image_flipped = mpi->stride[0] < 0;
-    for (n = 0; n < p->plane_count; n++) {
+    for (int n = 0; n < p->plane_count; n++) {
         struct texplane *plane = &vimg->planes[n];
         void *plane_ptr = mpi->planes[n];
         if (pbo) {
@@ -1598,10 +1646,11 @@ struct mp_image *gl_video_download_image(struct gl_video *p)
 
     struct video_image *vimg = &p->image;
 
-    if (!p->have_image || !vimg->planes[0].gl_texture)
+    if (!p->have_image)
         return NULL;
 
-    assert(p->image_format == p->image_params.imgfmt);
+    set_image_textures(p, vimg, NULL);
+
     assert(p->texture_w >= p->image_params.w);
     assert(p->texture_h >= p->image_params.h);
 
@@ -1611,12 +1660,12 @@ struct mp_image *gl_video_download_image(struct gl_video *p)
     for (int n = 0; n < p->plane_count; n++) {
         struct texplane *plane = &vimg->planes[n];
         gl->ActiveTexture(GL_TEXTURE0 + n);
-        gl->BindTexture(GL_TEXTURE_2D, plane->gl_texture);
         glDownloadTex(gl, GL_TEXTURE_2D, plane->gl_format, plane->gl_type,
                       image->planes[n], image->stride[n]);
     }
-    gl->ActiveTexture(GL_TEXTURE0);
-    mp_image_set_params(image, &p->image_params);
+    mp_image_set_attributes(image, &p->image_params);
+
+    unset_image_textures(p);
 
     return image;
 }
@@ -1884,6 +1933,12 @@ static bool init_format(int fmt, struct gl_video *init)
     if (!init)
         init = &dummy;
 
+    init->hwdec_active = false;
+    if (init->hwdec && init->hwdec->driver->query_format(fmt)) {
+        fmt = init->hwdec->converted_imgfmt;
+        init->hwdec_active = true;
+    }
+
     struct mp_imgfmt_desc desc = mp_imgfmt_get_desc(fmt);
     if (!desc.id)
         return false;
@@ -1985,9 +2040,10 @@ static bool init_format(int fmt, struct gl_video *init)
     return true;
 }
 
-bool gl_video_check_format(int mp_format)
+bool gl_video_check_format(struct gl_video *p, int mp_format)
 {
-    return init_format(mp_format, NULL);
+    struct gl_video tmp = *p;
+    return init_format(mp_format, &tmp);
 }
 
 void gl_video_config(struct gl_video *p, struct mp_image_params *params)
@@ -2012,6 +2068,7 @@ void gl_video_config(struct gl_video *p, struct mp_image_params *params)
     p->colorspace = csp;
 
     p->have_image = false;
+    mp_image_unrefp(&p->image.hwimage);
 }
 
 void gl_video_set_output_depth(struct gl_video *p, int r, int g, int b)
@@ -2132,3 +2189,9 @@ void gl_video_resize_redraw(struct gl_video *p, int w, int h)
     gl_video_render_frame(p);
     mpgl_osd_redraw_cb(p->osd, draw_osd_cb, p);
 }
+
+void gl_video_set_hwdec(struct gl_video *p, struct gl_hwdec *hwdec)
+{
+    p->hwdec = hwdec;
+    mp_image_unrefp(&p->image.hwimage);
+}

video/out/gl_video.h

@@ -56,6 +56,7 @@ struct gl_video;
 struct gl_video *gl_video_init(GL *gl, struct mp_log *log);
 void gl_video_uninit(struct gl_video *p);
 void gl_video_set_options(struct gl_video *p, struct gl_video_opts *opts);
+bool gl_video_check_format(struct gl_video *p, int mp_format);
 void gl_video_config(struct gl_video *p, struct mp_image_params *params);
 void gl_video_set_output_depth(struct gl_video *p, int r, int g, int b);
 void gl_video_set_lut3d(struct gl_video *p, struct lut3d *lut3d);
@@ -73,6 +74,7 @@ bool gl_video_get_equalizer(struct gl_video *p, const char *name, int *val);
 void gl_video_set_debug(struct gl_video *p, bool enable);
 void gl_video_resize_redraw(struct gl_video *p, int w, int h);
 
-bool gl_video_check_format(int mp_format);
+struct gl_hwdec;
+void gl_video_set_hwdec(struct gl_video *p, struct gl_hwdec *hwdec);
 
 #endif

video/out/vo_opengl.c

@@ -48,6 +48,7 @@
 #include "gl_osd.h"
 #include "filter_kernels.h"
 #include "video/memcpy_pic.h"
+#include "video/decode/dec_video.h"
 #include "gl_video.h"
 #include "gl_lcms.h"
 
@@ -58,6 +59,8 @@ struct gl_priv {
 
     struct gl_video *renderer;
 
+    struct gl_hwdec *hwdec;
+
     // Options
     struct gl_video_opts *renderer_opts;
     struct mp_icc_opts *icc_opts;
@@ -134,8 +137,9 @@ static void draw_image(struct vo *vo, mp_image_t *mpi)
 
 static int query_format(struct vo *vo, uint32_t format)
 {
+    struct gl_priv *p = vo->priv;
     int caps = VFCAP_CSP_SUPPORTED | VFCAP_CSP_SUPPORTED_BY_HW | VFCAP_FLIP;
-    if (!gl_video_check_format(format))
+    if (!gl_video_check_format(p->renderer, format))
         return 0;
     return caps;
 }
@@ -191,6 +195,68 @@ static int reconfig(struct vo *vo, struct mp_image_params *params, int flags)
     return 0;
 }
 
+
+static void load_hwdec_driver(struct gl_priv *p,
+                              const struct gl_hwdec_driver *drv)
+{
+    assert(!p->hwdec);
+    struct gl_hwdec *hwdec = talloc(NULL, struct gl_hwdec);
+    *hwdec = (struct gl_hwdec) {
+        .driver = drv,
+        .log = mp_log_new(hwdec, p->vo->log, drv->api_name),
+        .mpgl = p->glctx,
+        .info = talloc_zero(hwdec, struct mp_hwdec_info),
+    };
+    mpgl_lock(p->glctx);
+    if (hwdec->driver->create(hwdec) < 0) {
+        mpgl_unlock(p->glctx);
+        talloc_free(hwdec);
+        MP_ERR(p->vo, "Couldn't load hwdec driver '%s'\n", drv->api_name);
+        return;
+    }
+    p->hwdec = hwdec;
+    gl_video_set_hwdec(p->renderer, p->hwdec);
+    mpgl_unlock(p->glctx);
+}
+
+static void request_hwdec_api(struct mp_hwdec_info *info, const char *api_name)
+{
+    struct gl_priv *p = info->load_api_ctx;
+    // Load at most one hwdec API
+    if (p->hwdec)
+        return;
+    for (int n = 0; mpgl_hwdec_drivers[n]; n++) {
+        const struct gl_hwdec_driver *drv = mpgl_hwdec_drivers[n];
+        if (api_name && strcmp(drv->api_name, api_name) == 0) {
+            load_hwdec_driver(p, drv);
+            if (p->hwdec) {
+                *info = *p->hwdec->info;
+                return;
+            }
+        }
+    }
+}
+
+static void get_hwdec_info(struct gl_priv *p, struct mp_hwdec_info *info)
+{
+    info->load_api = request_hwdec_api;
+    info->load_api_ctx = p;
+    if (p->hwdec)
+        *info = *p->hwdec->info;
+}
+
+static void unload_hwdec_driver(struct gl_priv *p)
+{
+    if (p->hwdec) {
+        mpgl_lock(p->glctx);
+        gl_video_set_hwdec(p->renderer, NULL);
+        p->hwdec->driver->destroy(p->hwdec);
+        talloc_free(p->hwdec);
+        p->hwdec = NULL;
+        mpgl_unlock(p->glctx);
+    }
+}
+
 static bool reparse_cmdline(struct gl_priv *p, char *args)
 {
     struct m_config *cfg = NULL;
@@ -263,6 +329,10 @@ static int control(struct vo *vo, uint32_t request, void *data)
         mpgl_unlock(p->glctx);
         return true;
     }
+    case VOCTRL_GET_HWDEC_INFO: {
+        get_hwdec_info(p, data);
+        return true;
+    }
     case VOCTRL_REDRAW_FRAME:
         mpgl_lock(p->glctx);
         gl_video_render_frame(p->renderer);
@@ -291,6 +361,7 @@ static void uninit(struct vo *vo)
     struct gl_priv *p = vo->priv;
 
     if (p->glctx) {
+        unload_hwdec_driver(p);
         if (p->renderer)
             gl_video_uninit(p->renderer);
         mpgl_uninit(p->glctx);