Re: [FFmpeg-devel] [PATCH 2/2] avfilter/vf_vpp_qsv: apply 3D LUT from file.
Thanks for your comments.
I'll use MFX_RESOURCE_SYSTEM_SURFACE and send another patch.
While implementing this feature, I noticed that
vpp_set_frame_ext_params is called multiple times.
If using system memory for storing 3D LUT, is it possible the LUT
table copying to gfx memory will occur multiple times?
On Mon, Oct 16, 2023 at 4:05 PM Xiang, Haihao wrote:
>
> On Sa, 2023-09-23 at 23:36 +0800, Chen Yufei wrote:
> > Usage: "vpp_qsv=lut3d_file="
> >
> > Only enabled with VAAPI because using VASurface to store 3D LUT.
> >
> > Signed-off-by: Chen Yufei
> > ---
> > libavfilter/vf_vpp_qsv.c | 241 ++-
> > 1 file changed, 236 insertions(+), 5 deletions(-)
> >
> > diff --git a/libavfilter/vf_vpp_qsv.c b/libavfilter/vf_vpp_qsv.c
> > index c07b45fedb..cd913d3c40 100644
> > --- a/libavfilter/vf_vpp_qsv.c
> > +++ b/libavfilter/vf_vpp_qsv.c
> > @@ -23,6 +23,7 @@
> >
> > #include
> >
> > +#include "config.h"
> > #include "config_components.h"
> >
> > #include "libavutil/opt.h"
> > @@ -37,10 +38,15 @@
> > #include "internal.h"
> > #include "avfilter.h"
> > #include "filters.h"
> > +#include "lut3d.h"
> >
> > #include "qsvvpp.h"
> > #include "transpose.h"
> >
> > +#if QSV_ONEVPL && CONFIG_VAAPI
> > +#include
> > +#endif
>
> VA-API is available on Windows now, however oneVPL can't work with VA-API on
> Windows.
>
> I'd prefer to support MFX_RESOURCE_SYSTEM_SURFACE instead of
> MFX_RESOURCE_VA_SURFACE in FFmpeg because we neend't consider VA-API too much
> for MFX_RESOURCE_SYSTEM_SURFACE. oneVPL should be able to copy data from
> system
> memory to gfx memory internally.
>
> Thanks
> Haihao
>
>
> > +
> > #define OFFSET(x) offsetof(VPPContext, x)
> > #define FLAGS (AV_OPT_FLAG_VIDEO_PARAM | AV_OPT_FLAG_FILTERING_PARAM)
> >
> > @@ -67,6 +73,10 @@ typedef struct VPPContext{
> > /** HDR parameters attached on the input frame */
> > mfxExtMasteringDisplayColourVolume mdcv_conf;
> > mfxExtContentLightLevelInfo clli_conf;
> > +
> > +/** LUT parameters attached on the input frame */
> > +mfxExtVPP3DLut lut3d_conf;
> > +LUT3DContext lut3d;
> > #endif
> >
> > /**
> > @@ -260,6 +270,7 @@ static av_cold int vpp_preinit(AVFilterContext *ctx)
> >
> > static av_cold int vpp_init(AVFilterContext *ctx)
> > {
> > +int ret = 0;
> > VPPContext *vpp = ctx->priv;
> >
> > if (!vpp->output_format_str || !strcmp(vpp->output_format_str,
> > "same")) {
> > @@ -288,9 +299,9 @@ static av_cold int vpp_init(AVFilterContext *ctx)
> > STRING_OPTION(color_primaries, color_primaries, AVCOL_PRI_UNSPECIFIED);
> > STRING_OPTION(color_transfer, color_transfer, AVCOL_TRC_UNSPECIFIED);
> > STRING_OPTION(color_matrix,color_space, AVCOL_SPC_UNSPECIFIED);
> > -
> > #undef STRING_OPTION
> > -return 0;
> > +
> > +return ret;
> > }
> >
> > static int config_input(AVFilterLink *inlink)
> > @@ -388,6 +399,194 @@ static mfxStatus get_mfx_version(const AVFilterContext
> > *ctx, mfxVersion *mfx_ver
> > return MFXQueryVersion(device_hwctx->session, mfx_version);
> > }
> >
> > +#if QSV_ONEVPL && CONFIG_VAAPI
> > +static mfxStatus get_va_display(AVFilterContext *ctx, VADisplay
> > *va_display)
> > +{
> > +VPPContext *vpp = ctx->priv;
> > +QSVVPPContext *qsvvpp = >qsv;
> > +mfxHDL handle;
> > +mfxStatus ret;
> > +
> > +ret = MFXVideoCORE_GetHandle(qsvvpp->session, MFX_HANDLE_VA_DISPLAY,
> > );
> > +if (ret != MFX_ERR_NONE) {
> > +av_log(ctx, AV_LOG_ERROR, "MFXVideoCORE_GetHandle failed, status:
> > %d\n", ret);
> > +*va_display = NULL;
> > +return ret;
> > +}
> > +
> > +*va_display = (VADisplay)handle;
> > +return MFX_ERR_NONE;
> > +}
> > +
> > +// Allocate memory on device and copy 3D LUT table.
> > +// Reference
> > https://spec.oneapi.io/onevpl/2.9.0/programming_guide/VPL_prg_vpp.html#video-processing-3dlut
> > +static int init_3dlut_surface(AVFilterContext *ctx)
> > +{
> > +VPPContext *vpp = ctx->priv;
> > +LUT3DContext *lut3d = >lut3d;
> > +mfxExtVPP3DLut *lut3d_conf = >lut3d_conf;
> > +
> > +VAStatus ret = 0;
> > +VADisplay va_dpy = 0;
> > +VASurfaceID surface_id = 0;
> > +VASurfaceAttrib surface_attrib;
> > +VAImage surface_image;
> > +mfxU16 *surface_u16 = NULL;
> > +mfx3DLutMemoryLayout mem_layout;
> > +mfxMemId mem_id = 0;
> > +
> > +int lut_size = lut3d->lutsize;
> > +int mul_size = 0;
> > +
> > +int r, g, b, lut_idx, sf_idx;
> > +struct rgbvec *s = NULL;
> > +
> > +av_log(ctx, AV_LOG_VERBOSE, "create 3D LUT surface, size: %u.\n",
> > lut_size);
> > +
> > +switch (lut_size) {
> > +case 17:
> > +mul_size = 32;
> > +mem_layout = MFX_3DLUT_MEMORY_LAYOUT_INTEL_17LUT;
> > +break;
> > +case 33:
> > +mul_size = 64;
> > +mem_layout = MFX_3DLUT_MEMORY_LAYOUT_INTEL_33LUT;
> > +break;
> > +case 65:
> > +mul_size = 128;
> > +
Re: [FFmpeg-devel] [PATCH 2/2] avfilter/vf_vpp_qsv: apply 3D LUT from file.
On Sa, 2023-09-23 at 23:36 +0800, Chen Yufei wrote:
> Usage: "vpp_qsv=lut3d_file="
>
> Only enabled with VAAPI because using VASurface to store 3D LUT.
>
> Signed-off-by: Chen Yufei
> ---
> libavfilter/vf_vpp_qsv.c | 241 ++-
> 1 file changed, 236 insertions(+), 5 deletions(-)
>
> diff --git a/libavfilter/vf_vpp_qsv.c b/libavfilter/vf_vpp_qsv.c
> index c07b45fedb..cd913d3c40 100644
> --- a/libavfilter/vf_vpp_qsv.c
> +++ b/libavfilter/vf_vpp_qsv.c
> @@ -23,6 +23,7 @@
>
> #include
>
> +#include "config.h"
> #include "config_components.h"
>
> #include "libavutil/opt.h"
> @@ -37,10 +38,15 @@
> #include "internal.h"
> #include "avfilter.h"
> #include "filters.h"
> +#include "lut3d.h"
>
> #include "qsvvpp.h"
> #include "transpose.h"
>
> +#if QSV_ONEVPL && CONFIG_VAAPI
> +#include
> +#endif
VA-API is available on Windows now, however oneVPL can't work with VA-API on
Windows.
I'd prefer to support MFX_RESOURCE_SYSTEM_SURFACE instead of
MFX_RESOURCE_VA_SURFACE in FFmpeg because we neend't consider VA-API too much
for MFX_RESOURCE_SYSTEM_SURFACE. oneVPL should be able to copy data from system
memory to gfx memory internally.
Thanks
Haihao
> +
> #define OFFSET(x) offsetof(VPPContext, x)
> #define FLAGS (AV_OPT_FLAG_VIDEO_PARAM | AV_OPT_FLAG_FILTERING_PARAM)
>
> @@ -67,6 +73,10 @@ typedef struct VPPContext{
> /** HDR parameters attached on the input frame */
> mfxExtMasteringDisplayColourVolume mdcv_conf;
> mfxExtContentLightLevelInfo clli_conf;
> +
> + /** LUT parameters attached on the input frame */
> + mfxExtVPP3DLut lut3d_conf;
> + LUT3DContext lut3d;
> #endif
>
> /**
> @@ -260,6 +270,7 @@ static av_cold int vpp_preinit(AVFilterContext *ctx)
>
> static av_cold int vpp_init(AVFilterContext *ctx)
> {
> + int ret = 0;
> VPPContext *vpp = ctx->priv;
>
> if (!vpp->output_format_str || !strcmp(vpp->output_format_str, "same")) {
> @@ -288,9 +299,9 @@ static av_cold int vpp_init(AVFilterContext *ctx)
> STRING_OPTION(color_primaries, color_primaries, AVCOL_PRI_UNSPECIFIED);
> STRING_OPTION(color_transfer, color_transfer, AVCOL_TRC_UNSPECIFIED);
> STRING_OPTION(color_matrix, color_space, AVCOL_SPC_UNSPECIFIED);
> -
> #undef STRING_OPTION
> - return 0;
> +
> + return ret;
> }
>
> static int config_input(AVFilterLink *inlink)
> @@ -388,6 +399,194 @@ static mfxStatus get_mfx_version(const AVFilterContext
> *ctx, mfxVersion *mfx_ver
> return MFXQueryVersion(device_hwctx->session, mfx_version);
> }
>
> +#if QSV_ONEVPL && CONFIG_VAAPI
> +static mfxStatus get_va_display(AVFilterContext *ctx, VADisplay *va_display)
> +{
> + VPPContext *vpp = ctx->priv;
> + QSVVPPContext *qsvvpp = >qsv;
> + mfxHDL handle;
> + mfxStatus ret;
> +
> + ret = MFXVideoCORE_GetHandle(qsvvpp->session, MFX_HANDLE_VA_DISPLAY,
> );
> + if (ret != MFX_ERR_NONE) {
> + av_log(ctx, AV_LOG_ERROR, "MFXVideoCORE_GetHandle failed, status:
> %d\n", ret);
> + *va_display = NULL;
> + return ret;
> + }
> +
> + *va_display = (VADisplay)handle;
> + return MFX_ERR_NONE;
> +}
> +
> +// Allocate memory on device and copy 3D LUT table.
> +// Reference
> https://spec.oneapi.io/onevpl/2.9.0/programming_guide/VPL_prg_vpp.html#video-processing-3dlut
> +static int init_3dlut_surface(AVFilterContext *ctx)
> +{
> + VPPContext *vpp = ctx->priv;
> + LUT3DContext *lut3d = >lut3d;
> + mfxExtVPP3DLut *lut3d_conf = >lut3d_conf;
> +
> + VAStatus ret = 0;
> + VADisplay va_dpy = 0;
> + VASurfaceID surface_id = 0;
> + VASurfaceAttrib surface_attrib;
> + VAImage surface_image;
> + mfxU16 *surface_u16 = NULL;
> + mfx3DLutMemoryLayout mem_layout;
> + mfxMemId mem_id = 0;
> +
> + int lut_size = lut3d->lutsize;
> + int mul_size = 0;
> +
> + int r, g, b, lut_idx, sf_idx;
> + struct rgbvec *s = NULL;
> +
> + av_log(ctx, AV_LOG_VERBOSE, "create 3D LUT surface, size: %u.\n",
> lut_size);
> +
> + switch (lut_size) {
> + case 17:
> + mul_size = 32;
> + mem_layout = MFX_3DLUT_MEMORY_LAYOUT_INTEL_17LUT;
> + break;
> + case 33:
> + mul_size = 64;
> + mem_layout = MFX_3DLUT_MEMORY_LAYOUT_INTEL_33LUT;
> + break;
> + case 65:
> + mul_size = 128;
> + mem_layout = MFX_3DLUT_MEMORY_LAYOUT_INTEL_65LUT;
> + break;
> + default:
> + av_log(ctx, AV_LOG_ERROR, "3D LUT surface supports only LUT size: 17,
> 33, 65.");
> + return AVERROR(EINVAL);
> + }
> +
> + ret = get_va_display(ctx, _dpy);
> + if (ret != VA_STATUS_SUCCESS) {
> + av_log(ctx, AV_LOG_ERROR, "get VADisplay failed, unable to create 3D
> LUT surface.\n");
> + return ret;
> + }
> +
> + memset(_attrib, 0, sizeof(surface_attrib));
> + surface_attrib.type = VASurfaceAttribPixelFormat;
> + surface_attrib.flags = VA_SURFACE_ATTRIB_SETTABLE;
> + surface_attrib.value.type =
[FFmpeg-devel] [PATCH 2/2] avfilter/vf_vpp_qsv: apply 3D LUT from file.
Usage: "vpp_qsv=lut3d_file="
Only enabled with VAAPI because using VASurface to store 3D LUT.
Signed-off-by: Chen Yufei
---
libavfilter/vf_vpp_qsv.c | 241 ++-
1 file changed, 236 insertions(+), 5 deletions(-)
diff --git a/libavfilter/vf_vpp_qsv.c b/libavfilter/vf_vpp_qsv.c
index c07b45fedb..cd913d3c40 100644
--- a/libavfilter/vf_vpp_qsv.c
+++ b/libavfilter/vf_vpp_qsv.c
@@ -23,6 +23,7 @@
#include
+#include "config.h"
#include "config_components.h"
#include "libavutil/opt.h"
@@ -37,10 +38,15 @@
#include "internal.h"
#include "avfilter.h"
#include "filters.h"
+#include "lut3d.h"
#include "qsvvpp.h"
#include "transpose.h"
+#if QSV_ONEVPL && CONFIG_VAAPI
+#include
+#endif
+
#define OFFSET(x) offsetof(VPPContext, x)
#define FLAGS (AV_OPT_FLAG_VIDEO_PARAM | AV_OPT_FLAG_FILTERING_PARAM)
@@ -67,6 +73,10 @@ typedef struct VPPContext{
/** HDR parameters attached on the input frame */
mfxExtMasteringDisplayColourVolume mdcv_conf;
mfxExtContentLightLevelInfo clli_conf;
+
+/** LUT parameters attached on the input frame */
+mfxExtVPP3DLut lut3d_conf;
+LUT3DContext lut3d;
#endif
/**
@@ -260,6 +270,7 @@ static av_cold int vpp_preinit(AVFilterContext *ctx)
static av_cold int vpp_init(AVFilterContext *ctx)
{
+int ret = 0;
VPPContext *vpp = ctx->priv;
if (!vpp->output_format_str || !strcmp(vpp->output_format_str, "same")) {
@@ -288,9 +299,9 @@ static av_cold int vpp_init(AVFilterContext *ctx)
STRING_OPTION(color_primaries, color_primaries, AVCOL_PRI_UNSPECIFIED);
STRING_OPTION(color_transfer, color_transfer, AVCOL_TRC_UNSPECIFIED);
STRING_OPTION(color_matrix,color_space, AVCOL_SPC_UNSPECIFIED);
-
#undef STRING_OPTION
-return 0;
+
+return ret;
}
static int config_input(AVFilterLink *inlink)
@@ -388,6 +399,194 @@ static mfxStatus get_mfx_version(const AVFilterContext
*ctx, mfxVersion *mfx_ver
return MFXQueryVersion(device_hwctx->session, mfx_version);
}
+#if QSV_ONEVPL && CONFIG_VAAPI
+static mfxStatus get_va_display(AVFilterContext *ctx, VADisplay *va_display)
+{
+VPPContext *vpp = ctx->priv;
+QSVVPPContext *qsvvpp = >qsv;
+mfxHDL handle;
+mfxStatus ret;
+
+ret = MFXVideoCORE_GetHandle(qsvvpp->session, MFX_HANDLE_VA_DISPLAY,
);
+if (ret != MFX_ERR_NONE) {
+av_log(ctx, AV_LOG_ERROR, "MFXVideoCORE_GetHandle failed, status:
%d\n", ret);
+*va_display = NULL;
+return ret;
+}
+
+*va_display = (VADisplay)handle;
+return MFX_ERR_NONE;
+}
+
+// Allocate memory on device and copy 3D LUT table.
+// Reference
https://spec.oneapi.io/onevpl/2.9.0/programming_guide/VPL_prg_vpp.html#video-processing-3dlut
+static int init_3dlut_surface(AVFilterContext *ctx)
+{
+VPPContext *vpp = ctx->priv;
+LUT3DContext *lut3d = >lut3d;
+mfxExtVPP3DLut *lut3d_conf = >lut3d_conf;
+
+VAStatus ret = 0;
+VADisplay va_dpy = 0;
+VASurfaceID surface_id = 0;
+VASurfaceAttrib surface_attrib;
+VAImage surface_image;
+mfxU16 *surface_u16 = NULL;
+mfx3DLutMemoryLayout mem_layout;
+mfxMemId mem_id = 0;
+
+int lut_size = lut3d->lutsize;
+int mul_size = 0;
+
+int r, g, b, lut_idx, sf_idx;
+struct rgbvec *s = NULL;
+
+av_log(ctx, AV_LOG_VERBOSE, "create 3D LUT surface, size: %u.\n",
lut_size);
+
+switch (lut_size) {
+case 17:
+mul_size = 32;
+mem_layout = MFX_3DLUT_MEMORY_LAYOUT_INTEL_17LUT;
+break;
+case 33:
+mul_size = 64;
+mem_layout = MFX_3DLUT_MEMORY_LAYOUT_INTEL_33LUT;
+break;
+case 65:
+mul_size = 128;
+mem_layout = MFX_3DLUT_MEMORY_LAYOUT_INTEL_65LUT;
+break;
+default:
+av_log(ctx, AV_LOG_ERROR, "3D LUT surface supports only LUT size: 17,
33, 65.");
+return AVERROR(EINVAL);
+}
+
+ret = get_va_display(ctx, _dpy);
+if (ret != VA_STATUS_SUCCESS) {
+av_log(ctx, AV_LOG_ERROR, "get VADisplay failed, unable to create 3D
LUT surface.\n");
+return ret;
+}
+
+memset(_attrib, 0, sizeof(surface_attrib));
+surface_attrib.type = VASurfaceAttribPixelFormat;
+surface_attrib.flags = VA_SURFACE_ATTRIB_SETTABLE;
+surface_attrib.value.type = VAGenericValueTypeInteger;
+surface_attrib.value.value.i = VA_FOURCC_RGBA;
+
+ret = vaCreateSurfaces(va_dpy,
+ VA_RT_FORMAT_RGB32, // 4 bytes
+ lut_size * mul_size, // width
+ lut_size * 2,// height
+ _id, 1,
+ _attrib, 1);
+if (ret != VA_STATUS_SUCCESS) {
+av_log(ctx, AV_LOG_ERROR, "vaCreateSurfaces for 3D LUT surface failed,
status: %d %s\n", ret, vaErrorStr(ret));
+return AVERROR(ret);
+}
+av_log(ctx, AV_LOG_DEBUG, "3D LUT surface id %u\n", surface_id);
+
+ret = vaSyncSurface(va_dpy, surface_id);
+if (ret !=
