VPE(Video Pipeline Engine) is VeriSilicon's hardware engine for multi formats video encoding and decoding.
It is used with the VPE hwaccel codec API and library to initialize and use a VPE device which is within the hwcontext libavutil framework. These VPE codecs are used in transcoding project - multiple transcoding boards are installed on one x86 server VIA PCIE interface to do video transcoding job, FFmpeg will run on the x86 server to control the codes which exist on transcoding board. Signed-off-by: Qin.Wang <qin.w...@verisilicon.com> --- configure | 3 + libavutil/Makefile | 3 + libavutil/hwcontext.c | 4 + libavutil/hwcontext.h | 1 + libavutil/hwcontext_internal.h | 1 + libavutil/hwcontext_vpe.c | 402 +++++++++++++++++++++++++++++++++ libavutil/hwcontext_vpe.h | 52 +++++ libavutil/pixdesc.c | 4 + libavutil/pixfmt.h | 7 + libavutil/version.h | 2 +- 10 files changed, 478 insertions(+), 1 deletion(-) create mode 100644 libavutil/hwcontext_vpe.c create mode 100644 libavutil/hwcontext_vpe.h diff --git a/configure b/configure index f97cad0298..cf7d1d9fcc 100755 --- a/configure +++ b/configure @@ -322,6 +322,7 @@ External library support: --enable-vulkan enable Vulkan code [no] --disable-xlib disable xlib [autodetect] --disable-zlib disable zlib [autodetect] + --enable-vpe enable vpe codec [no] The following libraries provide various hardware acceleration features: --disable-amf disable AMF video encoding code [autodetect] @@ -1822,6 +1823,7 @@ EXTERNAL_LIBRARY_LIST=" opengl pocketsphinx vapoursynth + vpe " HWACCEL_AUTODETECT_LIBRARY_LIST=" @@ -6475,6 +6477,7 @@ enabled rkmpp && { require_pkg_config rkmpp rockchip_mpp rockchip/r die "ERROR: rkmpp requires --enable-libdrm"; } } enabled vapoursynth && require_pkg_config vapoursynth "vapoursynth-script >= 42" VSScript.h vsscript_init +enabled vpe && require_pkg_config libvpi libvpi "vpe/vpi_types.h vpe/vpi_api.h" vpi_create if enabled gcrypt; then diff --git a/libavutil/Makefile b/libavutil/Makefile index 9b08372eb2..5809b38591 100644 --- a/libavutil/Makefile +++ b/libavutil/Makefile @@ -46,6 +46,7 @@ HEADERS = adler32.h \ hwcontext_videotoolbox.h \ hwcontext_vdpau.h \ hwcontext_vulkan.h \ + hwcontext_vpe.h \ imgutils.h \ intfloat.h \ intreadwrite.h \ @@ -184,6 +185,7 @@ OBJS-$(CONFIG_VAAPI) += hwcontext_vaapi.o OBJS-$(CONFIG_VIDEOTOOLBOX) += hwcontext_videotoolbox.o OBJS-$(CONFIG_VDPAU) += hwcontext_vdpau.o OBJS-$(CONFIG_VULKAN) += hwcontext_vulkan.o +OBJS-$(CONFIG_VPE) += hwcontext_vpe.o OBJS += $(COMPAT_OBJS:%=../compat/%) @@ -201,6 +203,7 @@ SKIPHEADERS-$(CONFIG_VAAPI) += hwcontext_vaapi.h SKIPHEADERS-$(CONFIG_VIDEOTOOLBOX) += hwcontext_videotoolbox.h SKIPHEADERS-$(CONFIG_VDPAU) += hwcontext_vdpau.h SKIPHEADERS-$(CONFIG_VULKAN) += hwcontext_vulkan.h +SKIPHEADERS-$(CONFIG_VPE) += hwcontext_vpe.h TESTPROGS = adler32 \ aes \ diff --git a/libavutil/hwcontext.c b/libavutil/hwcontext.c index d13d0f7c9b..5b12013286 100644 --- a/libavutil/hwcontext.c +++ b/libavutil/hwcontext.c @@ -61,6 +61,9 @@ static const HWContextType * const hw_table[] = { #endif #if CONFIG_VULKAN &ff_hwcontext_type_vulkan, +#endif +#if CONFIG_VPE + &ff_hwcontext_type_vpe, #endif NULL, }; @@ -77,6 +80,7 @@ static const char *const hw_type_names[] = { [AV_HWDEVICE_TYPE_VIDEOTOOLBOX] = "videotoolbox", [AV_HWDEVICE_TYPE_MEDIACODEC] = "mediacodec", [AV_HWDEVICE_TYPE_VULKAN] = "vulkan", + [AV_HWDEVICE_TYPE_VPE] = "vpe", }; enum AVHWDeviceType av_hwdevice_find_type_by_name(const char *name) diff --git a/libavutil/hwcontext.h b/libavutil/hwcontext.h index 04d19d89c2..62d948ec6c 100644 --- a/libavutil/hwcontext.h +++ b/libavutil/hwcontext.h @@ -37,6 +37,7 @@ enum AVHWDeviceType { AV_HWDEVICE_TYPE_OPENCL, AV_HWDEVICE_TYPE_MEDIACODEC, AV_HWDEVICE_TYPE_VULKAN, + AV_HWDEVICE_TYPE_VPE, }; typedef struct AVHWDeviceInternal AVHWDeviceInternal; diff --git a/libavutil/hwcontext_internal.h b/libavutil/hwcontext_internal.h index e6266494ac..3190a3a80f 100644 --- a/libavutil/hwcontext_internal.h +++ b/libavutil/hwcontext_internal.h @@ -174,5 +174,6 @@ extern const HWContextType ff_hwcontext_type_vdpau; extern const HWContextType ff_hwcontext_type_videotoolbox; extern const HWContextType ff_hwcontext_type_mediacodec; extern const HWContextType ff_hwcontext_type_vulkan; +extern const HWContextType ff_hwcontext_type_vpe; #endif /* AVUTIL_HWCONTEXT_INTERNAL_H */ diff --git a/libavutil/hwcontext_vpe.c b/libavutil/hwcontext_vpe.c new file mode 100644 index 0000000000..7b218e30a7 --- /dev/null +++ b/libavutil/hwcontext_vpe.c @@ -0,0 +1,402 @@ +/* + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include <stdlib.h> +#include <stdio.h> +#include <string.h> +#include <unistd.h> +#include <fcntl.h> +#include <errno.h> +#include <sys/ioctl.h> + +#include "buffer.h" +#include "pixfmt.h" +#include "pixdesc.h" +#include "hwcontext.h" +#include "hwcontext_internal.h" +#include "hwcontext_vpe.h" +#include "libavutil/opt.h" + +typedef struct VpeDevicePriv { + VpiSysInfo *sys_info; +} VpeDevicePriv; + +typedef struct VpeDeviceContext { + VpiMediaProc *media_proc; +} VpeDeviceContext; + +static const enum AVPixelFormat supported_sw_formats[] = { + AV_PIX_FMT_NV12, + AV_PIX_FMT_P010LE, + AV_PIX_FMT_YUV420P, + AV_PIX_FMT_YUV422P, + AV_PIX_FMT_NV21, + AV_PIX_FMT_YUV420P10LE, + AV_PIX_FMT_YUV420P10BE, + AV_PIX_FMT_YUV422P10LE, + AV_PIX_FMT_YUV422P10BE, + AV_PIX_FMT_P010BE, + AV_PIX_FMT_YUV444P, + AV_PIX_FMT_RGB24, + AV_PIX_FMT_BGR24, + AV_PIX_FMT_ARGB, + AV_PIX_FMT_RGBA, + AV_PIX_FMT_ABGR, + AV_PIX_FMT_BGRA, +}; + +static const enum AVPixelFormat supported_hw_formats[] = { + AV_PIX_FMT_VPE, +}; + +static int vpe_frames_get_constraints(AVHWDeviceContext *ctx, + const void *hwconfig, + AVHWFramesConstraints *constraints) +{ + int i; + + constraints->valid_sw_formats = + av_malloc_array(FF_ARRAY_ELEMS(supported_sw_formats) + 1, + sizeof(*constraints->valid_sw_formats)); + if (!constraints->valid_sw_formats) + return AVERROR(ENOMEM); + + for (i = 0; i < FF_ARRAY_ELEMS(supported_sw_formats); i++) + constraints->valid_sw_formats[i] = supported_sw_formats[i]; + constraints->valid_sw_formats[FF_ARRAY_ELEMS(supported_sw_formats)] = + AV_PIX_FMT_NONE; + + constraints->valid_hw_formats = + av_malloc_array(2, sizeof(*constraints->valid_hw_formats)); + if (!constraints->valid_hw_formats) { + return AVERROR(ENOMEM); + } + + constraints->valid_hw_formats[0] = AV_PIX_FMT_VPE; + constraints->valid_hw_formats[1] = AV_PIX_FMT_NONE; + + constraints->min_width = 144; + constraints->min_height = 144; + constraints->max_width = 4096; + constraints->max_height = 4096; + + return 0; +} + +static int vpe_transfer_get_formats(AVHWFramesContext *ctx, + enum AVHWFrameTransferDirection dir, + enum AVPixelFormat **formats) +{ + enum AVPixelFormat *fmts; + + fmts = av_malloc_array(2, sizeof(*fmts)); + if (!fmts) + return AVERROR(ENOMEM); + + fmts[0] = ctx->sw_format; + fmts[1] = AV_PIX_FMT_NONE; + + *formats = fmts; + + return 0; +} + +static int vpe_transfer_data_from(AVHWFramesContext *ctx, AVFrame *dst, + const AVFrame *src) +{ + VpeDeviceContext *priv = ctx->device_ctx->internal->priv; + VpiFrame *in_frame; + VpiFrame out_frame; + VpiCtrlCmdParam cmd_param; + + VpiPicInfo *pic_info; + int pp_index; + int i, ret; + + in_frame = (VpiFrame *)src->data[0]; + for (i = 1; i < PIC_INDEX_MAX_NUMBER; i++) { + pic_info = (VpiPicInfo *)src->buf[i]->data; + if (pic_info->enabled == 1) { + pp_index = i; + break; + } + } + if (i == PIC_INDEX_MAX_NUMBER) { + return AVERROR_EXTERNAL; + } + + cmd_param.cmd = VPI_CMD_HWDW_SET_INDEX; + cmd_param.data = (void *)&pp_index; + priv->media_proc->vpi->control(priv->media_proc->ctx, + (void *)&cmd_param, NULL); + + out_frame.data[0] = dst->data[0]; + out_frame.data[1] = dst->data[1]; + out_frame.data[2] = dst->data[2]; + out_frame.linesize[0] = dst->linesize[0]; + out_frame.linesize[1] = dst->linesize[1]; + out_frame.linesize[2] = dst->linesize[2]; + ret = priv->media_proc->vpi->process(priv->media_proc->ctx, + in_frame, &out_frame); + + return ret; +} + +static void vpe_buffer_free(void *opaque, uint8_t *data) +{ + AVHWFramesContext *ctx = opaque; + AVVpeDeviceContext *device_hwctx = ctx->device_ctx->hwctx; + VpiCtrlCmdParam cmd_param; + + cmd_param.cmd = VPI_CMD_FREE_FRAME_BUFFER; + cmd_param.data = (void *)data; + device_hwctx->func->control(&device_hwctx->device, &cmd_param, NULL); +} + +static AVBufferRef *vpe_pool_alloc(void *opaque, int size) +{ + AVHWFramesContext *ctx = opaque; + AVVpeDeviceContext *device_hwctx = ctx->device_ctx->hwctx; + VpiFrame *v_frame = NULL; + VpiCtrlCmdParam cmd_param; + AVBufferRef *ret; + + cmd_param.cmd = VPI_CMD_GET_FRAME_BUFFER; + device_hwctx->func->control(&device_hwctx->device, &cmd_param, (void *)&v_frame); + if (!v_frame) { + return NULL; + } + ret = av_buffer_create((uint8_t*)v_frame, size, + vpe_buffer_free, ctx, 0); + + return ret; +} + +static int vpe_frames_init(AVHWFramesContext *hwfc) +{ + AVVpeDeviceContext *device_hwctx = hwfc->device_ctx->hwctx; + AVVpeFramesContext *frame_hwctx = hwfc->hwctx; + VpiCtrlCmdParam cmd_param; + int size = 0; + int picinfo_size = 0; + int i; + + for (i = 0; i < FF_ARRAY_ELEMS(supported_sw_formats); i++) { + if (hwfc->sw_format == supported_sw_formats[i]) { + break; + } + } + if (i == FF_ARRAY_ELEMS(supported_sw_formats)) { + av_log(hwfc, AV_LOG_ERROR, "Pixel format '%s' is not supported\n", + av_get_pix_fmt_name(hwfc->sw_format)); + return AVERROR(ENOSYS); + } + + if (!hwfc->pool) { + cmd_param.cmd = VPI_CMD_GET_VPEFRAME_SIZE; + device_hwctx->func->control(&device_hwctx->device, &cmd_param, (void *)&size); + if (size == 0) { + return AVERROR_EXTERNAL; + } + frame_hwctx->frame = (VpiFrame *)av_mallocz(size); + if (!frame_hwctx->frame) { + return AVERROR(ENOMEM); + } + frame_hwctx->frame_size = size; + + cmd_param.cmd = VPI_CMD_GET_PICINFO_SIZE; + device_hwctx->func->control(&device_hwctx->device, &cmd_param, (void *)&picinfo_size); + if (picinfo_size == 0) { + return AVERROR_EXTERNAL; + } + frame_hwctx->pic_info_size = picinfo_size; + + hwfc->internal->pool_internal = + av_buffer_pool_init2(size, hwfc, vpe_pool_alloc, NULL); + if (!hwfc->internal->pool_internal) + return AVERROR(ENOMEM); + } + return 0; +} + +static int vpe_get_buffer(AVHWFramesContext *hwfc, AVFrame *frame) +{ + AVVpeFramesContext *vpeframe_ctx = hwfc->hwctx; + VpiPicInfo *pic_info; + int i; + + frame->buf[0] = av_buffer_pool_get(hwfc->pool); + if (!frame->buf[0]) + return AVERROR(ENOMEM); + + frame->data[0] = frame->buf[0]->data; + memset(frame->data[0], 0, vpeframe_ctx->frame_size); + + /* suppor max 4 channel low res buffer */ + for (i = 1; i < PIC_INDEX_MAX_NUMBER; i++) { + frame->buf[i] = av_buffer_alloc(vpeframe_ctx->pic_info_size); + if (frame->buf[i] == 0) { + return AVERROR(ENOMEM); + } + } + /* always enable the first channel */ + pic_info = (VpiPicInfo *)frame->buf[1]->data; + pic_info->enabled = 1; + + frame->format = AV_PIX_FMT_VPE; + frame->width = hwfc->width; + frame->height = hwfc->height; + return 0; +} + +static void vpe_device_uninit(AVHWDeviceContext *device_ctx) +{ + VpeDeviceContext *priv = device_ctx->internal->priv; + + if (priv->media_proc) { + if (priv->media_proc->ctx) { + priv->media_proc->vpi->close(priv->media_proc->ctx); + vpi_destroy(priv->media_proc->ctx); + priv->media_proc->ctx = NULL; + } + vpi_freep(&priv->media_proc); + } +} + +static int vpe_device_init(AVHWDeviceContext *device_ctx) +{ + VpeDeviceContext *priv = device_ctx->internal->priv; + int ret = 0; + + ret = vpi_get_media_proc_struct(&priv->media_proc); + if (ret || !priv->media_proc) { + return AVERROR(ENOMEM); + } + + ret = vpi_create(&priv->media_proc->ctx, &priv->media_proc->vpi, + HWDOWNLOAD_VPE); + if (ret != 0) + return AVERROR_EXTERNAL; + + ret = priv->media_proc->vpi->init(priv->media_proc->ctx, NULL); + if (ret != 0) + return AVERROR_EXTERNAL; + + return 0; +} + +static void vpe_device_free(AVHWDeviceContext *device_ctx) +{ + AVVpeDeviceContext *hwctx = device_ctx->hwctx; + VpeDevicePriv *priv = (VpeDevicePriv *)device_ctx->user_opaque; + + vpi_destroy(priv->sys_info); + + vpi_freep(&priv->sys_info); + av_freep(&priv); + + if (hwctx->device >= 0) { + close(hwctx->device); + } +} + +static int vpe_device_create(AVHWDeviceContext *device_ctx, const char *device, + AVDictionary *opts, int flags) +{ + AVVpeDeviceContext *hwctx = device_ctx->hwctx; + AVDictionaryEntry *opt; + VpeDevicePriv *priv; + const char *path; + int ret; + + priv = av_mallocz(sizeof(*priv)); + if (!priv) + return AVERROR(ENOMEM); + ret = vpi_get_sys_info_struct(&priv->sys_info); + if (ret || !priv->sys_info) { + av_freep(&priv); + return AVERROR(ENOMEM); + } + + device_ctx->user_opaque = priv; + device_ctx->free = vpe_device_free; + + // Try to open the device as a transcode path. + // Default to using the first node if the user did not + // supply a path. + path = device ? device : "/dev/transcoder0"; + hwctx->device = open(path, O_RDWR); + if (hwctx->device == -1) { + av_log(device_ctx, AV_LOG_ERROR, "failed to open hw device\n"); + return AVERROR(errno); + } + + priv->sys_info->device = hwctx->device; + priv->sys_info->device_name = av_strdup(path); + if (!priv->sys_info->device_name) { + return AVERROR(ENOMEM); + } + priv->sys_info->priority = VPE_TASK_VOD; + priv->sys_info->sys_log_level = 0; + + if (opts) { + opt = av_dict_get(opts, "priority", NULL, 0); + if (opt) { + if (!strcmp(opt->value, "live")) { + priv->sys_info->priority = VPE_TASK_LIVE; + } else if (!strcmp(opt->value, "vod")) { + priv->sys_info->priority = VPE_TASK_VOD; + } else { + av_log(device_ctx, AV_LOG_ERROR, "Unknow priority : %s\n", + opt->value); + return AVERROR_INVALIDDATA; + } + } + + opt = av_dict_get(opts, "fbloglevel", NULL, 0); + if (opt) { + priv->sys_info->sys_log_level = atoi(opt->value); + } + } + + if (vpi_create(&priv->sys_info, &hwctx->func, HWCONTEXT_VPE) != 0) { + return AVERROR_EXTERNAL; + } + + return 0; +} + +const HWContextType ff_hwcontext_type_vpe = { + .type = AV_HWDEVICE_TYPE_VPE, + .name = "VPE", + + .device_hwctx_size = sizeof(AVVpeDeviceContext), + .device_priv_size = sizeof(VpeDeviceContext), + .frames_hwctx_size = sizeof(AVVpeFramesContext), + + .device_create = vpe_device_create, + .device_init = vpe_device_init, + .device_uninit = vpe_device_uninit, + .frames_get_constraints = vpe_frames_get_constraints, + .frames_init = vpe_frames_init, + .frames_get_buffer = vpe_get_buffer, + .transfer_get_formats = vpe_transfer_get_formats, + .transfer_data_from = vpe_transfer_data_from, + + .pix_fmts = (const enum AVPixelFormat[]){ AV_PIX_FMT_VPE, AV_PIX_FMT_NONE }, +}; diff --git a/libavutil/hwcontext_vpe.h b/libavutil/hwcontext_vpe.h new file mode 100644 index 0000000000..6fd5291e45 --- /dev/null +++ b/libavutil/hwcontext_vpe.h @@ -0,0 +1,52 @@ +/* + * Verisilicon VPI Video codec + * Copyright (c) 2019 VeriSilicon, Inc. + * + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#ifndef AVUTIL_HWCONTEXT_VPE_H +#define AVUTIL_HWCONTEXT_VPE_H + +#include <vpe/vpi_api.h> +#include <vpe/vpi_types.h> + +/** + * @file + * An API-specific header for AV_HWDEVICE_TYPE_VPE. + */ + +/** + * This struct is allocated as AVHWDeviceContext.hwctx + * It will save some device level info + */ + +typedef struct AVVpeDeviceContext { + int device; + VpiApi *func; +} AVVpeDeviceContext; + +/** + * This struct is allocated as AVHWFramesContext.hwctx + * It will save some frame level info + */ +typedef struct AVVpeFramesContext { + VpiFrame *frame; + int frame_size; + int pic_info_size; +} AVVpeFramesContext; +#endif /* AVUTIL_HWCONTEXT_VPE_H */ diff --git a/libavutil/pixdesc.c b/libavutil/pixdesc.c index 9d61c52567..f2dc9facd3 100644 --- a/libavutil/pixdesc.c +++ b/libavutil/pixdesc.c @@ -2371,6 +2371,10 @@ static const AVPixFmtDescriptor av_pix_fmt_descriptors[AV_PIX_FMT_NB] = { .name = "vulkan", .flags = AV_PIX_FMT_FLAG_HWACCEL, }, + [AV_PIX_FMT_VPE] = { + .name = "vpe", + .flags = AV_PIX_FMT_FLAG_HWACCEL, + }, }; #if FF_API_PLUS1_MINUS1 FF_ENABLE_DEPRECATION_WARNINGS diff --git a/libavutil/pixfmt.h b/libavutil/pixfmt.h index 1c625cfc8a..252261e867 100644 --- a/libavutil/pixfmt.h +++ b/libavutil/pixfmt.h @@ -358,6 +358,13 @@ enum AVPixelFormat { AV_PIX_FMT_Y210BE, ///< packed YUV 4:2:2 like YUYV422, 20bpp, data in the high bits, big-endian AV_PIX_FMT_Y210LE, ///< packed YUV 4:2:2 like YUYV422, 20bpp, data in the high bits, little-endian + /** + * VPE hardware images. + * + * data[0] points to a VpiFrame + */ + AV_PIX_FMT_VPE, + AV_PIX_FMT_NB ///< number of pixel formats, DO NOT USE THIS if you want to link with shared libav* because the number of formats might differ between versions }; diff --git a/libavutil/version.h b/libavutil/version.h index 7acecf5a97..5d5cec62ab 100644 --- a/libavutil/version.h +++ b/libavutil/version.h @@ -79,7 +79,7 @@ */ #define LIBAVUTIL_VERSION_MAJOR 56 -#define LIBAVUTIL_VERSION_MINOR 49 +#define LIBAVUTIL_VERSION_MINOR 50 #define LIBAVUTIL_VERSION_MICRO 100 #define LIBAVUTIL_VERSION_INT AV_VERSION_INT(LIBAVUTIL_VERSION_MAJOR, \ -- 2.19.1 _______________________________________________ ffmpeg-devel mailing list ffmpeg-devel@ffmpeg.org https://ffmpeg.org/mailman/listinfo/ffmpeg-devel To unsubscribe, visit link above, or email ffmpeg-devel-requ...@ffmpeg.org with subject "unsubscribe".