Set profile compatibility/constraint flags properly (including the
constraint flags used for RExt profiles, as all streams we can currently
generate are RExt-compatible), and use that to add support for the "Main
Intra" and "Main 10 Intra" RExt subprofiles (for which we can re-use the
existing Main and Main10 VAAPI profiles).
---
libavcodec/Makefile| 2 +-
libavcodec/vaapi_encode_h265.c | 70 ++
2 files changed, 56 insertions(+), 16 deletions(-)
diff --git a/libavcodec/Makefile b/libavcodec/Makefile
index d07a9073af..756779ec16 100644
--- a/libavcodec/Makefile
+++ b/libavcodec/Makefile
@@ -373,7 +373,7 @@ OBJS-$(CONFIG_HEVC_QSV_DECODER)+= qsvdec_h2645.o
OBJS-$(CONFIG_HEVC_QSV_ENCODER)+= qsvenc_hevc.o hevc_ps_enc.o \
hevc_data.o
OBJS-$(CONFIG_HEVC_RKMPP_DECODER) += rkmppdec.o
-OBJS-$(CONFIG_HEVC_VAAPI_ENCODER) += vaapi_encode_h265.o
+OBJS-$(CONFIG_HEVC_VAAPI_ENCODER) += h265_profile_level.o
vaapi_encode_h265.o
OBJS-$(CONFIG_HEVC_V4L2M2M_DECODER)+= v4l2_m2m_dec.o
OBJS-$(CONFIG_HEVC_V4L2M2M_ENCODER)+= v4l2_m2m_enc.o
OBJS-$(CONFIG_HNM4_VIDEO_DECODER) += hnm4video.o
diff --git a/libavcodec/vaapi_encode_h265.c b/libavcodec/vaapi_encode_h265.c
index 6940a59240..45576868df 100644
--- a/libavcodec/vaapi_encode_h265.c
+++ b/libavcodec/vaapi_encode_h265.c
@@ -23,6 +23,7 @@
#include "libavutil/avassert.h"
#include "libavutil/common.h"
+#include "libavutil/pixdesc.h"
#include "libavutil/opt.h"
#include "libavutil/mastering_display_metadata.h"
@@ -260,9 +261,12 @@ static int
vaapi_encode_h265_init_sequence_params(AVCodecContext *avctx)
H265RawVPS*vps = &priv->raw_vps;
H265RawSPS*sps = &priv->raw_sps;
H265RawPPS*pps = &priv->raw_pps;
+H265RawProfileTierLevel *ptl = &vps->profile_tier_level;
H265RawVUI*vui = &sps->vui;
VAEncSequenceParameterBufferHEVC *vseq = ctx->codec_sequence_params;
VAEncPictureParameterBufferHEVC *vpic = ctx->codec_picture_params;
+const AVPixFmtDescriptor *desc;
+int chroma_format, bit_depth;
int i;
memset(&priv->current_access_unit, 0,
@@ -273,6 +277,25 @@ static int
vaapi_encode_h265_init_sequence_params(AVCodecContext *avctx)
memset(pps, 0, sizeof(*pps));
+desc = av_pix_fmt_desc_get(priv->common.input_frames->sw_format);
+av_assert0(desc);
+if (desc->nb_components == 1) {
+chroma_format = 0;
+} else {
+if (desc->log2_chroma_w == 1 && desc->log2_chroma_h == 1) {
+chroma_format = 1;
+} else if (desc->log2_chroma_w == 1 && desc->log2_chroma_h == 0) {
+chroma_format = 2;
+} else if (desc->log2_chroma_w == 0 && desc->log2_chroma_h == 0) {
+chroma_format = 3;
+} else {
+av_log(avctx, AV_LOG_ERROR, "Chroma format of input pixel format "
+ "%s is not supported.\n", desc->name);
+}
+}
+bit_depth = desc->comp[0].depth;
+
+
// VPS
vps->nal_unit_header = (H265RawNALUnitHeader) {
@@ -289,19 +312,34 @@ static int
vaapi_encode_h265_init_sequence_params(AVCodecContext *avctx)
vps->vps_max_sub_layers_minus1 = 0;
vps->vps_temporal_id_nesting_flag = 1;
-vps->profile_tier_level = (H265RawProfileTierLevel) {
-.general_profile_space = 0,
-.general_profile_idc = avctx->profile,
-.general_tier_flag = 0,
+ptl->general_profile_space = 0;
+ptl->general_profile_idc = avctx->profile;
+ptl->general_tier_flag = 0;
-.general_progressive_source_flag= 1,
-.general_interlaced_source_flag = 0,
-.general_non_packed_constraint_flag = 1,
-.general_frame_only_constraint_flag = 1,
+if (chroma_format == 1) {
+ptl->general_profile_compatibility_flag[1] = bit_depth == 8;
+ptl->general_profile_compatibility_flag[2] = bit_depth <= 10;
+}
+ptl->general_profile_compatibility_flag[4] = 1;
-.general_level_idc = avctx->level,
-};
-vps->profile_tier_level.general_profile_compatibility_flag[avctx->profile
& 31] = 1;
+ptl->general_progressive_source_flag= 1;
+ptl->general_interlaced_source_flag = 0;
+ptl->general_non_packed_constraint_flag = 1;
+ptl->general_frame_only_constraint_flag = 1;
+
+ptl->general_max_12bit_constraint_flag = bit_depth <= 12;
+ptl->general_max_10bit_constraint_flag = bit_depth <= 10;
+ptl->general_max_8bit_constraint_flag = bit_depth == 8;
+
+ptl->general_max_422chroma_constraint_flag = chroma_format <= 2;
+ptl->general_max_420chroma_constraint_flag = chroma_format <= 1;
+ptl->general_max_monochrome_constraint_flag = chroma_format == 0;
+
+ptl->general_intra_constraint_flag = ctx->gop_size == 1;
+
+ptl->general_lower_bit_rate_constraint_fl
