PR #25824 opened by Zhao Zhili (quink)
URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/25824
Patch URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/25824.patch
# Summary of changes
Encoders often split a tile into several slices to bound slice size.
Such pictures were decoded in tile scan order, which completes no CTB
row before the last tile is decoded, so frame threading could not
overlap pictures.
A slice without entry points never crosses a tile boundary, so every
independent slice without entry points can be deferred. If the deferred
slices of a picture arrive in increasing tile scan address and every
tile begins with a slice at its first CTB, they are decoded together in
row major order. Otherwise they are decoded in their original order.
The decoded output is the same either way.
Frame threading decode fps, AMD EPYC 9K84 (gcc 13). Three 7680x3840
HEVC streams (A, B, C) with 3x1 tiles split into two slices per tile.
"before" is the tile scan order (tile_interleave off); "after" is
this change. The last row is the ratio of the 32-thread to the
1-thread fps.
```
| A | B | C
threads | before | after | before | after | before | after
1 | 16.3 | 16.4 | 15.3 | 15.4 | 11.8 | 11.9
2 | 22.8 | 31.2 | 21.2 | 29.0 | 16.8 | 22.8
4 | 24.3 | 58.8 | 22.5 | 51.7 | 17.9 | 42.1
8 | 28.0 | 100.2 | 25.8 | 83.6 | 20.6 | 72.2
16 | 39.4 | 127.8 | 36.2 | 106.8 | 29.3 | 99.8
32 | 41.4 | 149.1 | 38.9 | 135.2 | 30.6 | 123.7
32/1 | 2.5x | 9.1x | 2.5x | 8.8x | 2.6x | 10.4x
```
>From 2f4ade07e43374892a559e2d65ad662503cbdaf1 Mon Sep 17 00:00:00 2001
From: Zhao Zhili <[email protected]>
Date: Fri, 9 Oct 2026 15:30:19 +0800
Subject: [PATCH 1/2] lavc/hevcdec: describe each tile by a list of segments
A segment is one CABAC substream together with the state of the slice
it belongs to. decode_tiles_row_major() assumes that a tile consists of
a single segment, which holds when a picture's tiles share a slice
through entry points or carry one slice each. A tile coded as several
slices needs one segment per slice. Describe each tile by a list of
segments instead.
No functional change.
Signed-off-by: Zhao Zhili <[email protected]>
---
libavcodec/hevc/hevcdec.c | 92 ++++++++++++++++++++++++++++-----------
libavcodec/hevc/hevcdec.h | 25 ++++++++---
2 files changed, 84 insertions(+), 33 deletions(-)
diff --git a/libavcodec/hevc/hevcdec.c b/libavcodec/hevc/hevcdec.c
index faee53c5094..64cfdc481ed 100644
--- a/libavcodec/hevc/hevcdec.c
+++ b/libavcodec/hevc/hevcdec.c
@@ -3126,6 +3126,21 @@ static int alloc_tile_streams(HEVCContext *s, unsigned n)
return 0;
}
+static int alloc_tile_segments(HEVCContext *s, unsigned n)
+{
+ if (n > s->tile_segments_alloc) {
+ void *tmp = av_realloc_array(s->tile_segments, n,
sizeof(*s->tile_segments));
+
+ if (!tmp)
+ return AVERROR(ENOMEM);
+
+ s->tile_segments = tmp;
+ s->tile_segments_alloc = n;
+ }
+
+ return 0;
+}
+
static int alloc_substreams(HEVCContext *s, unsigned n)
{
if (n > s->nb_substreams) {
@@ -3144,9 +3159,10 @@ static int alloc_substreams(HEVCContext *s, unsigned n)
return 0;
}
-/* Walk the picture in raster order. A band of tile rows has one tile per
- * column, so one substream state per column is enough; lc switches state
- * when the column changes. */
+/* One CABAC state per tile column suffices: tile_id is assigned in tile row
+ * major order, so tile % num_tile_columns is the column. Within a tile the
+ * walk visits CTBs in increasing ts, so each segment is entered at its
+ * first CTB. */
static int decode_tiles_row_major(HEVCContext *s, const HEVCLayerContext *l,
HEVCTileStream *tiles)
{
@@ -3162,8 +3178,15 @@ static int decode_tiles_row_major(HEVCContext *s, const
HEVCLayerContext *l,
const int tile = pps->tile_id[ts];
HEVCTileStream *const t = &tiles[tile];
HEVCSubstream *const sub = &s->substreams[tile %
pps->num_tile_columns];
+ int enter = t->next_seg < t->end_seg &&
+ s->tile_segments[t->next_seg].start_ts == ts;
- if (t->ended)
+ if (enter)
+ t->seg = &s->tile_segments[t->next_seg++];
+
+ /* No segment covers this CTB: damaged input, either a segment
+ * ended early or a slice is missing */
+ if (!t->seg)
continue;
if (cur != sub) {
@@ -3173,10 +3196,11 @@ static int decode_tiles_row_major(HEVCContext *s, const
HEVCLayerContext *l,
cur = sub;
}
- bind_slice_ctx(lc, t->sh, t->rpl, t->collocated_ref);
+ bind_slice_ctx(lc, t->seg->sh, t->seg->rpl, t->seg->collocated_ref);
- if (ts == t->start_ts)
- substream_load(&lc->sub, t->data, t->size, t->qp_y_seed);
+ if (enter)
+ substream_load(&lc->sub, t->seg->data, t->seg->size,
+ t->seg->qp_y_seed);
ret = decode_ctb(s, l, lc, rs);
if (ret < 0)
@@ -3185,9 +3209,8 @@ static int decode_tiles_row_major(HEVCContext *s, const
HEVCLayerContext *l,
if (rs == sps->ctb_size - 1)
last_ctb_decoded = 1;
- /* End of substream: last CTB of the tile, or damaged input */
if (!ret)
- t->ended = 1;
+ t->seg = NULL;
}
/* lc still carries the slice of the last column */
@@ -3220,22 +3243,29 @@ static int hls_decode_entry_tiles(HEVCContext *s, const
H2645NAL *nal)
if (ret < 0)
return ret;
+ ret = alloc_tile_segments(s, nb_tiles);
+ if (ret < 0)
+ return ret;
+
for (int i = 0; i < nb_tiles; i++) {
- HEVCTileStream *const t = &s->tile_streams[i];
+ HEVCTileSegment *const seg = &s->tile_segments[i];
const int start_ts = pps->ctb_addr_rs_to_ts[pps->tile_pos_rs[i]];
if (s->sh.size[i] <= 0 || pps->tile_id[start_ts] != i)
return AVERROR_INVALIDDATA;
- t->sh = &s->sh;
- t->rpl = s->cur_frame->refPicList;
- t->collocated_ref = s->collocated_ref;
- t->data = s->data + s->sh.offset[i];
- t->size = s->sh.size[i];
- t->start_ts = start_ts;
+ seg->sh = &s->sh;
+ seg->rpl = s->cur_frame->refPicList;
+ seg->collocated_ref = s->collocated_ref;
+ seg->data = s->data + s->sh.offset[i];
+ seg->size = s->sh.size[i];
+ seg->start_ts = start_ts;
/* The QP seed is the value decode_slice_data() left in lc[0]; with
* cu_qp_delta enabled it must not be overwritten with slice_qp */
- t->qp_y_seed = s->local_ctx[0].sub.qp_y;
+ seg->qp_y_seed = s->local_ctx[0].sub.qp_y;
+
+ s->tile_streams[i].next_seg = i;
+ s->tile_streams[i].end_seg = i + 1;
}
ret = decode_tiles_row_major(s, l, s->tile_streams);
@@ -3350,9 +3380,14 @@ static int build_tile_streams(HEVCContext *s, unsigned
nb)
if (ret < 0)
return ret;
+ ret = alloc_tile_segments(s, nb_tiles);
+ if (ret < 0)
+ return ret;
+
for (unsigned i = 0; i < nb; i++) {
const HEVCTileSlice *const sl = &s->tile_slices[i];
HEVCTileStream *t;
+ HEVCTileSegment *seg;
const uint8_t *data;
size_t size;
int tile;
@@ -3370,23 +3405,27 @@ static int build_tile_streams(HEVCContext *s, unsigned
nb)
return 0;
t = &s->tile_streams[tile];
- if (t->sh)
+ if (t->end_seg)
return 0;
if (tile_slice_payload(sl, &data, &size) < 0)
return 0;
- t->sh = &sl->sh;
- t->rpl = sl->rpl;
- t->collocated_ref = sl->collocated_ref;
- t->data = data;
- t->size = size;
- t->start_ts = sl->start_ts;
- t->qp_y_seed = sl->qp_y_seed;
+ seg = &s->tile_segments[tile];
+ seg->sh = &sl->sh;
+ seg->rpl = sl->rpl;
+ seg->collocated_ref = sl->collocated_ref;
+ seg->data = data;
+ seg->size = size;
+ seg->start_ts = sl->start_ts;
+ seg->qp_y_seed = sl->qp_y_seed;
+
+ t->next_seg = tile;
+ t->end_seg = tile + 1;
}
for (unsigned i = 0; i < nb_tiles; i++)
- if (!s->tile_streams[i].sh)
+ if (!s->tile_streams[i].end_seg)
return 0;
return 1;
@@ -4718,6 +4757,7 @@ static av_cold int hevc_decode_free(AVCodecContext *avctx)
av_freep(&s->local_ctx);
av_freep(&s->tile_slices);
+ av_freep(&s->tile_segments);
av_freep(&s->tile_streams);
av_freep(&s->substreams);
diff --git a/libavcodec/hevc/hevcdec.h b/libavcodec/hevc/hevcdec.h
index bafc218d32b..1cc5434debb 100644
--- a/libavcodec/hevc/hevcdec.h
+++ b/libavcodec/hevc/hevcdec.h
@@ -515,17 +515,24 @@ typedef struct HEVCTileSlice {
const HEVCFrame *collocated_ref; ///< temporal MVP target of this slice
} HEVCTileSlice;
-/* Start and state of one tile, for tiles bound by entry points and for one
- * slice per tile alike. */
-typedef struct HEVCTileStream {
- const SliceHeader *sh; ///< header of the slice this tile belongs to
+/* One CABAC substream within a tile and the state of the slice it belongs
+ * to. A tile is decoded from one or more segments. */
+typedef struct HEVCTileSegment {
+ const SliceHeader *sh; ///< header of the slice this segment belongs to
const RefPicList *rpl; ///< reference lists of that slice, NULL for I
slices
const HEVCFrame *collocated_ref; ///< temporal MVP target of that slice
const uint8_t *data; ///< start of the CABAC substream
size_t size;
int qp_y_seed;
- int start_ts; ///< first CTB of the tile, in tile scan order
- int ended; ///< the substream signalled its end (damaged
input)
+ int start_ts; ///< first CTB of the segment, in tile scan order
+} HEVCTileSegment;
+
+/* Decoding position within one tile. The segments of the tile are
+ * tile_segments[next_seg, end_seg) in increasing start_ts. */
+typedef struct HEVCTileStream {
+ const HEVCTileSegment *seg; ///< segment being decoded, NULL if none
+ unsigned next_seg;
+ unsigned end_seg;
} HEVCTileStream;
typedef struct HEVCContext {
@@ -636,7 +643,11 @@ typedef struct HEVCContext {
const HEVCFrame *tile_slices_frame;
int tile_slices_layer;
- /* Per-tile substream table of the picture, one entry per tile */
+ /* Segments of the picture in the row major traversal, grouped by tile */
+ HEVCTileSegment *tile_segments;
+ unsigned tile_segments_alloc;
+
+ /* One entry per tile */
HEVCTileStream *tile_streams;
unsigned tile_streams_alloc;
--
2.54.0
>From 9aed3f4e5b6fd2e5724b4e63666be3894f88036c Mon Sep 17 00:00:00 2001
From: Zhao Zhili <[email protected]>
Date: Fri, 6 Nov 2026 19:50:46 +0800
Subject: [PATCH 2/2] lavc/hevcdec: decode tiles split into several slices in
CTB row order
Encoders often split a tile into several slices to bound slice size.
Such pictures were decoded in tile scan order, which completes no CTB
row before the last tile is decoded, so frame threading could not
overlap pictures.
A slice without entry points never crosses a tile boundary, so every
independent slice without entry points can be deferred. If the deferred
slices of a picture arrive in increasing tile scan address and every
tile begins with a slice at its first CTB, they are decoded together in
row major order. Otherwise they are decoded in their original order.
The decoded output is the same either way.
Frame threading decode fps, AMD EPYC 9K84 (gcc 13). Three 7680x3840
HEVC streams (A, B, C) with 3x1 tiles split into two slices per tile.
"before" is the tile scan order (tile_interleave off); "after" is
this change. The last row is the ratio of the 32-thread to the
1-thread fps.
| A | B | C
threads | before | after | before | after | before | after
1 | 16.3 | 16.4 | 15.3 | 15.4 | 11.8 | 11.9
2 | 22.8 | 31.2 | 21.2 | 29.0 | 16.8 | 22.8
4 | 24.3 | 58.8 | 22.5 | 51.7 | 17.9 | 42.1
8 | 28.0 | 100.2 | 25.8 | 83.6 | 20.6 | 72.2
16 | 39.4 | 127.8 | 36.2 | 106.8 | 29.3 | 99.8
32 | 41.4 | 149.1 | 38.9 | 135.2 | 30.6 | 123.7
32/1 | 2.5x | 9.1x | 2.5x | 8.8x | 2.6x | 10.4x
Signed-off-by: Zhao Zhili <[email protected]>
---
libavcodec/hevc/hevcdec.c | 57 ++++++++++++++++++---------------------
libavcodec/hevc/hevcdec.h | 7 ++---
2 files changed, 30 insertions(+), 34 deletions(-)
diff --git a/libavcodec/hevc/hevcdec.c b/libavcodec/hevc/hevcdec.c
index 64cfdc481ed..bbccc8af963 100644
--- a/libavcodec/hevc/hevcdec.c
+++ b/libavcodec/hevc/hevcdec.c
@@ -3288,7 +3288,8 @@ static int tile_slice_payload(const HEVCTileSlice *sl,
return 0;
}
-/* Only an independent slice covering exactly one tile may be deferred. */
+/* A slice without entry points lies within a single tile, so an independent
+ * one can be deferred; a dependent one cannot be reordered. */
static int slice_is_deferrable(const HEVCContext *s)
{
const HEVCPPS *const pps = s->pps;
@@ -3307,8 +3308,7 @@ static int slice_is_deferrable(const HEVCContext *s)
start_ts = pps->ctb_addr_rs_to_ts[s->sh.slice_ctb_addr_rs];
- if (start_ts < 0 || start_ts >= pps->sps->ctb_size ||
- (start_ts > 0 && pps->tile_id[start_ts - 1] == pps->tile_id[start_ts]))
+ if (start_ts < 0 || start_ts >= pps->sps->ctb_size)
return 0;
return 1;
@@ -3364,8 +3364,10 @@ static int defer_tile_slice(HEVCContext *s, const
H2645NAL *nal)
return 1;
}
-/* Fill the tile table and return 1 if the deferred slices map one to one
- * onto the tiles, 0 to decode them in their original order. */
+/* Build the segment table and the per-tile runs of segments from the
+ * deferred slices. Return 1 if start_ts increases strictly and every tile
+ * begins with a segment at its first CTB, 0 otherwise (the caller then
+ * decodes the slices in their original order), <0 on error. */
static int build_tile_streams(HEVCContext *s, unsigned nb)
{
const HEVCPPS *const pps = s->pps;
@@ -3373,45 +3375,41 @@ static int build_tile_streams(HEVCContext *s, unsigned
nb)
const unsigned nb_tiles = pps->num_tile_columns * pps->num_tile_rows;
int ret;
- if (nb != nb_tiles)
- return 0;
-
ret = alloc_tile_streams(s, nb_tiles);
if (ret < 0)
return ret;
- ret = alloc_tile_segments(s, nb_tiles);
+ ret = alloc_tile_segments(s, nb);
if (ret < 0)
return ret;
+ /* The check below keeps start_ts strictly increasing, so the segments
+ * of a tile form one run of the table. */
for (unsigned i = 0; i < nb; i++) {
const HEVCTileSlice *const sl = &s->tile_slices[i];
+ HEVCTileSegment *const seg = &s->tile_segments[i];
HEVCTileStream *t;
- HEVCTileSegment *seg;
const uint8_t *data;
size_t size;
int tile;
- if (sl->start_ts < 0 || sl->start_ts >= sps->ctb_size)
- return 0;
-
- tile = pps->tile_id[sl->start_ts];
-
- if (tile < 0 || tile >= (int)nb_tiles)
- return 0;
-
- /* Every slice has to start at a tile, one slice per tile */
- if (pps->ctb_addr_rs_to_ts[pps->tile_pos_rs[tile]] != sl->start_ts)
- return 0;
-
- t = &s->tile_streams[tile];
- if (t->end_seg)
+ if (sl->start_ts < 0 || sl->start_ts >= sps->ctb_size ||
+ (i && sl->start_ts <= sl[-1].start_ts))
return 0;
if (tile_slice_payload(sl, &data, &size) < 0)
return 0;
- seg = &s->tile_segments[tile];
+ tile = pps->tile_id[sl->start_ts];
+ t = &s->tile_streams[tile];
+
+ if (!t->end_seg) {
+ if (sl->start_ts != pps->ctb_addr_rs_to_ts[pps->tile_pos_rs[tile]])
+ return 0;
+ t->next_seg = i;
+ }
+ t->end_seg = i + 1;
+
seg->sh = &sl->sh;
seg->rpl = sl->rpl;
seg->collocated_ref = sl->collocated_ref;
@@ -3419,9 +3417,6 @@ static int build_tile_streams(HEVCContext *s, unsigned nb)
seg->size = size;
seg->start_ts = sl->start_ts;
seg->qp_y_seed = sl->qp_y_seed;
-
- t->next_seg = tile;
- t->end_seg = tile + 1;
}
for (unsigned i = 0; i < nb_tiles; i++)
@@ -3431,9 +3426,9 @@ static int build_tile_streams(HEVCContext *s, unsigned nb)
return 1;
}
-/* If picture_complete, the slices cover the picture and are decoded in row
- * major order. Otherwise more slices follow, so decode them in their
- * original order. */
+/* If picture_complete, no more slices follow and the deferred ones are
+ * decoded in row major order when build_tile_streams() accepts them.
+ * Otherwise decode them in their original order. */
static int decode_deferred_tile_slices(HEVCContext *s, int picture_complete)
{
const HEVCPPS *const pps = s->pps;
diff --git a/libavcodec/hevc/hevcdec.h b/libavcodec/hevc/hevcdec.h
index 1cc5434debb..e006f668343 100644
--- a/libavcodec/hevc/hevcdec.h
+++ b/libavcodec/hevc/hevcdec.h
@@ -634,9 +634,10 @@ typedef struct HEVCContext {
AVBufferRef *rpu_buf; ///< 0 or 1 Dolby Vision RPUs.
DOVIContext dovi_ctx; ///< Dolby Vision decoding context
- /* Slices deferred when every tile carries its own slice. Frame and layer
- * are kept so that leftovers are dropped, not decoded into another
- * frame. */
+ /* Independent slices of one picture that stay within single tiles,
+ * buffered for decoding in row major order. The frame and layer are
+ * kept so that leftovers from another picture are dropped instead of
+ * being decoded into the current frame. */
HEVCTileSlice *tile_slices;
unsigned nb_tile_slices;
unsigned tile_slices_alloc;
--
2.54.0
_______________________________________________
ffmpeg-devel mailing list -- [email protected]
To unsubscribe send an email to [email protected]
