Stitch the two HEVC EAC views from a GoPro Max 2 .360 file to 2:1 equirectangular. Based on gopromax_opencl (Max 1) by Ronan LE MEILLAT and TADANO Tokumei.
Co-authored-by: Cursor <[email protected]> Signed-off-by: Reinhard Nißl <[email protected]> --- Changelog | 1 + doc/filters.texi | 29 +++ libavfilter/Makefile | 1 + libavfilter/allfilters.c | 1 + libavfilter/vf_gopromax2.c | 482 +++++++++++++++++++++++++++++++++++++ 5 files changed, 514 insertions(+) create mode 100644 libavfilter/vf_gopromax2.c diff --git a/Changelog b/Changelog index 531fd76145..2348c65173 100644 --- a/Changelog +++ b/Changelog @@ -19,6 +19,7 @@ version <next>: - MJPEG encoder packed RGB input - mjpeg_nvjpeg NVIDIA JPEG encoder - yuvrgb_cuda filter +- gopromax2 GoPro Max 2 CPU stitch filter version 9.0: - Extend AMF Color Converter (vf_vpp_amf) HDR capabilities diff --git a/doc/filters.texi b/doc/filters.texi index 861c0b1053..14efd1d1b2 100644 --- a/doc/filters.texi +++ b/doc/filters.texi @@ -27646,6 +27646,35 @@ JPEG (full) range @section overlay_cuda @anchor{yuvrgb_cuda} @section yuvrgb_cuda +@anchor{gopromax2} +@section gopromax2 + +Convert the two GoPro Max 2 .360 EAC tracks to equirectangular projection +on the CPU. + +Each input track is @code{side} + @code{center} + @code{side} (GoPro Max 2: +5952x1920 = 2016 + 1920 + 2016). The duplicated overlap lives in the side +faces and is blended over @code{blend} pixels (default 48). Typical mapping +is front @code{0:0} and rear @code{0:4}. Default output size is 7680x3840. + +@table @option +@item w +@item h +Output equirectangular size. Must be 2:1. +@item side +Width of each side face in luma pixels. @code{0} is @code{(iw-center)/2}. +@item center +Width of the center face. @code{0} is the input height. +@item blend +Overlap blend width in luma pixels. Default 48. +@item yaw +Yaw rotation of the output equirect in degrees. Default 0. +@end table + +@example +ffmpeg -i INPUT.360 -filter_complex "[0:0][0:4]gopromax2=yaw=-90" -frames:v 1 out.jpg +@end example + Convert CUDA @code{yuv444p} / @code{yuv444p10} to packed @code{rgb0}, or @code{rgb0} back to @code{yuv444p}. @ref{scale_cuda} cannot do this diff --git a/libavfilter/Makefile b/libavfilter/Makefile index 7ceddf552c..4b65c28314 100644 --- a/libavfilter/Makefile +++ b/libavfilter/Makefile @@ -336,6 +336,7 @@ OBJS-$(CONFIG_FSYNC_FILTER) += vf_fsync.o OBJS-$(CONFIG_GBLUR_FILTER) += vf_gblur.o OBJS-$(CONFIG_GBLUR_VULKAN_FILTER) += vf_gblur_vulkan.o vulkan.o vulkan_filter.o OBJS-$(CONFIG_GEQ_FILTER) += vf_geq.o +OBJS-$(CONFIG_GOPROMAX2_FILTER) += vf_gopromax2.o framesync.o OBJS-$(CONFIG_GRADFUN_FILTER) += vf_gradfun.o OBJS-$(CONFIG_GRAPHMONITOR_FILTER) += f_graphmonitor.o OBJS-$(CONFIG_GRAYWORLD_FILTER) += vf_grayworld.o diff --git a/libavfilter/allfilters.c b/libavfilter/allfilters.c index 82a3f001e9..4ee16d9f51 100644 --- a/libavfilter/allfilters.c +++ b/libavfilter/allfilters.c @@ -310,6 +310,7 @@ extern const FFFilter ff_vf_fsync; extern const FFFilter ff_vf_gblur; extern const FFFilter ff_vf_gblur_vulkan; extern const FFFilter ff_vf_geq; +extern const FFFilter ff_vf_gopromax2; extern const FFFilter ff_vf_gradfun; extern const FFFilter ff_vf_graphmonitor; extern const FFFilter ff_vf_grayworld; diff --git a/libavfilter/vf_gopromax2.c b/libavfilter/vf_gopromax2.c new file mode 100644 index 0000000000..806d55a737 --- /dev/null +++ b/libavfilter/vf_gopromax2.c @@ -0,0 +1,482 @@ +/* + * GoPro Max 2 .360 to equirectangular (CPU). + * Dual-EAC side/center/side stitch to 2:1 equirectangular. + * + * Copyright (c) 2021 Ronan LE MEILLAT + * Copyright (c) 2024 TADANO Tokumei + * Copyright (c) 2026 Reinhard Nißl + * + * Based on gopromax_opencl (Max 1) by Ronan LE MEILLAT and TADANO Tokumei. + * + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include <math.h> + +#include "libavutil/common.h" +#include "libavutil/opt.h" +#include "libavutil/pixdesc.h" + +#ifndef M_PI +#define M_PI 3.14159265358979323846 +#endif + +#include "avfilter.h" +#include "filters.h" +#include "framesync.h" +#include "video.h" + +enum { + FACE_LEFT, + FACE_RIGHT, + FACE_FRONT, + FACE_BACK, + FACE_TOP, + FACE_DOWN +}; + +typedef struct GoProMax2Context { + const AVClass *class; + + FFFrameSync fs; + + int width, height; + int side, center, blend; + float yaw; + + int in_w, in_h; + int out_w, out_h; + int layout_side, layout_center, layout_blend; + int nb_planes; + int depth; + int maxval; + int log2_chroma_w, log2_chroma_h; +} GoProMax2Context; + +typedef struct ThreadData { + AVFrame *front, *rear, *dst; + float yaw; +} ThreadData; + +static const enum AVPixelFormat pix_fmts[] = { + AV_PIX_FMT_YUV420P, AV_PIX_FMT_YUV422P, AV_PIX_FMT_YUV444P, + AV_PIX_FMT_YUVJ420P, AV_PIX_FMT_YUVJ422P, AV_PIX_FMT_YUVJ444P, + AV_PIX_FMT_YUV420P10, AV_PIX_FMT_YUV422P10, AV_PIX_FMT_YUV444P10, + AV_PIX_FMT_YUV420P12, AV_PIX_FMT_YUV422P12, AV_PIX_FMT_YUV444P12, + AV_PIX_FMT_YUV420P16, AV_PIX_FMT_YUV422P16, AV_PIX_FMT_YUV444P16, + AV_PIX_FMT_GBRP, AV_PIX_FMT_GBRP10, AV_PIX_FMT_GBRP12, AV_PIX_FMT_GBRP16, + AV_PIX_FMT_NONE +}; + +static av_always_inline float sample_u8(const uint8_t *data, int linesize, + int w, int h, float x, float y) +{ + int x0, y0, x1, y1; + float fx, fy, a, b; + + x = av_clipf(x, 0.f, (float)(w - 1)); + y = av_clipf(y, 0.f, (float)(h - 1)); + x0 = (int)x; + y0 = (int)y; + x1 = FFMIN(x0 + 1, w - 1); + y1 = FFMIN(y0 + 1, h - 1); + fx = x - x0; + fy = y - y0; + a = data[y0 * linesize + x0] * (1.f - fx) + data[y0 * linesize + x1] * fx; + b = data[y1 * linesize + x0] * (1.f - fx) + data[y1 * linesize + x1] * fx; + return a * (1.f - fy) + b * fy; +} + +static av_always_inline float sample_u16(const uint8_t *data, int linesize, + int w, int h, float x, float y) +{ + const uint16_t *row0, *row1; + int x0, y0, x1, y1; + float fx, fy, a, b; + + x = av_clipf(x, 0.f, (float)(w - 1)); + y = av_clipf(y, 0.f, (float)(h - 1)); + x0 = (int)x; + y0 = (int)y; + x1 = FFMIN(x0 + 1, w - 1); + y1 = FFMIN(y0 + 1, h - 1); + fx = x - x0; + fy = y - y0; + row0 = (const uint16_t *)(data + (size_t)y0 * linesize); + row1 = (const uint16_t *)(data + (size_t)y1 * linesize); + a = row0[x0] * (1.f - fx) + row0[x1] * fx; + b = row1[x0] * (1.f - fx) + row1[x1] * fx; + return a * (1.f - fy) + b * fy; +} + +static int find_face(float lon, float lat, float *u, float *v) +{ + float cl = cosf(lat); + float px = cl * sinf(lon); + float py = cl * cosf(lon); + float pz = sinf(lat); + float ax = fabsf(px), ay = fabsf(py), az = fabsf(pz); + float dom, k, qx, qy, qz; + int face = FACE_FRONT; + + if (ax >= ay && ax >= az) + face = px < 0.f ? FACE_LEFT : FACE_RIGHT; + else if (ay > ax && ay >= az) + face = py >= 0.f ? FACE_FRONT : FACE_BACK; + else if (az > ax && az > ay) + face = pz >= 0.f ? FACE_TOP : FACE_DOWN; + + dom = ax; + if (face == FACE_FRONT || face == FACE_BACK) + dom = ay; + else if (face == FACE_TOP || face == FACE_DOWN) + dom = az; + if (dom < 1e-12f) + dom = 1e-12f; + + k = 4.f / (float)M_PI; + qx = atanf(px / dom) * k; + qy = atanf(py / dom) * k; + qz = atanf(pz / dom) * k; + + switch (face) { + case FACE_LEFT: *u = (qy + 1.f) * 0.5f; *v = (qz + 1.f) * 0.5f; break; + case FACE_RIGHT: *u = (1.f - qy) * 0.5f; *v = (qz + 1.f) * 0.5f; break; + case FACE_FRONT: *u = (qx + 1.f) * 0.5f; *v = (qz + 1.f) * 0.5f; break; + case FACE_BACK: *u = (1.f - qx) * 0.5f; *v = (qz + 1.f) * 0.5f; break; + case FACE_TOP: *u = (1.f - qx) * 0.5f; *v = (qy + 1.f) * 0.5f; break; + default: *u = (1.f - qx) * 0.5f; *v = (1.f - qy) * 0.5f; break; + } + *u = av_clipf(*u, 0.f, 1.f - 1e-6f); + *v = av_clipf(*v, 0.f, 1.f - 1e-6f); + return face; +} + +static float get_colour(int face, float u, float v, + const uint8_t *front, int front_ls, + const uint8_t *rear, int rear_ls, + int src_w, int src_h, + int side, int center, int blendw, int hi) +{ + const uint8_t *src; + int use_front, x0, w, src_ls; + float duv, uL, uR, y, c1, c2; + + if (face == FACE_DOWN || face == FACE_BACK || face == FACE_TOP) { + float nu = v; + float nv = 1.f - 1e-6f - u; + u = nu; + v = nv; + } + v = 1.f - 1e-6f - v; + u = av_clipf(u, 0.f, 1.f - 1e-6f); + v = av_clipf(v, 0.f, 1.f - 1e-6f); + + use_front = (face == FACE_FRONT || face == FACE_LEFT || face == FACE_RIGHT); + src = use_front ? front : rear; + src_ls = use_front ? front_ls : rear_ls; + + if (face == FACE_FRONT || face == FACE_BACK) { + x0 = side; + w = center; + if (hi) + return sample_u16(src, src_ls, src_w, src_h, + x0 + u * (float)w, v * (float)src_h); + return sample_u8(src, src_ls, src_w, src_h, + x0 + u * (float)w, v * (float)src_h); + } + + if (face == FACE_LEFT || face == FACE_DOWN) { + x0 = 0; + w = side; + } else { + x0 = side + center; + w = side; + } + + duv = (float)blendw / (float)w; + uL = 2.f * (0.5f - duv) * u; + uR = 2.f * (0.5f - duv) * (u - 0.5f) + 0.5f + duv; + y = v * (float)src_h; + if (uL <= 0.5f - 2.f * duv) { + if (hi) + return sample_u16(src, src_ls, src_w, src_h, x0 + uL * (float)w, y); + return sample_u8(src, src_ls, src_w, src_h, x0 + uL * (float)w, y); + } + if (uR >= 0.5f + 2.f * duv) { + if (hi) + return sample_u16(src, src_ls, src_w, src_h, x0 + uR * (float)w, y); + return sample_u8(src, src_ls, src_w, src_h, x0 + uR * (float)w, y); + } + if (hi) { + c1 = sample_u16(src, src_ls, src_w, src_h, x0 + uL * (float)w, y); + c2 = sample_u16(src, src_ls, src_w, src_h, x0 + uR * (float)w, y); + } else { + c1 = sample_u8(src, src_ls, src_w, src_h, x0 + uL * (float)w, y); + c2 = sample_u8(src, src_ls, src_w, src_h, x0 + uR * (float)w, y); + } + return c1 + (c2 - c1) * ((uL - 0.5f + 2.f * duv) / (2.f * duv)); +} + +static void plane_layout(const GoProMax2Context *s, int plane, + int *pw, int *ph, int *side, int *center, int *blend) +{ + *pw = s->in_w; + *ph = s->in_h; + if (plane == 1 || plane == 2) { + *pw = AV_CEIL_RSHIFT(s->in_w, s->log2_chroma_w); + *ph = AV_CEIL_RSHIFT(s->in_h, s->log2_chroma_h); + } + *side = s->layout_side * *pw / s->in_w; + *center = s->layout_center * *ph / s->in_h; + *blend = FFMAX(1, s->layout_blend * *pw / s->in_w); +} + +static int filter_slice(AVFilterContext *ctx, void *arg, int jobnr, int nb_jobs) +{ + GoProMax2Context *s = ctx->priv; + ThreadData *td = arg; + int plane, hi = s->depth > 8; + + for (plane = 0; plane < s->nb_planes; plane++) { + int src_w, src_h, side, center, blend; + int dst_w, dst_h, y0, y1, x, y; + uint8_t *dst; + + plane_layout(s, plane, &src_w, &src_h, &side, ¢er, &blend); + dst_w = plane == 1 || plane == 2 ? + AV_CEIL_RSHIFT(s->out_w, s->log2_chroma_w) : s->out_w; + dst_h = plane == 1 || plane == 2 ? + AV_CEIL_RSHIFT(s->out_h, s->log2_chroma_h) : s->out_h; + + y0 = (dst_h * jobnr) / nb_jobs; + y1 = (dst_h * (jobnr + 1)) / nb_jobs; + dst = td->dst->data[plane]; + + for (y = y0; y < y1; y++) { + for (x = 0; x < dst_w; x++) { + float lat = (float)M_PI * 0.5f - (y + 0.5f) / (float)dst_h * (float)M_PI; + float lon = (x + 0.5f) / (float)dst_w * 2.f * (float)M_PI - (float)M_PI + td->yaw; + float u, v, val; + int face; + + if (lon > (float)M_PI) + lon -= 2.f * (float)M_PI; + if (lon < -(float)M_PI) + lon += 2.f * (float)M_PI; + + face = find_face(lon, lat, &u, &v); + val = get_colour(face, u, v, + td->front->data[plane], td->front->linesize[plane], + td->rear->data[plane], td->rear->linesize[plane], + src_w, src_h, side, center, blend, hi); + if (hi) { + uint16_t *row = (uint16_t *)(dst + (size_t)y * td->dst->linesize[plane]); + row[x] = av_clip(lrintf(val), 0, s->maxval); + } else { + dst[y * td->dst->linesize[plane] + x] = + av_clip_uint8(lrintf(val)); + } + } + } + } + return 0; +} + +static int gopromax2_filter(FFFrameSync *fs) +{ + AVFilterContext *avctx = fs->parent; + AVFilterLink *outlink = avctx->outputs[0]; + GoProMax2Context *s = avctx->priv; + AVFrame *front, *rear, *out; + ThreadData td; + int ret; + + ret = ff_framesync_get_frame(fs, 0, &front, 0); + if (ret < 0) + return ret; + ret = ff_framesync_get_frame(fs, 1, &rear, 0); + if (ret < 0) + return ret; + + out = ff_get_video_buffer(outlink, outlink->w, outlink->h); + if (!out) + return AVERROR(ENOMEM); + + ret = av_frame_copy_props(out, front); + if (ret < 0) { + av_frame_free(&out); + return ret; + } + + td.front = front; + td.rear = rear; + td.dst = out; + td.yaw = s->yaw * (float)M_PI / 180.f; + + ff_filter_execute(avctx, filter_slice, &td, NULL, + FFMIN(s->out_h, ff_filter_get_nb_threads(avctx))); + + return ff_filter_frame(outlink, out); +} + +static int config_output(AVFilterLink *outlink) +{ + AVFilterContext *avctx = outlink->src; + GoProMax2Context *s = avctx->priv; + AVFilterLink *frontlink = avctx->inputs[0]; + AVFilterLink *rearlink = avctx->inputs[1]; + FilterLink *inl = ff_filter_link(frontlink); + FilterLink *outl = ff_filter_link(outlink); + const AVPixFmtDescriptor *desc; + int ret; + + if (frontlink->w != rearlink->w || frontlink->h != rearlink->h) { + av_log(avctx, AV_LOG_ERROR, "Front and rear must be the same size.\n"); + return AVERROR(EINVAL); + } + if (frontlink->format != rearlink->format) { + av_log(avctx, AV_LOG_ERROR, "Front and rear must be the same pixel format.\n"); + return AVERROR(EINVAL); + } + + s->in_w = frontlink->w; + s->in_h = frontlink->h; + s->layout_center = s->center > 0 ? s->center : s->in_h; + s->layout_side = s->side > 0 ? s->side : (s->in_w - s->layout_center) / 2; + s->layout_blend = s->blend > 0 ? s->blend : 48; + + if (s->layout_side <= 0 || s->layout_center <= 0 || + 2 * s->layout_side + s->layout_center != s->in_w) { + av_log(avctx, AV_LOG_ERROR, + "Max 2 layout mismatch: input %dx%d, side %d center %d " + "(need 2*side+center = width).\n", + s->in_w, s->in_h, s->layout_side, s->layout_center); + return AVERROR(EINVAL); + } + + if (s->width > 0 && s->height > 0) { + if (s->width != s->height * 2) { + av_log(avctx, AV_LOG_ERROR, + "Specified size (%dx%d) is not 2:1 equirect.\n", + s->width, s->height); + return AVERROR(EINVAL); + } + s->out_w = s->width; + s->out_h = s->height; + } else if (s->width > 0 || s->height > 0) { + av_log(avctx, AV_LOG_ERROR, + "Both width and height values should be specified.\n"); + return AVERROR(EINVAL); + } else { + s->out_w = 4 * s->in_h; + s->out_h = 2 * s->in_h; + } + + desc = av_pix_fmt_desc_get(frontlink->format); + s->nb_planes = av_pix_fmt_count_planes(frontlink->format); + s->depth = desc->comp[0].depth; + s->maxval = (1 << s->depth) - 1; + s->log2_chroma_w = desc->log2_chroma_w; + s->log2_chroma_h = desc->log2_chroma_h; + + outlink->w = s->out_w; + outlink->h = s->out_h; + outlink->sample_aspect_ratio = (AVRational){ 1, 1 }; + outlink->time_base = frontlink->time_base; + outl->frame_rate = inl->frame_rate; + + av_log(avctx, AV_LOG_INFO, + "gopromax2 in %dx%d side %d center %d blend %d yaw %.1f out %dx%d\n", + s->in_w, s->in_h, s->layout_side, s->layout_center, s->layout_blend, + s->yaw, s->out_w, s->out_h); + + ret = ff_framesync_init_dualinput(&s->fs, avctx); + if (ret < 0) + return ret; + + ret = ff_framesync_configure(&s->fs); + if (ret < 0) + return ret; + + outlink->time_base = s->fs.time_base; + return 0; +} + +static av_cold int gopromax2_init(AVFilterContext *avctx) +{ + GoProMax2Context *s = avctx->priv; + s->fs.on_event = &gopromax2_filter; + return 0; +} + +static int gopromax2_activate(AVFilterContext *avctx) +{ + GoProMax2Context *s = avctx->priv; + return ff_framesync_activate(&s->fs); +} + +static av_cold void gopromax2_uninit(AVFilterContext *avctx) +{ + GoProMax2Context *s = avctx->priv; + ff_framesync_uninit(&s->fs); +} + +#define OFFSET(x) offsetof(GoProMax2Context, x) +#define FLAGS (AV_OPT_FLAG_FILTERING_PARAM | AV_OPT_FLAG_VIDEO_PARAM) +static const AVOption gopromax2_options[] = { + { "w", "output width", OFFSET(width), AV_OPT_TYPE_INT, {.i64=0}, 0, INT16_MAX, FLAGS }, + { "h", "output height", OFFSET(height), AV_OPT_TYPE_INT, {.i64=0}, 0, INT16_MAX, FLAGS }, + { "side", "side face width in luma pixels (0 = (in_w-center)/2)", + OFFSET(side), AV_OPT_TYPE_INT, {.i64=0}, 0, INT16_MAX, FLAGS }, + { "center", "center face width in luma pixels (0 = in_h)", + OFFSET(center), AV_OPT_TYPE_INT, {.i64=0}, 0, INT16_MAX, FLAGS }, + { "blend", "overlap blend width in luma pixels", + OFFSET(blend), AV_OPT_TYPE_INT, {.i64=48}, 1, 1024, FLAGS }, + { "yaw", "yaw rotation of the equirect in degrees", + OFFSET(yaw), AV_OPT_TYPE_FLOAT, {.dbl=0}, -360, 360, FLAGS }, + { NULL }, +}; + +AVFILTER_DEFINE_CLASS(gopromax2); + +static const AVFilterPad gopromax2_inputs[] = { + { .name = "front", .type = AVMEDIA_TYPE_VIDEO }, + { .name = "rear", .type = AVMEDIA_TYPE_VIDEO }, +}; + +static const AVFilterPad gopromax2_outputs[] = { + { + .name = "default", + .type = AVMEDIA_TYPE_VIDEO, + .config_props = &config_output, + }, +}; + +const FFFilter ff_vf_gopromax2 = { + .p.name = "gopromax2", + .p.description = NULL_IF_CONFIG_SMALL("GoPro Max 2 .360 to equirectangular projection"), + .p.priv_class = &gopromax2_class, + .p.flags = AVFILTER_FLAG_SLICE_THREADS, + .priv_size = sizeof(GoProMax2Context), + .init = &gopromax2_init, + .uninit = &gopromax2_uninit, + .activate = &gopromax2_activate, + FILTER_INPUTS(gopromax2_inputs), + FILTER_OUTPUTS(gopromax2_outputs), + FILTER_PIXFMTS_ARRAY(pix_fmts), +}; -- 2.43.0 _______________________________________________ ffmpeg-devel mailing list -- [email protected] To unsubscribe send an email to [email protected]
