PR #23541 opened by Zhao Zhili (quink) URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/23541 Patch URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/23541.patch
# Summary of changes Add a muxer that wraps encoded image in the iTerm2 inline image protocol (OSC 1337) so ffmpeg can play video directly in an iTerm2 terminal. The output is a self-contained byte stream: it can be played live or saved to a file and replayed with cat. A screenshot of ssh to a remote machine then run `./ffmpeg -re -i ~/video/bunny.mp4 -f iterm2 -tmux 1 -` on that machine  >From 866e045f8b06160f18650039c29e7fbec64c7227 Mon Sep 17 00:00:00 2001 From: Zhao Zhili <[email protected]> Date: Fri, 19 Jun 2026 21:34:06 +0800 Subject: [PATCH] avformat: add iTerm2 inline image protocol muxer Add a muxer that wraps encoded image in the iTerm2 inline image protocol (OSC 1337) so ffmpeg can play video directly in an iTerm2 terminal. The output is a self-contained byte stream: it can be played live or saved to a file and replayed with cat. --- Changelog | 1 + doc/muxers.texi | 52 +++++++++++ libavformat/Makefile | 1 + libavformat/allformats.c | 1 + libavformat/iterm2enc.c | 180 +++++++++++++++++++++++++++++++++++++++ libavformat/version.h | 4 +- 6 files changed, 237 insertions(+), 2 deletions(-) create mode 100644 libavformat/iterm2enc.c diff --git a/Changelog b/Changelog index 2ad3ee255f..9782fae96c 100644 --- a/Changelog +++ b/Changelog @@ -18,6 +18,7 @@ version <next>: - Remove ogg/celt parsing - Bitstream filter to split Dolby Vision multi-layer HEVC - Add AMF hardware memory mapping support. +- iTerm2 inline image protocol muxer version 8.1: diff --git a/doc/muxers.texi b/doc/muxers.texi index 5056a3e3d6..092dc13ed9 100644 --- a/doc/muxers.texi +++ b/doc/muxers.texi @@ -2722,6 +2722,58 @@ computer-generated compositions. This muxer accepts a single audio stream containing PCM data. +@section iterm2 +iTerm2 inline image protocol muxer. + +This muxer writes video frames as OSC 1337 inline images for display in +terminals that support the iTerm2 image protocol. Use @option{-re} to limit +the output rate to the source framerate; without it, frames are emitted as +fast as they are encoded, which is usually not desired for live display. + +Frames are sent with the multipart form of the protocol, which splits each +image across several short control sequences. This avoids the per-sequence +size limit that otherwise discards large frames, and requires iTerm2 3.5 or +newer. + +The output is a self-contained byte stream and can be redirected to a file. +Replaying the file with @command{cat} displays the images in the terminal. + +@subsection Options +@table @option +@item display_width @var{size} +Set the displayed image width. @var{size} can be @samp{auto}, @var{N} terminal +cells, @var{N}px pixels, or @var{N}% of the terminal width. When unset, the +terminal derives the width from the image. + +@item display_height @var{size} +Set the displayed image height. @var{size} uses the same syntax as +@option{display_width}. When unset, the terminal derives the height from the +image. + +@item keep_aspect @var{bool} +Preserve the input aspect ratio when scaling. Default is enabled. + +@item tmux @var{bool} +Wrap image data in tmux DCS passthrough. This requires a tmux version whose +passthrough sequence size limit is large enough for image data, with +passthrough enabled via @command{tmux set -g allow-passthrough on}. Default is +disabled. +@end table + +@subsection Examples + +Display a video in an iTerm2 terminal: +@example +ffmpeg -re -i input.mp4 -f iterm2 - +@end example + +Scale the displayed image to 40 terminal cells tall. Inside tmux, enable +passthrough first with @command{tmux set -g allow-passthrough on}, then add +@option{tmux}: +@example +ffmpeg -re -i input.mp4 -f iterm2 -display_height 40 -tmux 1 - +@end example + @section ivf On2 IVF muxer. diff --git a/libavformat/Makefile b/libavformat/Makefile index 0db0c7c2a9..70b62f9724 100644 --- a/libavformat/Makefile +++ b/libavformat/Makefile @@ -337,6 +337,7 @@ OBJS-$(CONFIG_IPU_DEMUXER) += ipudec.o rawdec.o OBJS-$(CONFIG_IRCAM_DEMUXER) += ircamdec.o ircam.o pcm.o OBJS-$(CONFIG_IRCAM_MUXER) += ircamenc.o ircam.o rawenc.o OBJS-$(CONFIG_ISS_DEMUXER) += iss.o +OBJS-$(CONFIG_ITERM2_MUXER) += iterm2enc.o OBJS-$(CONFIG_IV8_DEMUXER) += iv8.o OBJS-$(CONFIG_IVF_DEMUXER) += ivfdec.o OBJS-$(CONFIG_IVF_MUXER) += ivfenc.o diff --git a/libavformat/allformats.c b/libavformat/allformats.c index af7eea5e5c..05624e66eb 100644 --- a/libavformat/allformats.c +++ b/libavformat/allformats.c @@ -242,6 +242,7 @@ extern const FFInputFormat ff_ircam_demuxer; extern const FFOutputFormat ff_ircam_muxer; extern const FFOutputFormat ff_ismv_muxer; extern const FFInputFormat ff_iss_demuxer; +extern const FFOutputFormat ff_iterm2_muxer; extern const FFInputFormat ff_iv8_demuxer; extern const FFInputFormat ff_ivf_demuxer; extern const FFOutputFormat ff_ivf_muxer; diff --git a/libavformat/iterm2enc.c b/libavformat/iterm2enc.c new file mode 100644 index 0000000000..5503b08266 --- /dev/null +++ b/libavformat/iterm2enc.c @@ -0,0 +1,180 @@ +/* + * This file is part of FFmpeg. + * + * Copyright (c) 2026 Zhao Zhili <[email protected]> + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include <string.h> + +#include "libavutil/base64.h" +#include "libavutil/macros.h" +#include "libavutil/mem.h" +#include "libavutil/opt.h" +#include "avformat.h" +#include "mux.h" + +/* iTerm2 inline image protocol: https://iterm2.com/documentation-images.html */ + +#define ESC "\033" + +#define SYNC_BEGIN ESC "[?2026h" +#define SYNC_END ESC "[?2026l" +#define CURSOR_SAVE ESC "7" +#define CURSOR_RESTORE ESC "8" +#define CURSOR_HOME ESC "[H" +#define CURSOR_BOTTOM ESC "[999H" + +#define OSC_START ESC "]1337;" +#define BEL "\a" +#define ST ESC "\\" + +/* tmux requires DCS passthrough with ST termination and ESC doubling */ +#define TMUX_DCS ESC "Ptmux;" + +/* iTerm2 and tmux silently drop a single OSC sequence >= 1 MiB, so split the + * image into chunks below that limit. Old tmux capped a sequence at 256 bytes, + * but the tiny chunks that would require flood the tmux parser and freeze the + * terminal, so we do not support such versions. */ +#define FILEPART_CHUNK ((1 << 20) - 4096) + +#define WRITE_LITERAL(pb, str) avio_write(pb, (const unsigned char *)(str), \ + sizeof(str) - 1) + +typedef struct ITerm2Context { + const AVClass *class; + char *display_width; + char *display_height; + int keep_aspect; + int tmux; + char *b64; + unsigned b64_size; +} ITerm2Context; + +static void osc_open(ITerm2Context *c, AVIOContext *pb) +{ + if (c->tmux) + WRITE_LITERAL(pb, TMUX_DCS ESC); + WRITE_LITERAL(pb, OSC_START); +} + +static void osc_close(ITerm2Context *c, AVIOContext *pb) +{ + WRITE_LITERAL(pb, BEL); + if (c->tmux) + WRITE_LITERAL(pb, ST); +} + +static void write_image(ITerm2Context *c, AVIOContext *pb, int size) +{ + size_t b64_len = strlen(c->b64); + + osc_open(c, pb); + WRITE_LITERAL(pb, "MultipartFile="); + avio_printf(pb, "inline=1;size=%d", size); + if (c->display_width && c->display_width[0]) + avio_printf(pb, ";width=%s", c->display_width); + if (c->display_height && c->display_height[0]) + avio_printf(pb, ";height=%s", c->display_height); + if (!c->keep_aspect) + WRITE_LITERAL(pb, ";preserveAspectRatio=0"); + osc_close(c, pb); + + for (size_t off = 0; off < b64_len; off += FILEPART_CHUNK) { + size_t n = FFMIN(FILEPART_CHUNK, b64_len - off); + + osc_open(c, pb); + WRITE_LITERAL(pb, "FilePart="); + avio_write(pb, c->b64 + off, n); + osc_close(c, pb); + } + + osc_open(c, pb); + WRITE_LITERAL(pb, "FileEnd"); + osc_close(c, pb); +} + +static int iterm2_write_packet(AVFormatContext *s, AVPacket *pkt) +{ + ITerm2Context *c = s->priv_data; + + av_fast_malloc(&c->b64, &c->b64_size, AV_BASE64_SIZE(pkt->size)); + if (!c->b64) + return AVERROR(ENOMEM); + if (!av_base64_encode(c->b64, c->b64_size, pkt->data, pkt->size)) + return AVERROR(EINVAL); + + /* Synchronized output swaps the frame in atomically. */ + WRITE_LITERAL(s->pb, SYNC_BEGIN CURSOR_SAVE CURSOR_HOME); + + write_image(c, s->pb, pkt->size); + + WRITE_LITERAL(s->pb, CURSOR_RESTORE SYNC_END); + + avio_flush(s->pb); + + return 0; +} + +static int iterm2_write_trailer(AVFormatContext *s) +{ + WRITE_LITERAL(s->pb, CURSOR_BOTTOM "\n"); + + return 0; +} + +static av_cold void iterm2_deinit(AVFormatContext *s) +{ + ITerm2Context *c = s->priv_data; + av_freep(&c->b64); +} + +#define OFFSET(x) offsetof(ITerm2Context, x) +#define ENC AV_OPT_FLAG_ENCODING_PARAM + +static const AVOption options[] = { + { "display_width", "on-screen width (auto, N cells, Npx, N%%)", + OFFSET(display_width), AV_OPT_TYPE_STRING, { .str = NULL }, 0, 0, ENC }, + { "display_height", "on-screen height (auto, N cells, Npx, N%%)", + OFFSET(display_height), AV_OPT_TYPE_STRING, { .str = NULL }, 0, 0, ENC }, + { "keep_aspect", "preserve aspect ratio when scaling", + OFFSET(keep_aspect), AV_OPT_TYPE_BOOL, { .i64 = 1 }, 0, 1, ENC }, + { "tmux", "wrap image in tmux DCS passthrough, requires tmux set -g allow-passthrough on", + OFFSET(tmux), AV_OPT_TYPE_BOOL, { .i64 = 0 }, 0, 1, ENC }, + { NULL }, +}; + +static const AVClass iterm2_class = { + .class_name = "iTerm2 muxer", + .item_name = av_default_item_name, + .option = options, + .version = LIBAVUTIL_VERSION_INT, + .category = AV_CLASS_CATEGORY_MUXER, +}; + +const FFOutputFormat ff_iterm2_muxer = { + .p.name = "iterm2", + .p.long_name = NULL_IF_CONFIG_SMALL("iTerm2 inline image protocol"), + .priv_data_size = sizeof(ITerm2Context), + .p.audio_codec = AV_CODEC_ID_NONE, + .p.video_codec = AV_CODEC_ID_MJPEG, + .write_packet = iterm2_write_packet, + .write_trailer = iterm2_write_trailer, + .deinit = iterm2_deinit, + .flags_internal = FF_OFMT_FLAG_MAX_ONE_OF_EACH, + .p.flags = AVFMT_NOTIMESTAMPS | AVFMT_NODIMENSIONS, + .p.priv_class = &iterm2_class, +}; diff --git a/libavformat/version.h b/libavformat/version.h index bbb2fc7d87..de9cc8e31d 100644 --- a/libavformat/version.h +++ b/libavformat/version.h @@ -31,8 +31,8 @@ #include "version_major.h" -#define LIBAVFORMAT_VERSION_MINOR 19 -#define LIBAVFORMAT_VERSION_MICRO 101 +#define LIBAVFORMAT_VERSION_MINOR 20 +#define LIBAVFORMAT_VERSION_MICRO 100 #define LIBAVFORMAT_VERSION_INT AV_VERSION_INT(LIBAVFORMAT_VERSION_MAJOR, \ LIBAVFORMAT_VERSION_MINOR, \ -- 2.52.0 _______________________________________________ ffmpeg-devel mailing list -- [email protected] To unsubscribe send an email to [email protected]
