PR #25017 opened by timrae URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/25017 Patch URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/25017.patch
libssh_read() called sftp_read(), which sends one SSH_FXP_READ request and blocks until its reply arrives, so throughput was limited to one request per network round trip regardless of bandwidth. Over a 24 ms round trip with a 90 Mbit/s link, mpv read an mkv at about 0.8 MB/s. With libssh 0.11.0 or later, keep up to 32 read requests of 32 KiB in flight using the sftp_aio API, the same approach OpenSSH's sftp client uses, with the requests queued in an AVFifo. The window starts at one request after opening, seeking or a short read, and doubles with each completed request, so the probing seeks done by demuxers do not wait for a full window of unwanted data. SFTP cannot cancel requests, so seeking drains the outstanding ones. A short read, including the end of the file, also drains and restarts from the actual position, since the following requests assumed a full read. Read-ahead is only used for read-only files. Older libssh versions and files opened for writing keep using the synchronous path. On the same link, reading a 910 MB file goes from 0.8 MB/s (mpv) or 3.8 MB/s (tools/aviocat) to 11.7 MB/s, with identical content. >From ae375401f45c2f255b589033ec6b4c2953fa2aad Mon Sep 17 00:00:00 2001 From: Tim Rae <[email protected]> Date: Sun, 11 Oct 2026 20:45:33 +0200 Subject: [PATCH] avformat/libssh: pipeline SFTP read requests libssh_read() called sftp_read(), which sends one SSH_FXP_READ request and blocks until its reply arrives, so throughput was limited to one request per network round trip regardless of bandwidth. Over a 24 ms round trip with a 90 Mbit/s link, mpv read an mkv at about 0.8 MB/s. With libssh 0.11.0 or later, keep up to 32 read requests of 32 KiB in flight using the sftp_aio API, the same approach OpenSSH's sftp client uses, with the requests queued in an AVFifo. The window starts at one request after opening, seeking or a short read, and doubles with each completed request, so the probing seeks done by demuxers do not wait for a full window of unwanted data. SFTP cannot cancel requests, so seeking drains the outstanding ones. A short read, including the end of the file, also drains and restarts from the actual position, since the following requests assumed a full read. Read-ahead is only used for read-only files. Older libssh versions and files opened for writing keep using the synchronous path. On the same link, reading a 910 MB file goes from 0.8 MB/s (mpv) or 3.8 MB/s (tools/aviocat) to 11.7 MB/s and 11.1 MB/s, with identical content. Signed-off-by: Tim Rae <[email protected]> --- libavformat/libssh.c | 104 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 103 insertions(+), 1 deletion(-) diff --git a/libavformat/libssh.c b/libavformat/libssh.c index 470175924d..f76c65c140 100644 --- a/libavformat/libssh.c +++ b/libavformat/libssh.c @@ -22,6 +22,7 @@ #define LIBSSH_STATIC #include <libssh/sftp.h> #include "libavutil/avstring.h" +#include "libavutil/fifo.h" #include "libavutil/mem.h" #include "libavutil/opt.h" #include "libavutil/attributes.h" @@ -30,6 +31,10 @@ #include "internal.h" #include "url.h" +#define USE_SFTP_AIO (LIBSSH_VERSION_INT >= SSH_VERSION_INT(0, 11, 0)) +#define READ_CHUNK_SIZE 32768 ///< every SFTP server must accept reads of this size +#define MAX_READS_IN_FLIGHT 32 + typedef struct { const AVClass *class; ssh_session session; @@ -41,6 +46,13 @@ typedef struct { int trunc; char *priv_key; int verify; + int64_t pos; +#if USE_SFTP_AIO + AVFifo *reads; ///< outstanding read requests, oldest first; NULL if not reading ahead + int max_reads; + uint8_t *buf; ///< READ_CHUNK_SIZE of data, then READ_CHUNK_SIZE of scratch space + int buf_pos, buf_len; +#endif } LIBSSHContext; #if LIBSSH_VERSION_INT < SSH_VERSION_INT(0, 8, 0) @@ -214,6 +226,13 @@ static av_cold void libssh_stat_file(LIBSSHContext *libssh) static av_cold int libssh_close(URLContext *h) { LIBSSHContext *libssh = h->priv_data; +#if USE_SFTP_AIO + sftp_aio aio; + while (libssh->reads && av_fifo_read(libssh->reads, &aio, 1) >= 0) + sftp_aio_free(aio); + av_fifo_freep2(&libssh->reads); + av_freep(&libssh->buf); +#endif if (libssh->file) { sftp_close(libssh->file); libssh->file = NULL; @@ -281,6 +300,18 @@ static av_cold int libssh_open(URLContext *h, const char *url, int flags) libssh_stat_file(libssh); +#if USE_SFTP_AIO + if (!(flags & AVIO_FLAG_WRITE)) { + libssh->reads = av_fifo_alloc2(MAX_READS_IN_FLIGHT, sizeof(sftp_aio), 0); + libssh->buf = av_malloc(2 * READ_CHUNK_SIZE); + if (!libssh->reads || !libssh->buf) { + ret = AVERROR(ENOMEM); + goto fail; + } + libssh->max_reads = 1; + } +#endif + return 0; fail: @@ -288,6 +319,60 @@ static av_cold int libssh_open(URLContext *h, const char *url, int flags) return ret; } +#if USE_SFTP_AIO +/* SFTP cannot cancel requests, so wait for and discard the outstanding ones. */ +static void libssh_drain_reads(LIBSSHContext *libssh) +{ + sftp_aio aio; + while (av_fifo_read(libssh->reads, &aio, 1) >= 0) + sftp_aio_wait_read(&aio, libssh->buf + READ_CHUNK_SIZE, READ_CHUNK_SIZE); + libssh->max_reads = 1; +} + +/* Keep up to max_reads requests in flight, as a single request per round + * trip is far slower than the network on all but the shortest links. */ +static int libssh_read_ahead(URLContext *h, unsigned char *buf, int size) +{ + LIBSSHContext *libssh = h->priv_data; + sftp_aio aio; + int len; + + if (libssh->buf_pos == libssh->buf_len) { + while (av_fifo_can_read(libssh->reads) < libssh->max_reads) { + if (sftp_aio_begin_read(libssh->file, READ_CHUNK_SIZE, &aio) < 0) + goto fail; + av_fifo_write(libssh->reads, &aio, 1); + } + av_fifo_read(libssh->reads, &aio, 1); + if ((len = sftp_aio_wait_read(&aio, libssh->buf, READ_CHUNK_SIZE)) < 0) + goto fail; + libssh->max_reads = FFMIN(2 * libssh->max_reads, MAX_READS_IN_FLIGHT); + + /* The later requests assumed this one was complete, so after a + * short read or the end of the file, restart from here. */ + if (len < READ_CHUNK_SIZE) { + libssh_drain_reads(libssh); + if (sftp_seek64(libssh->file, libssh->pos + len) < 0) + goto fail; + if (!len) + return AVERROR_EOF; + } + libssh->buf_pos = 0; + libssh->buf_len = len; + } + + len = FFMIN(size, libssh->buf_len - libssh->buf_pos); + memcpy(buf, libssh->buf + libssh->buf_pos, len); + libssh->buf_pos += len; + libssh->pos += len; + return len; + +fail: + av_log(libssh, AV_LOG_ERROR, "Read error: %s\n", ssh_get_error(libssh->session)); + return AVERROR(EIO); +} +#endif + static int64_t libssh_seek(URLContext *h, int64_t pos, int whence) { LIBSSHContext *libssh = h->priv_data; @@ -305,7 +390,7 @@ static int64_t libssh_seek(URLContext *h, int64_t pos, int whence) newpos = pos; break; case SEEK_CUR: - newpos = sftp_tell64(libssh->file) + pos; + newpos = libssh->pos + pos; break; case SEEK_END: newpos = libssh->filesize + pos; @@ -319,10 +404,20 @@ static int64_t libssh_seek(URLContext *h, int64_t pos, int whence) return AVERROR(EINVAL); } +#if USE_SFTP_AIO + if (libssh->reads) { + if (newpos == libssh->pos) + return newpos; + libssh_drain_reads(libssh); + libssh->buf_pos = libssh->buf_len = 0; + } +#endif + if (sftp_seek64(libssh->file, newpos)) { av_log(h, AV_LOG_ERROR, "Error during seeking.\n"); return AVERROR(EIO); } + libssh->pos = newpos; return newpos; } @@ -332,10 +427,16 @@ static int libssh_read(URLContext *h, unsigned char *buf, int size) LIBSSHContext *libssh = h->priv_data; int bytes_read; +#if USE_SFTP_AIO + if (libssh->reads) + return libssh_read_ahead(h, buf, size); +#endif + if ((bytes_read = sftp_read(libssh->file, buf, size)) < 0) { av_log(libssh, AV_LOG_ERROR, "Read error.\n"); return AVERROR(EIO); } + libssh->pos += bytes_read; return bytes_read ? bytes_read : AVERROR_EOF; } @@ -348,6 +449,7 @@ static int libssh_write(URLContext *h, const unsigned char *buf, int size) av_log(libssh, AV_LOG_ERROR, "Write error.\n"); return AVERROR(EIO); } + libssh->pos += bytes_written; return bytes_written; } -- 2.52.0 _______________________________________________ ffmpeg-devel mailing list -- [email protected] To unsubscribe send an email to [email protected]
