The branch, master has been updated via ae1f0029 Reduce memory usage (#235) from 3911c238 Tweak SIMD & ASM option defaults.
https://git.samba.org/?p=rsync.git;a=shortlog;h=master - Log ----------------------------------------------------------------- commit ae1f002999fc060178ef00d10cc021404a428e41 Author: JindÅich MakoviÄka <makov...@gmail.com> Date: Fri Oct 1 21:04:59 2021 +0200 Reduce memory usage (#235) In 2004, an allocation optimization has been added to the file list handling code, that preallocates 32k of file_struct pointers in a file_list. This optimization predates the incremental recursion feature, for which it is not appropriate anymore. When copying a tree containing a large number of small directories, using the incremental recursion, rsync allocates many short file_lists. Suddenly, the unused file_struct pointers can easily take 90-95% of the memory allocated by rsync. ----------------------------------------------------------------------- Summary of changes: flist.c | 11 +++++++++-- rsync.h | 5 +++-- 2 files changed, 12 insertions(+), 4 deletions(-) Changeset truncated at 500 lines: diff --git a/flist.c b/flist.c index 3442d868..803490bd 100644 --- a/flist.c +++ b/flist.c @@ -295,6 +295,8 @@ static void flist_expand(struct file_list *flist, int extra) flist->malloced = FLIST_START; else if (flist->malloced >= FLIST_LINEAR) flist->malloced += FLIST_LINEAR; + else if (flist->malloced < FLIST_START_LARGE/16) + flist->malloced *= 4; else flist->malloced *= 2; @@ -305,7 +307,7 @@ static void flist_expand(struct file_list *flist, int extra) new_ptr = realloc_array(flist->files, struct file_struct *, flist->malloced); - if (DEBUG_GTE(FLIST, 1) && flist->malloced != FLIST_START) { + if (DEBUG_GTE(FLIST, 1) && flist->files) { rprintf(FCLIENT, "[%s] expand file_list pointer array to %s bytes, did%s move\n", who_am_i(), big_num(sizeof flist->files[0] * flist->malloced), @@ -2186,8 +2188,10 @@ struct file_list *send_file_list(int f, int argc, char *argv[]) #endif flist = cur_flist = flist_new(0, "send_file_list"); + flist_expand(flist, FLIST_START_LARGE); if (inc_recurse) { dir_flist = flist_new(FLIST_TEMP, "send_file_list"); + flist_expand(dir_flist, FLIST_START_LARGE); flags |= FLAG_DIVERT_DIRS; } else dir_flist = cur_flist; @@ -2541,10 +2545,13 @@ struct file_list *recv_file_list(int f, int dir_ndx) #endif flist = flist_new(0, "recv_file_list"); + flist_expand(flist, FLIST_START_LARGE); if (inc_recurse) { - if (flist->ndx_start == 1) + if (flist->ndx_start == 1) { dir_flist = flist_new(FLIST_TEMP, "recv_file_list"); + flist_expand(dir_flist, FLIST_START_LARGE); + } dstart = dir_flist->used; } else { dir_flist = flist; diff --git a/rsync.h b/rsync.h index 88319732..f8fcbffb 100644 --- a/rsync.h +++ b/rsync.h @@ -918,8 +918,9 @@ extern int xattrs_ndx; * Start the flist array at FLIST_START entries and grow it * by doubling until FLIST_LINEAR then grow by FLIST_LINEAR */ -#define FLIST_START (32 * 1024) -#define FLIST_LINEAR (FLIST_START * 512) +#define FLIST_START (32) +#define FLIST_START_LARGE (32 * 1024) +#define FLIST_LINEAR (FLIST_START_LARGE * 512) /* * Extent size for allocation pools: A minimum size of 128KB -- The rsync repository.
_______________________________________________ rsync-cvs mailing list rsync-cvs@lists.samba.org https://lists.samba.org/mailman/listinfo/rsync-cvs