This version of the patch is still throwing warnings on the autobuilder: stdio: WARNING: lighttpd-1.4.55-r0 do_patch: Fuzz detected: stdio: WARNING: lighttpd-1.4.55-r0 do_patch: QA Issue: Patch log indicates that patches do not apply cleanly. [patch-fuzz]
Steve On Sun, Aug 8, 2021 at 9:20 PM Purushottam choudhary <purushottam.choudh...@kpit.com> wrote: > > Added 0001-core-reuse-large-mem-chunks-fix-mem-usage-fixes-3033.patch > to fix large memory usage for large file downloads > from dynamic backends reuse or release large memory chunks. > > This issue is caused by a bug in the lighttpd 1.4.55 version and > has been fixed in lighttpd 1.4.58. Hence, it is not needed for > master and hardknott branch because lighttpd has 1.4.59 version. > > Link: > https://redmine.lighttpd.net/projects/lighttpd/repository/14/revisions/7ba521ffb4959f6f74a609d5d4acafc29a038337 > > Signed-off-by: Purushottam Choudhary <purushottam.choudh...@kpit.com> > --- > ...large-mem-chunks-fix-mem-usage-fixes-3033.patch | 224 > +++++++++++++++++++++ > meta/recipes-extended/lighttpd/lighttpd_1.4.55.bb | 1 + > 2 files changed, 225 insertions(+) > create mode 100644 > meta/recipes-extended/lighttpd/lighttpd/0001-core-reuse-large-mem-chunks-fix-mem-usage-fixes-3033.patch > > diff --git > a/meta/recipes-extended/lighttpd/lighttpd/0001-core-reuse-large-mem-chunks-fix-mem-usage-fixes-3033.patch > > b/meta/recipes-extended/lighttpd/lighttpd/0001-core-reuse-large-mem-chunks-fix-mem-usage-fixes-3033.patch > new file mode 100644 > index 0000000..e3ede66 > --- /dev/null > +++ > b/meta/recipes-extended/lighttpd/lighttpd/0001-core-reuse-large-mem-chunks-fix-mem-usage-fixes-3033.patch > @@ -0,0 +1,224 @@ > +From f46bba11bf47ab6d6601e6ca653f1cdc7a2673ed Mon Sep 17 00:00:00 2001 > +From: Glenn Strauss <gstra...@gluelogic.com> > +Date: Wed, 23 Dec 2020 23:14:47 -0500 > +Subject: [PATCH] reuse large mem chunks (fix mem usage) (fixes #3033) > + > +(cherry picked from commit 7ba521ffb4959f6f74a609d5d4acafc29a038337) > + > +(thx flynn) > + > +fix large memory usage for large file downloads from dynamic backends > + > +reuse or release large memory chunks > + > +x-ref: > + "Memory Growth with PUT and full buffered streams" > + https://redmine.lighttpd.net/issues/3033 > + > +Upstream-Status: Backport > +Comment: No hunk refreshed > +https://redmine.lighttpd.net/projects/lighttpd/repository/14/revisions/7ba521ffb4959f6f74a609d5d4acafc29a038337 > +Signed-off-by: Purushottam Choudhary <purushottam.choudh...@kpit.com> > + > +--- > + src/chunk.c | 99 > ++++++++++++++++++++++++++++++++++++++++---------- > + src/chunk.h | 2 + > + src/http-header-glue.c | 2 +- > + 3 files changed, 82 insertions(+), 21 deletions(-) > + > +diff --git a/src/chunk.c b/src/chunk.c > +index 09dd3f1..ffb3795 100644 > +--- a/src/chunk.c > ++++ b/src/chunk.c > +@@ -28,16 +28,20 @@ > + static size_t chunk_buf_sz = 4096; > + static chunk *chunks, *chunks_oversized; > + static chunk *chunk_buffers; > ++static int chunks_oversized_n; > + static array *chunkqueue_default_tempdirs = NULL; > + static off_t chunkqueue_default_tempfile_size = DEFAULT_TEMPFILE_SIZE; > + > + void chunkqueue_set_chunk_size (size_t sz) > + { > +- chunk_buf_sz = sz > 0 ? ((sz + 1023) & ~1023uL) : 4096; > ++ size_t x = 1024; > ++ while (x < sz && x < (1u << 30)) x <<= 1; > ++ chunk_buf_sz = sz > 0 ? x : 4096; > + } > + > + void chunkqueue_set_tempdirs_default_reset (void) > + { > ++ chunk_buf_sz = 8192; > + chunkqueue_default_tempdirs = NULL; > + chunkqueue_default_tempfile_size = DEFAULT_TEMPFILE_SIZE; > + } > +@@ -120,15 +124,49 @@ static void chunk_free(chunk *c) { > + free(c); > + } > + > +-buffer * chunk_buffer_acquire(void) { > ++static chunk * chunk_pop_oversized(size_t sz) { > ++ /* future: might have buckets of certain sizes, up to socket buf sizes > */ > ++ if (chunks_oversized && chunks_oversized->mem->size >= sz) { > ++ --chunks_oversized_n; > ++ chunk *c = chunks_oversized; > ++ chunks_oversized = c->next; > ++ return c; > ++ } > ++ return NULL; > ++} > ++ > ++static void chunk_push_oversized(chunk * const c, const size_t sz) { > ++ if (chunks_oversized_n < 64 && chunk_buf_sz >= 4096) { > ++ ++chunks_oversized_n; > ++ chunk **co = &chunks_oversized; > ++ while (*co && sz < (*co)->mem->size) co = &(*co)->next; > ++ c->next = *co; > ++ *co = c; > ++ } > ++ else > ++ chunk_free(c); > ++} > ++ > ++static buffer * chunk_buffer_acquire_sz(size_t sz) { > + chunk *c; > + buffer *b; > +- if (chunks) { > +- c = chunks; > +- chunks = c->next; > ++ if (sz <= chunk_buf_sz) { > ++ if (chunks) { > ++ c = chunks; > ++ chunks = c->next; > ++ } > ++ else > ++ c = chunk_init(chunk_buf_sz); > ++ /* future: might choose to pop from chunks_oversized, if > available > ++ * (even if larger than sz) rather than allocating new chunk > ++ * (and if doing so, might replace chunks_oversized_n) */ > + } > + else { > +- c = chunk_init(chunk_buf_sz); > ++ /*(round up to nearest chunk_buf_sz)*/ > ++ sz = (sz + (chunk_buf_sz-1)) & ~(chunk_buf_sz-1); > ++ c = chunk_pop_oversized(sz); > ++ if (NULL == c) > ++ c = chunk_init(sz); > + } > + c->next = chunk_buffers; > + chunk_buffers = c; > +@@ -137,21 +175,47 @@ buffer * chunk_buffer_acquire(void) { > + return b; > + } > + > ++buffer * chunk_buffer_acquire(void) { > ++ return chunk_buffer_acquire_sz(chunk_buf_sz); > ++} > ++ > + void chunk_buffer_release(buffer *b) { > + if (NULL == b) return; > +- if (b->size >= chunk_buf_sz && chunk_buffers) { > ++ if (chunk_buffers) { > + chunk *c = chunk_buffers; > + chunk_buffers = c->next; > + c->mem = b; > +- c->next = chunks; > +- chunks = c; > + buffer_clear(b); > ++ if (b->size == chunk_buf_sz) { > ++ c->next = chunks; > ++ chunks = c; > ++ } > ++ else if (b->size > chunk_buf_sz) > ++ chunk_push_oversized(c, b->size); > ++ else > ++ chunk_free(c); > + } > + else { > + buffer_free(b); > + } > + } > + > ++size_t chunk_buffer_prepare_append(buffer * const b, size_t sz) { > ++ if (sz > chunk_buffer_string_space(b)) { > ++ sz += b->used ? b->used : 1; > ++ buffer * const cb = chunk_buffer_acquire_sz(sz); > ++ /* swap buffer contents and copy original b->ptr into larger b->ptr > */ > ++ /*(this does more than buffer_move())*/ > ++ buffer tb = *b; > ++ *b = *cb; > ++ *cb = tb; > ++ if ((b->used = tb.used)) > ++ memcpy(b->ptr, tb.ptr, tb.used); > ++ chunk_buffer_release(cb); > ++ } > ++ return chunk_buffer_string_space(b); > ++} > ++ > + static chunk * chunk_acquire(size_t sz) { > + if (sz <= chunk_buf_sz) { > + if (chunks) { > +@@ -162,13 +226,10 @@ static chunk * chunk_acquire(size_t sz) { > + sz = chunk_buf_sz; > + } > + else { > +- sz = (sz + 8191) & ~8191uL; > +- /* future: might have buckets of certain sizes, up to socket buf > sizes*/ > +- if (chunks_oversized && chunks_oversized->mem->size >= sz) { > +- chunk *c = chunks_oversized; > +- chunks_oversized = c->next; > +- return c; > +- } > ++ /*(round up to nearest chunk_buf_sz)*/ > ++ sz = (sz + (chunk_buf_sz-1)) & ~(chunk_buf_sz-1); > ++ chunk *c = chunk_pop_oversized(sz); > ++ if (c) return c; > + } > + > + return chunk_init(sz); > +@@ -183,10 +244,7 @@ static void chunk_release(chunk *c) { > + } > + else if (sz > chunk_buf_sz) { > + chunk_reset(c); > +- chunk **co = &chunks_oversized; > +- while (*co && sz < (*co)->mem->size) co = &(*co)->next; > +- c->next = *co; > +- *co = c; > ++ chunk_push_oversized(c, sz); > + } > + else { > + chunk_free(c); > +@@ -205,6 +263,7 @@ void chunkqueue_chunk_pool_clear(void) > + chunk_free(c); > + } > + chunks_oversized = NULL; > ++ chunks_oversized_n = 0; > + } > + > + void chunkqueue_chunk_pool_free(void) > +diff --git a/src/chunk.h b/src/chunk.h > +index 4c6b7e4..93f343c 100644 > +--- a/src/chunk.h > ++++ b/src/chunk.h > +@@ -50,6 +50,8 @@ typedef struct { > + buffer * chunk_buffer_acquire(void); > + void chunk_buffer_release(buffer *b); > + > ++size_t chunk_buffer_prepare_append (buffer *b, size_t sz); > ++ > + void chunkqueue_chunk_pool_clear(void); > + void chunkqueue_chunk_pool_free(void); > + > +diff --git a/src/http-header-glue.c b/src/http-header-glue.c > +index d54f00c..2231fba 100644 > +--- a/src/http-header-glue.c > ++++ b/src/http-header-glue.c > +@@ -1267,7 +1267,7 @@ handler_t http_response_read(server *srv, connection > *con, http_response_opts *o > + if (avail < toread) { > + /*(add avail+toread to reduce allocations when ioctl > EOPNOTSUPP)*/ > + avail = avail ? avail - 1 + toread : toread; > +- buffer_string_prepare_append(b, avail); > ++ avail = chunk_buffer_prepare_append(b, avail); > + } > + > + n = read(fd, b->ptr+buffer_string_length(b), avail); > diff --git a/meta/recipes-extended/lighttpd/lighttpd_1.4.55.bb > b/meta/recipes-extended/lighttpd/lighttpd_1.4.55.bb > index 35a268a..5329465 100644 > --- a/meta/recipes-extended/lighttpd/lighttpd_1.4.55.bb > +++ b/meta/recipes-extended/lighttpd/lighttpd_1.4.55.bb > @@ -18,6 +18,7 @@ SRC_URI = > "http://download.lighttpd.net/lighttpd/releases-1.4.x/lighttpd-${PV}.t > file://lighttpd.conf \ > file://lighttpd \ > > file://0001-Use-pkg-config-for-pcre-dependency-instead-of-config.patch \ > + > file://0001-core-reuse-large-mem-chunks-fix-mem-usage-fixes-3033.patch \ > " > > SRC_URI[md5sum] = "be4bda2c28bcbdac6eb941528f6edf03" > -- > 2.7.4 > > This message contains information that may be privileged or confidential and > is the property of the KPIT Technologies Ltd. It is intended only for the > person to whom it is addressed. If you are not the intended recipient, you > are not authorized to read, print, retain copy, disseminate, distribute, or > use this message or any part thereof. If you receive this message in error, > please notify the sender immediately and delete all copies of this message. > KPIT Technologies Ltd. does not accept any liability for virus infected mails. > > >
-=-=-=-=-=-=-=-=-=-=-=- Links: You receive all messages sent to this group. View/Reply Online (#154656): https://lists.openembedded.org/g/openembedded-core/message/154656 Mute This Topic: https://lists.openembedded.org/mt/84763682/21656 Group Owner: openembedded-core+ow...@lists.openembedded.org Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [arch...@mail-archive.com] -=-=-=-=-=-=-=-=-=-=-=-
