On Thu, Jun 24, 2021 at 11:09:05AM -0600, Matt Dowle wrote: > Hi, > > Is it intentional or is there any good reason that OpenBSD 6.9 released May > 2021 uses a 16 year old version of zlib (v1.2.3; July 2005)? The latest > version v1.2.11 (Jan 2017) is 4 years old. > > Background here: https://github.com/Rdatatable/data.table/pull/5049
If we want to go the cherry-picking route, here's a diff that fixes the test.csv.gz test case provided in the linked issue. All of test.data.table() now passes: All 9062 tests (last 2163) in tests/tests.Rraw.bz2 completed ok in 57.9s elapsed (49.9s cpu) I haven't done further testing as deflateBound() is not used in the base system. This backports zlib's deflate.c diff between 1.2.3 and 1.2.3.1. It adjusts the calculation of the bound by calculating the wrapper length instead of trying to bound it. The result can now be larger than the previous bound. It modifies the two copies of deflate.c in /usr/src where the change would apply cleanly. Perl's version has the change and the copy in gnu/usr.bin/cvs is even older... https://github.com/madler/zlib/commit/b1c19ca6d82c98a8be6cd9cad7a9c5fa5e8e634e#diff-7e5fd0aa55941ed3e3c9c564a713b3841f20f7defc549277cdf94f6c90882af8 Index: lib/libz/deflate.c =================================================================== RCS file: /cvs/src/lib/libz/deflate.c,v retrieving revision 1.11 diff -u -p -r1.11 deflate.c --- lib/libz/deflate.c 27 Oct 2009 23:59:31 -0000 1.11 +++ lib/libz/deflate.c 25 Jun 2021 03:03:40 -0000 @@ -479,33 +479,65 @@ int ZEXPORT deflateTune(strm, good_lengt * resulting from using fixed blocks instead of stored blocks, which deflate * can emit on compressed data for some combinations of the parameters. * - * This function could be more sophisticated to provide closer upper bounds - * for every combination of windowBits and memLevel, as well as wrap. - * But even the conservative upper bound of about 14% expansion does not - * seem onerous for output buffer allocation. + * This function could be more sophisticated to provide closer upper bounds for + * every combination of windowBits and memLevel. But even the conservative + * upper bound of about 14% expansion does not seem onerous for output buffer + * allocation. */ uLong ZEXPORT deflateBound(strm, sourceLen) z_streamp strm; uLong sourceLen; { deflate_state *s; - uLong destLen; + uLong complen, wraplen; + Bytef *str; - /* conservative upper bound */ - destLen = sourceLen + - ((sourceLen + 7) >> 3) + ((sourceLen + 63) >> 6) + 11; + /* conservative upper bound for compressed data */ + complen = sourceLen + + ((sourceLen + 7) >> 3) + ((sourceLen + 63) >> 6) + 5; - /* if can't get parameters, return conservative bound */ + /* if can't get parameters, return conservative bound plus zlib wrapper */ if (strm == Z_NULL || strm->state == Z_NULL) - return destLen; + return complen + 6; - /* if not default parameters, return conservative bound */ + /* compute wrapper length */ s = strm->state; + switch (s->wrap) { + case 0: /* raw deflate */ + wraplen = 0; + break; + case 1: /* zlib wrapper */ + wraplen = 6 + (s->strstart ? 4 : 0); + break; + case 2: /* gzip wrapper */ + wraplen = 18; + if (s->gzhead != NULL) { /* user-supplied gzip header */ + if (s->gzhead->extra != NULL) + wraplen += 2 + s->gzhead->extra_len; + str = s->gzhead->name; + if (str != NULL) + do { + wraplen++; + } while (*str++); + str = s->gzhead->comment; + if (str != NULL) + do { + wraplen++; + } while (*str++); + if (s->gzhead->hcrc) + wraplen += 2; + } + break; + default: /* for compiler happiness */ + wraplen = 6; + } + + /* if not default parameters, return conservative bound */ if (s->w_bits != 15 || s->hash_bits != 8 + 7) - return destLen; + return complen + wraplen; /* default settings: return tight bound for that case */ - return compressBound(sourceLen); + return compressBound(sourceLen) - 6 + wraplen; } /* ========================================================================= Index: sys/lib/libz/deflate.c =================================================================== RCS file: /cvs/src/sys/lib/libz/deflate.c,v retrieving revision 1.3 diff -u -p -r1.3 deflate.c --- sys/lib/libz/deflate.c 14 Mar 2016 23:08:06 -0000 1.3 +++ sys/lib/libz/deflate.c 25 Jun 2021 03:04:49 -0000 @@ -479,33 +479,65 @@ int ZEXPORT deflateTune(strm, good_lengt * resulting from using fixed blocks instead of stored blocks, which deflate * can emit on compressed data for some combinations of the parameters. * - * This function could be more sophisticated to provide closer upper bounds - * for every combination of windowBits and memLevel, as well as wrap. - * But even the conservative upper bound of about 14% expansion does not - * seem onerous for output buffer allocation. + * This function could be more sophisticated to provide closer upper bounds for + * every combination of windowBits and memLevel. But even the conservative + * upper bound of about 14% expansion does not seem onerous for output buffer + * allocation. */ uLong ZEXPORT deflateBound(strm, sourceLen) z_streamp strm; uLong sourceLen; { deflate_state *s; - uLong destLen; + uLong complen, wraplen; + Bytef *str; - /* conservative upper bound */ - destLen = sourceLen + - ((sourceLen + 7) >> 3) + ((sourceLen + 63) >> 6) + 11; + /* conservative upper bound for compressed data */ + complen = sourceLen + + ((sourceLen + 7) >> 3) + ((sourceLen + 63) >> 6) + 5; - /* if can't get parameters, return conservative bound */ + /* if can't get parameters, return conservative bound plus zlib wrapper */ if (strm == Z_NULL || strm->state == Z_NULL) - return destLen; + return complen + 6; - /* if not default parameters, return conservative bound */ + /* compute wrapper length */ s = strm->state; + switch (s->wrap) { + case 0: /* raw deflate */ + wraplen = 0; + break; + case 1: /* zlib wrapper */ + wraplen = 6 + (s->strstart ? 4 : 0); + break; + case 2: /* gzip wrapper */ + wraplen = 18; + if (s->gzhead != NULL) { /* user-supplied gzip header */ + if (s->gzhead->extra != NULL) + wraplen += 2 + s->gzhead->extra_len; + str = s->gzhead->name; + if (str != NULL) + do { + wraplen++; + } while (*str++); + str = s->gzhead->comment; + if (str != NULL) + do { + wraplen++; + } while (*str++); + if (s->gzhead->hcrc) + wraplen += 2; + } + break; + default: /* for compiler happiness */ + wraplen = 6; + } + + /* if not default parameters, return conservative bound */ if (s->w_bits != 15 || s->hash_bits != 8 + 7) - return destLen; + return complen + wraplen; /* default settings: return tight bound for that case */ - return compressBound(sourceLen); + return compressBound(sourceLen) - 6 + wraplen; } /* =========================================================================
