From: tb Date: Sat, 18 Nov 2023 22:43:25 +0000 (+0000) Subject: libz sync with develop branch, discussed with deraadt X-Git-Url: http://artulab.com/gitweb/?a=commitdiff_plain;h=62a6fda735da47fbe45d938a823cd676d7e0b71d;p=openbsd libz sync with develop branch, discussed with deraadt Apart from cosmetics, this includes the following commits: Add LIT_MEM define to use more memory for a small deflate speedup. A bug fix in zlib 1.2.12 resulted in a slight slowdown (1-2%) of deflate. This commit provides the option to #define LIT_MEM, which uses more memory to reverse most of that slowdown. The memory for the pending buffer and symbol buffers is increased by 25%, which increases the total memory usage with the default parameters by about 6%. https://github.com/madler/zlib/commit/ac8f12c97d1afd9bafa9c710f827d40a407d3266 Fix bug in inflateSync() for data held in bit buffer. https://github.com/madler/zlib/commit/5af7cef45eeef86ddf6ab00b4e363c1eecaf47b6 --- diff --git a/lib/libz/ChangeLog b/lib/libz/ChangeLog index 58fa08a5985..49de6135d10 100644 --- a/lib/libz/ChangeLog +++ b/lib/libz/ChangeLog @@ -1,6 +1,9 @@ ChangeLog file for zlib +Changes in 1.3.0.1 (xx Aug 2023) +- + Changes in 1.3 (18 Aug 2023) - Remove K&R function definitions and zlib2ansi - Fix bug in deflateBound() for level 0 and memLevel 9 diff --git a/lib/libz/README b/lib/libz/README index e02fc5aa206..20f7064a5fd 100644 --- a/lib/libz/README +++ b/lib/libz/README @@ -1,6 +1,6 @@ ZLIB DATA COMPRESSION LIBRARY -zlib 1.3 is a general purpose data compression library. All the code is +zlib 1.3.0.1 is a general purpose data compression library. All the code is thread safe. The data format used by the zlib library is described by RFCs (Request for Comments) 1950 to 1952 in the files http://tools.ietf.org/html/rfc1950 (zlib format), rfc1951 (deflate format) and @@ -31,7 +31,7 @@ Mark Nelson wrote an article about zlib for the Jan. 1997 issue of Dr. Dobb's Journal; a copy of the article is available at https://marknelson.us/posts/1997/01/01/zlib-engine.html . -The changes made in version 1.3 are documented in the file ChangeLog. +The changes made in version 1.3.0.1 are documented in the file ChangeLog. Unsupported third party contributions are provided in directory contrib/ . diff --git a/lib/libz/deflate.c b/lib/libz/deflate.c index 0f8130a6517..f878ebc99f2 100644 --- a/lib/libz/deflate.c +++ b/lib/libz/deflate.c @@ -489,7 +489,11 @@ int ZEXPORT deflateInit2_(z_streamp strm, int level, int method, * symbols from which it is being constructed. */ +#ifdef LIT_MEM + s->pending_buf = (uchf *) ZALLOC(strm, s->lit_bufsize, 5); +#else s->pending_buf = (uchf *) ZALLOC(strm, s->lit_bufsize, 4); +#endif s->pending_buf_size = (ulg)s->lit_bufsize * 4; if (s->window == Z_NULL || s->prev == Z_NULL || s->head == Z_NULL || @@ -499,8 +503,14 @@ int ZEXPORT deflateInit2_(z_streamp strm, int level, int method, deflateEnd (strm); return Z_MEM_ERROR; } +#ifdef LIT_MEM + s->d_buf = (ushf *)(s->pending_buf + (s->lit_bufsize << 1)); + s->l_buf = s->pending_buf + (s->lit_bufsize << 2); + s->sym_end = s->lit_bufsize - 1; +#else s->sym_buf = s->pending_buf + s->lit_bufsize; s->sym_end = (s->lit_bufsize - 1) * 3; +#endif /* We avoid equality with lit_bufsize*3 because of wraparound at 64K * on 16 bit machines and because stored blocks are restricted to * 64K-1 bytes. @@ -716,9 +726,15 @@ int ZEXPORT deflatePrime(z_streamp strm, int bits, int value) { if (deflateStateCheck(strm)) return Z_STREAM_ERROR; s = strm->state; +#ifdef LIT_MEM + if (bits < 0 || bits > 16 || + (uchf *)s->d_buf < s->pending_out + ((Buf_size + 7) >> 3)) + return Z_BUF_ERROR; +#else if (bits < 0 || bits > 16 || s->sym_buf < s->pending_out + ((Buf_size + 7) >> 3)) return Z_BUF_ERROR; +#endif do { put = Buf_size - s->bi_valid; if (put > bits) @@ -1304,7 +1320,12 @@ int ZEXPORT deflateCopy(z_streamp dest, z_streamp source) { zmemcpy(ds->pending_buf, ss->pending_buf, (uInt)ds->pending_buf_size); ds->pending_out = ds->pending_buf + (ss->pending_out - ss->pending_buf); +#ifdef LIT_MEM + ds->d_buf = (ushf *)(ds->pending_buf + (ds->lit_bufsize << 1)); + ds->l_buf = ds->pending_buf + (ds->lit_bufsize << 2); +#else ds->sym_buf = ds->pending_buf + ds->lit_bufsize; +#endif ds->l_desc.dyn_tree = ds->dyn_ltree; ds->d_desc.dyn_tree = ds->dyn_dtree; diff --git a/lib/libz/deflate.h b/lib/libz/deflate.h index fc66a6ffb88..eb28c625ab2 100644 --- a/lib/libz/deflate.h +++ b/lib/libz/deflate.h @@ -21,6 +21,10 @@ # define GZIP #endif +/* define LIT_MEM to slightly increase the speed of deflate (order 1% to 2%) at + the cost of a larger memory footprint */ +/* #define LIT_MEM */ + /* =========================================================================== * Internal compression state. */ @@ -215,7 +219,12 @@ typedef struct internal_state { /* Depth of each subtree used as tie breaker for trees of equal frequency */ +#ifdef LIT_MEM + ushf *d_buf; /* buffer for distances */ + uchf *l_buf; /* buffer for literals/lengths */ +#else uchf *sym_buf; /* buffer for distances and literals/lengths */ +#endif uInt lit_bufsize; /* Size of match buffer for literals/lengths. There are 4 reasons for @@ -237,7 +246,7 @@ typedef struct internal_state { * - I can't count above 4 */ - uInt sym_next; /* running index in sym_buf */ + uInt sym_next; /* running index in symbol buffer */ uInt sym_end; /* symbol table full when sym_next reaches this */ ulg opt_len; /* bit length of current block with optimal trees */ @@ -316,6 +325,25 @@ void ZLIB_INTERNAL _tr_stored_block(deflate_state *s, charf *buf, extern const uch ZLIB_INTERNAL _dist_code[]; #endif +#ifdef LIT_MEM +# define _tr_tally_lit(s, c, flush) \ + { uch cc = (c); \ + s->d_buf[s->sym_next] = 0; \ + s->l_buf[s->sym_next++] = cc; \ + s->dyn_ltree[cc].Freq++; \ + flush = (s->sym_next == s->sym_end); \ + } +# define _tr_tally_dist(s, distance, length, flush) \ + { uch len = (uch)(length); \ + ush dist = (ush)(distance); \ + s->d_buf[s->sym_next] = dist; \ + s->l_buf[s->sym_next++] = len; \ + dist--; \ + s->dyn_ltree[_length_code[len]+LITERALS+1].Freq++; \ + s->dyn_dtree[d_code(dist)].Freq++; \ + flush = (s->sym_next == s->sym_end); \ + } +#else # define _tr_tally_lit(s, c, flush) \ { uch cc = (c); \ s->sym_buf[s->sym_next++] = 0; \ @@ -335,6 +363,7 @@ void ZLIB_INTERNAL _tr_stored_block(deflate_state *s, charf *buf, s->dyn_dtree[d_code(dist)].Freq++; \ flush = (s->sym_next == s->sym_end); \ } +#endif #else # define _tr_tally_lit(s, c, flush) flush = _tr_tally(s, 0, c) # define _tr_tally_dist(s, distance, length, flush) \ diff --git a/lib/libz/inflate.c b/lib/libz/inflate.c index c73d8a27c25..caac5d61911 100644 --- a/lib/libz/inflate.c +++ b/lib/libz/inflate.c @@ -1473,7 +1473,7 @@ int ZEXPORT inflateSync(z_streamp strm) { /* if first time, start search in bit buffer */ if (state->mode != SYNC) { state->mode = SYNC; - state->hold <<= state->bits & 7; + state->hold >>= state->bits & 7; state->bits -= state->bits & 7; len = 0; while (state->bits >= 8) { diff --git a/lib/libz/inftrees.c b/lib/libz/inftrees.c index 9ff2067ee00..719e20d527f 100644 --- a/lib/libz/inftrees.c +++ b/lib/libz/inftrees.c @@ -55,7 +55,7 @@ int ZLIB_INTERNAL inflate_table(codetype type, unsigned short FAR *lens, 35, 43, 51, 59, 67, 83, 99, 115, 131, 163, 195, 227, 258, 0, 0}; static const unsigned short lext[31] = { /* Length codes 257..285 extra */ 16, 16, 16, 16, 16, 16, 16, 16, 17, 17, 17, 17, 18, 18, 18, 18, - 19, 19, 19, 19, 20, 20, 20, 20, 21, 21, 21, 21, 16, 198, 203}; + 19, 19, 19, 19, 20, 20, 20, 20, 21, 21, 21, 21, 16, 70, 200}; static const unsigned short dbase[32] = { /* Distance codes 0..29 base */ 1, 2, 3, 4, 5, 7, 9, 13, 17, 25, 33, 49, 65, 97, 129, 193, 257, 385, 513, 769, 1025, 1537, 2049, 3073, 4097, 6145, diff --git a/lib/libz/inftrees.h b/lib/libz/inftrees.h index a10712d8cb5..396f74b5da7 100644 --- a/lib/libz/inftrees.h +++ b/lib/libz/inftrees.h @@ -41,8 +41,8 @@ typedef struct { examples/enough.c found in the zlib distribution. The arguments to that program are the number of symbols, the initial root table size, and the maximum bit length of a code. "enough 286 9 15" for literal/length codes - returns returns 852, and "enough 30 6 15" for distance codes returns 592. - The initial root table size (9 or 6) is found in the fifth argument of the + returns 852, and "enough 30 6 15" for distance codes returns 592. The + initial root table size (9 or 6) is found in the fifth argument of the inflate_table() calls in inflate.c and infback.c. If the root table size is changed, then these maximum sizes would be need to be recalculated and updated. */ diff --git a/lib/libz/trees.c b/lib/libz/trees.c index 7d2ef3634e4..6bbbb21c377 100644 --- a/lib/libz/trees.c +++ b/lib/libz/trees.c @@ -897,14 +897,19 @@ local void compress_block(deflate_state *s, const ct_data *ltree, const ct_data *dtree) { unsigned dist; /* distance of matched string */ int lc; /* match length or unmatched char (if dist == 0) */ - unsigned sx = 0; /* running index in sym_buf */ + unsigned sx = 0; /* running index in symbol buffers */ unsigned code; /* the code to send */ int extra; /* number of extra bits to send */ if (s->sym_next != 0) do { +#ifdef LIT_MEM + dist = s->d_buf[sx]; + lc = s->l_buf[sx++]; +#else dist = s->sym_buf[sx++] & 0xff; dist += (unsigned)(s->sym_buf[sx++] & 0xff) << 8; lc = s->sym_buf[sx++]; +#endif if (dist == 0) { send_code(s, lc, ltree); /* send a literal byte */ Tracecv(isgraph(lc), (stderr," '%c' ", lc)); @@ -929,8 +934,12 @@ local void compress_block(deflate_state *s, const ct_data *ltree, } } /* literal or match pair ? */ - /* Check that the overlay between pending_buf and sym_buf is ok: */ + /* Check for no overlay of pending_buf on needed symbols */ +#ifdef LIT_MEM + Assert(s->pending < (s->lit_bufsize << 1) + sx, "pendingBuf overflow"); +#else Assert(s->pending < s->lit_bufsize + sx, "pendingBuf overflow"); +#endif } while (sx < s->sym_next); @@ -1080,9 +1089,14 @@ void ZLIB_INTERNAL _tr_flush_block(deflate_state *s, charf *buf, * the current block must be flushed. */ int ZLIB_INTERNAL _tr_tally(deflate_state *s, unsigned dist, unsigned lc) { +#ifdef LIT_MEM + s->d_buf[s->sym_next] = (ush)dist; + s->l_buf[s->sym_next++] = (uch)lc; +#else s->sym_buf[s->sym_next++] = (uch)dist; s->sym_buf[s->sym_next++] = (uch)(dist >> 8); s->sym_buf[s->sym_next++] = (uch)lc; +#endif if (dist == 0) { /* lc is the unmatched char */ s->dyn_ltree[lc].Freq++; diff --git a/lib/libz/zlib.h b/lib/libz/zlib.h index 6b7244f9943..b3e19fce8f4 100644 --- a/lib/libz/zlib.h +++ b/lib/libz/zlib.h @@ -1,5 +1,5 @@ /* zlib.h -- interface of the 'zlib' general purpose compression library - version 1.3, August 18th, 2023 + version 1.3.0.1, August xxth, 2023 Copyright (C) 1995-2023 Jean-loup Gailly and Mark Adler @@ -37,12 +37,12 @@ extern "C" { #endif -#define ZLIB_VERSION "1.3" -#define ZLIB_VERNUM 0x1300 +#define ZLIB_VERSION "1.3.0.1-motley" +#define ZLIB_VERNUM 0x1301 #define ZLIB_VER_MAJOR 1 #define ZLIB_VER_MINOR 3 #define ZLIB_VER_REVISION 0 -#define ZLIB_VER_SUBREVISION 0 +#define ZLIB_VER_SUBREVISION 1 /* The 'zlib' compression library provides in-memory compression and @@ -936,10 +936,10 @@ ZEXTERN int ZEXPORT inflateSync(z_streamp strm); inflateSync returns Z_OK if a possible full flush point has been found, Z_BUF_ERROR if no more input was provided, Z_DATA_ERROR if no flush point has been found, or Z_STREAM_ERROR if the stream structure was inconsistent. - In the success case, the application may save the current current value of - total_in which indicates where valid compressed data was found. In the - error case, the application may repeatedly call inflateSync, providing more - input each time, until success or end of the input data. + In the success case, the application may save the current value of total_in + which indicates where valid compressed data was found. In the error case, + the application may repeatedly call inflateSync, providing more input each + time, until success or end of the input data. */ ZEXTERN int ZEXPORT inflateCopy(z_streamp dest,