From d88dbaa71864c42df1394be25234d7c187a12f48 Mon Sep 17 00:00:00 2001 From: Elijah Newren Date: Tue, 11 Apr 2023 00:41:51 -0700 Subject: git-zlib: move declarations for git-zlib functions from cache.h Move functions from cache.h for zlib.c into a new header file. Since adding a "zlib.h" would cause issues with the real zlib, rename zlib.c to git-zlib.c while we are at it. Signed-off-by: Elijah Newren Acked-by: Calvin Wan Signed-off-by: Junio C Hamano --- Makefile | 2 +- archive-tar.c | 1 + archive-zip.c | 1 + builtin/unpack-objects.c | 1 + cache.h | 25 +---- git-zlib.c | 274 +++++++++++++++++++++++++++++++++++++++++++++++ git-zlib.h | 28 +++++ http-backend.c | 1 + http.h | 3 + zlib.c | 273 ---------------------------------------------- 10 files changed, 311 insertions(+), 298 deletions(-) create mode 100644 git-zlib.c create mode 100644 git-zlib.h delete mode 100644 zlib.c diff --git a/Makefile b/Makefile index 50ee51fde3..0c367881bf 100644 --- a/Makefile +++ b/Makefile @@ -1036,6 +1036,7 @@ LIB_OBJS += fsmonitor.o LIB_OBJS += fsmonitor-ipc.o LIB_OBJS += fsmonitor-settings.o LIB_OBJS += gettext.o +LIB_OBJS += git-zlib.o LIB_OBJS += gpg-interface.o LIB_OBJS += graph.o LIB_OBJS += grep.o @@ -1196,7 +1197,6 @@ LIB_OBJS += write-or-die.o LIB_OBJS += ws.o LIB_OBJS += wt-status.o LIB_OBJS += xdiff-interface.o -LIB_OBJS += zlib.o BUILTIN_OBJS += builtin/add.o BUILTIN_OBJS += builtin/am.o diff --git a/archive-tar.c b/archive-tar.c index 497dad0b3a..4cd81d8161 100644 --- a/archive-tar.c +++ b/archive-tar.c @@ -5,6 +5,7 @@ #include "alloc.h" #include "config.h" #include "gettext.h" +#include "git-zlib.h" #include "hex.h" #include "tar.h" #include "archive.h" diff --git a/archive-zip.c b/archive-zip.c index e6f5c10a14..ef538a90df 100644 --- a/archive-zip.c +++ b/archive-zip.c @@ -5,6 +5,7 @@ #include "config.h" #include "archive.h" #include "gettext.h" +#include "git-zlib.h" #include "hex.h" #include "streaming.h" #include "utf8.h" diff --git a/builtin/unpack-objects.c b/builtin/unpack-objects.c index f2c1323e66..ddbdb3dd22 100644 --- a/builtin/unpack-objects.c +++ b/builtin/unpack-objects.c @@ -4,6 +4,7 @@ #include "config.h" #include "environment.h" #include "gettext.h" +#include "git-zlib.h" #include "hex.h" #include "object-store.h" #include "object.h" diff --git a/cache.h b/cache.h index 21db5da39a..c06778b69c 100644 --- a/cache.h +++ b/cache.h @@ -3,6 +3,7 @@ #include "git-compat-util.h" #include "strbuf.h" +#include "git-zlib.h" #include "hashmap.h" #include "list.h" #include "gettext.h" @@ -14,30 +15,6 @@ #include "repository.h" #include "statinfo.h" -typedef struct git_zstream { - z_stream z; - unsigned long avail_in; - unsigned long avail_out; - unsigned long total_in; - unsigned long total_out; - unsigned char *next_in; - unsigned char *next_out; -} git_zstream; - -void git_inflate_init(git_zstream *); -void git_inflate_init_gzip_only(git_zstream *); -void git_inflate_end(git_zstream *); -int git_inflate(git_zstream *, int flush); - -void git_deflate_init(git_zstream *, int level); -void git_deflate_init_gzip(git_zstream *, int level); -void git_deflate_init_raw(git_zstream *, int level); -void git_deflate_end(git_zstream *); -int git_deflate_abort(git_zstream *); -int git_deflate_end_gently(git_zstream *); -int git_deflate(git_zstream *, int flush); -unsigned long git_deflate_bound(git_zstream *, unsigned long); - #if defined(DT_UNKNOWN) && !defined(NO_D_TYPE_IN_DIRENT) #define DTYPE(de) ((de)->d_type) #else diff --git a/git-zlib.c b/git-zlib.c new file mode 100644 index 0000000000..d43bbeb6da --- /dev/null +++ b/git-zlib.c @@ -0,0 +1,274 @@ +/* + * zlib wrappers to make sure we don't silently miss errors + * at init time. + */ +#include "git-compat-util.h" +#include "git-zlib.h" + +static const char *zerr_to_string(int status) +{ + switch (status) { + case Z_MEM_ERROR: + return "out of memory"; + case Z_VERSION_ERROR: + return "wrong version"; + case Z_NEED_DICT: + return "needs dictionary"; + case Z_DATA_ERROR: + return "data stream error"; + case Z_STREAM_ERROR: + return "stream consistency error"; + default: + return "unknown error"; + } +} + +/* + * avail_in and avail_out in zlib are counted in uInt, which typically + * limits the size of the buffer we can use to 4GB when interacting + * with zlib in a single call to inflate/deflate. + */ +/* #define ZLIB_BUF_MAX ((uInt)-1) */ +#define ZLIB_BUF_MAX ((uInt) 1024 * 1024 * 1024) /* 1GB */ +static inline uInt zlib_buf_cap(unsigned long len) +{ + return (ZLIB_BUF_MAX < len) ? ZLIB_BUF_MAX : len; +} + +static void zlib_pre_call(git_zstream *s) +{ + s->z.next_in = s->next_in; + s->z.next_out = s->next_out; + s->z.total_in = s->total_in; + s->z.total_out = s->total_out; + s->z.avail_in = zlib_buf_cap(s->avail_in); + s->z.avail_out = zlib_buf_cap(s->avail_out); +} + +static void zlib_post_call(git_zstream *s) +{ + unsigned long bytes_consumed; + unsigned long bytes_produced; + + bytes_consumed = s->z.next_in - s->next_in; + bytes_produced = s->z.next_out - s->next_out; + if (s->z.total_out != s->total_out + bytes_produced) + BUG("total_out mismatch"); + if (s->z.total_in != s->total_in + bytes_consumed) + BUG("total_in mismatch"); + + s->total_out = s->z.total_out; + s->total_in = s->z.total_in; + s->next_in = s->z.next_in; + s->next_out = s->z.next_out; + s->avail_in -= bytes_consumed; + s->avail_out -= bytes_produced; +} + +void git_inflate_init(git_zstream *strm) +{ + int status; + + zlib_pre_call(strm); + status = inflateInit(&strm->z); + zlib_post_call(strm); + if (status == Z_OK) + return; + die("inflateInit: %s (%s)", zerr_to_string(status), + strm->z.msg ? strm->z.msg : "no message"); +} + +void git_inflate_init_gzip_only(git_zstream *strm) +{ + /* + * Use default 15 bits, +16 is to accept only gzip and to + * yield Z_DATA_ERROR when fed zlib format. + */ + const int windowBits = 15 + 16; + int status; + + zlib_pre_call(strm); + status = inflateInit2(&strm->z, windowBits); + zlib_post_call(strm); + if (status == Z_OK) + return; + die("inflateInit2: %s (%s)", zerr_to_string(status), + strm->z.msg ? strm->z.msg : "no message"); +} + +void git_inflate_end(git_zstream *strm) +{ + int status; + + zlib_pre_call(strm); + status = inflateEnd(&strm->z); + zlib_post_call(strm); + if (status == Z_OK) + return; + error("inflateEnd: %s (%s)", zerr_to_string(status), + strm->z.msg ? strm->z.msg : "no message"); +} + +int git_inflate(git_zstream *strm, int flush) +{ + int status; + + for (;;) { + zlib_pre_call(strm); + /* Never say Z_FINISH unless we are feeding everything */ + status = inflate(&strm->z, + (strm->z.avail_in != strm->avail_in) + ? 0 : flush); + if (status == Z_MEM_ERROR) + die("inflate: out of memory"); + zlib_post_call(strm); + + /* + * Let zlib work another round, while we can still + * make progress. + */ + if ((strm->avail_out && !strm->z.avail_out) && + (status == Z_OK || status == Z_BUF_ERROR)) + continue; + break; + } + + switch (status) { + /* Z_BUF_ERROR: normal, needs more space in the output buffer */ + case Z_BUF_ERROR: + case Z_OK: + case Z_STREAM_END: + return status; + default: + break; + } + error("inflate: %s (%s)", zerr_to_string(status), + strm->z.msg ? strm->z.msg : "no message"); + return status; +} + +#if defined(NO_DEFLATE_BOUND) || ZLIB_VERNUM < 0x1200 +#define deflateBound(c,s) ((s) + (((s) + 7) >> 3) + (((s) + 63) >> 6) + 11) +#endif + +unsigned long git_deflate_bound(git_zstream *strm, unsigned long size) +{ + return deflateBound(&strm->z, size); +} + +void git_deflate_init(git_zstream *strm, int level) +{ + int status; + + memset(strm, 0, sizeof(*strm)); + zlib_pre_call(strm); + status = deflateInit(&strm->z, level); + zlib_post_call(strm); + if (status == Z_OK) + return; + die("deflateInit: %s (%s)", zerr_to_string(status), + strm->z.msg ? strm->z.msg : "no message"); +} + +static void do_git_deflate_init(git_zstream *strm, int level, int windowBits) +{ + int status; + + memset(strm, 0, sizeof(*strm)); + zlib_pre_call(strm); + status = deflateInit2(&strm->z, level, + Z_DEFLATED, windowBits, + 8, Z_DEFAULT_STRATEGY); + zlib_post_call(strm); + if (status == Z_OK) + return; + die("deflateInit2: %s (%s)", zerr_to_string(status), + strm->z.msg ? strm->z.msg : "no message"); +} + +void git_deflate_init_gzip(git_zstream *strm, int level) +{ + /* + * Use default 15 bits, +16 is to generate gzip header/trailer + * instead of the zlib wrapper. + */ + do_git_deflate_init(strm, level, 15 + 16); +} + +void git_deflate_init_raw(git_zstream *strm, int level) +{ + /* + * Use default 15 bits, negate the value to get raw compressed + * data without zlib header and trailer. + */ + do_git_deflate_init(strm, level, -15); +} + +int git_deflate_abort(git_zstream *strm) +{ + int status; + + zlib_pre_call(strm); + status = deflateEnd(&strm->z); + zlib_post_call(strm); + return status; +} + +void git_deflate_end(git_zstream *strm) +{ + int status = git_deflate_abort(strm); + + if (status == Z_OK) + return; + error("deflateEnd: %s (%s)", zerr_to_string(status), + strm->z.msg ? strm->z.msg : "no message"); +} + +int git_deflate_end_gently(git_zstream *strm) +{ + int status; + + zlib_pre_call(strm); + status = deflateEnd(&strm->z); + zlib_post_call(strm); + return status; +} + +int git_deflate(git_zstream *strm, int flush) +{ + int status; + + for (;;) { + zlib_pre_call(strm); + + /* Never say Z_FINISH unless we are feeding everything */ + status = deflate(&strm->z, + (strm->z.avail_in != strm->avail_in) + ? 0 : flush); + if (status == Z_MEM_ERROR) + die("deflate: out of memory"); + zlib_post_call(strm); + + /* + * Let zlib work another round, while we can still + * make progress. + */ + if ((strm->avail_out && !strm->z.avail_out) && + (status == Z_OK || status == Z_BUF_ERROR)) + continue; + break; + } + + switch (status) { + /* Z_BUF_ERROR: normal, needs more space in the output buffer */ + case Z_BUF_ERROR: + case Z_OK: + case Z_STREAM_END: + return status; + default: + break; + } + error("deflate: %s (%s)", zerr_to_string(status), + strm->z.msg ? strm->z.msg : "no message"); + return status; +} diff --git a/git-zlib.h b/git-zlib.h new file mode 100644 index 0000000000..d8a670aff9 --- /dev/null +++ b/git-zlib.h @@ -0,0 +1,28 @@ +#ifndef GIT_ZLIB_H +#define GIT_ZLIB_H + +typedef struct git_zstream { + z_stream z; + unsigned long avail_in; + unsigned long avail_out; + unsigned long total_in; + unsigned long total_out; + unsigned char *next_in; + unsigned char *next_out; +} git_zstream; + +void git_inflate_init(git_zstream *); +void git_inflate_init_gzip_only(git_zstream *); +void git_inflate_end(git_zstream *); +int git_inflate(git_zstream *, int flush); + +void git_deflate_init(git_zstream *, int level); +void git_deflate_init_gzip(git_zstream *, int level); +void git_deflate_init_raw(git_zstream *, int level); +void git_deflate_end(git_zstream *); +int git_deflate_abort(git_zstream *); +int git_deflate_end_gently(git_zstream *); +int git_deflate(git_zstream *, int flush); +unsigned long git_deflate_bound(git_zstream *, unsigned long); + +#endif /* GIT_ZLIB_H */ diff --git a/http-backend.c b/http-backend.c index 89aad1b42c..d41b3b9e1e 100644 --- a/http-backend.c +++ b/http-backend.c @@ -2,6 +2,7 @@ #include "alloc.h" #include "config.h" #include "environment.h" +#include "git-zlib.h" #include "hex.h" #include "repository.h" #include "refs.h" diff --git a/http.h b/http.h index 77c042706c..783b2b09b8 100644 --- a/http.h +++ b/http.h @@ -1,7 +1,10 @@ #ifndef HTTP_H #define HTTP_H +struct packed_git; + #include "cache.h" +#include "git-zlib.h" #include #include diff --git a/zlib.c b/zlib.c deleted file mode 100644 index d594cba3fc..0000000000 --- a/zlib.c +++ /dev/null @@ -1,273 +0,0 @@ -/* - * zlib wrappers to make sure we don't silently miss errors - * at init time. - */ -#include "cache.h" - -static const char *zerr_to_string(int status) -{ - switch (status) { - case Z_MEM_ERROR: - return "out of memory"; - case Z_VERSION_ERROR: - return "wrong version"; - case Z_NEED_DICT: - return "needs dictionary"; - case Z_DATA_ERROR: - return "data stream error"; - case Z_STREAM_ERROR: - return "stream consistency error"; - default: - return "unknown error"; - } -} - -/* - * avail_in and avail_out in zlib are counted in uInt, which typically - * limits the size of the buffer we can use to 4GB when interacting - * with zlib in a single call to inflate/deflate. - */ -/* #define ZLIB_BUF_MAX ((uInt)-1) */ -#define ZLIB_BUF_MAX ((uInt) 1024 * 1024 * 1024) /* 1GB */ -static inline uInt zlib_buf_cap(unsigned long len) -{ - return (ZLIB_BUF_MAX < len) ? ZLIB_BUF_MAX : len; -} - -static void zlib_pre_call(git_zstream *s) -{ - s->z.next_in = s->next_in; - s->z.next_out = s->next_out; - s->z.total_in = s->total_in; - s->z.total_out = s->total_out; - s->z.avail_in = zlib_buf_cap(s->avail_in); - s->z.avail_out = zlib_buf_cap(s->avail_out); -} - -static void zlib_post_call(git_zstream *s) -{ - unsigned long bytes_consumed; - unsigned long bytes_produced; - - bytes_consumed = s->z.next_in - s->next_in; - bytes_produced = s->z.next_out - s->next_out; - if (s->z.total_out != s->total_out + bytes_produced) - BUG("total_out mismatch"); - if (s->z.total_in != s->total_in + bytes_consumed) - BUG("total_in mismatch"); - - s->total_out = s->z.total_out; - s->total_in = s->z.total_in; - s->next_in = s->z.next_in; - s->next_out = s->z.next_out; - s->avail_in -= bytes_consumed; - s->avail_out -= bytes_produced; -} - -void git_inflate_init(git_zstream *strm) -{ - int status; - - zlib_pre_call(strm); - status = inflateInit(&strm->z); - zlib_post_call(strm); - if (status == Z_OK) - return; - die("inflateInit: %s (%s)", zerr_to_string(status), - strm->z.msg ? strm->z.msg : "no message"); -} - -void git_inflate_init_gzip_only(git_zstream *strm) -{ - /* - * Use default 15 bits, +16 is to accept only gzip and to - * yield Z_DATA_ERROR when fed zlib format. - */ - const int windowBits = 15 + 16; - int status; - - zlib_pre_call(strm); - status = inflateInit2(&strm->z, windowBits); - zlib_post_call(strm); - if (status == Z_OK) - return; - die("inflateInit2: %s (%s)", zerr_to_string(status), - strm->z.msg ? strm->z.msg : "no message"); -} - -void git_inflate_end(git_zstream *strm) -{ - int status; - - zlib_pre_call(strm); - status = inflateEnd(&strm->z); - zlib_post_call(strm); - if (status == Z_OK) - return; - error("inflateEnd: %s (%s)", zerr_to_string(status), - strm->z.msg ? strm->z.msg : "no message"); -} - -int git_inflate(git_zstream *strm, int flush) -{ - int status; - - for (;;) { - zlib_pre_call(strm); - /* Never say Z_FINISH unless we are feeding everything */ - status = inflate(&strm->z, - (strm->z.avail_in != strm->avail_in) - ? 0 : flush); - if (status == Z_MEM_ERROR) - die("inflate: out of memory"); - zlib_post_call(strm); - - /* - * Let zlib work another round, while we can still - * make progress. - */ - if ((strm->avail_out && !strm->z.avail_out) && - (status == Z_OK || status == Z_BUF_ERROR)) - continue; - break; - } - - switch (status) { - /* Z_BUF_ERROR: normal, needs more space in the output buffer */ - case Z_BUF_ERROR: - case Z_OK: - case Z_STREAM_END: - return status; - default: - break; - } - error("inflate: %s (%s)", zerr_to_string(status), - strm->z.msg ? strm->z.msg : "no message"); - return status; -} - -#if defined(NO_DEFLATE_BOUND) || ZLIB_VERNUM < 0x1200 -#define deflateBound(c,s) ((s) + (((s) + 7) >> 3) + (((s) + 63) >> 6) + 11) -#endif - -unsigned long git_deflate_bound(git_zstream *strm, unsigned long size) -{ - return deflateBound(&strm->z, size); -} - -void git_deflate_init(git_zstream *strm, int level) -{ - int status; - - memset(strm, 0, sizeof(*strm)); - zlib_pre_call(strm); - status = deflateInit(&strm->z, level); - zlib_post_call(strm); - if (status == Z_OK) - return; - die("deflateInit: %s (%s)", zerr_to_string(status), - strm->z.msg ? strm->z.msg : "no message"); -} - -static void do_git_deflate_init(git_zstream *strm, int level, int windowBits) -{ - int status; - - memset(strm, 0, sizeof(*strm)); - zlib_pre_call(strm); - status = deflateInit2(&strm->z, level, - Z_DEFLATED, windowBits, - 8, Z_DEFAULT_STRATEGY); - zlib_post_call(strm); - if (status == Z_OK) - return; - die("deflateInit2: %s (%s)", zerr_to_string(status), - strm->z.msg ? strm->z.msg : "no message"); -} - -void git_deflate_init_gzip(git_zstream *strm, int level) -{ - /* - * Use default 15 bits, +16 is to generate gzip header/trailer - * instead of the zlib wrapper. - */ - do_git_deflate_init(strm, level, 15 + 16); -} - -void git_deflate_init_raw(git_zstream *strm, int level) -{ - /* - * Use default 15 bits, negate the value to get raw compressed - * data without zlib header and trailer. - */ - do_git_deflate_init(strm, level, -15); -} - -int git_deflate_abort(git_zstream *strm) -{ - int status; - - zlib_pre_call(strm); - status = deflateEnd(&strm->z); - zlib_post_call(strm); - return status; -} - -void git_deflate_end(git_zstream *strm) -{ - int status = git_deflate_abort(strm); - - if (status == Z_OK) - return; - error("deflateEnd: %s (%s)", zerr_to_string(status), - strm->z.msg ? strm->z.msg : "no message"); -} - -int git_deflate_end_gently(git_zstream *strm) -{ - int status; - - zlib_pre_call(strm); - status = deflateEnd(&strm->z); - zlib_post_call(strm); - return status; -} - -int git_deflate(git_zstream *strm, int flush) -{ - int status; - - for (;;) { - zlib_pre_call(strm); - - /* Never say Z_FINISH unless we are feeding everything */ - status = deflate(&strm->z, - (strm->z.avail_in != strm->avail_in) - ? 0 : flush); - if (status == Z_MEM_ERROR) - die("deflate: out of memory"); - zlib_post_call(strm); - - /* - * Let zlib work another round, while we can still - * make progress. - */ - if ((strm->avail_out && !strm->z.avail_out) && - (status == Z_OK || status == Z_BUF_ERROR)) - continue; - break; - } - - switch (status) { - /* Z_BUF_ERROR: normal, needs more space in the output buffer */ - case Z_BUF_ERROR: - case Z_OK: - case Z_STREAM_END: - return status; - default: - break; - } - error("deflate: %s (%s)", zerr_to_string(status), - strm->z.msg ? strm->z.msg : "no message"); - return status; -} -- cgit v1.3-5-g45d5