@matheusbsbs / matheus-bagliano / commits / b9d6f64393

compat/zlib: allow use of zlib-ng as backend

The zlib-ng library is a hard fork of the old and venerable zlib library. It describes itself as zlib replacement with optimizations for "next generation" systems. As such, it contains several implementations of central algorithms using for example SSE2, AVX2 and other vectorized CPU intrinsics that supposedly speed up in- and deflating data. And indeed, compiling Git against zlib-ng leads to a significant speedup when reading objects. The following benchmark uses git-cat-file(1) with `--batch --batch-all-objects` in the Git repository: Benchmark 1: zlib Time (mean ± σ): 52.085 s ± 0.141 s [User: 51.500 s, System: 0.456 s] Range (min … max): 52.004 s … 52.335 s 5 runs Benchmark 2: zlib-ng Time (mean ± σ): 40.324 s ± 0.134 s [User: 39.731 s, System: 0.490 s] Range (min … max): 40.135 s … 40.484 s 5 runs Summary zlib-ng ran 1.29 ± 0.01 times faster than zlib So we're looking at a ~25% speedup compared to zlib. This is of course an extreme example, as it makes us read through all objects in the repository. But regardless, it should be possible to see some sort of speedup in most commands that end up accessing the object database. The zlib-ng library provides a compatibility layer that makes it a proper drop-in replacement for zlib: nothing needs to change in the build system to support it. Unfortunately though, this mode isn't easy to use on most systems because distributions do not allow you to install zlib-ng in that way, as that would mean that the zlib library would be globally replaced. Instead, many distributions provide a package that installs zlib-ng without the compatibility layer. This version does provide effectively the same APIs like zlib does, but all of the symbols are prefixed with `zng_` to avoid symbol collisions. Implement a new build option that allows us to link against zlib-ng directly. If set, we redefine zlib symbols so that we use the `zng_` prefixed versions thereof provided by that library. Like this, it becomes possible to install both zlib and zlib-ng (without the compat layer) and then pick whichever library one wants to link against for Git. Signed-off-by: Patrick Steinhardt <ps@pks.im> Signed-off-by: Junio C Hamano <gitster@pobox.com>

Patrick Steinhardt committed Jan 28, 2025 at 09:41 UTC b9d6f64393275b505937a8621a6cc4875adde8e0

4 files changed +64 -15

Makefile

+15 -5

@@ -183,7 +183,8 @@ include shared.mak

183 # byte-order mark (BOM) when writing UTF-16 or UTF-32 and always writes in

184 # big-endian format.

185 #

186 -# Define NO_DEFLATE_BOUND if your zlib does not have deflateBound.

186 +# Define NO_DEFLATE_BOUND if your zlib does not have deflateBound. Define

187 +# ZLIB_NG if you want to use zlib-ng instead of zlib.

188 #

189 # Define NO_NORETURN if using buggy versions of gcc 4.6+ and profile feedback,

190 # as the compiler can crash (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=49299)

@@ -1687,11 +1688,20 @@ else

1688 endif

1689 IMAP_SEND_LDFLAGS += $(OPENSSL_LINK) $(OPENSSL_LIBSSL) $(LIB_4_CRYPTO)

1690

1690 -ifdef ZLIB_PATH

1691 - BASIC_CFLAGS += -I$(ZLIB_PATH)/include

1692 - EXTLIBS += $(call libpath_template,$(ZLIB_PATH)/$(lib))

1691 +ifdef ZLIB_NG

1692 + BASIC_CFLAGS += -DHAVE_ZLIB_NG

1693 + ifdef ZLIB_NG_PATH

1694 + BASIC_CFLAGS += -I$(ZLIB_NG_PATH)/include

1695 + EXTLIBS += $(call libpath_template,$(ZLIB_NG_PATH)/$(lib))

1696 + endif

1697 + EXTLIBS += -lz-ng

1698 +else

1699 + ifdef ZLIB_PATH

1700 + BASIC_CFLAGS += -I$(ZLIB_PATH)/include

1701 + EXTLIBS += $(call libpath_template,$(ZLIB_PATH)/$(lib))

1702 + endif

1703 + EXTLIBS += -lz

1704 endif

1694 -EXTLIBS += -lz

1705

1706 ifndef NO_OPENSSL

1707 OPENSSL_LIBSSL = -lssl

compat/zlib-compat.h

+30 -6

@@ -1,11 +1,34 @@

1 #ifndef COMPAT_ZLIB_H

2 #define COMPAT_ZLIB_H

4 -#include <zlib.h>

4 +#ifdef HAVE_ZLIB_NG

5 +# include <zlib-ng.h>

6 -#if defined(NO_DEFLATE_BOUND) || ZLIB_VERNUM < 0x1200

7 -# define deflateBound(c,s) ((s) + (((s) + 7) >> 3) + (((s) + 63) >> 6) + 11)

8 -#endif

7 +# define z_stream zng_stream

8 +#define gz_header_s zng_gz_header_s

9 +

10 +# define crc32(crc, buf, len) zng_crc32(crc, buf, len)

11 +

12 +# define inflate(strm, bits) zng_inflate(strm, bits)

13 +# define inflateEnd(strm) zng_inflateEnd(strm)

14 +# define inflateInit(strm) zng_inflateInit(strm)

15 +# define inflateInit2(strm, bits) zng_inflateInit2(strm, bits)

16 +# define inflateReset(strm) zng_inflateReset(strm)

17 +

18 +# define deflate(strm, flush) zng_deflate(strm, flush)

19 +# define deflateBound(strm, source_len) zng_deflateBound(strm, source_len)

20 +# define deflateEnd(strm) zng_deflateEnd(strm)

21 +# define deflateInit(strm, level) zng_deflateInit(strm, level)

22 +# define deflateInit2(stream, level, method, window_bits, mem_level, strategy) zng_deflateInit2(stream, level, method, window_bits, mem_level, strategy)

23 +# define deflateReset(strm) zng_deflateReset(strm)

24 +# define deflateSetHeader(strm, head) zng_deflateSetHeader(strm, head)

25 +

26 +#else

27 +# include <zlib.h>

28 +

29 +# if defined(NO_DEFLATE_BOUND) || ZLIB_VERNUM < 0x1200

30 +# define deflateBound(c,s) ((s) + (((s) + 7) >> 3) + (((s) + 63) >> 6) + 11)

31 +# endif

33 /*

34 * zlib only gained support for setting up the gzip header in v1.2.2.1. In

@@ -13,7 +36,7 @@

36 * operating systems, so it's fine to simply make this a no-op when using a

37 * zlib version that doesn't support this yet.

38 */

16 -#if ZLIB_VERNUM < 0x1221

39 +# if ZLIB_VERNUM < 0x1221

40 struct gz_header_s {

41 int os;

42 };

@@ -24,6 +47,7 @@ static int deflateSetHeader(z_streamp strm, struct gz_header_s *head)

47 (void)(head);

48 return Z_OK;

49 }

27 -#endif

50 +# endif

51 +#endif /* HAVE_ZLIB_NG */

53 #endif /* COMPAT_ZLIB_H */

meson.build

+17 -4

@@ -792,11 +792,23 @@ else

792 build_options_config.set('NO_PERL_CPAN_FALLBACKS', '')

793 endif

794

795 -zlib = dependency('zlib', default_options: ['default_library=static', 'tests=disabled'])

796 -if zlib.version().version_compare('<1.2.0')

797 - libgit_c_args += '-DNO_DEFLATE_BOUND'

795 +zlib_backend = get_option('zlib_backend')

796 +if zlib_backend in ['auto', 'zlib-ng']

797 + zlib_ng = dependency('zlib-ng', required: zlib_backend == 'zlib-ng')

798 + if zlib_ng.found()

799 + zlib_backend = 'zlib-ng'

800 + libgit_c_args += '-DHAVE_ZLIB_NG'

801 + libgit_dependencies += zlib_ng

802 + endif

803 +endif

804 +if zlib_backend in ['auto', 'zlib']

805 + zlib = dependency('zlib', default_options: ['default_library=static', 'tests=disabled'])

806 + if zlib.version().version_compare('<1.2.0')

807 + libgit_c_args += '-DNO_DEFLATE_BOUND'

808 + endif

809 + zlib_backend = 'zlib'

810 + libgit_dependencies += zlib

811 endif

799 -libgit_dependencies += zlib

812

813 threads = dependency('threads', required: false)

814 if threads.found()

@@ -2002,4 +2014,5 @@ summary({

2014 'sha1': sha1_backend,

2015 'sha1_unsafe': sha1_unsafe_backend,

2016 'sha256': sha256_backend,

2017 + 'zlib': zlib_backend,

2018 }, section: 'Backends')

meson_options.txt

@@ -57,6 +57,8 @@ option('sha1_unsafe_backend', type: 'combo', choices: ['openssl', 'block', 'Comm

57 description: 'The backend used for hashing data with the SHA1 object format in case no cryptographic security is needed.')

58 option('sha256_backend', type: 'combo', choices: ['openssl', 'nettle', 'gcrypt', 'block'], value: 'block',

59 description: 'The backend used for hashing objects with the SHA256 object format.')

60 +option('zlib_backend', type: 'combo', choices: ['auto', 'zlib', 'zlib-ng'], value: 'auto',

61 + description: 'The backend used for compressing objects and other data.')

63 # Build tweaks.

64 option('macos_use_homebrew_gettext', type: 'boolean', value: true,