@matheusbsbs / matheus-bagliano / commits / 960ccca680
Fork 0

Custom compression levels for objects and packs

Add config variables pack.compression and core.loosecompression , and switch --compression=level to pack-objects. Loose objects will be compressed using core.loosecompression if set, else core.compression if set, else Z_BEST_SPEED. Packed objects will be compressed using --compression=level if seen, else pack.compression if set, else core.compression if set, else Z_DEFAULT_COMPRESSION. This is the "pack compression level". Loose objects added to a pack undeltified will be recompressed to the pack compression level if it is unequal to the current loose compression level by the preceding rules, or if the loose object was written while core.legacyheaders = true. Newly deltified loose objects are always compressed to the current pack compression level. Previously packed objects added to a pack are recompressed to the current pack compression level exactly when their deltification status changes, since the previous pack data cannot be reused. In either case, the --no-reuse-object switch from the first patch below will always force recompression to the current pack compression level, instead of assuming the pack compression level hasn't changed and pack data can be reused when possible. This applies on top of the following patches from Nicolas Pitre: [PATCH] allow for undeltified objects not to be reused [PATCH] make "repack -f" imply "pack-objects --no-reuse-object" Signed-off-by: Dana L. How <danahow@gmail.com> Signed-off-by: Junio C Hamano <junkio@cox.net>

Dana How committed May 9, 2007 at 13:56 UTC 960ccca6803c9fb57429d43572a9545a96107e32
8 files changed +84 -10
Documentation/config.txt
+15 -2
@@ -204,10 +204,16 @@ core.warnAmbiguousRefs::
204 and might match multiple refs in the .git/refs/ tree. True by default.
205
206 core.compression::
207 + An integer -1..9, indicating a default compression level.
208 + -1 is the zlib default. 0 means no compression,
209 + and 1..9 are various speed/size tradeoffs, 9 being slowest.
210 +
211 +core.loosecompression::
212 An integer -1..9, indicating the compression level for objects that
208 - are not in a pack file. -1 is the zlib and git default. 0 means no
213 + are not in a pack file. -1 is the zlib default. 0 means no
214 compression, and 1..9 are various speed/size tradeoffs, 9 being
210 - slowest.
215 + slowest. If not set, defaults to core.compression. If that is
216 + not set, defaults to 0 (best speed).
217
218 core.legacyheaders::
219 A boolean which
@@ -550,6 +556,13 @@ pack.depth::
556 The maximum delta depth used by gitlink:git-pack-objects[1] when no
557 maximum depth is given on the command line. Defaults to 50.
558
559 +pack.compression::
560 + An integer -1..9, indicating the compression level for objects
561 + in a pack file. -1 is the zlib default. 0 means no
562 + compression, and 1..9 are various speed/size tradeoffs, 9 being
563 + slowest. If not set, defaults to core.compression. If that is
564 + not set, defaults to -1.
565 +
566 pull.octopus::
567 The default merge strategy to use when pulling multiple branches
568 at once.
Documentation/git-pack-objects.txt
+13 -1
@@ -130,10 +130,22 @@ base-name::
130 --no-reuse-object::
131 This flag tells the command not to reuse existing object data at all,
132 including non deltified object, forcing recompression of everything.
133 - This implies --no-reuse-delta. Useful only in the obscur case where
133 + This implies --no-reuse-delta. Useful only in the obscure case where
134 wholesale enforcement of a different compression level on the
135 packed data is desired.
136
137 +--compression=[N]::
138 + Specifies compression level for newly-compressed data in the
139 + generated pack. If not specified, pack compression level is
140 + determined first by pack.compression, then by core.compression,
141 + and defaults to -1, the zlib default, if neither is set.
142 + Data copied from loose objects will be recompressed
143 + if core.legacyheaders was true when they were created or if
144 + the loose compression level (see core.loosecompression and
145 + core.compression) is now a different value than the pack
146 + compression level. Add --no-reuse-object if you want to force
147 + a uniform compression level on all data no matter the source.
148 +
149 --delta-base-offset::
150 A packed archive can express base object of a delta as
151 either 20-byte object name or as an offset in the
builtin-pack-objects.c
+31 -2
@@ -68,6 +68,8 @@ static int depth = 50;
68 static int pack_to_stdout;
69 static int num_preferred_base;
70 static struct progress progress_state;
71 +static int pack_compression_level = Z_DEFAULT_COMPRESSION;
72 +static int pack_compression_seen;
73
74 /*
75 * The object names in objects array are hashed with this hashtable,
@@ -427,7 +429,10 @@ static unsigned long write_object(struct sha1file *f,
429 * and we do not need to deltify it.
430 */
431
430 - if (!no_reuse_object && !entry->in_pack && !entry->delta) {
432 + /* differing core & pack compression when loose object -> must recompress */
433 + if (!entry->in_pack && pack_compression_level != zlib_compression_level)
434 + to_reuse = 0;
435 + else if (!no_reuse_object && !entry->in_pack && !entry->delta) {
436 unsigned char *map;
437 unsigned long mapsize;
438 map = map_sha1_file(entry->sha1, &mapsize);
@@ -487,7 +492,7 @@ static unsigned long write_object(struct sha1file *f,
492 sha1write(f, entry->delta->sha1, 20);
493 hdrlen += 20;
494 }
490 - datalen = sha1write_compressed(f, buf, size);
495 + datalen = sha1write_compressed(f, buf, size, pack_compression_level);
496 free(buf);
497 }
498 else {
@@ -1496,6 +1501,16 @@ static int git_pack_config(const char *k, const char *v)
1501 depth = git_config_int(k, v);
1502 return 0;
1503 }
1504 + if (!strcmp(k, "pack.compression")) {
1505 + int level = git_config_int(k, v);
1506 + if (level == -1)
1507 + level = Z_DEFAULT_COMPRESSION;
1508 + else if (level < 0 || level > Z_BEST_COMPRESSION)
1509 + die("bad pack compression level %d", level);
1510 + pack_compression_level = level;
1511 + pack_compression_seen = 1;
1512 + return 0;
1513 + }
1514 return git_default_config(k, v);
1515 }
1516
@@ -1607,6 +1622,8 @@ int cmd_pack_objects(int argc, const char **argv, const char *prefix)
1622 rp_ac = 2;
1623
1624 git_config(git_pack_config);
1625 + if (!pack_compression_seen && core_compression_seen)
1626 + pack_compression_level = core_compression_level;
1627
1628 progress = isatty(2);
1629 for (i = 1; i < argc; i++) {
@@ -1627,6 +1644,18 @@ int cmd_pack_objects(int argc, const char **argv, const char *prefix)
1644 incremental = 1;
1645 continue;
1646 }
1647 + if (!prefixcmp(arg, "--compression=")) {
1648 + char *end;
1649 + int level = strtoul(arg+14, &end, 0);
1650 + if (!arg[14] || *end)
1651 + usage(pack_usage);
1652 + if (level == -1)
1653 + level = Z_DEFAULT_COMPRESSION;
1654 + else if (level < 0 || level > Z_BEST_COMPRESSION)
1655 + die("bad pack compression level %d", level);
1656 + pack_compression_level = level;
1657 + continue;
1658 + }
1659 if (!prefixcmp(arg, "--window=")) {
1660 char *end;
1661 window = strtoul(arg+9, &end, 0);
cache.h
+2
@@ -283,6 +283,8 @@ extern int warn_ambiguous_refs;
283 extern int shared_repository;
284 extern const char *apply_default_whitespace;
285 extern int zlib_compression_level;
286 +extern int core_compression_level;
287 +extern int core_compression_seen;
288 extern size_t packed_git_window_size;
289 extern size_t packed_git_limit;
290 extern size_t delta_base_cache_limit;
config.c
+17 -1
@@ -12,6 +12,8 @@
12 static FILE *config_file;
13 static const char *config_file_name;
14 static int config_linenr;
15 +static int zlib_compression_seen;
16 +
17 static int get_next_char(void)
18 {
19 int c;
@@ -304,13 +306,27 @@ int git_default_config(const char *var, const char *value)
306 return 0;
307 }
308
307 - if (!strcmp(var, "core.compression")) {
309 + if (!strcmp(var, "core.loosecompression")) {
310 int level = git_config_int(var, value);
311 if (level == -1)
312 level = Z_DEFAULT_COMPRESSION;
313 else if (level < 0 || level > Z_BEST_COMPRESSION)
314 die("bad zlib compression level %d", level);
315 zlib_compression_level = level;
316 + zlib_compression_seen = 1;
317 + return 0;
318 + }
319 +
320 + if (!strcmp(var, "core.compression")) {
321 + int level = git_config_int(var, value);
322 + if (level == -1)
323 + level = Z_DEFAULT_COMPRESSION;
324 + else if (level < 0 || level > Z_BEST_COMPRESSION)
325 + die("bad zlib compression level %d", level);
326 + core_compression_level = level;
327 + core_compression_seen = 1;
328 + if (!zlib_compression_seen)
329 + zlib_compression_level = level;
330 return 0;
331 }
332
csum-file.c
+2 -2
@@ -119,14 +119,14 @@ struct sha1file *sha1fd(int fd, const char *name)
119 return f;
120 }
121
122 -int sha1write_compressed(struct sha1file *f, void *in, unsigned int size)
122 +int sha1write_compressed(struct sha1file *f, void *in, unsigned int size, int level)
123 {
124 z_stream stream;
125 unsigned long maxsize;
126 void *out;
127
128 memset(&stream, 0, sizeof(stream));
129 - deflateInit(&stream, zlib_compression_level);
129 + deflateInit(&stream, level);
130 maxsize = deflateBound(&stream, size);
131 out = xmalloc(maxsize);
132
csum-file.h
+1 -1
@@ -16,7 +16,7 @@ extern struct sha1file *sha1fd(int fd, const char *name);
16 extern struct sha1file *sha1create(const char *fmt, ...) __attribute__((format (printf, 1, 2)));
17 extern int sha1close(struct sha1file *, unsigned char *, int);
18 extern int sha1write(struct sha1file *, void *, unsigned int);
19 -extern int sha1write_compressed(struct sha1file *, void *, unsigned int);
19 +extern int sha1write_compressed(struct sha1file *, void *, unsigned int, int);
20 extern void crc32_begin(struct sha1file *);
21 extern uint32_t crc32_end(struct sha1file *);
22
environment.c
+3 -1
@@ -24,7 +24,9 @@ const char *git_commit_encoding;
24 const char *git_log_output_encoding;
25 int shared_repository = PERM_UMASK;
26 const char *apply_default_whitespace;
27 -int zlib_compression_level = Z_DEFAULT_COMPRESSION;
27 +int zlib_compression_level = Z_BEST_SPEED;
28 +int core_compression_level;
29 +int core_compression_seen;
30 size_t packed_git_window_size = DEFAULT_PACKED_GIT_WINDOW_SIZE;
31 size_t packed_git_limit = DEFAULT_PACKED_GIT_LIMIT;
32 size_t delta_base_cache_limit = 16 * 1024 * 1024;