@matheusbsbs / matheus-bagliano / commits / 00727249ec

range-diff: add configurable memory limit for cost matrix

When comparing large commit ranges (e.g., 250,000+ commits), range-diff attempts to allocate an n×n cost matrix that can exhaust available memory. For example, with 256,784 commits (n = 513,568), the matrix would require approximately 256GB of memory (513,568² × 4 bytes), causing either immediate segmentation faults due to integer overflow or system hangs. Add a memory limit check in get_correspondences() before allocating the cost matrix. This check uses the total size in bytes (n² × sizeof(int)) and compares it against a configurable maximum, preventing both excessive memory usage and integer overflow issues. The limit is configurable via a new --max-memory option that accepts human-readable sizes (e.g., "1G", "500M"). The default is 4GB for 64 bit systems and 2GB for 32 bit systems. This allows comparing ranges of approximately 32,000 (16,000) commits - generous for real-world use cases while preventing impractical operations. When the limit is exceeded, range-diff now displays a clear error message showing both the requested memory size and the maximum allowed, formatted in human-readable units for better user experience. Example usage: git range-diff --max-memory=1G branch1...branch2 git range-diff --max-memory=500M base..topic1 base..topic2 This approach was chosen over alternatives: - Pre-counting commits: Would require spawning additional git processes and reading all commits twice - Limiting by commit count: Less precise than actual memory usage - Streaming approach: Would require significant refactoring of the current algorithm This issue was previously discussed in: https://lore.kernel.org/git/RFC-cover-v2-0.5-00000000000-20211210T122901Z-avarab@gmail.com/ Acked-by: Johannes Schindelin <johannes.schindelin@gmx.de> Signed-off-by: Paulo Casaretto <pcasaretto@gmail.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>

Paulo Casaretto committed Aug 29, 2025 at 16:02 UTC 00727249ec8404c68391ec58e9c9f0d8a88d5ca0

5 files changed +44 -4

builtin/log.c

@@ -1404,6 +1404,7 @@ static void make_cover_letter(struct rev_info *rev, int use_separate_file,

1404 struct range_diff_options range_diff_opts = {

1405 .creation_factor = rev->creation_factor,

1406 .dual_color = 1,

1407 + .max_memory = RANGE_DIFF_MAX_MEMORY_DEFAULT,

1408 .diffopt = &opts,

1409 .other_arg = &other_arg

1410 };

builtin/range-diff.c

+21

@@ -6,6 +6,7 @@

6 #include "parse-options.h"

7 #include "range-diff.h"

8 #include "config.h"

9 +#include "parse.h"

12 static const char * const builtin_range_diff_usage[] = {

@@ -15,6 +16,21 @@ N_("git range-diff [<options>] <base> <old-tip> <new-tip>"),

16 NULL

17 };

19 +static int parse_max_memory(const struct option *opt, const char *arg, int unset)

20 +{

21 + size_t *max_memory = opt->value;

22 + uintmax_t val;

23 +

24 + if (unset)

25 + return 0;

26 +

27 + if (!git_parse_unsigned(arg, &val, SIZE_MAX))

28 + return error(_("invalid max-memory value: %s"), arg);

29 +

30 + *max_memory = (size_t)val;

31 + return 0;

32 +}

33 +

34 int cmd_range_diff(int argc,

35 const char **argv,

36 const char *prefix,

@@ -25,6 +41,7 @@ int cmd_range_diff(int argc,

41 struct strvec diff_merges_arg = STRVEC_INIT;

42 struct range_diff_options range_diff_opts = {

43 .creation_factor = RANGE_DIFF_CREATION_FACTOR_DEFAULT,

44 + .max_memory = RANGE_DIFF_MAX_MEMORY_DEFAULT,

45 .diffopt = &diffopt,

46 .other_arg = &other_arg

47 };

@@ -40,6 +57,10 @@ int cmd_range_diff(int argc,

57 PARSE_OPT_OPTARG),

58 OPT_PASSTHRU_ARGV(0, "diff-merges", &diff_merges_arg,

59 N_("style"), N_("passed to 'git log'"), 0),

60 + OPT_CALLBACK(0, "max-memory", &range_diff_opts.max_memory,

61 + N_("size"),

62 + N_("maximum memory for cost matrix (default 4G)"),

63 + parse_max_memory),

64 OPT_PASSTHRU_ARGV(0, "remerge-diff", &diff_merges_arg, NULL,

65 N_("passed to 'git log'"), PARSE_OPT_NOARG),

66 OPT_BOOL(0, "left-only", &left_only,

log-tree.c

@@ -717,6 +717,7 @@ static void show_diff_of_diff(struct rev_info *opt)

717 struct range_diff_options range_diff_opts = {

718 .creation_factor = opt->creation_factor,

719 .dual_color = 1,

720 + .max_memory = RANGE_DIFF_MAX_MEMORY_DEFAULT,

721 .diffopt = &opts

722 };

723

range-diff.c

+16 -4

@@ -325,13 +325,24 @@ static int diffsize(const char *a, const char *b)

325 }

326

327 static void get_correspondences(struct string_list *a, struct string_list *b,

328 - int creation_factor)

328 + int creation_factor, size_t max_memory)

329 {

330 int n = a->nr + b->nr;

331 int *cost, c, *a2b, *b2a;

332 int i, j;

333 -

334 - ALLOC_ARRAY(cost, st_mult(n, n));

333 + size_t cost_size = st_mult(n, n);

334 + size_t cost_bytes = st_mult(sizeof(int), cost_size);

335 + if (cost_bytes >= max_memory) {

336 + struct strbuf cost_str = STRBUF_INIT;

337 + struct strbuf max_str = STRBUF_INIT;

338 + strbuf_humanise_bytes(&cost_str, cost_bytes);

339 + strbuf_humanise_bytes(&max_str, max_memory);

340 + die(_("range-diff: unable to compute the range-diff, since it "

341 + "exceeds the maximum memory for the cost matrix: %s "

342 + "(%"PRIuMAX" bytes) needed, limited to %s (%"PRIuMAX" bytes)"),

343 + cost_str.buf, (uintmax_t)cost_bytes, max_str.buf, (uintmax_t)max_memory);

344 + }

345 + ALLOC_ARRAY(cost, cost_size);

346 ALLOC_ARRAY(a2b, n);

347 ALLOC_ARRAY(b2a, n);

348

@@ -591,7 +602,8 @@ int show_range_diff(const char *range1, const char *range2,

602 if (!res) {

603 find_exact_matches(&branch1, &branch2);

604 get_correspondences(&branch1, &branch2,

594 - range_diff_opts->creation_factor);

605 + range_diff_opts->creation_factor,

606 + range_diff_opts->max_memory);

607 output(&branch1, &branch2, range_diff_opts);

608 }

609

range-diff.h

@@ -5,6 +5,10 @@

5 #include "strvec.h"

7 #define RANGE_DIFF_CREATION_FACTOR_DEFAULT 60

8 +#define RANGE_DIFF_MAX_MEMORY_DEFAULT \

9 + (sizeof(void*) >= 8 ? \

10 + ((size_t)(1024L * 1024L) * (size_t)(4L * 1024L)) : /* 4GB on 64-bit */ \

11 + ((size_t)(1024L * 1024L) * (size_t)(2L * 1024L))) /* 2GB on 32-bit */

13 /*

14 * A much higher value than the default, when we KNOW we are comparing

@@ -17,6 +21,7 @@ struct range_diff_options {

21 unsigned dual_color:1;

22 unsigned left_only:1, right_only:1;

23 unsigned include_merges:1;

24 + size_t max_memory;

25 const struct diff_options *diffopt; /* may be NULL */

26 const struct strvec *other_arg; /* may be NULL */

27 };