test-tool: add pack-deltas helper
When trying to demonstrate certain behavior in tests, it can be helpful to create packfiles that have specific delta structures. 'git pack-objects' uses various algorithms to select deltas based on their compression rates, but that does not always demonstrate all possible packfile shapes. This becomes especially important when wanting to test 'git index-pack' and its ability to parse certain pack shapes. We have prior art in t/lib-pack.sh, where certain delta structures are produced by manually writing certain opaque pack contents. However, producing these script updates is cumbersome and difficult to do as a contributor. Instead, create a new test-tool, 'test-tool pack-deltas', that reads a list of instructions for which objects to include in a packfile and how those objects should be written in delta form. At the moment, this only supports REF_DELTAs as those are the kinds of deltas needed to exercise a bug in 'git index-pack'. Signed-off-by: Derrick Stolee <stolee@gmail.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>
Derrick Stolee committed
Apr 28, 2025 at 20:24 UTC
89d557b950c7a0581c12452e8f9576c45546246b
5 files changed
+152
Makefile
+1
@@ -821,6 +821,7 @@ TEST_BUILTINS_OBJS += test-mergesort.o
821
TEST_BUILTINS_OBJS += test-mktemp.o
822
TEST_BUILTINS_OBJS += test-name-hash.o
823
TEST_BUILTINS_OBJS += test-online-cpus.o
824
+TEST_BUILTINS_OBJS += test-pack-deltas.o
825
TEST_BUILTINS_OBJS += test-pack-mtimes.o
826
TEST_BUILTINS_OBJS += test-parse-options.o
827
TEST_BUILTINS_OBJS += test-parse-pathspec-file.o
t/helper/meson.build
+1
@@ -36,6 +36,7 @@ test_tool_sources = [
36
'test-mktemp.c',
37
'test-name-hash.c',
38
'test-online-cpus.c',
39
+ 'test-pack-deltas.c',
40
'test-pack-mtimes.c',
41
'test-parse-options.c',
42
'test-parse-pathspec-file.c',
t/helper/test-pack-deltas.c
new
+148
@@ -0,0 +1,148 @@
1
+#define USE_THE_REPOSITORY_VARIABLE
2
+
3
+#include "test-tool.h"
4
+#include "git-compat-util.h"
5
+#include "delta.h"
6
+#include "git-zlib.h"
7
+#include "hash.h"
8
+#include "hex.h"
9
+#include "pack.h"
10
+#include "pack-objects.h"
11
+#include "parse-options.h"
12
+#include "setup.h"
13
+#include "strbuf.h"
14
+#include "string-list.h"
15
+
16
+static const char *usage_str[] = {
17
+ "test-tool pack-deltas --num-objects <num-objects>",
18
+ NULL
19
+};
20
+
21
+static unsigned long do_compress(void **pptr, unsigned long size)
22
+{
23
+ git_zstream stream;
24
+ void *in, *out;
25
+ unsigned long maxsize;
26
+
27
+ git_deflate_init(&stream, 1);
28
+ maxsize = git_deflate_bound(&stream, size);
29
+
30
+ in = *pptr;
31
+ out = xmalloc(maxsize);
32
+ *pptr = out;
33
+
34
+ stream.next_in = in;
35
+ stream.avail_in = size;
36
+ stream.next_out = out;
37
+ stream.avail_out = maxsize;
38
+ while (git_deflate(&stream, Z_FINISH) == Z_OK)
39
+ ; /* nothing */
40
+ git_deflate_end(&stream);
41
+
42
+ free(in);
43
+ return stream.total_out;
44
+}
45
+
46
+static void write_ref_delta(struct hashfile *f,
47
+ struct object_id *oid,
48
+ struct object_id *base)
49
+{
50
+ unsigned char header[MAX_PACK_OBJECT_HEADER];
51
+ unsigned long size, base_size, delta_size, compressed_size, hdrlen;
52
+ enum object_type type;
53
+ void *base_buf, *delta_buf;
54
+ void *buf = repo_read_object_file(the_repository,
55
+ oid, &type,
56
+ &size);
57
+
58
+ if (!buf)
59
+ die("unable to read %s", oid_to_hex(oid));
60
+
61
+ base_buf = repo_read_object_file(the_repository,
62
+ base, &type,
63
+ &base_size);
64
+
65
+ if (!base_buf)
66
+ die("unable to read %s", oid_to_hex(base));
67
+
68
+ delta_buf = diff_delta(base_buf, base_size,
69
+ buf, size, &delta_size, 0);
70
+
71
+ compressed_size = do_compress(&delta_buf, delta_size);
72
+
73
+ hdrlen = encode_in_pack_object_header(header, sizeof(header),
74
+ OBJ_REF_DELTA, delta_size);
75
+ hashwrite(f, header, hdrlen);
76
+ hashwrite(f, base->hash, the_repository->hash_algo->rawsz);
77
+ hashwrite(f, delta_buf, compressed_size);
78
+
79
+ free(buf);
80
+ free(base_buf);
81
+ free(delta_buf);
82
+}
83
+
84
+int cmd__pack_deltas(int argc, const char **argv)
85
+{
86
+ int num_objects = -1;
87
+ struct hashfile *f;
88
+ struct strbuf line = STRBUF_INIT;
89
+ struct option options[] = {
90
+ OPT_INTEGER('n', "num-objects", &num_objects, N_("the number of objects to write")),
91
+ OPT_END()
92
+ };
93
+
94
+ argc = parse_options(argc, argv, NULL,
95
+ options, usage_str, 0);
96
+
97
+ if (argc || num_objects < 0)
98
+ usage_with_options(usage_str, options);
99
+
100
+ setup_git_directory();
101
+
102
+ f = hashfd(the_repository->hash_algo, 1, "<stdout>");
103
+ write_pack_header(f, num_objects);
104
+
105
+ /* Read each line from stdin into 'line' */
106
+ while (strbuf_getline_lf(&line, stdin) != EOF) {
107
+ const char *type_str, *content_oid_str, *base_oid_str = NULL;
108
+ struct object_id content_oid, base_oid;
109
+ struct string_list items = STRING_LIST_INIT_NODUP;
110
+ /*
111
+ * Tokenize into two or three parts:
112
+ * 1. REF_DELTA, OFS_DELTA, or FULL.
113
+ * 2. The object ID for the content object.
114
+ * 3. The object ID for the base object (optional).
115
+ */
116
+ if (string_list_split_in_place(&items, line.buf, " ", 3) < 0)
117
+ die("invalid input format: %s", line.buf);
118
+
119
+ if (items.nr < 2)
120
+ die("invalid input format: %s", line.buf);
121
+
122
+ type_str = items.items[0].string;
123
+ content_oid_str = items.items[1].string;
124
+
125
+ if (get_oid_hex(content_oid_str, &content_oid))
126
+ die("invalid object: %s", content_oid_str);
127
+ if (items.nr >= 3) {
128
+ base_oid_str = items.items[2].string;
129
+ if (get_oid_hex(base_oid_str, &base_oid))
130
+ die("invalid object: %s", base_oid_str);
131
+ }
132
+ string_list_clear(&items, 0);
133
+
134
+ if (!strcmp(type_str, "REF_DELTA"))
135
+ write_ref_delta(f, &content_oid, &base_oid);
136
+ else if (!strcmp(type_str, "OFS_DELTA"))
137
+ die("OFS_DELTA not implemented");
138
+ else if (!strcmp(type_str, "FULL"))
139
+ die("FULL not implemented");
140
+ else
141
+ die("unknown pack type: %s", type_str);
142
+ }
143
+
144
+ finalize_hashfile(f, NULL, FSYNC_COMPONENT_PACK,
145
+ CSUM_HASH_IN_STREAM | CSUM_FSYNC | CSUM_CLOSE);
146
+ strbuf_release(&line);
147
+ return 0;
148
+}
t/helper/test-tool.c
+1
@@ -46,6 +46,7 @@ static struct test_cmd cmds[] = {
46
{ "mktemp", cmd__mktemp },
47
{ "name-hash", cmd__name_hash },
48
{ "online-cpus", cmd__online_cpus },
49
+ { "pack-deltas", cmd__pack_deltas },
50
{ "pack-mtimes", cmd__pack_mtimes },
51
{ "parse-options", cmd__parse_options },
52
{ "parse-options-flags", cmd__parse_options_flags },
t/helper/test-tool.h
+1
@@ -39,6 +39,7 @@ int cmd__mergesort(int argc, const char **argv);
39
int cmd__mktemp(int argc, const char **argv);
40
int cmd__name_hash(int argc, const char **argv);
41
int cmd__online_cpus(int argc, const char **argv);
42
+int cmd__pack_deltas(int argc, const char **argv);
43
int cmd__pack_mtimes(int argc, const char **argv);
44
int cmd__parse_options(int argc, const char **argv);
45
int cmd__parse_options_flags(int argc, const char **argv);