pack-mtimes: support reading .mtimes files
To store the individual mtimes of objects in a cruft pack, introduce a new `.mtimes` format that can optionally accompany a single pack in the repository. The format is defined in Documentation/technical/pack-format.txt, and stores a 4-byte network order timestamp for each object in name (index) order. This patch prepares for cruft packs by defining the `.mtimes` format, and introducing a basic API that callers can use to read out individual mtimes. Signed-off-by: Taylor Blau <me@ttaylorr.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>
Taylor Blau committed
May 20, 2022 at 19:17 UTC
94cd775a6c52a99caeb1278c3d8044ee109e2d3e
7 files changed
+202
-3
Documentation/technical/pack-format.txt
+19
@@ -294,6 +294,25 @@ Pack file entry: <+
294
295
All 4-byte numbers are in network order.
296
297
+== pack-*.mtimes files have the format:
298
+
299
+All 4-byte numbers are in network byte order.
300
+
301
+ - A 4-byte magic number '0x4d544d45' ('MTME').
302
+
303
+ - A 4-byte version identifier (= 1).
304
+
305
+ - A 4-byte hash function identifier (= 1 for SHA-1, 2 for SHA-256).
306
+
307
+ - A table of 4-byte unsigned integers. The ith value is the
308
+ modification time (mtime) of the ith object in the corresponding
309
+ pack by lexicographic (index) order. The mtimes count standard
310
+ epoch seconds.
311
+
312
+ - A trailer, containing a checksum of the corresponding packfile,
313
+ and a checksum of all of the above (each having length according
314
+ to the specified hash function).
315
+
316
== multi-pack-index (MIDX) files have the following format:
317
318
The multi-pack-index files refer to multiple pack-files and loose objects.
Makefile
+1
@@ -993,6 +993,7 @@ LIB_OBJS += oidtree.o
993
LIB_OBJS += pack-bitmap-write.o
994
LIB_OBJS += pack-bitmap.o
995
LIB_OBJS += pack-check.o
996
+LIB_OBJS += pack-mtimes.o
997
LIB_OBJS += pack-objects.o
998
LIB_OBJS += pack-revindex.o
999
LIB_OBJS += pack-write.o
builtin/repack.c
+1
@@ -217,6 +217,7 @@ static struct {
217
} exts[] = {
218
{".pack"},
219
{".rev", 1},
220
+ {".mtimes", 1},
221
{".bitmap", 1},
222
{".promisor", 1},
223
{".idx"},
object-store.h
+9
-1
@@ -115,12 +115,20 @@ struct packed_git {
115
freshened:1,
116
do_not_close:1,
117
pack_promisor:1,
118
- multi_pack_index:1;
118
+ multi_pack_index:1,
119
+ is_cruft:1;
120
unsigned char hash[GIT_MAX_RAWSZ];
121
struct revindex_entry *revindex;
122
const uint32_t *revindex_data;
123
const uint32_t *revindex_map;
124
size_t revindex_size;
125
+ /*
126
+ * mtimes_map points at the beginning of the memory mapped region of
127
+ * this pack's corresponding .mtimes file, and mtimes_size is the size
128
+ * of that .mtimes file
129
+ */
130
+ const uint32_t *mtimes_map;
131
+ size_t mtimes_size;
132
/* something like ".git/objects/pack/xxxxx.pack" */
133
char pack_name[FLEX_ARRAY]; /* more */
134
};
pack-mtimes.c
new
+129
@@ -0,0 +1,129 @@
1
+#include "git-compat-util.h"
2
+#include "pack-mtimes.h"
3
+#include "object-store.h"
4
+#include "packfile.h"
5
+
6
+static char *pack_mtimes_filename(struct packed_git *p)
7
+{
8
+ size_t len;
9
+ if (!strip_suffix(p->pack_name, ".pack", &len))
10
+ BUG("pack_name does not end in .pack");
11
+ return xstrfmt("%.*s.mtimes", (int)len, p->pack_name);
12
+}
13
+
14
+#define MTIMES_HEADER_SIZE (12)
15
+
16
+struct mtimes_header {
17
+ uint32_t signature;
18
+ uint32_t version;
19
+ uint32_t hash_id;
20
+};
21
+
22
+static int load_pack_mtimes_file(char *mtimes_file,
23
+ uint32_t num_objects,
24
+ const uint32_t **data_p, size_t *len_p)
25
+{
26
+ int fd, ret = 0;
27
+ struct stat st;
28
+ uint32_t *data = NULL;
29
+ size_t mtimes_size, expected_size;
30
+ struct mtimes_header header;
31
+
32
+ fd = git_open(mtimes_file);
33
+
34
+ if (fd < 0) {
35
+ ret = -1;
36
+ goto cleanup;
37
+ }
38
+ if (fstat(fd, &st)) {
39
+ ret = error_errno(_("failed to read %s"), mtimes_file);
40
+ goto cleanup;
41
+ }
42
+
43
+ mtimes_size = xsize_t(st.st_size);
44
+
45
+ if (mtimes_size < MTIMES_HEADER_SIZE) {
46
+ ret = error(_("mtimes file %s is too small"), mtimes_file);
47
+ goto cleanup;
48
+ }
49
+
50
+ data = xmmap(NULL, mtimes_size, PROT_READ, MAP_PRIVATE, fd, 0);
51
+
52
+ header.signature = ntohl(data[0]);
53
+ header.version = ntohl(data[1]);
54
+ header.hash_id = ntohl(data[2]);
55
+
56
+ if (header.signature != MTIMES_SIGNATURE) {
57
+ ret = error(_("mtimes file %s has unknown signature"), mtimes_file);
58
+ goto cleanup;
59
+ }
60
+
61
+ if (header.version != 1) {
62
+ ret = error(_("mtimes file %s has unsupported version %"PRIu32),
63
+ mtimes_file, header.version);
64
+ goto cleanup;
65
+ }
66
+
67
+ if (!(header.hash_id == 1 || header.hash_id == 2)) {
68
+ ret = error(_("mtimes file %s has unsupported hash id %"PRIu32),
69
+ mtimes_file, header.hash_id);
70
+ goto cleanup;
71
+ }
72
+
73
+
74
+ expected_size = MTIMES_HEADER_SIZE;
75
+ expected_size = st_add(expected_size, st_mult(sizeof(uint32_t), num_objects));
76
+ expected_size = st_add(expected_size, 2 * (header.hash_id == 1 ? GIT_SHA1_RAWSZ : GIT_SHA256_RAWSZ));
77
+
78
+ if (mtimes_size != expected_size) {
79
+ ret = error(_("mtimes file %s is corrupt"), mtimes_file);
80
+ goto cleanup;
81
+ }
82
+
83
+cleanup:
84
+ if (ret) {
85
+ if (data)
86
+ munmap(data, mtimes_size);
87
+ } else {
88
+ *len_p = mtimes_size;
89
+ *data_p = data;
90
+ }
91
+
92
+ close(fd);
93
+ return ret;
94
+}
95
+
96
+int load_pack_mtimes(struct packed_git *p)
97
+{
98
+ char *mtimes_name = NULL;
99
+ int ret = 0;
100
+
101
+ if (!p->is_cruft)
102
+ return ret; /* not a cruft pack */
103
+ if (p->mtimes_map)
104
+ return ret; /* already loaded */
105
+
106
+ ret = open_pack_index(p);
107
+ if (ret < 0)
108
+ goto cleanup;
109
+
110
+ mtimes_name = pack_mtimes_filename(p);
111
+ ret = load_pack_mtimes_file(mtimes_name,
112
+ p->num_objects,
113
+ &p->mtimes_map,
114
+ &p->mtimes_size);
115
+cleanup:
116
+ free(mtimes_name);
117
+ return ret;
118
+}
119
+
120
+uint32_t nth_packed_mtime(struct packed_git *p, uint32_t pos)
121
+{
122
+ if (!p->mtimes_map)
123
+ BUG("pack .mtimes file not loaded for %s", p->pack_name);
124
+ if (p->num_objects <= pos)
125
+ BUG("pack .mtimes out-of-bounds (%"PRIu32" vs %"PRIu32")",
126
+ pos, p->num_objects);
127
+
128
+ return get_be32(p->mtimes_map + pos + 3);
129
+}
pack-mtimes.h
new
+26
@@ -0,0 +1,26 @@
1
+#ifndef PACK_MTIMES_H
2
+#define PACK_MTIMES_H
3
+
4
+#include "git-compat-util.h"
5
+
6
+#define MTIMES_SIGNATURE 0x4d544d45 /* "MTME" */
7
+#define MTIMES_VERSION 1
8
+
9
+struct packed_git;
10
+
11
+/*
12
+ * Loads the .mtimes file corresponding to "p", if any, returning zero
13
+ * on success.
14
+ */
15
+int load_pack_mtimes(struct packed_git *p);
16
+
17
+/* Returns the mtime associated with the object at position "pos" (in
18
+ * lexicographic/index order) in pack "p".
19
+ *
20
+ * Note that it is a BUG() to call this function if either (a) "p" does
21
+ * not have a corresponding .mtimes file, or (b) it does, but it hasn't
22
+ * been loaded
23
+ */
24
+uint32_t nth_packed_mtime(struct packed_git *p, uint32_t pos);
25
+
26
+#endif
packfile.c
+17
-2
@@ -334,12 +334,22 @@ static void close_pack_revindex(struct packed_git *p)
334
p->revindex_data = NULL;
335
}
336
337
+static void close_pack_mtimes(struct packed_git *p)
338
+{
339
+ if (!p->mtimes_map)
340
+ return;
341
+
342
+ munmap((void *)p->mtimes_map, p->mtimes_size);
343
+ p->mtimes_map = NULL;
344
+}
345
+
346
void close_pack(struct packed_git *p)
347
{
348
close_pack_windows(p);
349
close_pack_fd(p);
350
close_pack_index(p);
351
close_pack_revindex(p);
352
+ close_pack_mtimes(p);
353
oidset_clear(&p->bad_objects);
354
}
355
@@ -363,7 +373,7 @@ void close_object_store(struct raw_object_store *o)
373
374
void unlink_pack_path(const char *pack_name, int force_delete)
375
{
366
- static const char *exts[] = {".pack", ".idx", ".rev", ".keep", ".bitmap", ".promisor"};
376
+ static const char *exts[] = {".pack", ".idx", ".rev", ".keep", ".bitmap", ".promisor", ".mtimes"};
377
int i;
378
struct strbuf buf = STRBUF_INIT;
379
size_t plen;
@@ -718,6 +728,10 @@ struct packed_git *add_packed_git(const char *path, size_t path_len, int local)
728
if (!access(p->pack_name, F_OK))
729
p->pack_promisor = 1;
730
731
+ xsnprintf(p->pack_name + path_len, alloc - path_len, ".mtimes");
732
+ if (!access(p->pack_name, F_OK))
733
+ p->is_cruft = 1;
734
+
735
xsnprintf(p->pack_name + path_len, alloc - path_len, ".pack");
736
if (stat(p->pack_name, &st) || !S_ISREG(st.st_mode)) {
737
free(p);
@@ -869,7 +883,8 @@ static void prepare_pack(const char *full_name, size_t full_name_len,
883
ends_with(file_name, ".pack") ||
884
ends_with(file_name, ".bitmap") ||
885
ends_with(file_name, ".keep") ||
872
- ends_with(file_name, ".promisor"))
886
+ ends_with(file_name, ".promisor") ||
887
+ ends_with(file_name, ".mtimes"))
888
string_list_append(data->garbage, full_name);
889
else
890
report_garbage(PACKDIR_FILE_GARBAGE, full_name);