loose: add a mapping between SHA-1 and SHA-256 for loose objects
As part of the transition plan, we'd like to add a file in the .git directory that maps loose objects between SHA-1 and SHA-256. Let's implement the specification in the transition plan and store this data on a per-repository basis in struct repository. Signed-off-by: brian m. carlson <sandals@crustytoothpaste.net> Signed-off-by: Eric W. Biederman <ebiederm@xmission.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>
brian m. carlson committed
Oct 1, 2023 at 21:40 UTC
23b2c7e95b6f8f3045665835d2dc5028701eff18
7 files changed
+293
-1
Makefile
+1
@@ -1053,6 +1053,7 @@ LIB_OBJS += list-objects-filter.o
1053
LIB_OBJS += list-objects.o
1054
LIB_OBJS += lockfile.o
1055
LIB_OBJS += log-tree.o
1056
+LIB_OBJS += loose.o
1057
LIB_OBJS += ls-refs.o
1058
LIB_OBJS += mailinfo.o
1059
LIB_OBJS += mailmap.o
loose.c
new
+246
@@ -0,0 +1,246 @@
1
+#include "git-compat-util.h"
2
+#include "hash.h"
3
+#include "path.h"
4
+#include "object-store.h"
5
+#include "hex.h"
6
+#include "wrapper.h"
7
+#include "gettext.h"
8
+#include "loose.h"
9
+#include "lockfile.h"
10
+
11
+static const char *loose_object_header = "# loose-object-idx\n";
12
+
13
+static inline int should_use_loose_object_map(struct repository *repo)
14
+{
15
+ return repo->compat_hash_algo && repo->gitdir;
16
+}
17
+
18
+void loose_object_map_init(struct loose_object_map **map)
19
+{
20
+ struct loose_object_map *m;
21
+ m = xmalloc(sizeof(**map));
22
+ m->to_compat = kh_init_oid_map();
23
+ m->to_storage = kh_init_oid_map();
24
+ *map = m;
25
+}
26
+
27
+static int insert_oid_pair(kh_oid_map_t *map, const struct object_id *key, const struct object_id *value)
28
+{
29
+ khiter_t pos;
30
+ int ret;
31
+ struct object_id *stored;
32
+
33
+ pos = kh_put_oid_map(map, *key, &ret);
34
+
35
+ /* This item already exists in the map. */
36
+ if (ret == 0)
37
+ return 0;
38
+
39
+ stored = xmalloc(sizeof(*stored));
40
+ oidcpy(stored, value);
41
+ kh_value(map, pos) = stored;
42
+ return 1;
43
+}
44
+
45
+static int load_one_loose_object_map(struct repository *repo, struct object_directory *dir)
46
+{
47
+ struct strbuf buf = STRBUF_INIT, path = STRBUF_INIT;
48
+ FILE *fp;
49
+
50
+ if (!dir->loose_map)
51
+ loose_object_map_init(&dir->loose_map);
52
+
53
+ insert_oid_pair(dir->loose_map->to_compat, repo->hash_algo->empty_tree, repo->compat_hash_algo->empty_tree);
54
+ insert_oid_pair(dir->loose_map->to_storage, repo->compat_hash_algo->empty_tree, repo->hash_algo->empty_tree);
55
+
56
+ insert_oid_pair(dir->loose_map->to_compat, repo->hash_algo->empty_blob, repo->compat_hash_algo->empty_blob);
57
+ insert_oid_pair(dir->loose_map->to_storage, repo->compat_hash_algo->empty_blob, repo->hash_algo->empty_blob);
58
+
59
+ insert_oid_pair(dir->loose_map->to_compat, repo->hash_algo->null_oid, repo->compat_hash_algo->null_oid);
60
+ insert_oid_pair(dir->loose_map->to_storage, repo->compat_hash_algo->null_oid, repo->hash_algo->null_oid);
61
+
62
+ strbuf_git_common_path(&path, repo, "objects/loose-object-idx");
63
+ fp = fopen(path.buf, "rb");
64
+ if (!fp) {
65
+ strbuf_release(&path);
66
+ return 0;
67
+ }
68
+
69
+ errno = 0;
70
+ if (strbuf_getwholeline(&buf, fp, '\n') || strcmp(buf.buf, loose_object_header))
71
+ goto err;
72
+ while (!strbuf_getline_lf(&buf, fp)) {
73
+ const char *p;
74
+ struct object_id oid, compat_oid;
75
+ if (parse_oid_hex_algop(buf.buf, &oid, &p, repo->hash_algo) ||
76
+ *p++ != ' ' ||
77
+ parse_oid_hex_algop(p, &compat_oid, &p, repo->compat_hash_algo) ||
78
+ p != buf.buf + buf.len)
79
+ goto err;
80
+ insert_oid_pair(dir->loose_map->to_compat, &oid, &compat_oid);
81
+ insert_oid_pair(dir->loose_map->to_storage, &compat_oid, &oid);
82
+ }
83
+
84
+ strbuf_release(&buf);
85
+ strbuf_release(&path);
86
+ return errno ? -1 : 0;
87
+err:
88
+ strbuf_release(&buf);
89
+ strbuf_release(&path);
90
+ return -1;
91
+}
92
+
93
+int repo_read_loose_object_map(struct repository *repo)
94
+{
95
+ struct object_directory *dir;
96
+
97
+ if (!should_use_loose_object_map(repo))
98
+ return 0;
99
+
100
+ prepare_alt_odb(repo);
101
+
102
+ for (dir = repo->objects->odb; dir; dir = dir->next) {
103
+ if (load_one_loose_object_map(repo, dir) < 0) {
104
+ return -1;
105
+ }
106
+ }
107
+ return 0;
108
+}
109
+
110
+int repo_write_loose_object_map(struct repository *repo)
111
+{
112
+ kh_oid_map_t *map = repo->objects->odb->loose_map->to_compat;
113
+ struct lock_file lock;
114
+ int fd;
115
+ khiter_t iter;
116
+ struct strbuf buf = STRBUF_INIT, path = STRBUF_INIT;
117
+
118
+ if (!should_use_loose_object_map(repo))
119
+ return 0;
120
+
121
+ strbuf_git_common_path(&path, repo, "objects/loose-object-idx");
122
+ fd = hold_lock_file_for_update_timeout(&lock, path.buf, LOCK_DIE_ON_ERROR, -1);
123
+ iter = kh_begin(map);
124
+ if (write_in_full(fd, loose_object_header, strlen(loose_object_header)) < 0)
125
+ goto errout;
126
+
127
+ for (; iter != kh_end(map); iter++) {
128
+ if (kh_exist(map, iter)) {
129
+ if (oideq(&kh_key(map, iter), the_hash_algo->empty_tree) ||
130
+ oideq(&kh_key(map, iter), the_hash_algo->empty_blob))
131
+ continue;
132
+ strbuf_addf(&buf, "%s %s\n", oid_to_hex(&kh_key(map, iter)), oid_to_hex(kh_value(map, iter)));
133
+ if (write_in_full(fd, buf.buf, buf.len) < 0)
134
+ goto errout;
135
+ strbuf_reset(&buf);
136
+ }
137
+ }
138
+ strbuf_release(&buf);
139
+ if (commit_lock_file(&lock) < 0) {
140
+ error_errno(_("could not write loose object index %s"), path.buf);
141
+ strbuf_release(&path);
142
+ return -1;
143
+ }
144
+ strbuf_release(&path);
145
+ return 0;
146
+errout:
147
+ rollback_lock_file(&lock);
148
+ strbuf_release(&buf);
149
+ error_errno(_("failed to write loose object index %s\n"), path.buf);
150
+ strbuf_release(&path);
151
+ return -1;
152
+}
153
+
154
+static int write_one_object(struct repository *repo, const struct object_id *oid,
155
+ const struct object_id *compat_oid)
156
+{
157
+ struct lock_file lock;
158
+ int fd;
159
+ struct stat st;
160
+ struct strbuf buf = STRBUF_INIT, path = STRBUF_INIT;
161
+
162
+ strbuf_git_common_path(&path, repo, "objects/loose-object-idx");
163
+ hold_lock_file_for_update_timeout(&lock, path.buf, LOCK_DIE_ON_ERROR, -1);
164
+
165
+ fd = open(path.buf, O_WRONLY | O_CREAT | O_APPEND, 0666);
166
+ if (fd < 0)
167
+ goto errout;
168
+ if (fstat(fd, &st) < 0)
169
+ goto errout;
170
+ if (!st.st_size && write_in_full(fd, loose_object_header, strlen(loose_object_header)) < 0)
171
+ goto errout;
172
+
173
+ strbuf_addf(&buf, "%s %s\n", oid_to_hex(oid), oid_to_hex(compat_oid));
174
+ if (write_in_full(fd, buf.buf, buf.len) < 0)
175
+ goto errout;
176
+ if (close(fd))
177
+ goto errout;
178
+ adjust_shared_perm(path.buf);
179
+ rollback_lock_file(&lock);
180
+ strbuf_release(&buf);
181
+ strbuf_release(&path);
182
+ return 0;
183
+errout:
184
+ error_errno(_("failed to write loose object index %s\n"), path.buf);
185
+ close(fd);
186
+ rollback_lock_file(&lock);
187
+ strbuf_release(&buf);
188
+ strbuf_release(&path);
189
+ return -1;
190
+}
191
+
192
+int repo_add_loose_object_map(struct repository *repo, const struct object_id *oid,
193
+ const struct object_id *compat_oid)
194
+{
195
+ int inserted = 0;
196
+
197
+ if (!should_use_loose_object_map(repo))
198
+ return 0;
199
+
200
+ inserted |= insert_oid_pair(repo->objects->odb->loose_map->to_compat, oid, compat_oid);
201
+ inserted |= insert_oid_pair(repo->objects->odb->loose_map->to_storage, compat_oid, oid);
202
+ if (inserted)
203
+ return write_one_object(repo, oid, compat_oid);
204
+ return 0;
205
+}
206
+
207
+int repo_loose_object_map_oid(struct repository *repo,
208
+ const struct object_id *src,
209
+ const struct git_hash_algo *to,
210
+ struct object_id *dest)
211
+{
212
+ struct object_directory *dir;
213
+ kh_oid_map_t *map;
214
+ khiter_t pos;
215
+
216
+ for (dir = repo->objects->odb; dir; dir = dir->next) {
217
+ struct loose_object_map *loose_map = dir->loose_map;
218
+ if (!loose_map)
219
+ continue;
220
+ map = (to == repo->compat_hash_algo) ?
221
+ loose_map->to_compat :
222
+ loose_map->to_storage;
223
+ pos = kh_get_oid_map(map, *src);
224
+ if (pos < kh_end(map)) {
225
+ oidcpy(dest, kh_value(map, pos));
226
+ return 0;
227
+ }
228
+ }
229
+ return -1;
230
+}
231
+
232
+void loose_object_map_clear(struct loose_object_map **map)
233
+{
234
+ struct loose_object_map *m = *map;
235
+ struct object_id *oid;
236
+
237
+ if (!m)
238
+ return;
239
+
240
+ kh_foreach_value(m->to_compat, oid, free(oid));
241
+ kh_foreach_value(m->to_storage, oid, free(oid));
242
+ kh_destroy_oid_map(m->to_compat);
243
+ kh_destroy_oid_map(m->to_storage);
244
+ free(m);
245
+ *map = NULL;
246
+}
loose.h
new
+22
@@ -0,0 +1,22 @@
1
+#ifndef LOOSE_H
2
+#define LOOSE_H
3
+
4
+#include "khash.h"
5
+
6
+struct loose_object_map {
7
+ kh_oid_map_t *to_compat;
8
+ kh_oid_map_t *to_storage;
9
+};
10
+
11
+void loose_object_map_init(struct loose_object_map **map);
12
+void loose_object_map_clear(struct loose_object_map **map);
13
+int repo_loose_object_map_oid(struct repository *repo,
14
+ const struct object_id *src,
15
+ const struct git_hash_algo *dest_algo,
16
+ struct object_id *dest);
17
+int repo_add_loose_object_map(struct repository *repo, const struct object_id *oid,
18
+ const struct object_id *compat_oid);
19
+int repo_read_loose_object_map(struct repository *repo);
20
+int repo_write_loose_object_map(struct repository *repo);
21
+
22
+#endif
object-file-convert.c
+13
-1
@@ -4,6 +4,7 @@
4
#include "repository.h"
5
#include "hash-ll.h"
6
#include "object.h"
7
+#include "loose.h"
8
#include "object-file-convert.h"
9
10
int repo_oid_to_algop(struct repository *repo, const struct object_id *src,
@@ -21,7 +22,18 @@ int repo_oid_to_algop(struct repository *repo, const struct object_id *src,
22
oidcpy(dest, src);
23
return 0;
24
}
24
- return -1;
25
+ if (repo_loose_object_map_oid(repo, src, to, dest)) {
26
+ /*
27
+ * We may have loaded the object map at repo initialization but
28
+ * another process (perhaps upstream of a pipe from us) may have
29
+ * written a new object into the map. If the object is missing,
30
+ * let's reload the map to see if the object has appeared.
31
+ */
32
+ repo_read_loose_object_map(repo);
33
+ if (repo_loose_object_map_oid(repo, src, to, dest))
34
+ return -1;
35
+ }
36
+ return 0;
37
}
38
39
int convert_object_file(struct strbuf *outbuf,
object-store-ll.h
+3
@@ -26,6 +26,9 @@ struct object_directory {
26
uint32_t loose_objects_subdir_seen[8]; /* 256 bits */
27
struct oidtree *loose_objects_cache;
28
29
+ /* Map between object IDs for loose objects. */
30
+ struct loose_object_map *loose_map;
31
+
32
/*
33
* This is a temporary object store created by the tmp_objdir
34
* facility. Disable ref updates since the objects in the store
object.c
+2
@@ -13,6 +13,7 @@
13
#include "alloc.h"
14
#include "packfile.h"
15
#include "commit-graph.h"
16
+#include "loose.h"
17
18
unsigned int get_max_object_index(void)
19
{
@@ -540,6 +541,7 @@ void free_object_directory(struct object_directory *odb)
541
{
542
free(odb->path);
543
odb_clear_loose_cache(odb);
544
+ loose_object_map_clear(&odb->loose_map);
545
free(odb);
546
}
547
repository.c
+6
@@ -14,6 +14,7 @@
14
#include "read-cache-ll.h"
15
#include "remote.h"
16
#include "setup.h"
17
+#include "loose.h"
18
#include "submodule-config.h"
19
#include "sparse-index.h"
20
#include "trace2.h"
@@ -109,6 +110,8 @@ void repo_set_compat_hash_algo(struct repository *repo, int algo)
110
if (hash_algo_by_ptr(repo->hash_algo) == algo)
111
BUG("hash_algo and compat_hash_algo match");
112
repo->compat_hash_algo = algo ? &hash_algos[algo] : NULL;
113
+ if (repo->compat_hash_algo)
114
+ repo_read_loose_object_map(repo);
115
}
116
117
/*
@@ -201,6 +204,9 @@ int repo_init(struct repository *repo,
204
if (worktree)
205
repo_set_worktree(repo, worktree);
206
207
+ if (repo->compat_hash_algo)
208
+ repo_read_loose_object_map(repo);
209
+
210
clear_repository_format(&format);
211
return 0;
212