tmp-objdir: introduce API for temporary object directories
Once objects are added to the object database by a process, they cannot easily be deleted, as we don't know what other processes may have started referencing them. We have to clean them up with git-gc, which will apply the usual reachability and grace-period checks. This patch provides an alternative: it helps callers create a temporary directory inside the object directory, and a temporary environment which can be passed to sub-programs to ask them to write there (the original object directory remains accessible as an alternate of the temporary one). See tmp-objdir.h for details on the API. Signed-off-by: Jeff King <peff@peff.net> Signed-off-by: Junio C Hamano <gitster@pobox.com>
Jeff King committed
Oct 3, 2016 at 16:49 UTC
2564d994c9c91aea58d59565d68d42bbc017f536
3 files changed
+328
Makefile
+1
@@ -831,6 +831,7 @@ LIB_OBJS += submodule-config.o
831
LIB_OBJS += symlinks.o
832
LIB_OBJS += tag.o
833
LIB_OBJS += tempfile.o
834
+LIB_OBJS += tmp-objdir.o
835
LIB_OBJS += trace.o
836
LIB_OBJS += trailer.o
837
LIB_OBJS += transport.o
tmp-objdir.c
new
+273
@@ -0,0 +1,273 @@
1
+#include "cache.h"
2
+#include "tmp-objdir.h"
3
+#include "dir.h"
4
+#include "sigchain.h"
5
+#include "string-list.h"
6
+#include "strbuf.h"
7
+#include "argv-array.h"
8
+
9
+struct tmp_objdir {
10
+ struct strbuf path;
11
+ struct argv_array env;
12
+};
13
+
14
+/*
15
+ * Allow only one tmp_objdir at a time in a running process, which simplifies
16
+ * our signal/atexit cleanup routines. It's doubtful callers will ever need
17
+ * more than one, and we can expand later if so. You can have many such
18
+ * tmp_objdirs simultaneously in many processes, of course.
19
+ */
20
+static struct tmp_objdir *the_tmp_objdir;
21
+
22
+static void tmp_objdir_free(struct tmp_objdir *t)
23
+{
24
+ strbuf_release(&t->path);
25
+ argv_array_clear(&t->env);
26
+ free(t);
27
+}
28
+
29
+static int tmp_objdir_destroy_1(struct tmp_objdir *t, int on_signal)
30
+{
31
+ int err;
32
+
33
+ if (!t)
34
+ return 0;
35
+
36
+ if (t == the_tmp_objdir)
37
+ the_tmp_objdir = NULL;
38
+
39
+ /*
40
+ * This may use malloc via strbuf_grow(), but we should
41
+ * have pre-grown t->path sufficiently so that this
42
+ * doesn't happen in practice.
43
+ */
44
+ err = remove_dir_recursively(&t->path, 0);
45
+
46
+ /*
47
+ * When we are cleaning up due to a signal, we won't bother
48
+ * freeing memory; it may cause a deadlock if the signal
49
+ * arrived while libc's allocator lock is held.
50
+ */
51
+ if (!on_signal)
52
+ tmp_objdir_free(t);
53
+ return err;
54
+}
55
+
56
+int tmp_objdir_destroy(struct tmp_objdir *t)
57
+{
58
+ return tmp_objdir_destroy_1(t, 0);
59
+}
60
+
61
+static void remove_tmp_objdir(void)
62
+{
63
+ tmp_objdir_destroy(the_tmp_objdir);
64
+}
65
+
66
+static void remove_tmp_objdir_on_signal(int signo)
67
+{
68
+ tmp_objdir_destroy_1(the_tmp_objdir, 1);
69
+ sigchain_pop(signo);
70
+ raise(signo);
71
+}
72
+
73
+/*
74
+ * These env_* functions are for setting up the child environment; the
75
+ * "replace" variant overrides the value of any existing variable with that
76
+ * "key". The "append" variant puts our new value at the end of a list,
77
+ * separated by PATH_SEP (which is what separate values in
78
+ * GIT_ALTERNATE_OBJECT_DIRECTORIES).
79
+ */
80
+static void env_append(struct argv_array *env, const char *key, const char *val)
81
+{
82
+ const char *old = getenv(key);
83
+
84
+ if (!old)
85
+ argv_array_pushf(env, "%s=%s", key, val);
86
+ else
87
+ argv_array_pushf(env, "%s=%s%c%s", key, old, PATH_SEP, val);
88
+}
89
+
90
+static void env_replace(struct argv_array *env, const char *key, const char *val)
91
+{
92
+ argv_array_pushf(env, "%s=%s", key, val);
93
+}
94
+
95
+static int setup_tmp_objdir(const char *root)
96
+{
97
+ char *path;
98
+ int ret = 0;
99
+
100
+ path = xstrfmt("%s/pack", root);
101
+ ret = mkdir(path, 0777);
102
+ free(path);
103
+
104
+ return ret;
105
+}
106
+
107
+struct tmp_objdir *tmp_objdir_create(void)
108
+{
109
+ static int installed_handlers;
110
+ struct tmp_objdir *t;
111
+
112
+ if (the_tmp_objdir)
113
+ die("BUG: only one tmp_objdir can be used at a time");
114
+
115
+ t = xmalloc(sizeof(*t));
116
+ strbuf_init(&t->path, 0);
117
+ argv_array_init(&t->env);
118
+
119
+ strbuf_addf(&t->path, "%s/incoming-XXXXXX", get_object_directory());
120
+
121
+ /*
122
+ * Grow the strbuf beyond any filename we expect to be placed in it.
123
+ * If tmp_objdir_destroy() is called by a signal handler, then
124
+ * we should be able to use the strbuf to remove files without
125
+ * having to call malloc.
126
+ */
127
+ strbuf_grow(&t->path, 1024);
128
+
129
+ if (!mkdtemp(t->path.buf)) {
130
+ /* free, not destroy, as we never touched the filesystem */
131
+ tmp_objdir_free(t);
132
+ return NULL;
133
+ }
134
+
135
+ the_tmp_objdir = t;
136
+ if (!installed_handlers) {
137
+ atexit(remove_tmp_objdir);
138
+ sigchain_push_common(remove_tmp_objdir_on_signal);
139
+ installed_handlers++;
140
+ }
141
+
142
+ if (setup_tmp_objdir(t->path.buf)) {
143
+ tmp_objdir_destroy(t);
144
+ return NULL;
145
+ }
146
+
147
+ env_append(&t->env, ALTERNATE_DB_ENVIRONMENT,
148
+ absolute_path(get_object_directory()));
149
+ env_replace(&t->env, DB_ENVIRONMENT, absolute_path(t->path.buf));
150
+
151
+ return t;
152
+}
153
+
154
+/*
155
+ * Make sure we copy packfiles and their associated metafiles in the correct
156
+ * order. All of these ends_with checks are slightly expensive to do in
157
+ * the midst of a sorting routine, but in practice it shouldn't matter.
158
+ * We will have a relatively small number of packfiles to order, and loose
159
+ * objects exit early in the first line.
160
+ */
161
+static int pack_copy_priority(const char *name)
162
+{
163
+ if (!starts_with(name, "pack"))
164
+ return 0;
165
+ if (ends_with(name, ".keep"))
166
+ return 1;
167
+ if (ends_with(name, ".pack"))
168
+ return 2;
169
+ if (ends_with(name, ".idx"))
170
+ return 3;
171
+ return 4;
172
+}
173
+
174
+static int pack_copy_cmp(const char *a, const char *b)
175
+{
176
+ return pack_copy_priority(a) - pack_copy_priority(b);
177
+}
178
+
179
+static int read_dir_paths(struct string_list *out, const char *path)
180
+{
181
+ DIR *dh;
182
+ struct dirent *de;
183
+
184
+ dh = opendir(path);
185
+ if (!dh)
186
+ return -1;
187
+
188
+ while ((de = readdir(dh)))
189
+ if (!is_dot_or_dotdot(de->d_name))
190
+ string_list_append(out, de->d_name);
191
+
192
+ closedir(dh);
193
+ return 0;
194
+}
195
+
196
+static int migrate_paths(struct strbuf *src, struct strbuf *dst);
197
+
198
+static int migrate_one(struct strbuf *src, struct strbuf *dst)
199
+{
200
+ struct stat st;
201
+
202
+ if (stat(src->buf, &st) < 0)
203
+ return -1;
204
+ if (S_ISDIR(st.st_mode)) {
205
+ if (!mkdir(dst->buf, 0777)) {
206
+ if (adjust_shared_perm(dst->buf))
207
+ return -1;
208
+ } else if (errno != EEXIST)
209
+ return -1;
210
+ return migrate_paths(src, dst);
211
+ }
212
+ return finalize_object_file(src->buf, dst->buf);
213
+}
214
+
215
+static int migrate_paths(struct strbuf *src, struct strbuf *dst)
216
+{
217
+ size_t src_len = src->len, dst_len = dst->len;
218
+ struct string_list paths = STRING_LIST_INIT_DUP;
219
+ int i;
220
+ int ret = 0;
221
+
222
+ if (read_dir_paths(&paths, src->buf) < 0)
223
+ return -1;
224
+ paths.cmp = pack_copy_cmp;
225
+ string_list_sort(&paths);
226
+
227
+ for (i = 0; i < paths.nr; i++) {
228
+ const char *name = paths.items[i].string;
229
+
230
+ strbuf_addf(src, "/%s", name);
231
+ strbuf_addf(dst, "/%s", name);
232
+
233
+ ret |= migrate_one(src, dst);
234
+
235
+ strbuf_setlen(src, src_len);
236
+ strbuf_setlen(dst, dst_len);
237
+ }
238
+
239
+ string_list_clear(&paths, 0);
240
+ return ret;
241
+}
242
+
243
+int tmp_objdir_migrate(struct tmp_objdir *t)
244
+{
245
+ struct strbuf src = STRBUF_INIT, dst = STRBUF_INIT;
246
+ int ret;
247
+
248
+ if (!t)
249
+ return 0;
250
+
251
+ strbuf_addbuf(&src, &t->path);
252
+ strbuf_addstr(&dst, get_object_directory());
253
+
254
+ ret = migrate_paths(&src, &dst);
255
+
256
+ strbuf_release(&src);
257
+ strbuf_release(&dst);
258
+
259
+ tmp_objdir_destroy(t);
260
+ return ret;
261
+}
262
+
263
+const char **tmp_objdir_env(const struct tmp_objdir *t)
264
+{
265
+ if (!t)
266
+ return NULL;
267
+ return t->env.argv;
268
+}
269
+
270
+void tmp_objdir_add_as_alternate(const struct tmp_objdir *t)
271
+{
272
+ add_to_alternates_memory(t->path.buf);
273
+}
tmp-objdir.h
new
+54
@@ -0,0 +1,54 @@
1
+#ifndef TMP_OBJDIR_H
2
+#define TMP_OBJDIR_H
3
+
4
+/*
5
+ * This API allows you to create a temporary object directory, advertise it to
6
+ * sub-processes via GIT_OBJECT_DIRECTORY and GIT_ALTERNATE_OBJECT_DIRECTORIES,
7
+ * and then either migrate its object into the main object directory, or remove
8
+ * it. The library handles unexpected signal/exit death by cleaning up the
9
+ * temporary directory.
10
+ *
11
+ * Example:
12
+ *
13
+ * struct tmp_objdir *t = tmp_objdir_create();
14
+ * if (!run_command_v_opt_cd_env(cmd, 0, NULL, tmp_objdir_env(t)) &&
15
+ * !tmp_objdir_migrate(t))
16
+ * printf("success!\n");
17
+ * else
18
+ * die("failed...tmp_objdir will clean up for us");
19
+ *
20
+ */
21
+
22
+struct tmp_objdir;
23
+
24
+/*
25
+ * Create a new temporary object directory; returns NULL on failure.
26
+ */
27
+struct tmp_objdir *tmp_objdir_create(void);
28
+
29
+/*
30
+ * Return a list of environment strings, suitable for use with
31
+ * child_process.env, that can be passed to child programs to make use of the
32
+ * temporary object directory.
33
+ */
34
+const char **tmp_objdir_env(const struct tmp_objdir *);
35
+
36
+/*
37
+ * Finalize a temporary object directory by migrating its objects into the main
38
+ * object database, removing the temporary directory, and freeing any
39
+ * associated resources.
40
+ */
41
+int tmp_objdir_migrate(struct tmp_objdir *);
42
+
43
+/*
44
+ * Destroy a temporary object directory, discarding any objects it contains.
45
+ */
46
+int tmp_objdir_destroy(struct tmp_objdir *);
47
+
48
+/*
49
+ * Add the temporary object directory as an alternate object store in the
50
+ * current process.
51
+ */
52
+void tmp_objdir_add_as_alternate(const struct tmp_objdir *);
53
+
54
+#endif /* TMP_OBJDIR_H */