@matheusbsbs / matheus-bagliano / commits / b3cecf49ea

tmp-objdir: new API for creating temporary writable databases

The tmp_objdir API provides the ability to create temporary object directories, but was designed with the goal of having subprocesses access these object stores, followed by the main process migrating objects from it to the main object store or just deleting it. The subprocesses would view it as their primary datastore and write to it. Here we add the tmp_objdir_replace_primary_odb function that replaces the current process's writable "main" object directory with the specified one. The previous main object directory is restored in either tmp_objdir_migrate or tmp_objdir_destroy. For the --remerge-diff usecase, add a new `will_destroy` flag in `struct object_database` to mark ephemeral object databases that do not require fsync durability. Add 'git prune' support for removing temporary object databases, and make sure that they have a name starting with tmp_ and containing an operation-specific name. Based-on-patch-by: Elijah Newren <newren@gmail.com> Signed-off-by: Neeraj Singh <neerajsi@microsoft.com> Reviewed-by: Elijah Newren <newren@gmail.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>

Neeraj Singh committed Dec 6, 2021 at 22:05 UTC b3cecf49eac00d62e361bf6e6e81392f5a2fb571

8 files changed +162 -14

builtin/prune.c

+16 -4

@@ -26,10 +26,22 @@ static int prune_tmp_file(const char *fullpath)

26 return error("Could not stat '%s'", fullpath);

27 if (st.st_mtime > expire)

28 return 0;

29 - if (show_only || verbose)

30 - printf("Removing stale temporary file %s\n", fullpath);

31 - if (!show_only)

32 - unlink_or_warn(fullpath);

29 + if (S_ISDIR(st.st_mode)) {

30 + if (show_only || verbose)

31 + printf("Removing stale temporary directory %s\n", fullpath);

32 + if (!show_only) {

33 + struct strbuf remove_dir_buf = STRBUF_INIT;

34 +

35 + strbuf_addstr(&remove_dir_buf, fullpath);

36 + remove_dir_recursively(&remove_dir_buf, 0);

37 + strbuf_release(&remove_dir_buf);

38 + }

39 + } else {

40 + if (show_only || verbose)

41 + printf("Removing stale temporary file %s\n", fullpath);

42 + if (!show_only)

43 + unlink_or_warn(fullpath);

44 + }

45 return 0;

46 }

builtin/receive-pack.c

+1 -1

@@ -2208,7 +2208,7 @@ static const char *unpack(int err_fd, struct shallow_info *si)

2208 strvec_push(&child.args, alt_shallow_file);

2209 }

2210

2211 - tmp_objdir = tmp_objdir_create();

2211 + tmp_objdir = tmp_objdir_create("incoming");

2212 if (!tmp_objdir) {

2213 if (err_fd > 0)

2214 close(err_fd);

environment.c

@@ -17,6 +17,7 @@

17 #include "commit.h"

18 #include "strvec.h"

19 #include "object-store.h"

20 +#include "tmp-objdir.h"

21 #include "chdir-notify.h"

22 #include "shallow.h"

@@ -340,10 +341,14 @@ static void update_relative_gitdir(const char *name,

341 void *data)

342 {

343 char *path = reparent_relative_path(old_cwd, new_cwd, get_git_dir());

344 + struct tmp_objdir *tmp_objdir = tmp_objdir_unapply_primary_odb();

345 +

346 trace_printf_key(&trace_setup_key,

347 "setup: move $GIT_DIR to '%s'",

348 path);

349 set_git_dir_1(path);

350 + if (tmp_objdir)

351 + tmp_objdir_reapply_primary_odb(tmp_objdir, old_cwd, new_cwd);

352 free(path);

353 }

354

object-file.c

+42 -2

@@ -751,6 +751,43 @@ void add_to_alternates_memory(const char *reference)

751 '\n', NULL, 0);

752 }

753

754 +struct object_directory *set_temporary_primary_odb(const char *dir, int will_destroy)

755 +{

756 + struct object_directory *new_odb;

757 +

758 + /*

759 + * Make sure alternates are initialized, or else our entry may be

760 + * overwritten when they are.

761 + */

762 + prepare_alt_odb(the_repository);

763 +

764 + /*

765 + * Make a new primary odb and link the old primary ODB in as an

766 + * alternate

767 + */

768 + new_odb = xcalloc(1, sizeof(*new_odb));

769 + new_odb->path = xstrdup(dir);

770 + new_odb->will_destroy = will_destroy;

771 + new_odb->next = the_repository->objects->odb;

772 + the_repository->objects->odb = new_odb;

773 + return new_odb->next;

774 +}

775 +

776 +void restore_primary_odb(struct object_directory *restore_odb, const char *old_path)

777 +{

778 + struct object_directory *cur_odb = the_repository->objects->odb;

779 +

780 + if (strcmp(old_path, cur_odb->path))

781 + BUG("expected %s as primary object store; found %s",

782 + old_path, cur_odb->path);

783 +

784 + if (cur_odb->next != restore_odb)

785 + BUG("we expect the old primary object store to be the first alternate");

786 +

787 + the_repository->objects->odb = restore_odb;

788 + free_object_directory(cur_odb);

789 +}

790 +

791 /*

792 * Compute the exact path an alternate is at and returns it. In case of

793 * error NULL is returned and the human readable error is added to `err`

@@ -1888,8 +1925,11 @@ int hash_object_file(const struct git_hash_algo *algo, const void *buf,

1925 /* Finalize a file on disk, and close it. */

1926 static void close_loose_object(int fd)

1927 {

1891 - if (fsync_object_files)

1892 - fsync_or_die(fd, "loose object file");

1928 + if (!the_repository->objects->odb->will_destroy) {

1929 + if (fsync_object_files)

1930 + fsync_or_die(fd, "loose object file");

1931 + }

1932 +

1933 if (close(fd) != 0)

1934 die_errno(_("error when closing loose object file"));

1935 }

object-store.h

+19

@@ -27,6 +27,11 @@ struct object_directory {

27 uint32_t loose_objects_subdir_seen[8]; /* 256 bits */

28 struct oidtree *loose_objects_cache;

30 + /*

31 + * This object store is ephemeral, so there is no need to fsync.

32 + */

33 + int will_destroy;

34 +

35 /*

36 * Path to the alternative object store. If this is a relative path,

37 * it is relative to the current working directory.

@@ -58,6 +63,17 @@ void add_to_alternates_file(const char *dir);

63 */

64 void add_to_alternates_memory(const char *dir);

66 +/*

67 + * Replace the current writable object directory with the specified temporary

68 + * object directory; returns the former primary object directory.

69 + */

70 +struct object_directory *set_temporary_primary_odb(const char *dir, int will_destroy);

71 +

72 +/*

73 + * Restore a previous ODB replaced by set_temporary_main_odb.

74 + */

75 +void restore_primary_odb(struct object_directory *restore_odb, const char *old_path);

76 +

77 /*

78 * Populate and return the loose object cache array corresponding to the

79 * given object ID.

@@ -68,6 +84,9 @@ struct oidtree *odb_loose_cache(struct object_directory *odb,

84 /* Empty the loose object cache for the specified object directory. */

85 void odb_clear_loose_cache(struct object_directory *odb);

87 +/* Clear and free the specified object directory */

88 +void free_object_directory(struct object_directory *odb);

89 +

90 struct packed_git {

91 struct hashmap_entry packmap_ent;

92 struct packed_git *next;

object.c

+1 -1

@@ -513,7 +513,7 @@ struct raw_object_store *raw_object_store_new(void)

513 return o;

514 }

515

516 -static void free_object_directory(struct object_directory *odb)

516 +void free_object_directory(struct object_directory *odb)

517 {

518 free(odb->path);

519 odb_clear_loose_cache(odb);

tmp-objdir.c

+52 -3

@@ -1,5 +1,6 @@

1 #include "cache.h"

2 #include "tmp-objdir.h"

3 +#include "chdir-notify.h"

4 #include "dir.h"

5 #include "sigchain.h"

6 #include "string-list.h"

@@ -11,6 +12,8 @@

12 struct tmp_objdir {

13 struct strbuf path;

14 struct strvec env;

15 + struct object_directory *prev_odb;

16 + int will_destroy;

17 };

19 /*

@@ -38,6 +41,9 @@ static int tmp_objdir_destroy_1(struct tmp_objdir *t, int on_signal)

41 if (t == the_tmp_objdir)

42 the_tmp_objdir = NULL;

44 + if (!on_signal && t->prev_odb)

45 + restore_primary_odb(t->prev_odb, t->path.buf);

46 +

47 /*

48 * This may use malloc via strbuf_grow(), but we should

49 * have pre-grown t->path sufficiently so that this

@@ -52,6 +58,7 @@ static int tmp_objdir_destroy_1(struct tmp_objdir *t, int on_signal)

58 */

59 if (!on_signal)

60 tmp_objdir_free(t);

61 +

62 return err;

63 }

@@ -121,7 +128,7 @@ static int setup_tmp_objdir(const char *root)

128 return ret;

129 }

130

124 -struct tmp_objdir *tmp_objdir_create(void)

131 +struct tmp_objdir *tmp_objdir_create(const char *prefix)

132 {

133 static int installed_handlers;

134 struct tmp_objdir *t;

@@ -129,11 +136,16 @@ struct tmp_objdir *tmp_objdir_create(void)

136 if (the_tmp_objdir)

137 BUG("only one tmp_objdir can be used at a time");

138

132 - t = xmalloc(sizeof(*t));

139 + t = xcalloc(1, sizeof(*t));

140 strbuf_init(&t->path, 0);

141 strvec_init(&t->env);

142

136 - strbuf_addf(&t->path, "%s/incoming-XXXXXX", get_object_directory());

143 + /*

144 + * Use a string starting with tmp_ so that the builtin/prune.c code

145 + * can recognize any stale objdirs left behind by a crash and delete

146 + * them.

147 + */

148 + strbuf_addf(&t->path, "%s/tmp_objdir-%s-XXXXXX", get_object_directory(), prefix);

149

150 /*

151 * Grow the strbuf beyond any filename we expect to be placed in it.

@@ -269,6 +281,13 @@ int tmp_objdir_migrate(struct tmp_objdir *t)

281 if (!t)

282 return 0;

283

284 + if (t->prev_odb) {

285 + if (the_repository->objects->odb->will_destroy)

286 + BUG("migrating an ODB that was marked for destruction");

287 + restore_primary_odb(t->prev_odb, t->path.buf);

288 + t->prev_odb = NULL;

289 + }

290 +

291 strbuf_addbuf(&src, &t->path);

292 strbuf_addstr(&dst, get_object_directory());

293

@@ -292,3 +311,33 @@ void tmp_objdir_add_as_alternate(const struct tmp_objdir *t)

311 {

312 add_to_alternates_memory(t->path.buf);

313 }

314 +

315 +void tmp_objdir_replace_primary_odb(struct tmp_objdir *t, int will_destroy)

316 +{

317 + if (t->prev_odb)

318 + BUG("the primary object database is already replaced");

319 + t->prev_odb = set_temporary_primary_odb(t->path.buf, will_destroy);

320 + t->will_destroy = will_destroy;

321 +}

322 +

323 +struct tmp_objdir *tmp_objdir_unapply_primary_odb(void)

324 +{

325 + if (!the_tmp_objdir || !the_tmp_objdir->prev_odb)

326 + return NULL;

327 +

328 + restore_primary_odb(the_tmp_objdir->prev_odb, the_tmp_objdir->path.buf);

329 + the_tmp_objdir->prev_odb = NULL;

330 + return the_tmp_objdir;

331 +}

332 +

333 +void tmp_objdir_reapply_primary_odb(struct tmp_objdir *t, const char *old_cwd,

334 + const char *new_cwd)

335 +{

336 + char *path;

337 +

338 + path = reparent_relative_path(old_cwd, new_cwd, t->path.buf);

339 + strbuf_reset(&t->path);

340 + strbuf_addstr(&t->path, path);

341 + free(path);

342 + tmp_objdir_replace_primary_odb(t, t->will_destroy);

343 +}

tmp-objdir.h

+26 -3

@@ -10,7 +10,7 @@

10 *

11 * Example:

12 *

13 - * struct tmp_objdir *t = tmp_objdir_create();

13 + * struct tmp_objdir *t = tmp_objdir_create("incoming");

14 * if (!run_command_v_opt_cd_env(cmd, 0, NULL, tmp_objdir_env(t)) &&

15 * !tmp_objdir_migrate(t))

16 * printf("success!\n");

@@ -22,9 +22,10 @@

22 struct tmp_objdir;

24 /*

25 - * Create a new temporary object directory; returns NULL on failure.

25 + * Create a new temporary object directory with the specified prefix;

26 + * returns NULL on failure.

27 */

27 -struct tmp_objdir *tmp_objdir_create(void);

28 +struct tmp_objdir *tmp_objdir_create(const char *prefix);

30 /*

31 * Return a list of environment strings, suitable for use with

@@ -51,4 +52,26 @@ int tmp_objdir_destroy(struct tmp_objdir *);

52 */

53 void tmp_objdir_add_as_alternate(const struct tmp_objdir *);

55 +/*

56 + * Replaces the writable object store in the current process with the temporary

57 + * object directory and makes the former main object store an alternate.

58 + * If will_destroy is nonzero, the object directory may not be migrated.

59 + */

60 +void tmp_objdir_replace_primary_odb(struct tmp_objdir *, int will_destroy);

61 +

62 +/*

63 + * If the primary object database was replaced by a temporary object directory,

64 + * restore it to its original value while keeping the directory contents around.

65 + * Returns NULL if the primary object database was not replaced.

66 + */

67 +struct tmp_objdir *tmp_objdir_unapply_primary_odb(void);

68 +

69 +/*

70 + * Reapplies the former primary temporary object database, after potentially

71 + * changing its relative path.

72 + */

73 +void tmp_objdir_reapply_primary_odb(struct tmp_objdir *, const char *old_cwd,

74 + const char *new_cwd);

75 +

76 +

77 #endif /* TMP_OBJDIR_H */