@matheusbsbs / matheus-bagliano / commits / 99bf115c87

repository: introduce parsed objects field

Convert the existing global cache for parsed objects (obj_hash) into repository-specific parsed object caches. Existing code that uses obj_hash are modified to use the parsed object cache of the_repository; future patches will use the parsed object caches of other repositories. Another future use case for a pool of objects is ease of memory management in revision walking: If we can free the rev-list related memory early in pack-objects (e.g. part of repack operation) then it could lower memory pressure significantly when running on large repos. While this has been discussed on the mailing list lately, this series doesn't implement this. Signed-off-by: Stefan Beller <sbeller@google.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>

Stefan Beller committed May 8, 2018 at 12:37 UTC 99bf115c879af7e38ef0ca9596fc9db1d6598d5f

4 files changed +64 -23

object.c

+40 -23

@@ -8,17 +8,14 @@

8 #include "object-store.h"

9 #include "packfile.h"

11 -static struct object **obj_hash;

12 -static int nr_objs, obj_hash_size;

13 -

11 unsigned int get_max_object_index(void)

12 {

16 - return obj_hash_size;

13 + return the_repository->parsed_objects->obj_hash_size;

14 }

16 struct object *get_indexed_object(unsigned int idx)

17 {

21 - return obj_hash[idx];

18 + return the_repository->parsed_objects->obj_hash[idx];

19 }

21 static const char *object_type_strings[] = {

@@ -90,15 +87,16 @@ struct object *lookup_object(const unsigned char *sha1)

87 unsigned int i, first;

88 struct object *obj;

93 - if (!obj_hash)

90 + if (!the_repository->parsed_objects->obj_hash)

91 return NULL;

96 - first = i = hash_obj(sha1, obj_hash_size);

97 - while ((obj = obj_hash[i]) != NULL) {

93 + first = i = hash_obj(sha1,

94 + the_repository->parsed_objects->obj_hash_size);

95 + while ((obj = the_repository->parsed_objects->obj_hash[i]) != NULL) {

96 if (!hashcmp(sha1, obj->oid.hash))

97 break;

98 i++;

101 - if (i == obj_hash_size)

99 + if (i == the_repository->parsed_objects->obj_hash_size)

100 i = 0;

101 }

102 if (obj && i != first) {

@@ -107,7 +105,8 @@ struct object *lookup_object(const unsigned char *sha1)

105 * that we do not need to walk the hash table the next

106 * time we look for it.

107 */

110 - SWAP(obj_hash[i], obj_hash[first]);

108 + SWAP(the_repository->parsed_objects->obj_hash[i],

109 + the_repository->parsed_objects->obj_hash[first]);

110 }

111 return obj;

112 }

@@ -124,19 +123,19 @@ static void grow_object_hash(void)

123 * Note that this size must always be power-of-2 to match hash_obj

124 * above.

125 */

127 - int new_hash_size = obj_hash_size < 32 ? 32 : 2 * obj_hash_size;

126 + int new_hash_size = the_repository->parsed_objects->obj_hash_size < 32 ? 32 : 2 * the_repository->parsed_objects->obj_hash_size;

127 struct object **new_hash;

128

129 new_hash = xcalloc(new_hash_size, sizeof(struct object *));

131 - for (i = 0; i < obj_hash_size; i++) {

132 - struct object *obj = obj_hash[i];

130 + for (i = 0; i < the_repository->parsed_objects->obj_hash_size; i++) {

131 + struct object *obj = the_repository->parsed_objects->obj_hash[i];

132 if (!obj)

133 continue;

134 insert_obj_hash(obj, new_hash, new_hash_size);

135 }

137 - free(obj_hash);

138 - obj_hash = new_hash;

139 - obj_hash_size = new_hash_size;

136 + free(the_repository->parsed_objects->obj_hash);

137 + the_repository->parsed_objects->obj_hash = new_hash;

138 + the_repository->parsed_objects->obj_hash_size = new_hash_size;

139 }

140

141 void *create_object(const unsigned char *sha1, void *o)

@@ -147,11 +146,12 @@ void *create_object(const unsigned char *sha1, void *o)

146 obj->flags = 0;

147 hashcpy(obj->oid.hash, sha1);

148

150 - if (obj_hash_size - 1 <= nr_objs * 2)

149 + if (the_repository->parsed_objects->obj_hash_size - 1 <= the_repository->parsed_objects->nr_objs * 2)

150 grow_object_hash();

151

153 - insert_obj_hash(obj, obj_hash, obj_hash_size);

154 - nr_objs++;

152 + insert_obj_hash(obj, the_repository->parsed_objects->obj_hash,

153 + the_repository->parsed_objects->obj_hash_size);

154 + the_repository->parsed_objects->nr_objs++;

155 return obj;

156 }

157

@@ -431,8 +431,8 @@ void clear_object_flags(unsigned flags)

431 {

432 int i;

433

434 - for (i=0; i < obj_hash_size; i++) {

435 - struct object *obj = obj_hash[i];

434 + for (i=0; i < the_repository->parsed_objects->obj_hash_size; i++) {

435 + struct object *obj = the_repository->parsed_objects->obj_hash[i];

436 if (obj)

437 obj->flags &= ~flags;

438 }

@@ -442,13 +442,20 @@ void clear_commit_marks_all(unsigned int flags)

442 {

443 int i;

444

445 - for (i = 0; i < obj_hash_size; i++) {

446 - struct object *obj = obj_hash[i];

445 + for (i = 0; i < the_repository->parsed_objects->obj_hash_size; i++) {

446 + struct object *obj = the_repository->parsed_objects->obj_hash[i];

447 if (obj && obj->type == OBJ_COMMIT)

448 obj->flags &= ~flags;

449 }

450 }

451

452 +struct parsed_object_pool *parsed_object_pool_new(void)

453 +{

454 + struct parsed_object_pool *o = xmalloc(sizeof(*o));

455 + memset(o, 0, sizeof(*o));

456 + return o;

457 +}

458 +

459 struct raw_object_store *raw_object_store_new(void)

460 {

461 struct raw_object_store *o = xmalloc(sizeof(*o));

@@ -488,3 +495,13 @@ void raw_object_store_clear(struct raw_object_store *o)

495 close_all_packs(o);

496 o->packed_git = NULL;

497 }

498 +

499 +void parsed_object_pool_clear(struct parsed_object_pool *o)

500 +{

501 + /*

502 + * TOOD free objects in o->obj_hash.

503 + *

504 + * As objects are allocated in slabs (see alloc.c), we do

505 + * not need to free each object, but each slab instead.

506 + */

507 +}

object.h

@@ -1,6 +1,14 @@

1 #ifndef OBJECT_H

2 #define OBJECT_H

4 +struct parsed_object_pool {

5 + struct object **obj_hash;

6 + int nr_objs, obj_hash_size;

7 +};

8 +

9 +struct parsed_object_pool *parsed_object_pool_new(void);

10 +void parsed_object_pool_clear(struct parsed_object_pool *o);

11 +

12 struct object_list {

13 struct object *item;

14 struct object_list *next;

repository.c

@@ -2,6 +2,7 @@

2 #include "repository.h"

3 #include "object-store.h"

4 #include "config.h"

5 +#include "object.h"

6 #include "submodule-config.h"

8 /* The main repository */

@@ -14,6 +15,8 @@ void initialize_the_repository(void)

16 the_repo.index = &the_index;

17 the_repo.objects = raw_object_store_new();

18 + the_repo.parsed_objects = parsed_object_pool_new();

19 +

20 repo_set_hash_algo(&the_repo, GIT_HASH_SHA1);

21 }

@@ -143,6 +146,7 @@ static int repo_init(struct repository *repo,

146 memset(repo, 0, sizeof(*repo));

147

148 repo->objects = raw_object_store_new();

149 + repo->parsed_objects = parsed_object_pool_new();

150

151 if (repo_init_gitdir(repo, gitdir))

152 goto error;

@@ -226,6 +230,9 @@ void repo_clear(struct repository *repo)

230 raw_object_store_clear(repo->objects);

231 FREE_AND_NULL(repo->objects);

232

233 + parsed_object_pool_clear(repo->parsed_objects);

234 + FREE_AND_NULL(repo->parsed_objects);

235 +

236 if (repo->config) {

237 git_configset_clear(repo->config);

238 FREE_AND_NULL(repo->config);

repository.h

@@ -26,6 +26,15 @@ struct repository {

26 */

27 struct raw_object_store *objects;

29 + /*

30 + * All objects in this repository that have been parsed. This structure

31 + * owns all objects it references, so users of "struct object *"

32 + * generally do not need to free them; instead, when a repository is no

33 + * longer used, call parsed_object_pool_clear() on this structure, which

34 + * is called by the repositories repo_clear on its desconstruction.

35 + */

36 + struct parsed_object_pool *parsed_objects;

37 +

38 /* The store in which the refs are held. */

39 struct ref_store *refs;