@matheusbsbs / matheus-bagliano / commits / 03e79c88aa
Fork 0

Replace use_packed_git with window cursors.

Part of the implementation concept of the sliding mmap window for pack access is to permit multiple windows per pack to be mapped independently. Since the inuse_cnt is associated with the mmap and not with the file, this value is in struct pack_window and needs to be incremented/decremented for each pack_window accessed by any code. To faciliate that implementation we need to replace all uses of use_packed_git() and unuse_packed_git() with a different API that follows struct pack_window objects rather than struct packed_git. The way this works is when we need to start accessing a pack for the first time we should setup a new window 'cursor' by declaring a local and setting it to NULL: struct pack_windows *w_curs = NULL; To obtain the memory region which contains a specific section of the pack file we invoke use_pack(), supplying the address of our current window cursor: unsigned int len; unsigned char *addr = use_pack(p, &w_curs, offset, &len); the returned address `addr` will be the first byte at `offset` within the pack file. The optional variable len will also be updated with the number of bytes remaining following the address. Multiple calls to use_pack() with the same window cursor will update the window cursor, moving it from one window to another when necessary. In this way each window cursor variable maintains only one struct pack_window inuse at a time. Finally before exiting the scope which originally declared the window cursor we must invoke unuse_pack() to unuse the current window (which may be different from the one that was first obtained from use_pack): unuse_pack(&w_curs); This implementation is still not complete with regards to multiple windows, as only one window per pack file is supported right now. Signed-off-by: Shawn O. Pearce <spearce@spearce.org> Signed-off-by: Junio C Hamano <junkio@cox.net>

Shawn O. Pearce committed Dec 23, 2006 at 02:34 UTC 03e79c88aa34ce188eb4fb7509e6d127c79c507d
4 files changed +85 -67
builtin-pack-objects.c
+7 -9
@@ -416,6 +416,7 @@ static unsigned long write_object(struct sha1file *f,
416 }
417 else {
418 struct packed_git *p = entry->in_pack;
419 + struct pack_window *w_curs = NULL;
420
421 if (entry->delta) {
422 obj_type = (allow_ofs_delta && entry->delta->offset) ?
@@ -437,16 +438,14 @@ static unsigned long write_object(struct sha1file *f,
438 hdrlen += 20;
439 }
440
440 - use_packed_git(p);
441 - buf = p->windows->base
442 - + entry->in_pack_offset
443 - + entry->in_pack_header_size;
441 + buf = use_pack(p, &w_curs, entry->in_pack_offset
442 + + entry->in_pack_header_size, NULL);
443 datalen = find_packed_object_size(p, entry->in_pack_offset)
444 - entry->in_pack_header_size;
445 if (!pack_to_stdout && check_inflate(buf, datalen, entry->size))
446 die("corrupt delta in pack %s", sha1_to_hex(entry->sha1));
447 sha1write(f, buf, datalen);
449 - unuse_packed_git(p);
448 + unuse_pack(&w_curs);
449 reused++;
450 }
451 if (entry->delta)
@@ -937,14 +936,13 @@ static void check_object(struct object_entry *entry)
936
937 if (entry->in_pack && !entry->preferred_base) {
938 struct packed_git *p = entry->in_pack;
939 + struct pack_window *w_curs = NULL;
940 unsigned long left = p->pack_size - entry->in_pack_offset;
941 unsigned long size, used;
942 unsigned char *buf;
943 struct object_entry *base_entry = NULL;
944
945 - use_packed_git(p);
946 - buf = p->windows->base;
947 - buf += entry->in_pack_offset;
945 + buf = use_pack(p, &w_curs, entry->in_pack_offset, NULL);
946
947 /* We want in_pack_type even if we do not reuse delta.
948 * There is no point not reusing non-delta representations.
@@ -990,7 +988,7 @@ static void check_object(struct object_entry *entry)
988 if (base_name)
989 base_entry = locate_object_entry(base_name);
990 }
993 - unuse_packed_git(p);
991 + unuse_pack(&w_curs);
992 entry->in_pack_header_size = used;
993
994 if (base_entry) {
cache.h
+2 -2
@@ -397,8 +397,8 @@ extern void install_packed_git(struct packed_git *pack);
397 extern struct packed_git *find_sha1_pack(const unsigned char *sha1,
398 struct packed_git *packs);
399
400 -extern int use_packed_git(struct packed_git *);
401 -extern void unuse_packed_git(struct packed_git *);
400 +extern unsigned char* use_pack(struct packed_git *, struct pack_window **, unsigned long, unsigned int *);
401 +extern void unuse_pack(struct pack_window **);
402 extern struct packed_git *add_packed_git(char *, int, int);
403 extern int num_packed_objects(const struct packed_git *p);
404 extern int nth_packed_object_sha1(const struct packed_git *, int, unsigned char*);
pack-check.c
+12 -10
@@ -1,7 +1,8 @@
1 #include "cache.h"
2 #include "pack.h"
3
4 -static int verify_packfile(struct packed_git *p)
4 +static int verify_packfile(struct packed_git *p,
5 + struct pack_window **w_curs)
6 {
7 unsigned long index_size = p->index_size;
8 void *index_base = p->index_base;
@@ -13,7 +14,7 @@ static int verify_packfile(struct packed_git *p)
14 int nr_objects, err, i;
15
16 /* Header consistency check */
16 - pack_base = p->windows->base;
17 + pack_base = use_pack(p, w_curs, 0, NULL);
18 hdr = (struct pack_header*)pack_base;
19 if (hdr->hdr_signature != htonl(PACK_SIGNATURE))
20 return error("Packfile %s signature mismatch", p->pack_name);
@@ -72,13 +73,14 @@ static int verify_packfile(struct packed_git *p)
73
74 #define MAX_CHAIN 40
75
75 -static void show_pack_info(struct packed_git *p)
76 +static void show_pack_info(struct packed_git *p,
77 + struct pack_window **w_curs)
78 {
79 struct pack_header *hdr;
80 int nr_objects, i;
81 unsigned int chain_histogram[MAX_CHAIN];
82
81 - hdr = (struct pack_header*)p->windows->base;
83 + hdr = (struct pack_header*)use_pack(p, w_curs, 0, NULL);
84 nr_objects = ntohl(hdr->hdr_entries);
85 memset(chain_histogram, 0, sizeof(chain_histogram));
86
@@ -142,18 +144,18 @@ int verify_pack(struct packed_git *p, int verbose)
144
145 if (!ret) {
146 /* Verify pack file */
145 - use_packed_git(p);
146 - ret = verify_packfile(p);
147 - unuse_packed_git(p);
147 + struct pack_window *w_curs = NULL;
148 + ret = verify_packfile(p, &w_curs);
149 + unuse_pack(&w_curs);
150 }
151
152 if (verbose) {
153 if (ret)
154 printf("%s: bad\n", p->pack_name);
155 else {
154 - use_packed_git(p);
155 - show_pack_info(p);
156 - unuse_packed_git(p);
156 + struct pack_window *w_curs = NULL;
157 + show_pack_info(p, &w_curs);
158 + unuse_pack(&w_curs);
159 printf("%s: ok\n", p->pack_name);
160 }
161 }
sha1_file.c
+64 -46
@@ -470,10 +470,13 @@ static int unuse_one_packed_git(void)
470 return 1;
471 }
472
473 -void unuse_packed_git(struct packed_git *p)
473 +void unuse_pack(struct pack_window **w_cursor)
474 {
475 - if (p->windows)
476 - p->windows->inuse_cnt--;
475 + struct pack_window *w = *w_cursor;
476 + if (w) {
477 + w->inuse_cnt--;
478 + *w_cursor = NULL;
479 + }
480 }
481
482 static void open_packed_git(struct packed_git *p)
@@ -518,13 +521,16 @@ static void open_packed_git(struct packed_git *p)
521 die("packfile %s does not match index", p->pack_name);
522 }
523
521 -int use_packed_git(struct packed_git *p)
524 +unsigned char* use_pack(struct packed_git *p,
525 + struct pack_window **w_cursor,
526 + unsigned long offset,
527 + unsigned int *left)
528 {
529 + struct pack_window *win = p->windows;
530 +
531 if (p->pack_fd == -1)
532 open_packed_git(p);
525 - if (!p->windows) {
526 - struct pack_window *win;
527 -
533 + if (!win) {
534 pack_mapped += p->pack_size;
535 while (packed_git_limit < pack_mapped && unuse_one_packed_git())
536 ; /* nothing */
@@ -535,9 +541,14 @@ int use_packed_git(struct packed_git *p)
541 die("packfile %s cannot be mapped.", p->pack_name);
542 p->windows = win;
543 }
538 - p->windows->last_used = pack_used_ctr++;
539 - p->windows->inuse_cnt++;
540 - return 0;
544 + if (win != *w_cursor) {
545 + win->last_used = pack_used_ctr++;
546 + win->inuse_cnt++;
547 + *w_cursor = win;
548 + }
549 + if (left)
550 + *left = win->len - offset;
551 + return win->base + offset;
552 }
553
554 struct packed_git *add_packed_git(char *path, int path_len, int local)
@@ -883,12 +894,13 @@ void * unpack_sha1_file(void *map, unsigned long mapsize, char *type, unsigned l
894 }
895
896 static unsigned long get_delta_base(struct packed_git *p,
897 + struct pack_window **w_curs,
898 unsigned long offset,
899 enum object_type kind,
900 unsigned long delta_obj_offset,
901 unsigned long *base_obj_offset)
902 {
891 - unsigned char *base_info = p->windows->base + offset;
903 + unsigned char *base_info = use_pack(p, w_curs, offset, NULL);
904 unsigned long base_offset;
905
906 /* there must be at least 20 bytes left regardless of delta type */
@@ -928,6 +940,7 @@ static int packed_object_info(struct packed_git *p, unsigned long offset,
940 char *type, unsigned long *sizep);
941
942 static int packed_delta_info(struct packed_git *p,
943 + struct pack_window **w_curs,
944 unsigned long offset,
945 enum object_type kind,
946 unsigned long obj_offset,
@@ -936,7 +949,8 @@ static int packed_delta_info(struct packed_git *p,
949 {
950 unsigned long base_offset;
951
939 - offset = get_delta_base(p, offset, kind, obj_offset, &base_offset);
952 + offset = get_delta_base(p, w_curs, offset, kind,
953 + obj_offset, &base_offset);
954
955 /* We choose to only get the type of the base object and
956 * ignore potentially corrupt pack file that expects the delta
@@ -955,8 +969,7 @@ static int packed_delta_info(struct packed_git *p,
969
970 memset(&stream, 0, sizeof(stream));
971
958 - stream.next_in = p->windows->base + offset;
959 - stream.avail_in = p->pack_size - offset;
972 + stream.next_in = use_pack(p, w_curs, offset, &stream.avail_in);
973 stream.next_out = delta_head;
974 stream.avail_out = sizeof(delta_head);
975
@@ -982,16 +995,18 @@ static int packed_delta_info(struct packed_git *p,
995 return 0;
996 }
997
985 -static unsigned long unpack_object_header(struct packed_git *p, unsigned long offset,
986 - enum object_type *type, unsigned long *sizep)
998 +static unsigned long unpack_object_header(struct packed_git *p,
999 + struct pack_window **w_curs,
1000 + unsigned long offset,
1001 + enum object_type *type,
1002 + unsigned long *sizep)
1003 {
1004 + unsigned char *base;
1005 + unsigned int left;
1006 unsigned long used;
1007
990 - if (p->pack_size <= offset)
991 - die("object offset outside of pack file");
992 -
993 - used = unpack_object_header_gently(p->windows->base + offset,
994 - p->pack_size - offset, type, sizep);
1008 + base = use_pack(p, w_curs, offset, &left);
1009 + used = unpack_object_header_gently(base, left, type, sizep);
1010 if (!used)
1011 die("object offset outside of pack file");
1012
@@ -1006,13 +1021,14 @@ void packed_object_info_detail(struct packed_git *p,
1021 unsigned int *delta_chain_length,
1022 unsigned char *base_sha1)
1023 {
1024 + struct pack_window *w_curs = NULL;
1025 unsigned long obj_offset, val;
1026 unsigned char *next_sha1;
1027 enum object_type kind;
1028
1029 *delta_chain_length = 0;
1030 obj_offset = offset;
1015 - offset = unpack_object_header(p, offset, &kind, size);
1031 + offset = unpack_object_header(p, &w_curs, offset, &kind, size);
1032
1033 for (;;) {
1034 switch (kind) {
@@ -1025,25 +1041,24 @@ void packed_object_info_detail(struct packed_git *p,
1041 case OBJ_TAG:
1042 strcpy(type, type_names[kind]);
1043 *store_size = 0; /* notyet */
1044 + unuse_pack(&w_curs);
1045 return;
1046 case OBJ_OFS_DELTA:
1030 - get_delta_base(p, offset, kind, obj_offset, &offset);
1047 + get_delta_base(p, &w_curs, offset, kind,
1048 + obj_offset, &offset);
1049 if (*delta_chain_length == 0) {
1050 /* TODO: find base_sha1 as pointed by offset */
1051 }
1052 break;
1053 case OBJ_REF_DELTA:
1036 - if (p->pack_size <= offset + 20)
1037 - die("pack file %s records an incomplete delta base",
1038 - p->pack_name);
1039 - next_sha1 = p->windows->base + offset;
1054 + next_sha1 = use_pack(p, &w_curs, offset, NULL);
1055 if (*delta_chain_length == 0)
1056 hashcpy(base_sha1, next_sha1);
1057 offset = find_pack_entry_one(next_sha1, p);
1058 break;
1059 }
1060 obj_offset = offset;
1046 - offset = unpack_object_header(p, offset, &kind, &val);
1061 + offset = unpack_object_header(p, &w_curs, offset, &kind, &val);
1062 (*delta_chain_length)++;
1063 }
1064 }
@@ -1051,20 +1066,26 @@ void packed_object_info_detail(struct packed_git *p,
1066 static int packed_object_info(struct packed_git *p, unsigned long offset,
1067 char *type, unsigned long *sizep)
1068 {
1069 + struct pack_window *w_curs = NULL;
1070 unsigned long size, obj_offset = offset;
1071 enum object_type kind;
1072 + int r;
1073
1057 - offset = unpack_object_header(p, offset, &kind, &size);
1074 + offset = unpack_object_header(p, &w_curs, offset, &kind, &size);
1075
1076 switch (kind) {
1077 case OBJ_OFS_DELTA:
1078 case OBJ_REF_DELTA:
1062 - return packed_delta_info(p, offset, kind, obj_offset, type, sizep);
1079 + r = packed_delta_info(p, &w_curs, offset, kind,
1080 + obj_offset, type, sizep);
1081 + unuse_pack(&w_curs);
1082 + return r;
1083 case OBJ_COMMIT:
1084 case OBJ_TREE:
1085 case OBJ_BLOB:
1086 case OBJ_TAG:
1087 strcpy(type, type_names[kind]);
1088 + unuse_pack(&w_curs);
1089 break;
1090 default:
1091 die("pack %s contains unknown object type %d",
@@ -1076,6 +1097,7 @@ static int packed_object_info(struct packed_git *p, unsigned long offset,
1097 }
1098
1099 static void *unpack_compressed_entry(struct packed_git *p,
1100 + struct pack_window **w_curs,
1101 unsigned long offset,
1102 unsigned long size)
1103 {
@@ -1086,8 +1108,7 @@ static void *unpack_compressed_entry(struct packed_git *p,
1108 buffer = xmalloc(size + 1);
1109 buffer[size] = 0;
1110 memset(&stream, 0, sizeof(stream));
1089 - stream.next_in = p->windows->base + offset;
1090 - stream.avail_in = p->pack_size - offset;
1111 + stream.next_in = use_pack(p, w_curs, offset, &stream.avail_in);
1112 stream.next_out = buffer;
1113 stream.avail_out = size;
1114
@@ -1103,6 +1124,7 @@ static void *unpack_compressed_entry(struct packed_git *p,
1124 }
1125
1126 static void *unpack_delta_entry(struct packed_git *p,
1127 + struct pack_window **w_curs,
1128 unsigned long offset,
1129 unsigned long delta_size,
1130 enum object_type kind,
@@ -1113,13 +1135,14 @@ static void *unpack_delta_entry(struct packed_git *p,
1135 void *delta_data, *result, *base;
1136 unsigned long result_size, base_size, base_offset;
1137
1116 - offset = get_delta_base(p, offset, kind, obj_offset, &base_offset);
1138 + offset = get_delta_base(p, w_curs, offset, kind,
1139 + obj_offset, &base_offset);
1140 base = unpack_entry(p, base_offset, type, &base_size);
1141 if (!base)
1142 die("failed to read delta base object at %lu from %s",
1143 base_offset, p->pack_name);
1144
1122 - delta_data = unpack_compressed_entry(p, offset, delta_size);
1145 + delta_data = unpack_compressed_entry(p, w_curs, offset, delta_size);
1146 result = patch_delta(base, base_size,
1147 delta_data, delta_size,
1148 &result_size);
@@ -1134,17 +1157,17 @@ static void *unpack_delta_entry(struct packed_git *p,
1157 void *unpack_entry(struct packed_git *p, unsigned long offset,
1158 char *type, unsigned long *sizep)
1159 {
1160 + struct pack_window *w_curs = NULL;
1161 unsigned long size, obj_offset = offset;
1162 enum object_type kind;
1163 void *retval;
1164
1141 - if (use_packed_git(p))
1142 - die("cannot map packed file");
1143 - offset = unpack_object_header(p, offset, &kind, &size);
1165 + offset = unpack_object_header(p, &w_curs, offset, &kind, &size);
1166 switch (kind) {
1167 case OBJ_OFS_DELTA:
1168 case OBJ_REF_DELTA:
1147 - retval = unpack_delta_entry(p, offset, size, kind, obj_offset, type, sizep);
1169 + retval = unpack_delta_entry(p, &w_curs, offset, size,
1170 + kind, obj_offset, type, sizep);
1171 break;
1172 case OBJ_COMMIT:
1173 case OBJ_TREE:
@@ -1152,12 +1175,12 @@ void *unpack_entry(struct packed_git *p, unsigned long offset,
1175 case OBJ_TAG:
1176 strcpy(type, type_names[kind]);
1177 *sizep = size;
1155 - retval = unpack_compressed_entry(p, offset, size);
1178 + retval = unpack_compressed_entry(p, &w_curs, offset, size);
1179 break;
1180 default:
1181 die("unknown object type %i in %s", kind, p->pack_name);
1182 }
1160 - unuse_packed_git(p);
1183 + unuse_pack(&w_curs);
1184 return retval;
1185 }
1186
@@ -1284,7 +1307,6 @@ static int sha1_loose_object_info(const unsigned char *sha1, char *type, unsigne
1307
1308 int sha1_object_info(const unsigned char *sha1, char *type, unsigned long *sizep)
1309 {
1287 - int status;
1310 struct pack_entry e;
1311
1312 if (!find_pack_entry(sha1, &e, NULL)) {
@@ -1292,11 +1314,7 @@ int sha1_object_info(const unsigned char *sha1, char *type, unsigned long *sizep
1314 if (!find_pack_entry(sha1, &e, NULL))
1315 return sha1_loose_object_info(sha1, type, sizep);
1316 }
1295 - if (use_packed_git(e.p))
1296 - die("cannot map packed file");
1297 - status = packed_object_info(e.p, e.offset, type, sizep);
1298 - unuse_packed_git(e.p);
1299 - return status;
1317 + return packed_object_info(e.p, e.offset, type, sizep);
1318 }
1319
1320 static void *read_packed_sha1(const unsigned char *sha1, char *type, unsigned long *size)