Raw
1 #include "git-compat-util.h"
2 #include "abspath.h"
3 #include "chdir-notify.h"
4 #include "gettext.h"
5 #include "hex.h"
6 #include "loose.h"
7 #include "object-file.h"
8 #include "object-file-convert.h"
9 #include "odb.h"
10 #include "odb/source-files.h"
11 #include "odb/source-loose.h"
12 #include "odb/streaming.h"
13 #include "oidtree.h"
14 #include "repository.h"
15 #include "strbuf.h"
16
17 static int append_loose_object(const struct object_id *oid,
18 const char *path UNUSED,
19 void *data)
20 {
21 oidtree_insert(data, oid, NULL);
22 return 0;
23 }
24
25 static struct oidtree *odb_source_loose_cache(struct odb_source_loose *loose,
26 const struct object_id *oid)
27 {
28 int subdir_nr = oid->hash[0];
29 struct strbuf buf = STRBUF_INIT;
30 size_t word_bits = bitsizeof(loose->subdir_seen[0]);
31 size_t word_index = subdir_nr / word_bits;
32 size_t mask = (size_t)1u << (subdir_nr % word_bits);
33 uint32_t *bitmap;
34
35 if (subdir_nr < 0 ||
36 (size_t) subdir_nr >= bitsizeof(loose->subdir_seen))
37 BUG("subdir_nr out of range");
38
39 bitmap = &loose->subdir_seen[word_index];
40 if (*bitmap & mask)
41 return loose->cache;
42 if (!loose->cache) {
43 ALLOC_ARRAY(loose->cache, 1);
44 oidtree_init(loose->cache);
45 }
46 strbuf_addstr(&buf, loose->base.path);
47 for_each_file_in_obj_subdir(subdir_nr, &buf,
48 loose->base.odb->repo->hash_algo,
49 append_loose_object,
50 NULL, NULL,
51 loose->cache);
52 *bitmap |= mask;
53 strbuf_release(&buf);
54 return loose->cache;
55 }
56
57 static int quick_has_loose(struct odb_source_loose *loose,
58 const struct object_id *oid)
59 {
60 return !!oidtree_contains(odb_source_loose_cache(loose, oid), oid);
61 }
62
63 static int read_object_info_from_path(struct odb_source_loose *loose,
64 const char *path,
65 const struct object_id *oid,
66 struct object_info *oi,
67 enum object_info_flags flags)
68 {
69 int ret;
70 int fd;
71 unsigned long mapsize;
72 void *map = NULL;
73 git_zstream stream, *stream_to_end = NULL;
74 char hdr[MAX_HEADER_LEN];
75 size_t size_scratch;
76 enum object_type type_scratch;
77 struct stat st;
78
79 /*
80 * If we don't care about type or size, then we don't
81 * need to look inside the object at all. Note that we
82 * do not optimize out the stat call, even if the
83 * caller doesn't care about the disk-size, since our
84 * return value implicitly indicates whether the
85 * object even exists.
86 */
87 if (!oi || (!oi->typep && !oi->sizep && !oi->contentp)) {
88 struct stat st;
89
90 if ((!oi || (!oi->disk_sizep && !oi->mtimep)) && (flags & OBJECT_INFO_QUICK)) {
91 ret = quick_has_loose(loose, oid) ? 0 : -1;
92 goto out;
93 }
94
95 if (lstat(path, &st) < 0) {
96 ret = -1;
97 goto out;
98 }
99
100 if (oi) {
101 if (oi->disk_sizep)
102 *oi->disk_sizep = st.st_size;
103 if (oi->mtimep)
104 *oi->mtimep = st.st_mtime;
105 }
106
107 ret = 0;
108 goto out;
109 }
110
111 fd = git_open(path);
112 if (fd < 0) {
113 if (errno != ENOENT)
114 error_errno(_("unable to open loose object %s"), oid_to_hex(oid));
115 ret = -1;
116 goto out;
117 }
118
119 if (fstat(fd, &st)) {
120 close(fd);
121 ret = -1;
122 goto out;
123 }
124
125 mapsize = xsize_t(st.st_size);
126 if (!mapsize) {
127 close(fd);
128 ret = error(_("object file %s is empty"), path);
129 goto out;
130 }
131
132 map = xmmap(NULL, mapsize, PROT_READ, MAP_PRIVATE, fd, 0);
133 close(fd);
134 if (!map) {
135 ret = -1;
136 goto out;
137 }
138
139 if (oi->disk_sizep)
140 *oi->disk_sizep = mapsize;
141 if (oi->mtimep)
142 *oi->mtimep = st.st_mtime;
143
144 stream_to_end = &stream;
145
146 switch (unpack_loose_header(&stream, map, mapsize, hdr, sizeof(hdr))) {
147 case ULHR_OK:
148 if (!oi->sizep)
149 oi->sizep = &size_scratch;
150 if (!oi->typep)
151 oi->typep = &type_scratch;
152
153 if (parse_loose_header(hdr, oi) < 0) {
154 ret = error(_("unable to parse %s header"), oid_to_hex(oid));
155 goto corrupt;
156 }
157
158 if (*oi->typep < 0)
159 die(_("invalid object type"));
160
161 if (oi->contentp) {
162 *oi->contentp = unpack_loose_rest(&stream, hdr, *oi->sizep, oid);
163 if (!*oi->contentp) {
164 ret = -1;
165 goto corrupt;
166 }
167 }
168
169 break;
170 case ULHR_BAD:
171 ret = error(_("unable to unpack %s header"),
172 oid_to_hex(oid));
173 goto corrupt;
174 case ULHR_TOO_LONG:
175 ret = error(_("header for %s too long, exceeds %d bytes"),
176 oid_to_hex(oid), MAX_HEADER_LEN);
177 goto corrupt;
178 }
179
180 ret = 0;
181
182 corrupt:
183 if (ret && (flags & OBJECT_INFO_DIE_IF_CORRUPT))
184 die(_("loose object %s (stored in %s) is corrupt"),
185 oid_to_hex(oid), path);
186
187 out:
188 if (stream_to_end)
189 git_inflate_end(stream_to_end);
190 if (map)
191 munmap(map, mapsize);
192 if (oi) {
193 if (oi->sizep == &size_scratch)
194 oi->sizep = NULL;
195 if (oi->typep == &type_scratch)
196 oi->typep = NULL;
197 if (oi->delta_base_oid)
198 oidclr(oi->delta_base_oid, loose->base.odb->repo->hash_algo);
199 if (!ret)
200 oi->whence = OI_LOOSE;
201 }
202
203 return ret;
204 }
205
206 static int odb_source_loose_read_object_info(struct odb_source *source,
207 const struct object_id *oid,
208 struct object_info *oi,
209 enum object_info_flags flags)
210 {
211 struct odb_source_loose *loose = odb_source_loose_downcast(source);
212 static struct strbuf buf = STRBUF_INIT;
213
214 /*
215 * The second read shouldn't cause new loose objects to show up, unless
216 * there was a race condition with a secondary process. We don't care
217 * about this case though, so we simply skip reading loose objects a
218 * second time.
219 */
220 if (flags & OBJECT_INFO_SECOND_READ)
221 return -1;
222
223 odb_loose_path(loose, &buf, oid);
224 return read_object_info_from_path(loose, buf.buf, oid, oi, flags);
225 }
226
227 /*
228 * Find "oid" as a loose object in given source, open the object and return its
229 * file descriptor. Returns the file descriptor on success, negative on failure.
230 *
231 * The "path" out-parameter will give the path of the object we found (if any).
232 * Note that it may point to static storage and is only valid until another
233 * call to open_loose_object().
234 */
235 static int open_loose_object(struct odb_source_loose *loose,
236 const struct object_id *oid, const char **path)
237 {
238 static struct strbuf buf = STRBUF_INIT;
239 int fd;
240
241 *path = odb_loose_path(loose, &buf, oid);
242 fd = git_open(*path);
243 if (fd >= 0)
244 return fd;
245
246 return -1;
247 }
248
249 static void *odb_source_loose_map_object(struct odb_source_loose *loose,
250 const struct object_id *oid,
251 unsigned long *size)
252 {
253 const char *p;
254 int fd = open_loose_object(loose, oid, &p);
255 void *map = NULL;
256 struct stat st;
257
258 if (fd < 0)
259 return NULL;
260
261 if (!fstat(fd, &st)) {
262 *size = xsize_t(st.st_size);
263 if (!*size) {
264 /* mmap() is forbidden on empty files */
265 error(_("object file %s is empty"), p);
266 goto out;
267 }
268
269 map = xmmap(NULL, *size, PROT_READ, MAP_PRIVATE, fd, 0);
270 }
271
272 out:
273 close(fd);
274 return map;
275 }
276
277 struct odb_loose_read_stream {
278 struct odb_read_stream base;
279 git_zstream z;
280 enum {
281 ODB_LOOSE_READ_STREAM_INUSE,
282 ODB_LOOSE_READ_STREAM_DONE,
283 ODB_LOOSE_READ_STREAM_ERROR,
284 } z_state;
285 void *mapped;
286 unsigned long mapsize;
287 char hdr[32];
288 int hdr_avail;
289 int hdr_used;
290 };
291
292 static ssize_t read_istream_loose(struct odb_read_stream *_st, char *buf, size_t sz)
293 {
294 struct odb_loose_read_stream *st =
295 container_of(_st, struct odb_loose_read_stream, base);
296 size_t total_read = 0;
297
298 switch (st->z_state) {
299 case ODB_LOOSE_READ_STREAM_DONE:
300 return 0;
301 case ODB_LOOSE_READ_STREAM_ERROR:
302 return -1;
303 default:
304 break;
305 }
306
307 if (st->hdr_used < st->hdr_avail) {
308 size_t to_copy = st->hdr_avail - st->hdr_used;
309 if (sz < to_copy)
310 to_copy = sz;
311 memcpy(buf, st->hdr + st->hdr_used, to_copy);
312 st->hdr_used += to_copy;
313 total_read += to_copy;
314 }
315
316 while (total_read < sz) {
317 int status;
318
319 st->z.next_out = (unsigned char *)buf + total_read;
320 st->z.avail_out = sz - total_read;
321 status = git_inflate(&st->z, Z_FINISH);
322
323 total_read = st->z.next_out - (unsigned char *)buf;
324
325 if (status == Z_STREAM_END) {
326 git_inflate_end(&st->z);
327 st->z_state = ODB_LOOSE_READ_STREAM_DONE;
328 break;
329 }
330 if (status != Z_OK && (status != Z_BUF_ERROR || total_read < sz)) {
331 git_inflate_end(&st->z);
332 st->z_state = ODB_LOOSE_READ_STREAM_ERROR;
333 return -1;
334 }
335 }
336 return total_read;
337 }
338
339 static int close_istream_loose(struct odb_read_stream *_st)
340 {
341 struct odb_loose_read_stream *st =
342 container_of(_st, struct odb_loose_read_stream, base);
343
344 if (st->z_state == ODB_LOOSE_READ_STREAM_INUSE)
345 git_inflate_end(&st->z);
346 munmap(st->mapped, st->mapsize);
347 return 0;
348 }
349
350 static int odb_source_loose_read_object_stream(struct odb_read_stream **out,
351 struct odb_source *source,
352 const struct object_id *oid)
353 {
354 struct odb_source_loose *loose = odb_source_loose_downcast(source);
355 struct object_info oi = OBJECT_INFO_INIT;
356 struct odb_loose_read_stream *st;
357 unsigned long mapsize;
358 void *mapped;
359
360 mapped = odb_source_loose_map_object(loose, oid, &mapsize);
361 if (!mapped)
362 return -1;
363
364 /*
365 * Note: we must allocate this structure early even though we may still
366 * fail. This is because we need to initialize the zlib stream, and it
367 * is not possible to copy the stream around after the fact because it
368 * has self-referencing pointers.
369 */
370 CALLOC_ARRAY(st, 1);
371
372 switch (unpack_loose_header(&st->z, mapped, mapsize, st->hdr,
373 sizeof(st->hdr))) {
374 case ULHR_OK:
375 break;
376 case ULHR_BAD:
377 case ULHR_TOO_LONG:
378 goto error;
379 }
380
381 oi.sizep = &st->base.size;
382 oi.typep = &st->base.type;
383
384 if (parse_loose_header(st->hdr, &oi) < 0 || st->base.type < 0)
385 goto error;
386
387 st->mapped = mapped;
388 st->mapsize = mapsize;
389 st->hdr_used = strlen(st->hdr) + 1;
390 st->hdr_avail = st->z.total_out;
391 st->z_state = ODB_LOOSE_READ_STREAM_INUSE;
392 st->base.close = close_istream_loose;
393 st->base.read = read_istream_loose;
394
395 *out = &st->base;
396
397 return 0;
398 error:
399 git_inflate_end(&st->z);
400 munmap(mapped, mapsize);
401 free(st);
402 return -1;
403 }
404
405 struct for_each_object_wrapper_data {
406 struct odb_source_loose *loose;
407 const struct object_info *request;
408 odb_for_each_object_cb cb;
409 void *cb_data;
410 };
411
412 static int for_each_object_wrapper_cb(const struct object_id *oid,
413 const char *path,
414 void *cb_data)
415 {
416 struct for_each_object_wrapper_data *data = cb_data;
417
418 if (data->request) {
419 struct object_info oi = *data->request;
420
421 if (read_object_info_from_path(data->loose, path, oid, &oi, 0) < 0)
422 return -1;
423
424 return data->cb(oid, &oi, data->cb_data);
425 } else {
426 return data->cb(oid, NULL, data->cb_data);
427 }
428 }
429
430 static int for_each_prefixed_object_wrapper_cb(const struct object_id *oid,
431 void *node_data UNUSED,
432 void *cb_data)
433 {
434 struct for_each_object_wrapper_data *data = cb_data;
435 if (data->request) {
436 struct object_info oi = *data->request;
437
438 if (odb_source_read_object_info(&data->loose->base,
439 oid, &oi, 0) < 0)
440 return -1;
441
442 return data->cb(oid, &oi, data->cb_data);
443 } else {
444 return data->cb(oid, NULL, data->cb_data);
445 }
446 }
447
448 static int odb_source_loose_for_each_object(struct odb_source *source,
449 const struct object_info *request,
450 odb_for_each_object_cb cb,
451 void *cb_data,
452 const struct odb_for_each_object_options *opts)
453 {
454 struct odb_source_loose *loose = odb_source_loose_downcast(source);
455 struct for_each_object_wrapper_data data = {
456 .loose = loose,
457 .request = request,
458 .cb = cb,
459 .cb_data = cb_data,
460 };
461
462 /* There are no loose promisor objects, so we can return immediately. */
463 if ((opts->flags & ODB_FOR_EACH_OBJECT_PROMISOR_ONLY))
464 return 0;
465 if ((opts->flags & ODB_FOR_EACH_OBJECT_LOCAL_ONLY) && !source->local)
466 return 0;
467
468 if (opts->prefix)
469 return oidtree_each(odb_source_loose_cache(loose, opts->prefix),
470 opts->prefix, opts->prefix_hex_len,
471 for_each_prefixed_object_wrapper_cb, &data);
472
473 return for_each_loose_file_in_source(source, for_each_object_wrapper_cb,
474 NULL, NULL, &data);
475 }
476
477 struct find_abbrev_len_data {
478 const struct object_id *oid;
479 unsigned len;
480 };
481
482 static int find_abbrev_len_cb(const struct object_id *oid,
483 struct object_info *oi UNUSED,
484 void *cb_data)
485 {
486 struct find_abbrev_len_data *data = cb_data;
487 unsigned len = oid_common_prefix_hexlen(oid, data->oid);
488 if (len != hash_algos[oid->algo].hexsz && len >= data->len)
489 data->len = len + 1;
490 return 0;
491 }
492
493 static int odb_source_loose_find_abbrev_len(struct odb_source *source,
494 const struct object_id *oid,
495 unsigned min_len,
496 unsigned *out)
497 {
498 struct odb_source_loose *loose = odb_source_loose_downcast(source);
499 struct odb_for_each_object_options opts = {
500 .prefix = oid,
501 .prefix_hex_len = min_len,
502 };
503 struct find_abbrev_len_data data = {
504 .oid = oid,
505 .len = min_len,
506 };
507 int ret;
508
509 ret = odb_source_for_each_object(&loose->base, NULL, find_abbrev_len_cb,
510 &data, &opts);
511 *out = data.len;
512
513 return ret;
514 }
515
516 static int count_loose_object(const struct object_id *oid UNUSED,
517 struct object_info *oi UNUSED,
518 void *payload)
519 {
520 unsigned long *count = payload;
521 (*count)++;
522 return 0;
523 }
524
525 static int odb_source_loose_count_objects(struct odb_source *source,
526 enum odb_count_objects_flags flags,
527 unsigned long *out)
528 {
529 struct odb_source_loose *loose = odb_source_loose_downcast(source);
530 const unsigned hexsz = source->odb->repo->hash_algo->hexsz - 2;
531 char *path = NULL;
532 DIR *dir = NULL;
533 int ret;
534
535 if (flags & ODB_COUNT_OBJECTS_APPROXIMATE) {
536 unsigned long count = 0;
537 struct dirent *ent;
538
539 path = xstrfmt("%s/17", source->path);
540
541 dir = opendir(path);
542 if (!dir) {
543 if (errno == ENOENT) {
544 *out = 0;
545 ret = 0;
546 goto out;
547 }
548
549 ret = error_errno("cannot open object shard '%s'", path);
550 goto out;
551 }
552
553 while ((ent = readdir(dir)) != NULL) {
554 if (strspn(ent->d_name, "0123456789abcdef") != hexsz ||
555 ent->d_name[hexsz] != '\0')
556 continue;
557 count++;
558 }
559
560 *out = count * 256;
561 ret = 0;
562 } else {
563 struct odb_for_each_object_options opts = { 0 };
564 *out = 0;
565 ret = odb_source_for_each_object(&loose->base, NULL, count_loose_object,
566 out, &opts);
567 }
568
569 out:
570 if (dir)
571 closedir(dir);
572 free(path);
573 return ret;
574 }
575
576 static int odb_source_loose_freshen_object(struct odb_source *source,
577 const struct object_id *oid)
578 {
579 struct odb_source_loose *loose = odb_source_loose_downcast(source);
580 static struct strbuf path = STRBUF_INIT;
581 odb_loose_path(loose, &path, oid);
582 return !!check_and_freshen_file(path.buf, 1);
583 }
584
585 static int odb_source_loose_write_object(struct odb_source *source,
586 const void *buf, unsigned long len,
587 enum object_type type, struct object_id *oid,
588 struct object_id *compat_oid_in,
589 enum odb_write_object_flags flags)
590 {
591 struct odb_source_loose *loose = odb_source_loose_downcast(source);
592 const struct git_hash_algo *algo = source->odb->repo->hash_algo;
593 const struct git_hash_algo *compat = source->odb->repo->compat_hash_algo;
594 struct object_id compat_oid;
595 char hdr[MAX_HEADER_LEN];
596 int hdrlen = sizeof(hdr);
597
598 /* Generate compat_oid */
599 if (compat) {
600 if (compat_oid_in)
601 oidcpy(&compat_oid, compat_oid_in);
602 else if (type == OBJ_BLOB)
603 hash_object_file(compat, buf, len, type, &compat_oid);
604 else {
605 struct strbuf converted = STRBUF_INIT;
606 convert_object_file(source->odb->repo, &converted, algo, compat,
607 buf, len, type, 0);
608 hash_object_file(compat, converted.buf, converted.len,
609 type, &compat_oid);
610 strbuf_release(&converted);
611 }
612 }
613
614 /* Normally if we have it in the pack then we do not bother writing
615 * it out into .git/objects/??/?{38} file.
616 */
617 write_object_file_prepare(algo, buf, len, type, oid, hdr, &hdrlen);
618 if (odb_freshen_object(source->odb, oid))
619 return 0;
620 if (write_loose_object(loose, oid, hdr, hdrlen, buf, len, 0, flags))
621 return -1;
622 if (compat)
623 return repo_add_loose_object_map(loose, oid, &compat_oid);
624 return 0;
625 }
626
627 static int odb_source_loose_write_object_stream(struct odb_source *source,
628 struct odb_write_stream *in_stream,
629 size_t len,
630 struct object_id *oid)
631 {
632 /*
633 * TODO: the implementation should be moved here, see the comment on
634 * the called function in "object-file.h".
635 */
636 struct odb_source_loose *loose = odb_source_loose_downcast(source);
637 return odb_source_loose_write_stream(loose, in_stream, len, oid);
638 }
639
640 static int odb_source_loose_begin_transaction(struct odb_source *source UNUSED,
641 struct odb_transaction **out UNUSED)
642 {
643 /* TODO: this is a known omission that we'll want to address eventually. */
644 return error("loose source does not support transactions");
645 }
646
647 static int odb_source_loose_read_alternates(struct odb_source *source UNUSED,
648 struct strvec *out UNUSED)
649 {
650 return 0;
651 }
652
653 static int odb_source_loose_write_alternate(struct odb_source *source UNUSED,
654 const char *alternate UNUSED)
655 {
656 return error("loose source does not support alternates");
657 }
658
659 static void odb_source_loose_clear_cache(struct odb_source_loose *loose)
660 {
661 oidtree_clear(loose->cache);
662 FREE_AND_NULL(loose->cache);
663 memset(&loose->subdir_seen, 0,
664 sizeof(loose->subdir_seen));
665 }
666
667 static void odb_source_loose_reprepare(struct odb_source *source)
668 {
669 struct odb_source_loose *loose = odb_source_loose_downcast(source);
670 odb_source_loose_clear_cache(loose);
671 }
672
673 static void odb_source_loose_close(struct odb_source *source UNUSED)
674 {
675 /* Nothing to do. */
676 }
677
678 static void odb_source_loose_reparent(const char *name UNUSED,
679 const char *old_cwd,
680 const char *new_cwd,
681 void *cb_data)
682 {
683 struct odb_source_loose *loose = cb_data;
684 char *path = reparent_relative_path(old_cwd, new_cwd,
685 loose->base.path);
686 free(loose->base.path);
687 loose->base.path = path;
688 }
689
690 static void odb_source_loose_free(struct odb_source *source)
691 {
692 struct odb_source_loose *loose = odb_source_loose_downcast(source);
693 odb_source_loose_clear_cache(loose);
694 loose_object_map_clear(&loose->map);
695 chdir_notify_unregister(NULL, odb_source_loose_reparent, loose);
696 odb_source_release(&loose->base);
697 free(loose);
698 }
699
700 struct odb_source_loose *odb_source_loose_new(struct object_database *odb,
701 const char *path,
702 bool local)
703 {
704 struct odb_source_loose *loose;
705
706 CALLOC_ARRAY(loose, 1);
707 odb_source_init(&loose->base, odb, ODB_SOURCE_LOOSE, path, local);
708
709 loose->base.free = odb_source_loose_free;
710 loose->base.close = odb_source_loose_close;
711 loose->base.reprepare = odb_source_loose_reprepare;
712 loose->base.read_object_info = odb_source_loose_read_object_info;
713 loose->base.read_object_stream = odb_source_loose_read_object_stream;
714 loose->base.for_each_object = odb_source_loose_for_each_object;
715 loose->base.find_abbrev_len = odb_source_loose_find_abbrev_len;
716 loose->base.count_objects = odb_source_loose_count_objects;
717 loose->base.freshen_object = odb_source_loose_freshen_object;
718 loose->base.write_object = odb_source_loose_write_object;
719 loose->base.write_object_stream = odb_source_loose_write_object_stream;
720 loose->base.begin_transaction = odb_source_loose_begin_transaction;
721 loose->base.read_alternates = odb_source_loose_read_alternates;
722 loose->base.write_alternate = odb_source_loose_write_alternate;
723
724 if (!is_absolute_path(loose->base.path))
725 chdir_notify_register(NULL, odb_source_loose_reparent, loose);
726
727 return loose;
728 }