@leroysheep / LeeRoySheep
Code Commits Pulls CI/CD 81,348
maint
bisect jch maint master next seen test todo
c 1,696 lines 45.6 KB
Raw
1 /*
2 * GIT - The information manager from hell
3 *
4 * Copyright (C) Linus Torvalds, 2005
5 *
6 * This handles basic git object files - packing, unpacking,
7 * creation etc.
8 */
9
10 #define USE_THE_REPOSITORY_VARIABLE
11
12 #include "git-compat-util.h"
13 #include "convert.h"
14 #include "dir.h"
15 #include "environment.h"
16 #include "fsck.h"
17 #include "gettext.h"
18 #include "hex.h"
19 #include "loose.h"
20 #include "object-file-convert.h"
21 #include "object-file.h"
22 #include "odb.h"
23 #include "odb/streaming.h"
24 #include "odb/transaction.h"
25 #include "pack.h"
26 #include "packfile.h"
27 #include "path.h"
28 #include "read-cache-ll.h"
29 #include "setup.h"
30 #include "tempfile.h"
31 #include "tmp-objdir.h"
32
33 static int get_conv_flags(unsigned flags)
34 {
35 if (flags & INDEX_RENORMALIZE)
36 return CONV_EOL_RENORMALIZE;
37 else if (flags & INDEX_WRITE_OBJECT)
38 return global_conv_flags_eol | CONV_WRITE_OBJECT;
39 else
40 return 0;
41 }
42
43 static void fill_loose_path(struct strbuf *buf,
44 const struct object_id *oid,
45 const struct git_hash_algo *algop)
46 {
47 for (size_t i = 0; i < algop->rawsz; i++) {
48 static char hex[] = "0123456789abcdef";
49 unsigned int val = oid->hash[i];
50 strbuf_addch(buf, hex[val >> 4]);
51 strbuf_addch(buf, hex[val & 0xf]);
52 if (!i)
53 strbuf_addch(buf, '/');
54 }
55 }
56
57 const char *odb_loose_path(struct odb_source_loose *loose,
58 struct strbuf *buf,
59 const struct object_id *oid)
60 {
61 strbuf_reset(buf);
62 strbuf_addstr(buf, loose->base.path);
63 strbuf_addch(buf, '/');
64 fill_loose_path(buf, oid, loose->base.odb->repo->hash_algo);
65 return buf->buf;
66 }
67
68 /* Returns 1 if we have successfully freshened the file, 0 otherwise. */
69 static int freshen_file(const char *fn)
70 {
71 return !utime(fn, NULL);
72 }
73
74 /*
75 * All of the check_and_freshen functions return 1 if the file exists and was
76 * freshened (if freshening was requested), 0 otherwise. If they return
77 * 0, you should not assume that it is safe to skip a write of the object (it
78 * either does not exist on disk, or has a stale mtime and may be subject to
79 * pruning).
80 */
81 int check_and_freshen_file(const char *fn, int freshen)
82 {
83 if (access(fn, F_OK))
84 return 0;
85 if (freshen && !freshen_file(fn))
86 return 0;
87 return 1;
88 }
89
90 int format_object_header(char *str, size_t size, enum object_type type,
91 size_t objsize)
92 {
93 const char *name = type_name(type);
94
95 if (!name)
96 BUG("could not get a type name for 'enum object_type' value %d", type);
97
98 return xsnprintf(str, size, "%s %"PRIuMAX, name, (uintmax_t)objsize) + 1;
99 }
100
101 int check_object_signature(struct repository *r, const struct object_id *oid,
102 void *buf, unsigned long size,
103 enum object_type type)
104 {
105 const struct git_hash_algo *algo =
106 oid->algo ? &hash_algos[oid->algo] : r->hash_algo;
107 struct object_id real_oid;
108
109 hash_object_file(algo, buf, size, type, &real_oid);
110
111 return !oideq(oid, &real_oid) ? -1 : 0;
112 }
113
114 int stream_object_signature(struct repository *r,
115 struct odb_read_stream *st,
116 const struct object_id *oid)
117 {
118 struct object_id real_oid;
119 struct git_hash_ctx c;
120 char hdr[MAX_HEADER_LEN];
121 int hdrlen;
122
123 /* Generate the header */
124 hdrlen = format_object_header(hdr, sizeof(hdr), st->type, st->size);
125
126 /* Sha1.. */
127 r->hash_algo->init_fn(&c);
128 git_hash_update(&c, hdr, hdrlen);
129 for (;;) {
130 char buf[1024 * 16];
131 ssize_t readlen = odb_read_stream_read(st, buf, sizeof(buf));
132
133 if (readlen < 0) {
134 odb_read_stream_close(st);
135 return -1;
136 }
137 if (!readlen)
138 break;
139 git_hash_update(&c, buf, readlen);
140 }
141 git_hash_final_oid(&real_oid, &c);
142 return !oideq(oid, &real_oid) ? -1 : 0;
143 }
144
145 /*
146 * Map and close the given loose object fd. The path argument is used for
147 * error reporting.
148 */
149 static void *map_fd(int fd, const char *path, unsigned long *size)
150 {
151 void *map = NULL;
152 struct stat st;
153
154 if (!fstat(fd, &st)) {
155 *size = xsize_t(st.st_size);
156 if (!*size) {
157 /* mmap() is forbidden on empty files */
158 error(_("object file %s is empty"), path);
159 close(fd);
160 return NULL;
161 }
162 map = xmmap(NULL, *size, PROT_READ, MAP_PRIVATE, fd, 0);
163 }
164 close(fd);
165 return map;
166 }
167
168 enum unpack_loose_header_result unpack_loose_header(git_zstream *stream,
169 unsigned char *map,
170 unsigned long mapsize,
171 void *buffer,
172 unsigned long bufsiz)
173 {
174 int status;
175
176 /* Get the data stream */
177 memset(stream, 0, sizeof(*stream));
178 stream->next_in = map;
179 stream->avail_in = mapsize;
180 stream->next_out = buffer;
181 stream->avail_out = bufsiz;
182
183 git_inflate_init(stream);
184 obj_read_unlock();
185 status = git_inflate(stream, 0);
186 obj_read_lock();
187 if (status != Z_OK && status != Z_STREAM_END)
188 return ULHR_BAD;
189
190 /*
191 * Check if entire header is unpacked in the first iteration.
192 */
193 if (memchr(buffer, '\0', stream->next_out - (unsigned char *)buffer))
194 return ULHR_OK;
195
196 /*
197 * We have a header longer than MAX_HEADER_LEN.
198 */
199 return ULHR_TOO_LONG;
200 }
201
202 void *unpack_loose_rest(git_zstream *stream,
203 void *buffer, unsigned long size,
204 const struct object_id *oid)
205 {
206 size_t bytes = strlen(buffer) + 1, n;
207 unsigned char *buf = xmallocz(size);
208 int status = Z_OK;
209
210 n = stream->total_out - bytes;
211 if (n > size)
212 n = size;
213 memcpy(buf, (char *) buffer + bytes, n);
214 bytes = n;
215 if (bytes <= size) {
216 /*
217 * The above condition must be (bytes <= size), not
218 * (bytes < size). In other words, even though we
219 * expect no more output and set avail_out to zero,
220 * the input zlib stream may have bytes that express
221 * "this concludes the stream", and we *do* want to
222 * eat that input.
223 *
224 * Otherwise we would not be able to test that we
225 * consumed all the input to reach the expected size;
226 * we also want to check that zlib tells us that all
227 * went well with status == Z_STREAM_END at the end.
228 */
229 stream->next_out = buf + bytes;
230 stream->avail_out = size - bytes;
231 while (status == Z_OK) {
232 obj_read_unlock();
233 status = git_inflate(stream, Z_FINISH);
234 obj_read_lock();
235 }
236 }
237
238 if (status != Z_STREAM_END) {
239 error(_("corrupt loose object '%s'"), oid_to_hex(oid));
240 FREE_AND_NULL(buf);
241 } else if (stream->avail_in) {
242 error(_("garbage at end of loose object '%s'"),
243 oid_to_hex(oid));
244 FREE_AND_NULL(buf);
245 }
246
247 return buf;
248 }
249
250 /*
251 * parse_loose_header() parses the starting "<type> <len>\0" of an
252 * object. If it doesn't follow that format -1 is returned. To check
253 * the validity of the <type> populate the "typep" in the "struct
254 * object_info". It will be OBJ_BAD if the object type is unknown. The
255 * parsed <len> can be retrieved via "oi->sizep", and from there
256 * passed to unpack_loose_rest().
257 *
258 * We used to just use "sscanf()", but that's actually way
259 * too permissive for what we want to check. So do an anal
260 * object header parse by hand.
261 */
262 int parse_loose_header(const char *hdr, struct object_info *oi)
263 {
264 const char *type_buf = hdr;
265 size_t size;
266 int type, type_len = 0;
267
268 /*
269 * The type can be of any size but is followed by
270 * a space.
271 */
272 for (;;) {
273 char c = *hdr++;
274 if (!c)
275 return -1;
276 if (c == ' ')
277 break;
278 type_len++;
279 }
280
281 type = type_from_string_gently(type_buf, type_len, 1);
282 if (oi->typep)
283 *oi->typep = type;
284
285 /*
286 * The length must follow immediately, and be in canonical
287 * decimal format (ie "010" is not valid).
288 */
289 size = *hdr++ - '0';
290 if (size > 9)
291 return -1;
292 if (size) {
293 for (;;) {
294 unsigned long c = *hdr - '0';
295 if (c > 9)
296 break;
297 hdr++;
298 size = st_add(st_mult(size, 10), c);
299 }
300 }
301
302 if (oi->sizep)
303 *oi->sizep = size;
304
305 /*
306 * The length must be followed by a zero byte
307 */
308 if (*hdr)
309 return -1;
310
311 /*
312 * The format is valid, but the type may still be bogus. The
313 * Caller needs to check its oi->typep.
314 */
315 return 0;
316 }
317
318 static void hash_object_body(const struct git_hash_algo *algo, struct git_hash_ctx *c,
319 const void *buf, unsigned long len,
320 struct object_id *oid,
321 char *hdr, int *hdrlen)
322 {
323 algo->init_fn(c);
324 git_hash_update(c, hdr, *hdrlen);
325 git_hash_update(c, buf, len);
326 git_hash_final_oid(oid, c);
327 }
328
329 void write_object_file_prepare(const struct git_hash_algo *algo,
330 const void *buf, unsigned long len,
331 enum object_type type, struct object_id *oid,
332 char *hdr, int *hdrlen)
333 {
334 struct git_hash_ctx c;
335
336 /* Generate the header */
337 *hdrlen = format_object_header(hdr, *hdrlen, type, len);
338
339 /* Sha1.. */
340 hash_object_body(algo, &c, buf, len, oid, hdr, hdrlen);
341 }
342
343 #define CHECK_COLLISION_DEST_VANISHED -2
344
345 static int check_collision(const char *source, const char *dest)
346 {
347 char buf_source[4096], buf_dest[4096];
348 int fd_source = -1, fd_dest = -1;
349 int ret = 0;
350
351 fd_source = open(source, O_RDONLY);
352 if (fd_source < 0) {
353 ret = error_errno(_("unable to open %s"), source);
354 goto out;
355 }
356
357 fd_dest = open(dest, O_RDONLY);
358 if (fd_dest < 0) {
359 if (errno != ENOENT)
360 ret = error_errno(_("unable to open %s"), dest);
361 else
362 ret = CHECK_COLLISION_DEST_VANISHED;
363 goto out;
364 }
365
366 while (1) {
367 ssize_t sz_a, sz_b;
368
369 sz_a = read_in_full(fd_source, buf_source, sizeof(buf_source));
370 if (sz_a < 0) {
371 ret = error_errno(_("unable to read %s"), source);
372 goto out;
373 }
374
375 sz_b = read_in_full(fd_dest, buf_dest, sizeof(buf_dest));
376 if (sz_b < 0) {
377 ret = error_errno(_("unable to read %s"), dest);
378 goto out;
379 }
380
381 if (sz_a != sz_b || memcmp(buf_source, buf_dest, sz_a)) {
382 ret = error(_("files '%s' and '%s' differ in contents"),
383 source, dest);
384 goto out;
385 }
386
387 if ((size_t) sz_a < sizeof(buf_source))
388 break;
389 }
390
391 out:
392 if (fd_source > -1)
393 close(fd_source);
394 if (fd_dest > -1)
395 close(fd_dest);
396 return ret;
397 }
398
399 /*
400 * Move the just written object into its final resting place.
401 */
402 int finalize_object_file(struct repository *repo,
403 const char *tmpfile, const char *filename)
404 {
405 return finalize_object_file_flags(repo, tmpfile, filename, 0);
406 }
407
408 int finalize_object_file_flags(struct repository *repo,
409 const char *tmpfile, const char *filename,
410 enum finalize_object_file_flags flags)
411 {
412 unsigned retries = 0;
413 int ret;
414
415 retry:
416 ret = 0;
417
418 if (object_creation_mode == OBJECT_CREATION_USES_RENAMES)
419 goto try_rename;
420 else if (link(tmpfile, filename))
421 ret = errno;
422 else
423 unlink_or_warn(tmpfile);
424
425 /*
426 * Coda hack - coda doesn't like cross-directory links,
427 * so we fall back to a rename, which will mean that it
428 * won't be able to check collisions, but that's not a
429 * big deal.
430 *
431 * The same holds for FAT formatted media.
432 *
433 * When this succeeds, we just return. We have nothing
434 * left to unlink.
435 */
436 if (ret && ret != EEXIST) {
437 struct stat st;
438
439 try_rename:
440 if (!stat(filename, &st))
441 ret = EEXIST;
442 else if (!rename(tmpfile, filename))
443 goto out;
444 else
445 ret = errno;
446 }
447 if (ret) {
448 if (ret != EEXIST) {
449 int saved_errno = errno;
450 unlink_or_warn(tmpfile);
451 errno = saved_errno;
452 return error_errno(_("unable to write file %s"), filename);
453 }
454 if (!(flags & FOF_SKIP_COLLISION_CHECK)) {
455 ret = check_collision(tmpfile, filename);
456 if (ret == CHECK_COLLISION_DEST_VANISHED) {
457 if (retries++ > 5)
458 return error(_("unable to write repeatedly vanishing file %s"),
459 filename);
460 goto retry;
461 }
462 else if (ret)
463 return -1;
464 }
465 unlink_or_warn(tmpfile);
466 }
467
468 out:
469 if (adjust_shared_perm(repo, filename))
470 return error(_("unable to set permission to '%s'"), filename);
471 return 0;
472 }
473
474 void hash_object_file(const struct git_hash_algo *algo, const void *buf,
475 unsigned long len, enum object_type type,
476 struct object_id *oid)
477 {
478 char hdr[MAX_HEADER_LEN];
479 int hdrlen = sizeof(hdr);
480
481 write_object_file_prepare(algo, buf, len, type, oid, hdr, &hdrlen);
482 }
483
484 struct transaction_packfile {
485 char *pack_tmp_name;
486 struct hashfile *f;
487 off_t offset;
488 struct pack_idx_option pack_idx_opts;
489
490 struct pack_idx_entry **written;
491 uint32_t alloc_written;
492 uint32_t nr_written;
493 };
494
495 struct odb_transaction_files {
496 struct odb_transaction base;
497
498 struct tmp_objdir *objdir;
499 struct transaction_packfile packfile;
500 };
501
502 static void prepare_loose_object_transaction(struct odb_transaction *base)
503 {
504 struct odb_transaction_files *transaction =
505 container_of_or_null(base, struct odb_transaction_files, base);
506
507 /*
508 * We lazily create the temporary object directory
509 * the first time an object might be added, since
510 * callers may not know whether any objects will be
511 * added at the time they call odb_transaction_files_begin.
512 */
513 if (!transaction || transaction->objdir)
514 return;
515
516 transaction->objdir = tmp_objdir_create(base->source->odb->repo, "bulk-fsync");
517 if (transaction->objdir)
518 tmp_objdir_replace_primary_odb(transaction->objdir, 0);
519 }
520
521 static void fsync_loose_object_transaction(struct odb_transaction *base,
522 int fd, const char *filename)
523 {
524 struct odb_transaction_files *transaction =
525 container_of_or_null(base, struct odb_transaction_files, base);
526
527 /*
528 * If we have an active ODB transaction, we issue a call that
529 * cleans the filesystem page cache but avoids a hardware flush
530 * command. Later on we will issue a single hardware flush
531 * before renaming the objects to their final names as part of
532 * flush_batch_fsync.
533 */
534 if (!transaction || !transaction->objdir ||
535 git_fsync(fd, FSYNC_WRITEOUT_ONLY) < 0) {
536 if (errno == ENOSYS)
537 warning(_("core.fsyncMethod = batch is unsupported on this platform"));
538 fsync_or_die(fd, filename);
539 }
540 }
541
542 /*
543 * Cleanup after batch-mode fsync_object_files.
544 */
545 static void flush_loose_object_transaction(struct odb_transaction_files *transaction)
546 {
547 struct strbuf temp_path = STRBUF_INIT;
548 struct tempfile *temp;
549
550 if (!transaction->objdir)
551 return;
552
553 /*
554 * Issue a full hardware flush against a temporary file to ensure
555 * that all objects are durable before any renames occur. The code in
556 * fsync_loose_object_transaction has already issued a writeout
557 * request, but it has not flushed any writeback cache in the storage
558 * hardware or any filesystem logs. This fsync call acts as a barrier
559 * to ensure that the data in each new object file is durable before
560 * the final name is visible.
561 */
562 strbuf_addf(&temp_path, "%s/bulk_fsync_XXXXXX",
563 repo_get_object_directory(transaction->base.source->odb->repo));
564 temp = xmks_tempfile(temp_path.buf);
565 fsync_or_die(get_tempfile_fd(temp), get_tempfile_path(temp));
566 delete_tempfile(&temp);
567 strbuf_release(&temp_path);
568
569 /*
570 * Make the object files visible in the primary ODB after their data is
571 * fully durable.
572 */
573 tmp_objdir_migrate(transaction->objdir);
574 transaction->objdir = NULL;
575 }
576
577 /* Finalize a file on disk, and close it. */
578 static void close_loose_object(struct odb_source_loose *loose,
579 int fd, const char *filename)
580 {
581 if (loose->base.will_destroy)
582 goto out;
583
584 if (batch_fsync_enabled(FSYNC_COMPONENT_LOOSE_OBJECT))
585 fsync_loose_object_transaction(loose->base.odb->transaction, fd, filename);
586 else if (fsync_object_files > 0)
587 fsync_or_die(fd, filename);
588 else
589 fsync_component_or_die(FSYNC_COMPONENT_LOOSE_OBJECT, fd,
590 filename);
591
592 out:
593 if (close(fd) != 0)
594 die_errno(_("error when closing loose object file"));
595 }
596
597 /* Size of directory component, including the ending '/' */
598 static inline int directory_size(const char *filename)
599 {
600 const char *s = strrchr(filename, '/');
601 if (!s)
602 return 0;
603 return s - filename + 1;
604 }
605
606 /*
607 * This creates a temporary file in the same directory as the final
608 * 'filename'
609 *
610 * We want to avoid cross-directory filename renames, because those
611 * can have problems on various filesystems (FAT, NFS, Coda).
612 */
613 static int create_tmpfile(struct repository *repo,
614 struct strbuf *tmp, const char *filename)
615 {
616 int fd, dirlen = directory_size(filename);
617
618 strbuf_reset(tmp);
619 strbuf_add(tmp, filename, dirlen);
620 strbuf_addstr(tmp, "tmp_obj_XXXXXX");
621 fd = git_mkstemp_mode(tmp->buf, 0444);
622 if (fd < 0 && dirlen && errno == ENOENT) {
623 /*
624 * Make sure the directory exists; note that the contents
625 * of the buffer are undefined after mkstemp returns an
626 * error, so we have to rewrite the whole buffer from
627 * scratch.
628 */
629 strbuf_reset(tmp);
630 strbuf_add(tmp, filename, dirlen - 1);
631 if (mkdir(tmp->buf, 0777) && errno != EEXIST)
632 return -1;
633 if (adjust_shared_perm(repo, tmp->buf))
634 return -1;
635
636 /* Try again */
637 strbuf_addstr(tmp, "/tmp_obj_XXXXXX");
638 fd = git_mkstemp_mode(tmp->buf, 0444);
639 }
640 return fd;
641 }
642
643 /**
644 * Common steps for loose object writers to start writing loose
645 * objects:
646 *
647 * - Create tmpfile for the loose object.
648 * - Setup zlib stream for compression.
649 * - Start to feed header to zlib stream.
650 *
651 * Returns a "fd", which should later be provided to
652 * end_loose_object_common().
653 */
654 static int start_loose_object_common(struct odb_source_loose *loose,
655 struct strbuf *tmp_file,
656 const char *filename, unsigned flags,
657 git_zstream *stream,
658 unsigned char *buf, size_t buflen,
659 struct git_hash_ctx *c, struct git_hash_ctx *compat_c,
660 char *hdr, int hdrlen)
661 {
662 const struct git_hash_algo *algo = loose->base.odb->repo->hash_algo;
663 const struct git_hash_algo *compat = loose->base.odb->repo->compat_hash_algo;
664 int fd;
665 struct repo_config_values *cfg = repo_config_values(the_repository);
666
667 fd = create_tmpfile(loose->base.odb->repo, tmp_file, filename);
668 if (fd < 0) {
669 if (flags & ODB_WRITE_OBJECT_SILENT)
670 return -1;
671 else if (errno == EACCES)
672 return error(_("insufficient permission for adding "
673 "an object to repository database %s"),
674 loose->base.path);
675 else
676 return error_errno(
677 _("unable to create temporary file"));
678 }
679
680 /* Setup zlib stream for compression */
681 git_deflate_init(stream, cfg->zlib_compression_level);
682 stream->next_out = buf;
683 stream->avail_out = buflen;
684 algo->init_fn(c);
685 if (compat && compat_c)
686 compat->init_fn(compat_c);
687
688 /* Start to feed header to zlib stream */
689 stream->next_in = (unsigned char *)hdr;
690 stream->avail_in = hdrlen;
691 while (git_deflate(stream, 0) == Z_OK)
692 ; /* nothing */
693 git_hash_update(c, hdr, hdrlen);
694 if (compat && compat_c)
695 git_hash_update(compat_c, hdr, hdrlen);
696
697 return fd;
698 }
699
700 /**
701 * Common steps for the inner git_deflate() loop for writing loose
702 * objects. Returns what git_deflate() returns.
703 */
704 static int write_loose_object_common(struct odb_source_loose *loose,
705 struct git_hash_ctx *c, struct git_hash_ctx *compat_c,
706 git_zstream *stream, const int flush,
707 unsigned char *in0, const int fd,
708 unsigned char *compressed,
709 const size_t compressed_len)
710 {
711 const struct git_hash_algo *compat = loose->base.odb->repo->compat_hash_algo;
712 int ret;
713
714 ret = git_deflate(stream, flush ? Z_FINISH : 0);
715 git_hash_update(c, in0, stream->next_in - in0);
716 if (compat && compat_c)
717 git_hash_update(compat_c, in0, stream->next_in - in0);
718 if (write_in_full(fd, compressed, stream->next_out - compressed) < 0)
719 die_errno(_("unable to write loose object file"));
720 stream->next_out = compressed;
721 stream->avail_out = compressed_len;
722
723 return ret;
724 }
725
726 /**
727 * Common steps for loose object writers to end writing loose objects:
728 *
729 * - End the compression of zlib stream.
730 * - Get the calculated oid to "oid".
731 */
732 static int end_loose_object_common(struct odb_source_loose *loose,
733 struct git_hash_ctx *c, struct git_hash_ctx *compat_c,
734 git_zstream *stream, struct object_id *oid,
735 struct object_id *compat_oid)
736 {
737 const struct git_hash_algo *compat = loose->base.odb->repo->compat_hash_algo;
738 int ret;
739
740 ret = git_deflate_end_gently(stream);
741 if (ret != Z_OK)
742 return ret;
743 git_hash_final_oid(oid, c);
744 if (compat && compat_c)
745 git_hash_final_oid(compat_oid, compat_c);
746
747 return Z_OK;
748 }
749
750 int write_loose_object(struct odb_source_loose *loose,
751 const struct object_id *oid, char *hdr,
752 int hdrlen, const void *buf, unsigned long len,
753 time_t mtime, unsigned flags)
754 {
755 int fd, ret;
756 unsigned char compressed[4096];
757 git_zstream stream;
758 struct git_hash_ctx c;
759 struct object_id parano_oid;
760 static struct strbuf tmp_file = STRBUF_INIT;
761 static struct strbuf filename = STRBUF_INIT;
762
763 if (batch_fsync_enabled(FSYNC_COMPONENT_LOOSE_OBJECT))
764 prepare_loose_object_transaction(loose->base.odb->transaction);
765
766 odb_loose_path(loose, &filename, oid);
767
768 fd = start_loose_object_common(loose, &tmp_file, filename.buf, flags,
769 &stream, compressed, sizeof(compressed),
770 &c, NULL, hdr, hdrlen);
771 if (fd < 0)
772 return -1;
773
774 /* Then the data itself.. */
775 stream.next_in = (void *)buf;
776 stream.avail_in = len;
777 do {
778 unsigned char *in0 = stream.next_in;
779
780 ret = write_loose_object_common(loose, &c, NULL, &stream, 1, in0, fd,
781 compressed, sizeof(compressed));
782 } while (ret == Z_OK);
783
784 if (ret != Z_STREAM_END)
785 die(_("unable to deflate new object %s (%d)"), oid_to_hex(oid),
786 ret);
787 ret = end_loose_object_common(loose, &c, NULL, &stream, &parano_oid, NULL);
788 if (ret != Z_OK)
789 die(_("deflateEnd on object %s failed (%d)"), oid_to_hex(oid),
790 ret);
791 if (!oideq(oid, &parano_oid))
792 die(_("confused by unstable object source data for %s"),
793 oid_to_hex(oid));
794
795 close_loose_object(loose, fd, tmp_file.buf);
796
797 if (mtime) {
798 struct utimbuf utb;
799 utb.actime = mtime;
800 utb.modtime = mtime;
801 if (utime(tmp_file.buf, &utb) < 0 &&
802 !(flags & ODB_WRITE_OBJECT_SILENT))
803 warning_errno(_("failed utime() on %s"), tmp_file.buf);
804 }
805
806 return finalize_object_file_flags(loose->base.odb->repo, tmp_file.buf, filename.buf,
807 FOF_SKIP_COLLISION_CHECK);
808 }
809
810 int odb_source_loose_write_stream(struct odb_source_loose *loose,
811 struct odb_write_stream *in_stream, size_t len,
812 struct object_id *oid)
813 {
814 const struct git_hash_algo *compat = loose->base.odb->repo->compat_hash_algo;
815 struct object_id compat_oid;
816 int fd, ret, err = 0, flush = 0;
817 unsigned char compressed[4096];
818 git_zstream stream;
819 struct git_hash_ctx c, compat_c;
820 struct strbuf tmp_file = STRBUF_INIT;
821 struct strbuf filename = STRBUF_INIT;
822 unsigned char buf[8192];
823 int dirlen;
824 char hdr[MAX_HEADER_LEN];
825 int hdrlen;
826
827 if (batch_fsync_enabled(FSYNC_COMPONENT_LOOSE_OBJECT))
828 prepare_loose_object_transaction(loose->base.odb->transaction);
829
830 /* Since oid is not determined, save tmp file to odb path. */
831 strbuf_addf(&filename, "%s/", loose->base.path);
832 hdrlen = format_object_header(hdr, sizeof(hdr), OBJ_BLOB, len);
833
834 /*
835 * Common steps for write_loose_object and stream_loose_object to
836 * start writing loose objects:
837 *
838 * - Create tmpfile for the loose object.
839 * - Setup zlib stream for compression.
840 * - Start to feed header to zlib stream.
841 */
842 fd = start_loose_object_common(loose, &tmp_file, filename.buf, 0,
843 &stream, compressed, sizeof(compressed),
844 &c, &compat_c, hdr, hdrlen);
845 if (fd < 0) {
846 err = -1;
847 goto cleanup;
848 }
849
850 /* Then the data itself.. */
851 do {
852 unsigned char *in0 = stream.next_in;
853
854 if (!stream.avail_in && !in_stream->is_finished) {
855 ssize_t read_len = odb_write_stream_read(in_stream, buf,
856 sizeof(buf));
857 if (read_len < 0) {
858 close(fd);
859 err = -1;
860 goto cleanup;
861 }
862
863 stream.avail_in = read_len;
864 stream.next_in = buf;
865 in0 = buf;
866 /* All data has been read. */
867 if (in_stream->is_finished)
868 flush = 1;
869 }
870 ret = write_loose_object_common(loose, &c, &compat_c, &stream, flush, in0, fd,
871 compressed, sizeof(compressed));
872 /*
873 * Unlike write_loose_object(), we do not have the entire
874 * buffer. If we get Z_BUF_ERROR due to too few input bytes,
875 * then we'll replenish them in the next input_stream->read()
876 * call when we loop.
877 */
878 } while (ret == Z_OK || ret == Z_BUF_ERROR);
879
880 if (stream.total_in != len + hdrlen)
881 die(_("write stream object %"PRIuMAX" != %"PRIuMAX), (uintmax_t)stream.total_in,
882 (uintmax_t)len + hdrlen);
883
884 /*
885 * Common steps for write_loose_object and stream_loose_object to
886 * end writing loose object:
887 *
888 * - End the compression of zlib stream.
889 * - Get the calculated oid.
890 */
891 if (ret != Z_STREAM_END)
892 die(_("unable to stream deflate new object (%d)"), ret);
893 ret = end_loose_object_common(loose, &c, &compat_c, &stream, oid, &compat_oid);
894 if (ret != Z_OK)
895 die(_("deflateEnd on stream object failed (%d)"), ret);
896 close_loose_object(loose, fd, tmp_file.buf);
897
898 if (odb_freshen_object(loose->base.odb, oid)) {
899 unlink_or_warn(tmp_file.buf);
900 goto cleanup;
901 }
902 odb_loose_path(loose, &filename, oid);
903
904 /* We finally know the object path, and create the missing dir. */
905 dirlen = directory_size(filename.buf);
906 if (dirlen) {
907 struct strbuf dir = STRBUF_INIT;
908 strbuf_add(&dir, filename.buf, dirlen);
909
910 if (safe_create_dir_in_gitdir(loose->base.odb->repo, dir.buf) &&
911 errno != EEXIST) {
912 err = error_errno(_("unable to create directory %s"), dir.buf);
913 strbuf_release(&dir);
914 goto cleanup;
915 }
916 strbuf_release(&dir);
917 }
918
919 err = finalize_object_file_flags(loose->base.odb->repo, tmp_file.buf, filename.buf,
920 FOF_SKIP_COLLISION_CHECK);
921 if (!err && compat)
922 err = repo_add_loose_object_map(loose, oid, &compat_oid);
923 cleanup:
924 strbuf_release(&tmp_file);
925 strbuf_release(&filename);
926 return err;
927 }
928
929 int force_object_loose(struct odb_source *source,
930 const struct object_id *oid, time_t mtime)
931 {
932 struct odb_source_files *files = odb_source_files_downcast(source);
933 const struct git_hash_algo *compat = source->odb->repo->compat_hash_algo;
934 void *buf;
935 size_t len;
936 struct object_info oi = OBJECT_INFO_INIT;
937 struct object_id compat_oid;
938 enum object_type type;
939 char hdr[MAX_HEADER_LEN];
940 int hdrlen;
941 int ret;
942
943 for (struct odb_source *s = source->odb->sources; s; s = s->next) {
944 struct odb_source_files *files = odb_source_files_downcast(s);
945 if (!odb_source_read_object_info(&files->loose->base, oid, NULL, 0))
946 return 0;
947 }
948
949 oi.typep = &type;
950 oi.sizep = &len;
951 oi.contentp = &buf;
952 if (odb_read_object_info_extended(source->odb, oid, &oi, 0))
953 return error(_("cannot read object for %s"), oid_to_hex(oid));
954 if (compat) {
955 if (repo_oid_to_algop(source->odb->repo, oid, compat, &compat_oid))
956 return error(_("cannot map object %s to %s"),
957 oid_to_hex(oid), compat->name);
958 }
959 hdrlen = format_object_header(hdr, sizeof(hdr), type, len);
960 ret = write_loose_object(files->loose, oid, hdr, hdrlen, buf, len, mtime, 0);
961 if (!ret && compat)
962 ret = repo_add_loose_object_map(files->loose, oid, &compat_oid);
963 free(buf);
964
965 return ret;
966 }
967
968 /*
969 * We can't use the normal fsck_error_function() for index_mem(),
970 * because we don't yet have a valid oid for it to report. Instead,
971 * report the minimal fsck error here, and rely on the caller to
972 * give more context.
973 */
974 static int hash_format_check_report(struct fsck_options *opts UNUSED,
975 void *fsck_report UNUSED,
976 enum fsck_msg_type msg_type UNUSED,
977 enum fsck_msg_id msg_id UNUSED,
978 const char *message)
979 {
980 error(_("object fails fsck: %s"), message);
981 return 1;
982 }
983
984 static int index_mem(struct index_state *istate,
985 struct object_id *oid,
986 const void *buf, size_t size,
987 enum object_type type,
988 const char *path, unsigned flags)
989 {
990 struct strbuf nbuf = STRBUF_INIT;
991 int ret = 0;
992 int write_object = flags & INDEX_WRITE_OBJECT;
993
994 if (!type)
995 type = OBJ_BLOB;
996
997 /*
998 * Convert blobs to git internal format
999 */
1000 if ((type == OBJ_BLOB) && path) {
1001 if (convert_to_git(istate, path, buf, size, &nbuf,
1002 get_conv_flags(flags))) {
1003 buf = nbuf.buf;
1004 size = nbuf.len;
1005 }
1006 }
1007 if (flags & INDEX_FORMAT_CHECK) {
1008 struct fsck_options opts;
1009
1010 fsck_options_init(&opts, the_repository, FSCK_OPTIONS_DEFAULT);
1011 opts.strict = 1;
1012 opts.error_func = hash_format_check_report;
1013 if (fsck_buffer(null_oid(istate->repo->hash_algo), type, buf, size, &opts))
1014 die(_("refusing to create malformed object"));
1015 fsck_finish(&opts);
1016 }
1017
1018 if (write_object)
1019 ret = odb_write_object(istate->repo->objects, buf, size, type, oid);
1020 else
1021 hash_object_file(istate->repo->hash_algo, buf, size, type, oid);
1022
1023 strbuf_release(&nbuf);
1024 return ret;
1025 }
1026
1027 static int index_stream_convert_blob(struct index_state *istate,
1028 struct object_id *oid,
1029 int fd,
1030 const char *path,
1031 unsigned flags)
1032 {
1033 int ret = 0;
1034 const int write_object = flags & INDEX_WRITE_OBJECT;
1035 struct strbuf sbuf = STRBUF_INIT;
1036
1037 assert(path);
1038 ASSERT(would_convert_to_git_filter_fd(istate, path));
1039
1040 convert_to_git_filter_fd(istate, path, fd, &sbuf,
1041 get_conv_flags(flags));
1042
1043 if (write_object)
1044 ret = odb_write_object(istate->repo->objects, sbuf.buf, sbuf.len, OBJ_BLOB,
1045 oid);
1046 else
1047 hash_object_file(istate->repo->hash_algo, sbuf.buf, sbuf.len, OBJ_BLOB,
1048 oid);
1049 strbuf_release(&sbuf);
1050 return ret;
1051 }
1052
1053 static int index_pipe(struct index_state *istate, struct object_id *oid,
1054 int fd, enum object_type type,
1055 const char *path, unsigned flags)
1056 {
1057 struct strbuf sbuf = STRBUF_INIT;
1058 int ret;
1059
1060 if (strbuf_read(&sbuf, fd, 4096) >= 0)
1061 ret = index_mem(istate, oid, sbuf.buf, sbuf.len, type, path, flags);
1062 else
1063 ret = -1;
1064 strbuf_release(&sbuf);
1065 return ret;
1066 }
1067
1068 #define SMALL_FILE_SIZE (32*1024)
1069
1070 static int index_core(struct index_state *istate,
1071 struct object_id *oid, int fd, size_t size,
1072 enum object_type type, const char *path,
1073 unsigned flags)
1074 {
1075 int ret;
1076
1077 if (!size) {
1078 ret = index_mem(istate, oid, "", size, type, path, flags);
1079 } else if (size <= SMALL_FILE_SIZE) {
1080 char *buf = xmalloc(size);
1081 ssize_t read_result = read_in_full(fd, buf, size);
1082 if (read_result < 0)
1083 ret = error_errno(_("read error while indexing %s"),
1084 path ? path : "<unknown>");
1085 else if ((size_t) read_result != size)
1086 ret = error(_("short read while indexing %s"),
1087 path ? path : "<unknown>");
1088 else
1089 ret = index_mem(istate, oid, buf, size, type, path, flags);
1090 free(buf);
1091 } else {
1092 void *buf = xmmap(NULL, size, PROT_READ, MAP_PRIVATE, fd, 0);
1093 ret = index_mem(istate, oid, buf, size, type, path, flags);
1094 munmap(buf, size);
1095 }
1096 return ret;
1097 }
1098
1099 static int already_written(struct odb_transaction_files *transaction,
1100 struct object_id *oid)
1101 {
1102 /* The object may already exist in the repository */
1103 if (odb_has_object(transaction->base.source->odb, oid,
1104 ODB_HAS_OBJECT_RECHECK_PACKED | ODB_HAS_OBJECT_FETCH_PROMISOR))
1105 return 1;
1106
1107 /* Might want to keep the list sorted */
1108 for (uint32_t i = 0; i < transaction->packfile.nr_written; i++)
1109 if (oideq(&transaction->packfile.written[i]->oid, oid))
1110 return 1;
1111
1112 /* This is a new object we need to keep */
1113 return 0;
1114 }
1115
1116 /* Lazily create backing packfile for the state */
1117 static void prepare_packfile_transaction(struct odb_transaction_files *transaction)
1118 {
1119 struct transaction_packfile *state = &transaction->packfile;
1120 if (state->f)
1121 return;
1122
1123 state->f = create_tmp_packfile(transaction->base.source->odb->repo,
1124 &state->pack_tmp_name);
1125 reset_pack_idx_option(&state->pack_idx_opts);
1126
1127 /* Pretend we are going to write only one object */
1128 state->offset = write_pack_header(state->f, 1);
1129 if (!state->offset)
1130 die_errno("unable to write pack header");
1131 }
1132
1133 static int hash_blob_stream(struct odb_write_stream *stream,
1134 const struct git_hash_algo *hash_algo,
1135 struct object_id *result_oid, size_t size)
1136 {
1137 unsigned char buf[16384];
1138 struct git_hash_ctx ctx;
1139 unsigned header_len;
1140 size_t bytes_hashed = 0;
1141
1142 header_len = format_object_header((char *)buf, sizeof(buf),
1143 OBJ_BLOB, size);
1144 hash_algo->init_fn(&ctx);
1145 git_hash_update(&ctx, buf, header_len);
1146
1147 while (!stream->is_finished) {
1148 ssize_t read_result = odb_write_stream_read(stream, buf,
1149 sizeof(buf));
1150
1151 if (read_result < 0)
1152 return -1;
1153
1154 git_hash_update(&ctx, buf, read_result);
1155 bytes_hashed += read_result;
1156 }
1157
1158 if (bytes_hashed != size)
1159 return -1;
1160
1161 git_hash_final_oid(result_oid, &ctx);
1162
1163 return 0;
1164 }
1165
1166 /*
1167 * Read the contents from the stream provided, streaming it to the
1168 * packfile in state while updating the hash in ctx.
1169 */
1170 static void stream_blob_to_pack(struct transaction_packfile *state,
1171 struct git_hash_ctx *ctx, size_t size,
1172 struct odb_write_stream *stream)
1173 {
1174 git_zstream s;
1175 unsigned char ibuf[16384];
1176 unsigned char obuf[16384];
1177 unsigned hdrlen;
1178 int status = Z_OK;
1179 struct repo_config_values *cfg = repo_config_values(the_repository);
1180 size_t bytes_read = 0;
1181
1182 git_deflate_init(&s, cfg->pack_compression_level);
1183
1184 hdrlen = encode_in_pack_object_header(obuf, sizeof(obuf), OBJ_BLOB, size);
1185 s.next_out = obuf + hdrlen;
1186 s.avail_out = sizeof(obuf) - hdrlen;
1187
1188 while (status != Z_STREAM_END) {
1189 if (!stream->is_finished && !s.avail_in) {
1190 ssize_t rsize = odb_write_stream_read(stream, ibuf,
1191 sizeof(ibuf));
1192
1193 if (rsize < 0)
1194 die("failed to read blob data");
1195
1196 git_hash_update(ctx, ibuf, rsize);
1197
1198 s.next_in = ibuf;
1199 s.avail_in = rsize;
1200 bytes_read += rsize;
1201 }
1202
1203 status = git_deflate(&s, stream->is_finished ? Z_FINISH : 0);
1204
1205 if (!s.avail_out || status == Z_STREAM_END) {
1206 size_t written = s.next_out - obuf;
1207
1208 hashwrite(state->f, obuf, written);
1209 state->offset += written;
1210 s.next_out = obuf;
1211 s.avail_out = sizeof(obuf);
1212 }
1213
1214 switch (status) {
1215 case Z_OK:
1216 case Z_BUF_ERROR:
1217 case Z_STREAM_END:
1218 continue;
1219 default:
1220 die("unexpected deflate failure: %d", status);
1221 }
1222 }
1223
1224 if (bytes_read != size)
1225 die("read %" PRIuMAX " bytes of blob data, but expected %" PRIuMAX " bytes",
1226 (uintmax_t)bytes_read, (uintmax_t)size);
1227
1228 git_deflate_end(&s);
1229 }
1230
1231 static void flush_packfile_transaction(struct odb_transaction_files *transaction)
1232 {
1233 struct transaction_packfile *state = &transaction->packfile;
1234 struct repository *repo = transaction->base.source->odb->repo;
1235 unsigned char hash[GIT_MAX_RAWSZ];
1236 struct strbuf packname = STRBUF_INIT;
1237 char *idx_tmp_name = NULL;
1238
1239 if (!state->f)
1240 return;
1241
1242 if (state->nr_written == 0) {
1243 close(state->f->fd);
1244 free_hashfile(state->f);
1245 unlink(state->pack_tmp_name);
1246 goto clear_exit;
1247 } else if (state->nr_written == 1) {
1248 finalize_hashfile(state->f, hash, FSYNC_COMPONENT_PACK,
1249 CSUM_HASH_IN_STREAM | CSUM_FSYNC | CSUM_CLOSE);
1250 } else {
1251 int fd = finalize_hashfile(state->f, hash, FSYNC_COMPONENT_PACK, 0);
1252 fixup_pack_header_footer(repo->hash_algo, fd, hash, state->pack_tmp_name,
1253 state->nr_written, hash,
1254 state->offset);
1255 close(fd);
1256 }
1257
1258 strbuf_addf(&packname, "%s/pack/pack-%s.",
1259 repo_get_object_directory(transaction->base.source->odb->repo),
1260 hash_to_hex_algop(hash, repo->hash_algo));
1261
1262 stage_tmp_packfiles(repo, &packname, state->pack_tmp_name,
1263 state->written, state->nr_written, NULL,
1264 &state->pack_idx_opts, hash, &idx_tmp_name);
1265 rename_tmp_packfile_idx(repo, &packname, &idx_tmp_name);
1266
1267 for (uint32_t i = 0; i < state->nr_written; i++)
1268 free(state->written[i]);
1269
1270 clear_exit:
1271 free(idx_tmp_name);
1272 free(state->pack_tmp_name);
1273 free(state->written);
1274 memset(state, 0, sizeof(*state));
1275
1276 strbuf_release(&packname);
1277 /* Make objects we just wrote available to ourselves */
1278 odb_reprepare(repo->objects);
1279 }
1280
1281 /*
1282 * This writes the specified object to a packfile. Objects written here
1283 * during the same transaction are written to the same packfile. The
1284 * packfile is not flushed until the transaction is flushed. The caller
1285 * is expected to ensure a valid transaction is setup for objects to be
1286 * recorded to.
1287 *
1288 * This also bypasses the usual "convert-to-git" dance, and that is on
1289 * purpose. We could write a streaming version of the converting
1290 * functions and insert that before feeding the data to fast-import
1291 * (or equivalent in-core API described above). However, that is
1292 * somewhat complicated, as we do not know the size of the filter
1293 * result, which we need to know beforehand when writing a git object.
1294 * Since the primary motivation for trying to stream from the working
1295 * tree file and to avoid mmaping it in core is to deal with large
1296 * binary blobs, they generally do not want to get any conversion, and
1297 * callers should avoid this code path when filters are requested.
1298 */
1299 static int odb_transaction_files_write_object_stream(struct odb_transaction *base,
1300 struct odb_write_stream *stream,
1301 size_t size,
1302 struct object_id *result_oid)
1303 {
1304 struct odb_transaction_files *transaction = container_of(base,
1305 struct odb_transaction_files,
1306 base);
1307 struct transaction_packfile *state = &transaction->packfile;
1308 struct git_hash_ctx ctx;
1309 unsigned char obuf[16384];
1310 unsigned header_len;
1311 struct hashfile_checkpoint checkpoint;
1312 struct pack_idx_entry *idx;
1313
1314 header_len = format_object_header((char *)obuf, sizeof(obuf),
1315 OBJ_BLOB, size);
1316 transaction->base.source->odb->repo->hash_algo->init_fn(&ctx);
1317 git_hash_update(&ctx, obuf, header_len);
1318
1319 /*
1320 * If writing another object to the packfile could result in it
1321 * exceeding the configured size limit, flush the current packfile
1322 * transaction.
1323 *
1324 * Note that this uses the inflated object size as an approximation.
1325 * Blob objects written in this manner are not delta-compressed, so
1326 * the difference between the inflated and on-disk size is limited
1327 * to zlib compression and is sufficient for this check.
1328 */
1329 if (state->nr_written && pack_size_limit_cfg &&
1330 pack_size_limit_cfg < state->offset + size)
1331 flush_packfile_transaction(transaction);
1332
1333 CALLOC_ARRAY(idx, 1);
1334 prepare_packfile_transaction(transaction);
1335 hashfile_checkpoint_init(state->f, &checkpoint);
1336
1337 hashfile_checkpoint(state->f, &checkpoint);
1338 idx->offset = state->offset;
1339 crc32_begin(state->f);
1340 stream_blob_to_pack(state, &ctx, size, stream);
1341 git_hash_final_oid(result_oid, &ctx);
1342
1343 idx->crc32 = crc32_end(state->f);
1344 if (already_written(transaction, result_oid)) {
1345 hashfile_truncate(state->f, &checkpoint);
1346 state->offset = checkpoint.offset;
1347 free(idx);
1348 } else {
1349 oidcpy(&idx->oid, result_oid);
1350 ALLOC_GROW(state->written,
1351 state->nr_written + 1,
1352 state->alloc_written);
1353 state->written[state->nr_written++] = idx;
1354 }
1355 return 0;
1356 }
1357
1358 int index_fd(struct index_state *istate, struct object_id *oid,
1359 int fd, struct stat *st,
1360 enum object_type type, const char *path, unsigned flags)
1361 {
1362 int ret;
1363
1364 /*
1365 * Call xsize_t() only when needed to avoid potentially unnecessary
1366 * die() for large files.
1367 */
1368 if (type == OBJ_BLOB && path && would_convert_to_git_filter_fd(istate, path)) {
1369 ret = index_stream_convert_blob(istate, oid, fd, path, flags);
1370 } else if (!S_ISREG(st->st_mode)) {
1371 ret = index_pipe(istate, oid, fd, type, path, flags);
1372 } else if ((st->st_size >= 0 &&
1373 (size_t)st->st_size <= repo_settings_get_big_file_threshold(istate->repo)) ||
1374 type != OBJ_BLOB ||
1375 (path && would_convert_to_git(istate, path))) {
1376 ret = index_core(istate, oid, fd, xsize_t(st->st_size),
1377 type, path, flags);
1378 } else {
1379 struct odb_write_stream stream;
1380 odb_write_stream_from_fd(&stream, fd, xsize_t(st->st_size));
1381
1382 if (flags & INDEX_WRITE_OBJECT) {
1383 struct object_database *odb = the_repository->objects;
1384 struct odb_transaction *transaction = odb_transaction_begin(odb);
1385
1386 ret = odb_transaction_write_object_stream(odb->transaction,
1387 &stream,
1388 xsize_t(st->st_size),
1389 oid);
1390 odb_transaction_commit(transaction);
1391 } else {
1392 ret = hash_blob_stream(&stream,
1393 the_repository->hash_algo, oid,
1394 xsize_t(st->st_size));
1395 }
1396
1397 odb_write_stream_release(&stream);
1398 }
1399
1400 close(fd);
1401 return ret;
1402 }
1403
1404 int index_path(struct index_state *istate, struct object_id *oid,
1405 const char *path, struct stat *st, unsigned flags)
1406 {
1407 int fd;
1408 struct strbuf sb = STRBUF_INIT;
1409 int rc = 0;
1410
1411 switch (st->st_mode & S_IFMT) {
1412 case S_IFREG:
1413 fd = open(path, O_RDONLY);
1414 if (fd < 0)
1415 return error_errno("open(\"%s\")", path);
1416 if (index_fd(istate, oid, fd, st, OBJ_BLOB, path, flags) < 0)
1417 return error(_("%s: failed to insert into database"),
1418 path);
1419 break;
1420 case S_IFLNK:
1421 if (strbuf_readlink(&sb, path, st->st_size))
1422 return error_errno("readlink(\"%s\")", path);
1423 if (!(flags & INDEX_WRITE_OBJECT))
1424 hash_object_file(istate->repo->hash_algo, sb.buf, sb.len,
1425 OBJ_BLOB, oid);
1426 else if (odb_write_object(istate->repo->objects, sb.buf, sb.len, OBJ_BLOB, oid))
1427 rc = error(_("%s: failed to insert into database"), path);
1428 strbuf_release(&sb);
1429 break;
1430 case S_IFDIR:
1431 if (repo_resolve_gitlink_ref(istate->repo, path, "HEAD", oid))
1432 return error(_("'%s' does not have a commit checked out"), path);
1433 if (&hash_algos[oid->algo] != istate->repo->hash_algo)
1434 return error(_("cannot add a submodule of a different hash algorithm"));
1435 break;
1436 default:
1437 return error(_("%s: unsupported file type"), path);
1438 }
1439 return rc;
1440 }
1441
1442 int read_pack_header(int fd, struct pack_header *header)
1443 {
1444 if (read_in_full(fd, header, sizeof(*header)) != sizeof(*header))
1445 /* "eof before pack header was fully read" */
1446 return PH_ERROR_EOF;
1447
1448 if (header->hdr_signature != htonl(PACK_SIGNATURE))
1449 /* "protocol error (pack signature mismatch detected)" */
1450 return PH_ERROR_PACK_SIGNATURE;
1451 if (!pack_version_ok(header->hdr_version))
1452 /* "protocol error (pack version unsupported)" */
1453 return PH_ERROR_PROTOCOL;
1454 return 0;
1455 }
1456
1457 int for_each_file_in_obj_subdir(unsigned int subdir_nr,
1458 struct strbuf *path,
1459 const struct git_hash_algo *algop,
1460 each_loose_object_fn obj_cb,
1461 each_loose_cruft_fn cruft_cb,
1462 each_loose_subdir_fn subdir_cb,
1463 void *data)
1464 {
1465 size_t origlen, baselen;
1466 DIR *dir;
1467 struct dirent *de;
1468 int r = 0;
1469 struct object_id oid;
1470
1471 if (subdir_nr > 0xff)
1472 BUG("invalid loose object subdirectory: %x", subdir_nr);
1473
1474 origlen = path->len;
1475 strbuf_complete(path, '/');
1476 strbuf_addf(path, "%02x", subdir_nr);
1477
1478 dir = opendir(path->buf);
1479 if (!dir) {
1480 if (errno != ENOENT)
1481 r = error_errno(_("unable to open %s"), path->buf);
1482 strbuf_setlen(path, origlen);
1483 return r;
1484 }
1485
1486 oid.hash[0] = subdir_nr;
1487 strbuf_addch(path, '/');
1488 baselen = path->len;
1489
1490 while ((de = readdir_skip_dot_and_dotdot(dir))) {
1491 size_t namelen;
1492
1493 namelen = strlen(de->d_name);
1494 strbuf_setlen(path, baselen);
1495 strbuf_add(path, de->d_name, namelen);
1496 if (namelen == algop->hexsz - 2 &&
1497 !hex_to_bytes(oid.hash + 1, de->d_name,
1498 algop->rawsz - 1)) {
1499 oid_set_algo(&oid, algop);
1500 memset(oid.hash + algop->rawsz, 0,
1501 GIT_MAX_RAWSZ - algop->rawsz);
1502 if (obj_cb) {
1503 r = obj_cb(&oid, path->buf, data);
1504 if (r)
1505 break;
1506 }
1507 continue;
1508 }
1509
1510 if (cruft_cb) {
1511 r = cruft_cb(de->d_name, path->buf, data);
1512 if (r)
1513 break;
1514 }
1515 }
1516 closedir(dir);
1517
1518 strbuf_setlen(path, baselen - 1);
1519 if (!r && subdir_cb)
1520 r = subdir_cb(subdir_nr, path->buf, data);
1521
1522 strbuf_setlen(path, origlen);
1523
1524 return r;
1525 }
1526
1527 int for_each_loose_file_in_source(struct odb_source *source,
1528 each_loose_object_fn obj_cb,
1529 each_loose_cruft_fn cruft_cb,
1530 each_loose_subdir_fn subdir_cb,
1531 void *data)
1532 {
1533 struct strbuf buf = STRBUF_INIT;
1534 int r;
1535
1536 strbuf_addstr(&buf, source->path);
1537 for (int i = 0; i < 256; i++) {
1538 r = for_each_file_in_obj_subdir(i, &buf, source->odb->repo->hash_algo,
1539 obj_cb, cruft_cb, subdir_cb, data);
1540 if (r)
1541 break;
1542 }
1543
1544 strbuf_release(&buf);
1545 return r;
1546 }
1547
1548 static int check_stream_oid(git_zstream *stream,
1549 const char *hdr,
1550 unsigned long size,
1551 const char *path,
1552 const struct object_id *expected_oid,
1553 const struct git_hash_algo *algop)
1554 {
1555 struct git_hash_ctx c;
1556 struct object_id real_oid;
1557 unsigned char buf[4096];
1558 unsigned long total_read;
1559 int status = Z_OK;
1560
1561 algop->init_fn(&c);
1562 git_hash_update(&c, hdr, stream->total_out);
1563
1564 /*
1565 * We already read some bytes into hdr, but the ones up to the NUL
1566 * do not count against the object's content size.
1567 */
1568 total_read = stream->total_out - strlen(hdr) - 1;
1569
1570 /*
1571 * This size comparison must be "<=" to read the final zlib packets;
1572 * see the comment in unpack_loose_rest for details.
1573 */
1574 while (total_read <= size &&
1575 (status == Z_OK ||
1576 (status == Z_BUF_ERROR && !stream->avail_out))) {
1577 stream->next_out = buf;
1578 stream->avail_out = sizeof(buf);
1579 if (size - total_read < stream->avail_out)
1580 stream->avail_out = size - total_read;
1581 status = git_inflate(stream, Z_FINISH);
1582 git_hash_update(&c, buf, stream->next_out - buf);
1583 total_read += stream->next_out - buf;
1584 }
1585
1586 if (status != Z_STREAM_END) {
1587 error(_("corrupt loose object '%s'"), oid_to_hex(expected_oid));
1588 return -1;
1589 }
1590 if (stream->avail_in) {
1591 error(_("garbage at end of loose object '%s'"),
1592 oid_to_hex(expected_oid));
1593 return -1;
1594 }
1595
1596 git_hash_final_oid(&real_oid, &c);
1597 if (!oideq(expected_oid, &real_oid)) {
1598 error(_("hash mismatch for %s (expected %s)"), path,
1599 oid_to_hex(expected_oid));
1600 return -1;
1601 }
1602
1603 return 0;
1604 }
1605
1606 int read_loose_object(struct repository *repo,
1607 const char *path,
1608 const struct object_id *expected_oid,
1609 struct object_id *real_oid,
1610 void **contents,
1611 struct object_info *oi)
1612 {
1613 int ret = -1;
1614 int fd;
1615 void *map = NULL;
1616 unsigned long mapsize;
1617 git_zstream stream;
1618 char hdr[MAX_HEADER_LEN];
1619 size_t *size = oi->sizep;
1620
1621 fd = git_open(path);
1622 if (fd >= 0)
1623 map = map_fd(fd, path, &mapsize);
1624 if (!map) {
1625 error_errno(_("unable to mmap %s"), path);
1626 goto out;
1627 }
1628
1629 if (unpack_loose_header(&stream, map, mapsize, hdr, sizeof(hdr)) != ULHR_OK) {
1630 error(_("unable to unpack header of %s"), path);
1631 goto out_inflate;
1632 }
1633
1634 if (parse_loose_header(hdr, oi) < 0) {
1635 error(_("unable to parse header of %s"), path);
1636 goto out_inflate;
1637 }
1638
1639 if (*oi->typep < 0) {
1640 error(_("unable to parse type from header '%s' of %s"),
1641 hdr, path);
1642 goto out_inflate;
1643 }
1644
1645 if (*oi->typep == OBJ_BLOB &&
1646 *size > repo_settings_get_big_file_threshold(repo)) {
1647 if (check_stream_oid(&stream, hdr, *size, path, expected_oid,
1648 repo->hash_algo) < 0)
1649 goto out_inflate;
1650 } else {
1651 *contents = unpack_loose_rest(&stream, hdr, *size, expected_oid);
1652 if (!*contents) {
1653 error(_("unable to unpack contents of %s"), path);
1654 goto out_inflate;
1655 }
1656 hash_object_file(repo->hash_algo,
1657 *contents, *size,
1658 *oi->typep, real_oid);
1659 if (!oideq(expected_oid, real_oid))
1660 goto out_inflate;
1661 }
1662
1663 ret = 0; /* everything checks out */
1664
1665 out_inflate:
1666 git_inflate_end(&stream);
1667 out:
1668 if (map)
1669 munmap(map, mapsize);
1670 return ret;
1671 }
1672
1673 static void odb_transaction_files_commit(struct odb_transaction *base)
1674 {
1675 struct odb_transaction_files *transaction =
1676 container_of(base, struct odb_transaction_files, base);
1677
1678 flush_loose_object_transaction(transaction);
1679 flush_packfile_transaction(transaction);
1680 }
1681
1682 struct odb_transaction *odb_transaction_files_begin(struct odb_source *source)
1683 {
1684 struct odb_transaction_files *transaction;
1685 struct object_database *odb = source->odb;
1686
1687 if (odb->transaction)
1688 return NULL;
1689
1690 transaction = xcalloc(1, sizeof(*transaction));
1691 transaction->base.source = source;
1692 transaction->base.commit = odb_transaction_files_commit;
1693 transaction->base.write_object_stream = odb_transaction_files_write_object_stream;
1694
1695 return &transaction->base;
1696 }