Raw
1 #define USE_THE_REPOSITORY_VARIABLE
2
3 #include "git-compat-util.h"
4 #include "odb.h"
5 #include "odb/streaming.h"
6 #include "dir.h"
7 #include "environment.h"
8 #include "gettext.h"
9 #include "hex.h"
10 #include "name-hash.h"
11 #include "sparse-index.h"
12 #include "submodule.h"
13 #include "symlinks.h"
14 #include "progress.h"
15 #include "fsmonitor.h"
16 #include "entry.h"
17 #include "parallel-checkout.h"
18
19 static void create_directories(const char *path, int path_len,
20 const struct checkout *state)
21 {
22 char *buf = xmallocz(path_len);
23 int len = 0;
24
25 while (len < path_len) {
26 do {
27 buf[len] = path[len];
28 len++;
29 } while (len < path_len && path[len] != '/');
30 if (len >= path_len)
31 break;
32 buf[len] = 0;
33
34 /*
35 * For 'checkout-index --prefix=<dir>', <dir> is
36 * allowed to be a symlink to an existing directory,
37 * and we set 'state->base_dir_len' below, such that
38 * we test the path components of the prefix with the
39 * stat() function instead of the lstat() function.
40 */
41 if (has_dirs_only_path(buf, len, state->base_dir_len))
42 continue; /* ok, it is already a directory. */
43
44 /*
45 * If this mkdir() would fail, it could be that there
46 * is already a symlink or something else exists
47 * there, therefore we then try to unlink it and try
48 * one more time to create the directory.
49 */
50 if (mkdir(buf, 0777)) {
51 if (errno == EEXIST && state->force &&
52 !unlink_or_warn(buf) && !mkdir(buf, 0777))
53 continue;
54 die_errno("cannot create directory at '%s'", buf);
55 }
56 }
57 free(buf);
58 }
59
60 static void remove_subtree(struct strbuf *path)
61 {
62 DIR *dir = opendir(path->buf);
63 struct dirent *de;
64 int origlen = path->len;
65
66 if (!dir)
67 die_errno("cannot opendir '%s'", path->buf);
68 while ((de = readdir_skip_dot_and_dotdot(dir)) != NULL) {
69 struct stat st;
70
71 strbuf_addch(path, '/');
72 strbuf_addstr(path, de->d_name);
73 if (lstat(path->buf, &st))
74 die_errno("cannot lstat '%s'", path->buf);
75 if (S_ISDIR(st.st_mode))
76 remove_subtree(path);
77 else if (unlink(path->buf))
78 die_errno("cannot unlink '%s'", path->buf);
79 strbuf_setlen(path, origlen);
80 }
81 closedir(dir);
82 if (rmdir(path->buf))
83 die_errno("cannot rmdir '%s'", path->buf);
84 }
85
86 static int create_file(const char *path, unsigned int mode)
87 {
88 mode = (mode & 0100) ? 0777 : 0666;
89 return open(path, O_WRONLY | O_CREAT | O_EXCL, mode);
90 }
91
92 void *read_blob_entry(const struct cache_entry *ce, size_t *size)
93 {
94 enum object_type type;
95 void *blob_data = odb_read_object(the_repository->objects, &ce->oid,
96 &type, size);
97
98 if (blob_data) {
99 if (type == OBJ_BLOB)
100 return blob_data;
101 free(blob_data);
102 }
103 return NULL;
104 }
105
106 static int open_output_fd(char *path, const struct cache_entry *ce, int to_tempfile)
107 {
108 int symlink = (ce->ce_mode & S_IFMT) != S_IFREG;
109 if (to_tempfile) {
110 xsnprintf(path, TEMPORARY_FILENAME_LENGTH, "%s",
111 symlink ? ".merge_link_XXXXXX" : ".merge_file_XXXXXX");
112 return mkstemp(path);
113 } else {
114 return create_file(path, !symlink ? ce->ce_mode : 0666);
115 }
116 }
117
118 int fstat_checkout_output(int fd, const struct checkout *state, struct stat *st)
119 {
120 /* use fstat() only when path == ce->name */
121 if (fstat_is_reliable() &&
122 state->refresh_cache && !state->base_dir_len) {
123 return !fstat(fd, st);
124 }
125 return 0;
126 }
127
128 static int streaming_write_entry(const struct cache_entry *ce, char *path,
129 struct stream_filter *filter,
130 const struct checkout *state, int to_tempfile,
131 int *fstat_done, struct stat *statbuf)
132 {
133 int result = 0;
134 int fd;
135
136 fd = open_output_fd(path, ce, to_tempfile);
137 if (fd < 0)
138 return -1;
139
140 result |= odb_stream_blob_to_fd(the_repository->objects, fd, &ce->oid, filter, 1);
141 *fstat_done = fstat_checkout_output(fd, state, statbuf);
142 result |= close(fd);
143
144 if (result)
145 unlink(path);
146 return result;
147 }
148
149 void enable_delayed_checkout(struct checkout *state)
150 {
151 if (!state->delayed_checkout) {
152 state->delayed_checkout = xmalloc(sizeof(*state->delayed_checkout));
153 state->delayed_checkout->state = CE_CAN_DELAY;
154 string_list_init_nodup(&state->delayed_checkout->filters);
155 string_list_init_nodup(&state->delayed_checkout->paths);
156 }
157 }
158
159 static int remove_available_paths(struct string_list_item *item, void *cb_data)
160 {
161 struct string_list *available_paths = cb_data;
162 struct string_list_item *available;
163
164 available = string_list_lookup(available_paths, item->string);
165 if (available)
166 available->util = item->util;
167 return !available;
168 }
169
170 static int string_is_not_null(struct string_list_item *item, void *data UNUSED)
171 {
172 return !!item->string;
173 }
174
175 int finish_delayed_checkout(struct checkout *state, int show_progress)
176 {
177 int errs = 0;
178 unsigned processed_paths = 0;
179 off_t filtered_bytes = 0;
180 struct string_list_item *filter, *path;
181 struct progress *progress = NULL;
182 struct delayed_checkout *dco = state->delayed_checkout;
183
184 if (!state->delayed_checkout)
185 return errs;
186
187 dco->state = CE_RETRY;
188 if (show_progress)
189 progress = start_delayed_progress(the_repository,
190 _("Filtering content"),
191 dco->paths.nr);
192 while (dco->filters.nr > 0) {
193 for_each_string_list_item(filter, &dco->filters) {
194 struct string_list available_paths = STRING_LIST_INIT_DUP;
195
196 if (!async_query_available_blobs(filter->string, &available_paths)) {
197 /* Filter reported an error */
198 errs = 1;
199 filter->string = NULL;
200 continue;
201 }
202 if (available_paths.nr <= 0) {
203 /*
204 * Filter responded with no entries. That means
205 * the filter is done and we can remove the
206 * filter from the list (see
207 * "string_list_remove_empty_items" call below).
208 */
209 filter->string = NULL;
210 continue;
211 }
212
213 /*
214 * In dco->paths we store a list of all delayed paths.
215 * The filter just send us a list of available paths.
216 * Remove them from the list.
217 */
218 filter_string_list(&dco->paths, 0,
219 &remove_available_paths, &available_paths);
220
221 for_each_string_list_item(path, &available_paths) {
222 struct cache_entry* ce;
223
224 if (!path->util) {
225 error("external filter '%s' signaled that '%s' "
226 "is now available although it has not been "
227 "delayed earlier",
228 filter->string, path->string);
229 errs |= 1;
230
231 /*
232 * Do not ask the filter for available blobs,
233 * again, as the filter is likely buggy.
234 */
235 filter->string = NULL;
236 continue;
237 }
238 ce = index_file_exists(state->istate, path->string,
239 strlen(path->string), 0);
240 if (ce) {
241 display_progress(progress, ++processed_paths);
242 errs |= checkout_entry(ce, state, NULL, path->util);
243 filtered_bytes += ce->ce_stat_data.sd_size;
244 display_throughput(progress, filtered_bytes);
245 } else
246 errs = 1;
247 }
248
249 string_list_clear(&available_paths, 0);
250 }
251
252 filter_string_list(&dco->filters, 0, string_is_not_null, NULL);
253 }
254 stop_progress(&progress);
255 string_list_clear(&dco->filters, 0);
256
257 /* At this point we should not have any delayed paths anymore. */
258 errs |= dco->paths.nr;
259 for_each_string_list_item(path, &dco->paths) {
260 error("'%s' was not filtered properly", path->string);
261 }
262 string_list_clear(&dco->paths, 0);
263
264 free(dco);
265 state->delayed_checkout = NULL;
266
267 return errs;
268 }
269
270 void update_ce_after_write(const struct checkout *state, struct cache_entry *ce,
271 struct stat *st)
272 {
273 if (state->refresh_cache) {
274 assert(state->istate);
275 fill_stat_cache_info(state->istate, ce, st);
276 ce->ce_flags |= CE_UPDATE_IN_BASE;
277 mark_fsmonitor_invalid(state->istate, ce);
278 state->istate->cache_changed |= CE_ENTRY_CHANGED;
279 }
280 }
281
282 /* Note: ca is used (and required) iff the entry refers to a regular file. */
283 static int write_entry(struct cache_entry *ce, char *path, struct conv_attrs *ca,
284 const struct checkout *state, int to_tempfile,
285 int *nr_checkouts)
286 {
287 unsigned int ce_mode_s_ifmt = ce->ce_mode & S_IFMT;
288 struct delayed_checkout *dco = state->delayed_checkout;
289 int fd, ret, fstat_done = 0;
290 char *new_blob;
291 struct strbuf buf = STRBUF_INIT;
292 size_t size;
293 ssize_t wrote;
294 size_t newsize = 0;
295 struct stat st;
296 const struct submodule *sub;
297 struct checkout_metadata meta;
298 static int scratch_nr_checkouts;
299
300 clone_checkout_metadata(&meta, &state->meta, &ce->oid);
301
302 if (ce_mode_s_ifmt == S_IFREG) {
303 struct stream_filter *filter = get_stream_filter_ca(ca, &ce->oid);
304 if (filter &&
305 !streaming_write_entry(ce, path, filter,
306 state, to_tempfile,
307 &fstat_done, &st))
308 goto finish;
309 }
310
311 switch (ce_mode_s_ifmt) {
312 case S_IFLNK:
313 new_blob = read_blob_entry(ce, &size);
314 if (!new_blob)
315 return error("unable to read sha1 file of %s (%s)",
316 ce->name, oid_to_hex(&ce->oid));
317
318 /*
319 * We can't make a real symlink; write out a regular file entry
320 * with the symlink destination as its contents.
321 */
322 if (!has_symlinks || to_tempfile)
323 goto write_file_entry;
324
325 ret = symlink(new_blob, path);
326 free(new_blob);
327 if (ret)
328 return error_errno("unable to create symlink %s", path);
329 break;
330
331 case S_IFREG:
332 /*
333 * We do not send the blob in case of a retry, so do not
334 * bother reading it at all.
335 */
336 if (dco && dco->state == CE_RETRY) {
337 new_blob = NULL;
338 size = 0;
339 } else {
340 new_blob = read_blob_entry(ce, &size);
341 if (!new_blob)
342 return error("unable to read sha1 file of %s (%s)",
343 ce->name, oid_to_hex(&ce->oid));
344 }
345
346 /*
347 * Convert from git internal format to working tree format
348 */
349 if (dco && dco->state != CE_NO_DELAY) {
350 ret = async_convert_to_working_tree_ca(ca, ce->name,
351 new_blob, size,
352 &buf, &meta, dco);
353 if (ret) {
354 struct string_list_item *item =
355 string_list_lookup(&dco->paths, ce->name);
356 if (item) {
357 item->util = nr_checkouts ? nr_checkouts
358 : &scratch_nr_checkouts;
359 free(new_blob);
360 goto delayed;
361 }
362 }
363 } else {
364 ret = convert_to_working_tree_ca(ca, ce->name, new_blob,
365 size, &buf, &meta);
366 }
367
368 if (ret) {
369 free(new_blob);
370 new_blob = strbuf_detach(&buf, &newsize);
371 size = newsize;
372 }
373 /*
374 * No "else" here as errors from convert are OK at this
375 * point. If the error would have been fatal (e.g.
376 * filter is required), then we would have died already.
377 */
378
379 write_file_entry:
380 fd = open_output_fd(path, ce, to_tempfile);
381 if (fd < 0) {
382 free(new_blob);
383 return error_errno("unable to create file %s", path);
384 }
385
386 wrote = write_in_full(fd, new_blob, size);
387 if (!to_tempfile)
388 fstat_done = fstat_checkout_output(fd, state, &st);
389 close(fd);
390 free(new_blob);
391 if (wrote < 0)
392 return error("unable to write file %s", path);
393 break;
394
395 case S_IFGITLINK:
396 if (to_tempfile)
397 return error("cannot create temporary submodule %s", ce->name);
398 if (mkdir(path, 0777) < 0)
399 return error("cannot create submodule directory %s", path);
400 sub = submodule_from_ce(ce);
401 if (sub)
402 return submodule_move_head(ce->name, state->super_prefix,
403 NULL, oid_to_hex(&ce->oid),
404 state->force ? SUBMODULE_MOVE_HEAD_FORCE : 0);
405 break;
406
407 default:
408 return error("unknown file mode for %s in index", ce->name);
409 }
410
411 finish:
412 if (state->refresh_cache) {
413 if (!fstat_done && lstat(ce->name, &st) < 0)
414 return error_errno("unable to stat just-written file %s",
415 ce->name);
416 update_ce_after_write(state, ce , &st);
417 }
418 if (nr_checkouts)
419 (*nr_checkouts)++;
420 delayed:
421 return 0;
422 }
423
424 /*
425 * This is like 'lstat()', except it refuses to follow symlinks
426 * in the path, after skipping "skiplen".
427 */
428 static int check_path(const char *path, int len, struct stat *st, int skiplen)
429 {
430 const char *slash = path + len;
431
432 while (path < slash && *slash != '/')
433 slash--;
434 if (!has_dirs_only_path(path, slash - path, skiplen)) {
435 errno = ENOENT;
436 return -1;
437 }
438 return lstat(path, st);
439 }
440
441 static void mark_colliding_entries(const struct checkout *state,
442 struct cache_entry *ce, struct stat *st)
443 {
444 struct repo_config_values *cfg = repo_config_values(the_repository);
445 int trust_ino = cfg->check_stat;
446
447 #if defined(GIT_WINDOWS_NATIVE) || defined(__CYGWIN__)
448 trust_ino = 0;
449 #endif
450
451 ce->ce_flags |= CE_MATCHED;
452
453 /* TODO: audit for interaction with sparse-index. */
454 ensure_full_index(state->istate);
455 for (size_t i = 0; i < state->istate->cache_nr; i++) {
456 struct cache_entry *dup = state->istate->cache[i];
457
458 if (dup == ce) {
459 /*
460 * Parallel checkout doesn't create the files in index
461 * order. So the other side of the collision may appear
462 * after the given cache_entry in the array.
463 */
464 if (parallel_checkout_status() == PC_RUNNING)
465 continue;
466 else
467 break;
468 }
469
470 if (dup->ce_flags & (CE_MATCHED | CE_VALID | CE_SKIP_WORKTREE))
471 continue;
472
473 if ((trust_ino && !match_stat_data(&dup->ce_stat_data, st)) ||
474 paths_collide(ce->name, dup->name)) {
475 dup->ce_flags |= CE_MATCHED;
476 break;
477 }
478 }
479 }
480
481 int checkout_entry_ca(struct cache_entry *ce, struct conv_attrs *ca,
482 const struct checkout *state, char *topath,
483 int *nr_checkouts)
484 {
485 static struct strbuf path = STRBUF_INIT;
486 struct stat st;
487 struct conv_attrs ca_buf;
488
489 if (ce->ce_flags & CE_WT_REMOVE) {
490 if (topath)
491 /*
492 * No content and thus no path to create, so we have
493 * no pathname to return.
494 */
495 BUG("Can't remove entry to a path");
496 unlink_entry(ce, state->super_prefix);
497 return 0;
498 }
499
500 if (topath) {
501 if (S_ISREG(ce->ce_mode) && !ca) {
502 convert_attrs(state->istate, &ca_buf, ce->name);
503 ca = &ca_buf;
504 }
505 return write_entry(ce, topath, ca, state, 1, nr_checkouts);
506 }
507
508 strbuf_reset(&path);
509 strbuf_add(&path, state->base_dir, state->base_dir_len);
510 strbuf_add(&path, ce->name, ce_namelen(ce));
511
512 if (!check_path(path.buf, path.len, &st, state->base_dir_len)) {
513 const struct submodule *sub;
514 unsigned changed = ie_match_stat(state->istate, ce, &st,
515 CE_MATCH_IGNORE_VALID | CE_MATCH_IGNORE_SKIP_WORKTREE);
516 /*
517 * Needs to be checked before !changed returns early,
518 * as the possibly empty directory was not changed
519 */
520 sub = submodule_from_ce(ce);
521 if (sub) {
522 int err;
523 if (!is_submodule_populated_gently(ce->name, &err)) {
524 struct stat sb;
525 if (lstat(ce->name, &sb))
526 die(_("could not stat file '%s'"), ce->name);
527 if (!(st.st_mode & S_IFDIR))
528 unlink_or_warn(ce->name);
529
530 return submodule_move_head(ce->name, state->super_prefix,
531 NULL, oid_to_hex(&ce->oid), 0);
532 } else
533 return submodule_move_head(ce->name, state->super_prefix,
534 "HEAD", oid_to_hex(&ce->oid),
535 state->force ? SUBMODULE_MOVE_HEAD_FORCE : 0);
536 }
537
538 if (!changed)
539 return 0;
540 if (!state->force) {
541 if (!state->quiet)
542 fprintf(stderr,
543 "%s already exists, no checkout\n",
544 path.buf);
545 return -1;
546 }
547
548 if (state->clone)
549 mark_colliding_entries(state, ce, &st);
550
551 /*
552 * We unlink the old file, to get the new one with the
553 * right permissions (including umask, which is nasty
554 * to emulate by hand - much easier to let the system
555 * just do the right thing)
556 */
557 if (S_ISDIR(st.st_mode)) {
558 /* If it is a gitlink, leave it alone! */
559 if (S_ISGITLINK(ce->ce_mode))
560 return 0;
561 /*
562 * We must avoid replacing submodules' leading
563 * directories with symbolic links, lest recursive
564 * clones can write into arbitrary locations.
565 *
566 * Technically, this logic is not limited
567 * to recursive clones, or for that matter to
568 * submodules' paths colliding with symbolic links'
569 * paths. Yet it strikes a balance in favor of
570 * simplicity, and if paths are colliding, we might
571 * just as well keep the directories during a clone.
572 */
573 if (state->clone && S_ISLNK(ce->ce_mode))
574 return 0;
575 remove_subtree(&path);
576 } else if (unlink(path.buf))
577 return error_errno("unable to unlink old '%s'", path.buf);
578 } else if (state->not_new)
579 return 0;
580
581 create_directories(path.buf, path.len, state);
582
583 if (S_ISREG(ce->ce_mode) && !ca) {
584 convert_attrs(state->istate, &ca_buf, ce->name);
585 ca = &ca_buf;
586 }
587
588 if (!enqueue_checkout(ce, ca, nr_checkouts))
589 return 0;
590
591 return write_entry(ce, path.buf, ca, state, 0, nr_checkouts);
592 }
593
594 void unlink_entry(const struct cache_entry *ce, const char *super_prefix)
595 {
596 const struct submodule *sub = submodule_from_ce(ce);
597 if (sub) {
598 /* state.force is set at the caller. */
599 submodule_move_head(ce->name, super_prefix, "HEAD", NULL,
600 SUBMODULE_MOVE_HEAD_FORCE);
601 }
602 if (check_leading_path(ce->name, ce_namelen(ce), 1) >= 0)
603 return;
604 if (remove_or_warn(ce->ce_mode, ce->name))
605 return;
606 schedule_dir_for_removal(ce->name, ce_namelen(ce));
607 }
608
609 int remove_or_warn(unsigned int mode, const char *file)
610 {
611 return S_ISGITLINK(mode) ? rmdir_or_warn(file) : unlink_or_warn(file);
612 }