Raw
1 #define USE_THE_REPOSITORY_VARIABLE
2
3 #include "git-compat-util.h"
4 #include "odb.h"
5 #include "odb/streaming.h"
6 #include "dir.h"
7 #include "environment.h"
8 #include "gettext.h"
9 #include "hex.h"
10 #include "name-hash.h"
11 #include "sparse-index.h"
12 #include "submodule.h"
13 #include "symlinks.h"
14 #include "progress.h"
15 #include "fsmonitor.h"
16 #include "entry.h"
17 #include "parallel-checkout.h"
18
19 static void create_directories(const char *path, int path_len,
20 const struct checkout *state)
21 {
22 char *buf = xmallocz(path_len);
23 int len = 0;
24
25 while (len < path_len) {
26 do {
27 buf[len] = path[len];
28 len++;
29 } while (len < path_len && path[len] != '/');
30 if (len >= path_len)
31 break;
32 buf[len] = 0;
33
34 /*
35 * For 'checkout-index --prefix=<dir>', <dir> is
36 * allowed to be a symlink to an existing directory,
37 * and we set 'state->base_dir_len' below, such that
38 * we test the path components of the prefix with the
39 * stat() function instead of the lstat() function.
40 */
41 if (has_dirs_only_path(buf, len, state->base_dir_len))
42 continue; /* ok, it is already a directory. */
43
44 /*
45 * If this mkdir() would fail, it could be that there
46 * is already a symlink or something else exists
47 * there, therefore we then try to unlink it and try
48 * one more time to create the directory.
49 */
50 if (mkdir(buf, 0777)) {
51 if (errno == EEXIST && state->force &&
52 !unlink_or_warn(buf) && !mkdir(buf, 0777))
53 continue;
54 die_errno("cannot create directory at '%s'", buf);
55 }
56 }
57 free(buf);
58 }
59
60 static void remove_subtree(struct strbuf *path)
61 {
62 DIR *dir = opendir(path->buf);
63 struct dirent *de;
64 int origlen = path->len;
65
66 if (!dir)
67 die_errno("cannot opendir '%s'", path->buf);
68 while ((de = readdir_skip_dot_and_dotdot(dir)) != NULL) {
69 struct stat st;
70
71 strbuf_addch(path, '/');
72 strbuf_addstr(path, de->d_name);
73 if (lstat(path->buf, &st))
74 die_errno("cannot lstat '%s'", path->buf);
75 if (S_ISDIR(st.st_mode))
76 remove_subtree(path);
77 else if (unlink(path->buf))
78 die_errno("cannot unlink '%s'", path->buf);
79 strbuf_setlen(path, origlen);
80 }
81 closedir(dir);
82 if (rmdir(path->buf))
83 die_errno("cannot rmdir '%s'", path->buf);
84 }
85
86 static int create_file(const char *path, unsigned int mode)
87 {
88 mode = (mode & 0100) ? 0777 : 0666;
89 return open(path, O_WRONLY | O_CREAT | O_EXCL, mode);
90 }
91
92 void *read_blob_entry(const struct cache_entry *ce, size_t *size)
93 {
94 enum object_type type;
95 unsigned long ul;
96 void *blob_data = odb_read_object(the_repository->objects, &ce->oid,
97 &type, &ul);
98
99 *size = ul;
100 if (blob_data) {
101 if (type == OBJ_BLOB)
102 return blob_data;
103 free(blob_data);
104 }
105 return NULL;
106 }
107
108 static int open_output_fd(char *path, const struct cache_entry *ce, int to_tempfile)
109 {
110 int symlink = (ce->ce_mode & S_IFMT) != S_IFREG;
111 if (to_tempfile) {
112 xsnprintf(path, TEMPORARY_FILENAME_LENGTH, "%s",
113 symlink ? ".merge_link_XXXXXX" : ".merge_file_XXXXXX");
114 return mkstemp(path);
115 } else {
116 return create_file(path, !symlink ? ce->ce_mode : 0666);
117 }
118 }
119
120 int fstat_checkout_output(int fd, const struct checkout *state, struct stat *st)
121 {
122 /* use fstat() only when path == ce->name */
123 if (fstat_is_reliable() &&
124 state->refresh_cache && !state->base_dir_len) {
125 return !fstat(fd, st);
126 }
127 return 0;
128 }
129
130 static int streaming_write_entry(const struct cache_entry *ce, char *path,
131 struct stream_filter *filter,
132 const struct checkout *state, int to_tempfile,
133 int *fstat_done, struct stat *statbuf)
134 {
135 int result = 0;
136 int fd;
137
138 fd = open_output_fd(path, ce, to_tempfile);
139 if (fd < 0)
140 return -1;
141
142 result |= odb_stream_blob_to_fd(the_repository->objects, fd, &ce->oid, filter, 1);
143 *fstat_done = fstat_checkout_output(fd, state, statbuf);
144 result |= close(fd);
145
146 if (result)
147 unlink(path);
148 return result;
149 }
150
151 void enable_delayed_checkout(struct checkout *state)
152 {
153 if (!state->delayed_checkout) {
154 state->delayed_checkout = xmalloc(sizeof(*state->delayed_checkout));
155 state->delayed_checkout->state = CE_CAN_DELAY;
156 string_list_init_nodup(&state->delayed_checkout->filters);
157 string_list_init_nodup(&state->delayed_checkout->paths);
158 }
159 }
160
161 static int remove_available_paths(struct string_list_item *item, void *cb_data)
162 {
163 struct string_list *available_paths = cb_data;
164 struct string_list_item *available;
165
166 available = string_list_lookup(available_paths, item->string);
167 if (available)
168 available->util = item->util;
169 return !available;
170 }
171
172 static int string_is_not_null(struct string_list_item *item, void *data UNUSED)
173 {
174 return !!item->string;
175 }
176
177 int finish_delayed_checkout(struct checkout *state, int show_progress)
178 {
179 int errs = 0;
180 unsigned processed_paths = 0;
181 off_t filtered_bytes = 0;
182 struct string_list_item *filter, *path;
183 struct progress *progress = NULL;
184 struct delayed_checkout *dco = state->delayed_checkout;
185
186 if (!state->delayed_checkout)
187 return errs;
188
189 dco->state = CE_RETRY;
190 if (show_progress)
191 progress = start_delayed_progress(the_repository,
192 _("Filtering content"),
193 dco->paths.nr);
194 while (dco->filters.nr > 0) {
195 for_each_string_list_item(filter, &dco->filters) {
196 struct string_list available_paths = STRING_LIST_INIT_DUP;
197
198 if (!async_query_available_blobs(filter->string, &available_paths)) {
199 /* Filter reported an error */
200 errs = 1;
201 filter->string = NULL;
202 continue;
203 }
204 if (available_paths.nr <= 0) {
205 /*
206 * Filter responded with no entries. That means
207 * the filter is done and we can remove the
208 * filter from the list (see
209 * "string_list_remove_empty_items" call below).
210 */
211 filter->string = NULL;
212 continue;
213 }
214
215 /*
216 * In dco->paths we store a list of all delayed paths.
217 * The filter just send us a list of available paths.
218 * Remove them from the list.
219 */
220 filter_string_list(&dco->paths, 0,
221 &remove_available_paths, &available_paths);
222
223 for_each_string_list_item(path, &available_paths) {
224 struct cache_entry* ce;
225
226 if (!path->util) {
227 error("external filter '%s' signaled that '%s' "
228 "is now available although it has not been "
229 "delayed earlier",
230 filter->string, path->string);
231 errs |= 1;
232
233 /*
234 * Do not ask the filter for available blobs,
235 * again, as the filter is likely buggy.
236 */
237 filter->string = NULL;
238 continue;
239 }
240 ce = index_file_exists(state->istate, path->string,
241 strlen(path->string), 0);
242 if (ce) {
243 display_progress(progress, ++processed_paths);
244 errs |= checkout_entry(ce, state, NULL, path->util);
245 filtered_bytes += ce->ce_stat_data.sd_size;
246 display_throughput(progress, filtered_bytes);
247 } else
248 errs = 1;
249 }
250
251 string_list_clear(&available_paths, 0);
252 }
253
254 filter_string_list(&dco->filters, 0, string_is_not_null, NULL);
255 }
256 stop_progress(&progress);
257 string_list_clear(&dco->filters, 0);
258
259 /* At this point we should not have any delayed paths anymore. */
260 errs |= dco->paths.nr;
261 for_each_string_list_item(path, &dco->paths) {
262 error("'%s' was not filtered properly", path->string);
263 }
264 string_list_clear(&dco->paths, 0);
265
266 free(dco);
267 state->delayed_checkout = NULL;
268
269 return errs;
270 }
271
272 void update_ce_after_write(const struct checkout *state, struct cache_entry *ce,
273 struct stat *st)
274 {
275 if (state->refresh_cache) {
276 assert(state->istate);
277 fill_stat_cache_info(state->istate, ce, st);
278 ce->ce_flags |= CE_UPDATE_IN_BASE;
279 mark_fsmonitor_invalid(state->istate, ce);
280 state->istate->cache_changed |= CE_ENTRY_CHANGED;
281 }
282 }
283
284 /* Note: ca is used (and required) iff the entry refers to a regular file. */
285 static int write_entry(struct cache_entry *ce, char *path, struct conv_attrs *ca,
286 const struct checkout *state, int to_tempfile,
287 int *nr_checkouts)
288 {
289 unsigned int ce_mode_s_ifmt = ce->ce_mode & S_IFMT;
290 struct delayed_checkout *dco = state->delayed_checkout;
291 int fd, ret, fstat_done = 0;
292 char *new_blob;
293 struct strbuf buf = STRBUF_INIT;
294 size_t size;
295 ssize_t wrote;
296 size_t newsize = 0;
297 struct stat st;
298 const struct submodule *sub;
299 struct checkout_metadata meta;
300 static int scratch_nr_checkouts;
301
302 clone_checkout_metadata(&meta, &state->meta, &ce->oid);
303
304 if (ce_mode_s_ifmt == S_IFREG) {
305 struct stream_filter *filter = get_stream_filter_ca(ca, &ce->oid);
306 if (filter &&
307 !streaming_write_entry(ce, path, filter,
308 state, to_tempfile,
309 &fstat_done, &st))
310 goto finish;
311 }
312
313 switch (ce_mode_s_ifmt) {
314 case S_IFLNK:
315 new_blob = read_blob_entry(ce, &size);
316 if (!new_blob)
317 return error("unable to read sha1 file of %s (%s)",
318 ce->name, oid_to_hex(&ce->oid));
319
320 /*
321 * We can't make a real symlink; write out a regular file entry
322 * with the symlink destination as its contents.
323 */
324 if (!has_symlinks || to_tempfile)
325 goto write_file_entry;
326
327 ret = symlink(new_blob, path);
328 free(new_blob);
329 if (ret)
330 return error_errno("unable to create symlink %s", path);
331 break;
332
333 case S_IFREG:
334 /*
335 * We do not send the blob in case of a retry, so do not
336 * bother reading it at all.
337 */
338 if (dco && dco->state == CE_RETRY) {
339 new_blob = NULL;
340 size = 0;
341 } else {
342 new_blob = read_blob_entry(ce, &size);
343 if (!new_blob)
344 return error("unable to read sha1 file of %s (%s)",
345 ce->name, oid_to_hex(&ce->oid));
346 }
347
348 /*
349 * Convert from git internal format to working tree format
350 */
351 if (dco && dco->state != CE_NO_DELAY) {
352 ret = async_convert_to_working_tree_ca(ca, ce->name,
353 new_blob, size,
354 &buf, &meta, dco);
355 if (ret) {
356 struct string_list_item *item =
357 string_list_lookup(&dco->paths, ce->name);
358 if (item) {
359 item->util = nr_checkouts ? nr_checkouts
360 : &scratch_nr_checkouts;
361 free(new_blob);
362 goto delayed;
363 }
364 }
365 } else {
366 ret = convert_to_working_tree_ca(ca, ce->name, new_blob,
367 size, &buf, &meta);
368 }
369
370 if (ret) {
371 free(new_blob);
372 new_blob = strbuf_detach(&buf, &newsize);
373 size = newsize;
374 }
375 /*
376 * No "else" here as errors from convert are OK at this
377 * point. If the error would have been fatal (e.g.
378 * filter is required), then we would have died already.
379 */
380
381 write_file_entry:
382 fd = open_output_fd(path, ce, to_tempfile);
383 if (fd < 0) {
384 free(new_blob);
385 return error_errno("unable to create file %s", path);
386 }
387
388 wrote = write_in_full(fd, new_blob, size);
389 if (!to_tempfile)
390 fstat_done = fstat_checkout_output(fd, state, &st);
391 close(fd);
392 free(new_blob);
393 if (wrote < 0)
394 return error("unable to write file %s", path);
395 break;
396
397 case S_IFGITLINK:
398 if (to_tempfile)
399 return error("cannot create temporary submodule %s", ce->name);
400 if (mkdir(path, 0777) < 0)
401 return error("cannot create submodule directory %s", path);
402 sub = submodule_from_ce(ce);
403 if (sub)
404 return submodule_move_head(ce->name, state->super_prefix,
405 NULL, oid_to_hex(&ce->oid),
406 state->force ? SUBMODULE_MOVE_HEAD_FORCE : 0);
407 break;
408
409 default:
410 return error("unknown file mode for %s in index", ce->name);
411 }
412
413 finish:
414 if (state->refresh_cache) {
415 if (!fstat_done && lstat(ce->name, &st) < 0)
416 return error_errno("unable to stat just-written file %s",
417 ce->name);
418 update_ce_after_write(state, ce , &st);
419 }
420 if (nr_checkouts)
421 (*nr_checkouts)++;
422 delayed:
423 return 0;
424 }
425
426 /*
427 * This is like 'lstat()', except it refuses to follow symlinks
428 * in the path, after skipping "skiplen".
429 */
430 static int check_path(const char *path, int len, struct stat *st, int skiplen)
431 {
432 const char *slash = path + len;
433
434 while (path < slash && *slash != '/')
435 slash--;
436 if (!has_dirs_only_path(path, slash - path, skiplen)) {
437 errno = ENOENT;
438 return -1;
439 }
440 return lstat(path, st);
441 }
442
443 static void mark_colliding_entries(const struct checkout *state,
444 struct cache_entry *ce, struct stat *st)
445 {
446 struct repo_config_values *cfg = repo_config_values(the_repository);
447 int trust_ino = cfg->check_stat;
448
449 #if defined(GIT_WINDOWS_NATIVE) || defined(__CYGWIN__)
450 trust_ino = 0;
451 #endif
452
453 ce->ce_flags |= CE_MATCHED;
454
455 /* TODO: audit for interaction with sparse-index. */
456 ensure_full_index(state->istate);
457 for (size_t i = 0; i < state->istate->cache_nr; i++) {
458 struct cache_entry *dup = state->istate->cache[i];
459
460 if (dup == ce) {
461 /*
462 * Parallel checkout doesn't create the files in index
463 * order. So the other side of the collision may appear
464 * after the given cache_entry in the array.
465 */
466 if (parallel_checkout_status() == PC_RUNNING)
467 continue;
468 else
469 break;
470 }
471
472 if (dup->ce_flags & (CE_MATCHED | CE_VALID | CE_SKIP_WORKTREE))
473 continue;
474
475 if ((trust_ino && !match_stat_data(&dup->ce_stat_data, st)) ||
476 paths_collide(ce->name, dup->name)) {
477 dup->ce_flags |= CE_MATCHED;
478 break;
479 }
480 }
481 }
482
483 int checkout_entry_ca(struct cache_entry *ce, struct conv_attrs *ca,
484 const struct checkout *state, char *topath,
485 int *nr_checkouts)
486 {
487 static struct strbuf path = STRBUF_INIT;
488 struct stat st;
489 struct conv_attrs ca_buf;
490
491 if (ce->ce_flags & CE_WT_REMOVE) {
492 if (topath)
493 /*
494 * No content and thus no path to create, so we have
495 * no pathname to return.
496 */
497 BUG("Can't remove entry to a path");
498 unlink_entry(ce, state->super_prefix);
499 return 0;
500 }
501
502 if (topath) {
503 if (S_ISREG(ce->ce_mode) && !ca) {
504 convert_attrs(state->istate, &ca_buf, ce->name);
505 ca = &ca_buf;
506 }
507 return write_entry(ce, topath, ca, state, 1, nr_checkouts);
508 }
509
510 strbuf_reset(&path);
511 strbuf_add(&path, state->base_dir, state->base_dir_len);
512 strbuf_add(&path, ce->name, ce_namelen(ce));
513
514 if (!check_path(path.buf, path.len, &st, state->base_dir_len)) {
515 const struct submodule *sub;
516 unsigned changed = ie_match_stat(state->istate, ce, &st,
517 CE_MATCH_IGNORE_VALID | CE_MATCH_IGNORE_SKIP_WORKTREE);
518 /*
519 * Needs to be checked before !changed returns early,
520 * as the possibly empty directory was not changed
521 */
522 sub = submodule_from_ce(ce);
523 if (sub) {
524 int err;
525 if (!is_submodule_populated_gently(ce->name, &err)) {
526 struct stat sb;
527 if (lstat(ce->name, &sb))
528 die(_("could not stat file '%s'"), ce->name);
529 if (!(st.st_mode & S_IFDIR))
530 unlink_or_warn(ce->name);
531
532 return submodule_move_head(ce->name, state->super_prefix,
533 NULL, oid_to_hex(&ce->oid), 0);
534 } else
535 return submodule_move_head(ce->name, state->super_prefix,
536 "HEAD", oid_to_hex(&ce->oid),
537 state->force ? SUBMODULE_MOVE_HEAD_FORCE : 0);
538 }
539
540 if (!changed)
541 return 0;
542 if (!state->force) {
543 if (!state->quiet)
544 fprintf(stderr,
545 "%s already exists, no checkout\n",
546 path.buf);
547 return -1;
548 }
549
550 if (state->clone)
551 mark_colliding_entries(state, ce, &st);
552
553 /*
554 * We unlink the old file, to get the new one with the
555 * right permissions (including umask, which is nasty
556 * to emulate by hand - much easier to let the system
557 * just do the right thing)
558 */
559 if (S_ISDIR(st.st_mode)) {
560 /* If it is a gitlink, leave it alone! */
561 if (S_ISGITLINK(ce->ce_mode))
562 return 0;
563 /*
564 * We must avoid replacing submodules' leading
565 * directories with symbolic links, lest recursive
566 * clones can write into arbitrary locations.
567 *
568 * Technically, this logic is not limited
569 * to recursive clones, or for that matter to
570 * submodules' paths colliding with symbolic links'
571 * paths. Yet it strikes a balance in favor of
572 * simplicity, and if paths are colliding, we might
573 * just as well keep the directories during a clone.
574 */
575 if (state->clone && S_ISLNK(ce->ce_mode))
576 return 0;
577 remove_subtree(&path);
578 } else if (unlink(path.buf))
579 return error_errno("unable to unlink old '%s'", path.buf);
580 } else if (state->not_new)
581 return 0;
582
583 create_directories(path.buf, path.len, state);
584
585 if (S_ISREG(ce->ce_mode) && !ca) {
586 convert_attrs(state->istate, &ca_buf, ce->name);
587 ca = &ca_buf;
588 }
589
590 if (!enqueue_checkout(ce, ca, nr_checkouts))
591 return 0;
592
593 return write_entry(ce, path.buf, ca, state, 0, nr_checkouts);
594 }
595
596 void unlink_entry(const struct cache_entry *ce, const char *super_prefix)
597 {
598 const struct submodule *sub = submodule_from_ce(ce);
599 if (sub) {
600 /* state.force is set at the caller. */
601 submodule_move_head(ce->name, super_prefix, "HEAD", NULL,
602 SUBMODULE_MOVE_HEAD_FORCE);
603 }
604 if (check_leading_path(ce->name, ce_namelen(ce), 1) >= 0)
605 return;
606 if (remove_or_warn(ce->ce_mode, ce->name))
607 return;
608 schedule_dir_for_removal(ce->name, ce_namelen(ce));
609 }
610
611 int remove_or_warn(unsigned int mode, const char *file)
612 {
613 return S_ISGITLINK(mode) ? rmdir_or_warn(file) : unlink_or_warn(file);
614 }