Raw
1 #define USE_THE_REPOSITORY_VARIABLE
2
3 #include "git-compat-util.h"
4 #include "odb.h"
5 #include "odb/streaming.h"
6 #include "dir.h"
7 #include "environment.h"
8 #include "gettext.h"
9 #include "hex.h"
10 #include "name-hash.h"
11 #include "sparse-index.h"
12 #include "submodule.h"
13 #include "symlinks.h"
14 #include "progress.h"
15 #include "fsmonitor.h"
16 #include "entry.h"
17 #include "parallel-checkout.h"
18
19 static void create_directories(const char *path, int path_len,
20 const struct checkout *state)
21 {
22 char *buf = xmallocz(path_len);
23 int len = 0;
24
25 while (len < path_len) {
26 do {
27 buf[len] = path[len];
28 len++;
29 } while (len < path_len && path[len] != '/');
30 if (len >= path_len)
31 break;
32 buf[len] = 0;
33
34 /*
35 * For 'checkout-index --prefix=<dir>', <dir> is
36 * allowed to be a symlink to an existing directory,
37 * and we set 'state->base_dir_len' below, such that
38 * we test the path components of the prefix with the
39 * stat() function instead of the lstat() function.
40 */
41 if (has_dirs_only_path(buf, len, state->base_dir_len))
42 continue; /* ok, it is already a directory. */
43
44 /*
45 * If this mkdir() would fail, it could be that there
46 * is already a symlink or something else exists
47 * there, therefore we then try to unlink it and try
48 * one more time to create the directory.
49 */
50 if (mkdir(buf, 0777)) {
51 if (errno == EEXIST && state->force &&
52 !unlink_or_warn(buf) && !mkdir(buf, 0777))
53 continue;
54 die_errno("cannot create directory at '%s'", buf);
55 }
56 }
57 free(buf);
58 }
59
60 static void remove_subtree(struct strbuf *path)
61 {
62 DIR *dir = opendir(path->buf);
63 struct dirent *de;
64 int origlen = path->len;
65
66 if (!dir)
67 die_errno("cannot opendir '%s'", path->buf);
68 while ((de = readdir_skip_dot_and_dotdot(dir)) != NULL) {
69 struct stat st;
70
71 strbuf_addch(path, '/');
72 strbuf_addstr(path, de->d_name);
73 if (lstat(path->buf, &st))
74 die_errno("cannot lstat '%s'", path->buf);
75 if (S_ISDIR(st.st_mode))
76 remove_subtree(path);
77 else if (unlink(path->buf))
78 die_errno("cannot unlink '%s'", path->buf);
79 strbuf_setlen(path, origlen);
80 }
81 closedir(dir);
82 if (rmdir(path->buf))
83 die_errno("cannot rmdir '%s'", path->buf);
84 }
85
86 static int create_file(const char *path, unsigned int mode)
87 {
88 mode = (mode & 0100) ? 0777 : 0666;
89 return open(path, O_WRONLY | O_CREAT | O_EXCL, mode);
90 }
91
92 void *read_blob_entry(const struct cache_entry *ce, size_t *size)
93 {
94 enum object_type type;
95 void *blob_data = odb_read_object(the_repository->objects, &ce->oid,
96 &type, size);
97
98 if (blob_data) {
99 if (type == OBJ_BLOB)
100 return blob_data;
101 free(blob_data);
102 }
103 return NULL;
104 }
105
106 static int open_output_fd(char *path, const struct cache_entry *ce, int to_tempfile)
107 {
108 int symlink = (ce->ce_mode & S_IFMT) != S_IFREG;
109 if (to_tempfile) {
110 xsnprintf(path, TEMPORARY_FILENAME_LENGTH, "%s",
111 symlink ? ".merge_link_XXXXXX" : ".merge_file_XXXXXX");
112 return mkstemp(path);
113 } else {
114 return create_file(path, !symlink ? ce->ce_mode : 0666);
115 }
116 }
117
118 int fstat_checkout_output(int fd, const struct checkout *state, struct stat *st)
119 {
120 /* use fstat() only when path == ce->name */
121 if (fstat_is_reliable() &&
122 state->refresh_cache && !state->base_dir_len) {
123 return !fstat(fd, st);
124 }
125 return 0;
126 }
127
128 static int streaming_write_entry(const struct cache_entry *ce, char *path,
129 struct stream_filter *filter,
130 const struct checkout *state, int to_tempfile,
131 int *fstat_done, struct stat *statbuf)
132 {
133 int result = 0;
134 int fd;
135
136 fd = open_output_fd(path, ce, to_tempfile);
137 if (fd < 0)
138 return -1;
139
140 result |= odb_stream_blob_to_fd(the_repository->objects, fd, &ce->oid, filter, 1);
141 *fstat_done = fstat_checkout_output(fd, state, statbuf);
142 result |= close(fd);
143
144 if (result)
145 unlink(path);
146 return result;
147 }
148
149 void enable_delayed_checkout(struct checkout *state)
150 {
151 if (!state->delayed_checkout) {
152 state->delayed_checkout = xmalloc(sizeof(*state->delayed_checkout));
153 state->delayed_checkout->state = CE_CAN_DELAY;
154 string_list_init_nodup(&state->delayed_checkout->filters);
155 string_list_init_nodup(&state->delayed_checkout->paths);
156 }
157 }
158
159 static int remove_available_paths(struct string_list_item *item, void *cb_data)
160 {
161 struct string_list *available_paths = cb_data;
162 struct string_list_item *available;
163
164 available = string_list_lookup(available_paths, item->string);
165 if (available)
166 available->util = item->util;
167 return !available;
168 }
169
170 static int string_is_not_null(struct string_list_item *item, void *data UNUSED)
171 {
172 return !!item->string;
173 }
174
175 int finish_delayed_checkout(struct checkout *state, int show_progress)
176 {
177 int errs = 0;
178 unsigned processed_paths = 0;
179 off_t filtered_bytes = 0;
180 struct string_list_item *filter, *path;
181 struct progress *progress = NULL;
182 struct delayed_checkout *dco = state->delayed_checkout;
183
184 if (!state->delayed_checkout)
185 return errs;
186
187 dco->state = CE_RETRY;
188 if (show_progress)
189 progress = start_delayed_progress(the_repository,
190 _("Filtering content"),
191 dco->paths.nr);
192 while (dco->filters.nr > 0) {
193 for_each_string_list_item(filter, &dco->filters) {
194 struct string_list available_paths = STRING_LIST_INIT_DUP;
195
196 if (!async_query_available_blobs(filter->string, &available_paths)) {
197 /* Filter reported an error */
198 errs = 1;
199 filter->string = NULL;
200 continue;
201 }
202 if (available_paths.nr <= 0) {
203 /*
204 * Filter responded with no entries. That means
205 * the filter is done and we can remove the
206 * filter from the list (see
207 * "string_list_remove_empty_items" call below).
208 */
209 filter->string = NULL;
210 continue;
211 }
212
213 /*
214 * In dco->paths we store a list of all delayed paths.
215 * The filter just send us a list of available paths.
216 * Remove them from the list.
217 */
218 filter_string_list(&dco->paths, 0,
219 &remove_available_paths, &available_paths);
220
221 for_each_string_list_item(path, &available_paths) {
222 struct cache_entry* ce;
223
224 if (!path->util) {
225 error("external filter '%s' signaled that '%s' "
226 "is now available although it has not been "
227 "delayed earlier",
228 filter->string, path->string);
229 errs |= 1;
230
231 /*
232 * Do not ask the filter for available blobs,
233 * again, as the filter is likely buggy.
234 */
235 filter->string = NULL;
236 continue;
237 }
238 ce = index_file_exists(state->istate, path->string,
239 strlen(path->string), 0);
240 if (ce) {
241 display_progress(progress, ++processed_paths);
242 errs |= checkout_entry(ce, state, NULL, path->util);
243 filtered_bytes += ce->ce_stat_data.sd_size;
244 display_throughput(progress, filtered_bytes);
245 } else
246 errs = 1;
247 }
248
249 string_list_clear(&available_paths, 0);
250 }
251
252 filter_string_list(&dco->filters, 0, string_is_not_null, NULL);
253 }
254 stop_progress(&progress);
255 string_list_clear(&dco->filters, 0);
256
257 /* At this point we should not have any delayed paths anymore. */
258 errs |= dco->paths.nr;
259 for_each_string_list_item(path, &dco->paths) {
260 error("'%s' was not filtered properly", path->string);
261 }
262 string_list_clear(&dco->paths, 0);
263
264 free(dco);
265 state->delayed_checkout = NULL;
266
267 return errs;
268 }
269
270 void update_ce_after_write(const struct checkout *state, struct cache_entry *ce,
271 struct stat *st)
272 {
273 if (state->refresh_cache) {
274 assert(state->istate);
275 fill_stat_cache_info(state->istate, ce, st);
276 ce->ce_flags |= CE_UPDATE_IN_BASE;
277 mark_fsmonitor_invalid(state->istate, ce);
278 state->istate->cache_changed |= CE_ENTRY_CHANGED;
279 }
280 }
281
282 /* Note: ca is used (and required) iff the entry refers to a regular file. */
283 static int write_entry(struct cache_entry *ce, char *path, struct conv_attrs *ca,
284 const struct checkout *state, int to_tempfile,
285 int *nr_checkouts)
286 {
287 unsigned int ce_mode_s_ifmt = ce->ce_mode & S_IFMT;
288 struct delayed_checkout *dco = state->delayed_checkout;
289 int fd, ret, fstat_done = 0;
290 char *new_blob;
291 struct strbuf buf = STRBUF_INIT;
292 size_t size;
293 ssize_t wrote;
294 size_t newsize = 0;
295 struct stat st;
296 const struct submodule *sub;
297 struct checkout_metadata meta;
298 static int scratch_nr_checkouts;
299
300 clone_checkout_metadata(&meta, &state->meta, &ce->oid);
301
302 if (ce_mode_s_ifmt == S_IFREG) {
303 struct stream_filter *filter = get_stream_filter_ca(ca, &ce->oid);
304 if (filter &&
305 !streaming_write_entry(ce, path, filter,
306 state, to_tempfile,
307 &fstat_done, &st))
308 goto finish;
309 }
310
311 switch (ce_mode_s_ifmt) {
312 case S_IFLNK:
313 new_blob = read_blob_entry(ce, &size);
314 if (!new_blob)
315 return error("unable to read sha1 file of %s (%s)",
316 ce->name, oid_to_hex(&ce->oid));
317
318 /*
319 * We can't make a real symlink; write out a regular file entry
320 * with the symlink destination as its contents.
321 */
322 if (!repo_has_symlinks(state->istate && state->istate->repo ?
323 state->istate->repo : the_repository) || to_tempfile)
324 goto write_file_entry;
325
326 ret = symlink(new_blob, path);
327 free(new_blob);
328 if (ret)
329 return error_errno("unable to create symlink %s", path);
330 break;
331
332 case S_IFREG:
333 /*
334 * We do not send the blob in case of a retry, so do not
335 * bother reading it at all.
336 */
337 if (dco && dco->state == CE_RETRY) {
338 new_blob = NULL;
339 size = 0;
340 } else {
341 new_blob = read_blob_entry(ce, &size);
342 if (!new_blob)
343 return error("unable to read sha1 file of %s (%s)",
344 ce->name, oid_to_hex(&ce->oid));
345 }
346
347 /*
348 * Convert from git internal format to working tree format
349 */
350 if (dco && dco->state != CE_NO_DELAY) {
351 ret = async_convert_to_working_tree_ca(ca, ce->name,
352 new_blob, size,
353 &buf, &meta, dco);
354 if (ret) {
355 struct string_list_item *item =
356 string_list_lookup(&dco->paths, ce->name);
357 if (item) {
358 item->util = nr_checkouts ? nr_checkouts
359 : &scratch_nr_checkouts;
360 free(new_blob);
361 goto delayed;
362 }
363 }
364 } else {
365 ret = convert_to_working_tree_ca(ca, ce->name, new_blob,
366 size, &buf, &meta);
367 }
368
369 if (ret) {
370 free(new_blob);
371 new_blob = strbuf_detach(&buf, &newsize);
372 size = newsize;
373 }
374 /*
375 * No "else" here as errors from convert are OK at this
376 * point. If the error would have been fatal (e.g.
377 * filter is required), then we would have died already.
378 */
379
380 write_file_entry:
381 fd = open_output_fd(path, ce, to_tempfile);
382 if (fd < 0) {
383 free(new_blob);
384 return error_errno("unable to create file %s", path);
385 }
386
387 wrote = write_in_full(fd, new_blob, size);
388 if (!to_tempfile)
389 fstat_done = fstat_checkout_output(fd, state, &st);
390 close(fd);
391 free(new_blob);
392 if (wrote < 0)
393 return error("unable to write file %s", path);
394 break;
395
396 case S_IFGITLINK:
397 if (to_tempfile)
398 return error("cannot create temporary submodule %s", ce->name);
399 if (mkdir(path, 0777) < 0)
400 return error("cannot create submodule directory %s", path);
401 sub = submodule_from_ce(ce);
402 if (sub)
403 return submodule_move_head(ce->name, state->super_prefix,
404 NULL, oid_to_hex(&ce->oid),
405 state->force ? SUBMODULE_MOVE_HEAD_FORCE : 0);
406 break;
407
408 default:
409 return error("unknown file mode for %s in index", ce->name);
410 }
411
412 finish:
413 if (state->refresh_cache) {
414 if (!fstat_done && lstat(ce->name, &st) < 0)
415 return error_errno("unable to stat just-written file %s",
416 ce->name);
417 update_ce_after_write(state, ce , &st);
418 }
419 if (nr_checkouts)
420 (*nr_checkouts)++;
421 delayed:
422 return 0;
423 }
424
425 /*
426 * This is like 'lstat()', except it refuses to follow symlinks
427 * in the path, after skipping "skiplen".
428 */
429 static int check_path(const char *path, int len, struct stat *st, int skiplen)
430 {
431 const char *slash = path + len;
432
433 while (path < slash && *slash != '/')
434 slash--;
435 if (!has_dirs_only_path(path, slash - path, skiplen)) {
436 errno = ENOENT;
437 return -1;
438 }
439 return lstat(path, st);
440 }
441
442 static void mark_colliding_entries(const struct checkout *state,
443 struct cache_entry *ce, struct stat *st)
444 {
445 struct repo_config_values *cfg = repo_config_values(the_repository);
446 int trust_ino = cfg->check_stat;
447
448 #if defined(GIT_WINDOWS_NATIVE) || defined(__CYGWIN__)
449 trust_ino = 0;
450 #endif
451
452 ce->ce_flags |= CE_MATCHED;
453
454 /* TODO: audit for interaction with sparse-index. */
455 ensure_full_index(state->istate);
456 for (size_t i = 0; i < state->istate->cache_nr; i++) {
457 struct cache_entry *dup = state->istate->cache[i];
458
459 if (dup == ce) {
460 /*
461 * Parallel checkout doesn't create the files in index
462 * order. So the other side of the collision may appear
463 * after the given cache_entry in the array.
464 */
465 if (parallel_checkout_status() == PC_RUNNING)
466 continue;
467 else
468 break;
469 }
470
471 if (dup->ce_flags & (CE_MATCHED | CE_VALID | CE_SKIP_WORKTREE))
472 continue;
473
474 if ((trust_ino && !match_stat_data(&dup->ce_stat_data, st)) ||
475 paths_collide(ce->name, dup->name)) {
476 dup->ce_flags |= CE_MATCHED;
477 break;
478 }
479 }
480 }
481
482 int checkout_entry_ca(struct cache_entry *ce, struct conv_attrs *ca,
483 const struct checkout *state, char *topath,
484 int *nr_checkouts)
485 {
486 static struct strbuf path = STRBUF_INIT;
487 struct stat st;
488 struct conv_attrs ca_buf;
489
490 if (ce->ce_flags & CE_WT_REMOVE) {
491 if (topath)
492 /*
493 * No content and thus no path to create, so we have
494 * no pathname to return.
495 */
496 BUG("Can't remove entry to a path");
497 unlink_entry(ce, state->super_prefix);
498 return 0;
499 }
500
501 if (topath) {
502 if (S_ISREG(ce->ce_mode) && !ca) {
503 convert_attrs(state->istate, &ca_buf, ce->name);
504 ca = &ca_buf;
505 }
506 return write_entry(ce, topath, ca, state, 1, nr_checkouts);
507 }
508
509 strbuf_reset(&path);
510 strbuf_add(&path, state->base_dir, state->base_dir_len);
511 strbuf_add(&path, ce->name, ce_namelen(ce));
512
513 if (!check_path(path.buf, path.len, &st, state->base_dir_len)) {
514 const struct submodule *sub;
515 unsigned changed = ie_match_stat(state->istate, ce, &st,
516 CE_MATCH_IGNORE_VALID | CE_MATCH_IGNORE_SKIP_WORKTREE);
517 /*
518 * Needs to be checked before !changed returns early,
519 * as the possibly empty directory was not changed
520 */
521 sub = submodule_from_ce(ce);
522 if (sub) {
523 int err;
524 if (!is_submodule_populated_gently(ce->name, &err)) {
525 struct stat sb;
526 if (lstat(ce->name, &sb))
527 die(_("could not stat file '%s'"), ce->name);
528 if (!(st.st_mode & S_IFDIR))
529 unlink_or_warn(ce->name);
530
531 return submodule_move_head(ce->name, state->super_prefix,
532 NULL, oid_to_hex(&ce->oid), 0);
533 } else
534 return submodule_move_head(ce->name, state->super_prefix,
535 "HEAD", oid_to_hex(&ce->oid),
536 state->force ? SUBMODULE_MOVE_HEAD_FORCE : 0);
537 }
538
539 if (!changed)
540 return 0;
541 if (!state->force) {
542 if (!state->quiet)
543 fprintf(stderr,
544 "%s already exists, no checkout\n",
545 path.buf);
546 return -1;
547 }
548
549 if (state->clone)
550 mark_colliding_entries(state, ce, &st);
551
552 /*
553 * We unlink the old file, to get the new one with the
554 * right permissions (including umask, which is nasty
555 * to emulate by hand - much easier to let the system
556 * just do the right thing)
557 */
558 if (S_ISDIR(st.st_mode)) {
559 /* If it is a gitlink, leave it alone! */
560 if (S_ISGITLINK(ce->ce_mode))
561 return 0;
562 /*
563 * We must avoid replacing submodules' leading
564 * directories with symbolic links, lest recursive
565 * clones can write into arbitrary locations.
566 *
567 * Technically, this logic is not limited
568 * to recursive clones, or for that matter to
569 * submodules' paths colliding with symbolic links'
570 * paths. Yet it strikes a balance in favor of
571 * simplicity, and if paths are colliding, we might
572 * just as well keep the directories during a clone.
573 */
574 if (state->clone && S_ISLNK(ce->ce_mode))
575 return 0;
576 remove_subtree(&path);
577 } else if (unlink(path.buf))
578 return error_errno("unable to unlink old '%s'", path.buf);
579 } else if (state->not_new)
580 return 0;
581
582 create_directories(path.buf, path.len, state);
583
584 if (S_ISREG(ce->ce_mode) && !ca) {
585 convert_attrs(state->istate, &ca_buf, ce->name);
586 ca = &ca_buf;
587 }
588
589 if (!enqueue_checkout(ce, ca, nr_checkouts))
590 return 0;
591
592 return write_entry(ce, path.buf, ca, state, 0, nr_checkouts);
593 }
594
595 void unlink_entry(const struct cache_entry *ce, const char *super_prefix)
596 {
597 const struct submodule *sub = submodule_from_ce(ce);
598 if (sub) {
599 /* state.force is set at the caller. */
600 submodule_move_head(ce->name, super_prefix, "HEAD", NULL,
601 SUBMODULE_MOVE_HEAD_FORCE);
602 }
603 if (check_leading_path(ce->name, ce_namelen(ce), 1) >= 0)
604 return;
605 if (remove_or_warn(ce->ce_mode, ce->name))
606 return;
607 schedule_dir_for_removal(ce->name, ce_namelen(ce));
608 }
609
610 int remove_or_warn(unsigned int mode, const char *file)
611 {
612 return S_ISGITLINK(mode) ? rmdir_or_warn(file) : unlink_or_warn(file);
613 }