Raw
1 #ifndef RUN_COMMAND_H
2 #define RUN_COMMAND_H
3
4 #include "thread-utils.h"
5
6 #include "strvec.h"
7
8 struct repository;
9
10 /**
11 * The run-command API offers a versatile tool to run sub-processes with
12 * redirected input and output as well as with a modified environment
13 * and an alternate current directory.
14 *
15 * A similar API offers the capability to run a function asynchronously,
16 * which is primarily used to capture the output that the function
17 * produces in the caller in order to process it.
18 */
19
20
21 /**
22 * This describes the arguments, redirections, and environment of a
23 * command to run in a sub-process.
24 *
25 * The caller:
26 *
27 * 1. allocates and clears (using child_process_init() or
28 * CHILD_PROCESS_INIT) a struct child_process variable;
29 * 2. initializes the members;
30 * 3. calls start_command();
31 * 4. processes the data;
32 * 5. closes file descriptors (if necessary; see below);
33 * 6. calls finish_command().
34 *
35 * Special forms of redirection are available by setting these members
36 * to 1:
37 *
38 * .no_stdin, .no_stdout, .no_stderr: The respective channel is
39 * redirected to /dev/null.
40 *
41 * .stdout_to_stderr: stdout of the child is redirected to its
42 * stderr. This happens after stderr is itself redirected.
43 * So stdout will follow stderr to wherever it is
44 * redirected.
45 */
46 struct child_process {
47
48 /**
49 * The .args is a `struct strvec', use that API to manipulate
50 * it, e.g. strvec_pushv() to add an existing "const char **"
51 * vector.
52 *
53 * If the command to run is a git command, set the first
54 * element in the strvec to the command name without the
55 * 'git-' prefix and set .git_cmd = 1.
56 *
57 * The memory in .args will be cleaned up automatically during
58 * `finish_command` (or during `start_command` when it is unsuccessful).
59 */
60 struct strvec args;
61
62 /**
63 * Like .args the .env is a `struct strvec'.
64 *
65 * To modify the environment of the sub-process, specify an array of
66 * environment settings. Each string in the array manipulates the
67 * environment.
68 *
69 * - If the string is of the form "VAR=value", i.e. it contains '='
70 * the variable is added to the child process's environment.
71 *
72 * - If the string does not contain '=', it names an environment
73 * variable that will be removed from the child process's environment.
74 *
75 * The memory in .env will be cleaned up automatically during
76 * `finish_command` (or during `start_command` when it is unsuccessful).
77 */
78 struct strvec env;
79 pid_t pid;
80
81 int trace2_child_id;
82 uint64_t trace2_child_us_start;
83 const char *trace2_child_class;
84 const char *trace2_hook_name;
85
86 /*
87 * Using .in, .out, .err:
88 * - Specify 0 for no redirections. No new file descriptor is allocated.
89 * (child inherits stdin, stdout, stderr from parent).
90 * - Specify -1 to have a pipe allocated as follows:
91 * .in: returns the writable pipe end; parent writes to it,
92 * the readable pipe end becomes child's stdin
93 * .out, .err: returns the readable pipe end; parent reads from
94 * it, the writable pipe end becomes child's stdout/stderr
95 * The caller of start_command() must close the returned FDs
96 * after it has completed reading from/writing to it!
97 * - Specify > 0 to set a channel to a particular FD as follows:
98 * .in: a readable FD, becomes child's stdin
99 * .out: a writable FD, becomes child's stdout/stderr
100 * .err: a writable FD, becomes child's stderr
101 * The specified FD is closed by start_command(), even in case
102 * of errors!
103 */
104 int in;
105 int out;
106 int err;
107
108 /**
109 * To specify a new initial working directory for the sub-process,
110 * specify it in the .dir member.
111 */
112 const char *dir;
113
114 unsigned no_stdin:1;
115 unsigned no_stdout:1;
116 unsigned no_stderr:1;
117 unsigned git_cmd:1; /* if this is to be git sub-command */
118
119 /**
120 * If the program cannot be found, the functions return -1 and set
121 * errno to ENOENT. Normally, an error message is printed, but if
122 * .silent_exec_failure is set to 1, no message is printed for this
123 * special error condition.
124 */
125 unsigned silent_exec_failure:1;
126
127 /**
128 * Run the command from argv[0] using a shell (but note that we may
129 * still optimize out the shell call if the command contains no
130 * metacharacters). Note that further arguments to the command in
131 * argv[1], etc, do not need to be shell-quoted.
132 */
133 unsigned use_shell:1;
134
135 /**
136 * Release any open file handles to the object store before running
137 * the command; This is necessary e.g. when the spawned process may
138 * want to repack because that would delete `.pack` files (and on
139 * Windows, you cannot delete files that are still in use).
140 */
141 struct object_database *odb_to_close;
142
143 unsigned stdout_to_stderr:1;
144 unsigned clean_on_exit:1;
145 unsigned wait_after_clean:1;
146
147 /**
148 * Close file descriptors 3 and above in the child after forking
149 * but before exec. This prevents the child from inheriting
150 * pipe endpoints or other descriptors from the parent
151 * environment (e.g., the test harness).
152 */
153 unsigned close_fd_above_stderr:1;
154
155 void (*clean_on_exit_handler)(struct child_process *process);
156 };
157
158 #define CHILD_PROCESS_INIT { \
159 .args = STRVEC_INIT, \
160 .env = STRVEC_INIT, \
161 }
162
163 /**
164 * The functions: start_command, finish_command, run_command do the following:
165 *
166 * - If a system call failed, errno is set and -1 is returned. A diagnostic
167 * is printed.
168 *
169 * - If the program was not found, then -1 is returned and errno is set to
170 * ENOENT; a diagnostic is printed only if .silent_exec_failure is 0.
171 *
172 * - Otherwise, the program is run. If it terminates regularly, its exit
173 * code is returned. No diagnostic is printed, even if the exit code is
174 * non-zero.
175 *
176 * - If the program terminated due to a signal, then the return value is the
177 * signal number + 128, ie. the same value that a POSIX shell's $? would
178 * report. A diagnostic is printed.
179 *
180 */
181
182 /**
183 * Initialize a struct child_process variable.
184 */
185 void child_process_init(struct child_process *);
186
187 /**
188 * Release the memory associated with the struct child_process.
189 * Most users of the run-command API don't need to call this
190 * function explicitly because `start_command` invokes it on
191 * failure and `finish_command` calls it automatically already.
192 */
193 void child_process_clear(struct child_process *);
194
195 int is_executable(const char *name);
196
197 /**
198 * Check if the command exists on $PATH. This emulates the path search that
199 * execvp would perform, without actually executing the command so it
200 * can be used before fork() to prepare to run a command using
201 * execve() or after execvp() to diagnose why it failed.
202 *
203 * The caller should ensure that command contains no directory separators.
204 *
205 * Returns 1 if it is found in $PATH or 0 if the command could not be found.
206 */
207 int exists_in_PATH(const char *command);
208
209 /**
210 * Return the path that is used to execute Unix shell command-lines.
211 */
212 char *git_shell_path(void);
213
214 /**
215 * Start a sub-process. Takes a pointer to a `struct child_process`
216 * that specifies the details and returns pipe FDs (if requested).
217 * See below for details.
218 */
219 int start_command(struct child_process *);
220
221 /**
222 * Wait for the completion of a sub-process that was started with
223 * start_command().
224 */
225 int finish_command(struct child_process *);
226
227 int finish_command_in_signal(struct child_process *);
228
229 /**
230 * A convenience function that encapsulates a sequence of
231 * start_command() followed by finish_command(). Takes a pointer
232 * to a `struct child_process` that specifies the details.
233 */
234 int run_command(struct child_process *);
235
236 /*
237 * Prepare a `struct child_process` to run auto-maintenance. Returns 1 if the
238 * process has been prepared and is ready to run, or 0 in case auto-maintenance
239 * should be skipped.
240 */
241 int prepare_auto_maintenance(struct repository *r, int quiet,
242 struct child_process *maint);
243
244 /*
245 * Trigger an auto-gc
246 */
247 int run_auto_maintenance(struct repository *r, int quiet);
248
249 /**
250 * Execute the given command, sending "in" to its stdin, and capturing its
251 * stdout and stderr in the "out" and "err" strbufs. Any of the three may
252 * be NULL to skip processing.
253 *
254 * Returns -1 if starting the command fails or reading fails, and otherwise
255 * returns the exit code of the command. Any output collected in the
256 * buffers is kept even if the command returns a non-zero exit. The hint fields
257 * gives starting sizes for the strbuf allocations.
258 *
259 * The fields of "cmd" should be set up as they would for a normal run_command
260 * invocation. But note that there is no need to set the in, out, or err
261 * fields; pipe_command handles that automatically.
262 */
263 int pipe_command(struct child_process *cmd,
264 const char *in, size_t in_len,
265 struct strbuf *out, size_t out_hint,
266 struct strbuf *err, size_t err_hint);
267
268 /**
269 * Convenience wrapper around pipe_command for the common case
270 * of capturing only stdout.
271 */
272 static inline int capture_command(struct child_process *cmd,
273 struct strbuf *out,
274 size_t hint)
275 {
276 return pipe_command(cmd, NULL, 0, out, hint, NULL, 0);
277 }
278
279 /*
280 * The purpose of the following functions is to feed a pipe by running
281 * a function asynchronously and providing output that the caller reads.
282 *
283 * It is expected that no synchronization and mutual exclusion between
284 * the caller and the feed function is necessary so that the function
285 * can run in a thread without interfering with the caller.
286 *
287 * The caller:
288 *
289 * 1. allocates and clears (memset(&asy, 0, sizeof(asy));) a
290 * struct async variable;
291 * 2. initializes .proc and .data;
292 * 3. calls start_async();
293 * 4. processes communicates with proc through .in and .out;
294 * 5. closes .in and .out;
295 * 6. calls finish_async().
296 *
297 * There are serious restrictions on what the asynchronous function can do
298 * because this facility is implemented by a thread in the same address
299 * space on most platforms (when pthreads is available), but by a pipe to
300 * a forked process otherwise:
301 *
302 * - It cannot change the program's state (global variables, environment,
303 * etc.) in a way that the caller notices; in other words, .in and .out
304 * are the only communication channels to the caller.
305 *
306 * - It must not change the program's state that the caller of the
307 * facility also uses.
308 *
309 */
310 struct async {
311
312 /**
313 * The function pointer in .proc has the following signature:
314 *
315 * int proc(int in, int out, void *data);
316 *
317 * - in, out specifies a set of file descriptors to which the function
318 * must read/write the data that it needs/produces. The function
319 * *must* close these descriptors before it returns. A descriptor
320 * may be -1 if the caller did not configure a descriptor for that
321 * direction.
322 *
323 * - data is the value that the caller has specified in the .data member
324 * of struct async.
325 *
326 * - The return value of the function is 0 on success and non-zero
327 * on failure. If the function indicates failure, finish_async() will
328 * report failure as well.
329 *
330 */
331 int (*proc)(int in, int out, void *data);
332
333 void *data;
334
335 /**
336 * The members .in, .out are used to provide a set of fd's for
337 * communication between the caller and the callee as follows:
338 *
339 * - Specify 0 to have no file descriptor passed. The callee will
340 * receive -1 in the corresponding argument.
341 *
342 * - Specify < 0 to have a pipe allocated; start_async() replaces
343 * with the pipe FD in the following way:
344 *
345 * .in: Returns the writable pipe end into which the caller
346 * writes; the readable end of the pipe becomes the function's
347 * in argument.
348 *
349 * .out: Returns the readable pipe end from which the caller
350 * reads; the writable end of the pipe becomes the function's
351 * out argument.
352 *
353 * The caller of start_async() must close the returned FDs after it
354 * has completed reading from/writing from them.
355 *
356 * - Specify a file descriptor > 0 to be used by the function:
357 *
358 * .in: The FD must be readable; it becomes the function's in.
359 * .out: The FD must be writable; it becomes the function's out.
360 *
361 * The specified FD is closed by start_async(), even if it fails to
362 * run the function.
363 */
364 int in; /* caller writes here and closes it */
365 int out; /* caller reads from here and closes it */
366 #ifdef NO_PTHREADS
367 pid_t pid;
368 #else
369 pthread_t tid;
370 int proc_in;
371 int proc_out;
372 #endif
373 int isolate_sigpipe;
374 };
375
376 /**
377 * Run a function asynchronously. Takes a pointer to a `struct
378 * async` that specifies the details and returns a set of pipe FDs
379 * for communication with the function. See below for details.
380 */
381 int start_async(struct async *async);
382
383 /**
384 * Wait for the completion of an asynchronous function that was
385 * started with start_async().
386 */
387 int finish_async(struct async *async);
388
389 int in_async(void);
390 int async_with_fork(void);
391 void check_pipe(int err);
392
393 /**
394 * This callback should initialize the child process and preload the
395 * error channel if desired. The preloading of is useful if you want to
396 * have a message printed directly before the output of the child process.
397 * pp_cb is the callback cookie as passed to run_processes_parallel.
398 * You can store a child process specific callback cookie in pp_task_cb.
399 *
400 * See run_processes_parallel() below for a discussion of the "struct
401 * strbuf *out" parameter.
402 *
403 * Even after returning 0 to indicate that there are no more processes,
404 * this function will be called again until there are no more running
405 * child processes.
406 *
407 * Return 1 if the next child is ready to run.
408 * Return 0 if there are currently no more tasks to be processed.
409 * To send a signal to other child processes for abortion,
410 * return the negative signal number.
411 */
412 typedef int (*get_next_task_fn)(struct child_process *cp,
413 struct strbuf *out,
414 void *pp_cb,
415 void **pp_task_cb);
416
417 /**
418 * This callback is called whenever there are problems starting
419 * a new process.
420 *
421 * See run_processes_parallel() below for a discussion of the "struct
422 * strbuf *out" parameter.
423 *
424 * pp_cb is the callback cookie as passed into run_processes_parallel,
425 * pp_task_cb is the callback cookie as passed into get_next_task_fn.
426 *
427 * Return 0 to continue the parallel processing. To abort return non zero.
428 * To send a signal to other child processes for abortion, return
429 * the negative signal number.
430 */
431 typedef int (*start_failure_fn)(struct strbuf *out,
432 void *pp_cb,
433 void *pp_task_cb);
434
435 /**
436 * This callback is repeatedly called on every child process who requests
437 * start_command() to create a pipe by setting child_process.in < 0.
438 *
439 * pp_cb is the callback cookie as passed into run_processes_parallel, and
440 * pp_task_cb is the callback cookie as passed into get_next_task_fn.
441 *
442 * Returns < 0 for error
443 * Returns == 0 when there is more data to be fed (will be called again)
444 * Returns > 0 when finished (child closed fd or no more data to be fed)
445 */
446 typedef int (*feed_pipe_fn)(int child_in,
447 void *pp_cb,
448 void *pp_task_cb);
449
450 /**
451 * This callback is called on every child process that finished processing.
452 *
453 * See run_processes_parallel() below for a discussion of the "struct
454 * strbuf *out" parameter.
455 *
456 * pp_cb is the callback cookie as passed into run_processes_parallel,
457 * pp_task_cb is the callback cookie as passed into get_next_task_fn.
458 *
459 * Return 0 to continue the parallel processing. To abort return non zero.
460 * To send a signal to other child processes for abortion, return
461 * the negative signal number.
462 */
463 typedef int (*task_finished_fn)(int result,
464 struct strbuf *out,
465 void *pp_cb,
466 void *pp_task_cb);
467
468 /**
469 * Option used by run_processes_parallel(), { 0 }-initialized means no
470 * options.
471 */
472 struct run_process_parallel_opts
473 {
474 /**
475 * tr2_category & tr2_label: sets the trace2 category and label for
476 * logging. These must either be unset, or both of them must be set.
477 */
478 const char *tr2_category;
479 const char *tr2_label;
480
481 /**
482 * processes: see 'processes' in run_processes_parallel() below.
483 */
484 size_t processes;
485
486 /**
487 * ungroup: see 'ungroup' in run_processes_parallel() below.
488 */
489 unsigned int ungroup:1;
490
491 /**
492 * get_next_task: See get_next_task_fn() above. This must be
493 * specified.
494 */
495 get_next_task_fn get_next_task;
496
497 /**
498 * start_failure: See start_failure_fn() above. This can be
499 * NULL to omit any special handling.
500 */
501 start_failure_fn start_failure;
502
503 /*
504 * feed_pipe: see feed_pipe_fn() above. This can be NULL to omit any
505 * special handling.
506 */
507 feed_pipe_fn feed_pipe;
508
509 /**
510 * task_finished: See task_finished_fn() above. This can be
511 * NULL to omit any special handling.
512 */
513 task_finished_fn task_finished;
514
515 /**
516 * data: user data, will be passed as "pp_cb" to the callback
517 * parameters.
518 */
519 void *data;
520 };
521
522 /**
523 * Options are passed via the "struct run_process_parallel_opts" above.
524 *
525 * Runs N 'processes' at the same time. Whenever a process can be
526 * started, the callback opts.get_next_task is called to obtain the data
527 * required to start another child process.
528 *
529 * The children started via this function run in parallel. Their output
530 * (both stdout and stderr) is routed to stderr in a manner that output
531 * from different tasks does not interleave (but see "ungroup" below).
532 *
533 * If the "ungroup" option isn't specified, the API will set the
534 * "stdout_to_stderr" parameter in "struct child_process" and provide
535 * the callbacks with a "struct strbuf *out" parameter to write output
536 * to. In this case the callbacks must not write to stdout or
537 * stderr as such output will mess up the output of the other parallel
538 * processes. If "ungroup" option is specified callbacks will get a
539 * NULL "struct strbuf *out" parameter, and are responsible for
540 * emitting their own output, including dealing with any race
541 * conditions due to writing in parallel to stdout and stderr.
542 */
543 void run_processes_parallel(const struct run_process_parallel_opts *opts);
544
545 /**
546 * Unset all local-repo GIT_* variables in env; see local_repo_env in
547 * environment.h. GIT_CONFIG_PARAMETERS and GIT_CONFIG_COUNT are preserved
548 * to pass -c and --config-env options from the parent process.
549 */
550 void sanitize_repo_env(struct strvec *env);
551
552 /**
553 * Convenience function which prepares env for a command to be run in a
554 * new repo. This removes variables pointing to the local repository (using
555 * sanitize_repo_env() above), and adds an environment variable pointing to
556 * new_git_dir.
557 */
558 void prepare_other_repo_env(struct strvec *env, const char *new_git_dir);
559
560 /**
561 * Possible return values for start_bg_command().
562 */
563 enum start_bg_result {
564 /* child process is "ready" */
565 SBGR_READY = 0,
566
567 /* child process could not be started */
568 SBGR_ERROR,
569
570 /* callback error when testing for "ready" */
571 SBGR_CB_ERROR,
572
573 /* timeout expired waiting for child to become "ready" */
574 SBGR_TIMEOUT,
575
576 /* child process exited or was signalled before becoming "ready" */
577 SBGR_DIED,
578 };
579
580 /**
581 * Callback used by start_bg_command() to ask whether the
582 * child process is ready or needs more time to become "ready".
583 *
584 * The callback will receive the cmd and cb_data arguments given to
585 * start_bg_command().
586 *
587 * Returns 1 is child needs more time (subject to the requested timeout).
588 * Returns 0 if child is "ready".
589 * Returns -1 on any error and cause start_bg_command() to also error out.
590 */
591 typedef int(start_bg_wait_cb)(const struct child_process *cmd, void *cb_data);
592
593 /**
594 * Start a command in the background. Wait long enough for the child
595 * to become "ready" (as defined by the provided callback). Capture
596 * immediate errors (like failure to start) and any immediate exit
597 * status (such as a shutdown/signal before the child became "ready")
598 * and return this like start_command().
599 *
600 * We run a custom wait loop using the provided callback to wait for
601 * the child to start and become "ready". This is limited by the given
602 * timeout value.
603 *
604 * If the child does successfully start and become "ready", we orphan
605 * it into the background.
606 *
607 * The caller must not call finish_command().
608 *
609 * The opaque cb_data argument will be forwarded to the callback for
610 * any instance data that it might require. This may be NULL.
611 */
612 enum start_bg_result start_bg_command(struct child_process *cmd,
613 start_bg_wait_cb *wait_cb,
614 void *cb_data,
615 unsigned int timeout_sec);
616
617 int sane_execvp(const char *file, char *const argv[]);
618
619 #endif