| 1 | /* |
| 2 | * test-run-command.c: test run command API. |
| 3 | * |
| 4 | * (C) 2009 Ilari Liusvaara <ilari.liusvaara@elisanet.fi> |
| 5 | * |
| 6 | * This code is free software; you can redistribute it and/or modify |
| 7 | * it under the terms of the GNU General Public License version 2 as |
| 8 | * published by the Free Software Foundation. |
| 9 | */ |
| 10 | |
| 11 | #define DISABLE_SIGN_COMPARE_WARNINGS |
| 12 | |
| 13 | #include "test-tool.h" |
| 14 | #include "run-command.h" |
| 15 | #include "strvec.h" |
| 16 | #include "strbuf.h" |
| 17 | #include "parse-options.h" |
| 18 | #include "string-list.h" |
| 19 | #include "thread-utils.h" |
| 20 | #include "wildmatch.h" |
| 21 | |
| 22 | static int number_callbacks; |
| 23 | static int parallel_next(struct child_process *cp, |
| 24 | struct strbuf *err, |
| 25 | void *cb, |
| 26 | void **task_cb) |
| 27 | { |
| 28 | struct child_process *d = cb; |
| 29 | if (number_callbacks >= 4) |
| 30 | return 0; |
| 31 | |
| 32 | strvec_pushv(&cp->args, d->args.v); |
| 33 | cp->in = d->in; |
| 34 | cp->no_stdin = d->no_stdin; |
| 35 | if (err) |
| 36 | strbuf_addstr(err, "preloaded output of a child\n"); |
| 37 | else |
| 38 | fprintf(stderr, "preloaded output of a child\n"); |
| 39 | |
| 40 | number_callbacks++; |
| 41 | |
| 42 | /* test_stdin callback will use this to count remaining lines */ |
| 43 | *task_cb = xmalloc(sizeof(int)); |
| 44 | *(int*)(*task_cb) = 2; |
| 45 | |
| 46 | return 1; |
| 47 | } |
| 48 | |
| 49 | static int no_job(struct child_process *cp UNUSED, |
| 50 | struct strbuf *err, |
| 51 | void *cb UNUSED, |
| 52 | void **task_cb UNUSED) |
| 53 | { |
| 54 | if (err) |
| 55 | strbuf_addstr(err, "no further jobs available\n"); |
| 56 | else |
| 57 | fprintf(stderr, "no further jobs available\n"); |
| 58 | return 0; |
| 59 | } |
| 60 | |
| 61 | static int task_finished(int result UNUSED, |
| 62 | struct strbuf *err, |
| 63 | void *pp_cb UNUSED, |
| 64 | void *pp_task_cb) |
| 65 | { |
| 66 | if (err) |
| 67 | strbuf_addstr(err, "asking for a quick stop\n"); |
| 68 | else |
| 69 | fprintf(stderr, "asking for a quick stop\n"); |
| 70 | |
| 71 | FREE_AND_NULL(pp_task_cb); |
| 72 | |
| 73 | return 1; |
| 74 | } |
| 75 | |
| 76 | static int task_finished_quiet(int result UNUSED, |
| 77 | struct strbuf *err UNUSED, |
| 78 | void *pp_cb UNUSED, |
| 79 | void *pp_task_cb) |
| 80 | { |
| 81 | FREE_AND_NULL(pp_task_cb); |
| 82 | return 0; |
| 83 | } |
| 84 | |
| 85 | static int test_stdin_pipe_feed(int hook_stdin_fd, void *cb UNUSED, void *task_cb) |
| 86 | { |
| 87 | int *lines_remaining = task_cb; |
| 88 | |
| 89 | if (*lines_remaining) { |
| 90 | struct strbuf buf = STRBUF_INIT; |
| 91 | strbuf_addf(&buf, "sample stdin %d\n", --(*lines_remaining)); |
| 92 | if (write_in_full(hook_stdin_fd, buf.buf, buf.len) < 0) { |
| 93 | if (errno == EPIPE) { |
| 94 | /* child closed stdin, nothing more to do */ |
| 95 | strbuf_release(&buf); |
| 96 | return 1; |
| 97 | } |
| 98 | die_errno("write"); |
| 99 | } |
| 100 | strbuf_release(&buf); |
| 101 | } |
| 102 | |
| 103 | return !(*lines_remaining); |
| 104 | } |
| 105 | |
| 106 | struct testsuite { |
| 107 | struct string_list tests, failed; |
| 108 | int next; |
| 109 | int quiet, immediate, verbose, verbose_log, trace, write_junit_xml; |
| 110 | const char *shell_path; |
| 111 | }; |
| 112 | #define TESTSUITE_INIT { \ |
| 113 | .tests = STRING_LIST_INIT_DUP, \ |
| 114 | .failed = STRING_LIST_INIT_DUP, \ |
| 115 | } |
| 116 | |
| 117 | static int next_test(struct child_process *cp, struct strbuf *err, void *cb, |
| 118 | void **task_cb) |
| 119 | { |
| 120 | struct testsuite *suite = cb; |
| 121 | const char *test; |
| 122 | if (suite->next >= suite->tests.nr) |
| 123 | return 0; |
| 124 | |
| 125 | test = suite->tests.items[suite->next++].string; |
| 126 | if (suite->shell_path) |
| 127 | strvec_push(&cp->args, suite->shell_path); |
| 128 | strvec_push(&cp->args, test); |
| 129 | if (suite->quiet) |
| 130 | strvec_push(&cp->args, "--quiet"); |
| 131 | if (suite->immediate) |
| 132 | strvec_push(&cp->args, "-i"); |
| 133 | if (suite->verbose) |
| 134 | strvec_push(&cp->args, "-v"); |
| 135 | if (suite->verbose_log) |
| 136 | strvec_push(&cp->args, "-V"); |
| 137 | if (suite->trace) |
| 138 | strvec_push(&cp->args, "-x"); |
| 139 | if (suite->write_junit_xml) |
| 140 | strvec_push(&cp->args, "--write-junit-xml"); |
| 141 | |
| 142 | strbuf_addf(err, "Output of '%s':\n", test); |
| 143 | *task_cb = (void *)test; |
| 144 | |
| 145 | return 1; |
| 146 | } |
| 147 | |
| 148 | static int test_finished(int result, struct strbuf *err, void *cb, |
| 149 | void *task_cb) |
| 150 | { |
| 151 | struct testsuite *suite = cb; |
| 152 | const char *name = (const char *)task_cb; |
| 153 | |
| 154 | if (result) |
| 155 | string_list_append(&suite->failed, name); |
| 156 | |
| 157 | strbuf_addf(err, "%s: '%s'\n", result ? "FAIL" : "SUCCESS", name); |
| 158 | |
| 159 | return 0; |
| 160 | } |
| 161 | |
| 162 | static int test_failed(struct strbuf *out, void *cb, void *task_cb) |
| 163 | { |
| 164 | struct testsuite *suite = cb; |
| 165 | const char *name = (const char *)task_cb; |
| 166 | |
| 167 | string_list_append(&suite->failed, name); |
| 168 | strbuf_addf(out, "FAILED TO START: '%s'\n", name); |
| 169 | |
| 170 | return 0; |
| 171 | } |
| 172 | |
| 173 | static const char * const testsuite_usage[] = { |
| 174 | "test-run-command testsuite [<options>] [<pattern>...]", |
| 175 | NULL |
| 176 | }; |
| 177 | |
| 178 | static int testsuite(int argc, const char **argv) |
| 179 | { |
| 180 | struct testsuite suite = TESTSUITE_INIT; |
| 181 | int max_jobs = 1, i, ret = 0; |
| 182 | DIR *dir; |
| 183 | struct dirent *d; |
| 184 | struct option options[] = { |
| 185 | OPT_BOOL('i', "immediate", &suite.immediate, |
| 186 | "stop at first failed test case(s)"), |
| 187 | OPT_INTEGER('j', "jobs", &max_jobs, "run <N> jobs in parallel"), |
| 188 | OPT_BOOL('q', "quiet", &suite.quiet, "be terse"), |
| 189 | OPT_BOOL('v', "verbose", &suite.verbose, "be verbose"), |
| 190 | OPT_BOOL('V', "verbose-log", &suite.verbose_log, |
| 191 | "be verbose, redirected to a file"), |
| 192 | OPT_BOOL('x', "trace", &suite.trace, "trace shell commands"), |
| 193 | OPT_BOOL(0, "write-junit-xml", &suite.write_junit_xml, |
| 194 | "write JUnit-style XML files"), |
| 195 | OPT_END() |
| 196 | }; |
| 197 | struct run_process_parallel_opts opts = { |
| 198 | .get_next_task = next_test, |
| 199 | .start_failure = test_failed, |
| 200 | .feed_pipe = test_stdin_pipe_feed, |
| 201 | .task_finished = test_finished, |
| 202 | .data = &suite, |
| 203 | }; |
| 204 | struct strbuf progpath = STRBUF_INIT; |
| 205 | size_t path_prefix_len; |
| 206 | |
| 207 | argc = parse_options(argc, argv, NULL, options, |
| 208 | testsuite_usage, PARSE_OPT_STOP_AT_NON_OPTION); |
| 209 | |
| 210 | if (max_jobs <= 0) |
| 211 | max_jobs = online_cpus(); |
| 212 | |
| 213 | /* |
| 214 | * If we run without a shell, execute the programs directly from CWD. |
| 215 | */ |
| 216 | suite.shell_path = getenv("TEST_SHELL_PATH"); |
| 217 | if (!suite.shell_path) |
| 218 | strbuf_addstr(&progpath, "./"); |
| 219 | path_prefix_len = progpath.len; |
| 220 | |
| 221 | dir = opendir("."); |
| 222 | if (!dir) |
| 223 | die("Could not open the current directory"); |
| 224 | while ((d = readdir(dir))) { |
| 225 | const char *p = d->d_name; |
| 226 | |
| 227 | if (!strcmp(p, ".") || !strcmp(p, "..")) |
| 228 | continue; |
| 229 | |
| 230 | /* No pattern: match all */ |
| 231 | if (!argc) { |
| 232 | strbuf_setlen(&progpath, path_prefix_len); |
| 233 | strbuf_addstr(&progpath, p); |
| 234 | string_list_append(&suite.tests, progpath.buf); |
| 235 | continue; |
| 236 | } |
| 237 | |
| 238 | for (i = 0; i < argc; i++) |
| 239 | if (!wildmatch(argv[i], p, 0)) { |
| 240 | strbuf_setlen(&progpath, path_prefix_len); |
| 241 | strbuf_addstr(&progpath, p); |
| 242 | string_list_append(&suite.tests, progpath.buf); |
| 243 | break; |
| 244 | } |
| 245 | } |
| 246 | closedir(dir); |
| 247 | |
| 248 | if (!suite.tests.nr) |
| 249 | die("No tests match!"); |
| 250 | if (max_jobs > suite.tests.nr) |
| 251 | max_jobs = suite.tests.nr; |
| 252 | |
| 253 | fprintf(stderr, "Running %"PRIuMAX" tests (%d at a time)\n", |
| 254 | (uintmax_t)suite.tests.nr, max_jobs); |
| 255 | |
| 256 | opts.processes = max_jobs; |
| 257 | run_processes_parallel(&opts); |
| 258 | |
| 259 | if (suite.failed.nr > 0) { |
| 260 | ret = 1; |
| 261 | fprintf(stderr, "%"PRIuMAX" tests failed:\n\n", |
| 262 | (uintmax_t)suite.failed.nr); |
| 263 | for (i = 0; i < suite.failed.nr; i++) |
| 264 | fprintf(stderr, "\t%s\n", suite.failed.items[i].string); |
| 265 | } |
| 266 | |
| 267 | string_list_clear(&suite.tests, 0); |
| 268 | string_list_clear(&suite.failed, 0); |
| 269 | strbuf_release(&progpath); |
| 270 | |
| 271 | return ret; |
| 272 | } |
| 273 | |
| 274 | static uint64_t my_random_next = 1234; |
| 275 | |
| 276 | static uint64_t my_random(void) |
| 277 | { |
| 278 | uint64_t res = my_random_next; |
| 279 | my_random_next = my_random_next * 1103515245 + 12345; |
| 280 | return res; |
| 281 | } |
| 282 | |
| 283 | static int quote_stress_test(int argc, const char **argv) |
| 284 | { |
| 285 | /* |
| 286 | * We are running a quote-stress test. |
| 287 | * spawn a subprocess that runs quote-stress with a |
| 288 | * special option that echoes back the arguments that |
| 289 | * were passed in. |
| 290 | */ |
| 291 | char special[] = ".?*\\^_\"'`{}()[]<>@~&+:;$%"; // \t\r\n\a"; |
| 292 | int i, j, k, trials = 100, skip = 0, msys2 = 0; |
| 293 | struct strbuf out = STRBUF_INIT; |
| 294 | struct strvec args = STRVEC_INIT; |
| 295 | struct option options[] = { |
| 296 | OPT_INTEGER('n', "trials", &trials, "number of trials"), |
| 297 | OPT_INTEGER('s', "skip", &skip, "skip <n> trials"), |
| 298 | OPT_BOOL('m', "msys2", &msys2, "test quoting for MSYS2's sh"), |
| 299 | OPT_END() |
| 300 | }; |
| 301 | const char * const usage[] = { |
| 302 | "test-tool run-command quote-stress-test <options>", |
| 303 | NULL |
| 304 | }; |
| 305 | |
| 306 | argc = parse_options(argc, argv, NULL, options, usage, 0); |
| 307 | |
| 308 | setenv("MSYS_NO_PATHCONV", "1", 0); |
| 309 | |
| 310 | for (i = 0; i < trials; i++) { |
| 311 | struct child_process cp = CHILD_PROCESS_INIT; |
| 312 | size_t arg_count, arg_offset; |
| 313 | int ret = 0; |
| 314 | |
| 315 | strvec_clear(&args); |
| 316 | if (msys2) |
| 317 | strvec_pushl(&args, "sh", "-c", |
| 318 | "printf %s\\\\0 \"$@\"", "skip", NULL); |
| 319 | else |
| 320 | strvec_pushl(&args, "test-tool", "run-command", |
| 321 | "quote-echo", NULL); |
| 322 | arg_offset = args.nr; |
| 323 | |
| 324 | if (argc > 0) { |
| 325 | trials = 1; |
| 326 | arg_count = argc; |
| 327 | for (j = 0; j < arg_count; j++) |
| 328 | strvec_push(&args, argv[j]); |
| 329 | } else { |
| 330 | arg_count = 1 + (my_random() % 5); |
| 331 | for (j = 0; j < arg_count; j++) { |
| 332 | char buf[20]; |
| 333 | size_t min_len = 1; |
| 334 | size_t arg_len = min_len + |
| 335 | (my_random() % (ARRAY_SIZE(buf) - min_len)); |
| 336 | |
| 337 | for (k = 0; k < arg_len; k++) |
| 338 | buf[k] = special[my_random() % |
| 339 | ARRAY_SIZE(special)]; |
| 340 | buf[arg_len] = '\0'; |
| 341 | |
| 342 | strvec_push(&args, buf); |
| 343 | } |
| 344 | } |
| 345 | |
| 346 | if (i < skip) |
| 347 | continue; |
| 348 | |
| 349 | strvec_pushv(&cp.args, args.v); |
| 350 | strbuf_reset(&out); |
| 351 | if (pipe_command(&cp, NULL, 0, &out, 0, NULL, 0) < 0) |
| 352 | return error("Failed to spawn child process"); |
| 353 | |
| 354 | for (j = 0, k = 0; j < arg_count; j++) { |
| 355 | const char *arg = args.v[j + arg_offset]; |
| 356 | |
| 357 | if (strcmp(arg, out.buf + k)) |
| 358 | ret = error("incorrectly quoted arg: '%s', " |
| 359 | "echoed back as '%s'", |
| 360 | arg, out.buf + k); |
| 361 | k += strlen(out.buf + k) + 1; |
| 362 | } |
| 363 | |
| 364 | if (k != out.len) |
| 365 | ret = error("got %d bytes, but consumed only %d", |
| 366 | (int)out.len, (int)k); |
| 367 | |
| 368 | if (ret) { |
| 369 | fprintf(stderr, "Trial #%d failed. Arguments:\n", i); |
| 370 | for (j = 0; j < arg_count; j++) |
| 371 | fprintf(stderr, "arg #%d: '%s'\n", |
| 372 | (int)j, args.v[j + arg_offset]); |
| 373 | |
| 374 | strbuf_release(&out); |
| 375 | strvec_clear(&args); |
| 376 | |
| 377 | return ret; |
| 378 | } |
| 379 | |
| 380 | if (i && (i % 100) == 0) |
| 381 | fprintf(stderr, "Trials completed: %d\n", (int)i); |
| 382 | } |
| 383 | |
| 384 | strbuf_release(&out); |
| 385 | strvec_clear(&args); |
| 386 | |
| 387 | return 0; |
| 388 | } |
| 389 | |
| 390 | static int quote_echo(int argc, const char **argv) |
| 391 | { |
| 392 | while (argc > 1) { |
| 393 | fwrite(argv[1], strlen(argv[1]), 1, stdout); |
| 394 | fputc('\0', stdout); |
| 395 | argv++; |
| 396 | argc--; |
| 397 | } |
| 398 | |
| 399 | return 0; |
| 400 | } |
| 401 | |
| 402 | static int inherit_handle(const char *argv0) |
| 403 | { |
| 404 | struct child_process cp = CHILD_PROCESS_INIT; |
| 405 | char path[PATH_MAX]; |
| 406 | int tmp; |
| 407 | |
| 408 | /* First, open an inheritable handle */ |
| 409 | xsnprintf(path, sizeof(path), "out-XXXXXX"); |
| 410 | tmp = xmkstemp(path); |
| 411 | |
| 412 | strvec_pushl(&cp.args, |
| 413 | "test-tool", argv0, "inherited-handle-child", NULL); |
| 414 | cp.in = -1; |
| 415 | cp.no_stdout = cp.no_stderr = 1; |
| 416 | if (start_command(&cp) < 0) |
| 417 | die("Could not start child process"); |
| 418 | |
| 419 | /* Then close it, and try to delete it. */ |
| 420 | close(tmp); |
| 421 | if (unlink(path)) |
| 422 | die("Could not delete '%s'", path); |
| 423 | |
| 424 | if (close(cp.in) < 0 || finish_command(&cp) < 0) |
| 425 | die("Child did not finish"); |
| 426 | |
| 427 | return 0; |
| 428 | } |
| 429 | |
| 430 | static int inherit_handle_child(void) |
| 431 | { |
| 432 | struct strbuf buf = STRBUF_INIT; |
| 433 | |
| 434 | if (strbuf_read(&buf, 0, 0) < 0) |
| 435 | die("Could not read stdin"); |
| 436 | printf("Received %s\n", buf.buf); |
| 437 | strbuf_release(&buf); |
| 438 | |
| 439 | return 0; |
| 440 | } |
| 441 | |
| 442 | int cmd__run_command(int argc, const char **argv) |
| 443 | { |
| 444 | struct child_process proc = CHILD_PROCESS_INIT; |
| 445 | int jobs; |
| 446 | int ret; |
| 447 | struct run_process_parallel_opts opts = { |
| 448 | .data = &proc, |
| 449 | }; |
| 450 | |
| 451 | if (argc > 1 && !strcmp(argv[1], "testsuite")) |
| 452 | return testsuite(argc - 1, argv + 1); |
| 453 | if (!strcmp(argv[1], "inherited-handle")) |
| 454 | return inherit_handle(argv[0]); |
| 455 | if (!strcmp(argv[1], "inherited-handle-child")) |
| 456 | return inherit_handle_child(); |
| 457 | |
| 458 | if (argc >= 2 && !strcmp(argv[1], "quote-stress-test")) |
| 459 | return !!quote_stress_test(argc - 1, argv + 1); |
| 460 | |
| 461 | if (argc >= 2 && !strcmp(argv[1], "quote-echo")) |
| 462 | return !!quote_echo(argc - 1, argv + 1); |
| 463 | |
| 464 | if (argc < 3) |
| 465 | return 1; |
| 466 | while (!strcmp(argv[1], "env")) { |
| 467 | if (!argv[2]) |
| 468 | die("env specifier without a value"); |
| 469 | strvec_push(&proc.env, argv[2]); |
| 470 | argv += 2; |
| 471 | argc -= 2; |
| 472 | } |
| 473 | if (argc < 3) { |
| 474 | ret = 1; |
| 475 | goto cleanup; |
| 476 | } |
| 477 | strvec_pushv(&proc.args, (const char **)argv + 2); |
| 478 | |
| 479 | if (!strcmp(argv[1], "start-command-ENOENT")) { |
| 480 | if (start_command(&proc) < 0 && errno == ENOENT) { |
| 481 | ret = 0; |
| 482 | goto cleanup; |
| 483 | } |
| 484 | fprintf(stderr, "FAIL %s\n", argv[1]); |
| 485 | return 1; |
| 486 | } |
| 487 | if (!strcmp(argv[1], "run-command")) { |
| 488 | ret = run_command(&proc); |
| 489 | goto cleanup; |
| 490 | } |
| 491 | |
| 492 | if (!strcmp(argv[1], "--ungroup")) { |
| 493 | argv += 1; |
| 494 | argc -= 1; |
| 495 | opts.ungroup = 1; |
| 496 | } |
| 497 | |
| 498 | jobs = atoi(argv[2]); |
| 499 | strvec_clear(&proc.args); |
| 500 | strvec_pushv(&proc.args, (const char **)argv + 3); |
| 501 | |
| 502 | if (!strcmp(argv[1], "run-command-parallel")) { |
| 503 | opts.get_next_task = parallel_next; |
| 504 | opts.task_finished = task_finished_quiet; |
| 505 | } else if (!strcmp(argv[1], "run-command-abort")) { |
| 506 | opts.get_next_task = parallel_next; |
| 507 | opts.task_finished = task_finished; |
| 508 | } else if (!strcmp(argv[1], "run-command-no-jobs")) { |
| 509 | opts.get_next_task = no_job; |
| 510 | opts.task_finished = task_finished; |
| 511 | } else if (!strcmp(argv[1], "run-command-stdin")) { |
| 512 | proc.in = -1; |
| 513 | proc.no_stdin = 0; |
| 514 | opts.get_next_task = parallel_next; |
| 515 | opts.task_finished = task_finished_quiet; |
| 516 | opts.feed_pipe = test_stdin_pipe_feed; |
| 517 | } else { |
| 518 | ret = 1; |
| 519 | fprintf(stderr, "check usage\n"); |
| 520 | goto cleanup; |
| 521 | } |
| 522 | opts.processes = jobs; |
| 523 | run_processes_parallel(&opts); |
| 524 | ret = 0; |
| 525 | cleanup: |
| 526 | child_process_clear(&proc); |
| 527 | return ret; |
| 528 | } |