Win32: reliably detect console pipe handles

As of "Win32: Thread-safe windows console output", child processes may print to the console even if stdout has been redirected to a file. E.g.: git config tar.cat.command "cat" git archive -o test.cat HEAD Detecting whether stdout / stderr point to our console pipe is currently based on the assumption that OS HANDLE values are never reused. This is apparently not true if stdout / stderr is replaced via dup2() (as in builtin/archive.c:17). Instead of comparing handle values, check if the file descriptor isatty() backed by a pipe OS handle. This is only possible by swapping the handles in MSVCRT's internal data structures, as we do in winansi_init(). Reported-by: Johannes Sixt <j6t@kdbg.org> Signed-off-by: Karsten Blees <blees@dcon.de> Signed-off-by: Junio C Hamano <gitster@pobox.com>

Karsten Blees committed Jun 14, 2014 at 00:09 UTC 51822653f58e6b9b6119f560ff864813fbde81ba
1 file changed +7 -18
compat/winansi.c
+7 -18
@@ -20,7 +20,6 @@ static WORD attr;
20 static int negative;
21 static int non_ascii_used = 0;
22 static HANDLE hthread, hread, hwrite;
23 -static HANDLE hwrite1 = INVALID_HANDLE_VALUE, hwrite2 = INVALID_HANDLE_VALUE;
23 static HANDLE hconsole1, hconsole2;
24
25 #ifdef __MINGW32__
@@ -435,10 +434,6 @@ static void winansi_exit(void)
434 WaitForSingleObject(hthread, INFINITE);
435
436 /* cleanup handles... */
438 - if (hwrite1 != INVALID_HANDLE_VALUE)
439 - CloseHandle(hwrite1);
440 - if (hwrite2 != INVALID_HANDLE_VALUE)
441 - CloseHandle(hwrite2);
437 CloseHandle(hwrite);
438 CloseHandle(hthread);
439 }
@@ -565,14 +560,9 @@ void winansi_init(void)
560
561 /* redirect stdout / stderr to the pipe */
562 if (con1)
568 - hconsole1 = swap_osfhnd(1, hwrite1 = duplicate_handle(hwrite));
563 + hconsole1 = swap_osfhnd(1, duplicate_handle(hwrite));
564 if (con2)
570 - hconsole2 = swap_osfhnd(2, hwrite2 = duplicate_handle(hwrite));
571 -}
572 -
573 -static int is_same_handle(HANDLE hnd, int fd)
574 -{
575 - return hnd != INVALID_HANDLE_VALUE && hnd == (HANDLE) _get_osfhandle(fd);
565 + hconsole2 = swap_osfhnd(2, duplicate_handle(hwrite));
566 }
567
568 /*
@@ -581,10 +571,9 @@ static int is_same_handle(HANDLE hnd, int fd)
571 */
572 HANDLE winansi_get_osfhandle(int fd)
573 {
584 - if (fd == 1 && is_same_handle(hwrite1, 1))
585 - return hconsole1;
586 - else if (fd == 2 && is_same_handle(hwrite2, 2))
587 - return hconsole2;
588 - else
589 - return (HANDLE) _get_osfhandle(fd);
574 + HANDLE hnd = (HANDLE) _get_osfhandle(fd);
575 + if ((fd == 1 || fd == 2) && isatty(fd)
576 + && GetFileType(hnd) == FILE_TYPE_PIPE)
577 + return (fd == 1) ? hconsole1 : hconsole2;
578 + return hnd;
579 }