mingw: make isatty() recognize MSYS2's pseudo terminals (/dev/pty*)

MSYS2 emulates pseudo terminals via named pipes, and isatty() returns 0 for such file descriptors. Therefore, some interactive functionality (such as launching a pager, asking if a failed unlink should be repeated etc.) doesn't work when run in a terminal emulator that uses MSYS2's ptys (such as mintty). However, MSYS2 uses special names for its pty pipes ('msys-*-pty*'), which allows us to distinguish them from normal piped input / output. On startup, check if stdin / stdout / stderr are connected to such pipes using the NtQueryObject API from NTDll.dll. If the names match, adjust the flags in MSVCRT's ioinfo structure accordingly. Signed-off-by: Karsten Blees <blees@dcon.de> Signed-off-by: Johannes Schindelin <johannes.schindelin@gmx.de> Signed-off-by: Junio C Hamano <gitster@pobox.com>

Karsten Blees committed Apr 27, 2016 at 17:16 UTC f7f90e0f4f58d493242078d17c0eba41dd3f1f79
2 files changed +56 -5
compat/winansi.c
+54 -4
@@ -483,6 +483,7 @@ static size_t sizeof_ioinfo = 0;
483 #define IOINFO_L2E 5
484 #define IOINFO_ARRAY_ELTS (1 << IOINFO_L2E)
485
486 +#define FPIPE 0x08
487 #define FDEV 0x40
488
489 static inline ioinfo* _pioinfo(int fd)
@@ -530,6 +531,45 @@ static HANDLE swap_osfhnd(int fd, HANDLE new_handle)
531 return old_handle;
532 }
533
534 +#ifdef DETECT_MSYS_TTY
535 +
536 +#include <winternl.h>
537 +#include <ntstatus.h>
538 +
539 +static void detect_msys_tty(int fd)
540 +{
541 + ULONG result;
542 + BYTE buffer[1024];
543 + POBJECT_NAME_INFORMATION nameinfo = (POBJECT_NAME_INFORMATION) buffer;
544 + PWSTR name;
545 +
546 + /* check if fd is a pipe */
547 + HANDLE h = (HANDLE) _get_osfhandle(fd);
548 + if (GetFileType(h) != FILE_TYPE_PIPE)
549 + return;
550 +
551 + /* get pipe name */
552 + if (!NT_SUCCESS(NtQueryObject(h, ObjectNameInformation,
553 + buffer, sizeof(buffer) - 2, &result)))
554 + return;
555 + name = nameinfo->Name.Buffer;
556 + name[nameinfo->Name.Length] = 0;
557 +
558 + /* check if this could be a MSYS2 pty pipe ('msys-XXXX-ptyN-XX') */
559 + if (!wcsstr(name, L"msys-") || !wcsstr(name, L"-pty"))
560 + return;
561 +
562 + /* init ioinfo size if we haven't done so */
563 + if (init_sizeof_ioinfo())
564 + return;
565 +
566 + /* set FDEV flag, reset FPIPE flag */
567 + _pioinfo(fd)->osflags &= ~FPIPE;
568 + _pioinfo(fd)->osflags |= FDEV;
569 +}
570 +
571 +#endif
572 +
573 void winansi_init(void)
574 {
575 int con1, con2;
@@ -538,8 +578,15 @@ void winansi_init(void)
578 /* check if either stdout or stderr is a console output screen buffer */
579 con1 = is_console(1);
580 con2 = is_console(2);
541 - if (!con1 && !con2)
581 + if (!con1 && !con2) {
582 +#ifdef DETECT_MSYS_TTY
583 + /* check if stdin / stdout / stderr are MSYS2 pty pipes */
584 + detect_msys_tty(0);
585 + detect_msys_tty(1);
586 + detect_msys_tty(2);
587 +#endif
588 return;
589 + }
590
591 /* create a named pipe to communicate with the console thread */
592 xsnprintf(name, sizeof(name), "\\\\.\\pipe\\winansi%lu", GetCurrentProcessId());
@@ -575,8 +622,11 @@ void winansi_init(void)
622 HANDLE winansi_get_osfhandle(int fd)
623 {
624 HANDLE hnd = (HANDLE) _get_osfhandle(fd);
578 - if ((fd == 1 || fd == 2) && isatty(fd)
579 - && GetFileType(hnd) == FILE_TYPE_PIPE)
580 - return (fd == 1) ? hconsole1 : hconsole2;
625 + if (isatty(fd) && GetFileType(hnd) == FILE_TYPE_PIPE) {
626 + if (fd == 1 && hconsole1)
627 + return hconsole1;
628 + else if (fd == 2 && hconsole2)
629 + return hconsole2;
630 + }
631 return hnd;
632 }
config.mak.uname
+2 -1
@@ -557,7 +557,8 @@ else
557 BASIC_LDFLAGS += -Wl,--large-address-aware
558 endif
559 CC = gcc
560 - COMPAT_CFLAGS += -D__USE_MINGW_ANSI_STDIO=0
560 + COMPAT_CFLAGS += -D__USE_MINGW_ANSI_STDIO=0 -DDETECT_MSYS_TTY
561 + EXTLIBS += -lntdll
562 INSTALL = /bin/install
563 NO_R_TO_GCC_LINKER = YesPlease
564 INTERNAL_QSORT = YesPlease