main: wait for interrupt to finish before ending command invocation
If a command invocation such as 'daemon' is interrupted, the interrupt handler asks the node to close. The closing of the node will result in the command invocation finishing, and possibly returning from main() before the interrupt handler is done. In particular, the info logging that a graceful shutdown was completed may never reach reach stdout. As the whole point of logging "Gracefully shut down." is to give confidence when debugging that the shutdown was clean, this is slightly unfortunate. The interrupt handler needs to be set up in main() instead of Run(), so that we can defer the closing of the interrupt handler until just before returning from main, not when Run() returns with a streaming result reader.
Tor Arne Vestbø committed
Apr 17, 2015 at 18:46 UTC
00a6e595ba49cec48dcf082f6140fbfc034e2049
1 file changed
+67
-37
cmd/ipfs/main.go
+67
-37
@@ -11,6 +11,7 @@ import (
11
"runtime"
12
"runtime/pprof"
13
"strings"
14
+ "sync"
15
"syscall"
16
"time"
17
@@ -39,7 +40,6 @@ const (
40
cpuProfile = "ipfs.cpuprof"
41
heapProfile = "ipfs.memprof"
42
errorFormat = "ERROR: %v\n\n"
42
- shutdownMessage = "Received interrupt signal, shutting down..."
43
)
44
45
type cmdInvocation struct {
@@ -141,6 +141,8 @@ func main() {
141
}
142
143
// ok, finally, run the command invocation.
144
+ intrh := invoc.SetupInterruptHandler()
145
+ defer intrh.Close()
146
output, err := invoc.Run(ctx)
147
if err != nil {
148
printErr(err)
@@ -157,8 +159,6 @@ func main() {
159
}
160
161
func (i *cmdInvocation) Run(ctx context.Context) (output io.Reader, err error) {
160
- // setup our global interrupt handler.
161
- i.setupInterruptHandler()
162
163
// check if user wants to debug. option OR env var.
164
debug, _, err := i.req.Option("debug").Bool()
@@ -474,57 +474,87 @@ func writeHeapProfileToFile() error {
474
return pprof.WriteHeapProfile(mprof)
475
}
476
477
-// listen for and handle SIGTERM
478
-func (i *cmdInvocation) setupInterruptHandler() {
477
+// IntrHandler helps set up an interrupt handler that can
478
+// be cleanly shut down through the io.Closer interface.
479
+type IntrHandler struct {
480
+ sig chan os.Signal
481
+ wg sync.WaitGroup
482
+}
483
+
484
+func NewIntrHandler() *IntrHandler {
485
+ ih := &IntrHandler{}
486
+ ih.sig = make(chan os.Signal, 1)
487
+ return ih
488
+}
489
+
490
+func (ih *IntrHandler) Close() error {
491
+ close(ih.sig)
492
+ ih.wg.Wait()
493
+ return nil
494
+}
495
480
- ctx := i.req.Context()
481
- sig := allInterruptSignals()
496
497
+// Handle starts handling the given signals, and will call the handler
498
+// callback function each time a signal is catched. The function is passed
499
+// the number of times the handler has been triggered in total, as
500
+// well as the handler itself, so that the handling logic can use the
501
+// handler's wait group to ensure clean shutdown when Close() is called.
502
+func (ih *IntrHandler) Handle(handler func(count int, ih *IntrHandler), sigs ...os.Signal) {
503
+ signal.Notify(ih.sig, sigs...)
504
+ ih.wg.Add(1)
505
go func() {
484
- // first time, try to shut down.
506
+ defer ih.wg.Done()
507
+ count := 0
508
+ for _ = range ih.sig {
509
+ count++
510
+ handler(count, ih)
511
+ }
512
+ signal.Stop(ih.sig)
513
+ }()
514
+}
515
+
516
+func (i *cmdInvocation) SetupInterruptHandler() io.Closer {
517
486
- // loop because we may be
487
- for count := 0; ; count++ {
488
- <-sig
518
+ intrh := NewIntrHandler()
519
+ handlerFunc := func(count int, ih *IntrHandler) {
520
+ switch count {
521
+ case 1:
522
+ // first time, try to shut down
523
+ fmt.Println("Received interrupt signal, shutting down...")
524
+
525
+ ctx := i.req.Context()
526
527
// if we're still initializing, cannot use `ctx.GetNode()`
528
select {
529
default: // initialization not done
493
- fmt.Println(shutdownMessage)
530
os.Exit(-1)
531
case <-ctx.InitDone:
532
}
533
498
- // TODO cancel the command context instead
534
+ ih.wg.Add(1)
535
+ go func() {
536
+ defer ih.wg.Done()
537
500
- n, err := ctx.GetNode()
501
- if err != nil {
502
- log.Error(err)
503
- fmt.Println(shutdownMessage)
504
- os.Exit(-1)
505
- }
538
+ // TODO cancel the command context instead
539
+ n, err := ctx.GetNode()
540
+ if err != nil {
541
+ log.Error(err)
542
+ os.Exit(-1)
543
+ }
544
507
- switch count {
508
- case 0:
509
- fmt.Println(shutdownMessage)
510
- go func() {
511
- n.Close()
512
- log.Info("Gracefully shut down.")
513
- }()
545
+ n.Close()
546
+ log.Info("Gracefully shut down.")
547
+ }()
548
515
- default:
516
- fmt.Println("Received another interrupt before graceful shutdown, terminating...")
517
- os.Exit(-1)
518
- }
549
+ default:
550
+ fmt.Println("Received another interrupt before graceful shutdown, terminating...")
551
+ os.Exit(-1)
552
}
520
- }()
521
-}
553
+ }
554
+
555
+ intrh.Handle(handlerFunc, syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM)
556
523
-func allInterruptSignals() chan os.Signal {
524
- sigc := make(chan os.Signal, 1)
525
- signal.Notify(sigc, syscall.SIGHUP, syscall.SIGINT,
526
- syscall.SIGTERM)
527
- return sigc
557
+ return intrh
558
}
559
560
func profileIfEnabled() (func(), error) {