HTRACE-278. htraced: dump thread stacks and GC statistics when SIGQUIT is sent (cmccabe)
diff --git a/htrace-htraced/go/src/org/apache/htrace/common/process.go b/htrace-htraced/go/src/org/apache/htrace/common/process.go
index d138178..419d6fe 100644
--- a/htrace-htraced/go/src/org/apache/htrace/common/process.go
+++ b/htrace-htraced/go/src/org/apache/htrace/common/process.go
@@ -24,6 +24,8 @@
"org/apache/htrace/conf"
"os"
"os/signal"
+ "runtime"
+ "runtime/debug"
"syscall"
)
@@ -51,17 +53,52 @@
syscall.SIGBUS,
syscall.SIGFPE,
syscall.SIGILL,
- syscall.SIGQUIT,
syscall.SIGSEGV,
syscall.SIGTERM,
}
- sigChan := make(chan os.Signal, len(fatalSigs))
- signal.Notify(sigChan, fatalSigs...)
- lg := NewLogger("exit", cnf)
+ fatalSigChan := make(chan os.Signal, 1)
+ signal.Notify(fatalSigChan, fatalSigs...)
+ lg := NewLogger("signal", cnf)
go func() {
- sig := <-sigChan
+ sig := <-fatalSigChan
lg.Errorf("Terminating on signal: %v\n", sig)
lg.Close()
os.Exit(1)
}()
+
+ sigQuitChan := make(chan os.Signal, 1)
+ signal.Notify(sigQuitChan, syscall.SIGQUIT)
+ go func() {
+ bufSize := 1<<20
+ buf := make([]byte, bufSize)
+ for {
+ <-sigQuitChan
+ neededBytes := runtime.Stack(buf, true)
+ if neededBytes > bufSize {
+ bufSize = neededBytes
+ buf = make([]byte, bufSize)
+ runtime.Stack(buf, true)
+ }
+ lg.Info("=== received SIGQUIT ===\n")
+ lg.Info("=== GOROUTINE STACKS ===\n")
+ lg.Info(string(buf[:neededBytes]))
+ lg.Info("\n=== END GOROUTINE STACKS ===\n")
+ gcs := debug.GCStats{}
+ debug.ReadGCStats(&gcs)
+ lg.Info("=== GC STATISTICS ===\n")
+ lg.Infof("LastGC: %s\n", gcs.LastGC.UTC().String())
+ lg.Infof("NumGC: %d\n", gcs.NumGC)
+ lg.Infof("PauseTotal: %v\n", gcs.PauseTotal)
+ if gcs.Pause != nil {
+ pauseStr := ""
+ prefix := ""
+ for p := range gcs.Pause {
+ pauseStr += prefix + gcs.Pause[p].String()
+ prefix = ", "
+ }
+ lg.Infof("Pause History: %s\n", pauseStr)
+ }
+ lg.Info("=== END GC STATISTICS ===\n")
+ }
+ }()
}
diff --git a/htrace-htraced/go/src/org/apache/htrace/common/process_test.go b/htrace-htraced/go/src/org/apache/htrace/common/process_test.go
new file mode 100644
index 0000000..7609133
--- /dev/null
+++ b/htrace-htraced/go/src/org/apache/htrace/common/process_test.go
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package common
+
+import (
+ "bufio"
+ "fmt"
+ "org/apache/htrace/conf"
+ "os"
+ "os/exec"
+ "strings"
+ "syscall"
+ "testing"
+ "time"
+)
+
+const HTRACED_TEST_HELPER_PROCESS = "HTRACED_TEST_HELPER_PROCESS"
+
+// This test runs a helper process which will install our htraced signal
+// handlers. We will send signals to the subprocess and verify that it has
+// caught them and responded appropriately.
+func TestSignals(t *testing.T) {
+ if os.Getenv(HTRACED_TEST_HELPER_PROCESS) == "1" {
+ runHelperProcess()
+ os.Exit(0)
+ }
+ helper := exec.Command(os.Args[0], "-test.run=TestSignals", "--")
+ helper.Env = []string { HTRACED_TEST_HELPER_PROCESS + "=1" }
+ stdoutPipe, err := helper.StdoutPipe()
+ if err != nil {
+ panic(fmt.Sprintf("Failed to open pipe to process stdout: %s",
+ err.Error()))
+ }
+ stderrPipe, err := helper.StderrPipe()
+ if err != nil {
+ panic(fmt.Sprintf("Failed to open pipe to process stderr: %s",
+ err.Error()))
+ }
+ err = helper.Start()
+ if err != nil {
+ t.Fatal("Failed to start command %s: %s\n", os.Args[0], err.Error())
+ }
+ t.Logf("Started suprocess...\n")
+ done := make(chan interface{})
+ go func() {
+ scanner := bufio.NewScanner(stdoutPipe)
+ for scanner.Scan() {
+ text := scanner.Text()
+ if strings.Contains(text, "=== GOROUTINE STACKS ===") {
+ break
+ }
+ }
+ t.Logf("Saw 'GOROUTINE STACKS on stdout.' Sending SIGINT.\n")
+ helper.Process.Signal(syscall.SIGINT)
+ for scanner.Scan() {
+ text := scanner.Text()
+ if strings.Contains(text, "Terminating on signal: SIGINT") {
+ break
+ }
+ }
+ t.Logf("Saw 'Terminating on signal: SIGINT'. " +
+ "Helper goroutine exiting.\n")
+ done<-nil
+ }()
+ scanner := bufio.NewScanner(stderrPipe)
+ for scanner.Scan() {
+ text := scanner.Text()
+ if strings.Contains(text, "Signal handler installed.") {
+ break
+ }
+ }
+ t.Logf("Saw 'Signal handler installed.' Sending SIGINT.")
+ helper.Process.Signal(syscall.SIGQUIT)
+ t.Logf("Waiting for helper goroutine to exit.\n")
+ <-done
+ t.Logf("Waiting for subprocess to exit.\n")
+ helper.Wait()
+ t.Logf("Done.")
+}
+
+// Run the helper process which TestSignals spawns.
+func runHelperProcess() {
+ cnfMap := map[string]string {
+ conf.HTRACE_LOG_LEVEL: "TRACE",
+ conf.HTRACE_LOG_PATH: "", // log to stdout
+ }
+ cnfBld := conf.Builder{Values: cnfMap, Defaults: conf.DEFAULTS}
+ cnf, err := cnfBld.Build()
+ if err != nil {
+ fmt.Printf("Error building configuration: %s\n", err.Error())
+ os.Exit(1)
+ }
+ InstallSignalHandlers(cnf)
+ fmt.Fprintf(os.Stderr, "Signal handler installed.\n")
+ // Wait for a signal to be delivered
+ for {
+ time.Sleep(time.Hour * 100)
+ }
+}
