8267893: Improve jtreg test failure handler do get native/mixed stack traces for cores and live processes

Reviewed-by: iignatyev

test/failure_handler/Makefile

@@ -1,5 +1,5 @@
 #
-# Copyright (c) 2015, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2015, 2021, Oracle and/or its affiliates. All rights reserved.
 # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 #
 # This code is free software; you can redistribute it and/or modify it
@@ -38,7 +38,7 @@ SOURCES := ${SRC_DIR}/jdk/test/failurehandler/*.java                   \
 
 CONF_DIR = src/share/conf
 
-JAVA_RELEASE = 7
+JAVA_RELEASE = 15
 
 TARGET_JAR = ${IMAGE_DIR}/lib/jtregFailureHandler.jar
 
@@ -116,4 +116,3 @@ build: classes native
 
 .PHONY: all build classes native test require_env clean
 .DEFAULT: all
-

test/failure_handler/README

@@ -1,4 +1,4 @@
-Copyright (c) 2015, Oracle and/or its affiliates. All rights reserved.
+Copyright (c) 2015, 2021, Oracle and/or its affiliates. All rights reserved.
 DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 
 This code is free software; you can redistribute it and/or modify it
@@ -32,7 +32,7 @@ The library uses JTHarness Observer and jtreg TimeoutHandler extensions points.
 
 DEPENDENCES
 
-The library requires jtreg 4b13+ and JDK 7+.
+The library requires jtreg 4b13+ and JDK 15+.
 
 BUILDING
 
@@ -102,4 +102,3 @@ $ ${JTREG_HOME}/bin/jtreg -jdk:${JAVA_HOME}                                   \
  -timeoutHandler:jdk.test.failurehandler.jtreg.GatherProcessInfoTimeoutHandler\
  -observer:jdk.test.failurehandler.jtreg.GatherDiagnosticInfoObserver         \
  ${WS}/hotspot/test/
-

test/failure_handler/src/share/classes/jdk/test/failurehandler/CoreInfoGatherer.java

@@ -0,0 +1,30 @@
+/*
+ * Copyright (c) 2021, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+package jdk.test.failurehandler;
+
+import java.nio.file.Path;
+
+public interface CoreInfoGatherer {
+    void gatherCoreInfo(HtmlSection section, Path core);
+}

test/failure_handler/src/share/classes/jdk/test/failurehandler/GathererFactory.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, 2018, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015, 2021, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -51,6 +51,10 @@ public final class GathererFactory {
         return create();
     }
 
+    public CoreInfoGatherer getCoreInfoGatherer() {
+        return create();
+    }
+
     private ToolKit create() {
         Properties osProperty = Utils.getProperties(osName);
         try {

test/failure_handler/src/share/classes/jdk/test/failurehandler/ToolKit.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, 2019, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015, 2021, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -26,19 +26,25 @@ package jdk.test.failurehandler;
 import jdk.test.failurehandler.action.ActionSet;
 import jdk.test.failurehandler.action.ActionHelper;
 
+import java.io.IOException;
 import java.io.PrintWriter;
+import java.nio.file.Files;
+import java.nio.file.Path;
 import java.util.ArrayList;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Queue;
 import java.util.Deque;
+import java.util.zip.GZIPInputStream;
 
-public class ToolKit implements EnvironmentInfoGatherer, ProcessInfoGatherer {
+public class ToolKit implements EnvironmentInfoGatherer, ProcessInfoGatherer, CoreInfoGatherer {
     private final List<ActionSet> actions = new ArrayList<>();
     private final ActionHelper helper;
+    private final PrintWriter log;
 
     public ToolKit(ActionHelper helper, PrintWriter log, String... names) {
         this.helper = helper;
+        this.log = log;
         for (String name : names) {
             actions.add(new ActionSet(helper, log, name));
         }
@@ -51,6 +57,27 @@ public class ToolKit implements EnvironmentInfoGatherer, ProcessInfoGatherer {
         }
     }
 
+    @Override
+    public void gatherCoreInfo(HtmlSection section, Path core) {
+        if (core.getFileName().toString().endsWith(".gz")) {
+            Path unpackedCore = Path.of(core.toString().replace(".gz", ""));
+            try (GZIPInputStream gzis = new GZIPInputStream(Files.newInputStream(core))) {
+                Files.copy(gzis, unpackedCore);
+                for (ActionSet set : actions) {
+                    set.gatherCoreInfo(section, unpackedCore);
+                }
+                Files.delete(unpackedCore);
+            } catch (IOException ioe) {
+                log.printf("Unexpected exception whilc opening %s", core.getFileName().toString());
+                ioe.printStackTrace(log);
+            }
+        } else {
+            for (ActionSet set : actions) {
+                set.gatherCoreInfo(section, core);
+            }
+        }
+    }
+
     @Override
     public void gatherProcessInfo(HtmlSection section, long pid) {
         // as some of actions can kill a process, we need to get children of all

test/failure_handler/src/share/classes/jdk/test/failurehandler/action/ActionHelper.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015, 2021, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -23,8 +23,6 @@
 
 package jdk.test.failurehandler.action;
 
-import com.sun.tools.attach.VirtualMachine;
-import com.sun.tools.attach.VirtualMachineDescriptor;
 import jdk.test.failurehandler.value.InvalidValueException;
 import jdk.test.failurehandler.value.Value;
 import jdk.test.failurehandler.value.ValueHandler;
@@ -134,7 +132,7 @@ public class ActionHelper {
                 .directory(workDir.toFile());
     }
 
-    private File findApp(String app) {
+    public File findApp(String app) {
         String name = app + executableSuffix;
         for (Path pathElem : paths) {
             File result = pathElem.resolve(name).toFile();

test/failure_handler/src/share/classes/jdk/test/failurehandler/action/ActionSet.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015, 2021, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -23,19 +23,26 @@
 
 package jdk.test.failurehandler.action;
 
+import jdk.test.failurehandler.CoreInfoGatherer;
 import jdk.test.failurehandler.ProcessInfoGatherer;
 import jdk.test.failurehandler.EnvironmentInfoGatherer;
 import jdk.test.failurehandler.HtmlSection;
 import jdk.test.failurehandler.Utils;
 
+import java.io.IOException;
 import java.io.PrintWriter;
+import java.nio.file.Files;
+import java.nio.file.Path;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Properties;
+import java.util.zip.GZIPInputStream;
 
-public class ActionSet implements ProcessInfoGatherer, EnvironmentInfoGatherer {
+public class ActionSet implements ProcessInfoGatherer, EnvironmentInfoGatherer, CoreInfoGatherer {
     private static final String ENVIRONMENT_PROPERTY = "environment";
     private static final String ON_PID_PROPERTY = "onTimeout";
+    private static final String CORES_PROPERTY = "cores";
+
 
     private final ActionHelper helper;
 
@@ -46,6 +53,7 @@ public class ActionSet implements ProcessInfoGatherer, EnvironmentInfoGatherer {
     private final String name;
     private final List<SimpleAction> environmentActions;
     private final List<PatternAction> processActions;
+    private final List<PatternAction> coreActions;
 
 
     public ActionSet(ActionHelper helper, PrintWriter log, String name) {
@@ -55,6 +63,7 @@ public class ActionSet implements ProcessInfoGatherer, EnvironmentInfoGatherer {
         Properties p = Utils.getProperties(name);
         environmentActions = getSimpleActions(log, p, ENVIRONMENT_PROPERTY);
         processActions = getPatternActions(log, p, ON_PID_PROPERTY);
+        coreActions = getPatternActions(log, p, CORES_PROPERTY);
     }
 
     private List<SimpleAction> getSimpleActions(PrintWriter log, Properties p,
@@ -123,4 +132,11 @@ public class ActionSet implements ProcessInfoGatherer, EnvironmentInfoGatherer {
             helper.runPatternAction(action, section);
         }
     }
+
+    @Override
+    public void gatherCoreInfo(HtmlSection section, Path core) {
+        for (PatternAction action : coreActions) {
+            helper.runPatternAction(action, section, core.toString());
+        }
+    }
 }

test/failure_handler/src/share/classes/jdk/test/failurehandler/action/PatternAction.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, 2019, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015, 2021, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -59,6 +59,9 @@ public class PatternAction implements Action {
         for (int i = 0, n = args.length; i < n; ++i) {
             args[i] = args[i].replace(pattern, value) ;
         }
+        for (int i = 0, n = args.length; i < n; ++i) {
+            args[i] = args[i].replace("%java", helper.findApp("java").getAbsolutePath());
+        }
         return action.prepareProcess(section.getWriter(), helper);
     }
 

test/failure_handler/src/share/classes/jdk/test/failurehandler/jtreg/GatherDiagnosticInfoObserver.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015, 2021, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -33,10 +33,9 @@ import java.io.File;
 import java.io.FileWriter;
 import java.io.IOException;
 import java.io.PrintWriter;
+import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
-import java.util.HashMap;
-import java.util.Map;
 
 /**
  * The jtreg test execution observer, which gathers info about
@@ -45,6 +44,8 @@ import java.util.Map;
 public class GatherDiagnosticInfoObserver implements Harness.Observer {
     public static final String LOG_FILENAME = "environment.log";
     public static final String ENVIRONMENT_OUTPUT = "environment.html";
+    public static final String CORES_OUTPUT = "cores.html";
+
 
     private Path compileJdk;
     private Path testJdk;
@@ -64,18 +65,19 @@ public class GatherDiagnosticInfoObserver implements Harness.Observer {
         workDir.toFile().mkdir();
 
         String name = getClass().getName();
-        PrintWriter log;
+        PrintWriter log1;
         boolean needClose = false;
         try {
-            log = new PrintWriter(new FileWriter(
+            log1 = new PrintWriter(new FileWriter(
                     workDir.resolve(LOG_FILENAME).toFile(), true), true);
             needClose = true;
         } catch (IOException e) {
-            log = new PrintWriter(System.out);
+            log1 = new PrintWriter(System.out);
-            log.printf("ERROR: %s cannot open log file %s", name,
+            log1.printf("ERROR: %s cannot open log file %s", name,
                     LOG_FILENAME);
-            e.printStackTrace(log);
+            e.printStackTrace(log1);
         }
+        final PrintWriter log = log1;
         try {
             log.printf("%s ---%n", name);
             GathererFactory gathererFactory = new GathererFactory(
@@ -83,6 +85,11 @@ public class GatherDiagnosticInfoObserver implements Harness.Observer {
                     testJdk, compileJdk);
             gatherEnvInfo(workDir, name, log,
                     gathererFactory.getEnvironmentInfoGatherer());
+            Files.walk(workDir)
+                    .filter(Files::isRegularFile)
+                    .filter(f -> (f.getFileName().toString().contains("core") || f.getFileName().toString().contains("mdmp")))
+                    .forEach(core -> gatherCoreInfo(workDir, name,
+                            core, log, gathererFactory.getCoreInfoGatherer()));
         } catch (Throwable e) {
             log.printf("ERROR: exception in observer %s:", name);
             e.printStackTrace(log);
@@ -96,6 +103,22 @@ public class GatherDiagnosticInfoObserver implements Harness.Observer {
         }
     }
 
+    private void gatherCoreInfo(Path workDir, String name, Path core, PrintWriter log,
+                               CoreInfoGatherer gatherer) {
+        File output = workDir.resolve(CORES_OUTPUT).toFile();
+        try (HtmlPage html = new HtmlPage(new PrintWriter(
+                new FileWriter(output, true), true))) {
+            try (ElapsedTimePrinter timePrinter
+                         = new ElapsedTimePrinter(new Stopwatch(), name, log)) {
+                gatherer.gatherCoreInfo(html.getRootSection(), core);
+            }
+        } catch (Throwable e) {
+            log.printf("ERROR: exception in observer on getting environment "
+                    + "information %s:", name);
+            e.printStackTrace(log);
+        }
+    }
+
     private void gatherEnvInfo(Path workDir, String name, PrintWriter log,
                                EnvironmentInfoGatherer gatherer) {
         File output = workDir.resolve(ENVIRONMENT_OUTPUT).toFile();

test/failure_handler/src/share/conf/common.properties

@@ -1,5 +1,5 @@
 #
-# Copyright (c) 2015, 2019, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2015, 2021, Oracle and/or its affiliates. All rights reserved.
 # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 #
 # This code is free software; you can redistribute it and/or modify it
@@ -35,7 +35,7 @@ onTimeout=\
         jcmd.vm.symboltable jcmd.vm.uptime jcmd.vm.dynlibs \
         jcmd.vm.system_properties \
   jcmd.gc.heap_info jcmd.gc.class_histogram jcmd.gc.finalizer_info \
-  jstack
+  jstack jhsdb.jstack
 
 jinfo.app=jinfo
 
@@ -51,6 +51,7 @@ jcmd.vm.symboltable.args=%p VM.symboltable
 jcmd.vm.uptime.args=%p VM.uptime
 jcmd.vm.dynlibs.args=%p VM.dynlibs
 jcmd.vm.system_properties.args=%p VM.system_properties
+jcmd.vm.info.args=%p VM.info
 
 jcmd.gc.class_histogram.args=%p GC.class_histogram
 jcmd.gc.finalizer_info.args=%p GC.finalizer_info
@@ -60,6 +61,15 @@ jstack.app=jstack
 jstack.args=-e -l %p
 jstack.params.repeat=6
 
+jhsdb.app=jhsdb
+jhsdb.jstack.args=jstack --mixed --pid %p
+jhsdb.jstack.params.repeat=6
+
+cores=jhsdb.jstack
+jhsdb.jstack.app=jhsdb
+# Assume that java standard laucher has been used
+jhsdb.jstack.args=jstack --mixed --core %p --exe %java
+
 ################################################################################
 # environment info to gather
 ################################################################################

test/failure_handler/src/share/conf/linux.properties

@@ -1,5 +1,5 @@
 #
-# Copyright (c) 2015, 2020, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2015, 2021, Oracle and/or its affiliates. All rights reserved.
 # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 #
 # This code is free software; you can redistribute it and/or modify it
@@ -32,6 +32,7 @@ onTimeout=\
   native.pmap.normal native.pmap.everything \
   native.files native.locks \
   native.stack native.core
+
 ################################################################################
 native.pattern=%p
 native.javaOnly=false
@@ -55,6 +56,13 @@ native.stack.params.repeat=6
 # has to be the last command
 native.core.app=kill
 native.core.args=-ABRT %p
+
+cores=native.gdb
+native.gdb.app=gdb
+# Assume that java standard laucher has been used
+native.gdb.args=%java\0-c\0%p\0-batch\0-ex\0thread apply all backtrace
+native.gdb.args.delimiter=\0
+
 ################################################################################
 # environment info to gather
 ################################################################################
@@ -123,4 +131,3 @@ screenshot.args=-c\0\
     ' | jshell -
 screenshot.args.delimiter=\0
 ################################################################################
-

test/failure_handler/src/share/conf/mac.properties

@@ -1,5 +1,5 @@
 #
-# Copyright (c) 2015, 2020, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2015, 2021, Oracle and/or its affiliates. All rights reserved.
 # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 #
 # This code is free software; you can redistribute it and/or modify it
@@ -64,6 +64,13 @@ native.stack.args=-c\0DevToolsSecurity --status | grep -q enabled && lldb -o 'at
 # has to be the last command
 native.core.app=kill
 native.core.args=-ABRT %p
+
+cores=native.lldb
+native.lldb.app=lldb
+native.lldb.delimiter=\0
+# Assume that java standard laucher has been used
+native.lldb.args=--core\0%p\0%java\0-o\0thread backtrace all\0-o\0quit
+
 ################################################################################
 # environment info to gather
 ################################################################################