8260282: Add option to compress heap dumps created by -XX:+HeapDumpOnOutOfMemoryError

Reviewed-by: cjplummer, stuefe

src/hotspot/share/runtime/globals.hpp

@@ -554,6 +554,12 @@ const intx ObjectAlignmentInBytes = 8;
           "directory) of the dump file (defaults to java_pid<pid>.hprof "   \
           "in the working directory)")                                      \
                                                                             \
+  product(intx, HeapDumpGzipLevel, 0, MANAGEABLE,                           \
+          "When HeapDumpOnOutOfMemoryError is on, the gzip compression "    \
+          "level of the dump file. 0 (the default) disables gzip "          \
+          "compression. Otherwise the level must be between 1 and 9.")      \
+          range(0, 9)                                                       \
+                                                                            \
   product(ccstr, NativeMemoryTracking, "off",                               \
           "Native memory tracking options")                                 \
                                                                             \

src/hotspot/share/services/heapDumper.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2005, 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2005, 2021, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -2029,7 +2029,7 @@ void HeapDumper::dump_heap(bool oome) {
   const int max_digit_chars = 20;
 
   const char* dump_file_name = "java_pid";
-  const char* dump_file_ext  = ".hprof";
+  const char* dump_file_ext  = HeapDumpGzipLevel > 0 ? ".hprof.gz" : ".hprof";
 
   // The dump file defaults to java_pid<pid>.hprof in the current working
   // directory. HeapDumpPath=<file> can be used to specify an alternative
@@ -2096,6 +2096,6 @@ void HeapDumper::dump_heap(bool oome) {
 
   HeapDumper dumper(false /* no GC before heap dump */,
                     oome  /* pass along out-of-memory-error flag */);
-  dumper.dump(my_path, tty);
+  dumper.dump(my_path, tty, HeapDumpGzipLevel);
   os::free(my_path);
 }

test/hotspot/jtreg/runtime/ErrorHandling/TestGZippedHeapDumpOnOutOfMemoryError.java

@@ -0,0 +1,83 @@
+/*
+ * Copyright (c) 2021 SAP SE. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+/*
+ * @test
+ * @summary Test verifies that -XX:HeapDumpGzipLevel=0 works
+ * @library /test/lib
+ * @run driver/timeout=240 TestGZippedHeapDumpOnOutOfMemoryError run 0
+ */
+
+/*
+ * @test
+ * @summary Test verifies that -XX:HeapDumpGzipLevel=1 works
+ * @library /test/lib
+ * @run driver/timeout=240 TestGZippedHeapDumpOnOutOfMemoryError run 1
+ */
+
+import jdk.test.lib.Asserts;
+import jdk.test.lib.hprof.HprofParser;
+import jdk.test.lib.process.ProcessTools;
+import jdk.test.lib.process.OutputAnalyzer;
+
+import java.io.File;
+
+public class TestGZippedHeapDumpOnOutOfMemoryError {
+
+    static volatile Object[] oa;
+
+    public static void main(String[] args) throws Exception {
+        if (args.length == 2) {
+            test(Integer.parseInt(args[1]));
+            return;
+        }
+
+        try {
+            oa = new Object[Integer.MAX_VALUE];
+            throw new Error("OOME not triggered");
+        } catch (OutOfMemoryError err) {
+            // Ignore
+        }
+    }
+
+    static void test(int level) throws Exception {
+        ProcessBuilder pb = ProcessTools.createJavaProcessBuilder(
+            "-XX:+HeapDumpOnOutOfMemoryError",
+            "-XX:HeapDumpGzipLevel=" + level,
+            "-Xmx128M",
+            TestGZippedHeapDumpOnOutOfMemoryError.class.getName());
+
+        Process proc = pb.start();
+        String heapdumpFilename = "java_pid" + proc.pid() + ".hprof" + (level > 0 ? ".gz" : "");
+        OutputAnalyzer output = new OutputAnalyzer(proc);
+        output.stdoutShouldNotBeEmpty();
+        output.shouldContain("Dumping heap to " + heapdumpFilename);
+        File dump = new File(heapdumpFilename);
+        Asserts.assertTrue(dump.exists() && dump.isFile(),
+                "Could not find dump file " + dump.getAbsolutePath());
+
+        HprofParser.parse(new File(heapdumpFilename));
+        System.out.println("PASSED");
+    }
+
+}