8149591: Prepare hotspot for GTest

Co-authored-by: Stefan Karlsson <stefan.karlsson@oracle.com>
Co-authored-by: Stefan Sarne <stefan.sarne@oracle.com>
Co-authored-by: Jesper Wilhelmsson <jesper.wilhelmsson@oracle.com>
Co-authored-by: Erik Helin <erik.helin@oracle.com>
Co-authored-by: Alexandre Iline <alexandre.iline@oracle.com>
Reviewed-by: jwilhelm

hotspot/src/os/posix/vm/os_posix.cpp

@@ -47,6 +47,12 @@
 
 // Check core dump limit and report possible place where core can be found
 void os::check_dump_limit(char* buffer, size_t bufferSize) {
+  if (!FLAG_IS_DEFAULT(CreateCoredumpOnCrash) && !CreateCoredumpOnCrash) {
+    jio_snprintf(buffer, bufferSize, "CreateCoredumpOnCrash is disabled from command line");
+    VMError::record_coredump_status(buffer, false);
+    return;
+  }
+
   int n;
   struct rlimit rlim;
   bool success;

hotspot/src/share/vm/memory/allocation.cpp

@@ -664,64 +664,6 @@ void* Arena::internal_malloc_4(size_t x) {
 // Non-product code
 
 #ifndef PRODUCT
-// The global operator new should never be called since it will usually indicate
-// a memory leak.  Use CHeapObj as the base class of such objects to make it explicit
-// that they're allocated on the C heap.
-// Commented out in product version to avoid conflicts with third-party C++ native code.
-//
-// In C++98/03 the throwing new operators are defined with the following signature:
-//
-// void* operator new(std::size_tsize) throw(std::bad_alloc);
-// void* operator new[](std::size_tsize) throw(std::bad_alloc);
-//
-// while all the other (non-throwing) new and delete operators are defined with an empty
-// throw clause (i.e. "operator delete(void* p) throw()") which means that they do not
-// throw any exceptions (see section 18.4 of the C++ standard).
-//
-// In the new C++11/14 standard, the signature of the throwing new operators was changed
-// by completely omitting the throw clause (which effectively means they could throw any
-// exception) while all the other new/delete operators where changed to have a 'nothrow'
-// clause instead of an empty throw clause.
-//
-// Unfortunately, the support for exception specifications among C++ compilers is still
-// very fragile. While some more strict compilers like AIX xlC or HP aCC reject to
-// override the default throwing new operator with a user operator with an empty throw()
-// clause, the MS Visual C++ compiler warns for every non-empty throw clause like
-// throw(std::bad_alloc) that it will ignore the exception specification. The following
-// operator definitions have been checked to correctly work with all currently supported
-// compilers and they should be upwards compatible with C++11/14. Therefore
-// PLEASE BE CAREFUL if you change the signature of the following operators!
-
-static void * zero = (void *) 0;
-
-void* operator new(size_t size) /* throw(std::bad_alloc) */ {
-  fatal("Should not call global operator new");
-  return zero;
-}
-
-void* operator new [](size_t size) /* throw(std::bad_alloc) */ {
-  fatal("Should not call global operator new[]");
-  return zero;
-}
-
-void* operator new(size_t size, const std::nothrow_t&  nothrow_constant) throw() {
-  fatal("Should not call global operator new");
-  return 0;
-}
-
-void* operator new [](size_t size, std::nothrow_t&  nothrow_constant) throw() {
-  fatal("Should not call global operator new[]");
-  return 0;
-}
-
-void operator delete(void* p) throw() {
-  fatal("Should not call global delete");
-}
-
-void operator delete [](void* p) throw() {
-  fatal("Should not call global delete []");
-}
-
 void AllocatedObj::print() const       { print_on(tty); }
 void AllocatedObj::print_value() const { print_value_on(tty); }
 

hotspot/src/share/vm/memory/operator_new.cpp

@@ -0,0 +1,92 @@
+/*
+ * Copyright (c) 1997, 2014, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#include "precompiled.hpp"
+#include "utilities/debug.hpp"
+
+#include <new>
+
+//--------------------------------------------------------------------------------------
+// Non-product code
+
+#ifndef PRODUCT
+// The global operator new should never be called since it will usually indicate
+// a memory leak.  Use CHeapObj as the base class of such objects to make it explicit
+// that they're allocated on the C heap.
+// Commented out in product version to avoid conflicts with third-party C++ native code.
+//
+// In C++98/03 the throwing new operators are defined with the following signature:
+//
+// void* operator new(std::size_tsize) throw(std::bad_alloc);
+// void* operator new[](std::size_tsize) throw(std::bad_alloc);
+//
+// while all the other (non-throwing) new and delete operators are defined with an empty
+// throw clause (i.e. "operator delete(void* p) throw()") which means that they do not
+// throw any exceptions (see section 18.4 of the C++ standard).
+//
+// In the new C++11/14 standard, the signature of the throwing new operators was changed
+// by completely omitting the throw clause (which effectively means they could throw any
+// exception) while all the other new/delete operators where changed to have a 'nothrow'
+// clause instead of an empty throw clause.
+//
+// Unfortunately, the support for exception specifications among C++ compilers is still
+// very fragile. While some more strict compilers like AIX xlC or HP aCC reject to
+// override the default throwing new operator with a user operator with an empty throw()
+// clause, the MS Visual C++ compiler warns for every non-empty throw clause like
+// throw(std::bad_alloc) that it will ignore the exception specification. The following
+// operator definitions have been checked to correctly work with all currently supported
+// compilers and they should be upwards compatible with C++11/14. Therefore
+// PLEASE BE CAREFUL if you change the signature of the following operators!
+
+static void * zero = (void *) 0;
+
+void* operator new(size_t size) /* throw(std::bad_alloc) */ {
+  fatal("Should not call global operator new");
+  return zero;
+}
+
+void* operator new [](size_t size) /* throw(std::bad_alloc) */ {
+  fatal("Should not call global operator new[]");
+  return zero;
+}
+
+void* operator new(size_t size, const std::nothrow_t&  nothrow_constant) throw() {
+  fatal("Should not call global operator new");
+  return 0;
+}
+
+void* operator new [](size_t size, std::nothrow_t&  nothrow_constant) throw() {
+  fatal("Should not call global operator new[]");
+  return 0;
+}
+
+void operator delete(void* p) throw() {
+  fatal("Should not call global delete");
+}
+
+void operator delete [](void* p) throw() {
+  fatal("Should not call global delete []");
+}
+
+#endif // Non-product

hotspot/src/share/vm/runtime/globals.hpp

@@ -2070,6 +2070,9 @@ public:
   notproduct(bool, VerboseInternalVMTests, false,                           \
           "Turn on logging for internal VM tests.")                         \
                                                                             \
+  product(bool, ExecutingUnitTests, false,                                  \
+          "Whether the JVM is running unit tests or not")                   \
+                                                                            \
   product_pd(bool, UseTLAB, "Use thread-local object allocation")           \
                                                                             \
   product_pd(bool, ResizeTLAB,                                              \

hotspot/src/share/vm/services/management.cpp

@@ -1609,8 +1609,8 @@ JVM_ENTRY(void, jmm_SetVMGlobal(JNIEnv *env, jstring flag_name, jvalue new_value
   }
   char* name = java_lang_String::as_utf8_string(fn);
 
-  FormatBuffer<80> err_msg("%s", "");
+  FormatBuffer<80> error_msg("%s", "");
-  int succeed = WriteableFlags::set_flag(name, new_value, Flag::MANAGEMENT, err_msg);
+  int succeed = WriteableFlags::set_flag(name, new_value, Flag::MANAGEMENT, error_msg);
 
   if (succeed != Flag::SUCCESS) {
     if (succeed == Flag::MISSING_VALUE) {
@@ -1619,7 +1619,7 @@ JVM_ENTRY(void, jmm_SetVMGlobal(JNIEnv *env, jstring flag_name, jvalue new_value
     } else {
       // all the other errors are reported as IAE with the appropriate error message
       THROW_MSG(vmSymbols::java_lang_IllegalArgumentException(),
-                err_msg.buffer());
+                error_msg.buffer());
     }
   }
   assert(succeed == Flag::SUCCESS, "Setting flag should succeed");

hotspot/src/share/vm/utilities/debug.cpp

@@ -58,6 +58,8 @@
 #include "trace/tracing.hpp"
 #endif
 
+#include <stdio.h>
+
 #ifndef ASSERT
 #  ifdef _DEBUG
    // NOTE: don't turn the lines below into a comment -- if you're getting
@@ -187,7 +189,7 @@ bool error_is_suppressed(const char* file_name, int line_no) {
     return true;
   }
 
-  if (!is_error_reported()) {
+  if (!is_error_reported() && !SuppressFatalErrorMessage) {
     // print a friendly hint:
     fdStream out(defaultStream::output_fd());
     out.print_raw_cr("# To suppress the following error report, specify this argument");
@@ -262,6 +264,21 @@ void report_unimplemented(const char* file, int line) {
   report_vm_error(file, line, "Unimplemented()");
 }
 
+#ifdef ASSERT
+bool is_executing_unit_tests() {
+  return ExecutingUnitTests;
+}
+
+void report_assert_msg(const char* msg, ...) {
+  va_list ap;
+  va_start(ap, msg);
+
+  fprintf(stderr, "assert failed: %s\n", err_msg(FormatBufferDummy(), msg, ap).buffer());
+
+  va_end(ap);
+}
+#endif // ASSERT
+
 void report_untested(const char* file, int line, const char* message) {
 #ifndef PRODUCT
   warning("Untested: %s in %s: %d\n", message, file, line);

hotspot/src/share/vm/utilities/debug.hpp

@@ -46,11 +46,15 @@ class FormatBufferResource : public FormatBufferBase {
   FormatBufferResource(const char * format, ...) ATTRIBUTE_PRINTF(2, 3);
 };
 
+class FormatBufferDummy {};
+
 // Use stack for buffer
 template <size_t bufsz = FormatBufferBase::BufferSize>
 class FormatBuffer : public FormatBufferBase {
  public:
   inline FormatBuffer(const char* format, ...) ATTRIBUTE_PRINTF(2, 3);
+  // since va_list is unspecified type (can be char*), we use FormatBufferDummy to disambiguate these constructors
+  inline FormatBuffer(FormatBufferDummy dummy, const char* format, va_list ap) ATTRIBUTE_PRINTF(3, 0);
   inline void append(const char* format, ...)  ATTRIBUTE_PRINTF(2, 3);
   inline void print(const char* format, ...)  ATTRIBUTE_PRINTF(2, 3);
   inline void printv(const char* format, va_list ap) ATTRIBUTE_PRINTF(2, 0);
@@ -74,6 +78,11 @@ FormatBuffer<bufsz>::FormatBuffer(const char * format, ...) : FormatBufferBase(_
   va_end(argp);
 }
 
+template <size_t bufsz>
+FormatBuffer<bufsz>::FormatBuffer(FormatBufferDummy dummy, const char * format, va_list ap) : FormatBufferBase(_buffer) {
+  jio_vsnprintf(_buf, bufsz, format, ap);
+}
+
 template <size_t bufsz>
 FormatBuffer<bufsz>::FormatBuffer() : FormatBufferBase(_buffer) {
   _buf[0] = '\0';
@@ -119,11 +128,13 @@ typedef FormatBuffer<> err_msg;
 #define vmassert(p, ...)                                                       \
 do {                                                                           \
   if (!(p)) {                                                                  \
+    if (is_executing_unit_tests()) {                                           \
+      report_assert_msg(__VA_ARGS__);                                          \
+    }                                                                          \
     report_vm_error(__FILE__, __LINE__, "assert(" #p ") failed", __VA_ARGS__); \
     BREAKPOINT;                                                                \
   }                                                                            \
 } while (0)
-
 #endif
 
 // For backward compatibility.
@@ -210,10 +221,16 @@ void report_vm_error(const char* file, int line, const char* error_msg);
 // ATTRIBUTE_PRINTF works with gcc >= 4.8 and any other compiler.
 void report_vm_error(const char* file, int line, const char* error_msg,
                      const char* detail_fmt, ...) ATTRIBUTE_PRINTF(4, 5);
+#ifdef ASSERT
+void report_assert_msg(const char* msg, ...) ATTRIBUTE_PRINTF(1, 2);
+#endif // ASSERT
 #else
 // GCC < 4.8 warns because of empty format string.  Warning can not be switched off selectively.
 void report_vm_error(const char* file, int line, const char* error_msg,
                      const char* detail_fmt, ...);
+#ifdef ASSERT
+void report_assert_msg(const char* msg, ...);
+#endif // ASSERT
 #endif
 void report_vm_status_error(const char* file, int line, const char* error_msg,
                             int status, const char* detail);
@@ -225,6 +242,11 @@ void report_should_not_reach_here(const char* file, int line);
 void report_unimplemented(const char* file, int line);
 void report_untested(const char* file, int line, const char* message);
 
+#ifdef ASSERT
+// unit test support
+bool is_executing_unit_tests();
+#endif // ASSERT
+
 void warning(const char* format, ...) ATTRIBUTE_PRINTF(1, 2);
 
 // Compile-time asserts.  Cond must be a compile-time constant expression that