From afc563751f00e5c992b7dc1bbcb13fa72f9d4dc5 Mon Sep 17 00:00:00 2001
From: Zhengyu Gu <zgu@openjdk.org>
Date: Thu, 30 Sep 2010 12:05:08 -0400
Subject: [PATCH 01/23] 6988363: Rebrand vm vendor property settings (jdk7
 only)

Vendor properties should be initialized after JDK version is determined.

Reviewed-by: kamg, ohair, dcubed, dholmes
---
 hotspot/src/share/vm/runtime/arguments.cpp | 11 ++++++++---
 hotspot/src/share/vm/runtime/arguments.hpp |  3 +++
 hotspot/src/share/vm/runtime/thread.cpp    |  3 +++
 3 files changed, 14 insertions(+), 3 deletions(-)

diff --git a/hotspot/src/share/vm/runtime/arguments.cpp b/hotspot/src/share/vm/runtime/arguments.cpp
index 78bccb66cf9..f62565a4192 100644
--- a/hotspot/src/share/vm/runtime/arguments.cpp
+++ b/hotspot/src/share/vm/runtime/arguments.cpp
@@ -119,11 +119,8 @@ void Arguments::init_system_properties() {
   PropertyList_add(&_system_properties, new SystemProperty("java.vm.specification.version", "1.0", false));
   PropertyList_add(&_system_properties, new SystemProperty("java.vm.specification.name",
                                                                  "Java Virtual Machine Specification",  false));
-  PropertyList_add(&_system_properties, new SystemProperty("java.vm.specification.vendor",
-        JDK_Version::is_gte_jdk17x_version() ? "Oracle Corporation" : "Sun Microsystems Inc.", false));
   PropertyList_add(&_system_properties, new SystemProperty("java.vm.version", VM_Version::vm_release(),  false));
   PropertyList_add(&_system_properties, new SystemProperty("java.vm.name", VM_Version::vm_name(),  false));
-  PropertyList_add(&_system_properties, new SystemProperty("java.vm.vendor", VM_Version::vm_vendor(),  false));
   PropertyList_add(&_system_properties, new SystemProperty("java.vm.info", VM_Version::vm_info_string(),  true));
 
   // following are JVMTI agent writeable properties.
@@ -151,6 +148,14 @@ void Arguments::init_system_properties() {
   os::init_system_properties_values();
 }
 
+
+  // Update/Initialize System properties after JDK version number is known
+void Arguments::init_version_specific_system_properties() {
+  PropertyList_add(&_system_properties, new SystemProperty("java.vm.specification.vendor",
+        JDK_Version::is_gte_jdk17x_version() ? "Oracle Corporation" : "Sun Microsystems Inc.", false));
+  PropertyList_add(&_system_properties, new SystemProperty("java.vm.vendor", VM_Version::vm_vendor(),  false));
+}
+
 /**
  * Provide a slightly more user-friendly way of eliminating -XX flags.
  * When a flag is eliminated, it can be added to this list in order to
diff --git a/hotspot/src/share/vm/runtime/arguments.hpp b/hotspot/src/share/vm/runtime/arguments.hpp
index 949d4560f97..2d278e18741 100644
--- a/hotspot/src/share/vm/runtime/arguments.hpp
+++ b/hotspot/src/share/vm/runtime/arguments.hpp
@@ -484,6 +484,9 @@ class Arguments : AllStatic {
   // System properties
   static void init_system_properties();
 
+  // Update/Initialize System properties after JDK version number is known
+  static void init_version_specific_system_properties();
+
   // Property List manipulation
   static void PropertyList_add(SystemProperty** plist, SystemProperty *element);
   static void PropertyList_add(SystemProperty** plist, const char* k, char* v);
diff --git a/hotspot/src/share/vm/runtime/thread.cpp b/hotspot/src/share/vm/runtime/thread.cpp
index 00e5e991efc..7e96ac60956 100644
--- a/hotspot/src/share/vm/runtime/thread.cpp
+++ b/hotspot/src/share/vm/runtime/thread.cpp
@@ -2898,6 +2898,9 @@ jint Threads::create_vm(JavaVMInitArgs* args, bool* canTryAgain) {
   // So that JDK version can be used as a discrimintor when parsing arguments
   JDK_Version_init();
 
+  // Update/Initialize System properties after JDK version number is known
+  Arguments::init_version_specific_system_properties();
+
   // Parse arguments
   jint parse_result = Arguments::parse(args);
   if (parse_result != JNI_OK) return parse_result;

From 9e4a795ff5a949b86d68aac961309da7d109eb62 Mon Sep 17 00:00:00 2001
From: Karen Kinnear <acorn@openjdk.org>
Date: Mon, 4 Oct 2010 13:11:10 -0400
Subject: [PATCH 02/23] 6763959:
 java.util.concurrent.locks.LockSupport.parkUntil(0) blocks forever

Absolute time 0 needs to return immediately.

Reviewed-by: phh, dcubed, dholmes
---
 hotspot/src/os/linux/vm/os_linux.cpp     | 4 ++--
 hotspot/src/os/solaris/vm/os_solaris.cpp | 4 ++--
 hotspot/src/os/windows/vm/os_windows.cpp | 2 +-
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/hotspot/src/os/linux/vm/os_linux.cpp b/hotspot/src/os/linux/vm/os_linux.cpp
index e3e117eb5fb..38eea57f743 100644
--- a/hotspot/src/os/linux/vm/os_linux.cpp
+++ b/hotspot/src/os/linux/vm/os_linux.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 1999, 2009, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1999, 2010, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -4839,7 +4839,7 @@ void Parker::park(bool isAbsolute, jlong time) {
 
   // Next, demultiplex/decode time arguments
   timespec absTime;
-  if (time < 0) { // don't wait at all
+  if (time < 0 || (isAbsolute && time == 0) ) { // don't wait at all
     return;
   }
   if (time > 0) {
diff --git a/hotspot/src/os/solaris/vm/os_solaris.cpp b/hotspot/src/os/solaris/vm/os_solaris.cpp
index 0245d6f609d..0db63b82f10 100644
--- a/hotspot/src/os/solaris/vm/os_solaris.cpp
+++ b/hotspot/src/os/solaris/vm/os_solaris.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 1997, 2009, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2010, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -5837,7 +5837,7 @@ void Parker::park(bool isAbsolute, jlong time) {
 
   // First, demultiplex/decode time arguments
   timespec absTime;
-  if (time < 0) { // don't wait at all
+  if (time < 0 || (isAbsolute && time == 0) ) { // don't wait at all
     return;
   }
   if (time > 0) {
diff --git a/hotspot/src/os/windows/vm/os_windows.cpp b/hotspot/src/os/windows/vm/os_windows.cpp
index f2b7eedcb5e..59b60df03f9 100644
--- a/hotspot/src/os/windows/vm/os_windows.cpp
+++ b/hotspot/src/os/windows/vm/os_windows.cpp
@@ -3992,7 +3992,7 @@ void Parker::park(bool isAbsolute, jlong time) {
   if (time < 0) { // don't wait
     return;
   }
-  else if (time == 0) {
+  else if (time == 0 && !isAbsolute) {
     time = INFINITE;
   }
   else if  (isAbsolute) {

From 39992cab03e5c391d32977f557fa22f7d6d7d512 Mon Sep 17 00:00:00 2001
From: Coleen Phillimore <coleenp@openjdk.org>
Date: Thu, 7 Oct 2010 08:06:06 -0700
Subject: [PATCH 03/23] 6983240: guarantee((Solaris::min_stack_allowed >=
 (StackYellowPages+StackRedPages...) wrong

Min_stack_allowed is a compile time constant and Stack*Pages are settable

Reviewed-by: dholmes, kvn
---
 hotspot/src/cpu/x86/vm/methodHandles_x86.cpp  |  2 +-
 hotspot/src/os/linux/vm/os_linux.cpp          | 19 +++++++++++++++----
 hotspot/src/os/solaris/vm/os_solaris.cpp      | 13 ++++++-------
 hotspot/src/os/windows/vm/os_windows.cpp      | 16 +++++++++++++++-
 hotspot/src/share/vm/runtime/arguments.cpp    |  3 ++-
 hotspot/src/share/vm/utilities/exceptions.cpp | 14 +++++++++++++-
 6 files changed, 52 insertions(+), 15 deletions(-)

diff --git a/hotspot/src/cpu/x86/vm/methodHandles_x86.cpp b/hotspot/src/cpu/x86/vm/methodHandles_x86.cpp
index e8cd888cd71..b6c08684fff 100644
--- a/hotspot/src/cpu/x86/vm/methodHandles_x86.cpp
+++ b/hotspot/src/cpu/x86/vm/methodHandles_x86.cpp
@@ -346,7 +346,7 @@ void trace_method_handle_stub(const char* adaptername,
     if (stack_dump_count > 64)  stack_dump_count = 48;
     for (i = 0; i < stack_dump_count; i += 4) {
       printf(" dump at SP[%d] "INTPTR_FORMAT": "INTPTR_FORMAT" "INTPTR_FORMAT" "INTPTR_FORMAT" "INTPTR_FORMAT"\n",
-             i, &entry_sp[i+0], entry_sp[i+0], entry_sp[i+1], entry_sp[i+2], entry_sp[i+3]);
+             i, (intptr_t)&entry_sp[i+0], entry_sp[i+0], entry_sp[i+1], entry_sp[i+2], entry_sp[i+3]);
     }
     print_method_handle(mh);
   }
diff --git a/hotspot/src/os/linux/vm/os_linux.cpp b/hotspot/src/os/linux/vm/os_linux.cpp
index 38eea57f743..e42507f4aa7 100644
--- a/hotspot/src/os/linux/vm/os_linux.cpp
+++ b/hotspot/src/os/linux/vm/os_linux.cpp
@@ -827,8 +827,10 @@ bool os::create_thread(Thread* thread, ThreadType thr_type, size_t stack_size) {
 
       switch (thr_type) {
       case os::java_thread:
-        // Java threads use ThreadStackSize which default value can be changed with the flag -Xss
-        if (JavaThread::stack_size_at_create() > 0) stack_size = JavaThread::stack_size_at_create();
+        // Java threads use ThreadStackSize which default value can be
+        // changed with the flag -Xss
+        assert (JavaThread::stack_size_at_create() > 0, "this should be set");
+        stack_size = JavaThread::stack_size_at_create();
         break;
       case os::compiler_thread:
         if (CompilerThreadStackSize > 0) {
@@ -3922,12 +3924,21 @@ jint os::init_2(void)
   Linux::signal_sets_init();
   Linux::install_signal_handlers();
 
+  // Check minimum allowable stack size for thread creation and to initialize
+  // the java system classes, including StackOverflowError - depends on page
+  // size.  Add a page for compiler2 recursion in main thread.
+  // Add in 2*BytesPerWord times page size to account for VM stack during
+  // class initialization depending on 32 or 64 bit VM.
+  os::Linux::min_stack_allowed = MAX2(os::Linux::min_stack_allowed,
+            (size_t)(StackYellowPages+StackRedPages+StackShadowPages+
+                    2*BytesPerWord COMPILER2_PRESENT(+1)) * Linux::page_size());
+
   size_t threadStackSizeInBytes = ThreadStackSize * K;
   if (threadStackSizeInBytes != 0 &&
-      threadStackSizeInBytes < Linux::min_stack_allowed) {
+      threadStackSizeInBytes < os::Linux::min_stack_allowed) {
         tty->print_cr("\nThe stack size specified is too small, "
                       "Specify at least %dk",
-                      Linux::min_stack_allowed / K);
+                      os::Linux::min_stack_allowed/ K);
         return JNI_ERR;
   }
 
diff --git a/hotspot/src/os/solaris/vm/os_solaris.cpp b/hotspot/src/os/solaris/vm/os_solaris.cpp
index 0db63b82f10..08a16b76369 100644
--- a/hotspot/src/os/solaris/vm/os_solaris.cpp
+++ b/hotspot/src/os/solaris/vm/os_solaris.cpp
@@ -4878,18 +4878,17 @@ jint os::init_2(void) {
   // Check minimum allowable stack size for thread creation and to initialize
   // the java system classes, including StackOverflowError - depends on page
   // size.  Add a page for compiler2 recursion in main thread.
-  // Add in BytesPerWord times page size to account for VM stack during
+  // Add in 2*BytesPerWord times page size to account for VM stack during
   // class initialization depending on 32 or 64 bit VM.
-  guarantee((Solaris::min_stack_allowed >=
-    (StackYellowPages+StackRedPages+StackShadowPages+BytesPerWord
-     COMPILER2_PRESENT(+1)) * page_size),
-    "need to increase Solaris::min_stack_allowed on this platform");
+  os::Solaris::min_stack_allowed = MAX2(os::Solaris::min_stack_allowed,
+            (size_t)(StackYellowPages+StackRedPages+StackShadowPages+
+                    2*BytesPerWord COMPILER2_PRESENT(+1)) * page_size);
 
   size_t threadStackSizeInBytes = ThreadStackSize * K;
   if (threadStackSizeInBytes != 0 &&
-    threadStackSizeInBytes < Solaris::min_stack_allowed) {
+    threadStackSizeInBytes < os::Solaris::min_stack_allowed) {
     tty->print_cr("\nThe stack size specified is too small, Specify at least %dk",
-                  Solaris::min_stack_allowed/K);
+                  os::Solaris::min_stack_allowed/K);
     return JNI_ERR;
   }
 
diff --git a/hotspot/src/os/windows/vm/os_windows.cpp b/hotspot/src/os/windows/vm/os_windows.cpp
index 59b60df03f9..d0837200194 100644
--- a/hotspot/src/os/windows/vm/os_windows.cpp
+++ b/hotspot/src/os/windows/vm/os_windows.cpp
@@ -3311,7 +3311,6 @@ extern "C" {
   }
 }
 
-
 // this is called _after_ the global arguments have been parsed
 jint os::init_2(void) {
   // Allocate a single page and mark it as readable for safepoint polling
@@ -3390,6 +3389,21 @@ jint os::init_2(void) {
     actual_reserve_size = default_reserve_size;
   }
 
+  // Check minimum allowable stack size for thread creation and to initialize
+  // the java system classes, including StackOverflowError - depends on page
+  // size.  Add a page for compiler2 recursion in main thread.
+  // Add in 2*BytesPerWord times page size to account for VM stack during
+  // class initialization depending on 32 or 64 bit VM.
+  size_t min_stack_allowed =
+            (size_t)(StackYellowPages+StackRedPages+StackShadowPages+
+            2*BytesPerWord COMPILER2_PRESENT(+1)) * os::vm_page_size();
+  if (actual_reserve_size < min_stack_allowed) {
+    tty->print_cr("\nThe stack size specified is too small, "
+                  "Specify at least %dk",
+                  min_stack_allowed / K);
+    return JNI_ERR;
+  }
+
   JavaThread::set_stack_size_at_create(stack_commit_size);
 
   // Calculate theoretical max. size of Threads to guard gainst artifical
diff --git a/hotspot/src/share/vm/runtime/arguments.cpp b/hotspot/src/share/vm/runtime/arguments.cpp
index f62565a4192..26d1f20e72d 100644
--- a/hotspot/src/share/vm/runtime/arguments.cpp
+++ b/hotspot/src/share/vm/runtime/arguments.cpp
@@ -1663,7 +1663,8 @@ bool Arguments::check_stack_pages()
   bool status = true;
   status = status && verify_min_value(StackYellowPages, 1, "StackYellowPages");
   status = status && verify_min_value(StackRedPages, 1, "StackRedPages");
-  status = status && verify_min_value(StackShadowPages, 1, "StackShadowPages");
+  // greater stack shadow pages can't generate instruction to bang stack
+  status = status && verify_interval(StackShadowPages, 1, 50, "StackShadowPages");
   return status;
 }
 
diff --git a/hotspot/src/share/vm/utilities/exceptions.cpp b/hotspot/src/share/vm/utilities/exceptions.cpp
index ef37af5071b..0c3cdf8ce32 100644
--- a/hotspot/src/share/vm/utilities/exceptions.cpp
+++ b/hotspot/src/share/vm/utilities/exceptions.cpp
@@ -61,6 +61,18 @@ bool Exceptions::special_exception(Thread* thread, const char* file, int line, H
    ShouldNotReachHere();
   }
 
+#ifdef ASSERT
+  // Check for trying to throw stack overflow before initialization is complete
+  // to prevent infinite recursion trying to initialize stack overflow without
+  // adequate stack space.
+  // This can happen with stress testing a large value of StackShadowPages
+  if (h_exception()->klass() == SystemDictionary::StackOverflowError_klass()) {
+    instanceKlass* ik = instanceKlass::cast(h_exception->klass());
+    assert(ik->is_initialized(),
+           "need to increase min_stack_allowed calculation");
+  }
+#endif // ASSERT
+
   if (thread->is_VM_thread()
       || thread->is_Compiler_thread() ) {
     // We do not care what kind of exception we get for the vm-thread or a thread which
@@ -91,7 +103,6 @@ bool Exceptions::special_exception(Thread* thread, const char* file, int line, s
     thread->set_pending_exception(Universe::vm_exception(), file, line);
     return true;
   }
-
   return false;
 }
 
@@ -193,6 +204,7 @@ void Exceptions::throw_stack_overflow_exception(Thread* THREAD, const char* file
     klassOop k = SystemDictionary::StackOverflowError_klass();
     oop e = instanceKlass::cast(k)->allocate_instance(CHECK);
     exception = Handle(THREAD, e);  // fill_in_stack trace does gc
+    assert(instanceKlass::cast(k)->is_initialized(), "need to increase min_stack_allowed calculation");
     if (StackTraceInThrowable) {
       java_lang_Throwable::fill_in_stack_trace(exception);
     }

From 72616ad2bc05a562919155be8b1a688c2e3774d3 Mon Sep 17 00:00:00 2001
From: Bob Vandette <bobv@openjdk.org>
Date: Thu, 7 Oct 2010 15:12:57 -0400
Subject: [PATCH 04/23] 6989297: Integrate additional portability improvements

Reviewed-by: vladidan, dholmes
---
 hotspot/src/cpu/sparc/vm/globals_sparc.hpp       | 2 ++
 hotspot/src/cpu/x86/vm/globals_x86.hpp           | 2 ++
 hotspot/src/cpu/x86/vm/methodHandles_x86.cpp     | 2 +-
 hotspot/src/cpu/zero/vm/globals_zero.hpp         | 2 ++
 hotspot/src/os/linux/vm/attachListener_linux.cpp | 4 ++--
 hotspot/src/share/vm/includeDB_core              | 1 +
 hotspot/src/share/vm/runtime/globals.hpp         | 4 ++--
 hotspot/src/share/vm/runtime/sharedRuntime.cpp   | 3 +++
 hotspot/src/share/vm/runtime/sharedRuntime.hpp   | 3 +++
 9 files changed, 18 insertions(+), 5 deletions(-)

diff --git a/hotspot/src/cpu/sparc/vm/globals_sparc.hpp b/hotspot/src/cpu/sparc/vm/globals_sparc.hpp
index 2478ed38415..071f6b58d68 100644
--- a/hotspot/src/cpu/sparc/vm/globals_sparc.hpp
+++ b/hotspot/src/cpu/sparc/vm/globals_sparc.hpp
@@ -62,3 +62,5 @@ define_pd_global(intx, PreInflateSpin,       40);  // Determined by running desi
 
 define_pd_global(bool, RewriteBytecodes,     true);
 define_pd_global(bool, RewriteFrequentPairs, true);
+
+define_pd_global(bool, UseMembar,            false);
diff --git a/hotspot/src/cpu/x86/vm/globals_x86.hpp b/hotspot/src/cpu/x86/vm/globals_x86.hpp
index 307762d866b..e62e6748210 100644
--- a/hotspot/src/cpu/x86/vm/globals_x86.hpp
+++ b/hotspot/src/cpu/x86/vm/globals_x86.hpp
@@ -63,3 +63,5 @@ define_pd_global(intx, PreInflateSpin,           10);
 
 define_pd_global(bool, RewriteBytecodes,     true);
 define_pd_global(bool, RewriteFrequentPairs, true);
+
+define_pd_global(bool, UseMembar,            false);
diff --git a/hotspot/src/cpu/x86/vm/methodHandles_x86.cpp b/hotspot/src/cpu/x86/vm/methodHandles_x86.cpp
index e8cd888cd71..b6c08684fff 100644
--- a/hotspot/src/cpu/x86/vm/methodHandles_x86.cpp
+++ b/hotspot/src/cpu/x86/vm/methodHandles_x86.cpp
@@ -346,7 +346,7 @@ void trace_method_handle_stub(const char* adaptername,
     if (stack_dump_count > 64)  stack_dump_count = 48;
     for (i = 0; i < stack_dump_count; i += 4) {
       printf(" dump at SP[%d] "INTPTR_FORMAT": "INTPTR_FORMAT" "INTPTR_FORMAT" "INTPTR_FORMAT" "INTPTR_FORMAT"\n",
-             i, &entry_sp[i+0], entry_sp[i+0], entry_sp[i+1], entry_sp[i+2], entry_sp[i+3]);
+             i, (intptr_t)&entry_sp[i+0], entry_sp[i+0], entry_sp[i+1], entry_sp[i+2], entry_sp[i+3]);
     }
     print_method_handle(mh);
   }
diff --git a/hotspot/src/cpu/zero/vm/globals_zero.hpp b/hotspot/src/cpu/zero/vm/globals_zero.hpp
index f9d1635cb4c..d55a9ca29ec 100644
--- a/hotspot/src/cpu/zero/vm/globals_zero.hpp
+++ b/hotspot/src/cpu/zero/vm/globals_zero.hpp
@@ -45,3 +45,5 @@ define_pd_global(intx,  StackShadowPages,     5 LP64_ONLY(+1) DEBUG_ONLY(+3));
 
 define_pd_global(bool,  RewriteBytecodes,     true);
 define_pd_global(bool,  RewriteFrequentPairs, true);
+
+define_pd_global(bool,  UseMembar,            false);
diff --git a/hotspot/src/os/linux/vm/attachListener_linux.cpp b/hotspot/src/os/linux/vm/attachListener_linux.cpp
index 837410ab3bd..a97f5869b61 100644
--- a/hotspot/src/os/linux/vm/attachListener_linux.cpp
+++ b/hotspot/src/os/linux/vm/attachListener_linux.cpp
@@ -176,10 +176,10 @@ int LinuxAttachListener::init() {
 
   int n = snprintf(path, UNIX_PATH_MAX, "%s/.java_pid%d",
                    os::get_temp_directory(), os::current_process_id());
-  if (n <= (int)UNIX_PATH_MAX) {
+  if (n < (int)UNIX_PATH_MAX) {
     n = snprintf(initial_path, UNIX_PATH_MAX, "%s.tmp", path);
   }
-  if (n > (int)UNIX_PATH_MAX) {
+  if (n >= (int)UNIX_PATH_MAX) {
     return -1;
   }
 
diff --git a/hotspot/src/share/vm/includeDB_core b/hotspot/src/share/vm/includeDB_core
index faf7843fcb2..f7362cdedc4 100644
--- a/hotspot/src/share/vm/includeDB_core
+++ b/hotspot/src/share/vm/includeDB_core
@@ -3229,6 +3229,7 @@ orderAccess.hpp                         allocation.hpp
 orderAccess.hpp                         os.hpp
 
 orderAccess_<os_arch>.inline.hpp        orderAccess.hpp
+orderAccess_<os_arch>.inline.hpp        vm_version_<arch>.hpp
 
 os.cpp                                  allocation.inline.hpp
 os.cpp                                  arguments.hpp
diff --git a/hotspot/src/share/vm/runtime/globals.hpp b/hotspot/src/share/vm/runtime/globals.hpp
index 1fe8eb9a36b..167edb5f796 100644
--- a/hotspot/src/share/vm/runtime/globals.hpp
+++ b/hotspot/src/share/vm/runtime/globals.hpp
@@ -323,10 +323,10 @@ class CommandLineFlags {
   /* UseMembar is theoretically a temp flag used for memory barrier         \
    * removal testing.  It was supposed to be removed before FCS but has     \
    * been re-added (see 6401008) */                                         \
-  product(bool, UseMembar, false,                                           \
+  product_pd(bool, UseMembar,                                               \
           "(Unstable) Issues membars on thread state transitions")          \
                                                                             \
-  /* Temporary: See 6948537 */                                             \
+  /* Temporary: See 6948537 */                                              \
   experimental(bool, UseMemSetInBOT, true,                                  \
           "(Unstable) uses memset in BOT updates in GC code")               \
                                                                             \
diff --git a/hotspot/src/share/vm/runtime/sharedRuntime.cpp b/hotspot/src/share/vm/runtime/sharedRuntime.cpp
index e35ecf2be61..0ecbdcc52ed 100644
--- a/hotspot/src/share/vm/runtime/sharedRuntime.cpp
+++ b/hotspot/src/share/vm/runtime/sharedRuntime.cpp
@@ -302,6 +302,9 @@ double SharedRuntime::dabs(double f)  {
   return (f <= (double)0.0) ? (double)0.0 - f : f;
 }
 
+#endif
+
+#if defined(__SOFTFP__) || defined(PPC)
 double SharedRuntime::dsqrt(double f) {
   return sqrt(f);
 }
diff --git a/hotspot/src/share/vm/runtime/sharedRuntime.hpp b/hotspot/src/share/vm/runtime/sharedRuntime.hpp
index 4f6d103f838..411d80eacf9 100644
--- a/hotspot/src/share/vm/runtime/sharedRuntime.hpp
+++ b/hotspot/src/share/vm/runtime/sharedRuntime.hpp
@@ -116,6 +116,9 @@ class SharedRuntime: AllStatic {
 
 #if defined(__SOFTFP__) || defined(E500V2)
   static double dabs(double f);
+#endif
+
+#if defined(__SOFTFP__) || defined(PPC)
   static double dsqrt(double f);
 #endif
 

From da95f5319ecd3ef0085a2c0b4947e2ab08191e30 Mon Sep 17 00:00:00 2001
From: Keith McGuigan <kamg@openjdk.org>
Date: Tue, 12 Oct 2010 10:57:33 -0400
Subject: [PATCH 05/23] 6392697: Additional flag needed to supress Hotspot
 warning messages

Apply PrintJvmWarnings flag to all warnings

Reviewed-by: coleenp, phh
---
 hotspot/src/share/vm/runtime/globals.hpp |  3 +++
 hotspot/src/share/vm/utilities/debug.cpp | 16 +++++++++-------
 2 files changed, 12 insertions(+), 7 deletions(-)

diff --git a/hotspot/src/share/vm/runtime/globals.hpp b/hotspot/src/share/vm/runtime/globals.hpp
index 167edb5f796..c2619205957 100644
--- a/hotspot/src/share/vm/runtime/globals.hpp
+++ b/hotspot/src/share/vm/runtime/globals.hpp
@@ -815,6 +815,9 @@ class CommandLineFlags {
   develop(bool, PrintJVMWarnings, false,                                    \
           "Prints warnings for unimplemented JVM functions")                \
                                                                             \
+  product(bool, PrintWarnings, true,                                        \
+          "Prints JVM warnings to output stream")                           \
+                                                                            \
   notproduct(uintx, WarnOnStalledSpinLock, 0,                               \
           "Prints warnings for stalled SpinLocks")                          \
                                                                             \
diff --git a/hotspot/src/share/vm/utilities/debug.cpp b/hotspot/src/share/vm/utilities/debug.cpp
index 54282bf0740..c535ff0e5ab 100644
--- a/hotspot/src/share/vm/utilities/debug.cpp
+++ b/hotspot/src/share/vm/utilities/debug.cpp
@@ -51,14 +51,16 @@
 
 
 void warning(const char* format, ...) {
-  // In case error happens before init or during shutdown
-  if (tty == NULL) ostream_init();
+  if (PrintWarnings) {
+    // In case error happens before init or during shutdown
+    if (tty == NULL) ostream_init();
 
-  tty->print("%s warning: ", VM_Version::vm_name());
-  va_list ap;
-  va_start(ap, format);
-  tty->vprint_cr(format, ap);
-  va_end(ap);
+    tty->print("%s warning: ", VM_Version::vm_name());
+    va_list ap;
+    va_start(ap, format);
+    tty->vprint_cr(format, ap);
+    va_end(ap);
+  }
   if (BreakAtWarning) BREAKPOINT;
 }
 

From 02bad20969e7532c6ea6ac3fbd03e344a6f04f9e Mon Sep 17 00:00:00 2001
From: John Cuthbertson <johnc@openjdk.org>
Date: Tue, 12 Oct 2010 09:36:48 -0700
Subject: [PATCH 06/23] 6971296: G1: simplify G1RemSet class hierarchy

Remove G1RemSet base class and StupidG1RemSet class; rename HRInto_G1RemSet to just G1RemSet.

Reviewed-by: ysr, tonyp
---
 .../gc_implementation/g1/g1CollectedHeap.cpp  |  19 +--
 .../vm/gc_implementation/g1/g1OopClosures.hpp |   2 -
 .../vm/gc_implementation/g1/g1RemSet.cpp      |  82 ++++------
 .../vm/gc_implementation/g1/g1RemSet.hpp      | 153 +++++-------------
 .../gc_implementation/g1/g1RemSet.inline.hpp  |  16 +-
 .../vm/gc_implementation/g1/g1_globals.hpp    |   3 -
 .../vm/gc_implementation/includeDB_gc_g1      |   6 +
 7 files changed, 99 insertions(+), 182 deletions(-)

diff --git a/hotspot/src/share/vm/gc_implementation/g1/g1CollectedHeap.cpp b/hotspot/src/share/vm/gc_implementation/g1/g1CollectedHeap.cpp
index c72e154c494..6a9114268fe 100644
--- a/hotspot/src/share/vm/gc_implementation/g1/g1CollectedHeap.cpp
+++ b/hotspot/src/share/vm/gc_implementation/g1/g1CollectedHeap.cpp
@@ -791,7 +791,7 @@ class RebuildRSOutOfRegionClosure: public HeapRegionClosure {
   int                _worker_i;
 public:
   RebuildRSOutOfRegionClosure(G1CollectedHeap* g1, int worker_i = 0) :
-    _cl(g1->g1_rem_set()->as_HRInto_G1RemSet(), worker_i),
+    _cl(g1->g1_rem_set(), worker_i),
     _worker_i(worker_i),
     _g1h(g1)
   { }
@@ -890,7 +890,7 @@ void G1CollectedHeap::do_collection(bool explicit_gc,
     abandon_cur_alloc_region();
     abandon_gc_alloc_regions();
     assert(_cur_alloc_region == NULL, "Invariant.");
-    g1_rem_set()->as_HRInto_G1RemSet()->cleanupHRRS();
+    g1_rem_set()->cleanupHRRS();
     tear_down_region_lists();
     set_used_regions_to_need_zero_fill();
 
@@ -1506,15 +1506,11 @@ jint G1CollectedHeap::initialize() {
   }
 
   // Also create a G1 rem set.
-  if (G1UseHRIntoRS) {
-    if (mr_bs()->is_a(BarrierSet::CardTableModRef)) {
-      _g1_rem_set = new HRInto_G1RemSet(this, (CardTableModRefBS*)mr_bs());
-    } else {
-      vm_exit_during_initialization("G1 requires a cardtable mod ref bs.");
-      return JNI_ENOMEM;
-    }
+  if (mr_bs()->is_a(BarrierSet::CardTableModRef)) {
+    _g1_rem_set = new G1RemSet(this, (CardTableModRefBS*)mr_bs());
   } else {
-    _g1_rem_set = new StupidG1RemSet(this);
+    vm_exit_during_initialization("G1 requires a cardtable mod ref bs.");
+    return JNI_ENOMEM;
   }
 
   // Carve out the G1 part of the heap.
@@ -2706,8 +2702,7 @@ size_t G1CollectedHeap::max_pending_card_num() {
 }
 
 size_t G1CollectedHeap::cards_scanned() {
-  HRInto_G1RemSet* g1_rset = (HRInto_G1RemSet*) g1_rem_set();
-  return g1_rset->cardsScanned();
+  return g1_rem_set()->cardsScanned();
 }
 
 void
diff --git a/hotspot/src/share/vm/gc_implementation/g1/g1OopClosures.hpp b/hotspot/src/share/vm/gc_implementation/g1/g1OopClosures.hpp
index 2aef2518734..542e7b02615 100644
--- a/hotspot/src/share/vm/gc_implementation/g1/g1OopClosures.hpp
+++ b/hotspot/src/share/vm/gc_implementation/g1/g1OopClosures.hpp
@@ -25,8 +25,6 @@
 class HeapRegion;
 class G1CollectedHeap;
 class G1RemSet;
-class HRInto_G1RemSet;
-class G1RemSet;
 class ConcurrentMark;
 class DirtyCardToOopClosure;
 class CMBitMap;
diff --git a/hotspot/src/share/vm/gc_implementation/g1/g1RemSet.cpp b/hotspot/src/share/vm/gc_implementation/g1/g1RemSet.cpp
index 6c79abcf8ba..ca796df2e91 100644
--- a/hotspot/src/share/vm/gc_implementation/g1/g1RemSet.cpp
+++ b/hotspot/src/share/vm/gc_implementation/g1/g1RemSet.cpp
@@ -97,13 +97,6 @@ public:
   }
 };
 
-void
-StupidG1RemSet::oops_into_collection_set_do(OopsInHeapRegionClosure* oc,
-                                            int worker_i) {
-  IntoCSRegionClosure rc(_g1, oc);
-  _g1->heap_region_iterate(&rc);
-}
-
 class VerifyRSCleanCardOopClosure: public OopClosure {
   G1CollectedHeap* _g1;
 public:
@@ -119,8 +112,9 @@ public:
   }
 };
 
-HRInto_G1RemSet::HRInto_G1RemSet(G1CollectedHeap* g1, CardTableModRefBS* ct_bs)
-  : G1RemSet(g1), _ct_bs(ct_bs), _g1p(_g1->g1_policy()),
+G1RemSet::G1RemSet(G1CollectedHeap* g1, CardTableModRefBS* ct_bs)
+  : _g1(g1), _conc_refine_cards(0),
+    _ct_bs(ct_bs), _g1p(_g1->g1_policy()),
     _cg1r(g1->concurrent_g1_refine()),
     _traversal_in_progress(false),
     _cset_rs_update_cl(NULL),
@@ -134,7 +128,7 @@ HRInto_G1RemSet::HRInto_G1RemSet(G1CollectedHeap* g1, CardTableModRefBS* ct_bs)
   }
 }
 
-HRInto_G1RemSet::~HRInto_G1RemSet() {
+G1RemSet::~G1RemSet() {
   delete _seq_task;
   for (uint i = 0; i < n_workers(); i++) {
     assert(_cset_rs_update_cl[i] == NULL, "it should be");
@@ -277,7 +271,7 @@ public:
 //          p threads
 // Then thread t will start at region t * floor (n/p)
 
-HeapRegion* HRInto_G1RemSet::calculateStartRegion(int worker_i) {
+HeapRegion* G1RemSet::calculateStartRegion(int worker_i) {
   HeapRegion* result = _g1p->collection_set();
   if (ParallelGCThreads > 0) {
     size_t cs_size = _g1p->collection_set_size();
@@ -290,7 +284,7 @@ HeapRegion* HRInto_G1RemSet::calculateStartRegion(int worker_i) {
   return result;
 }
 
-void HRInto_G1RemSet::scanRS(OopsInHeapRegionClosure* oc, int worker_i) {
+void G1RemSet::scanRS(OopsInHeapRegionClosure* oc, int worker_i) {
   double rs_time_start = os::elapsedTime();
   HeapRegion *startRegion = calculateStartRegion(worker_i);
 
@@ -340,7 +334,7 @@ public:
   }
 };
 
-void HRInto_G1RemSet::updateRS(DirtyCardQueue* into_cset_dcq, int worker_i) {
+void G1RemSet::updateRS(DirtyCardQueue* into_cset_dcq, int worker_i) {
   double start = os::elapsedTime();
   // Apply the given closure to all remaining log entries.
   RefineRecordRefsIntoCSCardTableEntryClosure into_cset_update_rs_cl(_g1, into_cset_dcq);
@@ -439,12 +433,11 @@ public:
   }
 };
 
-void HRInto_G1RemSet::cleanupHRRS() {
+void G1RemSet::cleanupHRRS() {
   HeapRegionRemSet::cleanup();
 }
 
-void
-HRInto_G1RemSet::oops_into_collection_set_do(OopsInHeapRegionClosure* oc,
+void G1RemSet::oops_into_collection_set_do(OopsInHeapRegionClosure* oc,
                                              int worker_i) {
 #if CARD_REPEAT_HISTO
   ct_freq_update_histo_and_reset();
@@ -508,8 +501,7 @@ HRInto_G1RemSet::oops_into_collection_set_do(OopsInHeapRegionClosure* oc,
   _cset_rs_update_cl[worker_i] = NULL;
 }
 
-void HRInto_G1RemSet::
-prepare_for_oops_into_collection_set_do() {
+void G1RemSet::prepare_for_oops_into_collection_set_do() {
 #if G1_REM_SET_LOGGING
   PrintRSClosure cl;
   _g1->collection_set_iterate(&cl);
@@ -581,7 +573,7 @@ public:
     //   RSet updating,
     // * the post-write barrier shouldn't be logging updates to young
     //   regions (but there is a situation where this can happen - see
-    //   the comment in HRInto_G1RemSet::concurrentRefineOneCard below -
+    //   the comment in G1RemSet::concurrentRefineOneCard below -
     //   that should not be applicable here), and
     // * during actual RSet updating, the filtering of cards in young
     //   regions in HeapRegion::oops_on_card_seq_iterate_careful is
@@ -601,7 +593,7 @@ public:
   }
 };
 
-void HRInto_G1RemSet::cleanup_after_oops_into_collection_set_do() {
+void G1RemSet::cleanup_after_oops_into_collection_set_do() {
   guarantee( _cards_scanned != NULL, "invariant" );
   _total_cards_scanned = 0;
   for (uint i = 0; i < n_workers(); ++i)
@@ -692,12 +684,12 @@ public:
   }
 };
 
-void HRInto_G1RemSet::scrub(BitMap* region_bm, BitMap* card_bm) {
+void G1RemSet::scrub(BitMap* region_bm, BitMap* card_bm) {
   ScrubRSClosure scrub_cl(region_bm, card_bm);
   _g1->heap_region_iterate(&scrub_cl);
 }
 
-void HRInto_G1RemSet::scrub_par(BitMap* region_bm, BitMap* card_bm,
+void G1RemSet::scrub_par(BitMap* region_bm, BitMap* card_bm,
                                 int worker_num, int claim_val) {
   ScrubRSClosure scrub_cl(region_bm, card_bm);
   _g1->heap_region_par_iterate_chunked(&scrub_cl, worker_num, claim_val);
@@ -741,7 +733,7 @@ public:
   virtual void do_oop(narrowOop* p)  { do_oop_nv(p); }
 };
 
-bool HRInto_G1RemSet::concurrentRefineOneCard_impl(jbyte* card_ptr, int worker_i,
+bool G1RemSet::concurrentRefineOneCard_impl(jbyte* card_ptr, int worker_i,
                                                    bool check_for_refs_into_cset) {
   // Construct the region representing the card.
   HeapWord* start = _ct_bs->addr_for(card_ptr);
@@ -820,7 +812,7 @@ bool HRInto_G1RemSet::concurrentRefineOneCard_impl(jbyte* card_ptr, int worker_i
   return trigger_cl.value();
 }
 
-bool HRInto_G1RemSet::concurrentRefineOneCard(jbyte* card_ptr, int worker_i,
+bool G1RemSet::concurrentRefineOneCard(jbyte* card_ptr, int worker_i,
                                               bool check_for_refs_into_cset) {
   // If the card is no longer dirty, nothing to do.
   if (*card_ptr != CardTableModRefBS::dirty_card_val()) {
@@ -995,7 +987,7 @@ public:
   }
 };
 
-void HRInto_G1RemSet::print_summary_info() {
+void G1RemSet::print_summary_info() {
   G1CollectedHeap* g1 = G1CollectedHeap::heap();
 
 #if CARD_REPEAT_HISTO
@@ -1029,30 +1021,26 @@ void HRInto_G1RemSet::print_summary_info() {
   g1->concurrent_g1_refine()->threads_do(&p);
   gclog_or_tty->print_cr("");
 
-  if (G1UseHRIntoRS) {
-    HRRSStatsIter blk;
-    g1->heap_region_iterate(&blk);
-    gclog_or_tty->print_cr("  Total heap region rem set sizes = " SIZE_FORMAT "K."
-                           "  Max = " SIZE_FORMAT "K.",
-                           blk.total_mem_sz()/K, blk.max_mem_sz()/K);
-    gclog_or_tty->print_cr("  Static structures = " SIZE_FORMAT "K,"
-                           " free_lists = " SIZE_FORMAT "K.",
-                           HeapRegionRemSet::static_mem_size()/K,
-                           HeapRegionRemSet::fl_mem_size()/K);
-    gclog_or_tty->print_cr("    %d occupied cards represented.",
-                           blk.occupied());
-    gclog_or_tty->print_cr("    Max sz region = [" PTR_FORMAT ", " PTR_FORMAT " )"
-                           ", cap = " SIZE_FORMAT "K, occ = " SIZE_FORMAT "K.",
-                           blk.max_mem_sz_region()->bottom(), blk.max_mem_sz_region()->end(),
-                           (blk.max_mem_sz_region()->rem_set()->mem_size() + K - 1)/K,
-                           (blk.max_mem_sz_region()->rem_set()->occupied() + K - 1)/K);
-    gclog_or_tty->print_cr("    Did %d coarsenings.",
-                  HeapRegionRemSet::n_coarsenings());
-
-  }
+  HRRSStatsIter blk;
+  g1->heap_region_iterate(&blk);
+  gclog_or_tty->print_cr("  Total heap region rem set sizes = " SIZE_FORMAT "K."
+                         "  Max = " SIZE_FORMAT "K.",
+                         blk.total_mem_sz()/K, blk.max_mem_sz()/K);
+  gclog_or_tty->print_cr("  Static structures = " SIZE_FORMAT "K,"
+                         " free_lists = " SIZE_FORMAT "K.",
+                         HeapRegionRemSet::static_mem_size()/K,
+                         HeapRegionRemSet::fl_mem_size()/K);
+  gclog_or_tty->print_cr("    %d occupied cards represented.",
+                         blk.occupied());
+  gclog_or_tty->print_cr("    Max sz region = [" PTR_FORMAT ", " PTR_FORMAT " )"
+                         ", cap = " SIZE_FORMAT "K, occ = " SIZE_FORMAT "K.",
+                         blk.max_mem_sz_region()->bottom(), blk.max_mem_sz_region()->end(),
+                         (blk.max_mem_sz_region()->rem_set()->mem_size() + K - 1)/K,
+                         (blk.max_mem_sz_region()->rem_set()->occupied() + K - 1)/K);
+  gclog_or_tty->print_cr("    Did %d coarsenings.", HeapRegionRemSet::n_coarsenings());
 }
 
-void HRInto_G1RemSet::prepare_for_verify() {
+void G1RemSet::prepare_for_verify() {
   if (G1HRRSFlushLogBuffersOnVerify &&
       (VerifyBeforeGC || VerifyAfterGC)
       &&  !_g1->full_collection()) {
diff --git a/hotspot/src/share/vm/gc_implementation/g1/g1RemSet.hpp b/hotspot/src/share/vm/gc_implementation/g1/g1RemSet.hpp
index fcb5ecd76cf..5e0e8ddb1c0 100644
--- a/hotspot/src/share/vm/gc_implementation/g1/g1RemSet.hpp
+++ b/hotspot/src/share/vm/gc_implementation/g1/g1RemSet.hpp
@@ -27,107 +27,18 @@
 
 class G1CollectedHeap;
 class CardTableModRefBarrierSet;
-class HRInto_G1RemSet;
 class ConcurrentG1Refine;
 
+// A G1RemSet in which each heap region has a rem set that records the
+// external heap references into it.  Uses a mod ref bs to track updates,
+// so that they can be used to update the individual region remsets.
+
 class G1RemSet: public CHeapObj {
 protected:
   G1CollectedHeap* _g1;
   unsigned _conc_refine_cards;
   size_t n_workers();
 
-public:
-  G1RemSet(G1CollectedHeap* g1) :
-    _g1(g1), _conc_refine_cards(0)
-  {}
-
-  // Invoke "blk->do_oop" on all pointers into the CS in object in regions
-  // outside the CS (having invoked "blk->set_region" to set the "from"
-  // region correctly beforehand.) The "worker_i" param is for the
-  // parallel case where the number of the worker thread calling this
-  // function can be helpful in partitioning the work to be done. It
-  // should be the same as the "i" passed to the calling thread's
-  // work(i) function. In the sequential case this param will be ingored.
-  virtual void oops_into_collection_set_do(OopsInHeapRegionClosure* blk,
-                                           int worker_i) = 0;
-
-  // Prepare for and cleanup after an oops_into_collection_set_do
-  // call.  Must call each of these once before and after (in sequential
-  // code) any threads call oops into collection set do.  (This offers an
-  // opportunity to sequential setup and teardown of structures needed by a
-  // parallel iteration over the CS's RS.)
-  virtual void prepare_for_oops_into_collection_set_do() = 0;
-  virtual void cleanup_after_oops_into_collection_set_do() = 0;
-
-  // If "this" is of the given subtype, return "this", else "NULL".
-  virtual HRInto_G1RemSet* as_HRInto_G1RemSet() { return NULL; }
-
-  // Record, if necessary, the fact that *p (where "p" is in region "from",
-  // and is, a fortiori, required to be non-NULL) has changed to its new value.
-  virtual void write_ref(HeapRegion* from, oop* p) = 0;
-  virtual void write_ref(HeapRegion* from, narrowOop* p) = 0;
-  virtual void par_write_ref(HeapRegion* from, oop* p, int tid) = 0;
-  virtual void par_write_ref(HeapRegion* from, narrowOop* p, int tid) = 0;
-
-  // Requires "region_bm" and "card_bm" to be bitmaps with 1 bit per region
-  // or card, respectively, such that a region or card with a corresponding
-  // 0 bit contains no part of any live object.  Eliminates any remembered
-  // set entries that correspond to dead heap ranges.
-  virtual void scrub(BitMap* region_bm, BitMap* card_bm) = 0;
-  // Like the above, but assumes is called in parallel: "worker_num" is the
-  // parallel thread id of the current thread, and "claim_val" is the
-  // value that should be used to claim heap regions.
-  virtual void scrub_par(BitMap* region_bm, BitMap* card_bm,
-                         int worker_num, int claim_val) = 0;
-
-  // Refine the card corresponding to "card_ptr".  If "sts" is non-NULL,
-  // join and leave around parts that must be atomic wrt GC.  (NULL means
-  // being done at a safepoint.)
-  // With some implementations of this routine, when check_for_refs_into_cset
-  // is true, a true result may be returned if the given card contains oops
-  // that have references into the current collection set.
-  virtual bool concurrentRefineOneCard(jbyte* card_ptr, int worker_i,
-                                       bool check_for_refs_into_cset) {
-    return false;
-  }
-
-  // Print any relevant summary info.
-  virtual void print_summary_info() {}
-
-  // Prepare remebered set for verification.
-  virtual void prepare_for_verify() {};
-};
-
-
-// The simplest possible G1RemSet: iterates over all objects in non-CS
-// regions, searching for pointers into the CS.
-class StupidG1RemSet: public G1RemSet {
-public:
-  StupidG1RemSet(G1CollectedHeap* g1) : G1RemSet(g1) {}
-
-  void oops_into_collection_set_do(OopsInHeapRegionClosure* blk,
-                                   int worker_i);
-
-  void prepare_for_oops_into_collection_set_do() {}
-  void cleanup_after_oops_into_collection_set_do() {}
-
-  // Nothing is necessary in the version below.
-  void write_ref(HeapRegion* from, oop* p) {}
-  void write_ref(HeapRegion* from, narrowOop* p) {}
-  void par_write_ref(HeapRegion* from, oop* p, int tid) {}
-  void par_write_ref(HeapRegion* from, narrowOop* p, int tid) {}
-
-  void scrub(BitMap* region_bm, BitMap* card_bm) {}
-  void scrub_par(BitMap* region_bm, BitMap* card_bm,
-                 int worker_num, int claim_val) {}
-
-};
-
-// A G1RemSet in which each heap region has a rem set that records the
-// external heap references into it.  Uses a mod ref bs to track updates,
-// so that they can be used to update the individual region remsets.
-
-class HRInto_G1RemSet: public G1RemSet {
 protected:
   enum SomePrivateConstants {
     UpdateRStoMergeSync  = 0,
@@ -175,27 +86,31 @@ public:
   // scanned.
   void cleanupHRRS();
 
-  HRInto_G1RemSet(G1CollectedHeap* g1, CardTableModRefBS* ct_bs);
-  ~HRInto_G1RemSet();
+  G1RemSet(G1CollectedHeap* g1, CardTableModRefBS* ct_bs);
+  ~G1RemSet();
 
+  // Invoke "blk->do_oop" on all pointers into the CS in objects in regions
+  // outside the CS (having invoked "blk->set_region" to set the "from"
+  // region correctly beforehand.) The "worker_i" param is for the
+  // parallel case where the number of the worker thread calling this
+  // function can be helpful in partitioning the work to be done. It
+  // should be the same as the "i" passed to the calling thread's
+  // work(i) function. In the sequential case this param will be ingored.
   void oops_into_collection_set_do(OopsInHeapRegionClosure* blk,
                                    int worker_i);
 
+  // Prepare for and cleanup after an oops_into_collection_set_do
+  // call.  Must call each of these once before and after (in sequential
+  // code) any threads call oops_into_collection_set_do.  (This offers an
+  // opportunity to sequential setup and teardown of structures needed by a
+  // parallel iteration over the CS's RS.)
   void prepare_for_oops_into_collection_set_do();
   void cleanup_after_oops_into_collection_set_do();
-  void scanRS(OopsInHeapRegionClosure* oc, int worker_i);
-  template <class T> void scanNewRefsRS_work(OopsInHeapRegionClosure* oc, int worker_i);
-  void scanNewRefsRS(OopsInHeapRegionClosure* oc, int worker_i) {
-    if (UseCompressedOops) {
-      scanNewRefsRS_work<narrowOop>(oc, worker_i);
-    } else {
-      scanNewRefsRS_work<oop>(oc, worker_i);
-    }
-  }
-  void updateRS(DirtyCardQueue* into_cset_dcq, int worker_i);
-  HeapRegion* calculateStartRegion(int i);
 
-  HRInto_G1RemSet* as_HRInto_G1RemSet() { return this; }
+  void scanRS(OopsInHeapRegionClosure* oc, int worker_i);
+  void updateRS(DirtyCardQueue* into_cset_dcq, int worker_i);
+
+  HeapRegion* calculateStartRegion(int i);
 
   CardTableModRefBS* ct_bs() { return _ct_bs; }
   size_t cardsScanned() { return _total_cards_scanned; }
@@ -219,17 +134,31 @@ public:
 
   bool self_forwarded(oop obj);
 
+  // Requires "region_bm" and "card_bm" to be bitmaps with 1 bit per region
+  // or card, respectively, such that a region or card with a corresponding
+  // 0 bit contains no part of any live object.  Eliminates any remembered
+  // set entries that correspond to dead heap ranges.
   void scrub(BitMap* region_bm, BitMap* card_bm);
+
+  // Like the above, but assumes is called in parallel: "worker_num" is the
+  // parallel thread id of the current thread, and "claim_val" is the
+  // value that should be used to claim heap regions.
   void scrub_par(BitMap* region_bm, BitMap* card_bm,
                  int worker_num, int claim_val);
 
-  // If check_for_refs_into_cset is true then a true result is returned
-  // if the card contains oops that have references into the current
-  // collection set.
+  // Refine the card corresponding to "card_ptr".  If "sts" is non-NULL,
+  // join and leave around parts that must be atomic wrt GC.  (NULL means
+  // being done at a safepoint.)
+  // If check_for_refs_into_cset is true, a true result is returned
+  // if the given card contains oops that have references into the
+  // current collection set.
   virtual bool concurrentRefineOneCard(jbyte* card_ptr, int worker_i,
                                        bool check_for_refs_into_cset);
 
+  // Print any relevant summary info.
   virtual void print_summary_info();
+
+  // Prepare remembered set for verification.
   virtual void prepare_for_verify();
 };
 
@@ -250,13 +179,13 @@ public:
 
 class UpdateRSOopClosure: public OopClosure {
   HeapRegion* _from;
-  HRInto_G1RemSet* _rs;
+  G1RemSet* _rs;
   int _worker_i;
 
   template <class T> void do_oop_work(T* p);
 
 public:
-  UpdateRSOopClosure(HRInto_G1RemSet* rs, int worker_i = 0) :
+  UpdateRSOopClosure(G1RemSet* rs, int worker_i = 0) :
     _from(NULL), _rs(rs), _worker_i(worker_i) {
     guarantee(_rs != NULL, "Requires an HRIntoG1RemSet");
   }
diff --git a/hotspot/src/share/vm/gc_implementation/g1/g1RemSet.inline.hpp b/hotspot/src/share/vm/gc_implementation/g1/g1RemSet.inline.hpp
index ce64065527b..672c1899683 100644
--- a/hotspot/src/share/vm/gc_implementation/g1/g1RemSet.inline.hpp
+++ b/hotspot/src/share/vm/gc_implementation/g1/g1RemSet.inline.hpp
@@ -30,16 +30,18 @@ inline size_t G1RemSet::n_workers() {
   }
 }
 
-template <class T> inline void HRInto_G1RemSet::write_ref_nv(HeapRegion* from, T* p) {
+template <class T>
+inline void G1RemSet::write_ref_nv(HeapRegion* from, T* p) {
   par_write_ref_nv(from, p, 0);
 }
 
-inline bool HRInto_G1RemSet::self_forwarded(oop obj) {
+inline bool G1RemSet::self_forwarded(oop obj) {
   bool result =  (obj->is_forwarded() && (obj->forwardee()== obj));
   return result;
 }
 
-template <class T> inline void HRInto_G1RemSet::par_write_ref_nv(HeapRegion* from, T* p, int tid) {
+template <class T>
+inline void G1RemSet::par_write_ref_nv(HeapRegion* from, T* p, int tid) {
   oop obj = oopDesc::load_decode_heap_oop(p);
 #ifdef ASSERT
   // can't do because of races
@@ -77,7 +79,7 @@ template <class T> inline void HRInto_G1RemSet::par_write_ref_nv(HeapRegion* fro
       // Deferred updates to the CSet are either discarded (in the normal case),
       // or processed (if an evacuation failure occurs) at the end
       // of the collection.
-      // See HRInto_G1RemSet::cleanup_after_oops_into_collection_set_do().
+      // See G1RemSet::cleanup_after_oops_into_collection_set_do().
     } else {
 #if G1_REM_SET_LOGGING
       gclog_or_tty->print_cr("Adding " PTR_FORMAT " (" PTR_FORMAT ") to RS"
@@ -91,12 +93,14 @@ template <class T> inline void HRInto_G1RemSet::par_write_ref_nv(HeapRegion* fro
   }
 }
 
-template <class T> inline void UpdateRSOopClosure::do_oop_work(T* p) {
+template <class T>
+inline void UpdateRSOopClosure::do_oop_work(T* p) {
   assert(_from != NULL, "from region must be non-NULL");
   _rs->par_write_ref(_from, p, _worker_i);
 }
 
-template <class T> inline void UpdateRSetImmediate::do_oop_work(T* p) {
+template <class T>
+inline void UpdateRSetImmediate::do_oop_work(T* p) {
   assert(_from->is_in_reserved(p), "paranoia");
   T heap_oop = oopDesc::load_heap_oop(p);
   if (!oopDesc::is_null(heap_oop) && !_from->is_survivor()) {
diff --git a/hotspot/src/share/vm/gc_implementation/g1/g1_globals.hpp b/hotspot/src/share/vm/gc_implementation/g1/g1_globals.hpp
index 0abac8a1855..67dc4d1c40c 100644
--- a/hotspot/src/share/vm/gc_implementation/g1/g1_globals.hpp
+++ b/hotspot/src/share/vm/gc_implementation/g1/g1_globals.hpp
@@ -40,9 +40,6 @@
   develop(intx, G1PolicyVerbose, 0,                                         \
           "The verbosity level on G1 policy decisions")                     \
                                                                             \
-  develop(bool, G1UseHRIntoRS, true,                                        \
-          "Determines whether the 'advanced' HR Into rem set is used.")     \
-                                                                            \
   develop(intx, G1MarkingVerboseLevel, 0,                                   \
           "Level (0-4) of verboseness of the marking code")                 \
                                                                             \
diff --git a/hotspot/src/share/vm/gc_implementation/includeDB_gc_g1 b/hotspot/src/share/vm/gc_implementation/includeDB_gc_g1
index d1476972275..073a48cad3c 100644
--- a/hotspot/src/share/vm/gc_implementation/includeDB_gc_g1
+++ b/hotspot/src/share/vm/gc_implementation/includeDB_gc_g1
@@ -310,10 +310,16 @@ heapRegionSeq.hpp                       heapRegion.hpp
 
 heapRegionSeq.inline.hpp                heapRegionSeq.hpp
 
+instanceKlass.cpp                       g1RemSet.inline.hpp
+
+instanceRefKlass.cpp                    g1RemSet.inline.hpp
+
 klass.hpp				g1OopClosures.hpp
 
 memoryService.cpp                       g1MemoryPool.hpp
 
+objArrayKlass.cpp                       g1RemSet.inline.hpp
+
 ptrQueue.cpp                            allocation.hpp
 ptrQueue.cpp                            allocation.inline.hpp
 ptrQueue.cpp                            mutex.hpp

From d85b042fcadbb20aba0a162e24543af18e72de82 Mon Sep 17 00:00:00 2001
From: John Coomes <jcoomes@openjdk.org>
Date: Tue, 12 Oct 2010 11:29:45 -0700
Subject: [PATCH 07/23] 6989448: G1: refactor and simplify G1ParScanThreadState

Reviewed-by: iveresov, tonyp
---
 .../gc_implementation/g1/g1CollectedHeap.cpp  | 112 +++++++++++++-----
 .../gc_implementation/g1/g1CollectedHeap.hpp  | 108 +++--------------
 2 files changed, 96 insertions(+), 124 deletions(-)

diff --git a/hotspot/src/share/vm/gc_implementation/g1/g1CollectedHeap.cpp b/hotspot/src/share/vm/gc_implementation/g1/g1CollectedHeap.cpp
index 6a9114268fe..d12e59fd316 100644
--- a/hotspot/src/share/vm/gc_implementation/g1/g1CollectedHeap.cpp
+++ b/hotspot/src/share/vm/gc_implementation/g1/g1CollectedHeap.cpp
@@ -3845,6 +3845,54 @@ G1ParScanThreadState::print_termination_stats(int i,
                undo_waste() * HeapWordSize / K);
 }
 
+#ifdef ASSERT
+bool G1ParScanThreadState::verify_ref(narrowOop* ref) const {
+  assert(ref != NULL, "invariant");
+  assert(UseCompressedOops, "sanity");
+  assert(!has_partial_array_mask(ref), err_msg("ref=" PTR_FORMAT, ref));
+  oop p = oopDesc::load_decode_heap_oop(ref);
+  assert(_g1h->is_in_g1_reserved(p),
+         err_msg("ref=" PTR_FORMAT " p=" PTR_FORMAT, ref, intptr_t(p)));
+  return true;
+}
+
+bool G1ParScanThreadState::verify_ref(oop* ref) const {
+  assert(ref != NULL, "invariant");
+  if (has_partial_array_mask(ref)) {
+    // Must be in the collection set--it's already been copied.
+    oop p = clear_partial_array_mask(ref);
+    assert(_g1h->obj_in_cs(p),
+           err_msg("ref=" PTR_FORMAT " p=" PTR_FORMAT, ref, intptr_t(p)));
+  } else {
+    oop p = oopDesc::load_decode_heap_oop(ref);
+    assert(_g1h->is_in_g1_reserved(p),
+           err_msg("ref=" PTR_FORMAT " p=" PTR_FORMAT, ref, intptr_t(p)));
+  }
+  return true;
+}
+
+bool G1ParScanThreadState::verify_task(StarTask ref) const {
+  if (ref.is_narrow()) {
+    return verify_ref((narrowOop*) ref);
+  } else {
+    return verify_ref((oop*) ref);
+  }
+}
+#endif // ASSERT
+
+void G1ParScanThreadState::trim_queue() {
+  StarTask ref;
+  do {
+    // Drain the overflow stack first, so other threads can steal.
+    while (refs()->pop_overflow(ref)) {
+      deal_with_reference(ref);
+    }
+    while (refs()->pop_local(ref)) {
+      deal_with_reference(ref);
+    }
+  } while (!refs()->is_empty());
+}
+
 G1ParClosureSuper::G1ParClosureSuper(G1CollectedHeap* g1, G1ParScanThreadState* par_scan_state) :
   _g1(g1), _g1_rem(_g1->g1_rem_set()), _cm(_g1->concurrent_mark()),
   _par_scan_state(par_scan_state) { }
@@ -4047,39 +4095,40 @@ public:
     : _g1h(g1h), _par_scan_state(par_scan_state),
       _queues(queues), _terminator(terminator) {}
 
-  void do_void() {
-    G1ParScanThreadState* pss = par_scan_state();
-    while (true) {
-      pss->trim_queue();
+  void do_void();
 
-      StarTask stolen_task;
-      if (queues()->steal(pss->queue_num(), pss->hash_seed(), stolen_task)) {
-        // slightly paranoid tests; I'm trying to catch potential
-        // problems before we go into push_on_queue to know where the
-        // problem is coming from
-        assert((oop*)stolen_task != NULL, "Error");
-        if (stolen_task.is_narrow()) {
-          assert(UseCompressedOops, "Error");
-          narrowOop* p = (narrowOop*) stolen_task;
-          assert(has_partial_array_mask(p) ||
-                 _g1h->is_in_g1_reserved(oopDesc::load_decode_heap_oop(p)), "Error");
-          pss->push_on_queue(p);
-        } else {
-          oop* p = (oop*) stolen_task;
-          assert(has_partial_array_mask(p) || _g1h->is_in_g1_reserved(*p), "Error");
-          pss->push_on_queue(p);
-        }
-        continue;
-      }
-      pss->start_term_time();
-      if (terminator()->offer_termination()) break;
-      pss->end_term_time();
-    }
-    pss->end_term_time();
-    pss->retire_alloc_buffers();
-  }
+private:
+  inline bool offer_termination();
 };
 
+bool G1ParEvacuateFollowersClosure::offer_termination() {
+  G1ParScanThreadState* const pss = par_scan_state();
+  pss->start_term_time();
+  const bool res = terminator()->offer_termination();
+  pss->end_term_time();
+  return res;
+}
+
+void G1ParEvacuateFollowersClosure::do_void() {
+  StarTask stolen_task;
+  G1ParScanThreadState* const pss = par_scan_state();
+  pss->trim_queue();
+
+  do {
+    while (queues()->steal(pss->queue_num(), pss->hash_seed(), stolen_task)) {
+      assert(pss->verify_task(stolen_task), "sanity");
+      if (stolen_task.is_narrow()) {
+        pss->push_on_queue((narrowOop*) stolen_task);
+      } else {
+        pss->push_on_queue((oop*) stolen_task);
+      }
+      pss->trim_queue();
+    }
+  } while (!offer_termination());
+
+  pss->retire_alloc_buffers();
+}
+
 class G1ParTask : public AbstractGangTask {
 protected:
   G1CollectedHeap*       _g1h;
@@ -4177,8 +4226,7 @@ public:
       pss.print_termination_stats(i);
     }
 
-    assert(pss.refs_to_scan() == 0, "Task queue should be empty");
-    assert(pss.overflowed_refs_to_scan() == 0, "Overflow queue should be empty");
+    assert(pss.refs()->is_empty(), "should be empty");
     double end_time_ms = os::elapsedTime() * 1000.0;
     _g1h->g1_policy()->record_gc_worker_end_time(i, end_time_ms);
   }
diff --git a/hotspot/src/share/vm/gc_implementation/g1/g1CollectedHeap.hpp b/hotspot/src/share/vm/gc_implementation/g1/g1CollectedHeap.hpp
index 50869e301fb..c474ce2ea5c 100644
--- a/hotspot/src/share/vm/gc_implementation/g1/g1CollectedHeap.hpp
+++ b/hotspot/src/share/vm/gc_implementation/g1/g1CollectedHeap.hpp
@@ -1651,49 +1651,17 @@ public:
   size_t alloc_buffer_waste() const              { return _alloc_buffer_waste; }
   size_t undo_waste() const                      { return _undo_waste; }
 
-  template <class T> void push_on_queue(T* ref) {
-    assert(ref != NULL, "invariant");
-    assert(has_partial_array_mask(ref) ||
-           _g1h->is_in_g1_reserved(oopDesc::load_decode_heap_oop(ref)), "invariant");
 #ifdef ASSERT
-    if (has_partial_array_mask(ref)) {
-      oop p = clear_partial_array_mask(ref);
-      // Verify that we point into the CS
-      assert(_g1h->obj_in_cs(p), "Should be in CS");
-    }
-#endif
+  bool verify_ref(narrowOop* ref) const;
+  bool verify_ref(oop* ref) const;
+  bool verify_task(StarTask ref) const;
+#endif // ASSERT
+
+  template <class T> void push_on_queue(T* ref) {
+    assert(verify_ref(ref), "sanity");
     refs()->push(ref);
   }
 
-  void pop_from_queue(StarTask& ref) {
-    if (refs()->pop_local(ref)) {
-      assert((oop*)ref != NULL, "pop_local() returned true");
-      assert(UseCompressedOops || !ref.is_narrow(), "Error");
-      assert(has_partial_array_mask((oop*)ref) ||
-             _g1h->is_in_g1_reserved(ref.is_narrow() ? oopDesc::load_decode_heap_oop((narrowOop*)ref)
-                                                     : oopDesc::load_decode_heap_oop((oop*)ref)),
-              "invariant");
-    } else {
-      StarTask null_task;
-      ref = null_task;
-    }
-  }
-
-  void pop_from_overflow_queue(StarTask& ref) {
-    StarTask new_ref;
-    refs()->pop_overflow(new_ref);
-    assert((oop*)new_ref != NULL, "pop() from a local non-empty stack");
-    assert(UseCompressedOops || !new_ref.is_narrow(), "Error");
-    assert(has_partial_array_mask((oop*)new_ref) ||
-           _g1h->is_in_g1_reserved(new_ref.is_narrow() ? oopDesc::load_decode_heap_oop((narrowOop*)new_ref)
-                                                       : oopDesc::load_decode_heap_oop((oop*)new_ref)),
-           "invariant");
-    ref = new_ref;
-  }
-
-  int refs_to_scan()            { return (int)refs()->size(); }
-  int overflowed_refs_to_scan() { return (int)refs()->overflow_stack()->size(); }
-
   template <class T> void update_rs(HeapRegion* from, T* p, int tid) {
     if (G1DeferredRSUpdate) {
       deferred_rs_update(from, p, tid);
@@ -1818,59 +1786,15 @@ private:
     }
   }
 
-public:
-  void trim_queue() {
-    // I've replicated the loop twice, first to drain the overflow
-    // queue, second to drain the task queue. This is better than
-    // having a single loop, which checks both conditions and, inside
-    // it, either pops the overflow queue or the task queue, as each
-    // loop is tighter. Also, the decision to drain the overflow queue
-    // first is not arbitrary, as the overflow queue is not visible
-    // to the other workers, whereas the task queue is. So, we want to
-    // drain the "invisible" entries first, while allowing the other
-    // workers to potentially steal the "visible" entries.
-
-    while (refs_to_scan() > 0 || overflowed_refs_to_scan() > 0) {
-      while (overflowed_refs_to_scan() > 0) {
-        StarTask ref_to_scan;
-        assert((oop*)ref_to_scan == NULL, "Constructed above");
-        pop_from_overflow_queue(ref_to_scan);
-        // We shouldn't have pushed it on the queue if it was not
-        // pointing into the CSet.
-        assert((oop*)ref_to_scan != NULL, "Follows from inner loop invariant");
-        if (ref_to_scan.is_narrow()) {
-          assert(UseCompressedOops, "Error");
-          narrowOop* p = (narrowOop*)ref_to_scan;
-          assert(!has_partial_array_mask(p) &&
-                 _g1h->is_in_g1_reserved(oopDesc::load_decode_heap_oop(p)), "sanity");
-          deal_with_reference(p);
-        } else {
-          oop* p = (oop*)ref_to_scan;
-          assert((has_partial_array_mask(p) && _g1h->is_in_g1_reserved(clear_partial_array_mask(p))) ||
-                 _g1h->is_in_g1_reserved(oopDesc::load_decode_heap_oop(p)), "sanity");
-          deal_with_reference(p);
-        }
-      }
-
-      while (refs_to_scan() > 0) {
-        StarTask ref_to_scan;
-        assert((oop*)ref_to_scan == NULL, "Constructed above");
-        pop_from_queue(ref_to_scan);
-        if ((oop*)ref_to_scan != NULL) {
-          if (ref_to_scan.is_narrow()) {
-            assert(UseCompressedOops, "Error");
-            narrowOop* p = (narrowOop*)ref_to_scan;
-            assert(!has_partial_array_mask(p) &&
-                    _g1h->is_in_g1_reserved(oopDesc::load_decode_heap_oop(p)), "sanity");
-            deal_with_reference(p);
-          } else {
-            oop* p = (oop*)ref_to_scan;
-            assert((has_partial_array_mask(p) && _g1h->obj_in_cs(clear_partial_array_mask(p))) ||
-                   _g1h->is_in_g1_reserved(oopDesc::load_decode_heap_oop(p)), "sanity");
-            deal_with_reference(p);
-          }
-        }
-      }
+  void deal_with_reference(StarTask ref) {
+    assert(verify_task(ref), "sanity");
+    if (ref.is_narrow()) {
+      deal_with_reference((narrowOop*)ref);
+    } else {
+      deal_with_reference((oop*)ref);
     }
   }
+
+public:
+  void trim_queue();
 };

From e79a62412fb64ca5783fb536aa2640bf814304d9 Mon Sep 17 00:00:00 2001
From: Antonios Printezis <tonyp@openjdk.org>
Date: Thu, 14 Oct 2010 10:38:14 -0400
Subject: [PATCH 08/23] 6990359: G1: don't push a stolen entry on the
 taskqueue, deal with it directly

When an entry is stolen, don't push it on the task queue but process it directly.

Reviewed-by: iveresov, ysr, jcoomes
---
 .../src/share/vm/gc_implementation/g1/g1CollectedHeap.cpp | 8 ++++++--
 .../src/share/vm/gc_implementation/g1/g1CollectedHeap.hpp | 1 -
 2 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/hotspot/src/share/vm/gc_implementation/g1/g1CollectedHeap.cpp b/hotspot/src/share/vm/gc_implementation/g1/g1CollectedHeap.cpp
index d12e59fd316..c6cd6607eb4 100644
--- a/hotspot/src/share/vm/gc_implementation/g1/g1CollectedHeap.cpp
+++ b/hotspot/src/share/vm/gc_implementation/g1/g1CollectedHeap.cpp
@@ -4118,10 +4118,14 @@ void G1ParEvacuateFollowersClosure::do_void() {
     while (queues()->steal(pss->queue_num(), pss->hash_seed(), stolen_task)) {
       assert(pss->verify_task(stolen_task), "sanity");
       if (stolen_task.is_narrow()) {
-        pss->push_on_queue((narrowOop*) stolen_task);
+        pss->deal_with_reference((narrowOop*) stolen_task);
       } else {
-        pss->push_on_queue((oop*) stolen_task);
+        pss->deal_with_reference((oop*) stolen_task);
       }
+
+      // We've just processed a reference and we might have made
+      // available new entries on the queues. So we have to make sure
+      // we drain the queues as necessary.
       pss->trim_queue();
     }
   } while (!offer_termination());
diff --git a/hotspot/src/share/vm/gc_implementation/g1/g1CollectedHeap.hpp b/hotspot/src/share/vm/gc_implementation/g1/g1CollectedHeap.hpp
index c474ce2ea5c..b97d8710a64 100644
--- a/hotspot/src/share/vm/gc_implementation/g1/g1CollectedHeap.hpp
+++ b/hotspot/src/share/vm/gc_implementation/g1/g1CollectedHeap.hpp
@@ -1772,7 +1772,6 @@ public:
     }
   }
 
-private:
   template <class T> void deal_with_reference(T* ref_to_scan) {
     if (has_partial_array_mask(ref_to_scan)) {
       _partial_scan_cl->do_oop_nv(ref_to_scan);

From e4f08bc1950f4f4c73bfd414bfaac585a10ecf46 Mon Sep 17 00:00:00 2001
From: Tom Rodriguez <never@openjdk.org>
Date: Fri, 15 Oct 2010 14:21:11 -0700
Subject: [PATCH 09/23] 6992477: fix for 6991512 broke sparc barriers

Reviewed-by: kvn, iveresov
---
 hotspot/src/cpu/sparc/vm/c1_LIRGenerator_sparc.cpp | 2 +-
 hotspot/src/cpu/x86/vm/c1_CodeStubs_x86.cpp        | 2 +-
 hotspot/src/cpu/x86/vm/c1_LIRGenerator_x86.cpp     | 2 +-
 hotspot/src/share/vm/c1/c1_LIRGenerator.cpp        | 1 -
 4 files changed, 3 insertions(+), 4 deletions(-)

diff --git a/hotspot/src/cpu/sparc/vm/c1_LIRGenerator_sparc.cpp b/hotspot/src/cpu/sparc/vm/c1_LIRGenerator_sparc.cpp
index c5385a984b7..2566f3f8227 100644
--- a/hotspot/src/cpu/sparc/vm/c1_LIRGenerator_sparc.cpp
+++ b/hotspot/src/cpu/sparc/vm/c1_LIRGenerator_sparc.cpp
@@ -664,7 +664,7 @@ void LIRGenerator::do_CompareAndSwap(Intrinsic* x, ValueType* type) {
   // Use temps to avoid kills
   LIR_Opr t1 = FrameMap::G1_opr;
   LIR_Opr t2 = FrameMap::G3_opr;
-  LIR_Opr addr = (type == objectType) ? new_register(T_OBJECT) : new_pointer_register();
+  LIR_Opr addr = new_pointer_register();
 
   // get address of field
   obj.load_item();
diff --git a/hotspot/src/cpu/x86/vm/c1_CodeStubs_x86.cpp b/hotspot/src/cpu/x86/vm/c1_CodeStubs_x86.cpp
index f5bea330c2f..b6f9eea591b 100644
--- a/hotspot/src/cpu/x86/vm/c1_CodeStubs_x86.cpp
+++ b/hotspot/src/cpu/x86/vm/c1_CodeStubs_x86.cpp
@@ -499,7 +499,7 @@ void G1PostBarrierStub::emit_code(LIR_Assembler* ce) {
   Register new_val_reg = new_val()->as_register();
   __ cmpptr(new_val_reg, (int32_t) NULL_WORD);
   __ jcc(Assembler::equal, _continuation);
-  ce->store_parameter(addr()->as_register(), 0);
+  ce->store_parameter(addr()->as_pointer_register(), 0);
   __ call(RuntimeAddress(Runtime1::entry_for(Runtime1::g1_post_barrier_slow_id)));
   __ jmp(_continuation);
 }
diff --git a/hotspot/src/cpu/x86/vm/c1_LIRGenerator_x86.cpp b/hotspot/src/cpu/x86/vm/c1_LIRGenerator_x86.cpp
index c9a26e7b1dd..dd8bc8d9962 100644
--- a/hotspot/src/cpu/x86/vm/c1_LIRGenerator_x86.cpp
+++ b/hotspot/src/cpu/x86/vm/c1_LIRGenerator_x86.cpp
@@ -765,7 +765,7 @@ void LIRGenerator::do_CompareAndSwap(Intrinsic* x, ValueType* type) {
     ShouldNotReachHere();
   }
 
-  LIR_Opr addr = (type == objectType) ? new_register(T_OBJECT) : new_pointer_register();
+  LIR_Opr addr = new_pointer_register();
   LIR_Address* a;
   if(offset.result()->is_constant()) {
     a = new LIR_Address(obj.result(),
diff --git a/hotspot/src/share/vm/c1/c1_LIRGenerator.cpp b/hotspot/src/share/vm/c1/c1_LIRGenerator.cpp
index dffc85c970d..332f193e28e 100644
--- a/hotspot/src/share/vm/c1/c1_LIRGenerator.cpp
+++ b/hotspot/src/share/vm/c1/c1_LIRGenerator.cpp
@@ -1350,7 +1350,6 @@ void LIRGenerator::G1SATBCardTableModRef_post_barrier(LIR_OprDesc* addr, LIR_Opr
     addr = ptr;
   }
   assert(addr->is_register(), "must be a register at this point");
-  assert(addr->type() == T_OBJECT, "addr should point to an object");
 
   LIR_Opr xor_res = new_pointer_register();
   LIR_Opr xor_shift_res = new_pointer_register();

From 0c9bfb6003cbd798f8e9a71be2334d415144f921 Mon Sep 17 00:00:00 2001
From: Antonios Printezis <tonyp@openjdk.org>
Date: Fri, 15 Oct 2010 17:26:56 -0400
Subject: [PATCH 10/23] 6992189: G1: inconsistent base used in sparse rem set
 iterator

The remembered set iterator for sparse tables incorrectly assumes that index 0 corresponds to the bottom of the heap, not address 0 as it is the case.

Reviewed-by: ysr, jmasa
---
 .../gc_implementation/g1/heapRegionRemSet.cpp  |  4 +---
 .../vm/gc_implementation/g1/sparsePRT.cpp      |  5 +----
 .../vm/gc_implementation/g1/sparsePRT.hpp      | 18 ++++--------------
 3 files changed, 6 insertions(+), 21 deletions(-)

diff --git a/hotspot/src/share/vm/gc_implementation/g1/heapRegionRemSet.cpp b/hotspot/src/share/vm/gc_implementation/g1/heapRegionRemSet.cpp
index c4c87327484..5d055f65597 100644
--- a/hotspot/src/share/vm/gc_implementation/g1/heapRegionRemSet.cpp
+++ b/hotspot/src/share/vm/gc_implementation/g1/heapRegionRemSet.cpp
@@ -1159,9 +1159,7 @@ HeapRegionRemSetIterator() :
   _hrrs(NULL),
   _g1h(G1CollectedHeap::heap()),
   _bosa(NULL),
-  _sparse_iter(size_t(G1CollectedHeap::heap()->reserved_region().start())
-               >> CardTableModRefBS::card_shift)
-{}
+  _sparse_iter() { }
 
 void HeapRegionRemSetIterator::initialize(const HeapRegionRemSet* hrrs) {
   _hrrs = hrrs;
diff --git a/hotspot/src/share/vm/gc_implementation/g1/sparsePRT.cpp b/hotspot/src/share/vm/gc_implementation/g1/sparsePRT.cpp
index 5bc3ab29c70..86eaa729d3f 100644
--- a/hotspot/src/share/vm/gc_implementation/g1/sparsePRT.cpp
+++ b/hotspot/src/share/vm/gc_implementation/g1/sparsePRT.cpp
@@ -323,10 +323,7 @@ CardIdx_t /* RSHashTable:: */ RSHashTableIter::find_first_card_in_list() {
 }
 
 size_t /* RSHashTable:: */ RSHashTableIter::compute_card_ind(CardIdx_t ci) {
-  return
-    _heap_bot_card_ind
-    + (_rsht->entry(_bl_ind)->r_ind() * HeapRegion::CardsPerRegion)
-    + ci;
+  return (_rsht->entry(_bl_ind)->r_ind() * HeapRegion::CardsPerRegion) + ci;
 }
 
 bool /* RSHashTable:: */ RSHashTableIter::has_next(size_t& card_index) {
diff --git a/hotspot/src/share/vm/gc_implementation/g1/sparsePRT.hpp b/hotspot/src/share/vm/gc_implementation/g1/sparsePRT.hpp
index 7812ac4fb53..92a69b4d5cc 100644
--- a/hotspot/src/share/vm/gc_implementation/g1/sparsePRT.hpp
+++ b/hotspot/src/share/vm/gc_implementation/g1/sparsePRT.hpp
@@ -169,7 +169,6 @@ class RSHashTableIter VALUE_OBJ_CLASS_SPEC {
   int _bl_ind;          // [-1, 0.._rsht->_capacity)
   short _card_ind;      // [0..SparsePRTEntry::cards_num())
   RSHashTable* _rsht;
-  size_t _heap_bot_card_ind;
 
   // If the bucket list pointed to by _bl_ind contains a card, sets
   // _bl_ind to the index of that entry, and returns the card.
@@ -183,13 +182,11 @@ class RSHashTableIter VALUE_OBJ_CLASS_SPEC {
   size_t compute_card_ind(CardIdx_t ci);
 
 public:
-  RSHashTableIter(size_t heap_bot_card_ind) :
+  RSHashTableIter() :
     _tbl_ind(RSHashTable::NullEntry),
     _bl_ind(RSHashTable::NullEntry),
     _card_ind((SparsePRTEntry::cards_num() - 1)),
-    _rsht(NULL),
-    _heap_bot_card_ind(heap_bot_card_ind)
-  {}
+    _rsht(NULL) {}
 
   void init(RSHashTable* rsht) {
     _rsht = rsht;
@@ -280,19 +277,12 @@ public:
   bool contains_card(RegionIdx_t region_id, CardIdx_t card_index) const {
     return _next->contains_card(region_id, card_index);
   }
-
-#if 0
-  void verify_is_cleared();
-  void print();
-#endif
 };
 
 
-class SparsePRTIter: public /* RSHashTable:: */RSHashTableIter {
+class SparsePRTIter: public RSHashTableIter {
 public:
-  SparsePRTIter(size_t heap_bot_card_ind) :
-    /* RSHashTable:: */RSHashTableIter(heap_bot_card_ind)
-  {}
+  SparsePRTIter() : RSHashTableIter() { }
 
   void init(const SparsePRT* sprt) {
     RSHashTableIter::init(sprt->cur());

From f6acb9efb026a05ee7f436fbf8f276857562ece4 Mon Sep 17 00:00:00 2001
From: John Cuthbertson <johnc@openjdk.org>
Date: Mon, 18 Oct 2010 15:01:41 -0700
Subject: [PATCH 11/23] 6988458: G1: assert(mr.end() <= _cm->finger()) failed:
 otherwise the region shouldn't be on the stack

The changes from 6941395 did not clear the CMTask::_aborted_region fields when concurrent marking aborted because of overflow. As a result, the next time around we could see a memory region whose start address was above the global finger and the assertion tripped. Moved the clearing of the aborted regions to ConcurrentMark::clear_marking_state, which is executed on all of the exit paths.

Reviewed-by: tonyp, ysr, jmasa
---
 hotspot/src/share/vm/gc_implementation/g1/concurrentMark.cpp | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/hotspot/src/share/vm/gc_implementation/g1/concurrentMark.cpp b/hotspot/src/share/vm/gc_implementation/g1/concurrentMark.cpp
index 5bf7faec667..ec9c9da4f81 100644
--- a/hotspot/src/share/vm/gc_implementation/g1/concurrentMark.cpp
+++ b/hotspot/src/share/vm/gc_implementation/g1/concurrentMark.cpp
@@ -2418,6 +2418,8 @@ void ConcurrentMark::clear_marking_state() {
   for (int i = 0; i < (int)_max_task_num; ++i) {
     OopTaskQueue* queue = _task_queues->queue(i);
     queue->set_empty();
+    // Clear any partial regions from the CMTasks
+    _tasks[i]->clear_aborted_region();
   }
 }
 
@@ -2706,7 +2708,6 @@ void ConcurrentMark::abort() {
   clear_marking_state();
   for (int i = 0; i < (int)_max_task_num; ++i) {
     _tasks[i]->clear_region_fields();
-    _tasks[i]->clear_aborted_region();
   }
   _has_aborted = true;
 
@@ -2985,7 +2986,7 @@ void CMTask::reset(CMBitMap* nextMarkBitMap) {
 
   _nextMarkBitMap                = nextMarkBitMap;
   clear_region_fields();
-  clear_aborted_region();
+  assert(_aborted_region.is_empty(), "should have been cleared");
 
   _calls                         = 0;
   _elapsed_time_ms               = 0.0;

From 0c660e1f60c4bd37086a92e807ce071a1d948efa Mon Sep 17 00:00:00 2001
From: Antonios Printezis <tonyp@openjdk.org>
Date: Sat, 16 Oct 2010 17:12:19 -0400
Subject: [PATCH 12/23] 6991377: G1: race between concurrent refinement and
 humongous object allocation

There is a race between the concurrent refinement threads and the humongous object allocation that can cause the concurrent refinement threads to corrupt the part of the BOT that it is being initialized by the humongous object allocation operation. The solution is to do the humongous object allocation in careful steps to ensure that the concurrent refinement threads always have a consistent view over the BOT, region contents, and top. The fix includes some very minor tidying up in sparsePRT.

Reviewed-by: jcoomes, johnc, ysr
---
 .../g1/g1BlockOffsetTable.cpp                 |  21 ++-
 .../g1/g1BlockOffsetTable.hpp                 |   4 +
 .../vm/gc_implementation/g1/heapRegion.cpp    |  37 +++---
 .../vm/gc_implementation/g1/heapRegion.hpp    |  11 +-
 .../vm/gc_implementation/g1/heapRegionSeq.cpp | 122 +++++++++++++++---
 .../vm/gc_implementation/g1/sparsePRT.cpp     |   6 +-
 .../vm/gc_implementation/g1/sparsePRT.hpp     |   2 -
 7 files changed, 149 insertions(+), 54 deletions(-)

diff --git a/hotspot/src/share/vm/gc_implementation/g1/g1BlockOffsetTable.cpp b/hotspot/src/share/vm/gc_implementation/g1/g1BlockOffsetTable.cpp
index 106c521698a..8eb6607ae14 100644
--- a/hotspot/src/share/vm/gc_implementation/g1/g1BlockOffsetTable.cpp
+++ b/hotspot/src/share/vm/gc_implementation/g1/g1BlockOffsetTable.cpp
@@ -175,7 +175,7 @@ G1BlockOffsetArray::set_remainder_to_point_to_start_incl(size_t start_card, size
   }
   assert(start_card > _array->index_for(_bottom), "Cannot be first card");
   assert(_array->offset_array(start_card-1) <= N_words,
-    "Offset card has an unexpected value");
+         "Offset card has an unexpected value");
   size_t start_card_for_region = start_card;
   u_char offset = max_jubyte;
   for (int i = 0; i < BlockOffsetArray::N_powers; i++) {
@@ -577,6 +577,16 @@ void G1BlockOffsetArray::alloc_block_work2(HeapWord** threshold_, size_t* index_
 #endif
 }
 
+void
+G1BlockOffsetArray::set_for_starts_humongous(HeapWord* new_end) {
+  assert(_end ==  new_end, "_end should have already been updated");
+
+  // The first BOT entry should have offset 0.
+  _array->set_offset_array(_array->index_for(_bottom), 0);
+  // The rest should point to the first one.
+  set_remainder_to_point_to_start(_bottom + N_words, new_end);
+}
+
 //////////////////////////////////////////////////////////////////////
 // G1BlockOffsetArrayContigSpace
 //////////////////////////////////////////////////////////////////////
@@ -626,3 +636,12 @@ void G1BlockOffsetArrayContigSpace::zero_bottom_entry() {
          "Precondition of call");
   _array->set_offset_array(bottom_index, 0);
 }
+
+void
+G1BlockOffsetArrayContigSpace::set_for_starts_humongous(HeapWord* new_end) {
+  G1BlockOffsetArray::set_for_starts_humongous(new_end);
+
+  // Make sure _next_offset_threshold and _next_offset_index point to new_end.
+  _next_offset_threshold = new_end;
+  _next_offset_index     = _array->index_for(new_end);
+}
diff --git a/hotspot/src/share/vm/gc_implementation/g1/g1BlockOffsetTable.hpp b/hotspot/src/share/vm/gc_implementation/g1/g1BlockOffsetTable.hpp
index 7ce92971839..28c29502c7c 100644
--- a/hotspot/src/share/vm/gc_implementation/g1/g1BlockOffsetTable.hpp
+++ b/hotspot/src/share/vm/gc_implementation/g1/g1BlockOffsetTable.hpp
@@ -436,6 +436,8 @@ public:
   }
 
   void check_all_cards(size_t left_card, size_t right_card) const;
+
+  virtual void set_for_starts_humongous(HeapWord* new_end);
 };
 
 // A subtype of BlockOffsetArray that takes advantage of the fact
@@ -484,4 +486,6 @@ class G1BlockOffsetArrayContigSpace: public G1BlockOffsetArray {
 
   HeapWord* block_start_unsafe(const void* addr);
   HeapWord* block_start_unsafe_const(const void* addr) const;
+
+  virtual void set_for_starts_humongous(HeapWord* new_end);
 };
diff --git a/hotspot/src/share/vm/gc_implementation/g1/heapRegion.cpp b/hotspot/src/share/vm/gc_implementation/g1/heapRegion.cpp
index 58b55123aae..dbe2d24a6ce 100644
--- a/hotspot/src/share/vm/gc_implementation/g1/heapRegion.cpp
+++ b/hotspot/src/share/vm/gc_implementation/g1/heapRegion.cpp
@@ -377,10 +377,26 @@ void HeapRegion::calc_gc_efficiency() {
 }
 // </PREDICTION>
 
-void HeapRegion::set_startsHumongous() {
+void HeapRegion::set_startsHumongous(HeapWord* new_end) {
+  assert(end() == _orig_end,
+         "Should be normal before the humongous object allocation");
+  assert(top() == bottom(), "should be empty");
+
   _humongous_type = StartsHumongous;
   _humongous_start_region = this;
-  assert(end() == _orig_end, "Should be normal before alloc.");
+
+  set_end(new_end);
+  _offsets.set_for_starts_humongous(new_end);
+}
+
+void HeapRegion::set_continuesHumongous(HeapRegion* start) {
+  assert(end() == _orig_end,
+         "Should be normal before the humongous object allocation");
+  assert(top() == bottom(), "should be empty");
+  assert(start->startsHumongous(), "pre-condition");
+
+  _humongous_type = ContinuesHumongous;
+  _humongous_start_region = start;
 }
 
 bool HeapRegion::claimHeapRegion(jint claimValue) {
@@ -500,23 +516,6 @@ CompactibleSpace* HeapRegion::next_compaction_space() const {
   return blk.result();
 }
 
-void HeapRegion::set_continuesHumongous(HeapRegion* start) {
-  // The order is important here.
-  start->add_continuingHumongousRegion(this);
-  _humongous_type = ContinuesHumongous;
-  _humongous_start_region = start;
-}
-
-void HeapRegion::add_continuingHumongousRegion(HeapRegion* cont) {
-  // Must join the blocks of the current H region seq with the block of the
-  // added region.
-  offsets()->join_blocks(bottom(), cont->bottom());
-  arrayOop obj = (arrayOop)(bottom());
-  obj->set_length((int) (obj->length() + cont->capacity()/jintSize));
-  set_end(cont->end());
-  set_top(cont->end());
-}
-
 void HeapRegion::save_marks() {
   set_saved_mark();
 }
diff --git a/hotspot/src/share/vm/gc_implementation/g1/heapRegion.hpp b/hotspot/src/share/vm/gc_implementation/g1/heapRegion.hpp
index 42e96bbfb50..48118ceb7e8 100644
--- a/hotspot/src/share/vm/gc_implementation/g1/heapRegion.hpp
+++ b/hotspot/src/share/vm/gc_implementation/g1/heapRegion.hpp
@@ -395,14 +395,12 @@ class HeapRegion: public G1OffsetTableContigSpace {
 
   // Causes the current region to represent a humongous object spanning "n"
   // regions.
-  virtual void set_startsHumongous();
+  void set_startsHumongous(HeapWord* new_end);
 
   // The regions that continue a humongous sequence should be added using
   // this method, in increasing address order.
   void set_continuesHumongous(HeapRegion* start);
 
-  void add_continuingHumongousRegion(HeapRegion* cont);
-
   // If the region has a remembered set, return a pointer to it.
   HeapRegionRemSet* rem_set() const {
     return _rem_set;
@@ -733,13 +731,6 @@ class HeapRegion: public G1OffsetTableContigSpace {
                                    FilterOutOfRegionClosure* cl,
                                    bool filter_young);
 
-  // The region "mr" is entirely in "this", and starts and ends at block
-  // boundaries. The caller declares that all the contained blocks are
-  // coalesced into one.
-  void declare_filled_region_to_BOT(MemRegion mr) {
-    _offsets.single_block(mr.start(), mr.end());
-  }
-
   // A version of block start that is guaranteed to find *some* block
   // boundary at or before "p", but does not object iteration, and may
   // therefore be used safely when the heap is unparseable.
diff --git a/hotspot/src/share/vm/gc_implementation/g1/heapRegionSeq.cpp b/hotspot/src/share/vm/gc_implementation/g1/heapRegionSeq.cpp
index b71cf1918c9..0640cbe0d39 100644
--- a/hotspot/src/share/vm/gc_implementation/g1/heapRegionSeq.cpp
+++ b/hotspot/src/share/vm/gc_implementation/g1/heapRegionSeq.cpp
@@ -91,34 +91,118 @@ HeapRegionSeq::alloc_obj_from_region_index(int ind, size_t word_size) {
   }
   if (sumSizes >= word_size) {
     _alloc_search_start = cur;
-    // Mark the allocated regions as allocated.
+
+    // We need to initialize the region(s) we just discovered. This is
+    // a bit tricky given that it can happen concurrently with
+    // refinement threads refining cards on these regions and
+    // potentially wanting to refine the BOT as they are scanning
+    // those cards (this can happen shortly after a cleanup; see CR
+    // 6991377). So we have to set up the region(s) carefully and in
+    // a specific order.
+
+    // Currently, allocs_are_zero_filled() returns false. The zero
+    // filling infrastructure will be going away soon (see CR 6977804).
+    // So no need to do anything else here.
     bool zf = G1CollectedHeap::heap()->allocs_are_zero_filled();
+    assert(!zf, "not supported");
+
+    // This will be the "starts humongous" region.
     HeapRegion* first_hr = _regions.at(first);
-    for (int i = first; i < cur; i++) {
-      HeapRegion* hr = _regions.at(i);
-      if (zf)
-        hr->ensure_zero_filled();
+    {
+      MutexLockerEx x(ZF_mon, Mutex::_no_safepoint_check_flag);
+      first_hr->set_zero_fill_allocated();
+    }
+    // The header of the new object will be placed at the bottom of
+    // the first region.
+    HeapWord* new_obj = first_hr->bottom();
+    // This will be the new end of the first region in the series that
+    // should also match the end of the last region in the seriers.
+    // (Note: sumSizes = "region size" x "number of regions we found").
+    HeapWord* new_end = new_obj + sumSizes;
+    // This will be the new top of the first region that will reflect
+    // this allocation.
+    HeapWord* new_top = new_obj + word_size;
+
+    // First, we need to zero the header of the space that we will be
+    // allocating. When we update top further down, some refinement
+    // threads might try to scan the region. By zeroing the header we
+    // ensure that any thread that will try to scan the region will
+    // come across the zero klass word and bail out.
+    //
+    // NOTE: It would not have been correct to have used
+    // CollectedHeap::fill_with_object() and make the space look like
+    // an int array. The thread that is doing the allocation will
+    // later update the object header to a potentially different array
+    // type and, for a very short period of time, the klass and length
+    // fields will be inconsistent. This could cause a refinement
+    // thread to calculate the object size incorrectly.
+    Copy::fill_to_words(new_obj, oopDesc::header_size(), 0);
+
+    // We will set up the first region as "starts humongous". This
+    // will also update the BOT covering all the regions to reflect
+    // that there is a single object that starts at the bottom of the
+    // first region.
+    first_hr->set_startsHumongous(new_end);
+
+    // Then, if there are any, we will set up the "continues
+    // humongous" regions.
+    HeapRegion* hr = NULL;
+    for (int i = first + 1; i < cur; ++i) {
+      hr = _regions.at(i);
       {
         MutexLockerEx x(ZF_mon, Mutex::_no_safepoint_check_flag);
         hr->set_zero_fill_allocated();
       }
-      size_t sz = hr->capacity() / HeapWordSize;
-      HeapWord* tmp = hr->allocate(sz);
-      assert(tmp != NULL, "Humongous allocation failure");
-      MemRegion mr = MemRegion(tmp, sz);
-      CollectedHeap::fill_with_object(mr);
-      hr->declare_filled_region_to_BOT(mr);
-      if (i == first) {
-        first_hr->set_startsHumongous();
+      hr->set_continuesHumongous(first_hr);
+    }
+    // If we have "continues humongous" regions (hr != NULL), then the
+    // end of the last one should match new_end.
+    assert(hr == NULL || hr->end() == new_end, "sanity");
+
+    // Up to this point no concurrent thread would have been able to
+    // do any scanning on any region in this series. All the top
+    // fields still point to bottom, so the intersection between
+    // [bottom,top] and [card_start,card_end] will be empty. Before we
+    // update the top fields, we'll do a storestore to make sure that
+    // no thread sees the update to top before the zeroing of the
+    // object header and the BOT initialization.
+    OrderAccess::storestore();
+
+    // Now that the BOT and the object header have been initialized,
+    // we can update top of the "starts humongous" region.
+    assert(first_hr->bottom() < new_top && new_top <= first_hr->end(),
+           "new_top should be in this region");
+    first_hr->set_top(new_top);
+
+    // Now, we will update the top fields of the "continues humongous"
+    // regions. The reason we need to do this is that, otherwise,
+    // these regions would look empty and this will confuse parts of
+    // G1. For example, the code that looks for a consecutive number
+    // of empty regions will consider them empty and try to
+    // re-allocate them. We can extend is_empty() to also include
+    // !continuesHumongous(), but it is easier to just update the top
+    // fields here.
+    hr = NULL;
+    for (int i = first + 1; i < cur; ++i) {
+      hr = _regions.at(i);
+      if ((i + 1) == cur) {
+        // last continues humongous region
+        assert(hr->bottom() < new_top && new_top <= hr->end(),
+               "new_top should fall on this region");
+        hr->set_top(new_top);
       } else {
-        assert(i > first, "sanity");
-        hr->set_continuesHumongous(first_hr);
+        // not last one
+        assert(new_top > hr->end(), "new_top should be above this region");
+        hr->set_top(hr->end());
       }
     }
-    HeapWord* first_hr_bot = first_hr->bottom();
-    HeapWord* obj_end = first_hr_bot + word_size;
-    first_hr->set_top(obj_end);
-    return first_hr_bot;
+    // If we have continues humongous regions (hr != NULL), then the
+    // end of the last one should match new_end and its top should
+    // match new_top.
+    assert(hr == NULL ||
+           (hr->end() == new_end && hr->top() == new_top), "sanity");
+
+    return new_obj;
   } else {
     // If we started from the beginning, we want to know why we can't alloc.
     return NULL;
diff --git a/hotspot/src/share/vm/gc_implementation/g1/sparsePRT.cpp b/hotspot/src/share/vm/gc_implementation/g1/sparsePRT.cpp
index 86eaa729d3f..896b5ee70b3 100644
--- a/hotspot/src/share/vm/gc_implementation/g1/sparsePRT.cpp
+++ b/hotspot/src/share/vm/gc_implementation/g1/sparsePRT.cpp
@@ -308,7 +308,7 @@ void RSHashTable::add_entry(SparsePRTEntry* e) {
   assert(e2->num_valid_cards() > 0, "Postcondition.");
 }
 
-CardIdx_t /* RSHashTable:: */ RSHashTableIter::find_first_card_in_list() {
+CardIdx_t RSHashTableIter::find_first_card_in_list() {
   CardIdx_t res;
   while (_bl_ind != RSHashTable::NullEntry) {
     res = _rsht->entry(_bl_ind)->card(0);
@@ -322,11 +322,11 @@ CardIdx_t /* RSHashTable:: */ RSHashTableIter::find_first_card_in_list() {
   return SparsePRTEntry::NullEntry;
 }
 
-size_t /* RSHashTable:: */ RSHashTableIter::compute_card_ind(CardIdx_t ci) {
+size_t RSHashTableIter::compute_card_ind(CardIdx_t ci) {
   return (_rsht->entry(_bl_ind)->r_ind() * HeapRegion::CardsPerRegion) + ci;
 }
 
-bool /* RSHashTable:: */ RSHashTableIter::has_next(size_t& card_index) {
+bool RSHashTableIter::has_next(size_t& card_index) {
   _card_ind++;
   CardIdx_t ci;
   if (_card_ind < SparsePRTEntry::cards_num() &&
diff --git a/hotspot/src/share/vm/gc_implementation/g1/sparsePRT.hpp b/hotspot/src/share/vm/gc_implementation/g1/sparsePRT.hpp
index 92a69b4d5cc..715a5515eba 100644
--- a/hotspot/src/share/vm/gc_implementation/g1/sparsePRT.hpp
+++ b/hotspot/src/share/vm/gc_implementation/g1/sparsePRT.hpp
@@ -282,8 +282,6 @@ public:
 
 class SparsePRTIter: public RSHashTableIter {
 public:
-  SparsePRTIter() : RSHashTableIter() { }
-
   void init(const SparsePRT* sprt) {
     RSHashTableIter::init(sprt->cur());
   }

From 6a17d9c416de42f6bdd98b848b780ec5ea66b197 Mon Sep 17 00:00:00 2001
From: Igor Veresov <iveresov@openjdk.org>
Date: Tue, 19 Oct 2010 11:14:52 -0700
Subject: [PATCH 13/23] 6989669: Coops: -Xshare:dump causes crash

Temporarily fix to disable compressed oops with CDS

Reviewed-by: dholmes, twisti, kvn, never
---
 hotspot/src/share/vm/runtime/arguments.cpp | 7 +++++++
 hotspot/src/share/vm/runtime/globals.hpp   | 2 +-
 2 files changed, 8 insertions(+), 1 deletion(-)

diff --git a/hotspot/src/share/vm/runtime/arguments.cpp b/hotspot/src/share/vm/runtime/arguments.cpp
index 2e1ac1e1655..e996510bb1e 100644
--- a/hotspot/src/share/vm/runtime/arguments.cpp
+++ b/hotspot/src/share/vm/runtime/arguments.cpp
@@ -2979,6 +2979,13 @@ jint Arguments::parse(const JavaVMInitArgs* args) {
   UseCompressedOops = false;
 #endif
 
+#if defined(_LP64)
+  if ((DumpSharedSpaces || RequireSharedSpaces) && UseCompressedOops) {
+    // Disable compressed oops with shared spaces
+    UseCompressedOops = false;
+  }
+#endif
+
   // Set object alignment values.
   set_object_alignment();
 
diff --git a/hotspot/src/share/vm/runtime/globals.hpp b/hotspot/src/share/vm/runtime/globals.hpp
index 8395bf9a373..12e10218510 100644
--- a/hotspot/src/share/vm/runtime/globals.hpp
+++ b/hotspot/src/share/vm/runtime/globals.hpp
@@ -3545,7 +3545,7 @@ class CommandLineFlags {
   product(uintx, SharedDummyBlockSize, 512*M,                               \
           "Size of dummy block used to shift heap addresses (in bytes)")    \
                                                                             \
-  product(uintx, SharedReadWriteSize,  12*M,                                \
+  product(uintx, SharedReadWriteSize,  NOT_LP64(12*M) LP64_ONLY(13*M),      \
           "Size of read-write space in permanent generation (in bytes)")    \
                                                                             \
   product(uintx, SharedReadOnlySize,   10*M,                                \

From c3401153c816e70bcb33c35028e00dbbd884126e Mon Sep 17 00:00:00 2001
From: Keith McGuigan <kamg@openjdk.org>
Date: Thu, 21 Oct 2010 10:10:23 -0400
Subject: [PATCH 14/23] 6991315: RedefineClasses fails with
 java.lang.VerifyError

Repair stackmap table attribute when relocating bytecode

Reviewed-by: acorn, never
---
 .../vm/classfile/stackMapTableFormat.hpp      | 916 ++++++++++++++++++
 hotspot/src/share/vm/includeDB_core           |   4 +
 hotspot/src/share/vm/oops/methodOop.hpp       |   4 +
 hotspot/src/share/vm/runtime/relocator.cpp    | 118 +++
 hotspot/src/share/vm/runtime/relocator.hpp    |   1 +
 5 files changed, 1043 insertions(+)
 create mode 100644 hotspot/src/share/vm/classfile/stackMapTableFormat.hpp

diff --git a/hotspot/src/share/vm/classfile/stackMapTableFormat.hpp b/hotspot/src/share/vm/classfile/stackMapTableFormat.hpp
new file mode 100644
index 00000000000..d20520abea0
--- /dev/null
+++ b/hotspot/src/share/vm/classfile/stackMapTableFormat.hpp
@@ -0,0 +1,916 @@
+/*
+ * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+// These classes represent the stack-map substructures described in the JVMS
+// (hence the non-conforming naming scheme).
+
+// These classes work with the types in their compressed form in-place (as they
+// would appear in the classfile).  No virtual methods or fields allowed.
+
+class verification_type_info {
+ private:
+  // u1 tag
+  // u2 cpool_index || u2 bci (for ITEM_Object & ITEM_Uninitailized only)
+
+  address tag_addr() const { return (address)this; }
+  address cpool_index_addr() const { return tag_addr() + sizeof(u1); }
+  address bci_addr() const { return cpool_index_addr(); }
+
+ protected:
+  // No constructors  - should be 'private', but GCC issues a warning if it is
+  verification_type_info() {}
+  verification_type_info(const verification_type_info&) {}
+
+ public:
+
+  static verification_type_info* at(address addr) {
+    return (verification_type_info*)addr;
+  }
+
+  static verification_type_info* create_at(address addr, u1 tag) {
+    verification_type_info* vti = (verification_type_info*)addr;
+    vti->set_tag(tag);
+    return vti;
+  }
+
+  static verification_type_info* create_object_at(address addr, u2 cp_idx) {
+    verification_type_info* vti = (verification_type_info*)addr;
+    vti->set_tag(ITEM_Object);
+    vti->set_cpool_index(cp_idx);
+    return vti;
+  }
+
+  static verification_type_info* create_uninit_at(address addr, u2 bci) {
+    verification_type_info* vti = (verification_type_info*)addr;
+    vti->set_tag(ITEM_Uninitialized);
+    vti->set_bci(bci);
+    return vti;
+  }
+
+  static size_t calculate_size(u1 tag) {
+    if (tag == ITEM_Object || tag == ITEM_Uninitialized) {
+      return sizeof(u1) + sizeof(u2);
+    } else {
+      return sizeof(u1);
+    }
+  }
+
+  static size_t max_size() { return sizeof(u1) + sizeof(u2); }
+
+  u1 tag() const { return *(u1*)tag_addr(); }
+  void set_tag(u1 tag) { *((u1*)tag_addr()) = tag; }
+
+  bool is_object() const { return tag() == ITEM_Object; }
+  bool is_uninitialized() const { return tag() == ITEM_Uninitialized; }
+
+  u2 cpool_index() const {
+    assert(is_object(), "This type has no cp_index");
+    return Bytes::get_Java_u2(cpool_index_addr());
+  }
+  void set_cpool_index(u2 idx) {
+    assert(is_object(), "This type has no cp_index");
+    Bytes::put_Java_u2(cpool_index_addr(), idx);
+  }
+
+  u2 bci() const {
+    assert(is_uninitialized(), "This type has no bci");
+    return Bytes::get_Java_u2(bci_addr());
+  }
+
+  void set_bci(u2 bci) {
+    assert(is_uninitialized(), "This type has no bci");
+    Bytes::put_Java_u2(bci_addr(), bci);
+  }
+
+  void copy_from(verification_type_info* from) {
+    set_tag(from->tag());
+    if (from->is_object()) {
+      set_cpool_index(from->cpool_index());
+    } else if (from->is_uninitialized()) {
+      set_bci(from->bci());
+    }
+  }
+
+  size_t size() const {
+    return calculate_size(tag());
+  }
+
+  verification_type_info* next() {
+    return (verification_type_info*)((address)this + size());
+  }
+
+  // This method is used when reading unverified data in order to ensure
+  // that we don't read past a particular memory limit.  It returns false
+  // if any part of the data structure is outside the specified memory bounds.
+  bool verify(address start, address end) {
+    return ((address)this >= start &&
+            (address)this < end &&
+            (bci_addr() + sizeof(u2) <= end ||
+                !is_object() && !is_uninitialized()));
+  }
+
+#ifdef ASSERT
+  void print_on(outputStream* st) {
+    switch (tag()) {
+      case ITEM_Top: st->print("Top"); break;
+      case ITEM_Integer: st->print("Integer"); break;
+      case ITEM_Float: st->print("Float"); break;
+      case ITEM_Double: st->print("Double"); break;
+      case ITEM_Long: st->print("Long"); break;
+      case ITEM_Null: st->print("Null"); break;
+      case ITEM_UninitializedThis:
+        st->print("UninitializedThis"); break;
+      case ITEM_Uninitialized:
+        st->print("Uninitialized[#%d]", bci()); break;
+      case ITEM_Object:
+        st->print("Object[#%d]", cpool_index()); break;
+      default:
+        assert(false, "Bad verification_type_info");
+    }
+  }
+#endif
+};
+
+#define FOR_EACH_STACKMAP_FRAME_TYPE(macro, arg1, arg2) \
+  macro(same_frame, arg1, arg2) \
+  macro(same_frame_extended, arg1, arg2) \
+  macro(same_frame_1_stack_item_frame, arg1, arg2) \
+  macro(same_frame_1_stack_item_extended, arg1, arg2) \
+  macro(chop_frame, arg1, arg2) \
+  macro(append_frame, arg1, arg2) \
+  macro(full_frame, arg1, arg2)
+
+#define SM_FORWARD_DECL(type, arg1, arg2) class type;
+FOR_EACH_STACKMAP_FRAME_TYPE(SM_FORWARD_DECL, x, x)
+#undef SM_FORWARD_DECL
+
+class stack_map_frame {
+ protected:
+  address frame_type_addr() const { return (address)this; }
+
+  // No constructors  - should be 'private', but GCC issues a warning if it is
+  stack_map_frame() {}
+  stack_map_frame(const stack_map_frame&) {}
+
+ public:
+
+  static stack_map_frame* at(address addr) {
+    return (stack_map_frame*)addr;
+  }
+
+  stack_map_frame* next() const {
+    return at((address)this + size());
+  }
+
+  u1 frame_type() const { return *(u1*)frame_type_addr(); }
+  void set_frame_type(u1 type) { *((u1*)frame_type_addr()) = type; }
+
+  // pseudo-virtual methods
+  inline size_t size() const;
+  inline int offset_delta() const;
+  inline void set_offset_delta(int offset_delta);
+  inline int number_of_types() const; // number of types contained in the frame
+  inline verification_type_info* types() const; // pointer to first type
+  inline bool is_valid_offset(int offset_delta) const;
+
+  // This method must be used when reading unverified data in order to ensure
+  // that we don't read past a particular memory limit.  It returns false
+  // if any part of the data structure is outside the specified memory bounds.
+  inline bool verify(address start, address end) const;
+#ifdef ASSERT
+  inline void print_on(outputStream* st) const;
+#endif
+
+  // Create as_xxx and is_xxx methods for the subtypes
+#define FRAME_TYPE_DECL(stackmap_frame_type, arg1, arg2) \
+  inline stackmap_frame_type* as_##stackmap_frame_type() const; \
+  bool is_##stackmap_frame_type() { \
+    return as_##stackmap_frame_type() != NULL; \
+  }
+
+  FOR_EACH_STACKMAP_FRAME_TYPE(FRAME_TYPE_DECL, x, x)
+#undef FRAME_TYPE_DECL
+};
+
+class same_frame : public stack_map_frame {
+ private:
+  static int frame_type_to_offset_delta(u1 frame_type) {
+      return frame_type + 1; }
+  static u1 offset_delta_to_frame_type(int offset_delta) {
+      return (u1)(offset_delta - 1); }
+
+ public:
+
+  static bool is_frame_type(u1 tag) {
+    return tag < 64;
+  }
+
+  static same_frame* at(address addr) {
+    assert(is_frame_type(*addr), "Wrong frame id");
+    return (same_frame*)addr;
+  }
+
+  static same_frame* create_at(address addr, int offset_delta) {
+    same_frame* sm = (same_frame*)addr;
+    sm->set_offset_delta(offset_delta);
+    return sm;
+  }
+
+  static size_t calculate_size() { return sizeof(u1); }
+
+  size_t size() const { return calculate_size(); }
+  int offset_delta() const { return frame_type_to_offset_delta(frame_type()); }
+
+  void set_offset_delta(int offset_delta) {
+    assert(offset_delta <= 64, "Offset too large for same_frame");
+    set_frame_type(offset_delta_to_frame_type(offset_delta));
+  }
+
+  int number_of_types() const { return 0; }
+  verification_type_info* types() const { return NULL; }
+
+  bool is_valid_offset(int offset_delta) const {
+    return is_frame_type(offset_delta_to_frame_type(offset_delta));
+  }
+
+  bool verify_subtype(address start, address end) const {
+    return true;
+  }
+
+#ifdef ASSERT
+  void print_on(outputStream* st) const {
+    st->print("same_frame(%d)", offset_delta());
+  }
+#endif
+};
+
+class same_frame_extended : public stack_map_frame {
+ private:
+  enum { _frame_id = 251 };
+  address offset_delta_addr() const { return frame_type_addr() + sizeof(u1); }
+
+ public:
+  static bool is_frame_type(u1 tag) {
+    return tag == _frame_id;
+  }
+
+  static same_frame_extended* at(address addr) {
+    assert(is_frame_type(*addr), "Wrong frame type");
+    return (same_frame_extended*)addr;
+  }
+
+  static same_frame_extended* create_at(address addr, u2 offset_delta) {
+    same_frame_extended* sm = (same_frame_extended*)addr;
+    sm->set_frame_type(_frame_id);
+    sm->set_offset_delta(offset_delta);
+    return sm;
+  }
+
+  static size_t calculate_size() { return sizeof(u1) + sizeof(u2); }
+
+  size_t size() const { return calculate_size(); }
+  int offset_delta() const {
+    return Bytes::get_Java_u2(offset_delta_addr()) + 1;
+  }
+
+  void set_offset_delta(int offset_delta) {
+    Bytes::put_Java_u2(offset_delta_addr(), offset_delta - 1);
+  }
+
+  int number_of_types() const { return 0; }
+  verification_type_info* types() const { return NULL; }
+  bool is_valid_offset(int offset) const { return true; }
+
+  bool verify_subtype(address start, address end) const {
+    return frame_type_addr() + size() <= end;
+  }
+
+#ifdef ASSERT
+  void print_on(outputStream* st) const {
+    st->print("same_frame_extended(%d)", offset_delta());
+  }
+#endif
+};
+
+class same_frame_1_stack_item_frame : public stack_map_frame {
+ private:
+  address type_addr() const { return frame_type_addr() + sizeof(u1); }
+
+  static int frame_type_to_offset_delta(u1 frame_type) {
+      return frame_type - 63; }
+  static u1 offset_delta_to_frame_type(int offset_delta) {
+      return (u1)(offset_delta + 63); }
+
+ public:
+  static bool is_frame_type(u1 tag) {
+    return tag >= 64 && tag < 128;
+  }
+
+  static same_frame_1_stack_item_frame* at(address addr) {
+    assert(is_frame_type(*addr), "Wrong frame id");
+    return (same_frame_1_stack_item_frame*)addr;
+  }
+
+  static same_frame_1_stack_item_frame* create_at(
+      address addr, int offset_delta, verification_type_info* vti) {
+    same_frame_1_stack_item_frame* sm = (same_frame_1_stack_item_frame*)addr;
+    sm->set_offset_delta(offset_delta);
+    if (vti != NULL) {
+      sm->set_type(vti);
+    }
+    return sm;
+  }
+
+  static size_t calculate_size(verification_type_info* vti) {
+    return sizeof(u1) + vti->size();
+  }
+
+  static size_t max_size() {
+    return sizeof(u1) + verification_type_info::max_size();
+  }
+
+  size_t size() const { return calculate_size(types()); }
+  int offset_delta() const { return frame_type_to_offset_delta(frame_type()); }
+
+  void set_offset_delta(int offset_delta) {
+    assert(offset_delta > 0 && offset_delta <= 64,
+           "Offset too large for this frame type");
+    set_frame_type(offset_delta_to_frame_type(offset_delta));
+  }
+
+  void set_type(verification_type_info* vti) {
+    verification_type_info* cur = types();
+    cur->copy_from(vti);
+  }
+
+  int number_of_types() const { return 1; }
+  verification_type_info* types() const {
+    return verification_type_info::at(type_addr());
+  }
+
+  bool is_valid_offset(int offset_delta) const {
+    return is_frame_type(offset_delta_to_frame_type(offset_delta));
+  }
+
+  bool verify_subtype(address start, address end) const {
+    return types()->verify(start, end);
+  }
+
+#ifdef ASSERT
+  void print_on(outputStream* st) const {
+    st->print("same_frame_1_stack_item_frame(%d,", offset_delta());
+    types()->print_on(st);
+    st->print(")");
+  }
+#endif
+};
+
+class same_frame_1_stack_item_extended : public stack_map_frame {
+ private:
+  address offset_delta_addr() const { return frame_type_addr() + sizeof(u1); }
+  address type_addr() const { return offset_delta_addr() + sizeof(u2); }
+
+  enum { _frame_id = 247 };
+
+ public:
+  static bool is_frame_type(u1 tag) {
+    return tag == _frame_id;
+  }
+
+  static same_frame_1_stack_item_extended* at(address addr) {
+    assert(is_frame_type(*addr), "Wrong frame id");
+    return (same_frame_1_stack_item_extended*)addr;
+  }
+
+  static same_frame_1_stack_item_extended* create_at(
+      address addr, int offset_delta, verification_type_info* vti) {
+    same_frame_1_stack_item_extended* sm =
+       (same_frame_1_stack_item_extended*)addr;
+    sm->set_frame_type(_frame_id);
+    sm->set_offset_delta(offset_delta);
+    if (vti != NULL) {
+      sm->set_type(vti);
+    }
+    return sm;
+  }
+
+  static size_t calculate_size(verification_type_info* vti) {
+    return sizeof(u1) + sizeof(u2) + vti->size();
+  }
+
+  size_t size() const { return calculate_size(types()); }
+  int offset_delta() const {
+    return Bytes::get_Java_u2(offset_delta_addr()) + 1;
+  }
+
+  void set_offset_delta(int offset_delta) {
+    Bytes::put_Java_u2(offset_delta_addr(), offset_delta - 1);
+  }
+
+  void set_type(verification_type_info* vti) {
+    verification_type_info* cur = types();
+    cur->copy_from(vti);
+  }
+
+  int number_of_types() const { return 1; }
+  verification_type_info* types() const {
+    return verification_type_info::at(type_addr());
+  }
+  bool is_valid_offset(int offset) { return true; }
+
+  bool verify_subtype(address start, address end) const {
+    return type_addr() < end && types()->verify(start, end);
+  }
+
+#ifdef ASSERT
+  void print_on(outputStream* st) const {
+    st->print("same_frame_1_stack_item_extended(%d,", offset_delta());
+    types()->print_on(st);
+    st->print(")");
+  }
+#endif
+};
+
+class chop_frame : public stack_map_frame {
+ private:
+  address offset_delta_addr() const { return frame_type_addr() + sizeof(u1); }
+
+  static int frame_type_to_chops(u1 frame_type) {
+    int chop = 251 - frame_type;
+    return chop;
+  }
+
+  static u1 chops_to_frame_type(int chop) {
+    return 251 - chop;
+  }
+
+ public:
+  static bool is_frame_type(u1 tag) {
+    return frame_type_to_chops(tag) > 0 && frame_type_to_chops(tag) < 4;
+  }
+
+  static chop_frame* at(address addr) {
+    assert(is_frame_type(*addr), "Wrong frame id");
+    return (chop_frame*)addr;
+  }
+
+  static chop_frame* create_at(address addr, int offset_delta, int chops) {
+    chop_frame* sm = (chop_frame*)addr;
+    sm->set_chops(chops);
+    sm->set_offset_delta(offset_delta);
+    return sm;
+  }
+
+  static size_t calculate_size() {
+    return sizeof(u1) + sizeof(u2);
+  }
+
+  size_t size() const { return calculate_size(); }
+  int offset_delta() const {
+    return Bytes::get_Java_u2(offset_delta_addr()) + 1;
+  }
+  void set_offset_delta(int offset_delta) {
+    Bytes::put_Java_u2(offset_delta_addr(), offset_delta - 1);
+  }
+
+  int chops() const {
+    int chops = frame_type_to_chops(frame_type());
+    assert(chops > 0 && chops < 4, "Invalid number of chops in frame");
+    return chops;
+  }
+  void set_chops(int chops) {
+    assert(chops > 0 && chops <= 3, "Bad number of chops");
+    set_frame_type(chops_to_frame_type(chops));
+  }
+
+  int number_of_types() const { return 0; }
+  verification_type_info* types() const { return NULL; }
+  bool is_valid_offset(int offset) { return true; }
+
+  bool verify_subtype(address start, address end) const {
+    return frame_type_addr() + size() <= end;
+  }
+
+#ifdef ASSERT
+  void print_on(outputStream* st) const {
+    st->print("chop_frame(%d,%d)", offset_delta(), chops());
+  }
+#endif
+};
+
+class append_frame : public stack_map_frame {
+ private:
+  address offset_delta_addr() const { return frame_type_addr() + sizeof(u1); }
+  address types_addr() const { return offset_delta_addr() + sizeof(u2); }
+
+  static int frame_type_to_appends(u1 frame_type) {
+    int append = frame_type - 251;
+    return append;
+  }
+
+  static u1 appends_to_frame_type(int appends) {
+    assert(appends > 0 && appends < 4, "Invalid append amount");
+    return 251 + appends;
+  }
+
+ public:
+  static bool is_frame_type(u1 tag) {
+    return frame_type_to_appends(tag) > 0 && frame_type_to_appends(tag) < 4;
+  }
+
+  static append_frame* at(address addr) {
+    assert(is_frame_type(*addr), "Wrong frame id");
+    return (append_frame*)addr;
+  }
+
+  static append_frame* create_at(
+      address addr, int offset_delta, int appends,
+      verification_type_info* types) {
+    append_frame* sm = (append_frame*)addr;
+    sm->set_appends(appends);
+    sm->set_offset_delta(offset_delta);
+    if (types != NULL) {
+      verification_type_info* cur = sm->types();
+      for (int i = 0; i < appends; ++i) {
+        cur->copy_from(types);
+        cur = cur->next();
+        types = types->next();
+      }
+    }
+    return sm;
+  }
+
+  static size_t calculate_size(int appends, verification_type_info* types) {
+    size_t sz = sizeof(u1) + sizeof(u2);
+    for (int i = 0; i < appends; ++i) {
+      sz += types->size();
+      types = types->next();
+    }
+    return sz;
+  }
+
+  static size_t max_size() {
+    return sizeof(u1) + sizeof(u2) + 3 * verification_type_info::max_size();
+  }
+
+  size_t size() const { return calculate_size(number_of_types(), types()); }
+  int offset_delta() const {
+    return Bytes::get_Java_u2(offset_delta_addr()) + 1;
+  }
+
+  void set_offset_delta(int offset_delta) {
+    Bytes::put_Java_u2(offset_delta_addr(), offset_delta - 1);
+  }
+
+  void set_appends(int appends) {
+    assert(appends > 0 && appends < 4, "Bad number of appends");
+    set_frame_type(appends_to_frame_type(appends));
+  }
+
+  int number_of_types() const {
+    int appends = frame_type_to_appends(frame_type());
+    assert(appends > 0 && appends < 4, "Invalid number of appends in frame");
+    return appends;
+  }
+  verification_type_info* types() const {
+    return verification_type_info::at(types_addr());
+  }
+  bool is_valid_offset(int offset) const { return true; }
+
+  bool verify_subtype(address start, address end) const {
+    verification_type_info* vti = types();
+    if ((address)vti < end && vti->verify(start, end)) {
+      int nof = number_of_types();
+      vti = vti->next();
+      if (nof < 2 || vti->verify(start, end)) {
+        vti = vti->next();
+        if (nof < 3 || vti->verify(start, end)) {
+          return true;
+        }
+      }
+    }
+    return false;
+  }
+
+#ifdef ASSERT
+  void print_on(outputStream* st) const {
+    st->print("append_frame(%d,", offset_delta());
+    verification_type_info* vti = types();
+    for (int i = 0; i < number_of_types(); ++i) {
+      vti->print_on(st);
+      if (i != number_of_types() - 1) {
+        st->print(",");
+      }
+      vti = vti->next();
+    }
+    st->print(")");
+  }
+#endif
+};
+
+class full_frame : public stack_map_frame {
+ private:
+  address offset_delta_addr() const { return frame_type_addr() + sizeof(u1); }
+  address num_locals_addr() const { return offset_delta_addr() + sizeof(u2); }
+  address locals_addr() const { return num_locals_addr() + sizeof(u2); }
+  address stack_slots_addr(address end_of_locals) const {
+      return end_of_locals; }
+  address stack_addr(address end_of_locals) const {
+      return stack_slots_addr(end_of_locals) + sizeof(u2); }
+
+  enum { _frame_id = 255 };
+
+ public:
+  static bool is_frame_type(u1 tag) {
+    return tag == _frame_id;
+  }
+
+  static full_frame* at(address addr) {
+    assert(is_frame_type(*addr), "Wrong frame id");
+    return (full_frame*)addr;
+  }
+
+  static full_frame* create_at(
+      address addr, int offset_delta, int num_locals,
+      verification_type_info* locals,
+      int stack_slots, verification_type_info* stack) {
+    full_frame* sm = (full_frame*)addr;
+    sm->set_frame_type(_frame_id);
+    sm->set_offset_delta(offset_delta);
+    sm->set_num_locals(num_locals);
+    if (locals != NULL) {
+      verification_type_info* cur = sm->locals();
+      for (int i = 0; i < num_locals; ++i) {
+        cur->copy_from(locals);
+        cur = cur->next();
+        locals = locals->next();
+      }
+      address end_of_locals = (address)cur;
+      sm->set_stack_slots(end_of_locals, stack_slots);
+      cur = sm->stack(end_of_locals);
+      for (int i = 0; i < stack_slots; ++i) {
+        cur->copy_from(stack);
+        cur = cur->next();
+        stack = stack->next();
+      }
+    }
+    return sm;
+  }
+
+  static size_t calculate_size(
+      int num_locals, verification_type_info* locals,
+      int stack_slots, verification_type_info* stack) {
+    size_t sz = sizeof(u1) + sizeof(u2) + sizeof(u2) + sizeof(u2);
+    verification_type_info* vti = locals;
+    for (int i = 0; i < num_locals; ++i) {
+      sz += vti->size();
+      vti = vti->next();
+    }
+    vti = stack;
+    for (int i = 0; i < stack_slots; ++i) {
+      sz += vti->size();
+      vti = vti->next();
+    }
+    return sz;
+  }
+
+  static size_t max_size(int locals, int stack) {
+    return sizeof(u1) + 3 * sizeof(u2) +
+        (locals + stack) * verification_type_info::max_size();
+  }
+
+  size_t size() const {
+    address eol = end_of_locals();
+    return calculate_size(num_locals(), locals(), stack_slots(eol), stack(eol));
+  }
+
+  int offset_delta() const {
+    return Bytes::get_Java_u2(offset_delta_addr()) + 1;
+  }
+  int num_locals() const { return Bytes::get_Java_u2(num_locals_addr()); }
+  verification_type_info* locals() const {
+    return verification_type_info::at(locals_addr());
+  }
+  address end_of_locals() const {
+    verification_type_info* vti = locals();
+    for (int i = 0; i < num_locals(); ++i) {
+      vti = vti->next();
+    }
+    return (address)vti;
+  }
+  int stack_slots(address end_of_locals) const {
+    return Bytes::get_Java_u2(stack_slots_addr(end_of_locals));
+  }
+  verification_type_info* stack(address end_of_locals) const {
+    return verification_type_info::at(stack_addr(end_of_locals));
+  }
+
+  void set_offset_delta(int offset_delta) {
+    Bytes::put_Java_u2(offset_delta_addr(), offset_delta - 1);
+  }
+  void set_num_locals(int num_locals) {
+    Bytes::put_Java_u2(num_locals_addr(), num_locals);
+  }
+  void set_stack_slots(address end_of_locals, int stack_slots) {
+    Bytes::put_Java_u2(stack_slots_addr(end_of_locals), stack_slots);
+  }
+
+  // These return only the locals.  Extra processing is required for stack
+  // types of full frames.
+  int number_of_types() const { return num_locals(); }
+  verification_type_info* types() const { return locals(); }
+  bool is_valid_offset(int offset) { return true; }
+
+  bool verify_subtype(address start, address end) const {
+    verification_type_info* vti = types();
+    if ((address)vti >= end) {
+      return false;
+    }
+    int count = number_of_types();
+    for (int i = 0; i < count; ++i) {
+      if (!vti->verify(start, end)) {
+        return false;
+      }
+      vti = vti->next();
+    }
+    address eol = (address)vti;
+    if (eol + sizeof(u2) > end) {
+      return false;
+    }
+    count = stack_slots(eol);
+    vti = stack(eol);
+    for (int i = 0; i < stack_slots(eol); ++i) {
+      if (!vti->verify(start, end)) {
+        return false;
+      }
+      vti = vti->next();
+    }
+    return true;
+  }
+
+#ifdef ASSERT
+  void print_on(outputStream* st) const {
+    st->print("full_frame(%d,{", offset_delta());
+    verification_type_info* vti = locals();
+    for (int i = 0; i < num_locals(); ++i) {
+      vti->print_on(st);
+      if (i != num_locals() - 1) {
+        st->print(",");
+      }
+      vti = vti->next();
+    }
+    st->print("},{");
+    address end_of_locals = (address)vti;
+    vti = stack(end_of_locals);
+    int ss = stack_slots(end_of_locals);
+    for (int i = 0; i < ss; ++i) {
+      vti->print_on(st);
+      if (i != ss - 1) {
+        st->print(",");
+      }
+      vti = vti->next();
+    }
+    st->print("})");
+  }
+#endif
+};
+
+#define VIRTUAL_DISPATCH(stack_frame_type, func_name, args) \
+  stack_frame_type* item_##stack_frame_type = as_##stack_frame_type(); \
+  if (item_##stack_frame_type != NULL) { \
+    return item_##stack_frame_type->func_name args;  \
+  }
+
+#define VOID_VIRTUAL_DISPATCH(stack_frame_type, func_name, args) \
+  stack_frame_type* item_##stack_frame_type = as_##stack_frame_type(); \
+  if (item_##stack_frame_type != NULL) { \
+    item_##stack_frame_type->func_name args;  \
+    return; \
+  }
+
+size_t stack_map_frame::size() const {
+  FOR_EACH_STACKMAP_FRAME_TYPE(VIRTUAL_DISPATCH, size, ());
+  return 0;
+}
+
+int stack_map_frame::offset_delta() const {
+  FOR_EACH_STACKMAP_FRAME_TYPE(VIRTUAL_DISPATCH, offset_delta, ());
+  return 0;
+}
+
+void stack_map_frame::set_offset_delta(int offset_delta) {
+  FOR_EACH_STACKMAP_FRAME_TYPE(
+      VOID_VIRTUAL_DISPATCH, set_offset_delta, (offset_delta));
+}
+
+int stack_map_frame::number_of_types() const {
+  FOR_EACH_STACKMAP_FRAME_TYPE(VIRTUAL_DISPATCH, number_of_types, ());
+  return 0;
+}
+
+verification_type_info* stack_map_frame::types() const {
+  FOR_EACH_STACKMAP_FRAME_TYPE(VIRTUAL_DISPATCH, types, ());
+  return NULL;
+}
+
+bool stack_map_frame::is_valid_offset(int offset) const {
+  FOR_EACH_STACKMAP_FRAME_TYPE(VIRTUAL_DISPATCH, is_valid_offset, (offset));
+  return true;
+}
+
+bool stack_map_frame::verify(address start, address end) const {
+  if (frame_type_addr() >= start && frame_type_addr() < end) {
+    FOR_EACH_STACKMAP_FRAME_TYPE(
+       VIRTUAL_DISPATCH, verify_subtype, (start, end));
+  }
+  return false;
+}
+
+#ifdef ASSERT
+void stack_map_frame::print_on(outputStream* st) const {
+  FOR_EACH_STACKMAP_FRAME_TYPE(VOID_VIRTUAL_DISPATCH, print_on, (st));
+}
+#endif
+
+#undef VIRTUAL_DISPATCH
+#undef VOID_VIRTUAL_DISPATCH
+
+#define AS_SUBTYPE_DEF(stack_frame_type, arg1, arg2) \
+stack_frame_type* stack_map_frame::as_##stack_frame_type() const { \
+  if (stack_frame_type::is_frame_type(frame_type())) { \
+    return (stack_frame_type*)this; \
+  } else { \
+    return NULL; \
+  } \
+}
+
+FOR_EACH_STACKMAP_FRAME_TYPE(AS_SUBTYPE_DEF, x, x)
+#undef AS_SUBTYPE_DEF
+
+class stack_map_table_attribute {
+ private:
+  address name_index_addr() const {
+      return (address)this; }
+  address attribute_length_addr() const {
+      return name_index_addr() + sizeof(u2); }
+  address number_of_entries_addr() const {
+      return attribute_length_addr() + sizeof(u4); }
+  address entries_addr() const {
+      return number_of_entries_addr() + sizeof(u2); }
+
+ protected:
+  // No constructors  - should be 'private', but GCC issues a warning if it is
+  stack_map_table_attribute() {}
+  stack_map_table_attribute(const stack_map_table_attribute&) {}
+
+ public:
+
+  static stack_map_table_attribute* at(address addr) {
+    return (stack_map_table_attribute*)addr;
+  }
+
+  u2 name_index() const {
+       return Bytes::get_Java_u2(name_index_addr()); }
+  u4 attribute_length() const {
+      return Bytes::get_Java_u4(attribute_length_addr()); }
+  u2 number_of_entries() const {
+      return Bytes::get_Java_u2(number_of_entries_addr()); }
+  stack_map_frame* entries() const {
+    return stack_map_frame::at(entries_addr());
+  }
+
+  static size_t header_size() {
+      return sizeof(u2) + sizeof(u4);
+  }
+
+  void set_name_index(u2 idx) {
+    Bytes::put_Java_u2(name_index_addr(), idx);
+  }
+  void set_attribute_length(u4 len) {
+    Bytes::put_Java_u4(attribute_length_addr(), len);
+  }
+  void set_number_of_entries(u2 num) {
+    Bytes::put_Java_u2(number_of_entries_addr(), num);
+  }
+};
diff --git a/hotspot/src/share/vm/includeDB_core b/hotspot/src/share/vm/includeDB_core
index 14d78e91d41..a16805a0a9f 100644
--- a/hotspot/src/share/vm/includeDB_core
+++ b/hotspot/src/share/vm/includeDB_core
@@ -3601,7 +3601,9 @@ relocInfo_<arch>.hpp                    generate_platform_dependent_include
 relocator.cpp                           bytecodes.hpp
 relocator.cpp                           handles.inline.hpp
 relocator.cpp                           oop.inline.hpp
+relocator.cpp                           oopFactory.hpp
 relocator.cpp                           relocator.hpp
+relocator.cpp                           stackMapTableFormat.hpp
 relocator.cpp                           universe.inline.hpp
 
 relocator.hpp                           bytecodes.hpp
@@ -3908,6 +3910,8 @@ stackMapTable.hpp                       globalDefinitions.hpp
 stackMapTable.hpp                       methodOop.hpp
 stackMapTable.hpp                       stackMapFrame.hpp
 
+stackMapTableFormat.hpp                 verificationType.hpp
+
 stackValue.cpp                          debugInfo.hpp
 stackValue.cpp                          frame.inline.hpp
 stackValue.cpp                          handles.inline.hpp
diff --git a/hotspot/src/share/vm/oops/methodOop.hpp b/hotspot/src/share/vm/oops/methodOop.hpp
index 827e58f1ab7..9ae2ca9076f 100644
--- a/hotspot/src/share/vm/oops/methodOop.hpp
+++ b/hotspot/src/share/vm/oops/methodOop.hpp
@@ -247,6 +247,10 @@ class methodOopDesc : public oopDesc {
     return constMethod()->stackmap_data();
   }
 
+  void set_stackmap_data(typeArrayOop sd) {
+    constMethod()->set_stackmap_data(sd);
+  }
+
   // exception handler table
   typeArrayOop exception_table() const
                                    { return constMethod()->exception_table(); }
diff --git a/hotspot/src/share/vm/runtime/relocator.cpp b/hotspot/src/share/vm/runtime/relocator.cpp
index bf796680f5d..483d21bd393 100644
--- a/hotspot/src/share/vm/runtime/relocator.cpp
+++ b/hotspot/src/share/vm/runtime/relocator.cpp
@@ -435,6 +435,120 @@ void Relocator::adjust_local_var_table(int bci, int delta) {
   }
 }
 
+// Create a new array, copying the src array but adding a hole at
+// the specified location
+static typeArrayOop insert_hole_at(
+    size_t where, int hole_sz, typeArrayOop src) {
+  Thread* THREAD = Thread::current();
+  Handle src_hnd(THREAD, src);
+  typeArrayOop dst =
+      oopFactory::new_permanent_byteArray(src->length() + hole_sz, CHECK_NULL);
+  src = (typeArrayOop)src_hnd();
+
+  address src_addr = (address)src->byte_at_addr(0);
+  address dst_addr = (address)dst->byte_at_addr(0);
+
+  memcpy(dst_addr, src_addr, where);
+  memcpy(dst_addr + where + hole_sz,
+         src_addr + where, src->length() - where);
+  return dst;
+}
+
+// The width of instruction at "bci" is changing by "delta".  Adjust the stack
+// map frames.
+void Relocator::adjust_stack_map_table(int bci, int delta) {
+  if (method()->has_stackmap_table()) {
+    typeArrayOop data = method()->stackmap_data();
+    // The data in the array is a classfile representation of the stackmap
+    // table attribute, less the initial u2 tag and u4 attribute_length fields.
+    stack_map_table_attribute* attr = stack_map_table_attribute::at(
+        (address)data->byte_at_addr(0) - (sizeof(u2) + sizeof(u4)));
+
+    int count = attr->number_of_entries();
+    stack_map_frame* frame = attr->entries();
+    int bci_iter = -1;
+    bool offset_adjusted = false; // only need to adjust one offset
+
+    for (int i = 0; i < count; ++i) {
+      int offset_delta = frame->offset_delta();
+      bci_iter += offset_delta;
+
+      if (!offset_adjusted && bci_iter > bci) {
+        int new_offset_delta = offset_delta + delta;
+
+        if (frame->is_valid_offset(new_offset_delta)) {
+          frame->set_offset_delta(new_offset_delta);
+        } else {
+          assert(frame->is_same_frame() ||
+                 frame->is_same_frame_1_stack_item_frame(),
+                 "Frame must be one of the compressed forms");
+          // The new delta exceeds the capacity of the 'same_frame' or
+          // 'same_frame_1_stack_item_frame' frame types.  We need to
+          // convert these frames to the extended versions, but the extended
+          // version is bigger and requires more room.  So we allocate a
+          // new array and copy the data, being sure to leave u2-sized hole
+          // right after the 'frame_type' for the new offset field.
+          //
+          // We can safely ignore the reverse situation as a small delta
+          // can still be used in an extended version of the frame.
+
+          size_t frame_offset = (address)frame - (address)data->byte_at_addr(0);
+
+          data = insert_hole_at(frame_offset + 1, 2, data);
+          if (data == NULL) {
+            return; // out-of-memory?
+          }
+
+          address frame_addr = (address)(data->byte_at_addr(0) + frame_offset);
+          frame = stack_map_frame::at(frame_addr);
+
+
+          // Now convert the frames in place
+          if (frame->is_same_frame()) {
+            same_frame_extended::create_at(frame_addr, new_offset_delta);
+          } else {
+            same_frame_1_stack_item_extended::create_at(
+              frame_addr, new_offset_delta, NULL);
+            // the verification_info_type should already be at the right spot
+          }
+        }
+        offset_adjusted = true; // needs to be done only once, since subsequent
+                                // values are offsets from the current
+      }
+
+      // The stack map frame may contain verification types, if so we need to
+      // check and update any Uninitialized type's bci (no matter where it is).
+      int number_of_types = frame->number_of_types();
+      verification_type_info* types = frame->types();
+
+      for (int i = 0; i < number_of_types; ++i) {
+        if (types->is_uninitialized() && types->bci() > bci) {
+          types->set_bci(types->bci() + delta);
+        }
+        types = types->next();
+      }
+
+      // Full frame has stack values too
+      full_frame* ff = frame->as_full_frame();
+      if (ff != NULL) {
+        address eol = (address)types;
+        number_of_types = ff->stack_slots(eol);
+        types = ff->stack(eol);
+        for (int i = 0; i < number_of_types; ++i) {
+          if (types->is_uninitialized() && types->bci() > bci) {
+            types->set_bci(types->bci() + delta);
+          }
+          types = types->next();
+        }
+      }
+
+      frame = frame->next();
+    }
+
+    method()->set_stackmap_data(data); // in case it has changed
+  }
+}
+
 
 bool Relocator::expand_code_array(int delta) {
   int length = MAX2(code_length() + delta, code_length() * (100+code_slop_pct()) / 100);
@@ -499,6 +613,9 @@ bool Relocator::relocate_code(int bci, int ilen, int delta) {
   // And local variable table...
   adjust_local_var_table(bci, delta);
 
+  // Adjust stack maps
+  adjust_stack_map_table(bci, delta);
+
   // Relocate the pending change stack...
   for (int j = 0; j < _changes->length(); j++) {
     ChangeItem* ci = _changes->at(j);
@@ -641,6 +758,7 @@ bool Relocator::handle_switch_pad(int bci, int old_pad, bool is_lookup_switch) {
       memmove(addr_at(bci +1 + new_pad),
               addr_at(bci +1 + old_pad),
               len * 4);
+      memset(addr_at(bci + 1), 0, new_pad); // pad must be 0
     }
   }
   return true;
diff --git a/hotspot/src/share/vm/runtime/relocator.hpp b/hotspot/src/share/vm/runtime/relocator.hpp
index be925e6a89a..3f163723d03 100644
--- a/hotspot/src/share/vm/runtime/relocator.hpp
+++ b/hotspot/src/share/vm/runtime/relocator.hpp
@@ -105,6 +105,7 @@ class Relocator : public ResourceObj {
   void adjust_exception_table(int bci, int delta);
   void adjust_line_no_table  (int bci, int delta);
   void adjust_local_var_table(int bci, int delta);
+  void adjust_stack_map_table(int bci, int delta);
   int  get_orig_switch_pad   (int bci, bool is_lookup_switch);
   int  rc_instr_len          (int bci);
   bool expand_code_array     (int delta);

From 554e77efb458c07c20446cc944cc61c34ad390b9 Mon Sep 17 00:00:00 2001
From: "Y. Srinivas Ramakrishna" <ysr@openjdk.org>
Date: Thu, 21 Oct 2010 17:29:24 -0700
Subject: [PATCH 15/23] 6992998: CMSWaitDuration=0 causes hangs with
 +ExplicitGCInvokesConcurrent

Closed a timing hole during which concurrent full gc requests can be missed. The hole can increase the latency of the response to a full gc request by up to the value of CMSWaitDuration. If CMSWaitDuration=0 is, as currently, interpreted as an unbounded wait, suitable in certain tuning scenarios, the application can potentially hang. Made two obscure tunables, including CMSWaitDuration, manageable.

Reviewed-by: jcoomes, tonyp
---
 .../concurrentMarkSweepThread.cpp                 | 15 ++++++++++-----
 .../concurrentMarkSweepThread.hpp                 |  6 ++++--
 hotspot/src/share/vm/runtime/globals.hpp          |  4 ++--
 3 files changed, 16 insertions(+), 9 deletions(-)

diff --git a/hotspot/src/share/vm/gc_implementation/concurrentMarkSweep/concurrentMarkSweepThread.cpp b/hotspot/src/share/vm/gc_implementation/concurrentMarkSweep/concurrentMarkSweepThread.cpp
index f0033815bcd..f9823af67d9 100644
--- a/hotspot/src/share/vm/gc_implementation/concurrentMarkSweep/concurrentMarkSweepThread.cpp
+++ b/hotspot/src/share/vm/gc_implementation/concurrentMarkSweep/concurrentMarkSweepThread.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2001, 2006, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2001, 2010, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -272,12 +272,16 @@ void ConcurrentMarkSweepThread::desynchronize(bool is_cms_thread) {
   }
 }
 
-// Wait until the next synchronous GC or a timeout, whichever is earlier.
-void ConcurrentMarkSweepThread::wait_on_cms_lock(long t) {
+// Wait until the next synchronous GC, a concurrent full gc request,
+// or a timeout, whichever is earlier.
+void ConcurrentMarkSweepThread::wait_on_cms_lock(long t_millis) {
   MutexLockerEx x(CGC_lock,
                   Mutex::_no_safepoint_check_flag);
+  if (_should_terminate || _collector->_full_gc_requested) {
+    return;
+  }
   set_CMS_flag(CMS_cms_wants_token);   // to provoke notifies
-  CGC_lock->wait(Mutex::_no_safepoint_check_flag, t);
+  CGC_lock->wait(Mutex::_no_safepoint_check_flag, t_millis);
   clear_CMS_flag(CMS_cms_wants_token);
   assert(!CMS_flag_is_set(CMS_cms_has_token | CMS_cms_wants_token),
          "Should not be set");
@@ -289,7 +293,8 @@ void ConcurrentMarkSweepThread::sleepBeforeNextCycle() {
       icms_wait();
       return;
     } else {
-      // Wait until the next synchronous GC or a timeout, whichever is earlier
+      // Wait until the next synchronous GC, a concurrent full gc
+      // request or a timeout, whichever is earlier.
       wait_on_cms_lock(CMSWaitDuration);
     }
     // Check if we should start a CMS collection cycle
diff --git a/hotspot/src/share/vm/gc_implementation/concurrentMarkSweep/concurrentMarkSweepThread.hpp b/hotspot/src/share/vm/gc_implementation/concurrentMarkSweep/concurrentMarkSweepThread.hpp
index 26eac7c37c0..1390b20a1da 100644
--- a/hotspot/src/share/vm/gc_implementation/concurrentMarkSweep/concurrentMarkSweepThread.hpp
+++ b/hotspot/src/share/vm/gc_implementation/concurrentMarkSweep/concurrentMarkSweepThread.hpp
@@ -120,8 +120,10 @@ class ConcurrentMarkSweepThread: public ConcurrentGCThread {
   }
 
   // Wait on CMS lock until the next synchronous GC
-  // or given timeout, whichever is earlier.
-  void    wait_on_cms_lock(long t); // milliseconds
+  // or given timeout, whichever is earlier. A timeout value
+  // of 0 indicates that there is no upper bound on the wait time.
+  // A concurrent full gc request terminates the wait.
+  void wait_on_cms_lock(long t_millis);
 
   // The CMS thread will yield during the work portion of its cycle
   // only when requested to.  Both synchronous and asychronous requests
diff --git a/hotspot/src/share/vm/runtime/globals.hpp b/hotspot/src/share/vm/runtime/globals.hpp
index 6005865544b..1fa26ce7fd6 100644
--- a/hotspot/src/share/vm/runtime/globals.hpp
+++ b/hotspot/src/share/vm/runtime/globals.hpp
@@ -1585,7 +1585,7 @@ class CommandLineFlags {
           "(Temporary, subject to experimentation)"                         \
           "Nominal minimum work per abortable preclean iteration")          \
                                                                             \
-  product(intx, CMSAbortablePrecleanWaitMillis, 100,                        \
+  manageable(intx, CMSAbortablePrecleanWaitMillis, 100,                     \
           "(Temporary, subject to experimentation)"                         \
           " Time that we sleep between iterations when not given"           \
           " enough work per iteration")                                     \
@@ -1677,7 +1677,7 @@ class CommandLineFlags {
   product(uintx, CMSWorkQueueDrainThreshold, 10,                            \
           "Don't drain below this size per parallel worker/thief")          \
                                                                             \
-  product(intx, CMSWaitDuration, 2000,                                      \
+  manageable(intx, CMSWaitDuration, 2000,                                   \
           "Time in milliseconds that CMS thread waits for young GC")        \
                                                                             \
   product(bool, CMSYield, true,                                             \

From 22929fb78f4606f726b652a25772980336398575 Mon Sep 17 00:00:00 2001
From: Karen Kinnear <acorn@openjdk.org>
Date: Fri, 22 Oct 2010 15:59:34 -0400
Subject: [PATCH 16/23] 6988353: refactor contended sync subsystem

Reduce complexity by factoring synchronizer.cpp

Reviewed-by: dholmes, never, coleenp
---
 .../src/os/linux/vm/objectMonitor_linux.cpp   |   24 -
 .../src/os/linux/vm/objectMonitor_linux.hpp   |   25 -
 .../linux/vm/objectMonitor_linux.inline.hpp   |   23 -
 .../os/solaris/vm/objectMonitor_solaris.cpp   |   23 -
 .../os/solaris/vm/objectMonitor_solaris.hpp   |   25 -
 .../vm/objectMonitor_solaris.inline.hpp       |   23 -
 .../os/windows/vm/objectMonitor_windows.cpp   |   25 -
 .../os/windows/vm/objectMonitor_windows.hpp   |   25 -
 .../vm/objectMonitor_windows.inline.hpp       |   23 -
 hotspot/src/share/vm/includeDB_compiler1      |    2 +-
 hotspot/src/share/vm/includeDB_core           |   66 +-
 hotspot/src/share/vm/includeDB_features       |    7 +
 hotspot/src/share/vm/includeDB_jvmti          |    8 +-
 hotspot/src/share/vm/prims/jvmtiImpl.cpp      |   71 -
 hotspot/src/share/vm/prims/jvmtiImpl.hpp      |   71 -
 .../src/share/vm/prims/jvmtiRawMonitor.cpp    |  420 ++
 .../src/share/vm/prims/jvmtiRawMonitor.hpp    |   99 +
 hotspot/src/share/vm/runtime/basicLock.cpp    |   76 +
 hotspot/src/share/vm/runtime/basicLock.hpp    |   72 +
 hotspot/src/share/vm/runtime/mutex.hpp        |   45 -
 .../src/share/vm/runtime/objectMonitor.cpp    | 2421 +++++++++
 .../src/share/vm/runtime/objectMonitor.hpp    |  148 +-
 .../share/vm/runtime/objectMonitor.inline.hpp |    4 -
 hotspot/src/share/vm/runtime/park.cpp         |  237 +
 hotspot/src/share/vm/runtime/park.hpp         |  169 +
 hotspot/src/share/vm/runtime/synchronizer.cpp | 4615 +++--------------
 hotspot/src/share/vm/runtime/synchronizer.hpp |   73 +-
 hotspot/src/share/vm/runtime/thread.cpp       |  429 +-
 hotspot/src/share/vm/runtime/thread.hpp       |   99 +-
 29 files changed, 4556 insertions(+), 4792 deletions(-)
 delete mode 100644 hotspot/src/os/linux/vm/objectMonitor_linux.cpp
 delete mode 100644 hotspot/src/os/linux/vm/objectMonitor_linux.hpp
 delete mode 100644 hotspot/src/os/linux/vm/objectMonitor_linux.inline.hpp
 delete mode 100644 hotspot/src/os/solaris/vm/objectMonitor_solaris.cpp
 delete mode 100644 hotspot/src/os/solaris/vm/objectMonitor_solaris.hpp
 delete mode 100644 hotspot/src/os/solaris/vm/objectMonitor_solaris.inline.hpp
 delete mode 100644 hotspot/src/os/windows/vm/objectMonitor_windows.cpp
 delete mode 100644 hotspot/src/os/windows/vm/objectMonitor_windows.hpp
 delete mode 100644 hotspot/src/os/windows/vm/objectMonitor_windows.inline.hpp
 create mode 100644 hotspot/src/share/vm/prims/jvmtiRawMonitor.cpp
 create mode 100644 hotspot/src/share/vm/prims/jvmtiRawMonitor.hpp
 create mode 100644 hotspot/src/share/vm/runtime/basicLock.cpp
 create mode 100644 hotspot/src/share/vm/runtime/basicLock.hpp
 create mode 100644 hotspot/src/share/vm/runtime/objectMonitor.cpp
 create mode 100644 hotspot/src/share/vm/runtime/park.cpp
 create mode 100644 hotspot/src/share/vm/runtime/park.hpp

diff --git a/hotspot/src/os/linux/vm/objectMonitor_linux.cpp b/hotspot/src/os/linux/vm/objectMonitor_linux.cpp
deleted file mode 100644
index 4e7878d22fe..00000000000
--- a/hotspot/src/os/linux/vm/objectMonitor_linux.cpp
+++ /dev/null
@@ -1,24 +0,0 @@
-
-/*
- * Copyright (c) 1999, 2005, Oracle and/or its affiliates. All rights reserved.
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
- * or visit www.oracle.com if you need additional information or have any
- * questions.
- *
- */
diff --git a/hotspot/src/os/linux/vm/objectMonitor_linux.hpp b/hotspot/src/os/linux/vm/objectMonitor_linux.hpp
deleted file mode 100644
index 663dd38a8a9..00000000000
--- a/hotspot/src/os/linux/vm/objectMonitor_linux.hpp
+++ /dev/null
@@ -1,25 +0,0 @@
-/*
- * Copyright (c) 1999, 2005, Oracle and/or its affiliates. All rights reserved.
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
- * or visit www.oracle.com if you need additional information or have any
- * questions.
- *
- */
-
- private:
diff --git a/hotspot/src/os/linux/vm/objectMonitor_linux.inline.hpp b/hotspot/src/os/linux/vm/objectMonitor_linux.inline.hpp
deleted file mode 100644
index 2e84f688281..00000000000
--- a/hotspot/src/os/linux/vm/objectMonitor_linux.inline.hpp
+++ /dev/null
@@ -1,23 +0,0 @@
-/*
- * Copyright (c) 1999, 2005, Oracle and/or its affiliates. All rights reserved.
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
- * or visit www.oracle.com if you need additional information or have any
- * questions.
- *
- */
diff --git a/hotspot/src/os/solaris/vm/objectMonitor_solaris.cpp b/hotspot/src/os/solaris/vm/objectMonitor_solaris.cpp
deleted file mode 100644
index 07fe1598f87..00000000000
--- a/hotspot/src/os/solaris/vm/objectMonitor_solaris.cpp
+++ /dev/null
@@ -1,23 +0,0 @@
-/*
- * Copyright (c) 1998, 2005, Oracle and/or its affiliates. All rights reserved.
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
- * or visit www.oracle.com if you need additional information or have any
- * questions.
- *
- */
diff --git a/hotspot/src/os/solaris/vm/objectMonitor_solaris.hpp b/hotspot/src/os/solaris/vm/objectMonitor_solaris.hpp
deleted file mode 100644
index 0b4fc2f7138..00000000000
--- a/hotspot/src/os/solaris/vm/objectMonitor_solaris.hpp
+++ /dev/null
@@ -1,25 +0,0 @@
-/*
- * Copyright (c) 1998, 2005, Oracle and/or its affiliates. All rights reserved.
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
- * or visit www.oracle.com if you need additional information or have any
- * questions.
- *
- */
-
- private:
diff --git a/hotspot/src/os/solaris/vm/objectMonitor_solaris.inline.hpp b/hotspot/src/os/solaris/vm/objectMonitor_solaris.inline.hpp
deleted file mode 100644
index 07fe1598f87..00000000000
--- a/hotspot/src/os/solaris/vm/objectMonitor_solaris.inline.hpp
+++ /dev/null
@@ -1,23 +0,0 @@
-/*
- * Copyright (c) 1998, 2005, Oracle and/or its affiliates. All rights reserved.
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
- * or visit www.oracle.com if you need additional information or have any
- * questions.
- *
- */
diff --git a/hotspot/src/os/windows/vm/objectMonitor_windows.cpp b/hotspot/src/os/windows/vm/objectMonitor_windows.cpp
deleted file mode 100644
index 0410186f51f..00000000000
--- a/hotspot/src/os/windows/vm/objectMonitor_windows.cpp
+++ /dev/null
@@ -1,25 +0,0 @@
-/*
- * Copyright (c) 1998, 2005, Oracle and/or its affiliates. All rights reserved.
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
- * or visit www.oracle.com if you need additional information or have any
- * questions.
- *
- */
-
-#include "incls/_precompiled.incl"
diff --git a/hotspot/src/os/windows/vm/objectMonitor_windows.hpp b/hotspot/src/os/windows/vm/objectMonitor_windows.hpp
deleted file mode 100644
index 0b4fc2f7138..00000000000
--- a/hotspot/src/os/windows/vm/objectMonitor_windows.hpp
+++ /dev/null
@@ -1,25 +0,0 @@
-/*
- * Copyright (c) 1998, 2005, Oracle and/or its affiliates. All rights reserved.
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
- * or visit www.oracle.com if you need additional information or have any
- * questions.
- *
- */
-
- private:
diff --git a/hotspot/src/os/windows/vm/objectMonitor_windows.inline.hpp b/hotspot/src/os/windows/vm/objectMonitor_windows.inline.hpp
deleted file mode 100644
index 07fe1598f87..00000000000
--- a/hotspot/src/os/windows/vm/objectMonitor_windows.inline.hpp
+++ /dev/null
@@ -1,23 +0,0 @@
-/*
- * Copyright (c) 1998, 2005, Oracle and/or its affiliates. All rights reserved.
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
- * or visit www.oracle.com if you need additional information or have any
- * questions.
- *
- */
diff --git a/hotspot/src/share/vm/includeDB_compiler1 b/hotspot/src/share/vm/includeDB_compiler1
index f578561609e..71fbea13d56 100644
--- a/hotspot/src/share/vm/includeDB_compiler1
+++ b/hotspot/src/share/vm/includeDB_compiler1
@@ -301,6 +301,7 @@ c1_MacroAssembler.hpp                   assembler.hpp
 c1_MacroAssembler.hpp                   assembler_<arch>.inline.hpp
 
 c1_MacroAssembler_<arch>.cpp            arrayOop.hpp
+c1_MacroAssembler_<arch>.cpp            basicLock.hpp
 c1_MacroAssembler_<arch>.cpp            biasedLocking.hpp
 c1_MacroAssembler_<arch>.cpp            c1_MacroAssembler.hpp
 c1_MacroAssembler_<arch>.cpp            c1_Runtime1.hpp
@@ -309,7 +310,6 @@ c1_MacroAssembler_<arch>.cpp            interpreter.hpp
 c1_MacroAssembler_<arch>.cpp            markOop.hpp
 c1_MacroAssembler_<arch>.cpp            os.hpp
 c1_MacroAssembler_<arch>.cpp            stubRoutines.hpp
-c1_MacroAssembler_<arch>.cpp            synchronizer.hpp
 c1_MacroAssembler_<arch>.cpp            systemDictionary.hpp
 
 c1_MacroAssembler_<arch>.hpp            generate_platform_dependent_include
diff --git a/hotspot/src/share/vm/includeDB_core b/hotspot/src/share/vm/includeDB_core
index 14d78e91d41..479363c61ec 100644
--- a/hotspot/src/share/vm/includeDB_core
+++ b/hotspot/src/share/vm/includeDB_core
@@ -300,10 +300,17 @@ barrierSet.hpp                          oopsHierarchy.hpp
 barrierSet.inline.hpp                   barrierSet.hpp
 barrierSet.inline.hpp                   cardTableModRefBS.hpp
 
+basicLock.cpp                           basicLock.hpp
+basicLock.cpp                           synchronizer.hpp
+
+basicLock.hpp                           handles.hpp
+basicLock.hpp                           markOop.hpp
+basicLock.hpp                           top.hpp
+
+biasedLocking.cpp                       basicLock.hpp
 biasedLocking.cpp                       biasedLocking.hpp
 biasedLocking.cpp                       klass.inline.hpp
 biasedLocking.cpp                       markOop.hpp
-biasedLocking.cpp                       synchronizer.hpp
 biasedLocking.cpp                       task.hpp
 biasedLocking.cpp                       vframe.hpp
 biasedLocking.cpp                       vmThread.hpp
@@ -404,13 +411,13 @@ bytecodeInterpreter_<arch>.cpp          vframeArray.hpp
 bytecodeInterpreterWithChecks.cpp       bytecodeInterpreter.cpp
 
 bytecodeInterpreter.hpp                 allocation.hpp
+bytecodeInterpreter.hpp                 basicLock.hpp
 bytecodeInterpreter.hpp                 bytes_<arch>.hpp
 bytecodeInterpreter.hpp                 frame.hpp
 bytecodeInterpreter.hpp                 globalDefinitions.hpp
 bytecodeInterpreter.hpp                 globals.hpp
 bytecodeInterpreter.hpp                 methodDataOop.hpp
 bytecodeInterpreter.hpp                 methodOop.hpp
-bytecodeInterpreter.hpp                 synchronizer.hpp
 
 bytecodeInterpreter.inline.hpp          bytecodeInterpreter.hpp
 bytecodeInterpreter.inline.hpp          stubRoutines.hpp
@@ -1667,10 +1674,10 @@ frame.cpp                               stubRoutines.hpp
 frame.cpp                               universe.inline.hpp
 
 frame.hpp                               assembler.hpp
+frame.hpp                               basicLock.hpp
 frame.hpp                               methodOop.hpp
 frame.hpp                               monitorChunk.hpp
 frame.hpp                               registerMap.hpp
-frame.hpp                               synchronizer.hpp
 frame.hpp                               top.hpp
 
 frame.inline.hpp                        bytecodeInterpreter.hpp
@@ -2120,6 +2127,7 @@ interfaceSupport.hpp                    vmThread.hpp
 interfaceSupport_<os_family>.hpp        generate_platform_dependent_include
 
 interp_masm_<arch_model>.cpp            arrayOop.hpp
+interp_masm_<arch_model>.cpp            basicLock.hpp
 interp_masm_<arch_model>.cpp            biasedLocking.hpp
 interp_masm_<arch_model>.cpp            interp_masm_<arch_model>.hpp
 interp_masm_<arch_model>.cpp            interpreterRuntime.hpp
@@ -2131,7 +2139,6 @@ interp_masm_<arch_model>.cpp            markOop.hpp
 interp_masm_<arch_model>.cpp            methodDataOop.hpp
 interp_masm_<arch_model>.cpp            methodOop.hpp
 interp_masm_<arch_model>.cpp            sharedRuntime.hpp
-interp_masm_<arch_model>.cpp            synchronizer.hpp
 interp_masm_<arch_model>.cpp            thread_<os_family>.inline.hpp
 
 interp_masm_<arch_model>.hpp            assembler_<arch>.inline.hpp
@@ -3094,25 +3101,26 @@ objArrayOop.cpp                         oop.inline.hpp
 
 objArrayOop.hpp                         arrayOop.hpp
 
+objectMonitor.cpp                       dtrace.hpp
+objectMonitor.cpp                       handles.inline.hpp
+objectMonitor.cpp                       interfaceSupport.hpp
+objectMonitor.cpp                       markOop.hpp
+objectMonitor.cpp                       mutexLocker.hpp
+objectMonitor.cpp                       objectMonitor.hpp
+objectMonitor.cpp                       objectMonitor.inline.hpp
+objectMonitor.cpp                       oop.inline.hpp
+objectMonitor.cpp                       osThread.hpp
+objectMonitor.cpp                       os_<os_family>.inline.hpp
+objectMonitor.cpp                       preserveException.hpp
+objectMonitor.cpp                       resourceArea.hpp
+objectMonitor.cpp                       stubRoutines.hpp
+objectMonitor.cpp                       thread.hpp
+objectMonitor.cpp                       thread_<os_family>.inline.hpp
+objectMonitor.cpp                       threadService.hpp
+objectMonitor.cpp                       vmSymbols.hpp
+
 objectMonitor.hpp                       os.hpp
-
-objectMonitor_<os_family>.cpp           dtrace.hpp
-objectMonitor_<os_family>.cpp           interfaceSupport.hpp
-objectMonitor_<os_family>.cpp           objectMonitor.hpp
-objectMonitor_<os_family>.cpp           objectMonitor.inline.hpp
-objectMonitor_<os_family>.cpp           oop.inline.hpp
-objectMonitor_<os_family>.cpp           osThread.hpp
-objectMonitor_<os_family>.cpp           os_<os_family>.inline.hpp
-objectMonitor_<os_family>.cpp           threadService.hpp
-objectMonitor_<os_family>.cpp           thread_<os_family>.inline.hpp
-objectMonitor_<os_family>.cpp           vmSymbols.hpp
-
-objectMonitor_<os_family>.hpp           generate_platform_dependent_include
-objectMonitor_<os_family>.hpp           os_<os_family>.inline.hpp
-objectMonitor_<os_family>.hpp           thread_<os_family>.inline.hpp
-objectMonitor_<os_family>.hpp           top.hpp
-
-objectMonitor_<os_family>.inline.hpp    generate_platform_dependent_include
+objectMonitor.hpp                       perfData.hpp
 
 oop.cpp                                 copy.hpp
 oop.cpp                                 handles.inline.hpp
@@ -3329,7 +3337,6 @@ os_<os_family>.cpp                      mutex_<os_family>.inline.hpp
 os_<os_family>.cpp                      nativeInst_<arch>.hpp
 os_<os_family>.cpp                      no_precompiled_headers
 os_<os_family>.cpp                      objectMonitor.hpp
-os_<os_family>.cpp                      objectMonitor.inline.hpp
 os_<os_family>.cpp                      oop.inline.hpp
 os_<os_family>.cpp                      osThread.hpp
 os_<os_family>.cpp                      os_share_<os_family>.hpp
@@ -3389,6 +3396,12 @@ ostream.cpp                             xmlstream.hpp
 ostream.hpp                             allocation.hpp
 ostream.hpp                             timer.hpp
 
+// include thread.hpp to prevent cyclic includes
+park.cpp                                thread.hpp
+
+park.hpp                                debug.hpp
+park.hpp                                globalDefinitions.hpp
+
 pcDesc.cpp                              debugInfoRec.hpp
 pcDesc.cpp                              nmethod.hpp
 pcDesc.cpp                              pcDesc.hpp
@@ -4063,10 +4076,10 @@ synchronizer.cpp                        preserveException.hpp
 synchronizer.cpp                        resourceArea.hpp
 synchronizer.cpp                        stubRoutines.hpp
 synchronizer.cpp                        synchronizer.hpp
-synchronizer.cpp                        threadService.hpp
 synchronizer.cpp                        thread_<os_family>.inline.hpp
 synchronizer.cpp                        vmSymbols.hpp
 
+synchronizer.hpp                        basicLock.hpp
 synchronizer.hpp                        handles.hpp
 synchronizer.hpp                        markOop.hpp
 synchronizer.hpp                        perfData.hpp
@@ -4238,7 +4251,6 @@ thread.cpp                              memprofiler.hpp
 thread.cpp                              mutexLocker.hpp
 thread.cpp                              objArrayOop.hpp
 thread.cpp                              objectMonitor.hpp
-thread.cpp                              objectMonitor.inline.hpp
 thread.cpp                              oop.inline.hpp
 thread.cpp                              oopFactory.hpp
 thread.cpp                              osThread.hpp
@@ -4276,6 +4288,7 @@ thread.hpp                              mutexLocker.hpp
 thread.hpp                              oop.hpp
 thread.hpp                              os.hpp
 thread.hpp                              osThread.hpp
+thread.hpp                              park.hpp
 thread.hpp                              safepoint.hpp
 thread.hpp                              stubRoutines.hpp
 thread.hpp                              threadLocalAllocBuffer.hpp
@@ -4587,6 +4600,7 @@ vframeArray.hpp                         frame.inline.hpp
 vframeArray.hpp                         growableArray.hpp
 vframeArray.hpp                         monitorChunk.hpp
 
+vframe_hp.cpp                           basicLock.hpp
 vframe_hp.cpp                           codeCache.hpp
 vframe_hp.cpp                           debugInfoRec.hpp
 vframe_hp.cpp                           handles.inline.hpp
@@ -4600,7 +4614,6 @@ vframe_hp.cpp                           pcDesc.hpp
 vframe_hp.cpp                           scopeDesc.hpp
 vframe_hp.cpp                           signature.hpp
 vframe_hp.cpp                           stubRoutines.hpp
-vframe_hp.cpp                           synchronizer.hpp
 vframe_hp.cpp                           vframeArray.hpp
 vframe_hp.cpp                           vframe_hp.hpp
 
@@ -4752,6 +4765,7 @@ workgroup.cpp                           os.hpp
 workgroup.cpp                           workgroup.hpp
 
 workgroup.hpp                           taskqueue.hpp
+
 workgroup.hpp                           thread_<os_family>.inline.hpp
 
 xmlstream.cpp                           allocation.hpp
diff --git a/hotspot/src/share/vm/includeDB_features b/hotspot/src/share/vm/includeDB_features
index 39e1149b06f..5e9b00c0cb9 100644
--- a/hotspot/src/share/vm/includeDB_features
+++ b/hotspot/src/share/vm/includeDB_features
@@ -184,6 +184,13 @@ jvmtiImpl.hpp                           stackValueCollection.hpp
 jvmtiImpl.hpp                           systemDictionary.hpp
 jvmtiImpl.hpp                           vm_operations.hpp
 
+jvmtiRawMonitor.cpp                     interfaceSupport.hpp
+jvmtiRawMonitor.cpp                     jvmtiRawMonitor.hpp
+jvmtiRawMonitor.cpp                     thread.hpp
+
+jvmtiRawMonitor.hpp                     growableArray.hpp
+jvmtiRawMonitor.hpp                     objectMonitor.hpp
+
 jvmtiTagMap.cpp                         biasedLocking.hpp
 jvmtiTagMap.cpp                         javaCalls.hpp
 jvmtiTagMap.cpp                         jniHandles.hpp
diff --git a/hotspot/src/share/vm/includeDB_jvmti b/hotspot/src/share/vm/includeDB_jvmti
index 704cf78afef..837c1b34df2 100644
--- a/hotspot/src/share/vm/includeDB_jvmti
+++ b/hotspot/src/share/vm/includeDB_jvmti
@@ -35,6 +35,7 @@ jvmtiClassFileReconstituter.hpp         jvmtiEnv.hpp
 // jvmtiCodeBlobEvents is jck optional, please put deps in includeDB_features
 
 jvmtiEnter.cpp                          jvmtiEnter.hpp
+jvmtiEnter.cpp                          jvmtiRawMonitor.hpp
 jvmtiEnter.cpp                          jvmtiUtil.hpp
 
 jvmtiEnter.hpp                          interfaceSupport.hpp
@@ -44,6 +45,7 @@ jvmtiEnter.hpp                          resourceArea.hpp
 jvmtiEnter.hpp                          systemDictionary.hpp
 
 jvmtiEnterTrace.cpp                     jvmtiEnter.hpp
+jvmtiEnterTrace.cpp                     jvmtiRawMonitor.hpp
 jvmtiEnterTrace.cpp                     jvmtiUtil.hpp
 
 jvmtiEnv.cpp                            arguments.hpp
@@ -66,11 +68,11 @@ jvmtiEnv.cpp                            jvmtiExtensions.hpp
 jvmtiEnv.cpp                            jvmtiGetLoadedClasses.hpp
 jvmtiEnv.cpp                            jvmtiImpl.hpp
 jvmtiEnv.cpp                            jvmtiManageCapabilities.hpp
+jvmtiEnv.cpp                            jvmtiRawMonitor.hpp
 jvmtiEnv.cpp                            jvmtiRedefineClasses.hpp
 jvmtiEnv.cpp                            jvmtiTagMap.hpp
 jvmtiEnv.cpp                            jvmtiThreadState.inline.hpp
 jvmtiEnv.cpp                            jvmtiUtil.hpp
-jvmtiEnv.cpp                            objectMonitor.inline.hpp
 jvmtiEnv.cpp                            osThread.hpp
 jvmtiEnv.cpp                            preserveException.hpp
 jvmtiEnv.cpp                            reflectionUtils.hpp
@@ -178,11 +180,13 @@ jvmtiExport.cpp                         jvmtiEventController.inline.hpp
 jvmtiExport.cpp                         jvmtiExport.hpp
 jvmtiExport.cpp                         jvmtiImpl.hpp
 jvmtiExport.cpp                         jvmtiManageCapabilities.hpp
+jvmtiExport.cpp                         jvmtiRawMonitor.hpp
 jvmtiExport.cpp                         jvmtiTagMap.hpp
 jvmtiExport.cpp                         jvmtiThreadState.inline.hpp
 jvmtiExport.cpp                         nmethod.hpp
 jvmtiExport.cpp                         objArrayKlass.hpp
 jvmtiExport.cpp                         objArrayOop.hpp
+jvmtiExport.cpp                         objectMonitor.hpp
 jvmtiExport.cpp                         objectMonitor.inline.hpp
 jvmtiExport.cpp                         pcDesc.hpp
 jvmtiExport.cpp                         resourceArea.hpp
@@ -210,6 +214,8 @@ jvmtiManageCapabilities.cpp             jvmtiManageCapabilities.hpp
 jvmtiManageCapabilities.hpp             allocation.hpp
 jvmtiManageCapabilities.hpp             jvmti.h
 
+// jvmtiRawMonitor is jck optional, please put deps in includeDB_features
+
 jvmtiRedefineClasses.cpp                bitMap.inline.hpp
 jvmtiRedefineClasses.cpp                codeCache.hpp
 jvmtiRedefineClasses.cpp                deoptimization.hpp
diff --git a/hotspot/src/share/vm/prims/jvmtiImpl.cpp b/hotspot/src/share/vm/prims/jvmtiImpl.cpp
index 0fb385d1e7f..d18478cf6c8 100644
--- a/hotspot/src/share/vm/prims/jvmtiImpl.cpp
+++ b/hotspot/src/share/vm/prims/jvmtiImpl.cpp
@@ -25,26 +25,6 @@
 # include "incls/_precompiled.incl"
 # include "incls/_jvmtiImpl.cpp.incl"
 
-GrowableArray<JvmtiRawMonitor*> *JvmtiPendingMonitors::_monitors = new (ResourceObj::C_HEAP) GrowableArray<JvmtiRawMonitor*>(1,true);
-
-void JvmtiPendingMonitors::transition_raw_monitors() {
-  assert((Threads::number_of_threads()==1),
-         "Java thread has not created yet or more than one java thread \
-is running. Raw monitor transition will not work");
-  JavaThread *current_java_thread = JavaThread::current();
-  assert(current_java_thread->thread_state() == _thread_in_vm, "Must be in vm");
-  {
-    ThreadBlockInVM __tbivm(current_java_thread);
-    for(int i=0; i< count(); i++) {
-      JvmtiRawMonitor *rmonitor = monitors()->at(i);
-      int r = rmonitor->raw_enter(current_java_thread);
-      assert(r == ObjectMonitor::OM_OK, "raw_enter should have worked");
-    }
-  }
-  // pending monitors are converted to real monitor so delete them all.
-  dispose();
-}
-
 //
 // class JvmtiAgentThread
 //
@@ -216,57 +196,6 @@ void GrowableCache::gc_epilogue() {
   }
 }
 
-
-//
-// class JvmtiRawMonitor
-//
-
-JvmtiRawMonitor::JvmtiRawMonitor(const char *name) {
-#ifdef ASSERT
-  _name = strcpy(NEW_C_HEAP_ARRAY(char, strlen(name) + 1), name);
-#else
-  _name = NULL;
-#endif
-  _magic = JVMTI_RM_MAGIC;
-}
-
-JvmtiRawMonitor::~JvmtiRawMonitor() {
-#ifdef ASSERT
-  FreeHeap(_name);
-#endif
-  _magic = 0;
-}
-
-
-bool
-JvmtiRawMonitor::is_valid() {
-  int value = 0;
-
-  // This object might not be a JvmtiRawMonitor so we can't assume
-  // the _magic field is properly aligned. Get the value in a safe
-  // way and then check against JVMTI_RM_MAGIC.
-
-  switch (sizeof(_magic)) {
-  case 2:
-    value = Bytes::get_native_u2((address)&_magic);
-    break;
-
-  case 4:
-    value = Bytes::get_native_u4((address)&_magic);
-    break;
-
-  case 8:
-    value = Bytes::get_native_u8((address)&_magic);
-    break;
-
-  default:
-    guarantee(false, "_magic field is an unexpected size");
-  }
-
-  return value == JVMTI_RM_MAGIC;
-}
-
-
 //
 // class JvmtiBreakpoint
 //
diff --git a/hotspot/src/share/vm/prims/jvmtiImpl.hpp b/hotspot/src/share/vm/prims/jvmtiImpl.hpp
index 0b649412d97..945cda5ac34 100644
--- a/hotspot/src/share/vm/prims/jvmtiImpl.hpp
+++ b/hotspot/src/share/vm/prims/jvmtiImpl.hpp
@@ -26,7 +26,6 @@
 // Forward Declarations
 //
 
-class JvmtiRawMonitor;
 class JvmtiBreakpoint;
 class JvmtiBreakpoints;
 
@@ -327,76 +326,6 @@ bool JvmtiCurrentBreakpoints::is_breakpoint(address bcp) {
     return false;
 }
 
-
-///////////////////////////////////////////////////////////////
-//
-// class JvmtiRawMonitor
-//
-// Used by JVMTI methods: All RawMonitor methods (CreateRawMonitor, EnterRawMonitor, etc.)
-//
-// Wrapper for ObjectMonitor class that saves the Monitor's name
-//
-
-class JvmtiRawMonitor : public ObjectMonitor  {
-private:
-  int           _magic;
-  char *        _name;
-  // JVMTI_RM_MAGIC is set in contructor and unset in destructor.
-  enum { JVMTI_RM_MAGIC = (int)(('T' << 24) | ('I' << 16) | ('R' << 8) | 'M') };
-
-public:
-  JvmtiRawMonitor(const char *name);
-  ~JvmtiRawMonitor();
-  int            magic()   { return _magic;  }
-  const char *get_name()   { return _name; }
-  bool        is_valid();
-};
-
-// Onload pending raw monitors
-// Class is used to cache onload or onstart monitor enter
-// which will transition into real monitor when
-// VM is fully initialized.
-class JvmtiPendingMonitors : public AllStatic {
-
-private:
-  static GrowableArray<JvmtiRawMonitor*> *_monitors; // Cache raw monitor enter
-
-  inline static GrowableArray<JvmtiRawMonitor*>* monitors() { return _monitors; }
-
-  static void dispose() {
-    delete monitors();
-  }
-
-public:
-  static void enter(JvmtiRawMonitor *monitor) {
-    monitors()->append(monitor);
-  }
-
-  static int count() {
-    return monitors()->length();
-  }
-
-  static void destroy(JvmtiRawMonitor *monitor) {
-    while (monitors()->contains(monitor)) {
-      monitors()->remove(monitor);
-    }
-  }
-
-  // Return false if monitor is not found in the list.
-  static bool exit(JvmtiRawMonitor *monitor) {
-    if (monitors()->contains(monitor)) {
-      monitors()->remove(monitor);
-      return true;
-    } else {
-      return false;
-    }
-  }
-
-  static void transition_raw_monitors();
-};
-
-
-
 ///////////////////////////////////////////////////////////////
 // The get/set local operations must only be done by the VM thread
 // because the interpreter version needs to access oop maps, which can
diff --git a/hotspot/src/share/vm/prims/jvmtiRawMonitor.cpp b/hotspot/src/share/vm/prims/jvmtiRawMonitor.cpp
new file mode 100644
index 00000000000..c7dc2939f07
--- /dev/null
+++ b/hotspot/src/share/vm/prims/jvmtiRawMonitor.cpp
@@ -0,0 +1,420 @@
+/*
+ * Copyright (c) 2003, 2007, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+# include "incls/_precompiled.incl"
+# include "incls/_jvmtiRawMonitor.cpp.incl"
+
+GrowableArray<JvmtiRawMonitor*> *JvmtiPendingMonitors::_monitors = new (ResourceObj::C_HEAP) GrowableArray<JvmtiRawMonitor*>(1,true);
+
+void JvmtiPendingMonitors::transition_raw_monitors() {
+  assert((Threads::number_of_threads()==1),
+         "Java thread has not created yet or more than one java thread \
+is running. Raw monitor transition will not work");
+  JavaThread *current_java_thread = JavaThread::current();
+  assert(current_java_thread->thread_state() == _thread_in_vm, "Must be in vm");
+  {
+    ThreadBlockInVM __tbivm(current_java_thread);
+    for(int i=0; i< count(); i++) {
+      JvmtiRawMonitor *rmonitor = monitors()->at(i);
+      int r = rmonitor->raw_enter(current_java_thread);
+      assert(r == ObjectMonitor::OM_OK, "raw_enter should have worked");
+    }
+  }
+  // pending monitors are converted to real monitor so delete them all.
+  dispose();
+}
+
+//
+// class JvmtiRawMonitor
+//
+
+JvmtiRawMonitor::JvmtiRawMonitor(const char *name) {
+#ifdef ASSERT
+  _name = strcpy(NEW_C_HEAP_ARRAY(char, strlen(name) + 1), name);
+#else
+  _name = NULL;
+#endif
+  _magic = JVMTI_RM_MAGIC;
+}
+
+JvmtiRawMonitor::~JvmtiRawMonitor() {
+#ifdef ASSERT
+  FreeHeap(_name);
+#endif
+  _magic = 0;
+}
+
+
+bool
+JvmtiRawMonitor::is_valid() {
+  int value = 0;
+
+  // This object might not be a JvmtiRawMonitor so we can't assume
+  // the _magic field is properly aligned. Get the value in a safe
+  // way and then check against JVMTI_RM_MAGIC.
+
+  switch (sizeof(_magic)) {
+  case 2:
+    value = Bytes::get_native_u2((address)&_magic);
+    break;
+
+  case 4:
+    value = Bytes::get_native_u4((address)&_magic);
+    break;
+
+  case 8:
+    value = Bytes::get_native_u8((address)&_magic);
+    break;
+
+  default:
+    guarantee(false, "_magic field is an unexpected size");
+  }
+
+  return value == JVMTI_RM_MAGIC;
+}
+
+// -------------------------------------------------------------------------
+// The raw monitor subsystem is entirely distinct from normal
+// java-synchronization or jni-synchronization.  raw monitors are not
+// associated with objects.  They can be implemented in any manner
+// that makes sense.  The original implementors decided to piggy-back
+// the raw-monitor implementation on the existing Java objectMonitor mechanism.
+// This flaw needs to fixed.  We should reimplement raw monitors as sui-generis.
+// Specifically, we should not implement raw monitors via java monitors.
+// Time permitting, we should disentangle and deconvolve the two implementations
+// and move the resulting raw monitor implementation over to the JVMTI directories.
+// Ideally, the raw monitor implementation would be built on top of
+// park-unpark and nothing else.
+//
+// raw monitors are used mainly by JVMTI
+// The raw monitor implementation borrows the ObjectMonitor structure,
+// but the operators are degenerate and extremely simple.
+//
+// Mixed use of a single objectMonitor instance -- as both a raw monitor
+// and a normal java monitor -- is not permissible.
+//
+// Note that we use the single RawMonitor_lock to protect queue operations for
+// _all_ raw monitors.  This is a scalability impediment, but since raw monitor usage
+// is deprecated and rare, this is not of concern.  The RawMonitor_lock can not
+// be held indefinitely.  The critical sections must be short and bounded.
+//
+// -------------------------------------------------------------------------
+
+int JvmtiRawMonitor::SimpleEnter (Thread * Self) {
+  for (;;) {
+    if (Atomic::cmpxchg_ptr (Self, &_owner, NULL) == NULL) {
+       return OS_OK ;
+    }
+
+    ObjectWaiter Node (Self) ;
+    Self->_ParkEvent->reset() ;     // strictly optional
+    Node.TState = ObjectWaiter::TS_ENTER ;
+
+    RawMonitor_lock->lock_without_safepoint_check() ;
+    Node._next  = _EntryList ;
+    _EntryList  = &Node ;
+    OrderAccess::fence() ;
+    if (_owner == NULL && Atomic::cmpxchg_ptr (Self, &_owner, NULL) == NULL) {
+        _EntryList = Node._next ;
+        RawMonitor_lock->unlock() ;
+        return OS_OK ;
+    }
+    RawMonitor_lock->unlock() ;
+    while (Node.TState == ObjectWaiter::TS_ENTER) {
+       Self->_ParkEvent->park() ;
+    }
+  }
+}
+
+int JvmtiRawMonitor::SimpleExit (Thread * Self) {
+  guarantee (_owner == Self, "invariant") ;
+  OrderAccess::release_store_ptr (&_owner, NULL) ;
+  OrderAccess::fence() ;
+  if (_EntryList == NULL) return OS_OK ;
+  ObjectWaiter * w ;
+
+  RawMonitor_lock->lock_without_safepoint_check() ;
+  w = _EntryList ;
+  if (w != NULL) {
+      _EntryList = w->_next ;
+  }
+  RawMonitor_lock->unlock() ;
+  if (w != NULL) {
+      guarantee (w ->TState == ObjectWaiter::TS_ENTER, "invariant") ;
+      ParkEvent * ev = w->_event ;
+      w->TState = ObjectWaiter::TS_RUN ;
+      OrderAccess::fence() ;
+      ev->unpark() ;
+  }
+  return OS_OK ;
+}
+
+int JvmtiRawMonitor::SimpleWait (Thread * Self, jlong millis) {
+  guarantee (_owner == Self  , "invariant") ;
+  guarantee (_recursions == 0, "invariant") ;
+
+  ObjectWaiter Node (Self) ;
+  Node._notified = 0 ;
+  Node.TState    = ObjectWaiter::TS_WAIT ;
+
+  RawMonitor_lock->lock_without_safepoint_check() ;
+  Node._next     = _WaitSet ;
+  _WaitSet       = &Node ;
+  RawMonitor_lock->unlock() ;
+
+  SimpleExit (Self) ;
+  guarantee (_owner != Self, "invariant") ;
+
+  int ret = OS_OK ;
+  if (millis <= 0) {
+    Self->_ParkEvent->park();
+  } else {
+    ret = Self->_ParkEvent->park(millis);
+  }
+
+  // If thread still resides on the waitset then unlink it.
+  // Double-checked locking -- the usage is safe in this context
+  // as we TState is volatile and the lock-unlock operators are
+  // serializing (barrier-equivalent).
+
+  if (Node.TState == ObjectWaiter::TS_WAIT) {
+    RawMonitor_lock->lock_without_safepoint_check() ;
+    if (Node.TState == ObjectWaiter::TS_WAIT) {
+      // Simple O(n) unlink, but performance isn't critical here.
+      ObjectWaiter * p ;
+      ObjectWaiter * q = NULL ;
+      for (p = _WaitSet ; p != &Node; p = p->_next) {
+         q = p ;
+      }
+      guarantee (p == &Node, "invariant") ;
+      if (q == NULL) {
+        guarantee (p == _WaitSet, "invariant") ;
+        _WaitSet = p->_next ;
+      } else {
+        guarantee (p == q->_next, "invariant") ;
+        q->_next = p->_next ;
+      }
+      Node.TState = ObjectWaiter::TS_RUN ;
+    }
+    RawMonitor_lock->unlock() ;
+  }
+
+  guarantee (Node.TState == ObjectWaiter::TS_RUN, "invariant") ;
+  SimpleEnter (Self) ;
+
+  guarantee (_owner == Self, "invariant") ;
+  guarantee (_recursions == 0, "invariant") ;
+  return ret ;
+}
+
+int JvmtiRawMonitor::SimpleNotify (Thread * Self, bool All) {
+  guarantee (_owner == Self, "invariant") ;
+  if (_WaitSet == NULL) return OS_OK ;
+
+  // We have two options:
+  // A. Transfer the threads from the WaitSet to the EntryList
+  // B. Remove the thread from the WaitSet and unpark() it.
+  //
+  // We use (B), which is crude and results in lots of futile
+  // context switching.  In particular (B) induces lots of contention.
+
+  ParkEvent * ev = NULL ;       // consider using a small auto array ...
+  RawMonitor_lock->lock_without_safepoint_check() ;
+  for (;;) {
+      ObjectWaiter * w = _WaitSet ;
+      if (w == NULL) break ;
+      _WaitSet = w->_next ;
+      if (ev != NULL) { ev->unpark(); ev = NULL; }
+      ev = w->_event ;
+      OrderAccess::loadstore() ;
+      w->TState = ObjectWaiter::TS_RUN ;
+      OrderAccess::storeload();
+      if (!All) break ;
+  }
+  RawMonitor_lock->unlock() ;
+  if (ev != NULL) ev->unpark();
+  return OS_OK ;
+}
+
+// Any JavaThread will enter here with state _thread_blocked
+int JvmtiRawMonitor::raw_enter(TRAPS) {
+  TEVENT (raw_enter) ;
+  void * Contended ;
+
+  // don't enter raw monitor if thread is being externally suspended, it will
+  // surprise the suspender if a "suspended" thread can still enter monitor
+  JavaThread * jt = (JavaThread *)THREAD;
+  if (THREAD->is_Java_thread()) {
+    jt->SR_lock()->lock_without_safepoint_check();
+    while (jt->is_external_suspend()) {
+      jt->SR_lock()->unlock();
+      jt->java_suspend_self();
+      jt->SR_lock()->lock_without_safepoint_check();
+    }
+    // guarded by SR_lock to avoid racing with new external suspend requests.
+    Contended = Atomic::cmpxchg_ptr (THREAD, &_owner, NULL) ;
+    jt->SR_lock()->unlock();
+  } else {
+    Contended = Atomic::cmpxchg_ptr (THREAD, &_owner, NULL) ;
+  }
+
+  if (Contended == THREAD) {
+     _recursions ++ ;
+     return OM_OK ;
+  }
+
+  if (Contended == NULL) {
+     guarantee (_owner == THREAD, "invariant") ;
+     guarantee (_recursions == 0, "invariant") ;
+     return OM_OK ;
+  }
+
+  THREAD->set_current_pending_monitor(this);
+
+  if (!THREAD->is_Java_thread()) {
+     // No other non-Java threads besides VM thread would acquire
+     // a raw monitor.
+     assert(THREAD->is_VM_thread(), "must be VM thread");
+     SimpleEnter (THREAD) ;
+   } else {
+     guarantee (jt->thread_state() == _thread_blocked, "invariant") ;
+     for (;;) {
+       jt->set_suspend_equivalent();
+       // cleared by handle_special_suspend_equivalent_condition() or
+       // java_suspend_self()
+       SimpleEnter (THREAD) ;
+
+       // were we externally suspended while we were waiting?
+       if (!jt->handle_special_suspend_equivalent_condition()) break ;
+
+       // This thread was externally suspended
+       //
+       // This logic isn't needed for JVMTI raw monitors,
+       // but doesn't hurt just in case the suspend rules change. This
+           // logic is needed for the JvmtiRawMonitor.wait() reentry phase.
+           // We have reentered the contended monitor, but while we were
+           // waiting another thread suspended us. We don't want to reenter
+           // the monitor while suspended because that would surprise the
+           // thread that suspended us.
+           //
+           // Drop the lock -
+       SimpleExit (THREAD) ;
+
+           jt->java_suspend_self();
+         }
+
+     assert(_owner == THREAD, "Fatal error with monitor owner!");
+     assert(_recursions == 0, "Fatal error with monitor recursions!");
+  }
+
+  THREAD->set_current_pending_monitor(NULL);
+  guarantee (_recursions == 0, "invariant") ;
+  return OM_OK;
+}
+
+// Used mainly for JVMTI raw monitor implementation
+// Also used for JvmtiRawMonitor::wait().
+int JvmtiRawMonitor::raw_exit(TRAPS) {
+  TEVENT (raw_exit) ;
+  if (THREAD != _owner) {
+    return OM_ILLEGAL_MONITOR_STATE;
+  }
+  if (_recursions > 0) {
+    --_recursions ;
+    return OM_OK ;
+  }
+
+  void * List = _EntryList ;
+  SimpleExit (THREAD) ;
+
+  return OM_OK;
+}
+
+// Used for JVMTI raw monitor implementation.
+// All JavaThreads will enter here with state _thread_blocked
+
+int JvmtiRawMonitor::raw_wait(jlong millis, bool interruptible, TRAPS) {
+  TEVENT (raw_wait) ;
+  if (THREAD != _owner) {
+    return OM_ILLEGAL_MONITOR_STATE;
+  }
+
+  // To avoid spurious wakeups we reset the parkevent -- This is strictly optional.
+  // The caller must be able to tolerate spurious returns from raw_wait().
+  THREAD->_ParkEvent->reset() ;
+  OrderAccess::fence() ;
+
+  // check interrupt event
+  if (interruptible && Thread::is_interrupted(THREAD, true)) {
+    return OM_INTERRUPTED;
+  }
+
+  intptr_t save = _recursions ;
+  _recursions = 0 ;
+  _waiters ++ ;
+  if (THREAD->is_Java_thread()) {
+    guarantee (((JavaThread *) THREAD)->thread_state() == _thread_blocked, "invariant") ;
+    ((JavaThread *)THREAD)->set_suspend_equivalent();
+  }
+  int rv = SimpleWait (THREAD, millis) ;
+  _recursions = save ;
+  _waiters -- ;
+
+  guarantee (THREAD == _owner, "invariant") ;
+  if (THREAD->is_Java_thread()) {
+     JavaThread * jSelf = (JavaThread *) THREAD ;
+     for (;;) {
+        if (!jSelf->handle_special_suspend_equivalent_condition()) break ;
+        SimpleExit (THREAD) ;
+        jSelf->java_suspend_self();
+        SimpleEnter (THREAD) ;
+        jSelf->set_suspend_equivalent() ;
+     }
+  }
+  guarantee (THREAD == _owner, "invariant") ;
+
+  if (interruptible && Thread::is_interrupted(THREAD, true)) {
+    return OM_INTERRUPTED;
+  }
+  return OM_OK ;
+}
+
+int JvmtiRawMonitor::raw_notify(TRAPS) {
+  TEVENT (raw_notify) ;
+  if (THREAD != _owner) {
+    return OM_ILLEGAL_MONITOR_STATE;
+  }
+  SimpleNotify (THREAD, false) ;
+  return OM_OK;
+}
+
+int JvmtiRawMonitor::raw_notifyAll(TRAPS) {
+  TEVENT (raw_notifyAll) ;
+  if (THREAD != _owner) {
+    return OM_ILLEGAL_MONITOR_STATE;
+  }
+  SimpleNotify (THREAD, true) ;
+  return OM_OK;
+}
+
diff --git a/hotspot/src/share/vm/prims/jvmtiRawMonitor.hpp b/hotspot/src/share/vm/prims/jvmtiRawMonitor.hpp
new file mode 100644
index 00000000000..41d481bf448
--- /dev/null
+++ b/hotspot/src/share/vm/prims/jvmtiRawMonitor.hpp
@@ -0,0 +1,99 @@
+/*
+ * Copyright (c) 1999, 2007, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+//
+// class JvmtiRawMonitor
+//
+// Used by JVMTI methods: All RawMonitor methods (CreateRawMonitor, EnterRawMonitor, etc.)
+//
+// Wrapper for ObjectMonitor class that saves the Monitor's name
+//
+
+class JvmtiRawMonitor : public ObjectMonitor  {
+private:
+  int           _magic;
+  char *        _name;
+  // JVMTI_RM_MAGIC is set in contructor and unset in destructor.
+  enum { JVMTI_RM_MAGIC = (int)(('T' << 24) | ('I' << 16) | ('R' << 8) | 'M') };
+
+  int       SimpleEnter (Thread * Self) ;
+  int       SimpleExit  (Thread * Self) ;
+  int       SimpleWait  (Thread * Self, jlong millis) ;
+  int       SimpleNotify (Thread * Self, bool All) ;
+
+public:
+  JvmtiRawMonitor(const char *name);
+  ~JvmtiRawMonitor();
+  int       raw_enter(TRAPS);
+  int       raw_exit(TRAPS);
+  int       raw_wait(jlong millis, bool interruptable, TRAPS);
+  int       raw_notify(TRAPS);
+  int       raw_notifyAll(TRAPS);
+  int            magic()   { return _magic;  }
+  const char *get_name()   { return _name; }
+  bool        is_valid();
+};
+
+// Onload pending raw monitors
+// Class is used to cache onload or onstart monitor enter
+// which will transition into real monitor when
+// VM is fully initialized.
+class JvmtiPendingMonitors : public AllStatic {
+
+private:
+  static GrowableArray<JvmtiRawMonitor*> *_monitors; // Cache raw monitor enter
+
+  inline static GrowableArray<JvmtiRawMonitor*>* monitors() { return _monitors; }
+
+  static void dispose() {
+    delete monitors();
+  }
+
+public:
+  static void enter(JvmtiRawMonitor *monitor) {
+    monitors()->append(monitor);
+  }
+
+  static int count() {
+    return monitors()->length();
+  }
+
+  static void destroy(JvmtiRawMonitor *monitor) {
+    while (monitors()->contains(monitor)) {
+      monitors()->remove(monitor);
+    }
+  }
+
+  // Return false if monitor is not found in the list.
+  static bool exit(JvmtiRawMonitor *monitor) {
+    if (monitors()->contains(monitor)) {
+      monitors()->remove(monitor);
+      return true;
+    } else {
+      return false;
+    }
+  }
+
+  static void transition_raw_monitors();
+};
diff --git a/hotspot/src/share/vm/runtime/basicLock.cpp b/hotspot/src/share/vm/runtime/basicLock.cpp
new file mode 100644
index 00000000000..12956597e79
--- /dev/null
+++ b/hotspot/src/share/vm/runtime/basicLock.cpp
@@ -0,0 +1,76 @@
+/*
+ * Copyright (c) 1997, 2010, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+# include "incls/_precompiled.incl"
+# include "incls/_basicLock.cpp.incl"
+
+void BasicLock::print_on(outputStream* st) const {
+  st->print("monitor");
+}
+
+void BasicLock::move_to(oop obj, BasicLock* dest) {
+  // Check to see if we need to inflate the lock. This is only needed
+  // if an object is locked using "this" lightweight monitor. In that
+  // case, the displaced_header() is unlocked, because the
+  // displaced_header() contains the header for the originally unlocked
+  // object. However the object could have already been inflated. But it
+  // does not matter, the inflation will just a no-op. For other cases,
+  // the displaced header will be either 0x0 or 0x3, which are location
+  // independent, therefore the BasicLock is free to move.
+  //
+  // During OSR we may need to relocate a BasicLock (which contains a
+  // displaced word) from a location in an interpreter frame to a
+  // new location in a compiled frame.  "this" refers to the source
+  // basiclock in the interpreter frame.  "dest" refers to the destination
+  // basiclock in the new compiled frame.  We *always* inflate in move_to().
+  // The always-Inflate policy works properly, but in 1.5.0 it can sometimes
+  // cause performance problems in code that makes heavy use of a small # of
+  // uncontended locks.   (We'd inflate during OSR, and then sync performance
+  // would subsequently plummet because the thread would be forced thru the slow-path).
+  // This problem has been made largely moot on IA32 by inlining the inflated fast-path
+  // operations in Fast_Lock and Fast_Unlock in i486.ad.
+  //
+  // Note that there is a way to safely swing the object's markword from
+  // one stack location to another.  This avoids inflation.  Obviously,
+  // we need to ensure that both locations refer to the current thread's stack.
+  // There are some subtle concurrency issues, however, and since the benefit is
+  // is small (given the support for inflated fast-path locking in the fast_lock, etc)
+  // we'll leave that optimization for another time.
+
+  if (displaced_header()->is_neutral()) {
+    ObjectSynchronizer::inflate_helper(obj);
+    // WARNING: We can not put check here, because the inflation
+    // will not update the displaced header. Once BasicLock is inflated,
+    // no one should ever look at its content.
+  } else {
+    // Typically the displaced header will be 0 (recursive stack lock) or
+    // unused_mark.  Naively we'd like to assert that the displaced mark
+    // value is either 0, neutral, or 3.  But with the advent of the
+    // store-before-CAS avoidance in fast_lock/compiler_lock_object
+    // we can find any flavor mark in the displaced mark.
+  }
+// [RGV] The next line appears to do nothing!
+  intptr_t dh = (intptr_t) displaced_header();
+  dest->set_displaced_header(displaced_header());
+}
diff --git a/hotspot/src/share/vm/runtime/basicLock.hpp b/hotspot/src/share/vm/runtime/basicLock.hpp
new file mode 100644
index 00000000000..a7061b417a1
--- /dev/null
+++ b/hotspot/src/share/vm/runtime/basicLock.hpp
@@ -0,0 +1,72 @@
+/*
+ * Copyright (c) 1998, 2007, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+class BasicLock VALUE_OBJ_CLASS_SPEC {
+  friend class VMStructs;
+ private:
+  volatile markOop _displaced_header;
+ public:
+  markOop      displaced_header() const               { return _displaced_header; }
+  void         set_displaced_header(markOop header)   { _displaced_header = header; }
+
+  void print_on(outputStream* st) const;
+
+  // move a basic lock (used during deoptimization
+  void move_to(oop obj, BasicLock* dest);
+
+  static int displaced_header_offset_in_bytes()       { return offset_of(BasicLock, _displaced_header); }
+};
+
+// A BasicObjectLock associates a specific Java object with a BasicLock.
+// It is currently embedded in an interpreter frame.
+
+// Because some machines have alignment restrictions on the control stack,
+// the actual space allocated by the interpreter may include padding words
+// after the end of the BasicObjectLock.  Also, in order to guarantee
+// alignment of the embedded BasicLock objects on such machines, we
+// put the embedded BasicLock at the beginning of the struct.
+
+class BasicObjectLock VALUE_OBJ_CLASS_SPEC {
+  friend class VMStructs;
+ private:
+  BasicLock _lock;                                    // the lock, must be double word aligned
+  oop       _obj;                                     // object holds the lock;
+
+ public:
+  // Manipulation
+  oop      obj() const                                { return _obj;  }
+  void set_obj(oop obj)                               { _obj = obj; }
+  BasicLock* lock()                                   { return &_lock; }
+
+  // Note: Use frame::interpreter_frame_monitor_size() for the size of BasicObjectLocks
+  //       in interpreter activation frames since it includes machine-specific padding.
+  static int size()                                   { return sizeof(BasicObjectLock)/wordSize; }
+
+  // GC support
+  void oops_do(OopClosure* f) { f->do_oop(&_obj); }
+
+  static int obj_offset_in_bytes()                    { return offset_of(BasicObjectLock, _obj);  }
+  static int lock_offset_in_bytes()                   { return offset_of(BasicObjectLock, _lock); }
+};
+
diff --git a/hotspot/src/share/vm/runtime/mutex.hpp b/hotspot/src/share/vm/runtime/mutex.hpp
index eb9b5dc8c01..ee0ca717bf8 100644
--- a/hotspot/src/share/vm/runtime/mutex.hpp
+++ b/hotspot/src/share/vm/runtime/mutex.hpp
@@ -265,48 +265,3 @@ class Mutex : public Monitor {      // degenerate Monitor
    }
 };
 
-/*
- * Per-thread blocking support for JSR166. See the Java-level
- * Documentation for rationale. Basically, park acts like wait, unpark
- * like notify.
- *
- * 6271289 --
- * To avoid errors where an os thread expires but the JavaThread still
- * exists, Parkers are immortal (type-stable) and are recycled across
- * new threads.  This parallels the ParkEvent implementation.
- * Because park-unpark allow spurious wakeups it is harmless if an
- * unpark call unparks a new thread using the old Parker reference.
- *
- * In the future we'll want to think about eliminating Parker and using
- * ParkEvent instead.  There's considerable duplication between the two
- * services.
- *
- */
-
-class Parker : public os::PlatformParker {
-private:
-  volatile int _counter ;
-  Parker * FreeNext ;
-  JavaThread * AssociatedWith ; // Current association
-
-public:
-  Parker() : PlatformParker() {
-    _counter       = 0 ;
-    FreeNext       = NULL ;
-    AssociatedWith = NULL ;
-  }
-protected:
-  ~Parker() { ShouldNotReachHere(); }
-public:
-  // For simplicity of interface with Java, all forms of park (indefinite,
-  // relative, and absolute) are multiplexed into one call.
-  void park(bool isAbsolute, jlong time);
-  void unpark();
-
-  // Lifecycle operators
-  static Parker * Allocate (JavaThread * t) ;
-  static void Release (Parker * e) ;
-private:
-  static Parker * volatile FreeList ;
-  static volatile int ListLock ;
-};
diff --git a/hotspot/src/share/vm/runtime/objectMonitor.cpp b/hotspot/src/share/vm/runtime/objectMonitor.cpp
new file mode 100644
index 00000000000..9a8178774fd
--- /dev/null
+++ b/hotspot/src/share/vm/runtime/objectMonitor.cpp
@@ -0,0 +1,2421 @@
+/*
+ * Copyright (c) 1998, 2009, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+# include "incls/_precompiled.incl"
+# include "incls/_objectMonitor.cpp.incl"
+
+#if defined(__GNUC__) && !defined(IA64)
+  // Need to inhibit inlining for older versions of GCC to avoid build-time failures
+  #define ATTR __attribute__((noinline))
+#else
+  #define ATTR
+#endif
+
+
+#ifdef DTRACE_ENABLED
+
+// Only bother with this argument setup if dtrace is available
+// TODO-FIXME: probes should not fire when caller is _blocked.  assert() accordingly.
+
+HS_DTRACE_PROBE_DECL4(hotspot, monitor__notify,
+  jlong, uintptr_t, char*, int);
+HS_DTRACE_PROBE_DECL4(hotspot, monitor__notifyAll,
+  jlong, uintptr_t, char*, int);
+HS_DTRACE_PROBE_DECL4(hotspot, monitor__contended__enter,
+  jlong, uintptr_t, char*, int);
+HS_DTRACE_PROBE_DECL4(hotspot, monitor__contended__entered,
+  jlong, uintptr_t, char*, int);
+HS_DTRACE_PROBE_DECL4(hotspot, monitor__contended__exit,
+  jlong, uintptr_t, char*, int);
+
+#define DTRACE_MONITOR_PROBE_COMMON(klassOop, thread)                      \
+  char* bytes = NULL;                                                      \
+  int len = 0;                                                             \
+  jlong jtid = SharedRuntime::get_java_tid(thread);                        \
+  symbolOop klassname = ((oop)(klassOop))->klass()->klass_part()->name();  \
+  if (klassname != NULL) {                                                 \
+    bytes = (char*)klassname->bytes();                                     \
+    len = klassname->utf8_length();                                        \
+  }
+
+#define DTRACE_MONITOR_WAIT_PROBE(monitor, klassOop, thread, millis)       \
+  {                                                                        \
+    if (DTraceMonitorProbes) {                                            \
+      DTRACE_MONITOR_PROBE_COMMON(klassOop, thread);                       \
+      HS_DTRACE_PROBE5(hotspot, monitor__wait, jtid,                       \
+                       (monitor), bytes, len, (millis));                   \
+    }                                                                      \
+  }
+
+#define DTRACE_MONITOR_PROBE(probe, monitor, klassOop, thread)             \
+  {                                                                        \
+    if (DTraceMonitorProbes) {                                            \
+      DTRACE_MONITOR_PROBE_COMMON(klassOop, thread);                       \
+      HS_DTRACE_PROBE4(hotspot, monitor__##probe, jtid,                    \
+                       (uintptr_t)(monitor), bytes, len);                  \
+    }                                                                      \
+  }
+
+#else //  ndef DTRACE_ENABLED
+
+#define DTRACE_MONITOR_WAIT_PROBE(klassOop, thread, millis, mon)    {;}
+#define DTRACE_MONITOR_PROBE(probe, klassOop, thread, mon)          {;}
+
+#endif // ndef DTRACE_ENABLED
+
+// Tunables ...
+// The knob* variables are effectively final.  Once set they should
+// never be modified hence.  Consider using __read_mostly with GCC.
+
+int ObjectMonitor::Knob_Verbose    = 0 ;
+int ObjectMonitor::Knob_SpinLimit  = 5000 ;    // derived by an external tool -
+static int Knob_LogSpins           = 0 ;       // enable jvmstat tally for spins
+static int Knob_HandOff            = 0 ;
+static int Knob_ReportSettings     = 0 ;
+
+static int Knob_SpinBase           = 0 ;       // Floor AKA SpinMin
+static int Knob_SpinBackOff        = 0 ;       // spin-loop backoff
+static int Knob_CASPenalty         = -1 ;      // Penalty for failed CAS
+static int Knob_OXPenalty          = -1 ;      // Penalty for observed _owner change
+static int Knob_SpinSetSucc        = 1 ;       // spinners set the _succ field
+static int Knob_SpinEarly          = 1 ;
+static int Knob_SuccEnabled        = 1 ;       // futile wake throttling
+static int Knob_SuccRestrict       = 0 ;       // Limit successors + spinners to at-most-one
+static int Knob_MaxSpinners        = -1 ;      // Should be a function of # CPUs
+static int Knob_Bonus              = 100 ;     // spin success bonus
+static int Knob_BonusB             = 100 ;     // spin success bonus
+static int Knob_Penalty            = 200 ;     // spin failure penalty
+static int Knob_Poverty            = 1000 ;
+static int Knob_SpinAfterFutile    = 1 ;       // Spin after returning from park()
+static int Knob_FixedSpin          = 0 ;
+static int Knob_OState             = 3 ;       // Spinner checks thread state of _owner
+static int Knob_UsePause           = 1 ;
+static int Knob_ExitPolicy         = 0 ;
+static int Knob_PreSpin            = 10 ;      // 20-100 likely better
+static int Knob_ResetEvent         = 0 ;
+static int BackOffMask             = 0 ;
+
+static int Knob_FastHSSEC          = 0 ;
+static int Knob_MoveNotifyee       = 2 ;       // notify() - disposition of notifyee
+static int Knob_QMode              = 0 ;       // EntryList-cxq policy - queue discipline
+static volatile int InitDone       = 0 ;
+
+#define TrySpin TrySpin_VaryDuration
+
+// -----------------------------------------------------------------------------
+// Theory of operations -- Monitors lists, thread residency, etc:
+//
+// * A thread acquires ownership of a monitor by successfully
+//   CAS()ing the _owner field from null to non-null.
+//
+// * Invariant: A thread appears on at most one monitor list --
+//   cxq, EntryList or WaitSet -- at any one time.
+//
+// * Contending threads "push" themselves onto the cxq with CAS
+//   and then spin/park.
+//
+// * After a contending thread eventually acquires the lock it must
+//   dequeue itself from either the EntryList or the cxq.
+//
+// * The exiting thread identifies and unparks an "heir presumptive"
+//   tentative successor thread on the EntryList.  Critically, the
+//   exiting thread doesn't unlink the successor thread from the EntryList.
+//   After having been unparked, the wakee will recontend for ownership of
+//   the monitor.   The successor (wakee) will either acquire the lock or
+//   re-park itself.
+//
+//   Succession is provided for by a policy of competitive handoff.
+//   The exiting thread does _not_ grant or pass ownership to the
+//   successor thread.  (This is also referred to as "handoff" succession").
+//   Instead the exiting thread releases ownership and possibly wakes
+//   a successor, so the successor can (re)compete for ownership of the lock.
+//   If the EntryList is empty but the cxq is populated the exiting
+//   thread will drain the cxq into the EntryList.  It does so by
+//   by detaching the cxq (installing null with CAS) and folding
+//   the threads from the cxq into the EntryList.  The EntryList is
+//   doubly linked, while the cxq is singly linked because of the
+//   CAS-based "push" used to enqueue recently arrived threads (RATs).
+//
+// * Concurrency invariants:
+//
+//   -- only the monitor owner may access or mutate the EntryList.
+//      The mutex property of the monitor itself protects the EntryList
+//      from concurrent interference.
+//   -- Only the monitor owner may detach the cxq.
+//
+// * The monitor entry list operations avoid locks, but strictly speaking
+//   they're not lock-free.  Enter is lock-free, exit is not.
+//   See http://j2se.east/~dice/PERSIST/040825-LockFreeQueues.html
+//
+// * The cxq can have multiple concurrent "pushers" but only one concurrent
+//   detaching thread.  This mechanism is immune from the ABA corruption.
+//   More precisely, the CAS-based "push" onto cxq is ABA-oblivious.
+//
+// * Taken together, the cxq and the EntryList constitute or form a
+//   single logical queue of threads stalled trying to acquire the lock.
+//   We use two distinct lists to improve the odds of a constant-time
+//   dequeue operation after acquisition (in the ::enter() epilog) and
+//   to reduce heat on the list ends.  (c.f. Michael Scott's "2Q" algorithm).
+//   A key desideratum is to minimize queue & monitor metadata manipulation
+//   that occurs while holding the monitor lock -- that is, we want to
+//   minimize monitor lock holds times.  Note that even a small amount of
+//   fixed spinning will greatly reduce the # of enqueue-dequeue operations
+//   on EntryList|cxq.  That is, spinning relieves contention on the "inner"
+//   locks and monitor metadata.
+//
+//   Cxq points to the the set of Recently Arrived Threads attempting entry.
+//   Because we push threads onto _cxq with CAS, the RATs must take the form of
+//   a singly-linked LIFO.  We drain _cxq into EntryList  at unlock-time when
+//   the unlocking thread notices that EntryList is null but _cxq is != null.
+//
+//   The EntryList is ordered by the prevailing queue discipline and
+//   can be organized in any convenient fashion, such as a doubly-linked list or
+//   a circular doubly-linked list.  Critically, we want insert and delete operations
+//   to operate in constant-time.  If we need a priority queue then something akin
+//   to Solaris' sleepq would work nicely.  Viz.,
+//   http://agg.eng/ws/on10_nightly/source/usr/src/uts/common/os/sleepq.c.
+//   Queue discipline is enforced at ::exit() time, when the unlocking thread
+//   drains the cxq into the EntryList, and orders or reorders the threads on the
+//   EntryList accordingly.
+//
+//   Barring "lock barging", this mechanism provides fair cyclic ordering,
+//   somewhat similar to an elevator-scan.
+//
+// * The monitor synchronization subsystem avoids the use of native
+//   synchronization primitives except for the narrow platform-specific
+//   park-unpark abstraction.  See the comments in os_solaris.cpp regarding
+//   the semantics of park-unpark.  Put another way, this monitor implementation
+//   depends only on atomic operations and park-unpark.  The monitor subsystem
+//   manages all RUNNING->BLOCKED and BLOCKED->READY transitions while the
+//   underlying OS manages the READY<->RUN transitions.
+//
+// * Waiting threads reside on the WaitSet list -- wait() puts
+//   the caller onto the WaitSet.
+//
+// * notify() or notifyAll() simply transfers threads from the WaitSet to
+//   either the EntryList or cxq.  Subsequent exit() operations will
+//   unpark the notifyee.  Unparking a notifee in notify() is inefficient -
+//   it's likely the notifyee would simply impale itself on the lock held
+//   by the notifier.
+//
+// * An interesting alternative is to encode cxq as (List,LockByte) where
+//   the LockByte is 0 iff the monitor is owned.  _owner is simply an auxiliary
+//   variable, like _recursions, in the scheme.  The threads or Events that form
+//   the list would have to be aligned in 256-byte addresses.  A thread would
+//   try to acquire the lock or enqueue itself with CAS, but exiting threads
+//   could use a 1-0 protocol and simply STB to set the LockByte to 0.
+//   Note that is is *not* word-tearing, but it does presume that full-word
+//   CAS operations are coherent with intermix with STB operations.  That's true
+//   on most common processors.
+//
+// * See also http://blogs.sun.com/dave
+
+
+// -----------------------------------------------------------------------------
+// Enter support
+
+bool ObjectMonitor::try_enter(Thread* THREAD) {
+  if (THREAD != _owner) {
+    if (THREAD->is_lock_owned ((address)_owner)) {
+       assert(_recursions == 0, "internal state error");
+       _owner = THREAD ;
+       _recursions = 1 ;
+       OwnerIsThread = 1 ;
+       return true;
+    }
+    if (Atomic::cmpxchg_ptr (THREAD, &_owner, NULL) != NULL) {
+      return false;
+    }
+    return true;
+  } else {
+    _recursions++;
+    return true;
+  }
+}
+
+void ATTR ObjectMonitor::enter(TRAPS) {
+  // The following code is ordered to check the most common cases first
+  // and to reduce RTS->RTO cache line upgrades on SPARC and IA32 processors.
+  Thread * const Self = THREAD ;
+  void * cur ;
+
+  cur = Atomic::cmpxchg_ptr (Self, &_owner, NULL) ;
+  if (cur == NULL) {
+     // Either ASSERT _recursions == 0 or explicitly set _recursions = 0.
+     assert (_recursions == 0   , "invariant") ;
+     assert (_owner      == Self, "invariant") ;
+     // CONSIDER: set or assert OwnerIsThread == 1
+     return ;
+  }
+
+  if (cur == Self) {
+     // TODO-FIXME: check for integer overflow!  BUGID 6557169.
+     _recursions ++ ;
+     return ;
+  }
+
+  if (Self->is_lock_owned ((address)cur)) {
+    assert (_recursions == 0, "internal state error");
+    _recursions = 1 ;
+    // Commute owner from a thread-specific on-stack BasicLockObject address to
+    // a full-fledged "Thread *".
+    _owner = Self ;
+    OwnerIsThread = 1 ;
+    return ;
+  }
+
+  // We've encountered genuine contention.
+  assert (Self->_Stalled == 0, "invariant") ;
+  Self->_Stalled = intptr_t(this) ;
+
+  // Try one round of spinning *before* enqueueing Self
+  // and before going through the awkward and expensive state
+  // transitions.  The following spin is strictly optional ...
+  // Note that if we acquire the monitor from an initial spin
+  // we forgo posting JVMTI events and firing DTRACE probes.
+  if (Knob_SpinEarly && TrySpin (Self) > 0) {
+     assert (_owner == Self      , "invariant") ;
+     assert (_recursions == 0    , "invariant") ;
+     assert (((oop)(object()))->mark() == markOopDesc::encode(this), "invariant") ;
+     Self->_Stalled = 0 ;
+     return ;
+  }
+
+  assert (_owner != Self          , "invariant") ;
+  assert (_succ  != Self          , "invariant") ;
+  assert (Self->is_Java_thread()  , "invariant") ;
+  JavaThread * jt = (JavaThread *) Self ;
+  assert (!SafepointSynchronize::is_at_safepoint(), "invariant") ;
+  assert (jt->thread_state() != _thread_blocked   , "invariant") ;
+  assert (this->object() != NULL  , "invariant") ;
+  assert (_count >= 0, "invariant") ;
+
+  // Prevent deflation at STW-time.  See deflate_idle_monitors() and is_busy().
+  // Ensure the object-monitor relationship remains stable while there's contention.
+  Atomic::inc_ptr(&_count);
+
+  { // Change java thread status to indicate blocked on monitor enter.
+    JavaThreadBlockedOnMonitorEnterState jtbmes(jt, this);
+
+    DTRACE_MONITOR_PROBE(contended__enter, this, object(), jt);
+    if (JvmtiExport::should_post_monitor_contended_enter()) {
+      JvmtiExport::post_monitor_contended_enter(jt, this);
+    }
+
+    OSThreadContendState osts(Self->osthread());
+    ThreadBlockInVM tbivm(jt);
+
+    Self->set_current_pending_monitor(this);
+
+    // TODO-FIXME: change the following for(;;) loop to straight-line code.
+    for (;;) {
+      jt->set_suspend_equivalent();
+      // cleared by handle_special_suspend_equivalent_condition()
+      // or java_suspend_self()
+
+      EnterI (THREAD) ;
+
+      if (!ExitSuspendEquivalent(jt)) break ;
+
+      //
+      // We have acquired the contended monitor, but while we were
+      // waiting another thread suspended us. We don't want to enter
+      // the monitor while suspended because that would surprise the
+      // thread that suspended us.
+      //
+          _recursions = 0 ;
+      _succ = NULL ;
+      exit (Self) ;
+
+      jt->java_suspend_self();
+    }
+    Self->set_current_pending_monitor(NULL);
+  }
+
+  Atomic::dec_ptr(&_count);
+  assert (_count >= 0, "invariant") ;
+  Self->_Stalled = 0 ;
+
+  // Must either set _recursions = 0 or ASSERT _recursions == 0.
+  assert (_recursions == 0     , "invariant") ;
+  assert (_owner == Self       , "invariant") ;
+  assert (_succ  != Self       , "invariant") ;
+  assert (((oop)(object()))->mark() == markOopDesc::encode(this), "invariant") ;
+
+  // The thread -- now the owner -- is back in vm mode.
+  // Report the glorious news via TI,DTrace and jvmstat.
+  // The probe effect is non-trivial.  All the reportage occurs
+  // while we hold the monitor, increasing the length of the critical
+  // section.  Amdahl's parallel speedup law comes vividly into play.
+  //
+  // Another option might be to aggregate the events (thread local or
+  // per-monitor aggregation) and defer reporting until a more opportune
+  // time -- such as next time some thread encounters contention but has
+  // yet to acquire the lock.  While spinning that thread could
+  // spinning we could increment JVMStat counters, etc.
+
+  DTRACE_MONITOR_PROBE(contended__entered, this, object(), jt);
+  if (JvmtiExport::should_post_monitor_contended_entered()) {
+    JvmtiExport::post_monitor_contended_entered(jt, this);
+  }
+  if (ObjectMonitor::_sync_ContendedLockAttempts != NULL) {
+     ObjectMonitor::_sync_ContendedLockAttempts->inc() ;
+  }
+}
+
+
+// Caveat: TryLock() is not necessarily serializing if it returns failure.
+// Callers must compensate as needed.
+
+int ObjectMonitor::TryLock (Thread * Self) {
+   for (;;) {
+      void * own = _owner ;
+      if (own != NULL) return 0 ;
+      if (Atomic::cmpxchg_ptr (Self, &_owner, NULL) == NULL) {
+         // Either guarantee _recursions == 0 or set _recursions = 0.
+         assert (_recursions == 0, "invariant") ;
+         assert (_owner == Self, "invariant") ;
+         // CONSIDER: set or assert that OwnerIsThread == 1
+         return 1 ;
+      }
+      // The lock had been free momentarily, but we lost the race to the lock.
+      // Interference -- the CAS failed.
+      // We can either return -1 or retry.
+      // Retry doesn't make as much sense because the lock was just acquired.
+      if (true) return -1 ;
+   }
+}
+
+void ATTR ObjectMonitor::EnterI (TRAPS) {
+    Thread * Self = THREAD ;
+    assert (Self->is_Java_thread(), "invariant") ;
+    assert (((JavaThread *) Self)->thread_state() == _thread_blocked   , "invariant") ;
+
+    // Try the lock - TATAS
+    if (TryLock (Self) > 0) {
+        assert (_succ != Self              , "invariant") ;
+        assert (_owner == Self             , "invariant") ;
+        assert (_Responsible != Self       , "invariant") ;
+        return ;
+    }
+
+    DeferredInitialize () ;
+
+    // We try one round of spinning *before* enqueueing Self.
+    //
+    // If the _owner is ready but OFFPROC we could use a YieldTo()
+    // operation to donate the remainder of this thread's quantum
+    // to the owner.  This has subtle but beneficial affinity
+    // effects.
+
+    if (TrySpin (Self) > 0) {
+        assert (_owner == Self        , "invariant") ;
+        assert (_succ != Self         , "invariant") ;
+        assert (_Responsible != Self  , "invariant") ;
+        return ;
+    }
+
+    // The Spin failed -- Enqueue and park the thread ...
+    assert (_succ  != Self            , "invariant") ;
+    assert (_owner != Self            , "invariant") ;
+    assert (_Responsible != Self      , "invariant") ;
+
+    // Enqueue "Self" on ObjectMonitor's _cxq.
+    //
+    // Node acts as a proxy for Self.
+    // As an aside, if were to ever rewrite the synchronization code mostly
+    // in Java, WaitNodes, ObjectMonitors, and Events would become 1st-class
+    // Java objects.  This would avoid awkward lifecycle and liveness issues,
+    // as well as eliminate a subset of ABA issues.
+    // TODO: eliminate ObjectWaiter and enqueue either Threads or Events.
+    //
+
+    ObjectWaiter node(Self) ;
+    Self->_ParkEvent->reset() ;
+    node._prev   = (ObjectWaiter *) 0xBAD ;
+    node.TState  = ObjectWaiter::TS_CXQ ;
+
+    // Push "Self" onto the front of the _cxq.
+    // Once on cxq/EntryList, Self stays on-queue until it acquires the lock.
+    // Note that spinning tends to reduce the rate at which threads
+    // enqueue and dequeue on EntryList|cxq.
+    ObjectWaiter * nxt ;
+    for (;;) {
+        node._next = nxt = _cxq ;
+        if (Atomic::cmpxchg_ptr (&node, &_cxq, nxt) == nxt) break ;
+
+        // Interference - the CAS failed because _cxq changed.  Just retry.
+        // As an optional optimization we retry the lock.
+        if (TryLock (Self) > 0) {
+            assert (_succ != Self         , "invariant") ;
+            assert (_owner == Self        , "invariant") ;
+            assert (_Responsible != Self  , "invariant") ;
+            return ;
+        }
+    }
+
+    // Check for cxq|EntryList edge transition to non-null.  This indicates
+    // the onset of contention.  While contention persists exiting threads
+    // will use a ST:MEMBAR:LD 1-1 exit protocol.  When contention abates exit
+    // operations revert to the faster 1-0 mode.  This enter operation may interleave
+    // (race) a concurrent 1-0 exit operation, resulting in stranding, so we
+    // arrange for one of the contending thread to use a timed park() operations
+    // to detect and recover from the race.  (Stranding is form of progress failure
+    // where the monitor is unlocked but all the contending threads remain parked).
+    // That is, at least one of the contended threads will periodically poll _owner.
+    // One of the contending threads will become the designated "Responsible" thread.
+    // The Responsible thread uses a timed park instead of a normal indefinite park
+    // operation -- it periodically wakes and checks for and recovers from potential
+    // strandings admitted by 1-0 exit operations.   We need at most one Responsible
+    // thread per-monitor at any given moment.  Only threads on cxq|EntryList may
+    // be responsible for a monitor.
+    //
+    // Currently, one of the contended threads takes on the added role of "Responsible".
+    // A viable alternative would be to use a dedicated "stranding checker" thread
+    // that periodically iterated over all the threads (or active monitors) and unparked
+    // successors where there was risk of stranding.  This would help eliminate the
+    // timer scalability issues we see on some platforms as we'd only have one thread
+    // -- the checker -- parked on a timer.
+
+    if ((SyncFlags & 16) == 0 && nxt == NULL && _EntryList == NULL) {
+        // Try to assume the role of responsible thread for the monitor.
+        // CONSIDER:  ST vs CAS vs { if (Responsible==null) Responsible=Self }
+        Atomic::cmpxchg_ptr (Self, &_Responsible, NULL) ;
+    }
+
+    // The lock have been released while this thread was occupied queueing
+    // itself onto _cxq.  To close the race and avoid "stranding" and
+    // progress-liveness failure we must resample-retry _owner before parking.
+    // Note the Dekker/Lamport duality: ST cxq; MEMBAR; LD Owner.
+    // In this case the ST-MEMBAR is accomplished with CAS().
+    //
+    // TODO: Defer all thread state transitions until park-time.
+    // Since state transitions are heavy and inefficient we'd like
+    // to defer the state transitions until absolutely necessary,
+    // and in doing so avoid some transitions ...
+
+    TEVENT (Inflated enter - Contention) ;
+    int nWakeups = 0 ;
+    int RecheckInterval = 1 ;
+
+    for (;;) {
+
+        if (TryLock (Self) > 0) break ;
+        assert (_owner != Self, "invariant") ;
+
+        if ((SyncFlags & 2) && _Responsible == NULL) {
+           Atomic::cmpxchg_ptr (Self, &_Responsible, NULL) ;
+        }
+
+        // park self
+        if (_Responsible == Self || (SyncFlags & 1)) {
+            TEVENT (Inflated enter - park TIMED) ;
+            Self->_ParkEvent->park ((jlong) RecheckInterval) ;
+            // Increase the RecheckInterval, but clamp the value.
+            RecheckInterval *= 8 ;
+            if (RecheckInterval > 1000) RecheckInterval = 1000 ;
+        } else {
+            TEVENT (Inflated enter - park UNTIMED) ;
+            Self->_ParkEvent->park() ;
+        }
+
+        if (TryLock(Self) > 0) break ;
+
+        // The lock is still contested.
+        // Keep a tally of the # of futile wakeups.
+        // Note that the counter is not protected by a lock or updated by atomics.
+        // That is by design - we trade "lossy" counters which are exposed to
+        // races during updates for a lower probe effect.
+        TEVENT (Inflated enter - Futile wakeup) ;
+        if (ObjectMonitor::_sync_FutileWakeups != NULL) {
+           ObjectMonitor::_sync_FutileWakeups->inc() ;
+        }
+        ++ nWakeups ;
+
+        // Assuming this is not a spurious wakeup we'll normally find _succ == Self.
+        // We can defer clearing _succ until after the spin completes
+        // TrySpin() must tolerate being called with _succ == Self.
+        // Try yet another round of adaptive spinning.
+        if ((Knob_SpinAfterFutile & 1) && TrySpin (Self) > 0) break ;
+
+        // We can find that we were unpark()ed and redesignated _succ while
+        // we were spinning.  That's harmless.  If we iterate and call park(),
+        // park() will consume the event and return immediately and we'll
+        // just spin again.  This pattern can repeat, leaving _succ to simply
+        // spin on a CPU.  Enable Knob_ResetEvent to clear pending unparks().
+        // Alternately, we can sample fired() here, and if set, forgo spinning
+        // in the next iteration.
+
+        if ((Knob_ResetEvent & 1) && Self->_ParkEvent->fired()) {
+           Self->_ParkEvent->reset() ;
+           OrderAccess::fence() ;
+        }
+        if (_succ == Self) _succ = NULL ;
+
+        // Invariant: after clearing _succ a thread *must* retry _owner before parking.
+        OrderAccess::fence() ;
+    }
+
+    // Egress :
+    // Self has acquired the lock -- Unlink Self from the cxq or EntryList.
+    // Normally we'll find Self on the EntryList .
+    // From the perspective of the lock owner (this thread), the
+    // EntryList is stable and cxq is prepend-only.
+    // The head of cxq is volatile but the interior is stable.
+    // In addition, Self.TState is stable.
+
+    assert (_owner == Self      , "invariant") ;
+    assert (object() != NULL    , "invariant") ;
+    // I'd like to write:
+    //   guarantee (((oop)(object()))->mark() == markOopDesc::encode(this), "invariant") ;
+    // but as we're at a safepoint that's not safe.
+
+    UnlinkAfterAcquire (Self, &node) ;
+    if (_succ == Self) _succ = NULL ;
+
+    assert (_succ != Self, "invariant") ;
+    if (_Responsible == Self) {
+        _Responsible = NULL ;
+        // Dekker pivot-point.
+        // Consider OrderAccess::storeload() here
+
+        // We may leave threads on cxq|EntryList without a designated
+        // "Responsible" thread.  This is benign.  When this thread subsequently
+        // exits the monitor it can "see" such preexisting "old" threads --
+        // threads that arrived on the cxq|EntryList before the fence, above --
+        // by LDing cxq|EntryList.  Newly arrived threads -- that is, threads
+        // that arrive on cxq after the ST:MEMBAR, above -- will set Responsible
+        // non-null and elect a new "Responsible" timer thread.
+        //
+        // This thread executes:
+        //    ST Responsible=null; MEMBAR    (in enter epilog - here)
+        //    LD cxq|EntryList               (in subsequent exit)
+        //
+        // Entering threads in the slow/contended path execute:
+        //    ST cxq=nonnull; MEMBAR; LD Responsible (in enter prolog)
+        //    The (ST cxq; MEMBAR) is accomplished with CAS().
+        //
+        // The MEMBAR, above, prevents the LD of cxq|EntryList in the subsequent
+        // exit operation from floating above the ST Responsible=null.
+        //
+        // In *practice* however, EnterI() is always followed by some atomic
+        // operation such as the decrement of _count in ::enter().  Those atomics
+        // obviate the need for the explicit MEMBAR, above.
+    }
+
+    // We've acquired ownership with CAS().
+    // CAS is serializing -- it has MEMBAR/FENCE-equivalent semantics.
+    // But since the CAS() this thread may have also stored into _succ,
+    // EntryList, cxq or Responsible.  These meta-data updates must be
+    // visible __before this thread subsequently drops the lock.
+    // Consider what could occur if we didn't enforce this constraint --
+    // STs to monitor meta-data and user-data could reorder with (become
+    // visible after) the ST in exit that drops ownership of the lock.
+    // Some other thread could then acquire the lock, but observe inconsistent
+    // or old monitor meta-data and heap data.  That violates the JMM.
+    // To that end, the 1-0 exit() operation must have at least STST|LDST
+    // "release" barrier semantics.  Specifically, there must be at least a
+    // STST|LDST barrier in exit() before the ST of null into _owner that drops
+    // the lock.   The barrier ensures that changes to monitor meta-data and data
+    // protected by the lock will be visible before we release the lock, and
+    // therefore before some other thread (CPU) has a chance to acquire the lock.
+    // See also: http://gee.cs.oswego.edu/dl/jmm/cookbook.html.
+    //
+    // Critically, any prior STs to _succ or EntryList must be visible before
+    // the ST of null into _owner in the *subsequent* (following) corresponding
+    // monitorexit.  Recall too, that in 1-0 mode monitorexit does not necessarily
+    // execute a serializing instruction.
+
+    if (SyncFlags & 8) {
+       OrderAccess::fence() ;
+    }
+    return ;
+}
+
+// ReenterI() is a specialized inline form of the latter half of the
+// contended slow-path from EnterI().  We use ReenterI() only for
+// monitor reentry in wait().
+//
+// In the future we should reconcile EnterI() and ReenterI(), adding
+// Knob_Reset and Knob_SpinAfterFutile support and restructuring the
+// loop accordingly.
+
+void ATTR ObjectMonitor::ReenterI (Thread * Self, ObjectWaiter * SelfNode) {
+    assert (Self != NULL                , "invariant") ;
+    assert (SelfNode != NULL            , "invariant") ;
+    assert (SelfNode->_thread == Self   , "invariant") ;
+    assert (_waiters > 0                , "invariant") ;
+    assert (((oop)(object()))->mark() == markOopDesc::encode(this) , "invariant") ;
+    assert (((JavaThread *)Self)->thread_state() != _thread_blocked, "invariant") ;
+    JavaThread * jt = (JavaThread *) Self ;
+
+    int nWakeups = 0 ;
+    for (;;) {
+        ObjectWaiter::TStates v = SelfNode->TState ;
+        guarantee (v == ObjectWaiter::TS_ENTER || v == ObjectWaiter::TS_CXQ, "invariant") ;
+        assert    (_owner != Self, "invariant") ;
+
+        if (TryLock (Self) > 0) break ;
+        if (TrySpin (Self) > 0) break ;
+
+        TEVENT (Wait Reentry - parking) ;
+
+        // State transition wrappers around park() ...
+        // ReenterI() wisely defers state transitions until
+        // it's clear we must park the thread.
+        {
+           OSThreadContendState osts(Self->osthread());
+           ThreadBlockInVM tbivm(jt);
+
+           // cleared by handle_special_suspend_equivalent_condition()
+           // or java_suspend_self()
+           jt->set_suspend_equivalent();
+           if (SyncFlags & 1) {
+              Self->_ParkEvent->park ((jlong)1000) ;
+           } else {
+              Self->_ParkEvent->park () ;
+           }
+
+           // were we externally suspended while we were waiting?
+           for (;;) {
+              if (!ExitSuspendEquivalent (jt)) break ;
+              if (_succ == Self) { _succ = NULL; OrderAccess::fence(); }
+              jt->java_suspend_self();
+              jt->set_suspend_equivalent();
+           }
+        }
+
+        // Try again, but just so we distinguish between futile wakeups and
+        // successful wakeups.  The following test isn't algorithmically
+        // necessary, but it helps us maintain sensible statistics.
+        if (TryLock(Self) > 0) break ;
+
+        // The lock is still contested.
+        // Keep a tally of the # of futile wakeups.
+        // Note that the counter is not protected by a lock or updated by atomics.
+        // That is by design - we trade "lossy" counters which are exposed to
+        // races during updates for a lower probe effect.
+        TEVENT (Wait Reentry - futile wakeup) ;
+        ++ nWakeups ;
+
+        // Assuming this is not a spurious wakeup we'll normally
+        // find that _succ == Self.
+        if (_succ == Self) _succ = NULL ;
+
+        // Invariant: after clearing _succ a contending thread
+        // *must* retry  _owner before parking.
+        OrderAccess::fence() ;
+
+        if (ObjectMonitor::_sync_FutileWakeups != NULL) {
+          ObjectMonitor::_sync_FutileWakeups->inc() ;
+        }
+    }
+
+    // Self has acquired the lock -- Unlink Self from the cxq or EntryList .
+    // Normally we'll find Self on the EntryList.
+    // Unlinking from the EntryList is constant-time and atomic-free.
+    // From the perspective of the lock owner (this thread), the
+    // EntryList is stable and cxq is prepend-only.
+    // The head of cxq is volatile but the interior is stable.
+    // In addition, Self.TState is stable.
+
+    assert (_owner == Self, "invariant") ;
+    assert (((oop)(object()))->mark() == markOopDesc::encode(this), "invariant") ;
+    UnlinkAfterAcquire (Self, SelfNode) ;
+    if (_succ == Self) _succ = NULL ;
+    assert (_succ != Self, "invariant") ;
+    SelfNode->TState = ObjectWaiter::TS_RUN ;
+    OrderAccess::fence() ;      // see comments at the end of EnterI()
+}
+
+// after the thread acquires the lock in ::enter().  Equally, we could defer
+// unlinking the thread until ::exit()-time.
+
+void ObjectMonitor::UnlinkAfterAcquire (Thread * Self, ObjectWaiter * SelfNode)
+{
+    assert (_owner == Self, "invariant") ;
+    assert (SelfNode->_thread == Self, "invariant") ;
+
+    if (SelfNode->TState == ObjectWaiter::TS_ENTER) {
+        // Normal case: remove Self from the DLL EntryList .
+        // This is a constant-time operation.
+        ObjectWaiter * nxt = SelfNode->_next ;
+        ObjectWaiter * prv = SelfNode->_prev ;
+        if (nxt != NULL) nxt->_prev = prv ;
+        if (prv != NULL) prv->_next = nxt ;
+        if (SelfNode == _EntryList ) _EntryList = nxt ;
+        assert (nxt == NULL || nxt->TState == ObjectWaiter::TS_ENTER, "invariant") ;
+        assert (prv == NULL || prv->TState == ObjectWaiter::TS_ENTER, "invariant") ;
+        TEVENT (Unlink from EntryList) ;
+    } else {
+        guarantee (SelfNode->TState == ObjectWaiter::TS_CXQ, "invariant") ;
+        // Inopportune interleaving -- Self is still on the cxq.
+        // This usually means the enqueue of self raced an exiting thread.
+        // Normally we'll find Self near the front of the cxq, so
+        // dequeueing is typically fast.  If needbe we can accelerate
+        // this with some MCS/CHL-like bidirectional list hints and advisory
+        // back-links so dequeueing from the interior will normally operate
+        // in constant-time.
+        // Dequeue Self from either the head (with CAS) or from the interior
+        // with a linear-time scan and normal non-atomic memory operations.
+        // CONSIDER: if Self is on the cxq then simply drain cxq into EntryList
+        // and then unlink Self from EntryList.  We have to drain eventually,
+        // so it might as well be now.
+
+        ObjectWaiter * v = _cxq ;
+        assert (v != NULL, "invariant") ;
+        if (v != SelfNode || Atomic::cmpxchg_ptr (SelfNode->_next, &_cxq, v) != v) {
+            // The CAS above can fail from interference IFF a "RAT" arrived.
+            // In that case Self must be in the interior and can no longer be
+            // at the head of cxq.
+            if (v == SelfNode) {
+                assert (_cxq != v, "invariant") ;
+                v = _cxq ;          // CAS above failed - start scan at head of list
+            }
+            ObjectWaiter * p ;
+            ObjectWaiter * q = NULL ;
+            for (p = v ; p != NULL && p != SelfNode; p = p->_next) {
+                q = p ;
+                assert (p->TState == ObjectWaiter::TS_CXQ, "invariant") ;
+            }
+            assert (v != SelfNode,  "invariant") ;
+            assert (p == SelfNode,  "Node not found on cxq") ;
+            assert (p != _cxq,      "invariant") ;
+            assert (q != NULL,      "invariant") ;
+            assert (q->_next == p,  "invariant") ;
+            q->_next = p->_next ;
+        }
+        TEVENT (Unlink from cxq) ;
+    }
+
+    // Diagnostic hygiene ...
+    SelfNode->_prev  = (ObjectWaiter *) 0xBAD ;
+    SelfNode->_next  = (ObjectWaiter *) 0xBAD ;
+    SelfNode->TState = ObjectWaiter::TS_RUN ;
+}
+
+// -----------------------------------------------------------------------------
+// Exit support
+//
+// exit()
+// ~~~~~~
+// Note that the collector can't reclaim the objectMonitor or deflate
+// the object out from underneath the thread calling ::exit() as the
+// thread calling ::exit() never transitions to a stable state.
+// This inhibits GC, which in turn inhibits asynchronous (and
+// inopportune) reclamation of "this".
+//
+// We'd like to assert that: (THREAD->thread_state() != _thread_blocked) ;
+// There's one exception to the claim above, however.  EnterI() can call
+// exit() to drop a lock if the acquirer has been externally suspended.
+// In that case exit() is called with _thread_state as _thread_blocked,
+// but the monitor's _count field is > 0, which inhibits reclamation.
+//
+// 1-0 exit
+// ~~~~~~~~
+// ::exit() uses a canonical 1-1 idiom with a MEMBAR although some of
+// the fast-path operators have been optimized so the common ::exit()
+// operation is 1-0.  See i486.ad fast_unlock(), for instance.
+// The code emitted by fast_unlock() elides the usual MEMBAR.  This
+// greatly improves latency -- MEMBAR and CAS having considerable local
+// latency on modern processors -- but at the cost of "stranding".  Absent the
+// MEMBAR, a thread in fast_unlock() can race a thread in the slow
+// ::enter() path, resulting in the entering thread being stranding
+// and a progress-liveness failure.   Stranding is extremely rare.
+// We use timers (timed park operations) & periodic polling to detect
+// and recover from stranding.  Potentially stranded threads periodically
+// wake up and poll the lock.  See the usage of the _Responsible variable.
+//
+// The CAS() in enter provides for safety and exclusion, while the CAS or
+// MEMBAR in exit provides for progress and avoids stranding.  1-0 locking
+// eliminates the CAS/MEMBAR from the exist path, but it admits stranding.
+// We detect and recover from stranding with timers.
+//
+// If a thread transiently strands it'll park until (a) another
+// thread acquires the lock and then drops the lock, at which time the
+// exiting thread will notice and unpark the stranded thread, or, (b)
+// the timer expires.  If the lock is high traffic then the stranding latency
+// will be low due to (a).  If the lock is low traffic then the odds of
+// stranding are lower, although the worst-case stranding latency
+// is longer.  Critically, we don't want to put excessive load in the
+// platform's timer subsystem.  We want to minimize both the timer injection
+// rate (timers created/sec) as well as the number of timers active at
+// any one time.  (more precisely, we want to minimize timer-seconds, which is
+// the integral of the # of active timers at any instant over time).
+// Both impinge on OS scalability.  Given that, at most one thread parked on
+// a monitor will use a timer.
+
+void ATTR ObjectMonitor::exit(TRAPS) {
+   Thread * Self = THREAD ;
+   if (THREAD != _owner) {
+     if (THREAD->is_lock_owned((address) _owner)) {
+       // Transmute _owner from a BasicLock pointer to a Thread address.
+       // We don't need to hold _mutex for this transition.
+       // Non-null to Non-null is safe as long as all readers can
+       // tolerate either flavor.
+       assert (_recursions == 0, "invariant") ;
+       _owner = THREAD ;
+       _recursions = 0 ;
+       OwnerIsThread = 1 ;
+     } else {
+       // NOTE: we need to handle unbalanced monitor enter/exit
+       // in native code by throwing an exception.
+       // TODO: Throw an IllegalMonitorStateException ?
+       TEVENT (Exit - Throw IMSX) ;
+       assert(false, "Non-balanced monitor enter/exit!");
+       if (false) {
+          THROW(vmSymbols::java_lang_IllegalMonitorStateException());
+       }
+       return;
+     }
+   }
+
+   if (_recursions != 0) {
+     _recursions--;        // this is simple recursive enter
+     TEVENT (Inflated exit - recursive) ;
+     return ;
+   }
+
+   // Invariant: after setting Responsible=null an thread must execute
+   // a MEMBAR or other serializing instruction before fetching EntryList|cxq.
+   if ((SyncFlags & 4) == 0) {
+      _Responsible = NULL ;
+   }
+
+   for (;;) {
+      assert (THREAD == _owner, "invariant") ;
+
+
+      if (Knob_ExitPolicy == 0) {
+         // release semantics: prior loads and stores from within the critical section
+         // must not float (reorder) past the following store that drops the lock.
+         // On SPARC that requires MEMBAR #loadstore|#storestore.
+         // But of course in TSO #loadstore|#storestore is not required.
+         // I'd like to write one of the following:
+         // A.  OrderAccess::release() ; _owner = NULL
+         // B.  OrderAccess::loadstore(); OrderAccess::storestore(); _owner = NULL;
+         // Unfortunately OrderAccess::release() and OrderAccess::loadstore() both
+         // store into a _dummy variable.  That store is not needed, but can result
+         // in massive wasteful coherency traffic on classic SMP systems.
+         // Instead, I use release_store(), which is implemented as just a simple
+         // ST on x64, x86 and SPARC.
+         OrderAccess::release_store_ptr (&_owner, NULL) ;   // drop the lock
+         OrderAccess::storeload() ;                         // See if we need to wake a successor
+         if ((intptr_t(_EntryList)|intptr_t(_cxq)) == 0 || _succ != NULL) {
+            TEVENT (Inflated exit - simple egress) ;
+            return ;
+         }
+         TEVENT (Inflated exit - complex egress) ;
+
+         // Normally the exiting thread is responsible for ensuring succession,
+         // but if other successors are ready or other entering threads are spinning
+         // then this thread can simply store NULL into _owner and exit without
+         // waking a successor.  The existence of spinners or ready successors
+         // guarantees proper succession (liveness).  Responsibility passes to the
+         // ready or running successors.  The exiting thread delegates the duty.
+         // More precisely, if a successor already exists this thread is absolved
+         // of the responsibility of waking (unparking) one.
+         //
+         // The _succ variable is critical to reducing futile wakeup frequency.
+         // _succ identifies the "heir presumptive" thread that has been made
+         // ready (unparked) but that has not yet run.  We need only one such
+         // successor thread to guarantee progress.
+         // See http://www.usenix.org/events/jvm01/full_papers/dice/dice.pdf
+         // section 3.3 "Futile Wakeup Throttling" for details.
+         //
+         // Note that spinners in Enter() also set _succ non-null.
+         // In the current implementation spinners opportunistically set
+         // _succ so that exiting threads might avoid waking a successor.
+         // Another less appealing alternative would be for the exiting thread
+         // to drop the lock and then spin briefly to see if a spinner managed
+         // to acquire the lock.  If so, the exiting thread could exit
+         // immediately without waking a successor, otherwise the exiting
+         // thread would need to dequeue and wake a successor.
+         // (Note that we'd need to make the post-drop spin short, but no
+         // shorter than the worst-case round-trip cache-line migration time.
+         // The dropped lock needs to become visible to the spinner, and then
+         // the acquisition of the lock by the spinner must become visible to
+         // the exiting thread).
+         //
+
+         // It appears that an heir-presumptive (successor) must be made ready.
+         // Only the current lock owner can manipulate the EntryList or
+         // drain _cxq, so we need to reacquire the lock.  If we fail
+         // to reacquire the lock the responsibility for ensuring succession
+         // falls to the new owner.
+         //
+         if (Atomic::cmpxchg_ptr (THREAD, &_owner, NULL) != NULL) {
+            return ;
+         }
+         TEVENT (Exit - Reacquired) ;
+      } else {
+         if ((intptr_t(_EntryList)|intptr_t(_cxq)) == 0 || _succ != NULL) {
+            OrderAccess::release_store_ptr (&_owner, NULL) ;   // drop the lock
+            OrderAccess::storeload() ;
+            // Ratify the previously observed values.
+            if (_cxq == NULL || _succ != NULL) {
+                TEVENT (Inflated exit - simple egress) ;
+                return ;
+            }
+
+            // inopportune interleaving -- the exiting thread (this thread)
+            // in the fast-exit path raced an entering thread in the slow-enter
+            // path.
+            // We have two choices:
+            // A.  Try to reacquire the lock.
+            //     If the CAS() fails return immediately, otherwise
+            //     we either restart/rerun the exit operation, or simply
+            //     fall-through into the code below which wakes a successor.
+            // B.  If the elements forming the EntryList|cxq are TSM
+            //     we could simply unpark() the lead thread and return
+            //     without having set _succ.
+            if (Atomic::cmpxchg_ptr (THREAD, &_owner, NULL) != NULL) {
+               TEVENT (Inflated exit - reacquired succeeded) ;
+               return ;
+            }
+            TEVENT (Inflated exit - reacquired failed) ;
+         } else {
+            TEVENT (Inflated exit - complex egress) ;
+         }
+      }
+
+      guarantee (_owner == THREAD, "invariant") ;
+
+      ObjectWaiter * w = NULL ;
+      int QMode = Knob_QMode ;
+
+      if (QMode == 2 && _cxq != NULL) {
+          // QMode == 2 : cxq has precedence over EntryList.
+          // Try to directly wake a successor from the cxq.
+          // If successful, the successor will need to unlink itself from cxq.
+          w = _cxq ;
+          assert (w != NULL, "invariant") ;
+          assert (w->TState == ObjectWaiter::TS_CXQ, "Invariant") ;
+          ExitEpilog (Self, w) ;
+          return ;
+      }
+
+      if (QMode == 3 && _cxq != NULL) {
+          // Aggressively drain cxq into EntryList at the first opportunity.
+          // This policy ensure that recently-run threads live at the head of EntryList.
+          // Drain _cxq into EntryList - bulk transfer.
+          // First, detach _cxq.
+          // The following loop is tantamount to: w = swap (&cxq, NULL)
+          w = _cxq ;
+          for (;;) {
+             assert (w != NULL, "Invariant") ;
+             ObjectWaiter * u = (ObjectWaiter *) Atomic::cmpxchg_ptr (NULL, &_cxq, w) ;
+             if (u == w) break ;
+             w = u ;
+          }
+          assert (w != NULL              , "invariant") ;
+
+          ObjectWaiter * q = NULL ;
+          ObjectWaiter * p ;
+          for (p = w ; p != NULL ; p = p->_next) {
+              guarantee (p->TState == ObjectWaiter::TS_CXQ, "Invariant") ;
+              p->TState = ObjectWaiter::TS_ENTER ;
+              p->_prev = q ;
+              q = p ;
+          }
+
+          // Append the RATs to the EntryList
+          // TODO: organize EntryList as a CDLL so we can locate the tail in constant-time.
+          ObjectWaiter * Tail ;
+          for (Tail = _EntryList ; Tail != NULL && Tail->_next != NULL ; Tail = Tail->_next) ;
+          if (Tail == NULL) {
+              _EntryList = w ;
+          } else {
+              Tail->_next = w ;
+              w->_prev = Tail ;
+          }
+
+          // Fall thru into code that tries to wake a successor from EntryList
+      }
+
+      if (QMode == 4 && _cxq != NULL) {
+          // Aggressively drain cxq into EntryList at the first opportunity.
+          // This policy ensure that recently-run threads live at the head of EntryList.
+
+          // Drain _cxq into EntryList - bulk transfer.
+          // First, detach _cxq.
+          // The following loop is tantamount to: w = swap (&cxq, NULL)
+          w = _cxq ;
+          for (;;) {
+             assert (w != NULL, "Invariant") ;
+             ObjectWaiter * u = (ObjectWaiter *) Atomic::cmpxchg_ptr (NULL, &_cxq, w) ;
+             if (u == w) break ;
+             w = u ;
+          }
+          assert (w != NULL              , "invariant") ;
+
+          ObjectWaiter * q = NULL ;
+          ObjectWaiter * p ;
+          for (p = w ; p != NULL ; p = p->_next) {
+              guarantee (p->TState == ObjectWaiter::TS_CXQ, "Invariant") ;
+              p->TState = ObjectWaiter::TS_ENTER ;
+              p->_prev = q ;
+              q = p ;
+          }
+
+          // Prepend the RATs to the EntryList
+          if (_EntryList != NULL) {
+              q->_next = _EntryList ;
+              _EntryList->_prev = q ;
+          }
+          _EntryList = w ;
+
+          // Fall thru into code that tries to wake a successor from EntryList
+      }
+
+      w = _EntryList  ;
+      if (w != NULL) {
+          // I'd like to write: guarantee (w->_thread != Self).
+          // But in practice an exiting thread may find itself on the EntryList.
+          // Lets say thread T1 calls O.wait().  Wait() enqueues T1 on O's waitset and
+          // then calls exit().  Exit release the lock by setting O._owner to NULL.
+          // Lets say T1 then stalls.  T2 acquires O and calls O.notify().  The
+          // notify() operation moves T1 from O's waitset to O's EntryList. T2 then
+          // release the lock "O".  T2 resumes immediately after the ST of null into
+          // _owner, above.  T2 notices that the EntryList is populated, so it
+          // reacquires the lock and then finds itself on the EntryList.
+          // Given all that, we have to tolerate the circumstance where "w" is
+          // associated with Self.
+          assert (w->TState == ObjectWaiter::TS_ENTER, "invariant") ;
+          ExitEpilog (Self, w) ;
+          return ;
+      }
+
+      // If we find that both _cxq and EntryList are null then just
+      // re-run the exit protocol from the top.
+      w = _cxq ;
+      if (w == NULL) continue ;
+
+      // Drain _cxq into EntryList - bulk transfer.
+      // First, detach _cxq.
+      // The following loop is tantamount to: w = swap (&cxq, NULL)
+      for (;;) {
+          assert (w != NULL, "Invariant") ;
+          ObjectWaiter * u = (ObjectWaiter *) Atomic::cmpxchg_ptr (NULL, &_cxq, w) ;
+          if (u == w) break ;
+          w = u ;
+      }
+      TEVENT (Inflated exit - drain cxq into EntryList) ;
+
+      assert (w != NULL              , "invariant") ;
+      assert (_EntryList  == NULL    , "invariant") ;
+
+      // Convert the LIFO SLL anchored by _cxq into a DLL.
+      // The list reorganization step operates in O(LENGTH(w)) time.
+      // It's critical that this step operate quickly as
+      // "Self" still holds the outer-lock, restricting parallelism
+      // and effectively lengthening the critical section.
+      // Invariant: s chases t chases u.
+      // TODO-FIXME: consider changing EntryList from a DLL to a CDLL so
+      // we have faster access to the tail.
+
+      if (QMode == 1) {
+         // QMode == 1 : drain cxq to EntryList, reversing order
+         // We also reverse the order of the list.
+         ObjectWaiter * s = NULL ;
+         ObjectWaiter * t = w ;
+         ObjectWaiter * u = NULL ;
+         while (t != NULL) {
+             guarantee (t->TState == ObjectWaiter::TS_CXQ, "invariant") ;
+             t->TState = ObjectWaiter::TS_ENTER ;
+             u = t->_next ;
+             t->_prev = u ;
+             t->_next = s ;
+             s = t;
+             t = u ;
+         }
+         _EntryList  = s ;
+         assert (s != NULL, "invariant") ;
+      } else {
+         // QMode == 0 or QMode == 2
+         _EntryList = w ;
+         ObjectWaiter * q = NULL ;
+         ObjectWaiter * p ;
+         for (p = w ; p != NULL ; p = p->_next) {
+             guarantee (p->TState == ObjectWaiter::TS_CXQ, "Invariant") ;
+             p->TState = ObjectWaiter::TS_ENTER ;
+             p->_prev = q ;
+             q = p ;
+         }
+      }
+
+      // In 1-0 mode we need: ST EntryList; MEMBAR #storestore; ST _owner = NULL
+      // The MEMBAR is satisfied by the release_store() operation in ExitEpilog().
+
+      // See if we can abdicate to a spinner instead of waking a thread.
+      // A primary goal of the implementation is to reduce the
+      // context-switch rate.
+      if (_succ != NULL) continue;
+
+      w = _EntryList  ;
+      if (w != NULL) {
+          guarantee (w->TState == ObjectWaiter::TS_ENTER, "invariant") ;
+          ExitEpilog (Self, w) ;
+          return ;
+      }
+   }
+}
+
+// ExitSuspendEquivalent:
+// A faster alternate to handle_special_suspend_equivalent_condition()
+//
+// handle_special_suspend_equivalent_condition() unconditionally
+// acquires the SR_lock.  On some platforms uncontended MutexLocker()
+// operations have high latency.  Note that in ::enter() we call HSSEC
+// while holding the monitor, so we effectively lengthen the critical sections.
+//
+// There are a number of possible solutions:
+//
+// A.  To ameliorate the problem we might also defer state transitions
+//     to as late as possible -- just prior to parking.
+//     Given that, we'd call HSSEC after having returned from park(),
+//     but before attempting to acquire the monitor.  This is only a
+//     partial solution.  It avoids calling HSSEC while holding the
+//     monitor (good), but it still increases successor reacquisition latency --
+//     the interval between unparking a successor and the time the successor
+//     resumes and retries the lock.  See ReenterI(), which defers state transitions.
+//     If we use this technique we can also avoid EnterI()-exit() loop
+//     in ::enter() where we iteratively drop the lock and then attempt
+//     to reacquire it after suspending.
+//
+// B.  In the future we might fold all the suspend bits into a
+//     composite per-thread suspend flag and then update it with CAS().
+//     Alternately, a Dekker-like mechanism with multiple variables
+//     would suffice:
+//       ST Self->_suspend_equivalent = false
+//       MEMBAR
+//       LD Self_>_suspend_flags
+//
+
+
+bool ObjectMonitor::ExitSuspendEquivalent (JavaThread * jSelf) {
+   int Mode = Knob_FastHSSEC ;
+   if (Mode && !jSelf->is_external_suspend()) {
+      assert (jSelf->is_suspend_equivalent(), "invariant") ;
+      jSelf->clear_suspend_equivalent() ;
+      if (2 == Mode) OrderAccess::storeload() ;
+      if (!jSelf->is_external_suspend()) return false ;
+      // We raced a suspension -- fall thru into the slow path
+      TEVENT (ExitSuspendEquivalent - raced) ;
+      jSelf->set_suspend_equivalent() ;
+   }
+   return jSelf->handle_special_suspend_equivalent_condition() ;
+}
+
+
+void ObjectMonitor::ExitEpilog (Thread * Self, ObjectWaiter * Wakee) {
+   assert (_owner == Self, "invariant") ;
+
+   // Exit protocol:
+   // 1. ST _succ = wakee
+   // 2. membar #loadstore|#storestore;
+   // 2. ST _owner = NULL
+   // 3. unpark(wakee)
+
+   _succ = Knob_SuccEnabled ? Wakee->_thread : NULL ;
+   ParkEvent * Trigger = Wakee->_event ;
+
+   // Hygiene -- once we've set _owner = NULL we can't safely dereference Wakee again.
+   // The thread associated with Wakee may have grabbed the lock and "Wakee" may be
+   // out-of-scope (non-extant).
+   Wakee  = NULL ;
+
+   // Drop the lock
+   OrderAccess::release_store_ptr (&_owner, NULL) ;
+   OrderAccess::fence() ;                               // ST _owner vs LD in unpark()
+
+   if (SafepointSynchronize::do_call_back()) {
+      TEVENT (unpark before SAFEPOINT) ;
+   }
+
+   DTRACE_MONITOR_PROBE(contended__exit, this, object(), Self);
+   Trigger->unpark() ;
+
+   // Maintain stats and report events to JVMTI
+   if (ObjectMonitor::_sync_Parks != NULL) {
+      ObjectMonitor::_sync_Parks->inc() ;
+   }
+}
+
+
+// -----------------------------------------------------------------------------
+// Class Loader deadlock handling.
+//
+// complete_exit exits a lock returning recursion count
+// complete_exit/reenter operate as a wait without waiting
+// complete_exit requires an inflated monitor
+// The _owner field is not always the Thread addr even with an
+// inflated monitor, e.g. the monitor can be inflated by a non-owning
+// thread due to contention.
+intptr_t ObjectMonitor::complete_exit(TRAPS) {
+   Thread * const Self = THREAD;
+   assert(Self->is_Java_thread(), "Must be Java thread!");
+   JavaThread *jt = (JavaThread *)THREAD;
+
+   DeferredInitialize();
+
+   if (THREAD != _owner) {
+    if (THREAD->is_lock_owned ((address)_owner)) {
+       assert(_recursions == 0, "internal state error");
+       _owner = THREAD ;   /* Convert from basiclock addr to Thread addr */
+       _recursions = 0 ;
+       OwnerIsThread = 1 ;
+    }
+   }
+
+   guarantee(Self == _owner, "complete_exit not owner");
+   intptr_t save = _recursions; // record the old recursion count
+   _recursions = 0;        // set the recursion level to be 0
+   exit (Self) ;           // exit the monitor
+   guarantee (_owner != Self, "invariant");
+   return save;
+}
+
+// reenter() enters a lock and sets recursion count
+// complete_exit/reenter operate as a wait without waiting
+void ObjectMonitor::reenter(intptr_t recursions, TRAPS) {
+   Thread * const Self = THREAD;
+   assert(Self->is_Java_thread(), "Must be Java thread!");
+   JavaThread *jt = (JavaThread *)THREAD;
+
+   guarantee(_owner != Self, "reenter already owner");
+   enter (THREAD);       // enter the monitor
+   guarantee (_recursions == 0, "reenter recursion");
+   _recursions = recursions;
+   return;
+}
+
+
+// -----------------------------------------------------------------------------
+// A macro is used below because there may already be a pending
+// exception which should not abort the execution of the routines
+// which use this (which is why we don't put this into check_slow and
+// call it with a CHECK argument).
+
+#define CHECK_OWNER()                                                             \
+  do {                                                                            \
+    if (THREAD != _owner) {                                                       \
+      if (THREAD->is_lock_owned((address) _owner)) {                              \
+        _owner = THREAD ;  /* Convert from basiclock addr to Thread addr */       \
+        _recursions = 0;                                                          \
+        OwnerIsThread = 1 ;                                                       \
+      } else {                                                                    \
+        TEVENT (Throw IMSX) ;                                                     \
+        THROW(vmSymbols::java_lang_IllegalMonitorStateException());               \
+      }                                                                           \
+    }                                                                             \
+  } while (false)
+
+// check_slow() is a misnomer.  It's called to simply to throw an IMSX exception.
+// TODO-FIXME: remove check_slow() -- it's likely dead.
+
+void ObjectMonitor::check_slow(TRAPS) {
+  TEVENT (check_slow - throw IMSX) ;
+  assert(THREAD != _owner && !THREAD->is_lock_owned((address) _owner), "must not be owner");
+  THROW_MSG(vmSymbols::java_lang_IllegalMonitorStateException(), "current thread not owner");
+}
+
+static int Adjust (volatile int * adr, int dx) {
+  int v ;
+  for (v = *adr ; Atomic::cmpxchg (v + dx, adr, v) != v; v = *adr) ;
+  return v ;
+}
+// -----------------------------------------------------------------------------
+// Wait/Notify/NotifyAll
+//
+// Note: a subset of changes to ObjectMonitor::wait()
+// will need to be replicated in complete_exit above
+void ObjectMonitor::wait(jlong millis, bool interruptible, TRAPS) {
+   Thread * const Self = THREAD ;
+   assert(Self->is_Java_thread(), "Must be Java thread!");
+   JavaThread *jt = (JavaThread *)THREAD;
+
+   DeferredInitialize () ;
+
+   // Throw IMSX or IEX.
+   CHECK_OWNER();
+
+   // check for a pending interrupt
+   if (interruptible && Thread::is_interrupted(Self, true) && !HAS_PENDING_EXCEPTION) {
+     // post monitor waited event.  Note that this is past-tense, we are done waiting.
+     if (JvmtiExport::should_post_monitor_waited()) {
+        // Note: 'false' parameter is passed here because the
+        // wait was not timed out due to thread interrupt.
+        JvmtiExport::post_monitor_waited(jt, this, false);
+     }
+     TEVENT (Wait - Throw IEX) ;
+     THROW(vmSymbols::java_lang_InterruptedException());
+     return ;
+   }
+   TEVENT (Wait) ;
+
+   assert (Self->_Stalled == 0, "invariant") ;
+   Self->_Stalled = intptr_t(this) ;
+   jt->set_current_waiting_monitor(this);
+
+   // create a node to be put into the queue
+   // Critically, after we reset() the event but prior to park(), we must check
+   // for a pending interrupt.
+   ObjectWaiter node(Self);
+   node.TState = ObjectWaiter::TS_WAIT ;
+   Self->_ParkEvent->reset() ;
+   OrderAccess::fence();          // ST into Event; membar ; LD interrupted-flag
+
+   // Enter the waiting queue, which is a circular doubly linked list in this case
+   // but it could be a priority queue or any data structure.
+   // _WaitSetLock protects the wait queue.  Normally the wait queue is accessed only
+   // by the the owner of the monitor *except* in the case where park()
+   // returns because of a timeout of interrupt.  Contention is exceptionally rare
+   // so we use a simple spin-lock instead of a heavier-weight blocking lock.
+
+   Thread::SpinAcquire (&_WaitSetLock, "WaitSet - add") ;
+   AddWaiter (&node) ;
+   Thread::SpinRelease (&_WaitSetLock) ;
+
+   if ((SyncFlags & 4) == 0) {
+      _Responsible = NULL ;
+   }
+   intptr_t save = _recursions; // record the old recursion count
+   _waiters++;                  // increment the number of waiters
+   _recursions = 0;             // set the recursion level to be 1
+   exit (Self) ;                    // exit the monitor
+   guarantee (_owner != Self, "invariant") ;
+
+   // As soon as the ObjectMonitor's ownership is dropped in the exit()
+   // call above, another thread can enter() the ObjectMonitor, do the
+   // notify(), and exit() the ObjectMonitor. If the other thread's
+   // exit() call chooses this thread as the successor and the unpark()
+   // call happens to occur while this thread is posting a
+   // MONITOR_CONTENDED_EXIT event, then we run the risk of the event
+   // handler using RawMonitors and consuming the unpark().
+   //
+   // To avoid the problem, we re-post the event. This does no harm
+   // even if the original unpark() was not consumed because we are the
+   // chosen successor for this monitor.
+   if (node._notified != 0 && _succ == Self) {
+      node._event->unpark();
+   }
+
+   // The thread is on the WaitSet list - now park() it.
+   // On MP systems it's conceivable that a brief spin before we park
+   // could be profitable.
+   //
+   // TODO-FIXME: change the following logic to a loop of the form
+   //   while (!timeout && !interrupted && _notified == 0) park()
+
+   int ret = OS_OK ;
+   int WasNotified = 0 ;
+   { // State transition wrappers
+     OSThread* osthread = Self->osthread();
+     OSThreadWaitState osts(osthread, true);
+     {
+       ThreadBlockInVM tbivm(jt);
+       // Thread is in thread_blocked state and oop access is unsafe.
+       jt->set_suspend_equivalent();
+
+       if (interruptible && (Thread::is_interrupted(THREAD, false) || HAS_PENDING_EXCEPTION)) {
+           // Intentionally empty
+       } else
+       if (node._notified == 0) {
+         if (millis <= 0) {
+            Self->_ParkEvent->park () ;
+         } else {
+            ret = Self->_ParkEvent->park (millis) ;
+         }
+       }
+
+       // were we externally suspended while we were waiting?
+       if (ExitSuspendEquivalent (jt)) {
+          // TODO-FIXME: add -- if succ == Self then succ = null.
+          jt->java_suspend_self();
+       }
+
+     } // Exit thread safepoint: transition _thread_blocked -> _thread_in_vm
+
+
+     // Node may be on the WaitSet, the EntryList (or cxq), or in transition
+     // from the WaitSet to the EntryList.
+     // See if we need to remove Node from the WaitSet.
+     // We use double-checked locking to avoid grabbing _WaitSetLock
+     // if the thread is not on the wait queue.
+     //
+     // Note that we don't need a fence before the fetch of TState.
+     // In the worst case we'll fetch a old-stale value of TS_WAIT previously
+     // written by the is thread. (perhaps the fetch might even be satisfied
+     // by a look-aside into the processor's own store buffer, although given
+     // the length of the code path between the prior ST and this load that's
+     // highly unlikely).  If the following LD fetches a stale TS_WAIT value
+     // then we'll acquire the lock and then re-fetch a fresh TState value.
+     // That is, we fail toward safety.
+
+     if (node.TState == ObjectWaiter::TS_WAIT) {
+         Thread::SpinAcquire (&_WaitSetLock, "WaitSet - unlink") ;
+         if (node.TState == ObjectWaiter::TS_WAIT) {
+            DequeueSpecificWaiter (&node) ;       // unlink from WaitSet
+            assert(node._notified == 0, "invariant");
+            node.TState = ObjectWaiter::TS_RUN ;
+         }
+         Thread::SpinRelease (&_WaitSetLock) ;
+     }
+
+     // The thread is now either on off-list (TS_RUN),
+     // on the EntryList (TS_ENTER), or on the cxq (TS_CXQ).
+     // The Node's TState variable is stable from the perspective of this thread.
+     // No other threads will asynchronously modify TState.
+     guarantee (node.TState != ObjectWaiter::TS_WAIT, "invariant") ;
+     OrderAccess::loadload() ;
+     if (_succ == Self) _succ = NULL ;
+     WasNotified = node._notified ;
+
+     // Reentry phase -- reacquire the monitor.
+     // re-enter contended monitor after object.wait().
+     // retain OBJECT_WAIT state until re-enter successfully completes
+     // Thread state is thread_in_vm and oop access is again safe,
+     // although the raw address of the object may have changed.
+     // (Don't cache naked oops over safepoints, of course).
+
+     // post monitor waited event. Note that this is past-tense, we are done waiting.
+     if (JvmtiExport::should_post_monitor_waited()) {
+       JvmtiExport::post_monitor_waited(jt, this, ret == OS_TIMEOUT);
+     }
+     OrderAccess::fence() ;
+
+     assert (Self->_Stalled != 0, "invariant") ;
+     Self->_Stalled = 0 ;
+
+     assert (_owner != Self, "invariant") ;
+     ObjectWaiter::TStates v = node.TState ;
+     if (v == ObjectWaiter::TS_RUN) {
+         enter (Self) ;
+     } else {
+         guarantee (v == ObjectWaiter::TS_ENTER || v == ObjectWaiter::TS_CXQ, "invariant") ;
+         ReenterI (Self, &node) ;
+         node.wait_reenter_end(this);
+     }
+
+     // Self has reacquired the lock.
+     // Lifecycle - the node representing Self must not appear on any queues.
+     // Node is about to go out-of-scope, but even if it were immortal we wouldn't
+     // want residual elements associated with this thread left on any lists.
+     guarantee (node.TState == ObjectWaiter::TS_RUN, "invariant") ;
+     assert    (_owner == Self, "invariant") ;
+     assert    (_succ != Self , "invariant") ;
+   } // OSThreadWaitState()
+
+   jt->set_current_waiting_monitor(NULL);
+
+   guarantee (_recursions == 0, "invariant") ;
+   _recursions = save;     // restore the old recursion count
+   _waiters--;             // decrement the number of waiters
+
+   // Verify a few postconditions
+   assert (_owner == Self       , "invariant") ;
+   assert (_succ  != Self       , "invariant") ;
+   assert (((oop)(object()))->mark() == markOopDesc::encode(this), "invariant") ;
+
+   if (SyncFlags & 32) {
+      OrderAccess::fence() ;
+   }
+
+   // check if the notification happened
+   if (!WasNotified) {
+     // no, it could be timeout or Thread.interrupt() or both
+     // check for interrupt event, otherwise it is timeout
+     if (interruptible && Thread::is_interrupted(Self, true) && !HAS_PENDING_EXCEPTION) {
+       TEVENT (Wait - throw IEX from epilog) ;
+       THROW(vmSymbols::java_lang_InterruptedException());
+     }
+   }
+
+   // NOTE: Spurious wake up will be consider as timeout.
+   // Monitor notify has precedence over thread interrupt.
+}
+
+
+// Consider:
+// If the lock is cool (cxq == null && succ == null) and we're on an MP system
+// then instead of transferring a thread from the WaitSet to the EntryList
+// we might just dequeue a thread from the WaitSet and directly unpark() it.
+
+void ObjectMonitor::notify(TRAPS) {
+  CHECK_OWNER();
+  if (_WaitSet == NULL) {
+     TEVENT (Empty-Notify) ;
+     return ;
+  }
+  DTRACE_MONITOR_PROBE(notify, this, object(), THREAD);
+
+  int Policy = Knob_MoveNotifyee ;
+
+  Thread::SpinAcquire (&_WaitSetLock, "WaitSet - notify") ;
+  ObjectWaiter * iterator = DequeueWaiter() ;
+  if (iterator != NULL) {
+     TEVENT (Notify1 - Transfer) ;
+     guarantee (iterator->TState == ObjectWaiter::TS_WAIT, "invariant") ;
+     guarantee (iterator->_notified == 0, "invariant") ;
+     if (Policy != 4) {
+        iterator->TState = ObjectWaiter::TS_ENTER ;
+     }
+     iterator->_notified = 1 ;
+
+     ObjectWaiter * List = _EntryList ;
+     if (List != NULL) {
+        assert (List->_prev == NULL, "invariant") ;
+        assert (List->TState == ObjectWaiter::TS_ENTER, "invariant") ;
+        assert (List != iterator, "invariant") ;
+     }
+
+     if (Policy == 0) {       // prepend to EntryList
+         if (List == NULL) {
+             iterator->_next = iterator->_prev = NULL ;
+             _EntryList = iterator ;
+         } else {
+             List->_prev = iterator ;
+             iterator->_next = List ;
+             iterator->_prev = NULL ;
+             _EntryList = iterator ;
+        }
+     } else
+     if (Policy == 1) {      // append to EntryList
+         if (List == NULL) {
+             iterator->_next = iterator->_prev = NULL ;
+             _EntryList = iterator ;
+         } else {
+            // CONSIDER:  finding the tail currently requires a linear-time walk of
+            // the EntryList.  We can make tail access constant-time by converting to
+            // a CDLL instead of using our current DLL.
+            ObjectWaiter * Tail ;
+            for (Tail = List ; Tail->_next != NULL ; Tail = Tail->_next) ;
+            assert (Tail != NULL && Tail->_next == NULL, "invariant") ;
+            Tail->_next = iterator ;
+            iterator->_prev = Tail ;
+            iterator->_next = NULL ;
+        }
+     } else
+     if (Policy == 2) {      // prepend to cxq
+         // prepend to cxq
+         if (List == NULL) {
+             iterator->_next = iterator->_prev = NULL ;
+             _EntryList = iterator ;
+         } else {
+            iterator->TState = ObjectWaiter::TS_CXQ ;
+            for (;;) {
+                ObjectWaiter * Front = _cxq ;
+                iterator->_next = Front ;
+                if (Atomic::cmpxchg_ptr (iterator, &_cxq, Front) == Front) {
+                    break ;
+                }
+            }
+         }
+     } else
+     if (Policy == 3) {      // append to cxq
+        iterator->TState = ObjectWaiter::TS_CXQ ;
+        for (;;) {
+            ObjectWaiter * Tail ;
+            Tail = _cxq ;
+            if (Tail == NULL) {
+                iterator->_next = NULL ;
+                if (Atomic::cmpxchg_ptr (iterator, &_cxq, NULL) == NULL) {
+                   break ;
+                }
+            } else {
+                while (Tail->_next != NULL) Tail = Tail->_next ;
+                Tail->_next = iterator ;
+                iterator->_prev = Tail ;
+                iterator->_next = NULL ;
+                break ;
+            }
+        }
+     } else {
+        ParkEvent * ev = iterator->_event ;
+        iterator->TState = ObjectWaiter::TS_RUN ;
+        OrderAccess::fence() ;
+        ev->unpark() ;
+     }
+
+     if (Policy < 4) {
+       iterator->wait_reenter_begin(this);
+     }
+
+     // _WaitSetLock protects the wait queue, not the EntryList.  We could
+     // move the add-to-EntryList operation, above, outside the critical section
+     // protected by _WaitSetLock.  In practice that's not useful.  With the
+     // exception of  wait() timeouts and interrupts the monitor owner
+     // is the only thread that grabs _WaitSetLock.  There's almost no contention
+     // on _WaitSetLock so it's not profitable to reduce the length of the
+     // critical section.
+  }
+
+  Thread::SpinRelease (&_WaitSetLock) ;
+
+  if (iterator != NULL && ObjectMonitor::_sync_Notifications != NULL) {
+     ObjectMonitor::_sync_Notifications->inc() ;
+  }
+}
+
+
+void ObjectMonitor::notifyAll(TRAPS) {
+  CHECK_OWNER();
+  ObjectWaiter* iterator;
+  if (_WaitSet == NULL) {
+      TEVENT (Empty-NotifyAll) ;
+      return ;
+  }
+  DTRACE_MONITOR_PROBE(notifyAll, this, object(), THREAD);
+
+  int Policy = Knob_MoveNotifyee ;
+  int Tally = 0 ;
+  Thread::SpinAcquire (&_WaitSetLock, "WaitSet - notifyall") ;
+
+  for (;;) {
+     iterator = DequeueWaiter () ;
+     if (iterator == NULL) break ;
+     TEVENT (NotifyAll - Transfer1) ;
+     ++Tally ;
+
+     // Disposition - what might we do with iterator ?
+     // a.  add it directly to the EntryList - either tail or head.
+     // b.  push it onto the front of the _cxq.
+     // For now we use (a).
+
+     guarantee (iterator->TState == ObjectWaiter::TS_WAIT, "invariant") ;
+     guarantee (iterator->_notified == 0, "invariant") ;
+     iterator->_notified = 1 ;
+     if (Policy != 4) {
+        iterator->TState = ObjectWaiter::TS_ENTER ;
+     }
+
+     ObjectWaiter * List = _EntryList ;
+     if (List != NULL) {
+        assert (List->_prev == NULL, "invariant") ;
+        assert (List->TState == ObjectWaiter::TS_ENTER, "invariant") ;
+        assert (List != iterator, "invariant") ;
+     }
+
+     if (Policy == 0) {       // prepend to EntryList
+         if (List == NULL) {
+             iterator->_next = iterator->_prev = NULL ;
+             _EntryList = iterator ;
+         } else {
+             List->_prev = iterator ;
+             iterator->_next = List ;
+             iterator->_prev = NULL ;
+             _EntryList = iterator ;
+        }
+     } else
+     if (Policy == 1) {      // append to EntryList
+         if (List == NULL) {
+             iterator->_next = iterator->_prev = NULL ;
+             _EntryList = iterator ;
+         } else {
+            // CONSIDER:  finding the tail currently requires a linear-time walk of
+            // the EntryList.  We can make tail access constant-time by converting to
+            // a CDLL instead of using our current DLL.
+            ObjectWaiter * Tail ;
+            for (Tail = List ; Tail->_next != NULL ; Tail = Tail->_next) ;
+            assert (Tail != NULL && Tail->_next == NULL, "invariant") ;
+            Tail->_next = iterator ;
+            iterator->_prev = Tail ;
+            iterator->_next = NULL ;
+        }
+     } else
+     if (Policy == 2) {      // prepend to cxq
+         // prepend to cxq
+         iterator->TState = ObjectWaiter::TS_CXQ ;
+         for (;;) {
+             ObjectWaiter * Front = _cxq ;
+             iterator->_next = Front ;
+             if (Atomic::cmpxchg_ptr (iterator, &_cxq, Front) == Front) {
+                 break ;
+             }
+         }
+     } else
+     if (Policy == 3) {      // append to cxq
+        iterator->TState = ObjectWaiter::TS_CXQ ;
+        for (;;) {
+            ObjectWaiter * Tail ;
+            Tail = _cxq ;
+            if (Tail == NULL) {
+                iterator->_next = NULL ;
+                if (Atomic::cmpxchg_ptr (iterator, &_cxq, NULL) == NULL) {
+                   break ;
+                }
+            } else {
+                while (Tail->_next != NULL) Tail = Tail->_next ;
+                Tail->_next = iterator ;
+                iterator->_prev = Tail ;
+                iterator->_next = NULL ;
+                break ;
+            }
+        }
+     } else {
+        ParkEvent * ev = iterator->_event ;
+        iterator->TState = ObjectWaiter::TS_RUN ;
+        OrderAccess::fence() ;
+        ev->unpark() ;
+     }
+
+     if (Policy < 4) {
+       iterator->wait_reenter_begin(this);
+     }
+
+     // _WaitSetLock protects the wait queue, not the EntryList.  We could
+     // move the add-to-EntryList operation, above, outside the critical section
+     // protected by _WaitSetLock.  In practice that's not useful.  With the
+     // exception of  wait() timeouts and interrupts the monitor owner
+     // is the only thread that grabs _WaitSetLock.  There's almost no contention
+     // on _WaitSetLock so it's not profitable to reduce the length of the
+     // critical section.
+  }
+
+  Thread::SpinRelease (&_WaitSetLock) ;
+
+  if (Tally != 0 && ObjectMonitor::_sync_Notifications != NULL) {
+     ObjectMonitor::_sync_Notifications->inc(Tally) ;
+  }
+}
+
+// -----------------------------------------------------------------------------
+// Adaptive Spinning Support
+//
+// Adaptive spin-then-block - rational spinning
+//
+// Note that we spin "globally" on _owner with a classic SMP-polite TATAS
+// algorithm.  On high order SMP systems it would be better to start with
+// a brief global spin and then revert to spinning locally.  In the spirit of MCS/CLH,
+// a contending thread could enqueue itself on the cxq and then spin locally
+// on a thread-specific variable such as its ParkEvent._Event flag.
+// That's left as an exercise for the reader.  Note that global spinning is
+// not problematic on Niagara, as the L2$ serves the interconnect and has both
+// low latency and massive bandwidth.
+//
+// Broadly, we can fix the spin frequency -- that is, the % of contended lock
+// acquisition attempts where we opt to spin --  at 100% and vary the spin count
+// (duration) or we can fix the count at approximately the duration of
+// a context switch and vary the frequency.   Of course we could also
+// vary both satisfying K == Frequency * Duration, where K is adaptive by monitor.
+// See http://j2se.east/~dice/PERSIST/040824-AdaptiveSpinning.html.
+//
+// This implementation varies the duration "D", where D varies with
+// the success rate of recent spin attempts. (D is capped at approximately
+// length of a round-trip context switch).  The success rate for recent
+// spin attempts is a good predictor of the success rate of future spin
+// attempts.  The mechanism adapts automatically to varying critical
+// section length (lock modality), system load and degree of parallelism.
+// D is maintained per-monitor in _SpinDuration and is initialized
+// optimistically.  Spin frequency is fixed at 100%.
+//
+// Note that _SpinDuration is volatile, but we update it without locks
+// or atomics.  The code is designed so that _SpinDuration stays within
+// a reasonable range even in the presence of races.  The arithmetic
+// operations on _SpinDuration are closed over the domain of legal values,
+// so at worst a race will install and older but still legal value.
+// At the very worst this introduces some apparent non-determinism.
+// We might spin when we shouldn't or vice-versa, but since the spin
+// count are relatively short, even in the worst case, the effect is harmless.
+//
+// Care must be taken that a low "D" value does not become an
+// an absorbing state.  Transient spinning failures -- when spinning
+// is overall profitable -- should not cause the system to converge
+// on low "D" values.  We want spinning to be stable and predictable
+// and fairly responsive to change and at the same time we don't want
+// it to oscillate, become metastable, be "too" non-deterministic,
+// or converge on or enter undesirable stable absorbing states.
+//
+// We implement a feedback-based control system -- using past behavior
+// to predict future behavior.  We face two issues: (a) if the
+// input signal is random then the spin predictor won't provide optimal
+// results, and (b) if the signal frequency is too high then the control
+// system, which has some natural response lag, will "chase" the signal.
+// (b) can arise from multimodal lock hold times.  Transient preemption
+// can also result in apparent bimodal lock hold times.
+// Although sub-optimal, neither condition is particularly harmful, as
+// in the worst-case we'll spin when we shouldn't or vice-versa.
+// The maximum spin duration is rather short so the failure modes aren't bad.
+// To be conservative, I've tuned the gain in system to bias toward
+// _not spinning.  Relatedly, the system can sometimes enter a mode where it
+// "rings" or oscillates between spinning and not spinning.  This happens
+// when spinning is just on the cusp of profitability, however, so the
+// situation is not dire.  The state is benign -- there's no need to add
+// hysteresis control to damp the transition rate between spinning and
+// not spinning.
+//
+
+intptr_t ObjectMonitor::SpinCallbackArgument = 0 ;
+int (*ObjectMonitor::SpinCallbackFunction)(intptr_t, int) = NULL ;
+
+// Spinning: Fixed frequency (100%), vary duration
+
+
+int ObjectMonitor::TrySpin_VaryDuration (Thread * Self) {
+
+    // Dumb, brutal spin.  Good for comparative measurements against adaptive spinning.
+    int ctr = Knob_FixedSpin ;
+    if (ctr != 0) {
+        while (--ctr >= 0) {
+            if (TryLock (Self) > 0) return 1 ;
+            SpinPause () ;
+        }
+        return 0 ;
+    }
+
+    for (ctr = Knob_PreSpin + 1; --ctr >= 0 ; ) {
+      if (TryLock(Self) > 0) {
+        // Increase _SpinDuration ...
+        // Note that we don't clamp SpinDuration precisely at SpinLimit.
+        // Raising _SpurDuration to the poverty line is key.
+        int x = _SpinDuration ;
+        if (x < Knob_SpinLimit) {
+           if (x < Knob_Poverty) x = Knob_Poverty ;
+           _SpinDuration = x + Knob_BonusB ;
+        }
+        return 1 ;
+      }
+      SpinPause () ;
+    }
+
+    // Admission control - verify preconditions for spinning
+    //
+    // We always spin a little bit, just to prevent _SpinDuration == 0 from
+    // becoming an absorbing state.  Put another way, we spin briefly to
+    // sample, just in case the system load, parallelism, contention, or lock
+    // modality changed.
+    //
+    // Consider the following alternative:
+    // Periodically set _SpinDuration = _SpinLimit and try a long/full
+    // spin attempt.  "Periodically" might mean after a tally of
+    // the # of failed spin attempts (or iterations) reaches some threshold.
+    // This takes us into the realm of 1-out-of-N spinning, where we
+    // hold the duration constant but vary the frequency.
+
+    ctr = _SpinDuration  ;
+    if (ctr < Knob_SpinBase) ctr = Knob_SpinBase ;
+    if (ctr <= 0) return 0 ;
+
+    if (Knob_SuccRestrict && _succ != NULL) return 0 ;
+    if (Knob_OState && NotRunnable (Self, (Thread *) _owner)) {
+       TEVENT (Spin abort - notrunnable [TOP]);
+       return 0 ;
+    }
+
+    int MaxSpin = Knob_MaxSpinners ;
+    if (MaxSpin >= 0) {
+       if (_Spinner > MaxSpin) {
+          TEVENT (Spin abort -- too many spinners) ;
+          return 0 ;
+       }
+       // Slighty racy, but benign ...
+       Adjust (&_Spinner, 1) ;
+    }
+
+    // We're good to spin ... spin ingress.
+    // CONSIDER: use Prefetch::write() to avoid RTS->RTO upgrades
+    // when preparing to LD...CAS _owner, etc and the CAS is likely
+    // to succeed.
+    int hits    = 0 ;
+    int msk     = 0 ;
+    int caspty  = Knob_CASPenalty ;
+    int oxpty   = Knob_OXPenalty ;
+    int sss     = Knob_SpinSetSucc ;
+    if (sss && _succ == NULL ) _succ = Self ;
+    Thread * prv = NULL ;
+
+    // There are three ways to exit the following loop:
+    // 1.  A successful spin where this thread has acquired the lock.
+    // 2.  Spin failure with prejudice
+    // 3.  Spin failure without prejudice
+
+    while (--ctr >= 0) {
+
+      // Periodic polling -- Check for pending GC
+      // Threads may spin while they're unsafe.
+      // We don't want spinning threads to delay the JVM from reaching
+      // a stop-the-world safepoint or to steal cycles from GC.
+      // If we detect a pending safepoint we abort in order that
+      // (a) this thread, if unsafe, doesn't delay the safepoint, and (b)
+      // this thread, if safe, doesn't steal cycles from GC.
+      // This is in keeping with the "no loitering in runtime" rule.
+      // We periodically check to see if there's a safepoint pending.
+      if ((ctr & 0xFF) == 0) {
+         if (SafepointSynchronize::do_call_back()) {
+            TEVENT (Spin: safepoint) ;
+            goto Abort ;           // abrupt spin egress
+         }
+         if (Knob_UsePause & 1) SpinPause () ;
+
+         int (*scb)(intptr_t,int) = SpinCallbackFunction ;
+         if (hits > 50 && scb != NULL) {
+            int abend = (*scb)(SpinCallbackArgument, 0) ;
+         }
+      }
+
+      if (Knob_UsePause & 2) SpinPause() ;
+
+      // Exponential back-off ...  Stay off the bus to reduce coherency traffic.
+      // This is useful on classic SMP systems, but is of less utility on
+      // N1-style CMT platforms.
+      //
+      // Trade-off: lock acquisition latency vs coherency bandwidth.
+      // Lock hold times are typically short.  A histogram
+      // of successful spin attempts shows that we usually acquire
+      // the lock early in the spin.  That suggests we want to
+      // sample _owner frequently in the early phase of the spin,
+      // but then back-off and sample less frequently as the spin
+      // progresses.  The back-off makes a good citizen on SMP big
+      // SMP systems.  Oversampling _owner can consume excessive
+      // coherency bandwidth.  Relatedly, if we _oversample _owner we
+      // can inadvertently interfere with the the ST m->owner=null.
+      // executed by the lock owner.
+      if (ctr & msk) continue ;
+      ++hits ;
+      if ((hits & 0xF) == 0) {
+        // The 0xF, above, corresponds to the exponent.
+        // Consider: (msk+1)|msk
+        msk = ((msk << 2)|3) & BackOffMask ;
+      }
+
+      // Probe _owner with TATAS
+      // If this thread observes the monitor transition or flicker
+      // from locked to unlocked to locked, then the odds that this
+      // thread will acquire the lock in this spin attempt go down
+      // considerably.  The same argument applies if the CAS fails
+      // or if we observe _owner change from one non-null value to
+      // another non-null value.   In such cases we might abort
+      // the spin without prejudice or apply a "penalty" to the
+      // spin count-down variable "ctr", reducing it by 100, say.
+
+      Thread * ox = (Thread *) _owner ;
+      if (ox == NULL) {
+         ox = (Thread *) Atomic::cmpxchg_ptr (Self, &_owner, NULL) ;
+         if (ox == NULL) {
+            // The CAS succeeded -- this thread acquired ownership
+            // Take care of some bookkeeping to exit spin state.
+            if (sss && _succ == Self) {
+               _succ = NULL ;
+            }
+            if (MaxSpin > 0) Adjust (&_Spinner, -1) ;
+
+            // Increase _SpinDuration :
+            // The spin was successful (profitable) so we tend toward
+            // longer spin attempts in the future.
+            // CONSIDER: factor "ctr" into the _SpinDuration adjustment.
+            // If we acquired the lock early in the spin cycle it
+            // makes sense to increase _SpinDuration proportionally.
+            // Note that we don't clamp SpinDuration precisely at SpinLimit.
+            int x = _SpinDuration ;
+            if (x < Knob_SpinLimit) {
+                if (x < Knob_Poverty) x = Knob_Poverty ;
+                _SpinDuration = x + Knob_Bonus ;
+            }
+            return 1 ;
+         }
+
+         // The CAS failed ... we can take any of the following actions:
+         // * penalize: ctr -= Knob_CASPenalty
+         // * exit spin with prejudice -- goto Abort;
+         // * exit spin without prejudice.
+         // * Since CAS is high-latency, retry again immediately.
+         prv = ox ;
+         TEVENT (Spin: cas failed) ;
+         if (caspty == -2) break ;
+         if (caspty == -1) goto Abort ;
+         ctr -= caspty ;
+         continue ;
+      }
+
+      // Did lock ownership change hands ?
+      if (ox != prv && prv != NULL ) {
+          TEVENT (spin: Owner changed)
+          if (oxpty == -2) break ;
+          if (oxpty == -1) goto Abort ;
+          ctr -= oxpty ;
+      }
+      prv = ox ;
+
+      // Abort the spin if the owner is not executing.
+      // The owner must be executing in order to drop the lock.
+      // Spinning while the owner is OFFPROC is idiocy.
+      // Consider: ctr -= RunnablePenalty ;
+      if (Knob_OState && NotRunnable (Self, ox)) {
+         TEVENT (Spin abort - notrunnable);
+         goto Abort ;
+      }
+      if (sss && _succ == NULL ) _succ = Self ;
+   }
+
+   // Spin failed with prejudice -- reduce _SpinDuration.
+   // TODO: Use an AIMD-like policy to adjust _SpinDuration.
+   // AIMD is globally stable.
+   TEVENT (Spin failure) ;
+   {
+     int x = _SpinDuration ;
+     if (x > 0) {
+        // Consider an AIMD scheme like: x -= (x >> 3) + 100
+        // This is globally sample and tends to damp the response.
+        x -= Knob_Penalty ;
+        if (x < 0) x = 0 ;
+        _SpinDuration = x ;
+     }
+   }
+
+ Abort:
+   if (MaxSpin >= 0) Adjust (&_Spinner, -1) ;
+   if (sss && _succ == Self) {
+      _succ = NULL ;
+      // Invariant: after setting succ=null a contending thread
+      // must recheck-retry _owner before parking.  This usually happens
+      // in the normal usage of TrySpin(), but it's safest
+      // to make TrySpin() as foolproof as possible.
+      OrderAccess::fence() ;
+      if (TryLock(Self) > 0) return 1 ;
+   }
+   return 0 ;
+}
+
+// NotRunnable() -- informed spinning
+//
+// Don't bother spinning if the owner is not eligible to drop the lock.
+// Peek at the owner's schedctl.sc_state and Thread._thread_values and
+// spin only if the owner thread is _thread_in_Java or _thread_in_vm.
+// The thread must be runnable in order to drop the lock in timely fashion.
+// If the _owner is not runnable then spinning will not likely be
+// successful (profitable).
+//
+// Beware -- the thread referenced by _owner could have died
+// so a simply fetch from _owner->_thread_state might trap.
+// Instead, we use SafeFetchXX() to safely LD _owner->_thread_state.
+// Because of the lifecycle issues the schedctl and _thread_state values
+// observed by NotRunnable() might be garbage.  NotRunnable must
+// tolerate this and consider the observed _thread_state value
+// as advisory.
+//
+// Beware too, that _owner is sometimes a BasicLock address and sometimes
+// a thread pointer.  We differentiate the two cases with OwnerIsThread.
+// Alternately, we might tag the type (thread pointer vs basiclock pointer)
+// with the LSB of _owner.  Another option would be to probablistically probe
+// the putative _owner->TypeTag value.
+//
+// Checking _thread_state isn't perfect.  Even if the thread is
+// in_java it might be blocked on a page-fault or have been preempted
+// and sitting on a ready/dispatch queue.  _thread state in conjunction
+// with schedctl.sc_state gives us a good picture of what the
+// thread is doing, however.
+//
+// TODO: check schedctl.sc_state.
+// We'll need to use SafeFetch32() to read from the schedctl block.
+// See RFE #5004247 and http://sac.sfbay.sun.com/Archives/CaseLog/arc/PSARC/2005/351/
+//
+// The return value from NotRunnable() is *advisory* -- the
+// result is based on sampling and is not necessarily coherent.
+// The caller must tolerate false-negative and false-positive errors.
+// Spinning, in general, is probabilistic anyway.
+
+
+int ObjectMonitor::NotRunnable (Thread * Self, Thread * ox) {
+    // Check either OwnerIsThread or ox->TypeTag == 2BAD.
+    if (!OwnerIsThread) return 0 ;
+
+    if (ox == NULL) return 0 ;
+
+    // Avoid transitive spinning ...
+    // Say T1 spins or blocks trying to acquire L.  T1._Stalled is set to L.
+    // Immediately after T1 acquires L it's possible that T2, also
+    // spinning on L, will see L.Owner=T1 and T1._Stalled=L.
+    // This occurs transiently after T1 acquired L but before
+    // T1 managed to clear T1.Stalled.  T2 does not need to abort
+    // its spin in this circumstance.
+    intptr_t BlockedOn = SafeFetchN ((intptr_t *) &ox->_Stalled, intptr_t(1)) ;
+
+    if (BlockedOn == 1) return 1 ;
+    if (BlockedOn != 0) {
+      return BlockedOn != intptr_t(this) && _owner == ox ;
+    }
+
+    assert (sizeof(((JavaThread *)ox)->_thread_state == sizeof(int)), "invariant") ;
+    int jst = SafeFetch32 ((int *) &((JavaThread *) ox)->_thread_state, -1) ; ;
+    // consider also: jst != _thread_in_Java -- but that's overspecific.
+    return jst == _thread_blocked || jst == _thread_in_native ;
+}
+
+
+// -----------------------------------------------------------------------------
+// WaitSet management ...
+
+ObjectWaiter::ObjectWaiter(Thread* thread) {
+  _next     = NULL;
+  _prev     = NULL;
+  _notified = 0;
+  TState    = TS_RUN ;
+  _thread   = thread;
+  _event    = thread->_ParkEvent ;
+  _active   = false;
+  assert (_event != NULL, "invariant") ;
+}
+
+void ObjectWaiter::wait_reenter_begin(ObjectMonitor *mon) {
+  JavaThread *jt = (JavaThread *)this->_thread;
+  _active = JavaThreadBlockedOnMonitorEnterState::wait_reenter_begin(jt, mon);
+}
+
+void ObjectWaiter::wait_reenter_end(ObjectMonitor *mon) {
+  JavaThread *jt = (JavaThread *)this->_thread;
+  JavaThreadBlockedOnMonitorEnterState::wait_reenter_end(jt, _active);
+}
+
+inline void ObjectMonitor::AddWaiter(ObjectWaiter* node) {
+  assert(node != NULL, "should not dequeue NULL node");
+  assert(node->_prev == NULL, "node already in list");
+  assert(node->_next == NULL, "node already in list");
+  // put node at end of queue (circular doubly linked list)
+  if (_WaitSet == NULL) {
+    _WaitSet = node;
+    node->_prev = node;
+    node->_next = node;
+  } else {
+    ObjectWaiter* head = _WaitSet ;
+    ObjectWaiter* tail = head->_prev;
+    assert(tail->_next == head, "invariant check");
+    tail->_next = node;
+    head->_prev = node;
+    node->_next = head;
+    node->_prev = tail;
+  }
+}
+
+inline ObjectWaiter* ObjectMonitor::DequeueWaiter() {
+  // dequeue the very first waiter
+  ObjectWaiter* waiter = _WaitSet;
+  if (waiter) {
+    DequeueSpecificWaiter(waiter);
+  }
+  return waiter;
+}
+
+inline void ObjectMonitor::DequeueSpecificWaiter(ObjectWaiter* node) {
+  assert(node != NULL, "should not dequeue NULL node");
+  assert(node->_prev != NULL, "node already removed from list");
+  assert(node->_next != NULL, "node already removed from list");
+  // when the waiter has woken up because of interrupt,
+  // timeout or other spurious wake-up, dequeue the
+  // waiter from waiting list
+  ObjectWaiter* next = node->_next;
+  if (next == node) {
+    assert(node->_prev == node, "invariant check");
+    _WaitSet = NULL;
+  } else {
+    ObjectWaiter* prev = node->_prev;
+    assert(prev->_next == node, "invariant check");
+    assert(next->_prev == node, "invariant check");
+    next->_prev = prev;
+    prev->_next = next;
+    if (_WaitSet == node) {
+      _WaitSet = next;
+    }
+  }
+  node->_next = NULL;
+  node->_prev = NULL;
+}
+
+// -----------------------------------------------------------------------------
+// PerfData support
+PerfCounter * ObjectMonitor::_sync_ContendedLockAttempts       = NULL ;
+PerfCounter * ObjectMonitor::_sync_FutileWakeups               = NULL ;
+PerfCounter * ObjectMonitor::_sync_Parks                       = NULL ;
+PerfCounter * ObjectMonitor::_sync_EmptyNotifications          = NULL ;
+PerfCounter * ObjectMonitor::_sync_Notifications               = NULL ;
+PerfCounter * ObjectMonitor::_sync_PrivateA                    = NULL ;
+PerfCounter * ObjectMonitor::_sync_PrivateB                    = NULL ;
+PerfCounter * ObjectMonitor::_sync_SlowExit                    = NULL ;
+PerfCounter * ObjectMonitor::_sync_SlowEnter                   = NULL ;
+PerfCounter * ObjectMonitor::_sync_SlowNotify                  = NULL ;
+PerfCounter * ObjectMonitor::_sync_SlowNotifyAll               = NULL ;
+PerfCounter * ObjectMonitor::_sync_FailedSpins                 = NULL ;
+PerfCounter * ObjectMonitor::_sync_SuccessfulSpins             = NULL ;
+PerfCounter * ObjectMonitor::_sync_MonInCirculation            = NULL ;
+PerfCounter * ObjectMonitor::_sync_MonScavenged                = NULL ;
+PerfCounter * ObjectMonitor::_sync_Inflations                  = NULL ;
+PerfCounter * ObjectMonitor::_sync_Deflations                  = NULL ;
+PerfLongVariable * ObjectMonitor::_sync_MonExtant              = NULL ;
+
+// One-shot global initialization for the sync subsystem.
+// We could also defer initialization and initialize on-demand
+// the first time we call inflate().  Initialization would
+// be protected - like so many things - by the MonitorCache_lock.
+
+void ObjectMonitor::Initialize () {
+  static int InitializationCompleted = 0 ;
+  assert (InitializationCompleted == 0, "invariant") ;
+  InitializationCompleted = 1 ;
+  if (UsePerfData) {
+      EXCEPTION_MARK ;
+      #define NEWPERFCOUNTER(n)   {n = PerfDataManager::create_counter(SUN_RT, #n, PerfData::U_Events,CHECK); }
+      #define NEWPERFVARIABLE(n)  {n = PerfDataManager::create_variable(SUN_RT, #n, PerfData::U_Events,CHECK); }
+      NEWPERFCOUNTER(_sync_Inflations) ;
+      NEWPERFCOUNTER(_sync_Deflations) ;
+      NEWPERFCOUNTER(_sync_ContendedLockAttempts) ;
+      NEWPERFCOUNTER(_sync_FutileWakeups) ;
+      NEWPERFCOUNTER(_sync_Parks) ;
+      NEWPERFCOUNTER(_sync_EmptyNotifications) ;
+      NEWPERFCOUNTER(_sync_Notifications) ;
+      NEWPERFCOUNTER(_sync_SlowEnter) ;
+      NEWPERFCOUNTER(_sync_SlowExit) ;
+      NEWPERFCOUNTER(_sync_SlowNotify) ;
+      NEWPERFCOUNTER(_sync_SlowNotifyAll) ;
+      NEWPERFCOUNTER(_sync_FailedSpins) ;
+      NEWPERFCOUNTER(_sync_SuccessfulSpins) ;
+      NEWPERFCOUNTER(_sync_PrivateA) ;
+      NEWPERFCOUNTER(_sync_PrivateB) ;
+      NEWPERFCOUNTER(_sync_MonInCirculation) ;
+      NEWPERFCOUNTER(_sync_MonScavenged) ;
+      NEWPERFVARIABLE(_sync_MonExtant) ;
+      #undef NEWPERFCOUNTER
+  }
+}
+
+
+// Compile-time asserts
+// When possible, it's better to catch errors deterministically at
+// compile-time than at runtime.  The down-side to using compile-time
+// asserts is that error message -- often something about negative array
+// indices -- is opaque.
+
+#define CTASSERT(x) { int tag[1-(2*!(x))]; printf ("Tag @" INTPTR_FORMAT "\n", (intptr_t)tag); }
+
+void ObjectMonitor::ctAsserts() {
+  CTASSERT(offset_of (ObjectMonitor, _header) == 0);
+}
+
+
+static char * kvGet (char * kvList, const char * Key) {
+    if (kvList == NULL) return NULL ;
+    size_t n = strlen (Key) ;
+    char * Search ;
+    for (Search = kvList ; *Search ; Search += strlen(Search) + 1) {
+        if (strncmp (Search, Key, n) == 0) {
+            if (Search[n] == '=') return Search + n + 1 ;
+            if (Search[n] == 0)   return (char *) "1" ;
+        }
+    }
+    return NULL ;
+}
+
+static int kvGetInt (char * kvList, const char * Key, int Default) {
+    char * v = kvGet (kvList, Key) ;
+    int rslt = v ? ::strtol (v, NULL, 0) : Default ;
+    if (Knob_ReportSettings && v != NULL) {
+        ::printf ("  SyncKnob: %s %d(%d)\n", Key, rslt, Default) ;
+        ::fflush (stdout) ;
+    }
+    return rslt ;
+}
+
+void ObjectMonitor::DeferredInitialize () {
+  if (InitDone > 0) return ;
+  if (Atomic::cmpxchg (-1, &InitDone, 0) != 0) {
+      while (InitDone != 1) ;
+      return ;
+  }
+
+  // One-shot global initialization ...
+  // The initialization is idempotent, so we don't need locks.
+  // In the future consider doing this via os::init_2().
+  // SyncKnobs consist of <Key>=<Value> pairs in the style
+  // of environment variables.  Start by converting ':' to NUL.
+
+  if (SyncKnobs == NULL) SyncKnobs = "" ;
+
+  size_t sz = strlen (SyncKnobs) ;
+  char * knobs = (char *) malloc (sz + 2) ;
+  if (knobs == NULL) {
+     vm_exit_out_of_memory (sz + 2, "Parse SyncKnobs") ;
+     guarantee (0, "invariant") ;
+  }
+  strcpy (knobs, SyncKnobs) ;
+  knobs[sz+1] = 0 ;
+  for (char * p = knobs ; *p ; p++) {
+     if (*p == ':') *p = 0 ;
+  }
+
+  #define SETKNOB(x) { Knob_##x = kvGetInt (knobs, #x, Knob_##x); }
+  SETKNOB(ReportSettings) ;
+  SETKNOB(Verbose) ;
+  SETKNOB(FixedSpin) ;
+  SETKNOB(SpinLimit) ;
+  SETKNOB(SpinBase) ;
+  SETKNOB(SpinBackOff);
+  SETKNOB(CASPenalty) ;
+  SETKNOB(OXPenalty) ;
+  SETKNOB(LogSpins) ;
+  SETKNOB(SpinSetSucc) ;
+  SETKNOB(SuccEnabled) ;
+  SETKNOB(SuccRestrict) ;
+  SETKNOB(Penalty) ;
+  SETKNOB(Bonus) ;
+  SETKNOB(BonusB) ;
+  SETKNOB(Poverty) ;
+  SETKNOB(SpinAfterFutile) ;
+  SETKNOB(UsePause) ;
+  SETKNOB(SpinEarly) ;
+  SETKNOB(OState) ;
+  SETKNOB(MaxSpinners) ;
+  SETKNOB(PreSpin) ;
+  SETKNOB(ExitPolicy) ;
+  SETKNOB(QMode);
+  SETKNOB(ResetEvent) ;
+  SETKNOB(MoveNotifyee) ;
+  SETKNOB(FastHSSEC) ;
+  #undef SETKNOB
+
+  if (os::is_MP()) {
+     BackOffMask = (1 << Knob_SpinBackOff) - 1 ;
+     if (Knob_ReportSettings) ::printf ("BackOffMask=%X\n", BackOffMask) ;
+     // CONSIDER: BackOffMask = ROUNDUP_NEXT_POWER2 (ncpus-1)
+  } else {
+     Knob_SpinLimit = 0 ;
+     Knob_SpinBase  = 0 ;
+     Knob_PreSpin   = 0 ;
+     Knob_FixedSpin = -1 ;
+  }
+
+  if (Knob_LogSpins == 0) {
+     ObjectMonitor::_sync_FailedSpins = NULL ;
+  }
+
+  free (knobs) ;
+  OrderAccess::fence() ;
+  InitDone = 1 ;
+}
+
+#ifndef PRODUCT
+void ObjectMonitor::verify() {
+}
+
+void ObjectMonitor::print() {
+}
+#endif
diff --git a/hotspot/src/share/vm/runtime/objectMonitor.hpp b/hotspot/src/share/vm/runtime/objectMonitor.hpp
index 8bff00c2230..20f013619a1 100644
--- a/hotspot/src/share/vm/runtime/objectMonitor.hpp
+++ b/hotspot/src/share/vm/runtime/objectMonitor.hpp
@@ -22,6 +22,32 @@
  *
  */
 
+
+// ObjectWaiter serves as a "proxy" or surrogate thread.
+// TODO-FIXME: Eliminate ObjectWaiter and use the thread-specific
+// ParkEvent instead.  Beware, however, that the JVMTI code
+// knows about ObjectWaiters, so we'll have to reconcile that code.
+// See next_waiter(), first_waiter(), etc.
+
+class ObjectWaiter : public StackObj {
+ public:
+  enum TStates { TS_UNDEF, TS_READY, TS_RUN, TS_WAIT, TS_ENTER, TS_CXQ } ;
+  enum Sorted  { PREPEND, APPEND, SORTED } ;
+  ObjectWaiter * volatile _next;
+  ObjectWaiter * volatile _prev;
+  Thread*       _thread;
+  ParkEvent *   _event;
+  volatile int  _notified ;
+  volatile TStates TState ;
+  Sorted        _Sorted ;           // List placement disposition
+  bool          _active ;           // Contention monitoring is enabled
+ public:
+  ObjectWaiter(Thread* thread);
+
+  void wait_reenter_begin(ObjectMonitor *mon);
+  void wait_reenter_end(ObjectMonitor *mon);
+};
+
 // WARNING:
 //   This is a very sensitive and fragile class. DO NOT make any
 // change unless you are fully aware of the underlying semantics.
@@ -38,8 +64,6 @@
 // It is also used as RawMonitor by the JVMTI
 
 
-class ObjectWaiter;
-
 class ObjectMonitor {
  public:
   enum {
@@ -74,13 +98,16 @@ class ObjectMonitor {
 
 
  public:
-  ObjectMonitor();
-  ~ObjectMonitor();
-
   markOop   header() const;
   void      set_header(markOop hdr);
 
-  intptr_t  is_busy() const;
+  intptr_t is_busy() const {
+    // TODO-FIXME: merge _count and _waiters.
+    // TODO-FIXME: assert _owner == null implies _recursions = 0
+    // TODO-FIXME: assert _WaitSet != null implies _count > 0
+    return _count|_waiters|intptr_t(_owner)|intptr_t(_cxq)|intptr_t(_EntryList ) ;
+  }
+
   intptr_t  is_entered(Thread* current) const;
 
   void*     owner() const;
@@ -91,13 +118,58 @@ class ObjectMonitor {
   intptr_t  count() const;
   void      set_count(intptr_t count);
   intptr_t  contentions() const ;
+  intptr_t  recursions() const                                         { return _recursions; }
 
   // JVM/DI GetMonitorInfo() needs this
-  Thread *  thread_of_waiter (ObjectWaiter *) ;
-  ObjectWaiter * first_waiter () ;
-  ObjectWaiter * next_waiter(ObjectWaiter* o);
+  ObjectWaiter* first_waiter()                                         { return _WaitSet; }
+  ObjectWaiter* next_waiter(ObjectWaiter* o)                           { return o->_next; }
+  Thread* thread_of_waiter(ObjectWaiter* o)                            { return o->_thread; }
 
-  intptr_t  recursions() const { return _recursions; }
+  // initialize the monitor, exception the semaphore, all other fields
+  // are simple integers or pointers
+  ObjectMonitor() {
+    _header       = NULL;
+    _count        = 0;
+    _waiters      = 0,
+    _recursions   = 0;
+    _object       = NULL;
+    _owner        = NULL;
+    _WaitSet      = NULL;
+    _WaitSetLock  = 0 ;
+    _Responsible  = NULL ;
+    _succ         = NULL ;
+    _cxq          = NULL ;
+    FreeNext      = NULL ;
+    _EntryList    = NULL ;
+    _SpinFreq     = 0 ;
+    _SpinClock    = 0 ;
+    OwnerIsThread = 0 ;
+  }
+
+  ~ObjectMonitor() {
+   // TODO: Add asserts ...
+   // _cxq == 0 _succ == NULL _owner == NULL _waiters == 0
+   // _count == 0 _EntryList  == NULL etc
+  }
+
+private:
+  void Recycle () {
+    // TODO: add stronger asserts ...
+    // _cxq == 0 _succ == NULL _owner == NULL _waiters == 0
+    // _count == 0 EntryList  == NULL
+    // _recursions == 0 _WaitSet == NULL
+    // TODO: assert (is_busy()|_recursions) == 0
+    _succ          = NULL ;
+    _EntryList     = NULL ;
+    _cxq           = NULL ;
+    _WaitSet       = NULL ;
+    _recursions    = 0 ;
+    _SpinFreq      = 0 ;
+    _SpinClock     = 0 ;
+    OwnerIsThread  = 0 ;
+  }
+
+public:
 
   void*     object() const;
   void*     object_addr();
@@ -122,22 +194,9 @@ class ObjectMonitor {
   intptr_t  complete_exit(TRAPS);
   void      reenter(intptr_t recursions, TRAPS);
 
-  int       raw_enter(TRAPS);
-  int       raw_exit(TRAPS);
-  int       raw_wait(jlong millis, bool interruptable, TRAPS);
-  int       raw_notify(TRAPS);
-  int       raw_notifyAll(TRAPS);
-
  private:
-  // JVMTI support -- remove ASAP
-  int       SimpleEnter (Thread * Self) ;
-  int       SimpleExit  (Thread * Self) ;
-  int       SimpleWait  (Thread * Self, jlong millis) ;
-  int       SimpleNotify (Thread * Self, bool All) ;
-
- private:
-  void      Recycle () ;
   void      AddWaiter (ObjectWaiter * waiter) ;
+  static    void DeferredInitialize();
 
   ObjectWaiter * DequeueWaiter () ;
   void      DequeueSpecificWaiter (ObjectWaiter * waiter) ;
@@ -172,13 +231,17 @@ class ObjectMonitor {
   // The VM assumes write ordering wrt these fields, which can be
   // read from other threads.
 
+ protected:                         // protected for jvmtiRawMonitor
   void *  volatile _owner;          // pointer to owning thread OR BasicLock
   volatile intptr_t  _recursions;   // recursion count, 0 for first entry
+ private:
   int OwnerIsThread ;               // _owner is (Thread *) vs SP/BasicLock
   ObjectWaiter * volatile _cxq ;    // LL of recently-arrived threads blocked on entry.
                                     // The list is actually composed of WaitNodes, acting
                                     // as proxies for Threads.
+ protected:
   ObjectWaiter * volatile _EntryList ;     // Threads blocked on entry or reentry.
+ private:
   Thread * volatile _succ ;          // Heir presumptive thread - used for futile wakeup throttling
   Thread * volatile _Responsible ;
   int _PromptDrain ;                // rqst to drain cxq into EntryList ASAP
@@ -196,8 +259,12 @@ class ObjectMonitor {
   volatile intptr_t  _count;        // reference count to prevent reclaimation/deflation
                                     // at stop-the-world time.  See deflate_idle_monitors().
                                     // _count is approximately |_WaitSet| + |_EntryList|
+ protected:
   volatile intptr_t  _waiters;      // number of waiting threads
+ private:
+ protected:
   ObjectWaiter * volatile _WaitSet; // LL of threads wait()ing on the monitor
+ private:
   volatile int _WaitSetLock;        // protects Wait Queue - simple spinlock
 
  public:
@@ -205,4 +272,37 @@ class ObjectMonitor {
   ObjectMonitor * FreeNext ;        // Free list linkage
   intptr_t StatA, StatsB ;
 
+ public:
+  static void Initialize () ;
+  static PerfCounter * _sync_ContendedLockAttempts ;
+  static PerfCounter * _sync_FutileWakeups ;
+  static PerfCounter * _sync_Parks ;
+  static PerfCounter * _sync_EmptyNotifications ;
+  static PerfCounter * _sync_Notifications ;
+  static PerfCounter * _sync_SlowEnter ;
+  static PerfCounter * _sync_SlowExit ;
+  static PerfCounter * _sync_SlowNotify ;
+  static PerfCounter * _sync_SlowNotifyAll ;
+  static PerfCounter * _sync_FailedSpins ;
+  static PerfCounter * _sync_SuccessfulSpins ;
+  static PerfCounter * _sync_PrivateA ;
+  static PerfCounter * _sync_PrivateB ;
+  static PerfCounter * _sync_MonInCirculation ;
+  static PerfCounter * _sync_MonScavenged ;
+  static PerfCounter * _sync_Inflations ;
+  static PerfCounter * _sync_Deflations ;
+  static PerfLongVariable * _sync_MonExtant ;
+
+ public:
+  static int Knob_Verbose;
+  static int Knob_SpinLimit;
 };
+
+#undef TEVENT
+#define TEVENT(nom) {if (SyncVerbose) FEVENT(nom); }
+
+#define FEVENT(nom) { static volatile int ctr = 0 ; int v = ++ctr ; if ((v & (v-1)) == 0) { ::printf (#nom " : %d \n", v); ::fflush(stdout); }}
+
+#undef  TEVENT
+#define TEVENT(nom) {;}
+
diff --git a/hotspot/src/share/vm/runtime/objectMonitor.inline.hpp b/hotspot/src/share/vm/runtime/objectMonitor.inline.hpp
index 3a4ee4e2aee..114b7c10c52 100644
--- a/hotspot/src/share/vm/runtime/objectMonitor.inline.hpp
+++ b/hotspot/src/share/vm/runtime/objectMonitor.inline.hpp
@@ -104,7 +104,3 @@ inline void ObjectMonitor::set_owner(void* owner) {
   _count = 0;
 }
 
-
-// here are the platform-dependent bodies:
-
-# include "incls/_objectMonitor_pd.inline.hpp.incl"
diff --git a/hotspot/src/share/vm/runtime/park.cpp b/hotspot/src/share/vm/runtime/park.cpp
new file mode 100644
index 00000000000..4fdb4fc541a
--- /dev/null
+++ b/hotspot/src/share/vm/runtime/park.cpp
@@ -0,0 +1,237 @@
+/*
+ * Copyright (c) 1997, 2010, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+
+# include "incls/_precompiled.incl"
+# include "incls/_park.cpp.incl"
+
+
+// Lifecycle management for TSM ParkEvents.
+// ParkEvents are type-stable (TSM).
+// In our particular implementation they happen to be immortal.
+//
+// We manage concurrency on the FreeList with a CAS-based
+// detach-modify-reattach idiom that avoids the ABA problems
+// that would otherwise be present in a simple CAS-based
+// push-pop implementation.   (push-one and pop-all)
+//
+// Caveat: Allocate() and Release() may be called from threads
+// other than the thread associated with the Event!
+// If we need to call Allocate() when running as the thread in
+// question then look for the PD calls to initialize native TLS.
+// Native TLS (Win32/Linux/Solaris) can only be initialized or
+// accessed by the associated thread.
+// See also pd_initialize().
+//
+// Note that we could defer associating a ParkEvent with a thread
+// until the 1st time the thread calls park().  unpark() calls to
+// an unprovisioned thread would be ignored.  The first park() call
+// for a thread would allocate and associate a ParkEvent and return
+// immediately.
+
+volatile int ParkEvent::ListLock = 0 ;
+ParkEvent * volatile ParkEvent::FreeList = NULL ;
+
+ParkEvent * ParkEvent::Allocate (Thread * t) {
+  // In rare cases -- JVM_RawMonitor* operations -- we can find t == null.
+  ParkEvent * ev ;
+
+  // Start by trying to recycle an existing but unassociated
+  // ParkEvent from the global free list.
+  for (;;) {
+    ev = FreeList ;
+    if (ev == NULL) break ;
+    // 1: Detach - sequester or privatize the list
+    // Tantamount to ev = Swap (&FreeList, NULL)
+    if (Atomic::cmpxchg_ptr (NULL, &FreeList, ev) != ev) {
+       continue ;
+    }
+
+    // We've detached the list.  The list in-hand is now
+    // local to this thread.   This thread can operate on the
+    // list without risk of interference from other threads.
+    // 2: Extract -- pop the 1st element from the list.
+    ParkEvent * List = ev->FreeNext ;
+    if (List == NULL) break ;
+    for (;;) {
+        // 3: Try to reattach the residual list
+        guarantee (List != NULL, "invariant") ;
+        ParkEvent * Arv =  (ParkEvent *) Atomic::cmpxchg_ptr (List, &FreeList, NULL) ;
+        if (Arv == NULL) break ;
+
+        // New nodes arrived.  Try to detach the recent arrivals.
+        if (Atomic::cmpxchg_ptr (NULL, &FreeList, Arv) != Arv) {
+            continue ;
+        }
+        guarantee (Arv != NULL, "invariant") ;
+        // 4: Merge Arv into List
+        ParkEvent * Tail = List ;
+        while (Tail->FreeNext != NULL) Tail = Tail->FreeNext ;
+        Tail->FreeNext = Arv ;
+    }
+    break ;
+  }
+
+  if (ev != NULL) {
+    guarantee (ev->AssociatedWith == NULL, "invariant") ;
+  } else {
+    // Do this the hard way -- materialize a new ParkEvent.
+    // In rare cases an allocating thread might detach a long list --
+    // installing null into FreeList -- and then stall or be obstructed.
+    // A 2nd thread calling Allocate() would see FreeList == null.
+    // The list held privately by the 1st thread is unavailable to the 2nd thread.
+    // In that case the 2nd thread would have to materialize a new ParkEvent,
+    // even though free ParkEvents existed in the system.  In this case we end up
+    // with more ParkEvents in circulation than we need, but the race is
+    // rare and the outcome is benign.  Ideally, the # of extant ParkEvents
+    // is equal to the maximum # of threads that existed at any one time.
+    // Because of the race mentioned above, segments of the freelist
+    // can be transiently inaccessible.  At worst we may end up with the
+    // # of ParkEvents in circulation slightly above the ideal.
+    // Note that if we didn't have the TSM/immortal constraint, then
+    // when reattaching, above, we could trim the list.
+    ev = new ParkEvent () ;
+    guarantee ((intptr_t(ev) & 0xFF) == 0, "invariant") ;
+  }
+  ev->reset() ;                     // courtesy to caller
+  ev->AssociatedWith = t ;          // Associate ev with t
+  ev->FreeNext       = NULL ;
+  return ev ;
+}
+
+void ParkEvent::Release (ParkEvent * ev) {
+  if (ev == NULL) return ;
+  guarantee (ev->FreeNext == NULL      , "invariant") ;
+  ev->AssociatedWith = NULL ;
+  for (;;) {
+    // Push ev onto FreeList
+    // The mechanism is "half" lock-free.
+    ParkEvent * List = FreeList ;
+    ev->FreeNext = List ;
+    if (Atomic::cmpxchg_ptr (ev, &FreeList, List) == List) break ;
+  }
+}
+
+// Override operator new and delete so we can ensure that the
+// least significant byte of ParkEvent addresses is 0.
+// Beware that excessive address alignment is undesirable
+// as it can result in D$ index usage imbalance as
+// well as bank access imbalance on Niagara-like platforms,
+// although Niagara's hash function should help.
+
+void * ParkEvent::operator new (size_t sz) {
+  return (void *) ((intptr_t (CHeapObj::operator new (sz + 256)) + 256) & -256) ;
+}
+
+void ParkEvent::operator delete (void * a) {
+  // ParkEvents are type-stable and immortal ...
+  ShouldNotReachHere();
+}
+
+
+// 6399321 As a temporary measure we copied & modified the ParkEvent::
+// allocate() and release() code for use by Parkers.  The Parker:: forms
+// will eventually be removed as we consolide and shift over to ParkEvents
+// for both builtin synchronization and JSR166 operations.
+
+volatile int Parker::ListLock = 0 ;
+Parker * volatile Parker::FreeList = NULL ;
+
+Parker * Parker::Allocate (JavaThread * t) {
+  guarantee (t != NULL, "invariant") ;
+  Parker * p ;
+
+  // Start by trying to recycle an existing but unassociated
+  // Parker from the global free list.
+  for (;;) {
+    p = FreeList ;
+    if (p  == NULL) break ;
+    // 1: Detach
+    // Tantamount to p = Swap (&FreeList, NULL)
+    if (Atomic::cmpxchg_ptr (NULL, &FreeList, p) != p) {
+       continue ;
+    }
+
+    // We've detached the list.  The list in-hand is now
+    // local to this thread.   This thread can operate on the
+    // list without risk of interference from other threads.
+    // 2: Extract -- pop the 1st element from the list.
+    Parker * List = p->FreeNext ;
+    if (List == NULL) break ;
+    for (;;) {
+        // 3: Try to reattach the residual list
+        guarantee (List != NULL, "invariant") ;
+        Parker * Arv =  (Parker *) Atomic::cmpxchg_ptr (List, &FreeList, NULL) ;
+        if (Arv == NULL) break ;
+
+        // New nodes arrived.  Try to detach the recent arrivals.
+        if (Atomic::cmpxchg_ptr (NULL, &FreeList, Arv) != Arv) {
+            continue ;
+        }
+        guarantee (Arv != NULL, "invariant") ;
+        // 4: Merge Arv into List
+        Parker * Tail = List ;
+        while (Tail->FreeNext != NULL) Tail = Tail->FreeNext ;
+        Tail->FreeNext = Arv ;
+    }
+    break ;
+  }
+
+  if (p != NULL) {
+    guarantee (p->AssociatedWith == NULL, "invariant") ;
+  } else {
+    // Do this the hard way -- materialize a new Parker..
+    // In rare cases an allocating thread might detach
+    // a long list -- installing null into FreeList --and
+    // then stall.  Another thread calling Allocate() would see
+    // FreeList == null and then invoke the ctor.  In this case we
+    // end up with more Parkers in circulation than we need, but
+    // the race is rare and the outcome is benign.
+    // Ideally, the # of extant Parkers is equal to the
+    // maximum # of threads that existed at any one time.
+    // Because of the race mentioned above, segments of the
+    // freelist can be transiently inaccessible.  At worst
+    // we may end up with the # of Parkers in circulation
+    // slightly above the ideal.
+    p = new Parker() ;
+  }
+  p->AssociatedWith = t ;          // Associate p with t
+  p->FreeNext       = NULL ;
+  return p ;
+}
+
+
+void Parker::Release (Parker * p) {
+  if (p == NULL) return ;
+  guarantee (p->AssociatedWith != NULL, "invariant") ;
+  guarantee (p->FreeNext == NULL      , "invariant") ;
+  p->AssociatedWith = NULL ;
+  for (;;) {
+    // Push p onto FreeList
+    Parker * List = FreeList ;
+    p->FreeNext = List ;
+    if (Atomic::cmpxchg_ptr (p, &FreeList, List) == List) break ;
+  }
+}
+
diff --git a/hotspot/src/share/vm/runtime/park.hpp b/hotspot/src/share/vm/runtime/park.hpp
new file mode 100644
index 00000000000..8979b4dc007
--- /dev/null
+++ b/hotspot/src/share/vm/runtime/park.hpp
@@ -0,0 +1,169 @@
+/*
+ * Copyright (c) 1997, 2010, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+/*
+ * Per-thread blocking support for JSR166. See the Java-level
+ * Documentation for rationale. Basically, park acts like wait, unpark
+ * like notify.
+ *
+ * 6271289 --
+ * To avoid errors where an os thread expires but the JavaThread still
+ * exists, Parkers are immortal (type-stable) and are recycled across
+ * new threads.  This parallels the ParkEvent implementation.
+ * Because park-unpark allow spurious wakeups it is harmless if an
+ * unpark call unparks a new thread using the old Parker reference.
+ *
+ * In the future we'll want to think about eliminating Parker and using
+ * ParkEvent instead.  There's considerable duplication between the two
+ * services.
+ *
+ */
+
+class Parker : public os::PlatformParker {
+private:
+  volatile int _counter ;
+  Parker * FreeNext ;
+  JavaThread * AssociatedWith ; // Current association
+
+public:
+  Parker() : PlatformParker() {
+    _counter       = 0 ;
+    FreeNext       = NULL ;
+    AssociatedWith = NULL ;
+  }
+protected:
+  ~Parker() { ShouldNotReachHere(); }
+public:
+  // For simplicity of interface with Java, all forms of park (indefinite,
+  // relative, and absolute) are multiplexed into one call.
+  void park(bool isAbsolute, jlong time);
+  void unpark();
+
+  // Lifecycle operators
+  static Parker * Allocate (JavaThread * t) ;
+  static void Release (Parker * e) ;
+private:
+  static Parker * volatile FreeList ;
+  static volatile int ListLock ;
+
+};
+
+/////////////////////////////////////////////////////////////
+//
+// ParkEvents are type-stable and immortal.
+//
+// Lifecycle: Once a ParkEvent is associated with a thread that ParkEvent remains
+// associated with the thread for the thread's entire lifetime - the relationship is
+// stable. A thread will be associated at most one ParkEvent.  When the thread
+// expires, the ParkEvent moves to the EventFreeList.  New threads attempt to allocate from
+// the EventFreeList before creating a new Event.  Type-stability frees us from
+// worrying about stale Event or Thread references in the objectMonitor subsystem.
+// (A reference to ParkEvent is always valid, even though the event may no longer be associated
+// with the desired or expected thread.  A key aspect of this design is that the callers of
+// park, unpark, etc must tolerate stale references and spurious wakeups).
+//
+// Only the "associated" thread can block (park) on the ParkEvent, although
+// any other thread can unpark a reachable parkevent.  Park() is allowed to
+// return spuriously.  In fact park-unpark a really just an optimization to
+// avoid unbounded spinning and surrender the CPU to be a polite system citizen.
+// A degenerate albeit "impolite" park-unpark implementation could simply return.
+// See http://blogs.sun.com/dave for more details.
+//
+// Eventually I'd like to eliminate Events and ObjectWaiters, both of which serve as
+// thread proxies, and simply make the THREAD structure type-stable and persistent.
+// Currently, we unpark events associated with threads, but ideally we'd just
+// unpark threads.
+//
+// The base-class, PlatformEvent, is platform-specific while the ParkEvent is
+// platform-independent.  PlatformEvent provides park(), unpark(), etc., and
+// is abstract -- that is, a PlatformEvent should never be instantiated except
+// as part of a ParkEvent.
+// Equivalently we could have defined a platform-independent base-class that
+// exported Allocate(), Release(), etc.  The platform-specific class would extend
+// that base-class, adding park(), unpark(), etc.
+//
+// A word of caution: The JVM uses 2 very similar constructs:
+// 1. ParkEvent are used for Java-level "monitor" synchronization.
+// 2. Parkers are used by JSR166-JUC park-unpark.
+//
+// We'll want to eventually merge these redundant facilities and use ParkEvent.
+
+
+class ParkEvent : public os::PlatformEvent {
+  private:
+    ParkEvent * FreeNext ;
+
+    // Current association
+    Thread * AssociatedWith ;
+    intptr_t RawThreadIdentity ;        // LWPID etc
+    volatile int Incarnation ;
+
+    // diagnostic : keep track of last thread to wake this thread.
+    // this is useful for construction of dependency graphs.
+    void * LastWaker ;
+
+  public:
+    // MCS-CLH list linkage and Native Mutex/Monitor
+    ParkEvent * volatile ListNext ;
+    ParkEvent * volatile ListPrev ;
+    volatile intptr_t OnList ;
+    volatile int TState ;
+    volatile int Notified ;             // for native monitor construct
+    volatile int IsWaiting ;            // Enqueued on WaitSet
+
+
+  private:
+    static ParkEvent * volatile FreeList ;
+    static volatile int ListLock ;
+
+    // It's prudent to mark the dtor as "private"
+    // ensuring that it's not visible outside the package.
+    // Unfortunately gcc warns about such usage, so
+    // we revert to the less desirable "protected" visibility.
+    // The other compilers accept private dtors.
+
+  protected:        // Ensure dtor is never invoked
+    ~ParkEvent() { guarantee (0, "invariant") ; }
+
+    ParkEvent() : PlatformEvent() {
+       AssociatedWith = NULL ;
+       FreeNext       = NULL ;
+       ListNext       = NULL ;
+       ListPrev       = NULL ;
+       OnList         = 0 ;
+       TState         = 0 ;
+       Notified       = 0 ;
+       IsWaiting      = 0 ;
+    }
+
+    // We use placement-new to force ParkEvent instances to be
+    // aligned on 256-byte address boundaries.  This ensures that the least
+    // significant byte of a ParkEvent address is always 0.
+
+    void * operator new (size_t sz) ;
+    void operator delete (void * a) ;
+
+  public:
+    static ParkEvent * Allocate (Thread * t) ;
+    static void Release (ParkEvent * e) ;
+} ;
diff --git a/hotspot/src/share/vm/runtime/synchronizer.cpp b/hotspot/src/share/vm/runtime/synchronizer.cpp
index c390c076a64..642c9815662 100644
--- a/hotspot/src/share/vm/runtime/synchronizer.cpp
+++ b/hotspot/src/share/vm/runtime/synchronizer.cpp
@@ -32,15 +32,12 @@
   #define ATTR
 #endif
 
-// Native markword accessors for synchronization and hashCode().
-//
 // The "core" versions of monitor enter and exit reside in this file.
 // The interpreter and compilers contain specialized transliterated
 // variants of the enter-exit fast-path operations.  See i486.ad fast_lock(),
 // for instance.  If you make changes here, make sure to modify the
 // interpreter, and both C1 and C2 fast-path inline locking code emission.
 //
-// TODO: merge the objectMonitor and synchronizer classes.
 //
 // -----------------------------------------------------------------------------
 
@@ -53,16 +50,6 @@ HS_DTRACE_PROBE_DECL5(hotspot, monitor__wait,
   jlong, uintptr_t, char*, int, long);
 HS_DTRACE_PROBE_DECL4(hotspot, monitor__waited,
   jlong, uintptr_t, char*, int);
-HS_DTRACE_PROBE_DECL4(hotspot, monitor__notify,
-  jlong, uintptr_t, char*, int);
-HS_DTRACE_PROBE_DECL4(hotspot, monitor__notifyAll,
-  jlong, uintptr_t, char*, int);
-HS_DTRACE_PROBE_DECL4(hotspot, monitor__contended__enter,
-  jlong, uintptr_t, char*, int);
-HS_DTRACE_PROBE_DECL4(hotspot, monitor__contended__entered,
-  jlong, uintptr_t, char*, int);
-HS_DTRACE_PROBE_DECL4(hotspot, monitor__contended__exit,
-  jlong, uintptr_t, char*, int);
 
 #define DTRACE_MONITOR_PROBE_COMMON(klassOop, thread)                      \
   char* bytes = NULL;                                                      \
@@ -99,61 +86,300 @@ HS_DTRACE_PROBE_DECL4(hotspot, monitor__contended__exit,
 
 #endif // ndef DTRACE_ENABLED
 
-// ObjectWaiter serves as a "proxy" or surrogate thread.
-// TODO-FIXME: Eliminate ObjectWaiter and use the thread-specific
-// ParkEvent instead.  Beware, however, that the JVMTI code
-// knows about ObjectWaiters, so we'll have to reconcile that code.
-// See next_waiter(), first_waiter(), etc.
+// This exists only as a workaround of dtrace bug 6254741
+int dtrace_waited_probe(ObjectMonitor* monitor, Handle obj, Thread* thr) {
+  DTRACE_MONITOR_PROBE(waited, monitor, obj(), thr);
+  return 0;
+}
 
-class ObjectWaiter : public StackObj {
- public:
-  enum TStates { TS_UNDEF, TS_READY, TS_RUN, TS_WAIT, TS_ENTER, TS_CXQ } ;
-  enum Sorted  { PREPEND, APPEND, SORTED } ;
-  ObjectWaiter * volatile _next;
-  ObjectWaiter * volatile _prev;
-  Thread*       _thread;
-  ParkEvent *   _event;
-  volatile int  _notified ;
-  volatile TStates TState ;
-  Sorted        _Sorted ;           // List placement disposition
-  bool          _active ;           // Contention monitoring is enabled
- public:
-  ObjectWaiter(Thread* thread) {
-    _next     = NULL;
-    _prev     = NULL;
-    _notified = 0;
-    TState    = TS_RUN ;
-    _thread   = thread;
-    _event    = thread->_ParkEvent ;
-    _active   = false;
-    assert (_event != NULL, "invariant") ;
+#define NINFLATIONLOCKS 256
+static volatile intptr_t InflationLocks [NINFLATIONLOCKS] ;
+
+ObjectMonitor * ObjectSynchronizer::gBlockList = NULL ;
+ObjectMonitor * volatile ObjectSynchronizer::gFreeList  = NULL ;
+ObjectMonitor * volatile ObjectSynchronizer::gOmInUseList  = NULL ;
+int ObjectSynchronizer::gOmInUseCount = 0;
+static volatile intptr_t ListLock = 0 ;      // protects global monitor free-list cache
+static volatile int MonitorFreeCount  = 0 ;      // # on gFreeList
+static volatile int MonitorPopulation = 0 ;      // # Extant -- in circulation
+#define CHAINMARKER ((oop)-1)
+
+// -----------------------------------------------------------------------------
+//  Fast Monitor Enter/Exit
+// This the fast monitor enter. The interpreter and compiler use
+// some assembly copies of this code. Make sure update those code
+// if the following function is changed. The implementation is
+// extremely sensitive to race condition. Be careful.
+
+void ObjectSynchronizer::fast_enter(Handle obj, BasicLock* lock, bool attempt_rebias, TRAPS) {
+ if (UseBiasedLocking) {
+    if (!SafepointSynchronize::is_at_safepoint()) {
+      BiasedLocking::Condition cond = BiasedLocking::revoke_and_rebias(obj, attempt_rebias, THREAD);
+      if (cond == BiasedLocking::BIAS_REVOKED_AND_REBIASED) {
+        return;
+      }
+    } else {
+      assert(!attempt_rebias, "can not rebias toward VM thread");
+      BiasedLocking::revoke_at_safepoint(obj);
+    }
+    assert(!obj->mark()->has_bias_pattern(), "biases should be revoked by now");
+ }
+
+ slow_enter (obj, lock, THREAD) ;
+}
+
+void ObjectSynchronizer::fast_exit(oop object, BasicLock* lock, TRAPS) {
+  assert(!object->mark()->has_bias_pattern(), "should not see bias pattern here");
+  // if displaced header is null, the previous enter is recursive enter, no-op
+  markOop dhw = lock->displaced_header();
+  markOop mark ;
+  if (dhw == NULL) {
+     // Recursive stack-lock.
+     // Diagnostics -- Could be: stack-locked, inflating, inflated.
+     mark = object->mark() ;
+     assert (!mark->is_neutral(), "invariant") ;
+     if (mark->has_locker() && mark != markOopDesc::INFLATING()) {
+        assert(THREAD->is_lock_owned((address)mark->locker()), "invariant") ;
+     }
+     if (mark->has_monitor()) {
+        ObjectMonitor * m = mark->monitor() ;
+        assert(((oop)(m->object()))->mark() == mark, "invariant") ;
+        assert(m->is_entered(THREAD), "invariant") ;
+     }
+     return ;
   }
 
-  void wait_reenter_begin(ObjectMonitor *mon) {
-    JavaThread *jt = (JavaThread *)this->_thread;
-    _active = JavaThreadBlockedOnMonitorEnterState::wait_reenter_begin(jt, mon);
+  mark = object->mark() ;
+
+  // If the object is stack-locked by the current thread, try to
+  // swing the displaced header from the box back to the mark.
+  if (mark == (markOop) lock) {
+     assert (dhw->is_neutral(), "invariant") ;
+     if ((markOop) Atomic::cmpxchg_ptr (dhw, object->mark_addr(), mark) == mark) {
+        TEVENT (fast_exit: release stacklock) ;
+        return;
+     }
   }
 
-  void wait_reenter_end(ObjectMonitor *mon) {
-    JavaThread *jt = (JavaThread *)this->_thread;
-    JavaThreadBlockedOnMonitorEnterState::wait_reenter_end(jt, _active);
+  ObjectSynchronizer::inflate(THREAD, object)->exit (THREAD) ;
+}
+
+// -----------------------------------------------------------------------------
+// Interpreter/Compiler Slow Case
+// This routine is used to handle interpreter/compiler slow case
+// We don't need to use fast path here, because it must have been
+// failed in the interpreter/compiler code.
+void ObjectSynchronizer::slow_enter(Handle obj, BasicLock* lock, TRAPS) {
+  markOop mark = obj->mark();
+  assert(!mark->has_bias_pattern(), "should not see bias pattern here");
+
+  if (mark->is_neutral()) {
+    // Anticipate successful CAS -- the ST of the displaced mark must
+    // be visible <= the ST performed by the CAS.
+    lock->set_displaced_header(mark);
+    if (mark == (markOop) Atomic::cmpxchg_ptr(lock, obj()->mark_addr(), mark)) {
+      TEVENT (slow_enter: release stacklock) ;
+      return ;
+    }
+    // Fall through to inflate() ...
+  } else
+  if (mark->has_locker() && THREAD->is_lock_owned((address)mark->locker())) {
+    assert(lock != mark->locker(), "must not re-lock the same lock");
+    assert(lock != (BasicLock*)obj->mark(), "don't relock with same BasicLock");
+    lock->set_displaced_header(NULL);
+    return;
   }
-};
 
-enum ManifestConstants {
-    ClearResponsibleAtSTW   = 0,
-    MaximumRecheckInterval  = 1000
-} ;
+#if 0
+  // The following optimization isn't particularly useful.
+  if (mark->has_monitor() && mark->monitor()->is_entered(THREAD)) {
+    lock->set_displaced_header (NULL) ;
+    return ;
+  }
+#endif
+
+  // The object header will never be displaced to this lock,
+  // so it does not matter what the value is, except that it
+  // must be non-zero to avoid looking like a re-entrant lock,
+  // and must not look locked either.
+  lock->set_displaced_header(markOopDesc::unused_mark());
+  ObjectSynchronizer::inflate(THREAD, obj())->enter(THREAD);
+}
+
+// This routine is used to handle interpreter/compiler slow case
+// We don't need to use fast path here, because it must have
+// failed in the interpreter/compiler code. Simply use the heavy
+// weight monitor should be ok, unless someone find otherwise.
+void ObjectSynchronizer::slow_exit(oop object, BasicLock* lock, TRAPS) {
+  fast_exit (object, lock, THREAD) ;
+}
+
+// -----------------------------------------------------------------------------
+// Class Loader  support to workaround deadlocks on the class loader lock objects
+// Also used by GC
+// complete_exit()/reenter() are used to wait on a nested lock
+// i.e. to give up an outer lock completely and then re-enter
+// Used when holding nested locks - lock acquisition order: lock1 then lock2
+//  1) complete_exit lock1 - saving recursion count
+//  2) wait on lock2
+//  3) when notified on lock2, unlock lock2
+//  4) reenter lock1 with original recursion count
+//  5) lock lock2
+// NOTE: must use heavy weight monitor to handle complete_exit/reenter()
+intptr_t ObjectSynchronizer::complete_exit(Handle obj, TRAPS) {
+  TEVENT (complete_exit) ;
+  if (UseBiasedLocking) {
+    BiasedLocking::revoke_and_rebias(obj, false, THREAD);
+    assert(!obj->mark()->has_bias_pattern(), "biases should be revoked by now");
+  }
+
+  ObjectMonitor* monitor = ObjectSynchronizer::inflate(THREAD, obj());
+
+  return monitor->complete_exit(THREAD);
+}
+
+// NOTE: must use heavy weight monitor to handle complete_exit/reenter()
+void ObjectSynchronizer::reenter(Handle obj, intptr_t recursion, TRAPS) {
+  TEVENT (reenter) ;
+  if (UseBiasedLocking) {
+    BiasedLocking::revoke_and_rebias(obj, false, THREAD);
+    assert(!obj->mark()->has_bias_pattern(), "biases should be revoked by now");
+  }
+
+  ObjectMonitor* monitor = ObjectSynchronizer::inflate(THREAD, obj());
+
+  monitor->reenter(recursion, THREAD);
+}
+// -----------------------------------------------------------------------------
+// JNI locks on java objects
+// NOTE: must use heavy weight monitor to handle jni monitor enter
+void ObjectSynchronizer::jni_enter(Handle obj, TRAPS) { // possible entry from jni enter
+  // the current locking is from JNI instead of Java code
+  TEVENT (jni_enter) ;
+  if (UseBiasedLocking) {
+    BiasedLocking::revoke_and_rebias(obj, false, THREAD);
+    assert(!obj->mark()->has_bias_pattern(), "biases should be revoked by now");
+  }
+  THREAD->set_current_pending_monitor_is_from_java(false);
+  ObjectSynchronizer::inflate(THREAD, obj())->enter(THREAD);
+  THREAD->set_current_pending_monitor_is_from_java(true);
+}
+
+// NOTE: must use heavy weight monitor to handle jni monitor enter
+bool ObjectSynchronizer::jni_try_enter(Handle obj, Thread* THREAD) {
+  if (UseBiasedLocking) {
+    BiasedLocking::revoke_and_rebias(obj, false, THREAD);
+    assert(!obj->mark()->has_bias_pattern(), "biases should be revoked by now");
+  }
+
+  ObjectMonitor* monitor = ObjectSynchronizer::inflate_helper(obj());
+  return monitor->try_enter(THREAD);
+}
 
 
-#undef TEVENT
-#define TEVENT(nom) {if (SyncVerbose) FEVENT(nom); }
+// NOTE: must use heavy weight monitor to handle jni monitor exit
+void ObjectSynchronizer::jni_exit(oop obj, Thread* THREAD) {
+  TEVENT (jni_exit) ;
+  if (UseBiasedLocking) {
+    BiasedLocking::revoke_and_rebias(obj, false, THREAD);
+  }
+  assert(!obj->mark()->has_bias_pattern(), "biases should be revoked by now");
 
-#define FEVENT(nom) { static volatile int ctr = 0 ; int v = ++ctr ; if ((v & (v-1)) == 0) { ::printf (#nom " : %d \n", v); ::fflush(stdout); }}
+  ObjectMonitor* monitor = ObjectSynchronizer::inflate(THREAD, obj);
+  // If this thread has locked the object, exit the monitor.  Note:  can't use
+  // monitor->check(CHECK); must exit even if an exception is pending.
+  if (monitor->check(THREAD)) {
+     monitor->exit(THREAD);
+  }
+}
 
-#undef  TEVENT
-#define TEVENT(nom) {;}
+// -----------------------------------------------------------------------------
+// Internal VM locks on java objects
+// standard constructor, allows locking failures
+ObjectLocker::ObjectLocker(Handle obj, Thread* thread, bool doLock) {
+  _dolock = doLock;
+  _thread = thread;
+  debug_only(if (StrictSafepointChecks) _thread->check_for_valid_safepoint_state(false);)
+  _obj = obj;
 
+  if (_dolock) {
+    TEVENT (ObjectLocker) ;
+
+    ObjectSynchronizer::fast_enter(_obj, &_lock, false, _thread);
+  }
+}
+
+ObjectLocker::~ObjectLocker() {
+  if (_dolock) {
+    ObjectSynchronizer::fast_exit(_obj(), &_lock, _thread);
+  }
+}
+
+
+// -----------------------------------------------------------------------------
+//  Wait/Notify/NotifyAll
+// NOTE: must use heavy weight monitor to handle wait()
+void ObjectSynchronizer::wait(Handle obj, jlong millis, TRAPS) {
+  if (UseBiasedLocking) {
+    BiasedLocking::revoke_and_rebias(obj, false, THREAD);
+    assert(!obj->mark()->has_bias_pattern(), "biases should be revoked by now");
+  }
+  if (millis < 0) {
+    TEVENT (wait - throw IAX) ;
+    THROW_MSG(vmSymbols::java_lang_IllegalArgumentException(), "timeout value is negative");
+  }
+  ObjectMonitor* monitor = ObjectSynchronizer::inflate(THREAD, obj());
+  DTRACE_MONITOR_WAIT_PROBE(monitor, obj(), THREAD, millis);
+  monitor->wait(millis, true, THREAD);
+
+  /* This dummy call is in place to get around dtrace bug 6254741.  Once
+     that's fixed we can uncomment the following line and remove the call */
+  // DTRACE_MONITOR_PROBE(waited, monitor, obj(), THREAD);
+  dtrace_waited_probe(monitor, obj, THREAD);
+}
+
+void ObjectSynchronizer::waitUninterruptibly (Handle obj, jlong millis, TRAPS) {
+  if (UseBiasedLocking) {
+    BiasedLocking::revoke_and_rebias(obj, false, THREAD);
+    assert(!obj->mark()->has_bias_pattern(), "biases should be revoked by now");
+  }
+  if (millis < 0) {
+    TEVENT (wait - throw IAX) ;
+    THROW_MSG(vmSymbols::java_lang_IllegalArgumentException(), "timeout value is negative");
+  }
+  ObjectSynchronizer::inflate(THREAD, obj()) -> wait(millis, false, THREAD) ;
+}
+
+void ObjectSynchronizer::notify(Handle obj, TRAPS) {
+ if (UseBiasedLocking) {
+    BiasedLocking::revoke_and_rebias(obj, false, THREAD);
+    assert(!obj->mark()->has_bias_pattern(), "biases should be revoked by now");
+  }
+
+  markOop mark = obj->mark();
+  if (mark->has_locker() && THREAD->is_lock_owned((address)mark->locker())) {
+    return;
+  }
+  ObjectSynchronizer::inflate(THREAD, obj())->notify(THREAD);
+}
+
+// NOTE: see comment of notify()
+void ObjectSynchronizer::notifyall(Handle obj, TRAPS) {
+  if (UseBiasedLocking) {
+    BiasedLocking::revoke_and_rebias(obj, false, THREAD);
+    assert(!obj->mark()->has_bias_pattern(), "biases should be revoked by now");
+  }
+
+  markOop mark = obj->mark();
+  if (mark->has_locker() && THREAD->is_lock_owned((address)mark->locker())) {
+    return;
+  }
+  ObjectSynchronizer::inflate(THREAD, obj())->notifyAll(THREAD);
+}
+
+// -----------------------------------------------------------------------------
+// Hash Code handling
+//
 // Performance concern:
 // OrderAccess::storestore() calls release() which STs 0 into the global volatile
 // OrderAccess::Dummy variable.  This store is unnecessary for correctness.
@@ -188,44 +414,73 @@ static SharedGlobals GVars ;
 static int MonitorScavengeThreshold = 1000000 ;
 static volatile int ForceMonitorScavenge = 0 ; // Scavenge required and pending
 
+static markOop ReadStableMark (oop obj) {
+  markOop mark = obj->mark() ;
+  if (!mark->is_being_inflated()) {
+    return mark ;       // normal fast-path return
+  }
 
-// Tunables ...
-// The knob* variables are effectively final.  Once set they should
-// never be modified hence.  Consider using __read_mostly with GCC.
+  int its = 0 ;
+  for (;;) {
+    markOop mark = obj->mark() ;
+    if (!mark->is_being_inflated()) {
+      return mark ;    // normal fast-path return
+    }
 
-static int Knob_LogSpins           = 0 ;       // enable jvmstat tally for spins
-static int Knob_HandOff            = 0 ;
-static int Knob_Verbose            = 0 ;
-static int Knob_ReportSettings     = 0 ;
-
-static int Knob_SpinLimit          = 5000 ;    // derived by an external tool -
-static int Knob_SpinBase           = 0 ;       // Floor AKA SpinMin
-static int Knob_SpinBackOff        = 0 ;       // spin-loop backoff
-static int Knob_CASPenalty         = -1 ;      // Penalty for failed CAS
-static int Knob_OXPenalty          = -1 ;      // Penalty for observed _owner change
-static int Knob_SpinSetSucc        = 1 ;       // spinners set the _succ field
-static int Knob_SpinEarly          = 1 ;
-static int Knob_SuccEnabled        = 1 ;       // futile wake throttling
-static int Knob_SuccRestrict       = 0 ;       // Limit successors + spinners to at-most-one
-static int Knob_MaxSpinners        = -1 ;      // Should be a function of # CPUs
-static int Knob_Bonus              = 100 ;     // spin success bonus
-static int Knob_BonusB             = 100 ;     // spin success bonus
-static int Knob_Penalty            = 200 ;     // spin failure penalty
-static int Knob_Poverty            = 1000 ;
-static int Knob_SpinAfterFutile    = 1 ;       // Spin after returning from park()
-static int Knob_FixedSpin          = 0 ;
-static int Knob_OState             = 3 ;       // Spinner checks thread state of _owner
-static int Knob_UsePause           = 1 ;
-static int Knob_ExitPolicy         = 0 ;
-static int Knob_PreSpin            = 10 ;      // 20-100 likely better
-static int Knob_ResetEvent         = 0 ;
-static int BackOffMask             = 0 ;
-
-static int Knob_FastHSSEC          = 0 ;
-static int Knob_MoveNotifyee       = 2 ;       // notify() - disposition of notifyee
-static int Knob_QMode              = 0 ;       // EntryList-cxq policy - queue discipline
-static volatile int InitDone       = 0 ;
+    // The object is being inflated by some other thread.
+    // The caller of ReadStableMark() must wait for inflation to complete.
+    // Avoid live-lock
+    // TODO: consider calling SafepointSynchronize::do_call_back() while
+    // spinning to see if there's a safepoint pending.  If so, immediately
+    // yielding or blocking would be appropriate.  Avoid spinning while
+    // there is a safepoint pending.
+    // TODO: add inflation contention performance counters.
+    // TODO: restrict the aggregate number of spinners.
 
+    ++its ;
+    if (its > 10000 || !os::is_MP()) {
+       if (its & 1) {
+         os::NakedYield() ;
+         TEVENT (Inflate: INFLATING - yield) ;
+       } else {
+         // Note that the following code attenuates the livelock problem but is not
+         // a complete remedy.  A more complete solution would require that the inflating
+         // thread hold the associated inflation lock.  The following code simply restricts
+         // the number of spinners to at most one.  We'll have N-2 threads blocked
+         // on the inflationlock, 1 thread holding the inflation lock and using
+         // a yield/park strategy, and 1 thread in the midst of inflation.
+         // A more refined approach would be to change the encoding of INFLATING
+         // to allow encapsulation of a native thread pointer.  Threads waiting for
+         // inflation to complete would use CAS to push themselves onto a singly linked
+         // list rooted at the markword.  Once enqueued, they'd loop, checking a per-thread flag
+         // and calling park().  When inflation was complete the thread that accomplished inflation
+         // would detach the list and set the markword to inflated with a single CAS and
+         // then for each thread on the list, set the flag and unpark() the thread.
+         // This is conceptually similar to muxAcquire-muxRelease, except that muxRelease
+         // wakes at most one thread whereas we need to wake the entire list.
+         int ix = (intptr_t(obj) >> 5) & (NINFLATIONLOCKS-1) ;
+         int YieldThenBlock = 0 ;
+         assert (ix >= 0 && ix < NINFLATIONLOCKS, "invariant") ;
+         assert ((NINFLATIONLOCKS & (NINFLATIONLOCKS-1)) == 0, "invariant") ;
+         Thread::muxAcquire (InflationLocks + ix, "InflationLock") ;
+         while (obj->mark() == markOopDesc::INFLATING()) {
+           // Beware: NakedYield() is advisory and has almost no effect on some platforms
+           // so we periodically call Self->_ParkEvent->park(1).
+           // We use a mixed spin/yield/block mechanism.
+           if ((YieldThenBlock++) >= 16) {
+              Thread::current()->_ParkEvent->park(1) ;
+           } else {
+              os::NakedYield() ;
+           }
+         }
+         Thread::muxRelease (InflationLocks + ix ) ;
+         TEVENT (Inflate: INFLATING - yield/park) ;
+       }
+    } else {
+       SpinPause() ;       // SMP-polite spinning
+    }
+  }
+}
 
 // hashCode() generation :
 //
@@ -290,416 +545,272 @@ static inline intptr_t get_next_hash(Thread * Self, oop obj) {
   TEVENT (hashCode: GENERATE) ;
   return value;
 }
-
-void BasicLock::print_on(outputStream* st) const {
-  st->print("monitor");
-}
-
-void BasicLock::move_to(oop obj, BasicLock* dest) {
-  // Check to see if we need to inflate the lock. This is only needed
-  // if an object is locked using "this" lightweight monitor. In that
-  // case, the displaced_header() is unlocked, because the
-  // displaced_header() contains the header for the originally unlocked
-  // object. However the object could have already been inflated. But it
-  // does not matter, the inflation will just a no-op. For other cases,
-  // the displaced header will be either 0x0 or 0x3, which are location
-  // independent, therefore the BasicLock is free to move.
-  //
-  // During OSR we may need to relocate a BasicLock (which contains a
-  // displaced word) from a location in an interpreter frame to a
-  // new location in a compiled frame.  "this" refers to the source
-  // basiclock in the interpreter frame.  "dest" refers to the destination
-  // basiclock in the new compiled frame.  We *always* inflate in move_to().
-  // The always-Inflate policy works properly, but in 1.5.0 it can sometimes
-  // cause performance problems in code that makes heavy use of a small # of
-  // uncontended locks.   (We'd inflate during OSR, and then sync performance
-  // would subsequently plummet because the thread would be forced thru the slow-path).
-  // This problem has been made largely moot on IA32 by inlining the inflated fast-path
-  // operations in Fast_Lock and Fast_Unlock in i486.ad.
-  //
-  // Note that there is a way to safely swing the object's markword from
-  // one stack location to another.  This avoids inflation.  Obviously,
-  // we need to ensure that both locations refer to the current thread's stack.
-  // There are some subtle concurrency issues, however, and since the benefit is
-  // is small (given the support for inflated fast-path locking in the fast_lock, etc)
-  // we'll leave that optimization for another time.
-
-  if (displaced_header()->is_neutral()) {
-    ObjectSynchronizer::inflate_helper(obj);
-    // WARNING: We can not put check here, because the inflation
-    // will not update the displaced header. Once BasicLock is inflated,
-    // no one should ever look at its content.
-  } else {
-    // Typically the displaced header will be 0 (recursive stack lock) or
-    // unused_mark.  Naively we'd like to assert that the displaced mark
-    // value is either 0, neutral, or 3.  But with the advent of the
-    // store-before-CAS avoidance in fast_lock/compiler_lock_object
-    // we can find any flavor mark in the displaced mark.
-  }
-// [RGV] The next line appears to do nothing!
-  intptr_t dh = (intptr_t) displaced_header();
-  dest->set_displaced_header(displaced_header());
-}
-
-// -----------------------------------------------------------------------------
-
-// standard constructor, allows locking failures
-ObjectLocker::ObjectLocker(Handle obj, Thread* thread, bool doLock) {
-  _dolock = doLock;
-  _thread = thread;
-  debug_only(if (StrictSafepointChecks) _thread->check_for_valid_safepoint_state(false);)
-  _obj = obj;
-
-  if (_dolock) {
-    TEVENT (ObjectLocker) ;
-
-    ObjectSynchronizer::fast_enter(_obj, &_lock, false, _thread);
-  }
-}
-
-ObjectLocker::~ObjectLocker() {
-  if (_dolock) {
-    ObjectSynchronizer::fast_exit(_obj(), &_lock, _thread);
-  }
-}
-
-// -----------------------------------------------------------------------------
-
-
-PerfCounter * ObjectSynchronizer::_sync_Inflations                  = NULL ;
-PerfCounter * ObjectSynchronizer::_sync_Deflations                  = NULL ;
-PerfCounter * ObjectSynchronizer::_sync_ContendedLockAttempts       = NULL ;
-PerfCounter * ObjectSynchronizer::_sync_FutileWakeups               = NULL ;
-PerfCounter * ObjectSynchronizer::_sync_Parks                       = NULL ;
-PerfCounter * ObjectSynchronizer::_sync_EmptyNotifications          = NULL ;
-PerfCounter * ObjectSynchronizer::_sync_Notifications               = NULL ;
-PerfCounter * ObjectSynchronizer::_sync_PrivateA                    = NULL ;
-PerfCounter * ObjectSynchronizer::_sync_PrivateB                    = NULL ;
-PerfCounter * ObjectSynchronizer::_sync_SlowExit                    = NULL ;
-PerfCounter * ObjectSynchronizer::_sync_SlowEnter                   = NULL ;
-PerfCounter * ObjectSynchronizer::_sync_SlowNotify                  = NULL ;
-PerfCounter * ObjectSynchronizer::_sync_SlowNotifyAll               = NULL ;
-PerfCounter * ObjectSynchronizer::_sync_FailedSpins                 = NULL ;
-PerfCounter * ObjectSynchronizer::_sync_SuccessfulSpins             = NULL ;
-PerfCounter * ObjectSynchronizer::_sync_MonInCirculation            = NULL ;
-PerfCounter * ObjectSynchronizer::_sync_MonScavenged                = NULL ;
-PerfLongVariable * ObjectSynchronizer::_sync_MonExtant              = NULL ;
-
-// One-shot global initialization for the sync subsystem.
-// We could also defer initialization and initialize on-demand
-// the first time we call inflate().  Initialization would
-// be protected - like so many things - by the MonitorCache_lock.
-
-void ObjectSynchronizer::Initialize () {
-  static int InitializationCompleted = 0 ;
-  assert (InitializationCompleted == 0, "invariant") ;
-  InitializationCompleted = 1 ;
-  if (UsePerfData) {
-      EXCEPTION_MARK ;
-      #define NEWPERFCOUNTER(n)   {n = PerfDataManager::create_counter(SUN_RT, #n, PerfData::U_Events,CHECK); }
-      #define NEWPERFVARIABLE(n)  {n = PerfDataManager::create_variable(SUN_RT, #n, PerfData::U_Events,CHECK); }
-      NEWPERFCOUNTER(_sync_Inflations) ;
-      NEWPERFCOUNTER(_sync_Deflations) ;
-      NEWPERFCOUNTER(_sync_ContendedLockAttempts) ;
-      NEWPERFCOUNTER(_sync_FutileWakeups) ;
-      NEWPERFCOUNTER(_sync_Parks) ;
-      NEWPERFCOUNTER(_sync_EmptyNotifications) ;
-      NEWPERFCOUNTER(_sync_Notifications) ;
-      NEWPERFCOUNTER(_sync_SlowEnter) ;
-      NEWPERFCOUNTER(_sync_SlowExit) ;
-      NEWPERFCOUNTER(_sync_SlowNotify) ;
-      NEWPERFCOUNTER(_sync_SlowNotifyAll) ;
-      NEWPERFCOUNTER(_sync_FailedSpins) ;
-      NEWPERFCOUNTER(_sync_SuccessfulSpins) ;
-      NEWPERFCOUNTER(_sync_PrivateA) ;
-      NEWPERFCOUNTER(_sync_PrivateB) ;
-      NEWPERFCOUNTER(_sync_MonInCirculation) ;
-      NEWPERFCOUNTER(_sync_MonScavenged) ;
-      NEWPERFVARIABLE(_sync_MonExtant) ;
-      #undef NEWPERFCOUNTER
-  }
-}
-
-// Compile-time asserts
-// When possible, it's better to catch errors deterministically at
-// compile-time than at runtime.  The down-side to using compile-time
-// asserts is that error message -- often something about negative array
-// indices -- is opaque.
-
-#define CTASSERT(x) { int tag[1-(2*!(x))]; printf ("Tag @" INTPTR_FORMAT "\n", (intptr_t)tag); }
-
-void ObjectMonitor::ctAsserts() {
-  CTASSERT(offset_of (ObjectMonitor, _header) == 0);
-}
-
-static int Adjust (volatile int * adr, int dx) {
-  int v ;
-  for (v = *adr ; Atomic::cmpxchg (v + dx, adr, v) != v; v = *adr) ;
-  return v ;
-}
-
-// Ad-hoc mutual exclusion primitives: SpinLock and Mux
 //
-// We employ SpinLocks _only for low-contention, fixed-length
-// short-duration critical sections where we're concerned
-// about native mutex_t or HotSpot Mutex:: latency.
-// The mux construct provides a spin-then-block mutual exclusion
-// mechanism.
-//
-// Testing has shown that contention on the ListLock guarding gFreeList
-// is common.  If we implement ListLock as a simple SpinLock it's common
-// for the JVM to devolve to yielding with little progress.  This is true
-// despite the fact that the critical sections protected by ListLock are
-// extremely short.
-//
-// TODO-FIXME: ListLock should be of type SpinLock.
-// We should make this a 1st-class type, integrated into the lock
-// hierarchy as leaf-locks.  Critically, the SpinLock structure
-// should have sufficient padding to avoid false-sharing and excessive
-// cache-coherency traffic.
-
-
-typedef volatile int SpinLockT ;
-
-void Thread::SpinAcquire (volatile int * adr, const char * LockName) {
-  if (Atomic::cmpxchg (1, adr, 0) == 0) {
-     return ;   // normal fast-path return
+intptr_t ObjectSynchronizer::FastHashCode (Thread * Self, oop obj) {
+  if (UseBiasedLocking) {
+    // NOTE: many places throughout the JVM do not expect a safepoint
+    // to be taken here, in particular most operations on perm gen
+    // objects. However, we only ever bias Java instances and all of
+    // the call sites of identity_hash that might revoke biases have
+    // been checked to make sure they can handle a safepoint. The
+    // added check of the bias pattern is to avoid useless calls to
+    // thread-local storage.
+    if (obj->mark()->has_bias_pattern()) {
+      // Box and unbox the raw reference just in case we cause a STW safepoint.
+      Handle hobj (Self, obj) ;
+      // Relaxing assertion for bug 6320749.
+      assert (Universe::verify_in_progress() ||
+              !SafepointSynchronize::is_at_safepoint(),
+             "biases should not be seen by VM thread here");
+      BiasedLocking::revoke_and_rebias(hobj, false, JavaThread::current());
+      obj = hobj() ;
+      assert(!obj->mark()->has_bias_pattern(), "biases should be revoked by now");
+    }
   }
 
-  // Slow-path : We've encountered contention -- Spin/Yield/Block strategy.
-  TEVENT (SpinAcquire - ctx) ;
-  int ctr = 0 ;
-  int Yields = 0 ;
-  for (;;) {
-     while (*adr != 0) {
-        ++ctr ;
-        if ((ctr & 0xFFF) == 0 || !os::is_MP()) {
-           if (Yields > 5) {
-             // Consider using a simple NakedSleep() instead.
-             // Then SpinAcquire could be called by non-JVM threads
-             Thread::current()->_ParkEvent->park(1) ;
-           } else {
-             os::NakedYield() ;
-             ++Yields ;
-           }
-        } else {
-           SpinPause() ;
-        }
-     }
-     if (Atomic::cmpxchg (1, adr, 0) == 0) return ;
+  // hashCode() is a heap mutator ...
+  // Relaxing assertion for bug 6320749.
+  assert (Universe::verify_in_progress() ||
+          !SafepointSynchronize::is_at_safepoint(), "invariant") ;
+  assert (Universe::verify_in_progress() ||
+          Self->is_Java_thread() , "invariant") ;
+  assert (Universe::verify_in_progress() ||
+         ((JavaThread *)Self)->thread_state() != _thread_blocked, "invariant") ;
+
+  ObjectMonitor* monitor = NULL;
+  markOop temp, test;
+  intptr_t hash;
+  markOop mark = ReadStableMark (obj);
+
+  // object should remain ineligible for biased locking
+  assert (!mark->has_bias_pattern(), "invariant") ;
+
+  if (mark->is_neutral()) {
+    hash = mark->hash();              // this is a normal header
+    if (hash) {                       // if it has hash, just return it
+      return hash;
+    }
+    hash = get_next_hash(Self, obj);  // allocate a new hash code
+    temp = mark->copy_set_hash(hash); // merge the hash code into header
+    // use (machine word version) atomic operation to install the hash
+    test = (markOop) Atomic::cmpxchg_ptr(temp, obj->mark_addr(), mark);
+    if (test == mark) {
+      return hash;
+    }
+    // If atomic operation failed, we must inflate the header
+    // into heavy weight monitor. We could add more code here
+    // for fast path, but it does not worth the complexity.
+  } else if (mark->has_monitor()) {
+    monitor = mark->monitor();
+    temp = monitor->header();
+    assert (temp->is_neutral(), "invariant") ;
+    hash = temp->hash();
+    if (hash) {
+      return hash;
+    }
+    // Skip to the following code to reduce code size
+  } else if (Self->is_lock_owned((address)mark->locker())) {
+    temp = mark->displaced_mark_helper(); // this is a lightweight monitor owned
+    assert (temp->is_neutral(), "invariant") ;
+    hash = temp->hash();              // by current thread, check if the displaced
+    if (hash) {                       // header contains hash code
+      return hash;
+    }
+    // WARNING:
+    //   The displaced header is strictly immutable.
+    // It can NOT be changed in ANY cases. So we have
+    // to inflate the header into heavyweight monitor
+    // even the current thread owns the lock. The reason
+    // is the BasicLock (stack slot) will be asynchronously
+    // read by other threads during the inflate() function.
+    // Any change to stack may not propagate to other threads
+    // correctly.
   }
+
+  // Inflate the monitor to set hash code
+  monitor = ObjectSynchronizer::inflate(Self, obj);
+  // Load displaced header and check it has hash code
+  mark = monitor->header();
+  assert (mark->is_neutral(), "invariant") ;
+  hash = mark->hash();
+  if (hash == 0) {
+    hash = get_next_hash(Self, obj);
+    temp = mark->copy_set_hash(hash); // merge hash code into header
+    assert (temp->is_neutral(), "invariant") ;
+    test = (markOop) Atomic::cmpxchg_ptr(temp, monitor, mark);
+    if (test != mark) {
+      // The only update to the header in the monitor (outside GC)
+      // is install the hash code. If someone add new usage of
+      // displaced header, please update this code
+      hash = test->hash();
+      assert (test->is_neutral(), "invariant") ;
+      assert (hash != 0, "Trivial unexpected object/monitor header usage.");
+    }
+  }
+  // We finally get the hash
+  return hash;
 }
 
-void Thread::SpinRelease (volatile int * adr) {
-  assert (*adr != 0, "invariant") ;
-  OrderAccess::fence() ;      // guarantee at least release consistency.
-  // Roach-motel semantics.
-  // It's safe if subsequent LDs and STs float "up" into the critical section,
-  // but prior LDs and STs within the critical section can't be allowed
-  // to reorder or float past the ST that releases the lock.
-  *adr = 0 ;
+// Deprecated -- use FastHashCode() instead.
+
+intptr_t ObjectSynchronizer::identity_hash_value_for(Handle obj) {
+  return FastHashCode (Thread::current(), obj()) ;
 }
 
-// muxAcquire and muxRelease:
-//
-// *  muxAcquire and muxRelease support a single-word lock-word construct.
-//    The LSB of the word is set IFF the lock is held.
-//    The remainder of the word points to the head of a singly-linked list
-//    of threads blocked on the lock.
-//
-// *  The current implementation of muxAcquire-muxRelease uses its own
-//    dedicated Thread._MuxEvent instance.  If we're interested in
-//    minimizing the peak number of extant ParkEvent instances then
-//    we could eliminate _MuxEvent and "borrow" _ParkEvent as long
-//    as certain invariants were satisfied.  Specifically, care would need
-//    to be taken with regards to consuming unpark() "permits".
-//    A safe rule of thumb is that a thread would never call muxAcquire()
-//    if it's enqueued (cxq, EntryList, WaitList, etc) and will subsequently
-//    park().  Otherwise the _ParkEvent park() operation in muxAcquire() could
-//    consume an unpark() permit intended for monitorenter, for instance.
-//    One way around this would be to widen the restricted-range semaphore
-//    implemented in park().  Another alternative would be to provide
-//    multiple instances of the PlatformEvent() for each thread.  One
-//    instance would be dedicated to muxAcquire-muxRelease, for instance.
-//
-// *  Usage:
-//    -- Only as leaf locks
-//    -- for short-term locking only as muxAcquire does not perform
-//       thread state transitions.
-//
-// Alternatives:
-// *  We could implement muxAcquire and muxRelease with MCS or CLH locks
-//    but with parking or spin-then-park instead of pure spinning.
-// *  Use Taura-Oyama-Yonenzawa locks.
-// *  It's possible to construct a 1-0 lock if we encode the lockword as
-//    (List,LockByte).  Acquire will CAS the full lockword while Release
-//    will STB 0 into the LockByte.  The 1-0 scheme admits stranding, so
-//    acquiring threads use timers (ParkTimed) to detect and recover from
-//    the stranding window.  Thread/Node structures must be aligned on 256-byte
-//    boundaries by using placement-new.
-// *  Augment MCS with advisory back-link fields maintained with CAS().
-//    Pictorially:  LockWord -> T1 <-> T2 <-> T3 <-> ... <-> Tn <-> Owner.
-//    The validity of the backlinks must be ratified before we trust the value.
-//    If the backlinks are invalid the exiting thread must back-track through the
-//    the forward links, which are always trustworthy.
-// *  Add a successor indication.  The LockWord is currently encoded as
-//    (List, LOCKBIT:1).  We could also add a SUCCBIT or an explicit _succ variable
-//    to provide the usual futile-wakeup optimization.
-//    See RTStt for details.
-// *  Consider schedctl.sc_nopreempt to cover the critical section.
-//
 
-
-typedef volatile intptr_t MutexT ;      // Mux Lock-word
-enum MuxBits { LOCKBIT = 1 } ;
-
-void Thread::muxAcquire (volatile intptr_t * Lock, const char * LockName) {
-  intptr_t w = Atomic::cmpxchg_ptr (LOCKBIT, Lock, 0) ;
-  if (w == 0) return ;
-  if ((w & LOCKBIT) == 0 && Atomic::cmpxchg_ptr (w|LOCKBIT, Lock, w) == w) {
-     return ;
+bool ObjectSynchronizer::current_thread_holds_lock(JavaThread* thread,
+                                                   Handle h_obj) {
+  if (UseBiasedLocking) {
+    BiasedLocking::revoke_and_rebias(h_obj, false, thread);
+    assert(!h_obj->mark()->has_bias_pattern(), "biases should be revoked by now");
   }
 
-  TEVENT (muxAcquire - Contention) ;
-  ParkEvent * const Self = Thread::current()->_MuxEvent ;
-  assert ((intptr_t(Self) & LOCKBIT) == 0, "invariant") ;
-  for (;;) {
-     int its = (os::is_MP() ? 100 : 0) + 1 ;
+  assert(thread == JavaThread::current(), "Can only be called on current thread");
+  oop obj = h_obj();
 
-     // Optional spin phase: spin-then-park strategy
-     while (--its >= 0) {
-       w = *Lock ;
-       if ((w & LOCKBIT) == 0 && Atomic::cmpxchg_ptr (w|LOCKBIT, Lock, w) == w) {
-          return ;
-       }
-     }
+  markOop mark = ReadStableMark (obj) ;
 
-     Self->reset() ;
-     Self->OnList = intptr_t(Lock) ;
-     // The following fence() isn't _strictly necessary as the subsequent
-     // CAS() both serializes execution and ratifies the fetched *Lock value.
-     OrderAccess::fence();
-     for (;;) {
-        w = *Lock ;
-        if ((w & LOCKBIT) == 0) {
-            if (Atomic::cmpxchg_ptr (w|LOCKBIT, Lock, w) == w) {
-                Self->OnList = 0 ;   // hygiene - allows stronger asserts
-                return ;
-            }
-            continue ;      // Interference -- *Lock changed -- Just retry
-        }
-        assert (w & LOCKBIT, "invariant") ;
-        Self->ListNext = (ParkEvent *) (w & ~LOCKBIT );
-        if (Atomic::cmpxchg_ptr (intptr_t(Self)|LOCKBIT, Lock, w) == w) break ;
-     }
-
-     while (Self->OnList != 0) {
-        Self->park() ;
-     }
+  // Uncontended case, header points to stack
+  if (mark->has_locker()) {
+    return thread->is_lock_owned((address)mark->locker());
   }
+  // Contended case, header points to ObjectMonitor (tagged pointer)
+  if (mark->has_monitor()) {
+    ObjectMonitor* monitor = mark->monitor();
+    return monitor->is_entered(thread) != 0 ;
+  }
+  // Unlocked case, header in place
+  assert(mark->is_neutral(), "sanity check");
+  return false;
 }
 
-void Thread::muxAcquireW (volatile intptr_t * Lock, ParkEvent * ev) {
-  intptr_t w = Atomic::cmpxchg_ptr (LOCKBIT, Lock, 0) ;
-  if (w == 0) return ;
-  if ((w & LOCKBIT) == 0 && Atomic::cmpxchg_ptr (w|LOCKBIT, Lock, w) == w) {
-    return ;
+// Be aware of this method could revoke bias of the lock object.
+// This method querys the ownership of the lock handle specified by 'h_obj'.
+// If the current thread owns the lock, it returns owner_self. If no
+// thread owns the lock, it returns owner_none. Otherwise, it will return
+// ower_other.
+ObjectSynchronizer::LockOwnership ObjectSynchronizer::query_lock_ownership
+(JavaThread *self, Handle h_obj) {
+  // The caller must beware this method can revoke bias, and
+  // revocation can result in a safepoint.
+  assert (!SafepointSynchronize::is_at_safepoint(), "invariant") ;
+  assert (self->thread_state() != _thread_blocked , "invariant") ;
+
+  // Possible mark states: neutral, biased, stack-locked, inflated
+
+  if (UseBiasedLocking && h_obj()->mark()->has_bias_pattern()) {
+    // CASE: biased
+    BiasedLocking::revoke_and_rebias(h_obj, false, self);
+    assert(!h_obj->mark()->has_bias_pattern(),
+           "biases should be revoked by now");
   }
 
-  TEVENT (muxAcquire - Contention) ;
-  ParkEvent * ReleaseAfter = NULL ;
-  if (ev == NULL) {
-    ev = ReleaseAfter = ParkEvent::Allocate (NULL) ;
-  }
-  assert ((intptr_t(ev) & LOCKBIT) == 0, "invariant") ;
-  for (;;) {
-    guarantee (ev->OnList == 0, "invariant") ;
-    int its = (os::is_MP() ? 100 : 0) + 1 ;
+  assert(self == JavaThread::current(), "Can only be called on current thread");
+  oop obj = h_obj();
+  markOop mark = ReadStableMark (obj) ;
 
-    // Optional spin phase: spin-then-park strategy
-    while (--its >= 0) {
-      w = *Lock ;
-      if ((w & LOCKBIT) == 0 && Atomic::cmpxchg_ptr (w|LOCKBIT, Lock, w) == w) {
-        if (ReleaseAfter != NULL) {
-          ParkEvent::Release (ReleaseAfter) ;
-        }
-        return ;
+  // CASE: stack-locked.  Mark points to a BasicLock on the owner's stack.
+  if (mark->has_locker()) {
+    return self->is_lock_owned((address)mark->locker()) ?
+      owner_self : owner_other;
+  }
+
+  // CASE: inflated. Mark (tagged pointer) points to an objectMonitor.
+  // The Object:ObjectMonitor relationship is stable as long as we're
+  // not at a safepoint.
+  if (mark->has_monitor()) {
+    void * owner = mark->monitor()->_owner ;
+    if (owner == NULL) return owner_none ;
+    return (owner == self ||
+            self->is_lock_owned((address)owner)) ? owner_self : owner_other;
+  }
+
+  // CASE: neutral
+  assert(mark->is_neutral(), "sanity check");
+  return owner_none ;           // it's unlocked
+}
+
+// FIXME: jvmti should call this
+JavaThread* ObjectSynchronizer::get_lock_owner(Handle h_obj, bool doLock) {
+  if (UseBiasedLocking) {
+    if (SafepointSynchronize::is_at_safepoint()) {
+      BiasedLocking::revoke_at_safepoint(h_obj);
+    } else {
+      BiasedLocking::revoke_and_rebias(h_obj, false, JavaThread::current());
+    }
+    assert(!h_obj->mark()->has_bias_pattern(), "biases should be revoked by now");
+  }
+
+  oop obj = h_obj();
+  address owner = NULL;
+
+  markOop mark = ReadStableMark (obj) ;
+
+  // Uncontended case, header points to stack
+  if (mark->has_locker()) {
+    owner = (address) mark->locker();
+  }
+
+  // Contended case, header points to ObjectMonitor (tagged pointer)
+  if (mark->has_monitor()) {
+    ObjectMonitor* monitor = mark->monitor();
+    assert(monitor != NULL, "monitor should be non-null");
+    owner = (address) monitor->owner();
+  }
+
+  if (owner != NULL) {
+    return Threads::owning_thread_from_monitor_owner(owner, doLock);
+  }
+
+  // Unlocked case, header in place
+  // Cannot have assertion since this object may have been
+  // locked by another thread when reaching here.
+  // assert(mark->is_neutral(), "sanity check");
+
+  return NULL;
+}
+// Visitors ...
+
+void ObjectSynchronizer::monitors_iterate(MonitorClosure* closure) {
+  ObjectMonitor* block = gBlockList;
+  ObjectMonitor* mid;
+  while (block) {
+    assert(block->object() == CHAINMARKER, "must be a block header");
+    for (int i = _BLOCKSIZE - 1; i > 0; i--) {
+      mid = block + i;
+      oop object = (oop) mid->object();
+      if (object != NULL) {
+        closure->do_monitor(mid);
       }
     }
+    block = (ObjectMonitor*) block->FreeNext;
+  }
+}
 
-    ev->reset() ;
-    ev->OnList = intptr_t(Lock) ;
-    // The following fence() isn't _strictly necessary as the subsequent
-    // CAS() both serializes execution and ratifies the fetched *Lock value.
-    OrderAccess::fence();
-    for (;;) {
-      w = *Lock ;
-      if ((w & LOCKBIT) == 0) {
-        if (Atomic::cmpxchg_ptr (w|LOCKBIT, Lock, w) == w) {
-          ev->OnList = 0 ;
-          // We call ::Release while holding the outer lock, thus
-          // artificially lengthening the critical section.
-          // Consider deferring the ::Release() until the subsequent unlock(),
-          // after we've dropped the outer lock.
-          if (ReleaseAfter != NULL) {
-            ParkEvent::Release (ReleaseAfter) ;
-          }
-          return ;
-        }
-        continue ;      // Interference -- *Lock changed -- Just retry
+// Get the next block in the block list.
+static inline ObjectMonitor* next(ObjectMonitor* block) {
+  assert(block->object() == CHAINMARKER, "must be a block header");
+  block = block->FreeNext ;
+  assert(block == NULL || block->object() == CHAINMARKER, "must be a block header");
+  return block;
+}
+
+
+void ObjectSynchronizer::oops_do(OopClosure* f) {
+  assert(SafepointSynchronize::is_at_safepoint(), "must be at safepoint");
+  for (ObjectMonitor* block = gBlockList; block != NULL; block = next(block)) {
+    assert(block->object() == CHAINMARKER, "must be a block header");
+    for (int i = 1; i < _BLOCKSIZE; i++) {
+      ObjectMonitor* mid = &block[i];
+      if (mid->object() != NULL) {
+        f->do_oop((oop*)mid->object_addr());
       }
-      assert (w & LOCKBIT, "invariant") ;
-      ev->ListNext = (ParkEvent *) (w & ~LOCKBIT );
-      if (Atomic::cmpxchg_ptr (intptr_t(ev)|LOCKBIT, Lock, w) == w) break ;
-    }
-
-    while (ev->OnList != 0) {
-      ev->park() ;
     }
   }
 }
 
-// Release() must extract a successor from the list and then wake that thread.
-// It can "pop" the front of the list or use a detach-modify-reattach (DMR) scheme
-// similar to that used by ParkEvent::Allocate() and ::Release().  DMR-based
-// Release() would :
-// (A) CAS() or swap() null to *Lock, releasing the lock and detaching the list.
-// (B) Extract a successor from the private list "in-hand"
-// (C) attempt to CAS() the residual back into *Lock over null.
-//     If there were any newly arrived threads and the CAS() would fail.
-//     In that case Release() would detach the RATs, re-merge the list in-hand
-//     with the RATs and repeat as needed.  Alternately, Release() might
-//     detach and extract a successor, but then pass the residual list to the wakee.
-//     The wakee would be responsible for reattaching and remerging before it
-//     competed for the lock.
-//
-// Both "pop" and DMR are immune from ABA corruption -- there can be
-// multiple concurrent pushers, but only one popper or detacher.
-// This implementation pops from the head of the list.  This is unfair,
-// but tends to provide excellent throughput as hot threads remain hot.
-// (We wake recently run threads first).
-
-void Thread::muxRelease (volatile intptr_t * Lock)  {
-  for (;;) {
-    const intptr_t w = Atomic::cmpxchg_ptr (0, Lock, LOCKBIT) ;
-    assert (w & LOCKBIT, "invariant") ;
-    if (w == LOCKBIT) return ;
-    ParkEvent * List = (ParkEvent *) (w & ~LOCKBIT) ;
-    assert (List != NULL, "invariant") ;
-    assert (List->OnList == intptr_t(Lock), "invariant") ;
-    ParkEvent * nxt = List->ListNext ;
-
-    // The following CAS() releases the lock and pops the head element.
-    if (Atomic::cmpxchg_ptr (intptr_t(nxt), Lock, w) != w) {
-      continue ;
-    }
-    List->OnList = 0 ;
-    OrderAccess::fence() ;
-    List->unpark () ;
-    return ;
-  }
-}
 
+// -----------------------------------------------------------------------------
 // ObjectMonitor Lifecycle
 // -----------------------
 // Inflation unlinks monitors from the global gFreeList and
@@ -718,41 +829,7 @@ void Thread::muxRelease (volatile intptr_t * Lock)  {
 // --   assigned to an object.  The object is inflated and the mark refers
 //      to the objectmonitor.
 //
-// TODO-FIXME:
-//
-// *  We currently protect the gFreeList with a simple lock.
-//    An alternate lock-free scheme would be to pop elements from the gFreeList
-//    with CAS.  This would be safe from ABA corruption as long we only
-//    recycled previously appearing elements onto the list in deflate_idle_monitors()
-//    at STW-time.  Completely new elements could always be pushed onto the gFreeList
-//    with CAS.  Elements that appeared previously on the list could only
-//    be installed at STW-time.
-//
-// *  For efficiency and to help reduce the store-before-CAS penalty
-//    the objectmonitors on gFreeList or local free lists should be ready to install
-//    with the exception of _header and _object.  _object can be set after inflation.
-//    In particular, keep all objectMonitors on a thread's private list in ready-to-install
-//    state with m.Owner set properly.
-//
-// *  We could all diffuse contention by using multiple global (FreeList, Lock)
-//    pairs -- threads could use trylock() and a cyclic-scan strategy to search for
-//    an unlocked free list.
-//
-// *  Add lifecycle tags and assert()s.
-//
-// *  Be more consistent about when we clear an objectmonitor's fields:
-//    A.  After extracting the objectmonitor from a free list.
-//    B.  After adding an objectmonitor to a free list.
-//
 
-ObjectMonitor * ObjectSynchronizer::gBlockList = NULL ;
-ObjectMonitor * volatile ObjectSynchronizer::gFreeList  = NULL ;
-ObjectMonitor * volatile ObjectSynchronizer::gOmInUseList  = NULL ;
-int ObjectSynchronizer::gOmInUseCount = 0;
-static volatile intptr_t ListLock = 0 ;      // protects global monitor free-list cache
-static volatile int MonitorFreeCount  = 0 ;      // # on gFreeList
-static volatile int MonitorPopulation = 0 ;      // # Extant -- in circulation
-#define CHAINMARKER ((oop)-1)
 
 // Constraining monitor pool growth via MonitorBound ...
 //
@@ -768,41 +845,8 @@ static volatile int MonitorPopulation = 0 ;      // # Extant -- in circulation
 // we'll incur more safepoints, which are harmful to performance.
 // See also: GuaranteedSafepointInterval
 //
-// As noted elsewhere, the correct long-term solution is to deflate at
-// monitorexit-time, in which case the number of inflated objects is bounded
-// by the number of threads.  That policy obviates the need for scavenging at
-// STW safepoint time.   As an aside, scavenging can be time-consuming when the
-// # of extant monitors is large.   Unfortunately there's a day-1 assumption baked
-// into much HotSpot code that the object::monitor relationship, once established
-// or observed, will remain stable except over potential safepoints.
-//
-// We can use either a blocking synchronous VM operation or an async VM operation.
-// -- If we use a blocking VM operation :
-//    Calls to ScavengeCheck() should be inserted only into 'safe' locations in paths
-//    that lead to ::inflate() or ::omAlloc().
-//    Even though the safepoint will not directly induce GC, a GC might
-//    piggyback on the safepoint operation, so the caller should hold no naked oops.
-//    Furthermore, monitor::object relationships are NOT necessarily stable over this call
-//    unless the caller has made provisions to "pin" the object to the monitor, say
-//    by incrementing the monitor's _count field.
-// -- If we use a non-blocking asynchronous VM operation :
-//    the constraints above don't apply.  The safepoint will fire in the future
-//    at a more convenient time.  On the other hand the latency between posting and
-//    running the safepoint introduces or admits "slop" or laxity during which the
-//    monitor population can climb further above the threshold.  The monitor population,
-//    however, tends to converge asymptotically over time to a count that's slightly
-//    above the target value specified by MonitorBound.   That is, we avoid unbounded
-//    growth, albeit with some imprecision.
-//
 // The current implementation uses asynchronous VM operations.
 //
-// Ideally we'd check if (MonitorPopulation > MonitorBound) in omAlloc()
-// immediately before trying to grow the global list via allocation.
-// If the predicate was true then we'd induce a synchronous safepoint, wait
-// for the safepoint to complete, and then again to allocate from the global
-// free list.  This approach is much simpler and precise, admitting no "slop".
-// Unfortunately we can't safely safepoint in the midst of omAlloc(), so
-// instead we use asynchronous safepoints.
 
 static void InduceScavenge (Thread * Self, const char * Whence) {
   // Induce STW safepoint to trim monitors
@@ -812,7 +856,7 @@ static void InduceScavenge (Thread * Self, const char * Whence) {
   // TODO: assert thread state is reasonable
 
   if (ForceMonitorScavenge == 0 && Atomic::xchg (1, &ForceMonitorScavenge) == 0) {
-    if (Knob_Verbose) {
+    if (ObjectMonitor::Knob_Verbose) {
       ::printf ("Monitor scavenge - Induced STW @%s (%d)\n", Whence, ForceMonitorScavenge) ;
       ::fflush(stdout) ;
     }
@@ -822,7 +866,7 @@ static void InduceScavenge (Thread * Self, const char * Whence) {
     // The VMThread will delete the op when completed.
     VMThread::execute (new VM_ForceAsyncSafepoint()) ;
 
-    if (Knob_Verbose) {
+    if (ObjectMonitor::Knob_Verbose) {
       ::printf ("Monitor scavenge - STW posted @%s (%d)\n", Whence, ForceMonitorScavenge) ;
       ::fflush(stdout) ;
     }
@@ -844,7 +888,6 @@ void ObjectSynchronizer::verifyInUse (Thread *Self) {
    assert(freetally == Self->omFreeCount, "free count off");
 }
 */
-
 ObjectMonitor * ATTR ObjectSynchronizer::omAlloc (Thread * Self) {
     // A large MAXPRIVATE value reduces both list lock contention
     // and list coherency traffic, but also tends to increase the
@@ -974,12 +1017,6 @@ ObjectMonitor * ATTR ObjectSynchronizer::omAlloc (Thread * Self) {
 // attempt failed.  This doesn't allow unbounded #s of monitors to
 // accumulate on a thread's free list.
 //
-// In the future the usage of omRelease() might change and monitors
-// could migrate between free lists.  In that case to avoid excessive
-// accumulation we could  limit omCount to (omProvision*2), otherwise return
-// the objectMonitor to the global list.  We should drain (return) in reasonable chunks.
-// That is, *not* one-at-a-time.
-
 
 void ObjectSynchronizer::omRelease (Thread * Self, ObjectMonitor * m, bool fromPerThreadAlloc) {
     guarantee (m->object() == NULL, "invariant") ;
@@ -1082,15 +1119,6 @@ void ObjectSynchronizer::omFlush (Thread * Self) {
     TEVENT (omFlush) ;
 }
 
-
-// Get the next block in the block list.
-static inline ObjectMonitor* next(ObjectMonitor* block) {
-  assert(block->object() == CHAINMARKER, "must be a block header");
-  block = block->FreeNext ;
-  assert(block == NULL || block->object() == CHAINMARKER, "must be a block header");
-  return block;
-}
-
 // Fast path code shared by multiple functions
 ObjectMonitor* ObjectSynchronizer::inflate_helper(oop obj) {
   markOop mark = obj->mark();
@@ -1102,79 +1130,10 @@ ObjectMonitor* ObjectSynchronizer::inflate_helper(oop obj) {
   return ObjectSynchronizer::inflate(Thread::current(), obj);
 }
 
+
 // Note that we could encounter some performance loss through false-sharing as
 // multiple locks occupy the same $ line.  Padding might be appropriate.
 
-#define NINFLATIONLOCKS 256
-static volatile intptr_t InflationLocks [NINFLATIONLOCKS] ;
-
-static markOop ReadStableMark (oop obj) {
-  markOop mark = obj->mark() ;
-  if (!mark->is_being_inflated()) {
-    return mark ;       // normal fast-path return
-  }
-
-  int its = 0 ;
-  for (;;) {
-    markOop mark = obj->mark() ;
-    if (!mark->is_being_inflated()) {
-      return mark ;    // normal fast-path return
-    }
-
-    // The object is being inflated by some other thread.
-    // The caller of ReadStableMark() must wait for inflation to complete.
-    // Avoid live-lock
-    // TODO: consider calling SafepointSynchronize::do_call_back() while
-    // spinning to see if there's a safepoint pending.  If so, immediately
-    // yielding or blocking would be appropriate.  Avoid spinning while
-    // there is a safepoint pending.
-    // TODO: add inflation contention performance counters.
-    // TODO: restrict the aggregate number of spinners.
-
-    ++its ;
-    if (its > 10000 || !os::is_MP()) {
-       if (its & 1) {
-         os::NakedYield() ;
-         TEVENT (Inflate: INFLATING - yield) ;
-       } else {
-         // Note that the following code attenuates the livelock problem but is not
-         // a complete remedy.  A more complete solution would require that the inflating
-         // thread hold the associated inflation lock.  The following code simply restricts
-         // the number of spinners to at most one.  We'll have N-2 threads blocked
-         // on the inflationlock, 1 thread holding the inflation lock and using
-         // a yield/park strategy, and 1 thread in the midst of inflation.
-         // A more refined approach would be to change the encoding of INFLATING
-         // to allow encapsulation of a native thread pointer.  Threads waiting for
-         // inflation to complete would use CAS to push themselves onto a singly linked
-         // list rooted at the markword.  Once enqueued, they'd loop, checking a per-thread flag
-         // and calling park().  When inflation was complete the thread that accomplished inflation
-         // would detach the list and set the markword to inflated with a single CAS and
-         // then for each thread on the list, set the flag and unpark() the thread.
-         // This is conceptually similar to muxAcquire-muxRelease, except that muxRelease
-         // wakes at most one thread whereas we need to wake the entire list.
-         int ix = (intptr_t(obj) >> 5) & (NINFLATIONLOCKS-1) ;
-         int YieldThenBlock = 0 ;
-         assert (ix >= 0 && ix < NINFLATIONLOCKS, "invariant") ;
-         assert ((NINFLATIONLOCKS & (NINFLATIONLOCKS-1)) == 0, "invariant") ;
-         Thread::muxAcquire (InflationLocks + ix, "InflationLock") ;
-         while (obj->mark() == markOopDesc::INFLATING()) {
-           // Beware: NakedYield() is advisory and has almost no effect on some platforms
-           // so we periodically call Self->_ParkEvent->park(1).
-           // We use a mixed spin/yield/block mechanism.
-           if ((YieldThenBlock++) >= 16) {
-              Thread::current()->_ParkEvent->park(1) ;
-           } else {
-              os::NakedYield() ;
-           }
-         }
-         Thread::muxRelease (InflationLocks + ix ) ;
-         TEVENT (Inflate: INFLATING - yield/park) ;
-       }
-    } else {
-       SpinPause() ;       // SMP-polite spinning
-    }
-  }
-}
 
 ObjectMonitor * ATTR ObjectSynchronizer::inflate (Thread * Self, oop object) {
   // Inflate mutates the heap ...
@@ -1242,7 +1201,7 @@ ObjectMonitor * ATTR ObjectSynchronizer::inflate (Thread * Self, oop object) {
           m->_Responsible  = NULL ;
           m->OwnerIsThread = 0 ;
           m->_recursions   = 0 ;
-          m->_SpinDuration = Knob_SpinLimit ;   // Consider: maintain by type/class
+          m->_SpinDuration = ObjectMonitor::Knob_SpinLimit ;   // Consider: maintain by type/class
 
           markOop cmp = (markOop) Atomic::cmpxchg_ptr (markOopDesc::INFLATING(), object->mark_addr(), mark) ;
           if (cmp != mark) {
@@ -1302,7 +1261,7 @@ ObjectMonitor * ATTR ObjectSynchronizer::inflate (Thread * Self, oop object) {
 
           // Hopefully the performance counters are allocated on distinct cache lines
           // to avoid false sharing on MP systems ...
-          if (_sync_Inflations != NULL) _sync_Inflations->inc() ;
+          if (ObjectMonitor::_sync_Inflations != NULL) ObjectMonitor::_sync_Inflations->inc() ;
           TEVENT(Inflate: overwrite stacklock) ;
           if (TraceMonitorInflation) {
             if (object->is_instance()) {
@@ -1335,7 +1294,7 @@ ObjectMonitor * ATTR ObjectSynchronizer::inflate (Thread * Self, oop object) {
       m->OwnerIsThread = 1 ;
       m->_recursions   = 0 ;
       m->_Responsible  = NULL ;
-      m->_SpinDuration = Knob_SpinLimit ;       // consider: keep metastats by type/class
+      m->_SpinDuration = ObjectMonitor::Knob_SpinLimit ;       // consider: keep metastats by type/class
 
       if (Atomic::cmpxchg_ptr (markOopDesc::encode(m), object->mark_addr(), mark) != mark) {
           m->set_object (NULL) ;
@@ -1352,7 +1311,7 @@ ObjectMonitor * ATTR ObjectSynchronizer::inflate (Thread * Self, oop object) {
 
       // Hopefully the performance counters are allocated on distinct
       // cache lines to avoid false sharing on MP systems ...
-      if (_sync_Inflations != NULL) _sync_Inflations->inc() ;
+      if (ObjectMonitor::_sync_Inflations != NULL) ObjectMonitor::_sync_Inflations->inc() ;
       TEVENT(Inflate: overwrite neutral) ;
       if (TraceMonitorInflation) {
         if (object->is_instance()) {
@@ -1366,547 +1325,9 @@ ObjectMonitor * ATTR ObjectSynchronizer::inflate (Thread * Self, oop object) {
   }
 }
 
+// Note that we could encounter some performance loss through false-sharing as
+// multiple locks occupy the same $ line.  Padding might be appropriate.
 
-// This the fast monitor enter. The interpreter and compiler use
-// some assembly copies of this code. Make sure update those code
-// if the following function is changed. The implementation is
-// extremely sensitive to race condition. Be careful.
-
-void ObjectSynchronizer::fast_enter(Handle obj, BasicLock* lock, bool attempt_rebias, TRAPS) {
- if (UseBiasedLocking) {
-    if (!SafepointSynchronize::is_at_safepoint()) {
-      BiasedLocking::Condition cond = BiasedLocking::revoke_and_rebias(obj, attempt_rebias, THREAD);
-      if (cond == BiasedLocking::BIAS_REVOKED_AND_REBIASED) {
-        return;
-      }
-    } else {
-      assert(!attempt_rebias, "can not rebias toward VM thread");
-      BiasedLocking::revoke_at_safepoint(obj);
-    }
-    assert(!obj->mark()->has_bias_pattern(), "biases should be revoked by now");
- }
-
- slow_enter (obj, lock, THREAD) ;
-}
-
-void ObjectSynchronizer::fast_exit(oop object, BasicLock* lock, TRAPS) {
-  assert(!object->mark()->has_bias_pattern(), "should not see bias pattern here");
-  // if displaced header is null, the previous enter is recursive enter, no-op
-  markOop dhw = lock->displaced_header();
-  markOop mark ;
-  if (dhw == NULL) {
-     // Recursive stack-lock.
-     // Diagnostics -- Could be: stack-locked, inflating, inflated.
-     mark = object->mark() ;
-     assert (!mark->is_neutral(), "invariant") ;
-     if (mark->has_locker() && mark != markOopDesc::INFLATING()) {
-        assert(THREAD->is_lock_owned((address)mark->locker()), "invariant") ;
-     }
-     if (mark->has_monitor()) {
-        ObjectMonitor * m = mark->monitor() ;
-        assert(((oop)(m->object()))->mark() == mark, "invariant") ;
-        assert(m->is_entered(THREAD), "invariant") ;
-     }
-     return ;
-  }
-
-  mark = object->mark() ;
-
-  // If the object is stack-locked by the current thread, try to
-  // swing the displaced header from the box back to the mark.
-  if (mark == (markOop) lock) {
-     assert (dhw->is_neutral(), "invariant") ;
-     if ((markOop) Atomic::cmpxchg_ptr (dhw, object->mark_addr(), mark) == mark) {
-        TEVENT (fast_exit: release stacklock) ;
-        return;
-     }
-  }
-
-  ObjectSynchronizer::inflate(THREAD, object)->exit (THREAD) ;
-}
-
-// This routine is used to handle interpreter/compiler slow case
-// We don't need to use fast path here, because it must have been
-// failed in the interpreter/compiler code.
-void ObjectSynchronizer::slow_enter(Handle obj, BasicLock* lock, TRAPS) {
-  markOop mark = obj->mark();
-  assert(!mark->has_bias_pattern(), "should not see bias pattern here");
-
-  if (mark->is_neutral()) {
-    // Anticipate successful CAS -- the ST of the displaced mark must
-    // be visible <= the ST performed by the CAS.
-    lock->set_displaced_header(mark);
-    if (mark == (markOop) Atomic::cmpxchg_ptr(lock, obj()->mark_addr(), mark)) {
-      TEVENT (slow_enter: release stacklock) ;
-      return ;
-    }
-    // Fall through to inflate() ...
-  } else
-  if (mark->has_locker() && THREAD->is_lock_owned((address)mark->locker())) {
-    assert(lock != mark->locker(), "must not re-lock the same lock");
-    assert(lock != (BasicLock*)obj->mark(), "don't relock with same BasicLock");
-    lock->set_displaced_header(NULL);
-    return;
-  }
-
-#if 0
-  // The following optimization isn't particularly useful.
-  if (mark->has_monitor() && mark->monitor()->is_entered(THREAD)) {
-    lock->set_displaced_header (NULL) ;
-    return ;
-  }
-#endif
-
-  // The object header will never be displaced to this lock,
-  // so it does not matter what the value is, except that it
-  // must be non-zero to avoid looking like a re-entrant lock,
-  // and must not look locked either.
-  lock->set_displaced_header(markOopDesc::unused_mark());
-  ObjectSynchronizer::inflate(THREAD, obj())->enter(THREAD);
-}
-
-// This routine is used to handle interpreter/compiler slow case
-// We don't need to use fast path here, because it must have
-// failed in the interpreter/compiler code. Simply use the heavy
-// weight monitor should be ok, unless someone find otherwise.
-void ObjectSynchronizer::slow_exit(oop object, BasicLock* lock, TRAPS) {
-  fast_exit (object, lock, THREAD) ;
-}
-
-// NOTE: must use heavy weight monitor to handle jni monitor enter
-void ObjectSynchronizer::jni_enter(Handle obj, TRAPS) { // possible entry from jni enter
-  // the current locking is from JNI instead of Java code
-  TEVENT (jni_enter) ;
-  if (UseBiasedLocking) {
-    BiasedLocking::revoke_and_rebias(obj, false, THREAD);
-    assert(!obj->mark()->has_bias_pattern(), "biases should be revoked by now");
-  }
-  THREAD->set_current_pending_monitor_is_from_java(false);
-  ObjectSynchronizer::inflate(THREAD, obj())->enter(THREAD);
-  THREAD->set_current_pending_monitor_is_from_java(true);
-}
-
-// NOTE: must use heavy weight monitor to handle jni monitor enter
-bool ObjectSynchronizer::jni_try_enter(Handle obj, Thread* THREAD) {
-  if (UseBiasedLocking) {
-    BiasedLocking::revoke_and_rebias(obj, false, THREAD);
-    assert(!obj->mark()->has_bias_pattern(), "biases should be revoked by now");
-  }
-
-  ObjectMonitor* monitor = ObjectSynchronizer::inflate_helper(obj());
-  return monitor->try_enter(THREAD);
-}
-
-
-// NOTE: must use heavy weight monitor to handle jni monitor exit
-void ObjectSynchronizer::jni_exit(oop obj, Thread* THREAD) {
-  TEVENT (jni_exit) ;
-  if (UseBiasedLocking) {
-    BiasedLocking::revoke_and_rebias(obj, false, THREAD);
-  }
-  assert(!obj->mark()->has_bias_pattern(), "biases should be revoked by now");
-
-  ObjectMonitor* monitor = ObjectSynchronizer::inflate(THREAD, obj);
-  // If this thread has locked the object, exit the monitor.  Note:  can't use
-  // monitor->check(CHECK); must exit even if an exception is pending.
-  if (monitor->check(THREAD)) {
-     monitor->exit(THREAD);
-  }
-}
-
-// complete_exit()/reenter() are used to wait on a nested lock
-// i.e. to give up an outer lock completely and then re-enter
-// Used when holding nested locks - lock acquisition order: lock1 then lock2
-//  1) complete_exit lock1 - saving recursion count
-//  2) wait on lock2
-//  3) when notified on lock2, unlock lock2
-//  4) reenter lock1 with original recursion count
-//  5) lock lock2
-// NOTE: must use heavy weight monitor to handle complete_exit/reenter()
-intptr_t ObjectSynchronizer::complete_exit(Handle obj, TRAPS) {
-  TEVENT (complete_exit) ;
-  if (UseBiasedLocking) {
-    BiasedLocking::revoke_and_rebias(obj, false, THREAD);
-    assert(!obj->mark()->has_bias_pattern(), "biases should be revoked by now");
-  }
-
-  ObjectMonitor* monitor = ObjectSynchronizer::inflate(THREAD, obj());
-
-  return monitor->complete_exit(THREAD);
-}
-
-// NOTE: must use heavy weight monitor to handle complete_exit/reenter()
-void ObjectSynchronizer::reenter(Handle obj, intptr_t recursion, TRAPS) {
-  TEVENT (reenter) ;
-  if (UseBiasedLocking) {
-    BiasedLocking::revoke_and_rebias(obj, false, THREAD);
-    assert(!obj->mark()->has_bias_pattern(), "biases should be revoked by now");
-  }
-
-  ObjectMonitor* monitor = ObjectSynchronizer::inflate(THREAD, obj());
-
-  monitor->reenter(recursion, THREAD);
-}
-
-// This exists only as a workaround of dtrace bug 6254741
-int dtrace_waited_probe(ObjectMonitor* monitor, Handle obj, Thread* thr) {
-  DTRACE_MONITOR_PROBE(waited, monitor, obj(), thr);
-  return 0;
-}
-
-// NOTE: must use heavy weight monitor to handle wait()
-void ObjectSynchronizer::wait(Handle obj, jlong millis, TRAPS) {
-  if (UseBiasedLocking) {
-    BiasedLocking::revoke_and_rebias(obj, false, THREAD);
-    assert(!obj->mark()->has_bias_pattern(), "biases should be revoked by now");
-  }
-  if (millis < 0) {
-    TEVENT (wait - throw IAX) ;
-    THROW_MSG(vmSymbols::java_lang_IllegalArgumentException(), "timeout value is negative");
-  }
-  ObjectMonitor* monitor = ObjectSynchronizer::inflate(THREAD, obj());
-  DTRACE_MONITOR_WAIT_PROBE(monitor, obj(), THREAD, millis);
-  monitor->wait(millis, true, THREAD);
-
-  /* This dummy call is in place to get around dtrace bug 6254741.  Once
-     that's fixed we can uncomment the following line and remove the call */
-  // DTRACE_MONITOR_PROBE(waited, monitor, obj(), THREAD);
-  dtrace_waited_probe(monitor, obj, THREAD);
-}
-
-void ObjectSynchronizer::waitUninterruptibly (Handle obj, jlong millis, TRAPS) {
-  if (UseBiasedLocking) {
-    BiasedLocking::revoke_and_rebias(obj, false, THREAD);
-    assert(!obj->mark()->has_bias_pattern(), "biases should be revoked by now");
-  }
-  if (millis < 0) {
-    TEVENT (wait - throw IAX) ;
-    THROW_MSG(vmSymbols::java_lang_IllegalArgumentException(), "timeout value is negative");
-  }
-  ObjectSynchronizer::inflate(THREAD, obj()) -> wait(millis, false, THREAD) ;
-}
-
-void ObjectSynchronizer::notify(Handle obj, TRAPS) {
- if (UseBiasedLocking) {
-    BiasedLocking::revoke_and_rebias(obj, false, THREAD);
-    assert(!obj->mark()->has_bias_pattern(), "biases should be revoked by now");
-  }
-
-  markOop mark = obj->mark();
-  if (mark->has_locker() && THREAD->is_lock_owned((address)mark->locker())) {
-    return;
-  }
-  ObjectSynchronizer::inflate(THREAD, obj())->notify(THREAD);
-}
-
-// NOTE: see comment of notify()
-void ObjectSynchronizer::notifyall(Handle obj, TRAPS) {
-  if (UseBiasedLocking) {
-    BiasedLocking::revoke_and_rebias(obj, false, THREAD);
-    assert(!obj->mark()->has_bias_pattern(), "biases should be revoked by now");
-  }
-
-  markOop mark = obj->mark();
-  if (mark->has_locker() && THREAD->is_lock_owned((address)mark->locker())) {
-    return;
-  }
-  ObjectSynchronizer::inflate(THREAD, obj())->notifyAll(THREAD);
-}
-
-intptr_t ObjectSynchronizer::FastHashCode (Thread * Self, oop obj) {
-  if (UseBiasedLocking) {
-    // NOTE: many places throughout the JVM do not expect a safepoint
-    // to be taken here, in particular most operations on perm gen
-    // objects. However, we only ever bias Java instances and all of
-    // the call sites of identity_hash that might revoke biases have
-    // been checked to make sure they can handle a safepoint. The
-    // added check of the bias pattern is to avoid useless calls to
-    // thread-local storage.
-    if (obj->mark()->has_bias_pattern()) {
-      // Box and unbox the raw reference just in case we cause a STW safepoint.
-      Handle hobj (Self, obj) ;
-      // Relaxing assertion for bug 6320749.
-      assert (Universe::verify_in_progress() ||
-              !SafepointSynchronize::is_at_safepoint(),
-             "biases should not be seen by VM thread here");
-      BiasedLocking::revoke_and_rebias(hobj, false, JavaThread::current());
-      obj = hobj() ;
-      assert(!obj->mark()->has_bias_pattern(), "biases should be revoked by now");
-    }
-  }
-
-  // hashCode() is a heap mutator ...
-  // Relaxing assertion for bug 6320749.
-  assert (Universe::verify_in_progress() ||
-          !SafepointSynchronize::is_at_safepoint(), "invariant") ;
-  assert (Universe::verify_in_progress() ||
-          Self->is_Java_thread() , "invariant") ;
-  assert (Universe::verify_in_progress() ||
-         ((JavaThread *)Self)->thread_state() != _thread_blocked, "invariant") ;
-
-  ObjectMonitor* monitor = NULL;
-  markOop temp, test;
-  intptr_t hash;
-  markOop mark = ReadStableMark (obj);
-
-  // object should remain ineligible for biased locking
-  assert (!mark->has_bias_pattern(), "invariant") ;
-
-  if (mark->is_neutral()) {
-    hash = mark->hash();              // this is a normal header
-    if (hash) {                       // if it has hash, just return it
-      return hash;
-    }
-    hash = get_next_hash(Self, obj);  // allocate a new hash code
-    temp = mark->copy_set_hash(hash); // merge the hash code into header
-    // use (machine word version) atomic operation to install the hash
-    test = (markOop) Atomic::cmpxchg_ptr(temp, obj->mark_addr(), mark);
-    if (test == mark) {
-      return hash;
-    }
-    // If atomic operation failed, we must inflate the header
-    // into heavy weight monitor. We could add more code here
-    // for fast path, but it does not worth the complexity.
-  } else if (mark->has_monitor()) {
-    monitor = mark->monitor();
-    temp = monitor->header();
-    assert (temp->is_neutral(), "invariant") ;
-    hash = temp->hash();
-    if (hash) {
-      return hash;
-    }
-    // Skip to the following code to reduce code size
-  } else if (Self->is_lock_owned((address)mark->locker())) {
-    temp = mark->displaced_mark_helper(); // this is a lightweight monitor owned
-    assert (temp->is_neutral(), "invariant") ;
-    hash = temp->hash();              // by current thread, check if the displaced
-    if (hash) {                       // header contains hash code
-      return hash;
-    }
-    // WARNING:
-    //   The displaced header is strictly immutable.
-    // It can NOT be changed in ANY cases. So we have
-    // to inflate the header into heavyweight monitor
-    // even the current thread owns the lock. The reason
-    // is the BasicLock (stack slot) will be asynchronously
-    // read by other threads during the inflate() function.
-    // Any change to stack may not propagate to other threads
-    // correctly.
-  }
-
-  // Inflate the monitor to set hash code
-  monitor = ObjectSynchronizer::inflate(Self, obj);
-  // Load displaced header and check it has hash code
-  mark = monitor->header();
-  assert (mark->is_neutral(), "invariant") ;
-  hash = mark->hash();
-  if (hash == 0) {
-    hash = get_next_hash(Self, obj);
-    temp = mark->copy_set_hash(hash); // merge hash code into header
-    assert (temp->is_neutral(), "invariant") ;
-    test = (markOop) Atomic::cmpxchg_ptr(temp, monitor, mark);
-    if (test != mark) {
-      // The only update to the header in the monitor (outside GC)
-      // is install the hash code. If someone add new usage of
-      // displaced header, please update this code
-      hash = test->hash();
-      assert (test->is_neutral(), "invariant") ;
-      assert (hash != 0, "Trivial unexpected object/monitor header usage.");
-    }
-  }
-  // We finally get the hash
-  return hash;
-}
-
-// Deprecated -- use FastHashCode() instead.
-
-intptr_t ObjectSynchronizer::identity_hash_value_for(Handle obj) {
-  return FastHashCode (Thread::current(), obj()) ;
-}
-
-bool ObjectSynchronizer::current_thread_holds_lock(JavaThread* thread,
-                                                   Handle h_obj) {
-  if (UseBiasedLocking) {
-    BiasedLocking::revoke_and_rebias(h_obj, false, thread);
-    assert(!h_obj->mark()->has_bias_pattern(), "biases should be revoked by now");
-  }
-
-  assert(thread == JavaThread::current(), "Can only be called on current thread");
-  oop obj = h_obj();
-
-  markOop mark = ReadStableMark (obj) ;
-
-  // Uncontended case, header points to stack
-  if (mark->has_locker()) {
-    return thread->is_lock_owned((address)mark->locker());
-  }
-  // Contended case, header points to ObjectMonitor (tagged pointer)
-  if (mark->has_monitor()) {
-    ObjectMonitor* monitor = mark->monitor();
-    return monitor->is_entered(thread) != 0 ;
-  }
-  // Unlocked case, header in place
-  assert(mark->is_neutral(), "sanity check");
-  return false;
-}
-
-// Be aware of this method could revoke bias of the lock object.
-// This method querys the ownership of the lock handle specified by 'h_obj'.
-// If the current thread owns the lock, it returns owner_self. If no
-// thread owns the lock, it returns owner_none. Otherwise, it will return
-// ower_other.
-ObjectSynchronizer::LockOwnership ObjectSynchronizer::query_lock_ownership
-(JavaThread *self, Handle h_obj) {
-  // The caller must beware this method can revoke bias, and
-  // revocation can result in a safepoint.
-  assert (!SafepointSynchronize::is_at_safepoint(), "invariant") ;
-  assert (self->thread_state() != _thread_blocked , "invariant") ;
-
-  // Possible mark states: neutral, biased, stack-locked, inflated
-
-  if (UseBiasedLocking && h_obj()->mark()->has_bias_pattern()) {
-    // CASE: biased
-    BiasedLocking::revoke_and_rebias(h_obj, false, self);
-    assert(!h_obj->mark()->has_bias_pattern(),
-           "biases should be revoked by now");
-  }
-
-  assert(self == JavaThread::current(), "Can only be called on current thread");
-  oop obj = h_obj();
-  markOop mark = ReadStableMark (obj) ;
-
-  // CASE: stack-locked.  Mark points to a BasicLock on the owner's stack.
-  if (mark->has_locker()) {
-    return self->is_lock_owned((address)mark->locker()) ?
-      owner_self : owner_other;
-  }
-
-  // CASE: inflated. Mark (tagged pointer) points to an objectMonitor.
-  // The Object:ObjectMonitor relationship is stable as long as we're
-  // not at a safepoint.
-  if (mark->has_monitor()) {
-    void * owner = mark->monitor()->_owner ;
-    if (owner == NULL) return owner_none ;
-    return (owner == self ||
-            self->is_lock_owned((address)owner)) ? owner_self : owner_other;
-  }
-
-  // CASE: neutral
-  assert(mark->is_neutral(), "sanity check");
-  return owner_none ;           // it's unlocked
-}
-
-// FIXME: jvmti should call this
-JavaThread* ObjectSynchronizer::get_lock_owner(Handle h_obj, bool doLock) {
-  if (UseBiasedLocking) {
-    if (SafepointSynchronize::is_at_safepoint()) {
-      BiasedLocking::revoke_at_safepoint(h_obj);
-    } else {
-      BiasedLocking::revoke_and_rebias(h_obj, false, JavaThread::current());
-    }
-    assert(!h_obj->mark()->has_bias_pattern(), "biases should be revoked by now");
-  }
-
-  oop obj = h_obj();
-  address owner = NULL;
-
-  markOop mark = ReadStableMark (obj) ;
-
-  // Uncontended case, header points to stack
-  if (mark->has_locker()) {
-    owner = (address) mark->locker();
-  }
-
-  // Contended case, header points to ObjectMonitor (tagged pointer)
-  if (mark->has_monitor()) {
-    ObjectMonitor* monitor = mark->monitor();
-    assert(monitor != NULL, "monitor should be non-null");
-    owner = (address) monitor->owner();
-  }
-
-  if (owner != NULL) {
-    return Threads::owning_thread_from_monitor_owner(owner, doLock);
-  }
-
-  // Unlocked case, header in place
-  // Cannot have assertion since this object may have been
-  // locked by another thread when reaching here.
-  // assert(mark->is_neutral(), "sanity check");
-
-  return NULL;
-}
-
-// Iterate through monitor cache and attempt to release thread's monitors
-// Gives up on a particular monitor if an exception occurs, but continues
-// the overall iteration, swallowing the exception.
-class ReleaseJavaMonitorsClosure: public MonitorClosure {
-private:
-  TRAPS;
-
-public:
-  ReleaseJavaMonitorsClosure(Thread* thread) : THREAD(thread) {}
-  void do_monitor(ObjectMonitor* mid) {
-    if (mid->owner() == THREAD) {
-      (void)mid->complete_exit(CHECK);
-    }
-  }
-};
-
-// Release all inflated monitors owned by THREAD.  Lightweight monitors are
-// ignored.  This is meant to be called during JNI thread detach which assumes
-// all remaining monitors are heavyweight.  All exceptions are swallowed.
-// Scanning the extant monitor list can be time consuming.
-// A simple optimization is to add a per-thread flag that indicates a thread
-// called jni_monitorenter() during its lifetime.
-//
-// Instead of No_Savepoint_Verifier it might be cheaper to
-// use an idiom of the form:
-//   auto int tmp = SafepointSynchronize::_safepoint_counter ;
-//   <code that must not run at safepoint>
-//   guarantee (((tmp ^ _safepoint_counter) | (tmp & 1)) == 0) ;
-// Since the tests are extremely cheap we could leave them enabled
-// for normal product builds.
-
-void ObjectSynchronizer::release_monitors_owned_by_thread(TRAPS) {
-  assert(THREAD == JavaThread::current(), "must be current Java thread");
-  No_Safepoint_Verifier nsv ;
-  ReleaseJavaMonitorsClosure rjmc(THREAD);
-  Thread::muxAcquire(&ListLock, "release_monitors_owned_by_thread");
-  ObjectSynchronizer::monitors_iterate(&rjmc);
-  Thread::muxRelease(&ListLock);
-  THREAD->clear_pending_exception();
-}
-
-// Visitors ...
-
-void ObjectSynchronizer::monitors_iterate(MonitorClosure* closure) {
-  ObjectMonitor* block = gBlockList;
-  ObjectMonitor* mid;
-  while (block) {
-    assert(block->object() == CHAINMARKER, "must be a block header");
-    for (int i = _BLOCKSIZE - 1; i > 0; i--) {
-      mid = block + i;
-      oop object = (oop) mid->object();
-      if (object != NULL) {
-        closure->do_monitor(mid);
-      }
-    }
-    block = (ObjectMonitor*) block->FreeNext;
-  }
-}
-
-void ObjectSynchronizer::oops_do(OopClosure* f) {
-  assert(SafepointSynchronize::is_at_safepoint(), "must be at safepoint");
-  for (ObjectMonitor* block = gBlockList; block != NULL; block = next(block)) {
-    assert(block->object() == CHAINMARKER, "must be a block header");
-    for (int i = 1; i < _BLOCKSIZE; i++) {
-      ObjectMonitor* mid = &block[i];
-      if (mid->object() != NULL) {
-        f->do_oop((oop*)mid->object_addr());
-      }
-    }
-  }
-}
 
 // Deflate_idle_monitors() is called at all safepoints, immediately
 // after all mutators are stopped, but before any objects have moved.
@@ -1936,12 +1357,11 @@ void ObjectSynchronizer::oops_do(OopClosure* f) {
 // which in turn can mean large(r) numbers of objectmonitors in circulation.
 // This is an unfortunate aspect of this design.
 //
-// Another refinement would be to refrain from calling deflate_idle_monitors()
-// except at stop-the-world points associated with garbage collections.
-//
-// An even better solution would be to deflate on-the-fly, aggressively,
-// at monitorexit-time as is done in EVM's metalock or Relaxed Locks.
 
+enum ManifestConstants {
+    ClearResponsibleAtSTW   = 0,
+    MaximumRecheckInterval  = 1000
+} ;
 
 // Deflate a single monitor if not in use
 // Return true if deflated, false if in use
@@ -2088,7 +1508,7 @@ void ObjectSynchronizer::deflate_idle_monitors() {
 
   // Consider: audit gFreeList to ensure that MonitorFreeCount and list agree.
 
-  if (Knob_Verbose) {
+  if (ObjectMonitor::Knob_Verbose) {
     ::printf ("Deflate: InCirc=%d InUse=%d Scavenged=%d ForceMonitorScavenge=%d : pop=%d free=%d\n",
         nInCirculation, nInuse, nScavenged, ForceMonitorScavenge,
         MonitorPopulation, MonitorFreeCount) ;
@@ -2107,8 +1527,8 @@ void ObjectSynchronizer::deflate_idle_monitors() {
   }
   Thread::muxRelease (&ListLock) ;
 
-  if (_sync_Deflations != NULL) _sync_Deflations->inc(nScavenged) ;
-  if (_sync_MonExtant  != NULL) _sync_MonExtant ->set_value(nInCirculation);
+  if (ObjectMonitor::_sync_Deflations != NULL) ObjectMonitor::_sync_Deflations->inc(nScavenged) ;
+  if (ObjectMonitor::_sync_MonExtant  != NULL) ObjectMonitor::_sync_MonExtant ->set_value(nInCirculation);
 
   // TODO: Add objectMonitor leak detection.
   // Audit/inventory the objectMonitors -- make sure they're all accounted for.
@@ -2116,2810 +1536,49 @@ void ObjectSynchronizer::deflate_idle_monitors() {
   GVars.stwCycle ++ ;
 }
 
-// A macro is used below because there may already be a pending
-// exception which should not abort the execution of the routines
-// which use this (which is why we don't put this into check_slow and
-// call it with a CHECK argument).
+// Monitor cleanup on JavaThread::exit
 
-#define CHECK_OWNER()                                                             \
-  do {                                                                            \
-    if (THREAD != _owner) {                                                       \
-      if (THREAD->is_lock_owned((address) _owner)) {                              \
-        _owner = THREAD ;  /* Convert from basiclock addr to Thread addr */       \
-        _recursions = 0;                                                          \
-        OwnerIsThread = 1 ;                                                       \
-      } else {                                                                    \
-        TEVENT (Throw IMSX) ;                                                     \
-        THROW(vmSymbols::java_lang_IllegalMonitorStateException());               \
-      }                                                                           \
-    }                                                                             \
-  } while (false)
+// Iterate through monitor cache and attempt to release thread's monitors
+// Gives up on a particular monitor if an exception occurs, but continues
+// the overall iteration, swallowing the exception.
+class ReleaseJavaMonitorsClosure: public MonitorClosure {
+private:
+  TRAPS;
 
-// TODO-FIXME: eliminate ObjectWaiters.  Replace this visitor/enumerator
-// interface with a simple FirstWaitingThread(), NextWaitingThread() interface.
-
-ObjectWaiter* ObjectMonitor::first_waiter() {
-  return _WaitSet;
-}
-
-ObjectWaiter* ObjectMonitor::next_waiter(ObjectWaiter* o) {
-  return o->_next;
-}
-
-Thread* ObjectMonitor::thread_of_waiter(ObjectWaiter* o) {
-  return o->_thread;
-}
-
-// initialize the monitor, exception the semaphore, all other fields
-// are simple integers or pointers
-ObjectMonitor::ObjectMonitor() {
-  _header       = NULL;
-  _count        = 0;
-  _waiters      = 0,
-  _recursions   = 0;
-  _object       = NULL;
-  _owner        = NULL;
-  _WaitSet      = NULL;
-  _WaitSetLock  = 0 ;
-  _Responsible  = NULL ;
-  _succ         = NULL ;
-  _cxq          = NULL ;
-  FreeNext      = NULL ;
-  _EntryList    = NULL ;
-  _SpinFreq     = 0 ;
-  _SpinClock    = 0 ;
-  OwnerIsThread = 0 ;
-}
-
-ObjectMonitor::~ObjectMonitor() {
-   // TODO: Add asserts ...
-   // _cxq == 0 _succ == NULL _owner == NULL _waiters == 0
-   // _count == 0 _EntryList  == NULL etc
-}
-
-intptr_t ObjectMonitor::is_busy() const {
-  // TODO-FIXME: merge _count and _waiters.
-  // TODO-FIXME: assert _owner == null implies _recursions = 0
-  // TODO-FIXME: assert _WaitSet != null implies _count > 0
-  return _count|_waiters|intptr_t(_owner)|intptr_t(_cxq)|intptr_t(_EntryList ) ;
-}
-
-void ObjectMonitor::Recycle () {
-  // TODO: add stronger asserts ...
-  // _cxq == 0 _succ == NULL _owner == NULL _waiters == 0
-  // _count == 0 EntryList  == NULL
-  // _recursions == 0 _WaitSet == NULL
-  // TODO: assert (is_busy()|_recursions) == 0
-  _succ          = NULL ;
-  _EntryList     = NULL ;
-  _cxq           = NULL ;
-  _WaitSet       = NULL ;
-  _recursions    = 0 ;
-  _SpinFreq      = 0 ;
-  _SpinClock     = 0 ;
-  OwnerIsThread  = 0 ;
-}
-
-// WaitSet management ...
-
-inline void ObjectMonitor::AddWaiter(ObjectWaiter* node) {
-  assert(node != NULL, "should not dequeue NULL node");
-  assert(node->_prev == NULL, "node already in list");
-  assert(node->_next == NULL, "node already in list");
-  // put node at end of queue (circular doubly linked list)
-  if (_WaitSet == NULL) {
-    _WaitSet = node;
-    node->_prev = node;
-    node->_next = node;
-  } else {
-    ObjectWaiter* head = _WaitSet ;
-    ObjectWaiter* tail = head->_prev;
-    assert(tail->_next == head, "invariant check");
-    tail->_next = node;
-    head->_prev = node;
-    node->_next = head;
-    node->_prev = tail;
-  }
-}
-
-inline ObjectWaiter* ObjectMonitor::DequeueWaiter() {
-  // dequeue the very first waiter
-  ObjectWaiter* waiter = _WaitSet;
-  if (waiter) {
-    DequeueSpecificWaiter(waiter);
-  }
-  return waiter;
-}
-
-inline void ObjectMonitor::DequeueSpecificWaiter(ObjectWaiter* node) {
-  assert(node != NULL, "should not dequeue NULL node");
-  assert(node->_prev != NULL, "node already removed from list");
-  assert(node->_next != NULL, "node already removed from list");
-  // when the waiter has woken up because of interrupt,
-  // timeout or other spurious wake-up, dequeue the
-  // waiter from waiting list
-  ObjectWaiter* next = node->_next;
-  if (next == node) {
-    assert(node->_prev == node, "invariant check");
-    _WaitSet = NULL;
-  } else {
-    ObjectWaiter* prev = node->_prev;
-    assert(prev->_next == node, "invariant check");
-    assert(next->_prev == node, "invariant check");
-    next->_prev = prev;
-    prev->_next = next;
-    if (_WaitSet == node) {
-      _WaitSet = next;
+public:
+  ReleaseJavaMonitorsClosure(Thread* thread) : THREAD(thread) {}
+  void do_monitor(ObjectMonitor* mid) {
+    if (mid->owner() == THREAD) {
+      (void)mid->complete_exit(CHECK);
     }
   }
-  node->_next = NULL;
-  node->_prev = NULL;
+};
+
+// Release all inflated monitors owned by THREAD.  Lightweight monitors are
+// ignored.  This is meant to be called during JNI thread detach which assumes
+// all remaining monitors are heavyweight.  All exceptions are swallowed.
+// Scanning the extant monitor list can be time consuming.
+// A simple optimization is to add a per-thread flag that indicates a thread
+// called jni_monitorenter() during its lifetime.
+//
+// Instead of No_Savepoint_Verifier it might be cheaper to
+// use an idiom of the form:
+//   auto int tmp = SafepointSynchronize::_safepoint_counter ;
+//   <code that must not run at safepoint>
+//   guarantee (((tmp ^ _safepoint_counter) | (tmp & 1)) == 0) ;
+// Since the tests are extremely cheap we could leave them enabled
+// for normal product builds.
+
+void ObjectSynchronizer::release_monitors_owned_by_thread(TRAPS) {
+  assert(THREAD == JavaThread::current(), "must be current Java thread");
+  No_Safepoint_Verifier nsv ;
+  ReleaseJavaMonitorsClosure rjmc(THREAD);
+  Thread::muxAcquire(&ListLock, "release_monitors_owned_by_thread");
+  ObjectSynchronizer::monitors_iterate(&rjmc);
+  Thread::muxRelease(&ListLock);
+  THREAD->clear_pending_exception();
 }
 
-static char * kvGet (char * kvList, const char * Key) {
-    if (kvList == NULL) return NULL ;
-    size_t n = strlen (Key) ;
-    char * Search ;
-    for (Search = kvList ; *Search ; Search += strlen(Search) + 1) {
-        if (strncmp (Search, Key, n) == 0) {
-            if (Search[n] == '=') return Search + n + 1 ;
-            if (Search[n] == 0)   return (char *) "1" ;
-        }
-    }
-    return NULL ;
-}
-
-static int kvGetInt (char * kvList, const char * Key, int Default) {
-    char * v = kvGet (kvList, Key) ;
-    int rslt = v ? ::strtol (v, NULL, 0) : Default ;
-    if (Knob_ReportSettings && v != NULL) {
-        ::printf ("  SyncKnob: %s %d(%d)\n", Key, rslt, Default) ;
-        ::fflush (stdout) ;
-    }
-    return rslt ;
-}
-
-// By convention we unlink a contending thread from EntryList|cxq immediately
-// after the thread acquires the lock in ::enter().  Equally, we could defer
-// unlinking the thread until ::exit()-time.
-
-void ObjectMonitor::UnlinkAfterAcquire (Thread * Self, ObjectWaiter * SelfNode)
-{
-    assert (_owner == Self, "invariant") ;
-    assert (SelfNode->_thread == Self, "invariant") ;
-
-    if (SelfNode->TState == ObjectWaiter::TS_ENTER) {
-        // Normal case: remove Self from the DLL EntryList .
-        // This is a constant-time operation.
-        ObjectWaiter * nxt = SelfNode->_next ;
-        ObjectWaiter * prv = SelfNode->_prev ;
-        if (nxt != NULL) nxt->_prev = prv ;
-        if (prv != NULL) prv->_next = nxt ;
-        if (SelfNode == _EntryList ) _EntryList = nxt ;
-        assert (nxt == NULL || nxt->TState == ObjectWaiter::TS_ENTER, "invariant") ;
-        assert (prv == NULL || prv->TState == ObjectWaiter::TS_ENTER, "invariant") ;
-        TEVENT (Unlink from EntryList) ;
-    } else {
-        guarantee (SelfNode->TState == ObjectWaiter::TS_CXQ, "invariant") ;
-        // Inopportune interleaving -- Self is still on the cxq.
-        // This usually means the enqueue of self raced an exiting thread.
-        // Normally we'll find Self near the front of the cxq, so
-        // dequeueing is typically fast.  If needbe we can accelerate
-        // this with some MCS/CHL-like bidirectional list hints and advisory
-        // back-links so dequeueing from the interior will normally operate
-        // in constant-time.
-        // Dequeue Self from either the head (with CAS) or from the interior
-        // with a linear-time scan and normal non-atomic memory operations.
-        // CONSIDER: if Self is on the cxq then simply drain cxq into EntryList
-        // and then unlink Self from EntryList.  We have to drain eventually,
-        // so it might as well be now.
-
-        ObjectWaiter * v = _cxq ;
-        assert (v != NULL, "invariant") ;
-        if (v != SelfNode || Atomic::cmpxchg_ptr (SelfNode->_next, &_cxq, v) != v) {
-            // The CAS above can fail from interference IFF a "RAT" arrived.
-            // In that case Self must be in the interior and can no longer be
-            // at the head of cxq.
-            if (v == SelfNode) {
-                assert (_cxq != v, "invariant") ;
-                v = _cxq ;          // CAS above failed - start scan at head of list
-            }
-            ObjectWaiter * p ;
-            ObjectWaiter * q = NULL ;
-            for (p = v ; p != NULL && p != SelfNode; p = p->_next) {
-                q = p ;
-                assert (p->TState == ObjectWaiter::TS_CXQ, "invariant") ;
-            }
-            assert (v != SelfNode,  "invariant") ;
-            assert (p == SelfNode,  "Node not found on cxq") ;
-            assert (p != _cxq,      "invariant") ;
-            assert (q != NULL,      "invariant") ;
-            assert (q->_next == p,  "invariant") ;
-            q->_next = p->_next ;
-        }
-        TEVENT (Unlink from cxq) ;
-    }
-
-    // Diagnostic hygiene ...
-    SelfNode->_prev  = (ObjectWaiter *) 0xBAD ;
-    SelfNode->_next  = (ObjectWaiter *) 0xBAD ;
-    SelfNode->TState = ObjectWaiter::TS_RUN ;
-}
-
-// Caveat: TryLock() is not necessarily serializing if it returns failure.
-// Callers must compensate as needed.
-
-int ObjectMonitor::TryLock (Thread * Self) {
-   for (;;) {
-      void * own = _owner ;
-      if (own != NULL) return 0 ;
-      if (Atomic::cmpxchg_ptr (Self, &_owner, NULL) == NULL) {
-         // Either guarantee _recursions == 0 or set _recursions = 0.
-         assert (_recursions == 0, "invariant") ;
-         assert (_owner == Self, "invariant") ;
-         // CONSIDER: set or assert that OwnerIsThread == 1
-         return 1 ;
-      }
-      // The lock had been free momentarily, but we lost the race to the lock.
-      // Interference -- the CAS failed.
-      // We can either return -1 or retry.
-      // Retry doesn't make as much sense because the lock was just acquired.
-      if (true) return -1 ;
-   }
-}
-
-// NotRunnable() -- informed spinning
-//
-// Don't bother spinning if the owner is not eligible to drop the lock.
-// Peek at the owner's schedctl.sc_state and Thread._thread_values and
-// spin only if the owner thread is _thread_in_Java or _thread_in_vm.
-// The thread must be runnable in order to drop the lock in timely fashion.
-// If the _owner is not runnable then spinning will not likely be
-// successful (profitable).
-//
-// Beware -- the thread referenced by _owner could have died
-// so a simply fetch from _owner->_thread_state might trap.
-// Instead, we use SafeFetchXX() to safely LD _owner->_thread_state.
-// Because of the lifecycle issues the schedctl and _thread_state values
-// observed by NotRunnable() might be garbage.  NotRunnable must
-// tolerate this and consider the observed _thread_state value
-// as advisory.
-//
-// Beware too, that _owner is sometimes a BasicLock address and sometimes
-// a thread pointer.  We differentiate the two cases with OwnerIsThread.
-// Alternately, we might tag the type (thread pointer vs basiclock pointer)
-// with the LSB of _owner.  Another option would be to probablistically probe
-// the putative _owner->TypeTag value.
-//
-// Checking _thread_state isn't perfect.  Even if the thread is
-// in_java it might be blocked on a page-fault or have been preempted
-// and sitting on a ready/dispatch queue.  _thread state in conjunction
-// with schedctl.sc_state gives us a good picture of what the
-// thread is doing, however.
-//
-// TODO: check schedctl.sc_state.
-// We'll need to use SafeFetch32() to read from the schedctl block.
-// See RFE #5004247 and http://sac.sfbay.sun.com/Archives/CaseLog/arc/PSARC/2005/351/
-//
-// The return value from NotRunnable() is *advisory* -- the
-// result is based on sampling and is not necessarily coherent.
-// The caller must tolerate false-negative and false-positive errors.
-// Spinning, in general, is probabilistic anyway.
-
-
-int ObjectMonitor::NotRunnable (Thread * Self, Thread * ox) {
-    // Check either OwnerIsThread or ox->TypeTag == 2BAD.
-    if (!OwnerIsThread) return 0 ;
-
-    if (ox == NULL) return 0 ;
-
-    // Avoid transitive spinning ...
-    // Say T1 spins or blocks trying to acquire L.  T1._Stalled is set to L.
-    // Immediately after T1 acquires L it's possible that T2, also
-    // spinning on L, will see L.Owner=T1 and T1._Stalled=L.
-    // This occurs transiently after T1 acquired L but before
-    // T1 managed to clear T1.Stalled.  T2 does not need to abort
-    // its spin in this circumstance.
-    intptr_t BlockedOn = SafeFetchN ((intptr_t *) &ox->_Stalled, intptr_t(1)) ;
-
-    if (BlockedOn == 1) return 1 ;
-    if (BlockedOn != 0) {
-      return BlockedOn != intptr_t(this) && _owner == ox ;
-    }
-
-    assert (sizeof(((JavaThread *)ox)->_thread_state == sizeof(int)), "invariant") ;
-    int jst = SafeFetch32 ((int *) &((JavaThread *) ox)->_thread_state, -1) ; ;
-    // consider also: jst != _thread_in_Java -- but that's overspecific.
-    return jst == _thread_blocked || jst == _thread_in_native ;
-}
-
-
-// Adaptive spin-then-block - rational spinning
-//
-// Note that we spin "globally" on _owner with a classic SMP-polite TATAS
-// algorithm.  On high order SMP systems it would be better to start with
-// a brief global spin and then revert to spinning locally.  In the spirit of MCS/CLH,
-// a contending thread could enqueue itself on the cxq and then spin locally
-// on a thread-specific variable such as its ParkEvent._Event flag.
-// That's left as an exercise for the reader.  Note that global spinning is
-// not problematic on Niagara, as the L2$ serves the interconnect and has both
-// low latency and massive bandwidth.
-//
-// Broadly, we can fix the spin frequency -- that is, the % of contended lock
-// acquisition attempts where we opt to spin --  at 100% and vary the spin count
-// (duration) or we can fix the count at approximately the duration of
-// a context switch and vary the frequency.   Of course we could also
-// vary both satisfying K == Frequency * Duration, where K is adaptive by monitor.
-// See http://j2se.east/~dice/PERSIST/040824-AdaptiveSpinning.html.
-//
-// This implementation varies the duration "D", where D varies with
-// the success rate of recent spin attempts. (D is capped at approximately
-// length of a round-trip context switch).  The success rate for recent
-// spin attempts is a good predictor of the success rate of future spin
-// attempts.  The mechanism adapts automatically to varying critical
-// section length (lock modality), system load and degree of parallelism.
-// D is maintained per-monitor in _SpinDuration and is initialized
-// optimistically.  Spin frequency is fixed at 100%.
-//
-// Note that _SpinDuration is volatile, but we update it without locks
-// or atomics.  The code is designed so that _SpinDuration stays within
-// a reasonable range even in the presence of races.  The arithmetic
-// operations on _SpinDuration are closed over the domain of legal values,
-// so at worst a race will install and older but still legal value.
-// At the very worst this introduces some apparent non-determinism.
-// We might spin when we shouldn't or vice-versa, but since the spin
-// count are relatively short, even in the worst case, the effect is harmless.
-//
-// Care must be taken that a low "D" value does not become an
-// an absorbing state.  Transient spinning failures -- when spinning
-// is overall profitable -- should not cause the system to converge
-// on low "D" values.  We want spinning to be stable and predictable
-// and fairly responsive to change and at the same time we don't want
-// it to oscillate, become metastable, be "too" non-deterministic,
-// or converge on or enter undesirable stable absorbing states.
-//
-// We implement a feedback-based control system -- using past behavior
-// to predict future behavior.  We face two issues: (a) if the
-// input signal is random then the spin predictor won't provide optimal
-// results, and (b) if the signal frequency is too high then the control
-// system, which has some natural response lag, will "chase" the signal.
-// (b) can arise from multimodal lock hold times.  Transient preemption
-// can also result in apparent bimodal lock hold times.
-// Although sub-optimal, neither condition is particularly harmful, as
-// in the worst-case we'll spin when we shouldn't or vice-versa.
-// The maximum spin duration is rather short so the failure modes aren't bad.
-// To be conservative, I've tuned the gain in system to bias toward
-// _not spinning.  Relatedly, the system can sometimes enter a mode where it
-// "rings" or oscillates between spinning and not spinning.  This happens
-// when spinning is just on the cusp of profitability, however, so the
-// situation is not dire.  The state is benign -- there's no need to add
-// hysteresis control to damp the transition rate between spinning and
-// not spinning.
-//
-// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
-//
-// Spin-then-block strategies ...
-//
-// Thoughts on ways to improve spinning :
-//
-// *  Periodically call {psr_}getloadavg() while spinning, and
-//    permit unbounded spinning if the load average is <
-//    the number of processors.  Beware, however, that getloadavg()
-//    is exceptionally fast on solaris (about 1/10 the cost of a full
-//    spin cycle, but quite expensive on linux.  Beware also, that
-//    multiple JVMs could "ring" or oscillate in a feedback loop.
-//    Sufficient damping would solve that problem.
-//
-// *  We currently use spin loops with iteration counters to approximate
-//    spinning for some interval.  Given the availability of high-precision
-//    time sources such as gethrtime(), %TICK, %STICK, RDTSC, etc., we should
-//    someday reimplement the spin loops to duration-based instead of iteration-based.
-//
-// *  Don't spin if there are more than N = (CPUs/2) threads
-//        currently spinning on the monitor (or globally).
-//    That is, limit the number of concurrent spinners.
-//    We might also limit the # of spinners in the JVM, globally.
-//
-// *  If a spinning thread observes _owner change hands it should
-//    abort the spin (and park immediately) or at least debit
-//    the spin counter by a large "penalty".
-//
-// *  Classically, the spin count is either K*(CPUs-1) or is a
-//        simple constant that approximates the length of a context switch.
-//    We currently use a value -- computed by a special utility -- that
-//    approximates round-trip context switch times.
-//
-// *  Normally schedctl_start()/_stop() is used to advise the kernel
-//    to avoid preempting threads that are running in short, bounded
-//    critical sections.  We could use the schedctl hooks in an inverted
-//    sense -- spinners would set the nopreempt flag, but poll the preempt
-//    pending flag.  If a spinner observed a pending preemption it'd immediately
-//    abort the spin and park.   As such, the schedctl service acts as
-//    a preemption warning mechanism.
-//
-// *  In lieu of spinning, if the system is running below saturation
-//    (that is, loadavg() << #cpus), we can instead suppress futile
-//    wakeup throttling, or even wake more than one successor at exit-time.
-//    The net effect is largely equivalent to spinning.  In both cases,
-//    contending threads go ONPROC and opportunistically attempt to acquire
-//    the lock, decreasing lock handover latency at the expense of wasted
-//    cycles and context switching.
-//
-// *  We might to spin less after we've parked as the thread will
-//    have less $ and TLB affinity with the processor.
-//    Likewise, we might spin less if we come ONPROC on a different
-//    processor or after a long period (>> rechose_interval).
-//
-// *  A table-driven state machine similar to Solaris' dispadmin scheduling
-//    tables might be a better design.  Instead of encoding information in
-//    _SpinDuration, _SpinFreq and _SpinClock we'd just use explicit,
-//    discrete states.   Success or failure during a spin would drive
-//    state transitions, and each state node would contain a spin count.
-//
-// *  If the processor is operating in a mode intended to conserve power
-//    (such as Intel's SpeedStep) or to reduce thermal output (thermal
-//    step-down mode) then the Java synchronization subsystem should
-//    forgo spinning.
-//
-// *  The minimum spin duration should be approximately the worst-case
-//    store propagation latency on the platform.  That is, the time
-//    it takes a store on CPU A to become visible on CPU B, where A and
-//    B are "distant".
-//
-// *  We might want to factor a thread's priority in the spin policy.
-//    Threads with a higher priority might spin for slightly longer.
-//    Similarly, if we use back-off in the TATAS loop, lower priority
-//    threads might back-off longer.  We don't currently use a
-//    thread's priority when placing it on the entry queue.  We may
-//    want to consider doing so in future releases.
-//
-// *  We might transiently drop a thread's scheduling priority while it spins.
-//    SCHED_BATCH on linux and FX scheduling class at priority=0 on Solaris
-//    would suffice.  We could even consider letting the thread spin indefinitely at
-//    a depressed or "idle" priority.  This brings up fairness issues, however --
-//    in a saturated system a thread would with a reduced priority could languish
-//    for extended periods on the ready queue.
-//
-// *  While spinning try to use the otherwise wasted time to help the VM make
-//    progress:
-//
-//    -- YieldTo() the owner, if the owner is OFFPROC but ready
-//       Done our remaining quantum directly to the ready thread.
-//       This helps "push" the lock owner through the critical section.
-//       It also tends to improve affinity/locality as the lock
-//       "migrates" less frequently between CPUs.
-//    -- Walk our own stack in anticipation of blocking.  Memoize the roots.
-//    -- Perform strand checking for other thread.  Unpark potential strandees.
-//    -- Help GC: trace or mark -- this would need to be a bounded unit of work.
-//       Unfortunately this will pollute our $ and TLBs.  Recall that we
-//       spin to avoid context switching -- context switching has an
-//       immediate cost in latency, a disruptive cost to other strands on a CMT
-//       processor, and an amortized cost because of the D$ and TLB cache
-//       reload transient when the thread comes back ONPROC and repopulates
-//       $s and TLBs.
-//    -- call getloadavg() to see if the system is saturated.  It'd probably
-//       make sense to call getloadavg() half way through the spin.
-//       If the system isn't at full capacity the we'd simply reset
-//       the spin counter to and extend the spin attempt.
-//    -- Doug points out that we should use the same "helping" policy
-//       in thread.yield().
-//
-// *  Try MONITOR-MWAIT on systems that support those instructions.
-//
-// *  The spin statistics that drive spin decisions & frequency are
-//    maintained in the objectmonitor structure so if we deflate and reinflate
-//    we lose spin state.  In practice this is not usually a concern
-//    as the default spin state after inflation is aggressive (optimistic)
-//    and tends toward spinning.  So in the worst case for a lock where
-//    spinning is not profitable we may spin unnecessarily for a brief
-//    period.  But then again, if a lock is contended it'll tend not to deflate
-//    in the first place.
-
-
-intptr_t ObjectMonitor::SpinCallbackArgument = 0 ;
-int (*ObjectMonitor::SpinCallbackFunction)(intptr_t, int) = NULL ;
-
-// Spinning: Fixed frequency (100%), vary duration
-
-int ObjectMonitor::TrySpin_VaryDuration (Thread * Self) {
-
-    // Dumb, brutal spin.  Good for comparative measurements against adaptive spinning.
-    int ctr = Knob_FixedSpin ;
-    if (ctr != 0) {
-        while (--ctr >= 0) {
-            if (TryLock (Self) > 0) return 1 ;
-            SpinPause () ;
-        }
-        return 0 ;
-    }
-
-    for (ctr = Knob_PreSpin + 1; --ctr >= 0 ; ) {
-      if (TryLock(Self) > 0) {
-        // Increase _SpinDuration ...
-        // Note that we don't clamp SpinDuration precisely at SpinLimit.
-        // Raising _SpurDuration to the poverty line is key.
-        int x = _SpinDuration ;
-        if (x < Knob_SpinLimit) {
-           if (x < Knob_Poverty) x = Knob_Poverty ;
-           _SpinDuration = x + Knob_BonusB ;
-        }
-        return 1 ;
-      }
-      SpinPause () ;
-    }
-
-    // Admission control - verify preconditions for spinning
-    //
-    // We always spin a little bit, just to prevent _SpinDuration == 0 from
-    // becoming an absorbing state.  Put another way, we spin briefly to
-    // sample, just in case the system load, parallelism, contention, or lock
-    // modality changed.
-    //
-    // Consider the following alternative:
-    // Periodically set _SpinDuration = _SpinLimit and try a long/full
-    // spin attempt.  "Periodically" might mean after a tally of
-    // the # of failed spin attempts (or iterations) reaches some threshold.
-    // This takes us into the realm of 1-out-of-N spinning, where we
-    // hold the duration constant but vary the frequency.
-
-    ctr = _SpinDuration  ;
-    if (ctr < Knob_SpinBase) ctr = Knob_SpinBase ;
-    if (ctr <= 0) return 0 ;
-
-    if (Knob_SuccRestrict && _succ != NULL) return 0 ;
-    if (Knob_OState && NotRunnable (Self, (Thread *) _owner)) {
-       TEVENT (Spin abort - notrunnable [TOP]);
-       return 0 ;
-    }
-
-    int MaxSpin = Knob_MaxSpinners ;
-    if (MaxSpin >= 0) {
-       if (_Spinner > MaxSpin) {
-          TEVENT (Spin abort -- too many spinners) ;
-          return 0 ;
-       }
-       // Slighty racy, but benign ...
-       Adjust (&_Spinner, 1) ;
-    }
-
-    // We're good to spin ... spin ingress.
-    // CONSIDER: use Prefetch::write() to avoid RTS->RTO upgrades
-    // when preparing to LD...CAS _owner, etc and the CAS is likely
-    // to succeed.
-    int hits    = 0 ;
-    int msk     = 0 ;
-    int caspty  = Knob_CASPenalty ;
-    int oxpty   = Knob_OXPenalty ;
-    int sss     = Knob_SpinSetSucc ;
-    if (sss && _succ == NULL ) _succ = Self ;
-    Thread * prv = NULL ;
-
-    // There are three ways to exit the following loop:
-    // 1.  A successful spin where this thread has acquired the lock.
-    // 2.  Spin failure with prejudice
-    // 3.  Spin failure without prejudice
-
-    while (--ctr >= 0) {
-
-      // Periodic polling -- Check for pending GC
-      // Threads may spin while they're unsafe.
-      // We don't want spinning threads to delay the JVM from reaching
-      // a stop-the-world safepoint or to steal cycles from GC.
-      // If we detect a pending safepoint we abort in order that
-      // (a) this thread, if unsafe, doesn't delay the safepoint, and (b)
-      // this thread, if safe, doesn't steal cycles from GC.
-      // This is in keeping with the "no loitering in runtime" rule.
-      // We periodically check to see if there's a safepoint pending.
-      if ((ctr & 0xFF) == 0) {
-         if (SafepointSynchronize::do_call_back()) {
-            TEVENT (Spin: safepoint) ;
-            goto Abort ;           // abrupt spin egress
-         }
-         if (Knob_UsePause & 1) SpinPause () ;
-
-         int (*scb)(intptr_t,int) = SpinCallbackFunction ;
-         if (hits > 50 && scb != NULL) {
-            int abend = (*scb)(SpinCallbackArgument, 0) ;
-         }
-      }
-
-      if (Knob_UsePause & 2) SpinPause() ;
-
-      // Exponential back-off ...  Stay off the bus to reduce coherency traffic.
-      // This is useful on classic SMP systems, but is of less utility on
-      // N1-style CMT platforms.
-      //
-      // Trade-off: lock acquisition latency vs coherency bandwidth.
-      // Lock hold times are typically short.  A histogram
-      // of successful spin attempts shows that we usually acquire
-      // the lock early in the spin.  That suggests we want to
-      // sample _owner frequently in the early phase of the spin,
-      // but then back-off and sample less frequently as the spin
-      // progresses.  The back-off makes a good citizen on SMP big
-      // SMP systems.  Oversampling _owner can consume excessive
-      // coherency bandwidth.  Relatedly, if we _oversample _owner we
-      // can inadvertently interfere with the the ST m->owner=null.
-      // executed by the lock owner.
-      if (ctr & msk) continue ;
-      ++hits ;
-      if ((hits & 0xF) == 0) {
-        // The 0xF, above, corresponds to the exponent.
-        // Consider: (msk+1)|msk
-        msk = ((msk << 2)|3) & BackOffMask ;
-      }
-
-      // Probe _owner with TATAS
-      // If this thread observes the monitor transition or flicker
-      // from locked to unlocked to locked, then the odds that this
-      // thread will acquire the lock in this spin attempt go down
-      // considerably.  The same argument applies if the CAS fails
-      // or if we observe _owner change from one non-null value to
-      // another non-null value.   In such cases we might abort
-      // the spin without prejudice or apply a "penalty" to the
-      // spin count-down variable "ctr", reducing it by 100, say.
-
-      Thread * ox = (Thread *) _owner ;
-      if (ox == NULL) {
-         ox = (Thread *) Atomic::cmpxchg_ptr (Self, &_owner, NULL) ;
-         if (ox == NULL) {
-            // The CAS succeeded -- this thread acquired ownership
-            // Take care of some bookkeeping to exit spin state.
-            if (sss && _succ == Self) {
-               _succ = NULL ;
-            }
-            if (MaxSpin > 0) Adjust (&_Spinner, -1) ;
-
-            // Increase _SpinDuration :
-            // The spin was successful (profitable) so we tend toward
-            // longer spin attempts in the future.
-            // CONSIDER: factor "ctr" into the _SpinDuration adjustment.
-            // If we acquired the lock early in the spin cycle it
-            // makes sense to increase _SpinDuration proportionally.
-            // Note that we don't clamp SpinDuration precisely at SpinLimit.
-            int x = _SpinDuration ;
-            if (x < Knob_SpinLimit) {
-                if (x < Knob_Poverty) x = Knob_Poverty ;
-                _SpinDuration = x + Knob_Bonus ;
-            }
-            return 1 ;
-         }
-
-         // The CAS failed ... we can take any of the following actions:
-         // * penalize: ctr -= Knob_CASPenalty
-         // * exit spin with prejudice -- goto Abort;
-         // * exit spin without prejudice.
-         // * Since CAS is high-latency, retry again immediately.
-         prv = ox ;
-         TEVENT (Spin: cas failed) ;
-         if (caspty == -2) break ;
-         if (caspty == -1) goto Abort ;
-         ctr -= caspty ;
-         continue ;
-      }
-
-      // Did lock ownership change hands ?
-      if (ox != prv && prv != NULL ) {
-          TEVENT (spin: Owner changed)
-          if (oxpty == -2) break ;
-          if (oxpty == -1) goto Abort ;
-          ctr -= oxpty ;
-      }
-      prv = ox ;
-
-      // Abort the spin if the owner is not executing.
-      // The owner must be executing in order to drop the lock.
-      // Spinning while the owner is OFFPROC is idiocy.
-      // Consider: ctr -= RunnablePenalty ;
-      if (Knob_OState && NotRunnable (Self, ox)) {
-         TEVENT (Spin abort - notrunnable);
-         goto Abort ;
-      }
-      if (sss && _succ == NULL ) _succ = Self ;
-   }
-
-   // Spin failed with prejudice -- reduce _SpinDuration.
-   // TODO: Use an AIMD-like policy to adjust _SpinDuration.
-   // AIMD is globally stable.
-   TEVENT (Spin failure) ;
-   {
-     int x = _SpinDuration ;
-     if (x > 0) {
-        // Consider an AIMD scheme like: x -= (x >> 3) + 100
-        // This is globally sample and tends to damp the response.
-        x -= Knob_Penalty ;
-        if (x < 0) x = 0 ;
-        _SpinDuration = x ;
-     }
-   }
-
- Abort:
-   if (MaxSpin >= 0) Adjust (&_Spinner, -1) ;
-   if (sss && _succ == Self) {
-      _succ = NULL ;
-      // Invariant: after setting succ=null a contending thread
-      // must recheck-retry _owner before parking.  This usually happens
-      // in the normal usage of TrySpin(), but it's safest
-      // to make TrySpin() as foolproof as possible.
-      OrderAccess::fence() ;
-      if (TryLock(Self) > 0) return 1 ;
-   }
-   return 0 ;
-}
-
-#define TrySpin TrySpin_VaryDuration
-
-static void DeferredInitialize () {
-  if (InitDone > 0) return ;
-  if (Atomic::cmpxchg (-1, &InitDone, 0) != 0) {
-      while (InitDone != 1) ;
-      return ;
-  }
-
-  // One-shot global initialization ...
-  // The initialization is idempotent, so we don't need locks.
-  // In the future consider doing this via os::init_2().
-  // SyncKnobs consist of <Key>=<Value> pairs in the style
-  // of environment variables.  Start by converting ':' to NUL.
-
-  if (SyncKnobs == NULL) SyncKnobs = "" ;
-
-  size_t sz = strlen (SyncKnobs) ;
-  char * knobs = (char *) malloc (sz + 2) ;
-  if (knobs == NULL) {
-     vm_exit_out_of_memory (sz + 2, "Parse SyncKnobs") ;
-     guarantee (0, "invariant") ;
-  }
-  strcpy (knobs, SyncKnobs) ;
-  knobs[sz+1] = 0 ;
-  for (char * p = knobs ; *p ; p++) {
-     if (*p == ':') *p = 0 ;
-  }
-
-  #define SETKNOB(x) { Knob_##x = kvGetInt (knobs, #x, Knob_##x); }
-  SETKNOB(ReportSettings) ;
-  SETKNOB(Verbose) ;
-  SETKNOB(FixedSpin) ;
-  SETKNOB(SpinLimit) ;
-  SETKNOB(SpinBase) ;
-  SETKNOB(SpinBackOff);
-  SETKNOB(CASPenalty) ;
-  SETKNOB(OXPenalty) ;
-  SETKNOB(LogSpins) ;
-  SETKNOB(SpinSetSucc) ;
-  SETKNOB(SuccEnabled) ;
-  SETKNOB(SuccRestrict) ;
-  SETKNOB(Penalty) ;
-  SETKNOB(Bonus) ;
-  SETKNOB(BonusB) ;
-  SETKNOB(Poverty) ;
-  SETKNOB(SpinAfterFutile) ;
-  SETKNOB(UsePause) ;
-  SETKNOB(SpinEarly) ;
-  SETKNOB(OState) ;
-  SETKNOB(MaxSpinners) ;
-  SETKNOB(PreSpin) ;
-  SETKNOB(ExitPolicy) ;
-  SETKNOB(QMode);
-  SETKNOB(ResetEvent) ;
-  SETKNOB(MoveNotifyee) ;
-  SETKNOB(FastHSSEC) ;
-  #undef SETKNOB
-
-  if (os::is_MP()) {
-     BackOffMask = (1 << Knob_SpinBackOff) - 1 ;
-     if (Knob_ReportSettings) ::printf ("BackOffMask=%X\n", BackOffMask) ;
-     // CONSIDER: BackOffMask = ROUNDUP_NEXT_POWER2 (ncpus-1)
-  } else {
-     Knob_SpinLimit = 0 ;
-     Knob_SpinBase  = 0 ;
-     Knob_PreSpin   = 0 ;
-     Knob_FixedSpin = -1 ;
-  }
-
-  if (Knob_LogSpins == 0) {
-     ObjectSynchronizer::_sync_FailedSpins = NULL ;
-  }
-
-  free (knobs) ;
-  OrderAccess::fence() ;
-  InitDone = 1 ;
-}
-
-// Theory of operations -- Monitors lists, thread residency, etc:
-//
-// * A thread acquires ownership of a monitor by successfully
-//   CAS()ing the _owner field from null to non-null.
-//
-// * Invariant: A thread appears on at most one monitor list --
-//   cxq, EntryList or WaitSet -- at any one time.
-//
-// * Contending threads "push" themselves onto the cxq with CAS
-//   and then spin/park.
-//
-// * After a contending thread eventually acquires the lock it must
-//   dequeue itself from either the EntryList or the cxq.
-//
-// * The exiting thread identifies and unparks an "heir presumptive"
-//   tentative successor thread on the EntryList.  Critically, the
-//   exiting thread doesn't unlink the successor thread from the EntryList.
-//   After having been unparked, the wakee will recontend for ownership of
-//   the monitor.   The successor (wakee) will either acquire the lock or
-//   re-park itself.
-//
-//   Succession is provided for by a policy of competitive handoff.
-//   The exiting thread does _not_ grant or pass ownership to the
-//   successor thread.  (This is also referred to as "handoff" succession").
-//   Instead the exiting thread releases ownership and possibly wakes
-//   a successor, so the successor can (re)compete for ownership of the lock.
-//   If the EntryList is empty but the cxq is populated the exiting
-//   thread will drain the cxq into the EntryList.  It does so by
-//   by detaching the cxq (installing null with CAS) and folding
-//   the threads from the cxq into the EntryList.  The EntryList is
-//   doubly linked, while the cxq is singly linked because of the
-//   CAS-based "push" used to enqueue recently arrived threads (RATs).
-//
-// * Concurrency invariants:
-//
-//   -- only the monitor owner may access or mutate the EntryList.
-//      The mutex property of the monitor itself protects the EntryList
-//      from concurrent interference.
-//   -- Only the monitor owner may detach the cxq.
-//
-// * The monitor entry list operations avoid locks, but strictly speaking
-//   they're not lock-free.  Enter is lock-free, exit is not.
-//   See http://j2se.east/~dice/PERSIST/040825-LockFreeQueues.html
-//
-// * The cxq can have multiple concurrent "pushers" but only one concurrent
-//   detaching thread.  This mechanism is immune from the ABA corruption.
-//   More precisely, the CAS-based "push" onto cxq is ABA-oblivious.
-//
-// * Taken together, the cxq and the EntryList constitute or form a
-//   single logical queue of threads stalled trying to acquire the lock.
-//   We use two distinct lists to improve the odds of a constant-time
-//   dequeue operation after acquisition (in the ::enter() epilog) and
-//   to reduce heat on the list ends.  (c.f. Michael Scott's "2Q" algorithm).
-//   A key desideratum is to minimize queue & monitor metadata manipulation
-//   that occurs while holding the monitor lock -- that is, we want to
-//   minimize monitor lock holds times.  Note that even a small amount of
-//   fixed spinning will greatly reduce the # of enqueue-dequeue operations
-//   on EntryList|cxq.  That is, spinning relieves contention on the "inner"
-//   locks and monitor metadata.
-//
-//   Cxq points to the the set of Recently Arrived Threads attempting entry.
-//   Because we push threads onto _cxq with CAS, the RATs must take the form of
-//   a singly-linked LIFO.  We drain _cxq into EntryList  at unlock-time when
-//   the unlocking thread notices that EntryList is null but _cxq is != null.
-//
-//   The EntryList is ordered by the prevailing queue discipline and
-//   can be organized in any convenient fashion, such as a doubly-linked list or
-//   a circular doubly-linked list.  Critically, we want insert and delete operations
-//   to operate in constant-time.  If we need a priority queue then something akin
-//   to Solaris' sleepq would work nicely.  Viz.,
-//   http://agg.eng/ws/on10_nightly/source/usr/src/uts/common/os/sleepq.c.
-//   Queue discipline is enforced at ::exit() time, when the unlocking thread
-//   drains the cxq into the EntryList, and orders or reorders the threads on the
-//   EntryList accordingly.
-//
-//   Barring "lock barging", this mechanism provides fair cyclic ordering,
-//   somewhat similar to an elevator-scan.
-//
-// * The monitor synchronization subsystem avoids the use of native
-//   synchronization primitives except for the narrow platform-specific
-//   park-unpark abstraction.  See the comments in os_solaris.cpp regarding
-//   the semantics of park-unpark.  Put another way, this monitor implementation
-//   depends only on atomic operations and park-unpark.  The monitor subsystem
-//   manages all RUNNING->BLOCKED and BLOCKED->READY transitions while the
-//   underlying OS manages the READY<->RUN transitions.
-//
-// * Waiting threads reside on the WaitSet list -- wait() puts
-//   the caller onto the WaitSet.
-//
-// * notify() or notifyAll() simply transfers threads from the WaitSet to
-//   either the EntryList or cxq.  Subsequent exit() operations will
-//   unpark the notifyee.  Unparking a notifee in notify() is inefficient -
-//   it's likely the notifyee would simply impale itself on the lock held
-//   by the notifier.
-//
-// * An interesting alternative is to encode cxq as (List,LockByte) where
-//   the LockByte is 0 iff the monitor is owned.  _owner is simply an auxiliary
-//   variable, like _recursions, in the scheme.  The threads or Events that form
-//   the list would have to be aligned in 256-byte addresses.  A thread would
-//   try to acquire the lock or enqueue itself with CAS, but exiting threads
-//   could use a 1-0 protocol and simply STB to set the LockByte to 0.
-//   Note that is is *not* word-tearing, but it does presume that full-word
-//   CAS operations are coherent with intermix with STB operations.  That's true
-//   on most common processors.
-//
-// * See also http://blogs.sun.com/dave
-
-
-void ATTR ObjectMonitor::EnterI (TRAPS) {
-    Thread * Self = THREAD ;
-    assert (Self->is_Java_thread(), "invariant") ;
-    assert (((JavaThread *) Self)->thread_state() == _thread_blocked   , "invariant") ;
-
-    // Try the lock - TATAS
-    if (TryLock (Self) > 0) {
-        assert (_succ != Self              , "invariant") ;
-        assert (_owner == Self             , "invariant") ;
-        assert (_Responsible != Self       , "invariant") ;
-        return ;
-    }
-
-    DeferredInitialize () ;
-
-    // We try one round of spinning *before* enqueueing Self.
-    //
-    // If the _owner is ready but OFFPROC we could use a YieldTo()
-    // operation to donate the remainder of this thread's quantum
-    // to the owner.  This has subtle but beneficial affinity
-    // effects.
-
-    if (TrySpin (Self) > 0) {
-        assert (_owner == Self        , "invariant") ;
-        assert (_succ != Self         , "invariant") ;
-        assert (_Responsible != Self  , "invariant") ;
-        return ;
-    }
-
-    // The Spin failed -- Enqueue and park the thread ...
-    assert (_succ  != Self            , "invariant") ;
-    assert (_owner != Self            , "invariant") ;
-    assert (_Responsible != Self      , "invariant") ;
-
-    // Enqueue "Self" on ObjectMonitor's _cxq.
-    //
-    // Node acts as a proxy for Self.
-    // As an aside, if were to ever rewrite the synchronization code mostly
-    // in Java, WaitNodes, ObjectMonitors, and Events would become 1st-class
-    // Java objects.  This would avoid awkward lifecycle and liveness issues,
-    // as well as eliminate a subset of ABA issues.
-    // TODO: eliminate ObjectWaiter and enqueue either Threads or Events.
-    //
-
-    ObjectWaiter node(Self) ;
-    Self->_ParkEvent->reset() ;
-    node._prev   = (ObjectWaiter *) 0xBAD ;
-    node.TState  = ObjectWaiter::TS_CXQ ;
-
-    // Push "Self" onto the front of the _cxq.
-    // Once on cxq/EntryList, Self stays on-queue until it acquires the lock.
-    // Note that spinning tends to reduce the rate at which threads
-    // enqueue and dequeue on EntryList|cxq.
-    ObjectWaiter * nxt ;
-    for (;;) {
-        node._next = nxt = _cxq ;
-        if (Atomic::cmpxchg_ptr (&node, &_cxq, nxt) == nxt) break ;
-
-        // Interference - the CAS failed because _cxq changed.  Just retry.
-        // As an optional optimization we retry the lock.
-        if (TryLock (Self) > 0) {
-            assert (_succ != Self         , "invariant") ;
-            assert (_owner == Self        , "invariant") ;
-            assert (_Responsible != Self  , "invariant") ;
-            return ;
-        }
-    }
-
-    // Check for cxq|EntryList edge transition to non-null.  This indicates
-    // the onset of contention.  While contention persists exiting threads
-    // will use a ST:MEMBAR:LD 1-1 exit protocol.  When contention abates exit
-    // operations revert to the faster 1-0 mode.  This enter operation may interleave
-    // (race) a concurrent 1-0 exit operation, resulting in stranding, so we
-    // arrange for one of the contending thread to use a timed park() operations
-    // to detect and recover from the race.  (Stranding is form of progress failure
-    // where the monitor is unlocked but all the contending threads remain parked).
-    // That is, at least one of the contended threads will periodically poll _owner.
-    // One of the contending threads will become the designated "Responsible" thread.
-    // The Responsible thread uses a timed park instead of a normal indefinite park
-    // operation -- it periodically wakes and checks for and recovers from potential
-    // strandings admitted by 1-0 exit operations.   We need at most one Responsible
-    // thread per-monitor at any given moment.  Only threads on cxq|EntryList may
-    // be responsible for a monitor.
-    //
-    // Currently, one of the contended threads takes on the added role of "Responsible".
-    // A viable alternative would be to use a dedicated "stranding checker" thread
-    // that periodically iterated over all the threads (or active monitors) and unparked
-    // successors where there was risk of stranding.  This would help eliminate the
-    // timer scalability issues we see on some platforms as we'd only have one thread
-    // -- the checker -- parked on a timer.
-
-    if ((SyncFlags & 16) == 0 && nxt == NULL && _EntryList == NULL) {
-        // Try to assume the role of responsible thread for the monitor.
-        // CONSIDER:  ST vs CAS vs { if (Responsible==null) Responsible=Self }
-        Atomic::cmpxchg_ptr (Self, &_Responsible, NULL) ;
-    }
-
-    // The lock have been released while this thread was occupied queueing
-    // itself onto _cxq.  To close the race and avoid "stranding" and
-    // progress-liveness failure we must resample-retry _owner before parking.
-    // Note the Dekker/Lamport duality: ST cxq; MEMBAR; LD Owner.
-    // In this case the ST-MEMBAR is accomplished with CAS().
-    //
-    // TODO: Defer all thread state transitions until park-time.
-    // Since state transitions are heavy and inefficient we'd like
-    // to defer the state transitions until absolutely necessary,
-    // and in doing so avoid some transitions ...
-
-    TEVENT (Inflated enter - Contention) ;
-    int nWakeups = 0 ;
-    int RecheckInterval = 1 ;
-
-    for (;;) {
-
-        if (TryLock (Self) > 0) break ;
-        assert (_owner != Self, "invariant") ;
-
-        if ((SyncFlags & 2) && _Responsible == NULL) {
-           Atomic::cmpxchg_ptr (Self, &_Responsible, NULL) ;
-        }
-
-        // park self
-        if (_Responsible == Self || (SyncFlags & 1)) {
-            TEVENT (Inflated enter - park TIMED) ;
-            Self->_ParkEvent->park ((jlong) RecheckInterval) ;
-            // Increase the RecheckInterval, but clamp the value.
-            RecheckInterval *= 8 ;
-            if (RecheckInterval > 1000) RecheckInterval = 1000 ;
-        } else {
-            TEVENT (Inflated enter - park UNTIMED) ;
-            Self->_ParkEvent->park() ;
-        }
-
-        if (TryLock(Self) > 0) break ;
-
-        // The lock is still contested.
-        // Keep a tally of the # of futile wakeups.
-        // Note that the counter is not protected by a lock or updated by atomics.
-        // That is by design - we trade "lossy" counters which are exposed to
-        // races during updates for a lower probe effect.
-        TEVENT (Inflated enter - Futile wakeup) ;
-        if (ObjectSynchronizer::_sync_FutileWakeups != NULL) {
-           ObjectSynchronizer::_sync_FutileWakeups->inc() ;
-        }
-        ++ nWakeups ;
-
-        // Assuming this is not a spurious wakeup we'll normally find _succ == Self.
-        // We can defer clearing _succ until after the spin completes
-        // TrySpin() must tolerate being called with _succ == Self.
-        // Try yet another round of adaptive spinning.
-        if ((Knob_SpinAfterFutile & 1) && TrySpin (Self) > 0) break ;
-
-        // We can find that we were unpark()ed and redesignated _succ while
-        // we were spinning.  That's harmless.  If we iterate and call park(),
-        // park() will consume the event and return immediately and we'll
-        // just spin again.  This pattern can repeat, leaving _succ to simply
-        // spin on a CPU.  Enable Knob_ResetEvent to clear pending unparks().
-        // Alternately, we can sample fired() here, and if set, forgo spinning
-        // in the next iteration.
-
-        if ((Knob_ResetEvent & 1) && Self->_ParkEvent->fired()) {
-           Self->_ParkEvent->reset() ;
-           OrderAccess::fence() ;
-        }
-        if (_succ == Self) _succ = NULL ;
-
-        // Invariant: after clearing _succ a thread *must* retry _owner before parking.
-        OrderAccess::fence() ;
-    }
-
-    // Egress :
-    // Self has acquired the lock -- Unlink Self from the cxq or EntryList.
-    // Normally we'll find Self on the EntryList .
-    // From the perspective of the lock owner (this thread), the
-    // EntryList is stable and cxq is prepend-only.
-    // The head of cxq is volatile but the interior is stable.
-    // In addition, Self.TState is stable.
-
-    assert (_owner == Self      , "invariant") ;
-    assert (object() != NULL    , "invariant") ;
-    // I'd like to write:
-    //   guarantee (((oop)(object()))->mark() == markOopDesc::encode(this), "invariant") ;
-    // but as we're at a safepoint that's not safe.
-
-    UnlinkAfterAcquire (Self, &node) ;
-    if (_succ == Self) _succ = NULL ;
-
-    assert (_succ != Self, "invariant") ;
-    if (_Responsible == Self) {
-        _Responsible = NULL ;
-        // Dekker pivot-point.
-        // Consider OrderAccess::storeload() here
-
-        // We may leave threads on cxq|EntryList without a designated
-        // "Responsible" thread.  This is benign.  When this thread subsequently
-        // exits the monitor it can "see" such preexisting "old" threads --
-        // threads that arrived on the cxq|EntryList before the fence, above --
-        // by LDing cxq|EntryList.  Newly arrived threads -- that is, threads
-        // that arrive on cxq after the ST:MEMBAR, above -- will set Responsible
-        // non-null and elect a new "Responsible" timer thread.
-        //
-        // This thread executes:
-        //    ST Responsible=null; MEMBAR    (in enter epilog - here)
-        //    LD cxq|EntryList               (in subsequent exit)
-        //
-        // Entering threads in the slow/contended path execute:
-        //    ST cxq=nonnull; MEMBAR; LD Responsible (in enter prolog)
-        //    The (ST cxq; MEMBAR) is accomplished with CAS().
-        //
-        // The MEMBAR, above, prevents the LD of cxq|EntryList in the subsequent
-        // exit operation from floating above the ST Responsible=null.
-        //
-        // In *practice* however, EnterI() is always followed by some atomic
-        // operation such as the decrement of _count in ::enter().  Those atomics
-        // obviate the need for the explicit MEMBAR, above.
-    }
-
-    // We've acquired ownership with CAS().
-    // CAS is serializing -- it has MEMBAR/FENCE-equivalent semantics.
-    // But since the CAS() this thread may have also stored into _succ,
-    // EntryList, cxq or Responsible.  These meta-data updates must be
-    // visible __before this thread subsequently drops the lock.
-    // Consider what could occur if we didn't enforce this constraint --
-    // STs to monitor meta-data and user-data could reorder with (become
-    // visible after) the ST in exit that drops ownership of the lock.
-    // Some other thread could then acquire the lock, but observe inconsistent
-    // or old monitor meta-data and heap data.  That violates the JMM.
-    // To that end, the 1-0 exit() operation must have at least STST|LDST
-    // "release" barrier semantics.  Specifically, there must be at least a
-    // STST|LDST barrier in exit() before the ST of null into _owner that drops
-    // the lock.   The barrier ensures that changes to monitor meta-data and data
-    // protected by the lock will be visible before we release the lock, and
-    // therefore before some other thread (CPU) has a chance to acquire the lock.
-    // See also: http://gee.cs.oswego.edu/dl/jmm/cookbook.html.
-    //
-    // Critically, any prior STs to _succ or EntryList must be visible before
-    // the ST of null into _owner in the *subsequent* (following) corresponding
-    // monitorexit.  Recall too, that in 1-0 mode monitorexit does not necessarily
-    // execute a serializing instruction.
-
-    if (SyncFlags & 8) {
-       OrderAccess::fence() ;
-    }
-    return ;
-}
-
-// ExitSuspendEquivalent:
-// A faster alternate to handle_special_suspend_equivalent_condition()
-//
-// handle_special_suspend_equivalent_condition() unconditionally
-// acquires the SR_lock.  On some platforms uncontended MutexLocker()
-// operations have high latency.  Note that in ::enter() we call HSSEC
-// while holding the monitor, so we effectively lengthen the critical sections.
-//
-// There are a number of possible solutions:
-//
-// A.  To ameliorate the problem we might also defer state transitions
-//     to as late as possible -- just prior to parking.
-//     Given that, we'd call HSSEC after having returned from park(),
-//     but before attempting to acquire the monitor.  This is only a
-//     partial solution.  It avoids calling HSSEC while holding the
-//     monitor (good), but it still increases successor reacquisition latency --
-//     the interval between unparking a successor and the time the successor
-//     resumes and retries the lock.  See ReenterI(), which defers state transitions.
-//     If we use this technique we can also avoid EnterI()-exit() loop
-//     in ::enter() where we iteratively drop the lock and then attempt
-//     to reacquire it after suspending.
-//
-// B.  In the future we might fold all the suspend bits into a
-//     composite per-thread suspend flag and then update it with CAS().
-//     Alternately, a Dekker-like mechanism with multiple variables
-//     would suffice:
-//       ST Self->_suspend_equivalent = false
-//       MEMBAR
-//       LD Self_>_suspend_flags
-//
-
-
-bool ObjectMonitor::ExitSuspendEquivalent (JavaThread * jSelf) {
-   int Mode = Knob_FastHSSEC ;
-   if (Mode && !jSelf->is_external_suspend()) {
-      assert (jSelf->is_suspend_equivalent(), "invariant") ;
-      jSelf->clear_suspend_equivalent() ;
-      if (2 == Mode) OrderAccess::storeload() ;
-      if (!jSelf->is_external_suspend()) return false ;
-      // We raced a suspension -- fall thru into the slow path
-      TEVENT (ExitSuspendEquivalent - raced) ;
-      jSelf->set_suspend_equivalent() ;
-   }
-   return jSelf->handle_special_suspend_equivalent_condition() ;
-}
-
-
-// ReenterI() is a specialized inline form of the latter half of the
-// contended slow-path from EnterI().  We use ReenterI() only for
-// monitor reentry in wait().
-//
-// In the future we should reconcile EnterI() and ReenterI(), adding
-// Knob_Reset and Knob_SpinAfterFutile support and restructuring the
-// loop accordingly.
-
-void ATTR ObjectMonitor::ReenterI (Thread * Self, ObjectWaiter * SelfNode) {
-    assert (Self != NULL                , "invariant") ;
-    assert (SelfNode != NULL            , "invariant") ;
-    assert (SelfNode->_thread == Self   , "invariant") ;
-    assert (_waiters > 0                , "invariant") ;
-    assert (((oop)(object()))->mark() == markOopDesc::encode(this) , "invariant") ;
-    assert (((JavaThread *)Self)->thread_state() != _thread_blocked, "invariant") ;
-    JavaThread * jt = (JavaThread *) Self ;
-
-    int nWakeups = 0 ;
-    for (;;) {
-        ObjectWaiter::TStates v = SelfNode->TState ;
-        guarantee (v == ObjectWaiter::TS_ENTER || v == ObjectWaiter::TS_CXQ, "invariant") ;
-        assert    (_owner != Self, "invariant") ;
-
-        if (TryLock (Self) > 0) break ;
-        if (TrySpin (Self) > 0) break ;
-
-        TEVENT (Wait Reentry - parking) ;
-
-        // State transition wrappers around park() ...
-        // ReenterI() wisely defers state transitions until
-        // it's clear we must park the thread.
-        {
-           OSThreadContendState osts(Self->osthread());
-           ThreadBlockInVM tbivm(jt);
-
-           // cleared by handle_special_suspend_equivalent_condition()
-           // or java_suspend_self()
-           jt->set_suspend_equivalent();
-           if (SyncFlags & 1) {
-              Self->_ParkEvent->park ((jlong)1000) ;
-           } else {
-              Self->_ParkEvent->park () ;
-           }
-
-           // were we externally suspended while we were waiting?
-           for (;;) {
-              if (!ExitSuspendEquivalent (jt)) break ;
-              if (_succ == Self) { _succ = NULL; OrderAccess::fence(); }
-              jt->java_suspend_self();
-              jt->set_suspend_equivalent();
-           }
-        }
-
-        // Try again, but just so we distinguish between futile wakeups and
-        // successful wakeups.  The following test isn't algorithmically
-        // necessary, but it helps us maintain sensible statistics.
-        if (TryLock(Self) > 0) break ;
-
-        // The lock is still contested.
-        // Keep a tally of the # of futile wakeups.
-        // Note that the counter is not protected by a lock or updated by atomics.
-        // That is by design - we trade "lossy" counters which are exposed to
-        // races during updates for a lower probe effect.
-        TEVENT (Wait Reentry - futile wakeup) ;
-        ++ nWakeups ;
-
-        // Assuming this is not a spurious wakeup we'll normally
-        // find that _succ == Self.
-        if (_succ == Self) _succ = NULL ;
-
-        // Invariant: after clearing _succ a contending thread
-        // *must* retry  _owner before parking.
-        OrderAccess::fence() ;
-
-        if (ObjectSynchronizer::_sync_FutileWakeups != NULL) {
-          ObjectSynchronizer::_sync_FutileWakeups->inc() ;
-        }
-    }
-
-    // Self has acquired the lock -- Unlink Self from the cxq or EntryList .
-    // Normally we'll find Self on the EntryList.
-    // Unlinking from the EntryList is constant-time and atomic-free.
-    // From the perspective of the lock owner (this thread), the
-    // EntryList is stable and cxq is prepend-only.
-    // The head of cxq is volatile but the interior is stable.
-    // In addition, Self.TState is stable.
-
-    assert (_owner == Self, "invariant") ;
-    assert (((oop)(object()))->mark() == markOopDesc::encode(this), "invariant") ;
-    UnlinkAfterAcquire (Self, SelfNode) ;
-    if (_succ == Self) _succ = NULL ;
-    assert (_succ != Self, "invariant") ;
-    SelfNode->TState = ObjectWaiter::TS_RUN ;
-    OrderAccess::fence() ;      // see comments at the end of EnterI()
-}
-
-bool ObjectMonitor::try_enter(Thread* THREAD) {
-  if (THREAD != _owner) {
-    if (THREAD->is_lock_owned ((address)_owner)) {
-       assert(_recursions == 0, "internal state error");
-       _owner = THREAD ;
-       _recursions = 1 ;
-       OwnerIsThread = 1 ;
-       return true;
-    }
-    if (Atomic::cmpxchg_ptr (THREAD, &_owner, NULL) != NULL) {
-      return false;
-    }
-    return true;
-  } else {
-    _recursions++;
-    return true;
-  }
-}
-
-void ATTR ObjectMonitor::enter(TRAPS) {
-  // The following code is ordered to check the most common cases first
-  // and to reduce RTS->RTO cache line upgrades on SPARC and IA32 processors.
-  Thread * const Self = THREAD ;
-  void * cur ;
-
-  cur = Atomic::cmpxchg_ptr (Self, &_owner, NULL) ;
-  if (cur == NULL) {
-     // Either ASSERT _recursions == 0 or explicitly set _recursions = 0.
-     assert (_recursions == 0   , "invariant") ;
-     assert (_owner      == Self, "invariant") ;
-     // CONSIDER: set or assert OwnerIsThread == 1
-     return ;
-  }
-
-  if (cur == Self) {
-     // TODO-FIXME: check for integer overflow!  BUGID 6557169.
-     _recursions ++ ;
-     return ;
-  }
-
-  if (Self->is_lock_owned ((address)cur)) {
-    assert (_recursions == 0, "internal state error");
-    _recursions = 1 ;
-    // Commute owner from a thread-specific on-stack BasicLockObject address to
-    // a full-fledged "Thread *".
-    _owner = Self ;
-    OwnerIsThread = 1 ;
-    return ;
-  }
-
-  // We've encountered genuine contention.
-  assert (Self->_Stalled == 0, "invariant") ;
-  Self->_Stalled = intptr_t(this) ;
-
-  // Try one round of spinning *before* enqueueing Self
-  // and before going through the awkward and expensive state
-  // transitions.  The following spin is strictly optional ...
-  // Note that if we acquire the monitor from an initial spin
-  // we forgo posting JVMTI events and firing DTRACE probes.
-  if (Knob_SpinEarly && TrySpin (Self) > 0) {
-     assert (_owner == Self      , "invariant") ;
-     assert (_recursions == 0    , "invariant") ;
-     assert (((oop)(object()))->mark() == markOopDesc::encode(this), "invariant") ;
-     Self->_Stalled = 0 ;
-     return ;
-  }
-
-  assert (_owner != Self          , "invariant") ;
-  assert (_succ  != Self          , "invariant") ;
-  assert (Self->is_Java_thread()  , "invariant") ;
-  JavaThread * jt = (JavaThread *) Self ;
-  assert (!SafepointSynchronize::is_at_safepoint(), "invariant") ;
-  assert (jt->thread_state() != _thread_blocked   , "invariant") ;
-  assert (this->object() != NULL  , "invariant") ;
-  assert (_count >= 0, "invariant") ;
-
-  // Prevent deflation at STW-time.  See deflate_idle_monitors() and is_busy().
-  // Ensure the object-monitor relationship remains stable while there's contention.
-  Atomic::inc_ptr(&_count);
-
-  { // Change java thread status to indicate blocked on monitor enter.
-    JavaThreadBlockedOnMonitorEnterState jtbmes(jt, this);
-
-    DTRACE_MONITOR_PROBE(contended__enter, this, object(), jt);
-    if (JvmtiExport::should_post_monitor_contended_enter()) {
-      JvmtiExport::post_monitor_contended_enter(jt, this);
-    }
-
-    OSThreadContendState osts(Self->osthread());
-    ThreadBlockInVM tbivm(jt);
-
-    Self->set_current_pending_monitor(this);
-
-    // TODO-FIXME: change the following for(;;) loop to straight-line code.
-    for (;;) {
-      jt->set_suspend_equivalent();
-      // cleared by handle_special_suspend_equivalent_condition()
-      // or java_suspend_self()
-
-      EnterI (THREAD) ;
-
-      if (!ExitSuspendEquivalent(jt)) break ;
-
-      //
-      // We have acquired the contended monitor, but while we were
-      // waiting another thread suspended us. We don't want to enter
-      // the monitor while suspended because that would surprise the
-      // thread that suspended us.
-      //
-          _recursions = 0 ;
-      _succ = NULL ;
-      exit (Self) ;
-
-      jt->java_suspend_self();
-    }
-    Self->set_current_pending_monitor(NULL);
-  }
-
-  Atomic::dec_ptr(&_count);
-  assert (_count >= 0, "invariant") ;
-  Self->_Stalled = 0 ;
-
-  // Must either set _recursions = 0 or ASSERT _recursions == 0.
-  assert (_recursions == 0     , "invariant") ;
-  assert (_owner == Self       , "invariant") ;
-  assert (_succ  != Self       , "invariant") ;
-  assert (((oop)(object()))->mark() == markOopDesc::encode(this), "invariant") ;
-
-  // The thread -- now the owner -- is back in vm mode.
-  // Report the glorious news via TI,DTrace and jvmstat.
-  // The probe effect is non-trivial.  All the reportage occurs
-  // while we hold the monitor, increasing the length of the critical
-  // section.  Amdahl's parallel speedup law comes vividly into play.
-  //
-  // Another option might be to aggregate the events (thread local or
-  // per-monitor aggregation) and defer reporting until a more opportune
-  // time -- such as next time some thread encounters contention but has
-  // yet to acquire the lock.  While spinning that thread could
-  // spinning we could increment JVMStat counters, etc.
-
-  DTRACE_MONITOR_PROBE(contended__entered, this, object(), jt);
-  if (JvmtiExport::should_post_monitor_contended_entered()) {
-    JvmtiExport::post_monitor_contended_entered(jt, this);
-  }
-  if (ObjectSynchronizer::_sync_ContendedLockAttempts != NULL) {
-     ObjectSynchronizer::_sync_ContendedLockAttempts->inc() ;
-  }
-}
-
-void ObjectMonitor::ExitEpilog (Thread * Self, ObjectWaiter * Wakee) {
-   assert (_owner == Self, "invariant") ;
-
-   // Exit protocol:
-   // 1. ST _succ = wakee
-   // 2. membar #loadstore|#storestore;
-   // 2. ST _owner = NULL
-   // 3. unpark(wakee)
-
-   _succ = Knob_SuccEnabled ? Wakee->_thread : NULL ;
-   ParkEvent * Trigger = Wakee->_event ;
-
-   // Hygiene -- once we've set _owner = NULL we can't safely dereference Wakee again.
-   // The thread associated with Wakee may have grabbed the lock and "Wakee" may be
-   // out-of-scope (non-extant).
-   Wakee  = NULL ;
-
-   // Drop the lock
-   OrderAccess::release_store_ptr (&_owner, NULL) ;
-   OrderAccess::fence() ;                               // ST _owner vs LD in unpark()
-
-   // TODO-FIXME:
-   // If there's a safepoint pending the best policy would be to
-   // get _this thread to a safepoint and only wake the successor
-   // after the safepoint completed.  monitorexit uses a "leaf"
-   // state transition, however, so this thread can't become
-   // safe at this point in time.  (Its stack isn't walkable).
-   // The next best thing is to defer waking the successor by
-   // adding to a list of thread to be unparked after at the
-   // end of the forthcoming STW).
-   if (SafepointSynchronize::do_call_back()) {
-      TEVENT (unpark before SAFEPOINT) ;
-   }
-
-   // Possible optimizations ...
-   //
-   // * Consider: set Wakee->UnparkTime = timeNow()
-   //   When the thread wakes up it'll compute (timeNow() - Self->UnparkTime()).
-   //   By measuring recent ONPROC latency we can approximate the
-   //   system load.  In turn, we can feed that information back
-   //   into the spinning & succession policies.
-   //   (ONPROC latency correlates strongly with load).
-   //
-   // * Pull affinity:
-   //   If the wakee is cold then transiently setting it's affinity
-   //   to the current CPU is a good idea.
-   //   See http://j2se.east/~dice/PERSIST/050624-PullAffinity.txt
-   DTRACE_MONITOR_PROBE(contended__exit, this, object(), Self);
-   Trigger->unpark() ;
-
-   // Maintain stats and report events to JVMTI
-   if (ObjectSynchronizer::_sync_Parks != NULL) {
-      ObjectSynchronizer::_sync_Parks->inc() ;
-   }
-}
-
-
-// exit()
-// ~~~~~~
-// Note that the collector can't reclaim the objectMonitor or deflate
-// the object out from underneath the thread calling ::exit() as the
-// thread calling ::exit() never transitions to a stable state.
-// This inhibits GC, which in turn inhibits asynchronous (and
-// inopportune) reclamation of "this".
-//
-// We'd like to assert that: (THREAD->thread_state() != _thread_blocked) ;
-// There's one exception to the claim above, however.  EnterI() can call
-// exit() to drop a lock if the acquirer has been externally suspended.
-// In that case exit() is called with _thread_state as _thread_blocked,
-// but the monitor's _count field is > 0, which inhibits reclamation.
-//
-// 1-0 exit
-// ~~~~~~~~
-// ::exit() uses a canonical 1-1 idiom with a MEMBAR although some of
-// the fast-path operators have been optimized so the common ::exit()
-// operation is 1-0.  See i486.ad fast_unlock(), for instance.
-// The code emitted by fast_unlock() elides the usual MEMBAR.  This
-// greatly improves latency -- MEMBAR and CAS having considerable local
-// latency on modern processors -- but at the cost of "stranding".  Absent the
-// MEMBAR, a thread in fast_unlock() can race a thread in the slow
-// ::enter() path, resulting in the entering thread being stranding
-// and a progress-liveness failure.   Stranding is extremely rare.
-// We use timers (timed park operations) & periodic polling to detect
-// and recover from stranding.  Potentially stranded threads periodically
-// wake up and poll the lock.  See the usage of the _Responsible variable.
-//
-// The CAS() in enter provides for safety and exclusion, while the CAS or
-// MEMBAR in exit provides for progress and avoids stranding.  1-0 locking
-// eliminates the CAS/MEMBAR from the exist path, but it admits stranding.
-// We detect and recover from stranding with timers.
-//
-// If a thread transiently strands it'll park until (a) another
-// thread acquires the lock and then drops the lock, at which time the
-// exiting thread will notice and unpark the stranded thread, or, (b)
-// the timer expires.  If the lock is high traffic then the stranding latency
-// will be low due to (a).  If the lock is low traffic then the odds of
-// stranding are lower, although the worst-case stranding latency
-// is longer.  Critically, we don't want to put excessive load in the
-// platform's timer subsystem.  We want to minimize both the timer injection
-// rate (timers created/sec) as well as the number of timers active at
-// any one time.  (more precisely, we want to minimize timer-seconds, which is
-// the integral of the # of active timers at any instant over time).
-// Both impinge on OS scalability.  Given that, at most one thread parked on
-// a monitor will use a timer.
-
-void ATTR ObjectMonitor::exit(TRAPS) {
-   Thread * Self = THREAD ;
-   if (THREAD != _owner) {
-     if (THREAD->is_lock_owned((address) _owner)) {
-       // Transmute _owner from a BasicLock pointer to a Thread address.
-       // We don't need to hold _mutex for this transition.
-       // Non-null to Non-null is safe as long as all readers can
-       // tolerate either flavor.
-       assert (_recursions == 0, "invariant") ;
-       _owner = THREAD ;
-       _recursions = 0 ;
-       OwnerIsThread = 1 ;
-     } else {
-       // NOTE: we need to handle unbalanced monitor enter/exit
-       // in native code by throwing an exception.
-       // TODO: Throw an IllegalMonitorStateException ?
-       TEVENT (Exit - Throw IMSX) ;
-       assert(false, "Non-balanced monitor enter/exit!");
-       if (false) {
-          THROW(vmSymbols::java_lang_IllegalMonitorStateException());
-       }
-       return;
-     }
-   }
-
-   if (_recursions != 0) {
-     _recursions--;        // this is simple recursive enter
-     TEVENT (Inflated exit - recursive) ;
-     return ;
-   }
-
-   // Invariant: after setting Responsible=null an thread must execute
-   // a MEMBAR or other serializing instruction before fetching EntryList|cxq.
-   if ((SyncFlags & 4) == 0) {
-      _Responsible = NULL ;
-   }
-
-   for (;;) {
-      assert (THREAD == _owner, "invariant") ;
-
-      // Fast-path monitor exit:
-      //
-      // Observe the Dekker/Lamport duality:
-      // A thread in ::exit() executes:
-      //   ST Owner=null; MEMBAR; LD EntryList|cxq.
-      // A thread in the contended ::enter() path executes the complementary:
-      //   ST EntryList|cxq = nonnull; MEMBAR; LD Owner.
-      //
-      // Note that there's a benign race in the exit path.  We can drop the
-      // lock, another thread can reacquire the lock immediately, and we can
-      // then wake a thread unnecessarily (yet another flavor of futile wakeup).
-      // This is benign, and we've structured the code so the windows are short
-      // and the frequency of such futile wakeups is low.
-      //
-      // We could eliminate the race by encoding both the "LOCKED" state and
-      // the queue head in a single word.  Exit would then use either CAS to
-      // clear the LOCKED bit/byte.  This precludes the desirable 1-0 optimization,
-      // however.
-      //
-      // Possible fast-path ::exit() optimization:
-      // The current fast-path exit implementation fetches both cxq and EntryList.
-      // See also i486.ad fast_unlock().  Testing has shown that two LDs
-      // isn't measurably slower than a single LD on any platforms.
-      // Still, we could reduce the 2 LDs to one or zero by one of the following:
-      //
-      // - Use _count instead of cxq|EntryList
-      //   We intend to eliminate _count, however, when we switch
-      //   to on-the-fly deflation in ::exit() as is used in
-      //   Metalocks and RelaxedLocks.
-      //
-      // - Establish the invariant that cxq == null implies EntryList == null.
-      //   set cxq == EMPTY (1) to encode the state where cxq is empty
-      //   by EntryList != null.  EMPTY is a distinguished value.
-      //   The fast-path exit() would fetch cxq but not EntryList.
-      //
-      // - Encode succ as follows:
-      //   succ = t :  Thread t is the successor -- t is ready or is spinning.
-      //               Exiting thread does not need to wake a successor.
-      //   succ = 0 :  No successor required -> (EntryList|cxq) == null
-      //               Exiting thread does not need to wake a successor
-      //   succ = 1 :  Successor required    -> (EntryList|cxq) != null and
-      //               logically succ == null.
-      //               Exiting thread must wake a successor.
-      //
-      //   The 1-1 fast-exit path would appear as :
-      //     _owner = null ; membar ;
-      //     if (_succ == 1 && CAS (&_owner, null, Self) == null) goto SlowPath
-      //     goto FastPathDone ;
-      //
-      //   and the 1-0 fast-exit path would appear as:
-      //      if (_succ == 1) goto SlowPath
-      //      Owner = null ;
-      //      goto FastPathDone
-      //
-      // - Encode the LSB of _owner as 1 to indicate that exit()
-      //   must use the slow-path and make a successor ready.
-      //   (_owner & 1) == 0 IFF succ != null || (EntryList|cxq) == null
-      //   (_owner & 1) == 0 IFF succ == null && (EntryList|cxq) != null (obviously)
-      //   The 1-0 fast exit path would read:
-      //      if (_owner != Self) goto SlowPath
-      //      _owner = null
-      //      goto FastPathDone
-
-      if (Knob_ExitPolicy == 0) {
-         // release semantics: prior loads and stores from within the critical section
-         // must not float (reorder) past the following store that drops the lock.
-         // On SPARC that requires MEMBAR #loadstore|#storestore.
-         // But of course in TSO #loadstore|#storestore is not required.
-         // I'd like to write one of the following:
-         // A.  OrderAccess::release() ; _owner = NULL
-         // B.  OrderAccess::loadstore(); OrderAccess::storestore(); _owner = NULL;
-         // Unfortunately OrderAccess::release() and OrderAccess::loadstore() both
-         // store into a _dummy variable.  That store is not needed, but can result
-         // in massive wasteful coherency traffic on classic SMP systems.
-         // Instead, I use release_store(), which is implemented as just a simple
-         // ST on x64, x86 and SPARC.
-         OrderAccess::release_store_ptr (&_owner, NULL) ;   // drop the lock
-         OrderAccess::storeload() ;                         // See if we need to wake a successor
-         if ((intptr_t(_EntryList)|intptr_t(_cxq)) == 0 || _succ != NULL) {
-            TEVENT (Inflated exit - simple egress) ;
-            return ;
-         }
-         TEVENT (Inflated exit - complex egress) ;
-
-         // Normally the exiting thread is responsible for ensuring succession,
-         // but if other successors are ready or other entering threads are spinning
-         // then this thread can simply store NULL into _owner and exit without
-         // waking a successor.  The existence of spinners or ready successors
-         // guarantees proper succession (liveness).  Responsibility passes to the
-         // ready or running successors.  The exiting thread delegates the duty.
-         // More precisely, if a successor already exists this thread is absolved
-         // of the responsibility of waking (unparking) one.
-         //
-         // The _succ variable is critical to reducing futile wakeup frequency.
-         // _succ identifies the "heir presumptive" thread that has been made
-         // ready (unparked) but that has not yet run.  We need only one such
-         // successor thread to guarantee progress.
-         // See http://www.usenix.org/events/jvm01/full_papers/dice/dice.pdf
-         // section 3.3 "Futile Wakeup Throttling" for details.
-         //
-         // Note that spinners in Enter() also set _succ non-null.
-         // In the current implementation spinners opportunistically set
-         // _succ so that exiting threads might avoid waking a successor.
-         // Another less appealing alternative would be for the exiting thread
-         // to drop the lock and then spin briefly to see if a spinner managed
-         // to acquire the lock.  If so, the exiting thread could exit
-         // immediately without waking a successor, otherwise the exiting
-         // thread would need to dequeue and wake a successor.
-         // (Note that we'd need to make the post-drop spin short, but no
-         // shorter than the worst-case round-trip cache-line migration time.
-         // The dropped lock needs to become visible to the spinner, and then
-         // the acquisition of the lock by the spinner must become visible to
-         // the exiting thread).
-         //
-
-         // It appears that an heir-presumptive (successor) must be made ready.
-         // Only the current lock owner can manipulate the EntryList or
-         // drain _cxq, so we need to reacquire the lock.  If we fail
-         // to reacquire the lock the responsibility for ensuring succession
-         // falls to the new owner.
-         //
-         if (Atomic::cmpxchg_ptr (THREAD, &_owner, NULL) != NULL) {
-            return ;
-         }
-         TEVENT (Exit - Reacquired) ;
-      } else {
-         if ((intptr_t(_EntryList)|intptr_t(_cxq)) == 0 || _succ != NULL) {
-            OrderAccess::release_store_ptr (&_owner, NULL) ;   // drop the lock
-            OrderAccess::storeload() ;
-            // Ratify the previously observed values.
-            if (_cxq == NULL || _succ != NULL) {
-                TEVENT (Inflated exit - simple egress) ;
-                return ;
-            }
-
-            // inopportune interleaving -- the exiting thread (this thread)
-            // in the fast-exit path raced an entering thread in the slow-enter
-            // path.
-            // We have two choices:
-            // A.  Try to reacquire the lock.
-            //     If the CAS() fails return immediately, otherwise
-            //     we either restart/rerun the exit operation, or simply
-            //     fall-through into the code below which wakes a successor.
-            // B.  If the elements forming the EntryList|cxq are TSM
-            //     we could simply unpark() the lead thread and return
-            //     without having set _succ.
-            if (Atomic::cmpxchg_ptr (THREAD, &_owner, NULL) != NULL) {
-               TEVENT (Inflated exit - reacquired succeeded) ;
-               return ;
-            }
-            TEVENT (Inflated exit - reacquired failed) ;
-         } else {
-            TEVENT (Inflated exit - complex egress) ;
-         }
-      }
-
-      guarantee (_owner == THREAD, "invariant") ;
-
-      // Select an appropriate successor ("heir presumptive") from the EntryList
-      // and make it ready.  Generally we just wake the head of EntryList .
-      // There's no algorithmic constraint that we use the head - it's just
-      // a policy decision.   Note that the thread at head of the EntryList
-      // remains at the head until it acquires the lock.  This means we'll
-      // repeatedly wake the same thread until it manages to grab the lock.
-      // This is generally a good policy - if we're seeing lots of futile wakeups
-      // at least we're waking/rewaking a thread that's like to be hot or warm
-      // (have residual D$ and TLB affinity).
-      //
-      // "Wakeup locality" optimization:
-      // http://j2se.east/~dice/PERSIST/040825-WakeLocality.txt
-      // In the future we'll try to bias the selection mechanism
-      // to preferentially pick a thread that recently ran on
-      // a processor element that shares cache with the CPU on which
-      // the exiting thread is running.   We need access to Solaris'
-      // schedctl.sc_cpu to make that work.
-      //
-      ObjectWaiter * w = NULL ;
-      int QMode = Knob_QMode ;
-
-      if (QMode == 2 && _cxq != NULL) {
-          // QMode == 2 : cxq has precedence over EntryList.
-          // Try to directly wake a successor from the cxq.
-          // If successful, the successor will need to unlink itself from cxq.
-          w = _cxq ;
-          assert (w != NULL, "invariant") ;
-          assert (w->TState == ObjectWaiter::TS_CXQ, "Invariant") ;
-          ExitEpilog (Self, w) ;
-          return ;
-      }
-
-      if (QMode == 3 && _cxq != NULL) {
-          // Aggressively drain cxq into EntryList at the first opportunity.
-          // This policy ensure that recently-run threads live at the head of EntryList.
-          // Drain _cxq into EntryList - bulk transfer.
-          // First, detach _cxq.
-          // The following loop is tantamount to: w = swap (&cxq, NULL)
-          w = _cxq ;
-          for (;;) {
-             assert (w != NULL, "Invariant") ;
-             ObjectWaiter * u = (ObjectWaiter *) Atomic::cmpxchg_ptr (NULL, &_cxq, w) ;
-             if (u == w) break ;
-             w = u ;
-          }
-          assert (w != NULL              , "invariant") ;
-
-          ObjectWaiter * q = NULL ;
-          ObjectWaiter * p ;
-          for (p = w ; p != NULL ; p = p->_next) {
-              guarantee (p->TState == ObjectWaiter::TS_CXQ, "Invariant") ;
-              p->TState = ObjectWaiter::TS_ENTER ;
-              p->_prev = q ;
-              q = p ;
-          }
-
-          // Append the RATs to the EntryList
-          // TODO: organize EntryList as a CDLL so we can locate the tail in constant-time.
-          ObjectWaiter * Tail ;
-          for (Tail = _EntryList ; Tail != NULL && Tail->_next != NULL ; Tail = Tail->_next) ;
-          if (Tail == NULL) {
-              _EntryList = w ;
-          } else {
-              Tail->_next = w ;
-              w->_prev = Tail ;
-          }
-
-          // Fall thru into code that tries to wake a successor from EntryList
-      }
-
-      if (QMode == 4 && _cxq != NULL) {
-          // Aggressively drain cxq into EntryList at the first opportunity.
-          // This policy ensure that recently-run threads live at the head of EntryList.
-
-          // Drain _cxq into EntryList - bulk transfer.
-          // First, detach _cxq.
-          // The following loop is tantamount to: w = swap (&cxq, NULL)
-          w = _cxq ;
-          for (;;) {
-             assert (w != NULL, "Invariant") ;
-             ObjectWaiter * u = (ObjectWaiter *) Atomic::cmpxchg_ptr (NULL, &_cxq, w) ;
-             if (u == w) break ;
-             w = u ;
-          }
-          assert (w != NULL              , "invariant") ;
-
-          ObjectWaiter * q = NULL ;
-          ObjectWaiter * p ;
-          for (p = w ; p != NULL ; p = p->_next) {
-              guarantee (p->TState == ObjectWaiter::TS_CXQ, "Invariant") ;
-              p->TState = ObjectWaiter::TS_ENTER ;
-              p->_prev = q ;
-              q = p ;
-          }
-
-          // Prepend the RATs to the EntryList
-          if (_EntryList != NULL) {
-              q->_next = _EntryList ;
-              _EntryList->_prev = q ;
-          }
-          _EntryList = w ;
-
-          // Fall thru into code that tries to wake a successor from EntryList
-      }
-
-      w = _EntryList  ;
-      if (w != NULL) {
-          // I'd like to write: guarantee (w->_thread != Self).
-          // But in practice an exiting thread may find itself on the EntryList.
-          // Lets say thread T1 calls O.wait().  Wait() enqueues T1 on O's waitset and
-          // then calls exit().  Exit release the lock by setting O._owner to NULL.
-          // Lets say T1 then stalls.  T2 acquires O and calls O.notify().  The
-          // notify() operation moves T1 from O's waitset to O's EntryList. T2 then
-          // release the lock "O".  T2 resumes immediately after the ST of null into
-          // _owner, above.  T2 notices that the EntryList is populated, so it
-          // reacquires the lock and then finds itself on the EntryList.
-          // Given all that, we have to tolerate the circumstance where "w" is
-          // associated with Self.
-          assert (w->TState == ObjectWaiter::TS_ENTER, "invariant") ;
-          ExitEpilog (Self, w) ;
-          return ;
-      }
-
-      // If we find that both _cxq and EntryList are null then just
-      // re-run the exit protocol from the top.
-      w = _cxq ;
-      if (w == NULL) continue ;
-
-      // Drain _cxq into EntryList - bulk transfer.
-      // First, detach _cxq.
-      // The following loop is tantamount to: w = swap (&cxq, NULL)
-      for (;;) {
-          assert (w != NULL, "Invariant") ;
-          ObjectWaiter * u = (ObjectWaiter *) Atomic::cmpxchg_ptr (NULL, &_cxq, w) ;
-          if (u == w) break ;
-          w = u ;
-      }
-      TEVENT (Inflated exit - drain cxq into EntryList) ;
-
-      assert (w != NULL              , "invariant") ;
-      assert (_EntryList  == NULL    , "invariant") ;
-
-      // Convert the LIFO SLL anchored by _cxq into a DLL.
-      // The list reorganization step operates in O(LENGTH(w)) time.
-      // It's critical that this step operate quickly as
-      // "Self" still holds the outer-lock, restricting parallelism
-      // and effectively lengthening the critical section.
-      // Invariant: s chases t chases u.
-      // TODO-FIXME: consider changing EntryList from a DLL to a CDLL so
-      // we have faster access to the tail.
-
-      if (QMode == 1) {
-         // QMode == 1 : drain cxq to EntryList, reversing order
-         // We also reverse the order of the list.
-         ObjectWaiter * s = NULL ;
-         ObjectWaiter * t = w ;
-         ObjectWaiter * u = NULL ;
-         while (t != NULL) {
-             guarantee (t->TState == ObjectWaiter::TS_CXQ, "invariant") ;
-             t->TState = ObjectWaiter::TS_ENTER ;
-             u = t->_next ;
-             t->_prev = u ;
-             t->_next = s ;
-             s = t;
-             t = u ;
-         }
-         _EntryList  = s ;
-         assert (s != NULL, "invariant") ;
-      } else {
-         // QMode == 0 or QMode == 2
-         _EntryList = w ;
-         ObjectWaiter * q = NULL ;
-         ObjectWaiter * p ;
-         for (p = w ; p != NULL ; p = p->_next) {
-             guarantee (p->TState == ObjectWaiter::TS_CXQ, "Invariant") ;
-             p->TState = ObjectWaiter::TS_ENTER ;
-             p->_prev = q ;
-             q = p ;
-         }
-      }
-
-      // In 1-0 mode we need: ST EntryList; MEMBAR #storestore; ST _owner = NULL
-      // The MEMBAR is satisfied by the release_store() operation in ExitEpilog().
-
-      // See if we can abdicate to a spinner instead of waking a thread.
-      // A primary goal of the implementation is to reduce the
-      // context-switch rate.
-      if (_succ != NULL) continue;
-
-      w = _EntryList  ;
-      if (w != NULL) {
-          guarantee (w->TState == ObjectWaiter::TS_ENTER, "invariant") ;
-          ExitEpilog (Self, w) ;
-          return ;
-      }
-   }
-}
-// complete_exit exits a lock returning recursion count
-// complete_exit/reenter operate as a wait without waiting
-// complete_exit requires an inflated monitor
-// The _owner field is not always the Thread addr even with an
-// inflated monitor, e.g. the monitor can be inflated by a non-owning
-// thread due to contention.
-intptr_t ObjectMonitor::complete_exit(TRAPS) {
-   Thread * const Self = THREAD;
-   assert(Self->is_Java_thread(), "Must be Java thread!");
-   JavaThread *jt = (JavaThread *)THREAD;
-
-   DeferredInitialize();
-
-   if (THREAD != _owner) {
-    if (THREAD->is_lock_owned ((address)_owner)) {
-       assert(_recursions == 0, "internal state error");
-       _owner = THREAD ;   /* Convert from basiclock addr to Thread addr */
-       _recursions = 0 ;
-       OwnerIsThread = 1 ;
-    }
-   }
-
-   guarantee(Self == _owner, "complete_exit not owner");
-   intptr_t save = _recursions; // record the old recursion count
-   _recursions = 0;        // set the recursion level to be 0
-   exit (Self) ;           // exit the monitor
-   guarantee (_owner != Self, "invariant");
-   return save;
-}
-
-// reenter() enters a lock and sets recursion count
-// complete_exit/reenter operate as a wait without waiting
-void ObjectMonitor::reenter(intptr_t recursions, TRAPS) {
-   Thread * const Self = THREAD;
-   assert(Self->is_Java_thread(), "Must be Java thread!");
-   JavaThread *jt = (JavaThread *)THREAD;
-
-   guarantee(_owner != Self, "reenter already owner");
-   enter (THREAD);       // enter the monitor
-   guarantee (_recursions == 0, "reenter recursion");
-   _recursions = recursions;
-   return;
-}
-
-// Note: a subset of changes to ObjectMonitor::wait()
-// will need to be replicated in complete_exit above
-void ObjectMonitor::wait(jlong millis, bool interruptible, TRAPS) {
-   Thread * const Self = THREAD ;
-   assert(Self->is_Java_thread(), "Must be Java thread!");
-   JavaThread *jt = (JavaThread *)THREAD;
-
-   DeferredInitialize () ;
-
-   // Throw IMSX or IEX.
-   CHECK_OWNER();
-
-   // check for a pending interrupt
-   if (interruptible && Thread::is_interrupted(Self, true) && !HAS_PENDING_EXCEPTION) {
-     // post monitor waited event.  Note that this is past-tense, we are done waiting.
-     if (JvmtiExport::should_post_monitor_waited()) {
-        // Note: 'false' parameter is passed here because the
-        // wait was not timed out due to thread interrupt.
-        JvmtiExport::post_monitor_waited(jt, this, false);
-     }
-     TEVENT (Wait - Throw IEX) ;
-     THROW(vmSymbols::java_lang_InterruptedException());
-     return ;
-   }
-   TEVENT (Wait) ;
-
-   assert (Self->_Stalled == 0, "invariant") ;
-   Self->_Stalled = intptr_t(this) ;
-   jt->set_current_waiting_monitor(this);
-
-   // create a node to be put into the queue
-   // Critically, after we reset() the event but prior to park(), we must check
-   // for a pending interrupt.
-   ObjectWaiter node(Self);
-   node.TState = ObjectWaiter::TS_WAIT ;
-   Self->_ParkEvent->reset() ;
-   OrderAccess::fence();          // ST into Event; membar ; LD interrupted-flag
-
-   // Enter the waiting queue, which is a circular doubly linked list in this case
-   // but it could be a priority queue or any data structure.
-   // _WaitSetLock protects the wait queue.  Normally the wait queue is accessed only
-   // by the the owner of the monitor *except* in the case where park()
-   // returns because of a timeout of interrupt.  Contention is exceptionally rare
-   // so we use a simple spin-lock instead of a heavier-weight blocking lock.
-
-   Thread::SpinAcquire (&_WaitSetLock, "WaitSet - add") ;
-   AddWaiter (&node) ;
-   Thread::SpinRelease (&_WaitSetLock) ;
-
-   if ((SyncFlags & 4) == 0) {
-      _Responsible = NULL ;
-   }
-   intptr_t save = _recursions; // record the old recursion count
-   _waiters++;                  // increment the number of waiters
-   _recursions = 0;             // set the recursion level to be 1
-   exit (Self) ;                    // exit the monitor
-   guarantee (_owner != Self, "invariant") ;
-
-   // As soon as the ObjectMonitor's ownership is dropped in the exit()
-   // call above, another thread can enter() the ObjectMonitor, do the
-   // notify(), and exit() the ObjectMonitor. If the other thread's
-   // exit() call chooses this thread as the successor and the unpark()
-   // call happens to occur while this thread is posting a
-   // MONITOR_CONTENDED_EXIT event, then we run the risk of the event
-   // handler using RawMonitors and consuming the unpark().
-   //
-   // To avoid the problem, we re-post the event. This does no harm
-   // even if the original unpark() was not consumed because we are the
-   // chosen successor for this monitor.
-   if (node._notified != 0 && _succ == Self) {
-      node._event->unpark();
-   }
-
-   // The thread is on the WaitSet list - now park() it.
-   // On MP systems it's conceivable that a brief spin before we park
-   // could be profitable.
-   //
-   // TODO-FIXME: change the following logic to a loop of the form
-   //   while (!timeout && !interrupted && _notified == 0) park()
-
-   int ret = OS_OK ;
-   int WasNotified = 0 ;
-   { // State transition wrappers
-     OSThread* osthread = Self->osthread();
-     OSThreadWaitState osts(osthread, true);
-     {
-       ThreadBlockInVM tbivm(jt);
-       // Thread is in thread_blocked state and oop access is unsafe.
-       jt->set_suspend_equivalent();
-
-       if (interruptible && (Thread::is_interrupted(THREAD, false) || HAS_PENDING_EXCEPTION)) {
-           // Intentionally empty
-       } else
-       if (node._notified == 0) {
-         if (millis <= 0) {
-            Self->_ParkEvent->park () ;
-         } else {
-            ret = Self->_ParkEvent->park (millis) ;
-         }
-       }
-
-       // were we externally suspended while we were waiting?
-       if (ExitSuspendEquivalent (jt)) {
-          // TODO-FIXME: add -- if succ == Self then succ = null.
-          jt->java_suspend_self();
-       }
-
-     } // Exit thread safepoint: transition _thread_blocked -> _thread_in_vm
-
-
-     // Node may be on the WaitSet, the EntryList (or cxq), or in transition
-     // from the WaitSet to the EntryList.
-     // See if we need to remove Node from the WaitSet.
-     // We use double-checked locking to avoid grabbing _WaitSetLock
-     // if the thread is not on the wait queue.
-     //
-     // Note that we don't need a fence before the fetch of TState.
-     // In the worst case we'll fetch a old-stale value of TS_WAIT previously
-     // written by the is thread. (perhaps the fetch might even be satisfied
-     // by a look-aside into the processor's own store buffer, although given
-     // the length of the code path between the prior ST and this load that's
-     // highly unlikely).  If the following LD fetches a stale TS_WAIT value
-     // then we'll acquire the lock and then re-fetch a fresh TState value.
-     // That is, we fail toward safety.
-
-     if (node.TState == ObjectWaiter::TS_WAIT) {
-         Thread::SpinAcquire (&_WaitSetLock, "WaitSet - unlink") ;
-         if (node.TState == ObjectWaiter::TS_WAIT) {
-            DequeueSpecificWaiter (&node) ;       // unlink from WaitSet
-            assert(node._notified == 0, "invariant");
-            node.TState = ObjectWaiter::TS_RUN ;
-         }
-         Thread::SpinRelease (&_WaitSetLock) ;
-     }
-
-     // The thread is now either on off-list (TS_RUN),
-     // on the EntryList (TS_ENTER), or on the cxq (TS_CXQ).
-     // The Node's TState variable is stable from the perspective of this thread.
-     // No other threads will asynchronously modify TState.
-     guarantee (node.TState != ObjectWaiter::TS_WAIT, "invariant") ;
-     OrderAccess::loadload() ;
-     if (_succ == Self) _succ = NULL ;
-     WasNotified = node._notified ;
-
-     // Reentry phase -- reacquire the monitor.
-     // re-enter contended monitor after object.wait().
-     // retain OBJECT_WAIT state until re-enter successfully completes
-     // Thread state is thread_in_vm and oop access is again safe,
-     // although the raw address of the object may have changed.
-     // (Don't cache naked oops over safepoints, of course).
-
-     // post monitor waited event. Note that this is past-tense, we are done waiting.
-     if (JvmtiExport::should_post_monitor_waited()) {
-       JvmtiExport::post_monitor_waited(jt, this, ret == OS_TIMEOUT);
-     }
-     OrderAccess::fence() ;
-
-     assert (Self->_Stalled != 0, "invariant") ;
-     Self->_Stalled = 0 ;
-
-     assert (_owner != Self, "invariant") ;
-     ObjectWaiter::TStates v = node.TState ;
-     if (v == ObjectWaiter::TS_RUN) {
-         enter (Self) ;
-     } else {
-         guarantee (v == ObjectWaiter::TS_ENTER || v == ObjectWaiter::TS_CXQ, "invariant") ;
-         ReenterI (Self, &node) ;
-         node.wait_reenter_end(this);
-     }
-
-     // Self has reacquired the lock.
-     // Lifecycle - the node representing Self must not appear on any queues.
-     // Node is about to go out-of-scope, but even if it were immortal we wouldn't
-     // want residual elements associated with this thread left on any lists.
-     guarantee (node.TState == ObjectWaiter::TS_RUN, "invariant") ;
-     assert    (_owner == Self, "invariant") ;
-     assert    (_succ != Self , "invariant") ;
-   } // OSThreadWaitState()
-
-   jt->set_current_waiting_monitor(NULL);
-
-   guarantee (_recursions == 0, "invariant") ;
-   _recursions = save;     // restore the old recursion count
-   _waiters--;             // decrement the number of waiters
-
-   // Verify a few postconditions
-   assert (_owner == Self       , "invariant") ;
-   assert (_succ  != Self       , "invariant") ;
-   assert (((oop)(object()))->mark() == markOopDesc::encode(this), "invariant") ;
-
-   if (SyncFlags & 32) {
-      OrderAccess::fence() ;
-   }
-
-   // check if the notification happened
-   if (!WasNotified) {
-     // no, it could be timeout or Thread.interrupt() or both
-     // check for interrupt event, otherwise it is timeout
-     if (interruptible && Thread::is_interrupted(Self, true) && !HAS_PENDING_EXCEPTION) {
-       TEVENT (Wait - throw IEX from epilog) ;
-       THROW(vmSymbols::java_lang_InterruptedException());
-     }
-   }
-
-   // NOTE: Spurious wake up will be consider as timeout.
-   // Monitor notify has precedence over thread interrupt.
-}
-
-
-// Consider:
-// If the lock is cool (cxq == null && succ == null) and we're on an MP system
-// then instead of transferring a thread from the WaitSet to the EntryList
-// we might just dequeue a thread from the WaitSet and directly unpark() it.
-
-void ObjectMonitor::notify(TRAPS) {
-  CHECK_OWNER();
-  if (_WaitSet == NULL) {
-     TEVENT (Empty-Notify) ;
-     return ;
-  }
-  DTRACE_MONITOR_PROBE(notify, this, object(), THREAD);
-
-  int Policy = Knob_MoveNotifyee ;
-
-  Thread::SpinAcquire (&_WaitSetLock, "WaitSet - notify") ;
-  ObjectWaiter * iterator = DequeueWaiter() ;
-  if (iterator != NULL) {
-     TEVENT (Notify1 - Transfer) ;
-     guarantee (iterator->TState == ObjectWaiter::TS_WAIT, "invariant") ;
-     guarantee (iterator->_notified == 0, "invariant") ;
-     // Disposition - what might we do with iterator ?
-     // a.  add it directly to the EntryList - either tail or head.
-     // b.  push it onto the front of the _cxq.
-     // For now we use (a).
-     if (Policy != 4) {
-        iterator->TState = ObjectWaiter::TS_ENTER ;
-     }
-     iterator->_notified = 1 ;
-
-     ObjectWaiter * List = _EntryList ;
-     if (List != NULL) {
-        assert (List->_prev == NULL, "invariant") ;
-        assert (List->TState == ObjectWaiter::TS_ENTER, "invariant") ;
-        assert (List != iterator, "invariant") ;
-     }
-
-     if (Policy == 0) {       // prepend to EntryList
-         if (List == NULL) {
-             iterator->_next = iterator->_prev = NULL ;
-             _EntryList = iterator ;
-         } else {
-             List->_prev = iterator ;
-             iterator->_next = List ;
-             iterator->_prev = NULL ;
-             _EntryList = iterator ;
-        }
-     } else
-     if (Policy == 1) {      // append to EntryList
-         if (List == NULL) {
-             iterator->_next = iterator->_prev = NULL ;
-             _EntryList = iterator ;
-         } else {
-            // CONSIDER:  finding the tail currently requires a linear-time walk of
-            // the EntryList.  We can make tail access constant-time by converting to
-            // a CDLL instead of using our current DLL.
-            ObjectWaiter * Tail ;
-            for (Tail = List ; Tail->_next != NULL ; Tail = Tail->_next) ;
-            assert (Tail != NULL && Tail->_next == NULL, "invariant") ;
-            Tail->_next = iterator ;
-            iterator->_prev = Tail ;
-            iterator->_next = NULL ;
-        }
-     } else
-     if (Policy == 2) {      // prepend to cxq
-         // prepend to cxq
-         if (List == NULL) {
-             iterator->_next = iterator->_prev = NULL ;
-             _EntryList = iterator ;
-         } else {
-            iterator->TState = ObjectWaiter::TS_CXQ ;
-            for (;;) {
-                ObjectWaiter * Front = _cxq ;
-                iterator->_next = Front ;
-                if (Atomic::cmpxchg_ptr (iterator, &_cxq, Front) == Front) {
-                    break ;
-                }
-            }
-         }
-     } else
-     if (Policy == 3) {      // append to cxq
-        iterator->TState = ObjectWaiter::TS_CXQ ;
-        for (;;) {
-            ObjectWaiter * Tail ;
-            Tail = _cxq ;
-            if (Tail == NULL) {
-                iterator->_next = NULL ;
-                if (Atomic::cmpxchg_ptr (iterator, &_cxq, NULL) == NULL) {
-                   break ;
-                }
-            } else {
-                while (Tail->_next != NULL) Tail = Tail->_next ;
-                Tail->_next = iterator ;
-                iterator->_prev = Tail ;
-                iterator->_next = NULL ;
-                break ;
-            }
-        }
-     } else {
-        ParkEvent * ev = iterator->_event ;
-        iterator->TState = ObjectWaiter::TS_RUN ;
-        OrderAccess::fence() ;
-        ev->unpark() ;
-     }
-
-     if (Policy < 4) {
-       iterator->wait_reenter_begin(this);
-     }
-
-     // _WaitSetLock protects the wait queue, not the EntryList.  We could
-     // move the add-to-EntryList operation, above, outside the critical section
-     // protected by _WaitSetLock.  In practice that's not useful.  With the
-     // exception of  wait() timeouts and interrupts the monitor owner
-     // is the only thread that grabs _WaitSetLock.  There's almost no contention
-     // on _WaitSetLock so it's not profitable to reduce the length of the
-     // critical section.
-  }
-
-  Thread::SpinRelease (&_WaitSetLock) ;
-
-  if (iterator != NULL && ObjectSynchronizer::_sync_Notifications != NULL) {
-     ObjectSynchronizer::_sync_Notifications->inc() ;
-  }
-}
-
-
-void ObjectMonitor::notifyAll(TRAPS) {
-  CHECK_OWNER();
-  ObjectWaiter* iterator;
-  if (_WaitSet == NULL) {
-      TEVENT (Empty-NotifyAll) ;
-      return ;
-  }
-  DTRACE_MONITOR_PROBE(notifyAll, this, object(), THREAD);
-
-  int Policy = Knob_MoveNotifyee ;
-  int Tally = 0 ;
-  Thread::SpinAcquire (&_WaitSetLock, "WaitSet - notifyall") ;
-
-  for (;;) {
-     iterator = DequeueWaiter () ;
-     if (iterator == NULL) break ;
-     TEVENT (NotifyAll - Transfer1) ;
-     ++Tally ;
-
-     // Disposition - what might we do with iterator ?
-     // a.  add it directly to the EntryList - either tail or head.
-     // b.  push it onto the front of the _cxq.
-     // For now we use (a).
-     //
-     // TODO-FIXME: currently notifyAll() transfers the waiters one-at-a-time from the waitset
-     // to the EntryList.  This could be done more efficiently with a single bulk transfer,
-     // but in practice it's not time-critical.  Beware too, that in prepend-mode we invert the
-     // order of the waiters.  Lets say that the waitset is "ABCD" and the EntryList is "XYZ".
-     // After a notifyAll() in prepend mode the waitset will be empty and the EntryList will
-     // be "DCBAXYZ".
-
-     guarantee (iterator->TState == ObjectWaiter::TS_WAIT, "invariant") ;
-     guarantee (iterator->_notified == 0, "invariant") ;
-     iterator->_notified = 1 ;
-     if (Policy != 4) {
-        iterator->TState = ObjectWaiter::TS_ENTER ;
-     }
-
-     ObjectWaiter * List = _EntryList ;
-     if (List != NULL) {
-        assert (List->_prev == NULL, "invariant") ;
-        assert (List->TState == ObjectWaiter::TS_ENTER, "invariant") ;
-        assert (List != iterator, "invariant") ;
-     }
-
-     if (Policy == 0) {       // prepend to EntryList
-         if (List == NULL) {
-             iterator->_next = iterator->_prev = NULL ;
-             _EntryList = iterator ;
-         } else {
-             List->_prev = iterator ;
-             iterator->_next = List ;
-             iterator->_prev = NULL ;
-             _EntryList = iterator ;
-        }
-     } else
-     if (Policy == 1) {      // append to EntryList
-         if (List == NULL) {
-             iterator->_next = iterator->_prev = NULL ;
-             _EntryList = iterator ;
-         } else {
-            // CONSIDER:  finding the tail currently requires a linear-time walk of
-            // the EntryList.  We can make tail access constant-time by converting to
-            // a CDLL instead of using our current DLL.
-            ObjectWaiter * Tail ;
-            for (Tail = List ; Tail->_next != NULL ; Tail = Tail->_next) ;
-            assert (Tail != NULL && Tail->_next == NULL, "invariant") ;
-            Tail->_next = iterator ;
-            iterator->_prev = Tail ;
-            iterator->_next = NULL ;
-        }
-     } else
-     if (Policy == 2) {      // prepend to cxq
-         // prepend to cxq
-         iterator->TState = ObjectWaiter::TS_CXQ ;
-         for (;;) {
-             ObjectWaiter * Front = _cxq ;
-             iterator->_next = Front ;
-             if (Atomic::cmpxchg_ptr (iterator, &_cxq, Front) == Front) {
-                 break ;
-             }
-         }
-     } else
-     if (Policy == 3) {      // append to cxq
-        iterator->TState = ObjectWaiter::TS_CXQ ;
-        for (;;) {
-            ObjectWaiter * Tail ;
-            Tail = _cxq ;
-            if (Tail == NULL) {
-                iterator->_next = NULL ;
-                if (Atomic::cmpxchg_ptr (iterator, &_cxq, NULL) == NULL) {
-                   break ;
-                }
-            } else {
-                while (Tail->_next != NULL) Tail = Tail->_next ;
-                Tail->_next = iterator ;
-                iterator->_prev = Tail ;
-                iterator->_next = NULL ;
-                break ;
-            }
-        }
-     } else {
-        ParkEvent * ev = iterator->_event ;
-        iterator->TState = ObjectWaiter::TS_RUN ;
-        OrderAccess::fence() ;
-        ev->unpark() ;
-     }
-
-     if (Policy < 4) {
-       iterator->wait_reenter_begin(this);
-     }
-
-     // _WaitSetLock protects the wait queue, not the EntryList.  We could
-     // move the add-to-EntryList operation, above, outside the critical section
-     // protected by _WaitSetLock.  In practice that's not useful.  With the
-     // exception of  wait() timeouts and interrupts the monitor owner
-     // is the only thread that grabs _WaitSetLock.  There's almost no contention
-     // on _WaitSetLock so it's not profitable to reduce the length of the
-     // critical section.
-  }
-
-  Thread::SpinRelease (&_WaitSetLock) ;
-
-  if (Tally != 0 && ObjectSynchronizer::_sync_Notifications != NULL) {
-     ObjectSynchronizer::_sync_Notifications->inc(Tally) ;
-  }
-}
-
-// check_slow() is a misnomer.  It's called to simply to throw an IMSX exception.
-// TODO-FIXME: remove check_slow() -- it's likely dead.
-
-void ObjectMonitor::check_slow(TRAPS) {
-  TEVENT (check_slow - throw IMSX) ;
-  assert(THREAD != _owner && !THREAD->is_lock_owned((address) _owner), "must not be owner");
-  THROW_MSG(vmSymbols::java_lang_IllegalMonitorStateException(), "current thread not owner");
-}
-
-
-// -------------------------------------------------------------------------
-// The raw monitor subsystem is entirely distinct from normal
-// java-synchronization or jni-synchronization.  raw monitors are not
-// associated with objects.  They can be implemented in any manner
-// that makes sense.  The original implementors decided to piggy-back
-// the raw-monitor implementation on the existing Java objectMonitor mechanism.
-// This flaw needs to fixed.  We should reimplement raw monitors as sui-generis.
-// Specifically, we should not implement raw monitors via java monitors.
-// Time permitting, we should disentangle and deconvolve the two implementations
-// and move the resulting raw monitor implementation over to the JVMTI directories.
-// Ideally, the raw monitor implementation would be built on top of
-// park-unpark and nothing else.
-//
-// raw monitors are used mainly by JVMTI
-// The raw monitor implementation borrows the ObjectMonitor structure,
-// but the operators are degenerate and extremely simple.
-//
-// Mixed use of a single objectMonitor instance -- as both a raw monitor
-// and a normal java monitor -- is not permissible.
-//
-// Note that we use the single RawMonitor_lock to protect queue operations for
-// _all_ raw monitors.  This is a scalability impediment, but since raw monitor usage
-// is deprecated and rare, this is not of concern.  The RawMonitor_lock can not
-// be held indefinitely.  The critical sections must be short and bounded.
-//
-// -------------------------------------------------------------------------
-
-int ObjectMonitor::SimpleEnter (Thread * Self) {
-  for (;;) {
-    if (Atomic::cmpxchg_ptr (Self, &_owner, NULL) == NULL) {
-       return OS_OK ;
-    }
-
-    ObjectWaiter Node (Self) ;
-    Self->_ParkEvent->reset() ;     // strictly optional
-    Node.TState = ObjectWaiter::TS_ENTER ;
-
-    RawMonitor_lock->lock_without_safepoint_check() ;
-    Node._next  = _EntryList ;
-    _EntryList  = &Node ;
-    OrderAccess::fence() ;
-    if (_owner == NULL && Atomic::cmpxchg_ptr (Self, &_owner, NULL) == NULL) {
-        _EntryList = Node._next ;
-        RawMonitor_lock->unlock() ;
-        return OS_OK ;
-    }
-    RawMonitor_lock->unlock() ;
-    while (Node.TState == ObjectWaiter::TS_ENTER) {
-       Self->_ParkEvent->park() ;
-    }
-  }
-}
-
-int ObjectMonitor::SimpleExit (Thread * Self) {
-  guarantee (_owner == Self, "invariant") ;
-  OrderAccess::release_store_ptr (&_owner, NULL) ;
-  OrderAccess::fence() ;
-  if (_EntryList == NULL) return OS_OK ;
-  ObjectWaiter * w ;
-
-  RawMonitor_lock->lock_without_safepoint_check() ;
-  w = _EntryList ;
-  if (w != NULL) {
-      _EntryList = w->_next ;
-  }
-  RawMonitor_lock->unlock() ;
-  if (w != NULL) {
-      guarantee (w ->TState == ObjectWaiter::TS_ENTER, "invariant") ;
-      ParkEvent * ev = w->_event ;
-      w->TState = ObjectWaiter::TS_RUN ;
-      OrderAccess::fence() ;
-      ev->unpark() ;
-  }
-  return OS_OK ;
-}
-
-int ObjectMonitor::SimpleWait (Thread * Self, jlong millis) {
-  guarantee (_owner == Self  , "invariant") ;
-  guarantee (_recursions == 0, "invariant") ;
-
-  ObjectWaiter Node (Self) ;
-  Node._notified = 0 ;
-  Node.TState    = ObjectWaiter::TS_WAIT ;
-
-  RawMonitor_lock->lock_without_safepoint_check() ;
-  Node._next     = _WaitSet ;
-  _WaitSet       = &Node ;
-  RawMonitor_lock->unlock() ;
-
-  SimpleExit (Self) ;
-  guarantee (_owner != Self, "invariant") ;
-
-  int ret = OS_OK ;
-  if (millis <= 0) {
-    Self->_ParkEvent->park();
-  } else {
-    ret = Self->_ParkEvent->park(millis);
-  }
-
-  // If thread still resides on the waitset then unlink it.
-  // Double-checked locking -- the usage is safe in this context
-  // as we TState is volatile and the lock-unlock operators are
-  // serializing (barrier-equivalent).
-
-  if (Node.TState == ObjectWaiter::TS_WAIT) {
-    RawMonitor_lock->lock_without_safepoint_check() ;
-    if (Node.TState == ObjectWaiter::TS_WAIT) {
-      // Simple O(n) unlink, but performance isn't critical here.
-      ObjectWaiter * p ;
-      ObjectWaiter * q = NULL ;
-      for (p = _WaitSet ; p != &Node; p = p->_next) {
-         q = p ;
-      }
-      guarantee (p == &Node, "invariant") ;
-      if (q == NULL) {
-        guarantee (p == _WaitSet, "invariant") ;
-        _WaitSet = p->_next ;
-      } else {
-        guarantee (p == q->_next, "invariant") ;
-        q->_next = p->_next ;
-      }
-      Node.TState = ObjectWaiter::TS_RUN ;
-    }
-    RawMonitor_lock->unlock() ;
-  }
-
-  guarantee (Node.TState == ObjectWaiter::TS_RUN, "invariant") ;
-  SimpleEnter (Self) ;
-
-  guarantee (_owner == Self, "invariant") ;
-  guarantee (_recursions == 0, "invariant") ;
-  return ret ;
-}
-
-int ObjectMonitor::SimpleNotify (Thread * Self, bool All) {
-  guarantee (_owner == Self, "invariant") ;
-  if (_WaitSet == NULL) return OS_OK ;
-
-  // We have two options:
-  // A. Transfer the threads from the WaitSet to the EntryList
-  // B. Remove the thread from the WaitSet and unpark() it.
-  //
-  // We use (B), which is crude and results in lots of futile
-  // context switching.  In particular (B) induces lots of contention.
-
-  ParkEvent * ev = NULL ;       // consider using a small auto array ...
-  RawMonitor_lock->lock_without_safepoint_check() ;
-  for (;;) {
-      ObjectWaiter * w = _WaitSet ;
-      if (w == NULL) break ;
-      _WaitSet = w->_next ;
-      if (ev != NULL) { ev->unpark(); ev = NULL; }
-      ev = w->_event ;
-      OrderAccess::loadstore() ;
-      w->TState = ObjectWaiter::TS_RUN ;
-      OrderAccess::storeload();
-      if (!All) break ;
-  }
-  RawMonitor_lock->unlock() ;
-  if (ev != NULL) ev->unpark();
-  return OS_OK ;
-}
-
-// Any JavaThread will enter here with state _thread_blocked
-int ObjectMonitor::raw_enter(TRAPS) {
-  TEVENT (raw_enter) ;
-  void * Contended ;
-
-  // don't enter raw monitor if thread is being externally suspended, it will
-  // surprise the suspender if a "suspended" thread can still enter monitor
-  JavaThread * jt = (JavaThread *)THREAD;
-  if (THREAD->is_Java_thread()) {
-    jt->SR_lock()->lock_without_safepoint_check();
-    while (jt->is_external_suspend()) {
-      jt->SR_lock()->unlock();
-      jt->java_suspend_self();
-      jt->SR_lock()->lock_without_safepoint_check();
-    }
-    // guarded by SR_lock to avoid racing with new external suspend requests.
-    Contended = Atomic::cmpxchg_ptr (THREAD, &_owner, NULL) ;
-    jt->SR_lock()->unlock();
-  } else {
-    Contended = Atomic::cmpxchg_ptr (THREAD, &_owner, NULL) ;
-  }
-
-  if (Contended == THREAD) {
-     _recursions ++ ;
-     return OM_OK ;
-  }
-
-  if (Contended == NULL) {
-     guarantee (_owner == THREAD, "invariant") ;
-     guarantee (_recursions == 0, "invariant") ;
-     return OM_OK ;
-  }
-
-  THREAD->set_current_pending_monitor(this);
-
-  if (!THREAD->is_Java_thread()) {
-     // No other non-Java threads besides VM thread would acquire
-     // a raw monitor.
-     assert(THREAD->is_VM_thread(), "must be VM thread");
-     SimpleEnter (THREAD) ;
-   } else {
-     guarantee (jt->thread_state() == _thread_blocked, "invariant") ;
-     for (;;) {
-       jt->set_suspend_equivalent();
-       // cleared by handle_special_suspend_equivalent_condition() or
-       // java_suspend_self()
-       SimpleEnter (THREAD) ;
-
-       // were we externally suspended while we were waiting?
-       if (!jt->handle_special_suspend_equivalent_condition()) break ;
-
-       // This thread was externally suspended
-       //
-       // This logic isn't needed for JVMTI raw monitors,
-       // but doesn't hurt just in case the suspend rules change. This
-           // logic is needed for the ObjectMonitor.wait() reentry phase.
-           // We have reentered the contended monitor, but while we were
-           // waiting another thread suspended us. We don't want to reenter
-           // the monitor while suspended because that would surprise the
-           // thread that suspended us.
-           //
-           // Drop the lock -
-       SimpleExit (THREAD) ;
-
-           jt->java_suspend_self();
-         }
-
-     assert(_owner == THREAD, "Fatal error with monitor owner!");
-     assert(_recursions == 0, "Fatal error with monitor recursions!");
-  }
-
-  THREAD->set_current_pending_monitor(NULL);
-  guarantee (_recursions == 0, "invariant") ;
-  return OM_OK;
-}
-
-// Used mainly for JVMTI raw monitor implementation
-// Also used for ObjectMonitor::wait().
-int ObjectMonitor::raw_exit(TRAPS) {
-  TEVENT (raw_exit) ;
-  if (THREAD != _owner) {
-    return OM_ILLEGAL_MONITOR_STATE;
-  }
-  if (_recursions > 0) {
-    --_recursions ;
-    return OM_OK ;
-  }
-
-  void * List = _EntryList ;
-  SimpleExit (THREAD) ;
-
-  return OM_OK;
-}
-
-// Used for JVMTI raw monitor implementation.
-// All JavaThreads will enter here with state _thread_blocked
-
-int ObjectMonitor::raw_wait(jlong millis, bool interruptible, TRAPS) {
-  TEVENT (raw_wait) ;
-  if (THREAD != _owner) {
-    return OM_ILLEGAL_MONITOR_STATE;
-  }
-
-  // To avoid spurious wakeups we reset the parkevent -- This is strictly optional.
-  // The caller must be able to tolerate spurious returns from raw_wait().
-  THREAD->_ParkEvent->reset() ;
-  OrderAccess::fence() ;
-
-  // check interrupt event
-  if (interruptible && Thread::is_interrupted(THREAD, true)) {
-    return OM_INTERRUPTED;
-  }
-
-  intptr_t save = _recursions ;
-  _recursions = 0 ;
-  _waiters ++ ;
-  if (THREAD->is_Java_thread()) {
-    guarantee (((JavaThread *) THREAD)->thread_state() == _thread_blocked, "invariant") ;
-    ((JavaThread *)THREAD)->set_suspend_equivalent();
-  }
-  int rv = SimpleWait (THREAD, millis) ;
-  _recursions = save ;
-  _waiters -- ;
-
-  guarantee (THREAD == _owner, "invariant") ;
-  if (THREAD->is_Java_thread()) {
-     JavaThread * jSelf = (JavaThread *) THREAD ;
-     for (;;) {
-        if (!jSelf->handle_special_suspend_equivalent_condition()) break ;
-        SimpleExit (THREAD) ;
-        jSelf->java_suspend_self();
-        SimpleEnter (THREAD) ;
-        jSelf->set_suspend_equivalent() ;
-     }
-  }
-  guarantee (THREAD == _owner, "invariant") ;
-
-  if (interruptible && Thread::is_interrupted(THREAD, true)) {
-    return OM_INTERRUPTED;
-  }
-  return OM_OK ;
-}
-
-int ObjectMonitor::raw_notify(TRAPS) {
-  TEVENT (raw_notify) ;
-  if (THREAD != _owner) {
-    return OM_ILLEGAL_MONITOR_STATE;
-  }
-  SimpleNotify (THREAD, false) ;
-  return OM_OK;
-}
-
-int ObjectMonitor::raw_notifyAll(TRAPS) {
-  TEVENT (raw_notifyAll) ;
-  if (THREAD != _owner) {
-    return OM_ILLEGAL_MONITOR_STATE;
-  }
-  SimpleNotify (THREAD, true) ;
-  return OM_OK;
-}
-
-#ifndef PRODUCT
-void ObjectMonitor::verify() {
-}
-
-void ObjectMonitor::print() {
-}
-#endif
-
 //------------------------------------------------------------------------------
 // Non-product code
 
diff --git a/hotspot/src/share/vm/runtime/synchronizer.hpp b/hotspot/src/share/vm/runtime/synchronizer.hpp
index 90af13ab00e..8c7132a9f31 100644
--- a/hotspot/src/share/vm/runtime/synchronizer.hpp
+++ b/hotspot/src/share/vm/runtime/synchronizer.hpp
@@ -22,53 +22,6 @@
  *
  */
 
-class BasicLock VALUE_OBJ_CLASS_SPEC {
-  friend class VMStructs;
- private:
-  volatile markOop _displaced_header;
- public:
-  markOop      displaced_header() const               { return _displaced_header; }
-  void         set_displaced_header(markOop header)   { _displaced_header = header; }
-
-  void print_on(outputStream* st) const;
-
-  // move a basic lock (used during deoptimization
-  void move_to(oop obj, BasicLock* dest);
-
-  static int displaced_header_offset_in_bytes()       { return offset_of(BasicLock, _displaced_header); }
-};
-
-// A BasicObjectLock associates a specific Java object with a BasicLock.
-// It is currently embedded in an interpreter frame.
-
-// Because some machines have alignment restrictions on the control stack,
-// the actual space allocated by the interpreter may include padding words
-// after the end of the BasicObjectLock.  Also, in order to guarantee
-// alignment of the embedded BasicLock objects on such machines, we
-// put the embedded BasicLock at the beginning of the struct.
-
-class BasicObjectLock VALUE_OBJ_CLASS_SPEC {
-  friend class VMStructs;
- private:
-  BasicLock _lock;                                    // the lock, must be double word aligned
-  oop       _obj;                                     // object holds the lock;
-
- public:
-  // Manipulation
-  oop      obj() const                                { return _obj;  }
-  void set_obj(oop obj)                               { _obj = obj; }
-  BasicLock* lock()                                   { return &_lock; }
-
-  // Note: Use frame::interpreter_frame_monitor_size() for the size of BasicObjectLocks
-  //       in interpreter activation frames since it includes machine-specific padding.
-  static int size()                                   { return sizeof(BasicObjectLock)/wordSize; }
-
-  // GC support
-  void oops_do(OopClosure* f) { f->do_oop(&_obj); }
-
-  static int obj_offset_in_bytes()                    { return offset_of(BasicObjectLock, _obj);  }
-  static int lock_offset_in_bytes()                   { return offset_of(BasicObjectLock, _lock); }
-};
 
 class ObjectMonitor;
 
@@ -163,6 +116,8 @@ class ObjectSynchronizer : AllStatic {
   static void verify() PRODUCT_RETURN;
   static int  verify_objmon_isinpool(ObjectMonitor *addr) PRODUCT_RETURN0;
 
+  static void RegisterSpinCallback (int (*)(intptr_t, int), intptr_t) ;
+
  private:
   enum { _BLOCKSIZE = 128 };
   static ObjectMonitor* gBlockList;
@@ -170,30 +125,6 @@ class ObjectSynchronizer : AllStatic {
   static ObjectMonitor * volatile gOmInUseList; // for moribund thread, so monitors they inflated still get scanned
   static int gOmInUseCount;
 
- public:
-  static void Initialize () ;
-  static PerfCounter * _sync_ContendedLockAttempts ;
-  static PerfCounter * _sync_FutileWakeups ;
-  static PerfCounter * _sync_Parks ;
-  static PerfCounter * _sync_EmptyNotifications ;
-  static PerfCounter * _sync_Notifications ;
-  static PerfCounter * _sync_SlowEnter ;
-  static PerfCounter * _sync_SlowExit ;
-  static PerfCounter * _sync_SlowNotify ;
-  static PerfCounter * _sync_SlowNotifyAll ;
-  static PerfCounter * _sync_FailedSpins ;
-  static PerfCounter * _sync_SuccessfulSpins ;
-  static PerfCounter * _sync_PrivateA ;
-  static PerfCounter * _sync_PrivateB ;
-  static PerfCounter * _sync_MonInCirculation ;
-  static PerfCounter * _sync_MonScavenged ;
-  static PerfCounter * _sync_Inflations ;
-  static PerfCounter * _sync_Deflations ;
-  static PerfLongVariable * _sync_MonExtant ;
-
- public:
-  static void RegisterSpinCallback (int (*)(intptr_t, int), intptr_t) ;
-
 };
 
 // ObjectLocker enforced balanced locking and can never thrown an
diff --git a/hotspot/src/share/vm/runtime/thread.cpp b/hotspot/src/share/vm/runtime/thread.cpp
index 27ea367302b..1abd9cc61cc 100644
--- a/hotspot/src/share/vm/runtime/thread.cpp
+++ b/hotspot/src/share/vm/runtime/thread.cpp
@@ -2995,8 +2995,8 @@ jint Threads::create_vm(JavaVMInitArgs* args, bool* canTryAgain) {
   // crash Linux VM, see notes in os_linux.cpp.
   main_thread->create_stack_guard_pages();
 
-  // Initialize Java-Leve synchronization subsystem
-  ObjectSynchronizer::Initialize() ;
+  // Initialize Java-Level synchronization subsystem
+  ObjectMonitor::Initialize() ;
 
   // Initialize global modules
   jint status = init_globals();
@@ -3965,215 +3965,272 @@ void Threads::print_on_error(outputStream* st, Thread* current, char* buf, int b
   }
 }
 
+// Internal SpinLock and Mutex
+// Based on ParkEvent
 
-// Lifecycle management for TSM ParkEvents.
-// ParkEvents are type-stable (TSM).
-// In our particular implementation they happen to be immortal.
+// Ad-hoc mutual exclusion primitives: SpinLock and Mux
 //
-// We manage concurrency on the FreeList with a CAS-based
-// detach-modify-reattach idiom that avoids the ABA problems
-// that would otherwise be present in a simple CAS-based
-// push-pop implementation.   (push-one and pop-all)
+// We employ SpinLocks _only for low-contention, fixed-length
+// short-duration critical sections where we're concerned
+// about native mutex_t or HotSpot Mutex:: latency.
+// The mux construct provides a spin-then-block mutual exclusion
+// mechanism.
 //
-// Caveat: Allocate() and Release() may be called from threads
-// other than the thread associated with the Event!
-// If we need to call Allocate() when running as the thread in
-// question then look for the PD calls to initialize native TLS.
-// Native TLS (Win32/Linux/Solaris) can only be initialized or
-// accessed by the associated thread.
-// See also pd_initialize().
+// Testing has shown that contention on the ListLock guarding gFreeList
+// is common.  If we implement ListLock as a simple SpinLock it's common
+// for the JVM to devolve to yielding with little progress.  This is true
+// despite the fact that the critical sections protected by ListLock are
+// extremely short.
 //
-// Note that we could defer associating a ParkEvent with a thread
-// until the 1st time the thread calls park().  unpark() calls to
-// an unprovisioned thread would be ignored.  The first park() call
-// for a thread would allocate and associate a ParkEvent and return
-// immediately.
+// TODO-FIXME: ListLock should be of type SpinLock.
+// We should make this a 1st-class type, integrated into the lock
+// hierarchy as leaf-locks.  Critically, the SpinLock structure
+// should have sufficient padding to avoid false-sharing and excessive
+// cache-coherency traffic.
 
-volatile int ParkEvent::ListLock = 0 ;
-ParkEvent * volatile ParkEvent::FreeList = NULL ;
 
-ParkEvent * ParkEvent::Allocate (Thread * t) {
-  // In rare cases -- JVM_RawMonitor* operations -- we can find t == null.
-  ParkEvent * ev ;
+typedef volatile int SpinLockT ;
 
-  // Start by trying to recycle an existing but unassociated
-  // ParkEvent from the global free list.
+void Thread::SpinAcquire (volatile int * adr, const char * LockName) {
+  if (Atomic::cmpxchg (1, adr, 0) == 0) {
+     return ;   // normal fast-path return
+  }
+
+  // Slow-path : We've encountered contention -- Spin/Yield/Block strategy.
+  TEVENT (SpinAcquire - ctx) ;
+  int ctr = 0 ;
+  int Yields = 0 ;
   for (;;) {
-    ev = FreeList ;
-    if (ev == NULL) break ;
-    // 1: Detach - sequester or privatize the list
-    // Tantamount to ev = Swap (&FreeList, NULL)
-    if (Atomic::cmpxchg_ptr (NULL, &FreeList, ev) != ev) {
-       continue ;
-    }
-
-    // We've detached the list.  The list in-hand is now
-    // local to this thread.   This thread can operate on the
-    // list without risk of interference from other threads.
-    // 2: Extract -- pop the 1st element from the list.
-    ParkEvent * List = ev->FreeNext ;
-    if (List == NULL) break ;
-    for (;;) {
-        // 3: Try to reattach the residual list
-        guarantee (List != NULL, "invariant") ;
-        ParkEvent * Arv =  (ParkEvent *) Atomic::cmpxchg_ptr (List, &FreeList, NULL) ;
-        if (Arv == NULL) break ;
-
-        // New nodes arrived.  Try to detach the recent arrivals.
-        if (Atomic::cmpxchg_ptr (NULL, &FreeList, Arv) != Arv) {
-            continue ;
+     while (*adr != 0) {
+        ++ctr ;
+        if ((ctr & 0xFFF) == 0 || !os::is_MP()) {
+           if (Yields > 5) {
+             // Consider using a simple NakedSleep() instead.
+             // Then SpinAcquire could be called by non-JVM threads
+             Thread::current()->_ParkEvent->park(1) ;
+           } else {
+             os::NakedYield() ;
+             ++Yields ;
+           }
+        } else {
+           SpinPause() ;
         }
-        guarantee (Arv != NULL, "invariant") ;
-        // 4: Merge Arv into List
-        ParkEvent * Tail = List ;
-        while (Tail->FreeNext != NULL) Tail = Tail->FreeNext ;
-        Tail->FreeNext = Arv ;
-    }
-    break ;
+     }
+     if (Atomic::cmpxchg (1, adr, 0) == 0) return ;
   }
-
-  if (ev != NULL) {
-    guarantee (ev->AssociatedWith == NULL, "invariant") ;
-  } else {
-    // Do this the hard way -- materialize a new ParkEvent.
-    // In rare cases an allocating thread might detach a long list --
-    // installing null into FreeList -- and then stall or be obstructed.
-    // A 2nd thread calling Allocate() would see FreeList == null.
-    // The list held privately by the 1st thread is unavailable to the 2nd thread.
-    // In that case the 2nd thread would have to materialize a new ParkEvent,
-    // even though free ParkEvents existed in the system.  In this case we end up
-    // with more ParkEvents in circulation than we need, but the race is
-    // rare and the outcome is benign.  Ideally, the # of extant ParkEvents
-    // is equal to the maximum # of threads that existed at any one time.
-    // Because of the race mentioned above, segments of the freelist
-    // can be transiently inaccessible.  At worst we may end up with the
-    // # of ParkEvents in circulation slightly above the ideal.
-    // Note that if we didn't have the TSM/immortal constraint, then
-    // when reattaching, above, we could trim the list.
-    ev = new ParkEvent () ;
-    guarantee ((intptr_t(ev) & 0xFF) == 0, "invariant") ;
-  }
-  ev->reset() ;                     // courtesy to caller
-  ev->AssociatedWith = t ;          // Associate ev with t
-  ev->FreeNext       = NULL ;
-  return ev ;
 }
 
-void ParkEvent::Release (ParkEvent * ev) {
-  if (ev == NULL) return ;
-  guarantee (ev->FreeNext == NULL      , "invariant") ;
-  ev->AssociatedWith = NULL ;
+void Thread::SpinRelease (volatile int * adr) {
+  assert (*adr != 0, "invariant") ;
+  OrderAccess::fence() ;      // guarantee at least release consistency.
+  // Roach-motel semantics.
+  // It's safe if subsequent LDs and STs float "up" into the critical section,
+  // but prior LDs and STs within the critical section can't be allowed
+  // to reorder or float past the ST that releases the lock.
+  *adr = 0 ;
+}
+
+// muxAcquire and muxRelease:
+//
+// *  muxAcquire and muxRelease support a single-word lock-word construct.
+//    The LSB of the word is set IFF the lock is held.
+//    The remainder of the word points to the head of a singly-linked list
+//    of threads blocked on the lock.
+//
+// *  The current implementation of muxAcquire-muxRelease uses its own
+//    dedicated Thread._MuxEvent instance.  If we're interested in
+//    minimizing the peak number of extant ParkEvent instances then
+//    we could eliminate _MuxEvent and "borrow" _ParkEvent as long
+//    as certain invariants were satisfied.  Specifically, care would need
+//    to be taken with regards to consuming unpark() "permits".
+//    A safe rule of thumb is that a thread would never call muxAcquire()
+//    if it's enqueued (cxq, EntryList, WaitList, etc) and will subsequently
+//    park().  Otherwise the _ParkEvent park() operation in muxAcquire() could
+//    consume an unpark() permit intended for monitorenter, for instance.
+//    One way around this would be to widen the restricted-range semaphore
+//    implemented in park().  Another alternative would be to provide
+//    multiple instances of the PlatformEvent() for each thread.  One
+//    instance would be dedicated to muxAcquire-muxRelease, for instance.
+//
+// *  Usage:
+//    -- Only as leaf locks
+//    -- for short-term locking only as muxAcquire does not perform
+//       thread state transitions.
+//
+// Alternatives:
+// *  We could implement muxAcquire and muxRelease with MCS or CLH locks
+//    but with parking or spin-then-park instead of pure spinning.
+// *  Use Taura-Oyama-Yonenzawa locks.
+// *  It's possible to construct a 1-0 lock if we encode the lockword as
+//    (List,LockByte).  Acquire will CAS the full lockword while Release
+//    will STB 0 into the LockByte.  The 1-0 scheme admits stranding, so
+//    acquiring threads use timers (ParkTimed) to detect and recover from
+//    the stranding window.  Thread/Node structures must be aligned on 256-byte
+//    boundaries by using placement-new.
+// *  Augment MCS with advisory back-link fields maintained with CAS().
+//    Pictorially:  LockWord -> T1 <-> T2 <-> T3 <-> ... <-> Tn <-> Owner.
+//    The validity of the backlinks must be ratified before we trust the value.
+//    If the backlinks are invalid the exiting thread must back-track through the
+//    the forward links, which are always trustworthy.
+// *  Add a successor indication.  The LockWord is currently encoded as
+//    (List, LOCKBIT:1).  We could also add a SUCCBIT or an explicit _succ variable
+//    to provide the usual futile-wakeup optimization.
+//    See RTStt for details.
+// *  Consider schedctl.sc_nopreempt to cover the critical section.
+//
+
+
+typedef volatile intptr_t MutexT ;      // Mux Lock-word
+enum MuxBits { LOCKBIT = 1 } ;
+
+void Thread::muxAcquire (volatile intptr_t * Lock, const char * LockName) {
+  intptr_t w = Atomic::cmpxchg_ptr (LOCKBIT, Lock, 0) ;
+  if (w == 0) return ;
+  if ((w & LOCKBIT) == 0 && Atomic::cmpxchg_ptr (w|LOCKBIT, Lock, w) == w) {
+     return ;
+  }
+
+  TEVENT (muxAcquire - Contention) ;
+  ParkEvent * const Self = Thread::current()->_MuxEvent ;
+  assert ((intptr_t(Self) & LOCKBIT) == 0, "invariant") ;
   for (;;) {
-    // Push ev onto FreeList
-    // The mechanism is "half" lock-free.
-    ParkEvent * List = FreeList ;
-    ev->FreeNext = List ;
-    if (Atomic::cmpxchg_ptr (ev, &FreeList, List) == List) break ;
-  }
-}
+     int its = (os::is_MP() ? 100 : 0) + 1 ;
 
-// Override operator new and delete so we can ensure that the
-// least significant byte of ParkEvent addresses is 0.
-// Beware that excessive address alignment is undesirable
-// as it can result in D$ index usage imbalance as
-// well as bank access imbalance on Niagara-like platforms,
-// although Niagara's hash function should help.
+     // Optional spin phase: spin-then-park strategy
+     while (--its >= 0) {
+       w = *Lock ;
+       if ((w & LOCKBIT) == 0 && Atomic::cmpxchg_ptr (w|LOCKBIT, Lock, w) == w) {
+          return ;
+       }
+     }
 
-void * ParkEvent::operator new (size_t sz) {
-  return (void *) ((intptr_t (CHeapObj::operator new (sz + 256)) + 256) & -256) ;
-}
-
-void ParkEvent::operator delete (void * a) {
-  // ParkEvents are type-stable and immortal ...
-  ShouldNotReachHere();
-}
-
-
-// 6399321 As a temporary measure we copied & modified the ParkEvent::
-// allocate() and release() code for use by Parkers.  The Parker:: forms
-// will eventually be removed as we consolide and shift over to ParkEvents
-// for both builtin synchronization and JSR166 operations.
-
-volatile int Parker::ListLock = 0 ;
-Parker * volatile Parker::FreeList = NULL ;
-
-Parker * Parker::Allocate (JavaThread * t) {
-  guarantee (t != NULL, "invariant") ;
-  Parker * p ;
-
-  // Start by trying to recycle an existing but unassociated
-  // Parker from the global free list.
-  for (;;) {
-    p = FreeList ;
-    if (p  == NULL) break ;
-    // 1: Detach
-    // Tantamount to p = Swap (&FreeList, NULL)
-    if (Atomic::cmpxchg_ptr (NULL, &FreeList, p) != p) {
-       continue ;
-    }
-
-    // We've detached the list.  The list in-hand is now
-    // local to this thread.   This thread can operate on the
-    // list without risk of interference from other threads.
-    // 2: Extract -- pop the 1st element from the list.
-    Parker * List = p->FreeNext ;
-    if (List == NULL) break ;
-    for (;;) {
-        // 3: Try to reattach the residual list
-        guarantee (List != NULL, "invariant") ;
-        Parker * Arv =  (Parker *) Atomic::cmpxchg_ptr (List, &FreeList, NULL) ;
-        if (Arv == NULL) break ;
-
-        // New nodes arrived.  Try to detach the recent arrivals.
-        if (Atomic::cmpxchg_ptr (NULL, &FreeList, Arv) != Arv) {
-            continue ;
+     Self->reset() ;
+     Self->OnList = intptr_t(Lock) ;
+     // The following fence() isn't _strictly necessary as the subsequent
+     // CAS() both serializes execution and ratifies the fetched *Lock value.
+     OrderAccess::fence();
+     for (;;) {
+        w = *Lock ;
+        if ((w & LOCKBIT) == 0) {
+            if (Atomic::cmpxchg_ptr (w|LOCKBIT, Lock, w) == w) {
+                Self->OnList = 0 ;   // hygiene - allows stronger asserts
+                return ;
+            }
+            continue ;      // Interference -- *Lock changed -- Just retry
         }
-        guarantee (Arv != NULL, "invariant") ;
-        // 4: Merge Arv into List
-        Parker * Tail = List ;
-        while (Tail->FreeNext != NULL) Tail = Tail->FreeNext ;
-        Tail->FreeNext = Arv ;
-    }
-    break ;
-  }
+        assert (w & LOCKBIT, "invariant") ;
+        Self->ListNext = (ParkEvent *) (w & ~LOCKBIT );
+        if (Atomic::cmpxchg_ptr (intptr_t(Self)|LOCKBIT, Lock, w) == w) break ;
+     }
 
-  if (p != NULL) {
-    guarantee (p->AssociatedWith == NULL, "invariant") ;
-  } else {
-    // Do this the hard way -- materialize a new Parker..
-    // In rare cases an allocating thread might detach
-    // a long list -- installing null into FreeList --and
-    // then stall.  Another thread calling Allocate() would see
-    // FreeList == null and then invoke the ctor.  In this case we
-    // end up with more Parkers in circulation than we need, but
-    // the race is rare and the outcome is benign.
-    // Ideally, the # of extant Parkers is equal to the
-    // maximum # of threads that existed at any one time.
-    // Because of the race mentioned above, segments of the
-    // freelist can be transiently inaccessible.  At worst
-    // we may end up with the # of Parkers in circulation
-    // slightly above the ideal.
-    p = new Parker() ;
+     while (Self->OnList != 0) {
+        Self->park() ;
+     }
   }
-  p->AssociatedWith = t ;          // Associate p with t
-  p->FreeNext       = NULL ;
-  return p ;
 }
 
+void Thread::muxAcquireW (volatile intptr_t * Lock, ParkEvent * ev) {
+  intptr_t w = Atomic::cmpxchg_ptr (LOCKBIT, Lock, 0) ;
+  if (w == 0) return ;
+  if ((w & LOCKBIT) == 0 && Atomic::cmpxchg_ptr (w|LOCKBIT, Lock, w) == w) {
+    return ;
+  }
 
-void Parker::Release (Parker * p) {
-  if (p == NULL) return ;
-  guarantee (p->AssociatedWith != NULL, "invariant") ;
-  guarantee (p->FreeNext == NULL      , "invariant") ;
-  p->AssociatedWith = NULL ;
+  TEVENT (muxAcquire - Contention) ;
+  ParkEvent * ReleaseAfter = NULL ;
+  if (ev == NULL) {
+    ev = ReleaseAfter = ParkEvent::Allocate (NULL) ;
+  }
+  assert ((intptr_t(ev) & LOCKBIT) == 0, "invariant") ;
   for (;;) {
-    // Push p onto FreeList
-    Parker * List = FreeList ;
-    p->FreeNext = List ;
-    if (Atomic::cmpxchg_ptr (p, &FreeList, List) == List) break ;
+    guarantee (ev->OnList == 0, "invariant") ;
+    int its = (os::is_MP() ? 100 : 0) + 1 ;
+
+    // Optional spin phase: spin-then-park strategy
+    while (--its >= 0) {
+      w = *Lock ;
+      if ((w & LOCKBIT) == 0 && Atomic::cmpxchg_ptr (w|LOCKBIT, Lock, w) == w) {
+        if (ReleaseAfter != NULL) {
+          ParkEvent::Release (ReleaseAfter) ;
+        }
+        return ;
+      }
+    }
+
+    ev->reset() ;
+    ev->OnList = intptr_t(Lock) ;
+    // The following fence() isn't _strictly necessary as the subsequent
+    // CAS() both serializes execution and ratifies the fetched *Lock value.
+    OrderAccess::fence();
+    for (;;) {
+      w = *Lock ;
+      if ((w & LOCKBIT) == 0) {
+        if (Atomic::cmpxchg_ptr (w|LOCKBIT, Lock, w) == w) {
+          ev->OnList = 0 ;
+          // We call ::Release while holding the outer lock, thus
+          // artificially lengthening the critical section.
+          // Consider deferring the ::Release() until the subsequent unlock(),
+          // after we've dropped the outer lock.
+          if (ReleaseAfter != NULL) {
+            ParkEvent::Release (ReleaseAfter) ;
+          }
+          return ;
+        }
+        continue ;      // Interference -- *Lock changed -- Just retry
+      }
+      assert (w & LOCKBIT, "invariant") ;
+      ev->ListNext = (ParkEvent *) (w & ~LOCKBIT );
+      if (Atomic::cmpxchg_ptr (intptr_t(ev)|LOCKBIT, Lock, w) == w) break ;
+    }
+
+    while (ev->OnList != 0) {
+      ev->park() ;
+    }
   }
 }
 
+// Release() must extract a successor from the list and then wake that thread.
+// It can "pop" the front of the list or use a detach-modify-reattach (DMR) scheme
+// similar to that used by ParkEvent::Allocate() and ::Release().  DMR-based
+// Release() would :
+// (A) CAS() or swap() null to *Lock, releasing the lock and detaching the list.
+// (B) Extract a successor from the private list "in-hand"
+// (C) attempt to CAS() the residual back into *Lock over null.
+//     If there were any newly arrived threads and the CAS() would fail.
+//     In that case Release() would detach the RATs, re-merge the list in-hand
+//     with the RATs and repeat as needed.  Alternately, Release() might
+//     detach and extract a successor, but then pass the residual list to the wakee.
+//     The wakee would be responsible for reattaching and remerging before it
+//     competed for the lock.
+//
+// Both "pop" and DMR are immune from ABA corruption -- there can be
+// multiple concurrent pushers, but only one popper or detacher.
+// This implementation pops from the head of the list.  This is unfair,
+// but tends to provide excellent throughput as hot threads remain hot.
+// (We wake recently run threads first).
+
+void Thread::muxRelease (volatile intptr_t * Lock)  {
+  for (;;) {
+    const intptr_t w = Atomic::cmpxchg_ptr (0, Lock, LOCKBIT) ;
+    assert (w & LOCKBIT, "invariant") ;
+    if (w == LOCKBIT) return ;
+    ParkEvent * List = (ParkEvent *) (w & ~LOCKBIT) ;
+    assert (List != NULL, "invariant") ;
+    assert (List->OnList == intptr_t(Lock), "invariant") ;
+    ParkEvent * nxt = List->ListNext ;
+
+    // The following CAS() releases the lock and pops the head element.
+    if (Atomic::cmpxchg_ptr (intptr_t(nxt), Lock, w) != w) {
+      continue ;
+    }
+    List->OnList = 0 ;
+    OrderAccess::fence() ;
+    List->unpark () ;
+    return ;
+  }
+}
+
+
 void Threads::verify() {
   ALL_JAVA_THREADS(p) {
     p->verify();
diff --git a/hotspot/src/share/vm/runtime/thread.hpp b/hotspot/src/share/vm/runtime/thread.hpp
index 548c836ebb3..f02f5c6cb63 100644
--- a/hotspot/src/share/vm/runtime/thread.hpp
+++ b/hotspot/src/share/vm/runtime/thread.hpp
@@ -30,6 +30,7 @@ class JvmtiGetLoadedClassesClosure;
 class ThreadStatistics;
 class ConcurrentLocksDump;
 class ParkEvent ;
+class Parker;
 
 class ciEnv;
 class CompileThread;
@@ -544,7 +545,6 @@ public:
   static void muxAcquire  (volatile intptr_t * Lock, const char * Name) ;
   static void muxAcquireW (volatile intptr_t * Lock, ParkEvent * ev) ;
   static void muxRelease  (volatile intptr_t * Lock) ;
-
 };
 
 // Inline implementation of Thread::current()
@@ -1769,100 +1769,3 @@ public:
   }
 };
 
-// ParkEvents are type-stable and immortal.
-//
-// Lifecycle: Once a ParkEvent is associated with a thread that ParkEvent remains
-// associated with the thread for the thread's entire lifetime - the relationship is
-// stable. A thread will be associated at most one ParkEvent.  When the thread
-// expires, the ParkEvent moves to the EventFreeList.  New threads attempt to allocate from
-// the EventFreeList before creating a new Event.  Type-stability frees us from
-// worrying about stale Event or Thread references in the objectMonitor subsystem.
-// (A reference to ParkEvent is always valid, even though the event may no longer be associated
-// with the desired or expected thread.  A key aspect of this design is that the callers of
-// park, unpark, etc must tolerate stale references and spurious wakeups).
-//
-// Only the "associated" thread can block (park) on the ParkEvent, although
-// any other thread can unpark a reachable parkevent.  Park() is allowed to
-// return spuriously.  In fact park-unpark a really just an optimization to
-// avoid unbounded spinning and surrender the CPU to be a polite system citizen.
-// A degenerate albeit "impolite" park-unpark implementation could simply return.
-// See http://blogs.sun.com/dave for more details.
-//
-// Eventually I'd like to eliminate Events and ObjectWaiters, both of which serve as
-// thread proxies, and simply make the THREAD structure type-stable and persistent.
-// Currently, we unpark events associated with threads, but ideally we'd just
-// unpark threads.
-//
-// The base-class, PlatformEvent, is platform-specific while the ParkEvent is
-// platform-independent.  PlatformEvent provides park(), unpark(), etc., and
-// is abstract -- that is, a PlatformEvent should never be instantiated except
-// as part of a ParkEvent.
-// Equivalently we could have defined a platform-independent base-class that
-// exported Allocate(), Release(), etc.  The platform-specific class would extend
-// that base-class, adding park(), unpark(), etc.
-//
-// A word of caution: The JVM uses 2 very similar constructs:
-// 1. ParkEvent are used for Java-level "monitor" synchronization.
-// 2. Parkers are used by JSR166-JUC park-unpark.
-//
-// We'll want to eventually merge these redundant facilities and use ParkEvent.
-
-
-class ParkEvent : public os::PlatformEvent {
-  private:
-    ParkEvent * FreeNext ;
-
-    // Current association
-    Thread * AssociatedWith ;
-    intptr_t RawThreadIdentity ;        // LWPID etc
-    volatile int Incarnation ;
-
-    // diagnostic : keep track of last thread to wake this thread.
-    // this is useful for construction of dependency graphs.
-    void * LastWaker ;
-
-  public:
-    // MCS-CLH list linkage and Native Mutex/Monitor
-    ParkEvent * volatile ListNext ;
-    ParkEvent * volatile ListPrev ;
-    volatile intptr_t OnList ;
-    volatile int TState ;
-    volatile int Notified ;             // for native monitor construct
-    volatile int IsWaiting ;            // Enqueued on WaitSet
-
-
-  private:
-    static ParkEvent * volatile FreeList ;
-    static volatile int ListLock ;
-
-    // It's prudent to mark the dtor as "private"
-    // ensuring that it's not visible outside the package.
-    // Unfortunately gcc warns about such usage, so
-    // we revert to the less desirable "protected" visibility.
-    // The other compilers accept private dtors.
-
-  protected:        // Ensure dtor is never invoked
-    ~ParkEvent() { guarantee (0, "invariant") ; }
-
-    ParkEvent() : PlatformEvent() {
-       AssociatedWith = NULL ;
-       FreeNext       = NULL ;
-       ListNext       = NULL ;
-       ListPrev       = NULL ;
-       OnList         = 0 ;
-       TState         = 0 ;
-       Notified       = 0 ;
-       IsWaiting      = 0 ;
-    }
-
-    // We use placement-new to force ParkEvent instances to be
-    // aligned on 256-byte address boundaries.  This ensures that the least
-    // significant byte of a ParkEvent address is always 0.
-
-    void * operator new (size_t sz) ;
-    void operator delete (void * a) ;
-
-  public:
-    static ParkEvent * Allocate (Thread * t) ;
-    static void Release (ParkEvent * e) ;
-} ;

From cbc7f8756a7e9569bbe1a38ce7cab0c0c6002bf7 Mon Sep 17 00:00:00 2001
From: "Y. Srinivas Ramakrishna" <ysr@openjdk.org>
Date: Sat, 23 Oct 2010 23:03:49 -0700
Subject: [PATCH 17/23] 6896603: CMS/GCH: collection_attempt_is_safe() ergo
 should use more recent data

Deprecated HandlePromotionFailure, removing the ability to turn off that feature, did away with one epoch look-ahead when deciding if a scavenge is likely to fail, relying on current data.

Reviewed-by: jmasa, johnc, poonam
---
 .../concurrentMarkSweepGeneration.cpp         | 84 ++++---------------
 .../concurrentMarkSweepGeneration.hpp         |  3 +-
 .../parNew/parNewGeneration.cpp               | 17 +---
 .../src/share/vm/memory/collectorPolicy.cpp   |  8 +-
 .../src/share/vm/memory/defNewGeneration.cpp  | 56 ++++---------
 .../src/share/vm/memory/defNewGeneration.hpp  | 14 ++--
 .../src/share/vm/memory/genCollectedHeap.cpp  | 14 +---
 .../src/share/vm/memory/genCollectedHeap.hpp  | 39 +++++----
 hotspot/src/share/vm/memory/generation.cpp    | 15 ++--
 hotspot/src/share/vm/memory/generation.hpp    | 22 +++--
 .../src/share/vm/memory/tenuredGeneration.cpp | 35 +++-----
 .../src/share/vm/memory/tenuredGeneration.hpp |  3 +-
 hotspot/src/share/vm/runtime/arguments.cpp    |  6 +-
 hotspot/src/share/vm/runtime/globals.hpp      |  7 --
 14 files changed, 101 insertions(+), 222 deletions(-)

diff --git a/hotspot/src/share/vm/gc_implementation/concurrentMarkSweep/concurrentMarkSweepGeneration.cpp b/hotspot/src/share/vm/gc_implementation/concurrentMarkSweep/concurrentMarkSweepGeneration.cpp
index 2573d31a808..ceb7ad5dfd4 100644
--- a/hotspot/src/share/vm/gc_implementation/concurrentMarkSweep/concurrentMarkSweepGeneration.cpp
+++ b/hotspot/src/share/vm/gc_implementation/concurrentMarkSweep/concurrentMarkSweepGeneration.cpp
@@ -354,12 +354,8 @@ void CMSStats::adjust_cms_free_adjustment_factor(bool fail, size_t free) {
 double CMSStats::time_until_cms_gen_full() const {
   size_t cms_free = _cms_gen->cmsSpace()->free();
   GenCollectedHeap* gch = GenCollectedHeap::heap();
-  size_t expected_promotion = gch->get_gen(0)->capacity();
-  if (HandlePromotionFailure) {
-    expected_promotion = MIN2(
-        (size_t) _cms_gen->gc_stats()->avg_promoted()->padded_average(),
-        expected_promotion);
-  }
+  size_t expected_promotion = MIN2(gch->get_gen(0)->capacity(),
+                                   (size_t) _cms_gen->gc_stats()->avg_promoted()->padded_average());
   if (cms_free > expected_promotion) {
     // Start a cms collection if there isn't enough space to promote
     // for the next minor collection.  Use the padded average as
@@ -865,57 +861,18 @@ size_t ConcurrentMarkSweepGeneration::max_available() const {
   return free() + _virtual_space.uncommitted_size();
 }
 
-bool ConcurrentMarkSweepGeneration::promotion_attempt_is_safe(
-    size_t max_promotion_in_bytes,
-    bool younger_handles_promotion_failure) const {
-
-  // This is the most conservative test.  Full promotion is
-  // guaranteed if this is used. The multiplicative factor is to
-  // account for the worst case "dilatation".
-  double adjusted_max_promo_bytes = _dilatation_factor * max_promotion_in_bytes;
-  if (adjusted_max_promo_bytes > (double)max_uintx) { // larger than size_t
-    adjusted_max_promo_bytes = (double)max_uintx;
+bool ConcurrentMarkSweepGeneration::promotion_attempt_is_safe(size_t max_promotion_in_bytes) const {
+  size_t available = max_available();
+  size_t av_promo  = (size_t)gc_stats()->avg_promoted()->padded_average();
+  bool   res = (available >= av_promo) || (available >= max_promotion_in_bytes);
+  if (PrintGC && Verbose) {
+    gclog_or_tty->print_cr(
+      "CMS: promo attempt is%s safe: available("SIZE_FORMAT") %s av_promo("SIZE_FORMAT"),"
+      "max_promo("SIZE_FORMAT")",
+      res? "":" not", available, res? ">=":"<",
+      av_promo, max_promotion_in_bytes);
   }
-  bool result = (max_contiguous_available() >= (size_t)adjusted_max_promo_bytes);
-
-  if (younger_handles_promotion_failure && !result) {
-    // Full promotion is not guaranteed because fragmentation
-    // of the cms generation can prevent the full promotion.
-    result = (max_available() >= (size_t)adjusted_max_promo_bytes);
-
-    if (!result) {
-      // With promotion failure handling the test for the ability
-      // to support the promotion does not have to be guaranteed.
-      // Use an average of the amount promoted.
-      result = max_available() >= (size_t)
-        gc_stats()->avg_promoted()->padded_average();
-      if (PrintGC && Verbose && result) {
-        gclog_or_tty->print_cr(
-          "\nConcurrentMarkSweepGeneration::promotion_attempt_is_safe"
-          " max_available: " SIZE_FORMAT
-          " avg_promoted: " SIZE_FORMAT,
-          max_available(), (size_t)
-          gc_stats()->avg_promoted()->padded_average());
-      }
-    } else {
-      if (PrintGC && Verbose) {
-        gclog_or_tty->print_cr(
-          "\nConcurrentMarkSweepGeneration::promotion_attempt_is_safe"
-          " max_available: " SIZE_FORMAT
-          " adj_max_promo_bytes: " SIZE_FORMAT,
-          max_available(), (size_t)adjusted_max_promo_bytes);
-      }
-    }
-  } else {
-    if (PrintGC && Verbose) {
-      gclog_or_tty->print_cr(
-        "\nConcurrentMarkSweepGeneration::promotion_attempt_is_safe"
-        " contiguous_available: " SIZE_FORMAT
-        " adj_max_promo_bytes: " SIZE_FORMAT,
-        max_contiguous_available(), (size_t)adjusted_max_promo_bytes);
-    }
-  }
-  return result;
+  return res;
 }
 
 // At a promotion failure dump information on block layout in heap
@@ -6091,23 +6048,14 @@ void CMSCollector::sweep(bool asynch) {
   assert(_collectorState == Resizing, "Change of collector state to"
     " Resizing must be done under the freelistLocks (plural)");
 
-  // Now that sweeping has been completed, if the GCH's
-  // incremental_collection_will_fail flag is set, clear it,
+  // Now that sweeping has been completed, we clear
+  // the incremental_collection_failed flag,
   // thus inviting a younger gen collection to promote into
   // this generation. If such a promotion may still fail,
   // the flag will be set again when a young collection is
   // attempted.
-  // I think the incremental_collection_will_fail flag's use
-  // is specific to a 2 generation collection policy, so i'll
-  // assert that that's the configuration we are operating within.
-  // The use of the flag can and should be generalized appropriately
-  // in the future to deal with a general n-generation system.
-
   GenCollectedHeap* gch = GenCollectedHeap::heap();
-  assert(gch->collector_policy()->is_two_generation_policy(),
-         "Resetting of incremental_collection_will_fail flag"
-         " may be incorrect otherwise");
-  gch->clear_incremental_collection_will_fail();
+  gch->clear_incremental_collection_failed();  // Worth retrying as fresh space may have been freed up
   gch->update_full_collections_completed(_collection_count_start);
 }
 
diff --git a/hotspot/src/share/vm/gc_implementation/concurrentMarkSweep/concurrentMarkSweepGeneration.hpp b/hotspot/src/share/vm/gc_implementation/concurrentMarkSweep/concurrentMarkSweepGeneration.hpp
index 7a0670734e4..dce44e39729 100644
--- a/hotspot/src/share/vm/gc_implementation/concurrentMarkSweep/concurrentMarkSweepGeneration.hpp
+++ b/hotspot/src/share/vm/gc_implementation/concurrentMarkSweep/concurrentMarkSweepGeneration.hpp
@@ -1185,8 +1185,7 @@ class ConcurrentMarkSweepGeneration: public CardGeneration {
   virtual void par_promote_alloc_done(int thread_num);
   virtual void par_oop_since_save_marks_iterate_done(int thread_num);
 
-  virtual bool promotion_attempt_is_safe(size_t promotion_in_bytes,
-    bool younger_handles_promotion_failure) const;
+  virtual bool promotion_attempt_is_safe(size_t promotion_in_bytes) const;
 
   // Inform this (non-young) generation that a promotion failure was
   // encountered during a collection of a younger generation that
diff --git a/hotspot/src/share/vm/gc_implementation/parNew/parNewGeneration.cpp b/hotspot/src/share/vm/gc_implementation/parNew/parNewGeneration.cpp
index 2aa4e7f4447..12064fa7aa0 100644
--- a/hotspot/src/share/vm/gc_implementation/parNew/parNewGeneration.cpp
+++ b/hotspot/src/share/vm/gc_implementation/parNew/parNewGeneration.cpp
@@ -846,7 +846,7 @@ void ParNewGeneration::collect(bool   full,
   // from this generation, pass on collection; let the next generation
   // do it.
   if (!collection_attempt_is_safe()) {
-    gch->set_incremental_collection_will_fail();
+    gch->set_incremental_collection_failed();  // slight lie, in that we did not even attempt one
     return;
   }
   assert(to()->is_empty(), "Else not collection_attempt_is_safe");
@@ -935,8 +935,6 @@ void ParNewGeneration::collect(bool   full,
 
     assert(to()->is_empty(), "to space should be empty now");
   } else {
-    assert(HandlePromotionFailure,
-      "Should only be here if promotion failure handling is on");
     assert(_promo_failure_scan_stack.is_empty(), "post condition");
     _promo_failure_scan_stack.clear(true); // Clear cached segments.
 
@@ -947,7 +945,7 @@ void ParNewGeneration::collect(bool   full,
     // All the spaces are in play for mark-sweep.
     swap_spaces();  // Make life simpler for CMS || rescan; see 6483690.
     from()->set_next_compaction_space(to());
-    gch->set_incremental_collection_will_fail();
+    gch->set_incremental_collection_failed();
     // Inform the next generation that a promotion failure occurred.
     _next_gen->promotion_failure_occurred();
 
@@ -1092,11 +1090,6 @@ oop ParNewGeneration::copy_to_survivor_space_avoiding_promotion_undo(
                                        old, m, sz);
 
     if (new_obj == NULL) {
-      if (!HandlePromotionFailure) {
-        // A failed promotion likely means the MaxLiveObjectEvacuationRatio flag
-        // is incorrectly set. In any case, its seriously wrong to be here!
-        vm_exit_out_of_memory(sz*wordSize, "promotion");
-      }
       // promotion failed, forward to self
       _promotion_failed = true;
       new_obj = old;
@@ -1206,12 +1199,6 @@ oop ParNewGeneration::copy_to_survivor_space_with_undo(
                                        old, m, sz);
 
     if (new_obj == NULL) {
-      if (!HandlePromotionFailure) {
-        // A failed promotion likely means the MaxLiveObjectEvacuationRatio
-        // flag is incorrectly set. In any case, its seriously wrong to be
-        // here!
-        vm_exit_out_of_memory(sz*wordSize, "promotion");
-      }
       // promotion failed, forward to self
       forward_ptr = old->forward_to_atomic(old);
       new_obj = old;
diff --git a/hotspot/src/share/vm/memory/collectorPolicy.cpp b/hotspot/src/share/vm/memory/collectorPolicy.cpp
index c6b8e3875e4..3ddb0f65bc5 100644
--- a/hotspot/src/share/vm/memory/collectorPolicy.cpp
+++ b/hotspot/src/share/vm/memory/collectorPolicy.cpp
@@ -659,9 +659,6 @@ HeapWord* GenCollectorPolicy::satisfy_failed_allocation(size_t size,
     }
     return result;   // could be null if we are out of space
   } else if (!gch->incremental_collection_will_fail()) {
-    // The gc_prologues have not executed yet.  The value
-    // for incremental_collection_will_fail() is the remanent
-    // of the last collection.
     // Do an incremental collection.
     gch->do_collection(false            /* full */,
                        false            /* clear_all_soft_refs */,
@@ -739,9 +736,8 @@ bool GenCollectorPolicy::should_try_older_generation_allocation(
   GenCollectedHeap* gch = GenCollectedHeap::heap();
   size_t gen0_capacity = gch->get_gen(0)->capacity_before_gc();
   return    (word_size > heap_word_size(gen0_capacity))
-         || (GC_locker::is_active_and_needs_gc())
-         || (   gch->last_incremental_collection_failed()
-             && gch->incremental_collection_will_fail());
+         || GC_locker::is_active_and_needs_gc()
+         || gch->incremental_collection_failed();
 }
 
 
diff --git a/hotspot/src/share/vm/memory/defNewGeneration.cpp b/hotspot/src/share/vm/memory/defNewGeneration.cpp
index ac7cc267f23..6976e1660f2 100644
--- a/hotspot/src/share/vm/memory/defNewGeneration.cpp
+++ b/hotspot/src/share/vm/memory/defNewGeneration.cpp
@@ -510,7 +510,7 @@ void DefNewGeneration::collect(bool   full,
   // from this generation, pass on collection; let the next generation
   // do it.
   if (!collection_attempt_is_safe()) {
-    gch->set_incremental_collection_will_fail();
+    gch->set_incremental_collection_failed(); // Slight lie: we did not even attempt one
     return;
   }
   assert(to()->is_empty(), "Else not collection_attempt_is_safe");
@@ -596,9 +596,8 @@ void DefNewGeneration::collect(bool   full,
     if (PrintGC && !PrintGCDetails) {
       gch->print_heap_change(gch_prev_used);
     }
+    assert(!gch->incremental_collection_failed(), "Should be clear");
   } else {
-    assert(HandlePromotionFailure,
-      "Should not be here unless promotion failure handling is on");
     assert(_promo_failure_scan_stack.is_empty(), "post condition");
     _promo_failure_scan_stack.clear(true); // Clear cached segments.
 
@@ -613,7 +612,7 @@ void DefNewGeneration::collect(bool   full,
     // and from-space.
     swap_spaces();   // For uniformity wrt ParNewGeneration.
     from()->set_next_compaction_space(to());
-    gch->set_incremental_collection_will_fail();
+    gch->set_incremental_collection_failed();
 
     // Inform the next generation that a promotion failure occurred.
     _next_gen->promotion_failure_occurred();
@@ -700,12 +699,6 @@ oop DefNewGeneration::copy_to_survivor_space(oop old) {
   if (obj == NULL) {
     obj = _next_gen->promote(old, s);
     if (obj == NULL) {
-      if (!HandlePromotionFailure) {
-        // A failed promotion likely means the MaxLiveObjectEvacuationRatio flag
-        // is incorrectly set. In any case, its seriously wrong to be here!
-        vm_exit_out_of_memory(s*wordSize, "promotion");
-      }
-
       handle_promotion_failure(old);
       return old;
     }
@@ -812,31 +805,7 @@ bool DefNewGeneration::collection_attempt_is_safe() {
     assert(_next_gen != NULL,
            "This must be the youngest gen, and not the only gen");
   }
-
-  // Decide if there's enough room for a full promotion
-  // When using extremely large edens, we effectively lose a
-  // large amount of old space.  Use the "MaxLiveObjectEvacuationRatio"
-  // flag to reduce the minimum evacuation space requirements. If
-  // there is not enough space to evacuate eden during a scavenge,
-  // the VM will immediately exit with an out of memory error.
-  // This flag has not been tested
-  // with collectors other than simple mark & sweep.
-  //
-  // Note that with the addition of promotion failure handling, the
-  // VM will not immediately exit but will undo the young generation
-  // collection.  The parameter is left here for compatibility.
-  const double evacuation_ratio = MaxLiveObjectEvacuationRatio / 100.0;
-
-  // worst_case_evacuation is based on "used()".  For the case where this
-  // method is called after a collection, this is still appropriate because
-  // the case that needs to be detected is one in which a full collection
-  // has been done and has overflowed into the young generation.  In that
-  // case a minor collection will fail (the overflow of the full collection
-  // means there is no space in the old generation for any promotion).
-  size_t worst_case_evacuation = (size_t)(used() * evacuation_ratio);
-
-  return _next_gen->promotion_attempt_is_safe(worst_case_evacuation,
-                                              HandlePromotionFailure);
+  return _next_gen->promotion_attempt_is_safe(used());
 }
 
 void DefNewGeneration::gc_epilogue(bool full) {
@@ -845,14 +814,17 @@ void DefNewGeneration::gc_epilogue(bool full) {
   // a minimum at the end of a collection.  If it is not, then
   // the heap is approaching full.
   GenCollectedHeap* gch = GenCollectedHeap::heap();
-  clear_should_allocate_from_space();
-  if (collection_attempt_is_safe()) {
-    gch->clear_incremental_collection_will_fail();
-  } else {
-    gch->set_incremental_collection_will_fail();
-    if (full) { // we seem to be running out of space
-      set_should_allocate_from_space();
+  if (full) {
+    assert(!GC_locker::is_active(), "We should not be executing here");
+    if (!collection_attempt_is_safe()) {
+      gch->set_incremental_collection_failed(); // Slight lie: a full gc left us in that state
+      set_should_allocate_from_space(); // we seem to be running out of space
+    } else {
+      gch->clear_incremental_collection_failed(); // We just did a full collection
+      clear_should_allocate_from_space(); // if set
     }
+  } else {
+    assert(!gch->incremental_collection_failed(), "Error");
   }
 
   if (ZapUnusedHeapArea) {
diff --git a/hotspot/src/share/vm/memory/defNewGeneration.hpp b/hotspot/src/share/vm/memory/defNewGeneration.hpp
index 166510b0a38..6f9d9a7e68e 100644
--- a/hotspot/src/share/vm/memory/defNewGeneration.hpp
+++ b/hotspot/src/share/vm/memory/defNewGeneration.hpp
@@ -82,12 +82,6 @@ protected:
   Stack<oop>     _objs_with_preserved_marks;
   Stack<markOop> _preserved_marks_of_objs;
 
-  // Returns true if the collection can be safely attempted.
-  // If this method returns false, a collection is not
-  // guaranteed to fail but the system may not be able
-  // to recover from the failure.
-  bool collection_attempt_is_safe();
-
   // Promotion failure handling
   OopClosure *_promo_failure_scan_stack_closure;
   void set_promo_failure_scan_stack_closure(OopClosure *scan_stack_closure) {
@@ -304,6 +298,14 @@ protected:
 
   // GC support
   virtual void compute_new_size();
+
+  // Returns true if the collection is likely to be safely
+  // completed. Even if this method returns true, a collection
+  // may not be guaranteed to succeed, and the system should be
+  // able to safely unwind and recover from that failure, albeit
+  // at some additional cost. Override superclass's implementation.
+  virtual bool collection_attempt_is_safe();
+
   virtual void collect(bool   full,
                        bool   clear_all_soft_refs,
                        size_t size,
diff --git a/hotspot/src/share/vm/memory/genCollectedHeap.cpp b/hotspot/src/share/vm/memory/genCollectedHeap.cpp
index c2b8dcb3fab..432f64737fe 100644
--- a/hotspot/src/share/vm/memory/genCollectedHeap.cpp
+++ b/hotspot/src/share/vm/memory/genCollectedHeap.cpp
@@ -142,8 +142,7 @@ jint GenCollectedHeap::initialize() {
   }
   _perm_gen = perm_gen_spec->init(heap_rs, PermSize, rem_set());
 
-  clear_incremental_collection_will_fail();
-  clear_last_incremental_collection_failed();
+  clear_incremental_collection_failed();
 
 #ifndef SERIALGC
   // If we are running CMS, create the collector responsible
@@ -1347,17 +1346,6 @@ class GenGCEpilogueClosure: public GenCollectedHeap::GenClosure {
 };
 
 void GenCollectedHeap::gc_epilogue(bool full) {
-  // Remember if a partial collection of the heap failed, and
-  // we did a complete collection.
-  if (full && incremental_collection_will_fail()) {
-    set_last_incremental_collection_failed();
-  } else {
-    clear_last_incremental_collection_failed();
-  }
-  // Clear the flag, if set; the generation gc_epilogues will set the
-  // flag again if the condition persists despite the collection.
-  clear_incremental_collection_will_fail();
-
 #ifdef COMPILER2
   assert(DerivedPointerTable::is_empty(), "derived pointer present");
   size_t actual_gap = pointer_delta((HeapWord*) (max_uintx-3), *(end_addr()));
diff --git a/hotspot/src/share/vm/memory/genCollectedHeap.hpp b/hotspot/src/share/vm/memory/genCollectedHeap.hpp
index 75266a2d9a9..859f47936a7 100644
--- a/hotspot/src/share/vm/memory/genCollectedHeap.hpp
+++ b/hotspot/src/share/vm/memory/genCollectedHeap.hpp
@@ -62,11 +62,10 @@ public:
   // The generational collector policy.
   GenCollectorPolicy* _gen_policy;
 
-  // If a generation would bail out of an incremental collection,
-  // it sets this flag.  If the flag is set, satisfy_failed_allocation
-  // will attempt allocating in all generations before doing a full GC.
-  bool _incremental_collection_will_fail;
-  bool _last_incremental_collection_failed;
+  // Indicates that the most recent previous incremental collection failed.
+  // The flag is cleared when an action is taken that might clear the
+  // condition that caused that incremental collection to fail.
+  bool _incremental_collection_failed;
 
   // In support of ExplicitGCInvokesConcurrent functionality
   unsigned int _full_collections_completed;
@@ -469,26 +468,26 @@ public:
   // call to "save_marks".
   bool no_allocs_since_save_marks(int level);
 
-  // If a generation bails out of an incremental collection,
-  // it sets this flag.
+  // Returns true if an incremental collection is likely to fail.
   bool incremental_collection_will_fail() {
-    return _incremental_collection_will_fail;
-  }
-  void set_incremental_collection_will_fail() {
-    _incremental_collection_will_fail = true;
-  }
-  void clear_incremental_collection_will_fail() {
-    _incremental_collection_will_fail = false;
+    // Assumes a 2-generation system; the first disjunct remembers if an
+    // incremental collection failed, even when we thought (second disjunct)
+    // that it would not.
+    assert(heap()->collector_policy()->is_two_generation_policy(),
+           "the following definition may not be suitable for an n(>2)-generation system");
+    return incremental_collection_failed() || !get_gen(0)->collection_attempt_is_safe();
   }
 
-  bool last_incremental_collection_failed() const {
-    return _last_incremental_collection_failed;
+  // If a generation bails out of an incremental collection,
+  // it sets this flag.
+  bool incremental_collection_failed() const {
+    return _incremental_collection_failed;
   }
-  void set_last_incremental_collection_failed() {
-    _last_incremental_collection_failed = true;
+  void set_incremental_collection_failed() {
+    _incremental_collection_failed = true;
   }
-  void clear_last_incremental_collection_failed() {
-    _last_incremental_collection_failed = false;
+  void clear_incremental_collection_failed() {
+    _incremental_collection_failed = false;
   }
 
   // Promotion of obj into gen failed.  Try to promote obj to higher non-perm
diff --git a/hotspot/src/share/vm/memory/generation.cpp b/hotspot/src/share/vm/memory/generation.cpp
index 6d0d48596b3..c2f51593778 100644
--- a/hotspot/src/share/vm/memory/generation.cpp
+++ b/hotspot/src/share/vm/memory/generation.cpp
@@ -165,15 +165,16 @@ size_t Generation::max_contiguous_available() const {
   return max;
 }
 
-bool Generation::promotion_attempt_is_safe(size_t promotion_in_bytes,
-                                           bool not_used) const {
+bool Generation::promotion_attempt_is_safe(size_t max_promotion_in_bytes) const {
+  size_t available = max_contiguous_available();
+  bool   res = (available >= max_promotion_in_bytes);
   if (PrintGC && Verbose) {
-    gclog_or_tty->print_cr("Generation::promotion_attempt_is_safe"
-                " contiguous_available: " SIZE_FORMAT
-                " promotion_in_bytes: " SIZE_FORMAT,
-                max_contiguous_available(), promotion_in_bytes);
+    gclog_or_tty->print_cr(
+      "Generation: promo attempt is%s safe: available("SIZE_FORMAT") %s max_promo("SIZE_FORMAT")",
+      res? "":" not", available, res? ">=":"<",
+      max_promotion_in_bytes);
   }
-  return max_contiguous_available() >= promotion_in_bytes;
+  return res;
 }
 
 // Ignores "ref" and calls allocate().
diff --git a/hotspot/src/share/vm/memory/generation.hpp b/hotspot/src/share/vm/memory/generation.hpp
index af0bdf2b3f5..4398730a350 100644
--- a/hotspot/src/share/vm/memory/generation.hpp
+++ b/hotspot/src/share/vm/memory/generation.hpp
@@ -173,15 +173,11 @@ class Generation: public CHeapObj {
   // The largest number of contiguous free bytes in this or any higher generation.
   virtual size_t max_contiguous_available() const;
 
-  // Returns true if promotions of the specified amount can
-  // be attempted safely (without a vm failure).
+  // Returns true if promotions of the specified amount are
+  // likely to succeed without a promotion failure.
   // Promotion of the full amount is not guaranteed but
-  // can be attempted.
-  //   younger_handles_promotion_failure
-  // is true if the younger generation handles a promotion
-  // failure.
-  virtual bool promotion_attempt_is_safe(size_t promotion_in_bytes,
-    bool younger_handles_promotion_failure) const;
+  // might be attempted in the worst case.
+  virtual bool promotion_attempt_is_safe(size_t max_promotion_in_bytes) const;
 
   // For a non-young generation, this interface can be used to inform a
   // generation that a promotion attempt into that generation failed.
@@ -358,6 +354,16 @@ class Generation: public CHeapObj {
     return (full || should_allocate(word_size, is_tlab));
   }
 
+  // Returns true if the collection is likely to be safely
+  // completed. Even if this method returns true, a collection
+  // may not be guaranteed to succeed, and the system should be
+  // able to safely unwind and recover from that failure, albeit
+  // at some additional cost.
+  virtual bool collection_attempt_is_safe() {
+    guarantee(false, "Are you sure you want to call this method?");
+    return true;
+  }
+
   // Perform a garbage collection.
   // If full is true attempt a full garbage collection of this generation.
   // Otherwise, attempting to (at least) free enough space to support an
diff --git a/hotspot/src/share/vm/memory/tenuredGeneration.cpp b/hotspot/src/share/vm/memory/tenuredGeneration.cpp
index 593cd6c025b..ea474825db0 100644
--- a/hotspot/src/share/vm/memory/tenuredGeneration.cpp
+++ b/hotspot/src/share/vm/memory/tenuredGeneration.cpp
@@ -419,29 +419,16 @@ void TenuredGeneration::retire_alloc_buffers_before_full_gc() {}
 void TenuredGeneration::verify_alloc_buffers_clean() {}
 #endif // SERIALGC
 
-bool TenuredGeneration::promotion_attempt_is_safe(
-    size_t max_promotion_in_bytes,
-    bool younger_handles_promotion_failure) const {
-
-  bool result = max_contiguous_available() >= max_promotion_in_bytes;
-
-  if (younger_handles_promotion_failure && !result) {
-    result = max_contiguous_available() >=
-      (size_t) gc_stats()->avg_promoted()->padded_average();
-    if (PrintGC && Verbose && result) {
-      gclog_or_tty->print_cr("TenuredGeneration::promotion_attempt_is_safe"
-                  " contiguous_available: " SIZE_FORMAT
-                  " avg_promoted: " SIZE_FORMAT,
-                  max_contiguous_available(),
-                  gc_stats()->avg_promoted()->padded_average());
-    }
-  } else {
-    if (PrintGC && Verbose) {
-      gclog_or_tty->print_cr("TenuredGeneration::promotion_attempt_is_safe"
-                  " contiguous_available: " SIZE_FORMAT
-                  " promotion_in_bytes: " SIZE_FORMAT,
-                  max_contiguous_available(), max_promotion_in_bytes);
-    }
+bool TenuredGeneration::promotion_attempt_is_safe(size_t max_promotion_in_bytes) const {
+  size_t available = max_contiguous_available();
+  size_t av_promo  = (size_t)gc_stats()->avg_promoted()->padded_average();
+  bool   res = (available >= av_promo) || (available >= max_promotion_in_bytes);
+  if (PrintGC && Verbose) {
+    gclog_or_tty->print_cr(
+      "Tenured: promo attempt is%s safe: available("SIZE_FORMAT") %s av_promo("SIZE_FORMAT"),"
+      "max_promo("SIZE_FORMAT")",
+      res? "":" not", available, res? ">=":"<",
+      av_promo, max_promotion_in_bytes);
   }
-  return result;
+  return res;
 }
diff --git a/hotspot/src/share/vm/memory/tenuredGeneration.hpp b/hotspot/src/share/vm/memory/tenuredGeneration.hpp
index 82cc06fae44..3677867532f 100644
--- a/hotspot/src/share/vm/memory/tenuredGeneration.hpp
+++ b/hotspot/src/share/vm/memory/tenuredGeneration.hpp
@@ -101,8 +101,7 @@ class TenuredGeneration: public OneContigSpaceCardGeneration {
 
   virtual void update_gc_stats(int level, bool full);
 
-  virtual bool promotion_attempt_is_safe(size_t max_promoted_in_bytes,
-    bool younger_handles_promotion_failure) const;
+  virtual bool promotion_attempt_is_safe(size_t max_promoted_in_bytes) const;
 
   void verify_alloc_buffers_clean();
 };
diff --git a/hotspot/src/share/vm/runtime/arguments.cpp b/hotspot/src/share/vm/runtime/arguments.cpp
index 4b6d69f0566..4ef33935e32 100644
--- a/hotspot/src/share/vm/runtime/arguments.cpp
+++ b/hotspot/src/share/vm/runtime/arguments.cpp
@@ -185,6 +185,10 @@ static ObsoleteFlag obsolete_jvm_flags[] = {
                            JDK_Version::jdk_update(6,18), JDK_Version::jdk(7) },
   { "UseDepthFirstScavengeOrder",
                            JDK_Version::jdk_update(6,22), JDK_Version::jdk(7) },
+  { "HandlePromotionFailure",
+                           JDK_Version::jdk_update(6,24), JDK_Version::jdk(8) },
+  { "MaxLiveObjectEvacuationRatio",
+                           JDK_Version::jdk_update(6,24), JDK_Version::jdk(8) },
   { NULL, JDK_Version(0), JDK_Version(0) }
 };
 
@@ -1722,8 +1726,6 @@ bool Arguments::check_vm_args_consistency() {
     status = false;
   }
 
-  status = status && verify_percentage(MaxLiveObjectEvacuationRatio,
-                              "MaxLiveObjectEvacuationRatio");
   status = status && verify_percentage(AdaptiveSizePolicyWeight,
                               "AdaptiveSizePolicyWeight");
   status = status && verify_percentage(AdaptivePermSizeWeight, "AdaptivePermSizeWeight");
diff --git a/hotspot/src/share/vm/runtime/globals.hpp b/hotspot/src/share/vm/runtime/globals.hpp
index 1fa26ce7fd6..00f16350436 100644
--- a/hotspot/src/share/vm/runtime/globals.hpp
+++ b/hotspot/src/share/vm/runtime/globals.hpp
@@ -1786,10 +1786,6 @@ class CommandLineFlags {
   notproduct(bool, GCALotAtAllSafepoints, false,                            \
           "Enforce ScavengeALot/GCALot at all potential safepoints")        \
                                                                             \
-  product(bool, HandlePromotionFailure, true,                               \
-          "The youngest generation collection does not require "            \
-          "a guarantee of full promotion of all live objects.")             \
-                                                                            \
   product(bool, PrintPromotionFailure, false,                               \
           "Print additional diagnostic information following "              \
           " promotion failure")                                             \
@@ -3003,9 +2999,6 @@ class CommandLineFlags {
   product(intx, NewRatio, 2,                                                \
           "Ratio of new/old generation sizes")                              \
                                                                             \
-  product(uintx, MaxLiveObjectEvacuationRatio, 100,                         \
-          "Max percent of eden objects that will be live at scavenge")      \
-                                                                            \
   product_pd(uintx, NewSizeThreadIncrease,                                  \
           "Additional size added to desired new generation size per "       \
           "non-daemon thread (in bytes)")                                   \

From e36f1e36ebc0ca50f7b9125a22d8bd6eb4eb2c71 Mon Sep 17 00:00:00 2001
From: Keith McGuigan <kamg@openjdk.org>
Date: Tue, 26 Oct 2010 14:08:49 -0400
Subject: [PATCH 18/23] 6891959: HotSpot should not throw ClassFormatError if a
 class has a field with '>' and/or '<' in its name

Class file parser needs to look for and disallow '[' in names.

Reviewed-by: coleenp, never
---
 .../src/share/vm/classfile/classFileParser.cpp  | 17 +++++++++--------
 1 file changed, 9 insertions(+), 8 deletions(-)

diff --git a/hotspot/src/share/vm/classfile/classFileParser.cpp b/hotspot/src/share/vm/classfile/classFileParser.cpp
index a8de5afc5b1..b86e9fa892d 100644
--- a/hotspot/src/share/vm/classfile/classFileParser.cpp
+++ b/hotspot/src/share/vm/classfile/classFileParser.cpp
@@ -4309,20 +4309,21 @@ int ClassFileParser::verify_legal_method_signature(symbolHandle name, symbolHand
 }
 
 
-// Unqualified names may not contain the characters '.', ';', or '/'.
-// Method names also may not contain the characters '<' or '>', unless <init> or <clinit>.
-// Note that method names may not be <init> or <clinit> in this method.
-// Because these names have been checked as special cases before calling this method
-// in verify_legal_method_name.
-bool ClassFileParser::verify_unqualified_name(char* name, unsigned int length, int type) {
+// Unqualified names may not contain the characters '.', ';', '[', or '/'.
+// Method names also may not contain the characters '<' or '>', unless <init>
+// or <clinit>.  Note that method names may not be <init> or <clinit> in this
+// method.  Because these names have been checked as special cases before
+// calling this method in verify_legal_method_name.
+bool ClassFileParser::verify_unqualified_name(
+    char* name, unsigned int length, int type) {
   jchar ch;
 
   for (char* p = name; p != name + length; ) {
     ch = *p;
     if (ch < 128) {
       p++;
-      if (ch == '.' || ch == ';') {
-        return false;   // do not permit '.' or ';'
+      if (ch == '.' || ch == ';' || ch == '[' ) {
+        return false;   // do not permit '.', ';', or '['
       }
       if (type != LegalClass && ch == '/') {
         return false;   // do not permit '/' unless it's class name

From 9333ab239954e8e1a9a5b39ed0e2655abe1e0ca6 Mon Sep 17 00:00:00 2001
From: "Y. Srinivas Ramakrishna" <ysr@openjdk.org>
Date: Thu, 28 Oct 2010 14:46:29 -0700
Subject: [PATCH 19/23] 6995045: assert(!gch->incremental_collection_failed())
 failed: Error, defNewGeneration.cpp:827

Sharpened an assert, introduced in 6896603, that intended to check that the incremental_collection_failed() predicate on the heap was being reset "soon enough".

Reviewed-by: jmasa
---
 .../src/share/vm/memory/defNewGeneration.cpp  | 21 +++++++++++++++++--
 1 file changed, 19 insertions(+), 2 deletions(-)

diff --git a/hotspot/src/share/vm/memory/defNewGeneration.cpp b/hotspot/src/share/vm/memory/defNewGeneration.cpp
index 6976e1660f2..19f265a6729 100644
--- a/hotspot/src/share/vm/memory/defNewGeneration.cpp
+++ b/hotspot/src/share/vm/memory/defNewGeneration.cpp
@@ -809,13 +809,16 @@ bool DefNewGeneration::collection_attempt_is_safe() {
 }
 
 void DefNewGeneration::gc_epilogue(bool full) {
+  DEBUG_ONLY(static bool seen_incremental_collection_failed = false;)
+
+  assert(!GC_locker::is_active(), "We should not be executing here");
   // Check if the heap is approaching full after a collection has
   // been done.  Generally the young generation is empty at
   // a minimum at the end of a collection.  If it is not, then
   // the heap is approaching full.
   GenCollectedHeap* gch = GenCollectedHeap::heap();
   if (full) {
-    assert(!GC_locker::is_active(), "We should not be executing here");
+    DEBUG_ONLY(seen_incremental_collection_failed = false;)
     if (!collection_attempt_is_safe()) {
       gch->set_incremental_collection_failed(); // Slight lie: a full gc left us in that state
       set_should_allocate_from_space(); // we seem to be running out of space
@@ -824,7 +827,21 @@ void DefNewGeneration::gc_epilogue(bool full) {
       clear_should_allocate_from_space(); // if set
     }
   } else {
-    assert(!gch->incremental_collection_failed(), "Error");
+#ifdef ASSERT
+    // It is possible that incremental_collection_failed() == true
+    // here, because an attempted scavenge did not succeed. The policy
+    // is normally expected to cause a full collection which should
+    // clear that condition, so we should not be here twice in a row
+    // with incremental_collection_failed() == true without having done
+    // a full collection in between.
+    if (!seen_incremental_collection_failed &&
+        gch->incremental_collection_failed()) {
+      seen_incremental_collection_failed = true;
+    } else if (seen_incremental_collection_failed) {
+      assert(!gch->incremental_collection_failed(), "Twice in a row");
+      seen_incremental_collection_failed = false;
+    }
+#endif // ASSERT
   }
 
   if (ZapUnusedHeapArea) {

From 308952f81ee9e470f8a6c3543044bc96e04d2839 Mon Sep 17 00:00:00 2001
From: Igor Veresov <iveresov@openjdk.org>
Date: Tue, 2 Nov 2010 16:02:46 -0700
Subject: [PATCH 20/23] 6996136: VM crash in
 src/share/vm/runtime/virtualspace.cpp:424

Turn CDS off if compressed oops is on

Reviewed-by: ysr, kvn, jcoomes, phh
---
 hotspot/src/share/vm/runtime/arguments.cpp | 84 ++++++++++++++--------
 hotspot/src/share/vm/runtime/arguments.hpp |  2 +
 2 files changed, 57 insertions(+), 29 deletions(-)

diff --git a/hotspot/src/share/vm/runtime/arguments.cpp b/hotspot/src/share/vm/runtime/arguments.cpp
index a41abe26e72..b5704d32f3c 100644
--- a/hotspot/src/share/vm/runtime/arguments.cpp
+++ b/hotspot/src/share/vm/runtime/arguments.cpp
@@ -957,26 +957,65 @@ static void no_shared_spaces() {
   }
 }
 
+void Arguments::check_compressed_oops_compat() {
+#ifdef _LP64
+  assert(UseCompressedOops, "Precondition");
+#  if defined(COMPILER1) && !defined(TIERED)
+  // Until c1 supports compressed oops turn them off.
+  FLAG_SET_DEFAULT(UseCompressedOops, false);
+#  else
+  // Is it on by default or set on ergonomically
+  bool is_on_by_default = FLAG_IS_DEFAULT(UseCompressedOops) || FLAG_IS_ERGO(UseCompressedOops);
+
+  // Tiered currently doesn't work with compressed oops
+  if (TieredCompilation) {
+    if (is_on_by_default) {
+      FLAG_SET_DEFAULT(UseCompressedOops, false);
+      return;
+    } else {
+      vm_exit_during_initialization(
+        "Tiered compilation is not supported with compressed oops yet", NULL);
+    }
+  }
+
+  // XXX JSR 292 currently does not support compressed oops
+  if (EnableMethodHandles) {
+    if (is_on_by_default) {
+      FLAG_SET_DEFAULT(UseCompressedOops, false);
+      return;
+    } else {
+      vm_exit_during_initialization(
+        "JSR292 is not supported with compressed oops yet", NULL);
+    }
+  }
+
+  // If dumping an archive or forcing its use, disable compressed oops if possible
+  if (DumpSharedSpaces || RequireSharedSpaces) {
+    if (is_on_by_default) {
+      FLAG_SET_DEFAULT(UseCompressedOops, false);
+      return;
+    } else {
+      vm_exit_during_initialization(
+        "Class Data Sharing is not supported with compressed oops yet", NULL);
+    }
+  } else if (UseSharedSpaces) {
+    // UseSharedSpaces is on by default. With compressed oops, we turn it off.
+    FLAG_SET_DEFAULT(UseSharedSpaces, false);
+  }
+
+#  endif // defined(COMPILER1) && !defined(TIERED)
+#endif // _LP64
+}
+
 void Arguments::set_tiered_flags() {
   if (FLAG_IS_DEFAULT(CompilationPolicyChoice)) {
     FLAG_SET_DEFAULT(CompilationPolicyChoice, 2);
   }
-
   if (CompilationPolicyChoice < 2) {
     vm_exit_during_initialization(
       "Incompatible compilation policy selected", NULL);
   }
-
-#ifdef _LP64
-  if (FLAG_IS_DEFAULT(UseCompressedOops) || FLAG_IS_ERGO(UseCompressedOops)) {
-    UseCompressedOops = false;
-  }
-  if (UseCompressedOops) {
-    vm_exit_during_initialization(
-      "Tiered compilation is not supported with compressed oops yet", NULL);
-  }
-#endif
- // Increase the code cache size - tiered compiles a lot more.
+  // Increase the code cache size - tiered compiles a lot more.
   if (FLAG_IS_DEFAULT(ReservedCodeCacheSize)) {
     FLAG_SET_DEFAULT(ReservedCodeCacheSize, ReservedCodeCacheSize * 2);
   }
@@ -2835,6 +2874,7 @@ jint Arguments::parse_options_environment_variable(const char* name, SysClassPat
   return JNI_OK;
 }
 
+
 // Parse entry point called from JNI_CreateJavaVM
 
 jint Arguments::parse(const JavaVMInitArgs* args) {
@@ -2977,17 +3017,6 @@ jint Arguments::parse(const JavaVMInitArgs* args) {
     PrintGC = true;
   }
 
-#if defined(_LP64) && defined(COMPILER1) && !defined(TIERED)
-  UseCompressedOops = false;
-#endif
-
-#if defined(_LP64)
-  if ((DumpSharedSpaces || RequireSharedSpaces) && UseCompressedOops) {
-    // Disable compressed oops with shared spaces
-    UseCompressedOops = false;
-  }
-#endif
-
   // Set object alignment values.
   set_object_alignment();
 
@@ -3002,13 +3031,10 @@ jint Arguments::parse(const JavaVMInitArgs* args) {
   set_ergonomics_flags();
 
 #ifdef _LP64
-  // XXX JSR 292 currently does not support compressed oops.
-  if (EnableMethodHandles && UseCompressedOops) {
-    if (FLAG_IS_DEFAULT(UseCompressedOops) || FLAG_IS_ERGO(UseCompressedOops)) {
-      UseCompressedOops = false;
-    }
+  if (UseCompressedOops) {
+    check_compressed_oops_compat();
   }
-#endif // _LP64
+#endif
 
   // Check the GC selections again.
   if (!check_gc_consistency()) {
diff --git a/hotspot/src/share/vm/runtime/arguments.hpp b/hotspot/src/share/vm/runtime/arguments.hpp
index 2d278e18741..8229584ca3f 100644
--- a/hotspot/src/share/vm/runtime/arguments.hpp
+++ b/hotspot/src/share/vm/runtime/arguments.hpp
@@ -291,6 +291,8 @@ class Arguments : AllStatic {
 
   // Tiered
   static void set_tiered_flags();
+  // Check compressed oops compatibility with other flags
+  static void check_compressed_oops_compat();
   // CMS/ParNew garbage collectors
   static void set_parnew_gc_flags();
   static void set_cms_and_parnew_gc_flags();

From d6009ec7714f33c1668621f21d7551a936852d92 Mon Sep 17 00:00:00 2001
From: Erik Trimble <trims@openjdk.org>
Date: Thu, 4 Nov 2010 15:19:16 -0700
Subject: [PATCH 21/23] Added tag hs20-b02 for changeset 9353ae6eae7d

---
 hotspot/.hgtags | 1 +
 1 file changed, 1 insertion(+)

diff --git a/hotspot/.hgtags b/hotspot/.hgtags
index f26dd5097cb..bd02b5f617a 100644
--- a/hotspot/.hgtags
+++ b/hotspot/.hgtags
@@ -127,3 +127,4 @@ beef35b96b81129c375d572357fb9548d9020db1 jdk7-b113
 5511edd5d719f3fc9fdd04879482026a3d2c8652 hs20-b01
 bdbc48857210a509b3c50a3291ecb9dd6a72e016 jdk7-b115
 96b3f2a7add0b445b8aa421f6823cff5a2e2fe03 jdk7-b116
+52f19c724d9634af79044a2e0defbe4a5f1adbda hs20-b02

From 16225759ef39f39cae72051a976a3d2e6230409a Mon Sep 17 00:00:00 2001
From: Christine Lu <cl@openjdk.org>
Date: Thu, 4 Nov 2010 15:54:14 -0700
Subject: [PATCH 22/23] Added tag jdk7-b117 for changeset 0a086f450822

---
 .hgtags-top-repo | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.hgtags-top-repo b/.hgtags-top-repo
index e310c504710..333d1450108 100644
--- a/.hgtags-top-repo
+++ b/.hgtags-top-repo
@@ -91,3 +91,4 @@ c1df968c4527bfab5f97662a89245f15d12d378b jdk7-b113
 27985a5c6e5268014d25d55886e0ecb96af4763d jdk7-b114
 e8ebdf41b9c01a26642848f4134f5504e8fb3233 jdk7-b115
 94e9a1bfba8b8d1fe0bfd43b88629b1f27b02a76 jdk7-b116
+7220e60b097fa027e922f1aeecdd330f3e37409f jdk7-b117

From 59efb01f6ad63ee993669e95b4ce02d4919a4510 Mon Sep 17 00:00:00 2001
From: Christine Lu <cl@openjdk.org>
Date: Thu, 4 Nov 2010 15:54:20 -0700
Subject: [PATCH 23/23] Added tag jdk7-b117 for changeset fa3a454c72a6

---
 hotspot/.hgtags | 1 +
 1 file changed, 1 insertion(+)

diff --git a/hotspot/.hgtags b/hotspot/.hgtags
index f26dd5097cb..4abe098eb32 100644
--- a/hotspot/.hgtags
+++ b/hotspot/.hgtags
@@ -127,3 +127,4 @@ beef35b96b81129c375d572357fb9548d9020db1 jdk7-b113
 5511edd5d719f3fc9fdd04879482026a3d2c8652 hs20-b01
 bdbc48857210a509b3c50a3291ecb9dd6a72e016 jdk7-b115
 96b3f2a7add0b445b8aa421f6823cff5a2e2fe03 jdk7-b116
+806d0c037e6bbb88dac0699673f4ba55ee8c02da jdk7-b117