Merge

hotspot/src/share/vm/code/codeCache.cpp

@@ -637,16 +637,19 @@ void CodeCache::blobs_do(CodeBlobClosure* f) {
 }
 
 // Walk the list of methods which might contain non-perm oops.
-void CodeCache::scavenge_root_nmethods_do(CodeBlobClosure* f) {
+void CodeCache::scavenge_root_nmethods_do(CodeBlobToOopClosure* f) {
   assert_locked_or_safepoint(CodeCache_lock);
 
   if (UseG1GC) {
     return;
   }
 
+  const bool fix_relocations = f->fix_relocations();
   debug_only(mark_scavenge_root_nmethods());
 
-  for (nmethod* cur = scavenge_root_nmethods(); cur != NULL; cur = cur->scavenge_root_link()) {
+  nmethod* prev = NULL;
+  nmethod* cur = scavenge_root_nmethods();
+  while (cur != NULL) {
     debug_only(cur->clear_scavenge_root_marked());
     assert(cur->scavenge_root_not_marked(), "");
     assert(cur->on_scavenge_root_list(), "else shouldn't be on this list");
@@ -659,6 +662,18 @@ void CodeCache::scavenge_root_nmethods_do(CodeBlobClosure* f) {
       // Perform cur->oops_do(f), maybe just once per nmethod.
       f->do_code_blob(cur);
     }
+    nmethod* const next = cur->scavenge_root_link();
+    // The scavengable nmethod list must contain all methods with scavengable
+    // oops. It is safe to include more nmethod on the list, but we do not
+    // expect any live non-scavengable nmethods on the list.
+    if (fix_relocations) {
+      if (!is_live || !cur->detect_scavenge_root_oops()) {
+        unlink_scavenge_root_nmethod(cur, prev);
+      } else {
+        prev = cur;
+      }
+    }
+    cur = next;
   }
 
   // Check for stray marks.
@@ -678,6 +693,24 @@ void CodeCache::add_scavenge_root_nmethod(nmethod* nm) {
   print_trace("add_scavenge_root", nm);
 }
 
+void CodeCache::unlink_scavenge_root_nmethod(nmethod* nm, nmethod* prev) {
+  assert_locked_or_safepoint(CodeCache_lock);
+
+  assert((prev == NULL && scavenge_root_nmethods() == nm) ||
+         (prev != NULL && prev->scavenge_root_link() == nm), "precondition");
+
+  assert(!UseG1GC, "G1 does not use the scavenge_root_nmethods list");
+
+  print_trace("unlink_scavenge_root", nm);
+  if (prev == NULL) {
+    set_scavenge_root_nmethods(nm->scavenge_root_link());
+  } else {
+    prev->set_scavenge_root_link(nm->scavenge_root_link());
+  }
+  nm->set_scavenge_root_link(NULL);
+  nm->clear_on_scavenge_root_list();
+}
+
 void CodeCache::drop_scavenge_root_nmethod(nmethod* nm) {
   assert_locked_or_safepoint(CodeCache_lock);
 
@@ -686,20 +719,13 @@ void CodeCache::drop_scavenge_root_nmethod(nmethod* nm) {
   }
 
   print_trace("drop_scavenge_root", nm);
-  nmethod* last = NULL;
-  nmethod* cur = scavenge_root_nmethods();
-  while (cur != NULL) {
-    nmethod* next = cur->scavenge_root_link();
+  nmethod* prev = NULL;
+  for (nmethod* cur = scavenge_root_nmethods(); cur != NULL; cur = cur->scavenge_root_link()) {
     if (cur == nm) {
-      if (last != NULL)
-            last->set_scavenge_root_link(next);
-      else  set_scavenge_root_nmethods(next);
-      nm->set_scavenge_root_link(NULL);
-      nm->clear_on_scavenge_root_list();
+      unlink_scavenge_root_nmethod(cur, prev);
       return;
     }
-    last = cur;
-    cur = next;
+    prev = cur;
   }
   assert(false, "should have been on list");
 }
@@ -728,11 +754,7 @@ void CodeCache::prune_scavenge_root_nmethods() {
     } else {
       // Prune it from the list, so we don't have to look at it any more.
       print_trace("prune_scavenge_root", cur);
-      cur->set_scavenge_root_link(NULL);
-      cur->clear_on_scavenge_root_list();
-      if (last != NULL)
-            last->set_scavenge_root_link(next);
-      else  set_scavenge_root_nmethods(next);
+      unlink_scavenge_root_nmethod(cur, last);
     }
     cur = next;
   }

hotspot/src/share/vm/code/codeCache.hpp

@@ -116,6 +116,10 @@ class CodeCache : AllStatic {
   static int    allocated_segments();
   static size_t freelists_length();
 
+  static void set_scavenge_root_nmethods(nmethod* nm) { _scavenge_root_nmethods = nm; }
+  static void prune_scavenge_root_nmethods();
+  static void unlink_scavenge_root_nmethod(nmethod* nm, nmethod* prev);
+
  public:
   // Initialization
   static void initialize();
@@ -153,13 +157,17 @@ class CodeCache : AllStatic {
   // to "true" iff some code got unloaded.
   static void do_unloading(BoolObjectClosure* is_alive, bool unloading_occurred);
   static void asserted_non_scavengable_nmethods_do(CodeBlobClosure* f = NULL) PRODUCT_RETURN;
-  static void scavenge_root_nmethods_do(CodeBlobClosure* f);
+
+  // Apply f to every live code blob in scavengable nmethods. Prune nmethods
+  // from the list of scavengable nmethods if f->fix_relocations() and a nmethod
+  // no longer has scavengable oops.  If f->fix_relocations(), then f must copy
+  // objects to their new location immediately to avoid fixing nmethods on the
+  // basis of the old object locations.
+  static void scavenge_root_nmethods_do(CodeBlobToOopClosure* f);
 
   static nmethod* scavenge_root_nmethods()            { return _scavenge_root_nmethods; }
-  static void set_scavenge_root_nmethods(nmethod* nm) { _scavenge_root_nmethods = nm; }
   static void add_scavenge_root_nmethod(nmethod* nm);
   static void drop_scavenge_root_nmethod(nmethod* nm);
-  static void prune_scavenge_root_nmethods();
 
   // Printing/debugging
   static void print();                           // prints summary

hotspot/src/share/vm/code/nmethod.cpp

@@ -1381,7 +1381,6 @@ void nmethod::make_unloaded(BoolObjectClosure* is_alive, oop cause) {
   assert(_method == NULL, "Tautology");
 
   set_osr_link(NULL);
-  //set_scavenge_root_link(NULL); // done by prune_scavenge_root_nmethods
   NMethodSweeper::report_state_change(this);
 }
 

hotspot/src/share/vm/gc/parallel/psScavenge.cpp

@@ -600,12 +600,6 @@ bool PSScavenge::invoke_no_policy() {
 
     NOT_PRODUCT(reference_processor()->verify_no_references_recorded());
 
-    {
-      GCTraceTime(Debug, gc, phases) tm("Prune Scavenge Root Methods", &_gc_timer);
-
-      CodeCache::prune_scavenge_root_nmethods();
-    }
-
     // Re-verify object start arrays
     if (VerifyObjectStartArray &&
         VerifyAfterGC) {

hotspot/src/share/vm/gc/shared/genCollectedHeap.cpp

@@ -561,7 +561,7 @@ void GenCollectedHeap::process_roots(StrongRootsScope* scope,
                                      OopClosure* weak_roots,
                                      CLDClosure* strong_cld_closure,
                                      CLDClosure* weak_cld_closure,
-                                     CodeBlobClosure* code_roots) {
+                                     CodeBlobToOopClosure* code_roots) {
   // General roots.
   assert(Threads::thread_claim_parity() != 0, "must have called prologue code");
   assert(code_roots != NULL, "code root closure should always be set");
@@ -578,7 +578,7 @@ void GenCollectedHeap::process_roots(StrongRootsScope* scope,
   // Don't process them if they will be processed during the ClassLoaderDataGraph phase.
   CLDClosure* roots_from_clds_p = (strong_cld_closure != weak_cld_closure) ? strong_cld_closure : NULL;
   // Only process code roots from thread stacks if we aren't visiting the entire CodeCache anyway
-  CodeBlobClosure* roots_from_code_p = (so & SO_AllCodeCache) ? NULL : code_roots;
+  CodeBlobToOopClosure* roots_from_code_p = (so & SO_AllCodeCache) ? NULL : code_roots;
 
   bool is_par = scope->n_threads() > 1;
   Threads::possibly_parallel_oops_do(is_par, strong_roots, roots_from_clds_p, roots_from_code_p);

hotspot/src/share/vm/gc/shared/genCollectedHeap.hpp

@@ -399,7 +399,7 @@ public:
                      OopClosure* weak_roots,
                      CLDClosure* strong_cld_closure,
                      CLDClosure* weak_cld_closure,
-                     CodeBlobClosure* code_roots);
+                     CodeBlobToOopClosure* code_roots);
 
  public:
   static const bool StrongAndWeakRoots = false;

hotspot/src/share/vm/memory/iterator.hpp

@@ -285,9 +285,12 @@ class CodeBlobToOopClosure : public CodeBlobClosure {
  protected:
   void do_nmethod(nmethod* nm);
  public:
+  // If fix_relocations(), then cl must copy objects to their new location immediately to avoid
+  // patching nmethods with the old locations.
   CodeBlobToOopClosure(OopClosure* cl, bool fix_relocations) : _cl(cl), _fix_relocations(fix_relocations) {}
   virtual void do_code_blob(CodeBlob* cb);
 
+  bool fix_relocations() const { return _fix_relocations; }
   const static bool FixRelocations = true;
 };