Merge

hotspot/src/os/solaris/vm/os_solaris.cpp

@@ -122,6 +122,13 @@ struct memcntl_mha {
 # define  MADV_ACCESS_MANY        8       /* many processes to access heavily */
 #endif
 
+#ifndef LGRP_RSRC_CPU
+# define LGRP_RSRC_CPU           0       /* CPU resources */
+#endif
+#ifndef LGRP_RSRC_MEM
+# define LGRP_RSRC_MEM           1       /* memory resources */
+#endif
+
 // Some more macros from sys/mman.h that are not present in Solaris 8.
 
 #ifndef MAX_MEMINFO_CNT
@@ -2640,8 +2647,13 @@ size_t os::numa_get_leaf_groups(int *ids, size_t size) {
        return 1;
      }
      if (!r) {
+       // That's a leaf node.
        assert (bottom <= cur, "Sanity check");
-       ids[bottom++] = ids[cur];
+       // Check if the node has memory
+       if (Solaris::lgrp_resources(Solaris::lgrp_cookie(), ids[cur],
+                                   NULL, 0, LGRP_RSRC_MEM) > 0) {
+         ids[bottom++] = ids[cur];
+       }
      }
      top += r;
      cur++;
@@ -2664,11 +2676,20 @@ bool os::numa_topology_changed() {
 
 // Get the group id of the current LWP.
 int os::numa_get_group_id() {
-  int lgrp_id = os::Solaris::lgrp_home(P_LWPID, P_MYID);
+  int lgrp_id = Solaris::lgrp_home(P_LWPID, P_MYID);
   if (lgrp_id == -1) {
     return 0;
   }
-  return lgrp_id;
+  const int size = os::numa_get_groups_num();
+  int *ids = (int*)alloca(size * sizeof(int));
+
+  // Get the ids of all lgroups with memory; r is the count.
+  int r = Solaris::lgrp_resources(Solaris::lgrp_cookie(), lgrp_id,
+                                  (Solaris::lgrp_id_t*)ids, size, LGRP_RSRC_MEM);
+  if (r <= 0) {
+    return 0;
+  }
+  return ids[os::random() % r];
 }
 
 // Request information about the page.
@@ -4353,6 +4374,7 @@ os::Solaris::lgrp_init_func_t os::Solaris::_lgrp_init;
 os::Solaris::lgrp_fini_func_t os::Solaris::_lgrp_fini;
 os::Solaris::lgrp_root_func_t os::Solaris::_lgrp_root;
 os::Solaris::lgrp_children_func_t os::Solaris::_lgrp_children;
+os::Solaris::lgrp_resources_func_t os::Solaris::_lgrp_resources;
 os::Solaris::lgrp_nlgrps_func_t os::Solaris::_lgrp_nlgrps;
 os::Solaris::lgrp_cookie_stale_func_t os::Solaris::_lgrp_cookie_stale;
 os::Solaris::lgrp_cookie_t os::Solaris::_lgrp_cookie = 0;
@@ -4555,6 +4577,7 @@ void os::Solaris::liblgrp_init() {
     os::Solaris::set_lgrp_fini(CAST_TO_FN_PTR(lgrp_fini_func_t, dlsym(handle, "lgrp_fini")));
     os::Solaris::set_lgrp_root(CAST_TO_FN_PTR(lgrp_root_func_t, dlsym(handle, "lgrp_root")));
     os::Solaris::set_lgrp_children(CAST_TO_FN_PTR(lgrp_children_func_t, dlsym(handle, "lgrp_children")));
+    os::Solaris::set_lgrp_resources(CAST_TO_FN_PTR(lgrp_resources_func_t, dlsym(handle, "lgrp_resources")));
     os::Solaris::set_lgrp_nlgrps(CAST_TO_FN_PTR(lgrp_nlgrps_func_t, dlsym(handle, "lgrp_nlgrps")));
     os::Solaris::set_lgrp_cookie_stale(CAST_TO_FN_PTR(lgrp_cookie_stale_func_t,
                                        dlsym(handle, "lgrp_cookie_stale")));

hotspot/src/os/solaris/vm/os_solaris.hpp

@@ -66,6 +66,7 @@ class Solaris {
 
   typedef uintptr_t       lgrp_cookie_t;
   typedef id_t            lgrp_id_t;
+  typedef int             lgrp_rsrc_t;
   typedef enum lgrp_view {
         LGRP_VIEW_CALLER,       /* what's available to the caller */
         LGRP_VIEW_OS            /* what's available to operating system */
@@ -77,6 +78,9 @@ class Solaris {
   typedef lgrp_id_t (*lgrp_root_func_t)(lgrp_cookie_t cookie);
   typedef int (*lgrp_children_func_t)(lgrp_cookie_t  cookie,  lgrp_id_t  parent,
                                       lgrp_id_t *lgrp_array, uint_t lgrp_array_size);
+  typedef int (*lgrp_resources_func_t)(lgrp_cookie_t  cookie,  lgrp_id_t  lgrp,
+                                      lgrp_id_t *lgrp_array, uint_t lgrp_array_size,
+                                      lgrp_rsrc_t type);
   typedef int (*lgrp_nlgrps_func_t)(lgrp_cookie_t cookie);
   typedef int (*lgrp_cookie_stale_func_t)(lgrp_cookie_t cookie);
   typedef int (*meminfo_func_t)(const uint64_t inaddr[],   int addr_count,
@@ -88,6 +92,7 @@ class Solaris {
   static lgrp_fini_func_t _lgrp_fini;
   static lgrp_root_func_t _lgrp_root;
   static lgrp_children_func_t _lgrp_children;
+  static lgrp_resources_func_t _lgrp_resources;
   static lgrp_nlgrps_func_t _lgrp_nlgrps;
   static lgrp_cookie_stale_func_t _lgrp_cookie_stale;
   static lgrp_cookie_t _lgrp_cookie;
@@ -109,7 +114,6 @@ class Solaris {
   static int (*get_libjsig_version)();
   static void save_preinstalled_handler(int, struct sigaction&);
   static void check_signal_handler(int sig);
-
   // For overridable signals
   static int _SIGinterrupt;                  // user-overridable INTERRUPT_SIGNAL
   static int _SIGasync;                      // user-overridable ASYNC_SIGNAL
@@ -253,8 +257,9 @@ class Solaris {
   static void set_lgrp_init(lgrp_init_func_t func) { _lgrp_init = func; }
   static void set_lgrp_fini(lgrp_fini_func_t func) { _lgrp_fini = func; }
   static void set_lgrp_root(lgrp_root_func_t func) { _lgrp_root = func; }
-  static void set_lgrp_children(lgrp_children_func_t func) { _lgrp_children = func; }
+  static void set_lgrp_children(lgrp_children_func_t func)   { _lgrp_children = func; }
-  static void set_lgrp_nlgrps(lgrp_nlgrps_func_t func)     { _lgrp_nlgrps = func; }
+  static void set_lgrp_resources(lgrp_resources_func_t func) { _lgrp_resources = func; }
+  static void set_lgrp_nlgrps(lgrp_nlgrps_func_t func)       { _lgrp_nlgrps = func; }
   static void set_lgrp_cookie_stale(lgrp_cookie_stale_func_t func) { _lgrp_cookie_stale = func; }
   static void set_lgrp_cookie(lgrp_cookie_t cookie)  { _lgrp_cookie = cookie; }
 
@@ -266,6 +271,12 @@ class Solaris {
                     lgrp_id_t *lgrp_array, uint_t lgrp_array_size) {
     return _lgrp_children != NULL ? _lgrp_children(cookie, parent, lgrp_array, lgrp_array_size) : -1;
   }
+  static int lgrp_resources(lgrp_cookie_t  cookie,  lgrp_id_t  lgrp,
+                            lgrp_id_t *lgrp_array, uint_t lgrp_array_size,
+                            lgrp_rsrc_t type) {
+    return _lgrp_resources != NULL ? _lgrp_resources(cookie, lgrp, lgrp_array, lgrp_array_size, type) : -1;
+  }
+
   static int lgrp_nlgrps(lgrp_cookie_t cookie)       { return _lgrp_nlgrps != NULL ? _lgrp_nlgrps(cookie) : -1; }
   static int lgrp_cookie_stale(lgrp_cookie_t cookie) {
     return _lgrp_cookie_stale != NULL ? _lgrp_cookie_stale(cookie) : -1;

hotspot/src/share/vm/gc_implementation/concurrentMarkSweep/cmsOopClosures.hpp

@@ -329,7 +329,7 @@ class Par_PushOrMarkClosure: public OopClosure {
 class CMSKeepAliveClosure: public OopClosure {
  private:
   CMSCollector* _collector;
-  MemRegion     _span;
+  const MemRegion _span;
   CMSMarkStack* _mark_stack;
   CMSBitMap*    _bit_map;
  protected:
@@ -340,7 +340,9 @@ class CMSKeepAliveClosure: public OopClosure {
     _collector(collector),
     _span(span),
     _bit_map(bit_map),
-    _mark_stack(mark_stack) { }
+    _mark_stack(mark_stack) {
+    assert(!_span.is_empty(), "Empty span could spell trouble");
+  }
   virtual void do_oop(oop* p);
   virtual void do_oop(narrowOop* p);
   inline void do_oop_nv(oop* p)       { CMSKeepAliveClosure::do_oop_work(p); }

hotspot/src/share/vm/gc_implementation/concurrentMarkSweep/concurrentMarkSweepGeneration.cpp

@@ -520,7 +520,10 @@ CMSCollector::CMSCollector(ConcurrentMarkSweepGeneration* cmsGen,
                  -1 /* lock-free */, "No_lock" /* dummy */),
   _modUnionClosure(&_modUnionTable),
   _modUnionClosurePar(&_modUnionTable),
-  _is_alive_closure(&_markBitMap),
+  // Adjust my span to cover old (cms) gen and perm gen
+  _span(cmsGen->reserved()._union(permGen->reserved())),
+  // Construct the is_alive_closure with _span & markBitMap
+  _is_alive_closure(_span, &_markBitMap),
   _restart_addr(NULL),
   _overflow_list(NULL),
   _preserved_oop_stack(NULL),
@@ -572,11 +575,6 @@ CMSCollector::CMSCollector(ConcurrentMarkSweepGeneration* cmsGen,
   _cmsGen->cmsSpace()->set_collector(this);
   _permGen->cmsSpace()->set_collector(this);
 
-  // Adjust my span to cover old (cms) gen and perm gen
-  _span = _cmsGen->reserved()._union(_permGen->reserved());
-  // Initialize the span of is_alive_closure
-  _is_alive_closure.set_span(_span);
-
   // Allocate MUT and marking bit map
   {
     MutexLockerEx x(_markBitMap.lock(), Mutex::_no_safepoint_check_flag);
@@ -5496,7 +5494,7 @@ class CMSRefProcTaskProxy: public AbstractGangTask {
   typedef AbstractRefProcTaskExecutor::ProcessTask ProcessTask;
   CMSCollector*          _collector;
   CMSBitMap*             _mark_bit_map;
-  MemRegion              _span;
+  const MemRegion        _span;
   OopTaskQueueSet*       _task_queues;
   ParallelTaskTerminator _term;
   ProcessTask&           _task;
@@ -5513,7 +5511,10 @@ public:
     _collector(collector), _span(span), _mark_bit_map(mark_bit_map),
     _task_queues(task_queues),
     _term(total_workers, task_queues)
-    { }
+    {
+      assert(_collector->_span.equals(_span) && !_span.is_empty(),
+             "Inconsistency in _span");
+    }
 
   OopTaskQueueSet* task_queues() { return _task_queues; }
 
@@ -5530,11 +5531,12 @@ public:
 };
 
 void CMSRefProcTaskProxy::work(int i) {
+  assert(_collector->_span.equals(_span), "Inconsistency in _span");
   CMSParKeepAliveClosure par_keep_alive(_collector, _span,
                                         _mark_bit_map, work_queue(i));
   CMSParDrainMarkingStackClosure par_drain_stack(_collector, _span,
                                                  _mark_bit_map, work_queue(i));
-  CMSIsAliveClosure is_alive_closure(_mark_bit_map);
+  CMSIsAliveClosure is_alive_closure(_span, _mark_bit_map);
   _task.work(i, is_alive_closure, par_keep_alive, par_drain_stack);
   if (_task.marks_oops_alive()) {
     do_work_steal(i, &par_drain_stack, &par_keep_alive,

hotspot/src/share/vm/gc_implementation/concurrentMarkSweep/concurrentMarkSweepGeneration.hpp

@@ -435,23 +435,22 @@ class CMSStats VALUE_OBJ_CLASS_SPEC {
 // if the object is "live" (reachable). Used in weak
 // reference processing.
 class CMSIsAliveClosure: public BoolObjectClosure {
-  MemRegion  _span;
+  const MemRegion  _span;
   const CMSBitMap* _bit_map;
 
   friend class CMSCollector;
- protected:
-  void set_span(MemRegion span) { _span = span; }
  public:
-  CMSIsAliveClosure(CMSBitMap* bit_map):
-    _bit_map(bit_map) { }
-
   CMSIsAliveClosure(MemRegion span,
                     CMSBitMap* bit_map):
     _span(span),
-    _bit_map(bit_map) { }
+    _bit_map(bit_map) {
+    assert(!span.is_empty(), "Empty span could spell trouble");
+  }
+
   void do_object(oop obj) {
     assert(false, "not to be invoked");
   }
+
   bool do_object_b(oop obj);
 };
 
@@ -600,7 +599,7 @@ class CMSCollector: public CHeapObj {
   // ("Weak") Reference processing support
   ReferenceProcessor*            _ref_processor;
   CMSIsAliveClosure              _is_alive_closure;
-      // keep this textually after _markBitMap; c'tor dependency
+      // keep this textually after _markBitMap and _span; c'tor dependency
 
   ConcurrentMarkSweepThread*     _cmsThread;   // the thread doing the work
   ModUnionClosure    _modUnionClosure;

hotspot/src/share/vm/gc_implementation/shared/mutableNUMASpace.cpp

@@ -62,10 +62,10 @@ void MutableNUMASpace::ensure_parsability() {
   for (int i = 0; i < lgrp_spaces()->length(); i++) {
     LGRPSpace *ls = lgrp_spaces()->at(i);
     MutableSpace *s = ls->space();
-    if (!s->contains(top())) {
+    if (s->top() < top()) { // For all spaces preceeding the one containing top()
       if (s->free_in_words() > 0) {
         SharedHeap::fill_region_with_object(MemRegion(s->top(), s->end()));
-        size_t area_touched_words = pointer_delta(s->end(), s->top(), sizeof(HeapWordSize));
+        size_t area_touched_words = pointer_delta(s->end(), s->top());
 #ifndef ASSERT
         if (!ZapUnusedHeapArea) {
           area_touched_words = MIN2((size_t)align_object_size(typeArrayOopDesc::header_size(T_INT)),
@@ -88,7 +88,6 @@ void MutableNUMASpace::ensure_parsability() {
 
           ls->add_invalid_region(invalid);
         }
-        s->set_top(s->end());
       }
     } else {
       if (!os::numa_has_static_binding()) {
@@ -99,8 +98,12 @@ void MutableNUMASpace::ensure_parsability() {
         if (ZapUnusedHeapArea) {
           MemRegion invalid(s->top(), s->end());
           ls->add_invalid_region(invalid);
-        } else break;
+        } else {
+          return;
+        }
 #endif
+      } else {
+          return;
       }
     }
   }
@@ -658,9 +661,12 @@ HeapWord* MutableNUMASpace::allocate(size_t size) {
   MutableSpace *s = lgrp_spaces()->at(i)->space();
   HeapWord *p = s->allocate(size);
 
-  if (p != NULL && s->free_in_words() < (size_t)oopDesc::header_size()) {
+  if (p != NULL) {
-    s->set_top(s->top() - size);
+    size_t remainder = s->free_in_words();
-    p = NULL;
+    if (remainder < (size_t)oopDesc::header_size() && remainder > 0) {
+      s->set_top(s->top() - size);
+      p = NULL;
+    }
   }
   if (p != NULL) {
     if (top() < s->top()) { // Keep _top updated.
@@ -693,11 +699,14 @@ HeapWord* MutableNUMASpace::cas_allocate(size_t size) {
   }
   MutableSpace *s = lgrp_spaces()->at(i)->space();
   HeapWord *p = s->cas_allocate(size);
-  if (p != NULL && s->free_in_words() < (size_t)oopDesc::header_size()) {
+  if (p != NULL) {
-    if (s->cas_deallocate(p, size)) {
+    size_t remainder = pointer_delta(s->end(), p);
-      // We were the last to allocate and created a fragment less than
+    if (remainder < (size_t)oopDesc::header_size() && remainder > 0) {
-      // a minimal object.
+      if (s->cas_deallocate(p, size)) {
-      p = NULL;
+        // We were the last to allocate and created a fragment less than
+        // a minimal object.
+        p = NULL;
+      }
     }
   }
   if (p != NULL) {
@@ -738,6 +747,9 @@ void MutableNUMASpace::print_on(outputStream* st) const {
     st->print("    lgrp %d", ls->lgrp_id());
     ls->space()->print_on(st);
     if (NUMAStats) {
+      for (int i = 0; i < lgrp_spaces()->length(); i++) {
+        lgrp_spaces()->at(i)->accumulate_statistics(page_size());
+      }
       st->print("    local/remote/unbiased/uncommitted: %dK/%dK/%dK/%dK, large/small pages: %d/%d\n",
                 ls->space_stats()->_local_space / K,
                 ls->space_stats()->_remote_space / K,