6819061: G1: eliminate serial Other times that are proportional to the collection set length

6871109: G1: remove the concept of the scan only prefix

Removed scan only regions and associated code. The young portion of the collection set is now constructed incrementally - when a young region is retired as the current allocation region it is added to the collection set.

Reviewed-by: apetrusenko, iveresov, tonyp

hotspot/src/share/vm/gc_implementation/g1/concurrentG1RefineThread.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright 2001-2009 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2001-2010 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -69,9 +69,9 @@ void ConcurrentG1RefineThread::sample_young_list_rs_lengths() {
   G1CollectorPolicy* g1p = g1h->g1_policy();
   if (g1p->adaptive_young_list_length()) {
     int regions_visited = 0;
-    g1h->young_list_rs_length_sampling_init();
+    g1h->young_list()->rs_length_sampling_init();
-    while (g1h->young_list_rs_length_sampling_more()) {
+    while (g1h->young_list()->rs_length_sampling_more()) {
-      g1h->young_list_rs_length_sampling_next();
+      g1h->young_list()->rs_length_sampling_next();
       ++regions_visited;
 
       // we try to yield every time we visit 10 regions
@@ -162,6 +162,7 @@ void ConcurrentG1RefineThread::run() {
   if (_worker_id >= cg1r()->worker_thread_num()) {
     run_young_rs_sampling();
     terminate();
+    return;
   }
 
   _vtime_start = os::elapsedVTime();

hotspot/src/share/vm/gc_implementation/g1/concurrentMark.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright 2001-2009 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2001-2010 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -733,6 +733,19 @@ public:
   // to determine whether any heap regions are located above the finger.
   void registerCSetRegion(HeapRegion* hr);
 
+  // Registers the maximum region-end associated with a set of
+  // regions with CM. Again this is used to determine whether any
+  // heap regions are located above the finger.
+  void register_collection_set_finger(HeapWord* max_finger) {
+    // max_finger is the highest heap region end of the regions currently
+    // contained in the collection set. If this value is larger than
+    // _min_finger then we need to gray objects.
+    // This routine is like registerCSetRegion but for an entire
+    // collection of regions.
+    if (max_finger > _min_finger)
+      _should_gray_objects = true;
+  }
+
   // Returns "true" if at least one mark has been completed.
   bool at_least_one_mark_complete() { return _at_least_one_mark_complete; }
 

hotspot/src/share/vm/gc_implementation/g1/g1CollectedHeap.cpp

@@ -30,7 +30,7 @@ size_t G1CollectedHeap::_humongous_object_threshold_in_words = 0;
 // turn it on so that the contents of the young list (scan-only /
 // to-be-collected) are printed at "strategic" points before / during
 // / after the collection --- this is useful for debugging
-#define SCAN_ONLY_VERBOSE 0
+#define YOUNG_LIST_VERBOSE 0
 // CURRENT STATUS
 // This file is under construction.  Search for "FIXME".
 
@@ -133,8 +133,7 @@ public:
 
 YoungList::YoungList(G1CollectedHeap* g1h)
   : _g1h(g1h), _head(NULL),
-    _scan_only_head(NULL), _scan_only_tail(NULL), _curr_scan_only(NULL),
+    _length(0),
-    _length(0), _scan_only_length(0),
     _last_sampled_rs_lengths(0),
     _survivor_head(NULL), _survivor_tail(NULL), _survivor_length(0)
 {
@@ -166,48 +165,6 @@ void YoungList::add_survivor_region(HeapRegion* hr) {
   ++_survivor_length;
 }
 
-HeapRegion* YoungList::pop_region() {
-  while (_head != NULL) {
-    assert( length() > 0, "list should not be empty" );
-    HeapRegion* ret = _head;
-    _head = ret->get_next_young_region();
-    ret->set_next_young_region(NULL);
-    --_length;
-    assert(ret->is_young(), "region should be very young");
-
-    // Replace 'Survivor' region type with 'Young'. So the region will
-    // be treated as a young region and will not be 'confused' with
-    // newly created survivor regions.
-    if (ret->is_survivor()) {
-      ret->set_young();
-    }
-
-    if (!ret->is_scan_only()) {
-      return ret;
-    }
-
-    // scan-only, we'll add it to the scan-only list
-    if (_scan_only_tail == NULL) {
-      guarantee( _scan_only_head == NULL, "invariant" );
-
-      _scan_only_head = ret;
-      _curr_scan_only = ret;
-    } else {
-      guarantee( _scan_only_head != NULL, "invariant" );
-      _scan_only_tail->set_next_young_region(ret);
-    }
-    guarantee( ret->get_next_young_region() == NULL, "invariant" );
-    _scan_only_tail = ret;
-
-    // no need to be tagged as scan-only any more
-    ret->set_young();
-
-    ++_scan_only_length;
-  }
-  assert( length() == 0, "list should be empty" );
-  return NULL;
-}
-
 void YoungList::empty_list(HeapRegion* list) {
   while (list != NULL) {
     HeapRegion* next = list->get_next_young_region();
@@ -225,12 +182,6 @@ void YoungList::empty_list() {
   _head = NULL;
   _length = 0;
 
-  empty_list(_scan_only_head);
-  _scan_only_head = NULL;
-  _scan_only_tail = NULL;
-  _scan_only_length = 0;
-  _curr_scan_only = NULL;
-
   empty_list(_survivor_head);
   _survivor_head = NULL;
   _survivor_tail = NULL;
@@ -248,11 +199,11 @@ bool YoungList::check_list_well_formed() {
   HeapRegion* curr = _head;
   HeapRegion* last = NULL;
   while (curr != NULL) {
-    if (!curr->is_young() || curr->is_scan_only()) {
+    if (!curr->is_young()) {
       gclog_or_tty->print_cr("### YOUNG REGION "PTR_FORMAT"-"PTR_FORMAT" "
-                             "incorrectly tagged (%d, %d)",
+                             "incorrectly tagged (y: %d, surv: %d)",
                              curr->bottom(), curr->end(),
-                             curr->is_young(), curr->is_scan_only());
+                             curr->is_young(), curr->is_survivor());
       ret = false;
     }
     ++length;
@@ -267,47 +218,10 @@ bool YoungList::check_list_well_formed() {
                            length, _length);
   }
 
-  bool scan_only_ret = true;
+  return ret;
-  length = 0;
-  curr = _scan_only_head;
-  last = NULL;
-  while (curr != NULL) {
-    if (!curr->is_young() || curr->is_scan_only()) {
-      gclog_or_tty->print_cr("### SCAN-ONLY REGION "PTR_FORMAT"-"PTR_FORMAT" "
-                             "incorrectly tagged (%d, %d)",
-                             curr->bottom(), curr->end(),
-                             curr->is_young(), curr->is_scan_only());
-      scan_only_ret = false;
-    }
-    ++length;
-    last = curr;
-    curr = curr->get_next_young_region();
-  }
-  scan_only_ret = scan_only_ret && (length == _scan_only_length);
-
-  if ( (last != _scan_only_tail) ||
-       (_scan_only_head == NULL && _scan_only_tail != NULL) ||
-       (_scan_only_head != NULL && _scan_only_tail == NULL) ) {
-     gclog_or_tty->print_cr("## _scan_only_tail is set incorrectly");
-     scan_only_ret = false;
-  }
-
-  if (_curr_scan_only != NULL && _curr_scan_only != _scan_only_head) {
-    gclog_or_tty->print_cr("### _curr_scan_only is set incorrectly");
-    scan_only_ret = false;
-   }
-
-  if (!scan_only_ret) {
-    gclog_or_tty->print_cr("### SCAN-ONLY LIST seems not well formed!");
-    gclog_or_tty->print_cr("###   list has %d entries, _scan_only_length is %d",
-                  length, _scan_only_length);
-  }
-
-  return ret && scan_only_ret;
 }
 
-bool YoungList::check_list_empty(bool ignore_scan_only_list,
+bool YoungList::check_list_empty(bool check_sample) {
-                                 bool check_sample) {
   bool ret = true;
 
   if (_length != 0) {
@@ -327,28 +241,7 @@ bool YoungList::check_list_empty(bool ignore_scan_only_list,
     gclog_or_tty->print_cr("### YOUNG LIST does not seem empty");
   }
 
-  if (ignore_scan_only_list)
+  return ret;
-    return ret;
-
-  bool scan_only_ret = true;
-  if (_scan_only_length != 0) {
-    gclog_or_tty->print_cr("### SCAN-ONLY LIST should have 0 length, not %d",
-                  _scan_only_length);
-    scan_only_ret = false;
-  }
-  if (_scan_only_head != NULL) {
-    gclog_or_tty->print_cr("### SCAN-ONLY LIST does not have a NULL head");
-     scan_only_ret = false;
-  }
-  if (_scan_only_tail != NULL) {
-    gclog_or_tty->print_cr("### SCAN-ONLY LIST does not have a NULL tail");
-    scan_only_ret = false;
-  }
-  if (!scan_only_ret) {
-    gclog_or_tty->print_cr("### SCAN-ONLY LIST does not seem empty");
-  }
-
-  return ret && scan_only_ret;
 }
 
 void
@@ -365,7 +258,18 @@ YoungList::rs_length_sampling_more() {
 void
 YoungList::rs_length_sampling_next() {
   assert( _curr != NULL, "invariant" );
-  _sampled_rs_lengths += _curr->rem_set()->occupied();
+  size_t rs_length = _curr->rem_set()->occupied();
+
+  _sampled_rs_lengths += rs_length;
+
+  // The current region may not yet have been added to the
+  // incremental collection set (it gets added when it is
+  // retired as the current allocation region).
+  if (_curr->in_collection_set()) {
+    // Update the collection set policy information for this region
+    _g1h->g1_policy()->update_incremental_cset_info(_curr, rs_length);
+  }
+
   _curr = _curr->get_next_young_region();
   if (_curr == NULL) {
     _last_sampled_rs_lengths = _sampled_rs_lengths;
@@ -375,54 +279,46 @@ YoungList::rs_length_sampling_next() {
 
 void
 YoungList::reset_auxilary_lists() {
-  // We could have just "moved" the scan-only list to the young list.
-  // However, the scan-only list is ordered according to the region
-  // age in descending order, so, by moving one entry at a time, we
-  // ensure that it is recreated in ascending order.
-
   guarantee( is_empty(), "young list should be empty" );
   assert(check_list_well_formed(), "young list should be well formed");
 
   // Add survivor regions to SurvRateGroup.
   _g1h->g1_policy()->note_start_adding_survivor_regions();
   _g1h->g1_policy()->finished_recalculating_age_indexes(true /* is_survivors */);
+
   for (HeapRegion* curr = _survivor_head;
        curr != NULL;
        curr = curr->get_next_young_region()) {
     _g1h->g1_policy()->set_region_survivors(curr);
+
+    // The region is a non-empty survivor so let's add it to
+    // the incremental collection set for the next evacuation
+    // pause.
+    _g1h->g1_policy()->add_region_to_incremental_cset_rhs(curr);
   }
   _g1h->g1_policy()->note_stop_adding_survivor_regions();
 
+  _head   = _survivor_head;
+  _length = _survivor_length;
   if (_survivor_head != NULL) {
-    _head           = _survivor_head;
+    assert(_survivor_tail != NULL, "cause it shouldn't be");
-    _length         = _survivor_length + _scan_only_length;
+    assert(_survivor_length > 0, "invariant");
-    _survivor_tail->set_next_young_region(_scan_only_head);
+    _survivor_tail->set_next_young_region(NULL);
-  } else {
-    _head           = _scan_only_head;
-    _length         = _scan_only_length;
   }
 
-  for (HeapRegion* curr = _scan_only_head;
+  // Don't clear the survivor list handles until the start of
-       curr != NULL;
+  // the next evacuation pause - we need it in order to re-tag
-       curr = curr->get_next_young_region()) {
+  // the survivor regions from this evacuation pause as 'young'
-    curr->recalculate_age_in_surv_rate_group();
+  // at the start of the next.
-  }
-  _scan_only_head   = NULL;
-  _scan_only_tail   = NULL;
-  _scan_only_length = 0;
-  _curr_scan_only   = NULL;
 
-  _survivor_head    = NULL;
-  _survivor_tail   = NULL;
-  _survivor_length  = 0;
   _g1h->g1_policy()->finished_recalculating_age_indexes(false /* is_survivors */);
 
   assert(check_list_well_formed(), "young list should be well formed");
 }
 
 void YoungList::print() {
-  HeapRegion* lists[] = {_head,   _scan_only_head, _survivor_head};
+  HeapRegion* lists[] = {_head,   _survivor_head};
-  const char* names[] = {"YOUNG", "SCAN-ONLY",     "SURVIVOR"};
+  const char* names[] = {"YOUNG", "SURVIVOR"};
 
   for (unsigned int list = 0; list < ARRAY_SIZE(lists); ++list) {
     gclog_or_tty->print_cr("%s LIST CONTENTS", names[list]);
@@ -431,7 +327,7 @@ void YoungList::print() {
       gclog_or_tty->print_cr("  empty");
     while (curr != NULL) {
       gclog_or_tty->print_cr("  [%08x-%08x], t: %08x, P: %08x, N: %08x, C: %08x, "
-                             "age: %4d, y: %d, s-o: %d, surv: %d",
+                             "age: %4d, y: %d, surv: %d",
                              curr->bottom(), curr->end(),
                              curr->top(),
                              curr->prev_top_at_mark_start(),
@@ -439,7 +335,6 @@ void YoungList::print() {
                              curr->top_at_conc_mark_count(),
                              curr->age_in_surv_rate_group_cond(),
                              curr->is_young(),
-                             curr->is_scan_only(),
                              curr->is_survivor());
       curr = curr->get_next_young_region();
     }
@@ -707,6 +602,12 @@ G1CollectedHeap::attempt_allocation_slow(size_t word_size,
     // region below.
     if (_cur_alloc_region != NULL) {
       // We're finished with the _cur_alloc_region.
+      // As we're builing (at least the young portion) of the collection
+      // set incrementally we'll add the current allocation region to
+      // the collection set here.
+      if (_cur_alloc_region->is_young()) {
+        g1_policy()->add_region_to_incremental_cset_lhs(_cur_alloc_region);
+      }
       _summary_bytes_used += _cur_alloc_region->used();
       _cur_alloc_region = NULL;
     }
@@ -820,6 +721,12 @@ void G1CollectedHeap::abandon_cur_alloc_region() {
       _free_regions++;
       free_region(_cur_alloc_region);
     } else {
+      // As we're builing (at least the young portion) of the collection
+      // set incrementally we'll add the current allocation region to
+      // the collection set here.
+      if (_cur_alloc_region->is_young()) {
+        g1_policy()->add_region_to_incremental_cset_lhs(_cur_alloc_region);
+      }
       _summary_bytes_used += _cur_alloc_region->used();
     }
     _cur_alloc_region = NULL;
@@ -975,6 +882,15 @@ void G1CollectedHeap::do_collection(bool full, bool clear_all_soft_refs,
     g1_rem_set()->as_HRInto_G1RemSet()->cleanupHRRS();
     tear_down_region_lists();
     set_used_regions_to_need_zero_fill();
+
+    // We may have added regions to the current incremental collection
+    // set between the last GC or pause and now. We need to clear the
+    // incremental collection set and then start rebuilding it afresh
+    // after this full GC.
+    abandon_collection_set(g1_policy()->inc_cset_head());
+    g1_policy()->clear_incremental_cset();
+    g1_policy()->stop_incremental_cset_building();
+
     if (g1_policy()->in_young_gc_mode()) {
       empty_young_list();
       g1_policy()->set_full_young_gcs(true);
@@ -1058,6 +974,15 @@ void G1CollectedHeap::do_collection(bool full, bool clear_all_soft_refs,
       perm()->compute_new_size();
     }
 
+    // Start a new incremental collection set for the next pause
+    assert(g1_policy()->collection_set() == NULL, "must be");
+    g1_policy()->start_incremental_cset_building();
+
+    // Clear the _cset_fast_test bitmap in anticipation of adding
+    // regions to the incremental collection set for the next
+    // evacuation pause.
+    clear_cset_fast_test();
+
     double end = os::elapsedTime();
     g1_policy()->record_full_collection_end();
 
@@ -1076,7 +1001,9 @@ void G1CollectedHeap::do_collection(bool full, bool clear_all_soft_refs,
 
   if (g1_policy()->in_young_gc_mode()) {
     _young_list->reset_sampled_info();
-    assert( check_young_list_empty(false, false),
+    // At this point there should be no regions in the
+    // entire heap tagged as young.
+    assert( check_young_list_empty(true /* check_heap */),
             "young list should be empty at this point");
   }
 
@@ -1573,6 +1500,20 @@ jint G1CollectedHeap::initialize() {
 
   _g1h = this;
 
+   _in_cset_fast_test_length = max_regions();
+   _in_cset_fast_test_base = NEW_C_HEAP_ARRAY(bool, _in_cset_fast_test_length);
+
+   // We're biasing _in_cset_fast_test to avoid subtracting the
+   // beginning of the heap every time we want to index; basically
+   // it's the same with what we do with the card table.
+   _in_cset_fast_test = _in_cset_fast_test_base -
+                ((size_t) _g1_reserved.start() >> HeapRegion::LogOfHRGrainBytes);
+
+   // Clear the _cset_fast_test bitmap in anticipation of adding
+   // regions to the incremental collection set for the first
+   // evacuation pause.
+   clear_cset_fast_test();
+
   // Create the ConcurrentMark data structure and thread.
   // (Must do this late, so that "max_regions" is defined.)
   _cm       = new ConcurrentMark(heap_rs, (int) max_regions());
@@ -2751,25 +2692,19 @@ G1CollectedHeap::do_collection_pause_at_safepoint() {
       double start_time_sec = os::elapsedTime();
       size_t start_used_bytes = used();
 
+#if YOUNG_LIST_VERBOSE
+      gclog_or_tty->print_cr("\nBefore recording pause start.\nYoung_list:");
+      _young_list->print();
+      g1_policy()->print_collection_set(g1_policy()->inc_cset_head(), gclog_or_tty);
+#endif // YOUNG_LIST_VERBOSE
+
       g1_policy()->record_collection_pause_start(start_time_sec,
                                                  start_used_bytes);
 
-      guarantee(_in_cset_fast_test == NULL, "invariant");
+#if YOUNG_LIST_VERBOSE
-      guarantee(_in_cset_fast_test_base == NULL, "invariant");
+      gclog_or_tty->print_cr("\nAfter recording pause start.\nYoung_list:");
-      _in_cset_fast_test_length = max_regions();
-      _in_cset_fast_test_base =
-                             NEW_C_HEAP_ARRAY(bool, _in_cset_fast_test_length);
-      memset(_in_cset_fast_test_base, false,
-                                     _in_cset_fast_test_length * sizeof(bool));
-      // We're biasing _in_cset_fast_test to avoid subtracting the
-      // beginning of the heap every time we want to index; basically
-      // it's the same with what we do with the card table.
-      _in_cset_fast_test = _in_cset_fast_test_base -
-              ((size_t) _g1_reserved.start() >> HeapRegion::LogOfHRGrainBytes);
-
-#if SCAN_ONLY_VERBOSE
       _young_list->print();
-#endif // SCAN_ONLY_VERBOSE
+#endif // YOUNG_LIST_VERBOSE
 
       if (g1_policy()->during_initial_mark_pause()) {
         concurrent_mark()->checkpointRootsInitialPre();
@@ -2796,12 +2731,15 @@ G1CollectedHeap::do_collection_pause_at_safepoint() {
       if (mark_in_progress())
         concurrent_mark()->newCSet();
 
-      // Now choose the CS.
+#if YOUNG_LIST_VERBOSE
-      g1_policy()->choose_collection_set();
+      gclog_or_tty->print_cr("\nBefore choosing collection set.\nYoung_list:");
+      _young_list->print();
+      g1_policy()->print_collection_set(g1_policy()->inc_cset_head(), gclog_or_tty);
+#endif // YOUNG_LIST_VERBOSE
 
-      // We may abandon a pause if we find no region that will fit in the MMU
+      // Now choose the CS. We may abandon a pause if we find no
-      // pause.
+      // region that will fit in the MMU pause.
-      bool abandoned = (g1_policy()->collection_set() == NULL);
+      bool abandoned = g1_policy()->choose_collection_set();
 
       // Nothing to do if we were unable to choose a collection set.
       if (!abandoned) {
@@ -2819,40 +2757,64 @@ G1CollectedHeap::do_collection_pause_at_safepoint() {
 
         // Actually do the work...
         evacuate_collection_set();
+
         free_collection_set(g1_policy()->collection_set());
         g1_policy()->clear_collection_set();
 
-        FREE_C_HEAP_ARRAY(bool, _in_cset_fast_test_base);
-        // this is more for peace of mind; we're nulling them here and
-        // we're expecting them to be null at the beginning of the next GC
-        _in_cset_fast_test = NULL;
-        _in_cset_fast_test_base = NULL;
-
         cleanup_surviving_young_words();
 
+        // Start a new incremental collection set for the next pause.
+        g1_policy()->start_incremental_cset_building();
+
+        // Clear the _cset_fast_test bitmap in anticipation of adding
+        // regions to the incremental collection set for the next
+        // evacuation pause.
+        clear_cset_fast_test();
+
         if (g1_policy()->in_young_gc_mode()) {
           _young_list->reset_sampled_info();
-          assert(check_young_list_empty(true),
-                 "young list should be empty");
 
-#if SCAN_ONLY_VERBOSE
+          // Don't check the whole heap at this point as the
+          // GC alloc regions from this pause have been tagged
+          // as survivors and moved on to the survivor list.
+          // Survivor regions will fail the !is_young() check.
+          assert(check_young_list_empty(false /* check_heap */),
+              "young list should be empty");
+
+#if YOUNG_LIST_VERBOSE
+          gclog_or_tty->print_cr("Before recording survivors.\nYoung List:");
           _young_list->print();
-#endif // SCAN_ONLY_VERBOSE
+#endif // YOUNG_LIST_VERBOSE
 
           g1_policy()->record_survivor_regions(_young_list->survivor_length(),
                                           _young_list->first_survivor_region(),
                                           _young_list->last_survivor_region());
+
           _young_list->reset_auxilary_lists();
         }
       } else {
-        if (_in_cset_fast_test != NULL) {
+        // We have abandoned the current collection. This can only happen
-          assert(_in_cset_fast_test_base != NULL, "Since _in_cset_fast_test isn't");
+        // if we're not doing young or partially young collections, and
-          FREE_C_HEAP_ARRAY(bool, _in_cset_fast_test_base);
+        // we didn't find an old region that we're able to collect within
-          //  this is more for peace of mind; we're nulling them here and
+        // the allowed time.
-          // we're expecting them to be null at the beginning of the next GC
+
-          _in_cset_fast_test = NULL;
+        assert(g1_policy()->collection_set() == NULL, "should be");
-          _in_cset_fast_test_base = NULL;
+        assert(_young_list->length() == 0, "because it should be");
-        }
+
+        // This should be a no-op.
+        abandon_collection_set(g1_policy()->inc_cset_head());
+
+        g1_policy()->clear_incremental_cset();
+        g1_policy()->stop_incremental_cset_building();
+
+        // Start a new incremental collection set for the next pause.
+        g1_policy()->start_incremental_cset_building();
+
+        // Clear the _cset_fast_test bitmap in anticipation of adding
+        // regions to the incremental collection set for the next
+        // evacuation pause.
+        clear_cset_fast_test();
+
         // This looks confusing, because the DPT should really be empty
         // at this point -- since we have not done any collection work,
         // there should not be any derived pointers in the table to update;
@@ -2886,9 +2848,11 @@ G1CollectedHeap::do_collection_pause_at_safepoint() {
         doConcurrentMark();
       }
 
-#if SCAN_ONLY_VERBOSE
+#if YOUNG_LIST_VERBOSE
+      gclog_or_tty->print_cr("\nEnd of the pause.\nYoung_list:");
       _young_list->print();
-#endif // SCAN_ONLY_VERBOSE
+      g1_policy()->print_collection_set(g1_policy()->inc_cset_head(), gclog_or_tty);
+#endif // YOUNG_LIST_VERBOSE
 
       double end_time_sec = os::elapsedTime();
       double pause_time_ms = (end_time_sec - start_time_sec) * MILLIUNITS;
@@ -4027,16 +3991,13 @@ public:
 
     OopsInHeapRegionClosure        *scan_root_cl;
     OopsInHeapRegionClosure        *scan_perm_cl;
-    OopsInHeapRegionClosure        *scan_so_cl;
 
     if (_g1h->g1_policy()->during_initial_mark_pause()) {
       scan_root_cl = &scan_mark_root_cl;
       scan_perm_cl = &scan_mark_perm_cl;
-      scan_so_cl   = &scan_mark_heap_rs_cl;
     } else {
       scan_root_cl = &only_scan_root_cl;
       scan_perm_cl = &only_scan_perm_cl;
-      scan_so_cl   = &only_scan_heap_rs_cl;
     }
 
     pss.start_strong_roots();
@@ -4044,7 +4005,6 @@ public:
                                   SharedHeap::SO_AllClasses,
                                   scan_root_cl,
                                   &push_heap_rs_cl,
-                                  scan_so_cl,
                                   scan_perm_cl,
                                   i);
     pss.end_strong_roots();
@@ -4106,7 +4066,6 @@ g1_process_strong_roots(bool collecting_perm_gen,
                         SharedHeap::ScanningOption so,
                         OopClosure* scan_non_heap_roots,
                         OopsInHeapRegionClosure* scan_rs,
-                        OopsInHeapRegionClosure* scan_so,
                         OopsInGenClosure* scan_perm,
                         int worker_i) {
   // First scan the strong roots, including the perm gen.
@@ -4126,6 +4085,7 @@ g1_process_strong_roots(bool collecting_perm_gen,
                        &buf_scan_non_heap_roots,
                        &eager_scan_code_roots,
                        &buf_scan_perm);
+
   // Finish up any enqueued closure apps.
   buf_scan_non_heap_roots.done();
   buf_scan_perm.done();
@@ -4148,9 +4108,6 @@ g1_process_strong_roots(bool collecting_perm_gen,
 
   // XXX What should this be doing in the parallel case?
   g1_policy()->record_collection_pause_end_CH_strong_roots();
-  if (scan_so != NULL) {
-    scan_scan_only_set(scan_so, worker_i);
-  }
   // Now scan the complement of the collection set.
   if (scan_rs != NULL) {
     g1_rem_set()->oops_into_collection_set_do(scan_rs, worker_i);
@@ -4163,54 +4120,6 @@ g1_process_strong_roots(bool collecting_perm_gen,
   _process_strong_tasks->all_tasks_completed();
 }
 
-void
-G1CollectedHeap::scan_scan_only_region(HeapRegion* r,
-                                       OopsInHeapRegionClosure* oc,
-                                       int worker_i) {
-  HeapWord* startAddr = r->bottom();
-  HeapWord* endAddr = r->used_region().end();
-
-  oc->set_region(r);
-
-  HeapWord* p = r->bottom();
-  HeapWord* t = r->top();
-  guarantee( p == r->next_top_at_mark_start(), "invariant" );
-  while (p < t) {
-    oop obj = oop(p);
-    p += obj->oop_iterate(oc);
-  }
-}
-
-void
-G1CollectedHeap::scan_scan_only_set(OopsInHeapRegionClosure* oc,
-                                    int worker_i) {
-  double start = os::elapsedTime();
-
-  BufferingOopsInHeapRegionClosure boc(oc);
-
-  FilterInHeapRegionAndIntoCSClosure scan_only(this, &boc);
-  FilterAndMarkInHeapRegionAndIntoCSClosure scan_and_mark(this, &boc, concurrent_mark());
-
-  OopsInHeapRegionClosure *foc;
-  if (g1_policy()->during_initial_mark_pause())
-    foc = &scan_and_mark;
-  else
-    foc = &scan_only;
-
-  HeapRegion* hr;
-  int n = 0;
-  while ((hr = _young_list->par_get_next_scan_only_region()) != NULL) {
-    scan_scan_only_region(hr, foc, worker_i);
-    ++n;
-  }
-  boc.done();
-
-  double closure_app_s = boc.closure_app_seconds();
-  g1_policy()->record_obj_copy_time(worker_i, closure_app_s * 1000.0);
-  double ms = (os::elapsedTime() - start - closure_app_s)*1000.0;
-  g1_policy()->record_scan_only_time(worker_i, ms, n);
-}
-
 void
 G1CollectedHeap::g1_process_weak_roots(OopClosure* root_closure,
                                        OopClosure* non_root_closure) {
@@ -4409,17 +4318,14 @@ void G1CollectedHeap::dirtyCardsForYoungRegions(CardTableModRefBS* ct_bs, HeapRe
 class G1ParCleanupCTTask : public AbstractGangTask {
   CardTableModRefBS* _ct_bs;
   G1CollectedHeap* _g1h;
-  HeapRegion* volatile _so_head;
   HeapRegion* volatile _su_head;
 public:
   G1ParCleanupCTTask(CardTableModRefBS* ct_bs,
                      G1CollectedHeap* g1h,
-                     HeapRegion* scan_only_list,
                      HeapRegion* survivor_list) :
     AbstractGangTask("G1 Par Cleanup CT Task"),
     _ct_bs(ct_bs),
     _g1h(g1h),
-    _so_head(scan_only_list),
     _su_head(survivor_list)
   { }
 
@@ -4428,14 +4334,13 @@ public:
     while (r = _g1h->pop_dirty_cards_region()) {
       clear_cards(r);
     }
-    // Redirty the cards of the scan-only and survivor regions.
+    // Redirty the cards of the survivor regions.
-    dirty_list(&this->_so_head);
     dirty_list(&this->_su_head);
   }
 
   void clear_cards(HeapRegion* r) {
-    // Cards for Survivor and Scan-Only regions will be dirtied later.
+    // Cards for Survivor regions will be dirtied later.
-    if (!r->is_scan_only() && !r->is_survivor()) {
+    if (!r->is_survivor()) {
       _ct_bs->clear(MemRegion(r->bottom(), r->end()));
     }
   }
@@ -4468,7 +4373,7 @@ public:
   virtual bool doHeapRegion(HeapRegion* r)
   {
     MemRegion mr(r->bottom(), r->end());
-    if (r->is_scan_only() || r->is_survivor()) {
+    if (r->is_survivor()) {
       _ct_bs->verify_dirty_region(mr);
     } else {
       _ct_bs->verify_clean_region(mr);
@@ -4484,8 +4389,8 @@ void G1CollectedHeap::cleanUpCardTable() {
 
   // Iterate over the dirty cards region list.
   G1ParCleanupCTTask cleanup_task(ct_bs, this,
-                                  _young_list->first_scan_only_region(),
                                   _young_list->first_survivor_region());
+
   if (ParallelGCThreads > 0) {
     set_par_threads(workers()->total_workers());
     workers()->run_task(&cleanup_task);
@@ -4501,12 +4406,12 @@ void G1CollectedHeap::cleanUpCardTable() {
       }
       r->set_next_dirty_cards_region(NULL);
     }
-    // now, redirty the cards of the scan-only and survivor regions
+    // now, redirty the cards of the survivor regions
     // (it seemed faster to do it this way, instead of iterating over
     // all regions and then clearing / dirtying as appropriate)
-    dirtyCardsForYoungRegions(ct_bs, _young_list->first_scan_only_region());
     dirtyCardsForYoungRegions(ct_bs, _young_list->first_survivor_region());
   }
+
   double elapsed = os::elapsedTime() - start;
   g1_policy()->record_clear_ct_time( elapsed * 1000.0);
 #ifndef PRODUCT
@@ -4527,6 +4432,11 @@ void G1CollectedHeap::free_collection_set(HeapRegion* cs_head) {
   double young_time_ms     = 0.0;
   double non_young_time_ms = 0.0;
 
+  // Since the collection set is a superset of the the young list,
+  // all we need to do to clear the young list is clear its
+  // head and length, and unlink any young regions in the code below
+  _young_list->clear();
+
   G1CollectorPolicy* policy = g1_policy();
 
   double start_sec = os::elapsedTime();
@@ -4570,6 +4480,12 @@ void G1CollectedHeap::free_collection_set(HeapRegion* cs_head) {
       guarantee( (size_t)index < policy->young_cset_length(), "invariant" );
       size_t words_survived = _surviving_young_words[index];
       cur->record_surv_words_in_group(words_survived);
+
+      // At this point the we have 'popped' cur from the collection set
+      // (linked via next_in_collection_set()) but it is still in the
+      // young list (linked via next_young_region()). Clear the
+      // _next_young_region field.
+      cur->set_next_young_region(NULL);
     } else {
       int index = cur->young_index_in_cset();
       guarantee( index == -1, "invariant" );
@@ -4585,7 +4501,6 @@ void G1CollectedHeap::free_collection_set(HeapRegion* cs_head) {
              "Should not have empty regions in a CS.");
       free_region(cur);
     } else {
-      guarantee( !cur->is_scan_only(), "should not be scan only" );
       cur->uninstall_surv_rate_group();
       if (cur->is_young())
         cur->set_young_index_in_cset(-1);
@@ -4609,6 +4524,27 @@ void G1CollectedHeap::free_collection_set(HeapRegion* cs_head) {
   policy->record_non_young_free_cset_time_ms(non_young_time_ms);
 }
 
+// This routine is similar to the above but does not record
+// any policy statistics or update free lists; we are abandoning
+// the current incremental collection set in preparation of a
+// full collection. After the full GC we will start to build up
+// the incremental collection set again.
+// This is only called when we're doing a full collection
+// and is immediately followed by the tearing down of the young list.
+
+void G1CollectedHeap::abandon_collection_set(HeapRegion* cs_head) {
+  HeapRegion* cur = cs_head;
+
+  while (cur != NULL) {
+    HeapRegion* next = cur->next_in_collection_set();
+    assert(cur->in_collection_set(), "bad CS");
+    cur->set_next_in_collection_set(NULL);
+    cur->set_in_collection_set(false);
+    cur->set_young_index_in_cset(-1);
+    cur = next;
+  }
+}
+
 HeapRegion*
 G1CollectedHeap::alloc_region_from_unclean_list_locked(bool zero_filled) {
   assert(ZF_mon->owned_by_self(), "Precondition");
@@ -4975,12 +4911,10 @@ public:
   bool success() { return _success; }
 };
 
-bool G1CollectedHeap::check_young_list_empty(bool ignore_scan_only_list,
+bool G1CollectedHeap::check_young_list_empty(bool check_heap, bool check_sample) {
-                                             bool check_sample) {
+  bool ret = _young_list->check_list_empty(check_sample);
-  bool ret = true;
 
-  ret = _young_list->check_list_empty(ignore_scan_only_list, check_sample);
+  if (check_heap) {
-  if (!ignore_scan_only_list) {
     NoYoungRegionsClosure closure;
     heap_region_iterate(&closure);
     ret = ret && closure.success();

hotspot/src/share/vm/gc_implementation/g1/g1CollectedHeap.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright 2001-2009 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2001-2010 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -81,33 +81,29 @@ private:
 
   HeapRegion* _head;
 
-  HeapRegion* _scan_only_head;
+  HeapRegion* _survivor_head;
-  HeapRegion* _scan_only_tail;
+  HeapRegion* _survivor_tail;
+
+  HeapRegion* _curr;
+
   size_t      _length;
-  size_t      _scan_only_length;
+  size_t      _survivor_length;
 
   size_t      _last_sampled_rs_lengths;
   size_t      _sampled_rs_lengths;
-  HeapRegion* _curr;
-  HeapRegion* _curr_scan_only;
 
-  HeapRegion* _survivor_head;
+  void         empty_list(HeapRegion* list);
-  HeapRegion* _survivor_tail;
-  size_t      _survivor_length;
-
-  void          empty_list(HeapRegion* list);
 
 public:
   YoungList(G1CollectedHeap* g1h);
 
-  void          push_region(HeapRegion* hr);
+  void         push_region(HeapRegion* hr);
-  void          add_survivor_region(HeapRegion* hr);
+  void         add_survivor_region(HeapRegion* hr);
-  HeapRegion*   pop_region();
+
-  void          empty_list();
+  void         empty_list();
-  bool          is_empty() { return _length == 0; }
+  bool         is_empty() { return _length == 0; }
-  size_t        length() { return _length; }
+  size_t       length() { return _length; }
-  size_t        scan_only_length() { return _scan_only_length; }
+  size_t       survivor_length() { return _survivor_length; }
-  size_t        survivor_length() { return _survivor_length; }
 
   void rs_length_sampling_init();
   bool rs_length_sampling_more();
@@ -120,22 +116,21 @@ public:
 
   // for development purposes
   void reset_auxilary_lists();
+  void clear() { _head = NULL; _length = 0; }
+
+  void clear_survivors() {
+    _survivor_head    = NULL;
+    _survivor_tail    = NULL;
+    _survivor_length  = 0;
+  }
+
   HeapRegion* first_region() { return _head; }
-  HeapRegion* first_scan_only_region() { return _scan_only_head; }
   HeapRegion* first_survivor_region() { return _survivor_head; }
   HeapRegion* last_survivor_region() { return _survivor_tail; }
-  HeapRegion* par_get_next_scan_only_region() {
-    MutexLockerEx x(ParGCRareEvent_lock, Mutex::_no_safepoint_check_flag);
-    HeapRegion* ret = _curr_scan_only;
-    if (ret != NULL)
-      _curr_scan_only = ret->get_next_young_region();
-    return ret;
-  }
 
   // debugging
   bool          check_list_well_formed();
-  bool          check_list_empty(bool ignore_scan_only_list,
+  bool          check_list_empty(bool check_sample = true);
-                                 bool check_sample = true);
   void          print();
 };
 
@@ -405,8 +400,7 @@ public:
     assert(_in_cset_fast_test_base != NULL, "sanity");
     assert(r->in_collection_set(), "invariant");
     int index = r->hrs_index();
-    assert(0 <= (size_t) index && (size_t) index < _in_cset_fast_test_length,
+    assert(0 <= index && (size_t) index < _in_cset_fast_test_length, "invariant");
-           "invariant");
     assert(!_in_cset_fast_test_base[index], "invariant");
     _in_cset_fast_test_base[index] = true;
   }
@@ -431,6 +425,12 @@ public:
     }
   }
 
+  void clear_cset_fast_test() {
+    assert(_in_cset_fast_test_base != NULL, "sanity");
+    memset(_in_cset_fast_test_base, false,
+        _in_cset_fast_test_length * sizeof(bool));
+  }
+
 protected:
 
   // Shrink the garbage-first heap by at most the given size (in bytes!).
@@ -476,6 +476,10 @@ protected:
   // regions.
   void free_collection_set(HeapRegion* cs_head);
 
+  // Abandon the current collection set without recording policy
+  // statistics or updating free lists.
+  void abandon_collection_set(HeapRegion* cs_head);
+
   // Applies "scan_non_heap_roots" to roots outside the heap,
   // "scan_rs" to roots inside the heap (having done "set_region" to
   // indicate the region in which the root resides), and does "scan_perm"
@@ -488,16 +492,9 @@ protected:
                                SharedHeap::ScanningOption so,
                                OopClosure* scan_non_heap_roots,
                                OopsInHeapRegionClosure* scan_rs,
-                               OopsInHeapRegionClosure* scan_so,
                                OopsInGenClosure* scan_perm,
                                int worker_i);
 
-  void scan_scan_only_set(OopsInHeapRegionClosure* oc,
-                          int worker_i);
-  void scan_scan_only_region(HeapRegion* hr,
-                             OopsInHeapRegionClosure* oc,
-                             int worker_i);
-
   // Apply "blk" to all the weak roots of the system.  These include
   // JNI weak roots, the code cache, system dictionary, symbol table,
   // string table, and referents of reachable weak refs.
@@ -1136,36 +1133,14 @@ public:
   void set_region_short_lived_locked(HeapRegion* hr);
   // add appropriate methods for any other surv rate groups
 
-  void young_list_rs_length_sampling_init() {
+  YoungList* young_list() { return _young_list; }
-    _young_list->rs_length_sampling_init();
-  }
-  bool young_list_rs_length_sampling_more() {
-    return _young_list->rs_length_sampling_more();
-  }
-  void young_list_rs_length_sampling_next() {
-    _young_list->rs_length_sampling_next();
-  }
-  size_t young_list_sampled_rs_lengths() {
-    return _young_list->sampled_rs_lengths();
-  }
-
-  size_t young_list_length()   { return _young_list->length(); }
-  size_t young_list_scan_only_length() {
-                                      return _young_list->scan_only_length(); }
-
-  HeapRegion* pop_region_from_young_list() {
-    return _young_list->pop_region();
-  }
-
-  HeapRegion* young_list_first_region() {
-    return _young_list->first_region();
-  }
 
   // debugging
   bool check_young_list_well_formed() {
     return _young_list->check_list_well_formed();
   }
-  bool check_young_list_empty(bool ignore_scan_only_list,
+
+  bool check_young_list_empty(bool check_heap,
                               bool check_sample = true);
 
   // *** Stuff related to concurrent marking.  It's not clear to me that so

hotspot/src/share/vm/gc_implementation/g1/g1CollectorPolicy.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright 2001-2009 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2001-2010 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -61,7 +61,6 @@ class MainBodySummary: public CHeapObj {
   define_num_seq(parallel) // parallel only
     define_num_seq(ext_root_scan)
     define_num_seq(mark_stack_scan)
-    define_num_seq(scan_only)
     define_num_seq(update_rs)
     define_num_seq(scan_rs)
     define_num_seq(scan_new_refs) // Only for temp use; added to
@@ -174,8 +173,6 @@ protected:
 
   double* _par_last_ext_root_scan_times_ms;
   double* _par_last_mark_stack_scan_times_ms;
-  double* _par_last_scan_only_times_ms;
-  double* _par_last_scan_only_regions_scanned;
   double* _par_last_update_rs_start_times_ms;
   double* _par_last_update_rs_times_ms;
   double* _par_last_update_rs_processed_buffers;
@@ -196,7 +193,6 @@ protected:
   bool _adaptive_young_list_length;
   size_t _young_list_min_length;
   size_t _young_list_target_length;
-  size_t _young_list_so_prefix_length;
   size_t _young_list_fixed_length;
 
   size_t _young_cset_length;
@@ -234,7 +230,6 @@ private:
   TruncatedSeq* _pending_card_diff_seq;
   TruncatedSeq* _rs_length_diff_seq;
   TruncatedSeq* _cost_per_card_ms_seq;
-  TruncatedSeq* _cost_per_scan_only_region_ms_seq;
   TruncatedSeq* _fully_young_cards_per_entry_ratio_seq;
   TruncatedSeq* _partially_young_cards_per_entry_ratio_seq;
   TruncatedSeq* _cost_per_entry_ms_seq;
@@ -249,19 +244,16 @@ private:
   TruncatedSeq* _rs_lengths_seq;
 
   TruncatedSeq* _cost_per_byte_ms_during_cm_seq;
-  TruncatedSeq* _cost_per_scan_only_region_ms_during_cm_seq;
 
   TruncatedSeq* _young_gc_eff_seq;
 
   TruncatedSeq* _max_conc_overhead_seq;
 
   size_t _recorded_young_regions;
-  size_t _recorded_scan_only_regions;
   size_t _recorded_non_young_regions;
   size_t _recorded_region_num;
 
   size_t _free_regions_at_end_of_collection;
-  size_t _scan_only_regions_at_end_of_collection;
 
   size_t _recorded_rs_lengths;
   size_t _max_rs_lengths;
@@ -277,7 +269,6 @@ private:
   double _predicted_survival_ratio;
   double _predicted_rs_update_time_ms;
   double _predicted_rs_scan_time_ms;
-  double _predicted_scan_only_scan_time_ms;
   double _predicted_object_copy_time_ms;
   double _predicted_constant_other_time_ms;
   double _predicted_young_other_time_ms;
@@ -344,8 +335,6 @@ public:
   bool verify_young_ages();
 #endif // PRODUCT
 
-  void tag_scan_only(size_t short_lived_scan_only_length);
-
   double get_new_prediction(TruncatedSeq* seq) {
     return MAX2(seq->davg() + sigma() * seq->dsd(),
                 seq->davg() * confidence_factor(seq->num()));
@@ -431,23 +420,6 @@ public:
         get_new_prediction(_partially_young_cost_per_entry_ms_seq);
   }
 
-  double predict_scan_only_time_ms_during_cm(size_t scan_only_region_num) {
-    if (_cost_per_scan_only_region_ms_during_cm_seq->num() < 3)
-      return 1.5 * (double) scan_only_region_num *
-        get_new_prediction(_cost_per_scan_only_region_ms_seq);
-    else
-      return (double) scan_only_region_num *
-        get_new_prediction(_cost_per_scan_only_region_ms_during_cm_seq);
-  }
-
-  double predict_scan_only_time_ms(size_t scan_only_region_num) {
-    if (_in_marking_window_im)
-      return predict_scan_only_time_ms_during_cm(scan_only_region_num);
-    else
-      return (double) scan_only_region_num *
-        get_new_prediction(_cost_per_scan_only_region_ms_seq);
-  }
-
   double predict_object_copy_time_ms_during_cm(size_t bytes_to_copy) {
     if (_cost_per_byte_ms_during_cm_seq->num() < 3)
       return 1.1 * (double) bytes_to_copy *
@@ -490,24 +462,21 @@ public:
   size_t predict_bytes_to_copy(HeapRegion* hr);
   double predict_region_elapsed_time_ms(HeapRegion* hr, bool young);
 
-  // for use by: calculate_optimal_so_length(length)
+    // for use by: calculate_young_list_target_length(rs_length)
-  void predict_gc_eff(size_t young_region_num,
+  bool predict_will_fit(size_t young_region_num,
-                      size_t so_length,
+                        double base_time_ms,
-                      double base_time_ms,
+                        size_t init_free_regions,
-                      double *gc_eff,
+                        double target_pause_time_ms);
-                      double *pause_time_ms);
-
-  // for use by: calculate_young_list_target_config(rs_length)
-  bool predict_gc_eff(size_t young_region_num,
-                      size_t so_length,
-                      double base_time_with_so_ms,
-                      size_t init_free_regions,
-                      double target_pause_time_ms,
-                      double* gc_eff);
 
   void start_recording_regions();
-  void record_cset_region(HeapRegion* hr, bool young);
+  void record_cset_region_info(HeapRegion* hr, bool young);
-  void record_scan_only_regions(size_t scan_only_length);
+  void record_non_young_cset_region(HeapRegion* hr);
+
+  void set_recorded_young_regions(size_t n_regions);
+  void set_recorded_young_bytes(size_t bytes);
+  void set_recorded_rs_lengths(size_t rs_lengths);
+  void set_predicted_bytes_to_copy(size_t bytes);
+
   void end_recording_regions();
 
   void record_vtime_diff_ms(double vtime_diff_ms) {
@@ -638,11 +607,74 @@ protected:
   void update_recent_gc_times(double end_time_sec, double elapsed_ms);
 
   // The head of the list (via "next_in_collection_set()") representing the
-  // current collection set.
+  // current collection set. Set from the incrementally built collection
+  // set at the start of the pause.
   HeapRegion* _collection_set;
+
+  // The number of regions in the collection set. Set from the incrementally
+  // built collection set at the start of an evacuation pause.
   size_t _collection_set_size;
+
+  // The number of bytes in the collection set before the pause. Set from
+  // the incrementally built collection set at the start of an evacuation
+  // pause.
   size_t _collection_set_bytes_used_before;
 
+  // The associated information that is maintained while the incremental
+  // collection set is being built with young regions. Used to populate
+  // the recorded info for the evacuation pause.
+
+  enum CSetBuildType {
+    Active,             // We are actively building the collection set
+    Inactive            // We are not actively building the collection set
+  };
+
+  CSetBuildType _inc_cset_build_state;
+
+  // The head of the incrementally built collection set.
+  HeapRegion* _inc_cset_head;
+
+  // The tail of the incrementally built collection set.
+  HeapRegion* _inc_cset_tail;
+
+  // The number of regions in the incrementally built collection set.
+  // Used to set _collection_set_size at the start of an evacuation
+  // pause.
+  size_t _inc_cset_size;
+
+  // Used as the index in the surving young words structure
+  // which tracks the amount of space, for each young region,
+  // that survives the pause.
+  size_t _inc_cset_young_index;
+
+  // The number of bytes in the incrementally built collection set.
+  // Used to set _collection_set_bytes_used_before at the start of
+  // an evacuation pause.
+  size_t _inc_cset_bytes_used_before;
+
+  // Used to record the highest end of heap region in collection set
+  HeapWord* _inc_cset_max_finger;
+
+  // The number of recorded used bytes in the young regions
+  // of the collection set. This is the sum of the used() bytes
+  // of retired young regions in the collection set.
+  size_t _inc_cset_recorded_young_bytes;
+
+  // The RSet lengths recorded for regions in the collection set
+  // (updated by the periodic sampling of the regions in the
+  // young list/collection set).
+  size_t _inc_cset_recorded_rs_lengths;
+
+  // The predicted elapsed time it will take to collect the regions
+  // in the collection set (updated by the periodic sampling of the
+  // regions in the young list/collection set).
+  double _inc_cset_predicted_elapsed_time_ms;
+
+  // The predicted bytes to copy for the regions in the collection
+  // set (updated by the periodic sampling of the regions in the
+  // young list/collection set).
+  size_t _inc_cset_predicted_bytes_to_copy;
+
   // Info about marking.
   int _n_marks; // Sticky at 2, so we know when we've done at least 2.
 
@@ -761,9 +793,8 @@ protected:
   double _mark_closure_time_ms;
 
   void   calculate_young_list_min_length();
-  void   calculate_young_list_target_config();
+  void   calculate_young_list_target_length();
-  void   calculate_young_list_target_config(size_t rs_lengths);
+  void   calculate_young_list_target_length(size_t rs_lengths);
-  size_t calculate_optimal_so_length(size_t young_list_length);
 
 public:
 
@@ -868,11 +899,6 @@ public:
     _par_last_mark_stack_scan_times_ms[worker_i] = ms;
   }
 
-  void record_scan_only_time(int worker_i, double ms, int n) {
-    _par_last_scan_only_times_ms[worker_i] = ms;
-    _par_last_scan_only_regions_scanned[worker_i] = (double) n;
-  }
-
   void record_satb_drain_time(double ms) {
     _cur_satb_drain_time_ms = ms;
     _satb_drain_time_set    = true;
@@ -987,20 +1013,67 @@ public:
   // Choose a new collection set.  Marks the chosen regions as being
   // "in_collection_set", and links them together.  The head and number of
   // the collection set are available via access methods.
-  virtual void choose_collection_set() = 0;
+  virtual bool choose_collection_set() = 0;
-
-  void clear_collection_set() { _collection_set = NULL; }
 
   // The head of the list (via "next_in_collection_set()") representing the
   // current collection set.
   HeapRegion* collection_set() { return _collection_set; }
 
+  void clear_collection_set() { _collection_set = NULL; }
+
   // The number of elements in the current collection set.
   size_t collection_set_size() { return _collection_set_size; }
 
   // Add "hr" to the CS.
   void add_to_collection_set(HeapRegion* hr);
 
+  // Incremental CSet Support
+
+  // The head of the incrementally built collection set.
+  HeapRegion* inc_cset_head() { return _inc_cset_head; }
+
+  // The tail of the incrementally built collection set.
+  HeapRegion* inc_set_tail() { return _inc_cset_tail; }
+
+  // The number of elements in the incrementally built collection set.
+  size_t inc_cset_size() { return _inc_cset_size; }
+
+  // Initialize incremental collection set info.
+  void start_incremental_cset_building();
+
+  void clear_incremental_cset() {
+    _inc_cset_head = NULL;
+    _inc_cset_tail = NULL;
+  }
+
+  // Stop adding regions to the incremental collection set
+  void stop_incremental_cset_building() { _inc_cset_build_state = Inactive; }
+
+  // Add/remove information about hr to the aggregated information
+  // for the incrementally built collection set.
+  void add_to_incremental_cset_info(HeapRegion* hr, size_t rs_length);
+  void remove_from_incremental_cset_info(HeapRegion* hr);
+
+  // Update information about hr in the aggregated information for
+  // the incrementally built collection set.
+  void update_incremental_cset_info(HeapRegion* hr, size_t new_rs_length);
+
+private:
+  // Update the incremental cset information when adding a region
+  // (should not be called directly).
+  void add_region_to_incremental_cset_common(HeapRegion* hr);
+
+public:
+  // Add hr to the LHS of the incremental collection set.
+  void add_region_to_incremental_cset_lhs(HeapRegion* hr);
+
+  // Add hr to the RHS of the incremental collection set.
+  void add_region_to_incremental_cset_rhs(HeapRegion* hr);
+
+#ifndef PRODUCT
+  void print_collection_set(HeapRegion* list_head, outputStream* st);
+#endif // !PRODUCT
+
   bool initiate_conc_mark_if_possible()       { return _initiate_conc_mark_if_possible;  }
   void set_initiate_conc_mark_if_possible()   { _initiate_conc_mark_if_possible = true;  }
   void clear_initiate_conc_mark_if_possible() { _initiate_conc_mark_if_possible = false; }
@@ -1191,7 +1264,7 @@ class G1CollectorPolicy_BestRegionsFirst: public G1CollectorPolicy {
   // If the estimated is less then desirable, resize if possible.
   void expand_if_possible(size_t numRegions);
 
-  virtual void choose_collection_set();
+  virtual bool choose_collection_set();
   virtual void record_collection_pause_start(double start_time_sec,
                                              size_t start_used);
   virtual void record_concurrent_mark_cleanup_end(size_t freed_bytes,

hotspot/src/share/vm/gc_implementation/g1/g1_globals.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright 2001-2009 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2001-2010 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -226,10 +226,6 @@
           "the number of regions for which we'll print a surv rate "        \
           "summary.")                                                       \
                                                                             \
-  develop(bool, G1UseScanOnlyPrefix, false,                                 \
-          "It determines whether the system will calculate an optimum "     \
-          "scan-only set.")                                                 \
-                                                                            \
   product(intx, G1ReservePercent, 10,                                       \
           "It determines the minimum reserve we should have in the heap "   \
           "to minimize the probability of promotion failure.")              \

hotspot/src/share/vm/gc_implementation/g1/heapRegion.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright 2001-2009 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2001-2010 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -450,7 +450,9 @@ HeapRegion(G1BlockOffsetSharedArray* sharedOffsetArray,
     _young_type(NotYoung), _next_young_region(NULL),
     _next_dirty_cards_region(NULL),
     _young_index_in_cset(-1), _surv_rate_group(NULL), _age_index(-1),
-    _rem_set(NULL), _zfs(NotZeroFilled)
+    _rem_set(NULL), _zfs(NotZeroFilled),
+    _recorded_rs_length(0), _predicted_elapsed_time_ms(0),
+    _predicted_bytes_to_copy(0)
 {
   _orig_end = mr.end();
   // Note that initialize() will set the start of the unmarked area of the
@@ -733,7 +735,7 @@ void HeapRegion::print_on(outputStream* st) const {
   else
     st->print("   ");
   if (is_young())
-    st->print(is_scan_only() ? " SO" : (is_survivor() ? " SU" : " Y "));
+    st->print(is_survivor() ? " SU" : " Y ");
   else
     st->print("   ");
   if (is_empty())

hotspot/src/share/vm/gc_implementation/g1/heapRegion.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright 2001-2009 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2001-2010 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -247,7 +247,6 @@ class HeapRegion: public G1OffsetTableContigSpace {
 
   enum YoungType {
     NotYoung,                   // a region is not young
-    ScanOnly,                   // a region is young and scan-only
     Young,                      // a region is young
     Survivor                    // a region is young and it contains
                                 // survivor
@@ -292,6 +291,20 @@ class HeapRegion: public G1OffsetTableContigSpace {
     _young_type = new_type;
   }
 
+  // Cached attributes used in the collection set policy information
+
+  // The RSet length that was added to the total value
+  // for the collection set.
+  size_t _recorded_rs_length;
+
+  // The predicted elapsed time that was added to total value
+  // for the collection set.
+  double _predicted_elapsed_time_ms;
+
+  // The predicted number of bytes to copy that was added to
+  // the total value for the collection set.
+  size_t _predicted_bytes_to_copy;
+
  public:
   // If "is_zeroed" is "true", the region "mr" can be assumed to contain zeros.
   HeapRegion(G1BlockOffsetSharedArray* sharedOffsetArray,
@@ -614,7 +627,6 @@ class HeapRegion: public G1OffsetTableContigSpace {
   // </PREDICTION>
 
   bool is_young() const     { return _young_type != NotYoung; }
-  bool is_scan_only() const { return _young_type == ScanOnly; }
   bool is_survivor() const  { return _young_type == Survivor; }
 
   int  young_index_in_cset() const { return _young_index_in_cset; }
@@ -629,12 +641,6 @@ class HeapRegion: public G1OffsetTableContigSpace {
     return _surv_rate_group->age_in_group(_age_index);
   }
 
-  void recalculate_age_in_surv_rate_group() {
-    assert( _surv_rate_group != NULL, "pre-condition" );
-    assert( _age_index > -1, "pre-condition" );
-    _age_index = _surv_rate_group->recalculate_age_index(_age_index);
-  }
-
   void record_surv_words_in_group(size_t words_survived) {
     assert( _surv_rate_group != NULL, "pre-condition" );
     assert( _age_index > -1, "pre-condition" );
@@ -676,8 +682,6 @@ class HeapRegion: public G1OffsetTableContigSpace {
 
   void set_young() { set_young_type(Young); }
 
-  void set_scan_only() { set_young_type(ScanOnly); }
-
   void set_survivor() { set_young_type(Survivor); }
 
   void set_not_young() { set_young_type(NotYoung); }
@@ -775,6 +779,22 @@ class HeapRegion: public G1OffsetTableContigSpace {
     _zero_filler = NULL;
   }
 
+  size_t recorded_rs_length() const        { return _recorded_rs_length; }
+  double predicted_elapsed_time_ms() const { return _predicted_elapsed_time_ms; }
+  size_t predicted_bytes_to_copy() const   { return _predicted_bytes_to_copy; }
+
+  void set_recorded_rs_length(size_t rs_length) {
+    _recorded_rs_length = rs_length;
+  }
+
+  void set_predicted_elapsed_time_ms(double ms) {
+    _predicted_elapsed_time_ms = ms;
+  }
+
+  void set_predicted_bytes_to_copy(size_t bytes) {
+    _predicted_bytes_to_copy = bytes;
+  }
+
 #define HeapRegion_OOP_SINCE_SAVE_MARKS_DECL(OopClosureType, nv_suffix)  \
   virtual void oop_since_save_marks_iterate##nv_suffix(OopClosureType* cl);
   SPECIALIZED_SINCE_SAVE_MARKS_CLOSURES(HeapRegion_OOP_SINCE_SAVE_MARKS_DECL)

hotspot/src/share/vm/gc_implementation/g1/survRateGroup.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright 2001-2009 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2001-2010 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -55,7 +55,6 @@ SurvRateGroup::SurvRateGroup(G1CollectorPolicy* g1p,
 void SurvRateGroup::reset()
 {
   _all_regions_allocated = 0;
-  _scan_only_prefix      = 0;
   _setup_seq_num         = 0;
   _stats_arrays_length   = 0;
   _accum_surv_rate       = 0.0;
@@ -74,7 +73,7 @@ void SurvRateGroup::reset()
 void
 SurvRateGroup::start_adding_regions() {
   _setup_seq_num   = _stats_arrays_length;
-  _region_num      = _scan_only_prefix;
+  _region_num      = 0;
   _accum_surv_rate = 0.0;
 
 #if 0
@@ -163,12 +162,6 @@ SurvRateGroup::next_age_index() {
   return (int) ++_all_regions_allocated;
 }
 
-void
-SurvRateGroup::record_scan_only_prefix(size_t scan_only_prefix) {
-  guarantee( scan_only_prefix <= _region_num, "pre-condition" );
-  _scan_only_prefix = scan_only_prefix;
-}
-
 void
 SurvRateGroup::record_surviving_words(int age_in_group, size_t surv_words) {
   guarantee( 0 <= age_in_group && (size_t) age_in_group < _region_num,
@@ -218,13 +211,12 @@ SurvRateGroup::all_surviving_words_recorded(bool propagate) {
 #ifndef PRODUCT
 void
 SurvRateGroup::print() {
-  gclog_or_tty->print_cr("Surv Rate Group: %s (%d entries, %d scan-only)",
+  gclog_or_tty->print_cr("Surv Rate Group: %s (%d entries)",
-                _name, _region_num, _scan_only_prefix);
+                _name, _region_num);
   for (size_t i = 0; i < _region_num; ++i) {
-    gclog_or_tty->print_cr("    age %4d   surv rate %6.2lf %%   pred %6.2lf %%%s",
+    gclog_or_tty->print_cr("    age %4d   surv rate %6.2lf %%   pred %6.2lf %%",
                   i, _surv_rate[i] * 100.0,
-                  _g1p->get_new_prediction(_surv_rate_pred[i]) * 100.0,
+                  _g1p->get_new_prediction(_surv_rate_pred[i]) * 100.0);
-                  (i < _scan_only_prefix) ? " S-O" : "    ");
   }
 }
 

hotspot/src/share/vm/gc_implementation/g1/survRateGroup.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright 2001-2009 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2001-2010 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -41,7 +41,6 @@ private:
 
   int _all_regions_allocated;
   size_t _region_num;
-  size_t _scan_only_prefix;
   size_t _setup_seq_num;
 
 public:
@@ -51,13 +50,11 @@ public:
   void reset();
   void start_adding_regions();
   void stop_adding_regions();
-  void record_scan_only_prefix(size_t scan_only_prefix);
   void record_surviving_words(int age_in_group, size_t surv_words);
   void all_surviving_words_recorded(bool propagate);
   const char* name() { return _name; }
 
   size_t region_num() { return _region_num; }
-  size_t scan_only_length() { return _scan_only_prefix; }
   double accum_surv_rate_pred(int age) {
     assert(age >= 0, "must be");
     if ((size_t)age < _stats_arrays_length)
@@ -82,17 +79,12 @@ public:
 
   int next_age_index();
   int age_in_group(int age_index) {
-    int ret = (int) (_all_regions_allocated -  age_index);
+    int ret = (int) (_all_regions_allocated - age_index);
     assert( ret >= 0, "invariant" );
     return ret;
   }
-  int recalculate_age_index(int age_index) {
-    int new_age_index = (int) _scan_only_prefix - age_in_group(age_index);
-    guarantee( new_age_index >= 0, "invariant" );
-    return new_age_index;
-  }
   void finished_recalculating_age_indexes() {
-    _all_regions_allocated = (int) _scan_only_prefix;
+    _all_regions_allocated = 0;
   }
 
 #ifndef PRODUCT

hotspot/src/share/vm/services/g1MemoryPool.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2007 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2007-2010 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -45,7 +45,7 @@ size_t G1MemoryPoolSuper::eden_space_committed(G1CollectedHeap* g1h) {
 
 // See the comment at the top of g1MemoryPool.hpp
 size_t G1MemoryPoolSuper::eden_space_used(G1CollectedHeap* g1h) {
-  size_t young_list_length = g1h->young_list_length();
+  size_t young_list_length = g1h->young_list()->length();
   size_t eden_used = young_list_length * HeapRegion::GrainBytes;
   size_t survivor_used = survivor_space_used(g1h);
   eden_used = subtract_up_to_zero(eden_used, survivor_used);