8241141: Restructure humongous object allocation in G1

Reviewed-by: tschatzl, kbarrett

src/hotspot/share/gc/g1/g1CollectedHeap.cpp

@@ -209,14 +209,15 @@ HeapRegion* G1CollectedHeap::new_region(size_t word_size,
 }
 
 HeapWord*
-G1CollectedHeap::humongous_obj_allocate_initialize_regions(uint first,
+G1CollectedHeap::humongous_obj_allocate_initialize_regions(HeapRegion* first_hr,
                                                            uint num_regions,
                                                            size_t word_size) {
-  assert(first != G1_NO_HRM_INDEX, "pre-condition");
+  assert(first_hr != NULL, "pre-condition");
   assert(is_humongous(word_size), "word_size should be humongous");
   assert(num_regions * HeapRegion::GrainWords >= word_size, "pre-condition");
 
   // Index of last region in the series.
+  uint first = first_hr->hrm_index();
   uint last = first + num_regions - 1;
 
   // We need to initialize the region(s) we just discovered. This is
@@ -231,10 +232,8 @@ G1CollectedHeap::humongous_obj_allocate_initialize_regions(uint first,
   size_t word_size_sum = (size_t) num_regions * HeapRegion::GrainWords;
   assert(word_size <= word_size_sum, "sanity");
 
-  // This will be the "starts humongous" region.
-  HeapRegion* first_hr = region_at(first);
-  // The header of the new object will be placed at the bottom of
-  // the first region.
+  // The passed in hr will be the "starts humongous" region. The header
+  // of the new object will be placed at the bottom of this region.
   HeapWord* new_obj = first_hr->bottom();
   // This will be the new top of the new object.
   HeapWord* obj_top = new_obj + word_size;
@@ -340,57 +339,28 @@ HeapWord* G1CollectedHeap::humongous_obj_allocate(size_t word_size) {
 
   _verifier->verify_region_sets_optional();
 
-  uint first = G1_NO_HRM_INDEX;
   uint obj_regions = (uint) humongous_obj_size_in_regions(word_size);
 
-  if (obj_regions == 1) {
-    // Only one region to allocate, try to use a fast path by directly allocating
-    // from the free lists. Do not try to expand here, we will potentially do that
-    // later.
-    HeapRegion* hr = new_region(word_size, HeapRegionType::Humongous, false /* do_expand */);
-    if (hr != NULL) {
-      first = hr->hrm_index();
-    }
-  } else {
-    // Policy: Try only empty regions (i.e. already committed first). Maybe we
-    // are lucky enough to find some.
-    first = _hrm->find_contiguous_only_empty(obj_regions);
-    if (first != G1_NO_HRM_INDEX) {
-      _hrm->allocate_free_regions_starting_at(first, obj_regions);
-    }
-  }
-
-  if (first == G1_NO_HRM_INDEX) {
+  // Policy: First try to allocate a humongous object in the free list.
+  HeapRegion* humongous_start = _hrm->allocate_humongous(obj_regions);
+  if (humongous_start == NULL) {
     // Policy: We could not find enough regions for the humongous object in the
     // free list. Look through the heap to find a mix of free and uncommitted regions.
-    // If so, try expansion.
-    first = _hrm->find_contiguous_empty_or_unavailable(obj_regions);
-    if (first != G1_NO_HRM_INDEX) {
-      // We found something. Make sure these regions are committed, i.e. expand
-      // the heap. Alternatively we could do a defragmentation GC.
-      log_debug(gc, ergo, heap)("Attempt heap expansion (humongous allocation request failed). Allocation request: " SIZE_FORMAT "B",
-                                    word_size * HeapWordSize);
-
-      _hrm->expand_at(first, obj_regions, workers());
+    // If so, expand the heap and allocate the humongous object.
+    humongous_start = _hrm->expand_and_allocate_humongous(obj_regions);
+    if (humongous_start != NULL) {
+      // We managed to find a region by expanding the heap.
+      log_debug(gc, ergo, heap)("Heap expansion (humongous allocation request). Allocation request: " SIZE_FORMAT "B",
+                                word_size * HeapWordSize);
       policy()->record_new_heap_size(num_regions());
-
-#ifdef ASSERT
-      for (uint i = first; i < first + obj_regions; ++i) {
-        HeapRegion* hr = region_at(i);
-        assert(hr->is_free(), "sanity");
-        assert(hr->is_empty(), "sanity");
-        assert(is_on_master_free_list(hr), "sanity");
-      }
-#endif
-      _hrm->allocate_free_regions_starting_at(first, obj_regions);
     } else {
       // Policy: Potentially trigger a defragmentation GC.
     }
   }
 
   HeapWord* result = NULL;
-  if (first != G1_NO_HRM_INDEX) {
-    result = humongous_obj_allocate_initialize_regions(first, obj_regions, word_size);
+  if (humongous_start != NULL) {
+    result = humongous_obj_allocate_initialize_regions(humongous_start, obj_regions, word_size);
     assert(result != NULL, "it should always return a valid result");
 
     // A successful humongous object allocation changes the used space
@@ -4892,8 +4862,7 @@ HeapRegion* G1CollectedHeap::alloc_highest_free_region() {
       log_debug(gc, ergo, heap)("Attempt heap expansion (requested address range outside heap bounds). region size: " SIZE_FORMAT "B",
                                 HeapRegion::GrainWords * HeapWordSize);
     }
-    _hrm->allocate_free_regions_starting_at(index, 1);
-    return region_at(index);
+    return _hrm->allocate_free_regions_starting_at(index, 1);
   }
   return NULL;
 }

src/hotspot/share/gc/g1/g1CollectedHeap.hpp

@@ -409,7 +409,7 @@ private:
   // Initialize a contiguous set of free regions of length num_regions
   // and starting at index first so that they appear as a single
   // humongous region.
-  HeapWord* humongous_obj_allocate_initialize_regions(uint first,
+  HeapWord* humongous_obj_allocate_initialize_regions(HeapRegion* first_hr,
                                                       uint num_regions,
                                                       size_t word_size);
 

src/hotspot/share/gc/g1/heapRegionManager.cpp

@@ -135,6 +135,35 @@ HeapRegion* HeapRegionManager::allocate_free_region(HeapRegionType type, uint re
   return hr;
 }
 
+HeapRegion* HeapRegionManager::allocate_humongous_from_free_list(uint num_regions) {
+  uint candidate = find_contiguous_in_free_list(num_regions);
+  if (candidate == G1_NO_HRM_INDEX) {
+    return NULL;
+  }
+  return allocate_free_regions_starting_at(candidate, num_regions);
+}
+
+HeapRegion* HeapRegionManager::allocate_humongous_allow_expand(uint num_regions) {
+  uint candidate = find_contiguous_allow_expand(num_regions);
+  if (candidate == G1_NO_HRM_INDEX) {
+    return NULL;
+  }
+  expand_exact(candidate, num_regions, G1CollectedHeap::heap()->workers());
+  return allocate_free_regions_starting_at(candidate, num_regions);
+}
+
+HeapRegion* HeapRegionManager::allocate_humongous(uint num_regions) {
+  // Special case a single region to avoid expensive search.
+  if (num_regions == 1) {
+    return allocate_free_region(HeapRegionType::Humongous, G1NUMA::AnyNodeIndex);
+  }
+  return allocate_humongous_from_free_list(num_regions);
+}
+
+HeapRegion* HeapRegionManager::expand_and_allocate_humongous(uint num_regions) {
+  return allocate_humongous_allow_expand(num_regions);
+}
+
 #ifdef ASSERT
 bool HeapRegionManager::is_free(HeapRegion* hr) const {
   return _free_list.contains(hr);
@@ -271,6 +300,19 @@ uint HeapRegionManager::expand_at(uint start, uint num_regions, WorkGang* pretou
   return expanded;
 }
 
+void HeapRegionManager::expand_exact(uint start, uint num_regions, WorkGang* pretouch_workers) {
+  assert(num_regions != 0, "Need to request at least one region");
+  uint end = start + num_regions;
+
+  for (uint i = start; i < end; i++) {
+    if (!is_available(i)) {
+      make_regions_available(i, 1, pretouch_workers);
+    }
+  }
+
+  verify_optional();
+}
+
 uint HeapRegionManager::expand_on_preferred_node(uint preferred_index) {
   uint expand_candidate = UINT_MAX;
   for (uint i = 0; i < max_length(); i++) {
@@ -291,7 +333,7 @@ uint HeapRegionManager::expand_on_preferred_node(uint preferred_index) {
     return 0;
   }
 
-  make_regions_available(expand_candidate, 1, NULL);
+  expand_exact(expand_candidate, 1, NULL);
   return 1;
 }
 
@@ -300,36 +342,61 @@ bool HeapRegionManager::is_on_preferred_index(uint region_index, uint preferred_
   return region_node_index == preferred_node_index;
 }
 
-uint HeapRegionManager::find_contiguous(size_t num, bool empty_only) {
-  uint found = 0;
-  size_t length_found = 0;
-  uint cur = 0;
-
-  while (length_found < num && cur < max_length()) {
-    HeapRegion* hr = _regions.get_by_index(cur);
-    if ((!empty_only && !is_available(cur)) || (is_available(cur) && hr != NULL && hr->is_empty())) {
-      // This region is a potential candidate for allocation into.
-      length_found++;
-    } else {
-      // This region is not a candidate. The next region is the next possible one.
-      found = cur + 1;
-      length_found = 0;
-    }
-    cur++;
+void HeapRegionManager::guarantee_contiguous_range(uint start, uint num_regions) {
+  // General sanity check, regions found should either be available and empty
+  // or not available so that we can make them available and use them.
+  for (uint i = start; i < (start + num_regions); i++) {
+    HeapRegion* hr = _regions.get_by_index(i);
+    guarantee(!is_available(i) || hr->is_free(),
+              "Found region sequence starting at " UINT32_FORMAT ", length " UINT32_FORMAT
+              " that is not free at " UINT32_FORMAT ". Hr is " PTR_FORMAT ", type is %s",
+              start, num_regions, i, p2i(hr), hr->get_type_str());
   }
+}
 
-  if (length_found == num) {
-    for (uint i = found; i < (found + num); i++) {
-      HeapRegion* hr = _regions.get_by_index(i);
-      // sanity check
-      guarantee((!empty_only && !is_available(i)) || (is_available(i) && hr != NULL && hr->is_empty()),
-                "Found region sequence starting at " UINT32_FORMAT ", length " SIZE_FORMAT
-                " that is not empty at " UINT32_FORMAT ". Hr is " PTR_FORMAT, found, num, i, p2i(hr));
+uint HeapRegionManager::find_contiguous_in_range(uint start, uint end, uint num_regions) {
+  assert(start <= end, "precondition");
+  assert(num_regions >= 1, "precondition");
+  uint candidate = start;       // First region in candidate sequence.
+  uint unchecked = candidate;   // First unchecked region in candidate.
+  // While the candidate sequence fits in the range...
+  while (num_regions <= (end - candidate)) {
+    // Walk backward over the regions for the current candidate.
+    for (uint i = candidate + num_regions - 1; true; --i) {
+      if (is_available(i) && !at(i)->is_free()) {
+        // Region i can't be used, so restart with i+1 as the start
+        // of a new candidate sequence, and with the region after the
+        // old candidate sequence being the first unchecked region.
+        unchecked = candidate + num_regions;
+        candidate = i + 1;
+        break;
+      } else if (i == unchecked) {
+        // All regions of candidate sequence have passed check.
+        guarantee_contiguous_range(candidate, num_regions);
+        return candidate;
+      }
     }
-    return found;
-  } else {
-    return G1_NO_HRM_INDEX;
   }
+  return G1_NO_HRM_INDEX;
+}
+
+uint HeapRegionManager::find_contiguous_in_free_list(uint num_regions) {
+  BitMap::idx_t range_start = 0;
+  BitMap::idx_t range_end = range_start;
+  uint candidate = G1_NO_HRM_INDEX;
+
+  do {
+    range_start = _available_map.get_next_one_offset(range_end);
+    range_end = _available_map.get_next_zero_offset(range_start);
+    candidate = find_contiguous_in_range((uint) range_start, (uint) range_end, num_regions);
+  } while (candidate == G1_NO_HRM_INDEX && range_end < max_length());
+
+  return candidate;
+}
+
+uint HeapRegionManager::find_contiguous_allow_expand(uint num_regions) {
+  // Find any candidate.
+  return find_contiguous_in_range(0, max_length(), num_regions);
 }
 
 HeapRegion* HeapRegionManager::next_region_in_heap(const HeapRegion* r) const {

src/hotspot/share/gc/g1/heapRegionManager.hpp

@@ -94,11 +94,19 @@ class HeapRegionManager: public CHeapObj<mtGC> {
   // Notify other data structures about change in the heap layout.
   void update_committed_space(HeapWord* old_end, HeapWord* new_end);
 
-  // Find a contiguous set of empty or uncommitted regions of length num and return
+  // Find a contiguous set of empty or uncommitted regions of length num_regions and return
   // the index of the first region or G1_NO_HRM_INDEX if the search was unsuccessful.
-  // If only_empty is true, only empty regions are considered.
-  // Searches from bottom to top of the heap, doing a first-fit.
-  uint find_contiguous(size_t num, bool only_empty);
+  // Start and end defines the range to seek in, policy is first-fit.
+  uint find_contiguous_in_range(uint start, uint end, uint num_regions);
+  // Find a contiguous set of empty regions of length num_regions. Returns the start index
+  // of that set, or G1_NO_HRM_INDEX.
+  uint find_contiguous_in_free_list(uint num_regions);
+  // Find a contiguous set of empty or unavailable regions of length num_regions. Returns the
+  // start index of that set, or G1_NO_HRM_INDEX.
+  uint find_contiguous_allow_expand(uint num_regions);
+
+  void guarantee_contiguous_range(uint start, uint num_regions) ;
+
   // Finds the next sequence of unavailable regions starting from start_idx. Returns the
   // length of the sequence found. If this result is zero, no such sequence could be found,
   // otherwise res_idx indicates the start index of these regions.
@@ -122,6 +130,14 @@ protected:
   void uncommit_regions(uint index, size_t num_regions = 1);
   // Allocate a new HeapRegion for the given index.
   HeapRegion* new_heap_region(uint hrm_index);
+
+  // Humongous allocation helpers
+  virtual HeapRegion* allocate_humongous_from_free_list(uint num_regions);
+  virtual HeapRegion* allocate_humongous_allow_expand(uint num_regions);
+
+  // Expand helper for cases when the regions to expand are well defined.
+  void expand_exact(uint start, uint num_regions, WorkGang* pretouch_workers);
+
 #ifdef ASSERT
 public:
   bool is_free(HeapRegion* hr) const;
@@ -183,7 +199,13 @@ public:
   // Allocate a free region with specific node index. If fails allocate with next node index.
   virtual HeapRegion* allocate_free_region(HeapRegionType type, uint requested_node_index);
 
-  inline void allocate_free_regions_starting_at(uint first, uint num_regions);
+  // Allocate a humongous object from the free list
+  HeapRegion* allocate_humongous(uint num_regions);
+
+  // Allocate a humongous object by expanding the heap
+  HeapRegion* expand_and_allocate_humongous(uint num_regions);
+
+  inline HeapRegion* allocate_free_regions_starting_at(uint first, uint num_regions);
 
   // Remove all regions from the free list.
   void remove_all_free_regions() {
@@ -233,13 +255,6 @@ public:
   // Try to expand on the given node index.
   virtual uint expand_on_preferred_node(uint node_index);
 
-  // Find a contiguous set of empty regions of length num. Returns the start index of
-  // that set, or G1_NO_HRM_INDEX.
-  virtual uint find_contiguous_only_empty(size_t num) { return find_contiguous(num, true); }
-  // Find a contiguous set of empty or unavailable regions of length num. Returns the
-  // start index of that set, or G1_NO_HRM_INDEX.
-  virtual uint find_contiguous_empty_or_unavailable(size_t num) { return find_contiguous(num, false); }
-
   HeapRegion* next_region_in_heap(const HeapRegion* r) const;
 
   // Find the highest free or uncommitted region in the reserved heap,

src/hotspot/share/gc/g1/heapRegionManager.inline.hpp

@@ -73,8 +73,10 @@ inline void HeapRegionManager::insert_into_free_list(HeapRegion* hr) {
   _free_list.add_ordered(hr);
 }
 
-inline void HeapRegionManager::allocate_free_regions_starting_at(uint first, uint num_regions) {
-  _free_list.remove_starting_at(at(first), num_regions);
+inline HeapRegion* HeapRegionManager::allocate_free_regions_starting_at(uint first, uint num_regions) {
+  HeapRegion* start = at(first);
+  _free_list.remove_starting_at(start, num_regions);
+  return start;
 }
 
 #endif // SHARE_GC_G1_HEAPREGIONMANAGER_INLINE_HPP

src/hotspot/share/gc/g1/heapRegionSet.hpp

@@ -229,7 +229,7 @@ public:
 
   // Remove all (contiguous) regions from first to first + num_regions -1 from
   // this list.
-  // Num_regions must be > 1.
+  // Num_regions must be >= 1.
   void remove_starting_at(HeapRegion* first, uint num_regions);
 
   virtual void verify();

src/hotspot/share/gc/g1/heterogeneousHeapRegionManager.cpp

@@ -325,18 +325,28 @@ HeapRegion* HeterogeneousHeapRegionManager::allocate_free_region(HeapRegionType
   return hr;
 }
 
-uint HeterogeneousHeapRegionManager::find_contiguous_only_empty(size_t num) {
+HeapRegion* HeterogeneousHeapRegionManager::allocate_humongous_from_free_list(uint num_regions) {
   if (has_borrowed_regions()) {
-      return G1_NO_HRM_INDEX;
+      return NULL;
   }
-  return find_contiguous(start_index_of_nvdimm(), end_index_of_nvdimm(), num, true);
+  uint candidate = find_contiguous(start_index_of_nvdimm(), end_index_of_nvdimm(), num_regions, true);
+  if (candidate == G1_NO_HRM_INDEX) {
+    return NULL;
+  }
+  return allocate_free_regions_starting_at(candidate, num_regions);
 }
 
-uint HeterogeneousHeapRegionManager::find_contiguous_empty_or_unavailable(size_t num) {
+HeapRegion* HeterogeneousHeapRegionManager::allocate_humongous_allow_expand(uint num_regions) {
   if (has_borrowed_regions()) {
-    return G1_NO_HRM_INDEX;
+    return NULL;
   }
-  return find_contiguous(start_index_of_nvdimm(), end_index_of_nvdimm(), num, false);
+  uint candidate = find_contiguous(start_index_of_nvdimm(), end_index_of_nvdimm(), num_regions, false);
+  if (candidate == G1_NO_HRM_INDEX) {
+    return NULL;
+  }
+
+  expand_exact(candidate, num_regions, NULL);
+  return allocate_free_regions_starting_at(candidate, num_regions);
 }
 
 uint HeterogeneousHeapRegionManager::find_contiguous(size_t start, size_t end, size_t num, bool empty_only) {

src/hotspot/share/gc/g1/heterogeneousHeapRegionManager.hpp

@@ -120,6 +120,8 @@ public:
   void prepare_for_full_collection_end();
 
   virtual HeapRegion* allocate_free_region(HeapRegionType type, uint node_index);
+  virtual HeapRegion* allocate_humongous_from_free_list(uint num_regions);
+  virtual HeapRegion* allocate_humongous_allow_expand(uint num_regions);
 
   // Return maximum number of regions that heap can expand to.
   uint max_expandable_length() const;
@@ -130,12 +132,6 @@ public:
   // Override.
   uint expand_at(uint start, uint num_regions, WorkGang* pretouch_workers);
 
-  // Override. This function is called for humongous allocation, so we need to find empty regions in nv-dimm.
-  uint find_contiguous_only_empty(size_t num);
-
-  // Override. This function is called for humongous allocation, so we need to find empty or unavailable regions in nv-dimm.
-  uint find_contiguous_empty_or_unavailable(size_t num);
-
   // Overrides base class implementation to find highest free region in dram.
   uint find_highest_free(bool* expanded);