8137022: Concurrent refinement thread adjustment and (de-)activation suboptimal

8155996: Improve concurrent refinement green zone control
8134303: Introduce -XX:-G1UseConcRefinement

Reviewed-by: sjohanss, tschatzl, iwalulya, ayang

src/hotspot/share/gc/g1/g1Analytics.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016, 2021, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2016, 2022, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -82,6 +82,7 @@ G1Analytics::G1Analytics(const G1Predictions* predictor) :
     _mixed_rs_length_diff_seq(new TruncatedSeq(TruncatedSeqLength)),
     _concurrent_refine_rate_ms_seq(new TruncatedSeq(TruncatedSeqLength)),
     _dirtied_cards_rate_ms_seq(new TruncatedSeq(TruncatedSeqLength)),
+    _dirtied_cards_in_thread_buffers_seq(new TruncatedSeq(TruncatedSeqLength)),
     _young_card_scan_to_merge_ratio_seq(new TruncatedSeq(TruncatedSeqLength)),
     _mixed_card_scan_to_merge_ratio_seq(new TruncatedSeq(TruncatedSeqLength)),
     _young_cost_per_card_scan_ms_seq(new TruncatedSeq(TruncatedSeqLength)),
@@ -172,6 +173,10 @@ void G1Analytics::report_dirtied_cards_rate_ms(double cards_per_ms) {
   _dirtied_cards_rate_ms_seq->add(cards_per_ms);
 }
 
+void G1Analytics::report_dirtied_cards_in_thread_buffers(size_t cards) {
+  _dirtied_cards_in_thread_buffers_seq->add(double(cards));
+}
+
 void G1Analytics::report_cost_per_card_scan_ms(double cost_per_card_ms, bool for_young_only_phase) {
   if  (for_young_only_phase) {
     _young_cost_per_card_scan_ms_seq->add(cost_per_card_ms);
@@ -256,6 +261,10 @@ double G1Analytics::predict_dirtied_cards_rate_ms() const {
   return predict_zero_bounded(_dirtied_cards_rate_ms_seq);
 }
 
+size_t G1Analytics::predict_dirtied_cards_in_thread_buffers() const {
+  return predict_size(_dirtied_cards_in_thread_buffers_seq);
+}
+
 size_t G1Analytics::predict_scan_card_num(size_t rs_length, bool for_young_only_phase) const {
   if (for_young_only_phase || !enough_samples_available(_mixed_card_scan_to_merge_ratio_seq)) {
     return (size_t)(rs_length * predict_in_unit_interval(_young_card_scan_to_merge_ratio_seq));

src/hotspot/share/gc/g1/g1Analytics.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016, 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2016, 2022, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -49,6 +49,7 @@ class G1Analytics: public CHeapObj<mtGC> {
   TruncatedSeq* _mixed_rs_length_diff_seq;
   TruncatedSeq* _concurrent_refine_rate_ms_seq;
   TruncatedSeq* _dirtied_cards_rate_ms_seq;
+  TruncatedSeq* _dirtied_cards_in_thread_buffers_seq;
   // The ratio between the number of scanned cards and actually merged cards, for
   // young-only and mixed gcs.
   TruncatedSeq* _young_card_scan_to_merge_ratio_seq;
@@ -126,6 +127,7 @@ public:
   void report_alloc_rate_ms(double alloc_rate);
   void report_concurrent_refine_rate_ms(double cards_per_ms);
   void report_dirtied_cards_rate_ms(double cards_per_ms);
+  void report_dirtied_cards_in_thread_buffers(size_t num_cards);
   void report_cost_per_card_scan_ms(double cost_per_remset_card_ms, bool for_young_only_phase);
   void report_cost_per_card_merge_ms(double cost_per_card_ms, bool for_young_only_phase);
   void report_card_scan_to_merge_ratio(double cards_per_entry_ratio, bool for_young_only_phase);
@@ -142,6 +144,7 @@ public:
 
   double predict_concurrent_refine_rate_ms() const;
   double predict_dirtied_cards_rate_ms() const;
+  size_t predict_dirtied_cards_in_thread_buffers() const;
 
   // Predict how many of the given remembered set of length rs_length will add to
   // the number of total cards scanned.

src/hotspot/share/gc/g1/g1Arguments.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2018, 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2018, 2022, Oracle and/or its affiliates. All rights reserved.
  * Copyright (c) 2017, Red Hat, Inc. and/or its affiliates.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
@@ -177,7 +177,13 @@ void G1Arguments::initialize() {
     FLAG_SET_ERGO(ParallelGCThreads, 1);
   }
 
-  if (FLAG_IS_DEFAULT(G1ConcRefinementThreads)) {
+  if (!G1UseConcRefinement) {
+    if (!FLAG_IS_DEFAULT(G1ConcRefinementThreads)) {
+      log_warning(gc, ergo)("Ignoring -XX:G1ConcRefinementThreads "
+                            "because of -XX:-G1UseConcRefinement");
+    }
+    FLAG_SET_DEFAULT(G1ConcRefinementThreads, 0);
+  } else if (FLAG_IS_DEFAULT(G1ConcRefinementThreads)) {
     FLAG_SET_ERGO(G1ConcRefinementThreads, ParallelGCThreads);
   }
 

src/hotspot/share/gc/g1/g1CollectedHeap.cpp

@@ -1533,7 +1533,7 @@ G1RegionToSpaceMapper* G1CollectedHeap::create_aux_memory_mapper(const char* des
 
 jint G1CollectedHeap::initialize_concurrent_refinement() {
   jint ecode = JNI_OK;
-  _cr = G1ConcurrentRefine::create(&ecode);
+  _cr = G1ConcurrentRefine::create(policy(), &ecode);
   return ecode;
 }
 
@@ -1713,9 +1713,6 @@ jint G1CollectedHeap::initialize() {
     return ecode;
   }
 
-  // Initialize and schedule sampling task on service thread.
-  _rem_set->initialize_sampling_task(service_thread());
-
   // Create and schedule the periodic gc task on the service thread.
   _periodic_gc_task = new G1PeriodicGCTask("Periodic GC Task");
   _service_thread->register_task(_periodic_gc_task);

src/hotspot/share/gc/g1/g1ConcurrentRefine.cpp

@@ -24,71 +24,85 @@
 
 #include "precompiled.hpp"
 #include "gc/g1/g1BarrierSet.hpp"
+#include "gc/g1/g1CollectionSet.hpp"
 #include "gc/g1/g1ConcurrentRefine.hpp"
 #include "gc/g1/g1ConcurrentRefineThread.hpp"
 #include "gc/g1/g1DirtyCardQueue.hpp"
+#include "gc/g1/g1Policy.hpp"
+#include "gc/g1/heapRegion.inline.hpp"
+#include "gc/g1/heapRegionRemSet.inline.hpp"
+#include "gc/shared/gc_globals.hpp"
 #include "logging/log.hpp"
 #include "memory/allocation.inline.hpp"
 #include "memory/iterator.hpp"
-#include "runtime/globals_extension.hpp"
 #include "runtime/java.hpp"
-#include "runtime/javaThread.hpp"
+#include "runtime/mutexLocker.hpp"
 #include "utilities/debug.hpp"
-#include "utilities/formatBuffer.hpp"
 #include "utilities/globalDefinitions.hpp"
-#include "utilities/pair.hpp"
 #include <math.h>
 
 G1ConcurrentRefineThread* G1ConcurrentRefineThreadControl::create_refinement_thread(uint worker_id, bool initializing) {
-  G1ConcurrentRefineThread* result = NULL;
+  G1ConcurrentRefineThread* result = nullptr;
   if (initializing || !InjectGCWorkerCreationFailure) {
     result = G1ConcurrentRefineThread::create(_cr, worker_id);
   }
-  if (result == NULL || result->osthread() == NULL) {
+  if (result == nullptr || result->osthread() == nullptr) {
     log_warning(gc)("Failed to create refinement thread %u, no more %s",
                     worker_id,
-                    result == NULL ? "memory" : "OS threads");
+                    result == nullptr ? "memory" : "OS threads");
+    if (result != nullptr) {
+      delete result;
+      result = nullptr;
+    }
   }
   return result;
 }
 
 G1ConcurrentRefineThreadControl::G1ConcurrentRefineThreadControl() :
   _cr(nullptr),
-  _primary_thread(nullptr),
   _threads(nullptr),
-  _num_max_threads(0)
+  _max_num_threads(0)
-{
+{}
-}
 
 G1ConcurrentRefineThreadControl::~G1ConcurrentRefineThreadControl() {
-  for (uint i = 0; i < _num_max_threads; i++) {
+  if (_threads != nullptr) {
+    for (uint i = 0; i < _max_num_threads; i++) {
       G1ConcurrentRefineThread* t = _threads[i];
-    if (t != NULL) {
+      if (t == nullptr) {
+#ifdef ASSERT
+        for (uint j = i + 1; j < _max_num_threads; ++j) {
+          assert(_threads[j] == nullptr, "invariant");
+        }
+#endif // ASSERT
+        break;
+      } else {
         delete t;
       }
     }
     FREE_C_HEAP_ARRAY(G1ConcurrentRefineThread*, _threads);
+  }
 }
 
-jint G1ConcurrentRefineThreadControl::initialize(G1ConcurrentRefine* cr, uint num_max_threads) {
+jint G1ConcurrentRefineThreadControl::initialize(G1ConcurrentRefine* cr, uint max_num_threads) {
   assert(cr != NULL, "G1ConcurrentRefine must not be NULL");
   _cr = cr;
-  _num_max_threads = num_max_threads;
+  _max_num_threads = max_num_threads;
 
-  _threads = NEW_C_HEAP_ARRAY(G1ConcurrentRefineThread*, num_max_threads, mtGC);
+  if (max_num_threads > 0) {
+    _threads = NEW_C_HEAP_ARRAY(G1ConcurrentRefineThread*, max_num_threads, mtGC);
 
-  if (num_max_threads > 0) {
+    _threads[0] = create_refinement_thread(0, true);
-    auto primary = G1PrimaryConcurrentRefineThread::create(cr);
+    if (_threads[0] == nullptr) {
-    if (primary == nullptr) {
       vm_shutdown_during_initialization("Could not allocate primary refinement thread");
       return JNI_ENOMEM;
     }
-    _threads[0] = _primary_thread = primary;
 
-    for (uint i = 1; i < num_max_threads; ++i) {
     if (UseDynamicNumberOfGCThreads) {
+      for (uint i = 1; i < max_num_threads; ++i) {
         _threads[i] = nullptr;
+      }
     } else {
+      for (uint i = 1; i < max_num_threads; ++i) {
         _threads[i] = create_refinement_thread(i, true);
         if (_threads[i] == nullptr) {
           vm_shutdown_during_initialization("Could not allocate refinement threads.");
@@ -101,29 +115,29 @@ jint G1ConcurrentRefineThreadControl::initialize(G1ConcurrentRefine* cr, uint nu
   return JNI_OK;
 }
 
-void G1ConcurrentRefineThreadControl::maybe_activate_next(uint cur_worker_id) {
+#ifdef ASSERT
-  assert(cur_worker_id < _num_max_threads,
+void G1ConcurrentRefineThreadControl::assert_current_thread_is_primary_refinement_thread() const {
-         "Activating another thread from %u not allowed since there can be at most %u",
+  assert(_threads != nullptr, "No threads");
-         cur_worker_id, _num_max_threads);
+  assert(Thread::current() == _threads[0], "Not primary thread");
-  if (cur_worker_id == (_num_max_threads - 1)) {
+}
-    // Already the last thread, there is no more thread to activate.
+#endif // ASSERT
-    return;
-  }
 
-  uint worker_id = cur_worker_id + 1;
+bool G1ConcurrentRefineThreadControl::activate(uint worker_id) {
+  assert(worker_id < _max_num_threads, "precondition");
   G1ConcurrentRefineThread* thread_to_activate = _threads[worker_id];
-  if (thread_to_activate == NULL) {
+  if (thread_to_activate == nullptr) {
-    // Still need to create the thread...
+    thread_to_activate = create_refinement_thread(worker_id, false);
-    _threads[worker_id] = create_refinement_thread(worker_id, false);
+    if (thread_to_activate == nullptr) {
-    thread_to_activate = _threads[worker_id];
+      return false;
+    }
+    _threads[worker_id] = thread_to_activate;
   }
-  if (thread_to_activate != NULL) {
   thread_to_activate->activate();
-  }
+  return true;
 }
 
 void G1ConcurrentRefineThreadControl::worker_threads_do(ThreadClosure* tc) {
-  for (uint i = 0; i < _num_max_threads; i++) {
+  for (uint i = 0; i < _max_num_threads; i++) {
     if (_threads[i] != NULL) {
       tc->do_thread(_threads[i]);
     }
@@ -131,202 +145,46 @@ void G1ConcurrentRefineThreadControl::worker_threads_do(ThreadClosure* tc) {
 }
 
 void G1ConcurrentRefineThreadControl::stop() {
-  for (uint i = 0; i < _num_max_threads; i++) {
+  for (uint i = 0; i < _max_num_threads; i++) {
     if (_threads[i] != NULL) {
       _threads[i]->stop();
     }
   }
 }
 
-// Arbitrary but large limits, to simplify some of the zone calculations.
+uint64_t G1ConcurrentRefine::adjust_threads_period_ms() const {
-// The general idea is to allow expressions like
+  // Instead of a fixed value, this could be a command line option.  But then
-//   MIN2(x OP y, max_XXX_zone)
+  // we might also want to allow configuration of adjust_threads_wait_ms().
-// without needing to check for overflow in "x OP y", because the
+  return 50;
-// ranges for x and y have been restricted.
-STATIC_ASSERT(sizeof(LP64_ONLY(jint) NOT_LP64(jshort)) <= (sizeof(size_t)/2));
-const size_t max_yellow_zone = LP64_ONLY(max_jint) NOT_LP64(max_jshort);
-const size_t max_green_zone = max_yellow_zone / 2;
-const size_t max_red_zone = INT_MAX; // For dcqs.set_max_cards.
-STATIC_ASSERT(max_yellow_zone <= max_red_zone);
-
-// Range check assertions for green zone values.
-#define assert_zone_constraints_g(green)                        \
-  do {                                                          \
-    size_t azc_g_green = (green);                               \
-    assert(azc_g_green <= max_green_zone,                       \
-           "green exceeds max: " SIZE_FORMAT, azc_g_green);     \
-  } while (0)
-
-// Range check assertions for green and yellow zone values.
-#define assert_zone_constraints_gy(green, yellow)                       \
-  do {                                                                  \
-    size_t azc_gy_green = (green);                                      \
-    size_t azc_gy_yellow = (yellow);                                    \
-    assert_zone_constraints_g(azc_gy_green);                            \
-    assert(azc_gy_yellow <= max_yellow_zone,                            \
-           "yellow exceeds max: " SIZE_FORMAT, azc_gy_yellow);          \
-    assert(azc_gy_green <= azc_gy_yellow,                               \
-           "green (" SIZE_FORMAT ") exceeds yellow (" SIZE_FORMAT ")",  \
-           azc_gy_green, azc_gy_yellow);                                \
-  } while (0)
-
-// Range check assertions for green, yellow, and red zone values.
-#define assert_zone_constraints_gyr(green, yellow, red)                 \
-  do {                                                                  \
-    size_t azc_gyr_green = (green);                                     \
-    size_t azc_gyr_yellow = (yellow);                                   \
-    size_t azc_gyr_red = (red);                                         \
-    assert_zone_constraints_gy(azc_gyr_green, azc_gyr_yellow);          \
-    assert(azc_gyr_red <= max_red_zone,                                 \
-           "red exceeds max: " SIZE_FORMAT, azc_gyr_red);               \
-    assert(azc_gyr_yellow <= azc_gyr_red,                               \
-           "yellow (" SIZE_FORMAT ") exceeds red (" SIZE_FORMAT ")",    \
-           azc_gyr_yellow, azc_gyr_red);                                \
-  } while (0)
-
-// Logging tag sequence for refinement control updates.
-#define CTRL_TAGS gc, ergo, refine
-
-// For logging zone values, ensuring consistency of level and tags.
-#define LOG_ZONES(...) log_debug( CTRL_TAGS )(__VA_ARGS__)
-
-// Convert configuration values in units of buffers to number of cards.
-static size_t configuration_buffers_to_cards(size_t value, const char* value_name) {
-  if (value == 0) return 0;
-  size_t res = value * G1UpdateBufferSize;
-
-  if (res / value != G1UpdateBufferSize) { // Check overflow
-    vm_exit_during_initialization(err_msg("configuration_buffers_to_cards: "
-      "(%s = " SIZE_FORMAT ") * (G1UpdateBufferSize = " SIZE_FORMAT ") overflow!", value_name, value, G1UpdateBufferSize));
-  }
-  return res;
 }
 
-// Package for pair of refinement thread activation and deactivation
+static size_t minimum_pending_cards_target() {
-// thresholds.  The activation and deactivation levels are resp. the first
+  // One buffer per thread.
-// and second values of the pair.
+  return ParallelGCThreads * G1UpdateBufferSize;
-typedef Pair<size_t, size_t> Thresholds;
-inline size_t activation_level(const Thresholds& t) { return t.first; }
-inline size_t deactivation_level(const Thresholds& t) { return t.second; }
-
-static Thresholds calc_thresholds(size_t green_zone,
-                                  size_t yellow_zone,
-                                  uint worker_id) {
-  double yellow_size = yellow_zone - green_zone;
-  double step = yellow_size / G1ConcurrentRefine::max_num_threads();
-  if (worker_id == 0) {
-    // Potentially activate worker 0 more aggressively, to keep
-    // available buffers near green_zone value.  When yellow_size is
-    // large we don't want to allow a full step to accumulate before
-    // doing any processing, as that might lead to significantly more
-    // than green_zone buffers to be processed during pause.  So limit
-    // to an extra half buffer per pause-time processing thread.
-    step = MIN2(step, configuration_buffers_to_cards(ParallelGCThreads, "ParallelGCThreads") / 2.0);
-  }
-  size_t activate_offset = static_cast<size_t>(ceil(step * (worker_id + 1)));
-  size_t deactivate_offset = static_cast<size_t>(floor(step * worker_id));
-  return Thresholds(green_zone + activate_offset,
-                    green_zone + deactivate_offset);
 }
 
-G1ConcurrentRefine::G1ConcurrentRefine(size_t green_zone,
+G1ConcurrentRefine::G1ConcurrentRefine(G1Policy* policy) :
-                                       size_t yellow_zone,
+  _policy(policy),
-                                       size_t red_zone,
+  _threads_wanted(0),
-                                       size_t min_yellow_zone_size) :
+  _pending_cards_target(PendingCardsTargetUninitialized),
+  _last_adjust(),
+  _needs_adjust(false),
+  _threads_needed(policy, adjust_threads_period_ms()),
   _thread_control(),
-  _green_zone(green_zone),
+  _dcqs(G1BarrierSet::dirty_card_queue_set())
-  _yellow_zone(yellow_zone),
+{}
-  _red_zone(red_zone),
-  _min_yellow_zone_size(min_yellow_zone_size)
-{
-  assert_zone_constraints_gyr(green_zone, yellow_zone, red_zone);
-}
 
 jint G1ConcurrentRefine::initialize() {
-  jint result = _thread_control.initialize(this, max_num_threads());
+  return _thread_control.initialize(this, max_num_threads());
-  if (result != JNI_OK) return result;
-
-  G1DirtyCardQueueSet& dcqs = G1BarrierSet::dirty_card_queue_set();
-  dcqs.set_max_cards(red_zone());
-  if (max_num_threads() > 0) {
-    G1PrimaryConcurrentRefineThread* primary_thread = _thread_control.primary_thread();
-    primary_thread->update_notify_threshold(primary_activation_threshold());
-    dcqs.set_refinement_notification_thread(primary_thread);
-  }
-
-  return JNI_OK;
 }
 
-static size_t calc_min_yellow_zone_size() {
+G1ConcurrentRefine* G1ConcurrentRefine::create(G1Policy* policy, jint* ecode) {
-  size_t step = configuration_buffers_to_cards(G1ConcRefinementThresholdStep, "G1ConcRefinementThresholdStep");
+  G1ConcurrentRefine* cr = new G1ConcurrentRefine(policy);
-  uint n_workers = G1ConcurrentRefine::max_num_threads();
-  if ((max_yellow_zone / step) < n_workers) {
-    return max_yellow_zone;
-  } else {
-    return step * n_workers;
-  }
-}
-
-// An initial guess at the rate for pause-time card refinement for one
-// thread, used when computing the default initial green zone value.
-const double InitialPauseTimeCardRefinementRate = 200.0;
-
-static size_t calc_init_green_zone() {
-  size_t green;
-  if (FLAG_IS_DEFAULT(G1ConcRefinementGreenZone)) {
-    const double rate = InitialPauseTimeCardRefinementRate * ParallelGCThreads;
-    // The time budget for pause-time card refinement.
-    const double ms = MaxGCPauseMillis * (G1RSetUpdatingPauseTimePercent / 100.0);
-    green = rate * ms;
-  } else {
-    green = configuration_buffers_to_cards(G1ConcRefinementGreenZone,
-                                           "G1ConcRefinementGreenZone");
-  }
-  return MIN2(green, max_green_zone);
-}
-
-static size_t calc_init_yellow_zone(size_t green, size_t min_size) {
-  size_t config = configuration_buffers_to_cards(G1ConcRefinementYellowZone, "G1ConcRefinementYellowZone");
-  size_t size = 0;
-  if (FLAG_IS_DEFAULT(G1ConcRefinementYellowZone)) {
-    size = green * 2;
-  } else if (green < config) {
-    size = config - green;
-  }
-  size = MAX2(size, min_size);
-  size = MIN2(size, max_yellow_zone);
-  return MIN2(green + size, max_yellow_zone);
-}
-
-static size_t calc_init_red_zone(size_t green, size_t yellow) {
-  size_t size = yellow - green;
-  if (!FLAG_IS_DEFAULT(G1ConcRefinementRedZone)) {
-    size_t config = configuration_buffers_to_cards(G1ConcRefinementRedZone, "G1ConcRefinementRedZone");
-    if (yellow < config) {
-      size = MAX2(size, config - yellow);
-    }
-  }
-  return MIN2(yellow + size, max_red_zone);
-}
-
-G1ConcurrentRefine* G1ConcurrentRefine::create(jint* ecode) {
-  size_t min_yellow_zone_size = calc_min_yellow_zone_size();
-  size_t green_zone = calc_init_green_zone();
-  size_t yellow_zone = calc_init_yellow_zone(green_zone, min_yellow_zone_size);
-  size_t red_zone = calc_init_red_zone(green_zone, yellow_zone);
-
-  LOG_ZONES("Initial Refinement Zones: "
-            "green: " SIZE_FORMAT ", "
-            "yellow: " SIZE_FORMAT ", "
-            "red: " SIZE_FORMAT ", "
-            "min yellow size: " SIZE_FORMAT,
-            green_zone, yellow_zone, red_zone, min_yellow_zone_size);
-
-  G1ConcurrentRefine* cr = new G1ConcurrentRefine(green_zone,
-                                                  yellow_zone,
-                                                  red_zone,
-                                                  min_yellow_zone_size);
   *ecode = cr->initialize();
+  if (*ecode != 0) {
+    delete cr;
+    cr = nullptr;
+  }
   return cr;
 }
 
@@ -345,84 +203,234 @@ uint G1ConcurrentRefine::max_num_threads() {
   return G1ConcRefinementThreads;
 }
 
-static size_t calc_new_green_zone(size_t green,
+void G1ConcurrentRefine::update_pending_cards_target(double logged_cards_time_ms,
-                                  double logged_cards_scan_time,
                                                      size_t processed_logged_cards,
+                                                     size_t predicted_thread_buffer_cards,
                                                      double goal_ms) {
-  // Adjust green zone based on whether we're meeting the time goal.
+  size_t minimum = minimum_pending_cards_target();
-  // Limit to max_green_zone.
+  if ((processed_logged_cards < minimum) || (logged_cards_time_ms == 0.0)) {
-  const double inc_k = 1.1, dec_k = 0.9;
+    log_debug(gc, ergo, refine)("Unchanged pending cards target: %zu",
-  if (logged_cards_scan_time > goal_ms) {
+                                _pending_cards_target);
-    if (green > 0) {
+    return;
-      green = static_cast<size_t>(green * dec_k);
   }
-  } else if (logged_cards_scan_time < goal_ms &&
+
-             processed_logged_cards > green) {
+  // Base the pending cards budget on the measured rate.
-    green = static_cast<size_t>(MAX2(green * inc_k, green + 1.0));
+  double rate = processed_logged_cards / logged_cards_time_ms;
-    green = MIN2(green, max_green_zone);
+  size_t budget = static_cast<size_t>(goal_ms * rate);
+  // Deduct predicted cards in thread buffers to get target.
+  size_t new_target = budget - MIN2(budget, predicted_thread_buffer_cards);
+  // Add some hysteresis with previous values.
+  if (is_pending_cards_target_initialized()) {
+    new_target = (new_target + _pending_cards_target) / 2;
   }
-  return green;
+  // Apply minimum target.
+  new_target = MAX2(new_target, minimum_pending_cards_target());
+  _pending_cards_target = new_target;
+  log_debug(gc, ergo, refine)("New pending cards target: %zu", new_target);
 }
 
-static size_t calc_new_yellow_zone(size_t green, size_t min_yellow_size) {
+void G1ConcurrentRefine::adjust_after_gc(double logged_cards_time_ms,
-  size_t size = green * 2;
-  size = MAX2(size, min_yellow_size);
-  return MIN2(green + size, max_yellow_zone);
-}
-
-static size_t calc_new_red_zone(size_t green, size_t yellow) {
-  return MIN2(yellow + (yellow - green), max_red_zone);
-}
-
-void G1ConcurrentRefine::update_zones(double logged_cards_scan_time,
                                          size_t processed_logged_cards,
+                                         size_t predicted_thread_buffer_cards,
                                          double goal_ms) {
-  log_trace( CTRL_TAGS )("Updating Refinement Zones: "
+  if (!G1UseConcRefinement) return;
-                         "logged cards scan time: %.3fms, "
+
-                         "processed cards: " SIZE_FORMAT ", "
+  update_pending_cards_target(logged_cards_time_ms,
-                         "goal time: %.3fms",
-                         logged_cards_scan_time,
                               processed_logged_cards,
+                              predicted_thread_buffer_cards,
                               goal_ms);
-
+  if (_thread_control.max_num_threads() == 0) {
-  _green_zone = calc_new_green_zone(_green_zone,
+    // If no refinement threads then the mutator threshold is the target.
-                                    logged_cards_scan_time,
+    _dcqs.set_mutator_refinement_threshold(_pending_cards_target);
-                                    processed_logged_cards,
-                                    goal_ms);
-  _yellow_zone = calc_new_yellow_zone(_green_zone, _min_yellow_zone_size);
-  _red_zone = calc_new_red_zone(_green_zone, _yellow_zone);
-
-  assert_zone_constraints_gyr(_green_zone, _yellow_zone, _red_zone);
-  LOG_ZONES("Updated Refinement Zones: "
-            "green: " SIZE_FORMAT ", "
-            "yellow: " SIZE_FORMAT ", "
-            "red: " SIZE_FORMAT,
-            _green_zone, _yellow_zone, _red_zone);
-}
-
-void G1ConcurrentRefine::adjust(double logged_cards_scan_time,
-                                size_t processed_logged_cards,
-                                double goal_ms) {
-  G1DirtyCardQueueSet& dcqs = G1BarrierSet::dirty_card_queue_set();
-
-  if (G1UseAdaptiveConcRefinement) {
-    update_zones(logged_cards_scan_time, processed_logged_cards, goal_ms);
-
-    // Change the barrier params
-    if (max_num_threads() > 0) {
-      size_t threshold = primary_activation_threshold();
-      _thread_control.primary_thread()->update_notify_threshold(threshold);
-    }
-    dcqs.set_max_cards(red_zone());
-  }
-
-  size_t curr_queue_size = dcqs.num_cards();
-  if ((dcqs.max_cards() > 0) &&
-      (curr_queue_size >= yellow_zone())) {
-    dcqs.set_max_cards_padding(curr_queue_size);
   } else {
-    dcqs.set_max_cards_padding(0);
+    // Provisionally make the mutator threshold unlimited, to be updated by
+    // the next periodic adjustment.  Because card state may have changed
+    // drastically, record that adjustment is needed and kick the primary
+    // thread, in case it is waiting.
+    _dcqs.set_mutator_refinement_threshold(SIZE_MAX);
+    _needs_adjust = true;
+    if (is_pending_cards_target_initialized()) {
+      _thread_control.activate(0);
     }
+  }
+}
+
+// Wake up the primary thread less frequently when the time available until
+// the next GC is longer.  But don't increase the wait time too rapidly.
+// This reduces the number of primary thread wakeups that just immediately
+// go back to waiting, while still being responsive to behavior changes.
+static uint64_t compute_adjust_wait_time_ms(double available_ms) {
+  return static_cast<uint64_t>(sqrt(available_ms) * 4.0);
+}
+
+uint64_t G1ConcurrentRefine::adjust_threads_wait_ms() const {
+  assert_current_thread_is_primary_refinement_thread();
+  if (is_pending_cards_target_initialized()) {
+    double available_ms = _threads_needed.predicted_time_until_next_gc_ms();
+    uint64_t wait_time_ms = compute_adjust_wait_time_ms(available_ms);
+    return MAX2(wait_time_ms, adjust_threads_period_ms());
+  } else {
+    // If target not yet initialized then wait forever (until explicitly
+    // activated).  This happens during startup, when we don't bother with
+    // refinement.
+    return 0;
+  }
+}
+
+class G1ConcurrentRefine::RemSetSamplingClosure : public HeapRegionClosure {
+  G1CollectionSet* _cset;
+  size_t _sampled_rs_length;
+
+public:
+  explicit RemSetSamplingClosure(G1CollectionSet* cset) :
+    _cset(cset), _sampled_rs_length(0) {}
+
+  bool do_heap_region(HeapRegion* r) override {
+    size_t rs_length = r->rem_set()->occupied();
+    _sampled_rs_length += rs_length;
+    // Update the collection set policy information for this region.
+    _cset->update_young_region_prediction(r, rs_length);
+    return false;
+  }
+
+  size_t sampled_rs_length() const { return _sampled_rs_length; }
+};
+
+// Adjust the target length (in regions) of the young gen, based on the the
+// current length of the remembered sets.
+//
+// At the end of the GC G1 determines the length of the young gen based on
+// how much time the next GC can take, and when the next GC may occur
+// according to the MMU.
+//
+// The assumption is that a significant part of the GC is spent on scanning
+// the remembered sets (and many other components), so this thread constantly
+// reevaluates the prediction for the remembered set scanning costs, and potentially
+// resizes the young gen. This may do a premature GC or even increase the young
+// gen size to keep pause time length goal.
+void G1ConcurrentRefine::adjust_young_list_target_length() {
+  if (_policy->use_adaptive_young_list_length()) {
+    G1CollectionSet* cset = G1CollectedHeap::heap()->collection_set();
+    RemSetSamplingClosure cl{cset};
+    cset->iterate(&cl);
+    _policy->revise_young_list_target_length(cl.sampled_rs_length());
+  }
+}
+
+bool G1ConcurrentRefine::adjust_threads_periodically() {
+  assert_current_thread_is_primary_refinement_thread();
+
+  // Check whether it's time to do a periodic adjustment.
+  if (!_needs_adjust) {
+    Tickspan since_adjust = Ticks::now() - _last_adjust;
+    if (since_adjust.milliseconds() >= adjust_threads_period_ms()) {
+      _needs_adjust = true;
+    }
+  }
+
+  // If needed, try to adjust threads wanted.
+  if (_needs_adjust) {
+    // Getting used young bytes requires holding Heap_lock.  But we can't use
+    // normal lock and block until available.  Blocking on the lock could
+    // deadlock with a GC VMOp that is holding the lock and requesting a
+    // safepoint.  Instead try to lock, and if fail then skip adjustment for
+    // this iteration of the thread, do some refinement work, and retry the
+    // adjustment later.
+    if (Heap_lock->try_lock()) {
+      size_t used_bytes = _policy->estimate_used_young_bytes_locked();
+      Heap_lock->unlock();
+      adjust_young_list_target_length();
+      size_t young_bytes = _policy->young_list_target_length() * HeapRegion::GrainBytes;
+      size_t available_bytes = young_bytes - MIN2(young_bytes, used_bytes);
+      adjust_threads_wanted(available_bytes);
+      _needs_adjust = false;
+      _last_adjust = Ticks::now();
+      return true;
+    }
+  }
+
+  return false;
+}
+
+bool G1ConcurrentRefine::is_in_last_adjustment_period() const {
+  return _threads_needed.predicted_time_until_next_gc_ms() <= adjust_threads_period_ms();
+}
+
+void G1ConcurrentRefine::adjust_threads_wanted(size_t available_bytes) {
+  assert_current_thread_is_primary_refinement_thread();
+  size_t num_cards = _dcqs.num_cards();
+  size_t mutator_threshold = SIZE_MAX;
+  uint old_wanted = Atomic::load(&_threads_wanted);
+
+  _threads_needed.update(old_wanted,
+                         available_bytes,
+                         num_cards,
+                         _pending_cards_target);
+  uint new_wanted = _threads_needed.threads_needed();
+  if (new_wanted > _thread_control.max_num_threads()) {
+    // If running all the threads can't reach goal, turn on refinement by
+    // mutator threads.  Using target as the threshold may be stronger
+    // than required, but will do the most to get us under goal, and we'll
+    // reevaluate with the next adjustment.
+    mutator_threshold = _pending_cards_target;
+    new_wanted = _thread_control.max_num_threads();
+  } else if (is_in_last_adjustment_period()) {
+    // If very little time remains until GC, enable mutator refinement.  If
+    // the target has been reached, this keeps the number of pending cards on
+    // target even if refinement threads deactivate in the meantime.  And if
+    // the target hasn't been reached, this prevents things from getting
+    // worse.
+    mutator_threshold = _pending_cards_target;
+  }
+  Atomic::store(&_threads_wanted, new_wanted);
+  _dcqs.set_mutator_refinement_threshold(mutator_threshold);
+  log_debug(gc, refine)("Concurrent refinement: wanted %u, cards: %zu, "
+                        "predicted: %zu, time: %1.2fms",
+                        new_wanted,
+                        num_cards,
+                        _threads_needed.predicted_cards_at_next_gc(),
+                        _threads_needed.predicted_time_until_next_gc_ms());
+  // Activate newly wanted threads.  The current thread is the primary
+  // refinement thread, so is already active.
+  for (uint i = MAX2(old_wanted, 1u); i < new_wanted; ++i) {
+    if (!_thread_control.activate(i)) {
+      // Failed to allocate and activate thread.  Stop trying to activate, and
+      // instead use mutator threads to make up the gap.
+      Atomic::store(&_threads_wanted, i);
+      _dcqs.set_mutator_refinement_threshold(_pending_cards_target);
+      break;
+    }
+  }
+}
+
+void G1ConcurrentRefine::reduce_threads_wanted() {
+  assert_current_thread_is_primary_refinement_thread();
+  if (!_needs_adjust) {         // Defer if adjustment request is active.
+    uint wanted = Atomic::load(&_threads_wanted);
+    if (wanted > 0) {
+      Atomic::store(&_threads_wanted, --wanted);
+    }
+    // If very little time remains until GC, enable mutator refinement.  If
+    // the target has been reached, this keeps the number of pending cards on
+    // target even as refinement threads deactivate in the meantime.
+    if (is_in_last_adjustment_period()) {
+      _dcqs.set_mutator_refinement_threshold(_pending_cards_target);
+    }
+  }
+}
+
+bool G1ConcurrentRefine::is_thread_wanted(uint worker_id) const {
+  return worker_id < Atomic::load(&_threads_wanted);
+}
+
+bool G1ConcurrentRefine::is_thread_adjustment_needed() const {
+  assert_current_thread_is_primary_refinement_thread();
+  return _needs_adjust;
+}
+
+void G1ConcurrentRefine::record_thread_adjustment_needed() {
+  assert_current_thread_is_primary_refinement_thread();
+  _needs_adjust = true;
 }
 
 G1ConcurrentRefineStats G1ConcurrentRefine::get_and_reset_refinement_stats() {
@@ -439,46 +447,13 @@ G1ConcurrentRefineStats G1ConcurrentRefine::get_and_reset_refinement_stats() {
   return collector._total_stats;
 }
 
-size_t G1ConcurrentRefine::activation_threshold(uint worker_id) const {
-  Thresholds thresholds = calc_thresholds(_green_zone, _yellow_zone, worker_id);
-  return activation_level(thresholds);
-}
-
-size_t G1ConcurrentRefine::deactivation_threshold(uint worker_id) const {
-  Thresholds thresholds = calc_thresholds(_green_zone, _yellow_zone, worker_id);
-  return deactivation_level(thresholds);
-}
-
-size_t G1ConcurrentRefine::primary_activation_threshold() const {
-  assert(max_num_threads() > 0, "No primary refinement thread");
-  return activation_threshold(0);
-}
-
 uint G1ConcurrentRefine::worker_id_offset() {
   return G1DirtyCardQueueSet::num_par_ids();
 }
 
-void G1ConcurrentRefine::maybe_activate_more_threads(uint worker_id, size_t num_cur_cards) {
+bool G1ConcurrentRefine::try_refinement_step(uint worker_id,
-  if (num_cur_cards > activation_threshold(worker_id + 1)) {
+                                             size_t stop_at,
-    _thread_control.maybe_activate_next(worker_id);
-  }
-}
-
-bool G1ConcurrentRefine::do_refinement_step(uint worker_id,
                                              G1ConcurrentRefineStats* stats) {
-  G1DirtyCardQueueSet& dcqs = G1BarrierSet::dirty_card_queue_set();
+  uint adjusted_id = worker_id + worker_id_offset();
-
+  return _dcqs.refine_completed_buffer_concurrently(adjusted_id, stop_at, stats);
-  size_t curr_cards = dcqs.num_cards();
-  // If the number of the cards falls down into the yellow zone,
-  // that means that the transition period after the evacuation pause has ended.
-  if (curr_cards <= yellow_zone()) {
-    dcqs.discard_max_cards_padding();
-  }
-
-  maybe_activate_more_threads(worker_id, curr_cards);
-
-  // Process the next buffer, if there are enough left.
-  return dcqs.refine_completed_buffer_concurrently(worker_id + worker_id_offset(),
-                                                   deactivation_threshold(worker_id),
-                                                   stats);
 }

src/hotspot/share/gc/g1/g1ConcurrentRefine.hpp

@@ -26,132 +26,198 @@
 #define SHARE_GC_G1_G1CONCURRENTREFINE_HPP
 
 #include "gc/g1/g1ConcurrentRefineStats.hpp"
+#include "gc/g1/g1ConcurrentRefineThreadsNeeded.hpp"
 #include "memory/allocation.hpp"
 #include "utilities/debug.hpp"
 #include "utilities/globalDefinitions.hpp"
+#include "utilities/macros.hpp"
 
 // Forward decl
 class G1ConcurrentRefine;
 class G1ConcurrentRefineThread;
-class G1PrimaryConcurrentRefineThread;
+class G1DirtyCardQueueSet;
+class G1Policy;
 class ThreadClosure;
 
 // Helper class for refinement thread management. Used to start, stop and
 // iterate over them.
 class G1ConcurrentRefineThreadControl {
   G1ConcurrentRefine* _cr;
-  G1PrimaryConcurrentRefineThread* _primary_thread;
   G1ConcurrentRefineThread** _threads;
-  uint _num_max_threads;
+  uint _max_num_threads;
 
   // Create the refinement thread for the given worker id.
   // If initializing is true, ignore InjectGCWorkerCreationFailure.
   G1ConcurrentRefineThread* create_refinement_thread(uint worker_id, bool initializing);
+
+  NONCOPYABLE(G1ConcurrentRefineThreadControl);
+
 public:
   G1ConcurrentRefineThreadControl();
   ~G1ConcurrentRefineThreadControl();
 
-  jint initialize(G1ConcurrentRefine* cr, uint num_max_threads);
+  jint initialize(G1ConcurrentRefine* cr, uint max_num_threads);
 
-  G1PrimaryConcurrentRefineThread* primary_thread() const {
+  void assert_current_thread_is_primary_refinement_thread() const NOT_DEBUG_RETURN;
-    assert(_num_max_threads > 0, "precondition");
-    assert(_primary_thread != nullptr, "uninitialized");
-    return _primary_thread;
-  }
 
-  // If there is a "successor" thread that can be activated given the current id,
+  uint max_num_threads() const { return _max_num_threads; }
-  // activate it.
+
-  void maybe_activate_next(uint cur_worker_id);
+  // Activate the indicated thread.  If the thread has not yet been allocated,
+  // allocate and then activate.  If allocation is needed and fails, return
+  // false.  Otherwise return true.
+  // precondition: worker_id < max_num_threads().
+  // precondition: current thread is not the designated worker.
+  bool activate(uint worker_id);
 
   void worker_threads_do(ThreadClosure* tc);
   void stop();
 };
 
-// Controls refinement threads and their activation based on the number of
+// Controls concurrent refinement.
-// cards currently available in the global dirty card queue.
+//
-// Refinement threads obtain work from the queue (a buffer at a time) based
+// Mutator threads produce dirty cards, which need to be examined for updates
-// on these thresholds. They are activated gradually based on the amount of
+// to the remembered sets (refinement).  There is a pause-time budget for
-// work to do.
+// processing these dirty cards (see -XX:G1RSetUpdatingPauseTimePercent).  The
-// Refinement thread n activates thread n+1 if the instance of this class determines there
+// purpose of concurrent refinement is to (attempt to) ensure the number of
-// is enough work available. Threads deactivate themselves if the current amount of
+// pending dirty cards at the start of a GC can be processed within that time
-// available cards falls below their individual threshold.
+// budget.
+//
+// Concurrent refinement is performed by a combination of dedicated threads
+// and by mutator threads as they produce dirty cards.  If configured to not
+// have any dedicated threads (-XX:G1ConcRefinementThreads=0) then all
+// concurrent refinement work is performed by mutator threads.  When there are
+// dedicated threads, they generally do most of the concurrent refinement
+// work, to minimize throughput impact of refinement work on mutator threads.
+//
+// This class determines the target number of dirty cards pending for the next
+// GC.  It also owns the dedicated refinement threads and controls their
+// activation in order to achieve that target.
+//
+// There are two kinds of dedicated refinement threads, a single primary
+// thread and some number of secondary threads.  When active, all refinement
+// threads take buffers of dirty cards from the dirty card queue and process
+// them.  Between buffers they query this owning object to find out whether
+// they should continue running, deactivating themselves if not.
+//
+// The primary thread drives the control system that determines how many
+// refinement threads should be active.  If inactive, it wakes up periodically
+// to recalculate the number of active threads needed, and activates
+// additional threads as necessary.  While active it also periodically
+// recalculates the number wanted and activates more threads if needed.  It
+// also reduces the number of wanted threads when the target has been reached,
+// triggering deactivations.
 class G1ConcurrentRefine : public CHeapObj<mtGC> {
+  G1Policy* _policy;
+  volatile uint _threads_wanted;
+  size_t _pending_cards_target;
+  Ticks _last_adjust;
+  Ticks _last_deactivate;
+  bool _needs_adjust;
+  G1ConcurrentRefineThreadsNeeded _threads_needed;
   G1ConcurrentRefineThreadControl _thread_control;
-  /*
+  G1DirtyCardQueueSet& _dcqs;
-   * The value of the completed dirty card queue length falls into one of 3 zones:
-   * green, yellow, red. If the value is in [0, green) nothing is
-   * done, the buffered cards are left unprocessed to enable the caching effect of the
-   * dirtied cards. In the yellow zone [green, yellow) the concurrent refinement
-   * threads are gradually activated. In [yellow, red) all threads are
-   * running. If the length becomes red (max queue length) the mutators start
-   * processing cards too.
-   *
-   * There are some interesting cases (when G1UseAdaptiveConcRefinement
-   * is turned off):
-   * 1) green = yellow = red = 0. In this case the mutator will process all
-   *    cards. Except for those that are created by the deferred updates
-   *    machinery during a collection.
-   * 2) green = 0. Means no caching. Can be a good way to minimize the
-   *    amount of time spent updating remembered sets during a collection.
-   */
-  size_t _green_zone;
-  size_t _yellow_zone;
-  size_t _red_zone;
-  size_t _min_yellow_zone_size;
 
-  G1ConcurrentRefine(size_t green_zone,
+  G1ConcurrentRefine(G1Policy* policy);
-                     size_t yellow_zone,
-                     size_t red_zone,
-                     size_t min_yellow_zone_size);
-
-  // Update green/yellow/red zone values based on how well goals are being met.
-  void update_zones(double logged_cards_scan_time,
-                    size_t processed_logged_cards,
-                    double goal_ms);
 
   static uint worker_id_offset();
-  void maybe_activate_more_threads(uint worker_id, size_t num_cur_cards);
 
   jint initialize();
+
+  void assert_current_thread_is_primary_refinement_thread() const {
+    _thread_control.assert_current_thread_is_primary_refinement_thread();
+  }
+
+  // For the first few collection cycles we don't have a target (and so don't
+  // do any concurrent refinement), because there hasn't been enough pause
+  // time refinement work to be done to make useful predictions.  We use
+  // SIZE_MAX as a special marker value to indicate we're in this state.
+  static const size_t PendingCardsTargetUninitialized = SIZE_MAX;
+  bool is_pending_cards_target_initialized() const {
+    return _pending_cards_target != PendingCardsTargetUninitialized;
+  }
+
+  void update_pending_cards_target(double logged_cards_scan_time_ms,
+                                   size_t processed_logged_cards,
+                                   size_t predicted_thread_buffer_cards,
+                                   double goal_ms);
+
+  uint64_t adjust_threads_period_ms() const;
+  bool is_in_last_adjustment_period() const;
+
+  class RemSetSamplingClosure;  // Helper class for adjusting young length.
+  void adjust_young_list_target_length();
+
+  void adjust_threads_wanted(size_t available_bytes);
+
+  NONCOPYABLE(G1ConcurrentRefine);
+
 public:
   ~G1ConcurrentRefine();
 
   // Returns a G1ConcurrentRefine instance if succeeded to create/initialize the
-  // G1ConcurrentRefine instance. Otherwise, returns NULL with error code.
+  // G1ConcurrentRefine instance. Otherwise, returns nullptr with error code.
-  static G1ConcurrentRefine* create(jint* ecode);
+  static G1ConcurrentRefine* create(G1Policy* policy, jint* ecode);
 
+  // Stop all the refinement threads.
   void stop();
 
-  // The minimum number of pending cards for activation of the primary
+  // Called at the end of a GC to prepare for refinement during the next
-  // refinement thread.
+  // concurrent phase.  Updates the target for the number of pending dirty
-  size_t primary_activation_threshold() const;
+  // cards.  Updates the mutator refinement threshold.  Ensures the primary
+  // refinement thread (if it exists) is active, so it will adjust the number
+  // of running threads.
+  void adjust_after_gc(double logged_cards_scan_time_ms,
+                       size_t processed_logged_cards,
+                       size_t predicted_thread_buffer_cards,
+                       double goal_ms);
 
-  // Adjust refinement thresholds based on work done during the pause and the goal time.
+  // Target number of pending dirty cards at the start of the next GC.
-  void adjust(double logged_cards_scan_time, size_t processed_logged_cards, double goal_ms);
+  size_t pending_cards_target() const { return _pending_cards_target; }
+
+  // May recalculate the number of refinement threads that should be active in
+  // order to meet the pending cards target.  Returns true if adjustment was
+  // performed, and clears any pending request.  Returns false if the
+  // adjustment period has not expired, or because a timed or requested
+  // adjustment could not be performed immediately and so was deferred.
+  // precondition: current thread is the primary refinement thread.
+  bool adjust_threads_periodically();
+
+  // The amount of time (in ms) the primary refinement thread should sleep
+  // when it is inactive.  It requests adjustment whenever it is reactivated.
+  // precondition: current thread is the primary refinement thread.
+  uint64_t adjust_threads_wait_ms() const;
+
+  // Record a request for thread adjustment as soon as possible.
+  // precondition: current thread is the primary refinement thread.
+  void record_thread_adjustment_needed();
+
+  // Test whether there is a pending request for thread adjustment.
+  // precondition: current thread is the primary refinement thread.
+  bool is_thread_adjustment_needed() const;
+
+  // Reduce the number of active threads wanted.
+  // precondition: current thread is the primary refinement thread.
+  void reduce_threads_wanted();
+
+  // Test whether the thread designated by worker_id should be active.
+  bool is_thread_wanted(uint worker_id) const;
 
   // Return total of concurrent refinement stats for the
   // ConcurrentRefineThreads.  Also reset the stats for the threads.
   G1ConcurrentRefineStats get_and_reset_refinement_stats();
 
-  // Cards in the dirty card queue set.
-  size_t activation_threshold(uint worker_id) const;
-  size_t deactivation_threshold(uint worker_id) const;
-
   // Perform a single refinement step; called by the refinement
   // threads.  Returns true if there was refinement work available.
   // Updates stats.
-  bool do_refinement_step(uint worker_id, G1ConcurrentRefineStats* stats);
+  bool try_refinement_step(uint worker_id,
+                           size_t stop_at,
+                           G1ConcurrentRefineStats* stats);
 
   // Iterate over all concurrent refinement threads applying the given closure.
   void threads_do(ThreadClosure *tc);
 
   // Maximum number of refinement threads.
   static uint max_num_threads();
-
-  // Cards in the dirty card queue set.
-  size_t green_zone() const      { return _green_zone;  }
-  size_t yellow_zone() const     { return _yellow_zone; }
-  size_t red_zone() const        { return _red_zone;    }
 };
 
 #endif // SHARE_GC_G1_G1CONCURRENTREFINE_HPP

src/hotspot/share/gc/g1/g1ConcurrentRefineThread.cpp

@@ -30,17 +30,21 @@
 #include "gc/g1/g1DirtyCardQueue.hpp"
 #include "gc/shared/suspendibleThreadSet.hpp"
 #include "logging/log.hpp"
-#include "runtime/atomic.hpp"
-#include "runtime/init.hpp"
-#include "runtime/javaThread.hpp"
 #include "runtime/mutexLocker.hpp"
-#include "runtime/safepoint.hpp"
+#include "runtime/os.hpp"
+#include "runtime/thread.hpp"
+#include "utilities/debug.hpp"
+#include "utilities/formatBuffer.hpp"
+#include "utilities/globalDefinitions.hpp"
+#include "utilities/ticks.hpp"
 
 G1ConcurrentRefineThread::G1ConcurrentRefineThread(G1ConcurrentRefine* cr, uint worker_id) :
   ConcurrentGCThread(),
   _vtime_start(0.0),
   _vtime_accum(0.0),
-  _refinement_stats(new G1ConcurrentRefineStats()),
+  _notifier(Mutex::nosafepoint, FormatBuffer<>("G1 Refine#%d", worker_id), true),
+  _requested_active(false),
+  _refinement_stats(),
   _worker_id(worker_id),
   _cr(cr)
 {
@@ -48,51 +52,28 @@ G1ConcurrentRefineThread::G1ConcurrentRefineThread(G1ConcurrentRefine* cr, uint
   set_name("G1 Refine#%d", worker_id);
 }
 
-G1ConcurrentRefineThread::~G1ConcurrentRefineThread() {
-  delete _refinement_stats;
-}
-
 void G1ConcurrentRefineThread::run_service() {
   _vtime_start = os::elapsedVTime();
 
   while (wait_for_completed_buffers()) {
-    // For logging.
-    G1ConcurrentRefineStats start_stats = *_refinement_stats;
-    G1ConcurrentRefineStats total_stats; // Accumulate over activation.
-
-    {
     SuspendibleThreadSetJoiner sts_join;
-
+    G1ConcurrentRefineStats active_stats_start = _refinement_stats;
-      log_debug(gc, refine)("Activated worker %d, on threshold: %zu, current: %zu",
+    report_active("Activated");
-                            _worker_id, _cr->activation_threshold(_worker_id),
-                            G1BarrierSet::dirty_card_queue_set().num_cards());
-
     while (!should_terminate()) {
       if (sts_join.should_yield()) {
-          // Accumulate changed stats before possible GC that resets stats.
+        report_inactive("Paused", _refinement_stats - active_stats_start);
-          total_stats += *_refinement_stats - start_stats;
         sts_join.yield();
-          // Reinitialize baseline stats after safepoint.
+        // Reset after yield rather than accumulating across yields, else a
-          start_stats = *_refinement_stats;
+        // very long running thread could overflow.
-          continue;             // Re-check for termination after yield delay.
+        active_stats_start = _refinement_stats;
-        }
+        report_active("Resumed");
-
+      } else if (maybe_deactivate()) {
-        if (!_cr->do_refinement_step(_worker_id, _refinement_stats)) {
-          if (maybe_deactivate()) {
         break;
+      } else {
+        do_refinement_step();
       }
     }
-      }
+    report_inactive("Deactivated", _refinement_stats - active_stats_start);
-    }
-
-    total_stats += *_refinement_stats - start_stats;
-    log_debug(gc, refine)("Deactivated worker %d, off threshold: %zu, "
-                          "cards: %zu, refined %zu, rate %1.2fc/ms",
-                          _worker_id, _cr->deactivation_threshold(_worker_id),
-                          G1BarrierSet::dirty_card_queue_set().num_cards(),
-                          total_stats.refined_cards(),
-                          total_stats.refinement_rate_ms());
-
     if (os::supports_vtime()) {
       _vtime_accum = (os::elapsedVTime() - _vtime_start);
     } else {
@@ -103,144 +84,25 @@ void G1ConcurrentRefineThread::run_service() {
   log_debug(gc, refine)("Stopping %d", _worker_id);
 }
 
-void G1ConcurrentRefineThread::stop_service() {
+void G1ConcurrentRefineThread::report_active(const char* reason) const {
-  activate();
+  log_trace(gc, refine)("%s worker %u, current: %zu",
+                        reason,
+                        _worker_id,
+                        G1BarrierSet::dirty_card_queue_set().num_cards());
 }
 
-G1PrimaryConcurrentRefineThread*
+void G1ConcurrentRefineThread::report_inactive(const char* reason,
-G1PrimaryConcurrentRefineThread::create(G1ConcurrentRefine* cr) {
+                                               const G1ConcurrentRefineStats& stats) const {
-  G1PrimaryConcurrentRefineThread* crt =
+  log_trace(gc, refine)
-    new (std::nothrow) G1PrimaryConcurrentRefineThread(cr);
+           ("%s worker %u, cards: %zu, refined %zu, rate %1.2fc/ms",
-  if (crt != nullptr) {
+            reason,
-    crt->create_and_start();
+            _worker_id,
-  }
+            G1BarrierSet::dirty_card_queue_set().num_cards(),
-  return crt;
+            stats.refined_cards(),
+            stats.refinement_rate_ms());
 }
 
-G1PrimaryConcurrentRefineThread::G1PrimaryConcurrentRefineThread(G1ConcurrentRefine* cr) :
+void G1ConcurrentRefineThread::activate() {
-  G1ConcurrentRefineThread(cr, 0),
-  _notifier(0),
-  _threshold(0)
-{}
-
-void G1PrimaryConcurrentRefineThread::stop_service() {
-  G1DirtyCardQueueSet& dcqs = G1BarrierSet::dirty_card_queue_set();
-  dcqs.set_refinement_notification_thread(nullptr);
-  G1ConcurrentRefineThread::stop_service();
-}
-
-// The primary refinement thread is notified when buffers/cards are added to
-// the dirty card queue.  That can happen in fairly arbitrary contexts.
-// This means there may be arbitrary other locks held when notifying.  We
-// also don't want to have to take a lock on the fairly common notification
-// path, as contention for that lock can significantly impact performance.
-//
-// We use a semaphore to implement waiting and unblocking, to avoid
-// lock rank checking issues.  (We could alternatively use an
-// arbitrarily low ranked mutex.)  The atomic variable _threshold is
-// used to decide when to signal the semaphore.  When its value is
-// SIZE_MAX then the thread is running.  Otherwise, the thread should
-// be requested to run when notified that the number of cards has
-// exceeded the threshold value.
-
-bool G1PrimaryConcurrentRefineThread::wait_for_completed_buffers() {
-  assert(this == Thread::current(), "precondition");
-  _notifier.wait();
-  assert(Atomic::load(&_threshold) == SIZE_MAX || should_terminate(), "incorrect state");
-  return !should_terminate();
-}
-
-bool G1PrimaryConcurrentRefineThread::maybe_deactivate() {
-  assert(this == Thread::current(), "precondition");
-  assert(Atomic::load(&_threshold) == SIZE_MAX, "incorrect state");
-  Atomic::store(&_threshold, cr()->primary_activation_threshold());
-  // Always deactivate when no refinement work found.  New refinement
-  // work may have arrived after we tried, but checking for that would
-  // still be racy.  Instead, the next time additional work is made
-  // available we'll get reactivated.
-  return true;
-}
-
-void G1PrimaryConcurrentRefineThread::activate() {
-  assert(this != Thread::current(), "precondition");
-  // The thread is running when notifications are disabled, so shouldn't
-  // signal is this case.  But there's a race between stop requests and
-  // maybe_deactivate, so also signal if stop requested.
-  size_t threshold = Atomic::load(&_threshold);
-  if (((threshold != SIZE_MAX) &&
-       (threshold == Atomic::cmpxchg(&_threshold, threshold, SIZE_MAX))) ||
-      should_terminate()) {
-    _notifier.signal();
-  }
-}
-
-void G1PrimaryConcurrentRefineThread::notify(size_t num_cards) {
-  // Only activate if the number of pending cards exceeds the activation
-  // threshold.  Notification is disabled when the thread is running, by
-  // setting _threshold to SIZE_MAX.  A relaxed load is sufficient; we don't
-  // need to be precise about this.
-  if (num_cards > Atomic::load(&_threshold)) {
-    // Discard notifications occurring during a safepoint.  A GC safepoint
-    // may dirty some cards (such as during reference processing), possibly
-    // leading to notification.  End-of-GC update_notify_threshold activates
-    // the primary thread if needed.  Non-GC safepoints are expected to
-    // rarely (if ever) dirty cards, so defer activation to a post-safepoint
-    // notification.
-    if (!SafepointSynchronize::is_at_safepoint()) {
-      activate();
-    }
-  }
-}
-
-void G1PrimaryConcurrentRefineThread::update_notify_threshold(size_t threshold) {
-#ifdef ASSERT
-  if (is_init_completed()) {
-    assert_at_safepoint();
-    assert(Thread::current()->is_VM_thread(), "precondition");
-  }
-#endif // ASSERT
-  // If _threshold is SIZE_MAX then the thread is active and the value
-  // of _threshold shouldn't be changed.
-  if (Atomic::load(&_threshold) != SIZE_MAX) {
-    Atomic::store(&_threshold, threshold);
-    if (G1BarrierSet::dirty_card_queue_set().num_cards() > threshold) {
-      activate();
-    }
-  }
-}
-
-class G1SecondaryConcurrentRefineThread final : public G1ConcurrentRefineThread {
-  Monitor _notifier;
-  bool _requested_active;
-
-  bool wait_for_completed_buffers() override;
-  bool maybe_deactivate() override;
-
-public:
-  G1SecondaryConcurrentRefineThread(G1ConcurrentRefine* cr, uint worker_id);
-
-  void activate() override;
-};
-
-G1SecondaryConcurrentRefineThread::G1SecondaryConcurrentRefineThread(G1ConcurrentRefine* cr,
-                                                                     uint worker_id) :
-  G1ConcurrentRefineThread(cr, worker_id),
-  _notifier(Mutex::nosafepoint, this->name(), true),
-  _requested_active(false)
-{
-  assert(worker_id > 0, "precondition");
-}
-
-bool G1SecondaryConcurrentRefineThread::wait_for_completed_buffers() {
-  assert(this == Thread::current(), "precondition");
-  MonitorLocker ml(&_notifier, Mutex::_no_safepoint_check_flag);
-  while (!_requested_active && !should_terminate()) {
-    ml.wait();
-  }
-  return !should_terminate();
-}
-
-void G1SecondaryConcurrentRefineThread::activate() {
   assert(this != Thread::current(), "precondition");
   MonitorLocker ml(&_notifier, Mutex::_no_safepoint_check_flag);
   if (!_requested_active || should_terminate()) {
@@ -249,19 +111,118 @@ void G1SecondaryConcurrentRefineThread::activate() {
   }
 }
 
-bool G1SecondaryConcurrentRefineThread::maybe_deactivate() {
+bool G1ConcurrentRefineThread::maybe_deactivate() {
   assert(this == Thread::current(), "precondition");
+  if (cr()->is_thread_wanted(_worker_id)) {
+    return false;
+  } else {
     MutexLocker ml(&_notifier, Mutex::_no_safepoint_check_flag);
     bool requested = _requested_active;
     _requested_active = false;
-  return !requested;            // Deactivate if not recently requested active.
+    return !requested;  // Deactivate only if not recently requested active.
+  }
+}
+
+bool G1ConcurrentRefineThread::try_refinement_step(size_t stop_at) {
+  assert(this == Thread::current(), "precondition");
+  return _cr->try_refinement_step(_worker_id, stop_at, &_refinement_stats);
+}
+
+void G1ConcurrentRefineThread::stop_service() {
+  activate();
+}
+
+// The (single) primary thread drives the controller for the refinement threads.
+class G1PrimaryConcurrentRefineThread final : public G1ConcurrentRefineThread {
+  bool wait_for_completed_buffers() override;
+  bool maybe_deactivate() override;
+  void do_refinement_step() override;
+
+public:
+  G1PrimaryConcurrentRefineThread(G1ConcurrentRefine* cr) :
+    G1ConcurrentRefineThread(cr, 0)
+  {}
+};
+
+// When inactive, the primary thread periodically wakes up and requests
+// adjustment of the number of active refinement threads.
+bool G1PrimaryConcurrentRefineThread::wait_for_completed_buffers() {
+  assert(this == Thread::current(), "precondition");
+  MonitorLocker ml(notifier(), Mutex::_no_safepoint_check_flag);
+  if (!requested_active() && !should_terminate()) {
+    // Rather than trying to be smart about spurious wakeups, we just treat
+    // them as timeouts.
+    ml.wait(cr()->adjust_threads_wait_ms());
+  }
+  // Record adjustment needed whenever reactivating.
+  cr()->record_thread_adjustment_needed();
+  return !should_terminate();
+}
+
+bool G1PrimaryConcurrentRefineThread::maybe_deactivate() {
+  // Don't deactivate while needing to adjust the number of active threads.
+  return !cr()->is_thread_adjustment_needed() &&
+         G1ConcurrentRefineThread::maybe_deactivate();
+}
+
+void G1PrimaryConcurrentRefineThread::do_refinement_step() {
+  // Try adjustment first.  If it succeeds then don't do any refinement this
+  // round.  This thread may have just woken up but no threads are currently
+  // needed, which is common.  In this case we want to just go back to
+  // waiting, with a minimum of fuss; in particular, don't do any "premature"
+  // refinement.  However, adjustment may be pending but temporarily
+  // blocked. In that case we *do* try refinement, rather than possibly
+  // uselessly spinning while waiting for adjustment to succeed.
+  if (!cr()->adjust_threads_periodically()) {
+    // No adjustment, so try refinement, with the target as a cuttoff.
+    if (!try_refinement_step(cr()->pending_cards_target())) {
+      // Refinement was cut off, so proceed with fewer threads.
+      cr()->reduce_threads_wanted();
+    }
+  }
+}
+
+class G1SecondaryConcurrentRefineThread final : public G1ConcurrentRefineThread {
+  bool wait_for_completed_buffers() override;
+  void do_refinement_step() override;
+
+public:
+  G1SecondaryConcurrentRefineThread(G1ConcurrentRefine* cr, uint worker_id) :
+    G1ConcurrentRefineThread(cr, worker_id)
+  {
+    assert(worker_id > 0, "precondition");
+  }
+};
+
+bool G1SecondaryConcurrentRefineThread::wait_for_completed_buffers() {
+  assert(this == Thread::current(), "precondition");
+  MonitorLocker ml(notifier(), Mutex::_no_safepoint_check_flag);
+  while (!requested_active() && !should_terminate()) {
+    ml.wait();
+  }
+  return !should_terminate();
+}
+
+void G1SecondaryConcurrentRefineThread::do_refinement_step() {
+  assert(this == Thread::current(), "precondition");
+  // Secondary threads ignore the target and just drive the number of pending
+  // dirty cards down.  The primary thread is responsible for noticing the
+  // target has been reached and reducing the number of wanted threads.  This
+  // makes the control of wanted threads all under the primary, while avoiding
+  // useless spinning by secondary threads until the primary thread notices.
+  // (Useless spinning is still possible if there are no pending cards, but
+  // that should rarely happen.)
+  try_refinement_step(0);
 }
 
 G1ConcurrentRefineThread*
 G1ConcurrentRefineThread::create(G1ConcurrentRefine* cr, uint worker_id) {
-  assert(worker_id > 0, "precondition");
+  G1ConcurrentRefineThread* crt;
-  G1ConcurrentRefineThread* crt =
+  if (worker_id == 0) {
-    new (std::nothrow) G1SecondaryConcurrentRefineThread(cr, worker_id);
+    crt = new (std::nothrow) G1PrimaryConcurrentRefineThread(cr);
+  } else {
+    crt = new (std::nothrow) G1SecondaryConcurrentRefineThread(cr, worker_id);
+  }
   if (crt != nullptr) {
     crt->create_and_start();
   }

src/hotspot/share/gc/g1/g1ConcurrentRefineThread.hpp

@@ -25,14 +25,13 @@
 #ifndef SHARE_GC_G1_G1CONCURRENTREFINETHREAD_HPP
 #define SHARE_GC_G1_G1CONCURRENTREFINETHREAD_HPP
 
+#include "gc/g1/g1ConcurrentRefineStats.hpp"
 #include "gc/shared/concurrentGCThread.hpp"
-#include "memory/padded.hpp"
+#include "runtime/mutex.hpp"
-#include "runtime/semaphore.hpp"
+#include "utilities/globalDefinitions.hpp"
-#include "utilities/macros.hpp"
 
 // Forward Decl.
 class G1ConcurrentRefine;
-class G1ConcurrentRefineStats;
 
 // One or more G1 Concurrent Refinement Threads may be active if concurrent
 // refinement is in progress.
@@ -43,7 +42,10 @@ class G1ConcurrentRefineThread: public ConcurrentGCThread {
   double _vtime_start;  // Initial virtual time.
   double _vtime_accum;  // Accumulated virtual time.
 
-  G1ConcurrentRefineStats* _refinement_stats;
+  Monitor _notifier;
+  bool _requested_active;
+
+  G1ConcurrentRefineStats _refinement_stats;
 
   uint _worker_id;
 
@@ -54,14 +56,29 @@ class G1ConcurrentRefineThread: public ConcurrentGCThread {
 protected:
   G1ConcurrentRefineThread(G1ConcurrentRefine* cr, uint worker_id);
 
+  Monitor* notifier() { return &_notifier; }
+  bool requested_active() const { return _requested_active; }
+
   // Returns !should_terminate().
   // precondition: this is the current thread.
   virtual bool wait_for_completed_buffers() = 0;
 
-  // Called when no refinement work found for this thread.
+  // Deactivate if appropriate.  Returns true if deactivated.
-  // Returns true if should deactivate.
   // precondition: this is the current thread.
-  virtual bool maybe_deactivate() = 0;
+  virtual bool maybe_deactivate();
+
+  // Attempt to do some refinement work.
+  // precondition: this is the current thread.
+  virtual void do_refinement_step() = 0;
+
+  // Helper for do_refinement_step implementations.  Try to perform some
+  // refinement work, limited by stop_at.  Returns true if any refinement work
+  // was performed, false if no work available per stop_at.
+  // precondition: this is the current thread.
+  bool try_refinement_step(size_t stop_at);
+
+  void report_active(const char* reason) const;
+  void report_inactive(const char* reason, const G1ConcurrentRefineStats& stats) const;
 
   G1ConcurrentRefine* cr() const { return _cr; }
 
@@ -70,51 +87,24 @@ protected:
 
 public:
   static G1ConcurrentRefineThread* create(G1ConcurrentRefine* cr, uint worker_id);
-  virtual ~G1ConcurrentRefineThread();
+  virtual ~G1ConcurrentRefineThread() = default;
+
+  uint worker_id() const { return _worker_id; }
 
   // Activate this thread.
   // precondition: this is not the current thread.
-  virtual void activate() = 0;
+  void activate();
 
-  G1ConcurrentRefineStats* refinement_stats() const {
+  G1ConcurrentRefineStats* refinement_stats() {
-    return _refinement_stats;
+    return &_refinement_stats;
+  }
+
+  const G1ConcurrentRefineStats* refinement_stats() const {
+    return &_refinement_stats;
   }
 
   // Total virtual time so far.
   double vtime_accum() { return _vtime_accum; }
 };
 
-// Singleton special refinement thread, registered with the dirty card queue.
-// This thread supports notification of increases to the number of cards in
-// the dirty card queue, which may trigger activation of this thread when it
-// is not already running.
-class G1PrimaryConcurrentRefineThread final : public G1ConcurrentRefineThread {
-  // Support for activation.  The thread waits on this semaphore when idle.
-  // Calls to activate signal it to wake the thread.
-  Semaphore _notifier;
-  DEFINE_PAD_MINUS_SIZE(0, DEFAULT_CACHE_LINE_SIZE, 0);
-  // Used as both the activation threshold and also the "is active" state.
-  // The value is SIZE_MAX when the thread is active, otherwise the threshold
-  // for signaling the semaphore.
-  volatile size_t _threshold;
-  DEFINE_PAD_MINUS_SIZE(1, DEFAULT_CACHE_LINE_SIZE, sizeof(size_t));
-
-  bool wait_for_completed_buffers() override;
-  bool maybe_deactivate() override;
-
-  G1PrimaryConcurrentRefineThread(G1ConcurrentRefine* cr);
-
-  void stop_service() override;
-
-public:
-  static G1PrimaryConcurrentRefineThread* create(G1ConcurrentRefine* cr);
-
-  void activate() override;
-
-  // Used by the write barrier support to activate the thread if needed when
-  // there are new refinement buffers.
-  void notify(size_t num_cards);
-  void update_notify_threshold(size_t threshold);
-};
-
 #endif // SHARE_GC_G1_G1CONCURRENTREFINETHREAD_HPP

src/hotspot/share/gc/g1/g1ConcurrentRefineThreadsNeeded.cpp

@@ -0,0 +1,143 @@
+/*
+ * Copyright (c) 2022, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#include "precompiled.hpp"
+#include "gc/g1/g1Analytics.hpp"
+#include "gc/g1/g1ConcurrentRefineThreadsNeeded.hpp"
+#include "gc/g1/heapRegion.hpp"
+#include "gc/g1/g1Policy.hpp"
+#include "utilities/globalDefinitions.hpp"
+#include <math.h>
+
+G1ConcurrentRefineThreadsNeeded::G1ConcurrentRefineThreadsNeeded(G1Policy* policy,
+                                                                 double update_period_ms) :
+  _policy(policy),
+  _update_period_ms(update_period_ms),
+  _predicted_time_until_next_gc_ms(0.0),
+  _predicted_cards_at_next_gc(0),
+  _threads_needed(0)
+{}
+
+// Estimate how many concurrent refinement threads we need to run to achieve
+// the target number of card by the time the next GC happens.  There are
+// several secondary goals we'd like to achieve while meeting that goal.
+//
+// 1. Minimize the number of refinement threads running at once.
+//
+// 2. Minimize the number of activations and deactivations for the
+// refinement threads that run.
+//
+// 3. Delay performing refinement work.  Having more dirty cards waiting to
+// be refined can be beneficial, as further writes to the same card don't
+// create more work.
+void G1ConcurrentRefineThreadsNeeded::update(uint active_threads,
+                                             size_t available_bytes,
+                                             size_t num_cards,
+                                             size_t target_num_cards) {
+  const G1Analytics* analytics = _policy->analytics();
+
+  // Estimate time until next GC, based on remaining bytes available for
+  // allocation and the allocation rate.
+  double alloc_region_rate = analytics->predict_alloc_rate_ms();
+  double alloc_bytes_rate = alloc_region_rate * HeapRegion::GrainBytes;
+  if (alloc_bytes_rate == 0.0) {
+    // A zero rate indicates we don't yet have data to use for predictions.
+    // Since we don't have any idea how long until the next GC, use a time of
+    // zero.
+    _predicted_time_until_next_gc_ms = 0.0;
+  } else {
+    // If the heap size is large and the allocation rate is small, we can get
+    // a predicted time until next GC that is so large it can cause problems
+    // (such as overflow) in other calculations.  Limit the prediction to one
+    // hour, which is still large in this context.
+    const double one_hour_ms = 60.0 * 60.0 * MILLIUNITS;
+    double raw_time_ms = available_bytes / alloc_bytes_rate;
+    _predicted_time_until_next_gc_ms = MIN2(raw_time_ms, one_hour_ms);
+  }
+
+  // Estimate number of cards that need to be processed before next GC.  There
+  // are no incoming cards when time is short, because in that case the
+  // controller activates refinement by mutator threads to stay on target even
+  // if threads deactivate in the meantime.  This also covers the case of not
+  // having a real prediction of time until GC.
+  size_t incoming_cards = 0;
+  if (_predicted_time_until_next_gc_ms > _update_period_ms) {
+    double incoming_rate = analytics->predict_dirtied_cards_rate_ms();
+    double raw_cards = incoming_rate * _predicted_time_until_next_gc_ms;
+    incoming_cards = static_cast<size_t>(raw_cards);
+  }
+  size_t total_cards = num_cards + incoming_cards;
+  _predicted_cards_at_next_gc = total_cards;
+
+  // No concurrent refinement needed.
+  if (total_cards <= target_num_cards) {
+    // We don't expect to exceed the target before the next GC.
+    _threads_needed = 0;
+    return;
+  }
+
+  // The calculation of the number of threads needed isn't very stable when
+  // time is short, and can lead to starting up lots of threads for not much
+  // profit.  If we're in the last update period, don't change the number of
+  // threads running, other than to treat the current thread as running.  That
+  // might not be sufficient, but hopefully we were already reasonably close.
+  // We won't accumulate more because mutator refinement will be activated.
+  if (_predicted_time_until_next_gc_ms <= _update_period_ms) {
+    _threads_needed = MAX2(active_threads, 1u);
+    return;
+  }
+
+  // Estimate the number of cards that need to be refined before the next GC
+  // to meet the goal.
+  size_t cards_needed = total_cards - target_num_cards;
+
+  // Estimate the rate at which a thread can refine cards.  If we don't yet
+  // have an estimate then only request one running thread, since we do have
+  // excess cards to process.  Just one thread might not be sufficient, but
+  // we don't have any idea how many we actually need.  Eventually the
+  // prediction machinery will warm up and we'll be able to get estimates.
+  double refine_rate = analytics->predict_concurrent_refine_rate_ms();
+  if (refine_rate == 0.0) {
+    _threads_needed = 1;
+    return;
+  }
+
+  // Estimate the number of refinement threads we need to run in order to
+  // reach the goal in time.
+  double thread_capacity = refine_rate * _predicted_time_until_next_gc_ms;
+  double nthreads = cards_needed / thread_capacity;
+
+  // Decide how to round nthreads to an integral number of threads.  Always
+  // rounding up is contrary to delaying refinement work.  But when we're
+  // close to the next GC we want to drive toward the target, so round up
+  // then.  The rest of the time we round to nearest, trying to remain near
+  // the middle of the range.
+  if (_predicted_time_until_next_gc_ms <= _update_period_ms * 5.0) {
+    nthreads = ::ceil(nthreads);
+  } else {
+    nthreads = ::round(nthreads);
+  }
+
+  _threads_needed = static_cast<uint>(MIN2<size_t>(nthreads, UINT_MAX));
+}

src/hotspot/share/gc/g1/g1ConcurrentRefineThreadsNeeded.hpp

@@ -0,0 +1,70 @@
+/*
+ * Copyright (c) 2022, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#ifndef SHARE_GC_G1_G1CONCURRENTREFINETHREADSNEEDED_HPP
+#define SHARE_GC_G1_G1CONCURRENTREFINETHREADSNEEDED_HPP
+
+#include "memory/allocation.hpp"
+#include "utilities/globalDefinitions.hpp"
+
+class G1Analytics;
+class G1Policy;
+
+// Used to compute the number of refinement threads that need to be running
+// in order to have the number of pending cards below the policy-directed
+// goal when the next GC occurs.
+class G1ConcurrentRefineThreadsNeeded : public CHeapObj<mtGC> {
+  G1Policy* _policy;
+  double _update_period_ms;
+  double _predicted_time_until_next_gc_ms;
+  size_t _predicted_cards_at_next_gc;
+  uint _threads_needed;
+
+public:
+  G1ConcurrentRefineThreadsNeeded(G1Policy* policy, double update_period_ms);
+
+  // Update the number of running refinement threads needed to reach the
+  // target before the next GC.
+  void update(uint active_threads,
+              size_t available_bytes,
+              size_t num_cards,
+              size_t target_num_cards);
+
+  // Estimate of the number of active refinement threads needed to reach the
+  // target before the next GC.
+  uint threads_needed() const { return _threads_needed; }
+
+  // Estimate of the time until the next GC.
+  double predicted_time_until_next_gc_ms() const {
+    return _predicted_time_until_next_gc_ms;
+  }
+
+  // Estimate of the number of pending cards at the next GC if no further
+  // refinement is performed.
+  size_t predicted_cards_at_next_gc() const {
+    return _predicted_cards_at_next_gc;
+  }
+};
+
+#endif // SHARE_GC_G1_G1CONCURRENTREFINETHREADSNEEDED_HPP

src/hotspot/share/gc/g1/g1DirtyCardQueue.cpp

@@ -67,13 +67,11 @@ static uint par_ids_start() { return 0; }
 
 G1DirtyCardQueueSet::G1DirtyCardQueueSet(BufferNode::Allocator* allocator) :
   PtrQueueSet(allocator),
-  _refinement_notification_thread(nullptr),
   _num_cards(0),
+  _mutator_refinement_threshold(SIZE_MAX),
   _completed(),
   _paused(),
   _free_ids(par_ids_start(), num_par_ids()),
-  _max_cards(MaxCardsUnlimited),
-  _padded_max_cards(MaxCardsUnlimited),
   _detached_refinement_stats()
 {}
 
@@ -126,17 +124,12 @@ void G1DirtyCardQueueSet::enqueue_completed_buffer(BufferNode* cbn) {
   assert(cbn != NULL, "precondition");
   // Increment _num_cards before adding to queue, so queue removal doesn't
   // need to deal with _num_cards possibly going negative.
-  size_t new_num_cards = Atomic::add(&_num_cards, buffer_size() - cbn->index());
+  Atomic::add(&_num_cards, buffer_size() - cbn->index());
-  {
   // Perform push in CS.  The old tail may be popped while the push is
   // observing it (attaching it to the new buffer).  We need to ensure it
   // can't be reused until the push completes, to avoid ABA problems.
   GlobalCounter::CriticalSection cs(Thread::current());
   _completed.push(*cbn);
-  }
-  if (_refinement_notification_thread != nullptr) {
-    _refinement_notification_thread->notify(new_num_cards);
-  }
 }
 
 // Thread-safe attempt to remove and return the first buffer from
@@ -493,7 +486,7 @@ void G1DirtyCardQueueSet::handle_completed_buffer(BufferNode* new_node,
   enqueue_completed_buffer(new_node);
 
   // No need for mutator refinement if number of cards is below limit.
-  if (Atomic::load(&_num_cards) <= Atomic::load(&_padded_max_cards)) {
+  if (Atomic::load(&_num_cards) <= Atomic::load(&_mutator_refinement_threshold)) {
     return;
   }
 
@@ -542,6 +535,9 @@ void G1DirtyCardQueueSet::abandon_logs() {
   abandon_completed_buffers();
   _detached_refinement_stats.reset();
 
+  // Disable mutator refinement until concurrent refinement decides otherwise.
+  set_mutator_refinement_threshold(SIZE_MAX);
+
   // Since abandon is done only at safepoints, we can safely manipulate
   // these queues.
   struct AbandonThreadLogClosure : public ThreadClosure {
@@ -557,13 +553,13 @@ void G1DirtyCardQueueSet::abandon_logs() {
 }
 
 void G1DirtyCardQueueSet::concatenate_logs() {
-  // Iterate over all the threads, if we find a partial log add it to
-  // the global list of logs.  Temporarily turn off the limit on the number
-  // of outstanding buffers.
   assert_at_safepoint();
-  size_t old_limit = max_cards();
-  set_max_cards(MaxCardsUnlimited);
 
+  // Disable mutator refinement until concurrent refinement decides otherwise.
+  set_mutator_refinement_threshold(SIZE_MAX);
+
+  // Iterate over all the threads, if we find a partial log add it to
+  // the global list of logs.
   struct ConcatenateThreadLogClosure : public ThreadClosure {
     G1DirtyCardQueueSet& _qset;
     ConcatenateThreadLogClosure(G1DirtyCardQueueSet& qset) : _qset(qset) {}
@@ -579,7 +575,6 @@ void G1DirtyCardQueueSet::concatenate_logs() {
 
   enqueue_all_paused_buffers();
   verify_num_cards();
-  set_max_cards(old_limit);
 }
 
 G1ConcurrentRefineStats G1DirtyCardQueueSet::get_and_reset_refinement_stats() {
@@ -616,27 +611,10 @@ void G1DirtyCardQueueSet::record_detached_refinement_stats(G1ConcurrentRefineSta
   stats->reset();
 }
 
-size_t G1DirtyCardQueueSet::max_cards() const {
+size_t G1DirtyCardQueueSet::mutator_refinement_threshold() const {
-  return _max_cards;
+  return Atomic::load(&_mutator_refinement_threshold);
 }
 
-void G1DirtyCardQueueSet::set_max_cards(size_t value) {
+void G1DirtyCardQueueSet::set_mutator_refinement_threshold(size_t value) {
-  _max_cards = value;
+  Atomic::store(&_mutator_refinement_threshold, value);
-  Atomic::store(&_padded_max_cards, value);
-}
-
-void G1DirtyCardQueueSet::set_max_cards_padding(size_t padding) {
-  // Compute sum, clipping to max.
-  size_t limit = _max_cards + padding;
-  if (limit < padding) {        // Check for overflow.
-    limit = MaxCardsUnlimited;
-  }
-  Atomic::store(&_padded_max_cards, limit);
-}
-
-void G1DirtyCardQueueSet::discard_max_cards_padding() {
-  // Being racy here is okay, since all threads store the same value.
-  if (_max_cards != Atomic::load(&_padded_max_cards)) {
-    Atomic::store(&_padded_max_cards, _max_cards);
-  }
 }

src/hotspot/share/gc/g1/g1DirtyCardQueue.hpp

@@ -156,12 +156,13 @@ class G1DirtyCardQueueSet: public PtrQueueSet {
     HeadTail take_all();
   };
 
-  // The refinement notification thread, for activation when the notification
+  DEFINE_PAD_MINUS_SIZE(0, DEFAULT_CACHE_LINE_SIZE, 0);
-  // threshold is reached.  nullptr if there aren't any refinement threads.
-  G1PrimaryConcurrentRefineThread* _refinement_notification_thread;
-  DEFINE_PAD_MINUS_SIZE(1, DEFAULT_CACHE_LINE_SIZE, sizeof(G1PrimaryConcurrentRefineThread*));
   // Upper bound on the number of cards in the completed and paused buffers.
   volatile size_t _num_cards;
+  DEFINE_PAD_MINUS_SIZE(1, DEFAULT_CACHE_LINE_SIZE, sizeof(size_t));
+  // If the queue contains more cards than configured here, the
+  // mutator must start doing some of the concurrent refinement work.
+  volatile size_t _mutator_refinement_threshold;
   DEFINE_PAD_MINUS_SIZE(2, DEFAULT_CACHE_LINE_SIZE, sizeof(size_t));
   // Buffers ready for refinement.
   // NonblockingQueue has inner padding of one cache line.
@@ -174,12 +175,6 @@ class G1DirtyCardQueueSet: public PtrQueueSet {
 
   G1FreeIdSet _free_ids;
 
-  // If the queue contains more cards than configured here, the
-  // mutator must start doing some of the concurrent refinement work.
-  size_t _max_cards;
-  volatile size_t _padded_max_cards;
-  static const size_t MaxCardsUnlimited = SIZE_MAX;
-
   G1ConcurrentRefineStats _detached_refinement_stats;
 
   // Verify _num_cards == sum of cards in the completed queue.
@@ -227,8 +222,8 @@ class G1DirtyCardQueueSet: public PtrQueueSet {
 
   // Enqueue the buffer, and optionally perform refinement by the mutator.
   // Mutator refinement is only done by Java threads, and only if there
-  // are more than max_cards (possibly padded) cards in the completed
+  // are more than mutator_refinement_threshold cards in the completed buffers.
-  // buffers.  Updates stats.
+  // Updates stats.
   //
   // Mutator refinement, if performed, stops processing a buffer if
   // SuspendibleThreadSet::should_yield(), recording the incompletely
@@ -252,12 +247,6 @@ public:
   // is a concurrent modification of the set of buffers.
   size_t num_cards() const;
 
-  // Record the primary concurrent refinement thread.  This is the thread to
-  // be notified when num_cards() exceeds the refinement notification threshold.
-  void set_refinement_notification_thread(G1PrimaryConcurrentRefineThread* thread) {
-    _refinement_notification_thread = thread;
-  }
-
   void merge_bufferlists(G1RedirtyCardsQueueSet* src);
 
   BufferNodeList take_all_completed_buffers();
@@ -293,18 +282,11 @@ public:
   // Accumulate refinement stats from threads that are detaching.
   void record_detached_refinement_stats(G1ConcurrentRefineStats* stats);
 
-  // Threshold for mutator threads to also do refinement when there
+  // Number of cards above which mutator threads should do refinement.
-  // are concurrent refinement threads.
+  size_t mutator_refinement_threshold() const;
-  size_t max_cards() const;
 
-  // Set threshold for mutator threads to also do refinement.
+  // Set number of cards above which mutator threads should do refinement.
-  void set_max_cards(size_t value);
+  void set_mutator_refinement_threshold(size_t value);
-
-  // Artificially increase mutator refinement threshold.
-  void set_max_cards_padding(size_t padding);
-
-  // Discard artificial increase of mutator refinement threshold.
-  void discard_max_cards_padding();
 };
 
 #endif // SHARE_GC_G1_G1DIRTYCARDQUEUE_HPP

src/hotspot/share/gc/g1/g1Policy.cpp

@@ -23,6 +23,7 @@
  */
 
 #include "precompiled.hpp"
+#include "gc/g1/g1Allocator.hpp"
 #include "gc/g1/g1Analytics.hpp"
 #include "gc/g1/g1Arguments.hpp"
 #include "gc/g1/g1CollectedHeap.inline.hpp"
@@ -518,10 +519,10 @@ G1GCPhaseTimes* G1Policy::phase_times() const {
 void G1Policy::revise_young_list_target_length(size_t rs_length) {
   guarantee(use_adaptive_young_list_length(), "should not call this otherwise" );
 
+  size_t thread_buffer_cards = _analytics->predict_dirtied_cards_in_thread_buffers();
   G1DirtyCardQueueSet& dcqs = G1BarrierSet::dirty_card_queue_set();
-  // We have no measure of the number of cards in the thread buffers, assume
+  size_t pending_cards = dcqs.num_cards() + thread_buffer_cards;
-  // these are very few compared to the ones in the DCQS.
+  update_young_length_bounds(pending_cards, rs_length);
-  update_young_length_bounds(dcqs.num_cards(), rs_length);
 }
 
 void G1Policy::record_full_collection_start() {
@@ -615,6 +616,11 @@ void G1Policy::record_concurrent_refinement_stats() {
   }
 }
 
+void G1Policy::record_concatenate_dirty_card_logs(Tickspan concat_time, size_t num_cards) {
+  _analytics->report_dirtied_cards_in_thread_buffers(num_cards);
+  phase_times()->record_concatenate_dirty_card_logs_time_ms(concat_time.seconds() * MILLIUNITS);
+}
+
 void G1Policy::record_young_collection_start() {
   Ticks now = Ticks::now();
   // We only need to do this here as the policy will only be applied
@@ -898,26 +904,44 @@ void G1Policy::record_young_collection_end(bool concurrent_operation_is_full_mar
   }
 
   // Note that _mmu_tracker->max_gc_time() returns the time in seconds.
-  double scan_logged_cards_time_goal_ms = _mmu_tracker->max_gc_time() * MILLIUNITS * G1RSetUpdatingPauseTimePercent / 100.0;
+  double logged_cards_time_goal_ms = _mmu_tracker->max_gc_time() * MILLIUNITS * G1RSetUpdatingPauseTimePercent / 100.0;
 
-  if (scan_logged_cards_time_goal_ms < merge_hcc_time_ms) {
+  if (logged_cards_time_goal_ms < merge_hcc_time_ms) {
     log_debug(gc, ergo, refine)("Adjust concurrent refinement thresholds (scanning the HCC expected to take longer than Update RS time goal)."
                                 "Logged Cards Scan time goal: %1.2fms Scan HCC time: %1.2fms",
-                                scan_logged_cards_time_goal_ms, merge_hcc_time_ms);
+                                logged_cards_time_goal_ms, merge_hcc_time_ms);
 
-    scan_logged_cards_time_goal_ms = 0;
+    logged_cards_time_goal_ms = 0;
   } else {
-    scan_logged_cards_time_goal_ms -= merge_hcc_time_ms;
+    logged_cards_time_goal_ms -= merge_hcc_time_ms;
   }
 
-  double const logged_cards_time = logged_cards_processing_time();
+  double const logged_cards_time_ms = logged_cards_processing_time();
+  size_t logged_cards =
+    phase_times()->sum_thread_work_items(G1GCPhaseTimes::MergeLB,
+                                         G1GCPhaseTimes::MergeLBDirtyCards);
+  size_t hcc_cards =
+    phase_times()->sum_thread_work_items(G1GCPhaseTimes::MergeHCC,
+                                         G1GCPhaseTimes::MergeHCCDirtyCards);
+  bool exceeded_goal = logged_cards_time_goal_ms < logged_cards_time_ms;
+  size_t predicted_thread_buffer_cards = _analytics->predict_dirtied_cards_in_thread_buffers();
+  G1ConcurrentRefine* cr = _g1h->concurrent_refine();
 
-  log_debug(gc, ergo, refine)("Concurrent refinement times: Logged Cards Scan time goal: %1.2fms Logged Cards Scan time: %1.2fms HCC time: %1.2fms",
+  log_debug(gc, ergo, refine)
-                              scan_logged_cards_time_goal_ms, logged_cards_time, merge_hcc_time_ms);
+           ("GC refinement: goal: %zu + %zu / %1.2fms, actual: %zu / %1.2fms, HCC: %zu / %1.2fms%s",
+            cr->pending_cards_target(),
+            predicted_thread_buffer_cards,
+            logged_cards_time_goal_ms,
+            logged_cards,
+            logged_cards_time_ms,
+            hcc_cards,
+            merge_hcc_time_ms,
+            (exceeded_goal ? " (exceeded goal)" : ""));
 
-  _g1h->concurrent_refine()->adjust(logged_cards_time,
+  cr->adjust_after_gc(logged_cards_time_ms,
-                                    phase_times()->sum_thread_work_items(G1GCPhaseTimes::MergeLB, G1GCPhaseTimes::MergeLBDirtyCards),
+                      logged_cards,
-                                    scan_logged_cards_time_goal_ms);
+                      predicted_thread_buffer_cards,
+                      logged_cards_time_goal_ms);
 }
 
 G1IHOPControl* G1Policy::create_ihop_control(const G1OldGenAllocationTracker* old_gen_alloc_tracker,
@@ -1078,6 +1102,16 @@ bool G1Policy::use_adaptive_young_list_length() const {
   return _young_gen_sizer.use_adaptive_young_list_length();
 }
 
+size_t G1Policy::estimate_used_young_bytes_locked() const {
+  assert_lock_strong(Heap_lock);
+  G1Allocator* allocator = _g1h->allocator();
+  uint used = _g1h->young_regions_count();
+  uint alloc = allocator->num_nodes();
+  uint full = used - MIN2(used, alloc);
+  size_t bytes_used = full * HeapRegion::GrainBytes;
+  return bytes_used + allocator->used_in_alloc_regions();
+}
+
 size_t G1Policy::desired_survivor_size(uint max_regions) const {
   size_t const survivor_capacity = HeapRegion::GrainWords * max_regions;
   return (size_t)((((double)survivor_capacity) * TargetSurvivorRatio) / 100);

src/hotspot/share/gc/g1/g1Policy.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016, 2021, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2016, 2022, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -36,6 +36,7 @@
 #include "gc/g1/g1YoungGenSizer.hpp"
 #include "gc/shared/gcCause.hpp"
 #include "utilities/pair.hpp"
+#include "utilities/ticks.hpp"
 
 // A G1Policy makes policy decisions that determine the
 // characteristics of the collector.  Examples include:
@@ -295,6 +296,8 @@ public:
   // This should be called after the heap is resized.
   void record_new_heap_size(uint new_number_of_regions);
 
+  void record_concatenate_dirty_card_logs(Tickspan concat_time, size_t num_cards);
+
   void init(G1CollectedHeap* g1h, G1CollectionSet* collection_set);
 
   // Record the start and end of the young gc pause.
@@ -389,6 +392,10 @@ public:
 
   bool use_adaptive_young_list_length() const;
 
+  // Return an estimate of the number of bytes used in young gen.
+  // precondition: holding Heap_lock
+  size_t estimate_used_young_bytes_locked() const;
+
   void transfer_survivors_to_cset(const G1SurvivorRegions* survivors);
 
 private:

src/hotspot/share/gc/g1/g1RemSet.cpp

@@ -40,7 +40,6 @@
 #include "gc/g1/g1Policy.hpp"
 #include "gc/g1/g1RootClosures.hpp"
 #include "gc/g1/g1RemSet.hpp"
-#include "gc/g1/g1ServiceThread.hpp"
 #include "gc/g1/g1_globals.hpp"
 #include "gc/g1/heapRegion.inline.hpp"
 #include "gc/g1/heapRegionManager.inline.hpp"
@@ -48,7 +47,6 @@
 #include "gc/shared/bufferNodeList.hpp"
 #include "gc/shared/gcTraceTime.inline.hpp"
 #include "gc/shared/ptrQueue.hpp"
-#include "gc/shared/suspendibleThreadSet.hpp"
 #include "jfr/jfrEvents.hpp"
 #include "memory/iterator.hpp"
 #include "memory/resourceArea.hpp"
@@ -468,118 +466,6 @@ public:
   }
 };
 
-class G1YoungRemSetSamplingClosure : public HeapRegionClosure {
-  SuspendibleThreadSetJoiner* _sts;
-  size_t _regions_visited;
-  size_t _sampled_rs_length;
-public:
-  G1YoungRemSetSamplingClosure(SuspendibleThreadSetJoiner* sts) :
-    HeapRegionClosure(), _sts(sts), _regions_visited(0), _sampled_rs_length(0) { }
-
-  virtual bool do_heap_region(HeapRegion* r) {
-    size_t rs_length = r->rem_set()->occupied();
-    _sampled_rs_length += rs_length;
-
-    // Update the collection set policy information for this region
-    G1CollectedHeap::heap()->collection_set()->update_young_region_prediction(r, rs_length);
-
-    _regions_visited++;
-
-    if (_regions_visited == 10) {
-      if (_sts->should_yield()) {
-        _sts->yield();
-        // A gc may have occurred and our sampling data is stale and further
-        // traversal of the collection set is unsafe
-        return true;
-      }
-      _regions_visited = 0;
-    }
-    return false;
-  }
-
-  size_t sampled_rs_length() const { return _sampled_rs_length; }
-};
-
-// Task handling young gen remembered set sampling.
-class G1RemSetSamplingTask : public G1ServiceTask {
-  // Helper to account virtual time.
-  class VTimer {
-    double _start;
-  public:
-    VTimer() : _start(os::elapsedVTime()) { }
-    double duration() { return os::elapsedVTime() - _start; }
-  };
-
-  double _vtime_accum;  // Accumulated virtual time.
-  void update_vtime_accum(double duration) {
-    _vtime_accum += duration;
-  }
-
-  // Sample the current length of remembered sets for young.
-  //
-  // At the end of the GC G1 determines the length of the young gen based on
-  // how much time the next GC can take, and when the next GC may occur
-  // according to the MMU.
-  //
-  // The assumption is that a significant part of the GC is spent on scanning
-  // the remembered sets (and many other components), so this thread constantly
-  // reevaluates the prediction for the remembered set scanning costs, and potentially
-  // G1Policy resizes the young gen. This may do a premature GC or even
-  // increase the young gen size to keep pause time length goal.
-  void sample_young_list_rs_length(SuspendibleThreadSetJoiner* sts){
-    G1CollectedHeap* g1h = G1CollectedHeap::heap();
-    G1Policy* policy = g1h->policy();
-    VTimer vtime;
-
-    if (policy->use_adaptive_young_list_length()) {
-      G1YoungRemSetSamplingClosure cl(sts);
-
-      G1CollectionSet* g1cs = g1h->collection_set();
-      g1cs->iterate(&cl);
-
-      if (cl.is_complete()) {
-        policy->revise_young_list_target_length(cl.sampled_rs_length());
-      }
-    }
-    update_vtime_accum(vtime.duration());
-  }
-
-  // There is no reason to do the sampling if a GC occurred recently. We use the
-  // G1ConcRefinementServiceIntervalMillis as the metric for recently and calculate
-  // the diff to the last GC. If the last GC occurred longer ago than the interval
-  // 0 is returned.
-  jlong reschedule_delay_ms() {
-    Tickspan since_last_gc = G1CollectedHeap::heap()->time_since_last_collection();
-    jlong delay = (jlong) (G1ConcRefinementServiceIntervalMillis - since_last_gc.milliseconds());
-    return MAX2<jlong>(0L, delay);
-  }
-
-public:
-  G1RemSetSamplingTask(const char* name) : G1ServiceTask(name) { }
-  virtual void execute() {
-    SuspendibleThreadSetJoiner sts;
-
-    // Reschedule if a GC happened too recently.
-    jlong delay_ms = reschedule_delay_ms();
-    if (delay_ms > 0) {
-      schedule(delay_ms);
-      return;
-    }
-
-    // Do the actual sampling.
-    sample_young_list_rs_length(&sts);
-    schedule(G1ConcRefinementServiceIntervalMillis);
-  }
-
-  double vtime_accum() {
-    // Only report vtime if supported by the os.
-    if (!os::supports_vtime()) {
-      return 0.0;
-    }
-    return _vtime_accum;
-  }
-};
-
 G1RemSet::G1RemSet(G1CollectedHeap* g1h,
                    G1CardTable* ct,
                    G1HotCardCache* hot_card_cache) :
@@ -588,30 +474,17 @@ G1RemSet::G1RemSet(G1CollectedHeap* g1h,
   _g1h(g1h),
   _ct(ct),
   _g1p(_g1h->policy()),
-  _hot_card_cache(hot_card_cache),
+  _hot_card_cache(hot_card_cache) {
-  _sampling_task(NULL) {
 }
 
 G1RemSet::~G1RemSet() {
   delete _scan_state;
-  delete _sampling_task;
 }
 
 void G1RemSet::initialize(uint max_reserved_regions) {
   _scan_state->initialize(max_reserved_regions);
 }
 
-void G1RemSet::initialize_sampling_task(G1ServiceThread* thread) {
-  assert(_sampling_task == NULL, "Sampling task already initialized");
-  _sampling_task = new G1RemSetSamplingTask("Remembered Set Sampling Task");
-  thread->register_task(_sampling_task);
-}
-
-double G1RemSet::sampling_task_vtime() {
-  assert(_sampling_task != NULL, "Must have been initialized");
-  return _sampling_task->vtime_accum();
-}
-
 // Helper class to scan and detect ranges of cards that need to be scanned on the
 // card table.
 class G1CardTableScanner : public StackObj {

src/hotspot/share/gc/g1/g1RemSet.hpp

@@ -70,7 +70,6 @@ private:
   G1CardTable*           _ct;
   G1Policy*              _g1p;
   G1HotCardCache*        _hot_card_cache;
-  G1RemSetSamplingTask*  _sampling_task;
 
   void print_merge_heap_roots_stats();
 
@@ -87,12 +86,6 @@ public:
            G1HotCardCache* hot_card_cache);
   ~G1RemSet();
 
-  // Initialize and schedule young remembered set sampling task.
-  void initialize_sampling_task(G1ServiceThread* thread);
-
-  // Accumulated vtime used by the sampling task.
-  double sampling_task_vtime();
-
   // Scan all cards in the non-collection set regions that potentially contain
   // references into the current whole collection set.
   void scan_heap_roots(G1ParScanThreadState* pss,

src/hotspot/share/gc/g1/g1RemSetSummary.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2013, 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2013, 2022, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -52,8 +52,6 @@ void G1RemSetSummary::update() {
 
   G1CollectedHeap* g1h = G1CollectedHeap::heap();
   g1h->concurrent_refine()->threads_do(&collector);
-
-  set_sampling_task_vtime(g1h->rem_set()->sampling_task_vtime());
 }
 
 void G1RemSetSummary::set_rs_thread_vtime(uint thread, double value) {
@@ -70,8 +68,7 @@ double G1RemSetSummary::rs_thread_vtime(uint thread) const {
 
 G1RemSetSummary::G1RemSetSummary(bool should_update) :
   _num_vtimes(G1ConcurrentRefine::max_num_threads()),
-  _rs_threads_vtimes(NEW_C_HEAP_ARRAY(double, _num_vtimes, mtGC)),
+  _rs_threads_vtimes(NEW_C_HEAP_ARRAY(double, _num_vtimes, mtGC)) {
-  _sampling_task_vtime(0.0f) {
 
   memset(_rs_threads_vtimes, 0, sizeof(double) * _num_vtimes);
 
@@ -89,8 +86,6 @@ void G1RemSetSummary::set(G1RemSetSummary* other) {
   assert(_num_vtimes == other->_num_vtimes, "just checking");
 
   memcpy(_rs_threads_vtimes, other->_rs_threads_vtimes, sizeof(double) * _num_vtimes);
-
-  set_sampling_task_vtime(other->sampling_task_vtime());
 }
 
 void G1RemSetSummary::subtract_from(G1RemSetSummary* other) {
@@ -100,8 +95,6 @@ void G1RemSetSummary::subtract_from(G1RemSetSummary* other) {
   for (uint i = 0; i < _num_vtimes; i++) {
     set_rs_thread_vtime(i, other->rs_thread_vtime(i) - rs_thread_vtime(i));
   }
-
-  _sampling_task_vtime = other->sampling_task_vtime() - _sampling_task_vtime;
 }
 
 class RegionTypeCounter {
@@ -329,8 +322,6 @@ void G1RemSetSummary::print_on(outputStream* out, bool show_thread_times) {
       out->print("    %5.2f", rs_thread_vtime(i));
     }
     out->cr();
-    out->print_cr(" Sampling task time (ms)");
-    out->print_cr("         %5.3f", sampling_task_vtime() * MILLIUNITS);
   }
 
   HRRSStatsIter blk;

src/hotspot/share/gc/g1/g1RemSetSummary.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2013, 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2013, 2022, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -37,12 +37,7 @@ class G1RemSetSummary {
   size_t _num_vtimes;
   double* _rs_threads_vtimes;
 
-  double _sampling_task_vtime;
-
   void set_rs_thread_vtime(uint thread, double value);
-  void set_sampling_task_vtime(double value) {
-    _sampling_task_vtime = value;
-  }
 
   // update this summary with current data from various places
   void update();
@@ -60,10 +55,6 @@ public:
   void print_on(outputStream* out, bool show_thread_times);
 
   double rs_thread_vtime(uint thread) const;
-
-  double sampling_task_vtime() const {
-    return _sampling_task_vtime;
-  }
 };
 
 #endif // SHARE_GC_G1_G1REMSETSUMMARY_HPP

src/hotspot/share/gc/g1/g1YoungCollector.cpp

@@ -467,6 +467,16 @@ void G1YoungCollector::set_young_collection_default_active_worker_threads(){
   log_info(gc,task)("Using %u workers of %u for evacuation", active_workers, workers()->max_workers());
 }
 
+void G1YoungCollector::flush_dirty_card_queues() {
+  Ticks start = Ticks::now();
+  G1DirtyCardQueueSet& qset = G1BarrierSet::dirty_card_queue_set();
+  size_t old_cards = qset.num_cards();
+  qset.concatenate_logs();
+  size_t added_cards = qset.num_cards() - old_cards;
+  Tickspan concat_time = Ticks::now() - start;
+  policy()->record_concatenate_dirty_card_logs(concat_time, added_cards);
+}
+
 void G1YoungCollector::pre_evacuate_collection_set(G1EvacInfo* evacuation_info, G1ParScanThreadStateSet* per_thread_states) {
   // Please see comment in g1CollectedHeap.hpp and
   // G1CollectedHeap::ref_processing_init() to see how
@@ -483,14 +493,9 @@ void G1YoungCollector::pre_evacuate_collection_set(G1EvacInfo* evacuation_info,
     phase_times()->record_prepare_tlab_time_ms((Ticks::now() - start).seconds() * 1000.0);
   }
 
-  {
   // Flush dirty card queues to qset, so later phases don't need to account
   // for partially filled per-thread queues and such.
-    Ticks start = Ticks::now();
+  flush_dirty_card_queues();
-    G1BarrierSet::dirty_card_queue_set().concatenate_logs();
-    Tickspan dt = Ticks::now() - start;
-    phase_times()->record_concatenate_dirty_card_logs_time_ms(dt.seconds() * MILLIUNITS);
-  }
 
   hot_card_cache()->reset_hot_cache_claimed_index();
 

src/hotspot/share/gc/g1/g1YoungCollector.hpp

@@ -99,6 +99,8 @@ class G1YoungCollector {
 
   void set_young_collection_default_active_worker_threads();
 
+  void flush_dirty_card_queues();
+
   void pre_evacuate_collection_set(G1EvacInfo* evacuation_info, G1ParScanThreadStateSet* pss);
   // Actually do the work of evacuating the parts of the collection set.
   // The has_optional_evacuation_work flag for the initial collection set

src/hotspot/share/gc/g1/g1_globals.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2001, 2021, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2001, 2022, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -168,45 +168,15 @@
           "Size of an update buffer")                                       \
           range(1, NOT_LP64(32*M) LP64_ONLY(1*G))                           \
                                                                             \
-  product(size_t, G1ConcRefinementYellowZone, 0,                            \
-          "Number of enqueued update buffers that will "                    \
-          "trigger concurrent processing. Will be selected ergonomically "  \
-          "by default.")                                                    \
-          range(0, max_intx)                                                \
-                                                                            \
-  product(size_t, G1ConcRefinementRedZone, 0,                               \
-          "Maximum number of enqueued update buffers before mutator "       \
-          "threads start processing new ones instead of enqueueing them. "  \
-          "Will be selected ergonomically by default.")                     \
-          range(0, max_intx)                                                \
-                                                                            \
-  product(size_t, G1ConcRefinementGreenZone, 0,                             \
-          "The number of update buffers that are left in the queue by the " \
-          "concurrent processing threads. Will be selected ergonomically "  \
-          "by default.")                                                    \
-          range(0, max_intx)                                                \
-                                                                            \
-  product(uintx, G1ConcRefinementServiceIntervalMillis, 300,                \
-          "The G1 service thread wakes up every specified number of "       \
-          "milliseconds to do miscellaneous work.")                         \
-          range(0, max_jint)                                                \
-                                                                            \
-  product(size_t, G1ConcRefinementThresholdStep, 2,                         \
-          "Each time the remembered set update queue increases by this "    \
-          "amount activate the next refinement thread if available. "       \
-          "The actual step size will be selected ergonomically by "         \
-          "default, with this value used to determine a lower bound.")      \
-          range(1, SIZE_MAX)                                                \
-                                                                            \
   product(intx, G1RSetUpdatingPauseTimePercent, 10,                         \
           "A target percentage of time that is allowed to be spend on "     \
           "processing remembered set update buffers during the collection " \
           "pause.")                                                         \
           range(0, 100)                                                     \
                                                                             \
-  product(bool, G1UseAdaptiveConcRefinement, true,                          \
+  product(bool, G1UseConcRefinement, true, DIAGNOSTIC,                      \
-          "Select green, yellow and red zones adaptively to meet the "      \
+          "Control whether concurrent refinement is performed. "            \
-          "the pause requirements.")                                        \
+          "Disabling effectively ignores G1RSetUpdatingPauseTimePercent")   \
                                                                             \
   product(size_t, G1ConcRSLogCacheSize, 10,                                 \
           "Log base 2 of the length of conc RS hot-card cache.")            \

src/hotspot/share/runtime/arguments.cpp

@@ -552,6 +552,13 @@ static SpecialFlag const special_jvm_flags[] = {
   { "AliasLevel",                   JDK_Version::jdk(19), JDK_Version::jdk(20), JDK_Version::jdk(21) },
   { "UseCodeAging",                 JDK_Version::undefined(), JDK_Version::jdk(20), JDK_Version::jdk(21) },
 
+  { "G1ConcRefinementGreenZone",    JDK_Version::undefined(), JDK_Version::jdk(20), JDK_Version::undefined() },
+  { "G1ConcRefinementYellowZone",   JDK_Version::undefined(), JDK_Version::jdk(20), JDK_Version::undefined() },
+  { "G1ConcRefinementRedZone",      JDK_Version::undefined(), JDK_Version::jdk(20), JDK_Version::undefined() },
+  { "G1ConcRefinementThresholdStep", JDK_Version::undefined(), JDK_Version::jdk(20), JDK_Version::undefined() },
+  { "G1UseAdaptiveConcRefinement",  JDK_Version::undefined(), JDK_Version::jdk(20), JDK_Version::undefined() },
+  { "G1ConcRefinementServiceIntervalMillis", JDK_Version::undefined(), JDK_Version::jdk(20), JDK_Version::undefined() },
+
 #ifdef ASSERT
   { "DummyObsoleteTestFlag",        JDK_Version::undefined(), JDK_Version::jdk(18), JDK_Version::undefined() },
 #endif

test/hotspot/gtest/gc/g1/test_g1ServiceThread.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2021, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2021, 2022, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -76,7 +76,6 @@ TEST_VM(G1ServiceThread, test_add_while_waiting) {
   // Make sure default tasks use long intervals so that the service thread
   // is doing a long wait for the next execution.
   AutoModifyRestore<uintx> f1(G1PeriodicGCInterval, 100000);
-  AutoModifyRestore<uintx> f2(G1ConcRefinementServiceIntervalMillis, 100000);
 
   // Create thread and let it start.
   G1ServiceThread* st = new G1ServiceThread();

test/hotspot/jtreg/gc/g1/TestBuffersToCardsOverflow.java

@@ -1,46 +0,0 @@
-/*
- * Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved.
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
- * or visit www.oracle.com if you need additional information or have any
- * questions.
- */
-
-/**
- * @test
- * @bug 8257228
- * @library /test/lib
- * @requires vm.bits == 64
- * @build gc.g1.TestBuffersToCardsOverflow jdk.test.lib.process.*
- * @run main gc.g1.TestBuffersToCardsOverflow
- */
-
-package gc.g1;
-
-import jdk.test.lib.process.ProcessTools;
-
-public class TestBuffersToCardsOverflow {
-    public static void main(String... args) throws Exception {
-        ProcessTools.executeTestJava("-XX:G1ConcRefinementThresholdStep=16G",
-                                     "-XX:G1UpdateBufferSize=1G")
-                .outputTo(System.out)
-                .errorTo(System.out)
-                .stdoutShouldNotContain("SIGFPE")
-                .stdoutShouldNotContain("hs_err");
-    }
-}