From c157b744edc089486b7427af0ad41d6da751b4ba Mon Sep 17 00:00:00 2001 From: Antonios Printezis Date: Fri, 5 Feb 2010 11:05:50 -0500 Subject: [PATCH 1/4] 6920977: G1: guarantee(k == probe->klass(),"klass should be in dictionary") fails The guarantee is too strict and the test will fail (incorrectly) if the class is not in the system dictionary but in the placeholders. Reviewed-by: acorn, phh --- .../share/vm/classfile/loaderConstraints.cpp | 21 +++++++++++++++++-- .../share/vm/classfile/loaderConstraints.hpp | 2 +- .../share/vm/classfile/systemDictionary.cpp | 2 +- hotspot/src/share/vm/includeDB_core | 1 + 4 files changed, 22 insertions(+), 4 deletions(-) diff --git a/hotspot/src/share/vm/classfile/loaderConstraints.cpp b/hotspot/src/share/vm/classfile/loaderConstraints.cpp index daaaa327556..0a8693403d1 100644 --- a/hotspot/src/share/vm/classfile/loaderConstraints.cpp +++ b/hotspot/src/share/vm/classfile/loaderConstraints.cpp @@ -457,7 +457,8 @@ void LoaderConstraintTable::merge_loader_constraints( } -void LoaderConstraintTable::verify(Dictionary* dictionary) { +void LoaderConstraintTable::verify(Dictionary* dictionary, + PlaceholderTable* placeholders) { Thread *thread = Thread::current(); for (int cindex = 0; cindex < _loader_constraint_size; cindex++) { for (LoaderConstraintEntry* probe = bucket(cindex); @@ -472,7 +473,23 @@ void LoaderConstraintTable::verify(Dictionary* dictionary) { unsigned int d_hash = dictionary->compute_hash(name, loader); int d_index = dictionary->hash_to_index(d_hash); klassOop k = dictionary->find_class(d_index, d_hash, name, loader); - guarantee(k == probe->klass(), "klass should be in dictionary"); + if (k != NULL) { + // We found the class in the system dictionary, so we should + // make sure that the klassOop matches what we already have. + guarantee(k == probe->klass(), "klass should be in dictionary"); + } else { + // If we don't find the class in the system dictionary, it + // has to be in the placeholders table. + unsigned int p_hash = placeholders->compute_hash(name, loader); + int p_index = placeholders->hash_to_index(p_hash); + PlaceholderEntry* entry = placeholders->get_entry(p_index, p_hash, + name, loader); + + // The instanceKlass might not be on the entry, so the only + // thing we can check here is whether we were successful in + // finding the class in the placeholders table. + guarantee(entry != NULL, "klass should be in the placeholders"); + } } for (int n = 0; n< probe->num_loaders(); n++) { guarantee(probe->loader(n)->is_oop_or_null(), "should be oop"); diff --git a/hotspot/src/share/vm/classfile/loaderConstraints.hpp b/hotspot/src/share/vm/classfile/loaderConstraints.hpp index 6928180d22a..53ce146b168 100644 --- a/hotspot/src/share/vm/classfile/loaderConstraints.hpp +++ b/hotspot/src/share/vm/classfile/loaderConstraints.hpp @@ -84,7 +84,7 @@ public: void purge_loader_constraints(BoolObjectClosure* is_alive); - void verify(Dictionary* dictionary); + void verify(Dictionary* dictionary, PlaceholderTable* placeholders); #ifndef PRODUCT void print(); #endif diff --git a/hotspot/src/share/vm/classfile/systemDictionary.cpp b/hotspot/src/share/vm/classfile/systemDictionary.cpp index fb22282e97c..1db0a5cfb15 100644 --- a/hotspot/src/share/vm/classfile/systemDictionary.cpp +++ b/hotspot/src/share/vm/classfile/systemDictionary.cpp @@ -2573,7 +2573,7 @@ void SystemDictionary::verify() { // Verify constraint table guarantee(constraints() != NULL, "Verify of loader constraints failed"); - constraints()->verify(dictionary()); + constraints()->verify(dictionary(), placeholders()); } diff --git a/hotspot/src/share/vm/includeDB_core b/hotspot/src/share/vm/includeDB_core index 675a6657e0b..bf994522030 100644 --- a/hotspot/src/share/vm/includeDB_core +++ b/hotspot/src/share/vm/includeDB_core @@ -2654,6 +2654,7 @@ loaderConstraints.cpp resourceArea.hpp loaderConstraints.cpp safepoint.hpp loaderConstraints.hpp dictionary.hpp +loaderConstraints.hpp placeholders.hpp loaderConstraints.hpp hashtable.hpp location.cpp debugInfo.hpp From a88853347fa2a62d3670800026225e5b032aa0bd Mon Sep 17 00:00:00 2001 From: Antonios Printezis Date: Mon, 8 Feb 2010 14:23:01 -0500 Subject: [PATCH 2/4] 6802453: G1: hr()->is_in_reserved(from),"Precondition." The operations of re-using a RSet component and expanding the same RSet component were not mutually exlusive, and this could lead to RSets getting corrupted and entries being dropped. Reviewed-by: iveresov, johnc --- .../gc_implementation/g1/heapRegionRemSet.cpp | 96 +++++++++++-------- 1 file changed, 55 insertions(+), 41 deletions(-) diff --git a/hotspot/src/share/vm/gc_implementation/g1/heapRegionRemSet.cpp b/hotspot/src/share/vm/gc_implementation/g1/heapRegionRemSet.cpp index dd04bc8dcb8..aac289e67d6 100644 --- a/hotspot/src/share/vm/gc_implementation/g1/heapRegionRemSet.cpp +++ b/hotspot/src/share/vm/gc_implementation/g1/heapRegionRemSet.cpp @@ -258,42 +258,6 @@ class PosParPRT: public PerRegionTable { ReserveParTableExpansion = 1 }; - void par_expand() { - int n = HeapRegionRemSet::num_par_rem_sets()-1; - if (n <= 0) return; - if (_par_tables == NULL) { - PerRegionTable* res = - (PerRegionTable*) - Atomic::cmpxchg_ptr((PerRegionTable*)ReserveParTableExpansion, - &_par_tables, NULL); - if (res != NULL) return; - // Otherwise, we reserved the right to do the expansion. - - PerRegionTable** ptables = NEW_C_HEAP_ARRAY(PerRegionTable*, n); - for (int i = 0; i < n; i++) { - PerRegionTable* ptable = PerRegionTable::alloc(hr()); - ptables[i] = ptable; - } - // Here we do not need an atomic. - _par_tables = ptables; -#if COUNT_PAR_EXPANDS - print_par_expand(); -#endif - // We must put this table on the expanded list. - PosParPRT* exp_head = _par_expanded_list; - while (true) { - set_next_par_expanded(exp_head); - PosParPRT* res = - (PosParPRT*) - Atomic::cmpxchg_ptr(this, &_par_expanded_list, exp_head); - if (res == exp_head) return; - // Otherwise. - exp_head = res; - } - ShouldNotReachHere(); - } - } - void par_contract() { assert(_par_tables != NULL, "Precondition."); int n = HeapRegionRemSet::num_par_rem_sets()-1; @@ -391,13 +355,49 @@ public: void set_next(PosParPRT* nxt) { _next = nxt; } PosParPRT** next_addr() { return &_next; } + bool should_expand(int tid) { + return par_tables() == NULL && tid > 0 && hr()->is_gc_alloc_region(); + } + + void par_expand() { + int n = HeapRegionRemSet::num_par_rem_sets()-1; + if (n <= 0) return; + if (_par_tables == NULL) { + PerRegionTable* res = + (PerRegionTable*) + Atomic::cmpxchg_ptr((PerRegionTable*)ReserveParTableExpansion, + &_par_tables, NULL); + if (res != NULL) return; + // Otherwise, we reserved the right to do the expansion. + + PerRegionTable** ptables = NEW_C_HEAP_ARRAY(PerRegionTable*, n); + for (int i = 0; i < n; i++) { + PerRegionTable* ptable = PerRegionTable::alloc(hr()); + ptables[i] = ptable; + } + // Here we do not need an atomic. + _par_tables = ptables; +#if COUNT_PAR_EXPANDS + print_par_expand(); +#endif + // We must put this table on the expanded list. + PosParPRT* exp_head = _par_expanded_list; + while (true) { + set_next_par_expanded(exp_head); + PosParPRT* res = + (PosParPRT*) + Atomic::cmpxchg_ptr(this, &_par_expanded_list, exp_head); + if (res == exp_head) return; + // Otherwise. + exp_head = res; + } + ShouldNotReachHere(); + } + } + void add_reference(OopOrNarrowOopStar from, int tid) { // Expand if necessary. PerRegionTable** pt = par_tables(); - if (par_tables() == NULL && tid > 0 && hr()->is_gc_alloc_region()) { - par_expand(); - pt = par_tables(); - } if (pt != NULL) { // We always have to assume that mods to table 0 are in parallel, // because of the claiming scheme in parallel expansion. A thread @@ -696,7 +696,21 @@ void OtherRegionsTable::add_reference(OopOrNarrowOopStar from, int tid) { // OtherRegionsTable for why this is OK. assert(prt != NULL, "Inv"); - prt->add_reference(from, tid); + if (prt->should_expand(tid)) { + MutexLockerEx x(&_m, Mutex::_no_safepoint_check_flag); + HeapRegion* prt_hr = prt->hr(); + if (prt_hr == from_hr) { + // Make sure the table still corresponds to the same region + prt->par_expand(); + prt->add_reference(from, tid); + } + // else: The table has been concurrently coarsened, evicted, and + // the table data structure re-used for another table. So, we + // don't need to add the reference any more given that the table + // has been coarsened and the whole region will be scanned anyway. + } else { + prt->add_reference(from, tid); + } if (G1RecordHRRSOops) { HeapRegionRemSet::record(hr(), from); #if HRRS_VERBOSE From 0356567ed8ca3f41b7acca83a9fd98b045424699 Mon Sep 17 00:00:00 2001 From: John Cuthbertson Date: Tue, 9 Feb 2010 13:56:09 -0800 Subject: [PATCH 3/4] 6782663: Data produced by PrintGCApplicationConcurrentTime and PrintGCApplicationStoppedTime is not accurate Update and display the timers associated with these flags for all safepoints. Reviewed-by: ysr, jcoomes --- hotspot/src/share/vm/runtime/vmThread.cpp | 13 +------------ .../src/share/vm/services/runtimeService.cpp | 18 +++++++++++++++++- 2 files changed, 18 insertions(+), 13 deletions(-) diff --git a/hotspot/src/share/vm/runtime/vmThread.cpp b/hotspot/src/share/vm/runtime/vmThread.cpp index 26dad9a689c..ae7377b3ca1 100644 --- a/hotspot/src/share/vm/runtime/vmThread.cpp +++ b/hotspot/src/share/vm/runtime/vmThread.cpp @@ -1,5 +1,5 @@ /* - * Copyright 1998-2009 Sun Microsystems, Inc. All Rights Reserved. + * Copyright 1998-2010 Sun Microsystems, Inc. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -426,11 +426,6 @@ void VMThread::loop() { // follow that also require a safepoint if (_cur_vm_operation->evaluate_at_safepoint()) { - if (PrintGCApplicationConcurrentTime) { - gclog_or_tty->print_cr("Application time: %3.7f seconds", - RuntimeService::last_application_time_sec()); - } - _vm_queue->set_drain_list(safepoint_ops); // ensure ops can be scanned SafepointSynchronize::begin(); @@ -477,12 +472,6 @@ void VMThread::loop() { // Complete safepoint synchronization SafepointSynchronize::end(); - if (PrintGCApplicationStoppedTime) { - gclog_or_tty->print_cr("Total time for which application threads " - "were stopped: %3.7f seconds", - RuntimeService::last_safepoint_time_sec()); - } - } else { // not a safepoint operation if (TraceLongCompiles) { elapsedTimer t; diff --git a/hotspot/src/share/vm/services/runtimeService.cpp b/hotspot/src/share/vm/services/runtimeService.cpp index 16811a2aa5b..3e966009bef 100644 --- a/hotspot/src/share/vm/services/runtimeService.cpp +++ b/hotspot/src/share/vm/services/runtimeService.cpp @@ -1,5 +1,5 @@ /* - * Copyright 2003-2006 Sun Microsystems, Inc. All Rights Reserved. + * Copyright 2003-2010 Sun Microsystems, Inc. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -104,6 +104,13 @@ void RuntimeService::init() { void RuntimeService::record_safepoint_begin() { HS_DTRACE_PROBE(hs_private, safepoint__begin); + + // Print the time interval in which the app was executing + if (PrintGCApplicationConcurrentTime) { + gclog_or_tty->print_cr("Application time: %3.7f seconds", + last_application_time_sec()); + } + // update the time stamp to begin recording safepoint time _safepoint_timer.update(); if (UsePerfData) { @@ -122,6 +129,15 @@ void RuntimeService::record_safepoint_synchronized() { void RuntimeService::record_safepoint_end() { HS_DTRACE_PROBE(hs_private, safepoint__end); + + // Print the time interval for which the app was stopped + // during the current safepoint operation. + if (PrintGCApplicationStoppedTime) { + gclog_or_tty->print_cr("Total time for which application threads " + "were stopped: %3.7f seconds", + last_safepoint_time_sec()); + } + // update the time stamp to begin recording app time _app_timer.update(); if (UsePerfData) { From 52b92d5593c491a394d94b1a437a3ee0254ac55d Mon Sep 17 00:00:00 2001 From: Igor Veresov Date: Thu, 11 Feb 2010 15:52:19 -0800 Subject: [PATCH 4/4] 6923991: G1: improve scalability of RSet scanning Implemented block-based work stealing. Moved copying during the rset scanning phase to the main copying phase. Made the size of rset table depend on the region size. Reviewed-by: apetrusenko, tonyp --- .../gc_implementation/g1/g1CollectedHeap.cpp | 33 ++-- .../gc_implementation/g1/g1CollectedHeap.hpp | 24 +-- .../g1/g1CollectorPolicy.cpp | 1 + .../vm/gc_implementation/g1/g1OopClosures.hpp | 26 +++- .../g1/g1OopClosures.inline.hpp | 13 ++ .../vm/gc_implementation/g1/g1RemSet.cpp | 61 +++----- .../vm/gc_implementation/g1/g1_globals.hpp | 20 ++- .../g1/g1_specialized_oop_closures.hpp | 6 +- .../gc_implementation/g1/heapRegionRemSet.cpp | 35 +++-- .../gc_implementation/g1/heapRegionRemSet.hpp | 17 ++- .../vm/gc_implementation/g1/sparsePRT.cpp | 143 ++++++++++-------- .../vm/gc_implementation/g1/sparsePRT.hpp | 38 +++-- .../src/share/vm/memory/cardTableModRefBS.hpp | 10 ++ .../share/vm/utilities/globalDefinitions.hpp | 4 + 14 files changed, 263 insertions(+), 168 deletions(-) diff --git a/hotspot/src/share/vm/gc_implementation/g1/g1CollectedHeap.cpp b/hotspot/src/share/vm/gc_implementation/g1/g1CollectedHeap.cpp index 0c73f4d6c67..9b2b71292d8 100644 --- a/hotspot/src/share/vm/gc_implementation/g1/g1CollectedHeap.cpp +++ b/hotspot/src/share/vm/gc_implementation/g1/g1CollectedHeap.cpp @@ -2646,6 +2646,13 @@ G1CollectedHeap::cleanup_surviving_young_words() { // +struct PrepareForRSScanningClosure : public HeapRegionClosure { + bool doHeapRegion(HeapRegion *r) { + r->rem_set()->set_iter_claimed(0); + return false; + } +}; + void G1CollectedHeap::do_collection_pause_at_safepoint() { if (PrintHeapAtGC) { @@ -2784,6 +2791,8 @@ G1CollectedHeap::do_collection_pause_at_safepoint() { gclog_or_tty->print_cr("\nAfter pause, heap:"); print(); #endif + PrepareForRSScanningClosure prepare_for_rs_scan; + collection_set_iterate(&prepare_for_rs_scan); setup_surviving_young_words(); @@ -3781,22 +3790,16 @@ oop G1ParCopyHelper::copy_to_survivor_space(oop old) { return obj; } -template +template template -void G1ParCopyClosure +void G1ParCopyClosure ::do_oop_work(T* p) { oop obj = oopDesc::load_decode_heap_oop(p); assert(barrier != G1BarrierRS || obj != NULL, "Precondition: G1BarrierRS implies obj is nonNull"); - // The only time we skip the cset test is when we're scanning - // references popped from the queue. And we only push on the queue - // references that we know point into the cset, so no point in - // checking again. But we'll leave an assert here for peace of mind. - assert(!skip_cset_test || _g1->obj_in_cs(obj), "invariant"); - // here the null check is implicit in the cset_fast_test() test - if (skip_cset_test || _g1->in_cset_fast_test(obj)) { + if (_g1->in_cset_fast_test(obj)) { #if G1_REM_SET_LOGGING gclog_or_tty->print_cr("Loc "PTR_FORMAT" contains pointer "PTR_FORMAT" " "into CS.", p, (void*) obj); @@ -3813,7 +3816,6 @@ void G1ParCopyClosure update_rs(_from, p, _par_scan_state->queue_num()); } @@ -3823,8 +3825,8 @@ void G1ParCopyClosure ::do_oop_work(oop* p); -template void G1ParCopyClosure::do_oop_work(narrowOop* p); +template void G1ParCopyClosure::do_oop_work(oop* p); +template void G1ParCopyClosure::do_oop_work(narrowOop* p); template void G1ParScanPartialArrayClosure::do_oop_nv(T* p) { assert(has_partial_array_mask(p), "invariant"); @@ -3896,11 +3898,11 @@ public: assert(UseCompressedOops, "Error"); narrowOop* p = (narrowOop*) stolen_task; assert(has_partial_array_mask(p) || - _g1h->obj_in_cs(oopDesc::load_decode_heap_oop(p)), "Error"); + _g1h->is_in_g1_reserved(oopDesc::load_decode_heap_oop(p)), "Error"); pss->push_on_queue(p); } else { oop* p = (oop*) stolen_task; - assert(has_partial_array_mask(p) || _g1h->obj_in_cs(*p), "Error"); + assert(has_partial_array_mask(p) || _g1h->is_in_g1_reserved(*p), "Error"); pss->push_on_queue(p); } continue; @@ -3962,6 +3964,7 @@ public: G1ParScanExtRootClosure only_scan_root_cl(_g1h, &pss); G1ParScanPermClosure only_scan_perm_cl(_g1h, &pss); G1ParScanHeapRSClosure only_scan_heap_rs_cl(_g1h, &pss); + G1ParPushHeapRSClosure push_heap_rs_cl(_g1h, &pss); G1ParScanAndMarkExtRootClosure scan_mark_root_cl(_g1h, &pss); G1ParScanAndMarkPermClosure scan_mark_perm_cl(_g1h, &pss); @@ -3985,7 +3988,7 @@ public: _g1h->g1_process_strong_roots(/* not collecting perm */ false, SharedHeap::SO_AllClasses, scan_root_cl, - &only_scan_heap_rs_cl, + &push_heap_rs_cl, scan_so_cl, scan_perm_cl, i); diff --git a/hotspot/src/share/vm/gc_implementation/g1/g1CollectedHeap.hpp b/hotspot/src/share/vm/gc_implementation/g1/g1CollectedHeap.hpp index 51325935385..692b3e7f32a 100644 --- a/hotspot/src/share/vm/gc_implementation/g1/g1CollectedHeap.hpp +++ b/hotspot/src/share/vm/gc_implementation/g1/g1CollectedHeap.hpp @@ -1623,7 +1623,7 @@ public: template void push_on_queue(T* ref) { assert(ref != NULL, "invariant"); assert(has_partial_array_mask(ref) || - _g1h->obj_in_cs(oopDesc::load_decode_heap_oop(ref)), "invariant"); + _g1h->is_in_g1_reserved(oopDesc::load_decode_heap_oop(ref)), "invariant"); #ifdef ASSERT if (has_partial_array_mask(ref)) { oop p = clear_partial_array_mask(ref); @@ -1644,9 +1644,9 @@ public: assert((oop*)ref != NULL, "pop_local() returned true"); assert(UseCompressedOops || !ref.is_narrow(), "Error"); assert(has_partial_array_mask((oop*)ref) || - _g1h->obj_in_cs(ref.is_narrow() ? oopDesc::load_decode_heap_oop((narrowOop*)ref) - : oopDesc::load_decode_heap_oop((oop*)ref)), - "invariant"); + _g1h->is_in_g1_reserved(ref.is_narrow() ? oopDesc::load_decode_heap_oop((narrowOop*)ref) + : oopDesc::load_decode_heap_oop((oop*)ref)), + "invariant"); IF_G1_DETAILED_STATS(note_pop()); } else { StarTask null_task; @@ -1659,9 +1659,9 @@ public: assert((oop*)new_ref != NULL, "pop() from a local non-empty stack"); assert(UseCompressedOops || !new_ref.is_narrow(), "Error"); assert(has_partial_array_mask((oop*)new_ref) || - _g1h->obj_in_cs(new_ref.is_narrow() ? oopDesc::load_decode_heap_oop((narrowOop*)new_ref) - : oopDesc::load_decode_heap_oop((oop*)new_ref)), - "invariant"); + _g1h->is_in_g1_reserved(new_ref.is_narrow() ? oopDesc::load_decode_heap_oop((narrowOop*)new_ref) + : oopDesc::load_decode_heap_oop((oop*)new_ref)), + "invariant"); ref = new_ref; } @@ -1825,12 +1825,12 @@ public: assert(UseCompressedOops, "Error"); narrowOop* p = (narrowOop*)ref_to_scan; assert(!has_partial_array_mask(p) && - _g1h->obj_in_cs(oopDesc::load_decode_heap_oop(p)), "sanity"); + _g1h->is_in_g1_reserved(oopDesc::load_decode_heap_oop(p)), "sanity"); deal_with_reference(p); } else { oop* p = (oop*)ref_to_scan; - assert((has_partial_array_mask(p) && _g1h->obj_in_cs(clear_partial_array_mask(p))) || - _g1h->obj_in_cs(oopDesc::load_decode_heap_oop(p)), "sanity"); + assert((has_partial_array_mask(p) && _g1h->is_in_g1_reserved(clear_partial_array_mask(p))) || + _g1h->is_in_g1_reserved(oopDesc::load_decode_heap_oop(p)), "sanity"); deal_with_reference(p); } } @@ -1844,12 +1844,12 @@ public: assert(UseCompressedOops, "Error"); narrowOop* p = (narrowOop*)ref_to_scan; assert(!has_partial_array_mask(p) && - _g1h->obj_in_cs(oopDesc::load_decode_heap_oop(p)), "sanity"); + _g1h->is_in_g1_reserved(oopDesc::load_decode_heap_oop(p)), "sanity"); deal_with_reference(p); } else { oop* p = (oop*)ref_to_scan; assert((has_partial_array_mask(p) && _g1h->obj_in_cs(clear_partial_array_mask(p))) || - _g1h->obj_in_cs(oopDesc::load_decode_heap_oop(p)), "sanity"); + _g1h->is_in_g1_reserved(oopDesc::load_decode_heap_oop(p)), "sanity"); deal_with_reference(p); } } diff --git a/hotspot/src/share/vm/gc_implementation/g1/g1CollectorPolicy.cpp b/hotspot/src/share/vm/gc_implementation/g1/g1CollectorPolicy.cpp index 487fb6d954a..7702c788f21 100644 --- a/hotspot/src/share/vm/gc_implementation/g1/g1CollectorPolicy.cpp +++ b/hotspot/src/share/vm/gc_implementation/g1/g1CollectorPolicy.cpp @@ -205,6 +205,7 @@ G1CollectorPolicy::G1CollectorPolicy() : // policy is created before the heap, we have to set this up here, // so it's done as soon as possible. HeapRegion::setup_heap_region_size(Arguments::min_heap_size()); + HeapRegionRemSet::setup_remset_size(); _recent_prev_end_times_for_all_gcs_sec->add(os::elapsedTime()); _prev_collection_pause_end_ms = os::elapsedTime() * 1000.0; diff --git a/hotspot/src/share/vm/gc_implementation/g1/g1OopClosures.hpp b/hotspot/src/share/vm/gc_implementation/g1/g1OopClosures.hpp index f710203a1e9..fa143ba9bb3 100644 --- a/hotspot/src/share/vm/gc_implementation/g1/g1OopClosures.hpp +++ b/hotspot/src/share/vm/gc_implementation/g1/g1OopClosures.hpp @@ -53,6 +53,15 @@ public: bool apply_to_weak_ref_discovered_field() { return true; } }; +class G1ParPushHeapRSClosure : public G1ParClosureSuper { +public: + G1ParPushHeapRSClosure(G1CollectedHeap* g1, G1ParScanThreadState* par_scan_state) : + G1ParClosureSuper(g1, par_scan_state) { } + template void do_oop_nv(T* p); + virtual void do_oop(oop* p) { do_oop_nv(p); } + virtual void do_oop(narrowOop* p) { do_oop_nv(p); } +}; + class G1ParScanClosure : public G1ParClosureSuper { public: G1ParScanClosure(G1CollectedHeap* g1, G1ParScanThreadState* par_scan_state) : @@ -100,7 +109,7 @@ public: }; template + bool do_mark_forwardee> class G1ParCopyClosure : public G1ParCopyHelper { G1ParScanClosure _scanner; template void do_oop_work(T* p); @@ -116,12 +125,13 @@ public: virtual void do_oop(narrowOop* p) { do_oop_nv(p); } }; -typedef G1ParCopyClosure G1ParScanExtRootClosure; -typedef G1ParCopyClosure G1ParScanPermClosure; -typedef G1ParCopyClosure G1ParScanHeapRSClosure; -typedef G1ParCopyClosure G1ParScanAndMarkExtRootClosure; -typedef G1ParCopyClosure G1ParScanAndMarkPermClosure; -typedef G1ParCopyClosure G1ParScanAndMarkHeapRSClosure; +typedef G1ParCopyClosure G1ParScanExtRootClosure; +typedef G1ParCopyClosure G1ParScanPermClosure; +typedef G1ParCopyClosure G1ParScanHeapRSClosure; +typedef G1ParCopyClosure G1ParScanAndMarkExtRootClosure; +typedef G1ParCopyClosure G1ParScanAndMarkPermClosure; +typedef G1ParCopyClosure G1ParScanAndMarkHeapRSClosure; + // This is the only case when we set skip_cset_test. Basically, this // closure is (should?) only be called directly while we're draining // the overflow and task queues. In that case we know that the @@ -132,7 +142,7 @@ typedef G1ParCopyClosure G1ParScanAndMarkHea // We need a separate closure to handle references during evacuation // failure processing, as we cannot asume that the reference already // points into the collection set (like G1ParScanHeapEvacClosure does). -typedef G1ParCopyClosure G1ParScanHeapEvacFailureClosure; +typedef G1ParCopyClosure G1ParScanHeapEvacFailureClosure; class FilterIntoCSClosure: public OopClosure { G1CollectedHeap* _g1; diff --git a/hotspot/src/share/vm/gc_implementation/g1/g1OopClosures.inline.hpp b/hotspot/src/share/vm/gc_implementation/g1/g1OopClosures.inline.hpp index 2ba92a7f8d3..32697da3648 100644 --- a/hotspot/src/share/vm/gc_implementation/g1/g1OopClosures.inline.hpp +++ b/hotspot/src/share/vm/gc_implementation/g1/g1OopClosures.inline.hpp @@ -104,3 +104,16 @@ template inline void G1ParScanClosure::do_oop_nv(T* p) { } } } + +template inline void G1ParPushHeapRSClosure::do_oop_nv(T* p) { + T heap_oop = oopDesc::load_heap_oop(p); + + if (!oopDesc::is_null(heap_oop)) { + oop obj = oopDesc::decode_heap_oop_not_null(heap_oop); + if (_g1->in_cset_fast_test(obj)) { + Prefetch::write(obj->mark_addr(), 0); + Prefetch::read(obj->mark_addr(), (HeapWordSize*2)); + _par_scan_state->push_on_queue(p); + } + } +} diff --git a/hotspot/src/share/vm/gc_implementation/g1/g1RemSet.cpp b/hotspot/src/share/vm/gc_implementation/g1/g1RemSet.cpp index 224263e54ea..fa8916e52c5 100644 --- a/hotspot/src/share/vm/gc_implementation/g1/g1RemSet.cpp +++ b/hotspot/src/share/vm/gc_implementation/g1/g1RemSet.cpp @@ -155,8 +155,8 @@ class ScanRSClosure : public HeapRegionClosure { G1BlockOffsetSharedArray* _bot_shared; CardTableModRefBS *_ct_bs; int _worker_i; + int _block_size; bool _try_claimed; - size_t _min_skip_distance, _max_skip_distance; public: ScanRSClosure(OopsInHeapRegionClosure* oc, int worker_i) : _oc(oc), @@ -168,8 +168,7 @@ public: _g1h = G1CollectedHeap::heap(); _bot_shared = _g1h->bot_shared(); _ct_bs = (CardTableModRefBS*) (_g1h->barrier_set()); - _min_skip_distance = 16; - _max_skip_distance = 2 * _g1h->n_par_threads() * _min_skip_distance; + _block_size = MAX2(G1RSetScanBlockSize, 1); } void set_try_claimed() { _try_claimed = true; } @@ -225,12 +224,15 @@ public: HeapRegionRemSetIterator* iter = _g1h->rem_set_iterator(_worker_i); hrrs->init_iterator(iter); size_t card_index; - size_t skip_distance = 0, current_card = 0, jump_to_card = 0; - while (iter->has_next(card_index)) { - if (current_card < jump_to_card) { - ++current_card; - continue; + + // We claim cards in block so as to recude the contention. The block size is determined by + // the G1RSetScanBlockSize parameter. + size_t jump_to_card = hrrs->iter_claimed_next(_block_size); + for (size_t current_card = 0; iter->has_next(card_index); current_card++) { + if (current_card >= jump_to_card + _block_size) { + jump_to_card = hrrs->iter_claimed_next(_block_size); } + if (current_card < jump_to_card) continue; HeapWord* card_start = _g1h->bot_shared()->address_for_index(card_index); #if 0 gclog_or_tty->print("Rem set iteration yielded card [" PTR_FORMAT ", " PTR_FORMAT ").\n", @@ -247,22 +249,14 @@ public: // If the card is dirty, then we will scan it during updateRS. if (!card_region->in_collection_set() && !_ct_bs->is_card_dirty(card_index)) { - if (!_ct_bs->is_card_claimed(card_index) && _ct_bs->claim_card(card_index)) { - scanCard(card_index, card_region); - } else if (_try_claimed) { - if (jump_to_card == 0 || jump_to_card != current_card) { - // We did some useful work in the previous iteration. - // Decrease the distance. - skip_distance = MAX2(skip_distance >> 1, _min_skip_distance); - } else { - // Previous iteration resulted in a claim failure. - // Increase the distance. - skip_distance = MIN2(skip_distance << 1, _max_skip_distance); - } - jump_to_card = current_card + skip_distance; - } + // We make the card as "claimed" lazily (so races are possible but they're benign), + // which reduces the number of duplicate scans (the rsets of the regions in the cset + // can intersect). + if (!_ct_bs->is_card_claimed(card_index)) { + _ct_bs->set_card_claimed(card_index); + scanCard(card_index, card_region); + } } - ++current_card; } if (!_try_claimed) { hrrs->set_iter_complete(); @@ -299,30 +293,18 @@ void HRInto_G1RemSet::scanRS(OopsInHeapRegionClosure* oc, int worker_i) { double rs_time_start = os::elapsedTime(); HeapRegion *startRegion = calculateStartRegion(worker_i); - BufferingOopsInHeapRegionClosure boc(oc); - ScanRSClosure scanRScl(&boc, worker_i); + ScanRSClosure scanRScl(oc, worker_i); _g1->collection_set_iterate_from(startRegion, &scanRScl); scanRScl.set_try_claimed(); _g1->collection_set_iterate_from(startRegion, &scanRScl); - boc.done(); - double closure_app_time_sec = boc.closure_app_seconds(); - double scan_rs_time_sec = (os::elapsedTime() - rs_time_start) - - closure_app_time_sec; - double closure_app_time_ms = closure_app_time_sec * 1000.0; + double scan_rs_time_sec = os::elapsedTime() - rs_time_start; assert( _cards_scanned != NULL, "invariant" ); _cards_scanned[worker_i] = scanRScl.cards_done(); _g1p->record_scan_rs_start_time(worker_i, rs_time_start * 1000.0); _g1p->record_scan_rs_time(worker_i, scan_rs_time_sec * 1000.0); - - double scan_new_refs_time_ms = _g1p->get_scan_new_refs_time(worker_i); - if (scan_new_refs_time_ms > 0.0) { - closure_app_time_ms += scan_new_refs_time_ms; - } - - _g1p->record_obj_copy_time(worker_i, closure_app_time_ms); } void HRInto_G1RemSet::updateRS(int worker_i) { @@ -449,9 +431,8 @@ HRInto_G1RemSet::scanNewRefsRS_work(OopsInHeapRegionClosure* oc, oc->do_oop(p); } } - _g1p->record_scan_new_refs_time(worker_i, - (os::elapsedTime() - scan_new_refs_start_sec) - * 1000.0); + double scan_new_refs_time_ms = (os::elapsedTime() - scan_new_refs_start_sec) * 1000.0; + _g1p->record_scan_new_refs_time(worker_i, scan_new_refs_time_ms); } void HRInto_G1RemSet::cleanupHRRS() { diff --git a/hotspot/src/share/vm/gc_implementation/g1/g1_globals.hpp b/hotspot/src/share/vm/gc_implementation/g1/g1_globals.hpp index 4e6ce1361bd..7e084731a2d 100644 --- a/hotspot/src/share/vm/gc_implementation/g1/g1_globals.hpp +++ b/hotspot/src/share/vm/gc_implementation/g1/g1_globals.hpp @@ -207,8 +207,20 @@ develop(bool, G1PrintOopAppls, false, \ "When true, print applications of closures to external locs.") \ \ - develop(intx, G1LogRSRegionEntries, 7, \ - "Log_2 of max number of regions for which we keep bitmaps.") \ + develop(intx, G1RSetRegionEntriesBase, 256, \ + "Max number of regions in a fine-grain table per MB.") \ + \ + product(intx, G1RSetRegionEntries, 0, \ + "Max number of regions for which we keep bitmaps." \ + "Will be set ergonomically by default") \ + \ + develop(intx, G1RSetSparseRegionEntriesBase, 4, \ + "Max number of entries per region in a sparse table " \ + "per MB.") \ + \ + product(intx, G1RSetSparseRegionEntries, 0, \ + "Max number of entries per region in a sparse table." \ + "Will be set ergonomically by default.") \ \ develop(bool, G1RecordHRRSOops, false, \ "When true, record recent calls to rem set operations.") \ @@ -293,6 +305,10 @@ develop(bool, G1VerifyCTCleanup, false, \ "Verify card table cleanup.") \ \ + product(uintx, G1RSetScanBlockSize, 64, \ + "Size of a work unit of cards claimed by a worker thread" \ + "during RSet scanning.") \ + \ develop(bool, ReduceInitialCardMarksForG1, false, \ "When ReduceInitialCardMarks is true, this flag setting " \ " controls whether G1 allows the RICM optimization") diff --git a/hotspot/src/share/vm/gc_implementation/g1/g1_specialized_oop_closures.hpp b/hotspot/src/share/vm/gc_implementation/g1/g1_specialized_oop_closures.hpp index 101d330077e..266bbb5abdc 100644 --- a/hotspot/src/share/vm/gc_implementation/g1/g1_specialized_oop_closures.hpp +++ b/hotspot/src/share/vm/gc_implementation/g1/g1_specialized_oop_closures.hpp @@ -33,11 +33,12 @@ enum G1Barrier { }; template + bool do_mark_forwardee> class G1ParCopyClosure; class G1ParScanClosure; +class G1ParPushHeapRSClosure; -typedef G1ParCopyClosure G1ParScanHeapEvacClosure; +typedef G1ParCopyClosure G1ParScanHeapEvacClosure; class FilterIntoCSClosure; class FilterOutOfRegionClosure; @@ -51,6 +52,7 @@ class FilterAndMarkInHeapRegionAndIntoCSClosure; #define FURTHER_SPECIALIZED_OOP_OOP_ITERATE_CLOSURES(f) \ f(G1ParScanHeapEvacClosure,_nv) \ f(G1ParScanClosure,_nv) \ + f(G1ParPushHeapRSClosure,_nv) \ f(FilterIntoCSClosure,_nv) \ f(FilterOutOfRegionClosure,_nv) \ f(FilterInHeapRegionAndIntoCSClosure,_nv) \ diff --git a/hotspot/src/share/vm/gc_implementation/g1/heapRegionRemSet.cpp b/hotspot/src/share/vm/gc_implementation/g1/heapRegionRemSet.cpp index aac289e67d6..883b2b1ef26 100644 --- a/hotspot/src/share/vm/gc_implementation/g1/heapRegionRemSet.cpp +++ b/hotspot/src/share/vm/gc_implementation/g1/heapRegionRemSet.cpp @@ -505,12 +505,13 @@ OtherRegionsTable::OtherRegionsTable(HeapRegion* hr) : typedef PosParPRT* PosParPRTPtr; if (_max_fine_entries == 0) { assert(_mod_max_fine_entries_mask == 0, "Both or none."); - _max_fine_entries = (size_t)(1 << G1LogRSRegionEntries); + size_t max_entries_log = (size_t)log2_long((jlong)G1RSetRegionEntries); + _max_fine_entries = (size_t)(1 << max_entries_log); _mod_max_fine_entries_mask = _max_fine_entries - 1; #if SAMPLE_FOR_EVICTION assert(_fine_eviction_sample_size == 0 && _fine_eviction_stride == 0, "All init at same time."); - _fine_eviction_sample_size = MAX2((size_t)4, (size_t)G1LogRSRegionEntries); + _fine_eviction_sample_size = MAX2((size_t)4, max_entries_log); _fine_eviction_stride = _max_fine_entries / _fine_eviction_sample_size; #endif } @@ -655,13 +656,6 @@ void OtherRegionsTable::add_reference(OopOrNarrowOopStar from, int tid) { #endif } - // Otherwise, transfer from sparse to fine-grain. - CardIdx_t cards[SparsePRTEntry::CardsPerEntry]; - if (G1HRRSUseSparseTable) { - bool res = _sparse_table.get_cards(from_hrs_ind, &cards[0]); - assert(res, "There should have been an entry"); - } - if (_n_fine_entries == _max_fine_entries) { prt = delete_region_table(); } else { @@ -676,10 +670,12 @@ void OtherRegionsTable::add_reference(OopOrNarrowOopStar from, int tid) { _fine_grain_regions[ind] = prt; _n_fine_entries++; - // Add in the cards from the sparse table. if (G1HRRSUseSparseTable) { - for (int i = 0; i < SparsePRTEntry::CardsPerEntry; i++) { - CardIdx_t c = cards[i]; + // Transfer from sparse to fine-grain. + SparsePRTEntry *sprt_entry = _sparse_table.get_entry(from_hrs_ind); + assert(sprt_entry != NULL, "There should have been an entry"); + for (int i = 0; i < SparsePRTEntry::cards_num(); i++) { + CardIdx_t c = sprt_entry->card(i); if (c != SparsePRTEntry::NullEntry) { prt->add_card(c); } @@ -1084,6 +1080,19 @@ HeapRegionRemSet::HeapRegionRemSet(G1BlockOffsetSharedArray* bosa, {} +void HeapRegionRemSet::setup_remset_size() { + // Setup sparse and fine-grain tables sizes. + // table_size = base * (log(region_size / 1M) + 1) + int region_size_log_mb = MAX2((int)HeapRegion::LogOfHRGrainBytes - (int)LOG_M, 0); + if (FLAG_IS_DEFAULT(G1RSetSparseRegionEntries)) { + G1RSetSparseRegionEntries = G1RSetSparseRegionEntriesBase * (region_size_log_mb + 1); + } + if (FLAG_IS_DEFAULT(G1RSetRegionEntries)) { + G1RSetRegionEntries = G1RSetRegionEntriesBase * (region_size_log_mb + 1); + } + guarantee(G1RSetSparseRegionEntries > 0 && G1RSetRegionEntries > 0 , "Sanity"); +} + void HeapRegionRemSet::init_for_par_iteration() { _iter_state = Unclaimed; } @@ -1399,7 +1408,7 @@ void HeapRegionRemSet::test() { os::sleep(Thread::current(), (jlong)5000, false); G1CollectedHeap* g1h = G1CollectedHeap::heap(); - // Run with "-XX:G1LogRSRegionEntries=2", so that 1 and 5 end up in same + // Run with "-XX:G1LogRSetRegionEntries=2", so that 1 and 5 end up in same // hash bucket. HeapRegion* hr0 = g1h->region_at(0); HeapRegion* hr1 = g1h->region_at(1); diff --git a/hotspot/src/share/vm/gc_implementation/g1/heapRegionRemSet.hpp b/hotspot/src/share/vm/gc_implementation/g1/heapRegionRemSet.hpp index d525b5ad306..c51286290af 100644 --- a/hotspot/src/share/vm/gc_implementation/g1/heapRegionRemSet.hpp +++ b/hotspot/src/share/vm/gc_implementation/g1/heapRegionRemSet.hpp @@ -187,7 +187,8 @@ private: void clear_outgoing_entries(); enum ParIterState { Unclaimed, Claimed, Complete }; - ParIterState _iter_state; + volatile ParIterState _iter_state; + volatile jlong _iter_claimed; // Unused unless G1RecordHRRSOops is true. @@ -209,6 +210,7 @@ public: HeapRegion* hr); static int num_par_rem_sets(); + static void setup_remset_size(); HeapRegion* hr() const { return _other_regions.hr(); @@ -272,6 +274,19 @@ public: // Returns "true" iff the region's iteration is complete. bool iter_is_complete(); + // Support for claiming blocks of cards during iteration + void set_iter_claimed(size_t x) { _iter_claimed = (jlong)x; } + size_t iter_claimed() const { return (size_t)_iter_claimed; } + // Claim the next block of cards + size_t iter_claimed_next(size_t step) { + size_t current, next; + do { + current = iter_claimed(); + next = current + step; + } while (Atomic::cmpxchg((jlong)next, &_iter_claimed, (jlong)current) != (jlong)current); + return current; + } + // Initialize the given iterator to iterate over this rem set. void init_iterator(HeapRegionRemSetIterator* iter) const; diff --git a/hotspot/src/share/vm/gc_implementation/g1/sparsePRT.cpp b/hotspot/src/share/vm/gc_implementation/g1/sparsePRT.cpp index cef8297a367..c6b835b3513 100644 --- a/hotspot/src/share/vm/gc_implementation/g1/sparsePRT.cpp +++ b/hotspot/src/share/vm/gc_implementation/g1/sparsePRT.cpp @@ -27,7 +27,7 @@ #define SPARSE_PRT_VERBOSE 0 -#define UNROLL_CARD_LOOPS 1 +#define UNROLL_CARD_LOOPS 1 void SparsePRT::init_iterator(SparsePRTIter* sprt_iter) { sprt_iter->init(this); @@ -36,27 +36,32 @@ void SparsePRT::init_iterator(SparsePRTIter* sprt_iter) { void SparsePRTEntry::init(RegionIdx_t region_ind) { _region_ind = region_ind; _next_index = NullEntry; + #if UNROLL_CARD_LOOPS - assert(CardsPerEntry == 4, "Assumption. If changes, un-unroll."); - _cards[0] = NullEntry; - _cards[1] = NullEntry; - _cards[2] = NullEntry; - _cards[3] = NullEntry; + assert((cards_num() & (UnrollFactor - 1)) == 0, "Invalid number of cards in the entry"); + for (int i = 0; i < cards_num(); i += UnrollFactor) { + _cards[i] = NullEntry; + _cards[i + 1] = NullEntry; + _cards[i + 2] = NullEntry; + _cards[i + 3] = NullEntry; + } #else - for (int i = 0; i < CardsPerEntry; i++) + for (int i = 0; i < cards_num(); i++) _cards[i] = NullEntry; #endif } bool SparsePRTEntry::contains_card(CardIdx_t card_index) const { #if UNROLL_CARD_LOOPS - assert(CardsPerEntry == 4, "Assumption. If changes, un-unroll."); - if (_cards[0] == card_index) return true; - if (_cards[1] == card_index) return true; - if (_cards[2] == card_index) return true; - if (_cards[3] == card_index) return true; + assert((cards_num() & (UnrollFactor - 1)) == 0, "Invalid number of cards in the entry"); + for (int i = 0; i < cards_num(); i += UnrollFactor) { + if (_cards[i] == card_index || + _cards[i + 1] == card_index || + _cards[i + 2] == card_index || + _cards[i + 3] == card_index) return true; + } #else - for (int i = 0; i < CardsPerEntry; i++) { + for (int i = 0; i < cards_num(); i++) { if (_cards[i] == card_index) return true; } #endif @@ -67,14 +72,16 @@ bool SparsePRTEntry::contains_card(CardIdx_t card_index) const { int SparsePRTEntry::num_valid_cards() const { int sum = 0; #if UNROLL_CARD_LOOPS - assert(CardsPerEntry == 4, "Assumption. If changes, un-unroll."); - if (_cards[0] != NullEntry) sum++; - if (_cards[1] != NullEntry) sum++; - if (_cards[2] != NullEntry) sum++; - if (_cards[3] != NullEntry) sum++; + assert((cards_num() & (UnrollFactor - 1)) == 0, "Invalid number of cards in the entry"); + for (int i = 0; i < cards_num(); i += UnrollFactor) { + sum += (_cards[i] != NullEntry); + sum += (_cards[i + 1] != NullEntry); + sum += (_cards[i + 2] != NullEntry); + sum += (_cards[i + 3] != NullEntry); + } #else - for (int i = 0; i < CardsPerEntry; i++) { - if (_cards[i] != NulLEntry) sum++; + for (int i = 0; i < cards_num(); i++) { + sum += (_cards[i] != NullEntry); } #endif // Otherwise, we're full. @@ -83,27 +90,27 @@ int SparsePRTEntry::num_valid_cards() const { SparsePRTEntry::AddCardResult SparsePRTEntry::add_card(CardIdx_t card_index) { #if UNROLL_CARD_LOOPS - assert(CardsPerEntry == 4, "Assumption. If changes, un-unroll."); - CardIdx_t c = _cards[0]; - if (c == card_index) return found; - if (c == NullEntry) { _cards[0] = card_index; return added; } - c = _cards[1]; - if (c == card_index) return found; - if (c == NullEntry) { _cards[1] = card_index; return added; } - c = _cards[2]; - if (c == card_index) return found; - if (c == NullEntry) { _cards[2] = card_index; return added; } - c = _cards[3]; - if (c == card_index) return found; - if (c == NullEntry) { _cards[3] = card_index; return added; } + assert((cards_num() & (UnrollFactor - 1)) == 0, "Invalid number of cards in the entry"); + CardIdx_t c; + for (int i = 0; i < cards_num(); i += UnrollFactor) { + c = _cards[i]; + if (c == card_index) return found; + if (c == NullEntry) { _cards[i] = card_index; return added; } + c = _cards[i + 1]; + if (c == card_index) return found; + if (c == NullEntry) { _cards[i + 1] = card_index; return added; } + c = _cards[i + 2]; + if (c == card_index) return found; + if (c == NullEntry) { _cards[i + 2] = card_index; return added; } + c = _cards[i + 3]; + if (c == card_index) return found; + if (c == NullEntry) { _cards[i + 3] = card_index; return added; } + } #else - for (int i = 0; i < CardsPerEntry; i++) { + for (int i = 0; i < cards_num(); i++) { CardIdx_t c = _cards[i]; if (c == card_index) return found; - if (c == NullEntry) { - _cards[i] = card_index; - return added; - } + if (c == NullEntry) { _cards[i] = card_index; return added; } } #endif // Otherwise, we're full. @@ -112,13 +119,15 @@ SparsePRTEntry::AddCardResult SparsePRTEntry::add_card(CardIdx_t card_index) { void SparsePRTEntry::copy_cards(CardIdx_t* cards) const { #if UNROLL_CARD_LOOPS - assert(CardsPerEntry == 4, "Assumption. If changes, un-unroll."); - cards[0] = _cards[0]; - cards[1] = _cards[1]; - cards[2] = _cards[2]; - cards[3] = _cards[3]; + assert((cards_num() & (UnrollFactor - 1)) == 0, "Invalid number of cards in the entry"); + for (int i = 0; i < cards_num(); i += UnrollFactor) { + cards[i] = _cards[i]; + cards[i + 1] = _cards[i + 1]; + cards[i + 2] = _cards[i + 2]; + cards[i + 3] = _cards[i + 3]; + } #else - for (int i = 0; i < CardsPerEntry; i++) { + for (int i = 0; i < cards_num(); i++) { cards[i] = _cards[i]; } #endif @@ -133,7 +142,7 @@ void SparsePRTEntry::copy_cards(SparsePRTEntry* e) const { RSHashTable::RSHashTable(size_t capacity) : _capacity(capacity), _capacity_mask(capacity-1), _occupied_entries(0), _occupied_cards(0), - _entries(NEW_C_HEAP_ARRAY(SparsePRTEntry, capacity)), + _entries((SparsePRTEntry*)NEW_C_HEAP_ARRAY(char, SparsePRTEntry::size() * capacity)), _buckets(NEW_C_HEAP_ARRAY(int, capacity)), _free_list(NullEntry), _free_region(0) { @@ -161,8 +170,8 @@ void RSHashTable::clear() { "_capacity too large"); // This will put -1 == NullEntry in the key field of all entries. - memset(_entries, -1, _capacity * sizeof(SparsePRTEntry)); - memset(_buckets, -1, _capacity * sizeof(int)); + memset(_entries, NullEntry, _capacity * SparsePRTEntry::size()); + memset(_buckets, NullEntry, _capacity * sizeof(int)); _free_list = NullEntry; _free_region = 0; } @@ -175,8 +184,8 @@ bool RSHashTable::add_card(RegionIdx_t region_ind, CardIdx_t card_index) { if (res == SparsePRTEntry::added) _occupied_cards++; #if SPARSE_PRT_VERBOSE gclog_or_tty->print_cr(" after add_card[%d]: valid-cards = %d.", - pointer_delta(e, _entries, sizeof(SparsePRTEntry)), - e->num_valid_cards()); + pointer_delta(e, _entries, SparsePRTEntry::size()), + e->num_valid_cards()); #endif assert(e->num_valid_cards() > 0, "Postcondition"); return res != SparsePRTEntry::overflow; @@ -199,6 +208,22 @@ bool RSHashTable::get_cards(RegionIdx_t region_ind, CardIdx_t* cards) { return true; } +SparsePRTEntry* RSHashTable::get_entry(RegionIdx_t region_ind) { + int ind = (int) (region_ind & capacity_mask()); + int cur_ind = _buckets[ind]; + SparsePRTEntry* cur; + while (cur_ind != NullEntry && + (cur = entry(cur_ind))->r_ind() != region_ind) { + cur_ind = cur->next_index(); + } + + if (cur_ind == NullEntry) return NULL; + // Otherwise... + assert(cur->r_ind() == region_ind, "Postcondition of loop + test above."); + assert(cur->num_valid_cards() > 0, "Inv"); + return cur; +} + bool RSHashTable::delete_entry(RegionIdx_t region_ind) { int ind = (int) (region_ind & capacity_mask()); int* prev_loc = &_buckets[ind]; @@ -225,20 +250,8 @@ RSHashTable::entry_for_region_ind(RegionIdx_t region_ind) const { int ind = (int) (region_ind & capacity_mask()); int cur_ind = _buckets[ind]; SparsePRTEntry* cur; - // XXX - // int k = 0; while (cur_ind != NullEntry && (cur = entry(cur_ind))->r_ind() != region_ind) { - /* - k++; - if (k > 10) { - gclog_or_tty->print_cr("RSHashTable::entry_for_region_ind(%d): " - "k = %d, cur_ind = %d.", region_ind, k, cur_ind); - if (k >= 1000) { - while (1) ; - } - } - */ cur_ind = cur->next_index(); } @@ -319,7 +332,7 @@ size_t /* RSHashTable:: */ RSHashTableIter::compute_card_ind(CardIdx_t ci) { bool /* RSHashTable:: */ RSHashTableIter::has_next(size_t& card_index) { _card_ind++; CardIdx_t ci; - if (_card_ind < SparsePRTEntry::CardsPerEntry && + if (_card_ind < SparsePRTEntry::cards_num() && ((ci = _rsht->entry(_bl_ind)->card(_card_ind)) != SparsePRTEntry::NullEntry)) { card_index = compute_card_ind(ci); @@ -359,7 +372,7 @@ bool RSHashTable::contains_card(RegionIdx_t region_index, CardIdx_t card_index) size_t RSHashTable::mem_size() const { return sizeof(this) + - capacity() * (sizeof(SparsePRTEntry) + sizeof(int)); + capacity() * (SparsePRTEntry::size() + sizeof(int)); } // ---------------------------------------------------------------------- @@ -446,6 +459,10 @@ bool SparsePRT::get_cards(RegionIdx_t region_id, CardIdx_t* cards) { return _next->get_cards(region_id, cards); } +SparsePRTEntry* SparsePRT::get_entry(RegionIdx_t region_id) { + return _next->get_entry(region_id); +} + bool SparsePRT::delete_entry(RegionIdx_t region_id) { return _next->delete_entry(region_id); } diff --git a/hotspot/src/share/vm/gc_implementation/g1/sparsePRT.hpp b/hotspot/src/share/vm/gc_implementation/g1/sparsePRT.hpp index c5a39737c52..5c35c69b813 100644 --- a/hotspot/src/share/vm/gc_implementation/g1/sparsePRT.hpp +++ b/hotspot/src/share/vm/gc_implementation/g1/sparsePRT.hpp @@ -32,21 +32,28 @@ // insertions only enqueue old versions for deletions, but do not delete // old versions synchronously. - class SparsePRTEntry: public CHeapObj { public: - enum SomePublicConstants { - CardsPerEntry = 4, - NullEntry = -1 + NullEntry = -1, + UnrollFactor = 4 }; - private: RegionIdx_t _region_ind; int _next_index; - CardIdx_t _cards[CardsPerEntry]; - + CardIdx_t _cards[1]; + // WARNING: Don't put any data members beyond this line. Card array has, in fact, variable length. + // It should always be the last data member. public: + // Returns the size of the entry, used for entry allocation. + static size_t size() { return sizeof(SparsePRTEntry) + sizeof(CardIdx_t) * (cards_num() - 1); } + // Returns the size of the card array. + static int cards_num() { + // The number of cards should be a multiple of 4, because that's our current + // unrolling factor. + static const int s = MAX2(G1RSetSparseRegionEntries & ~(UnrollFactor - 1), UnrollFactor); + return s; + } // Set the region_ind to the given value, and delete all cards. inline void init(RegionIdx_t region_ind); @@ -134,12 +141,15 @@ public: bool add_card(RegionIdx_t region_id, CardIdx_t card_index); bool get_cards(RegionIdx_t region_id, CardIdx_t* cards); + bool delete_entry(RegionIdx_t region_id); bool contains_card(RegionIdx_t region_id, CardIdx_t card_index) const; void add_entry(SparsePRTEntry* e); + SparsePRTEntry* get_entry(RegionIdx_t region_id); + void clear(); size_t capacity() const { return _capacity; } @@ -148,7 +158,7 @@ public: size_t occupied_cards() const { return _occupied_cards; } size_t mem_size() const; - SparsePRTEntry* entry(int i) const { return &_entries[i]; } + SparsePRTEntry* entry(int i) const { return (SparsePRTEntry*)((char*)_entries + SparsePRTEntry::size() * i); } void print(); }; @@ -157,7 +167,7 @@ public: class RSHashTableIter VALUE_OBJ_CLASS_SPEC { int _tbl_ind; // [-1, 0.._rsht->_capacity) int _bl_ind; // [-1, 0.._rsht->_capacity) - short _card_ind; // [0..CardsPerEntry) + short _card_ind; // [0..SparsePRTEntry::cards_num()) RSHashTable* _rsht; size_t _heap_bot_card_ind; @@ -176,7 +186,7 @@ public: RSHashTableIter(size_t heap_bot_card_ind) : _tbl_ind(RSHashTable::NullEntry), _bl_ind(RSHashTable::NullEntry), - _card_ind((SparsePRTEntry::CardsPerEntry-1)), + _card_ind((SparsePRTEntry::cards_num() - 1)), _rsht(NULL), _heap_bot_card_ind(heap_bot_card_ind) {} @@ -185,7 +195,7 @@ public: _rsht = rsht; _tbl_ind = -1; // So that first increment gets to 0. _bl_ind = RSHashTable::NullEntry; - _card_ind = (SparsePRTEntry::CardsPerEntry-1); + _card_ind = (SparsePRTEntry::cards_num() - 1); } bool has_next(size_t& card_index); @@ -241,9 +251,13 @@ public: // If the table hold an entry for "region_ind", Copies its // cards into "cards", which must be an array of length at least - // "CardsPerEntry", and returns "true"; otherwise, returns "false". + // "SparePRTEntry::cards_num()", and returns "true"; otherwise, + // returns "false". bool get_cards(RegionIdx_t region_ind, CardIdx_t* cards); + // Return the pointer to the entry associated with the given region. + SparsePRTEntry* get_entry(RegionIdx_t region_ind); + // If there is an entry for "region_ind", removes it and return "true"; // otherwise returns "false." bool delete_entry(RegionIdx_t region_ind); diff --git a/hotspot/src/share/vm/memory/cardTableModRefBS.hpp b/hotspot/src/share/vm/memory/cardTableModRefBS.hpp index bffdd50926b..02fd91db16a 100644 --- a/hotspot/src/share/vm/memory/cardTableModRefBS.hpp +++ b/hotspot/src/share/vm/memory/cardTableModRefBS.hpp @@ -339,6 +339,16 @@ public: return (val & (clean_card_mask_val() | claimed_card_val())) == claimed_card_val(); } + void set_card_claimed(size_t card_index) { + jbyte val = _byte_map[card_index]; + if (val == clean_card_val()) { + val = (jbyte)claimed_card_val(); + } else { + val |= (jbyte)claimed_card_val(); + } + _byte_map[card_index] = val; + } + bool claim_card(size_t card_index); bool is_card_clean(size_t card_index) { diff --git a/hotspot/src/share/vm/utilities/globalDefinitions.hpp b/hotspot/src/share/vm/utilities/globalDefinitions.hpp index 32ecd64c880..a83f4ba23e9 100644 --- a/hotspot/src/share/vm/utilities/globalDefinitions.hpp +++ b/hotspot/src/share/vm/utilities/globalDefinitions.hpp @@ -139,6 +139,10 @@ const size_t M = K*K; const size_t G = M*K; const size_t HWperKB = K / sizeof(HeapWord); +const size_t LOG_K = 10; +const size_t LOG_M = 2 * LOG_K; +const size_t LOG_G = 2 * LOG_M; + const jint min_jint = (jint)1 << (sizeof(jint)*BitsPerByte-1); // 0x80000000 == smallest jint const jint max_jint = (juint)min_jint - 1; // 0x7FFFFFFF == largest jint