stan/jdk-24
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

8324649: Shenandoah: replace implementation of free set

Browse Source 
Reviewed-by: wkemper, ysr, rkennke
This commit is contained in:
Kelvin Nilsen 2024-05-16 16:47:09 +00:00 committed by Y. Srinivas Ramakrishna
parent 259915168d
commit dc184f1099
8 changed files with 2392 additions and 396 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1398
src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp
View File

File diff suppressed because it is too large Load Diff

361
src/hotspot/share/gc/shenandoah/shenandoahFreeSet.hpp
View File

@ -1,5 +1,6 @@
/*
* Copyright (c) 2016, 2019, Red Hat, Inc. All rights reserved.
* Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -27,70 +28,366 @@
#include "gc/shenandoah/shenandoahHeapRegionSet.hpp"
#include "gc/shenandoah/shenandoahHeap.hpp"
#include "gc/shenandoah/shenandoahSimpleBitMap.hpp"
// Each ShenandoahHeapRegion is associated with a ShenandoahFreeSetPartitionId.
enum class ShenandoahFreeSetPartitionId : uint8_t {
Mutator, // Region is in the Mutator free set: available memory is available to mutators.
Collector, // Region is in the Collector free set: available memory is reserved for evacuations.
NotFree // Region is in no free set: it has no available memory
};
// We do not maintain counts, capacity, or used for regions that are not free. Informally, if a region is NotFree, it is
// in no partition. NumPartitions represents the size of an array that may be indexed by Mutator or Collector.
#define NumPartitions (ShenandoahFreeSetPartitionId::NotFree)
#define IntNumPartitions int(ShenandoahFreeSetPartitionId::NotFree)
#define UIntNumPartitions uint(ShenandoahFreeSetPartitionId::NotFree)
// ShenandoahRegionPartitions provides an abstraction to help organize the implementation of ShenandoahFreeSet. This
// class implements partitioning of regions into distinct sets. Each ShenandoahHeapRegion is either in the Mutator free set,
// the Collector free set, or in neither free set (NotFree). When we speak of a "free partition", we mean partitions that
// for which the ShenandoahFreeSetPartitionId is not equal to NotFree.
class ShenandoahRegionPartitions {
private:
const ssize_t _max; // The maximum number of heap regions
const size_t _region_size_bytes;
const ShenandoahFreeSet* _free_set;
// For each partition, we maintain a bitmap of which regions are affiliated with his partition.
ShenandoahSimpleBitMap _membership[UIntNumPartitions];
// For each partition, we track an interval outside of which a region affiliated with that partition is guaranteed
// not to be found. This makes searches for free space more efficient. For each partition p, _leftmosts[p]
// represents its least index, and its _rightmosts[p] its greatest index. Empty intervals are indicated by the
// canonical [_max, -1].
ssize_t _leftmosts[UIntNumPartitions];
ssize_t _rightmosts[UIntNumPartitions];
// Allocation for humongous objects needs to find regions that are entirely empty. For each partion p, _leftmosts_empty[p]
// represents the first region belonging to this partition that is completely empty and _rightmosts_empty[p] represents the
// last region that is completely empty. If there is no completely empty region in this partition, this is represented
// by the canonical [_max, -1].
ssize_t _leftmosts_empty[UIntNumPartitions];
ssize_t _rightmosts_empty[UIntNumPartitions];
// For each partition p, _capacity[p] represents the total amount of memory within the partition at the time
// of the most recent rebuild, _used[p] represents the total amount of memory that has been allocated within this
// partition (either already allocated as of the rebuild, or allocated since the rebuild). _capacity[p] and _used[p]
// are denoted in bytes. Note that some regions that had been assigned to a particular partition at rebuild time
// may have been retired following the rebuild. The tallies for these regions are still reflected in _capacity[p]
// and _used[p], even though the region may have been removed from the free set.
size_t _capacity[UIntNumPartitions];
size_t _used[UIntNumPartitions];
size_t _region_counts[UIntNumPartitions];
// Shrink the intervals associated with partition when region idx is removed from this free set
inline void shrink_interval_if_boundary_modified(ShenandoahFreeSetPartitionId partition, ssize_t idx);
// Shrink the intervals associated with partition when regions low_idx through high_idx inclusive are removed from this free set
inline void shrink_interval_if_range_modifies_either_boundary(ShenandoahFreeSetPartitionId partition,
ssize_t low_idx, ssize_t high_idx);
inline void expand_interval_if_boundary_modified(ShenandoahFreeSetPartitionId partition, ssize_t idx, size_t capacity);
#ifndef PRODUCT
void dump_bitmap_row(ssize_t region_idx) const;
void dump_bitmap_range(ssize_t start_region_idx, ssize_t end_region_idx) const;
void dump_bitmap() const;
#endif
public:
ShenandoahRegionPartitions(size_t max_regions, ShenandoahFreeSet* free_set);
~ShenandoahRegionPartitions() {}
// Remove all regions from all partitions and reset all bounds
void make_all_regions_unavailable();
// Set the partition id for a particular region without adjusting interval bounds or usage/capacity tallies
inline void raw_assign_membership(size_t idx, ShenandoahFreeSetPartitionId p) {
_membership[int(p)].set_bit(idx);
}
// Set the Mutator intervals, usage, and capacity according to arguments. Reset the Collector intervals, used, capacity
// to represent empty Collector free set. We use this at the end of rebuild_free_set() to avoid the overhead of making
// many redundant incremental adjustments to the mutator intervals as the free set is being rebuilt.
void establish_mutator_intervals(ssize_t mutator_leftmost, ssize_t mutator_rightmost,
ssize_t mutator_leftmost_empty, ssize_t mutator_rightmost_empty,
size_t mutator_region_count, size_t mutator_used);
// Retire region idx from within partition, , leaving its capacity and used as part of the original free partition's totals.
// Requires that region idx is in in the Mutator or Collector partitions. Hereafter, identifies this region as NotFree.
// Any remnant of available memory at the time of retirement is added to the original partition's total of used bytes.
void retire_from_partition(ShenandoahFreeSetPartitionId p, ssize_t idx, size_t used_bytes);
// Retire all regions between low_idx and high_idx inclusive from within partition. Requires that each region idx is
// in the same Mutator or Collector partition. Hereafter, identifies each region as NotFree. Assumes that each region
// is now considered fully used, since the region is presumably used to represent a humongous object.
void retire_range_from_partition(ShenandoahFreeSetPartitionId partition, ssize_t low_idx, ssize_t high_idx);
// Place region idx into free set which_partition. Requires that idx is currently NotFree.
void make_free(ssize_t idx, ShenandoahFreeSetPartitionId which_partition, size_t region_capacity);
// Place region idx into free partition new_partition, adjusting used and capacity totals for the original and new partition
// given that available bytes can still be allocated within this region. Requires that idx is currently not NotFree.
void move_from_partition_to_partition(ssize_t idx, ShenandoahFreeSetPartitionId orig_partition,
ShenandoahFreeSetPartitionId new_partition, size_t available);
const char* partition_membership_name(ssize_t idx) const;
// Return the index of the next available region >= start_index, or maximum_regions if not found.
inline ssize_t find_index_of_next_available_region(ShenandoahFreeSetPartitionId which_partition, ssize_t start_index) const;
// Return the index of the previous available region <= last_index, or -1 if not found.
inline ssize_t find_index_of_previous_available_region(ShenandoahFreeSetPartitionId which_partition, ssize_t last_index) const;
// Return the index of the next available cluster of cluster_size regions >= start_index, or maximum_regions if not found.
inline ssize_t find_index_of_next_available_cluster_of_regions(ShenandoahFreeSetPartitionId which_partition,
ssize_t start_index, size_t cluster_size) const;
// Return the index of the previous available cluster of cluster_size regions <= last_index, or -1 if not found.
inline ssize_t find_index_of_previous_available_cluster_of_regions(ShenandoahFreeSetPartitionId which_partition,
ssize_t last_index, size_t cluster_size) const;
inline bool in_free_set(ShenandoahFreeSetPartitionId which_partition, ssize_t idx) const {
return _membership[int(which_partition)].is_set(idx);
}
// Returns the ShenandoahFreeSetPartitionId affiliation of region idx, NotFree if this region is not currently in any partition.
// This does not enforce that free_set membership implies allocation capacity.
inline ShenandoahFreeSetPartitionId membership(ssize_t idx) const;
#ifdef ASSERT
// Returns true iff region idx's membership is which_partition. If which_partition represents a free set, asserts
// that the region has allocation capacity.
inline bool partition_id_matches(ssize_t idx, ShenandoahFreeSetPartitionId which_partition) const;
#endif
inline size_t max_regions() const { return _max; }
inline size_t region_size_bytes() const { return _region_size_bytes; };
// The following four methods return the left-most and right-most bounds on ranges of regions representing
// the requested set. The _empty variants represent bounds on the range that holds completely empty
// regions, which are required for humongous allocations and desired for "very large" allocations.
// if the requested which_partition is empty:
// leftmost() and leftmost_empty() return _max, rightmost() and rightmost_empty() return 0
// otherwise, expect the following:
// 0 <= leftmost <= leftmost_empty <= rightmost_empty <= rightmost < _max
inline ssize_t leftmost(ShenandoahFreeSetPartitionId which_partition) const;
inline ssize_t rightmost(ShenandoahFreeSetPartitionId which_partition) const;
ssize_t leftmost_empty(ShenandoahFreeSetPartitionId which_partition);
ssize_t rightmost_empty(ShenandoahFreeSetPartitionId which_partition);
inline bool is_empty(ShenandoahFreeSetPartitionId which_partition) const;
inline void increase_used(ShenandoahFreeSetPartitionId which_partition, size_t bytes);
inline size_t capacity_of(ShenandoahFreeSetPartitionId which_partition) const {
assert (which_partition < NumPartitions, "selected free set must be valid");
return _capacity[int(which_partition)];
}
inline size_t used_by(ShenandoahFreeSetPartitionId which_partition) const {
assert (which_partition < NumPartitions, "selected free set must be valid");
return _used[int(which_partition)];
}
inline size_t available_in(ShenandoahFreeSetPartitionId which_partition) const {
assert (which_partition < NumPartitions, "selected free set must be valid");
return _capacity[int(which_partition)] - _used[int(which_partition)];
}
inline void set_capacity_of(ShenandoahFreeSetPartitionId which_partition, size_t value) {
assert (which_partition < NumPartitions, "selected free set must be valid");
_capacity[int(which_partition)] = value;
}
inline void set_used_by(ShenandoahFreeSetPartitionId which_partition, size_t value) {
assert (which_partition < NumPartitions, "selected free set must be valid");
_used[int(which_partition)] = value;
}
inline size_t count(ShenandoahFreeSetPartitionId which_partition) const { return _region_counts[int(which_partition)]; }
// Assure leftmost, rightmost, leftmost_empty, and rightmost_empty bounds are valid for all free sets.
// Valid bounds honor all of the following (where max is the number of heap regions):
// if the set is empty, leftmost equals max and rightmost equals 0
// Otherwise (the set is not empty):
// 0 <= leftmost < max and 0 <= rightmost < max
// the region at leftmost is in the set
// the region at rightmost is in the set
// rightmost >= leftmost
// for every idx that is in the set {
// idx >= leftmost &&
// idx <= rightmost
// }
// if the set has no empty regions, leftmost_empty equals max and rightmost_empty equals 0
// Otherwise (the region has empty regions):
// 0 <= leftmost_empty < max and 0 <= rightmost_empty < max
// rightmost_empty >= leftmost_empty
// for every idx that is in the set and is empty {
// idx >= leftmost &&
// idx <= rightmost
// }
void assert_bounds() NOT_DEBUG_RETURN;
};
// Publicly, ShenandoahFreeSet represents memory that is available to mutator threads. The public capacity(), used(),
// and available() methods represent this public notion of memory that is under control of the mutator. Separately,
// ShenandoahFreeSet also represents memory available to garbage collection activities for compaction purposes.
//
// The Shenandoah garbage collector evacuates live objects out of specific regions that are identified as members of the
// collection set (cset).
//
// The ShenandoahFreeSet endeavors to congregrate survivor objects (objects that have been evacuated at least once) at the
// high end of memory. New mutator allocations are taken from the low end of memory. Within the mutator's range of regions,
// humongous allocations are taken from the lowest addresses, and LAB (local allocation buffers) and regular shared allocations
// are taken from the higher address of the mutator's range of regions. This approach allows longer lasting survivor regions
// to congregate at the top of the heap and longer lasting humongous regions to congregate at the bottom of the heap, with
// short-lived frequently evacuated regions occupying the middle of the heap.
//
// Mutator and garbage collection activities tend to scramble the content of regions. Twice, during each GC pass, we rebuild
// the free set in an effort to restore the efficient segregation of Collector and Mutator regions:
//
// 1. At the start of evacuation, we know exactly how much memory is going to be evacuated, and this guides our
// sizing of the Collector free set.
//
// 2. At the end of GC, we have reclaimed all of the memory that was spanned by the cset. We rebuild here to make
// sure there is enough memory reserved at the high end of memory to hold the objects that might need to be evacuated
// during the next GC pass.
class ShenandoahFreeSet : public CHeapObj<mtGC> {
private:
ShenandoahHeap* const _heap;
CHeapBitMap _mutator_free_bitmap;
CHeapBitMap _collector_free_bitmap;
size_t _max;
ShenandoahRegionPartitions _partitions;
// Left-most and right-most region indexes. There are no free regions outside
// of [left-most; right-most] index intervals
size_t _mutator_leftmost, _mutator_rightmost;
size_t _collector_leftmost, _collector_rightmost;
// Mutator allocations are biased from left-to-right or from right-to-left based on which end of mutator range
// is most likely to hold partially used regions. In general, we want to finish consuming partially used
// regions and retire them in order to reduce the regions that must be searched for each allocation request.
bool _right_to_left_bias;
size_t _capacity;
size_t _used;
// We re-evaluate the left-to-right allocation bias whenever _alloc_bias_weight is less than zero. Each time
// we allocate an object, we decrement the count of this value. Each time we re-evaluate whether to allocate
// from right-to-left or left-to-right, we reset the value of this counter to _InitialAllocBiasWeight.
ssize_t _alloc_bias_weight;
void assert_bounds() const NOT_DEBUG_RETURN;
bool is_mutator_free(size_t idx) const;
bool is_collector_free(size_t idx) const;
const ssize_t _InitialAllocBiasWeight = 256;
HeapWord* try_allocate_in(ShenandoahHeapRegion* region, ShenandoahAllocRequest& req, bool& in_new_region);
// While holding the heap lock, allocate memory for a single object or LAB which is to be entirely contained
// within a single HeapRegion as characterized by req.
//
// Precondition: req.size() <= ShenandoahHeapRegion::humongous_threshold_words().
HeapWord* allocate_single(ShenandoahAllocRequest& req, bool& in_new_region);
// While holding the heap lock, allocate memory for a humongous object which spans one or more regions that
// were previously empty. Regions that represent humongous objects are entirely dedicated to the humongous
// object. No other objects are packed into these regions.
//
// Precondition: req.size() > ShenandoahHeapRegion::humongous_threshold_words().
HeapWord* allocate_contiguous(ShenandoahAllocRequest& req);
// Change region r from the Mutator partition to the GC's Collector partition. This requires that the region is entirely empty.
// Typical usage: During evacuation, the GC may find it needs more memory than had been reserved at the start of evacuation to
// hold evacuated objects. If this occurs and memory is still available in the Mutator's free set, we will flip a region from
// the Mutator free set into the Collector free set.
void flip_to_gc(ShenandoahHeapRegion* r);
void recompute_bounds();
void adjust_bounds();
bool touches_bounds(size_t num) const;
void increase_used(size_t amount);
void clear_internal();
size_t collector_count() const { return _collector_free_bitmap.count_one_bits(); }
size_t mutator_count() const { return _mutator_free_bitmap.count_one_bits(); }
void try_recycle_trashed(ShenandoahHeapRegion *r);
bool can_allocate_from(ShenandoahHeapRegion *r);
size_t alloc_capacity(ShenandoahHeapRegion *r);
bool has_no_alloc_capacity(ShenandoahHeapRegion *r);
// Returns true iff this region is entirely available, either because it is empty() or because it has been found to represent
// immediate trash and we'll be able to immediately recycle it. Note that we cannot recycle immediate trash if
// concurrent weak root processing is in progress.
inline bool can_allocate_from(ShenandoahHeapRegion *r) const;
inline bool can_allocate_from(size_t idx) const;
inline bool has_alloc_capacity(ShenandoahHeapRegion *r) const;
// This function places all regions that have allocation capacity into the mutator_partition, identifying regions
// that have no allocation capacity as NotFree. Subsequently, we will move some of the mutator regions into the
// collector partition with the intent of packing collector memory into the highest (rightmost) addresses of the
// heap, with mutator memory consuming the lowest addresses of the heap.
void find_regions_with_alloc_capacity(size_t &cset_regions);
// Having placed all regions that have allocation capacity into the mutator partition, move some of these regions from
// the mutator partition into the collector partition in order to assure that the memory available for allocations within
// the collector partition is at least to_reserve.
void reserve_regions(size_t to_reserve);
// Overwrite arguments to represent the number of regions to be reclaimed from the cset
void prepare_to_rebuild(size_t &cset_regions);
void finish_rebuild(size_t cset_regions);
public:
ShenandoahFreeSet(ShenandoahHeap* heap, size_t max_regions);
// Public because ShenandoahRegionPartitions assertions require access.
inline size_t alloc_capacity(ShenandoahHeapRegion *r) const;
inline size_t alloc_capacity(size_t idx) const;
void clear();
void rebuild();
void recycle_trash();
// Move up to cset_regions number of regions from being available to the collector to being available to the mutator.
//
// Typical usage: At the end of evacuation, when the collector no longer needs the regions that had been reserved
// for evacuation, invoke this to make regions available for mutator allocations.
//
// Note that we plan to replenish the Collector reserve at the end of update refs, at which time all
// of the regions recycled from the collection set will be available. If the very unlikely event that there
// are fewer regions in the collection set than remain in the collector set, we limit the transfer in order
// to assure that the replenished Collector reserve can be sufficiently large.
void move_regions_from_collector_to_mutator(size_t cset_regions);
void recycle_trash();
void log_status();
size_t capacity() const { return _capacity; }
size_t used() const { return _used; }
size_t available() const {
assert(_used <= _capacity, "must use less than capacity");
return _capacity - _used;
inline size_t capacity() const { return _partitions.capacity_of(ShenandoahFreeSetPartitionId::Mutator); }
inline size_t used() const { return _partitions.used_by(ShenandoahFreeSetPartitionId::Mutator); }
inline size_t available() const {
assert(used() <= capacity(), "must use less than capacity");
return capacity() - used();
}
HeapWord* allocate(ShenandoahAllocRequest& req, bool& in_new_region);
size_t unsafe_peek_free() const;
/*
* Internal fragmentation metric: describes how fragmented the heap regions are.
*
* It is derived as:
*
* sum(used[i]^2, i=0..k)
* IF = 1 - ------------------------------
* C * sum(used[i], i=0..k)
*
* ...where k is the number of regions in computation, C is the region capacity, and
* used[i] is the used space in the region.
*
* The non-linearity causes IF to be lower for the cases where the same total heap
* used is densely packed. For example:
* a) Heap is completely full => IF = 0
* b) Heap is half full, first 50% regions are completely full => IF = 0
* c) Heap is half full, each region is 50% full => IF = 1/2
* d) Heap is quarter full, first 50% regions are completely full => IF = 0
* e) Heap is quarter full, each region is 25% full => IF = 3/4
* f) Heap has one small object per each region => IF =~ 1
*/
double internal_fragmentation();
/*
* External fragmentation metric: describes how fragmented the heap is.
*
* It is derived as:
*
* EF = 1 - largest_contiguous_free / total_free
*
* For example:
* a) Heap is completely empty => EF = 0
* b) Heap is completely full => EF = 0
* c) Heap is first-half full => EF = 1/2
* d) Heap is half full, full and empty regions interleave => EF =~ 1
*/
double external_fragmentation();
void print_on(outputStream* out) const;

1
src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp
View File

@ -912,7 +912,6 @@ private:
public:
ShenandoahPostCompactClosure() : _heap(ShenandoahHeap::heap()), _live(0) {
_heap->free_set()->clear();
}
void heap_region_do(ShenandoahHeapRegion* r) {

16
src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp
View File

@ -2103,17 +2103,27 @@ public:
if (CONCURRENT) {
ShenandoahConcurrentWorkerSession worker_session(worker_id);
ShenandoahSuspendibleThreadSetJoiner stsj;
do_work<ShenandoahConcUpdateRefsClosure>();
do_work<ShenandoahConcUpdateRefsClosure>(worker_id);
} else {
ShenandoahParallelWorkerSession worker_session(worker_id);
do_work<ShenandoahSTWUpdateRefsClosure>();
do_work<ShenandoahSTWUpdateRefsClosure>(worker_id);
}
}
private:
template<class T>
void do_work() {
void do_work(uint worker_id) {
T cl;
if (CONCURRENT && (worker_id == 0)) {
// We ask the first worker to replenish the Mutator free set by moving regions previously reserved to hold the
// results of evacuation. These reserves are no longer necessary because evacuation has completed.
size_t cset_regions = _heap->collection_set()->count();
// We cannot transfer any more regions than will be reclaimed when the existing collection set is recycled because
// we need the reclaimed collection set regions to replenish the collector reserves
_heap->free_set()->move_regions_from_collector_to_mutator(cset_regions);
}
// If !CONCURRENT, there's no value in expanding Mutator free set
ShenandoahHeapRegion* r = _regions->next();
ShenandoahMarkingContext* const ctx = _heap->complete_marking_context();
while (r != nullptr) {

291
src/hotspot/share/gc/shenandoah/shenandoahSimpleBitMap.cpp Normal file
View File

@ -0,0 +1,291 @@
/*
* Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License version 2 only, as
* published by the Free Software Foundation.
*
* This code is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
* version 2 for more details (a copy is included in the LICENSE file that
* accompanied this code).
*
* You should have received a copy of the GNU General Public License version
* 2 along with this work; if not, write to the Free Software Foundation,
* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
*
* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
* or visit www.oracle.com if you need additional information or have any
* questions.
*
*/
#include "precompiled.hpp"
#include "gc/shenandoah/shenandoahSimpleBitMap.hpp"
ShenandoahSimpleBitMap::ShenandoahSimpleBitMap(size_t num_bits) :
_num_bits(num_bits),
_num_words(align_up(num_bits, BitsPerWord) / BitsPerWord),
_bitmap(NEW_C_HEAP_ARRAY(uintx, _num_words, mtGC))
{
clear_all();
}
ShenandoahSimpleBitMap::~ShenandoahSimpleBitMap() {
if (_bitmap != nullptr) {
FREE_C_HEAP_ARRAY(uintx, _bitmap);
}
}
size_t ShenandoahSimpleBitMap::count_leading_ones(idx_t start_idx) const {
assert((start_idx >= 0) && (start_idx < _num_bits), "precondition");
size_t array_idx = start_idx >> LogBitsPerWord;
uintx element_bits = _bitmap[array_idx];
uintx bit_number = start_idx & right_n_bits(LogBitsPerWord);
uintx mask = ~right_n_bits(bit_number);
size_t counted_ones = 0;
while ((element_bits & mask) == mask) {
// All bits numbered >= bit_number are set
size_t found_ones = BitsPerWord - bit_number;
counted_ones += found_ones;
// Dead code: do not need to compute: start_idx += found_ones;
// Strength reduction: array_idx = (start_idx >> LogBitsPerWord)
array_idx++;
element_bits = _bitmap[array_idx];
// Constant folding: bit_number = start_idx & right_n_bits(LogBitsPerWord);
bit_number = 0;
// Constant folding: mask = ~right_n_bits(bit_number);
mask = ~0;
}
// Add in number of consecutive ones starting with the_bit and including more significant bits and return result
uintx aligned = element_bits >> bit_number;
uintx complement = ~aligned;
return counted_ones + count_trailing_zeros<uintx>(complement);
}
size_t ShenandoahSimpleBitMap::count_trailing_ones(idx_t last_idx) const {
assert((last_idx >= 0) && (last_idx < _num_bits), "precondition");
size_t array_idx = last_idx >> LogBitsPerWord;
uintx element_bits = _bitmap[array_idx];
uintx bit_number = last_idx & right_n_bits(LogBitsPerWord);
// All ones from bit 0 to the_bit
uintx mask = right_n_bits(bit_number + 1);
size_t counted_ones = 0;
while ((element_bits & mask) == mask) {
// All bits numbered <= bit_number are set
size_t found_ones = bit_number + 1;
counted_ones += found_ones;
// Dead code: do not need to compute: last_idx -= found_ones;
array_idx--;
element_bits = _bitmap[array_idx];
// Constant folding: bit_number = last_idx & right_n_bits(LogBitsPerWord);
bit_number = BitsPerWord - 1;
// Constant folding: mask = right_n_bits(bit_number + 1);
mask = ~0;
}
// Add in number of consecutive ones starting with the_bit and including less significant bits and return result
uintx aligned = element_bits << (BitsPerWord - (bit_number + 1));
uintx complement = ~aligned;
return counted_ones + count_leading_zeros<uintx>(complement);
}
bool ShenandoahSimpleBitMap::is_forward_consecutive_ones(idx_t start_idx, idx_t count) const {
while (count > 0) {
assert((start_idx >= 0) && (start_idx < _num_bits), "precondition: start_idx: " SSIZE_FORMAT ", count: " SSIZE_FORMAT,
start_idx, count);
assert(start_idx + count <= (idx_t) _num_bits, "precondition");
size_t array_idx = start_idx >> LogBitsPerWord;
uintx bit_number = start_idx & right_n_bits(LogBitsPerWord);
uintx element_bits = _bitmap[array_idx];
uintx bits_to_examine = BitsPerWord - bit_number;
element_bits >>= bit_number;
uintx complement = ~element_bits;
uintx trailing_ones;
if (complement != 0) {
trailing_ones = count_trailing_zeros<uintx>(complement);
} else {
trailing_ones = bits_to_examine;
}
if (trailing_ones >= (uintx) count) {
return true;
} else if (trailing_ones == bits_to_examine) {
start_idx += bits_to_examine;
count -= bits_to_examine;
// Repeat search with smaller goal
} else {
return false;
}
}
return true;
}
bool ShenandoahSimpleBitMap::is_backward_consecutive_ones(idx_t last_idx, idx_t count) const {
while (count > 0) {
assert((last_idx >= 0) && (last_idx < _num_bits), "precondition");
assert(last_idx - count >= -1, "precondition");
size_t array_idx = last_idx >> LogBitsPerWord;
uintx bit_number = last_idx & right_n_bits(LogBitsPerWord);
uintx element_bits = _bitmap[array_idx];
uintx bits_to_examine = bit_number + 1;
element_bits <<= (BitsPerWord - bits_to_examine);
uintx complement = ~element_bits;
uintx leading_ones;
if (complement != 0) {
leading_ones = count_leading_zeros<uintx>(complement);
} else {
leading_ones = bits_to_examine;
}
if (leading_ones >= (uintx) count) {
return true;
} else if (leading_ones == bits_to_examine) {
last_idx -= leading_ones;
count -= leading_ones;
// Repeat search with smaller goal
} else {
return false;
}
}
return true;
}
idx_t ShenandoahSimpleBitMap::find_first_consecutive_set_bits(idx_t beg, idx_t end, size_t num_bits) const {
assert((beg >= 0) && (beg < _num_bits), "precondition");
// Stop looking if there are not num_bits remaining in probe space.
idx_t start_boundary = end - num_bits;
if (beg > start_boundary) {
return end;
}
uintx array_idx = beg >> LogBitsPerWord;
uintx bit_number = beg & right_n_bits(LogBitsPerWord);
uintx element_bits = _bitmap[array_idx];
if (bit_number > 0) {
uintx mask_out = right_n_bits(bit_number);
element_bits &= ~mask_out;
}
// The following loop minimizes the number of spans probed in order to find num_bits consecutive bits.
// For example, if bit_number = beg = 0, num_bits = 8, and element bits equals 00111111_11000000_00000000_10011000B,
// we need only 3 probes to find the match at bit offset 22.
//
// Let beg = 0
// element_bits = 00111111_11000000_00000000_10011000B;
// ________ (the searched span)
// ^ ^ ^- bit_number = beg = 0
// | +-- next_start_candidate_1 (where next 1 is found)
// +------ next_start_candidate_2 (start of the trailing 1s within span)
// Let beg = 7
// element_bits = 00111111_11000000_00000000_10011000B;
// ^ ^_________ (the searched span)
// | | ^- bit_number = beg = 7
// | +---------- next_start_candidate_2 (there are no trailing 1s within span)
// +------------------ next_start_candidate_1 (where next 1 is found)
// Let beg = 22
// Let beg = 22
// element_bits = 00111111_11000001_11111100_10011000B;
// _________ (the searched span)
// ^- bit_number = beg = 18
// Here, is_forward_consecutive_ones(22, 8) succeeds and we report the match
while (true) {
if (element_bits == 0) {
// move to the next element
beg += BitsPerWord - bit_number;
if (beg > start_boundary) {
// No match found.
return end;
}
array_idx++;
bit_number = 0;
element_bits = _bitmap[array_idx];
} else if (is_forward_consecutive_ones(beg, num_bits)) {
return beg;
} else {
// There is at least one non-zero bit within the masked element_bits. Arrange to skip over bits that
// cannot be part of a consecutive-ones match.
uintx next_set_bit = count_trailing_zeros<uintx>(element_bits);
uintx next_start_candidate_1 = (array_idx << LogBitsPerWord) + next_set_bit;
// There is at least one zero bit in this span. Align the next probe at the start of trailing ones for probed span,
// or align at end of span if this span has no trailing ones.
size_t trailing_ones = count_trailing_ones(beg + num_bits - 1);
uintx next_start_candidate_2 = beg + num_bits - trailing_ones;
beg = MAX2(next_start_candidate_1, next_start_candidate_2);
if (beg > start_boundary) {
// No match found.
return end;
}
array_idx = beg >> LogBitsPerWord;
element_bits = _bitmap[array_idx];
bit_number = beg & right_n_bits(LogBitsPerWord);
if (bit_number > 0) {
size_t mask_out = right_n_bits(bit_number);
element_bits &= ~mask_out;
}
}
}
}
idx_t ShenandoahSimpleBitMap::find_last_consecutive_set_bits(const idx_t beg, idx_t end, const size_t num_bits) const {
assert((end >= 0) && (end < _num_bits), "precondition");
// Stop looking if there are not num_bits remaining in probe space.
idx_t last_boundary = beg + num_bits;
if (end < last_boundary) {
return beg;
}
size_t array_idx = end >> LogBitsPerWord;
uintx bit_number = end & right_n_bits(LogBitsPerWord);
uintx element_bits = _bitmap[array_idx];
if (bit_number < BitsPerWord - 1) {
uintx mask_in = right_n_bits(bit_number + 1);
element_bits &= mask_in;
}
// See comment in find_first_consecutive_set_bits to understand how this loop works.
while (true) {
if (element_bits == 0) {
// move to the previous element
end -= bit_number + 1;
if (end < last_boundary) {
// No match found.
return beg;
}
array_idx--;
bit_number = BitsPerWord - 1;
element_bits = _bitmap[array_idx];
} else if (is_backward_consecutive_ones(end, num_bits)) {
return end + 1 - num_bits;
} else {
// There is at least one non-zero bit within the masked element_bits. Arrange to skip over bits that
// cannot be part of a consecutive-ones match.
uintx next_set_bit = BitsPerWord - (1 + count_leading_zeros<uintx>(element_bits));
uintx next_last_candidate_1 = (array_idx << LogBitsPerWord) + next_set_bit;
// There is at least one zero bit in this span. Align the next probe at the end of leading ones for probed span,
// or align before start of span if this span has no leading ones.
size_t leading_ones = count_leading_ones(end - (num_bits - 1));
uintx next_last_candidate_2 = end - (num_bits - leading_ones);
end = MIN2(next_last_candidate_1, next_last_candidate_2);
if (end < last_boundary) {
// No match found.
return beg;
}
array_idx = end >> LogBitsPerWord;
bit_number = end & right_n_bits(LogBitsPerWord);
element_bits = _bitmap[array_idx];
if (bit_number < BitsPerWord - 1){
size_t mask_in = right_n_bits(bit_number + 1);
element_bits &= mask_in;
}
}
}
}

170
src/hotspot/share/gc/shenandoah/shenandoahSimpleBitMap.hpp Normal file
View File

@ -0,0 +1,170 @@
/*
* Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License version 2 only, as
* published by the Free Software Foundation.
*
* This code is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
* version 2 for more details (a copy is included in the LICENSE file that
* accompanied this code).
*
* You should have received a copy of the GNU General Public License version
* 2 along with this work; if not, write to the Free Software Foundation,
* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
*
* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
* or visit www.oracle.com if you need additional information or have any
* questions.
*
*/
#ifndef SHARE_GC_SHENANDOAH_SHENANDOAHSIMPLEBITMAP_HPP
#define SHARE_GC_SHENANDOAH_SHENANDOAHSIMPLEBITMAP_HPP
#include <cstddef>
#include "gc/shenandoah/shenandoahAsserts.hpp"
// TODO: Merge the enhanced capabilities of ShenandoahSimpleBitMap into src/hotspot/share/utilities/bitMap.hpp
// and deprecate ShenandoahSimpleBitMap. The key enhanced capabilities to be integrated include:
//
// 1. Allow searches from high to low memory (when biasing allocations towards the top of the heap)
// 2. Allow searches for clusters of contiguous set bits (to expedite allocation for humongous objects)
//
// idx_t is defined here as ssize_t. In src/hotspot/share/utiliities/bitMap.hpp, idx is defined as size_t.
// This is a significant incompatibility.
//
// The API and internal implementation of ShenandoahSimpleBitMap and ShenandoahRegionPartitions use idx_t to
// represent index, even though index is "inherently" unsigned. There are two reasons for this choice:
// 1. We use -1 as a sentinel value to represent empty partitions. This same value may be used to represent
// failure to find a previous set bit or previous range of set bits.
// 2. Certain loops are written most naturally if the iterator, which may hold the sentinel -1 value, can be
// declared as signed and the terminating condition can be < 0.
typedef ssize_t idx_t;
// ShenandoahSimpleBitMap resembles CHeapBitMap but adds missing support for find_first_consecutive_set_bits() and
// find_last_consecutive_set_bits. An alternative refactoring of code would subclass CHeapBitMap, but this might
// break abstraction rules, because efficient implementation requires assumptions about superclass internals that
// might be violatee through future software maintenance.
class ShenandoahSimpleBitMap {
const idx_t _num_bits;
const size_t _num_words;
uintx* const _bitmap;
public:
ShenandoahSimpleBitMap(size_t num_bits);
~ShenandoahSimpleBitMap();
void clear_all() {
for (size_t i = 0; i < _num_words; i++) {
_bitmap[i] = 0;
}
}
private:
// Count consecutive ones in forward order, starting from start_idx. Requires that there is at least one zero
// between start_idx and index value (_num_bits - 1), inclusive.
size_t count_leading_ones(idx_t start_idx) const;
// Count consecutive ones in reverse order, starting from last_idx. Requires that there is at least one zero
// between last_idx and index value zero, inclusive.
size_t count_trailing_ones(idx_t last_idx) const;
bool is_forward_consecutive_ones(idx_t start_idx, idx_t count) const;
bool is_backward_consecutive_ones(idx_t last_idx, idx_t count) const;
public:
inline idx_t aligned_index(idx_t idx) const {
assert((idx >= 0) && (idx < _num_bits), "precondition");
idx_t array_idx = idx & ~right_n_bits(LogBitsPerWord);
return array_idx;
}
inline constexpr idx_t alignment() const {
return BitsPerWord;
}
// For testing
inline idx_t size() const {
return _num_bits;
}
// Return the word that holds idx bit and its neighboring bits.
inline uintx bits_at(idx_t idx) const {
assert((idx >= 0) && (idx < _num_bits), "precondition");
idx_t array_idx = idx >> LogBitsPerWord;
return _bitmap[array_idx];
}
inline void set_bit(idx_t idx) {
assert((idx >= 0) && (idx < _num_bits), "precondition");
size_t array_idx = idx >> LogBitsPerWord;
uintx bit_number = idx & right_n_bits(LogBitsPerWord);
uintx the_bit = nth_bit(bit_number);
_bitmap[array_idx] |= the_bit;
}
inline void clear_bit(idx_t idx) {
assert((idx >= 0) && (idx < _num_bits), "precondition");
assert(idx >= 0, "precondition");
size_t array_idx = idx >> LogBitsPerWord;
uintx bit_number = idx & right_n_bits(LogBitsPerWord);
uintx the_bit = nth_bit(bit_number);
_bitmap[array_idx] &= ~the_bit;
}
inline bool is_set(idx_t idx) const {
assert((idx >= 0) && (idx < _num_bits), "precondition");
assert(idx >= 0, "precondition");
size_t array_idx = idx >> LogBitsPerWord;
uintx bit_number = idx & right_n_bits(LogBitsPerWord);
uintx the_bit = nth_bit(bit_number);
return (_bitmap[array_idx] & the_bit)? true: false;
}
// Return the index of the first set bit in the range [beg, size()), or size() if none found.
// precondition: beg and end form a valid range for the bitmap.
inline idx_t find_first_set_bit(idx_t beg) const;
// Return the index of the first set bit in the range [beg, end), or end if none found.
// precondition: beg and end form a valid range for the bitmap.
inline idx_t find_first_set_bit(idx_t beg, idx_t end) const;
// Return the index of the last set bit in the range (-1, end], or -1 if none found.
// precondition: beg and end form a valid range for the bitmap.
inline idx_t find_last_set_bit(idx_t end) const;
// Return the index of the last set bit in the range (beg, end], or beg if none found.
// precondition: beg and end form a valid range for the bitmap.
inline idx_t find_last_set_bit(idx_t beg, idx_t end) const;
// Return the start index of the first run of <num_bits> consecutive set bits for which the first set bit is within
// the range [beg, size()), or size() if the run of <num_bits> is not found within this range.
// precondition: beg is within the valid range for the bitmap.
inline idx_t find_first_consecutive_set_bits(idx_t beg, size_t num_bits) const;
// Return the start index of the first run of <num_bits> consecutive set bits for which the first set bit is within
// the range [beg, end), or end if the run of <num_bits> is not found within this range.
// precondition: beg and end form a valid range for the bitmap.
idx_t find_first_consecutive_set_bits(idx_t beg, idx_t end, size_t num_bits) const;
// Return the start index of the last run of <num_bits> consecutive set bits for which the entire run of set bits is within
// the range (-1, end], or -1 if the run of <num_bits> is not found within this range.
// precondition: end is within the valid range for the bitmap.
inline idx_t find_last_consecutive_set_bits(idx_t end, size_t num_bits) const;
// Return the start index of the first run of <num_bits> consecutive set bits for which the entire run of set bits is within
// the range (beg, end], or beg if the run of <num_bits> is not found within this range.
// precondition: beg and end form a valid range for the bitmap.
idx_t find_last_consecutive_set_bits(idx_t beg, idx_t end, size_t num_bits) const;
};
#endif // SHARE_GC_SHENANDOAH_SHENANDOAHSIMPLEBITMAP_HPP

100
src/hotspot/share/gc/shenandoah/shenandoahSimpleBitMap.inline.hpp Normal file
View File

@ -0,0 +1,100 @@
/*
* Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License version 2 only, as
* published by the Free Software Foundation.
*
* This code is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
* version 2 for more details (a copy is included in the LICENSE file that
* accompanied this code).
*
* You should have received a copy of the GNU General Public License version
* 2 along with this work; if not, write to the Free Software Foundation,
* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
*
* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
* or visit www.oracle.com if you need additional information or have any
* questions.
*
*/
#ifndef SHARE_GC_SHENANDOAH_SHENANDOAHSIMPLEBITMAP_INLINE_HPP
#define SHARE_GC_SHENANDOAH_SHENANDOAHSIMPLEBITMAP_INLINE_HPP
#include "gc/shenandoah/shenandoahSimpleBitMap.hpp"
inline idx_t ShenandoahSimpleBitMap::find_first_set_bit(idx_t beg, idx_t end) const {
assert((beg >= 0) && (beg < _num_bits), "precondition");
assert((end > beg) && (end <= _num_bits), "precondition");
do {
size_t array_idx = beg >> LogBitsPerWord;
uintx bit_number = beg & right_n_bits(LogBitsPerWord);
uintx element_bits = _bitmap[array_idx];
if (bit_number > 0) {
uintx mask_out = right_n_bits(bit_number);
element_bits &= ~mask_out;
}
if (element_bits) {
// The next set bit is here. Find first set bit >= bit_number;
uintx aligned = element_bits >> bit_number;
uintx first_set_bit = count_trailing_zeros<uintx>(aligned);
idx_t candidate_result = (array_idx * BitsPerWord) + bit_number + first_set_bit;
return (candidate_result < end)? candidate_result: end;
} else {
// Next bit is not here. Try the next array element
beg += BitsPerWord - bit_number;
}
} while (beg < end);
return end;
}
inline idx_t ShenandoahSimpleBitMap::find_first_set_bit(idx_t beg) const {
assert((beg >= 0) && (beg < size()), "precondition");
return find_first_set_bit(beg, size());
}
inline idx_t ShenandoahSimpleBitMap::find_last_set_bit(idx_t beg, idx_t end) const {
assert((end >= 0) && (end < _num_bits), "precondition");
assert((beg >= -1) && (beg < end), "precondition");
do {
idx_t array_idx = end >> LogBitsPerWord;
uintx bit_number = end & right_n_bits(LogBitsPerWord);
uintx element_bits = _bitmap[array_idx];
if (bit_number < BitsPerWord - 1){
uintx mask_in = right_n_bits(bit_number + 1);
element_bits &= mask_in;
}
if (element_bits) {
// The prev set bit is here. Find the first set bit <= bit_number
uintx aligned = element_bits << (BitsPerWord - (bit_number + 1));
uintx first_set_bit = count_leading_zeros<uintx>(aligned);
idx_t candidate_result = array_idx * BitsPerWord + (bit_number - first_set_bit);
return (candidate_result > beg)? candidate_result: beg;
} else {
// Next bit is not here. Try the previous array element
end -= (bit_number + 1);
}
} while (end > beg);
return beg;
}
inline idx_t ShenandoahSimpleBitMap::find_last_set_bit(idx_t end) const {
assert((end >= 0) && (end < _num_bits), "precondition");
return find_last_set_bit(-1, end);
}
inline idx_t ShenandoahSimpleBitMap::find_first_consecutive_set_bits(idx_t beg, size_t num_bits) const {
assert((beg >= 0) && (beg < _num_bits), "precondition");
return find_first_consecutive_set_bits(beg, size(), num_bits);
}
inline idx_t ShenandoahSimpleBitMap::find_last_consecutive_set_bits(idx_t end, size_t num_bits) const {
assert((end >= 0) && (end < _num_bits), "precondition");
return find_last_consecutive_set_bits((idx_t) -1, end, num_bits);
}
#endif // SHARE_GC_SHENANDOAH_SHENANDOAHSIMPLEBITMAP_INLINE_HPP

451
test/hotspot/gtest/gc/shenandoah/test_shenandoahSimpleBitMap.cpp Normal file
View File

@ -0,0 +1,451 @@
/*
* Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License version 2 only, as
* published by the Free Software Foundation.
*
* This code is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
* version 2 for more details (a copy is included in the LICENSE file that
* accompanied this code).
*
* You should have received a copy of the GNU General Public License version
* 2 along with this work; if not, write to the Free Software Foundation,
* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
*
* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
* or visit www.oracle.com if you need additional information or have any
* questions.
*/
#include "precompiled.hpp"
#include "gc/shenandoah/shenandoahSimpleBitMap.hpp"
#include "gc/shenandoah/shenandoahSimpleBitMap.inline.hpp"
#include <iostream>
#include "utilities/ostream.hpp"
#include "utilities/vmassert_uninstall.hpp"
#include "utilities/vmassert_reinstall.hpp"
#include "unittest.hpp"
static bool _success;
static size_t _assertion_failures;
#define BitMapAssertEqual(a, b) ASSERT_EQ((a), (b)); if ((a) != (b)) { _assertion_failures++; }
class ShenandoahSimpleBitMapTest: public ::testing::Test {
protected:
static const ssize_t SMALL_BITMAP_SIZE = 512;
static const ssize_t LARGE_BITMAP_SIZE = 4096;
// set_bits[] is an array of indexes holding bits that are supposed to be set, in increasing order.
static void verifyBitMapState(ShenandoahSimpleBitMap& bm, ssize_t size, ssize_t set_bits[], ssize_t num_set_bits) {
// Verify number of bits
BitMapAssertEqual(bm.size(), size);
ssize_t set_bit_index = 0;
// Check that is_set(idx) for every possible idx
for (ssize_t i = 0; i < size; i++) {
bool is_set = bm.is_set(i);
bool intended_value = false;;
if (set_bit_index < num_set_bits) {
if (set_bits[set_bit_index] == i) {
intended_value = true;
set_bit_index++;
}
} else {
// If we've exhausted set_bits array, there should be no more set_bits
BitMapAssertEqual(is_set, false);
BitMapAssertEqual(set_bit_index, num_set_bits);
}
BitMapAssertEqual(is_set, intended_value);
}
BitMapAssertEqual(set_bit_index, num_set_bits);
// Check that bits_at(array_idx) matches intended value for every valid array_idx value
set_bit_index = 0;
ssize_t alignment = bm.alignment();
for (ssize_t i = 0; i < size; i += alignment) {
size_t bits = bm.bits_at(i);
for (ssize_t b = 0; b < alignment; b++) {
ssize_t bit_value = i + b;
bool intended_value = false;;
if (set_bit_index < num_set_bits) {
if (set_bits[set_bit_index] == bit_value) {
intended_value = true;
set_bit_index++;
}
}
size_t bit_mask = ((size_t) 0x01) << b;
bool is_set = (bits & bit_mask) != 0;
BitMapAssertEqual(is_set, intended_value);
}
}
// Make sure find_first_set_bit() works correctly
ssize_t probe_point = 0;
for (ssize_t i = 0; i < num_set_bits; i++) {
ssize_t next_expected_bit = set_bits[i];
probe_point = bm.find_first_set_bit(probe_point);
BitMapAssertEqual(probe_point, next_expected_bit);
probe_point++; // Prepare to look beyond the most recent bit.
}
if (probe_point < size) {
probe_point = bm.find_first_set_bit(probe_point);
BitMapAssertEqual(probe_point, size); // Verify that last failed search returns sentinel value: num bits in bit map
}
// Confirm that find_first_set_bit() with a bounded search space works correctly
// Limit this search to the first 3/4 of the full bit map
ssize_t boundary_idx = 3 * size / 4;
probe_point = 0;
for (ssize_t i = 0; i < num_set_bits; i++) {
ssize_t next_expected_bit = set_bits[i];
if (next_expected_bit >= boundary_idx) {
break;
} else {
probe_point = bm.find_first_set_bit(probe_point, boundary_idx);
BitMapAssertEqual(probe_point, next_expected_bit);
probe_point++; // Prepare to look beyond the most recent bit.
}
}
if (probe_point < boundary_idx) {
// In case there are no set bits in the last 1/4 of bit map, confirm that last failed search returns sentinel: boundary_idx
probe_point = bm.find_first_set_bit(probe_point, boundary_idx);
BitMapAssertEqual(probe_point, boundary_idx);
}
// Make sure find_last_set_bit() works correctly
probe_point = size - 1;
for (ssize_t i = num_set_bits - 1; i >= 0; i--) {
ssize_t next_expected_bit = set_bits[i];
probe_point = bm.find_last_set_bit(probe_point);
BitMapAssertEqual(probe_point, next_expected_bit);
probe_point--; // Prepare to look before the most recent bit.
}
if (probe_point >= 0) {
probe_point = bm.find_last_set_bit(probe_point);
BitMapAssertEqual(probe_point, (ssize_t) -1); // Verify that last failed search returns sentinel value: -1
}
// Confirm that find_last_set_bit() with a bounded search space works correctly
// Limit this search to the last 3/4 of the full bit map
boundary_idx = size / 4;
probe_point = size - 1;
for (ssize_t i = num_set_bits - 1; i >= 0; i--) {
ssize_t next_expected_bit = set_bits[i];
if (next_expected_bit > boundary_idx) {
probe_point = bm.find_last_set_bit(boundary_idx, probe_point);
BitMapAssertEqual(probe_point, next_expected_bit);
probe_point--;
} else {
break;
}
}
if (probe_point > boundary_idx) {
probe_point = bm.find_last_set_bit(boundary_idx, probe_point);
// Verify that last failed search returns sentinel value: boundary_idx
BitMapAssertEqual(probe_point, boundary_idx);
}
// What's the longest cluster of consecutive bits
ssize_t previous_value = -2;
ssize_t longest_run = 0;
ssize_t current_run = 0;
for (ssize_t i = 0; i < num_set_bits; i++) {
ssize_t next_expected_bit = set_bits[i];
if (next_expected_bit == previous_value + 1) {
current_run++;
} else {
previous_value = next_expected_bit;
current_run = 1;
}
if (current_run > longest_run) {
longest_run = current_run;
}
previous_value = next_expected_bit;
}
// Confirm that find_first_consecutive_set_bits() works for each cluster size known to have at least one match
for (ssize_t cluster_size = 1; cluster_size <= longest_run; cluster_size++) {
// Verify that find_first_consecutive_set_bits() works
ssize_t bit_idx = 0;
ssize_t probe_point = 0;
while ((probe_point <= size - cluster_size) && (bit_idx <= num_set_bits - cluster_size)) {
bool cluster_found = false;
while (!cluster_found && (bit_idx + cluster_size <= num_set_bits)) {
cluster_found = true;
for (ssize_t i = 1; i < cluster_size; i++) {
if (set_bits[bit_idx] + i != set_bits[bit_idx + i]) {
cluster_found = false;
bit_idx++;
break;
}
}
}
if (cluster_found) {
ssize_t next_expected_cluster = set_bits[bit_idx];
ssize_t orig_probe_point = probe_point;
probe_point = bm.find_first_consecutive_set_bits(orig_probe_point, cluster_size);
BitMapAssertEqual(next_expected_cluster, probe_point);
probe_point++;
bit_idx++;
} else {
bit_idx++;
break;
}
}
if (probe_point < size) {
// Confirm that the last request, which fails to find a cluster, returns sentinel value: num_bits
probe_point = bm.find_first_consecutive_set_bits(probe_point, cluster_size);
BitMapAssertEqual(probe_point, size);
}
// Repeat the above experiment, using 3/4 size as the search boundary_idx
bit_idx = 0;
probe_point = 0;
boundary_idx = 4 * size / 4;
while ((probe_point <= boundary_idx - cluster_size) && (bit_idx <= num_set_bits - cluster_size)) {
bool cluster_found = false;
while (!cluster_found && (bit_idx + cluster_size <= num_set_bits)) {
cluster_found = true;
for (int i = 1; i < cluster_size; i++) {
if (set_bits[bit_idx] + i != set_bits[bit_idx + i]) {
cluster_found = false;
bit_idx++;
break;
}
}
}
if (cluster_found) {
ssize_t next_expected_cluster = set_bits[bit_idx];
probe_point = bm.find_first_consecutive_set_bits(probe_point, boundary_idx, cluster_size);
BitMapAssertEqual(next_expected_cluster, probe_point);
probe_point++;
bit_idx++;
} else {
bit_idx++;
}
}
if (probe_point < boundary_idx) {
// Confirm that the last request, which fails to find a cluster, returns sentinel value: boundary_idx
probe_point = bm.find_first_consecutive_set_bits(probe_point, boundary_idx, cluster_size);
BitMapAssertEqual(probe_point, boundary_idx);
}
// Verify that find_last_consecutive_set_bits() works
bit_idx = num_set_bits - 1;
probe_point = size - 1;
// Iterate over all set bits in reverse order
while (bit_idx + 1 >= cluster_size) {
bool cluster_found = true;
for (int i = 1; i < cluster_size; i++) {
if (set_bits[bit_idx] - i != set_bits[bit_idx - i]) {
cluster_found = false;
break;
}
}
if (cluster_found) {
ssize_t next_expected_cluster = set_bits[bit_idx] + 1 - cluster_size;
probe_point = bm.find_last_consecutive_set_bits(probe_point, cluster_size);
BitMapAssertEqual(next_expected_cluster, probe_point);
probe_point = probe_point + cluster_size - 2;
bit_idx--;
} else {
bit_idx--;
}
}
if (probe_point >= 0) {
// Confirm that the last request, which fails to find a cluster, returns sentinel value: boundary_idx
probe_point = bm.find_last_consecutive_set_bits(boundary_idx, probe_point, cluster_size);
BitMapAssertEqual(probe_point, (ssize_t) boundary_idx);
}
// Verify that find_last_consecutive_set_bits() works with the search range bounded at 1/4 size
bit_idx = num_set_bits - 1;
probe_point = size - 1;
boundary_idx = size / 4;
while (bit_idx + 1 >= cluster_size) {
bool cluster_found = true;
for (int i = 1; i < cluster_size; i++) {
if (set_bits[bit_idx] - i != set_bits[bit_idx - i]) {
cluster_found = false;
break;
}
}
if (cluster_found && (set_bits[bit_idx] + 1 - cluster_size > boundary_idx)) {
ssize_t next_expected_cluster = set_bits[bit_idx] + 1 - cluster_size;
probe_point = bm.find_last_consecutive_set_bits(boundary_idx, probe_point, cluster_size);
BitMapAssertEqual(next_expected_cluster, probe_point);
probe_point = probe_point + cluster_size - 2;
bit_idx--;
} else if (set_bits[bit_idx] + 1 - cluster_size <= boundary_idx) {
break;
} else {
bit_idx--;
}
}
if (probe_point > boundary_idx) {
// Confirm that the last request, which fails to find a cluster, returns sentinel value: boundary_idx
probe_point = bm.find_last_consecutive_set_bits(boundary_idx, probe_point, cluster_size);
BitMapAssertEqual(probe_point, boundary_idx);
}
}
// Confirm that find_first_consecutive_set_bits() works for a cluster size known not to have any matches
probe_point = bm.find_first_consecutive_set_bits(0, longest_run + 1);
BitMapAssertEqual(probe_point, size); // Confirm: failed search returns sentinel: size
probe_point = bm.find_last_consecutive_set_bits(size - 1, longest_run + 1);
BitMapAssertEqual(probe_point, (ssize_t) -1); // Confirm: failed search returns sentinel: -1
boundary_idx = 3 * size / 4;
probe_point = bm.find_first_consecutive_set_bits(0, boundary_idx, longest_run + 1);
BitMapAssertEqual(probe_point, boundary_idx); // Confirm: failed search returns sentinel: boundary_idx
boundary_idx = size / 4;
probe_point = bm.find_last_consecutive_set_bits(boundary_idx, size - 1, longest_run + 1);
BitMapAssertEqual(probe_point, boundary_idx); // Confirm: failed search returns sentinel: boundary_idx
}
public:
static bool run_test() {
_success = false;
_assertion_failures = 0;
ShenandoahSimpleBitMap bm_small(SMALL_BITMAP_SIZE);
ShenandoahSimpleBitMap bm_large(LARGE_BITMAP_SIZE);
// Initial state of each bitmap is all bits are clear. Confirm this:
ssize_t set_bits_0[1] = { 0 };
verifyBitMapState(bm_small, SMALL_BITMAP_SIZE, set_bits_0, 0);
verifyBitMapState(bm_large, LARGE_BITMAP_SIZE, set_bits_0, 0);
bm_small.set_bit(5);
bm_small.set_bit(63);
bm_small.set_bit(128);
ssize_t set_bits_1[3] = { 5, 63, 128 };
verifyBitMapState(bm_small, SMALL_BITMAP_SIZE, set_bits_1, 3);
bm_large.set_bit(5);
bm_large.set_bit(63);
bm_large.set_bit(128);
verifyBitMapState(bm_large, LARGE_BITMAP_SIZE, set_bits_1, 3);
// Test some consecutive bits
bm_small.set_bit(140);
bm_small.set_bit(141);
bm_small.set_bit(142);
bm_small.set_bit(253);
bm_small.set_bit(254);
bm_small.set_bit(255);
bm_small.set_bit(271);
bm_small.set_bit(272);
bm_small.set_bit(320);
bm_small.set_bit(321);
bm_small.set_bit(322);
bm_small.set_bit(361);
ssize_t set_bits_2[15] = { 5, 63, 128, 140, 141, 142, 253, 254, 255, 271, 272, 320, 321, 322, 361 };
verifyBitMapState(bm_small, SMALL_BITMAP_SIZE, set_bits_2, 15);
bm_large.set_bit(140);
bm_large.set_bit(141);
bm_large.set_bit(142);
bm_large.set_bit(1021);
bm_large.set_bit(1022);
bm_large.set_bit(1023);
bm_large.set_bit(1051);
bm_large.set_bit(1280);
bm_large.set_bit(1281);
bm_large.set_bit(1282);
bm_large.set_bit(1300);
bm_large.set_bit(1301);
bm_large.set_bit(1302);
ssize_t set_bits_3[16] = { 5, 63, 128, 140, 141, 142, 1021, 1022, 1023, 1051, 1280, 1281, 1282, 1300, 1301, 1302 };
verifyBitMapState(bm_large, LARGE_BITMAP_SIZE, set_bits_3, 16);
// Test clear_bit
bm_small.clear_bit(141);
bm_small.clear_bit(253);
ssize_t set_bits_4[13] = { 5, 63, 128, 140, 142, 254, 255, 271, 272, 320, 321, 322, 361 };
verifyBitMapState(bm_small, SMALL_BITMAP_SIZE, set_bits_4, 13);
bm_large.clear_bit(5);
bm_large.clear_bit(63);
bm_large.clear_bit(128);
bm_large.clear_bit(141);
ssize_t set_bits_5[12] = { 140, 142, 1021, 1022, 1023, 1051, 1280, 1281, 1282, 1300, 1301, 1302 };
verifyBitMapState(bm_large, LARGE_BITMAP_SIZE, set_bits_5, 12);
// Look for large island of contiguous surrounded by smaller islands of contiguous
bm_large.set_bit(1024);
bm_large.set_bit(1025); // size-5 island from 1021 to 1025
bm_large.set_bit(1027);
bm_large.set_bit(1028);
bm_large.set_bit(1029);
bm_large.set_bit(1030);
bm_large.set_bit(1031);
bm_large.set_bit(1032); // size-6 island from 1027 to 1032
bm_large.set_bit(1034);
bm_large.set_bit(1035);
bm_large.set_bit(1036); // size-3 island from 1034 to 1036
ssize_t set_bits_6[23] = { 140, 142, 1021, 1022, 1023, 1024, 1025, 1027, 1028, 1029, 1030,
1031, 1032, 1034, 1035, 1036, 1051, 1280, 1281, 1282, 1300, 1301, 1302 };
verifyBitMapState(bm_large, LARGE_BITMAP_SIZE, set_bits_6, 23);
// Test that entire bitmap word (from 1024 to 1088) is 1's
ssize_t set_bits_7[76];
set_bits_7[0] = 140;
set_bits_7[1] = 142;
set_bits_7[2] = 1021;
set_bits_7[3] = 1022;
set_bits_7[4] = 1023;
size_t bit_idx = 5;
for (ssize_t i = 1024; i <= 1088; i++) {
bm_large.set_bit(i);
set_bits_7[bit_idx++] = i;
}
set_bits_7[bit_idx++] = 1280;
set_bits_7[bit_idx++] = 1281;
set_bits_7[bit_idx++] = 1282;
set_bits_7[bit_idx++] = 1300;
set_bits_7[bit_idx++] = 1301;
set_bits_7[bit_idx++] = 1302;
verifyBitMapState(bm_large, LARGE_BITMAP_SIZE, set_bits_7, bit_idx);
// Test clear_all()
bm_small.clear_all();
bm_large.clear_all();
verifyBitMapState(bm_small, SMALL_BITMAP_SIZE, set_bits_0, 0);
verifyBitMapState(bm_large, LARGE_BITMAP_SIZE, set_bits_0, 0);
_success = true;
return true;
}
};
TEST(BasicShenandoahSimpleBitMapTest, minimum_test) {
bool result = ShenandoahSimpleBitMapTest::run_test();
ASSERT_EQ(result, true);
ASSERT_EQ(_success, true);
ASSERT_EQ(_assertion_failures, (size_t) 0);
}