jdk-24/src/hotspot/share/gc/shenandoah/shenandoahCodeRoots.cpp

/*
 * Copyright (c) 2017, 2019, Red Hat, Inc. All rights reserved.
 *
 * This code is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License version 2 only, as
 * published by the Free Software Foundation.
 *
 * This code is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * version 2 for more details (a copy is included in the LICENSE file that
 * accompanied this code).
 *
 * You should have received a copy of the GNU General Public License version
 * 2 along with this work; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 *
 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
 * or visit www.oracle.com if you need additional information or have any
 * questions.
 *
 */

#include "precompiled.hpp"
#include "code/codeCache.hpp"
#include "code/nmethod.hpp"
#include "gc/shenandoah/shenandoahHeap.inline.hpp"
#include "gc/shenandoah/shenandoahCodeRoots.hpp"
#include "memory/resourceArea.hpp"

ShenandoahParallelCodeCacheIterator::ShenandoahParallelCodeCacheIterator(const GrowableArray<CodeHeap*>* heaps) {
  _length = heaps->length();
  _iters = NEW_C_HEAP_ARRAY(ShenandoahParallelCodeHeapIterator, _length, mtGC);
  for (int h = 0; h < _length; h++) {
    _iters[h] = ShenandoahParallelCodeHeapIterator(heaps->at(h));
  }
}

ShenandoahParallelCodeCacheIterator::~ShenandoahParallelCodeCacheIterator() {
  FREE_C_HEAP_ARRAY(ParallelCodeHeapIterator, _iters);
}

void ShenandoahParallelCodeCacheIterator::parallel_blobs_do(CodeBlobClosure* f) {
  for (int c = 0; c < _length; c++) {
    _iters[c].parallel_blobs_do(f);
  }
}

ShenandoahParallelCodeHeapIterator::ShenandoahParallelCodeHeapIterator(CodeHeap* heap) :
        _heap(heap), _claimed_idx(0), _finished(false) {
}

void ShenandoahParallelCodeHeapIterator::parallel_blobs_do(CodeBlobClosure* f) {
  assert(SafepointSynchronize::is_at_safepoint(), "Must be at safepoint");

  /*
   * Parallel code heap walk.
   *
   * This code makes all threads scan all code heaps, but only one thread would execute the
   * closure on given blob. This is achieved by recording the "claimed" blocks: if a thread
   * had claimed the block, it can process all blobs in it. Others have to fast-forward to
   * next attempt without processing.
   *
   * Late threads would return immediately if iterator is finished.
   */

  if (_finished) {
    return;
  }

  int stride = 256; // educated guess
  int stride_mask = stride - 1;
  assert (is_power_of_2(stride), "sanity");

  int count = 0;
  bool process_block = true;

  for (CodeBlob *cb = CodeCache::first_blob(_heap); cb != NULL; cb = CodeCache::next_blob(_heap, cb)) {
    int current = count++;
    if ((current & stride_mask) == 0) {
      process_block = (current >= _claimed_idx) &&
                      (Atomic::cmpxchg(current + stride, &_claimed_idx, current) == current);
    }
    if (process_block) {
      if (cb->is_alive()) {
        f->do_code_blob(cb);
#ifdef ASSERT
        if (cb->is_nmethod())
          Universe::heap()->verify_nmethod((nmethod*)cb);
#endif
      }
    }
  }

  _finished = true;
}

class ShenandoahNMethodOopDetector : public OopClosure {
private:
  ResourceMark rm; // For growable array allocation below.
  GrowableArray<oop*> _oops;

public:
  ShenandoahNMethodOopDetector() : _oops(10) {};

  void do_oop(oop* o) {
    _oops.append(o);
  }
  void do_oop(narrowOop* o) {
    fatal("NMethods should not have compressed oops embedded.");
  }

  GrowableArray<oop*>* oops() {
    return &_oops;
  }

  bool has_oops() {
    return !_oops.is_empty();
  }
};

class ShenandoahNMethodOopInitializer : public OopClosure {
private:
  ShenandoahHeap* const _heap;

public:
  ShenandoahNMethodOopInitializer() : _heap(ShenandoahHeap::heap()) {};

private:
  template <class T>
  inline void do_oop_work(T* p) {
    T o = RawAccess<>::oop_load(p);
    if (! CompressedOops::is_null(o)) {
      oop obj1 = CompressedOops::decode_not_null(o);
      oop obj2 = ShenandoahBarrierSet::barrier_set()->write_barrier(obj1);
      if (! oopDesc::equals_raw(obj1, obj2)) {
        shenandoah_assert_not_in_cset(NULL, obj2);
        RawAccess<IS_NOT_NULL>::oop_store(p, obj2);
        if (_heap->is_concurrent_traversal_in_progress()) {
          ShenandoahBarrierSet::barrier_set()->enqueue(obj2);
        }
      }
    }
  }

public:
  void do_oop(oop* o) {
    do_oop_work(o);
  }
  void do_oop(narrowOop* o) {
    do_oop_work(o);
  }
};

ShenandoahCodeRoots::PaddedLock ShenandoahCodeRoots::_recorded_nms_lock;
GrowableArray<ShenandoahNMethod*>* ShenandoahCodeRoots::_recorded_nms;

void ShenandoahCodeRoots::initialize() {
  _recorded_nms_lock._lock = 0;
  _recorded_nms = new (ResourceObj::C_HEAP, mtGC) GrowableArray<ShenandoahNMethod*>(100, true, mtGC);
}

void ShenandoahCodeRoots::add_nmethod(nmethod* nm) {
  switch (ShenandoahCodeRootsStyle) {
    case 0:
    case 1: {
      ShenandoahNMethodOopInitializer init;
      nm->oops_do(&init);
      nm->fix_oop_relocations();
      break;
    }
    case 2: {
      ShenandoahNMethodOopDetector detector;
      nm->oops_do(&detector);

      if (detector.has_oops()) {
        ShenandoahNMethodOopInitializer init;
        nm->oops_do(&init);
        nm->fix_oop_relocations();

        ShenandoahNMethod* nmr = new ShenandoahNMethod(nm, detector.oops());
        nmr->assert_alive_and_correct();

        ShenandoahCodeRootsLock lock(true);

        int idx = _recorded_nms->find(nm, ShenandoahNMethod::find_with_nmethod);
        if (idx != -1) {
          ShenandoahNMethod* old = _recorded_nms->at(idx);
          _recorded_nms->at_put(idx, nmr);
          delete old;
        } else {
          _recorded_nms->append(nmr);
        }
      }
      break;
    }
    default:
      ShouldNotReachHere();
  }
};

void ShenandoahCodeRoots::remove_nmethod(nmethod* nm) {
  switch (ShenandoahCodeRootsStyle) {
    case 0:
    case 1: {
      break;
    }
    case 2: {
      ShenandoahNMethodOopDetector detector;
      nm->oops_do(&detector, /* allow_zombie = */ true);

      if (detector.has_oops()) {
        ShenandoahCodeRootsLock lock(true);

        int idx = _recorded_nms->find(nm, ShenandoahNMethod::find_with_nmethod);
        assert(idx != -1, "nmethod " PTR_FORMAT " should be registered", p2i(nm));
        ShenandoahNMethod* old = _recorded_nms->at(idx);
        old->assert_same_oops(detector.oops());
        _recorded_nms->delete_at(idx);
        delete old;
      }
      break;
    }
    default:
      ShouldNotReachHere();
  }
}

ShenandoahCodeRootsIterator::ShenandoahCodeRootsIterator() :
        _heap(ShenandoahHeap::heap()),
        _par_iterator(CodeCache::heaps()),
        _claimed(0) {
  assert(SafepointSynchronize::is_at_safepoint(), "Must be at safepoint");
  assert(!Thread::current()->is_Worker_thread(), "Should not be acquired by workers");
  switch (ShenandoahCodeRootsStyle) {
    case 0:
    case 1: {
      // No need to do anything here
      break;
    }
    case 2: {
      ShenandoahCodeRoots::acquire_lock(false);
      break;
    }
    default:
      ShouldNotReachHere();
  }
}

ShenandoahCodeRootsIterator::~ShenandoahCodeRootsIterator() {
  switch (ShenandoahCodeRootsStyle) {
    case 0:
    case 1: {
      // No need to do anything here
      break;
    }
    case 2: {
      ShenandoahCodeRoots::release_lock(false);
      break;
    }
    default:
      ShouldNotReachHere();
  }
}

template<bool CSET_FILTER>
void ShenandoahCodeRootsIterator::dispatch_parallel_blobs_do(CodeBlobClosure *f) {
  switch (ShenandoahCodeRootsStyle) {
    case 0: {
      if (_seq_claimed.try_set()) {
        CodeCache::blobs_do(f);
      }
      break;
    }
    case 1: {
      _par_iterator.parallel_blobs_do(f);
      break;
    }
    case 2: {
      ShenandoahCodeRootsIterator::fast_parallel_blobs_do<CSET_FILTER>(f);
      break;
    }
    default:
      ShouldNotReachHere();
  }
}

ShenandoahAllCodeRootsIterator ShenandoahCodeRoots::iterator() {
  return ShenandoahAllCodeRootsIterator();
}

ShenandoahCsetCodeRootsIterator ShenandoahCodeRoots::cset_iterator() {
  return ShenandoahCsetCodeRootsIterator();
}

void ShenandoahAllCodeRootsIterator::possibly_parallel_blobs_do(CodeBlobClosure *f) {
  ShenandoahCodeRootsIterator::dispatch_parallel_blobs_do<false>(f);
}

void ShenandoahCsetCodeRootsIterator::possibly_parallel_blobs_do(CodeBlobClosure *f) {
  ShenandoahCodeRootsIterator::dispatch_parallel_blobs_do<true>(f);
}

template <bool CSET_FILTER>
void ShenandoahCodeRootsIterator::fast_parallel_blobs_do(CodeBlobClosure *f) {
  assert(SafepointSynchronize::is_at_safepoint(), "Must be at safepoint");

  size_t stride = 256; // educated guess

  GrowableArray<ShenandoahNMethod*>* list = ShenandoahCodeRoots::_recorded_nms;

  size_t max = (size_t)list->length();
  while (_claimed < max) {
    size_t cur = Atomic::add(stride, &_claimed) - stride;
    size_t start = cur;
    size_t end = MIN2(cur + stride, max);
    if (start >= max) break;

    for (size_t idx = start; idx < end; idx++) {
      ShenandoahNMethod* nmr = list->at((int) idx);
      nmr->assert_alive_and_correct();

      if (CSET_FILTER && !nmr->has_cset_oops(_heap)) {
        continue;
      }

      f->do_code_blob(nmr->nm());
    }
  }
}

ShenandoahNMethod::ShenandoahNMethod(nmethod* nm, GrowableArray<oop*>* oops) {
  _nm = nm;
  _oops = NEW_C_HEAP_ARRAY(oop*, oops->length(), mtGC);
  _oops_count = oops->length();
  for (int c = 0; c < _oops_count; c++) {
    _oops[c] = oops->at(c);
  }
}

ShenandoahNMethod::~ShenandoahNMethod() {
  if (_oops != NULL) {
    FREE_C_HEAP_ARRAY(oop*, _oops);
  }
}

bool ShenandoahNMethod::has_cset_oops(ShenandoahHeap *heap) {
  for (int c = 0; c < _oops_count; c++) {
    oop o = RawAccess<>::oop_load(_oops[c]);
    if (heap->in_collection_set(o)) {
      return true;
    }
  }
  return false;
}

#ifdef ASSERT
void ShenandoahNMethod::assert_alive_and_correct() {
  assert(_nm->is_alive(), "only alive nmethods here");
  assert(_oops_count > 0, "should have filtered nmethods without oops before");
  ShenandoahHeap* heap = ShenandoahHeap::heap();
  for (int c = 0; c < _oops_count; c++) {
    oop *loc = _oops[c];
    assert(_nm->code_contains((address) loc) || _nm->oops_contains(loc), "nmethod should contain the oop*");
    oop o = RawAccess<>::oop_load(loc);
    shenandoah_assert_correct_except(loc, o, o == NULL || heap->is_full_gc_move_in_progress());
  }
}

void ShenandoahNMethod::assert_same_oops(GrowableArray<oop*>* oops) {
  assert(_oops_count == oops->length(), "should have the same number of oop*");
  for (int c = 0; c < _oops_count; c++) {
    assert(_oops[c] == oops->at(c), "should be the same oop*");
  }
}
#endif