8157023: Integrate NMT with JFR

Reviewed-by: stuefe, mgronlun, egahlin
This commit is contained in:
Stefan Johansson 2022-12-07 18:43:42 +00:00
parent e86f31b5e7
commit 3b8c7ef8e7
14 changed files with 582 additions and 8 deletions

View File

@ -720,6 +720,19 @@
<Field type="OldObjectGcRoot" name="root" label="GC Root" />
</Event>
<Event name="NativeMemoryUsage" category="Java Virtual Machine, Memory" label="Native Memory Usage Per Type"
description="Native memory usage for a given memory type in the JVM" period="everyChunk">
<Field type="string" name="type" label="Memory Type" description="Type used for the native memory allocation" />
<Field type="ulong" contentType="bytes" name="reserved" label="Reserved Memory" description="Reserved bytes for this type" />
<Field type="ulong" contentType="bytes" name="committed" label="Committed Memory" description="Committed bytes for this type" />
</Event>
<Event name="NativeMemoryUsageTotal" category="Java Virtual Machine, Memory" label="Total Native Memory Usage"
description="Total native memory usage for the JVM. Might not be the exact sum of the NativeMemoryUsage events due to timeing." period="everyChunk">
<Field type="ulong" contentType="bytes" name="reserved" label="Reserved Memory" description="Total amount of reserved bytes for the JVM" />
<Field type="ulong" contentType="bytes" name="committed" label="Committed Memory" description="Total amount of committed bytes for the JVM" />
</Event>
<Event name="DumpReason" category="Flight Recorder" label="Recording Reason"
description="Who requested the recording and why"
startTime="false">

View File

@ -63,6 +63,7 @@
#include "runtime/vm_version.hpp"
#include "services/classLoadingService.hpp"
#include "services/management.hpp"
#include "services/memJfrReporter.hpp"
#include "services/threadService.hpp"
#include "utilities/exceptions.hpp"
#include "utilities/globalDefinitions.hpp"
@ -624,3 +625,11 @@ TRACE_REQUEST_FUNC(FinalizerStatistics) {
log_debug(jfr, system)("Unable to generate requestable event FinalizerStatistics. The required jvm feature 'management' is missing.");
#endif
}
TRACE_REQUEST_FUNC(NativeMemoryUsage) {
MemJFRReporter::send_type_events();
}
TRACE_REQUEST_FUNC(NativeMemoryUsageTotal) {
MemJFRReporter::send_total_event();
}

View File

@ -153,7 +153,12 @@ class MallocMemorySnapshot : public ResourceObj {
public:
inline MallocMemory* by_type(MEMFLAGS flags) {
inline MallocMemory* by_type(MEMFLAGS flags) {
int index = NMTUtil::flag_to_index(flags);
return &_malloc[index];
}
inline const MallocMemory* by_type(MEMFLAGS flags) const {
int index = NMTUtil::flag_to_index(flags);
return &_malloc[index];
}

View File

@ -0,0 +1,113 @@
/*
* Copyright (c) 2022, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License version 2 only, as
* published by the Free Software Foundation.
*
* This code is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
* version 2 for more details (a copy is included in the LICENSE file that
* accompanied this code).
*
* You should have received a copy of the GNU General Public License version
* 2 along with this work; if not, write to the Free Software Foundation,
* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
*
* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
* or visit www.oracle.com if you need additional information or have any
* questions.
*
*/
#include "precompiled.hpp"
#include "jfr/jfrEvents.hpp"
#include "services/memJfrReporter.hpp"
#include "services/memTracker.hpp"
#include "services/nmtUsage.hpp"
#include "utilities/globalDefinitions.hpp"
#include "utilities/ticks.hpp"
// Helper class to avoid refreshing the NMTUsage to often and allow
// the two JFR events to use the same data.
class MemJFRCurrentUsage : public AllStatic {
private:
// The age threshold in milliseconds. If older than this refresh the usage.
static const uint64_t AgeThreshold = 50;
static Ticks _timestamp;
static NMTUsage* _usage;
public:
static NMTUsage* get_usage();
static Ticks get_timestamp();
};
Ticks MemJFRCurrentUsage::_timestamp;
NMTUsage* MemJFRCurrentUsage::_usage = nullptr;
NMTUsage* MemJFRCurrentUsage::get_usage() {
Tickspan since_baselined = Ticks::now() - _timestamp;
if (_usage == nullptr) {
// First time, create a new NMTUsage.
_usage = new NMTUsage(NMTUsage::OptionsNoTS);
} else if (since_baselined.milliseconds() < AgeThreshold) {
// There is recent enough usage information, return it.
return _usage;
}
// Refresh the usage information.
_usage->refresh();
_timestamp.stamp();
return _usage;
}
Ticks MemJFRCurrentUsage::get_timestamp() {
return _timestamp;
}
void MemJFRReporter::send_total_event() {
if (!MemTracker::enabled()) {
return;
}
NMTUsage* usage = MemJFRCurrentUsage::get_usage();
Ticks timestamp = MemJFRCurrentUsage::get_timestamp();
EventNativeMemoryUsageTotal event(UNTIMED);
event.set_starttime(timestamp);
event.set_reserved(usage->total_reserved());
event.set_committed(usage->total_committed());
event.commit();
}
void MemJFRReporter::send_type_event(const Ticks& starttime, const char* type, size_t reserved, size_t committed) {
EventNativeMemoryUsage event(UNTIMED);
event.set_starttime(starttime);
event.set_type(type);
event.set_reserved(reserved);
event.set_committed(committed);
event.commit();
}
void MemJFRReporter::send_type_events() {
if (!MemTracker::enabled()) {
return;
}
NMTUsage* usage = MemJFRCurrentUsage::get_usage();
Ticks timestamp = MemJFRCurrentUsage::get_timestamp();
for (int index = 0; index < mt_number_of_types; index ++) {
MEMFLAGS flag = NMTUtil::index_to_flag(index);
if (flag == mtNone) {
// Skip mtNone since it is not really used.
continue;
}
send_type_event(timestamp, NMTUtil::flag_to_name(flag), usage->reserved(flag), usage->committed(flag));
}
}

View File

@ -0,0 +1,44 @@
/*
* Copyright (c) 2022, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License version 2 only, as
* published by the Free Software Foundation.
*
* This code is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
* version 2 for more details (a copy is included in the LICENSE file that
* accompanied this code).
*
* You should have received a copy of the GNU General Public License version
* 2 along with this work; if not, write to the Free Software Foundation,
* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
*
* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
* or visit www.oracle.com if you need additional information or have any
* questions.
*
*/
#ifndef SHARE_SERVICES_MEMJFRREPORTER_HPP
#define SHARE_SERVICES_MEMJFRREPORTER_HPP
#include "memory/allocation.hpp"
#include "services/nmtUsage.hpp"
#include "utilities/globalDefinitions.hpp"
#include "utilities/ticks.hpp"
// MemJFRReporter is only used by threads sending periodic JFR
// events. These threads are synchronized at a higher level,
// so no more synchronization is needed.
class MemJFRReporter : public AllStatic {
private:
static void send_type_event(const Ticks& starttime, const char* tag, size_t reserved, size_t committed);
public:
static void send_total_event();
static void send_type_events();
};
#endif //SHARE_SERVICES_MEMJFRREPORTER_HPP

View File

@ -31,11 +31,11 @@
#include "services/virtualMemoryTracker.hpp"
#include "utilities/globalDefinitions.hpp"
size_t MemReporterBase::reserved_total(const MallocMemory* malloc, const VirtualMemory* vm) const {
size_t MemReporterBase::reserved_total(const MallocMemory* malloc, const VirtualMemory* vm) {
return malloc->malloc_size() + malloc->arena_size() + vm->reserved();
}
size_t MemReporterBase::committed_total(const MallocMemory* malloc, const VirtualMemory* vm) const {
size_t MemReporterBase::committed_total(const MallocMemory* malloc, const VirtualMemory* vm) {
return malloc->malloc_size() + malloc->arena_size() + vm->committed();
}

View File

@ -49,6 +49,11 @@ class MemReporterBase : public StackObj {
_scale(scale), _output(out)
{}
// Helper functions
// Calculate total reserved and committed amount
static size_t reserved_total(const MallocMemory* malloc, const VirtualMemory* vm);
static size_t committed_total(const MallocMemory* malloc, const VirtualMemory* vm);
protected:
inline outputStream* output() const {
return _output;
@ -73,11 +78,6 @@ class MemReporterBase : public StackObj {
return amount / scale;
}
// Helper functions
// Calculate total reserved and committed amount
size_t reserved_total(const MallocMemory* malloc, const VirtualMemory* vm) const;
size_t committed_total(const MallocMemory* malloc, const VirtualMemory* vm) const;
// Print summary total, malloc and virtual memory
void print_total(size_t reserved, size_t committed) const;
void print_malloc(const MemoryCounter* c, MEMFLAGS flag = mtNone) const;

View File

@ -0,0 +1,129 @@
/*
* Copyright (c) 2022, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License version 2 only, as
* published by the Free Software Foundation.
*
* This code is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
* version 2 for more details (a copy is included in the LICENSE file that
* accompanied this code).
*
* You should have received a copy of the GNU General Public License version
* 2 along with this work; if not, write to the Free Software Foundation,
* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
*
* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
* or visit www.oracle.com if you need additional information or have any
* questions.
*
*/
#include "precompiled.hpp"
#include "runtime/threadCritical.hpp"
#include "services/nmtCommon.hpp"
#include "services/nmtUsage.hpp"
#include "services/mallocTracker.hpp"
#include "services/threadStackTracker.hpp"
#include "services/virtualMemoryTracker.hpp"
// Enabled all options for snapshot.
const NMTUsageOptions NMTUsage::OptionsAll = { true, true, true };
// Skip expensive thread stacks when refreshing usage.
const NMTUsageOptions NMTUsage::OptionsNoTS = { false, true, true };
NMTUsage::NMTUsage(NMTUsageOptions options) :
_malloc_by_type(),
_malloc_total(),
_vm_by_type(),
_vm_total(),
_usage_options(options) { }
void NMTUsage::walk_thread_stacks() {
// If backed by virtual memory, snapping the thread stacks involves walking
// them to to figure out how much memory is committed if they are backed by
// virtual memory. This needs ot happen before we take the snapshot of the
// virtual memory since it will update this information.
if (ThreadStackTracker::track_as_vm()) {
VirtualMemoryTracker::snapshot_thread_stacks();
}
}
void NMTUsage::update_malloc_usage() {
// Thread critical needed keep values in sync, total area size
// is deducted from mtChunk in the end to give correct values.
ThreadCritical tc;
const MallocMemorySnapshot* ms = MallocMemorySummary::as_snapshot();
size_t total_arena_size = 0;
for (int i = 0; i < mt_number_of_types; i++) {
MEMFLAGS flag = NMTUtil::index_to_flag(i);
const MallocMemory* mm = ms->by_type(flag);
_malloc_by_type[i] = mm->malloc_size() + mm->arena_size();
total_arena_size += mm->arena_size();
}
// Total malloc size.
_malloc_total = ms->total();
// Adjustment due to mtChunk double counting.
_malloc_by_type[NMTUtil::flag_to_index(mtChunk)] -= total_arena_size;
_malloc_total -= total_arena_size;
// Adjust mtNMT to include malloc overhead.
_malloc_by_type[NMTUtil::flag_to_index(mtNMT)] += ms->malloc_overhead();
}
void NMTUsage::update_vm_usage() {
const VirtualMemorySnapshot* vms = VirtualMemorySummary::as_snapshot();
// Reset total to allow recalculation.
_vm_total.committed = 0;
_vm_total.reserved = 0;
for (int i = 0; i < mt_number_of_types; i++) {
MEMFLAGS flag = NMTUtil::index_to_flag(i);
const VirtualMemory* vm = vms->by_type(flag);
_vm_by_type[i].reserved = vm->reserved();
_vm_by_type[i].committed = vm->committed();
_vm_total.reserved += vm->reserved();
_vm_total.committed += vm->committed();
}
}
void NMTUsage::refresh() {
if (_usage_options.include_malloc) {
update_malloc_usage();
}
if (_usage_options.include_vm) {
// Thread stacks only makes sense if virtual memory
// is also included. It must be executed before the
// over all usage is calculated.
if (_usage_options.update_thread_stacks) {
walk_thread_stacks();
}
update_vm_usage();
}
}
size_t NMTUsage::total_reserved() const {
return _malloc_total + _vm_total.reserved;
}
size_t NMTUsage::total_committed() const {
return _malloc_total + _vm_total.reserved;
}
size_t NMTUsage::reserved(MEMFLAGS flag) const {
int index = NMTUtil::flag_to_index(flag);
return _malloc_by_type[index] + _vm_by_type[index].reserved;
}
size_t NMTUsage::committed(MEMFLAGS flag) const {
int index = NMTUtil::flag_to_index(flag);
return _malloc_by_type[index] + _vm_by_type[index].committed;
}

View File

@ -0,0 +1,68 @@
/*
* Copyright (c) 2022, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License version 2 only, as
* published by the Free Software Foundation.
*
* This code is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
* version 2 for more details (a copy is included in the LICENSE file that
* accompanied this code).
*
* You should have received a copy of the GNU General Public License version
* 2 along with this work; if not, write to the Free Software Foundation,
* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
*
* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
* or visit www.oracle.com if you need additional information or have any
* questions.
*
*/
#ifndef SHARE_SERVICES_NMTUSAGE_HPP
#define SHARE_SERVICES_NMTUSAGE_HPP
#include "memory/allocation.hpp"
#include "utilities/globalDefinitions.hpp"
struct NMTUsagePair {
size_t reserved;
size_t committed;
};
struct NMTUsageOptions {
bool update_thread_stacks;
bool include_malloc;
bool include_vm;
};
class NMTUsage : public CHeapObj<mtNMT> {
private:
size_t _malloc_by_type[mt_number_of_types];
size_t _malloc_total;
NMTUsagePair _vm_by_type[mt_number_of_types];
NMTUsagePair _vm_total;
NMTUsageOptions _usage_options;
void walk_thread_stacks();
void update_malloc_usage();
void update_vm_usage();
public:
static const NMTUsageOptions OptionsAll;
static const NMTUsageOptions OptionsNoTS;
NMTUsage(NMTUsageOptions options = OptionsAll);
void refresh();
size_t total_reserved() const;
size_t total_committed() const;
size_t reserved(MEMFLAGS flag) const;
size_t committed(MEMFLAGS flag) const;
};
#endif // SHARE_SERVICES_NMTUSAGE_HPP

View File

@ -97,6 +97,11 @@ class VirtualMemorySnapshot : public ResourceObj {
return &_virtual_memory[index];
}
inline const VirtualMemory* by_type(MEMFLAGS flag) const {
int index = NMTUtil::flag_to_index(flag);
return &_virtual_memory[index];
}
inline size_t total_reserved() const {
size_t amount = 0;
for (int index = 0; index < mt_number_of_types; index ++) {

View File

@ -512,6 +512,16 @@
<setting name="cutoff" control="old-objects-cutoff">0 ns</setting>
</event>
<event name="jdk.NativeMemoryUsage">
<setting name="enabled" control="gc-enabled-normal">true</setting>
<setting name="period">1000 ms</setting>
</event>
<event name="jdk.NativeMemoryUsageTotal">
<setting name="enabled" control="gc-enabled-normal">true</setting>
<setting name="period">1000 ms</setting>
</event>
<event name="jdk.CompilerConfiguration">
<setting name="enabled" control="compiler-enabled">true</setting>
<setting name="period">beginChunk</setting>

View File

@ -512,6 +512,16 @@
<setting name="cutoff" control="old-objects-cutoff">0 ns</setting>
</event>
<event name="jdk.NativeMemoryUsage">
<setting name="enabled" control="gc-enabled-normal">true</setting>
<setting name="period">1000 ms</setting>
</event>
<event name="jdk.NativeMemoryUsageTotal">
<setting name="enabled" control="gc-enabled-normal">true</setting>
<setting name="period">1000 ms</setting>
</event>
<event name="jdk.CompilerConfiguration">
<setting name="enabled" control="compiler-enabled">true</setting>
<setting name="period">beginChunk</setting>

View File

@ -0,0 +1,166 @@
/*
* Copyright (c) 2022, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License version 2 only, as
* published by the Free Software Foundation.
*
* This code is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
* version 2 for more details (a copy is included in the LICENSE file that
* accompanied this code).
*
* You should have received a copy of the GNU General Public License version
* 2 along with this work; if not, write to the Free Software Foundation,
* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
*
* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
* or visit www.oracle.com if you need additional information or have any
* questions.
*/
package jdk.jfr.event.runtime;
import static jdk.test.lib.Asserts.assertGreaterThan;
import static jdk.test.lib.Asserts.assertTrue;
import java.util.ArrayList;
import java.util.List;
import jdk.jfr.Recording;
import jdk.jfr.consumer.RecordedEvent;
import jdk.test.lib.jfr.EventNames;
import jdk.test.lib.jfr.Events;
/**
* @test
* @key jfr
* @requires vm.opt.NativeMemoryTracking == null
* @requires vm.hasJFR
* @library /test/lib
* @modules jdk.jfr
* jdk.management
* @run main/othervm -XX:NativeMemoryTracking=summary -Xms16m -Xmx128m -Xlog:gc jdk.jfr.event.runtime.TestNativeMemoryUsageEvents true
* @run main/othervm -XX:NativeMemoryTracking=off -Xms16m -Xmx128m -Xlog:gc jdk.jfr.event.runtime.TestNativeMemoryUsageEvents false
*/
public class TestNativeMemoryUsageEvents {
private final static String UsageTotalEvent = EventNames.NativeMemoryUsageTotal;
private final static String UsageEvent = EventNames.NativeMemoryUsage;
private final static int UsagePeriod = 1000;
private final static int K = 1024;
private final static String[] UsageEventTypes = {
"Java Heap",
"Class",
"Thread",
"Thread Stack",
"Code",
"GC",
"GCCardSet",
"Compiler",
"JVMCI",
"Internal",
"Other",
"Symbol",
"Native Memory Tracking",
"Shared class space",
"Arena Chunk",
"Test",
"Tracing",
"Logging",
"Statistics",
"Arguments",
"Module",
"Safepoint",
"Synchronization",
"Serviceability",
"Metaspace",
"String Deduplication",
"Object Monitors"
};
private static ArrayList<byte[]> data = new ArrayList<byte[]>();
private static void generateHeapContents() {
for (int i = 0 ; i < 64; i++) {
for (int j = 0; j < K; j++) {
data.add(new byte[K]);
}
}
}
private static void generateEvents(Recording recording) throws Exception {
// Enable the two types of events for "everyChunk", it will give
// an event at the beginning of the chunk as well as at the end.
recording.enable(UsageEvent).with("period", "everyChunk");
recording.enable(UsageTotalEvent).with("period", "everyChunk");
recording.start();
// Generate data to force heap to grow.
generateHeapContents();
recording.stop();
}
private static void verifyExpectedEventTypes(List<RecordedEvent> events) throws Exception {
// First verify that the number of total usage events is greater than 0.
long numberOfTotal = events.stream()
.filter(e -> e.getEventType().getName().equals(UsageTotalEvent))
.count();
assertGreaterThan(numberOfTotal, 0L, "Should exist events of type: " + UsageTotalEvent);
// Now verify that we got the expected events.
List<String> uniqueEventTypes = events.stream()
.filter(e -> e.getEventType().getName().equals(UsageEvent))
.map(e -> e.getString("type"))
.distinct()
.toList();
for (String type : UsageEventTypes) {
assertTrue(uniqueEventTypes.contains(type), "Events should include: " + type);
}
}
private static void verifyHeapGrowth(List<RecordedEvent> events) throws Exception {
List<Long> javaHeapCommitted = events.stream()
.filter(e -> e.getEventType().getName().equals(UsageEvent))
.filter(e -> e.getString("type").equals("Java Heap"))
.map(e -> e.getLong("committed"))
.toList();
// Verify that the heap has grown between the first and last sample.
long firstSample = javaHeapCommitted.get(0);
long lastSample = javaHeapCommitted.get(javaHeapCommitted.size() - 1);
assertGreaterThan(lastSample, firstSample, "heap should have grown and NMT should show that");
}
private static void verifyNoUsageEvents(List<RecordedEvent> events) throws Exception {
Events.hasNotEvent(events, UsageEvent);
Events.hasNotEvent(events, UsageTotalEvent);
}
public static void main(String[] args) throws Exception {
// The tests takes a single boolean argument that states wether or not
// it is run with -XX:NativeMemoryTracking=summary. When tracking is
// enabled the tests verifies that the correct events are sent and
// the other way around when turned off.
assertTrue(args.length == 1, "Must have a single argument");
boolean nmtEnabled = Boolean.parseBoolean(args[0]);
try (Recording recording = new Recording()) {
generateEvents(recording);
var events = Events.fromRecording(recording);
if (nmtEnabled) {
verifyExpectedEventTypes(events);
verifyHeapGrowth(events);
} else {
verifyNoUsageEvents(events);
}
}
}
}

View File

@ -86,6 +86,8 @@ public class EventNames {
public static final String RetransformClasses = PREFIX + "RetransformClasses";
public static final String ClassRedefinition = PREFIX + "ClassRedefinition";
public static final String FinalizerStatistics = PREFIX + "FinalizerStatistics";
public static final String NativeMemoryUsage = PREFIX + "NativeMemoryUsage";
public static final String NativeMemoryUsageTotal = PREFIX + "NativeMemoryUsageTotal";
// This event is hard to test
public static final String ReservedStackActivation = PREFIX + "ReservedStackActivation";