8237649: ZGC: Improved NUMA support when using small pages
Reviewed-by: eosterlund, smonteith
This commit is contained in:
parent
06456a9773
commit
91d58b8f96
src/hotspot
@ -27,6 +27,7 @@
|
||||
#include "gc/z/zGlobals.hpp"
|
||||
#include "gc/z/zLargePages.inline.hpp"
|
||||
#include "gc/z/zMountPoint_linux.hpp"
|
||||
#include "gc/z/zNUMA.inline.hpp"
|
||||
#include "gc/z/zPhysicalMemoryBacking_linux.hpp"
|
||||
#include "gc/z/zSyscall_linux.hpp"
|
||||
#include "logging/log.hpp"
|
||||
@ -34,6 +35,7 @@
|
||||
#include "runtime/os.hpp"
|
||||
#include "utilities/align.hpp"
|
||||
#include "utilities/debug.hpp"
|
||||
#include "utilities/growableArray.hpp"
|
||||
|
||||
#include <fcntl.h>
|
||||
#include <stdio.h>
|
||||
@ -596,7 +598,38 @@ retry:
|
||||
return true;
|
||||
}
|
||||
|
||||
size_t ZPhysicalMemoryBacking::commit(size_t offset, size_t length) {
|
||||
static int offset_to_node(size_t offset) {
|
||||
const GrowableArray<int>* mapping = os::Linux::numa_nindex_to_node();
|
||||
const size_t nindex = (offset >> ZGranuleSizeShift) % mapping->length();
|
||||
return mapping->at((int)nindex);
|
||||
}
|
||||
|
||||
size_t ZPhysicalMemoryBacking::commit_numa_interleaved(size_t offset, size_t length) {
|
||||
size_t committed = 0;
|
||||
|
||||
// Commit one granule at a time, so that each granule
|
||||
// can be allocated from a different preferred node.
|
||||
while (committed < length) {
|
||||
const size_t granule_offset = offset + committed;
|
||||
|
||||
// Setup NUMA policy to allocate memory from a preferred node
|
||||
os::Linux::numa_set_preferred(offset_to_node(granule_offset));
|
||||
|
||||
if (!commit_inner(granule_offset, ZGranuleSize)) {
|
||||
// Failed
|
||||
break;
|
||||
}
|
||||
|
||||
committed += ZGranuleSize;
|
||||
}
|
||||
|
||||
// Restore NUMA policy
|
||||
os::Linux::numa_set_preferred(-1);
|
||||
|
||||
return committed;
|
||||
}
|
||||
|
||||
size_t ZPhysicalMemoryBacking::commit_default(size_t offset, size_t length) {
|
||||
// Try to commit the whole region
|
||||
if (commit_inner(offset, length)) {
|
||||
// Success
|
||||
@ -624,6 +657,16 @@ size_t ZPhysicalMemoryBacking::commit(size_t offset, size_t length) {
|
||||
}
|
||||
}
|
||||
|
||||
size_t ZPhysicalMemoryBacking::commit(size_t offset, size_t length) {
|
||||
if (ZNUMA::is_enabled() && !ZLargePages::is_explicit()) {
|
||||
// To get granule-level NUMA interleaving when using non-large pages,
|
||||
// we must explicitly interleave the memory at commit/fallocate time.
|
||||
return commit_numa_interleaved(offset, length);
|
||||
}
|
||||
|
||||
return commit_default(offset, length);
|
||||
}
|
||||
|
||||
size_t ZPhysicalMemoryBacking::uncommit(size_t offset, size_t length) {
|
||||
log_trace(gc, heap)("Uncommitting memory: " SIZE_FORMAT "M-" SIZE_FORMAT "M (" SIZE_FORMAT "M)",
|
||||
offset / M, (offset + length) / M, length / M);
|
||||
|
@ -57,6 +57,8 @@ private:
|
||||
ZErrno fallocate(bool punch_hole, size_t offset, size_t length);
|
||||
|
||||
bool commit_inner(size_t offset, size_t length);
|
||||
size_t commit_numa_interleaved(size_t offset, size_t length);
|
||||
size_t commit_default(size_t offset, size_t length);
|
||||
|
||||
public:
|
||||
ZPhysicalMemoryBacking();
|
||||
|
@ -3163,6 +3163,8 @@ bool os::Linux::libnuma_init() {
|
||||
libnuma_v2_dlsym(handle, "numa_get_interleave_mask")));
|
||||
set_numa_move_pages(CAST_TO_FN_PTR(numa_move_pages_func_t,
|
||||
libnuma_dlsym(handle, "numa_move_pages")));
|
||||
set_numa_set_preferred(CAST_TO_FN_PTR(numa_set_preferred_func_t,
|
||||
libnuma_dlsym(handle, "numa_set_preferred")));
|
||||
|
||||
if (numa_available() != -1) {
|
||||
set_numa_all_nodes((unsigned long*)libnuma_dlsym(handle, "numa_all_nodes"));
|
||||
@ -3298,6 +3300,7 @@ os::Linux::numa_distance_func_t os::Linux::_numa_distance;
|
||||
os::Linux::numa_get_membind_func_t os::Linux::_numa_get_membind;
|
||||
os::Linux::numa_get_interleave_mask_func_t os::Linux::_numa_get_interleave_mask;
|
||||
os::Linux::numa_move_pages_func_t os::Linux::_numa_move_pages;
|
||||
os::Linux::numa_set_preferred_func_t os::Linux::_numa_set_preferred;
|
||||
os::Linux::NumaAllocationPolicy os::Linux::_current_numa_policy;
|
||||
unsigned long* os::Linux::_numa_all_nodes;
|
||||
struct bitmask* os::Linux::_numa_all_nodes_ptr;
|
||||
|
@ -219,7 +219,7 @@ class Linux {
|
||||
typedef struct bitmask* (*numa_get_membind_func_t)(void);
|
||||
typedef struct bitmask* (*numa_get_interleave_mask_func_t)(void);
|
||||
typedef long (*numa_move_pages_func_t)(int pid, unsigned long count, void **pages, const int *nodes, int *status, int flags);
|
||||
|
||||
typedef void (*numa_set_preferred_func_t)(int node);
|
||||
typedef void (*numa_set_bind_policy_func_t)(int policy);
|
||||
typedef int (*numa_bitmask_isbitset_func_t)(struct bitmask *bmp, unsigned int n);
|
||||
typedef int (*numa_distance_func_t)(int node1, int node2);
|
||||
@ -238,6 +238,7 @@ class Linux {
|
||||
static numa_get_membind_func_t _numa_get_membind;
|
||||
static numa_get_interleave_mask_func_t _numa_get_interleave_mask;
|
||||
static numa_move_pages_func_t _numa_move_pages;
|
||||
static numa_set_preferred_func_t _numa_set_preferred;
|
||||
static unsigned long* _numa_all_nodes;
|
||||
static struct bitmask* _numa_all_nodes_ptr;
|
||||
static struct bitmask* _numa_nodes_ptr;
|
||||
@ -258,6 +259,7 @@ class Linux {
|
||||
static void set_numa_get_membind(numa_get_membind_func_t func) { _numa_get_membind = func; }
|
||||
static void set_numa_get_interleave_mask(numa_get_interleave_mask_func_t func) { _numa_get_interleave_mask = func; }
|
||||
static void set_numa_move_pages(numa_move_pages_func_t func) { _numa_move_pages = func; }
|
||||
static void set_numa_set_preferred(numa_set_preferred_func_t func) { _numa_set_preferred = func; }
|
||||
static void set_numa_all_nodes(unsigned long* ptr) { _numa_all_nodes = ptr; }
|
||||
static void set_numa_all_nodes_ptr(struct bitmask **ptr) { _numa_all_nodes_ptr = (ptr == NULL ? NULL : *ptr); }
|
||||
static void set_numa_nodes_ptr(struct bitmask **ptr) { _numa_nodes_ptr = (ptr == NULL ? NULL : *ptr); }
|
||||
@ -315,6 +317,11 @@ class Linux {
|
||||
_numa_interleave_memory(start, size, _numa_all_nodes);
|
||||
}
|
||||
}
|
||||
static void numa_set_preferred(int node) {
|
||||
if (_numa_set_preferred != NULL) {
|
||||
_numa_set_preferred(node);
|
||||
}
|
||||
}
|
||||
static void numa_set_bind_policy(int policy) {
|
||||
if (_numa_set_bind_policy != NULL) {
|
||||
_numa_set_bind_policy(policy);
|
||||
@ -392,6 +399,10 @@ class Linux {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
static const GrowableArray<int>* numa_nindex_to_node() {
|
||||
return _nindex_to_node;
|
||||
}
|
||||
};
|
||||
|
||||
#endif // OS_LINUX_OS_LINUX_HPP
|
||||
|
@ -24,7 +24,6 @@
|
||||
#include "precompiled.hpp"
|
||||
#include "gc/z/zNUMA.hpp"
|
||||
#include "logging/log.hpp"
|
||||
#include "runtime/os.hpp"
|
||||
|
||||
bool ZNUMA::_enabled;
|
||||
|
||||
|
@ -277,8 +277,11 @@ void ZPhysicalMemoryManager::map_view(const ZPhysicalMemory& pmem, uintptr_t add
|
||||
size += segment.size();
|
||||
}
|
||||
|
||||
// Setup NUMA interleaving
|
||||
if (ZNUMA::is_enabled()) {
|
||||
// Setup NUMA interleaving for large pages
|
||||
if (ZNUMA::is_enabled() && ZLargePages::is_explicit()) {
|
||||
// To get granule-level NUMA interleaving when using large pages,
|
||||
// we simply let the kernel interleave the memory for us at page
|
||||
// fault time.
|
||||
os::numa_make_global((char*)addr, size);
|
||||
}
|
||||
|
||||
|
Loading…
x
Reference in New Issue
Block a user