Commit c673fbd1 authored by Gisle Dankel's avatar Gisle Dankel Committed by Facebook Github Bot

Add option to set decay time to 0 for huge page allocator

Summary:
We observed some RSS regression for certain use cases - setting decay time to 0 should fix this as freed regions will be released back to the kernel immediately instead of delayed.

The risk is that we lose regions before we can reallocate them and thus over time end up with holes in the huge page region.
If we see this becoming an issue for some use cases, we can add a parameter to control the decay time.

Releasing immediately should be OK for huge pages in most cases. For cases with frequent allocation and deallocation of large areas there can be a perf impact however.

Reviewed By: interwq

Differential Revision: D13853171

fbshipit-source-id: 83fb95ed9c9bcb6ebdd1f80d78347e5eec61c084
parent 2ac8a368
...@@ -28,7 +28,7 @@ ...@@ -28,7 +28,7 @@
bool folly::JemallocHugePageAllocator::hugePagesSupported{true}; bool folly::JemallocHugePageAllocator::hugePagesSupported{true};
#endif #endif
#endif // defined(FOLLY_HAVE_LIBJEMALLOC) && !FOLLY_SANITIZE #endif // MADV_HUGEPAGE && defined(FOLLY_USE_JEMALLOC) && !FOLLY_SANITIZE
#ifndef FOLLY_JEMALLOC_HUGE_PAGE_ALLOCATOR_SUPPORTED #ifndef FOLLY_JEMALLOC_HUGE_PAGE_ALLOCATOR_SUPPORTED
// Some mocks when jemalloc.h is not included or version too old // Some mocks when jemalloc.h is not included or version too old
...@@ -39,6 +39,8 @@ bool folly::JemallocHugePageAllocator::hugePagesSupported{true}; ...@@ -39,6 +39,8 @@ bool folly::JemallocHugePageAllocator::hugePagesSupported{true};
#define MALLOCX_ARENA(x) 0 #define MALLOCX_ARENA(x) 0
#define MALLOCX_TCACHE_NONE 0 #define MALLOCX_TCACHE_NONE 0
#define MADV_HUGEPAGE 0 #define MADV_HUGEPAGE 0
#if !defined(JEMALLOC_VERSION_MAJOR) || (JEMALLOC_VERSION_MAJOR < 5)
typedef struct extent_hooks_s extent_hooks_t; typedef struct extent_hooks_s extent_hooks_t;
typedef void*(extent_alloc_t)( typedef void*(extent_alloc_t)(
extent_hooks_t*, extent_hooks_t*,
...@@ -51,6 +53,8 @@ typedef void*(extent_alloc_t)( ...@@ -51,6 +53,8 @@ typedef void*(extent_alloc_t)(
struct extent_hooks_s { struct extent_hooks_s {
extent_alloc_t* alloc; extent_alloc_t* alloc;
}; };
#endif // JEMALLOC_VERSION_MAJOR
bool folly::JemallocHugePageAllocator::hugePagesSupported{false}; bool folly::JemallocHugePageAllocator::hugePagesSupported{false};
#endif // FOLLY_JEMALLOC_HUGE_PAGE_ALLOCATOR_SUPPORTED #endif // FOLLY_JEMALLOC_HUGE_PAGE_ALLOCATOR_SUPPORTED
...@@ -65,7 +69,7 @@ static void print_error(int err, const char* msg) { ...@@ -65,7 +69,7 @@ static void print_error(int err, const char* msg) {
class HugePageArena { class HugePageArena {
public: public:
int init(int nr_pages); int init(int nr_pages, const JemallocHugePageAllocator::Options& options);
void* reserve(size_t size, size_t alignment); void* reserve(size_t size, size_t alignment);
bool addressInArena(void* address) { bool addressInArena(void* address) {
...@@ -77,6 +81,10 @@ class HugePageArena { ...@@ -77,6 +81,10 @@ class HugePageArena {
return end_ - freePtr_; return end_ - freePtr_;
} }
unsigned arenaIndex() {
return arenaIndex_;
}
private: private:
static void* allocHook( static void* allocHook(
extent_hooks_t* extent, extent_hooks_t* extent,
...@@ -92,6 +100,7 @@ class HugePageArena { ...@@ -92,6 +100,7 @@ class HugePageArena {
uintptr_t freePtr_{0}; uintptr_t freePtr_{0};
extent_alloc_t* originalAlloc_{nullptr}; extent_alloc_t* originalAlloc_{nullptr};
extent_hooks_t extentHooks_; extent_hooks_t extentHooks_;
unsigned arenaIndex_{0};
}; };
constexpr size_t kHugePageSize = 2 * 1024 * 1024; constexpr size_t kHugePageSize = 2 * 1024 * 1024;
...@@ -151,8 +160,6 @@ void* HugePageArena::allocHook( ...@@ -151,8 +160,6 @@ void* HugePageArena::allocHook(
if (new_addr == nullptr) { if (new_addr == nullptr) {
res = arena.reserve(size, alignment); res = arena.reserve(size, alignment);
} }
LOG(INFO) << "Extent request of size " << size << " alignment " << alignment
<< " = " << res << " (" << arena.freeSpace() << " bytes free)";
if (res == nullptr) { if (res == nullptr) {
LOG_IF(WARNING, new_addr != nullptr) << "Explicit address not supported"; LOG_IF(WARNING, new_addr != nullptr) << "Explicit address not supported";
res = arena.originalAlloc_( res = arena.originalAlloc_(
...@@ -163,16 +170,19 @@ void* HugePageArena::allocHook( ...@@ -163,16 +170,19 @@ void* HugePageArena::allocHook(
} }
*commit = true; *commit = true;
} }
VLOG(0) << "Extent request of size " << size << " alignment " << alignment
<< " = " << res << " (" << arena.freeSpace() << " bytes free)";
return res; return res;
} }
int HugePageArena::init(int nr_pages) { int HugePageArena::init(
int nr_pages,
const JemallocHugePageAllocator::Options& options) {
DCHECK(start_ == 0); DCHECK(start_ == 0);
DCHECK(usingJEMalloc()); DCHECK(usingJEMalloc());
unsigned arena_index; size_t len = sizeof(arenaIndex_);
size_t len = sizeof(arena_index); if (auto ret = mallctl("arenas.create", &arenaIndex_, &len, nullptr, 0)) {
if (auto ret = mallctl("arenas.create", &arena_index, &len, nullptr, 0)) {
print_error(ret, "Unable to create arena"); print_error(ret, "Unable to create arena");
return 0; return 0;
} }
...@@ -187,13 +197,13 @@ int HugePageArena::init(int nr_pages) { ...@@ -187,13 +197,13 @@ int HugePageArena::init(int nr_pages) {
size_t mib[3]; size_t mib[3];
size_t miblen = sizeof(mib) / sizeof(size_t); size_t miblen = sizeof(mib) / sizeof(size_t);
std::ostringstream rtl_key; std::ostringstream rtl_key;
rtl_key << "arena." << arena_index << ".retain_grow_limit"; rtl_key << "arena." << arenaIndex_ << ".retain_grow_limit";
if (auto ret = mallctlnametomib(rtl_key.str().c_str(), mib, &miblen)) { if (auto ret = mallctlnametomib(rtl_key.str().c_str(), mib, &miblen)) {
print_error(ret, "Unable to read growth limit"); print_error(ret, "Unable to read growth limit");
return 0; return 0;
} }
size_t grow_retained_limit = kHugePageSize; size_t grow_retained_limit = kHugePageSize;
mib[1] = arena_index; mib[1] = arenaIndex_;
if (auto ret = mallctlbymib( if (auto ret = mallctlbymib(
mib, mib,
miblen, miblen,
...@@ -206,7 +216,7 @@ int HugePageArena::init(int nr_pages) { ...@@ -206,7 +216,7 @@ int HugePageArena::init(int nr_pages) {
} }
std::ostringstream hooks_key; std::ostringstream hooks_key;
hooks_key << "arena." << arena_index << ".extent_hooks"; hooks_key << "arena." << arenaIndex_ << ".extent_hooks";
extent_hooks_t* hooks; extent_hooks_t* hooks;
len = sizeof(hooks); len = sizeof(hooks);
// Read the existing hooks // Read the existing hooks
...@@ -230,12 +240,29 @@ int HugePageArena::init(int nr_pages) { ...@@ -230,12 +240,29 @@ int HugePageArena::init(int nr_pages) {
return 0; return 0;
} }
if (options.noDecay) {
// Set decay time to 0, which will cause jemalloc to free memory
// back to kernel immediately.
ssize_t decay_ms = 0;
std::ostringstream dirty_decay_key;
dirty_decay_key << "arena." << arenaIndex_ << ".dirty_decay_ms";
if (auto ret = mallctl(
dirty_decay_key.str().c_str(),
nullptr,
nullptr,
&decay_ms,
sizeof(decay_ms))) {
print_error(ret, "Unable to set decay time");
return 0;
}
}
start_ = freePtr_ = map_pages(nr_pages); start_ = freePtr_ = map_pages(nr_pages);
if (start_ == 0) { if (start_ == 0) {
return false; return false;
} }
end_ = start_ + (nr_pages * kHugePageSize); end_ = start_ + (nr_pages * kHugePageSize);
return MALLOCX_ARENA(arena_index) | MALLOCX_TCACHE_NONE; return MALLOCX_ARENA(arenaIndex_) | MALLOCX_TCACHE_NONE;
} }
void* HugePageArena::reserve(size_t size, size_t alignment) { void* HugePageArena::reserve(size_t size, size_t alignment) {
...@@ -255,14 +282,14 @@ void* HugePageArena::reserve(size_t size, size_t alignment) { ...@@ -255,14 +282,14 @@ void* HugePageArena::reserve(size_t size, size_t alignment) {
int JemallocHugePageAllocator::flags_{0}; int JemallocHugePageAllocator::flags_{0};
bool JemallocHugePageAllocator::init(int nr_pages) { bool JemallocHugePageAllocator::init(int nr_pages, const Options& options) {
if (!usingJEMalloc()) { if (!usingJEMalloc()) {
LOG(ERROR) << "Not linked with jemalloc?"; LOG(ERROR) << "Not linked with jemalloc?";
hugePagesSupported = false; hugePagesSupported = false;
} }
if (hugePagesSupported) { if (hugePagesSupported) {
if (flags_ == 0) { if (flags_ == 0) {
flags_ = arena.init(nr_pages); flags_ = arena.init(nr_pages, options);
} else { } else {
LOG(WARNING) << "Already initialized"; LOG(WARNING) << "Already initialized";
} }
...@@ -280,4 +307,8 @@ bool JemallocHugePageAllocator::addressInArena(void* address) { ...@@ -280,4 +307,8 @@ bool JemallocHugePageAllocator::addressInArena(void* address) {
return arena.addressInArena(address); return arena.addressInArena(address);
} }
unsigned arenaIndex() {
return arena.arenaIndex();
}
} // namespace folly } // namespace folly
...@@ -56,7 +56,21 @@ namespace folly { ...@@ -56,7 +56,21 @@ namespace folly {
*/ */
class JemallocHugePageAllocator { class JemallocHugePageAllocator {
public: public:
static bool init(int nr_pages); struct Options {
// Set decay time to 0, which will cause jemalloc to free memory
// back to kernel immediately. It can still be reclaimed later if
// the kernel hasn't reused it for something else.
// This is primarily useful for preventing RSS regressions, but
// can cause the number of available pages to shrink over time
// as the likelihood they get reused by the kernel is increased.
bool noDecay = false;
};
static bool init(int nr_pages) {
return init(nr_pages, Options());
}
static bool init(int nr_pages, const Options& options);
static void* allocate(size_t size) { static void* allocate(size_t size) {
// If uninitialized, flags_ will be 0 and the mallocx behavior // If uninitialized, flags_ will be 0 and the mallocx behavior
......
...@@ -75,7 +75,7 @@ TEST(JemallocHugePageAllocatorTest, LargeAllocations) { ...@@ -75,7 +75,7 @@ TEST(JemallocHugePageAllocatorTest, LargeAllocations) {
} }
// This fits // This fits
void* ptr1 = jha::allocate(mb(1) + kb(512)); void* ptr1 = jha::allocate(mb(2));
EXPECT_NE(nullptr, ptr1); EXPECT_NE(nullptr, ptr1);
if (initialized) { if (initialized) {
...@@ -91,7 +91,7 @@ TEST(JemallocHugePageAllocatorTest, LargeAllocations) { ...@@ -91,7 +91,7 @@ TEST(JemallocHugePageAllocatorTest, LargeAllocations) {
// Free and reuse huge page area // Free and reuse huge page area
jha::deallocate(ptr2); jha::deallocate(ptr2);
jha::deallocate(ptr0); jha::deallocate(ptr0);
ptr2 = jha::allocate(mb(1)); ptr2 = jha::allocate(kb(64));
// No memory in the huge page arena was freed - ptr0 was allocated // No memory in the huge page arena was freed - ptr0 was allocated
// before init and ptr2 didn't fit // before init and ptr2 didn't fit
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment