Commit 17f682e4 authored by Nathan Bronson's avatar Nathan Bronson Committed by Facebook Github Bot

F14: add an API for enumerating dynamic allocation classes

Summary:
Internal memory fragmentation is responsible for a non-trivial
amount of the memory footprint of a dynamically-allocated hash table.
This diff adds an API visitAllocationClasses that allows the caller to
count exactly how many dynamic allocations were performed and of what
size, so that they can accurately account for overheads not included in
getAllocatedMemorySize's simple sum of the underlying malloc requests.

This diff also exposes F14Table's reset() via F14 maps and sets, which
was an unintentional omission.

Reviewed By: WillerZ

Differential Revision: D7894385

fbshipit-source-id: 2d7cadd3806e3f38a9e38cb94c90492a9965ff47
parent b31800c0
......@@ -40,6 +40,8 @@
#if !FOLLY_F14_VECTOR_INTRINSICS_AVAILABLE
//////// Compatibility for unsupported platforms (not x86_64 and not aarch64)
#include <string>
#include <unordered_map>
......@@ -55,13 +57,22 @@ class F14BasicMap : public std::unordered_map<K, M, H, E, A> {
using Super::Super;
F14BasicMap() : Super() {}
// Approximates allocated memory, does not include sizeof(*this), also does
// not account for memory allocator fragmentation
//// PUBLIC - F14 Extensions
typename Super::size_type getAllocatedMemorySize() const {
auto bc = this->bucket_count();
return (bc == 1 ? 0 : bc) * sizeof(typename Super::pointer) +
this->size() * sizeof(StdNodeReplica<K, typename Super::value_type, H>);
}
template <typename V>
void visitAllocationClasses(V&& visitor) const {
auto bc = this->bucket_count();
if (bc > 1) {
visitor(bc * sizeof(typename Super::pointer), 1);
}
visitor(size(), sizeof(StdNodeReplica<K, typename Super::value_type, H>));
}
};
} // namespace detail
} // namespace f14
......@@ -97,6 +108,8 @@ class F14VectorMap : public f14::detail::F14BasicMap<K, M, H, E, A> {
#else // FOLLY_F14_VECTOR_INTRINSICS_AVAILABLE
//////// Common case for supported platforms
namespace folly {
namespace f14 {
namespace detail {
......@@ -271,15 +284,6 @@ class F14BasicMap {
return table_.max_size();
}
// Accounts for allocated memory only, does not include sizeof(*this).
std::size_t getAllocatedMemorySize() const {
return table_.getAllocatedMemorySize();
}
F14TableStats computeStats() const noexcept {
return table_.computeStats();
}
//// PUBLIC - Modifiers
void clear() noexcept {
......@@ -728,6 +732,31 @@ class F14BasicMap {
return table_.keyEqual();
}
//// PUBLIC - F14 Extensions
// Get memory footprint, not including sizeof(*this).
std::size_t getAllocatedMemorySize() const {
return table_.getAllocatedMemorySize();
}
// Enumerates classes of allocated memory blocks currently owned
// by this table, calling visitor(allocationSize, allocationCount).
// This can be used to get a more accurate indication of memory footprint
// than getAllocatedMemorySize() if you have some way of computing the
// internal fragmentation of the allocator, such as JEMalloc's nallocx.
// The visitor might be called twice with the same allocationSize. The
// visitor's computation should produce the same result for visitor(8,
// 2) as for two calls to visitor(8, 1), for example. The visitor may
// be called with a zero allocationCount.
template <typename V>
void visitAllocationClasses(V&& visitor) const {
return table_.visitAllocationClasses(visitor);
}
F14TableStats computeStats() const noexcept {
return table_.computeStats();
}
private:
template <typename Self, typename K>
FOLLY_ALWAYS_INLINE static auto& at(Self& self, K const& key) {
......
......@@ -37,6 +37,8 @@
#if !FOLLY_F14_VECTOR_INTRINSICS_AVAILABLE
//////// Compatibility for unsupported platforms (not x86_64 and not aarch64)
#include <unordered_set>
namespace folly {
......@@ -51,13 +53,22 @@ class F14BasicSet : public std::unordered_set<K, H, E, A> {
using Super::Super;
F14BasicSet() : Super() {}
// Approximates allocated memory, does not include sizeof(*this), also does
// not account for memory allocator fragmentation
//// PUBLIC - F14 Extensions
typename Super::size_type getAllocatedMemorySize() const {
auto bc = this->bucket_count();
return (bc == 1 ? 0 : bc) * sizeof(typename Super::pointer) +
this->size() * sizeof(StdNodeReplica<K, typename Super::value_type, H>);
}
template <typename V>
void visitAllocationClasses(V&& visitor) const {
auto bc = this->bucket_count();
if (bc > 1) {
visitor(bc * sizeof(typename Super::pointer), 1);
}
visitor(size(), sizeof(StdNodeReplica<K, typename Super::value_type, H>));
}
};
} // namespace detail
} // namespace f14
......@@ -93,6 +104,8 @@ class F14VectorSet : public f14::detail::F14BasicSet<K, H, E, A> {
#else // FOLLY_F14_VECTOR_INTRINSICS_AVAILABLE
//////// Common case for supported platforms
namespace folly {
namespace f14 {
namespace detail {
......@@ -266,15 +279,6 @@ class F14BasicSet {
return table_.max_size();
}
// Accounts for allocated memory only, does not include sizeof(*this).
std::size_t getAllocatedMemorySize() const {
return table_.getAllocatedMemorySize();
}
F14TableStats computeStats() const {
return table_.computeStats();
}
//// PUBLIC - Modifiers
void clear() noexcept {
......@@ -406,10 +410,11 @@ class F14BasicSet {
// used to extract an item out of a F14Set while avoiding a copy.
template <typename BeforeDestroy>
FOLLY_ALWAYS_INLINE iterator
eraseInto(const_iterator pos, BeforeDestroy const& beforeDestroy) {
eraseInto(const_iterator pos, BeforeDestroy&& beforeDestroy) {
table_.eraseInto(table_.unwrapIter(pos), beforeDestroy);
// If we are inlined then gcc and clang can optimize away all of the
// work of ++pos if the caller discards it.
table_.eraseInto(table_.unwrapIter(pos), beforeDestroy);
return ++pos;
}
......@@ -417,7 +422,7 @@ class F14BasicSet {
iterator eraseInto(
const_iterator first,
const_iterator last,
BeforeDestroy const& beforeDestroy) {
BeforeDestroy&& beforeDestroy) {
while (first != last) {
first = eraseInto(first, beforeDestroy);
}
......@@ -425,7 +430,7 @@ class F14BasicSet {
}
template <typename BeforeDestroy>
size_type eraseInto(key_type const& key, BeforeDestroy const& beforeDestroy) {
size_type eraseInto(key_type const& key, BeforeDestroy&& beforeDestroy) {
return table_.eraseInto(key, beforeDestroy);
}
......@@ -571,6 +576,31 @@ class F14BasicSet {
return table_.keyEqual();
}
//// PUBLIC - F14 Extensions
// Get memory footprint, not including sizeof(*this).
std::size_t getAllocatedMemorySize() const {
return table_.getAllocatedMemorySize();
}
// Enumerates classes of allocated memory blocks currently owned
// by this table, calling visitor(allocationSize, allocationCount).
// This can be used to get a more accurate indication of memory footprint
// than getAllocatedMemorySize() if you have some way of computing the
// internal fragmentation of the allocator, such as JEMalloc's nallocx.
// The visitor might be called twice with the same allocationSize. The
// visitor's computation should produce the same result for visitor(8,
// 2) as for two calls to visitor(8, 1), for example. The visitor may
// be called with a zero allocationCount.
template <typename V>
void visitAllocationClasses(V&& visitor) const {
return table_.visitAllocationClasses(visitor);
}
F14TableStats computeStats() const noexcept {
return table_.computeStats();
}
private:
template <typename Self, typename K>
static auto equal_range(Self& self, K const& key) {
......@@ -812,7 +842,7 @@ class F14VectorSet
template <typename BeforeDestroy>
void eraseUnderlying(
typename Policy::ItemIter underlying,
BeforeDestroy const& beforeDestroy) {
BeforeDestroy&& beforeDestroy) {
Alloc& a = this->table_.alloc();
auto values = this->table_.values_;
......@@ -848,7 +878,7 @@ class F14VectorSet
template <typename BeforeDestroy>
FOLLY_ALWAYS_INLINE iterator
eraseInto(const_iterator pos, BeforeDestroy const& beforeDestroy) {
eraseInto(const_iterator pos, BeforeDestroy&& beforeDestroy) {
auto underlying = this->table_.find(
f14::detail::VectorContainerIndexSearch{this->table_.iterToIndex(pos)});
eraseUnderlying(underlying, beforeDestroy);
......@@ -859,7 +889,7 @@ class F14VectorSet
iterator eraseInto(
const_iterator first,
const_iterator last,
BeforeDestroy const& beforeDestroy) {
BeforeDestroy&& beforeDestroy) {
while (first != last) {
first = eraseInto(first, beforeDestroy);
}
......@@ -867,9 +897,7 @@ class F14VectorSet
}
template <typename BeforeDestroy>
std::size_t eraseInto(
key_type const& key,
BeforeDestroy const& beforeDestroy) {
std::size_t eraseInto(key_type const& key, BeforeDestroy&& beforeDestroy) {
auto underlying = this->table_.find(key);
if (underlying.atEnd()) {
return 0;
......
......@@ -426,9 +426,9 @@ class ValueContainerPolicy : public BasePolicy<
template <typename T>
[[deprecated(
"use F14NodeMap/Set or mark key and mapped type move constructor nothrow")]]
std::enable_if_t<!std::is_nothrow_move_constructible<
T>::value> complainUnlessNothrowMove() {}
"use F14NodeMap/Set or mark key and mapped type move constructor nothrow")]] std::
enable_if_t<!std::is_nothrow_move_constructible<T>::value>
complainUnlessNothrowMove() {}
template <typename Dummy = int>
void moveItemDuringRehash(
......@@ -478,10 +478,11 @@ class ValueContainerPolicy : public BasePolicy<
AllocTraits::destroy(a, std::addressof(item));
}
std::size_t indirectBytesUsed(std::size_t /*size*/, std::size_t /*capacity*/)
const {
return 0;
}
template <typename V>
void visitPolicyAllocationClasses(
std::size_t /*size*/,
std::size_t /*capacity*/,
V&& /*visitor*/) const {}
//////// F14BasicMap/Set policy
......@@ -692,9 +693,12 @@ class NodeContainerPolicy
item.~Item();
}
std::size_t indirectBytesUsed(std::size_t size, std::size_t /*capacity*/)
const {
return size * sizeof(Value);
template <typename V>
void visitPolicyAllocationClasses(
std::size_t size,
std::size_t /*capacity*/,
V&& visitor) const {
visitor(sizeof(Value), size);
}
//////// F14BasicMap/Set policy
......@@ -964,9 +968,9 @@ class VectorContainerPolicy : public BasePolicy<
template <typename T>
[[deprecated(
"use F14NodeMap/Set or mark key and mapped type move constructor nothrow")]]
std::enable_if_t<!std::is_nothrow_move_constructible<
T>::value> complainUnlessNothrowMove() {}
"use F14NodeMap/Set or mark key and mapped type move constructor nothrow")]] std::
enable_if_t<!std::is_nothrow_move_constructible<T>::value>
complainUnlessNothrowMove() {}
template <typename Dummy = int>
void transfer(
......@@ -1125,9 +1129,14 @@ class VectorContainerPolicy : public BasePolicy<
}
}
std::size_t indirectBytesUsed(std::size_t /*size*/, std::size_t capacity)
const {
return sizeof(Value) * capacity;
template <typename V>
void visitPolicyAllocationClasses(
std::size_t /*size*/,
std::size_t capacity,
V&& visitor) const {
if (capacity > 0) {
visitor(sizeof(Value) * capacity, 1);
}
}
// Iterator stuff
......
......@@ -1821,7 +1821,7 @@ class F14Table : public Policy {
// to intercept the value before it is destroyed (to extract it, for
// example), do so in the beforeDestroy callback.
template <typename BeforeDestroy>
void eraseInto(ItemIter pos, BeforeDestroy const& beforeDestroy) {
void eraseInto(ItemIter pos, BeforeDestroy&& beforeDestroy) {
HashPair hp{};
if (pos.chunk()->hostedOverflowCount() != 0) {
hp = splitHash(this->computeItemHash(pos.citem()));
......@@ -1836,7 +1836,7 @@ class F14Table : public Policy {
}
template <typename K, typename BeforeDestroy>
std::size_t eraseInto(K const& key, BeforeDestroy const& beforeDestroy) {
std::size_t eraseInto(K const& key, BeforeDestroy&& beforeDestroy) {
if (UNLIKELY(size() == 0)) {
return 0;
}
......@@ -1863,9 +1863,28 @@ class F14Table : public Policy {
// Get memory footprint, not including sizeof(*this).
std::size_t getAllocatedMemorySize() const {
std::size_t sum = 0;
visitAllocationClasses(
[&sum](std::size_t bytes, std::size_t n) { sum += bytes * n; });
return sum;
}
// Enumerates classes of allocated memory blocks currently owned
// by this table, calling visitor(allocationSize, allocationCount).
// This can be used to get a more accurate indication of memory footprint
// than getAllocatedMemorySize() if you have some way of computing the
// internal fragmentation of the allocator, such as JEMalloc's nallocx.
// The visitor might be called twice with the same allocationSize. The
// visitor's computation should produce the same result for visitor(8,
// 2) as for two calls to visitor(8, 1), for example. The visitor may
// be called with a zero allocationCount.
template <typename V>
void visitAllocationClasses(V&& visitor) const {
auto bc = bucket_count();
return (bc == 0 ? 0 : allocSize(chunkMask_ + 1, bc)) +
this->indirectBytesUsed(size(), bc);
if (bc != 0) {
visitor(allocSize(chunkMask_ + 1, bc), 1);
}
this->visitPolicyAllocationClasses(size(), bc, visitor);
}
private:
......
......@@ -62,8 +62,23 @@ void testAllocatedMemorySize() {
for (size_t i = 0; i < 1000; ++i) {
m.insert(std::make_pair(folly::to<K>(i), V{}));
m.erase(folly::to<K>(i / 10 + 2));
EXPECT_EQ(A::getAllocatedMemorySize(), m.getAllocatedMemorySize());
std::size_t size = 0;
std::size_t count = 0;
m.visitAllocationClasses([&](std::size_t, std::size_t) mutable {});
m.visitAllocationClasses([&](std::size_t bytes, std::size_t n) {
size += bytes * n;
count += n;
});
EXPECT_EQ(A::getAllocatedMemorySize(), size);
EXPECT_EQ(A::getAllocatedBlockCount(), count);
}
m = decltype(m){};
EXPECT_EQ(A::getAllocatedMemorySize(), 0);
EXPECT_EQ(A::getAllocatedBlockCount(), 0);
m.visitAllocationClasses([](std::size_t, std::size_t n) { EXPECT_EQ(n, 0); });
}
template <typename K, typename V>
......
......@@ -59,8 +59,23 @@ void testAllocatedMemorySize() {
for (size_t i = 0; i < 1000; ++i) {
m.insert(folly::to<K>(i));
m.erase(folly::to<K>(i / 10 + 2));
EXPECT_EQ(A::getAllocatedMemorySize(), m.getAllocatedMemorySize());
std::size_t size = 0;
std::size_t count = 0;
m.visitAllocationClasses([&](std::size_t, std::size_t) mutable {});
m.visitAllocationClasses([&](std::size_t bytes, std::size_t n) {
size += bytes * n;
count += n;
});
EXPECT_EQ(A::getAllocatedMemorySize(), size);
EXPECT_EQ(A::getAllocatedBlockCount(), count);
}
m = decltype(m){};
EXPECT_EQ(A::getAllocatedMemorySize(), 0);
EXPECT_EQ(A::getAllocatedBlockCount(), 0);
m.visitAllocationClasses([](std::size_t, std::size_t n) { EXPECT_EQ(n, 0); });
}
template <typename K>
......@@ -712,6 +727,7 @@ void runEraseIntoTest() {
EXPECT_FALSE(value.destroyed);
t0.emplace(std::move(value));
};
auto insertIntoT0Mut = [&](auto& value) mutable { insertIntoT0(value); };
t0.insert(10);
t1.insert(20);
......@@ -724,7 +740,7 @@ void runEraseIntoTest() {
t1.insert(20);
t1.insert(30);
t1.insert(40);
t1.eraseInto(t1.begin(), t1.end(), insertIntoT0);
t1.eraseInto(t1.begin(), t1.end(), insertIntoT0Mut);
EXPECT_TRUE(t1.empty());
EXPECT_EQ(t0.size(), 4);
EXPECT_TRUE(t0.find(30) != t0.end());
......@@ -738,7 +754,7 @@ void runEraseIntoTest() {
EXPECT_TRUE(t0.find(50) != t0.end());
typename S::value_type key{60};
erased = t1.eraseInto(key, insertIntoT0);
erased = t1.eraseInto(key, insertIntoT0Mut);
EXPECT_EQ(erased, 0);
EXPECT_EQ(t0.size(), 5);
}
......
......@@ -314,6 +314,7 @@ std::ostream& operator<<(std::ostream& xo, F14TableStats const& stats) {
}
thread_local size_t allocatedMemorySize{0};
thread_local size_t allocatedBlockCount{0};
template <class T>
class SwapTrackingAlloc {
......@@ -359,16 +360,23 @@ class SwapTrackingAlloc {
return allocatedMemorySize;
}
static size_t getAllocatedBlockCount() {
return allocatedBlockCount;
}
static void resetTracking() {
allocatedMemorySize = 0;
allocatedBlockCount = 0;
}
T* allocate(size_t n) {
allocatedMemorySize += n * sizeof(T);
++allocatedBlockCount;
return a_.allocate(n);
}
void deallocate(T* p, size_t n) {
allocatedMemorySize -= n * sizeof(T);
--allocatedBlockCount;
a_.deallocate(p, n);
}
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment