Commit 05fd9909 authored by Lucian Grijincu's avatar Lucian Grijincu Committed by Jordan DeLong

folly: speed up fastpath of StaticMeta::get()

Summary:
A smaller function makes it more likely it will be inlined
(it wasn't before, is now).

Test Plan: n/a

Reviewed By: tudorb@fb.com

FB internal diff: D760000
parent c246dd22
...@@ -260,59 +260,66 @@ struct StaticMeta { ...@@ -260,59 +260,66 @@ struct StaticMeta {
} }
} }
static ElementWrapper& get(int id) { /**
* Reserve enough space in the threadEntry_.elements for the item
* @id to fit in.
*/
static void reserve(int id) {
size_t prevSize = threadEntry_.elementsCapacity; size_t prevSize = threadEntry_.elementsCapacity;
if (prevSize <= id) { size_t newSize = static_cast<size_t>((id + 5) * 1.7);
size_t newSize = static_cast<size_t>((id + 5) * 1.7); auto& meta = instance();
auto & meta = instance(); ElementWrapper* ptr = nullptr;
ElementWrapper* ptr = NULL; // Rely on jemalloc to zero the memory if possible -- maybe it knows
// Rely on jemalloc to zero the memory if possible -- maybe it knows // it's already zeroed and saves us some work.
// it's already zeroed and saves us some work. if (!usingJEMalloc() ||
if (!usingJEMalloc() || prevSize < jemallocMinInPlaceExpandable ||
prevSize < jemallocMinInPlaceExpandable || (rallocm(
(rallocm( static_cast<void**>(static_cast<void*>(&threadEntry_.elements)),
static_cast<void**>(static_cast<void*>(&threadEntry_.elements)), NULL, newSize * sizeof(ElementWrapper), 0,
NULL, newSize * sizeof(ElementWrapper), 0, ALLOCM_NO_MOVE | ALLOCM_ZERO) != ALLOCM_SUCCESS)) {
ALLOCM_NO_MOVE | ALLOCM_ZERO) != ALLOCM_SUCCESS)) { // Sigh, must realloc, but we can't call realloc here, as elements is
// Sigh, must realloc, but we can't call realloc here, as elements is // still linked in meta, so another thread might access invalid memory
// still linked in meta, so another thread might access invalid memory // after realloc succeeds. We'll copy by hand and update threadEntry_
// after realloc succeeds. We'll copy by hand and update threadEntry_ // under the lock.
// under the lock. //
// // Note that we're using calloc instead of malloc in order to zero
// Note that we're using calloc instead of malloc in order to zero // the entire region. rallocm (ALLOCM_ZERO) will only zero newly
// the entire region. rallocm (ALLOCM_ZERO) will only zero newly // allocated memory, so if a previous allocation allocated more than
// allocated memory, so if a previous allocation allocated more than // we requested, it's our responsibility to guarantee that the tail
// we requested, it's our responsibility to guarantee that the tail // is zeroed. calloc() is simpler than malloc() followed by memset(),
// is zeroed. calloc() is simpler than malloc() followed by memset(), // and potentially faster when dealing with a lot of memory, as
// and potentially faster when dealing with a lot of memory, as // it can get already-zeroed pages from the kernel.
// it can get already-zeroed pages from the kernel. if ((ptr = static_cast<ElementWrapper*>(
if ((ptr = static_cast<ElementWrapper*>( calloc(newSize, sizeof(ElementWrapper)))) != nullptr) {
calloc(newSize, sizeof(ElementWrapper)))) != NULL) { memcpy(ptr, threadEntry_.elements, sizeof(ElementWrapper) * prevSize);
memcpy(ptr, threadEntry_.elements, } else {
sizeof(ElementWrapper) * prevSize); throw std::bad_alloc();
} else {
throw std::bad_alloc();
}
} }
}
// Success, update the entry // Success, update the entry
{ {
boost::lock_guard<boost::mutex> g(meta.lock_); boost::lock_guard<boost::mutex> g(meta.lock_);
if (prevSize == 0) { if (prevSize == 0) {
meta.push_back(&threadEntry_); meta.push_back(&threadEntry_);
} }
if (ptr) { if (ptr) {
using std::swap; using std::swap;
swap(ptr, threadEntry_.elements); swap(ptr, threadEntry_.elements);
}
threadEntry_.elementsCapacity = newSize;
} }
threadEntry_.elementsCapacity = newSize;
}
free(ptr); free(ptr);
if (prevSize == 0) { if (prevSize == 0) {
pthread_setspecific(meta.pthreadKey_, &meta); pthread_setspecific(meta.pthreadKey_, &meta);
} }
}
static ElementWrapper& get(int id) {
if (UNLIKELY(threadEntry_.elementsCapacity <= id)) {
reserve(id);
} }
return threadEntry_.elements[id]; return threadEntry_.elements[id];
} }
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment