Commit 4835f0e5 authored by Nathan Bronson's avatar Nathan Bronson Committed by Facebook Github Bot

support for unequal allocators

Summary:
The move constructor and move assignment operator for
allocator-aware containers is supposed to allocate new memory and
move values across when the underlying allocators are not equal.
Previously F14 just checked that this wasn't the case.  This diff
implements this feature.  This allows F14 tables to be moved into and
out of allocator-managed memory segments, such as arenas or shared
memory extents.

Reviewed By: yfeldblum

Differential Revision: D8022286

fbshipit-source-id: ff0b220450bb76822cc68cebad079a7ef43e2429
parent d1fdba5f
......@@ -136,9 +136,8 @@ class F14BasicMap {
using allocator_type = typename Policy::Alloc;
using reference = value_type&;
using const_reference = value_type const&;
using pointer = typename std::allocator_traits<allocator_type>::pointer;
using const_pointer =
typename std::allocator_traits<allocator_type>::const_pointer;
using pointer = typename Policy::AllocTraits::pointer;
using const_pointer = typename Policy::AllocTraits::const_pointer;
using iterator = typename Policy::Iter;
using const_iterator = typename Policy::ConstIter;
......@@ -148,7 +147,7 @@ class F14BasicMap {
public:
//// PUBLIC - Member functions
F14BasicMap() noexcept(F14Table<Policy>::kDefaultConstructIsNoexcept)
F14BasicMap() noexcept(Policy::kDefaultConstructIsNoexcept)
: F14BasicMap(0) {}
explicit F14BasicMap(
......@@ -210,7 +209,7 @@ class F14BasicMap {
F14BasicMap(F14BasicMap&& rhs) = default;
F14BasicMap(F14BasicMap&& rhs, allocator_type const& alloc) noexcept(
F14Table<Policy>::kAllocIsAlwaysEqual)
Policy::kAllocIsAlwaysEqual)
: table_{std::move(rhs.table_), alloc} {}
F14BasicMap(
......@@ -834,14 +833,11 @@ class F14ValueMap
using Super = f14::detail::F14BasicMap<Policy>;
public:
F14ValueMap() noexcept(
f14::detail::F14Table<Policy>::kDefaultConstructIsNoexcept)
: Super{} {}
F14ValueMap() noexcept(Policy::kDefaultConstructIsNoexcept) : Super{} {}
using Super::Super;
void swap(F14ValueMap& rhs) noexcept(
f14::detail::F14Table<Policy>::kSwapIsNoexcept) {
void swap(F14ValueMap& rhs) noexcept(Policy::kSwapIsNoexcept) {
this->table_.swap(rhs.table_);
}
};
......@@ -884,14 +880,11 @@ class F14NodeMap
using Super = f14::detail::F14BasicMap<Policy>;
public:
F14NodeMap() noexcept(
f14::detail::F14Table<Policy>::kDefaultConstructIsNoexcept)
: Super{} {}
F14NodeMap() noexcept(Policy::kDefaultConstructIsNoexcept) : Super{} {}
using Super::Super;
void swap(F14NodeMap& rhs) noexcept(
f14::detail::F14Table<Policy>::kSwapIsNoexcept) {
void swap(F14NodeMap& rhs) noexcept(Policy::kSwapIsNoexcept) {
this->table_.swap(rhs.table_);
}
......@@ -942,15 +935,12 @@ class F14VectorMap
using reverse_iterator = typename Policy::ReverseIter;
using const_reverse_iterator = typename Policy::ConstReverseIter;
F14VectorMap() noexcept(
f14::detail::F14Table<Policy>::kDefaultConstructIsNoexcept)
: Super{} {}
F14VectorMap() noexcept(Policy::kDefaultConstructIsNoexcept) : Super{} {}
// inherit constructors
using Super::Super;
void swap(F14VectorMap& rhs) noexcept(
f14::detail::F14Table<Policy>::kSwapIsNoexcept) {
void swap(F14VectorMap& rhs) noexcept(Policy::kSwapIsNoexcept) {
this->table_.swap(rhs.table_);
}
......@@ -1040,7 +1030,7 @@ class F14VectorMap
// The item still needs to be hashable during this call, so we must destroy
// the value _afterwards_.
this->table_.erase(underlying);
std::allocator_traits<Alloc>::destroy(a, std::addressof(values[index]));
Policy::AllocTraits::destroy(a, std::addressof(values[index]));
// move the last element in values_ down and fix up the inbound index
auto tailIndex = this->size();
......
......@@ -131,9 +131,8 @@ class F14BasicSet {
using allocator_type = typename Policy::Alloc;
using reference = value_type&;
using const_reference = value_type const&;
using pointer = typename std::allocator_traits<allocator_type>::pointer;
using const_pointer =
typename std::allocator_traits<allocator_type>::const_pointer;
using pointer = typename Policy::AllocTraits::pointer;
using const_pointer = typename Policy::AllocTraits::const_pointer;
using iterator = typename Policy::Iter;
using const_iterator = iterator;
......@@ -143,7 +142,7 @@ class F14BasicSet {
public:
//// PUBLIC - Member functions
F14BasicSet() noexcept(F14Table<Policy>::kDefaultConstructIsNoexcept)
F14BasicSet() noexcept(Policy::kDefaultConstructIsNoexcept)
: F14BasicSet(0) {}
explicit F14BasicSet(
......@@ -205,7 +204,7 @@ class F14BasicSet {
F14BasicSet(F14BasicSet&& rhs) = default;
F14BasicSet(F14BasicSet&& rhs, allocator_type const& alloc) noexcept(
F14Table<Policy>::kAllocIsAlwaysEqual)
Policy::kAllocIsAlwaysEqual)
: table_{std::move(rhs.table_), alloc} {}
F14BasicSet(
......@@ -657,14 +656,11 @@ class F14ValueSet
using Super = f14::detail::F14BasicSet<Policy>;
public:
F14ValueSet() noexcept(
f14::detail::F14Table<Policy>::kDefaultConstructIsNoexcept)
: Super{} {}
F14ValueSet() noexcept(Policy::kDefaultConstructIsNoexcept) : Super{} {}
using Super::Super;
void swap(F14ValueSet& rhs) noexcept(
f14::detail::F14Table<Policy>::kSwapIsNoexcept) {
void swap(F14ValueSet& rhs) noexcept(Policy::kSwapIsNoexcept) {
this->table_.swap(rhs.table_);
}
};
......@@ -700,14 +696,11 @@ class F14NodeSet
using Super = f14::detail::F14BasicSet<Policy>;
public:
F14NodeSet() noexcept(
f14::detail::F14Table<Policy>::kDefaultConstructIsNoexcept)
: Super{} {}
F14NodeSet() noexcept(Policy::kDefaultConstructIsNoexcept) : Super{} {}
using Super::Super;
void swap(F14NodeSet& rhs) noexcept(
f14::detail::F14Table<Policy>::kSwapIsNoexcept) {
void swap(F14NodeSet& rhs) noexcept(Policy::kSwapIsNoexcept) {
this->table_.swap(rhs.table_);
}
};
......@@ -750,15 +743,12 @@ class F14VectorSet
using reverse_iterator = typename Policy::ReverseIter;
using const_reverse_iterator = typename Policy::ConstReverseIter;
F14VectorSet() noexcept(
f14::detail::F14Table<Policy>::kDefaultConstructIsNoexcept)
: Super{} {}
F14VectorSet() noexcept(Policy::kDefaultConstructIsNoexcept) : Super{} {}
// inherit constructors
using Super::Super;
void swap(F14VectorSet& rhs) noexcept(
f14::detail::F14Table<Policy>::kSwapIsNoexcept) {
void swap(F14VectorSet& rhs) noexcept(Policy::kSwapIsNoexcept) {
this->table_.swap(rhs.table_);
}
......@@ -849,7 +839,7 @@ class F14VectorSet
// destroy the value and remove the ptr from the base table
auto index = underlying.item();
this->table_.eraseInto(underlying, beforeDestroy);
std::allocator_traits<Alloc>::destroy(a, std::addressof(values[index]));
Policy::AllocTraits::destroy(a, std::addressof(values[index]));
// move the last element in values_ down and fix up the inbound index
auto tailIndex = this->size();
......
......@@ -16,7 +16,10 @@
#pragma once
#include <memory>
#include <tuple>
#include <type_traits>
#include <utility>
#include <folly/container/detail/F14Table.h>
#include <folly/hash/Hash.h>
......@@ -71,6 +74,8 @@ struct BasePolicy
Defaulted<
AllocOrVoid,
DefaultAlloc<SetOrMapValueType<KeyType, MappedTypeOrVoid>>>> {
//////// user-supplied types
using Key = KeyType;
using Mapped = MappedTypeOrVoid;
using Value = SetOrMapValueType<Key, Mapped>;
......@@ -80,17 +85,53 @@ struct BasePolicy
using Alloc = Defaulted<AllocOrVoid, DefaultAlloc<Value>>;
using AllocTraits = std::allocator_traits<Alloc>;
using InternalSizeType = std::size_t;
//////// info about user-supplied types
using Super = std::tuple<Hasher, KeyEqual, Alloc>;
static_assert(
std::is_same<typename AllocTraits::value_type, Value>::value,
"wrong allocator value_type");
// if false, F14Table will be smaller but F14Table::begin() won't work
static constexpr bool kEnableItemIteration = true;
private:
// emulate c++17's std::allocator_traits<A>::is_always_equal
template <typename A, typename = void>
struct AllocIsAlwaysEqual : std::is_empty<A> {};
template <typename A>
struct AllocIsAlwaysEqual<A, typename A::is_always_equal>
: A::is_always_equal {};
// emulate c++17 has std::is_nothrow_swappable
template <typename T>
static constexpr bool isNothrowSwap() {
using std::swap;
return noexcept(swap(std::declval<T&>(), std::declval<T&>()));
}
public:
static constexpr bool kAllocIsAlwaysEqual = AllocIsAlwaysEqual<Alloc>::value;
static constexpr bool kDefaultConstructIsNoexcept =
std::is_nothrow_default_constructible<Hasher>::value &&
std::is_nothrow_default_constructible<KeyEqual>::value &&
std::is_nothrow_default_constructible<Alloc>::value;
static constexpr bool kSwapIsNoexcept = kAllocIsAlwaysEqual &&
isNothrowSwap<Hasher>() && isNothrowSwap<KeyEqual>();
static constexpr bool isAvalanchingHasher() {
return IsAvalanchingHasher<Hasher, Key>::value;
}
//////// internal types and constants
using InternalSizeType = std::size_t;
using Super = std::tuple<Hasher, KeyEqual, Alloc>;
// if false, F14Table will be smaller but F14Table::begin() won't work
static constexpr bool kEnableItemIteration = true;
using Chunk = F14Chunk<Item>;
using ChunkPtr = typename std::pointer_traits<
typename AllocTraits::pointer>::template rebind<Chunk>;
......@@ -101,9 +142,9 @@ struct BasePolicy
kIsMap == !std::is_void<MappedTypeOrVoid>::value,
"Assumption for the kIsMap check violated.");
static_assert(
std::is_same<typename AllocTraits::value_type, Value>::value,
"wrong allocator value_type");
using MappedOrBool = std::conditional_t<kIsMap, Mapped, bool>;
//////// methods
BasePolicy(Hasher const& hasher, KeyEqual const& keyEqual, Alloc const& alloc)
: Super{hasher, keyEqual, alloc} {}
......@@ -181,25 +222,45 @@ struct BasePolicy
Key const& keyForValue(Key const& v) const {
return v;
}
Key const& keyForValue(
std::pair<Key const, std::conditional_t<kIsMap, Mapped, bool>> const& p)
const {
Key const& keyForValue(std::pair<Key const, MappedOrBool> const& p) const {
return p.first;
}
Key const& keyForValue(std::pair<Key&&, MappedOrBool&&> const& p) const {
return p.first;
}
// map's choice of pair<K const, T> as value_type is unfortunate,
// because it means we either need a proxy iterator, a pointless key
// copy when moving items during rehash, or some sort of UB hack.
//
// This code implements the hack. Use moveValue(v) instead of
// std::move(v) as the source of a move construction. enable_if_t is
// used so that this works for maps while being a no-op for sets.
template <typename Dummy = int>
static std::pair<Key&&, MappedOrBool&&> moveValue(
std::pair<Key const, MappedOrBool>& value,
std::enable_if_t<kIsMap, Dummy> = 0) {
return {std::move(const_cast<Key&>(value.first)), std::move(value.second)};
}
template <typename Dummy = int>
static Value&& moveValue(Value& value, std::enable_if_t<!kIsMap, Dummy> = 0) {
return std::move(value);
}
template <typename P>
bool
beforeCopy(std::size_t /*size*/, std::size_t /*capacity*/, P const& /*rhs*/) {
beforeBuild(std::size_t /*size*/, std::size_t /*capacity*/, P&& /*rhs*/) {
return false;
}
template <typename P>
void afterCopy(
void afterBuild(
bool /*undoState*/,
bool /*success*/,
std::size_t /*size*/,
std::size_t /*capacity*/,
P const& /*rhs*/) {}
P&& /*rhs*/) {}
bool beforeRehash(
std::size_t /*size*/,
......@@ -349,13 +410,13 @@ class ValueContainerPolicy : public BasePolicy<
AllocOrVoid,
SetOrMapValueType<Key, MappedTypeOrVoid>>;
using typename Super::Alloc;
using typename Super::AllocTraits;
using typename Super::Item;
using typename Super::ItemIter;
using typename Super::Value;
private:
using Super::kIsMap;
using typename Super::AllocTraits;
public:
using ConstIter = ValueContainerIterator<typename AllocTraits::const_pointer>;
......@@ -404,11 +465,16 @@ class ValueContainerPolicy : public BasePolicy<
return this->keyEqual()(key, keyForValue(item));
}
Value const& valueAtItemForCopy(Item const& item) const {
Value const& buildArgForItem(Item const& item) const& {
return item;
}
Value&& valueAtItemForMove(Item& item) {
// buildArgForItem(Item&)&& is used when moving between unequal allocators
decltype(auto) buildArgForItem(Item& item) && {
return Super::moveValue(item);
}
Value&& valueAtItemForExtract(Item& item) {
return std::move(item);
}
......@@ -430,46 +496,23 @@ class ValueContainerPolicy : public BasePolicy<
enable_if_t<!std::is_nothrow_move_constructible<T>::value>
complainUnlessNothrowMove() {}
template <typename Dummy = int>
void moveItemDuringRehash(
Item* itemAddr,
Item& src,
typename std::enable_if_t<kIsMap, Dummy> = 0) {
void moveItemDuringRehash(Item* itemAddr, Item& src) {
complainUnlessNothrowMove<Key>();
complainUnlessNothrowMove<MappedTypeOrVoid>();
// map's choice of pair<K const,T> as value_type is unfortunate,
// because it means we either need a proxy iterator, a pointless key
// copy when moving items during rehash, or some sort of UB hack.
// See https://fb.quip.com/kKieAEtg0Pao for much more discussion of
// the possibilities.
//
// This code implements the hack.
// Laundering in the standard is only described as a solution for
// changes to const fields due to the creation of a new object
// lifetime (destroy and then placement new in the same location),
// but it seems highly likely that it will also cause the compiler
// to drop such assumptions that are violated due to our UB const_cast.
constructValueAtItem(
0,
itemAddr,
std::move(const_cast<Key&>(src.first)),
std::move(src.second));
constructValueAtItem(0, itemAddr, Super::moveValue(src));
if (destroyItemOnClear()) {
destroyItem(*folly::launder(std::addressof(src)));
}
}
template <typename Dummy = int>
void moveItemDuringRehash(
Item* itemAddr,
Item& src,
typename std::enable_if_t<!kIsMap, Dummy> = 0) {
complainUnlessNothrowMove<Item>();
constructValueAtItem(0, itemAddr, std::move(src));
if (destroyItemOnClear()) {
destroyItem(src);
if (kIsMap) {
// Laundering in the standard is only described as a solution
// for changes to const fields due to the creation of a new
// object lifetime (destroy and then placement new in the same
// location), but it seems highly likely that it will also cause
// the compiler to drop such assumptions that are violated due
// to our UB const_cast in moveValue.
destroyItem(*folly::launder(std::addressof(src)));
} else {
destroyItem(src);
}
}
}
......@@ -601,13 +644,13 @@ class NodeContainerPolicy
Key,
MapValueType<Key, MappedTypeOrVoid>>>>>::pointer>;
using typename Super::Alloc;
using typename Super::AllocTraits;
using typename Super::Item;
using typename Super::ItemIter;
using typename Super::Value;
private:
using Super::kIsMap;
using typename Super::AllocTraits;
public:
using ConstIter = NodeContainerIterator<typename AllocTraits::const_pointer>;
......@@ -652,11 +695,16 @@ class NodeContainerPolicy
return this->keyEqual()(key, keyForValue(*item));
}
Value const& valueAtItemForCopy(Item const& item) const {
Value const& buildArgForItem(Item const& item) const& {
return *item;
}
Value&& valueAtItemForMove(Item& item) {
// buildArgForItem(Item&)&& is used when moving between unequal allocators
decltype(auto) buildArgForItem(Item& item) && {
return Super::moveValue(*item);
}
Value&& valueAtItemForExtract(Item& item) {
return std::move(*item);
}
......@@ -819,15 +867,17 @@ class VectorContainerPolicy : public BasePolicy<
AllocOrVoid,
uint32_t>;
using typename Super::Alloc;
using typename Super::AllocTraits;
using typename Super::Hasher;
using typename Super::Item;
using typename Super::ItemIter;
using typename Super::KeyEqual;
using typename Super::Value;
using Super::kAllocIsAlwaysEqual;
private:
using Super::kIsMap;
using typename Super::AllocTraits;
public:
static constexpr bool kEnableItemIteration = false;
......@@ -887,8 +937,15 @@ class VectorContainerPolicy : public BasePolicy<
VectorContainerPolicy(
VectorContainerPolicy&& rhs,
Alloc const& alloc) noexcept
: Super{std::move(rhs), alloc}, values_{rhs.values_} {
rhs.values_ = nullptr;
: Super{std::move(rhs), alloc} {
if (kAllocIsAlwaysEqual || this->alloc() == rhs.alloc()) {
// common case
values_ = rhs.values_;
rhs.values_ = nullptr;
} else {
// table must be constructed in new memory
values_ = nullptr;
}
}
VectorContainerPolicy& operator=(VectorContainerPolicy const& rhs) {
......@@ -901,9 +958,15 @@ class VectorContainerPolicy : public BasePolicy<
VectorContainerPolicy& operator=(VectorContainerPolicy&& rhs) noexcept {
if (this != &rhs) {
FOLLY_SAFE_DCHECK(values_ == nullptr, "");
bool transfer =
AllocTraits::propagate_on_container_move_assignment::value ||
kAllocIsAlwaysEqual || this->alloc() == rhs.alloc();
Super::operator=(std::move(rhs));
values_ = rhs.values_;
rhs.values_ = nullptr;
if (transfer) {
values_ = rhs.values_;
rhs.values_ = nullptr;
}
}
return *this;
}
......@@ -947,11 +1010,11 @@ class VectorContainerPolicy : public BasePolicy<
return keyForValue(values_[arg.index_]);
}
VectorContainerIndexSearch valueAtItemForCopy(Item const& item) const {
VectorContainerIndexSearch buildArgForItem(Item const& item) const {
return {item};
}
Value&& valueAtItemForMove(Item& item) {
Value&& valueAtItemForExtract(Item& item) {
return std::move(values_[item]);
}
......@@ -990,13 +1053,7 @@ class VectorContainerPolicy : public BasePolicy<
enable_if_t<!std::is_nothrow_move_constructible<T>::value>
complainUnlessNothrowMove() {}
template <typename Dummy = int>
void transfer(
Alloc& a,
Value* src,
Value* dst,
std::size_t n,
typename std::enable_if_t<kIsMap, Dummy> = 0) {
void transfer(Alloc& a, Value* src, Value* dst, std::size_t n) {
complainUnlessNothrowMove<Key>();
complainUnlessNothrowMove<MappedTypeOrVoid>();
......@@ -1005,49 +1062,24 @@ class VectorContainerPolicy : public BasePolicy<
std::memcpy(dst, src, n * sizeof(Value));
} else {
for (std::size_t i = 0; i < n; ++i, ++src, ++dst) {
// See ValueContainerPolicy::moveItemDuringRehash for an explanation
// of // the strange const_cast and launder below
folly::assume(dst != nullptr);
AllocTraits::construct(
a,
dst,
std::move(const_cast<Key&>(src->first)),
std::move(src->second));
AllocTraits::destroy(a, folly::launder(src));
}
}
}
template <typename Dummy = int>
void transfer(
Alloc& a,
Value* src,
Value* dst,
std::size_t n,
typename std::enable_if_t<!kIsMap, Dummy> = 0) {
complainUnlessNothrowMove<Value>();
if (std::is_same<Alloc, std::allocator<Value>>::value &&
FOLLY_IS_TRIVIALLY_COPYABLE(Value)) {
std::memcpy(dst, src, n * sizeof(Value));
} else {
for (std::size_t i = 0; i < n; ++i, ++src, ++dst) {
folly::assume(dst != nullptr);
AllocTraits::construct(a, dst, std::move(*src));
AllocTraits::destroy(a, src);
AllocTraits::construct(a, dst, Super::moveValue(*src));
if (kIsMap) {
AllocTraits::destroy(a, folly::launder(src));
} else {
AllocTraits::destroy(a, src);
}
}
}
}
bool beforeCopy(
std::size_t size,
std::size_t /*capacity*/,
VectorContainerPolicy const& rhs) {
template <typename P, typename V>
bool beforeBuildImpl(std::size_t size, P&& rhs, V const& constructorArgFor) {
Alloc& a = this->alloc();
FOLLY_SAFE_DCHECK(values_ != nullptr, "");
Value const* src = std::addressof(rhs.values_[0]);
auto src = std::addressof(rhs.values_[0]);
Value* dst = std::addressof(values_[0]);
if (std::is_same<Alloc, std::allocator<Value>>::value &&
......@@ -1057,7 +1089,7 @@ class VectorContainerPolicy : public BasePolicy<
for (std::size_t i = 0; i < size; ++i, ++src, ++dst) {
try {
folly::assume(dst != nullptr);
AllocTraits::construct(a, dst, *src);
AllocTraits::construct(a, dst, constructorArgFor(*src));
} catch (...) {
for (Value* cleanup = std::addressof(values_[0]); cleanup != dst;
++cleanup) {
......@@ -1070,13 +1102,30 @@ class VectorContainerPolicy : public BasePolicy<
return true;
}
void afterCopy(
bool beforeBuild(
std::size_t size,
std::size_t /*capacity*/,
VectorContainerPolicy const& rhs) {
return beforeBuildImpl(size, rhs, [](Value const& v) { return v; });
}
bool beforeBuild(
std::size_t size,
std::size_t /*capacity*/,
VectorContainerPolicy&& rhs) {
return beforeBuildImpl(
size, rhs, [](Value& v) { return Super::moveValue(v); });
}
template <typename P>
void afterBuild(
bool /*undoState*/,
bool success,
std::size_t /*size*/,
std::size_t /*capacity*/,
VectorContainerPolicy const& /*rhs*/) {
// valueAtItemForCopy can be copied trivially, no failure should occur
P&& /*rhs*/) {
// buildArgForItem can be used to construct a new item trivially,
// so no failure between beforeBuild and afterBuild should be possible
FOLLY_SAFE_DCHECK(success, "");
}
......
......@@ -832,52 +832,23 @@ class F14Table : public Policy {
using allocator_type = typename Policy::Alloc;
private:
using HashPair = typename F14HashToken::HashPair;
using Chunk = F14Chunk<Item>;
using ChunkAlloc = typename std::allocator_traits<
allocator_type>::template rebind_alloc<Chunk>;
using ChunkPtr = typename std::allocator_traits<ChunkAlloc>::pointer;
using Policy::kAllocIsAlwaysEqual;
using Policy::kDefaultConstructIsNoexcept;
using Policy::kSwapIsNoexcept;
static constexpr bool kChunkAllocIsDefault =
std::is_same<ChunkAlloc, std::allocator<Chunk>>::value;
using typename Policy::AllocTraits;
using ByteAlloc = typename std::allocator_traits<
allocator_type>::template rebind_alloc<uint8_t>;
using ByteAlloc = typename AllocTraits::template rebind_alloc<uint8_t>;
using BytePtr = typename std::allocator_traits<ByteAlloc>::pointer;
public:
using ItemIter = F14ItemIter<ChunkPtr>;
private:
// emulate c++17's std::allocator_traits<A>::is_always_equal
template <typename A, typename = void>
struct AllocIsAlwaysEqual : std::is_empty<A> {};
template <typename A>
struct AllocIsAlwaysEqual<A, typename A::is_always_equal>
: A::is_always_equal {};
using Chunk = F14Chunk<Item>;
using ChunkPtr =
typename std::pointer_traits<BytePtr>::template rebind<Chunk>;
// emulate c++17 has std::is_nothrow_swappable
template <typename T>
static constexpr bool isNothrowSwap() {
using std::swap;
return noexcept(swap(std::declval<T&>(), std::declval<T&>()));
}
using HashPair = typename F14HashToken::HashPair;
public:
static constexpr bool kAllocIsAlwaysEqual =
AllocIsAlwaysEqual<allocator_type>::value;
static constexpr bool kDefaultConstructIsNoexcept =
std::is_nothrow_default_constructible<typename Policy::Hasher>::value &&
std::is_nothrow_default_constructible<typename Policy::KeyEqual>::value &&
std::is_nothrow_default_constructible<typename Policy::Alloc>::value;
static constexpr bool kSwapIsNoexcept = kAllocIsAlwaysEqual &&
isNothrowSwap<typename Policy::Hasher>() &&
isNothrowSwap<typename Policy::KeyEqual>();
using ItemIter = F14ItemIter<ChunkPtr>;
private:
//////// begin fields
......@@ -917,12 +888,12 @@ class F14Table : public Policy {
}
F14Table(F14Table const& rhs) : Policy{rhs} {
copyFromF14Table(rhs);
buildFromF14Table(rhs);
}
F14Table(F14Table const& rhs, typename Policy::Alloc const& alloc)
: Policy{rhs, alloc} {
copyFromF14Table(rhs);
buildFromF14Table(rhs);
}
F14Table(F14Table&& rhs) noexcept(
......@@ -936,17 +907,20 @@ class F14Table : public Policy {
F14Table(F14Table&& rhs, typename Policy::Alloc const& alloc) noexcept(
kAllocIsAlwaysEqual)
: Policy{std::move(rhs), alloc} {
FOLLY_SAFE_CHECK(
kAllocIsAlwaysEqual || this->alloc() == rhs.alloc(),
"F14 move with unequal allocators not yet supported");
swapContents(rhs);
if (kAllocIsAlwaysEqual || this->alloc() == rhs.alloc()) {
// move storage (common case)
swapContents(rhs);
} else {
// new storage because allocators unequal, move values (rare case)
buildFromF14Table(std::move(rhs));
}
}
F14Table& operator=(F14Table const& rhs) {
if (this != &rhs) {
reset();
static_cast<Policy&>(*this) = rhs;
copyFromF14Table(rhs);
buildFromF14Table(rhs);
}
return *this;
}
......@@ -955,18 +929,19 @@ class F14Table : public Policy {
std::is_nothrow_move_assignable<typename Policy::Hasher>::value&&
std::is_nothrow_move_assignable<typename Policy::KeyEqual>::value &&
(kAllocIsAlwaysEqual ||
(std::allocator_traits<typename Policy::Alloc>::
propagate_on_container_move_assignment::value &&
(AllocTraits::propagate_on_container_move_assignment::value &&
std::is_nothrow_move_assignable<typename Policy::Alloc>::value))) {
if (this != &rhs) {
reset();
static_cast<Policy&>(*this) = std::move(rhs);
FOLLY_SAFE_CHECK(
std::allocator_traits<typename Policy::Alloc>::
propagate_on_container_move_assignment::value ||
kAllocIsAlwaysEqual || this->alloc() == rhs.alloc(),
"F14 move with unequal allocators not yet supported");
swapContents(rhs);
if (AllocTraits::propagate_on_container_move_assignment::value ||
kAllocIsAlwaysEqual || this->alloc() == rhs.alloc()) {
// move storage (common case)
swapContents(rhs);
} else {
// new storage because allocators unequal, move values (rare case)
buildFromF14Table(std::move(rhs));
}
}
return *this;
}
......@@ -1113,7 +1088,7 @@ class F14Table : public Policy {
auto& a = this->alloc();
return std::min<std::size_t>(
(std::numeric_limits<typename Policy::InternalSizeType>::max)(),
std::allocator_traits<allocator_type>::max_size(a));
AllocTraits::max_size(a));
}
std::size_t bucket_count() const noexcept {
......@@ -1350,19 +1325,25 @@ class F14Table : public Policy {
}
}
void directCopyFrom(F14Table const& src) {
template <typename T>
void directBuildFrom(T&& src) {
FOLLY_SAFE_DCHECK(src.size() > 0 && chunkMask_ == src.chunkMask_, "");
Policy const& srcPolicy = src;
auto undoState = this->beforeCopy(src.size(), bucket_count(), srcPolicy);
// We use std::forward<T> to allow portions of src to be moved out by
// either beforeBuild or afterBuild, but we are just relying on good
// behavior of our Policy superclass to ensure that any particular
// field of this is a donor at most once.
auto undoState =
this->beforeBuild(src.size(), bucket_count(), std::forward<T>(src));
bool success = false;
SCOPE_EXIT {
this->afterCopy(
undoState, success, src.size(), bucket_count(), srcPolicy);
this->afterBuild(
undoState, success, src.size(), bucket_count(), std::forward<T>(src));
};
// Copy can fail part-way through if a Value copy constructor throws.
// Failing afterCopy is limited in its cleanup power in this case,
// Failing afterBuild is limited in its cleanup power in this case,
// because it can't enumerate the items that were actually copied.
// Fortunately we can divide the situation into cases where all of
// the state is owned by the table itself (F14Node and F14Value),
......@@ -1389,7 +1370,7 @@ class F14Table : public Policy {
// happy path, no rehash but pack items toward bottom of chunk and
// use copy constructor
Chunk const* srcChunk = &src.chunks_[maxChunkIndex];
auto srcChunk = &src.chunks_[maxChunkIndex];
Chunk* dstChunk = &chunks_[maxChunkIndex];
do {
dstChunk->copyOverflowInfoFrom(*srcChunk);
......@@ -1404,11 +1385,12 @@ class F14Table : public Policy {
std::size_t dstI = 0;
for (; iter.hasNext(); ++dstI) {
auto srcI = iter.next();
auto&& srcValue = src.valueAtItemForCopy(srcChunk->citem(srcI));
auto&& srcArg =
std::forward<T>(src).buildArgForItem(srcChunk->item(srcI));
auto dst = dstChunk->itemAddr(dstI);
folly::assume(dst != nullptr);
this->constructValueAtItem(
0, dst, std::forward<decltype(srcValue)>(srcValue));
0, dst, std::forward<decltype(srcArg)>(srcArg));
dstChunk->setTag(dstI, srcChunk->tag(srcI));
++sizeAndPackedBegin_.size_;
}
......@@ -1429,7 +1411,8 @@ class F14Table : public Policy {
success = true;
}
void rehashCopyFrom(F14Table const& src) {
template <typename T>
void rehashBuildFrom(T&& src) {
FOLLY_SAFE_DCHECK(src.chunkMask_ > chunkMask_, "");
// 1 byte per chunk means < 1 bit per value temporary overhead
......@@ -1454,30 +1437,35 @@ class F14Table : public Policy {
};
std::memset(fullness, '\0', cc);
// Exception safety requires beforeCopy to happen after all of the
// We use std::forward<T> to allow portions of src to be moved out by
// either beforeBuild or afterBuild, but we are just relying on good
// behavior of our Policy superclass to ensure that any particular
// field of this is a donor at most once.
// Exception safety requires beforeBuild to happen after all of the
// allocate() calls.
Policy const& srcPolicy = src;
auto undoState = this->beforeCopy(src.size(), bucket_count(), srcPolicy);
auto undoState =
this->beforeBuild(src.size(), bucket_count(), std::forward<T>(src));
bool success = false;
SCOPE_EXIT {
this->afterCopy(
undoState, success, src.size(), bucket_count(), srcPolicy);
this->afterBuild(
undoState, success, src.size(), bucket_count(), std::forward<T>(src));
};
// The current table is at a valid state at all points for policies
// in which non-trivial values are owned by the main table (F14Node
// and F14Value), so reset() will clean things up properly if we
// fail partway through. For the case that the policy manages value
// lifecycle (F14Vector) then nothing after beforeCopy can throw and
// lifecycle (F14Vector) then nothing after beforeBuild can throw and
// we don't have to worry about partial failure.
std::size_t srcChunkIndex = src.lastOccupiedChunk() - src.chunks_;
while (true) {
Chunk const* srcChunk = &src.chunks_[srcChunkIndex];
auto srcChunk = &src.chunks_[srcChunkIndex];
auto iter = srcChunk->occupiedIter();
if (Policy::prefetchBeforeRehash()) {
for (auto piter = iter; piter.hasNext();) {
this->prefetchValue(srcChunk->citem(piter.next()));
this->prefetchValue(srcChunk->item(piter.next()));
}
}
if (srcChunk->hostedOverflowCount() == 0) {
......@@ -1485,27 +1473,27 @@ class F14Table : public Policy {
// don't need to compute any hash values
while (iter.hasNext()) {
auto i = iter.next();
auto& srcItem = srcChunk->citem(i);
auto&& srcValue = src.valueAtItemForCopy(srcItem);
auto& srcItem = srcChunk->item(i);
auto&& srcArg = std::forward<T>(src).buildArgForItem(srcItem);
HashPair hp{srcChunkIndex, srcChunk->tag(i)};
insertAtBlank(
allocateTag(fullness, hp),
hp,
std::forward<decltype(srcValue)>(srcValue));
std::forward<decltype(srcArg)>(srcArg));
}
} else {
// any chunk's items might be in here
while (iter.hasNext()) {
auto i = iter.next();
auto& srcItem = srcChunk->citem(i);
auto&& srcValue = src.valueAtItemForCopy(srcItem);
auto const& srcKey = src.keyForValue(srcValue);
auto& srcItem = srcChunk->item(i);
auto&& srcArg = std::forward<T>(src).buildArgForItem(srcItem);
auto const& srcKey = src.keyForValue(srcArg);
auto hp = splitHash(this->computeKeyHash(srcKey));
FOLLY_SAFE_DCHECK(hp.second == srcChunk->tag(i), "");
insertAtBlank(
allocateTag(fullness, hp),
hp,
std::forward<decltype(srcValue)>(srcValue));
std::forward<decltype(srcArg)>(srcArg));
}
}
if (srcChunkIndex == 0) {
......@@ -1517,7 +1505,8 @@ class F14Table : public Policy {
success = true;
}
FOLLY_NOINLINE void copyFromF14Table(F14Table const& src) {
template <typename T>
FOLLY_NOINLINE void buildFromF14Table(T&& src) {
FOLLY_SAFE_DCHECK(size() == 0, "");
if (src.size() == 0) {
return;
......@@ -1526,9 +1515,9 @@ class F14Table : public Policy {
reserveForInsert(src.size());
try {
if (chunkMask_ == src.chunkMask_) {
directCopyFrom(src);
directBuildFrom(std::forward<T>(src));
} else {
rehashCopyFrom(src);
rehashBuildFrom(std::forward<T>(src));
}
} catch (...) {
reset();
......@@ -1826,7 +1815,7 @@ class F14Table : public Policy {
if (pos.chunk()->hostedOverflowCount() != 0) {
hp = splitHash(this->computeItemHash(pos.citem()));
}
beforeDestroy(this->valueAtItemForMove(pos.item()));
beforeDestroy(this->valueAtItemForExtract(pos.item()));
eraseImpl(pos, hp);
}
......@@ -1843,7 +1832,7 @@ class F14Table : public Policy {
auto hp = splitHash(this->computeKeyHash(key));
auto iter = findImpl(hp, key);
if (!iter.atEnd()) {
beforeDestroy(this->valueAtItemForMove(iter.item()));
beforeDestroy(this->valueAtItemForExtract(iter.item()));
eraseImpl(iter, hp);
return 1;
} else {
......
......@@ -364,11 +364,28 @@ void runMultiScopeTest() {
checkLocation("b1", segment1, *b1);
EXPECT_TRUE(*a1 == *c1);
// not yet supported by F14
// *a1 = std::move(*e2);
//
// checkLocation("a1", segment1, *a1);
// checkLocation("e2", segment2, *e2);
*a1 = std::move(*e2);
EXPECT_TRUE(*f2 == *a1);
checkLocation("a1", segment1, *a1);
checkLocation("e2", segment2, *e2);
auto g2 = segment2->construct<M>(anonymous_instance)(
std::move(*a1), typename M::allocator_type{mgr2});
EXPECT_TRUE(*f2 == *g2);
checkLocation("f2", segment2, *f2);
checkLocation("g2", segment2, *g2);
segment1->destroy_ptr(a1);
segment1->destroy_ptr(b1);
segment1->destroy_ptr(c1);
segment2->destroy_ptr(d2);
segment2->destroy_ptr(e2);
segment2->destroy_ptr(f2);
segment2->destroy_ptr(g2);
}
TEST(ShmF14ValueI2VVI, multiScope) {
......
......@@ -256,6 +256,8 @@ void runRandom() {
R r0;
R r1;
std::size_t rollbacks = 0;
std::size_t resizingSmallRollbacks = 0;
std::size_t resizingLargeRollbacks = 0;
for (std::size_t reps = 0; reps < 100000; ++reps) {
if (pctDist(gen) < 20) {
......@@ -476,6 +478,14 @@ void runRandom() {
}
}
if (t0.bucket_count() == t0.size() && t0.size() > 0) {
if (t0.size() < 10) {
++resizingSmallRollbacks;
} else {
++resizingLargeRollbacks;
}
}
assert(t0.size() == r0.size());
for (auto&& kv : r0) {
auto t = t0.find(kv.first);
......@@ -487,6 +497,8 @@ void runRandom() {
}
EXPECT_GE(rollbacks, 10);
EXPECT_GE(resizingSmallRollbacks, 1);
EXPECT_GE(resizingLargeRollbacks, 1);
}
EXPECT_EQ(testAllocatedMemorySize, 0);
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment