Commit 1aaded41 authored by Marc Celani's avatar Marc Celani Committed by Facebook Github Bot

BufferedStat

Summary:
BufferedStat hooks up the DigestBuilder and a Digest or a SlidingWindow

Every windowDuration time, the DigestBuilder builds a new digest that will be merged into the central data structure (either a digest or a sliding window of digests).

This diff removes the shared buffer from DigestBuilder, as the concurrency model was getting overly complex with it.

Reviewed By: simpkins

Differential Revision: D7629082

fbshipit-source-id: 54027363d37c0fdced287550eea5b12caf6b8138
parent bf3343eb
...@@ -615,6 +615,7 @@ if (BUILD_TESTS) ...@@ -615,6 +615,7 @@ if (BUILD_TESTS)
TEST openssl_hash_test SOURCES OpenSSLHashTest.cpp TEST openssl_hash_test SOURCES OpenSSLHashTest.cpp
DIRECTORY stats/test/ DIRECTORY stats/test/
TEST buffered_stat_test SOURCES BufferedStatTest.cpp
TEST digest_builder_test SOURCES DigestBuilderTest.cpp TEST digest_builder_test SOURCES DigestBuilderTest.cpp
TEST histogram_test SOURCES HistogramTest.cpp TEST histogram_test SOURCES HistogramTest.cpp
TEST sliding_window_test SOURCES SlidingWindowTest.cpp TEST sliding_window_test SOURCES SlidingWindowTest.cpp
......
...@@ -441,6 +441,8 @@ nobase_follyinclude_HEADERS = \ ...@@ -441,6 +441,8 @@ nobase_follyinclude_HEADERS = \
ssl/detail/OpenSSLThreading.h \ ssl/detail/OpenSSLThreading.h \
ssl/detail/SSLSessionImpl.h \ ssl/detail/SSLSessionImpl.h \
stats/detail/Bucket.h \ stats/detail/Bucket.h \
stats/detail/BufferedStat-defs.h \
stats/detail/BufferedStat.h \
stats/detail/DigestBuilder-defs.h \ stats/detail/DigestBuilder-defs.h \
stats/detail/DigestBuilder.h \ stats/detail/DigestBuilder.h \
stats/detail/SlidingWindow-defs.h \ stats/detail/SlidingWindow-defs.h \
......
...@@ -81,6 +81,10 @@ class TDigest { ...@@ -81,6 +81,10 @@ class TDigest {
return count_; return count_;
} }
bool empty() const {
return centroids_.empty();
}
private: private:
struct Centroid { struct Centroid {
public: public:
......
/*
* Copyright 2012-present Facebook, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#pragma once
#include <folly/stats/detail/BufferedStat.h>
#include <folly/stats/detail/DigestBuilder-defs.h>
#include <folly/stats/detail/SlidingWindow-defs.h>
namespace folly {
namespace detail {
template <typename DigestT, typename ClockT>
BufferedStat<DigestT, ClockT>::BufferedStat(
typename ClockT::duration bufferDuration,
size_t bufferSize,
size_t digestSize)
: bufferDuration_(bufferDuration), digestBuilder_(bufferSize, digestSize) {
expiry_.store(
TimePointHolder(roundUp(ClockT::now())), std::memory_order_relaxed);
}
template <typename DigestT, typename ClockT>
void BufferedStat<DigestT, ClockT>::append(double value) {
auto now = ClockT::now();
if (UNLIKELY(now > expiry_.load(std::memory_order_acquire).tp)) {
std::unique_lock<SharedMutex> g(mutex_, std::try_to_lock_t());
if (g.owns_lock()) {
doUpdate(now, g);
}
}
digestBuilder_.append(value);
}
template <typename DigestT, typename ClockT>
typename ClockT::time_point BufferedStat<DigestT, ClockT>::roundUp(
typename ClockT::time_point t) {
auto remainder = t.time_since_epoch() % bufferDuration_;
if (remainder.count() != 0) {
return t + bufferDuration_ - remainder;
}
return t;
}
template <typename DigestT, typename ClockT>
std::unique_lock<SharedMutex> BufferedStat<DigestT, ClockT>::updateIfExpired() {
auto now = ClockT::now();
std::unique_lock<SharedMutex> g(mutex_);
doUpdate(now, g);
return g;
}
template <typename DigestT, typename ClockT>
void BufferedStat<DigestT, ClockT>::doUpdate(
typename ClockT::time_point now,
const std::unique_lock<SharedMutex>& g) {
DCHECK(g.owns_lock());
// Check that no other thread has performed the slide after the check
auto oldExpiry = expiry_.load(std::memory_order_acquire).tp;
if (now > oldExpiry) {
now = roundUp(now);
expiry_.store(TimePointHolder(now), std::memory_order_release);
onNewDigest(digestBuilder_.build(), now, oldExpiry, g);
}
}
template <typename DigestT, typename ClockT>
BufferedDigest<DigestT, ClockT>::BufferedDigest(
typename ClockT::duration bufferDuration,
size_t bufferSize,
size_t digestSize)
: BufferedStat<DigestT, ClockT>(bufferDuration, bufferSize, digestSize),
digest_(digestSize) {}
template <typename DigestT, typename ClockT>
DigestT BufferedDigest<DigestT, ClockT>::get() {
auto g = this->updateIfExpired();
return digest_;
}
template <typename DigestT, typename ClockT>
void BufferedDigest<DigestT, ClockT>::onNewDigest(
DigestT digest,
typename ClockT::time_point /*newExpiry*/,
typename ClockT::time_point /*oldExpiry*/,
const std::unique_lock<SharedMutex>& /*g*/) {
std::array<DigestT, 2> a{digest_, std::move(digest)};
digest_ = DigestT::merge(a);
}
template <typename DigestT, typename ClockT>
BufferedSlidingWindow<DigestT, ClockT>::BufferedSlidingWindow(
size_t nBuckets,
typename ClockT::duration bufferDuration,
size_t bufferSize,
size_t digestSize)
: BufferedStat<DigestT, ClockT>(bufferDuration, bufferSize, digestSize),
slidingWindow_([=]() { return DigestT(digestSize); }, nBuckets) {}
template <typename DigestT, typename ClockT>
std::vector<DigestT> BufferedSlidingWindow<DigestT, ClockT>::get() {
std::vector<DigestT> digests;
{
auto g = this->updateIfExpired();
digests = slidingWindow_.get();
}
digests.erase(
std::remove_if(
digests.begin(),
digests.end(),
[](const DigestT& digest) { return digest.empty(); }),
digests.end());
return digests;
}
template <typename DigestT, typename ClockT>
void BufferedSlidingWindow<DigestT, ClockT>::onNewDigest(
DigestT digest,
typename ClockT::time_point newExpiry,
typename ClockT::time_point oldExpiry,
const std::unique_lock<SharedMutex>& /*g*/) {
auto diff = newExpiry - oldExpiry;
slidingWindow_.slide(diff / this->bufferDuration_);
diff -= this->bufferDuration_;
slidingWindow_.set(diff / this->bufferDuration_, std::move(digest));
}
} // namespace detail
} // namespace folly
/*
* Copyright 2012-present Facebook, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#pragma once
#include <folly/SharedMutex.h>
#include <folly/stats/detail/DigestBuilder.h>
#include <folly/stats/detail/SlidingWindow.h>
namespace folly {
namespace detail {
/*
* BufferedStat keeps a clock and every time period, will merge data from a
* DigestBuilder into a DigestT. Updates are made by the first appender after
* the expiry, or can be made at read time by calling update().
*/
template <typename DigestT, typename ClockT>
class BufferedStat {
public:
BufferedStat() = delete;
BufferedStat(
typename ClockT::duration bufferDuration,
size_t bufferSize,
size_t digestSize);
virtual ~BufferedStat() {}
void append(double value);
protected:
// https://www.mail-archive.com/llvm-bugs@lists.llvm.org/msg18280.html
// Wrap the time point in something with a noexcept constructor.
struct TimePointHolder {
public:
TimePointHolder() noexcept {}
TimePointHolder(typename ClockT::time_point t) : tp(t) {}
typename ClockT::time_point tp;
};
const typename ClockT::duration bufferDuration_;
std::atomic<TimePointHolder> expiry_;
SharedMutex mutex_;
virtual void onNewDigest(
DigestT digest,
typename ClockT::time_point newExpiry,
typename ClockT::time_point oldExpiry,
const std::unique_lock<SharedMutex>& g) = 0;
std::unique_lock<SharedMutex> updateIfExpired();
private:
DigestBuilder<DigestT> digestBuilder_;
void doUpdate(
typename ClockT::time_point now,
const std::unique_lock<SharedMutex>& g);
typename ClockT::time_point roundUp(typename ClockT::time_point t);
};
/*
* BufferedDigest is a BufferedStat that holds data in a single digest.
*/
template <typename DigestT, typename ClockT>
class BufferedDigest : public BufferedStat<DigestT, ClockT> {
public:
BufferedDigest(
typename ClockT::duration bufferDuration,
size_t bufferSize,
size_t digestSize);
DigestT get();
void onNewDigest(
DigestT digest,
typename ClockT::time_point newExpiry,
typename ClockT::time_point oldExpiry,
const std::unique_lock<SharedMutex>& g) final;
private:
DigestT digest_;
};
/*
* BufferedSlidingWindow is a BufferedStat that holds data in a SlidingWindow.
* onBufferSwap will slide the SlidingWindow and return the front of the list.
*/
template <typename DigestT, typename ClockT>
class BufferedSlidingWindow : public BufferedStat<DigestT, ClockT> {
public:
BufferedSlidingWindow(
size_t nBuckets,
typename ClockT::duration bufferDuration,
size_t bufferSize,
size_t digestSize);
std::vector<DigestT> get();
void onNewDigest(
DigestT digest,
typename ClockT::time_point newExpiry,
typename ClockT::time_point oldExpiry,
const std::unique_lock<SharedMutex>& g) final;
private:
SlidingWindow<DigestT> slidingWindow_;
};
} // namespace detail
} // namespace folly
...@@ -27,60 +27,55 @@ namespace detail { ...@@ -27,60 +27,55 @@ namespace detail {
template <typename DigestT> template <typename DigestT>
DigestBuilder<DigestT>::DigestBuilder(size_t bufferSize, size_t digestSize) DigestBuilder<DigestT>::DigestBuilder(size_t bufferSize, size_t digestSize)
: nextPos_(0), : bufferSize_(bufferSize), digestSize_(digestSize) {
digestSize_(digestSize), auto& cl = CacheLocality::system();
cpuLocalBuffersReady_(false), cpuLocalBuffers_.resize(cl.numCachesByLevel[0]);
buffer_(bufferSize) {} }
template <typename DigestT> template <typename DigestT>
DigestT DigestBuilder<DigestT>::buildSyncFree() const { DigestT DigestBuilder<DigestT>::build() {
std::vector<double> values; std::vector<std::vector<double>> valuesVec;
std::vector<std::unique_ptr<DigestT>> digestPtrs;
valuesVec.reserve(cpuLocalBuffers_.size());
digestPtrs.reserve(cpuLocalBuffers_.size());
for (auto& cpuLocalBuffer : cpuLocalBuffers_) {
SpinLockGuard g(cpuLocalBuffer.mutex);
valuesVec.push_back(std::move(cpuLocalBuffer.buffer));
if (cpuLocalBuffer.digest) {
digestPtrs.push_back(std::move(cpuLocalBuffer.digest));
}
}
std::vector<DigestT> digests; std::vector<DigestT> digests;
auto numElems = for (auto& digestPtr : digestPtrs) {
std::min(nextPos_.load(std::memory_order_relaxed), buffer_.size()); digests.push_back(std::move(*digestPtr));
values.insert(values.end(), buffer_.begin(), buffer_.begin() + numElems); }
if (cpuLocalBuffersReady_.load(std::memory_order_relaxed)) { size_t count = 0;
for (const auto& cpuLocalBuffer : cpuLocalBuffers_) { for (const auto& vec : valuesVec) {
if (cpuLocalBuffer.digest) { count += vec.size();
digests.push_back(*cpuLocalBuffer.digest); }
} if (count) {
values.insert( std::vector<double> values;
values.end(), values.reserve(count);
cpuLocalBuffer.buffer.begin(), for (const auto& vec : valuesVec) {
cpuLocalBuffer.buffer.end()); values.insert(values.end(), vec.begin(), vec.end());
} }
std::sort(values.begin(), values.end());
DigestT digest(digestSize_);
digests.push_back(digest.merge(values));
} }
std::sort(values.begin(), values.end());
DigestT digest(digestSize_);
digests.push_back(digest.merge(values));
return DigestT::merge(digests); return DigestT::merge(digests);
} }
template <typename DigestT> template <typename DigestT>
void DigestBuilder<DigestT>::append(double value) { void DigestBuilder<DigestT>::append(double value) {
auto pos = nextPos_.load(std::memory_order_relaxed);
if (pos < buffer_.size()) {
pos = nextPos_.fetch_add(1, std::memory_order_relaxed);
if (pos < buffer_.size()) {
buffer_[pos] = value;
if (pos == buffer_.size() - 1) {
// The shared buffer is full. From here on out, appends will go to a
// cpu local.
auto& cl = CacheLocality::system();
cpuLocalBuffers_.resize(cl.numCachesByLevel[0]);
cpuLocalBuffersReady_.store(true, std::memory_order_release);
}
return;
}
}
while (!cpuLocalBuffersReady_.load(std::memory_order_acquire)) {
}
auto which = AccessSpreader<>::current(cpuLocalBuffers_.size()); auto which = AccessSpreader<>::current(cpuLocalBuffers_.size());
auto& cpuLocalBuf = cpuLocalBuffers_[which]; auto& cpuLocalBuf = cpuLocalBuffers_[which];
SpinLockGuard g(cpuLocalBuf.mutex); SpinLockGuard g(cpuLocalBuf.mutex);
cpuLocalBuf.buffer.push_back(value); cpuLocalBuf.buffer.push_back(value);
if (cpuLocalBuf.buffer.size() > buffer_.size()) { if (cpuLocalBuf.buffer.size() == bufferSize_) {
std::sort(cpuLocalBuf.buffer.begin(), cpuLocalBuf.buffer.end()); std::sort(cpuLocalBuf.buffer.begin(), cpuLocalBuf.buffer.end());
if (!cpuLocalBuf.digest) { if (!cpuLocalBuf.digest) {
cpuLocalBuf.digest = std::make_unique<DigestT>(digestSize_); cpuLocalBuf.digest = std::make_unique<DigestT>(digestSize_);
......
...@@ -28,10 +28,13 @@ namespace detail { ...@@ -28,10 +28,13 @@ namespace detail {
* buffer writes and merge them in larger chunks. DigestBuilder buffers writes * buffer writes and merge them in larger chunks. DigestBuilder buffers writes
* to improve performance. * to improve performance.
* *
* The first bufferSize values are stored in a shared buffer. For cold stats, * Values are stored in a cpu local buffer. Hot stats will merge the cpu local
* this shared buffer minimizes memory usage at reasonable cpu cost. Warm stats * buffer into a cpu-local digest when the buffer size is reached.
* will fill the shared buffer, and begin to spill to cpu local buffers. Hot *
* stats will merge the cpu local buffers into cpu-local digests. * All methods in this class are thread safe, but it probably doesn't make sense
* for multiple threads to call build simultaneously. A typical usage is to
* buffer writes for a period of time, and then have one thread call build to
* merge the buffer into some other DigestT instance.
*/ */
template <typename DigestT> template <typename DigestT>
class DigestBuilder { class DigestBuilder {
...@@ -39,10 +42,10 @@ class DigestBuilder { ...@@ -39,10 +42,10 @@ class DigestBuilder {
explicit DigestBuilder(size_t bufferSize, size_t digestSize); explicit DigestBuilder(size_t bufferSize, size_t digestSize);
/* /*
* Builds a DigestT from the buffer in a sync free manner. It is the * Builds a DigestT from the buffer. All values used to build the DigestT are
* responsibility of the caller to synchronize with all appenders. * removed from the buffer.
*/ */
DigestT buildSyncFree() const; DigestT build();
/* /*
* Adds a value to the buffer. * Adds a value to the buffer.
...@@ -57,11 +60,9 @@ class DigestBuilder { ...@@ -57,11 +60,9 @@ class DigestBuilder {
std::unique_ptr<DigestT> digest; std::unique_ptr<DigestT> digest;
}; };
std::atomic<size_t> nextPos_;
std::vector<CpuLocalBuffer> cpuLocalBuffers_; std::vector<CpuLocalBuffer> cpuLocalBuffers_;
size_t bufferSize_;
size_t digestSize_; size_t digestSize_;
std::atomic<bool> cpuLocalBuffersReady_;
std::vector<double> buffer_;
}; };
} // namespace detail } // namespace detail
......
...@@ -51,7 +51,15 @@ std::vector<BucketT> SlidingWindow<BucketT>::get() const { ...@@ -51,7 +51,15 @@ std::vector<BucketT> SlidingWindow<BucketT>::get() const {
} }
template <typename BucketT> template <typename BucketT>
BucketT SlidingWindow<BucketT>::slide(size_t nBuckets) { void SlidingWindow<BucketT>::set(size_t idx, BucketT bucket) {
if (idx < buckets_.size()) {
idx = (curHead_ + idx) % buckets_.size();
buckets_[idx] = std::move(bucket);
}
}
template <typename BucketT>
void SlidingWindow<BucketT>::slide(size_t nBuckets) {
nBuckets = std::min(nBuckets, buckets_.size()); nBuckets = std::min(nBuckets, buckets_.size());
for (size_t i = 0; i < nBuckets; ++i) { for (size_t i = 0; i < nBuckets; ++i) {
if (curHead_ == 0) { if (curHead_ == 0) {
...@@ -61,7 +69,6 @@ BucketT SlidingWindow<BucketT>::slide(size_t nBuckets) { ...@@ -61,7 +69,6 @@ BucketT SlidingWindow<BucketT>::slide(size_t nBuckets) {
} }
buckets_[curHead_] = fn_(); buckets_[curHead_] = fn_();
} }
return buckets_[curHead_];
} }
} // namespace detail } // namespace detail
......
...@@ -38,11 +38,13 @@ class SlidingWindow { ...@@ -38,11 +38,13 @@ class SlidingWindow {
std::vector<BucketT> get() const; std::vector<BucketT> get() const;
void set(size_t idx, BucketT bucket);
/* /*
* Slides the SlidingWindow by nBuckets, inserting new buckets using the * Slides the SlidingWindow by nBuckets, inserting new buckets using the
* Function given during construction. Returns the new first bucket. * Function given during construction.
*/ */
BucketT slide(size_t nBuckets); void slide(size_t nBuckets);
private: private:
Function<BucketT(void)> fn_; Function<BucketT(void)> fn_;
......
/*
* Copyright 2013-present Facebook, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include <folly/stats/detail/BufferedStat-defs.h>
#include <folly/Range.h>
#include <folly/portability/GTest.h>
using namespace folly;
using namespace folly::detail;
const size_t kDigestSize = 100;
struct MockClock {
public:
using duration = std::chrono::steady_clock::duration;
using time_point = std::chrono::steady_clock::time_point;
static time_point now() {
return Now;
}
static time_point Now;
};
class SimpleDigest {
public:
explicit SimpleDigest(size_t sz) {
EXPECT_EQ(kDigestSize, sz);
}
SimpleDigest merge(Range<const double*> r) const {
SimpleDigest digest(100);
digest.values_ = values_;
for (auto it = r.begin(); it != r.end(); ++it) {
digest.values_.push_back(*it);
}
return digest;
}
static SimpleDigest merge(Range<const SimpleDigest*> r) {
SimpleDigest digest(100);
for (auto it = r.begin(); it != r.end(); ++it) {
for (auto value : it->values_) {
digest.values_.push_back(value);
}
}
return digest;
}
std::vector<double> getValues() const {
return values_;
}
bool empty() const {
return values_.empty();
}
private:
std::vector<double> values_;
};
MockClock::time_point MockClock::Now = MockClock::time_point{};
class BufferedDigestTest : public ::testing::Test {
protected:
std::unique_ptr<BufferedDigest<SimpleDigest, MockClock>> bd;
const size_t nBuckets = 60;
const size_t bufferSize = 1000;
const std::chrono::milliseconds bufferDuration{1000};
void SetUp() override {
MockClock::Now = MockClock::time_point{};
bd = std::make_unique<BufferedDigest<SimpleDigest, MockClock>>(
bufferDuration, bufferSize, kDigestSize);
}
};
TEST_F(BufferedDigestTest, Buffering) {
bd->append(0);
bd->append(1);
bd->append(2);
auto digest = bd->get();
EXPECT_TRUE(digest.empty());
}
TEST_F(BufferedDigestTest, PartiallyPassedExpiry) {
bd->append(0);
bd->append(1);
bd->append(2);
MockClock::Now += bufferDuration / 10;
auto digest = bd->get();
auto values = digest.getValues();
EXPECT_EQ(0, values[0]);
EXPECT_EQ(1, values[1]);
EXPECT_EQ(2, values[2]);
}
class BufferedSlidingWindowTest : public ::testing::Test {
protected:
std::unique_ptr<BufferedSlidingWindow<SimpleDigest, MockClock>> bsw;
const size_t nBuckets = 60;
const size_t bufferSize = 1000;
const std::chrono::milliseconds windowDuration{1000};
void SetUp() override {
MockClock::Now = MockClock::time_point{};
bsw = std::make_unique<BufferedSlidingWindow<SimpleDigest, MockClock>>(
nBuckets, windowDuration, bufferSize, kDigestSize);
}
};
TEST_F(BufferedSlidingWindowTest, Buffering) {
bsw->append(0);
bsw->append(1);
bsw->append(2);
auto digests = bsw->get();
EXPECT_EQ(0, digests.size());
}
TEST_F(BufferedSlidingWindowTest, PartiallyPassedExpiry) {
bsw->append(0);
bsw->append(1);
bsw->append(2);
MockClock::Now += windowDuration / 10;
auto digests = bsw->get();
EXPECT_EQ(1, digests.size());
EXPECT_EQ(3, digests[0].getValues().size());
for (double i = 0; i < 3; ++i) {
EXPECT_EQ(i, digests[0].getValues()[i]);
}
}
TEST_F(BufferedSlidingWindowTest, BufferingAfterSlide) {
MockClock::Now += std::chrono::milliseconds{1};
bsw->append(1);
auto digests = bsw->get();
EXPECT_EQ(0, digests.size());
}
TEST_F(BufferedSlidingWindowTest, TwoSlides) {
bsw->append(0);
MockClock::Now += windowDuration;
bsw->append(1);
MockClock::Now += windowDuration;
auto digests = bsw->get();
EXPECT_EQ(2, digests.size());
EXPECT_EQ(1, digests[0].getValues().size());
EXPECT_EQ(1, digests[0].getValues()[0]);
EXPECT_EQ(1, digests[1].getValues().size());
EXPECT_EQ(0, digests[1].getValues()[0]);
}
TEST_F(BufferedSlidingWindowTest, MultiWindowDurationSlide) {
bsw->append(0);
MockClock::Now += windowDuration * 2;
auto digests = bsw->get();
EXPECT_EQ(1, digests.size());
}
TEST_F(BufferedSlidingWindowTest, SlidePastWindow) {
bsw->append(0);
MockClock::Now += windowDuration * (nBuckets + 1);
auto digests = bsw->get();
EXPECT_EQ(0, digests.size());
}
...@@ -110,19 +110,19 @@ BENCHMARK_RELATIVE_NAMED_PARAM_MULTI(append, 10000x32, 10000, 32) ...@@ -110,19 +110,19 @@ BENCHMARK_RELATIVE_NAMED_PARAM_MULTI(append, 10000x32, 10000, 32)
* ============================================================================ * ============================================================================
* folly/stats/test/DigestBuilderBenchmark.cpp relative time/iter iters/s * folly/stats/test/DigestBuilderBenchmark.cpp relative time/iter iters/s
* ============================================================================ * ============================================================================
* append(1000x1) 45.44ns 22.01M * append(1000x1) 43.84ns 22.81M
* append(1000x2) 99.84% 45.52ns 21.97M * append(1000x2) 97.54% 44.95ns 22.25M
* append(1000x4) 96.65% 47.02ns 21.27M * append(1000x4) 96.14% 45.60ns 21.93M
* append(1000x8) 93.49% 48.61ns 20.57M * append(1000x8) 93.31% 46.99ns 21.28M
* append(1000x16) 46.88% 96.93ns 10.32M * append(1000x16) 44.73% 98.02ns 10.20M
* append(1000x32) 33.59% 135.30ns 7.39M * append(1000x32) 34.43% 127.33ns 7.85M
* ---------------------------------------------------------------------------- * ----------------------------------------------------------------------------
* append(10000x1) 46.12ns 21.68M * append(10000x1) 44.85ns 22.29M
* append(10000x2) 96.02% 48.03ns 20.82M * append(10000x2) 97.39% 46.06ns 21.71M
* append(10000x4) 95.39% 48.35ns 20.68M * append(10000x4) 94.25% 47.59ns 21.01M
* append(10000x8) 90.52% 50.95ns 19.63M * append(10000x8) 93.20% 48.13ns 20.78M
* append(10000x16) 43.39% 106.28ns 9.41M * append(10000x16) 50.75% 88.39ns 11.31M
* append(10000x32) 34.83% 132.41ns 7.55M * append(10000x32) 34.10% 131.52ns 7.60M
* ============================================================================ * ============================================================================
*/ */
......
...@@ -26,14 +26,15 @@ ...@@ -26,14 +26,15 @@
using namespace folly; using namespace folly;
using namespace folly::detail; using namespace folly::detail;
template <size_t MergeSize>
class SimpleDigest { class SimpleDigest {
public: public:
explicit SimpleDigest(size_t sz) : sz_(sz) {} explicit SimpleDigest(size_t sz) : sz_(sz) {}
SimpleDigest merge(Range<const double*> r) const { SimpleDigest merge(Range<const double*> r) const {
EXPECT_EQ(1000, r.size()); EXPECT_EQ(MergeSize, r.size());
for (size_t i = 0; i < 1000; ++i) { for (size_t i = 0; i < MergeSize; ++i) {
EXPECT_GE(1000, r[i]); EXPECT_GE(MergeSize, r[i]);
} }
return *this; return *this;
} }
...@@ -51,8 +52,24 @@ class SimpleDigest { ...@@ -51,8 +52,24 @@ class SimpleDigest {
int64_t sz_; int64_t sz_;
}; };
TEST(DigestBuilder, Basic) { TEST(DigestBuilder, SingleThreadUnfilledBuffer) {
DigestBuilder<SimpleDigest> builder(1000, 100); DigestBuilder<SimpleDigest<999>> builder(1000, 100);
for (int i = 0; i < 999; ++i) {
builder.append(i);
}
EXPECT_EQ(100, builder.build().getSize());
}
TEST(DigestBuilder, SingleThreadFilledBuffer) {
DigestBuilder<SimpleDigest<1000>> builder(1000, 100);
for (int i = 0; i < 1000; ++i) {
builder.append(i);
}
EXPECT_EQ(100, builder.build().getSize());
}
TEST(DigestBuilder, MultipleThreads) {
DigestBuilder<SimpleDigest<1000>> builder(1000, 100);
std::vector<std::thread> threads; std::vector<std::thread> threads;
for (int i = 0; i < 10; ++i) { for (int i = 0; i < 10; ++i) {
threads.push_back(std::thread([i, &builder]() { threads.push_back(std::thread([i, &builder]() {
...@@ -65,5 +82,5 @@ TEST(DigestBuilder, Basic) { ...@@ -65,5 +82,5 @@ TEST(DigestBuilder, Basic) {
thread.join(); thread.join();
} }
EXPECT_EQ(100, builder.buildSyncFree().getSize()); EXPECT_EQ(100, builder.build().getSize());
} }
...@@ -4,6 +4,7 @@ CPPFLAGS = -I$(top_srcdir)/test/gtest/googletest/include ...@@ -4,6 +4,7 @@ CPPFLAGS = -I$(top_srcdir)/test/gtest/googletest/include
ldadd = $(top_builddir)/test/libfollytestmain.la ldadd = $(top_builddir)/test/libfollytestmain.la
TESTS = \ TESTS = \
buffered_stat_test \
digest_builder_test \ digest_builder_test \
histogram_test \ histogram_test \
sliding_window_test \ sliding_window_test \
...@@ -11,6 +12,9 @@ TESTS = \ ...@@ -11,6 +12,9 @@ TESTS = \
check_PROGRAMS = $(TESTS) check_PROGRAMS = $(TESTS)
buffered_stat_test_SOURCES = BufferedStatTest.cpp
buffered_stat_test_LDADD = $(ldadd)
digest_builder_test_SOURCES = DigestBuilderTest.cpp digest_builder_test_SOURCES = DigestBuilderTest.cpp
digest_builder_test_LDADD = $(ldadd) digest_builder_test_LDADD = $(ldadd)
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment