Commit 69286490 authored by Marc Celani's avatar Marc Celani Committed by Facebook Github Bot

QuantileEstimator

Summary: Introduces two QuantileEstimators. Both estimators use the TDigest for estimating quantiles, and giving accurate sum, count, and mean. One estimator uses a single digest, and the other uses a SlidingWindow of 60 digests. All writes to the data structure are buffered for 1 second.

Reviewed By: anakryiko

Differential Revision: D7645824

fbshipit-source-id: 4f1f2c25388c3280f148591502449f857c6467d3
parent d767d0b5
......@@ -618,6 +618,7 @@ if (BUILD_TESTS)
TEST buffered_stat_test SOURCES BufferedStatTest.cpp
TEST digest_builder_test SOURCES DigestBuilderTest.cpp
TEST histogram_test SOURCES HistogramTest.cpp
TEST quantile_estimator_test SOURCES QuantileEstimatorTest.cpp
TEST sliding_window_test SOURCES SlidingWindowTest.cpp
TEST tdigest_test SOURCES TDigestTest.cpp
TEST timeseries_histogram_test SOURCES TimeseriesHistogramTest.cpp
......
......@@ -453,6 +453,8 @@ nobase_follyinclude_HEADERS = \
stats/Histogram.h \
stats/MultiLevelTimeSeries-defs.h \
stats/MultiLevelTimeSeries.h \
stats/QuantileEstimator-defs.h \
stats/QuantileEstimator.h \
stats/TDigest.h \
stats/TimeseriesHistogram-defs.h \
stats/TimeseriesHistogram.h \
......@@ -642,6 +644,7 @@ libfolly_la_SOURCES = \
stats/BucketedTimeSeries.cpp \
stats/Histogram.cpp \
stats/MultiLevelTimeSeries.cpp \
stats/QuantileEstimator.cpp \
stats/TDigest.cpp \
stats/TimeseriesHistogram.cpp \
synchronization/AsymmetricMemoryBarrier.cpp \
......
/*
* Copyright 2018-present Facebook, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include <folly/stats/QuantileEstimator.h>
#include <folly/stats/detail/BufferedStat-defs.h>
namespace folly {
namespace detail {
QuantileEstimates estimatesFromDigest(
const TDigest& digest,
Range<const double*> quantiles) {
QuantileEstimates result;
result.quantiles.reserve(quantiles.size());
result.sum = digest.sum();
result.count = digest.count();
for (auto it = quantiles.begin(); it != quantiles.end(); ++it) {
result.quantiles.push_back(
std::make_pair(*it, digest.estimateQuantile(*it)));
}
return result;
}
} // namespace detail
template <typename ClockT>
SimpleQuantileEstimator<ClockT>::SimpleQuantileEstimator()
: bufferedDigest_(std::chrono::seconds{1}, 1000, 100) {}
template <typename ClockT>
QuantileEstimates SimpleQuantileEstimator<ClockT>::estimateQuantiles(
Range<const double*> quantiles) {
auto digest = bufferedDigest_.get();
return detail::estimatesFromDigest(digest, quantiles);
}
template <typename ClockT>
void SimpleQuantileEstimator<ClockT>::addValue(double value) {
bufferedDigest_.append(value);
}
template <typename ClockT>
SlidingWindowQuantileEstimator<ClockT>::SlidingWindowQuantileEstimator(
std::chrono::seconds windowDuration,
size_t nWindows)
: bufferedSlidingWindow_(nWindows, windowDuration, 1000, 100) {}
template <typename ClockT>
QuantileEstimates SlidingWindowQuantileEstimator<ClockT>::estimateQuantiles(
Range<const double*> quantiles) {
auto digests = bufferedSlidingWindow_.get();
auto digest = TDigest::merge(digests);
return detail::estimatesFromDigest(digest, quantiles);
}
template <typename ClockT>
void SlidingWindowQuantileEstimator<ClockT>::addValue(double value) {
bufferedSlidingWindow_.append(value);
}
} // namespace folly
/*
* Copyright 2018-present Facebook, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include <folly/stats/QuantileEstimator-defs.h>
namespace folly {
template class SimpleQuantileEstimator<std::chrono::steady_clock>;
template class SlidingWindowQuantileEstimator<std::chrono::steady_clock>;
} // namespace folly
/*
* Copyright 2018-present Facebook, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#pragma once
#include <folly/stats/TDigest.h>
#include <folly/stats/detail/BufferedStat.h>
namespace folly {
struct QuantileEstimates {
public:
double sum;
double count;
// vector of {quantile, value}
std::vector<std::pair<double, double>> quantiles;
};
class QuantileEstimator {
public:
virtual ~QuantileEstimator() {}
virtual QuantileEstimates estimateQuantiles(
Range<const double*> quantiles) = 0;
virtual void addValue(double value) = 0;
};
/*
* A QuantileEstimator that buffers writes for 1 second.
*/
template <typename ClockT = std::chrono::steady_clock>
class SimpleQuantileEstimator : public QuantileEstimator {
public:
SimpleQuantileEstimator();
QuantileEstimates estimateQuantiles(Range<const double*> quantiles) override;
void addValue(double value) override;
private:
detail::BufferedDigest<TDigest, ClockT> bufferedDigest_;
};
/*
* A QuantileEstimator that keeps values for nWindows * windowDuration (see
* constructor). Values are buffered for windowDuration.
*/
template <typename ClockT = std::chrono::steady_clock>
class SlidingWindowQuantileEstimator : public QuantileEstimator {
public:
SlidingWindowQuantileEstimator(
std::chrono::seconds windowDuration,
size_t nWindows = 60);
QuantileEstimates estimateQuantiles(Range<const double*> quantiles) override;
void addValue(double value) override;
private:
detail::BufferedSlidingWindow<TDigest, ClockT> bufferedSlidingWindow_;
};
extern template class SimpleQuantileEstimator<std::chrono::steady_clock>;
extern template class SlidingWindowQuantileEstimator<std::chrono::steady_clock>;
} // namespace folly
......@@ -7,6 +7,7 @@ TESTS = \
buffered_stat_test \
digest_builder_test \
histogram_test \
quantile_estimator_test \
sliding_window_test \
tdigest_test
......@@ -21,6 +22,9 @@ digest_builder_test_LDADD = $(ldadd)
histogram_test_SOURCES = HistogramTest.cpp
histogram_test_LDADD = $(ldadd)
quantile_estimator_test_SOURCES = QuantileEstimatorTest.cpp
quantile_estimator_test_LDADD = $(ldadd)
sliding_window_test_SOURCES = SlidingWindowTest.cpp
sliding_window_test_LDADD = $(ldadd)
......
/*
* Copyright 2018-present Facebook, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include <folly/stats/QuantileEstimator-defs.h>
#include <folly/portability/GTest.h>
using namespace folly;
struct MockClock {
public:
using duration = std::chrono::steady_clock::duration;
using time_point = std::chrono::steady_clock::time_point;
static time_point now() {
return Now;
}
static time_point Now;
};
MockClock::time_point MockClock::Now = MockClock::time_point{};
class QuantileEstimatorTest
: public ::testing::TestWithParam<std::shared_ptr<QuantileEstimator>> {};
TEST_P(QuantileEstimatorTest, EstimateQuantiles) {
auto estimator = GetParam();
for (size_t i = 1; i <= 100; ++i) {
estimator->addValue(i);
}
MockClock::Now += std::chrono::seconds{1};
std::vector<double> quantiles = {0.001, 0.01, 0.5, 0.99, 0.999};
auto estimates = estimator->estimateQuantiles(quantiles);
EXPECT_EQ(5050, estimates.sum);
EXPECT_EQ(100, estimates.count);
EXPECT_EQ(0.001, estimates.quantiles[0].first);
EXPECT_EQ(0.01, estimates.quantiles[1].first);
EXPECT_EQ(0.5, estimates.quantiles[2].first);
EXPECT_EQ(0.99, estimates.quantiles[3].first);
EXPECT_EQ(0.999, estimates.quantiles[4].first);
EXPECT_EQ(0.6, estimates.quantiles[0].second);
EXPECT_EQ(2.0 - 0.5, estimates.quantiles[1].second);
EXPECT_EQ(50.375, estimates.quantiles[2].second);
EXPECT_EQ(100.0 - 0.5, estimates.quantiles[3].second);
EXPECT_EQ(100.4, estimates.quantiles[4].second);
}
INSTANTIATE_TEST_CASE_P(
SimpleQuantileEstimator,
QuantileEstimatorTest,
::testing::Values(new SimpleQuantileEstimator<MockClock>()));
INSTANTIATE_TEST_CASE_P(
SlidingWindowQuantileEstimator,
QuantileEstimatorTest,
::testing::Values(new SlidingWindowQuantileEstimator<MockClock>(
std::chrono::seconds{1})));
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment