Commit 21352de3 authored by Marc Celani's avatar Marc Celani Committed by Facebook Github Bot

Add sum, mean, and count to TDigest

Summary: This diff adds reads for sum, mean, and count on TDigest. The results are calculated at merge time, to make reads O(1).

Reviewed By: anakryiko

Differential Revision: D7639133

fbshipit-source-id: bb8f203ec0ed3197f090d2ce9223a2e9d1bca752
parent 87694f24
...@@ -73,13 +73,13 @@ TDigest TDigest::merge(Range<const double*> sortedValues) const { ...@@ -73,13 +73,13 @@ TDigest TDigest::merge(Range<const double*> sortedValues) const {
TDigest result(maxSize_); TDigest result(maxSize_);
result.total_ = total_ + sortedValues.size(); result.count_ = count_ + sortedValues.size();
std::vector<Centroid> compressed; std::vector<Centroid> compressed;
compressed.reserve(2 * maxSize_); compressed.reserve(2 * maxSize_);
double q_0_times_total = 0.0; double q_0_times_count = 0.0;
double q_limit_times_total = detail::k_to_q(1, maxSize_) * result.total_; double q_limit_times_count = detail::k_to_q(1, maxSize_) * result.count_;
auto it_centroids = centroids_.begin(); auto it_centroids = centroids_.begin();
auto it_sortedValues = sortedValues.begin(); auto it_sortedValues = sortedValues.begin();
...@@ -92,6 +92,8 @@ TDigest TDigest::merge(Range<const double*> sortedValues) const { ...@@ -92,6 +92,8 @@ TDigest TDigest::merge(Range<const double*> sortedValues) const {
cur = Centroid(*it_sortedValues++, 1.0); cur = Centroid(*it_sortedValues++, 1.0);
} }
result.sum_ += cur.mean() * cur.weight();
while (it_centroids != centroids_.end() || while (it_centroids != centroids_.end() ||
it_sortedValues != sortedValues.end()) { it_sortedValues != sortedValues.end()) {
Centroid next; Centroid next;
...@@ -104,17 +106,19 @@ TDigest TDigest::merge(Range<const double*> sortedValues) const { ...@@ -104,17 +106,19 @@ TDigest TDigest::merge(Range<const double*> sortedValues) const {
next = Centroid(*it_sortedValues++, 1.0); next = Centroid(*it_sortedValues++, 1.0);
} }
double q_times_total = q_0_times_total + cur.weight() + next.weight(); result.sum_ += next.mean() * next.weight();
double q_times_count = q_0_times_count + cur.weight() + next.weight();
if (q_times_total <= q_limit_times_total) { if (q_times_count <= q_limit_times_count) {
cur.add(next); cur.add(next);
} else { } else {
compressed.push_back(cur); compressed.push_back(cur);
q_0_times_total += cur.weight(); q_0_times_count += cur.weight();
double q_to_k_res = double q_to_k_res =
detail::q_to_k(q_0_times_total / result.total_, maxSize_); detail::q_to_k(q_0_times_count / result.count_, maxSize_);
q_limit_times_total = q_limit_times_count =
detail::k_to_q(q_to_k_res + 1, maxSize_) * result.total_; detail::k_to_q(q_to_k_res + 1, maxSize_) * result.count_;
cur = next; cur = next;
} }
} }
...@@ -136,12 +140,14 @@ TDigest TDigest::merge(Range<const TDigest*> digests) { ...@@ -136,12 +140,14 @@ TDigest TDigest::merge(Range<const TDigest*> digests) {
std::vector<Centroid> centroids; std::vector<Centroid> centroids;
centroids.reserve(nCentroids); centroids.reserve(nCentroids);
double total = 0; double count = 0;
double sum = 0;
for (auto it = digests.begin(); it != digests.end(); it++) { for (auto it = digests.begin(); it != digests.end(); it++) {
for (const auto& centroid : it->centroids_) { for (const auto& centroid : it->centroids_) {
centroids.push_back(centroid); centroids.push_back(centroid);
total += centroid.weight(); count += centroid.weight();
sum += centroid.mean() * centroid.weight();
} }
} }
std::sort(centroids.begin(), centroids.end()); std::sort(centroids.begin(), centroids.end());
...@@ -150,28 +156,29 @@ TDigest TDigest::merge(Range<const TDigest*> digests) { ...@@ -150,28 +156,29 @@ TDigest TDigest::merge(Range<const TDigest*> digests) {
std::vector<Centroid> compressed; std::vector<Centroid> compressed;
compressed.reserve(2 * maxSize); compressed.reserve(2 * maxSize);
double q_0_times_total = 0.0; double q_0_times_count = 0.0;
double q_limit_times_total = detail::k_to_q(1, maxSize) * total; double q_limit_times_count = detail::k_to_q(1, maxSize) * count;
Centroid cur = centroids.front(); Centroid cur = centroids.front();
for (auto it = centroids.begin() + 1; it != centroids.end(); ++it) { for (auto it = centroids.begin() + 1; it != centroids.end(); ++it) {
double q_times_total = q_0_times_total + cur.weight() + it->weight(); double q_times_count = q_0_times_count + cur.weight() + it->weight();
if (q_times_total <= q_limit_times_total) { if (q_times_count <= q_limit_times_count) {
cur.add(*it); cur.add(*it);
} else { } else {
compressed.push_back(cur); compressed.push_back(cur);
q_0_times_total += cur.weight(); q_0_times_count += cur.weight();
double q_to_k_res = detail::q_to_k(q_0_times_total / total, maxSize); double q_to_k_res = detail::q_to_k(q_0_times_count / count, maxSize);
q_limit_times_total = detail::k_to_q(q_to_k_res + 1, maxSize) * total; q_limit_times_count = detail::k_to_q(q_to_k_res + 1, maxSize) * count;
cur = *it; cur = *it;
} }
} }
compressed.push_back(cur); compressed.push_back(cur);
TDigest result(maxSize); TDigest result(maxSize);
result.total_ = total; result.count_ = count;
result.sum_ = sum;
result.centroids_ = std::move(compressed); result.centroids_ = std::move(compressed);
return result; return result;
} }
...@@ -180,7 +187,7 @@ double TDigest::estimateQuantile(double q) const { ...@@ -180,7 +187,7 @@ double TDigest::estimateQuantile(double q) const {
if (centroids_.empty()) { if (centroids_.empty()) {
return 0.0; return 0.0;
} }
double rank = q * total_; double rank = q * count_;
size_t pos; size_t pos;
double t; double t;
...@@ -189,7 +196,7 @@ double TDigest::estimateQuantile(double q) const { ...@@ -189,7 +196,7 @@ double TDigest::estimateQuantile(double q) const {
return centroids_.back().mean(); return centroids_.back().mean();
} }
pos = 0; pos = 0;
t = total_; t = count_;
for (auto rit = centroids_.rbegin(); rit != centroids_.rend(); ++rit) { for (auto rit = centroids_.rbegin(); rit != centroids_.rend(); ++rit) {
t -= rit->weight(); t -= rit->weight();
if (rank >= t) { if (rank >= t) {
......
...@@ -47,7 +47,8 @@ namespace folly { ...@@ -47,7 +47,8 @@ namespace folly {
*/ */
class TDigest { class TDigest {
public: public:
explicit TDigest(size_t maxSize = 100) : maxSize_(maxSize), total_(0.0) { explicit TDigest(size_t maxSize = 100)
: maxSize_(maxSize), sum_(0.0), count_(0.0) {
centroids_.reserve(maxSize); centroids_.reserve(maxSize);
} }
...@@ -68,6 +69,18 @@ class TDigest { ...@@ -68,6 +69,18 @@ class TDigest {
*/ */
double estimateQuantile(double q) const; double estimateQuantile(double q) const;
double mean() const {
return count_ ? sum_ / count_ : 0;
}
double sum() const {
return sum_;
}
double count() const {
return count_;
}
private: private:
struct Centroid { struct Centroid {
public: public:
...@@ -95,7 +108,8 @@ class TDigest { ...@@ -95,7 +108,8 @@ class TDigest {
std::vector<Centroid> centroids_; std::vector<Centroid> centroids_;
size_t maxSize_; size_t maxSize_;
double total_; double sum_;
double count_;
}; };
} // namespace folly } // namespace folly
...@@ -136,31 +136,31 @@ BENCHMARK_RELATIVE_NAMED_PARAM(estimateQuantile, 1000_p999, 1000, 0.999) ...@@ -136,31 +136,31 @@ BENCHMARK_RELATIVE_NAMED_PARAM(estimateQuantile, 1000_p999, 1000, 0.999)
* ============================================================================ * ============================================================================
* folly/stats/test/TDigestBenchmark.cpp relative time/iter iters/s * folly/stats/test/TDigestBenchmark.cpp relative time/iter iters/s
* ============================================================================ * ============================================================================
* merge(100x1) 4.08us 244.93K * merge(100x1) 4.11us 243.60K
* merge(100x5) 52.76% 7.74us 129.23K * merge(100x5) 52.31% 7.85us 127.43K
* merge(100x10) 30.58% 13.35us 74.91K * merge(100x10) 30.13% 13.63us 73.39K
* merge(1000x1) 12.42% 32.88us 30.42K * merge(1000x1) 12.26% 33.49us 29.86K
* merge(1000x5) 6.48% 63.03us 15.87K * merge(1000x5) 6.23% 65.94us 15.17K
* merge(1000x10) 3.81% 107.04us 9.34K * merge(1000x10) 3.64% 112.76us 8.87K
* ---------------------------------------------------------------------------- * ----------------------------------------------------------------------------
* mergeDigests(100x60) 382.79us 2.61K * mergeDigests(100x60) 381.44us 2.62K
* mergeDigests(1000x60) 9.16% 4.18ms 239.40 * mergeDigests(1000x60) 9.22% 4.14ms 241.73
* ---------------------------------------------------------------------------- * ----------------------------------------------------------------------------
* estimateQuantile(100x1_p001) 8.86ns 112.84M * estimateQuantile(100x1_p001) 8.48ns 117.96M
* estimateQuantile(100_p01) 57.10% 15.52ns 64.43M * estimateQuantile(100_p01) 57.14% 14.84ns 67.41M
* estimateQuantile(100_p25) 12.60% 70.34ns 14.22M * estimateQuantile(100_p25) 11.95% 70.96ns 14.09M
* estimateQuantile(100_p50) 9.83% 90.13ns 11.10M * estimateQuantile(100_p50) 9.19% 92.27ns 10.84M
* estimateQuantile(100_p75) 12.57% 70.52ns 14.18M * estimateQuantile(100_p75) 12.03% 70.49ns 14.19M
* estimateQuantile(100_p99) 60.29% 14.70ns 68.03M * estimateQuantile(100_p99) 65.68% 12.91ns 77.47M
* estimateQuantile(100_p999) 102.06% 8.68ns 115.16M * estimateQuantile(100_p999) 97.06% 8.73ns 114.49M
* ---------------------------------------------------------------------------- * ----------------------------------------------------------------------------
* estimateQuantile(1000_p001) 23.67% 37.43ns 26.71M * estimateQuantile(1000_p001) 23.30% 36.38ns 27.49M
* estimateQuantile(1000_p01) 6.80% 130.26ns 7.68M * estimateQuantile(1000_p01) 6.46% 131.30ns 7.62M
* estimateQuantile(1000_p25) 1.56% 569.73ns 1.76M * estimateQuantile(1000_p25) 1.51% 560.08ns 1.79M
* estimateQuantile(1000_p50) 1.13% 786.43ns 1.27M * estimateQuantile(1000_p50) 1.08% 781.97ns 1.28M
* estimateQuantile(1000_p75) 1.57% 564.81ns 1.77M * estimateQuantile(1000_p75) 1.49% 567.33ns 1.76M
* estimateQuantile(1000_p99) 6.86% 129.28ns 7.74M * estimateQuantile(1000_p99) 6.42% 131.99ns 7.58M
* estimateQuantile(1000_p999) 27.21% 32.57ns 30.71M * estimateQuantile(1000_p999) 27.09% 31.30ns 31.95M
* ============================================================================ * ============================================================================
*/ */
......
...@@ -41,6 +41,10 @@ TEST(TDigest, Basic) { ...@@ -41,6 +41,10 @@ TEST(TDigest, Basic) {
digest = digest.merge(values); digest = digest.merge(values);
EXPECT_EQ(100, digest.count());
EXPECT_EQ(5050, digest.sum());
EXPECT_EQ(50.5, digest.mean());
EXPECT_EQ(0.6, digest.estimateQuantile(0.001)); EXPECT_EQ(0.6, digest.estimateQuantile(0.001));
EXPECT_EQ(2.0 - 0.5, digest.estimateQuantile(0.01)); EXPECT_EQ(2.0 - 0.5, digest.estimateQuantile(0.01));
EXPECT_EQ(51.0 - 0.5, digest.estimateQuantile(0.5)); EXPECT_EQ(51.0 - 0.5, digest.estimateQuantile(0.5));
...@@ -65,6 +69,10 @@ TEST(TDigest, Merge) { ...@@ -65,6 +69,10 @@ TEST(TDigest, Merge) {
digest = digest.merge(values); digest = digest.merge(values);
EXPECT_EQ(200, digest.count());
EXPECT_EQ(20100, digest.sum());
EXPECT_EQ(100.5, digest.mean());
EXPECT_EQ(0.7, digest.estimateQuantile(0.001)); EXPECT_EQ(0.7, digest.estimateQuantile(0.001));
EXPECT_EQ(4.0 - 1.5, digest.estimateQuantile(0.01)); EXPECT_EQ(4.0 - 1.5, digest.estimateQuantile(0.01));
EXPECT_EQ(102.0 - 1.5, digest.estimateQuantile(0.5)); EXPECT_EQ(102.0 - 1.5, digest.estimateQuantile(0.5));
...@@ -80,6 +88,10 @@ TEST(TDigest, MergeSmall) { ...@@ -80,6 +88,10 @@ TEST(TDigest, MergeSmall) {
digest = digest.merge(values); digest = digest.merge(values);
EXPECT_EQ(1, digest.count());
EXPECT_EQ(1, digest.sum());
EXPECT_EQ(1, digest.mean());
EXPECT_EQ(1.0, digest.estimateQuantile(0.001)); EXPECT_EQ(1.0, digest.estimateQuantile(0.001));
EXPECT_EQ(1.0, digest.estimateQuantile(0.01)); EXPECT_EQ(1.0, digest.estimateQuantile(0.01));
EXPECT_EQ(1.0, digest.estimateQuantile(0.5)); EXPECT_EQ(1.0, digest.estimateQuantile(0.5));
...@@ -97,6 +109,10 @@ TEST(TDigest, MergeLarge) { ...@@ -97,6 +109,10 @@ TEST(TDigest, MergeLarge) {
digest = digest.merge(values); digest = digest.merge(values);
EXPECT_EQ(1000, digest.count());
EXPECT_EQ(500500, digest.sum());
EXPECT_EQ(500.5, digest.mean());
EXPECT_EQ(1.5, digest.estimateQuantile(0.001)); EXPECT_EQ(1.5, digest.estimateQuantile(0.001));
EXPECT_EQ(10.5, digest.estimateQuantile(0.01)); EXPECT_EQ(10.5, digest.estimateQuantile(0.01));
EXPECT_EQ(500.5, digest.estimateQuantile(0.5)); EXPECT_EQ(500.5, digest.estimateQuantile(0.5));
...@@ -118,6 +134,10 @@ TEST(TDigest, MergeLargeAsDigests) { ...@@ -118,6 +134,10 @@ TEST(TDigest, MergeLargeAsDigests) {
digest = TDigest::merge(digests); digest = TDigest::merge(digests);
EXPECT_EQ(1000, digest.count());
EXPECT_EQ(500500, digest.sum());
EXPECT_EQ(500.5, digest.mean());
EXPECT_EQ(1.5, digest.estimateQuantile(0.001)); EXPECT_EQ(1.5, digest.estimateQuantile(0.001));
EXPECT_EQ(10.5, digest.estimateQuantile(0.01)); EXPECT_EQ(10.5, digest.estimateQuantile(0.01));
EXPECT_EQ(500.5, digest.estimateQuantile(0.5)); EXPECT_EQ(500.5, digest.estimateQuantile(0.5));
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment