Add AccessSpreader<>::cachedCurrent()

Summary:
`AccessSpreader::cachedCurrent()` caches the result of `AccessSpreader::getcpuFunc()` for 32 calls in a thread-local. The cached function takes 2 ns, where the current call takes 12 ns. This comes at the cost of being imprecise when threads migrate to a new cpu.

I chose 32 as the number of calls because it only has a 10% overhead over never refreshing (2.05 ns vs 1.83 ns), where 16 has a 30% overhead, and it performs just as well as 64.

Reviewed By: ot

Differential Revision: D10151009

fbshipit-source-id: 07ed292dfdcdcedcb74c24279f7773a80ad09348

folly/concurrency/CacheLocality.h

@@ -239,6 +239,24 @@ struct AccessSpreader {
                               [cpu % kMaxCpus];
   }
 
+#ifdef FOLLY_TLS
+  /// Returns the stripe associated with the current CPU.  The returned
+  /// value will be < numStripes.
+  /// This function caches the current cpu in a thread-local variable for a
+  /// certain small number of calls, which can make the result imprecise, but
+  /// it is more efficient (amortized 2 ns on my dev box, compared to 12 ns for
+  /// current()).
+  static size_t cachedCurrent(size_t numStripes) {
+    return widthAndCpuToStripe[std::min(size_t(kMaxCpus), numStripes)]
+                              [cpuCache.cpu()];
+  }
+#else
+  /// Fallback implementation when thread-local storage isn't available.
+  static size_t cachedCurrent(size_t numStripes) {
+    return current(numStripes);
+  }
+#endif
+
  private:
   /// If there are more cpus than this nothing will crash, but there
   /// might be unnecessary sharing
@@ -267,6 +285,30 @@ struct AccessSpreader {
   /// array.
   static CompactStripe widthAndCpuToStripe[kMaxCpus + 1][kMaxCpus];
 
+  /// Caches the current CPU and refreshes the cache every so often.
+  class CpuCache {
+   public:
+    unsigned cpu() {
+      if (UNLIKELY(cachedCpuUses_-- == 0)) {
+        unsigned cpu;
+        AccessSpreader::getcpuFunc(&cpu, nullptr, nullptr);
+        cachedCpu_ = cpu % kMaxCpus;
+        cachedCpuUses_ = kMaxCachedCpuUses - 1;
+      }
+      return cachedCpu_;
+    }
+
+   private:
+    static constexpr unsigned kMaxCachedCpuUses = 32;
+
+    unsigned cachedCpu_{0};
+    unsigned cachedCpuUses_{0};
+  };
+
+#ifdef FOLLY_TLS
+  static FOLLY_TLS CpuCache cpuCache;
+#endif
+
   static bool initialized;
 
   /// Returns the best getcpu implementation for Atom
@@ -331,6 +373,12 @@ template <template <typename> class Atom>
 typename AccessSpreader<Atom>::CompactStripe
     AccessSpreader<Atom>::widthAndCpuToStripe[kMaxCpus + 1][kMaxCpus] = {};
 
+#ifdef FOLLY_TLS
+template <template <typename> class Atom>
+FOLLY_TLS
+    typename AccessSpreader<Atom>::CpuCache AccessSpreader<Atom>::cpuCache;
+#endif
+
 template <template <typename> class Atom>
 bool AccessSpreader<Atom>::initialized = AccessSpreader<Atom>::initialize();
 

folly/concurrency/test/CacheLocalityBenchmark.cpp

@@ -53,6 +53,18 @@ DECLARE_SPREADER_TAG(
     CacheLocality::system<>(),
     folly::FallbackGetcpu<HashingThreadId>::getcpu)
 
+// Allow us to run cachedCurrent() in the benchmark function easily.
+namespace {
+template <typename dummy>
+struct CachedCurrentTag {};
+} // namespace
+namespace folly {
+template <>
+size_t AccessSpreader<CachedCurrentTag>::current(size_t numStripes) {
+  return AccessSpreader<std::atomic>::cachedCurrent(numStripes);
+}
+} // namespace folly
+
 BENCHMARK(AccessSpreaderUse, iters) {
   for (unsigned long i = 0; i < iters; ++i) {
     auto x = AccessSpreader<>::current(16);
@@ -60,6 +72,30 @@ BENCHMARK(AccessSpreaderUse, iters) {
   }
 }
 
+BENCHMARK(CachedAccessSpreaderUse, iters) {
+  for (unsigned long i = 0; i < iters; ++i) {
+    auto x = AccessSpreader<>::cachedCurrent(16);
+    folly::doNotOptimizeAway(x);
+  }
+}
+
+BENCHMARK(BaselineAtomicIncrement, iters) {
+  std::atomic<int> value;
+  for (unsigned long i = 0; i < iters; ++i) {
+    ++value;
+    folly::doNotOptimizeAway(value);
+  }
+}
+
+BENCHMARK(CachedAccessSpreaderAtomicIncrement, iters) {
+  std::array<std::atomic<int>, 64> values;
+  for (unsigned long i = 0; i < iters; ++i) {
+    auto x = AccessSpreader<>::cachedCurrent(64);
+    ++values[x];
+    folly::doNotOptimizeAway(values[x]);
+  }
+}
+
 // Benchmark scores here reflect the time for 32 threads to perform an
 // atomic increment on a dual-socket E5-2660 @ 2.2Ghz.  Surprisingly,
 // if we don't separate the counters onto unique 128 byte stripes the
@@ -92,46 +128,55 @@ BENCHMARK(AccessSpreaderUse, iters) {
 // sudo nice -n -20 buck-out/gen/folly/test/cache_locality_test
 //     --benchmark --bm_min_iters=1000000
 // ============================================================================
-// folly/test/CacheLocalityTest.cpp                relative  time/iter  iters/s
+// folly/concurrency/test/CacheLocalityBenchmark.cpprelative  time/iter  iters/s
 // ============================================================================
-// AccessSpreaderUse                                           11.53ns   86.75M
+// AccessSpreaderUse                                           11.51ns   86.87M
+// CachedAccessSpreaderUse                                      1.98ns  490.03M
+// BaselineAtomicIncrement                                     10.37ns   96.43M
+// CachedAccessSpreaderAtomicIncrement                         11.43ns   87.50M
 // ----------------------------------------------------------------------------
-// contentionAtWidthGetcpu(1_stripe_0_work)                     1.05us  954.62K
-// contentionAtWidthGetcpu(2_stripe_0_work)                   521.93ns    1.92M
-// contentionAtWidthGetcpu(4_stripe_0_work)                   306.91ns    3.26M
-// contentionAtWidthGetcpu(8_stripe_0_work)                   150.86ns    6.63M
-// contentionAtWidthGetcpu(16_stripe_0_work)                   86.20ns   11.60M
-// contentionAtWidthGetcpu(32_stripe_0_work)                   35.90ns   27.85M
-// contentionAtWidthGetcpu(64_stripe_0_work)                   36.76ns   27.21M
-// contentionAtWidthThreadLocal(2_stripe_0_work)              303.88ns    3.29M
-// contentionAtWidthThreadLocal(4_stripe_0_work)              225.62ns    4.43M
-// contentionAtWidthThreadLocal(8_stripe_0_work)              106.88ns    9.36M
-// contentionAtWidthThreadLocal(16_stripe_0_work)              72.82ns   13.73M
-// contentionAtWidthThreadLocal(32_stripe_0_work)              26.53ns   37.70M
-// contentionAtWidthThreadLocal(64_stripe_0_work)              25.02ns   39.97M
-// contentionAtWidthPthreadSelf(2_stripe_0_work)              363.34ns    2.75M
-// contentionAtWidthPthreadSelf(4_stripe_0_work)              247.74ns    4.04M
-// contentionAtWidthPthreadSelf(8_stripe_0_work)              109.03ns    9.17M
-// contentionAtWidthPthreadSelf(16_stripe_0_work)              93.61ns   10.68M
-// contentionAtWidthPthreadSelf(32_stripe_0_work)              68.63ns   14.57M
-// contentionAtWidthPthreadSelf(64_stripe_0_work)              64.98ns   15.39M
-// atomicIncrBaseline(local_incr_0_work)                       13.64ns   73.33M
+// contentionAtWidthGetcpu(1_stripe_0_work)                   993.13ns    1.01M
+// contentionAtWidthGetcpu(2_stripe_0_work)                   551.45ns    1.81M
+// contentionAtWidthGetcpu(4_stripe_0_work)                   302.36ns    3.31M
+// contentionAtWidthGetcpu(8_stripe_0_work)                   156.57ns    6.39M
+// contentionAtWidthGetcpu(16_stripe_0_work)                   81.34ns   12.29M
+// contentionAtWidthGetcpu(32_stripe_0_work)                   37.90ns   26.39M
+// contentionAtWidthGetcpu(64_stripe_0_work)                   36.02ns   27.76M
+// contentionAtWidthCached(2_stripe_0_work)                   310.64ns    3.22M
+// contentionAtWidthCached(4_stripe_0_work)                   180.41ns    5.54M
+// contentionAtWidthCached(8_stripe_0_work)                    87.84ns   11.38M
+// contentionAtWidthCached(16_stripe_0_work)                   45.04ns   22.20M
+// contentionAtWidthCached(32_stripe_0_work)                   19.92ns   50.20M
+// contentionAtWidthCached(64_stripe_0_work)                   19.21ns   52.06M
+// contentionAtWidthThreadLocal(2_stripe_0_work)              321.14ns    3.11M
+// contentionAtWidthThreadLocal(4_stripe_0_work)              244.41ns    4.09M
+// contentionAtWidthThreadLocal(8_stripe_0_work)              103.47ns    9.66M
+// contentionAtWidthThreadLocal(16_stripe_0_work)              79.82ns   12.53M
+// contentionAtWidthThreadLocal(32_stripe_0_work)              20.41ns   49.01M
+// contentionAtWidthThreadLocal(64_stripe_0_work)              22.13ns   45.18M
+// contentionAtWidthPthreadSelf(2_stripe_0_work)              373.46ns    2.68M
+// contentionAtWidthPthreadSelf(4_stripe_0_work)              208.18ns    4.80M
+// contentionAtWidthPthreadSelf(8_stripe_0_work)              105.99ns    9.43M
+// contentionAtWidthPthreadSelf(16_stripe_0_work)             105.67ns    9.46M
+// contentionAtWidthPthreadSelf(32_stripe_0_work)              76.01ns   13.16M
+// contentionAtWidthPthreadSelf(64_stripe_0_work)              76.04ns   13.15M
+// atomicIncrBaseline(local_incr_0_work)                       13.43ns   74.47M
 // ----------------------------------------------------------------------------
-// contentionAtWidthGetcpu(1_stripe_500_work)                   1.87us  534.91K
-// contentionAtWidthGetcpu(2_stripe_500_work)                   1.58us  632.15K
-// contentionAtWidthGetcpu(4_stripe_500_work)                 622.80ns    1.61M
-// contentionAtWidthGetcpu(8_stripe_500_work)                 501.08ns    2.00M
-// contentionAtWidthGetcpu(16_stripe_500_work)                480.42ns    2.08M
-// contentionAtWidthGetcpu(32_stripe_500_work)                420.10ns    2.38M
-// atomicIncrBaseline(local_incr_500_work)                    407.74ns    2.45M
+// contentionAtWidthGetcpu(1_stripe_500_work)                   1.76us  567.20K
+// contentionAtWidthGetcpu(2_stripe_500_work)                   1.16us  863.67K
+// contentionAtWidthGetcpu(4_stripe_500_work)                 604.74ns    1.65M
+// contentionAtWidthGetcpu(8_stripe_500_work)                 524.16ns    1.91M
+// contentionAtWidthGetcpu(16_stripe_500_work)                478.92ns    2.09M
+// contentionAtWidthGetcpu(32_stripe_500_work)                480.64ns    2.08M
+// atomicIncrBaseline(local_incr_500_work)                    395.17ns    2.53M
 // ----------------------------------------------------------------------------
-// contentionAtWidthGetcpu(1_stripe_1000_work)                  1.93us  518.44K
-// contentionAtWidthGetcpu(2_stripe_1000_work)                  1.49us  669.39K
-// contentionAtWidthGetcpu(4_stripe_1000_work)                876.77ns    1.14M
-// contentionAtWidthGetcpu(8_stripe_1000_work)                749.22ns    1.33M
-// contentionAtWidthGetcpu(16_stripe_1000_work)               754.72ns    1.32M
-// contentionAtWidthGetcpu(32_stripe_1000_work)               591.59ns    1.69M
-// atomicIncrBaseline(local_incr_1000_work)                   611.62ns    1.64M
+// contentionAtWidthGetcpu(1_stripe_1000_work)                  2.16us  462.06K
+// contentionAtWidthGetcpu(2_stripe_1000_work)                  1.31us  764.80K
+// contentionAtWidthGetcpu(4_stripe_1000_work)                895.33ns    1.12M
+// contentionAtWidthGetcpu(8_stripe_1000_work)                833.98ns    1.20M
+// contentionAtWidthGetcpu(16_stripe_1000_work)               765.10ns    1.31M
+// contentionAtWidthGetcpu(32_stripe_1000_work)               646.85ns    1.55M
+// atomicIncrBaseline(local_incr_1000_work)                   656.15ns    1.52M
 // ============================================================================
 template <template <typename> class Tag>
 static void contentionAtWidth(size_t iters, size_t stripes, size_t work) {
@@ -249,6 +294,10 @@ contentionAtWidthPthreadSelf(size_t iters, size_t stripes, size_t work) {
   contentionAtWidth<PthreadSelfTag>(iters, stripes, work);
 }
 
+static void contentionAtWidthCached(size_t iters, size_t stripes, size_t work) {
+  contentionAtWidth<CachedCurrentTag>(iters, stripes, work);
+}
+
 BENCHMARK_DRAW_LINE();
 BENCHMARK_NAMED_PARAM(contentionAtWidthGetcpu, 1_stripe_0_work, 1, 0)
 BENCHMARK_NAMED_PARAM(contentionAtWidthGetcpu, 2_stripe_0_work, 2, 0)
@@ -257,6 +306,12 @@ BENCHMARK_NAMED_PARAM(contentionAtWidthGetcpu, 8_stripe_0_work, 8, 0)
 BENCHMARK_NAMED_PARAM(contentionAtWidthGetcpu, 16_stripe_0_work, 16, 0)
 BENCHMARK_NAMED_PARAM(contentionAtWidthGetcpu, 32_stripe_0_work, 32, 0)
 BENCHMARK_NAMED_PARAM(contentionAtWidthGetcpu, 64_stripe_0_work, 64, 0)
+BENCHMARK_NAMED_PARAM(contentionAtWidthCached, 2_stripe_0_work, 2, 0)
+BENCHMARK_NAMED_PARAM(contentionAtWidthCached, 4_stripe_0_work, 4, 0)
+BENCHMARK_NAMED_PARAM(contentionAtWidthCached, 8_stripe_0_work, 8, 0)
+BENCHMARK_NAMED_PARAM(contentionAtWidthCached, 16_stripe_0_work, 16, 0)
+BENCHMARK_NAMED_PARAM(contentionAtWidthCached, 32_stripe_0_work, 32, 0)
+BENCHMARK_NAMED_PARAM(contentionAtWidthCached, 64_stripe_0_work, 64, 0)
 BENCHMARK_NAMED_PARAM(contentionAtWidthThreadLocal, 2_stripe_0_work, 2, 0)
 BENCHMARK_NAMED_PARAM(contentionAtWidthThreadLocal, 4_stripe_0_work, 4, 0)
 BENCHMARK_NAMED_PARAM(contentionAtWidthThreadLocal, 8_stripe_0_work, 8, 0)

folly/concurrency/test/CacheLocalityTest.cpp

@@ -375,6 +375,30 @@ TEST(AccessSpreader, Simple) {
   }
 }
 
+TEST(AccessSpreader, SimpleCached) {
+  for (size_t s = 1; s < 200; ++s) {
+    EXPECT_LT(AccessSpreader<>::cachedCurrent(s), s);
+  }
+}
+
+TEST(AccessSpreader, ConcurrentAccessCached) {
+  std::vector<std::thread> threads;
+  for (size_t i = 0; i < 4; ++i) {
+    threads.emplace_back([]() {
+      for (size_t s : {16, 32, 64}) {
+        for (size_t j = 1; j < 200; ++j) {
+          EXPECT_LT(AccessSpreader<>::cachedCurrent(s), s);
+          EXPECT_LT(AccessSpreader<>::cachedCurrent(s), s);
+        }
+        std::this_thread::yield();
+      }
+    });
+  }
+  for (auto& thread : threads) {
+    thread.join();
+  }
+}
+
 #ifdef FOLLY_TLS
 #define DECLARE_SPREADER_TAG(tag, locality, func)      \
   namespace {                                          \