Utility to dispatch instructions

Summary: Add a simple function to dispatch the `instructions` types based on current CPU (overridable in tests).

Reviewed By: luciang

Differential Revision: D10244149

fbshipit-source-id: ac0c1ff0d14402fa77c939361398db6122dea728

CMakeLists.txt

@@ -405,8 +405,8 @@ if (BUILD_TESTS)
       TEST bits_test_2 SOURCES BitsTest.cpp
       TEST bitvector_test SOURCES BitVectorCodingTest.cpp
       TEST dynamic_parser_test SOURCES DynamicParserTest.cpp
-      TEST eliasfano_test SOURCES EliasFanoCodingTest.cpp
-      TEST event_count_test SOURCES EventCountTest.cpp
+      TEST eliasfano_test SOURCES EliasFanoCodingTest.cpp CodingTestUtils.cpp
+      TEST event_count_test SOURCES EventCountTest.cpp CodingTestUtils.cpp
       # FunctionSchedulerTest has a lot of timing-dependent checks,
       # and tends to fail on heavily loaded systems.
       TEST function_scheduler_test BROKEN SOURCES FunctionSchedulerTest.cpp

folly/experimental/Instructions.h

@@ -24,6 +24,7 @@
 
 #include <folly/CpuId.h>
 #include <folly/Portability.h>
+#include <folly/lang/Assume.h>
 #include <folly/portability/Builtins.h>
 
 namespace folly {
@@ -34,7 +35,7 @@ namespace instructions {
 // with Nehalem, Intel CPUs support POPCNT instruction and gcc will emit
 // it for __builtin_popcountll intrinsic.
 // But we provide an alternative way for the client code: it can switch to
-// the appropriate version of EliasFanoReader<> in realtime (client should
+// the appropriate version of EliasFanoReader<> at runtime (client should
 // implement this switching logic itself) by specifying instruction set to
 // use explicitly.
 
@@ -147,6 +148,48 @@ struct Haswell : public Nehalem {
 #endif
   }
 };
+
+enum class Type {
+  DEFAULT,
+  NEHALEM,
+  HASWELL,
+};
+
+inline Type detect() {
+  const static Type type = [] {
+    if (instructions::Haswell::supported()) {
+      VLOG(2) << "Will use folly::compression::instructions::Haswell";
+      return Type::HASWELL;
+    } else if (instructions::Nehalem::supported()) {
+      VLOG(2) << "Will use folly::compression::instructions::Nehalem";
+      return Type::NEHALEM;
+    } else {
+      VLOG(2) << "Will use folly::compression::instructions::Default";
+      return Type::DEFAULT;
+    }
+  }();
+  return type;
+}
+
+template <class F>
+auto dispatch(Type type, F&& f) -> decltype(f(std::declval<Default>())) {
+  switch (type) {
+    case Type::HASWELL:
+      return f(Haswell());
+    case Type::NEHALEM:
+      return f(Nehalem());
+    case Type::DEFAULT:
+      return f(Default());
+  }
+
+  assume_unreachable();
+}
+
+template <class F>
+auto dispatch(F&& f) -> decltype(f(std::declval<Default>())) {
+  return dispatch(detect(), std::forward<F>(f));
+}
+
 } // namespace instructions
 } // namespace compression
 } // namespace folly

folly/experimental/test/BitVectorCodingTest.cpp

@@ -23,18 +23,15 @@
 #include <folly/experimental/BitVectorCoding.h>
 #include <folly/experimental/Select64.h>
 #include <folly/experimental/test/CodingTestUtils.h>
+#include <folly/init/Init.h>
 
 using namespace folly::compression;
 
-#ifndef BV_TEST_ARCH
-#define BV_TEST_ARCH Default
-#endif // BV_TEST_ARCH
-
 class BitVectorCodingTest : public ::testing::Test {
  public:
   void doTestEmpty() {
     typedef BitVectorEncoder<uint32_t, size_t> Encoder;
-    typedef BitVectorReader<Encoder, instructions::BV_TEST_ARCH> Reader;
+    typedef BitVectorReader<Encoder, instructions::Default> Reader;
     testEmpty<Reader, Encoder>();
   }
 
@@ -71,7 +68,6 @@ TEST_F(BitVectorCodingTest, SkipForwardPointers) {
 namespace bm {
 
 typedef BitVectorEncoder<uint32_t, uint32_t, 128, 128> Encoder;
-typedef BitVectorReader<Encoder> Reader;
 
 std::vector<uint32_t> data;
 std::vector<size_t> order;
@@ -102,11 +98,17 @@ void free() {
 } // namespace bm
 
 BENCHMARK(Next, iters) {
-  bmNext<bm::Reader>(bm::list, bm::data, iters);
+  dispatchInstructions([&](auto instructions) {
+    bmNext<BitVectorReader<bm::Encoder, decltype(instructions)>>(
+        bm::list, bm::data, iters);
+  });
 }
 
 size_t Skip_ForwardQ128(size_t iters, size_t logAvgSkip) {
-  bmSkip<bm::Reader>(bm::list, bm::data, logAvgSkip, iters);
+  dispatchInstructions([&](auto instructions) {
+    bmSkip<BitVectorReader<bm::Encoder, decltype(instructions)>>(
+        bm::list, bm::data, logAvgSkip, iters);
+  });
   return iters;
 }
 
@@ -119,13 +121,19 @@ BENCHMARK_NAMED_PARAM_MULTI(Skip_ForwardQ128, 256_pm_64, 8)
 BENCHMARK_NAMED_PARAM_MULTI(Skip_ForwardQ128, 1024_pm_256, 10)
 
 BENCHMARK(Jump_ForwardQ128, iters) {
-  bmJump<bm::Reader>(bm::list, bm::data, bm::order, iters);
+  dispatchInstructions([&](auto instructions) {
+    bmJump<BitVectorReader<bm::Encoder, decltype(instructions)>>(
+        bm::list, bm::data, bm::order, iters);
+  });
 }
 
 BENCHMARK_DRAW_LINE();
 
 size_t SkipTo_SkipQ128(size_t iters, size_t logAvgSkip) {
-  bmSkipTo<bm::Reader>(bm::list, bm::data, logAvgSkip, iters);
+  dispatchInstructions([&](auto instructions) {
+    bmSkipTo<BitVectorReader<bm::Encoder, decltype(instructions)>>(
+        bm::list, bm::data, logAvgSkip, iters);
+  });
   return iters;
 }
 
@@ -138,7 +146,10 @@ BENCHMARK_NAMED_PARAM_MULTI(SkipTo_SkipQ128, 256_pm_64, 8)
 BENCHMARK_NAMED_PARAM_MULTI(SkipTo_SkipQ128, 1024_pm_256, 10)
 
 BENCHMARK(JumpTo_SkipQ128, iters) {
-  bmJumpTo<bm::Reader>(bm::list, bm::data, bm::order, iters);
+  dispatchInstructions([&](auto instructions) {
+    bmJumpTo<BitVectorReader<bm::Encoder, decltype(instructions)>>(
+        bm::list, bm::data, bm::order, iters);
+  });
 }
 
 BENCHMARK_DRAW_LINE();
@@ -156,37 +167,39 @@ BENCHMARK(Encode) {
 }
 
 #if 0
-Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz (turbo off),
-using instructions::Default and GCC 4.8 with --bm_min_usec 100000.
+// Intel(R) Xeon(R) CPU E5-2678 v3 @ 2.50GHz (turbo on),
+// Using GCC 5 with --bm_min_usec 100000.
+V1008 12:32:25.863286 101188 Instructions.h:161] Will use folly::compression::instructions::Haswell
 ============================================================================
 folly/experimental/test/BitVectorCodingTest.cpp relative  time/iter  iters/s
 ============================================================================
-Next                                                         9.59ns  104.25M
-Skip_ForwardQ128(1)                                         11.56ns   86.53M
-Skip_ForwardQ128(2)                                         23.30ns   42.93M
-Skip_ForwardQ128(4_pm_1)                                    52.99ns   18.87M
-Skip_ForwardQ128(16_pm_4)                                  200.85ns    4.98M
-Skip_ForwardQ128(64_pm_16)                                 733.20ns    1.36M
-Skip_ForwardQ128(256_pm_64)                                748.35ns    1.34M
-Skip_ForwardQ128(1024_pm_256)                              742.77ns    1.35M
-Jump_ForwardQ128                                           752.98ns    1.33M
+Next                                                         9.52ns  104.99M
+Skip_ForwardQ128(1)                                         13.90ns   71.96M
+Skip_ForwardQ128(2)                                         25.02ns   39.97M
+Skip_ForwardQ128(4_pm_1)                                    28.25ns   35.40M
+Skip_ForwardQ128(16_pm_4)                                   39.64ns   25.23M
+Skip_ForwardQ128(64_pm_16)                                 112.19ns    8.91M
+Skip_ForwardQ128(256_pm_64)                                137.75ns    7.26M
+Skip_ForwardQ128(1024_pm_256)                              131.56ns    7.60M
+Jump_ForwardQ128                                           133.30ns    7.50M
 ----------------------------------------------------------------------------
-SkipTo_SkipQ128(1)                                          23.47ns   42.62M
-SkipTo_SkipQ128(2)                                          24.48ns   40.85M
-SkipTo_SkipQ128(4_pm_1)                                     22.16ns   45.13M
-SkipTo_SkipQ128(16_pm_4)                                    28.43ns   35.17M
-SkipTo_SkipQ128(64_pm_16)                                   45.51ns   21.97M
-SkipTo_SkipQ128(256_pm_64)                                  44.03ns   22.71M
-SkipTo_SkipQ128(1024_pm_256)                                45.84ns   21.81M
-JumpTo_SkipQ128                                             15.33ns   65.25M
+SkipTo_SkipQ128(1)                                          13.30ns   75.16M
+SkipTo_SkipQ128(2)                                          13.81ns   72.40M
+SkipTo_SkipQ128(4_pm_1)                                     12.23ns   81.80M
+SkipTo_SkipQ128(16_pm_4)                                    13.72ns   72.89M
+SkipTo_SkipQ128(64_pm_16)                                   21.18ns   47.22M
+SkipTo_SkipQ128(256_pm_64)                                  20.15ns   49.63M
+SkipTo_SkipQ128(1024_pm_256)                                21.86ns   45.74M
+JumpTo_SkipQ128                                             23.10ns   43.30M
 ----------------------------------------------------------------------------
-Encode_10                                                    1.60us  624.33K
-Encode                                                      16.98ms    58.89
+Encode_10                                                  344.50ns    2.90M
+Encode                                                      10.88ms    91.90
 ============================================================================
 #endif
 
 int main(int argc, char** argv) {
   testing::InitGoogleTest(&argc, argv);
+  folly::init(&argc, &argv);
   gflags::ParseCommandLineFlags(&argc, &argv, true);
 
   auto ret = RUN_ALL_TESTS();

folly/experimental/test/CodingTestUtils.cpp

+/*
+ * Copyright 2013-present Facebook, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <folly/experimental/test/CodingTestUtils.h>
+
+#include <folly/portability/GFlags.h>
+#include <glog/logging.h>
+
+DEFINE_string(
+    coding_test_utils_instructions,
+    "",
+    "If non empty, forces the instruction set. Choices: Default, Nehalem, Haswell");
+
+namespace folly {
+namespace compression {
+
+folly::Optional<instructions::Type> instructionsOverride() {
+  if (FLAGS_coding_test_utils_instructions.empty()) {
+    return folly::none;
+  }
+
+  instructions::Type type;
+  if (FLAGS_coding_test_utils_instructions == "Default") {
+    type = instructions::Type::DEFAULT;
+  } else if (FLAGS_coding_test_utils_instructions == "Nehalem") {
+    type = instructions::Type::NEHALEM;
+  } else if (FLAGS_coding_test_utils_instructions == "Haswell") {
+    type = instructions::Type::HASWELL;
+  } else {
+    LOG(FATAL) << "Insupported instructions type "
+               << FLAGS_coding_test_utils_instructions;
+  }
+
+  instructions::dispatch(
+      type, [](auto instructions) { CHECK(instructions.supported()); });
+
+  return type;
+}
+
+} // namespace compression
+} // namespace folly

folly/experimental/test/CodingTestUtils.h

@@ -28,6 +28,8 @@
 
 #include <folly/Benchmark.h>
 #include <folly/Likely.h>
+#include <folly/Optional.h>
+#include <folly/experimental/Instructions.h>
 #include <folly/portability/GTest.h>
 
 namespace folly {
@@ -417,5 +419,17 @@ void bmJumpTo(
   }
 }
 
+folly::Optional<instructions::Type> instructionsOverride();
+
+template <class F>
+auto dispatchInstructions(F&& f)
+    -> decltype(f(std::declval<instructions::Default>())) {
+  if (auto type = instructionsOverride()) {
+    return instructions::dispatch(*type, std::forward<F>(f));
+  } else {
+    return instructions::dispatch(std::forward<F>(f));
+  }
+}
+
 } // namespace compression
 } // namespace folly

folly/experimental/test/EliasFanoCodingTest.cpp

@@ -24,13 +24,10 @@
 #include <folly/experimental/EliasFanoCoding.h>
 #include <folly/experimental/Select64.h>
 #include <folly/experimental/test/CodingTestUtils.h>
+#include <folly/init/Init.h>
 
 using namespace folly::compression;
 
-#ifndef EF_TEST_ARCH
-#define EF_TEST_ARCH Default
-#endif // EF_TEST_ARCH
-
 namespace {
 
 uint8_t slowDefaultNumLowerBits(size_t upperBound, size_t size) {
@@ -102,7 +99,7 @@ class EliasFanoCodingTest : public ::testing::Test {
         kForwardQuantum>
         Encoder;
     using Reader =
-        EliasFanoReader<Encoder, instructions::EF_TEST_ARCH, false, SizeType>;
+        EliasFanoReader<Encoder, instructions::Default, false, SizeType>;
     testAll<Reader, Encoder>({0});
     testAll<Reader, Encoder>(generateRandomList(100 * 1000, 10 * 1000 * 1000));
     testAll<Reader, Encoder>(generateSeqList(1, 100000, 100));
@@ -135,7 +132,7 @@ TEST_F(EliasFanoCodingTest, SkipForwardPointers) {
 
 TEST_F(EliasFanoCodingTest, BugLargeGapInUpperBits) { // t16274876
   typedef EliasFanoEncoderV2<uint32_t, uint32_t, 2, 2> Encoder;
-  typedef EliasFanoReader<Encoder, instructions::EF_TEST_ARCH> Reader;
+  typedef EliasFanoReader<Encoder, instructions::Default> Reader;
   constexpr uint32_t kLargeValue = 127;
 
   // Build a list where the upper bits have a large gap after the
@@ -160,7 +157,6 @@ TEST_F(EliasFanoCodingTest, BugLargeGapInUpperBits) { // t16274876
 namespace bm {
 
 typedef EliasFanoEncoderV2<uint32_t, uint32_t, 128, 128> Encoder;
-typedef EliasFanoReader<Encoder> Reader;
 
 std::vector<uint32_t> data;
 std::vector<size_t> order;
@@ -200,11 +196,17 @@ void free() {
 } // namespace bm
 
 BENCHMARK(Next, iters) {
-  bmNext<bm::Reader>(bm::list, bm::data, iters);
+  dispatchInstructions([&](auto instructions) {
+    bmNext<EliasFanoReader<bm::Encoder, decltype(instructions)>>(
+        bm::list, bm::data, iters);
+  });
 }
 
 size_t Skip_ForwardQ128(size_t iters, size_t logAvgSkip) {
-  bmSkip<bm::Reader>(bm::list, bm::data, logAvgSkip, iters);
+  dispatchInstructions([&](auto instructions) {
+    bmSkip<EliasFanoReader<bm::Encoder, decltype(instructions)>>(
+        bm::list, bm::data, logAvgSkip, iters);
+  });
   return iters;
 }
 
@@ -217,13 +219,19 @@ BENCHMARK_NAMED_PARAM_MULTI(Skip_ForwardQ128, 256_pm_64, 8)
 BENCHMARK_NAMED_PARAM_MULTI(Skip_ForwardQ128, 1024_pm_256, 10)
 
 BENCHMARK(Jump_ForwardQ128, iters) {
-  bmJump<bm::Reader>(bm::list, bm::data, bm::order, iters);
+  dispatchInstructions([&](auto instructions) {
+    bmJump<EliasFanoReader<bm::Encoder, decltype(instructions)>>(
+        bm::list, bm::data, bm::order, iters);
+  });
 }
 
 BENCHMARK_DRAW_LINE();
 
 size_t SkipTo_SkipQ128(size_t iters, size_t logAvgSkip) {
-  bmSkipTo<bm::Reader>(bm::list, bm::data, logAvgSkip, iters);
+  dispatchInstructions([&](auto instructions) {
+    bmSkipTo<EliasFanoReader<bm::Encoder, decltype(instructions)>>(
+        bm::list, bm::data, logAvgSkip, iters);
+  });
   return iters;
 }
 
@@ -236,7 +244,10 @@ BENCHMARK_NAMED_PARAM_MULTI(SkipTo_SkipQ128, 256_pm_64, 8)
 BENCHMARK_NAMED_PARAM_MULTI(SkipTo_SkipQ128, 1024_pm_256, 10)
 
 BENCHMARK(JumpTo_SkipQ128, iters) {
-  bmJumpTo<bm::Reader>(bm::list, bm::data, bm::order, iters);
+  dispatchInstructions([&](auto instructions) {
+    bmJumpTo<EliasFanoReader<bm::Encoder, decltype(instructions)>>(
+        bm::list, bm::data, bm::order, iters);
+  });
 }
 
 BENCHMARK_DRAW_LINE();
@@ -280,40 +291,42 @@ BENCHMARK(slowDefaultNumLowerBits, iters) {
 }
 
 #if 0
-Intel(R) Xeon(R) CPU E5-2678 v3 @ 2.50GHz (turbo on),
-using -DEF_TEST_ARCH Haswell and GCC 4.9 with --bm_min_usec 100000.
+// Intel(R) Xeon(R) CPU E5-2678 v3 @ 2.50GHz (turbo on),
+// Using GCC 5 with --bm_min_usec 100000.
+V1008 12:29:33.646595 87744 Instructions.h:161] Will use folly::compression::instructions::Haswell
 ============================================================================
 folly/experimental/test/EliasFanoCodingTest.cpp relative  time/iter  iters/s
 ============================================================================
-Next                                                         2.31ns  433.77M
-Skip_ForwardQ128(1)                                          3.73ns  267.93M
-Skip_ForwardQ128(2)                                          4.89ns  204.34M
-Skip_ForwardQ128(4_pm_1)                                     6.86ns  145.79M
-Skip_ForwardQ128(16_pm_4)                                   18.92ns   52.85M
-Skip_ForwardQ128(64_pm_16)                                  26.56ns   37.66M
-Skip_ForwardQ128(256_pm_64)                                 30.12ns   33.20M
-Skip_ForwardQ128(1024_pm_256)                               30.74ns   32.53M
-Jump_ForwardQ128                                            30.49ns   32.80M
+Next                                                         2.47ns  405.58M
+Skip_ForwardQ128(1)                                          6.68ns  149.67M
+Skip_ForwardQ128(2)                                          7.67ns  130.30M
+Skip_ForwardQ128(4_pm_1)                                     9.12ns  109.65M
+Skip_ForwardQ128(16_pm_4)                                    9.95ns  100.53M
+Skip_ForwardQ128(64_pm_16)                                  12.76ns   78.40M
+Skip_ForwardQ128(256_pm_64)                                 18.09ns   55.27M
+Skip_ForwardQ128(1024_pm_256)                               19.13ns   52.28M
+Jump_ForwardQ128                                            20.27ns   49.33M
 ----------------------------------------------------------------------------
-SkipTo_SkipQ128(1)                                           3.86ns  258.96M
-SkipTo_SkipQ128(2)                                           7.73ns  129.36M
-SkipTo_SkipQ128(4_pm_1)                                     10.29ns   97.18M
-SkipTo_SkipQ128(16_pm_4)                                    28.69ns   34.86M
-SkipTo_SkipQ128(64_pm_16)                                   39.73ns   25.17M
-SkipTo_SkipQ128(256_pm_64)                                  43.45ns   23.01M
-SkipTo_SkipQ128(1024_pm_256)                                44.66ns   22.39M
-JumpTo_SkipQ128                                             47.98ns   20.84M
+SkipTo_SkipQ128(1)                                           8.35ns  119.76M
+SkipTo_SkipQ128(2)                                          12.37ns   80.85M
+SkipTo_SkipQ128(4_pm_1)                                     15.05ns   66.44M
+SkipTo_SkipQ128(16_pm_4)                                    22.90ns   43.66M
+SkipTo_SkipQ128(64_pm_16)                                   34.11ns   29.31M
+SkipTo_SkipQ128(256_pm_64)                                  38.68ns   25.85M
+SkipTo_SkipQ128(1024_pm_256)                                41.75ns   23.95M
+JumpTo_SkipQ128                                             44.79ns   22.33M
 ----------------------------------------------------------------------------
-Encode_10                                                   77.92ns   12.83M
-Encode                                                       4.73ms   211.41
+Encode_10                                                  120.33ns    8.31M
+Encode                                                       7.61ms   131.32
 ----------------------------------------------------------------------------
-defaultNumLowerBits                                          2.20ns  455.01M
-slowDefaultNumLowerBits                                      7.90ns  126.59M
+defaultNumLowerBits                                          3.69ns  270.74M
+slowDefaultNumLowerBits                                     10.90ns   91.73M
 ============================================================================
 #endif
 
 int main(int argc, char** argv) {
   testing::InitGoogleTest(&argc, argv);
+  folly::init(&argc, &argv);
   gflags::ParseCommandLineFlags(&argc, &argv, true);
 
   auto ret = RUN_ALL_TESTS();