Commit cda4ae2a authored by Tudor Bosman's avatar Tudor Bosman Committed by Peter Griess

Improve CompressionTest

Summary: Test compressing low-entropy (constant) data as well.

Test Plan: ran it

Reviewed By: tuomas.pelkonen@fb.com

FB internal diff: D1061444

@override-unit-failures
parent 11b4d6ba
...@@ -313,6 +313,10 @@ class IOBuf { ...@@ -313,6 +313,10 @@ class IOBuf {
* On error, std::bad_alloc will be thrown. * On error, std::bad_alloc will be thrown.
*/ */
static std::unique_ptr<IOBuf> wrapBuffer(const void* buf, uint32_t capacity); static std::unique_ptr<IOBuf> wrapBuffer(const void* buf, uint32_t capacity);
static std::unique_ptr<IOBuf> wrapBuffer(ByteRange br) {
CHECK_LE(br.size(), std::numeric_limits<uint32_t>::max());
return wrapBuffer(br.data(), br.size());
}
/** /**
* Convenience function to create a new IOBuf object that copies data from a * Convenience function to create a new IOBuf object that copies data from a
...@@ -322,6 +326,12 @@ class IOBuf { ...@@ -322,6 +326,12 @@ class IOBuf {
static std::unique_ptr<IOBuf> copyBuffer(const void* buf, uint32_t size, static std::unique_ptr<IOBuf> copyBuffer(const void* buf, uint32_t size,
uint32_t headroom=0, uint32_t headroom=0,
uint32_t minTailroom=0); uint32_t minTailroom=0);
static std::unique_ptr<IOBuf> copyBuffer(ByteRange br,
uint32_t headroom=0,
uint32_t minTailroom=0) {
CHECK_LE(br.size(), std::numeric_limits<uint32_t>::max());
return copyBuffer(br.data(), br.size(), headroom, minTailroom);
}
/** /**
* Convenience function to create a new IOBuf object that copies data from a * Convenience function to create a new IOBuf object that copies data from a
......
...@@ -22,6 +22,7 @@ ...@@ -22,6 +22,7 @@
#include <tr1/tuple> #include <tr1/tuple>
#include <unordered_map> #include <unordered_map>
#include <boost/noncopyable.hpp>
#include <glog/logging.h> #include <glog/logging.h>
#include <gtest/gtest.h> #include <gtest/gtest.h>
...@@ -32,34 +33,55 @@ ...@@ -32,34 +33,55 @@
namespace folly { namespace io { namespace test { namespace folly { namespace io { namespace test {
constexpr size_t randomDataSizeLog2 = 27; // 128MiB class DataHolder : private boost::noncopyable {
constexpr size_t randomDataSize = size_t(1) << randomDataSizeLog2; public:
uint64_t hash(size_t size) const;
ByteRange data(size_t size) const;
std::unique_ptr<uint8_t[]> randomData; protected:
std::unordered_map<uint64_t, uint64_t> hashes; explicit DataHolder(size_t sizeLog2);
const size_t size_;
std::unique_ptr<uint8_t[]> data_;
mutable std::unordered_map<uint64_t, uint64_t> hashCache_;
};
uint64_t hashIOBuf(const IOBuf* buf) { DataHolder::DataHolder(size_t sizeLog2)
uint64_t h = folly::hash::FNV_64_HASH_START; : size_(size_t(1) << sizeLog2),
for (auto& range : *buf) { data_(new uint8_t[size_]) {
h = folly::hash::fnv64_buf(range.data(), range.size(), h);
}
return h;
} }
uint64_t getRandomDataHash(uint64_t size) { uint64_t DataHolder::hash(size_t size) const {
auto p = hashes.find(size); CHECK_LE(size, size_);
if (p != hashes.end()) { auto p = hashCache_.find(size);
if (p != hashCache_.end()) {
return p->second; return p->second;
} }
uint64_t h = folly::hash::fnv64_buf(randomData.get(), size); uint64_t h = folly::hash::fnv64_buf(data_.get(), size);
hashes[size] = h; hashCache_[size] = h;
return h; return h;
} }
void generateRandomData() { ByteRange DataHolder::data(size_t size) const {
randomData.reset(new uint8_t[size_t(1) << randomDataSizeLog2]); CHECK_LE(size, size_);
return ByteRange(data_.get(), size);
}
uint64_t hashIOBuf(const IOBuf* buf) {
uint64_t h = folly::hash::FNV_64_HASH_START;
for (auto& range : *buf) {
h = folly::hash::fnv64_buf(range.data(), range.size(), h);
}
return h;
}
class RandomDataHolder : public DataHolder {
public:
explicit RandomDataHolder(size_t sizeLog2);
};
RandomDataHolder::RandomDataHolder(size_t sizeLog2)
: DataHolder(sizeLog2) {
constexpr size_t numThreadsLog2 = 3; constexpr size_t numThreadsLog2 = 3;
constexpr size_t numThreads = size_t(1) << numThreadsLog2; constexpr size_t numThreads = size_t(1) << numThreadsLog2;
...@@ -69,12 +91,12 @@ void generateRandomData() { ...@@ -69,12 +91,12 @@ void generateRandomData() {
threads.reserve(numThreads); threads.reserve(numThreads);
for (size_t t = 0; t < numThreads; ++t) { for (size_t t = 0; t < numThreads; ++t) {
threads.emplace_back( threads.emplace_back(
[seed, t, numThreadsLog2] () { [this, seed, t, numThreadsLog2, sizeLog2] () {
std::mt19937 rng(seed + t); std::mt19937 rng(seed + t);
size_t countLog2 = size_t(1) << (randomDataSizeLog2 - numThreadsLog2); size_t countLog2 = size_t(1) << (sizeLog2 - numThreadsLog2);
size_t start = size_t(t) << countLog2; size_t start = size_t(t) << countLog2;
for (size_t i = 0; i < countLog2; ++i) { for (size_t i = 0; i < countLog2; ++i) {
randomData[start + i] = rng(); this->data_[start + i] = rng();
} }
}); });
} }
...@@ -84,6 +106,20 @@ void generateRandomData() { ...@@ -84,6 +106,20 @@ void generateRandomData() {
} }
} }
class ConstantDataHolder : public DataHolder {
public:
explicit ConstantDataHolder(size_t sizeLog2);
};
ConstantDataHolder::ConstantDataHolder(size_t sizeLog2)
: DataHolder(sizeLog2) {
memset(data_.get(), 'a', size_);
}
constexpr size_t dataSizeLog2 = 27; // 128MiB
RandomDataHolder randomDataHolder(dataSizeLog2);
ConstantDataHolder constantDataHolder(dataSizeLog2);
TEST(CompressionTestNeedsUncompressedLength, Simple) { TEST(CompressionTestNeedsUncompressedLength, Simple) {
EXPECT_FALSE(getCodec(CodecType::NO_COMPRESSION)->needsUncompressedLength()); EXPECT_FALSE(getCodec(CodecType::NO_COMPRESSION)->needsUncompressedLength());
EXPECT_TRUE(getCodec(CodecType::LZ4)->needsUncompressedLength()); EXPECT_TRUE(getCodec(CodecType::LZ4)->needsUncompressedLength());
...@@ -101,33 +137,41 @@ class CompressionTest : public testing::TestWithParam< ...@@ -101,33 +137,41 @@ class CompressionTest : public testing::TestWithParam<
codec_ = getCodec(std::tr1::get<1>(tup)); codec_ = getCodec(std::tr1::get<1>(tup));
} }
void runSimpleTest(const DataHolder& dh);
uint64_t uncompressedLength_; uint64_t uncompressedLength_;
std::unique_ptr<Codec> codec_; std::unique_ptr<Codec> codec_;
}; };
TEST_P(CompressionTest, Simple) { void CompressionTest::runSimpleTest(const DataHolder& dh) {
auto original = IOBuf::wrapBuffer(randomData.get(), uncompressedLength_); auto original = IOBuf::wrapBuffer(dh.data(uncompressedLength_));
auto compressed = codec_->compress(original.get()); auto compressed = codec_->compress(original.get());
if (!codec_->needsUncompressedLength()) { if (!codec_->needsUncompressedLength()) {
auto uncompressed = codec_->uncompress(compressed.get()); auto uncompressed = codec_->uncompress(compressed.get());
EXPECT_EQ(uncompressedLength_, uncompressed->computeChainDataLength()); EXPECT_EQ(uncompressedLength_, uncompressed->computeChainDataLength());
EXPECT_EQ(getRandomDataHash(uncompressedLength_), EXPECT_EQ(dh.hash(uncompressedLength_), hashIOBuf(uncompressed.get()));
hashIOBuf(uncompressed.get()));
} }
{ {
auto uncompressed = codec_->uncompress(compressed.get(), auto uncompressed = codec_->uncompress(compressed.get(),
uncompressedLength_); uncompressedLength_);
EXPECT_EQ(uncompressedLength_, uncompressed->computeChainDataLength()); EXPECT_EQ(uncompressedLength_, uncompressed->computeChainDataLength());
EXPECT_EQ(getRandomDataHash(uncompressedLength_), EXPECT_EQ(dh.hash(uncompressedLength_), hashIOBuf(uncompressed.get()));
hashIOBuf(uncompressed.get()));
} }
} }
TEST_P(CompressionTest, RandomData) {
runSimpleTest(randomDataHolder);
}
TEST_P(CompressionTest, ConstantData) {
runSimpleTest(constantDataHolder);
}
INSTANTIATE_TEST_CASE_P( INSTANTIATE_TEST_CASE_P(
CompressionTest, CompressionTest,
CompressionTest, CompressionTest,
testing::Combine( testing::Combine(
testing::Values(0, 1, 12, 22, int(randomDataSizeLog2)), testing::Values(0, 1, 12, 22, 25, 27),
testing::Values(CodecType::NO_COMPRESSION, testing::Values(CodecType::NO_COMPRESSION,
CodecType::LZ4, CodecType::LZ4,
CodecType::SNAPPY, CodecType::SNAPPY,
...@@ -140,26 +184,26 @@ class CompressionCorruptionTest : public testing::TestWithParam<CodecType> { ...@@ -140,26 +184,26 @@ class CompressionCorruptionTest : public testing::TestWithParam<CodecType> {
codec_ = getCodec(GetParam()); codec_ = getCodec(GetParam());
} }
void runSimpleTest(const DataHolder& dh);
std::unique_ptr<Codec> codec_; std::unique_ptr<Codec> codec_;
}; };
TEST_P(CompressionCorruptionTest, Simple) { void CompressionCorruptionTest::runSimpleTest(const DataHolder& dh) {
constexpr uint64_t uncompressedLength = 42; constexpr uint64_t uncompressedLength = 42;
auto original = IOBuf::wrapBuffer(randomData.get(), uncompressedLength); auto original = IOBuf::wrapBuffer(dh.data(uncompressedLength));
auto compressed = codec_->compress(original.get()); auto compressed = codec_->compress(original.get());
if (!codec_->needsUncompressedLength()) { if (!codec_->needsUncompressedLength()) {
auto uncompressed = codec_->uncompress(compressed.get()); auto uncompressed = codec_->uncompress(compressed.get());
EXPECT_EQ(uncompressedLength, uncompressed->computeChainDataLength()); EXPECT_EQ(uncompressedLength, uncompressed->computeChainDataLength());
EXPECT_EQ(getRandomDataHash(uncompressedLength), EXPECT_EQ(dh.hash(uncompressedLength), hashIOBuf(uncompressed.get()));
hashIOBuf(uncompressed.get()));
} }
{ {
auto uncompressed = codec_->uncompress(compressed.get(), auto uncompressed = codec_->uncompress(compressed.get(),
uncompressedLength); uncompressedLength);
EXPECT_EQ(uncompressedLength, uncompressed->computeChainDataLength()); EXPECT_EQ(uncompressedLength, uncompressed->computeChainDataLength());
EXPECT_EQ(getRandomDataHash(uncompressedLength), EXPECT_EQ(dh.hash(uncompressedLength), hashIOBuf(uncompressed.get()));
hashIOBuf(uncompressed.get()));
} }
EXPECT_THROW(codec_->uncompress(compressed.get(), uncompressedLength + 1), EXPECT_THROW(codec_->uncompress(compressed.get(), uncompressedLength + 1),
...@@ -177,6 +221,14 @@ TEST_P(CompressionCorruptionTest, Simple) { ...@@ -177,6 +221,14 @@ TEST_P(CompressionCorruptionTest, Simple) {
std::runtime_error); std::runtime_error);
} }
TEST_P(CompressionCorruptionTest, RandomData) {
runSimpleTest(randomDataHolder);
}
TEST_P(CompressionCorruptionTest, ConstantData) {
runSimpleTest(constantDataHolder);
}
INSTANTIATE_TEST_CASE_P( INSTANTIATE_TEST_CASE_P(
CompressionCorruptionTest, CompressionCorruptionTest,
CompressionCorruptionTest, CompressionCorruptionTest,
...@@ -192,8 +244,6 @@ int main(int argc, char *argv[]) { ...@@ -192,8 +244,6 @@ int main(int argc, char *argv[]) {
testing::InitGoogleTest(&argc, argv); testing::InitGoogleTest(&argc, argv);
google::ParseCommandLineFlags(&argc, &argv, true); google::ParseCommandLineFlags(&argc, &argv, true);
folly::io::test::generateRandomData(); // 4GB
auto ret = RUN_ALL_TESTS(); auto ret = RUN_ALL_TESTS();
if (!ret) { if (!ret) {
folly::runBenchmarksOnFlag(); folly::runBenchmarksOnFlag();
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment