Commit a5093cb5 authored by Felix Handte's avatar Felix Handte Committed by Facebook Github Bot

Add String Support to Compression Codec

Summary:
This diff adds string-based compress and uncompress support to
folly::io::Codec. By default, the string API is implemented by translating to
and from IOBufs. Room is left for individual implementations to override that
behavior and go straight to the underlying library.

Reviewed By: yfeldblum

Differential Revision: D4690770

fbshipit-source-id: 7e2615da9add8328654505cdda6a2c79583b4558
parent a1abbb7a
...@@ -56,13 +56,26 @@ std::unique_ptr<IOBuf> Codec::compress(const IOBuf* data) { ...@@ -56,13 +56,26 @@ std::unique_ptr<IOBuf> Codec::compress(const IOBuf* data) {
uint64_t len = data->computeChainDataLength(); uint64_t len = data->computeChainDataLength();
if (len == 0) { if (len == 0) {
return IOBuf::create(0); return IOBuf::create(0);
} else if (len > maxUncompressedLength()) { }
if (len > maxUncompressedLength()) {
throw std::runtime_error("Codec: uncompressed length too large"); throw std::runtime_error("Codec: uncompressed length too large");
} }
return doCompress(data); return doCompress(data);
} }
std::string Codec::compress(const StringPiece data) {
const uint64_t len = data.size();
if (len == 0) {
return "";
}
if (len > maxUncompressedLength()) {
throw std::runtime_error("Codec: uncompressed length too large");
}
return doCompressString(data);
}
std::unique_ptr<IOBuf> Codec::uncompress(const IOBuf* data, std::unique_ptr<IOBuf> Codec::uncompress(const IOBuf* data,
uint64_t uncompressedLength) { uint64_t uncompressedLength) {
if (uncompressedLength == UNKNOWN_UNCOMPRESSED_LENGTH) { if (uncompressedLength == UNKNOWN_UNCOMPRESSED_LENGTH) {
...@@ -84,6 +97,28 @@ std::unique_ptr<IOBuf> Codec::uncompress(const IOBuf* data, ...@@ -84,6 +97,28 @@ std::unique_ptr<IOBuf> Codec::uncompress(const IOBuf* data,
return doUncompress(data, uncompressedLength); return doUncompress(data, uncompressedLength);
} }
std::string Codec::uncompress(
const StringPiece data,
uint64_t uncompressedLength) {
if (uncompressedLength == UNKNOWN_UNCOMPRESSED_LENGTH) {
if (needsUncompressedLength()) {
throw std::invalid_argument("Codec: uncompressed length required");
}
} else if (uncompressedLength > maxUncompressedLength()) {
throw std::runtime_error("Codec: uncompressed length too large");
}
if (data.empty()) {
if (uncompressedLength != UNKNOWN_UNCOMPRESSED_LENGTH &&
uncompressedLength != 0) {
throw std::runtime_error("Codec: invalid uncompressed length");
}
return "";
}
return doUncompressString(data, uncompressedLength);
}
bool Codec::needsUncompressedLength() const { bool Codec::needsUncompressedLength() const {
return doNeedsUncompressedLength(); return doNeedsUncompressedLength();
} }
...@@ -100,6 +135,30 @@ uint64_t Codec::doMaxUncompressedLength() const { ...@@ -100,6 +135,30 @@ uint64_t Codec::doMaxUncompressedLength() const {
return UNLIMITED_UNCOMPRESSED_LENGTH; return UNLIMITED_UNCOMPRESSED_LENGTH;
} }
std::string Codec::doCompressString(const StringPiece data) {
const IOBuf inputBuffer{IOBuf::WRAP_BUFFER, data};
auto outputBuffer = doCompress(&inputBuffer);
std::string output;
output.reserve(outputBuffer->computeChainDataLength());
for (auto range : *outputBuffer) {
output.append(reinterpret_cast<const char*>(range.data()), range.size());
}
return output;
}
std::string Codec::doUncompressString(
const StringPiece data,
uint64_t uncompressedLength) {
const IOBuf inputBuffer{IOBuf::WRAP_BUFFER, data};
auto outputBuffer = doUncompress(&inputBuffer, uncompressedLength);
std::string output;
output.reserve(outputBuffer->computeChainDataLength());
for (auto range : *outputBuffer) {
output.append(reinterpret_cast<const char*>(range.data()), range.size());
}
return output;
}
namespace { namespace {
/** /**
......
...@@ -20,6 +20,7 @@ ...@@ -20,6 +20,7 @@
#include <limits> #include <limits>
#include <memory> #include <memory>
#include <folly/Range.h>
#include <folly/io/IOBuf.h> #include <folly/io/IOBuf.h>
/** /**
...@@ -119,6 +120,13 @@ class Codec { ...@@ -119,6 +120,13 @@ class Codec {
*/ */
std::unique_ptr<IOBuf> compress(const folly::IOBuf* data); std::unique_ptr<IOBuf> compress(const folly::IOBuf* data);
/**
* Compresses data. May involve additional copies compared to the overload
* that takes and returns IOBufs. Has the same error semantics as the IOBuf
* version.
*/
std::string compress(StringPiece data);
/** /**
* Uncompress data. Throws std::runtime_error on decompression error. * Uncompress data. Throws std::runtime_error on decompression error.
* *
...@@ -138,6 +146,15 @@ class Codec { ...@@ -138,6 +146,15 @@ class Codec {
const IOBuf* data, const IOBuf* data,
uint64_t uncompressedLength = UNKNOWN_UNCOMPRESSED_LENGTH); uint64_t uncompressedLength = UNKNOWN_UNCOMPRESSED_LENGTH);
/**
* Uncompresses data. May involve additional copies compared to the overload
* that takes and returns IOBufs. Has the same error semantics as the IOBuf
* version.
*/
std::string uncompress(
StringPiece data,
uint64_t uncompressedLength = UNKNOWN_UNCOMPRESSED_LENGTH);
protected: protected:
explicit Codec(CodecType type); explicit Codec(CodecType type);
...@@ -149,6 +166,14 @@ class Codec { ...@@ -149,6 +166,14 @@ class Codec {
virtual std::unique_ptr<IOBuf> doCompress(const folly::IOBuf* data) = 0; virtual std::unique_ptr<IOBuf> doCompress(const folly::IOBuf* data) = 0;
virtual std::unique_ptr<IOBuf> doUncompress(const folly::IOBuf* data, virtual std::unique_ptr<IOBuf> doUncompress(const folly::IOBuf* data,
uint64_t uncompressedLength) = 0; uint64_t uncompressedLength) = 0;
// default: an implementation is provided by default to wrap the strings into
// IOBufs and delegate to the IOBuf methods. This incurs a copy of the output
// from IOBuf to string. Implementers, at their discretion, can override
// these methods to avoid the copy.
virtual std::string doCompressString(StringPiece data);
virtual std::string doUncompressString(
StringPiece data,
uint64_t uncompressedLength);
CodecType type_; CodecType type_;
}; };
......
...@@ -143,7 +143,9 @@ class CompressionTest ...@@ -143,7 +143,9 @@ class CompressionTest
codec_ = getCodec(std::tr1::get<2>(tup)); codec_ = getCodec(std::tr1::get<2>(tup));
} }
void runSimpleTest(const DataHolder& dh); void runSimpleIOBufTest(const DataHolder& dh);
void runSimpleStringTest(const DataHolder& dh);
private: private:
std::unique_ptr<IOBuf> split(std::unique_ptr<IOBuf> data) const; std::unique_ptr<IOBuf> split(std::unique_ptr<IOBuf> data) const;
...@@ -153,7 +155,7 @@ class CompressionTest ...@@ -153,7 +155,7 @@ class CompressionTest
std::unique_ptr<Codec> codec_; std::unique_ptr<Codec> codec_;
}; };
void CompressionTest::runSimpleTest(const DataHolder& dh) { void CompressionTest::runSimpleIOBufTest(const DataHolder& dh) {
const auto original = split(IOBuf::wrapBuffer(dh.data(uncompressedLength_))); const auto original = split(IOBuf::wrapBuffer(dh.data(uncompressedLength_)));
const auto compressed = split(codec_->compress(original.get())); const auto compressed = split(codec_->compress(original.get()));
if (!codec_->needsUncompressedLength()) { if (!codec_->needsUncompressedLength()) {
...@@ -169,6 +171,23 @@ void CompressionTest::runSimpleTest(const DataHolder& dh) { ...@@ -169,6 +171,23 @@ void CompressionTest::runSimpleTest(const DataHolder& dh) {
} }
} }
void CompressionTest::runSimpleStringTest(const DataHolder& dh) {
const auto original = std::string(
reinterpret_cast<const char*>(dh.data(uncompressedLength_).data()),
uncompressedLength_);
const auto compressed = codec_->compress(original);
if (!codec_->needsUncompressedLength()) {
auto uncompressed = codec_->uncompress(compressed);
EXPECT_EQ(uncompressedLength_, uncompressed.length());
EXPECT_EQ(uncompressed, original);
}
{
auto uncompressed = codec_->uncompress(compressed, uncompressedLength_);
EXPECT_EQ(uncompressedLength_, uncompressed.length());
EXPECT_EQ(uncompressed, original);
}
}
// Uniformly split data into (potentially empty) chunks. // Uniformly split data into (potentially empty) chunks.
std::unique_ptr<IOBuf> CompressionTest::split( std::unique_ptr<IOBuf> CompressionTest::split(
std::unique_ptr<IOBuf> data) const { std::unique_ptr<IOBuf> data) const {
...@@ -196,11 +215,19 @@ std::unique_ptr<IOBuf> CompressionTest::split( ...@@ -196,11 +215,19 @@ std::unique_ptr<IOBuf> CompressionTest::split(
} }
TEST_P(CompressionTest, RandomData) { TEST_P(CompressionTest, RandomData) {
runSimpleTest(randomDataHolder); runSimpleIOBufTest(randomDataHolder);
} }
TEST_P(CompressionTest, ConstantData) { TEST_P(CompressionTest, ConstantData) {
runSimpleTest(constantDataHolder); runSimpleIOBufTest(constantDataHolder);
}
TEST_P(CompressionTest, RandomDataString) {
runSimpleStringTest(randomDataHolder);
}
TEST_P(CompressionTest, ConstantDataString) {
runSimpleStringTest(constantDataHolder);
} }
INSTANTIATE_TEST_CASE_P( INSTANTIATE_TEST_CASE_P(
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment