Commit 1e8e9c94 authored by Nick Terrell's avatar Nick Terrell Committed by Facebook Github Bot

Switch uncompressedLength to an Optional<uint64_t>

Summary: Instead of using `UNKNWON_UNCOMPRESSED_LENGTH` use an `Optional`.

Reviewed By: yfeldblum

Differential Revision: D5038919

fbshipit-source-id: 7fb60542277019996be3ff50509df5a5060cb1a0
parent 42c42914
...@@ -86,19 +86,19 @@ std::string Codec::compress(const StringPiece data) { ...@@ -86,19 +86,19 @@ std::string Codec::compress(const StringPiece data) {
return doCompressString(data); return doCompressString(data);
} }
std::unique_ptr<IOBuf> Codec::uncompress(const IOBuf* data, std::unique_ptr<IOBuf> Codec::uncompress(
uint64_t uncompressedLength) { const IOBuf* data,
if (uncompressedLength == UNKNOWN_UNCOMPRESSED_LENGTH) { Optional<uint64_t> uncompressedLength) {
if (!uncompressedLength) {
if (needsUncompressedLength()) { if (needsUncompressedLength()) {
throw std::invalid_argument("Codec: uncompressed length required"); throw std::invalid_argument("Codec: uncompressed length required");
} }
} else if (uncompressedLength > maxUncompressedLength()) { } else if (*uncompressedLength > maxUncompressedLength()) {
throw std::runtime_error("Codec: uncompressed length too large"); throw std::runtime_error("Codec: uncompressed length too large");
} }
if (data->empty()) { if (data->empty()) {
if (uncompressedLength != UNKNOWN_UNCOMPRESSED_LENGTH && if (uncompressedLength.value_or(0) != 0) {
uncompressedLength != 0) {
throw std::runtime_error("Codec: invalid uncompressed length"); throw std::runtime_error("Codec: invalid uncompressed length");
} }
return IOBuf::create(0); return IOBuf::create(0);
...@@ -109,18 +109,17 @@ std::unique_ptr<IOBuf> Codec::uncompress(const IOBuf* data, ...@@ -109,18 +109,17 @@ std::unique_ptr<IOBuf> Codec::uncompress(const IOBuf* data,
std::string Codec::uncompress( std::string Codec::uncompress(
const StringPiece data, const StringPiece data,
uint64_t uncompressedLength) { Optional<uint64_t> uncompressedLength) {
if (uncompressedLength == UNKNOWN_UNCOMPRESSED_LENGTH) { if (!uncompressedLength) {
if (needsUncompressedLength()) { if (needsUncompressedLength()) {
throw std::invalid_argument("Codec: uncompressed length required"); throw std::invalid_argument("Codec: uncompressed length required");
} }
} else if (uncompressedLength > maxUncompressedLength()) { } else if (*uncompressedLength > maxUncompressedLength()) {
throw std::runtime_error("Codec: uncompressed length too large"); throw std::runtime_error("Codec: uncompressed length too large");
} }
if (data.empty()) { if (data.empty()) {
if (uncompressedLength != UNKNOWN_UNCOMPRESSED_LENGTH && if (uncompressedLength.value_or(0) != 0) {
uncompressedLength != 0) {
throw std::runtime_error("Codec: invalid uncompressed length"); throw std::runtime_error("Codec: invalid uncompressed length");
} }
return ""; return "";
...@@ -149,7 +148,7 @@ std::vector<std::string> Codec::validPrefixes() const { ...@@ -149,7 +148,7 @@ std::vector<std::string> Codec::validPrefixes() const {
return {}; return {};
} }
bool Codec::canUncompress(const IOBuf*, uint64_t) const { bool Codec::canUncompress(const IOBuf*, Optional<uint64_t>) const {
return false; return false;
} }
...@@ -166,7 +165,7 @@ std::string Codec::doCompressString(const StringPiece data) { ...@@ -166,7 +165,7 @@ std::string Codec::doCompressString(const StringPiece data) {
std::string Codec::doUncompressString( std::string Codec::doUncompressString(
const StringPiece data, const StringPiece data,
uint64_t uncompressedLength) { Optional<uint64_t> uncompressedLength) {
const IOBuf inputBuffer{IOBuf::WRAP_BUFFER, data}; const IOBuf inputBuffer{IOBuf::WRAP_BUFFER, data};
auto outputBuffer = doUncompress(&inputBuffer, uncompressedLength); auto outputBuffer = doUncompress(&inputBuffer, uncompressedLength);
std::string output; std::string output;
...@@ -191,7 +190,7 @@ class NoCompressionCodec final : public Codec { ...@@ -191,7 +190,7 @@ class NoCompressionCodec final : public Codec {
std::unique_ptr<IOBuf> doCompress(const IOBuf* data) override; std::unique_ptr<IOBuf> doCompress(const IOBuf* data) override;
std::unique_ptr<IOBuf> doUncompress( std::unique_ptr<IOBuf> doUncompress(
const IOBuf* data, const IOBuf* data,
uint64_t uncompressedLength) override; Optional<uint64_t> uncompressedLength) override;
}; };
std::unique_ptr<Codec> NoCompressionCodec::create(int level, CodecType type) { std::unique_ptr<Codec> NoCompressionCodec::create(int level, CodecType type) {
...@@ -220,11 +219,11 @@ std::unique_ptr<IOBuf> NoCompressionCodec::doCompress( ...@@ -220,11 +219,11 @@ std::unique_ptr<IOBuf> NoCompressionCodec::doCompress(
std::unique_ptr<IOBuf> NoCompressionCodec::doUncompress( std::unique_ptr<IOBuf> NoCompressionCodec::doUncompress(
const IOBuf* data, const IOBuf* data,
uint64_t uncompressedLength) { Optional<uint64_t> uncompressedLength) {
if (uncompressedLength != UNKNOWN_UNCOMPRESSED_LENGTH && if (uncompressedLength &&
data->computeChainDataLength() != uncompressedLength) { data->computeChainDataLength() != *uncompressedLength) {
throw std::runtime_error(to<std::string>( throw std::runtime_error(
"NoCompressionCodec: invalid uncompressed length")); to<std::string>("NoCompressionCodec: invalid uncompressed length"));
} }
return data->clone(); return data->clone();
} }
...@@ -318,7 +317,7 @@ class LZ4Codec final : public Codec { ...@@ -318,7 +317,7 @@ class LZ4Codec final : public Codec {
std::unique_ptr<IOBuf> doCompress(const IOBuf* data) override; std::unique_ptr<IOBuf> doCompress(const IOBuf* data) override;
std::unique_ptr<IOBuf> doUncompress( std::unique_ptr<IOBuf> doUncompress(
const IOBuf* data, const IOBuf* data,
uint64_t uncompressedLength) override; Optional<uint64_t> uncompressedLength) override;
bool highCompression_; bool highCompression_;
}; };
...@@ -402,7 +401,7 @@ std::unique_ptr<IOBuf> LZ4Codec::doCompress(const IOBuf* data) { ...@@ -402,7 +401,7 @@ std::unique_ptr<IOBuf> LZ4Codec::doCompress(const IOBuf* data) {
std::unique_ptr<IOBuf> LZ4Codec::doUncompress( std::unique_ptr<IOBuf> LZ4Codec::doUncompress(
const IOBuf* data, const IOBuf* data,
uint64_t uncompressedLength) { Optional<uint64_t> uncompressedLength) {
IOBuf clone; IOBuf clone;
if (data->isChained()) { if (data->isChained()) {
// LZ4 doesn't support streaming, so we have to coalesce // LZ4 doesn't support streaming, so we have to coalesce
...@@ -414,16 +413,14 @@ std::unique_ptr<IOBuf> LZ4Codec::doUncompress( ...@@ -414,16 +413,14 @@ std::unique_ptr<IOBuf> LZ4Codec::doUncompress(
uint64_t actualUncompressedLength; uint64_t actualUncompressedLength;
if (encodeSize()) { if (encodeSize()) {
actualUncompressedLength = decodeVarintFromCursor(cursor); actualUncompressedLength = decodeVarintFromCursor(cursor);
if (uncompressedLength != UNKNOWN_UNCOMPRESSED_LENGTH && if (uncompressedLength && *uncompressedLength != actualUncompressedLength) {
uncompressedLength != actualUncompressedLength) {
throw std::runtime_error("LZ4Codec: invalid uncompressed length"); throw std::runtime_error("LZ4Codec: invalid uncompressed length");
} }
} else { } else {
actualUncompressedLength = uncompressedLength; // Invariants
if (actualUncompressedLength == UNKNOWN_UNCOMPRESSED_LENGTH || DCHECK(uncompressedLength.hasValue());
actualUncompressedLength > maxUncompressedLength()) { DCHECK(*uncompressedLength <= maxUncompressedLength());
throw std::runtime_error("LZ4Codec: invalid uncompressed length"); actualUncompressedLength = *uncompressedLength;
}
} }
auto sp = StringPiece{cursor.peekBytes()}; auto sp = StringPiece{cursor.peekBytes()};
...@@ -451,14 +448,14 @@ class LZ4FrameCodec final : public Codec { ...@@ -451,14 +448,14 @@ class LZ4FrameCodec final : public Codec {
~LZ4FrameCodec(); ~LZ4FrameCodec();
std::vector<std::string> validPrefixes() const override; std::vector<std::string> validPrefixes() const override;
bool canUncompress(const IOBuf* data, uint64_t uncompressedLength) bool canUncompress(const IOBuf* data, Optional<uint64_t> uncompressedLength)
const override; const override;
private: private:
std::unique_ptr<IOBuf> doCompress(const IOBuf* data) override; std::unique_ptr<IOBuf> doCompress(const IOBuf* data) override;
std::unique_ptr<IOBuf> doUncompress( std::unique_ptr<IOBuf> doUncompress(
const IOBuf* data, const IOBuf* data,
uint64_t uncompressedLength) override; Optional<uint64_t> uncompressedLength) override;
// Reset the dctx_ if it is dirty or null. // Reset the dctx_ if it is dirty or null.
void resetDCtx(); void resetDCtx();
...@@ -480,7 +477,7 @@ std::vector<std::string> LZ4FrameCodec::validPrefixes() const { ...@@ -480,7 +477,7 @@ std::vector<std::string> LZ4FrameCodec::validPrefixes() const {
return {prefixToStringLE(kLZ4FrameMagicLE)}; return {prefixToStringLE(kLZ4FrameMagicLE)};
} }
bool LZ4FrameCodec::canUncompress(const IOBuf* data, uint64_t) const { bool LZ4FrameCodec::canUncompress(const IOBuf* data, Optional<uint64_t>) const {
return dataStartsWithLE(data, kLZ4FrameMagicLE); return dataStartsWithLE(data, kLZ4FrameMagicLE);
} }
...@@ -551,7 +548,7 @@ std::unique_ptr<IOBuf> LZ4FrameCodec::doCompress(const IOBuf* data) { ...@@ -551,7 +548,7 @@ std::unique_ptr<IOBuf> LZ4FrameCodec::doCompress(const IOBuf* data) {
std::unique_ptr<IOBuf> LZ4FrameCodec::doUncompress( std::unique_ptr<IOBuf> LZ4FrameCodec::doUncompress(
const IOBuf* data, const IOBuf* data,
uint64_t uncompressedLength) { Optional<uint64_t> uncompressedLength) {
// Reset the dctx if any errors have occurred // Reset the dctx if any errors have occurred
resetDCtx(); resetDCtx();
// Coalesce the data // Coalesce the data
...@@ -569,12 +566,12 @@ std::unique_ptr<IOBuf> LZ4FrameCodec::doUncompress( ...@@ -569,12 +566,12 @@ std::unique_ptr<IOBuf> LZ4FrameCodec::doUncompress(
IOBufQueue queue(IOBufQueue::cacheChainLength()); IOBufQueue queue(IOBufQueue::cacheChainLength());
auto blockSize = uint64_t{64} << 10; auto blockSize = uint64_t{64} << 10;
auto growthSize = uint64_t{4} << 20; auto growthSize = uint64_t{4} << 20;
if (uncompressedLength != UNKNOWN_UNCOMPRESSED_LENGTH) { if (uncompressedLength) {
// Allocate uncompressedLength in one chunk (up to 64 MB) // Allocate uncompressedLength in one chunk (up to 64 MB)
const auto allocateSize = std::min(uncompressedLength, uint64_t{64} << 20); const auto allocateSize = std::min(*uncompressedLength, uint64_t{64} << 20);
queue.preallocate(allocateSize, allocateSize); queue.preallocate(allocateSize, allocateSize);
blockSize = std::min(uncompressedLength, blockSize); blockSize = std::min(*uncompressedLength, blockSize);
growthSize = std::min(uncompressedLength, growthSize); growthSize = std::min(*uncompressedLength, growthSize);
} else { } else {
// Reduce growthSize for small data // Reduce growthSize for small data
const auto guessUncompressedLen = const auto guessUncompressedLen =
...@@ -605,8 +602,7 @@ std::unique_ptr<IOBuf> LZ4FrameCodec::doUncompress( ...@@ -605,8 +602,7 @@ std::unique_ptr<IOBuf> LZ4FrameCodec::doUncompress(
} while (code != 0); } while (code != 0);
// At this point the decompression context can be reused // At this point the decompression context can be reused
dirty_ = false; dirty_ = false;
if (uncompressedLength != UNKNOWN_UNCOMPRESSED_LENGTH && if (uncompressedLength && queue.chainLength() != *uncompressedLength) {
queue.chainLength() != uncompressedLength) {
throw std::runtime_error("LZ4Frame error: Invalid uncompressedLength"); throw std::runtime_error("LZ4Frame error: Invalid uncompressedLength");
} }
return queue.move(); return queue.move();
...@@ -666,7 +662,7 @@ class SnappyCodec final : public Codec { ...@@ -666,7 +662,7 @@ class SnappyCodec final : public Codec {
std::unique_ptr<IOBuf> doCompress(const IOBuf* data) override; std::unique_ptr<IOBuf> doCompress(const IOBuf* data) override;
std::unique_ptr<IOBuf> doUncompress( std::unique_ptr<IOBuf> doUncompress(
const IOBuf* data, const IOBuf* data,
uint64_t uncompressedLength) override; Optional<uint64_t> uncompressedLength) override;
}; };
std::unique_ptr<Codec> SnappyCodec::create(int level, CodecType type) { std::unique_ptr<Codec> SnappyCodec::create(int level, CodecType type) {
...@@ -707,8 +703,9 @@ std::unique_ptr<IOBuf> SnappyCodec::doCompress(const IOBuf* data) { ...@@ -707,8 +703,9 @@ std::unique_ptr<IOBuf> SnappyCodec::doCompress(const IOBuf* data) {
return out; return out;
} }
std::unique_ptr<IOBuf> SnappyCodec::doUncompress(const IOBuf* data, std::unique_ptr<IOBuf> SnappyCodec::doUncompress(
uint64_t uncompressedLength) { const IOBuf* data,
Optional<uint64_t> uncompressedLength) {
uint32_t actualUncompressedLength = 0; uint32_t actualUncompressedLength = 0;
{ {
...@@ -716,8 +713,7 @@ std::unique_ptr<IOBuf> SnappyCodec::doUncompress(const IOBuf* data, ...@@ -716,8 +713,7 @@ std::unique_ptr<IOBuf> SnappyCodec::doUncompress(const IOBuf* data,
if (!snappy::GetUncompressedLength(&source, &actualUncompressedLength)) { if (!snappy::GetUncompressedLength(&source, &actualUncompressedLength)) {
throw std::runtime_error("snappy::GetUncompressedLength failed"); throw std::runtime_error("snappy::GetUncompressedLength failed");
} }
if (uncompressedLength != UNKNOWN_UNCOMPRESSED_LENGTH && if (uncompressedLength && *uncompressedLength != actualUncompressedLength) {
uncompressedLength != actualUncompressedLength) {
throw std::runtime_error("snappy: invalid uncompressed length"); throw std::runtime_error("snappy: invalid uncompressed length");
} }
} }
...@@ -748,14 +744,14 @@ class ZlibCodec final : public Codec { ...@@ -748,14 +744,14 @@ class ZlibCodec final : public Codec {
explicit ZlibCodec(int level, CodecType type); explicit ZlibCodec(int level, CodecType type);
std::vector<std::string> validPrefixes() const override; std::vector<std::string> validPrefixes() const override;
bool canUncompress(const IOBuf* data, uint64_t uncompressedLength) bool canUncompress(const IOBuf* data, Optional<uint64_t> uncompressedLength)
const override; const override;
private: private:
std::unique_ptr<IOBuf> doCompress(const IOBuf* data) override; std::unique_ptr<IOBuf> doCompress(const IOBuf* data) override;
std::unique_ptr<IOBuf> doUncompress( std::unique_ptr<IOBuf> doUncompress(
const IOBuf* data, const IOBuf* data,
uint64_t uncompressedLength) override; Optional<uint64_t> uncompressedLength) override;
std::unique_ptr<IOBuf> addOutputBuffer(z_stream* stream, uint32_t length); std::unique_ptr<IOBuf> addOutputBuffer(z_stream* stream, uint32_t length);
bool doInflate(z_stream* stream, IOBuf* head, uint32_t bufferLength); bool doInflate(z_stream* stream, IOBuf* head, uint32_t bufferLength);
...@@ -809,7 +805,7 @@ std::vector<std::string> ZlibCodec::validPrefixes() const { ...@@ -809,7 +805,7 @@ std::vector<std::string> ZlibCodec::validPrefixes() const {
} }
} }
bool ZlibCodec::canUncompress(const IOBuf* data, uint64_t) const { bool ZlibCodec::canUncompress(const IOBuf* data, Optional<uint64_t>) const {
if (type() == CodecType::ZLIB) { if (type() == CodecType::ZLIB) {
uint16_t value; uint16_t value;
Cursor cursor{data}; Cursor cursor{data};
...@@ -981,8 +977,9 @@ std::unique_ptr<IOBuf> ZlibCodec::doCompress(const IOBuf* data) { ...@@ -981,8 +977,9 @@ std::unique_ptr<IOBuf> ZlibCodec::doCompress(const IOBuf* data) {
return out; return out;
} }
std::unique_ptr<IOBuf> ZlibCodec::doUncompress(const IOBuf* data, std::unique_ptr<IOBuf> ZlibCodec::doUncompress(
uint64_t uncompressedLength) { const IOBuf* data,
Optional<uint64_t> uncompressedLength) {
z_stream stream; z_stream stream;
stream.zalloc = nullptr; stream.zalloc = nullptr;
stream.zfree = nullptr; stream.zfree = nullptr;
...@@ -1020,10 +1017,9 @@ std::unique_ptr<IOBuf> ZlibCodec::doUncompress(const IOBuf* data, ...@@ -1020,10 +1017,9 @@ std::unique_ptr<IOBuf> ZlibCodec::doUncompress(const IOBuf* data,
auto out = addOutputBuffer( auto out = addOutputBuffer(
&stream, &stream,
((uncompressedLength != UNKNOWN_UNCOMPRESSED_LENGTH && ((uncompressedLength && *uncompressedLength <= maxSingleStepLength)
uncompressedLength <= maxSingleStepLength) ? ? *uncompressedLength
uncompressedLength : : defaultBufferLength));
defaultBufferLength));
bool streamEnd = false; bool streamEnd = false;
for (auto& range : *data) { for (auto& range : *data) {
...@@ -1050,10 +1046,9 @@ std::unique_ptr<IOBuf> ZlibCodec::doUncompress(const IOBuf* data, ...@@ -1050,10 +1046,9 @@ std::unique_ptr<IOBuf> ZlibCodec::doUncompress(const IOBuf* data,
out->prev()->trimEnd(stream.avail_out); out->prev()->trimEnd(stream.avail_out);
if (uncompressedLength != UNKNOWN_UNCOMPRESSED_LENGTH && if (uncompressedLength && *uncompressedLength != stream.total_out) {
uncompressedLength != stream.total_out) { throw std::runtime_error(
throw std::runtime_error(to<std::string>( to<std::string>("ZlibCodec: invalid uncompressed length"));
"ZlibCodec: invalid uncompressed length"));
} }
success = true; // we survived success = true; // we survived
...@@ -1074,7 +1069,7 @@ class LZMA2Codec final : public Codec { ...@@ -1074,7 +1069,7 @@ class LZMA2Codec final : public Codec {
explicit LZMA2Codec(int level, CodecType type); explicit LZMA2Codec(int level, CodecType type);
std::vector<std::string> validPrefixes() const override; std::vector<std::string> validPrefixes() const override;
bool canUncompress(const IOBuf* data, uint64_t uncompressedLength) bool canUncompress(const IOBuf* data, Optional<uint64_t> uncompressedLength)
const override; const override;
private: private:
...@@ -1086,7 +1081,7 @@ class LZMA2Codec final : public Codec { ...@@ -1086,7 +1081,7 @@ class LZMA2Codec final : public Codec {
std::unique_ptr<IOBuf> doCompress(const IOBuf* data) override; std::unique_ptr<IOBuf> doCompress(const IOBuf* data) override;
std::unique_ptr<IOBuf> doUncompress( std::unique_ptr<IOBuf> doUncompress(
const IOBuf* data, const IOBuf* data,
uint64_t uncompressedLength) override; Optional<uint64_t> uncompressedLength) override;
std::unique_ptr<IOBuf> addOutputBuffer(lzma_stream* stream, size_t length); std::unique_ptr<IOBuf> addOutputBuffer(lzma_stream* stream, size_t length);
bool doInflate(lzma_stream* stream, IOBuf* head, size_t bufferLength); bool doInflate(lzma_stream* stream, IOBuf* head, size_t bufferLength);
...@@ -1104,7 +1099,7 @@ std::vector<std::string> LZMA2Codec::validPrefixes() const { ...@@ -1104,7 +1099,7 @@ std::vector<std::string> LZMA2Codec::validPrefixes() const {
return {prefixToStringLE(kLZMA2MagicLE, kLZMA2MagicBytes)}; return {prefixToStringLE(kLZMA2MagicLE, kLZMA2MagicBytes)};
} }
bool LZMA2Codec::canUncompress(const IOBuf* data, uint64_t) const { bool LZMA2Codec::canUncompress(const IOBuf* data, Optional<uint64_t>) const {
if (type() == CodecType::LZMA2_VARINT_SIZE) { if (type() == CodecType::LZMA2_VARINT_SIZE) {
return false; return false;
} }
...@@ -1255,8 +1250,9 @@ bool LZMA2Codec::doInflate(lzma_stream* stream, ...@@ -1255,8 +1250,9 @@ bool LZMA2Codec::doInflate(lzma_stream* stream,
return false; return false;
} }
std::unique_ptr<IOBuf> LZMA2Codec::doUncompress(const IOBuf* data, std::unique_ptr<IOBuf> LZMA2Codec::doUncompress(
uint64_t uncompressedLength) { const IOBuf* data,
Optional<uint64_t> uncompressedLength) {
lzma_ret rc; lzma_ret rc;
lzma_stream stream = LZMA_STREAM_INIT; lzma_stream stream = LZMA_STREAM_INIT;
...@@ -1275,8 +1271,7 @@ std::unique_ptr<IOBuf> LZMA2Codec::doUncompress(const IOBuf* data, ...@@ -1275,8 +1271,7 @@ std::unique_ptr<IOBuf> LZMA2Codec::doUncompress(const IOBuf* data,
folly::io::Cursor cursor(data); folly::io::Cursor cursor(data);
if (encodeSize()) { if (encodeSize()) {
const uint64_t actualUncompressedLength = decodeVarintFromCursor(cursor); const uint64_t actualUncompressedLength = decodeVarintFromCursor(cursor);
if (uncompressedLength != UNKNOWN_UNCOMPRESSED_LENGTH && if (uncompressedLength && *uncompressedLength != actualUncompressedLength) {
uncompressedLength != actualUncompressedLength) {
throw std::runtime_error("LZMA2Codec: invalid uncompressed length"); throw std::runtime_error("LZMA2Codec: invalid uncompressed length");
} }
uncompressedLength = actualUncompressedLength; uncompressedLength = actualUncompressedLength;
...@@ -1284,9 +1279,8 @@ std::unique_ptr<IOBuf> LZMA2Codec::doUncompress(const IOBuf* data, ...@@ -1284,9 +1279,8 @@ std::unique_ptr<IOBuf> LZMA2Codec::doUncompress(const IOBuf* data,
auto out = addOutputBuffer( auto out = addOutputBuffer(
&stream, &stream,
((uncompressedLength != UNKNOWN_UNCOMPRESSED_LENGTH && ((uncompressedLength && *uncompressedLength <= maxSingleStepLength)
uncompressedLength <= maxSingleStepLength) ? *uncompressedLength
? uncompressedLength
: defaultBufferLength)); : defaultBufferLength));
bool streamEnd = false; bool streamEnd = false;
...@@ -1314,8 +1308,7 @@ std::unique_ptr<IOBuf> LZMA2Codec::doUncompress(const IOBuf* data, ...@@ -1314,8 +1308,7 @@ std::unique_ptr<IOBuf> LZMA2Codec::doUncompress(const IOBuf* data,
out->prev()->trimEnd(stream.avail_out); out->prev()->trimEnd(stream.avail_out);
if (uncompressedLength != UNKNOWN_UNCOMPRESSED_LENGTH && if (uncompressedLength && *uncompressedLength != stream.total_out) {
uncompressedLength != stream.total_out) {
throw std::runtime_error( throw std::runtime_error(
to<std::string>("LZMA2Codec: invalid uncompressed length")); to<std::string>("LZMA2Codec: invalid uncompressed length"));
} }
...@@ -1336,7 +1329,7 @@ class ZSTDCodec final : public Codec { ...@@ -1336,7 +1329,7 @@ class ZSTDCodec final : public Codec {
explicit ZSTDCodec(int level, CodecType type); explicit ZSTDCodec(int level, CodecType type);
std::vector<std::string> validPrefixes() const override; std::vector<std::string> validPrefixes() const override;
bool canUncompress(const IOBuf* data, uint64_t uncompressedLength) bool canUncompress(const IOBuf* data, Optional<uint64_t> uncompressedLength)
const override; const override;
private: private:
...@@ -1344,7 +1337,7 @@ class ZSTDCodec final : public Codec { ...@@ -1344,7 +1337,7 @@ class ZSTDCodec final : public Codec {
std::unique_ptr<IOBuf> doCompress(const IOBuf* data) override; std::unique_ptr<IOBuf> doCompress(const IOBuf* data) override;
std::unique_ptr<IOBuf> doUncompress( std::unique_ptr<IOBuf> doUncompress(
const IOBuf* data, const IOBuf* data,
uint64_t uncompressedLength) override; Optional<uint64_t> uncompressedLength) override;
int level_; int level_;
}; };
...@@ -1355,7 +1348,7 @@ std::vector<std::string> ZSTDCodec::validPrefixes() const { ...@@ -1355,7 +1348,7 @@ std::vector<std::string> ZSTDCodec::validPrefixes() const {
return {prefixToStringLE(kZSTDMagicLE)}; return {prefixToStringLE(kZSTDMagicLE)};
} }
bool ZSTDCodec::canUncompress(const IOBuf* data, uint64_t) const { bool ZSTDCodec::canUncompress(const IOBuf* data, Optional<uint64_t>) const {
return dataStartsWithLE(data, kZSTDMagicLE); return dataStartsWithLE(data, kZSTDMagicLE);
} }
...@@ -1450,12 +1443,12 @@ std::unique_ptr<IOBuf> ZSTDCodec::doCompress(const IOBuf* data) { ...@@ -1450,12 +1443,12 @@ std::unique_ptr<IOBuf> ZSTDCodec::doCompress(const IOBuf* data) {
static std::unique_ptr<IOBuf> zstdUncompressBuffer( static std::unique_ptr<IOBuf> zstdUncompressBuffer(
const IOBuf* data, const IOBuf* data,
uint64_t uncompressedLength) { Optional<uint64_t> uncompressedLength) {
// Check preconditions // Check preconditions
DCHECK(!data->isChained()); DCHECK(!data->isChained());
DCHECK(uncompressedLength != Codec::UNKNOWN_UNCOMPRESSED_LENGTH); DCHECK(uncompressedLength.hasValue());
auto uncompressed = IOBuf::create(uncompressedLength); auto uncompressed = IOBuf::create(*uncompressedLength);
const auto decompressedSize = ZSTD_decompress( const auto decompressedSize = ZSTD_decompress(
uncompressed->writableTail(), uncompressed->writableTail(),
uncompressed->tailroom(), uncompressed->tailroom(),
...@@ -1471,7 +1464,7 @@ static std::unique_ptr<IOBuf> zstdUncompressBuffer( ...@@ -1471,7 +1464,7 @@ static std::unique_ptr<IOBuf> zstdUncompressBuffer(
static std::unique_ptr<IOBuf> zstdUncompressStream( static std::unique_ptr<IOBuf> zstdUncompressStream(
const IOBuf* data, const IOBuf* data,
uint64_t uncompressedLength) { Optional<uint64_t> uncompressedLength) {
auto zds = ZSTD_createDStream(); auto zds = ZSTD_createDStream();
SCOPE_EXIT { SCOPE_EXIT {
ZSTD_freeDStream(zds); ZSTD_freeDStream(zds);
...@@ -1483,10 +1476,7 @@ static std::unique_ptr<IOBuf> zstdUncompressStream( ...@@ -1483,10 +1476,7 @@ static std::unique_ptr<IOBuf> zstdUncompressStream(
ZSTD_outBuffer out{}; ZSTD_outBuffer out{};
ZSTD_inBuffer in{}; ZSTD_inBuffer in{};
auto outputSize = ZSTD_DStreamOutSize(); auto outputSize = uncompressedLength.value_or(ZSTD_DStreamOutSize());
if (uncompressedLength != Codec::UNKNOWN_UNCOMPRESSED_LENGTH) {
outputSize = uncompressedLength;
}
IOBufQueue queue(IOBufQueue::cacheChainLength()); IOBufQueue queue(IOBufQueue::cacheChainLength());
...@@ -1524,8 +1514,7 @@ static std::unique_ptr<IOBuf> zstdUncompressStream( ...@@ -1524,8 +1514,7 @@ static std::unique_ptr<IOBuf> zstdUncompressStream(
if (in.pos != in.size || !cursor.isAtEnd()) { if (in.pos != in.size || !cursor.isAtEnd()) {
throw std::runtime_error("ZSTD: junk after end of data"); throw std::runtime_error("ZSTD: junk after end of data");
} }
if (uncompressedLength != Codec::UNKNOWN_UNCOMPRESSED_LENGTH && if (uncompressedLength && queue.chainLength() != *uncompressedLength) {
queue.chainLength() != uncompressedLength) {
throw std::runtime_error("ZSTD: invalid uncompressed length"); throw std::runtime_error("ZSTD: invalid uncompressed length");
} }
...@@ -1534,21 +1523,20 @@ static std::unique_ptr<IOBuf> zstdUncompressStream( ...@@ -1534,21 +1523,20 @@ static std::unique_ptr<IOBuf> zstdUncompressStream(
std::unique_ptr<IOBuf> ZSTDCodec::doUncompress( std::unique_ptr<IOBuf> ZSTDCodec::doUncompress(
const IOBuf* data, const IOBuf* data,
uint64_t uncompressedLength) { Optional<uint64_t> uncompressedLength) {
{ {
// Read decompressed size from frame if available in first IOBuf. // Read decompressed size from frame if available in first IOBuf.
const auto decompressedSize = const auto decompressedSize =
ZSTD_getDecompressedSize(data->data(), data->length()); ZSTD_getDecompressedSize(data->data(), data->length());
if (decompressedSize != 0) { if (decompressedSize != 0) {
if (uncompressedLength != Codec::UNKNOWN_UNCOMPRESSED_LENGTH && if (uncompressedLength && *uncompressedLength != decompressedSize) {
uncompressedLength != decompressedSize) {
throw std::runtime_error("ZSTD: invalid uncompressed length"); throw std::runtime_error("ZSTD: invalid uncompressed length");
} }
uncompressedLength = decompressedSize; uncompressedLength = decompressedSize;
} }
} }
// Faster to decompress using ZSTD_decompress() if we can. // Faster to decompress using ZSTD_decompress() if we can.
if (uncompressedLength != UNKNOWN_UNCOMPRESSED_LENGTH && !data->isChained()) { if (uncompressedLength && !data->isChained()) {
return zstdUncompressBuffer(data, uncompressedLength); return zstdUncompressBuffer(data, uncompressedLength);
} }
// Fall back to slower streaming decompression. // Fall back to slower streaming decompression.
...@@ -1565,14 +1553,14 @@ class Bzip2Codec final : public Codec { ...@@ -1565,14 +1553,14 @@ class Bzip2Codec final : public Codec {
explicit Bzip2Codec(int level, CodecType type); explicit Bzip2Codec(int level, CodecType type);
std::vector<std::string> validPrefixes() const override; std::vector<std::string> validPrefixes() const override;
bool canUncompress(IOBuf const* data, uint64_t uncompressedLength) bool canUncompress(IOBuf const* data, Optional<uint64_t> uncompressedLength)
const override; const override;
private: private:
std::unique_ptr<IOBuf> doCompress(IOBuf const* data) override; std::unique_ptr<IOBuf> doCompress(IOBuf const* data) override;
std::unique_ptr<IOBuf> doUncompress( std::unique_ptr<IOBuf> doUncompress(
IOBuf const* data, IOBuf const* data,
uint64_t uncompressedLength) override; Optional<uint64_t> uncompressedLength) override;
int level_; int level_;
}; };
...@@ -1610,7 +1598,7 @@ std::vector<std::string> Bzip2Codec::validPrefixes() const { ...@@ -1610,7 +1598,7 @@ std::vector<std::string> Bzip2Codec::validPrefixes() const {
return {prefixToStringLE(kBzip2MagicLE, kBzip2MagicBytes)}; return {prefixToStringLE(kBzip2MagicLE, kBzip2MagicBytes)};
} }
bool Bzip2Codec::canUncompress(IOBuf const* data, uint64_t) const { bool Bzip2Codec::canUncompress(IOBuf const* data, Optional<uint64_t>) const {
return dataStartsWithLE(data, kBzip2MagicLE, kBzip2MagicBytes); return dataStartsWithLE(data, kBzip2MagicLE, kBzip2MagicBytes);
} }
...@@ -1706,7 +1694,7 @@ std::unique_ptr<IOBuf> Bzip2Codec::doCompress(IOBuf const* data) { ...@@ -1706,7 +1694,7 @@ std::unique_ptr<IOBuf> Bzip2Codec::doCompress(IOBuf const* data) {
std::unique_ptr<IOBuf> Bzip2Codec::doUncompress( std::unique_ptr<IOBuf> Bzip2Codec::doUncompress(
const IOBuf* data, const IOBuf* data,
uint64_t uncompressedLength) { Optional<uint64_t> uncompressedLength) {
bz_stream stream = createBzStream(); bz_stream stream = createBzStream();
bzCheck(BZ2_bzDecompressInit(&stream, 0, 0)); bzCheck(BZ2_bzDecompressInit(&stream, 0, 0));
SCOPE_EXIT { SCOPE_EXIT {
...@@ -1720,9 +1708,8 @@ std::unique_ptr<IOBuf> Bzip2Codec::doUncompress( ...@@ -1720,9 +1708,8 @@ std::unique_ptr<IOBuf> Bzip2Codec::doUncompress(
auto out = addOutputBuffer( auto out = addOutputBuffer(
&stream, &stream,
((uncompressedLength != UNKNOWN_UNCOMPRESSED_LENGTH && ((uncompressedLength && *uncompressedLength <= kMaxSingleStepLength)
uncompressedLength <= kMaxSingleStepLength) ? *uncompressedLength
? uncompressedLength
: kDefaultBufferLength)); : kDefaultBufferLength));
int rc = BZ_OK; int rc = BZ_OK;
...@@ -1753,8 +1740,7 @@ std::unique_ptr<IOBuf> Bzip2Codec::doUncompress( ...@@ -1753,8 +1740,7 @@ std::unique_ptr<IOBuf> Bzip2Codec::doUncompress(
uint64_t const totalOut = uint64_t const totalOut =
(uint64_t(stream.total_out_hi32) << 32) + stream.total_out_lo32; (uint64_t(stream.total_out_hi32) << 32) + stream.total_out_lo32;
if (uncompressedLength != UNKNOWN_UNCOMPRESSED_LENGTH && if (uncompressedLength && uncompressedLength != totalOut) {
uncompressedLength != totalOut) {
throw std::runtime_error("Bzip2 error: Invalid uncompressed length"); throw std::runtime_error("Bzip2 error: Invalid uncompressed length");
} }
...@@ -1773,7 +1759,7 @@ class AutomaticCodec final : public Codec { ...@@ -1773,7 +1759,7 @@ class AutomaticCodec final : public Codec {
explicit AutomaticCodec(std::vector<std::unique_ptr<Codec>> customCodecs); explicit AutomaticCodec(std::vector<std::unique_ptr<Codec>> customCodecs);
std::vector<std::string> validPrefixes() const override; std::vector<std::string> validPrefixes() const override;
bool canUncompress(const IOBuf* data, uint64_t uncompressedLength) bool canUncompress(const IOBuf* data, Optional<uint64_t> uncompressedLength)
const override; const override;
private: private:
...@@ -1785,7 +1771,7 @@ class AutomaticCodec final : public Codec { ...@@ -1785,7 +1771,7 @@ class AutomaticCodec final : public Codec {
} }
std::unique_ptr<IOBuf> doUncompress( std::unique_ptr<IOBuf> doUncompress(
const IOBuf* data, const IOBuf* data,
uint64_t uncompressedLength) override; Optional<uint64_t> uncompressedLength) override;
void addCodecIfSupported(CodecType type); void addCodecIfSupported(CodecType type);
...@@ -1808,7 +1794,7 @@ std::vector<std::string> AutomaticCodec::validPrefixes() const { ...@@ -1808,7 +1794,7 @@ std::vector<std::string> AutomaticCodec::validPrefixes() const {
bool AutomaticCodec::canUncompress( bool AutomaticCodec::canUncompress(
const IOBuf* data, const IOBuf* data,
uint64_t uncompressedLength) const { Optional<uint64_t> uncompressedLength) const {
return std::any_of( return std::any_of(
codecs_.begin(), codecs_.begin(),
codecs_.end(), codecs_.end(),
...@@ -1914,7 +1900,7 @@ uint64_t AutomaticCodec::doMaxUncompressedLength() const { ...@@ -1914,7 +1900,7 @@ uint64_t AutomaticCodec::doMaxUncompressedLength() const {
std::unique_ptr<IOBuf> AutomaticCodec::doUncompress( std::unique_ptr<IOBuf> AutomaticCodec::doUncompress(
const IOBuf* data, const IOBuf* data,
uint64_t uncompressedLength) { Optional<uint64_t> uncompressedLength) {
for (auto&& codec : codecs_) { for (auto&& codec : codecs_) {
if (codec->canUncompress(data, uncompressedLength)) { if (codec->canUncompress(data, uncompressedLength)) {
return codec->uncompress(data, uncompressedLength); return codec->uncompress(data, uncompressedLength);
......
...@@ -22,6 +22,7 @@ ...@@ -22,6 +22,7 @@
#include <string> #include <string>
#include <vector> #include <vector>
#include <folly/Optional.h>
#include <folly/Range.h> #include <folly/Range.h>
#include <folly/io/IOBuf.h> #include <folly/io/IOBuf.h>
...@@ -110,7 +111,7 @@ class Codec { ...@@ -110,7 +111,7 @@ class Codec {
* Return the maximum length of data that may be compressed with this codec. * Return the maximum length of data that may be compressed with this codec.
* NO_COMPRESSION and ZLIB support arbitrary lengths; * NO_COMPRESSION and ZLIB support arbitrary lengths;
* LZ4 supports up to 1.9GiB; SNAPPY supports up to 4GiB. * LZ4 supports up to 1.9GiB; SNAPPY supports up to 4GiB.
* May return UNLIMITED_UNCOMPRESSED_LENGTH if unlimited. * May return UNLIMITED_UNCOMPRESSED_LENGTH (uint64_t(-1)) if unlimited.
*/ */
uint64_t maxUncompressedLength() const; uint64_t maxUncompressedLength() const;
...@@ -153,12 +154,11 @@ class Codec { ...@@ -153,12 +154,11 @@ class Codec {
* Regardless of the behavior of the underlying compressor, uncompressing * Regardless of the behavior of the underlying compressor, uncompressing
* an empty IOBuf chain will return an empty IOBuf chain. * an empty IOBuf chain will return an empty IOBuf chain.
*/ */
static constexpr uint64_t UNKNOWN_UNCOMPRESSED_LENGTH = uint64_t(-1); static constexpr uint64_t UNLIMITED_UNCOMPRESSED_LENGTH = uint64_t(-1);
static constexpr uint64_t UNLIMITED_UNCOMPRESSED_LENGTH = uint64_t(-2);
std::unique_ptr<IOBuf> uncompress( std::unique_ptr<IOBuf> uncompress(
const IOBuf* data, const IOBuf* data,
uint64_t uncompressedLength = UNKNOWN_UNCOMPRESSED_LENGTH); folly::Optional<uint64_t> uncompressedLength = folly::none);
/** /**
* Uncompresses data. May involve additional copies compared to the overload * Uncompresses data. May involve additional copies compared to the overload
...@@ -167,7 +167,7 @@ class Codec { ...@@ -167,7 +167,7 @@ class Codec {
*/ */
std::string uncompress( std::string uncompress(
StringPiece data, StringPiece data,
uint64_t uncompressedLength = UNKNOWN_UNCOMPRESSED_LENGTH); folly::Optional<uint64_t> uncompressedLength = folly::none);
protected: protected:
explicit Codec(CodecType type); explicit Codec(CodecType type);
...@@ -189,7 +189,7 @@ class Codec { ...@@ -189,7 +189,7 @@ class Codec {
*/ */
virtual bool canUncompress( virtual bool canUncompress(
const folly::IOBuf* data, const folly::IOBuf* data,
uint64_t uncompressedLength = UNKNOWN_UNCOMPRESSED_LENGTH) const; folly::Optional<uint64_t> uncompressedLength = folly::none) const;
private: private:
// default: no limits (save for special value UNKNOWN_UNCOMPRESSED_LENGTH) // default: no limits (save for special value UNKNOWN_UNCOMPRESSED_LENGTH)
...@@ -197,8 +197,9 @@ class Codec { ...@@ -197,8 +197,9 @@ class Codec {
// default: doesn't need uncompressed length // default: doesn't need uncompressed length
virtual bool doNeedsUncompressedLength() const; virtual bool doNeedsUncompressedLength() const;
virtual std::unique_ptr<IOBuf> doCompress(const folly::IOBuf* data) = 0; virtual std::unique_ptr<IOBuf> doCompress(const folly::IOBuf* data) = 0;
virtual std::unique_ptr<IOBuf> doUncompress(const folly::IOBuf* data, virtual std::unique_ptr<IOBuf> doUncompress(
uint64_t uncompressedLength) = 0; const folly::IOBuf* data,
folly::Optional<uint64_t> uncompressedLength) = 0;
// default: an implementation is provided by default to wrap the strings into // default: an implementation is provided by default to wrap the strings into
// IOBufs and delegate to the IOBuf methods. This incurs a copy of the output // IOBufs and delegate to the IOBuf methods. This incurs a copy of the output
// from IOBuf to string. Implementers, at their discretion, can override // from IOBuf to string. Implementers, at their discretion, can override
...@@ -206,7 +207,7 @@ class Codec { ...@@ -206,7 +207,7 @@ class Codec {
virtual std::string doCompressString(StringPiece data); virtual std::string doCompressString(StringPiece data);
virtual std::string doUncompressString( virtual std::string doUncompressString(
StringPiece data, StringPiece data,
uint64_t uncompressedLength); folly::Optional<uint64_t> uncompressedLength);
CodecType type_; CodecType type_;
}; };
......
...@@ -500,7 +500,7 @@ class CustomCodec : public Codec { ...@@ -500,7 +500,7 @@ class CustomCodec : public Codec {
return {prefix_}; return {prefix_};
} }
bool canUncompress(const IOBuf* data, uint64_t) const override { bool canUncompress(const IOBuf* data, Optional<uint64_t>) const override {
auto clone = data->cloneCoalescedAsValue(); auto clone = data->cloneCoalescedAsValue();
if (clone.length() < prefix_.size()) { if (clone.length() < prefix_.size()) {
return false; return false;
...@@ -517,7 +517,7 @@ class CustomCodec : public Codec { ...@@ -517,7 +517,7 @@ class CustomCodec : public Codec {
std::unique_ptr<IOBuf> doUncompress( std::unique_ptr<IOBuf> doUncompress(
const IOBuf* data, const IOBuf* data,
uint64_t uncompressedLength) override { Optional<uint64_t> uncompressedLength) override {
EXPECT_TRUE(canUncompress(data, uncompressedLength)); EXPECT_TRUE(canUncompress(data, uncompressedLength));
auto clone = data->cloneCoalescedAsValue(); auto clone = data->cloneCoalescedAsValue();
clone.trimStart(prefix_.size()); clone.trimStart(prefix_.size());
...@@ -573,9 +573,9 @@ TEST_P(AutomaticCodecTest, canUncompressOneBytes) { ...@@ -573,9 +573,9 @@ TEST_P(AutomaticCodecTest, canUncompressOneBytes) {
IOBuf buf{IOBuf::CREATE, 1}; IOBuf buf{IOBuf::CREATE, 1};
buf.append(1); buf.append(1);
EXPECT_FALSE(codec_->canUncompress(&buf, 1)); EXPECT_FALSE(codec_->canUncompress(&buf, 1));
EXPECT_FALSE(codec_->canUncompress(&buf, Codec::UNKNOWN_UNCOMPRESSED_LENGTH)); EXPECT_FALSE(codec_->canUncompress(&buf, folly::none));
EXPECT_FALSE(auto_->canUncompress(&buf, 1)); EXPECT_FALSE(auto_->canUncompress(&buf, 1));
EXPECT_FALSE(auto_->canUncompress(&buf, Codec::UNKNOWN_UNCOMPRESSED_LENGTH)); EXPECT_FALSE(auto_->canUncompress(&buf, folly::none));
} }
INSTANTIATE_TEST_CASE_P( INSTANTIATE_TEST_CASE_P(
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment