Commit ad036b61 authored by Nick Terrell's avatar Nick Terrell Committed by Facebook Github Bot

Add support for zstd-1.3.8

Summary:
Adds support for zstd-1.3.8 so OSS builds work.
We can support zstd < 1.3.8 for some time with this small compatibility layer.
I plan on always supporting at minimum the latest 2 zstd versions.

Reviewed By: yfeldblum

Differential Revision: D13569550

fbshipit-source-id: 67d53c9ad0051a889b810c9ad46a2f349122cf7e
parent 8b1bbbf1
...@@ -39,6 +39,33 @@ namespace io { ...@@ -39,6 +39,33 @@ namespace io {
namespace zstd { namespace zstd {
namespace { namespace {
// Compatibility helpers for zstd versions < 1.3.8.
#if ZSTD_VERSION_NUMBER < 10308
#define ZSTD_compressStream2 ZSTD_compress_generic
#define ZSTD_c_compressionLevel ZSTD_p_compressionLevel
#define ZSTD_c_contentSizeFlag ZSTD_p_contentSizeFlag
void resetCCtxSessionAndParameters(ZSTD_CCtx* cctx) {
ZSTD_CCtx_reset(cctx);
}
void resetDCtxSessionAndParameters(ZSTD_DCtx* dctx) {
ZSTD_DCtx_reset(dctx);
}
#else
void resetCCtxSessionAndParameters(ZSTD_CCtx* cctx) {
ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
}
void resetDCtxSessionAndParameters(ZSTD_DCtx* dctx) {
ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters);
}
#endif
void zstdFreeCCtx(ZSTD_CCtx* zc) { void zstdFreeCCtx(ZSTD_CCtx* zc) {
ZSTD_freeCCtx(zc); ZSTD_freeCCtx(zc);
} }
...@@ -165,7 +192,7 @@ void ZSTDStreamCodec::resetCCtx() { ...@@ -165,7 +192,7 @@ void ZSTDStreamCodec::resetCCtx() {
throw std::bad_alloc{}; throw std::bad_alloc{};
} }
} }
ZSTD_CCtx_reset(cctx_.get()); resetCCtxSessionAndParameters(cctx_.get());
zstdThrowIfError( zstdThrowIfError(
ZSTD_CCtx_setParametersUsingCCtxParams(cctx_.get(), options_.params())); ZSTD_CCtx_setParametersUsingCCtxParams(cctx_.get(), options_.params()));
zstdThrowIfError(ZSTD_CCtx_setPledgedSrcSize( zstdThrowIfError(ZSTD_CCtx_setPledgedSrcSize(
...@@ -186,7 +213,7 @@ bool ZSTDStreamCodec::doCompressStream( ...@@ -186,7 +213,7 @@ bool ZSTDStreamCodec::doCompressStream(
input.uncheckedAdvance(in.pos); input.uncheckedAdvance(in.pos);
output.uncheckedAdvance(out.pos); output.uncheckedAdvance(out.pos);
}; };
size_t const rc = zstdThrowIfError(ZSTD_compress_generic( size_t const rc = zstdThrowIfError(ZSTD_compressStream2(
cctx_.get(), &out, &in, zstdTranslateFlush(flushOp))); cctx_.get(), &out, &in, zstdTranslateFlush(flushOp)));
switch (flushOp) { switch (flushOp) {
case StreamCodec::FlushOp::NONE: case StreamCodec::FlushOp::NONE:
...@@ -206,7 +233,7 @@ void ZSTDStreamCodec::resetDCtx() { ...@@ -206,7 +233,7 @@ void ZSTDStreamCodec::resetDCtx() {
throw std::bad_alloc{}; throw std::bad_alloc{};
} }
} }
ZSTD_DCtx_reset(dctx_.get()); resetDCtxSessionAndParameters(dctx_.get());
if (options_.maxWindowSize() != 0) { if (options_.maxWindowSize() != 0) {
zstdThrowIfError( zstdThrowIfError(
ZSTD_DCtx_setMaxWindowSize(dctx_.get(), options_.maxWindowSize())); ZSTD_DCtx_setMaxWindowSize(dctx_.get(), options_.maxWindowSize()));
...@@ -228,7 +255,7 @@ bool ZSTDStreamCodec::doUncompressStream( ...@@ -228,7 +255,7 @@ bool ZSTDStreamCodec::doUncompressStream(
output.uncheckedAdvance(out.pos); output.uncheckedAdvance(out.pos);
}; };
size_t const rc = size_t const rc =
zstdThrowIfError(ZSTD_decompress_generic(dctx_.get(), &out, &in)); zstdThrowIfError(ZSTD_decompressStream(dctx_.get(), &out, &in));
return rc == 0; return rc == 0;
} }
...@@ -242,17 +269,17 @@ Options::Options(int level) : params_(ZSTD_createCCtxParams()), level_(level) { ...@@ -242,17 +269,17 @@ Options::Options(int level) : params_(ZSTD_createCCtxParams()), level_(level) {
zstdThrowIfError(ZSTD_CCtxParams_init(params_.get(), level)); zstdThrowIfError(ZSTD_CCtxParams_init(params_.get(), level));
#else #else
zstdThrowIfError(ZSTD_initCCtxParams(params_.get(), level)); zstdThrowIfError(ZSTD_initCCtxParams(params_.get(), level));
set(ZSTD_p_contentSizeFlag, 1); set(ZSTD_c_contentSizeFlag, 1);
#endif #endif
// zstd-1.3.4 is buggy and only disables Huffman decompression for negative // zstd-1.3.4 is buggy and only disables Huffman decompression for negative
// compression levels if this call is present. This call is begign in other // compression levels if this call is present. This call is begign in other
// versions. // versions.
set(ZSTD_p_compressionLevel, level); set(ZSTD_c_compressionLevel, level);
} }
void Options::set(ZSTD_cParameter param, unsigned value) { void Options::set(ZSTD_cParameter param, unsigned value) {
zstdThrowIfError(ZSTD_CCtxParam_setParameter(params_.get(), param, value)); zstdThrowIfError(ZSTD_CCtxParam_setParameter(params_.get(), param, value));
if (param == ZSTD_p_compressionLevel) { if (param == ZSTD_c_compressionLevel) {
level_ = static_cast<int>(value); level_ = static_cast<int>(value);
} }
} }
......
...@@ -1386,6 +1386,12 @@ TEST(CheckCompatibleTest, ZlibIsPrefix) { ...@@ -1386,6 +1386,12 @@ TEST(CheckCompatibleTest, ZlibIsPrefix) {
#if FOLLY_HAVE_LIBZSTD #if FOLLY_HAVE_LIBZSTD
#if ZSTD_VERSION_NUMBER < 10308
#define ZSTD_c_contentSizeFlag ZSTD_p_contentSizeFlag
#define ZSTD_c_checksumFlag ZSTD_p_checksumFlag
#define ZSTD_c_windowLog ZSTD_p_windowLog
#endif
TEST(ZstdTest, BackwardCompatible) { TEST(ZstdTest, BackwardCompatible) {
auto codec = getCodec(CodecType::ZSTD); auto codec = getCodec(CodecType::ZSTD);
{ {
...@@ -1411,9 +1417,9 @@ TEST(ZstdTest, CustomOptions) { ...@@ -1411,9 +1417,9 @@ TEST(ZstdTest, CustomOptions) {
auto test = [](const DataHolder& dh, unsigned contentSizeFlag) { auto test = [](const DataHolder& dh, unsigned contentSizeFlag) {
unsigned const wlog = 23; unsigned const wlog = 23;
zstd::Options options(1); zstd::Options options(1);
options.set(ZSTD_p_contentSizeFlag, contentSizeFlag); options.set(ZSTD_c_contentSizeFlag, contentSizeFlag);
options.set(ZSTD_p_checksumFlag, 1); options.set(ZSTD_c_checksumFlag, 1);
options.set(ZSTD_p_windowLog, wlog); options.set(ZSTD_c_windowLog, wlog);
auto codec = zstd::getCodec(std::move(options)); auto codec = zstd::getCodec(std::move(options));
size_t const uncompressedLength = (size_t)1 << 27; size_t const uncompressedLength = (size_t)1 << 27;
auto const original = std::string( auto const original = std::string(
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment