diff --git a/.buckconfig b/.buckconfig index d698b35b..483f6053 100644 --- a/.buckconfig +++ b/.buckconfig @@ -1,7 +1,7 @@ [cxx] - cppflags = -DXXH_NAMESPACE=ZSTD_ -DZSTD_LEGACY_SUPPORT=1 + cppflags = -DXXH_NAMESPACE=ZSTD_ -DZSTD_LEGACY_SUPPORT=4 cflags = -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes -Wundef -Wpointer-arith - cxxppflags = -DXXH_NAMESPACE=ZSTD_ -DZSTD_LEGACY_SUPPORT=1 + cxxppflags = -DXXH_NAMESPACE=ZSTD_ -DZSTD_LEGACY_SUPPORT=4 cxxflags = -std=c++11 -Wno-deprecated-declarations gtest_dep = //contrib/pzstd:gtest diff --git a/build/VS2005/zstd/zstd.vcproj b/build/VS2005/zstd/zstd.vcproj index 58f254bc..1f4febea 100644 --- a/build/VS2005/zstd/zstd.vcproj +++ b/build/VS2005/zstd/zstd.vcproj @@ -44,7 +44,7 @@ Name="VCCLCompilerTool" Optimization="0" AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\dictBuilder;$(SolutionDir)..\..\lib\compress" - PreprocessorDefinitions="ZSTD_LEGACY_SUPPORT=1;WIN32;_DEBUG;_CONSOLE" + PreprocessorDefinitions="ZSTD_LEGACY_SUPPORT=4;WIN32;_DEBUG;_CONSOLE" MinimalRebuild="true" BasicRuntimeChecks="3" RuntimeLibrary="3" @@ -121,7 +121,7 @@ EnableIntrinsicFunctions="true" OmitFramePointers="true" AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\dictBuilder;$(SolutionDir)..\..\lib\compress" - PreprocessorDefinitions="ZSTD_LEGACY_SUPPORT=1;WIN32;NDEBUG;_CONSOLE" + PreprocessorDefinitions="ZSTD_LEGACY_SUPPORT=4;WIN32;NDEBUG;_CONSOLE" RuntimeLibrary="0" EnableFunctionLevelLinking="true" UsePrecompiledHeader="0" @@ -196,7 +196,7 @@ Name="VCCLCompilerTool" Optimization="0" AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\dictBuilder;$(SolutionDir)..\..\lib\compress" - PreprocessorDefinitions="ZSTD_LEGACY_SUPPORT=1;WIN32;_DEBUG;_CONSOLE" + PreprocessorDefinitions="ZSTD_LEGACY_SUPPORT=4;WIN32;_DEBUG;_CONSOLE" MinimalRebuild="true" BasicRuntimeChecks="3" RuntimeLibrary="3" @@ -274,7 +274,7 @@ EnableIntrinsicFunctions="true" OmitFramePointers="true" AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\dictBuilder;$(SolutionDir)..\..\lib\compress" - PreprocessorDefinitions="ZSTD_LEGACY_SUPPORT=1;WIN32;NDEBUG;_CONSOLE" + PreprocessorDefinitions="ZSTD_LEGACY_SUPPORT=4;WIN32;NDEBUG;_CONSOLE" RuntimeLibrary="0" EnableFunctionLevelLinking="true" UsePrecompiledHeader="0" diff --git a/build/VS2005/zstdlib/zstdlib.vcproj b/build/VS2005/zstdlib/zstdlib.vcproj index f4c9950f..8da31367 100644 --- a/build/VS2005/zstdlib/zstdlib.vcproj +++ b/build/VS2005/zstdlib/zstdlib.vcproj @@ -44,7 +44,7 @@ Name="VCCLCompilerTool" Optimization="0" AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\dictBuilder" - PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=1;WIN32;_DEBUG;_CONSOLE" + PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=4;WIN32;_DEBUG;_CONSOLE" MinimalRebuild="true" BasicRuntimeChecks="3" RuntimeLibrary="3" @@ -120,7 +120,7 @@ EnableIntrinsicFunctions="true" OmitFramePointers="true" AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\dictBuilder" - PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=1;WIN32;NDEBUG;_CONSOLE" + PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=4;WIN32;NDEBUG;_CONSOLE" RuntimeLibrary="0" EnableFunctionLevelLinking="true" UsePrecompiledHeader="0" @@ -194,7 +194,7 @@ Name="VCCLCompilerTool" Optimization="0" AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\dictBuilder" - PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=1;WIN32;_DEBUG;_CONSOLE" + PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=4;WIN32;_DEBUG;_CONSOLE" MinimalRebuild="true" BasicRuntimeChecks="3" RuntimeLibrary="3" @@ -271,7 +271,7 @@ EnableIntrinsicFunctions="true" OmitFramePointers="true" AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\dictBuilder" - PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=1;WIN32;NDEBUG;_CONSOLE" + PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=4;WIN32;NDEBUG;_CONSOLE" RuntimeLibrary="0" EnableFunctionLevelLinking="true" UsePrecompiledHeader="0" diff --git a/build/VS2008/zstd/zstd.vcproj b/build/VS2008/zstd/zstd.vcproj index 2dfaf393..468d2567 100644 --- a/build/VS2008/zstd/zstd.vcproj +++ b/build/VS2008/zstd/zstd.vcproj @@ -45,7 +45,7 @@ Name="VCCLCompilerTool" Optimization="0" AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\dictBuilder;$(SolutionDir)..\..\lib\compress" - PreprocessorDefinitions="ZSTD_LEGACY_SUPPORT=1;WIN32;_DEBUG;_CONSOLE" + PreprocessorDefinitions="ZSTD_LEGACY_SUPPORT=4;WIN32;_DEBUG;_CONSOLE" MinimalRebuild="true" BasicRuntimeChecks="3" RuntimeLibrary="3" @@ -122,7 +122,7 @@ EnableIntrinsicFunctions="true" OmitFramePointers="true" AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\dictBuilder;$(SolutionDir)..\..\lib\compress" - PreprocessorDefinitions="ZSTD_LEGACY_SUPPORT=1;WIN32;NDEBUG;_CONSOLE" + PreprocessorDefinitions="ZSTD_LEGACY_SUPPORT=4;WIN32;NDEBUG;_CONSOLE" RuntimeLibrary="0" EnableFunctionLevelLinking="true" UsePrecompiledHeader="0" @@ -197,7 +197,7 @@ Name="VCCLCompilerTool" Optimization="0" AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\dictBuilder;$(SolutionDir)..\..\lib\compress" - PreprocessorDefinitions="ZSTD_LEGACY_SUPPORT=1;WIN32;_DEBUG;_CONSOLE" + PreprocessorDefinitions="ZSTD_LEGACY_SUPPORT=4;WIN32;_DEBUG;_CONSOLE" MinimalRebuild="true" BasicRuntimeChecks="3" RuntimeLibrary="3" @@ -275,7 +275,7 @@ EnableIntrinsicFunctions="true" OmitFramePointers="true" AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\dictBuilder;$(SolutionDir)..\..\lib\compress" - PreprocessorDefinitions="ZSTD_LEGACY_SUPPORT=1;WIN32;NDEBUG;_CONSOLE" + PreprocessorDefinitions="ZSTD_LEGACY_SUPPORT=4;WIN32;NDEBUG;_CONSOLE" RuntimeLibrary="0" EnableFunctionLevelLinking="true" UsePrecompiledHeader="0" diff --git a/build/VS2008/zstdlib/zstdlib.vcproj b/build/VS2008/zstdlib/zstdlib.vcproj index cba0ff90..857e1463 100644 --- a/build/VS2008/zstdlib/zstdlib.vcproj +++ b/build/VS2008/zstdlib/zstdlib.vcproj @@ -45,7 +45,7 @@ Name="VCCLCompilerTool" Optimization="0" AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\dictBuilder" - PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=1;WIN32;_DEBUG;_CONSOLE" + PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=4;WIN32;_DEBUG;_CONSOLE" MinimalRebuild="true" BasicRuntimeChecks="3" RuntimeLibrary="3" @@ -121,7 +121,7 @@ EnableIntrinsicFunctions="true" OmitFramePointers="true" AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\dictBuilder" - PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=1;WIN32;NDEBUG;_CONSOLE" + PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=4;WIN32;NDEBUG;_CONSOLE" RuntimeLibrary="0" EnableFunctionLevelLinking="true" UsePrecompiledHeader="0" @@ -195,7 +195,7 @@ Name="VCCLCompilerTool" Optimization="0" AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\dictBuilder" - PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=1;WIN32;_DEBUG;_CONSOLE" + PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=4;WIN32;_DEBUG;_CONSOLE" MinimalRebuild="true" BasicRuntimeChecks="3" RuntimeLibrary="3" @@ -272,7 +272,7 @@ EnableIntrinsicFunctions="true" OmitFramePointers="true" AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\dictBuilder" - PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=1;WIN32;NDEBUG;_CONSOLE" + PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=4;WIN32;NDEBUG;_CONSOLE" RuntimeLibrary="0" EnableFunctionLevelLinking="true" UsePrecompiledHeader="0" diff --git a/build/VS2010/libzstd-dll/libzstd-dll.vcxproj b/build/VS2010/libzstd-dll/libzstd-dll.vcxproj index f78598fb..866d04a0 100644 --- a/build/VS2010/libzstd-dll/libzstd-dll.vcxproj +++ b/build/VS2010/libzstd-dll/libzstd-dll.vcxproj @@ -149,7 +149,7 @@ Level4 Disabled - ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=1;WIN32;_DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions) + ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=4;WIN32;_DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions) true EnableFastChecks MultiThreadedDebugDLL @@ -169,7 +169,7 @@ Level4 Disabled - ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=1;WIN32;_DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions) + ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=4;WIN32;_DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions) true EnableFastChecks MultiThreadedDebugDLL @@ -189,7 +189,7 @@ MaxSpeed true true - ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=1;WIN32;NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions) + ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=4;WIN32;NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions) false MultiThreaded ProgramDatabase @@ -211,7 +211,7 @@ MaxSpeed true true - ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=1;WIN32;NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions) + ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=4;WIN32;NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions) false false MultiThreaded diff --git a/build/VS2010/libzstd/libzstd.vcxproj b/build/VS2010/libzstd/libzstd.vcxproj index 72779551..186b4c4d 100644 --- a/build/VS2010/libzstd/libzstd.vcxproj +++ b/build/VS2010/libzstd/libzstd.vcxproj @@ -146,7 +146,7 @@ Level4 Disabled - ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=1;WIN32;_DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions) + ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=4;WIN32;_DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions) true EnableFastChecks MultiThreadedDebugDLL @@ -166,7 +166,7 @@ Level4 Disabled - ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=1;WIN32;_DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions) + ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=4;WIN32;_DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions) true EnableFastChecks MultiThreadedDebugDLL @@ -186,7 +186,7 @@ MaxSpeed true true - ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=1;WIN32;NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions) + ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=4;WIN32;NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions) false MultiThreaded ProgramDatabase @@ -208,7 +208,7 @@ MaxSpeed true true - ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=1;WIN32;NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions) + ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=4;WIN32;NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions) false false MultiThreaded diff --git a/build/VS2010/zstd/zstd.vcxproj b/build/VS2010/zstd/zstd.vcxproj index 62c0fe10..7568e490 100644 --- a/build/VS2010/zstd/zstd.vcxproj +++ b/build/VS2010/zstd/zstd.vcxproj @@ -155,7 +155,7 @@ Level4 Disabled - ZSTD_LEGACY_SUPPORT=1;WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) + ZSTD_LEGACY_SUPPORT=4;WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) true false @@ -171,7 +171,7 @@ Level4 Disabled - ZSTD_LEGACY_SUPPORT=1;WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) + ZSTD_LEGACY_SUPPORT=4;WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) true false @@ -189,7 +189,7 @@ MaxSpeed true true - ZSTD_LEGACY_SUPPORT=1;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) + ZSTD_LEGACY_SUPPORT=4;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) false false MultiThreaded @@ -210,7 +210,7 @@ MaxSpeed true true - ZSTD_LEGACY_SUPPORT=1;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) + ZSTD_LEGACY_SUPPORT=4;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) false false MultiThreaded diff --git a/build/cmake/CMakeLists.txt b/build/cmake/CMakeLists.txt index 6b7c2892..4805cc2c 100644 --- a/build/cmake/CMakeLists.txt +++ b/build/cmake/CMakeLists.txt @@ -16,7 +16,7 @@ OPTION(ZSTD_BUILD_CONTRIB "BUILD CONTRIB" OFF) IF (ZSTD_LEGACY_SUPPORT) MESSAGE(STATUS "ZSTD_LEGACY_SUPPORT defined!") - ADD_DEFINITIONS(-DZSTD_LEGACY_SUPPORT=1) + ADD_DEFINITIONS(-DZSTD_LEGACY_SUPPORT=4) ELSE (ZSTD_LEGACY_SUPPORT) MESSAGE(STATUS "ZSTD_LEGACY_SUPPORT not defined!") ADD_DEFINITIONS(-DZSTD_LEGACY_SUPPORT=0) diff --git a/contrib/meson/meson.build b/contrib/meson/meson.build index 36946133..8cbdcabe 100644 --- a/contrib/meson/meson.build +++ b/contrib/meson/meson.build @@ -15,7 +15,7 @@ libzstd_includes = [include_directories(common_dir, dictbuilder_dir, compress_di if get_option('legacy_support') message('Enabling legacy support') - libzstd_cflags = ['-DZSTD_LEGACY_SUPPORT=1'] + libzstd_cflags = ['-DZSTD_LEGACY_SUPPORT=4'] legacy_dir = join_paths(lib_dir, 'legacy') libzstd_includes += [include_directories(legacy_dir)] diff --git a/lib/Makefile b/lib/Makefile index 58f99baf..197fdeee 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -31,12 +31,15 @@ FLAGS = $(CPPFLAGS) $(CFLAGS) ZSTD_FILES := $(wildcard common/*.c compress/*.c decompress/*.c dictBuilder/*.c deprecated/*.c) -ifeq ($(ZSTD_LEGACY_SUPPORT), 0) -CPPFLAGS += -DZSTD_LEGACY_SUPPORT=0 -else -CPPFLAGS += -I./legacy -DZSTD_LEGACY_SUPPORT=1 -ZSTD_FILES+= $(wildcard legacy/*.c) +ZSTD_LEGACY_SUPPORT ?= 4 + +ifneq ($(ZSTD_LEGACY_SUPPORT), 0) +ifeq ($(shell test $(ZSTD_LEGACY_SUPPORT) -lt 8; echo $$?), 0) + ZSTD_FILES += $(shell ls legacy/*.c | grep 'v0[$(ZSTD_LEGACY_SUPPORT)-7]') endif + CPPFLAGS += -I./legacy +endif +CPPFLAGS += -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT) ZSTD_OBJ := $(patsubst %.c,%.o,$(ZSTD_FILES)) diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index 943bdf94..2aaa4a3d 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -320,7 +320,7 @@ size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t * - ZSTD_CONTENTSIZE_ERROR if an error occurred (e.g. invalid magic number, srcSize too small) */ unsigned long long ZSTD_getFrameContentSize(const void *src, size_t srcSize) { -#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1) +#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1) if (ZSTD_isLegacy(src, srcSize)) { unsigned long long const ret = ZSTD_getDecompressedSize_legacy(src, srcSize); return ret == 0 ? ZSTD_CONTENTSIZE_UNKNOWN : ret; @@ -1472,7 +1472,7 @@ size_t ZSTD_generateNxBytes(void* dst, size_t dstCapacity, BYTE byte, size_t len * @return : the compressed size of the frame starting at `src` */ size_t ZSTD_findFrameCompressedSize(const void *src, size_t srcSize) { -#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1) +#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1) if (ZSTD_isLegacy(src, srcSize)) return ZSTD_findFrameCompressedSizeLegacy(src, srcSize); #endif if (srcSize >= ZSTD_skippableHeaderSize && diff --git a/lib/legacy/zstd_legacy.h b/lib/legacy/zstd_legacy.h index 707e76f0..3c9798f8 100644 --- a/lib/legacy/zstd_legacy.h +++ b/lib/legacy/zstd_legacy.h @@ -20,14 +20,33 @@ extern "C" { #include "mem.h" /* MEM_STATIC */ #include "error_private.h" /* ERROR */ #include "zstd.h" /* ZSTD_inBuffer, ZSTD_outBuffer */ -#include "zstd_v01.h" -#include "zstd_v02.h" -#include "zstd_v03.h" -#include "zstd_v04.h" -#include "zstd_v05.h" -#include "zstd_v06.h" -#include "zstd_v07.h" +#if !defined (ZSTD_LEGACY_SUPPORT) || (ZSTD_LEGACY_SUPPORT == 0) +# undef ZSTD_LEGACY_SUPPORT +# define ZSTD_LEGACY_SUPPORT 8 +#endif + +#if (ZSTD_LEGACY_SUPPORT <= 1) +# include "zstd_v01.h" +#endif +#if (ZSTD_LEGACY_SUPPORT <= 2) +# include "zstd_v02.h" +#endif +#if (ZSTD_LEGACY_SUPPORT <= 3) +# include "zstd_v03.h" +#endif +#if (ZSTD_LEGACY_SUPPORT <= 4) +# include "zstd_v04.h" +#endif +#if (ZSTD_LEGACY_SUPPORT <= 5) +# include "zstd_v05.h" +#endif +#if (ZSTD_LEGACY_SUPPORT <= 6) +# include "zstd_v06.h" +#endif +#if (ZSTD_LEGACY_SUPPORT <= 7) +# include "zstd_v07.h" +#endif /** ZSTD_isLegacy() : @return : > 0 if supported by legacy decoder. 0 otherwise. @@ -40,13 +59,27 @@ MEM_STATIC unsigned ZSTD_isLegacy(const void* src, size_t srcSize) magicNumberLE = MEM_readLE32(src); switch(magicNumberLE) { +#if (ZSTD_LEGACY_SUPPORT <= 1) case ZSTDv01_magicNumberLE:return 1; +#endif +#if (ZSTD_LEGACY_SUPPORT <= 2) case ZSTDv02_magicNumber : return 2; +#endif +#if (ZSTD_LEGACY_SUPPORT <= 3) case ZSTDv03_magicNumber : return 3; +#endif +#if (ZSTD_LEGACY_SUPPORT <= 4) case ZSTDv04_magicNumber : return 4; +#endif +#if (ZSTD_LEGACY_SUPPORT <= 5) case ZSTDv05_MAGICNUMBER : return 5; +#endif +#if (ZSTD_LEGACY_SUPPORT <= 6) case ZSTDv06_MAGICNUMBER : return 6; +#endif +#if (ZSTD_LEGACY_SUPPORT <= 7) case ZSTDv07_MAGICNUMBER : return 7; +#endif default : return 0; } } @@ -56,24 +89,30 @@ MEM_STATIC unsigned long long ZSTD_getDecompressedSize_legacy(const void* src, s { U32 const version = ZSTD_isLegacy(src, srcSize); if (version < 5) return 0; /* no decompressed size in frame header, or not a legacy format */ +#if (ZSTD_LEGACY_SUPPORT <= 5) if (version==5) { ZSTDv05_parameters fParams; size_t const frResult = ZSTDv05_getFrameParams(&fParams, src, srcSize); if (frResult != 0) return 0; return fParams.srcSize; } +#endif +#if (ZSTD_LEGACY_SUPPORT <= 6) if (version==6) { ZSTDv06_frameParams fParams; size_t const frResult = ZSTDv06_getFrameParams(&fParams, src, srcSize); if (frResult != 0) return 0; return fParams.frameContentSize; } +#endif +#if (ZSTD_LEGACY_SUPPORT <= 7) if (version==7) { ZSTDv07_frameParams fParams; size_t const frResult = ZSTDv07_getFrameParams(&fParams, src, srcSize); if (frResult != 0) return 0; return fParams.frameContentSize; } +#endif return 0; /* should not be possible */ } @@ -86,14 +125,23 @@ MEM_STATIC size_t ZSTD_decompressLegacy( U32 const version = ZSTD_isLegacy(src, compressedSize); switch(version) { +#if (ZSTD_LEGACY_SUPPORT <= 1) case 1 : return ZSTDv01_decompress(dst, dstCapacity, src, compressedSize); +#endif +#if (ZSTD_LEGACY_SUPPORT <= 2) case 2 : return ZSTDv02_decompress(dst, dstCapacity, src, compressedSize); +#endif +#if (ZSTD_LEGACY_SUPPORT <= 3) case 3 : return ZSTDv03_decompress(dst, dstCapacity, src, compressedSize); +#endif +#if (ZSTD_LEGACY_SUPPORT <= 4) case 4 : return ZSTDv04_decompress(dst, dstCapacity, src, compressedSize); +#endif +#if (ZSTD_LEGACY_SUPPORT <= 5) case 5 : { size_t result; ZSTDv05_DCtx* const zd = ZSTDv05_createDCtx(); @@ -102,6 +150,8 @@ MEM_STATIC size_t ZSTD_decompressLegacy( ZSTDv05_freeDCtx(zd); return result; } +#endif +#if (ZSTD_LEGACY_SUPPORT <= 6) case 6 : { size_t result; ZSTDv06_DCtx* const zd = ZSTDv06_createDCtx(); @@ -110,6 +160,8 @@ MEM_STATIC size_t ZSTD_decompressLegacy( ZSTDv06_freeDCtx(zd); return result; } +#endif +#if (ZSTD_LEGACY_SUPPORT <= 7) case 7 : { size_t result; ZSTDv07_DCtx* const zd = ZSTDv07_createDCtx(); @@ -118,6 +170,7 @@ MEM_STATIC size_t ZSTD_decompressLegacy( ZSTDv07_freeDCtx(zd); return result; } +#endif default : return ERROR(prefix_unknown); } @@ -129,20 +182,34 @@ MEM_STATIC size_t ZSTD_findFrameCompressedSizeLegacy(const void *src, U32 const version = ZSTD_isLegacy(src, compressedSize); switch(version) { +#if (ZSTD_LEGACY_SUPPORT <= 1) case 1 : return ZSTDv01_findFrameCompressedSize(src, compressedSize); +#endif +#if (ZSTD_LEGACY_SUPPORT <= 2) case 2 : return ZSTDv02_findFrameCompressedSize(src, compressedSize); +#endif +#if (ZSTD_LEGACY_SUPPORT <= 3) case 3 : return ZSTDv03_findFrameCompressedSize(src, compressedSize); +#endif +#if (ZSTD_LEGACY_SUPPORT <= 4) case 4 : return ZSTDv04_findFrameCompressedSize(src, compressedSize); +#endif +#if (ZSTD_LEGACY_SUPPORT <= 5) case 5 : return ZSTDv05_findFrameCompressedSize(src, compressedSize); +#endif +#if (ZSTD_LEGACY_SUPPORT <= 6) case 6 : return ZSTDv06_findFrameCompressedSize(src, compressedSize); +#endif +#if (ZSTD_LEGACY_SUPPORT <= 7) case 7 : return ZSTDv07_findFrameCompressedSize(src, compressedSize); +#endif default : return ERROR(prefix_unknown); } @@ -157,10 +224,18 @@ MEM_STATIC size_t ZSTD_freeLegacyStreamContext(void* legacyContext, U32 version) case 2 : case 3 : return ERROR(version_unsupported); +#if (ZSTD_LEGACY_SUPPORT <= 4) case 4 : return ZBUFFv04_freeDCtx((ZBUFFv04_DCtx*)legacyContext); +#endif +#if (ZSTD_LEGACY_SUPPORT <= 5) case 5 : return ZBUFFv05_freeDCtx((ZBUFFv05_DCtx*)legacyContext); +#endif +#if (ZSTD_LEGACY_SUPPORT <= 6) case 6 : return ZBUFFv06_freeDCtx((ZBUFFv06_DCtx*)legacyContext); +#endif +#if (ZSTD_LEGACY_SUPPORT <= 7) case 7 : return ZBUFFv07_freeDCtx((ZBUFFv07_DCtx*)legacyContext); +#endif } } @@ -176,6 +251,7 @@ MEM_STATIC size_t ZSTD_initLegacyStream(void** legacyContext, U32 prevVersion, U case 2 : case 3 : return 0; +#if (ZSTD_LEGACY_SUPPORT <= 4) case 4 : { ZBUFFv04_DCtx* dctx = (prevVersion != newVersion) ? ZBUFFv04_createDCtx() : (ZBUFFv04_DCtx*)*legacyContext; @@ -185,6 +261,8 @@ MEM_STATIC size_t ZSTD_initLegacyStream(void** legacyContext, U32 prevVersion, U *legacyContext = dctx; return 0; } +#endif +#if (ZSTD_LEGACY_SUPPORT <= 5) case 5 : { ZBUFFv05_DCtx* dctx = (prevVersion != newVersion) ? ZBUFFv05_createDCtx() : (ZBUFFv05_DCtx*)*legacyContext; @@ -193,6 +271,8 @@ MEM_STATIC size_t ZSTD_initLegacyStream(void** legacyContext, U32 prevVersion, U *legacyContext = dctx; return 0; } +#endif +#if (ZSTD_LEGACY_SUPPORT <= 6) case 6 : { ZBUFFv06_DCtx* dctx = (prevVersion != newVersion) ? ZBUFFv06_createDCtx() : (ZBUFFv06_DCtx*)*legacyContext; @@ -201,6 +281,8 @@ MEM_STATIC size_t ZSTD_initLegacyStream(void** legacyContext, U32 prevVersion, U *legacyContext = dctx; return 0; } +#endif +#if (ZSTD_LEGACY_SUPPORT <= 7) case 7 : { ZBUFFv07_DCtx* dctx = (prevVersion != newVersion) ? ZBUFFv07_createDCtx() : (ZBUFFv07_DCtx*)*legacyContext; @@ -209,6 +291,7 @@ MEM_STATIC size_t ZSTD_initLegacyStream(void** legacyContext, U32 prevVersion, U *legacyContext = dctx; return 0; } +#endif } } @@ -224,6 +307,7 @@ MEM_STATIC size_t ZSTD_decompressLegacyStream(void* legacyContext, U32 version, case 2 : case 3 : return ERROR(version_unsupported); +#if (ZSTD_LEGACY_SUPPORT <= 4) case 4 : { ZBUFFv04_DCtx* dctx = (ZBUFFv04_DCtx*) legacyContext; @@ -236,6 +320,8 @@ MEM_STATIC size_t ZSTD_decompressLegacyStream(void* legacyContext, U32 version, input->pos += readSize; return hintSize; } +#endif +#if (ZSTD_LEGACY_SUPPORT <= 5) case 5 : { ZBUFFv05_DCtx* dctx = (ZBUFFv05_DCtx*) legacyContext; @@ -248,6 +334,8 @@ MEM_STATIC size_t ZSTD_decompressLegacyStream(void* legacyContext, U32 version, input->pos += readSize; return hintSize; } +#endif +#if (ZSTD_LEGACY_SUPPORT <= 6) case 6 : { ZBUFFv06_DCtx* dctx = (ZBUFFv06_DCtx*) legacyContext; @@ -260,6 +348,8 @@ MEM_STATIC size_t ZSTD_decompressLegacyStream(void* legacyContext, U32 version, input->pos += readSize; return hintSize; } +#endif +#if (ZSTD_LEGACY_SUPPORT <= 7) case 7 : { ZBUFFv07_DCtx* dctx = (ZBUFFv07_DCtx*) legacyContext; @@ -272,6 +362,7 @@ MEM_STATIC size_t ZSTD_decompressLegacyStream(void* legacyContext, U32 version, input->pos += readSize; return hintSize; } +#endif } } diff --git a/programs/Makefile b/programs/Makefile index a935c744..1475cb61 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -42,12 +42,14 @@ ZSTD_FILES := $(ZSTDDECOMP_FILES) $(ZSTDCOMMON_FILES) $(ZSTDCOMP_FILES) ZDICT_FILES := $(ZSTDDIR)/dictBuilder/*.c ZSTDDECOMP_O = $(ZSTDDIR)/decompress/zstd_decompress.o -ifeq ($(ZSTD_LEGACY_SUPPORT), 0) +ZSTD_LEGACY_SUPPORT ?= 4 ZSTDLEGACY_FILES:= +ifneq ($(ZSTD_LEGACY_SUPPORT), 0) +ifeq ($(shell test $(ZSTD_LEGACY_SUPPORT) -lt 8; echo $$?), 0) + ZSTDLEGACY_FILES += $(shell ls $(ZSTDDIR)/legacy/*.c | grep 'v0[$(ZSTD_LEGACY_SUPPORT)-7]') +endif + CPPFLAGS += -I$(ZSTDDIR)/legacy else -ZSTD_LEGACY_SUPPORT:=1 -CPPFLAGS += -I$(ZSTDDIR)/legacy -ZSTDLEGACY_FILES:= $(ZSTDDIR)/legacy/*.c endif ZSTDLIB_FILES := $(wildcard $(ZSTD_FILES)) $(wildcard $(ZSTDLEGACY_FILES)) $(wildcard $(ZDICT_FILES)) @@ -68,12 +70,27 @@ EXT = endif # zlib detection +NO_ZLIB_MSG := ==> no zlib, building zstd without .gz support VOID = /dev/null HAVE_ZLIB := $(shell printf '\#include \nint main(){}' | $(CC) -o have_zlib -x c - -lz 2> $(VOID) && rm have_zlib$(EXT) && echo 1 || echo 0) ifeq ($(HAVE_ZLIB), 1) +ZLIB_MSG := ==> building zstd with .gz compression support ZLIBCPP = -DZSTD_GZCOMPRESS -DZSTD_GZDECOMPRESS ZLIBLD = -lz +else +ZLIB_MSG := $(NO_ZLIB_MSG) endif +# lzma detection +NO_LZMA_MSG := ==> no liblzma, building zstd without .xz/.lzma support +HAVE_LZMA := $(shell printf '\#include \nint main(){}' | $(CC) -o have_lzma -x c - -llzma 2> $(VOID) && rm have_lzma$(EXT) && echo 1 || echo 0) +ifeq ($(HAVE_LZMA), 1) +LZMA_MSG := ==> building zstd with .xz/.lzma compression support +LZMACPP = -DZSTD_LZMACOMPRESS -DZSTD_LZMADECOMPRESS +LZMALD = -llzma +else +LZMA_MSG := $(NO_LZMA_MSG) +endif + .PHONY: default all clean clean_decomp_o install uninstall generate_res @@ -85,14 +102,15 @@ $(ZSTDDECOMP_O): CFLAGS += $(ALIGN_LOOP) zstd : CPPFLAGS += $(ZLIBCPP) zstd : LDFLAGS += $(ZLIBLD) -zstd-nogz : HAVE_ZLIB=0 -zstd zstd-nogz : CPPFLAGS += -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT) -zstd zstd-nogz : $(ZSTDLIB_OBJ) zstdcli.o fileio.o bench.o datagen.o dibio.o -ifeq ($(HAVE_ZLIB), 1) - @echo "==> building zstd with .gz decompression support " -else - @echo "==> no zlib, building zstd with .zst support only (no .gz support) " -endif +zstd : LZMA_MSG := $(NO_LZMA_MSG) +zstd-nogz : ZLIB_MSG := $(NO_ZLIB_MSG) +zstd-nogz : LZMA_MSG := $(NO_LZMA_MSG) +xzstd : CPPFLAGS += $(ZLIBCPP) $(LZMACPP) +xzstd : LDFLAGS += $(ZLIBLD) $(LZMALD) +zstd zstd-nogz xzstd : CPPFLAGS += -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT) +zstd zstd-nogz xzstd : $(ZSTDLIB_OBJ) zstdcli.o fileio.o bench.o datagen.o dibio.o + @echo "$(ZLIB_MSG)" + @echo "$(LZMA_MSG)" ifneq (,$(filter Windows%,$(OS))) windres/generate_res.bat endif diff --git a/programs/fileio.c b/programs/fileio.c index 41daa125..e6481f1f 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -44,6 +44,9 @@ # define z_const # endif #endif +#if defined(ZSTD_LZMACOMPRESS) || defined(ZSTD_LZMADECOMPRESS) +# include +#endif /*-************************************* @@ -71,7 +74,6 @@ #define MAX_DICT_SIZE (8 MB) /* protection against large input (attack scenario) */ #define FNSPACE 30 -#define GZ_EXTENSION ".gz" /*-************************************* @@ -434,6 +436,65 @@ static unsigned long long FIO_compressGzFrame(cRess_t* ress, const char* srcFile #endif +#ifdef ZSTD_LZMACOMPRESS +static unsigned long long FIO_compressLzmaFrame(cRess_t* ress, const char* srcFileName, U64 const srcFileSize, int compressionLevel, U64* readsize, int plain_lzma) +{ + unsigned long long inFileSize = 0, outFileSize = 0; + lzma_stream strm = LZMA_STREAM_INIT; + lzma_action action = LZMA_RUN; + lzma_ret ret; + + if (compressionLevel < 0) compressionLevel = 0; + if (compressionLevel > 9) compressionLevel = 9; + + if (plain_lzma) { + lzma_options_lzma opt_lzma; + if (lzma_lzma_preset(&opt_lzma, compressionLevel)) EXM_THROW(71, "zstd: %s: lzma_lzma_preset error", srcFileName); + ret = lzma_alone_encoder(&strm, &opt_lzma); /* LZMA */ + if (ret != LZMA_OK) EXM_THROW(71, "zstd: %s: lzma_alone_encoder error %d", srcFileName, ret); + } else { + ret = lzma_easy_encoder(&strm, compressionLevel, LZMA_CHECK_CRC64); /* XZ */ + if (ret != LZMA_OK) EXM_THROW(71, "zstd: %s: lzma_easy_encoder error %d", srcFileName, ret); + } + + strm.next_in = 0; + strm.avail_in = 0; + strm.next_out = ress->dstBuffer; + strm.avail_out = ress->dstBufferSize; + + while (1) { + if (strm.avail_in == 0) { + size_t const inSize = fread(ress->srcBuffer, 1, ress->srcBufferSize, ress->srcFile); + if (inSize == 0) action = LZMA_FINISH; + inFileSize += inSize; + strm.next_in = ress->srcBuffer; + strm.avail_in = inSize; + } + + ret = lzma_code(&strm, action); + + if (ret != LZMA_OK && ret != LZMA_STREAM_END) EXM_THROW(72, "zstd: %s: lzma_code encoding error %d", srcFileName, ret); + { size_t const compBytes = ress->dstBufferSize - strm.avail_out; + if (compBytes) { + if (fwrite(ress->dstBuffer, 1, compBytes, ress->dstFile) != compBytes) EXM_THROW(73, "Write error : cannot write to output file"); + outFileSize += compBytes; + strm.next_out = ress->dstBuffer; + strm.avail_out = ress->dstBufferSize; + } + } + if (!srcFileSize) DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%%", (U32)(inFileSize>>20), (double)outFileSize/inFileSize*100) + else DISPLAYUPDATE(2, "\rRead : %u / %u MB ==> %.2f%%", (U32)(inFileSize>>20), (U32)(srcFileSize>>20), (double)outFileSize/inFileSize*100); + if (ret == LZMA_STREAM_END) break; + } + + lzma_end(&strm); + *readsize = inFileSize; + + return outFileSize; +} +#endif + + /*! FIO_compressFilename_internal() : * same as FIO_compressFilename_extRess(), with `ress.desFile` already opened. * @return : 0 : compression completed correctly, @@ -448,14 +509,26 @@ static int FIO_compressFilename_internal(cRess_t ress, U64 compressedfilesize = 0; U64 const fileSize = UTIL_getFileSize(srcFileName); - if (g_compressionType) { + switch (g_compressionType) { + case FIO_zstdCompression: + break; + case FIO_gzipCompression: #ifdef ZSTD_GZCOMPRESS - compressedfilesize = FIO_compressGzFrame(&ress, srcFileName, fileSize, compressionLevel, &readsize); + compressedfilesize = FIO_compressGzFrame(&ress, srcFileName, fileSize, compressionLevel, &readsize); #else - (void)compressionLevel; - EXM_THROW(20, "zstd: %s: file cannot be compressed as gzip (zstd compiled without ZSTD_GZCOMPRESS) -- ignored \n", srcFileName); + (void)compressionLevel; + EXM_THROW(20, "zstd: %s: file cannot be compressed as gzip (zstd compiled without ZSTD_GZCOMPRESS) -- ignored \n", srcFileName); #endif - goto finish; + goto finish; + case FIO_xzCompression: + case FIO_lzmaCompression: +#ifdef ZSTD_LZMACOMPRESS + compressedfilesize = FIO_compressLzmaFrame(&ress, srcFileName, fileSize, compressionLevel, &readsize, g_compressionType==FIO_lzmaCompression); +#else + (void)compressionLevel; + EXM_THROW(20, "zstd: %s: file cannot be compressed as xz/lzma (zstd compiled without ZSTD_LZMACOMPRESS) -- ignored \n", srcFileName); +#endif + goto finish; } /* init */ @@ -763,10 +836,10 @@ static void FIO_fwriteSparseEnd(FILE* file, unsigned storedSkips) { if (storedSkips-->0) { /* implies g_sparseFileSupport>0 */ int const seekResult = LONG_SEEK(file, storedSkips, SEEK_CUR); - if (seekResult != 0) EXM_THROW(69, "Final skip error (sparse file)\n"); + if (seekResult != 0) EXM_THROW(69, "Final skip error (sparse file)"); { const char lastZeroByte[1] = { 0 }; size_t const sizeCheck = fwrite(lastZeroByte, 1, 1, file); - if (sizeCheck != 1) EXM_THROW(69, "Write error : cannot write last zero\n"); + if (sizeCheck != 1) EXM_THROW(69, "Write error : cannot write last zero"); } } } @@ -849,6 +922,7 @@ static unsigned long long FIO_decompressGzFrame(dRess_t* ress, FILE* srcFile, co { unsigned long long outFileSize = 0; z_stream strm; + int flush = Z_NO_FLUSH; int ret; strm.zalloc = Z_NULL; @@ -866,11 +940,12 @@ static unsigned long long FIO_decompressGzFrame(dRess_t* ress, FILE* srcFile, co for ( ; ; ) { if (strm.avail_in == 0) { ress->srcBufferLoaded = fread(ress->srcBuffer, 1, ress->srcBufferSize, srcFile); - if (ress->srcBufferLoaded == 0) break; + if (ress->srcBufferLoaded == 0) flush = Z_FINISH; strm.next_in = (z_const unsigned char*)ress->srcBuffer; strm.avail_in = (uInt)ress->srcBufferLoaded; } - ret = inflate(&strm, Z_NO_FLUSH); + ret = inflate(&strm, flush); + if (ret == Z_BUF_ERROR) EXM_THROW(39, "zstd: %s: premature end", srcFileName); if (ret != Z_OK && ret != Z_STREAM_END) { DISPLAY("zstd: %s: inflate error %d \n", srcFileName, ret); return 0; } { size_t const decompBytes = ress->dstBufferSize - strm.avail_out; if (decompBytes) { @@ -886,7 +961,60 @@ static unsigned long long FIO_decompressGzFrame(dRess_t* ress, FILE* srcFile, co if (strm.avail_in > 0) memmove(ress->srcBuffer, strm.next_in, strm.avail_in); ress->srcBufferLoaded = strm.avail_in; ret = inflateEnd(&strm); - if (ret != Z_OK) EXM_THROW(32, "zstd: %s: inflateEnd error %d \n", srcFileName, ret); + if (ret != Z_OK) EXM_THROW(32, "zstd: %s: inflateEnd error %d", srcFileName, ret); + return outFileSize; +} +#endif + + +#ifdef ZSTD_LZMADECOMPRESS +static unsigned long long FIO_decompressLzmaFrame(dRess_t* ress, FILE* srcFile, const char* srcFileName, int plain_lzma) +{ + unsigned long long outFileSize = 0; + lzma_stream strm = LZMA_STREAM_INIT; + lzma_action action = LZMA_RUN; + lzma_ret ret; + + strm.next_in = 0; + strm.avail_in = 0; + if (plain_lzma) { + ret = lzma_alone_decoder(&strm, UINT64_MAX); /* LZMA */ + } else { + ret = lzma_stream_decoder(&strm, UINT64_MAX, 0); /* XZ */ + } + + if (ret != LZMA_OK) EXM_THROW(71, "zstd: %s: lzma_alone_decoder/lzma_stream_decoder error %d", srcFileName, ret); + + strm.next_out = ress->dstBuffer; + strm.avail_out = ress->dstBufferSize; + strm.avail_in = ress->srcBufferLoaded; + strm.next_in = ress->srcBuffer; + + for ( ; ; ) { + if (strm.avail_in == 0) { + ress->srcBufferLoaded = fread(ress->srcBuffer, 1, ress->srcBufferSize, srcFile); + if (ress->srcBufferLoaded == 0) action = LZMA_FINISH; + strm.next_in = ress->srcBuffer; + strm.avail_in = ress->srcBufferLoaded; + } + ret = lzma_code(&strm, action); + + if (ret == LZMA_BUF_ERROR) EXM_THROW(39, "zstd: %s: premature end", srcFileName); + if (ret != LZMA_OK && ret != LZMA_STREAM_END) { DISPLAY("zstd: %s: lzma_code decoding error %d \n", srcFileName, ret); return 0; } + { size_t const decompBytes = ress->dstBufferSize - strm.avail_out; + if (decompBytes) { + if (fwrite(ress->dstBuffer, 1, decompBytes, ress->dstFile) != decompBytes) EXM_THROW(31, "Write error : cannot write to output file"); + outFileSize += decompBytes; + strm.next_out = ress->dstBuffer; + strm.avail_out = ress->dstBufferSize; + } + } + if (ret == LZMA_STREAM_END) break; + } + + if (strm.avail_in > 0) memmove(ress->srcBuffer, strm.next_in, strm.avail_in); + ress->srcBufferLoaded = strm.avail_in; + lzma_end(&strm); return outFileSize; } #endif @@ -924,7 +1052,7 @@ static int FIO_decompressSrcFile(dRess_t ress, const char* dstFileName, const ch } readSomething = 1; /* there is at least >= 4 bytes in srcFile */ if (ress.srcBufferLoaded < toRead) { DISPLAY("zstd: %s: unknown header \n", srcFileName); fclose(srcFile); return 1; } /* srcFileName is empty */ - if (buf[0] == 31 && buf[1] == 139) { /* gz header */ + if (buf[0] == 31 && buf[1] == 139) { /* gz magic number */ #ifdef ZSTD_GZDECOMPRESS unsigned long long const result = FIO_decompressGzFrame(&ress, srcFile, srcFileName); if (result == 0) return 1; @@ -932,6 +1060,16 @@ static int FIO_decompressSrcFile(dRess_t ress, const char* dstFileName, const ch #else DISPLAYLEVEL(1, "zstd: %s: gzip file cannot be uncompressed (zstd compiled without ZSTD_GZDECOMPRESS) -- ignored \n", srcFileName); return 1; +#endif + } else if ((buf[0] == 0xFD && buf[1] == 0x37) /* xz magic number */ + || (buf[0] == 0x5D && buf[1] == 0x00)) { /* lzma header (no magic number) */ +#ifdef ZSTD_LZMADECOMPRESS + unsigned long long const result = FIO_decompressLzmaFrame(&ress, srcFile, srcFileName, buf[0] != 0xFD); + if (result == 0) return 1; + filesize += result; +#else + DISPLAYLEVEL(1, "zstd: %s: xz/lzma file cannot be uncompressed (zstd compiled without ZSTD_LZMADECOMPRESS) -- ignored \n", srcFileName); + return 1; #endif } else { if (!ZSTD_isFrame(ress.srcBuffer, toRead)) { @@ -1020,32 +1158,31 @@ int FIO_decompressMultipleFilenames(const char** srcNamesTable, unsigned nbFiles missingFiles += FIO_decompressSrcFile(ress, suffix, srcNamesTable[u]); if (fclose(ress.dstFile)) EXM_THROW(72, "Write error : cannot properly close stdout"); } else { - size_t const suffixSize = strlen(suffix); - size_t const gzipSuffixSize = strlen(GZ_EXTENSION); + size_t suffixSize; size_t dfnSize = FNSPACE; unsigned u; char* dstFileName = (char*)malloc(FNSPACE); if (dstFileName==NULL) EXM_THROW(73, "not enough memory for dstFileName"); for (u=0; u tmp + $ZSTD -f --format=gzip tmp + $ZSTD -f tmp + cat tmp.gz tmp.zst tmp.gz tmp.zst | $ZSTD -d -f -o tmp + head -c -1 tmp.gz | $ZSTD -t && die "incomplete frame not detected !" + rm tmp* +else + $ECHO "gzip mode not supported" +fi + + +$ECHO "\n**** xz compatibility tests **** " + +LZMAMODE=1 +$ZSTD --format=xz -V || LZMAMODE=0 +if [ $LZMAMODE -eq 1 ]; then + $ECHO "xz support detected" + XZEXE=1 + xz -V && lzma -V || XZEXE=0 + if [ $XZEXE -eq 1 ]; then + ./datagen > tmp + $ZSTD --format=lzma -f tmp + $ZSTD --format=xz -f tmp + xz -t -v tmp.xz + xz -t -v tmp.lzma + xz -f -k tmp + lzma -f -k --lzma1 tmp + $ZSTD -d -f -v tmp.xz + $ZSTD -d -f -v tmp.lzma + rm tmp* + else + $ECHO "xz binary not detected" + fi +else + $ECHO "xz mode not supported" +fi + + +$ECHO "\n**** xz frame tests **** " + +if [ $LZMAMODE -eq 1 ]; then + ./datagen > tmp + $ZSTD -f --format=xz tmp + $ZSTD -f --format=lzma tmp + $ZSTD -f tmp + cat tmp.xz tmp.lzma tmp.zst tmp.lzma tmp.xz tmp.zst | $ZSTD -d -f -o tmp + head -c -1 tmp.xz | $ZSTD -t && die "incomplete frame not detected !" + head -c -1 tmp.lzma | $ZSTD -t && die "incomplete frame not detected !" + rm tmp* +else + $ECHO "xz mode not supported" +fi + + $ECHO "\n**** zstd round-trip tests **** " roundTripTest