Merge pull request #311 from Cyan4973/dev

Dev

.travis.yml

@@ -9,7 +9,7 @@ matrix:
       env: PLATFORM="Ubuntu 12.04 container" CMD="make test && make clean && make travis-install"
     - os: linux
       sudo: false
-      env: PLATFORM="Ubuntu 12.04 container" CMD="make -C programs test-zstd_nolegacy && make clean && make zlibwrapper && make clean && make cmaketest"
+      env: PLATFORM="Ubuntu 12.04 container" CMD="make -C tests test-zstd_nolegacy && make clean && make zlibwrapper && make clean && make cmaketest"
     - os: linux
       sudo: false
       env: PLATFORM="Ubuntu 12.04 container" CMD="make usan"
@@ -39,7 +39,7 @@ matrix:
             - qemu-user-static
     - os: linux
       sudo: required
-      env: PLATFORM="Ubuntu 12.04" CMD="make -C programs test32"
+      env: PLATFORM="Ubuntu 12.04" CMD="make -C tests test32"
       addons:
         apt:
           packages:
@@ -58,7 +58,7 @@ matrix:
             - gcc-multilib
     - os: linux
       sudo: required
-      env: PLATFORM="Ubuntu 12.04" CMD="make -C programs valgrindTest"
+      env: PLATFORM="Ubuntu 12.04" CMD="make -C tests valgrindTest"
       addons:
         apt:
           packages:

CONTRIBUTING.md

@@ -0,0 +1,42 @@
+# Contributing to Zstandard
+We want to make contributing to this project as easy and transparent as
+possible.
+
+## Our Development Process
+New versions are being developed in the "dev" branch,
+or in their own feature branch.
+When they are deemed ready for a release, they are merged into "master".
+
+As a consequences, all contributions must stage first through "dev"
+or their own feature branch.
+
+## Pull Requests
+We actively welcome your pull requests.
+
+1. Fork the repo and create your branch from `dev`.
+2. If you've added code that should be tested, add tests.
+3. If you've changed APIs, update the documentation.
+4. Ensure the test suite passes.
+5. Make sure your code lints.
+6. If you haven't already, complete the Contributor License Agreement ("CLA").
+
+## Contributor License Agreement ("CLA")
+In order to accept your pull request, we need you to submit a CLA. You only need
+to do this once to work on any of Facebook's open source projects.
+
+Complete your CLA here: <https://code.facebook.com/cla>
+
+## Issues
+We use GitHub issues to track public bugs. Please ensure your description is
+clear and has sufficient instructions to be able to reproduce the issue.
+
+Facebook has a [bounty program](https://www.facebook.com/whitehat/) for the safe
+disclosure of security bugs. In those cases, please go through the process
+outlined on that page and do not file a public issue.
+
+## Coding Style  
+* 4 spaces for indentation rather than tabs
+
+## License
+By contributing to Zstandard, you agree that your contributions will be licensed
+under the [LICENSE](LICENSE) file in the root directory of this source tree.

LICENSE

@@ -1,18 +1,22 @@
-ZSTD Library
-Copyright (c) 2014-2015, Yann Collet
-All rights reserved.
-
 BSD License
 
+For Zstandard software
+
+Copyright (c) 2016-present, Facebook, Inc. All rights reserved.
+
 Redistribution and use in source and binary forms, with or without modification,
 are permitted provided that the following conditions are met:
 
-* Redistributions of source code must retain the above copyright notice, this
+ * Redistributions of source code must retain the above copyright notice, this
-  list of conditions and the following disclaimer.
+   list of conditions and the following disclaimer.
 
-* Redistributions in binary form must reproduce the above copyright notice, this
+ * Redistributions in binary form must reproduce the above copyright notice,
-  list of conditions and the following disclaimer in the documentation and/or
+   this list of conditions and the following disclaimer in the documentation
-  other materials provided with the distribution.
+   and/or other materials provided with the distribution.
+
+ * Neither the name Facebook nor the names of its contributors may be used to
+   endorse or promote products derived from this software without specific
+   prior written permission.
 
 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
 ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED

LICENSE-examples

@@ -0,0 +1,11 @@
+Copyright (c) 2016-present, Facebook, Inc. All rights reserved.
+
+The examples provided by Facebook are for non-commercial testing and evaluation
+purposes only. Facebook reserves all rights not expressly granted.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+FACEBOOK BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

Makefile

@@ -1,38 +1,16 @@
 # ################################################################
-# zstd - Makefile
+# Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
-# Copyright (C) Yann Collet 2014-2016
 # All rights reserved.
 #
-# BSD license
+# This source code is licensed under the BSD-style license found in the
-#
+# LICENSE file in the root directory of this source tree. An additional grant
-# Redistribution and use in source and binary forms, with or without modification,
+# of patent rights can be found in the PATENTS file in the same directory.
-# are permitted provided that the following conditions are met:
-#
-# * Redistributions of source code must retain the above copyright notice, this
-#   list of conditions and the following disclaimer.
-#
-# * Redistributions in binary form must reproduce the above copyright notice, this
-#   list of conditions and the following disclaimer in the documentation and/or
-#   other materials provided with the distribution.
-#
-# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
-# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
-# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
-# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
-# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
-# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
-# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-#
-# You can contact the author at :
-#  - zstd homepage : http://www.zstd.net/
 # ################################################################
 
 PRGDIR  = programs
 ZSTDDIR = lib
 ZWRAPDIR = zlibWrapper
+TESTDIR  = tests
 
 # Define nul output
 ifneq (,$(filter Windows%,$(OS)))
@@ -47,9 +25,8 @@ default: zstd
 
 all:
 	$(MAKE) -C $(ZSTDDIR) $@
-	$(MAKE) -C $(PRGDIR) $@
+	$(MAKE) -C $(PRGDIR) $@ zstd32
-	@rm -f lib/decompress/*.o
+	$(MAKE) -C $(TESTDIR) $@ all32
-	$(MAKE) -C $(PRGDIR) all32
 
 zstd:
 	$(MAKE) -C $(PRGDIR)
@@ -60,11 +37,12 @@ zlibwrapper:
 	$(MAKE) -C $(ZWRAPDIR) all
 
 test:
-	$(MAKE) -C $(PRGDIR) $@
+	$(MAKE) -C $(TESTDIR) $@
 
 clean:
 	@$(MAKE) -C $(ZSTDDIR) $@ > $(VOID)
 	@$(MAKE) -C $(PRGDIR) $@ > $(VOID)
+	@$(MAKE) -C $(TESTDIR) $@ > $(VOID)
 	@$(MAKE) -C $(ZWRAPDIR) $@ > $(VOID)
 	@rm -f zstd
 	@echo Cleaning completed
@@ -99,19 +77,19 @@ gcc6test: clean
 
 clangtest: clean
 	clang -v
-	$(MAKE) all CC=clang MOREFLAGS="-Werror -Wconversion -Wno-sign-conversion"
+	$(MAKE) all CC=clang MOREFLAGS="-Werror -Wconversion -Wno-sign-conversion -Wdocumentation"
 
 armtest: clean
-	$(MAKE) -C $(PRGDIR) datagen   # use native, faster
+	$(MAKE) -C $(TESTDIR) datagen   # use native, faster
-	$(MAKE) -C $(PRGDIR) test CC=arm-linux-gnueabi-gcc ZSTDRTTEST= MOREFLAGS="-Werror -static"
+	$(MAKE) -C $(TESTDIR) test CC=arm-linux-gnueabi-gcc ZSTDRTTEST= MOREFLAGS="-Werror -static"
 
 ppctest: clean
-	$(MAKE) -C $(PRGDIR) datagen   # use native, faster
+	$(MAKE) -C $(TESTDIR) datagen   # use native, faster
-	$(MAKE) -C $(PRGDIR) test CC=powerpc-linux-gnu-gcc ZSTDRTTEST= MOREFLAGS="-Werror -Wno-attributes -static"
+	$(MAKE) -C $(TESTDIR) test CC=powerpc-linux-gnu-gcc ZSTDRTTEST= MOREFLAGS="-Werror -Wno-attributes -static"
 
 ppc64test: clean
-	$(MAKE) -C $(PRGDIR) datagen   # use native, faster
+	$(MAKE) -C $(TESTDIR) datagen   # use native, faster
-	$(MAKE) -C $(PRGDIR) test CC=powerpc-linux-gnu-gcc ZSTDRTTEST= MOREFLAGS="-m64 -static"
+	$(MAKE) -C $(TESTDIR) test CC=powerpc-linux-gnu-gcc ZSTDRTTEST= MOREFLAGS="-m64 -static"
 
 usan: clean
 	$(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=undefined"
@@ -123,7 +101,7 @@ msan: clean
 	$(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=memory -fno-omit-frame-pointer"   # datagen.c fails this test for no obvious reason
 
 asan32: clean
-	$(MAKE) -C $(PRGDIR) test32 CC=clang MOREFLAGS="-g -fsanitize=address"
+	$(MAKE) -C $(TESTDIR) test32 CC=clang MOREFLAGS="-g -fsanitize=address"
 
 uasan: clean
 	$(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=address -fsanitize=undefined"
@@ -163,13 +141,13 @@ c11test: clean
 	CFLAGS="-std=c11" $(MAKE) all
 
 bmix64test: clean
-	CFLAGS="-O3 -mbmi -Werror" $(MAKE) -C $(PRGDIR) test
+	CFLAGS="-O3 -mbmi -Werror" $(MAKE) -C $(TESTDIR) test
 
 bmix32test: clean
-	CFLAGS="-O3 -mbmi -mx32 -Werror" $(MAKE) -C $(PRGDIR) test
+	CFLAGS="-O3 -mbmi -mx32 -Werror" $(MAKE) -C $(TESTDIR) test
 
 bmi32test: clean
-	CFLAGS="-O3 -mbmi -m32 -Werror" $(MAKE) -C $(PRGDIR) test
+	CFLAGS="-O3 -mbmi -m32 -Werror" $(MAKE) -C $(TESTDIR) test
 
 staticAnalyze: clean
 	CPPFLAGS=-g scan-build --status-bugs -v $(MAKE) all

NEWS

@@ -1,3 +1,12 @@
+v1.0.0
+Change Licensing, all project is now BSD, Copyright Facebook
+Small decompression speed improvement
+API : Streaming API supports legacy format
+API : ZDICT_getDictID(), ZSTD_sizeof_{CCtx, DCtx, CStream, DStream}(), ZSTD_setDStreamParamter()
+CLI supports legacy formats v0.4+
+Fixed : compression fails on certain huge files, reported by Jesse McGrew
+Enhanced documentation, by Przemyslaw Skibinski
+
 v0.8.1
 New streaming API
 Changed : --ultra now enables levels beyond 19

PATENTS

@@ -0,0 +1,33 @@
+Additional Grant of Patent Rights Version 2
+
+"Software" means the Zstandard software distributed by Facebook, Inc.
+
+Facebook, Inc. ("Facebook") hereby grants to each recipient of the Software
+("you") a perpetual, worldwide, royalty-free, non-exclusive, irrevocable
+(subject to the termination provision below) license under any Necessary
+Claims, to make, have made, use, sell, offer to sell, import, and otherwise
+transfer the Software. For avoidance of doubt, no license is granted under
+Facebook’s rights in any patent claims that are infringed by (i) modifications
+to the Software made by you or any third party or (ii) the Software in
+combination with any software or other technology.
+
+The license granted hereunder will terminate, automatically and without notice,
+if you (or any of your subsidiaries, corporate affiliates or agents) initiate
+directly or indirectly, or take a direct financial interest in, any Patent
+Assertion: (i) against Facebook or any of its subsidiaries or corporate
+affiliates, (ii) against any party if such Patent Assertion arises in whole or
+in part from any software, technology, product or service of Facebook or any of
+its subsidiaries or corporate affiliates, or (iii) against any party relating
+to the Software. Notwithstanding the foregoing, if Facebook or any of its
+subsidiaries or corporate affiliates files a lawsuit alleging patent
+infringement against you in the first instance, and you respond by filing a
+patent infringement counterclaim in that lawsuit against that party that is
+unrelated to the Software, the license granted hereunder will not terminate
+under section (i) of this paragraph due to such counterclaim.
+
+A "Necessary Claim" is a claim of a patent owned by Facebook that is
+necessarily infringed by the Software standing alone.
+
+A "Patent Assertion" is any lawsuit or other action alleging direct, indirect,
+or contributory infringement or inducement to infringe any patent, including a
+cross-claim or counterclaim.

README.md

@@ -19,7 +19,7 @@ As a reference, several fast compression algorithms were tested and compared on
 |Name             | Ratio | C.speed | D.speed |
 |-----------------|-------|--------:|--------:|
 |                 |       |   MB/s  |  MB/s   |
-|**zstd 0.8.0 -1**|**2.877**|**330**| **930** |
+|**zstd 0.8.2 -1**|**2.877**|**330**| **940** |
 | [zlib] 1.2.8 -1 | 2.730 |    95   |   360   |
 | brotli 0.4 -0   | 2.708 |   320   |   375   |
 | QuickLZ 1.5     | 2.237 |   510   |   605   |
@@ -61,8 +61,6 @@ Hence, deploying one dictionary per type of data will provide the greater benefi
 
 #### Dictionary compression How To :
 
-##### _Using the Command Line Utility_ :
-
 1) Create the dictionary
 
 `zstd --train FullPathToTrainingSet/* -o dictionaryName`
@@ -77,14 +75,19 @@ Hence, deploying one dictionary per type of data will provide the greater benefi
 
 ### Status
 
-Zstd compression format has reached "Final status". It means it is planned to become the official stable zstd format tagged `v1.0`. The reason it's not yet tagged `v1.0` is that it currently performs its "validation period", making sure the format holds all its promises and nothing was missed.
+Zstandard is currently deployed within Facebook. It is used daily to compress and decompress very large amount of data in multiple formats and use cases.
-Zstd library also offers legacy decoder support. Any data compressed by any version >= `v0.1` is decodable now and in the future.
+Zstandard is considered safe for production environments.
-The library has been validated using strong [fuzzer tests](https://en.wikipedia.org/wiki/Fuzz_testing), including both [internal tools](programs/fuzzer.c) and [external ones](http://lcamtuf.coredump.cx/afl). It's able to withstand hazard situations, including invalid inputs.
-As a consequence, Zstandard is considered safe for, and is currently used in, production environments.
 
-### Branch Policy
+### License
 
-The "dev" branch is the one where all contributions will be merged before reaching "master". If you plan to propose a patch, please commit into the "dev" branch or its own feature branch. Direct commit to "master" are not permitted.
+Zstandard is [BSD-licensed](LICENSE). We also provide an [additional patent grant](PATENTS).
+
+### Contributing
+
+The "dev" branch is the one where all contributions will be merged before reaching "master".
+If you plan to propose a patch, please commit into the "dev" branch or its own feature branch.
+Direct commit to "master" are not permitted.
+For more information, please read [CONTRIBUTING](CONTRIBUTING.md).
 
 ### Miscellaneous
 

appveyor.yml

@@ -24,7 +24,7 @@ install:
   - ECHO Installing %COMPILER% %PLATFORM% %CONFIGURATION%
   - if [%COMPILER%]==[gcc] SET PATH_ORIGINAL=%PATH%
   - if [%COMPILER%]==[gcc] (
-      SET "CLANG_PARAMS=-C programs zstd fullbench fuzzer zbufftest paramgrill datagen CC=clang MOREFLAGS="--target=x86_64-w64-mingw32 -Werror -Wconversion -Wno-sign-conversion"" &&
+      SET "CLANG_PARAMS=-C tests zstd fullbench fuzzer zbufftest paramgrill datagen CC=clang MOREFLAGS="--target=x86_64-w64-mingw32 -Werror -Wconversion -Wno-sign-conversion"" &&
       SET "PATH_MINGW32=c:\MinGW\bin;c:\MinGW\usr\bin" &&
       SET "PATH_MINGW64=c:\msys64\mingw64\bin;c:\msys64\usr\bin" &&
       COPY C:\MinGW\bin\mingw32-make.exe C:\MinGW\bin\make.exe &&
@@ -44,7 +44,7 @@ build_script:
       ECHO *** &&
       ECHO make %CLANG_PARAMS% &&
       make %CLANG_PARAMS% &&
-      COPY programs\fuzzer.exe projects\fuzzer_clang.exe &&
+      COPY tests\fuzzer.exe tests\fuzzer_clang.exe &&
       make clean
     )
   - if [%COMPILER%]==[gcc] (
@@ -64,7 +64,7 @@ build_script:
       msbuild "projects\VS2008\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v90 /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" &&
       DIR projects\VS2008\bin\%PLATFORM%\%CONFIGURATION%\*.exe &&
       MD5sum projects/VS2008/bin/%PLATFORM%/%CONFIGURATION%/*.exe &&
-      COPY projects\VS2008\bin\%PLATFORM%\%CONFIGURATION%\fuzzer.exe projects\fuzzer_VS2008_%PLATFORM%_%CONFIGURATION%.exe &&
+      COPY projects\VS2008\bin\%PLATFORM%\%CONFIGURATION%\fuzzer.exe tests\fuzzer_VS2008_%PLATFORM%_%CONFIGURATION%.exe &&
       ECHO *** &&
       ECHO *** Building Visual Studio 2010 %PLATFORM%\%CONFIGURATION% &&
       ECHO *** &&
@@ -74,7 +74,7 @@ build_script:
       msbuild "projects\VS2010\zstd.sln" %ADDITIONALPARAM% /m /verbosity:minimal /property:PlatformToolset=v100 /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" &&
       DIR projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\*.exe &&
       MD5sum projects/VS2010/bin/%PLATFORM%/%CONFIGURATION%/*.exe &&
-      COPY projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\fuzzer.exe projects\fuzzer_VS2010_%PLATFORM%_%CONFIGURATION%.exe &&
+      COPY projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\fuzzer.exe tests\fuzzer_VS2010_%PLATFORM%_%CONFIGURATION%.exe &&
       ECHO *** &&
       ECHO *** Building Visual Studio 2012 %PLATFORM%\%CONFIGURATION% &&
       ECHO *** &&
@@ -84,7 +84,7 @@ build_script:
       msbuild "projects\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v110 /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" &&
       DIR projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\*.exe &&
       MD5sum projects/VS2010/bin/%PLATFORM%/%CONFIGURATION%/*.exe &&
-      COPY projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\fuzzer.exe projects\fuzzer_VS2012_%PLATFORM%_%CONFIGURATION%.exe &&
+      COPY projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\fuzzer.exe tests\fuzzer_VS2012_%PLATFORM%_%CONFIGURATION%.exe &&
       ECHO *** &&
       ECHO *** Building Visual Studio 2013 %PLATFORM%\%CONFIGURATION% &&
       ECHO *** &&
@@ -94,7 +94,7 @@ build_script:
       msbuild "projects\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v120 /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" &&
       DIR projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\*.exe &&
       MD5sum projects/VS2010/bin/%PLATFORM%/%CONFIGURATION%/*.exe &&
-      COPY projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\fuzzer.exe projects\fuzzer_VS2013_%PLATFORM%_%CONFIGURATION%.exe &&
+      COPY projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\fuzzer.exe tests\fuzzer_VS2013_%PLATFORM%_%CONFIGURATION%.exe &&
       ECHO *** &&
       ECHO *** Building Visual Studio 2015 %PLATFORM%\%CONFIGURATION% &&
       ECHO *** &&
@@ -104,26 +104,25 @@ build_script:
       msbuild "projects\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v140 /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" &&
       DIR projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\*.exe &&
       MD5sum projects/VS2010/bin/%PLATFORM%/%CONFIGURATION%/*.exe &&
-      COPY projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\fuzzer.exe projects\fuzzer_VS2015_%PLATFORM%_%CONFIGURATION%.exe &&
+      COPY projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\fuzzer.exe tests\fuzzer_VS2015_%PLATFORM%_%CONFIGURATION%.exe &&
-      COPY projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\*.exe programs\
+      COPY projects\VS2010\bin\%PLATFORM%\%CONFIGURATION%\*.exe tests\
     )
 
 test_script:
   - ECHO Testing %COMPILER% %PLATFORM% %CONFIGURATION%
   - SET FUZZERTEST=-T1mn
   - if [%COMPILER%]==[gcc] (
-      if [%PLATFORM%]==[mingw64] projects\fuzzer_clang.exe %FUZZERTEST%
+      if [%PLATFORM%]==[mingw64] tests\fuzzer_clang.exe %FUZZERTEST%
     )
   - if [%COMPILER%]==[visual] if [%CONFIGURATION%]==[Release] (
-      CD programs &&
+      CD tests &&
       SET ZSTD=./zstd.exe &&
       sh -e playTests.sh --test-large-data &&
       fullbench.exe -i1 &&
       fullbench.exe -i1 -P0 &&
-      CD .. &&
+      fuzzer_VS2008_%PLATFORM%_Release.exe %FUZZERTEST% &&
-      projects\fuzzer_VS2008_%PLATFORM%_Release.exe %FUZZERTEST% &&
+      fuzzer_VS2010_%PLATFORM%_Release.exe %FUZZERTEST% &&
-      projects\fuzzer_VS2010_%PLATFORM%_Release.exe %FUZZERTEST% &&
+      fuzzer_VS2012_%PLATFORM%_Release.exe %FUZZERTEST% &&
-      projects\fuzzer_VS2012_%PLATFORM%_Release.exe %FUZZERTEST% &&
+      fuzzer_VS2013_%PLATFORM%_Release.exe %FUZZERTEST% &&
-      projects\fuzzer_VS2013_%PLATFORM%_Release.exe %FUZZERTEST% &&
+      fuzzer_VS2015_%PLATFORM%_Release.exe %FUZZERTEST%
-      projects\fuzzer_VS2015_%PLATFORM%_Release.exe %FUZZERTEST%
     )

examples/dictionary_compression.c

@@ -1,27 +1,11 @@
-/*
+/**
-  Dictionary compression
+ * Copyright 2016-present, Yann Collet, Facebook, Inc.
-  Educational program using zstd library
+ * All rights reserved.
-  Copyright (C) Yann Collet 2016
+ *
+ * This source code is licensed under the license found in the
+ * LICENSE-examples file in the root directory of this source tree.
+ */
 
-  GPL v2 License
-
-  This program is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published by
-  the Free Software Foundation; either version 2 of the License, or
-  (at your option) any later version.
-
-  This program is distributed in the hope that it will be useful,
-  but WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-  GNU General Public License for more details.
-
-  You should have received a copy of the GNU General Public License along
-  with this program; if not, write to the Free Software Foundation, Inc.,
-  51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-  You can contact the author at :
-  - zstd homepage : http://www.zstd.net/
-*/
 
 #include <stdlib.h>    // malloc, exit
 #include <stdio.h>     // printf

examples/dictionary_decompression.c

@@ -1,27 +1,12 @@
-/*
+/**
-  Dictionary decompression
+ * Copyright 2016-present, Yann Collet, Facebook, Inc.
-  Educational program using zstd library
+ * All rights reserved.
-  Copyright (C) Yann Collet 2016
+ *
+ * This source code is licensed under the license found in the
+ * LICENSE-examples file in the root directory of this source tree.
+ */
 
-  GPL v2 License
 
-  This program is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published by
-  the Free Software Foundation; either version 2 of the License, or
-  (at your option) any later version.
-
-  This program is distributed in the hope that it will be useful,
-  but WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-  GNU General Public License for more details.
-
-  You should have received a copy of the GNU General Public License along
-  with this program; if not, write to the Free Software Foundation, Inc.,
-  51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-  You can contact the author at :
-  - zstd homepage : http://www.zstd.net/
-*/
 
 #include <stdlib.h>    // malloc, exit
 #include <stdio.h>     // printf

examples/simple_compression.c

@@ -1,27 +1,12 @@
-/*
+/**
-  Simple compression
+ * Copyright 2016-present, Yann Collet, Facebook, Inc.
-  Educational program using zstd library
+ * All rights reserved.
-  Copyright (C) Yann Collet 2016
+ *
+ * This source code is licensed under the license found in the
+ * LICENSE-examples file in the root directory of this source tree.
+ */
 
-  GPL v2 License
 
-  This program is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published by
-  the Free Software Foundation; either version 2 of the License, or
-  (at your option) any later version.
-
-  This program is distributed in the hope that it will be useful,
-  but WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-  GNU General Public License for more details.
-
-  You should have received a copy of the GNU General Public License along
-  with this program; if not, write to the Free Software Foundation, Inc.,
-  51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-  You can contact the author at :
-  - zstd homepage : http://www.zstd.net/
-*/
 
 #include <stdlib.h>    // malloc, exit
 #include <stdio.h>     // fprintf, perror

examples/simple_decompression.c

@@ -1,27 +1,12 @@
-/*
+/**
-  Simple decompression
+ * Copyright 2016-present, Yann Collet, Facebook, Inc.
-  Educational program using zstd library
+ * All rights reserved.
-  Copyright (C) Yann Collet 2016
+ *
+ * This source code is licensed under the license found in the
+ * LICENSE-examples file in the root directory of this source tree.
+ */
 
-  GPL v2 License
 
-  This program is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published by
-  the Free Software Foundation; either version 2 of the License, or
-  (at your option) any later version.
-
-  This program is distributed in the hope that it will be useful,
-  but WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-  GNU General Public License for more details.
-
-  You should have received a copy of the GNU General Public License along
-  with this program; if not, write to the Free Software Foundation, Inc.,
-  51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-  You can contact the author at :
-  - zstd homepage : http://www.zstd.net/
-*/
 
 #include <stdlib.h>    // malloc, exit
 #include <stdio.h>     // printf

examples/streaming_compression.c

@@ -1,27 +1,11 @@
-/*
+/**
-  Streaming compression
+ * Copyright 2016-present, Yann Collet, Facebook, Inc.
-  Educational program using zstd library
+ * All rights reserved.
-  Copyright (C) Yann Collet 2016
+ *
+ * This source code is licensed under the license found in the
+ * LICENSE-examples file in the root directory of this source tree.
+ */
 
-  GPL v2 License
-
-  This program is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published by
-  the Free Software Foundation; either version 2 of the License, or
-  (at your option) any later version.
-
-  This program is distributed in the hope that it will be useful,
-  but WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-  GNU General Public License for more details.
-
-  You should have received a copy of the GNU General Public License along
-  with this program; if not, write to the Free Software Foundation, Inc.,
-  51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-  You can contact the author at :
-  - zstd homepage : http://www.zstd.net/
-*/
 
 #include <stdlib.h>    // malloc, exit
 #include <stdio.h>     // fprintf, perror, feof

examples/streaming_decompression.c

@@ -1,33 +1,16 @@
-/*
+/**
-  Streaming compression
+ * Copyright 2016-present, Yann Collet, Facebook, Inc.
-  Educational program using zstd library
+ * All rights reserved.
-  Copyright (C) Yann Collet 2016
+ *
+ * This source code is licensed under the license found in the
+ * LICENSE-examples file in the root directory of this source tree.
+ */
 
-  GPL v2 License
-
-  This program is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published by
-  the Free Software Foundation; either version 2 of the License, or
-  (at your option) any later version.
-
-  This program is distributed in the hope that it will be useful,
-  but WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-  GNU General Public License for more details.
-
-  You should have received a copy of the GNU General Public License along
-  with this program; if not, write to the Free Software Foundation, Inc.,
-  51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-  You can contact the author at :
-  - zstd homepage : http://www.zstd.net/
-*/
 
 #include <stdlib.h>    // malloc, exit
 #include <stdio.h>     // fprintf, perror, feof
 #include <string.h>    // strerror
 #include <errno.h>     // errno
-#define ZSTD_STATIC_LINKING_ONLY  // streaming API defined as "experimental" for the time being
 #include <zstd.h>      // presumes zstd library is installed
 
 

lib/Makefile

@@ -1,33 +1,10 @@
 # ################################################################
-# ZSTD library - Makefile
+# Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
-# Copyright (C) Yann Collet 2015-2016
 # All rights reserved.
 #
-# BSD license
+# This source code is licensed under the BSD-style license found in the
-
+# LICENSE file in the root directory of this source tree. An additional grant
-# Redistribution and use in source and binary forms, with or without modification,
+# of patent rights can be found in the PATENTS file in the same directory.
-# are permitted provided that the following conditions are met:
-#
-# * Redistributions of source code must retain the above copyright notice, this
-#   list of conditions and the following disclaimer.
-#
-# * Redistributions in binary form must reproduce the above copyright notice, this
-#   list of conditions and the following disclaimer in the documentation and/or
-#   other materials provided with the distribution.
-#
-# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
-# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
-# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
-# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
-# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
-# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
-# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-#
-# You can contact the author at :
-#  - ZSTD homepage : http://www.zstd.net
 # ################################################################
 
 # Version numbers
@@ -125,14 +102,14 @@ install: libzstd libzstd.pc
 	@echo zstd static and shared library installed
 
 uninstall:
-	@rm -f $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT)
+	$(RM) $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT)
-	@rm -f $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT_MAJOR)
+	$(RM) $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT_MAJOR)
-	@rm -f $(DESTDIR)$(LIBDIR)/pkgconfig/libzstd.pc
+	$(RM) $(DESTDIR)$(LIBDIR)/pkgconfig/libzstd.pc
-	@[ -x $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT_VER) ] && rm -f $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT_VER)
+	[ -x $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT_VER) ] && $(RM) $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT_VER)
-	@[ -f $(DESTDIR)$(LIBDIR)/libzstd.a ] && rm -f $(DESTDIR)$(LIBDIR)/libzstd.a
+	@[ -f $(DESTDIR)$(LIBDIR)/libzstd.a ] && $(RM) $(DESTDIR)$(LIBDIR)/libzstd.a
-	@[ -f $(DESTDIR)$(INCLUDEDIR)/zstd.h ] && rm -f $(DESTDIR)$(INCLUDEDIR)/zstd.h
+	@[ -f $(DESTDIR)$(INCLUDEDIR)/zstd.h ] && $(RM) $(DESTDIR)$(INCLUDEDIR)/zstd.h
-	@[ -f $(DESTDIR)$(INCLUDEDIR)/zbuff.h ] && rm -f $(DESTDIR)$(INCLUDEDIR)/zbuff.h
+	@[ -f $(DESTDIR)$(INCLUDEDIR)/zbuff.h ] && $(RM) $(DESTDIR)$(INCLUDEDIR)/zbuff.h
-	@[ -f $(DESTDIR)$(INCLUDEDIR)/zdict.h ] && rm -f $(DESTDIR)$(INCLUDEDIR)/zdict.h
+	@[ -f $(DESTDIR)$(INCLUDEDIR)/zdict.h ] && $(RM) $(DESTDIR)$(INCLUDEDIR)/zdict.h
 	@echo zstd libraries successfully uninstalled
 
 endif

lib/README.md

@@ -1,4 +1,4 @@
-zstd - library files
+Zstandard library files
 ================================
 
 The __lib__ directory contains several directories.

lib/common/error_private.h

@@ -1,35 +1,12 @@
-/* ******************************************************************
+/**
-   Error codes and messages
+ * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
-   Copyright (C) 2013-2016, Yann Collet
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
 
-   BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
-
-   Redistribution and use in source and binary forms, with or without
-   modification, are permitted provided that the following conditions are
-   met:
-
-       * Redistributions of source code must retain the above copyright
-   notice, this list of conditions and the following disclaimer.
-       * Redistributions in binary form must reproduce the above
-   copyright notice, this list of conditions and the following disclaimer
-   in the documentation and/or other materials provided with the
-   distribution.
-
-   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-   You can contact the author at :
-   - Homepage : http://www.zstd.net
-****************************************************************** */
 /* Note : this module is expected to remain private, do not expose it */
 
 #ifndef ERROR_H_MODULE
@@ -93,6 +70,8 @@ ERR_STATIC const char* ERR_getErrorString(ERR_enum code)
     case PREFIX(no_error): return "No error detected";
     case PREFIX(GENERIC):  return "Error (generic)";
     case PREFIX(prefix_unknown): return "Unknown frame descriptor";
+    case PREFIX(version_unsupported): return "Version not supported";
+    case PREFIX(parameter_unknown): return "Unknown parameter type";
     case PREFIX(frameParameter_unsupported): return "Unsupported frame parameter";
     case PREFIX(frameParameter_unsupportedBy32bits): return "Frame parameter unsupported in 32-bits mode";
     case PREFIX(compressionParameter_unsupported): return "Compression parameter is out of bound";

lib/common/error_public.h

@@ -1,35 +1,12 @@
-/* ******************************************************************
+/**
-   Error codes list
+ * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
-   Copyright (C) 2016, Yann Collet
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
 
-   BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
-
-   Redistribution and use in source and binary forms, with or without
-   modification, are permitted provided that the following conditions are
-   met:
-
-       * Redistributions of source code must retain the above copyright
-   notice, this list of conditions and the following disclaimer.
-       * Redistributions in binary form must reproduce the above
-   copyright notice, this list of conditions and the following disclaimer
-   in the documentation and/or other materials provided with the
-   distribution.
-
-   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-   You can contact the author at :
-   - Homepage : http://www.zstd.net
-****************************************************************** */
 #ifndef ERROR_PUBLIC_H_MODULE
 #define ERROR_PUBLIC_H_MODULE
 
@@ -37,14 +14,19 @@
 extern "C" {
 #endif
 
+/*===== dependency =====*/
+#include <stddef.h>   /* size_t */
 
-/* ****************************************
+
+/*-****************************************
 *  error codes list
 ******************************************/
 typedef enum {
   ZSTD_error_no_error,
   ZSTD_error_GENERIC,
   ZSTD_error_prefix_unknown,
+  ZSTD_error_version_unsupported,
+  ZSTD_error_parameter_unknown,
   ZSTD_error_frameParameter_unsupported,
   ZSTD_error_frameParameter_unsupportedBy32bits,
   ZSTD_error_compressionParameter_unsupported,

lib/common/fse.h

@@ -503,7 +503,7 @@ MEM_STATIC void FSE_flushCState(BIT_CStream_t* bitC, const FSE_CState_t* statePt
     BIT_flushBits(bitC);
 }
 
-/*<=====    Decompression    =====>*/
+/* ======    Decompression    ====== */
 
 typedef struct {
     U16 tableLog;

lib/common/fse_decompress.c

@@ -68,6 +68,9 @@
 #define FSE_isError ERR_isError
 #define FSE_STATIC_ASSERT(c) { enum { FSE_static_assert = 1/(int)(!!(c)) }; }   /* use only *after* variable declarations */
 
+/* check and forward error code */
+#define CHECK_F(f) { size_t const e = f; if (FSE_isError(e)) return e; }
+
 
 /* **************************************************************
 *  Complex types
@@ -152,7 +155,6 @@ size_t FSE_buildDTable(FSE_DTable* dt, const short* normalizedCounter, unsigned
                 position = (position + step) & tableMask;
                 while (position > highThreshold) position = (position + step) & tableMask;   /* lowprob area */
         }   }
-
         if (position!=0) return ERROR(GENERIC);   /* position must reach all cells once, otherwise normalizedCounter is incorrect */
     }
 
@@ -169,7 +171,6 @@ size_t FSE_buildDTable(FSE_DTable* dt, const short* normalizedCounter, unsigned
 }
 
 
-
 #ifndef FSE_COMMONDEFS_ONLY
 
 /*-*******************************************************
@@ -234,8 +235,7 @@ FORCE_INLINE size_t FSE_decompress_usingDTable_generic(
     FSE_DState_t state2;
 
     /* Init */
-    { size_t const errorCode = BIT_initDStream(&bitD, cSrc, cSrcSize);   /* replaced last arg by maxCompressed Size */
+    CHECK_F(BIT_initDStream(&bitD, cSrc, cSrcSize));
-      if (FSE_isError(errorCode)) return errorCode; }
 
     FSE_initDState(&state1, &bitD, dt);
     FSE_initDState(&state2, &bitD, dt);
@@ -243,7 +243,7 @@ FORCE_INLINE size_t FSE_decompress_usingDTable_generic(
 #define FSE_GETSYMBOL(statePtr) fast ? FSE_decodeSymbolFast(statePtr, &bitD) : FSE_decodeSymbol(statePtr, &bitD)
 
     /* 4 symbols per loop */
-    for ( ; (BIT_reloadDStream(&bitD)==BIT_DStream_unfinished) && (op<olimit) ; op+=4) {
+    for ( ; (BIT_reloadDStream(&bitD)==BIT_DStream_unfinished) & (op<olimit) ; op+=4) {
         op[0] = FSE_GETSYMBOL(&state1);
 
         if (FSE_MAX_TABLELOG*2+7 > sizeof(bitD.bitContainer)*8)    /* This test must be static */
@@ -266,18 +266,14 @@ FORCE_INLINE size_t FSE_decompress_usingDTable_generic(
     /* note : BIT_reloadDStream(&bitD) >= FSE_DStream_partiallyFilled; Ends at exactly BIT_DStream_completed */
     while (1) {
         if (op>(omax-2)) return ERROR(dstSize_tooSmall);
-
         *op++ = FSE_GETSYMBOL(&state1);
-
         if (BIT_reloadDStream(&bitD)==BIT_DStream_overflow) {
             *op++ = FSE_GETSYMBOL(&state2);
             break;
         }
 
         if (op>(omax-2)) return ERROR(dstSize_tooSmall);
-
         *op++ = FSE_GETSYMBOL(&state2);
-
         if (BIT_reloadDStream(&bitD)==BIT_DStream_overflow) {
             *op++ = FSE_GETSYMBOL(&state1);
             break;
@@ -320,8 +316,7 @@ size_t FSE_decompress(void* dst, size_t maxDstSize, const void* cSrc, size_t cSr
         cSrcSize -= NCountLength;
     }
 
-    { size_t const errorCode = FSE_buildDTable (dt, counting, maxSymbolValue, tableLog);
+    CHECK_F( FSE_buildDTable (dt, counting, maxSymbolValue, tableLog) );
-      if (FSE_isError(errorCode)) return errorCode; }
 
     return FSE_decompress_usingDTable (dst, maxDstSize, ip, cSrcSize, dt);   /* always return, even if it is an error code */
 }

lib/common/mem.h

@@ -1,37 +1,12 @@
-/* ******************************************************************
+/**
-   mem.h
+ * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
-   low-level memory access routines
+ * All rights reserved.
-   Copyright (C) 2013-2015, Yann Collet.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
 
-   BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
-
-   Redistribution and use in source and binary forms, with or without
-   modification, are permitted provided that the following conditions are
-   met:
-
-       * Redistributions of source code must retain the above copyright
-   notice, this list of conditions and the following disclaimer.
-       * Redistributions in binary form must reproduce the above
-   copyright notice, this list of conditions and the following disclaimer
-   in the documentation and/or other materials provided with the
-   distribution.
-
-   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-    You can contact the author at :
-    - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy
-    - Public forum : https://groups.google.com/forum/#!forum/lz4c
-****************************************************************** */
 #ifndef MEM_H_MODULE
 #define MEM_H_MODULE
 

lib/common/zbuff.h

@@ -1,33 +1,20 @@
-/*
+/**
-    Buffered version of Zstd compression library
+ * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
-    Copyright (C) 2015-2016, Yann Collet.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
 
-    BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+/* ***************************************************************
+*  NOTES/WARNINGS
+*****************************************************************/
+/* The streaming API defined here will soon be deprecated by the
+* new one in 'zstd.h'; consider migrating towards newer streaming
+* API. See 'lib/README.md'.
+*****************************************************************/
 
-    Redistribution and use in source and binary forms, with or without
-    modification, are permitted provided that the following conditions are
-    met:
-    * Redistributions of source code must retain the above copyright
-    notice, this list of conditions and the following disclaimer.
-    * Redistributions in binary form must reproduce the above
-    copyright notice, this list of conditions and the following disclaimer
-    in the documentation and/or other materials provided with the
-    distribution.
-    THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-    "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-    LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-    A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-    OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-    SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-    LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-    DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-    THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-    (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-    OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-    You can contact the author at :
-    - zstd homepage : http://www.zstd.net/
-*/
 #ifndef ZSTD_BUFFERED_H_23987
 #define ZSTD_BUFFERED_H_23987
 

lib/common/zstd_common.c

@@ -1,33 +1,12 @@
-/*
+/**
-    Common functions of Zstd compression library
+ * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
-    Copyright (C) 2015-2016, Yann Collet.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
 
-    BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
-
-    Redistribution and use in source and binary forms, with or without
-    modification, are permitted provided that the following conditions are
-    met:
-    * Redistributions of source code must retain the above copyright
-    notice, this list of conditions and the following disclaimer.
-    * Redistributions in binary form must reproduce the above
-    copyright notice, this list of conditions and the following disclaimer
-    in the documentation and/or other materials provided with the
-    distribution.
-    THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-    "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-    LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-    A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-    OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-    SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-    LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-    DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-    THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-    (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-    OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-    You can contact the author at :
-    - zstd homepage : http://www.zstd.net/
-*/
 
 
 /*-*************************************
@@ -75,17 +54,30 @@ const char* ZBUFF_getErrorName(size_t errorCode) { return ERR_getErrorName(error
 
 
 
+/*=**************************************************************
+*  Custom allocator
+****************************************************************/
+/* default uses stdlib */
 void* ZSTD_defaultAllocFunction(void* opaque, size_t size)
 {
     void* address = malloc(size);
     (void)opaque;
-    /* printf("alloc %p, %d opaque=%p \n", address, (int)size, opaque); */
     return address;
 }
 
 void ZSTD_defaultFreeFunction(void* opaque, void* address)
 {
     (void)opaque;
-    /* if (address) printf("free %p opaque=%p \n", address, opaque); */
     free(address);
 }
+
+void* ZSTD_malloc(size_t size, ZSTD_customMem customMem)
+{
+    return customMem.customAlloc(customMem.opaque, size);
+}
+
+void ZSTD_free(void* ptr, ZSTD_customMem customMem)
+{
+    if (ptr!=NULL)
+        customMem.customFree(customMem.opaque, ptr);
+}

lib/common/zstd_internal.h

@@ -1,34 +1,12 @@
-/*
+/**
-    zstd_internal - common functions to include
+ * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
-    Header File for include
+ * All rights reserved.
-    Copyright (C) 2014-2016, Yann Collet.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
 
-    BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
-
-    Redistribution and use in source and binary forms, with or without
-    modification, are permitted provided that the following conditions are
-    met:
-    * Redistributions of source code must retain the above copyright
-    notice, this list of conditions and the following disclaimer.
-    * Redistributions in binary form must reproduce the above
-    copyright notice, this list of conditions and the following disclaimer
-    in the documentation and/or other materials provided with the
-    distribution.
-    THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-    "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-    LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-    A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-    OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-    SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-    LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-    DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-    THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-    (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-    OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-    You can contact the author at :
-    - zstd homepage : https://www.zstd.net
-*/
 #ifndef ZSTD_CCOMMON_H_MODULE
 #define ZSTD_CCOMMON_H_MODULE
 
@@ -51,19 +29,6 @@
 /*-*************************************
 *  Common constants
 ***************************************/
-#define ZSTD_OPT_DEBUG 0     /* 3 = compression stats;  5 = check encoded sequences;  9 = full logs */
-#if defined(ZSTD_OPT_DEBUG) && ZSTD_OPT_DEBUG>=9
-    #include <stdio.h>
-    #include <stdlib.h>
-    #define ZSTD_LOG_PARSER(...) printf(__VA_ARGS__)
-    #define ZSTD_LOG_ENCODE(...) printf(__VA_ARGS__)
-    #define ZSTD_LOG_BLOCK(...) printf(__VA_ARGS__)
-#else
-    #define ZSTD_LOG_PARSER(...)
-    #define ZSTD_LOG_ENCODE(...)
-    #define ZSTD_LOG_BLOCK(...)
-#endif
-
 #define ZSTD_OPT_NUM    (1<<12)
 #define ZSTD_DICT_MAGIC  0xEC30A437   /* v0.7+ */
 
@@ -184,16 +149,6 @@ typedef struct {
     U32 rep[ZSTD_REP_NUM];
 } ZSTD_optimal_t;
 
-#if ZSTD_OPT_DEBUG == 3
-    #include ".debug/zstd_stats.h"
-#else
-    struct ZSTD_stats_s { U32 unused; };
-    MEM_STATIC void ZSTD_statsPrint(ZSTD_stats_t* stats, U32 searchLength) { (void)stats; (void)searchLength; }
-    MEM_STATIC void ZSTD_statsInit(ZSTD_stats_t* stats) { (void)stats; }
-    MEM_STATIC void ZSTD_statsResetFreqs(ZSTD_stats_t* stats) { (void)stats; }
-    MEM_STATIC void ZSTD_statsUpdatePrices(ZSTD_stats_t* stats, size_t litLength, const BYTE* literals, size_t offset, size_t matchLength) { (void)stats; (void)litLength; (void)literals; (void)offset; (void)matchLength; }
-#endif   /* #if ZSTD_OPT_DEBUG == 3 */
-
 
 typedef struct seqDef_s {
     U32 offset;
@@ -233,7 +188,6 @@ typedef struct {
     U32  cachedPrice;
     U32  cachedLitLength;
     const BYTE* cachedLiterals;
-    ZSTD_stats_t stats;
 } seqStore_t;
 
 const seqStore_t* ZSTD_getSeqStore(const ZSTD_CCtx* ctx);
@@ -244,6 +198,9 @@ int ZSTD_isSkipFrame(ZSTD_DCtx* dctx);
 void* ZSTD_defaultAllocFunction(void* opaque, size_t size);
 void ZSTD_defaultFreeFunction(void* opaque, void* address);
 static const ZSTD_customMem defaultCustomMem = { ZSTD_defaultAllocFunction, ZSTD_defaultFreeFunction, NULL };
+void* ZSTD_malloc(size_t size, ZSTD_customMem customMem);
+void ZSTD_free(void* ptr, ZSTD_customMem customMem);
+
 
 /*======  common function  ======*/
 

lib/compress/.debug/zstd_stats.h

@@ -1,162 +0,0 @@
-/*
-    zstd - standard compression library
-    Header File for static linking only
-    Copyright (C) 2014-2016, Yann Collet.
-
-    BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
-
-    Redistribution and use in source and binary forms, with or without
-    modification, are permitted provided that the following conditions are
-    met:
-    * Redistributions of source code must retain the above copyright
-    notice, this list of conditions and the following disclaimer.
-    * Redistributions in binary form must reproduce the above
-    copyright notice, this list of conditions and the following disclaimer
-    in the documentation and/or other materials provided with the
-    distribution.
-    THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-    "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-    LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-    A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-    OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-    SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-    LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-    DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-    THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-    (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-    OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-    You can contact the author at :
-    - zstd homepage : http://www.zstd.net
-*/
-#ifndef ZSTD_STATS_H
-#define ZSTD_STATS_H
-
-
-#if defined (__cplusplus)
-extern "C" {
-#endif
-
-
-/*-*************************************
-*  Types
-***************************************/
-struct ZSTD_stats_s {
-    U32  priceOffset, priceOffCode, priceMatchLength, priceLiteral, priceLitLength;
-    U32  totalMatchSum, totalLitSum, totalSeqSum, totalRepSum;
-    U32  litSum, matchLengthSum, litLengthSum, offCodeSum;
-    U32  matchLengthFreq[MaxML+1];
-    U32  litLengthFreq[MaxLL+1];
-    U32  litFreq[1<<Litbits];
-    U32  offCodeFreq[MaxOff+1];
-};
-
-
-/*-*************************************
-*  Stats functions
-***************************************/
-MEM_STATIC ZSTD_stats_t* ZSTD_statsAlloc() { return malloc(sizeof(ZSTD_stats_t)); }
-MEM_STATIC void ZSTD_statsFree(struct ZSTD_stats_s* stats) { free(stats); }
-
-MEM_STATIC void ZSTD_statsPrint(ZSTD_stats_t* stats, U32 searchLength)
-{
-    stats->totalMatchSum += stats->totalSeqSum * ((searchLength == 3) ? 3 : 4);
-    printf("\navgMatchL=%.2f avgLitL=%.2f match=%.1f%% lit=%.1f%% reps=%d seq=%d\n", (float)stats->totalMatchSum/stats->totalSeqSum, (float)stats->totalLitSum/stats->totalSeqSum, 100.0*stats->totalMatchSum/(stats->totalMatchSum+stats->totalLitSum), 100.0*stats->totalLitSum/(stats->totalMatchSum+stats->totalLitSum), stats->totalRepSum, stats->totalSeqSum);
-    printf("SumBytes=%d Offset=%d OffCode=%d Match=%d Literal=%d LitLength=%d\n", (stats->priceOffset+stats->priceOffCode+stats->priceMatchLength+stats->priceLiteral+stats->priceLitLength)/8, stats->priceOffset/8, stats->priceOffCode/8, stats->priceMatchLength/8, stats->priceLiteral/8, stats->priceLitLength/8);
-}
-
-
-MEM_STATIC void ZSTD_statsInit(ZSTD_stats_t* stats)
-{
-    stats->totalLitSum = stats->totalMatchSum = stats->totalSeqSum = stats->totalRepSum = 1;
-    stats->priceOffset = stats->priceOffCode = stats->priceMatchLength = stats->priceLiteral = stats->priceLitLength = 0;
-}
-
-
-MEM_STATIC void ZSTD_statsResetFreqs(ZSTD_stats_t* stats)
-{
-    unsigned u;
-
-    stats->litSum = (2<<Litbits);
-    stats->litLengthSum = MaxLL+1;
-    stats->matchLengthSum = MaxML+1;
-    stats->offCodeSum = (MaxOff+1);
-
-    for (u=0; u<=MaxLit; u++)
-        stats->litFreq[u] = 1;
-    for (u=0; u<=MaxLL; u++)
-        stats->litLengthFreq[u] = 1;
-    for (u=0; u<=MaxML; u++)
-        stats->matchLengthFreq[u] = 1;
-    for (u=0; u<=MaxOff; u++)
-        stats->offCodeFreq[u] = 1;
-}
-
-
-MEM_STATIC void ZSTD_statsUpdatePrices(ZSTD_stats_t* stats, size_t litLength, const BYTE* literals, size_t offset, size_t matchLength)
-{
-    U32 u;
-    /* literals */
-    stats->priceLiteral += litLength * ZSTD_highbit(stats->litSum+1);
-    for (u=0; u < litLength; u++)
-        stats->priceLiteral -= ZSTD_highbit(stats->litFreq[literals[u]]+1);
-    stats->litSum += litLength;
-    for (u=0; u < litLength; u++)
-        stats->litFreq[literals[u]]++;
-
-    /* literal Length */
-    {   static const BYTE LL_Code[64] = {  0,  1,  2,  3,  4,  5,  6,  7,
-                                           8,  9, 10, 11, 12, 13, 14, 15,
-                                          16, 16, 17, 17, 18, 18, 19, 19,
-                                          20, 20, 20, 20, 21, 21, 21, 21,
-                                          22, 22, 22, 22, 22, 22, 22, 22,
-                                          23, 23, 23, 23, 23, 23, 23, 23,
-                                          24, 24, 24, 24, 24, 24, 24, 24,
-                                          24, 24, 24, 24, 24, 24, 24, 24 };
-        const BYTE LL_deltaCode = 19;
-        const BYTE llCode = (litLength>63) ? (BYTE)ZSTD_highbit(litLength) + LL_deltaCode : LL_Code[litLength];
-        if (litLength) {
-            stats->priceLitLength += LL_bits[llCode] + ZSTD_highbit(stats->litLengthSum+1) - ZSTD_highbit(stats->litLengthFreq[llCode]+1);
-        } else {
-            stats->priceLitLength += ZSTD_highbit(stats->litLengthSum+1) - ZSTD_highbit(stats->litLengthFreq[0]+1);
-        }
-        stats->litLengthFreq[llCode]++;
-        stats->litLengthSum++;
-    }
-
-    /* match offset */
-    {   BYTE offCode = (BYTE)ZSTD_highbit(offset+1);
-        stats->priceOffCode += ZSTD_highbit(stats->offCodeSum+1) - ZSTD_highbit(stats->offCodeFreq[offCode]+1);
-        stats->priceOffset += offCode;
-        stats->offCodeSum++;
-        stats->offCodeFreq[offCode]++;
-    }
-
-    /* match Length */
-    {   static const BYTE ML_Code[128] = { 0,  1,  2,  3,  4,  5,  6,  7,  8,  9, 10, 11, 12, 13, 14, 15,
-                                          16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31,
-                                          32, 32, 33, 33, 34, 34, 35, 35, 36, 36, 36, 36, 37, 37, 37, 37,
-                                          38, 38, 38, 38, 38, 38, 38, 38, 39, 39, 39, 39, 39, 39, 39, 39,
-                                          40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40,
-                                          41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41,
-                                          42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42,
-                                          42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42 };
-        const BYTE ML_deltaCode = 36;
-        const BYTE mlCode = (matchLength>127) ? (BYTE)ZSTD_highbit(matchLength) + ML_deltaCode : ML_Code[matchLength];
-        stats->priceMatchLength += ML_bits[mlCode] + ZSTD_highbit(stats->matchLengthSum+1) - ZSTD_highbit(stats->matchLengthFreq[mlCode]+1);
-        stats->matchLengthFreq[mlCode]++;
-        stats->matchLengthSum++;
-    }
-
-    if (offset == 0) stats->totalRepSum++;
-    stats->totalSeqSum++;
-    stats->totalMatchSum += matchLength;
-    stats->totalLitSum += litLength;
-}
-
-
-#if defined (__cplusplus)
-}
-#endif
-
-#endif  /* ZSTD_STATIC_H */

lib/compress/huf_compress.c

@@ -89,7 +89,7 @@ unsigned HUF_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxS
 struct HUF_CElt_s {
   U16  val;
   BYTE nbBits;
-};   /* typedef'd to HUF_CElt within huf_static.h */
+};   /* typedef'd to HUF_CElt within "huf.h" */
 
 typedef struct nodeElt_s {
     U32 count;
@@ -124,7 +124,8 @@ size_t HUF_writeCTable (void* dst, size_t maxDstSize,
         if ((size>1) & (size < maxSymbolValue/2)) {   /* FSE compressed */
             op[0] = (BYTE)size;
             return size+1;
-    }   }
+        }
+    }
 
     /* raw values */
     if (maxSymbolValue > (256-128)) return ERROR(GENERIC);   /* should not happen */
@@ -491,7 +492,7 @@ static size_t HUF_compress_internal (
     /* Scan input and build symbol stats */
     {   size_t const largest = FSE_count (count, &maxSymbolValue, (const BYTE*)src, srcSize);
         if (HUF_isError(largest)) return largest;
-        if (largest == srcSize) { *ostart = ((const BYTE*)src)[0]; return 1; }   /* rle */
+        if (largest == srcSize) { *ostart = ((const BYTE*)src)[0]; return 1; }   /* single symbol, rle */
         if (largest <= (srcSize >> 7)+1) return 0;   /* Fast heuristic : not compressible enough */
     }
 

lib/compress/zbuff_compress.c

@@ -1,33 +1,12 @@
-/*
+/**
-    Buffered version of Zstd compression library
+ * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
-    Copyright (C) 2015-2016, Yann Collet.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
 
-    BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
-
-    Redistribution and use in source and binary forms, with or without
-    modification, are permitted provided that the following conditions are
-    met:
-    * Redistributions of source code must retain the above copyright
-    notice, this list of conditions and the following disclaimer.
-    * Redistributions in binary form must reproduce the above
-    copyright notice, this list of conditions and the following disclaimer
-    in the documentation and/or other materials provided with the
-    distribution.
-    THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-    "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-    LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-    A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-    OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-    SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-    LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-    DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-    THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-    (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-    OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-    You can contact the author at :
-    - zstd homepage : http://www.zstd.net/
-*/
 
 
 /* *************************************
@@ -98,7 +77,7 @@ struct ZBUFF_CCtx_s {
     U32    checksum;
     U32    frameEnded;
     ZSTD_customMem customMem;
-};   /* typedef'd tp ZBUFF_CCtx within "zstd_buffered.h" */
+};   /* typedef'd tp ZBUFF_CCtx within "zbuff.h" */
 
 ZBUFF_CCtx* ZBUFF_createCCtx(void)
 {

lib/compress/zstd_compress.c

@@ -1,35 +1,12 @@
-/*
+/**
-    ZSTD HC - High Compression Mode of Zstandard
+ * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
-    Copyright (C) 2015-2016, Yann Collet.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
 
-    BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
-
-    Redistribution and use in source and binary forms, with or without
-    modification, are permitted provided that the following conditions are
-    met:
-
-    * Redistributions of source code must retain the above copyright
-    notice, this list of conditions and the following disclaimer.
-    * Redistributions in binary form must reproduce the above
-    copyright notice, this list of conditions and the following disclaimer
-    in the documentation and/or other materials provided with the
-    distribution.
-
-    THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-    "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-    LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-    A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-    OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-    SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-    LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-    DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-    THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-    (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-    OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-    You can contact the author at :
-       - Zstd source repository : https://www.zstd.net
-*/
 
 
 /*-*******************************************************
@@ -133,13 +110,10 @@ ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem)
 {
     ZSTD_CCtx* cctx;
 
-    if (!customMem.customAlloc && !customMem.customFree)
+    if (!customMem.customAlloc && !customMem.customFree) customMem = defaultCustomMem;
-        customMem = defaultCustomMem;
+    if (!customMem.customAlloc || !customMem.customFree) return NULL;
 
-    if (!customMem.customAlloc || !customMem.customFree)
+    cctx = (ZSTD_CCtx*) ZSTD_malloc(sizeof(ZSTD_CCtx), customMem);
-        return NULL;
-
-    cctx = (ZSTD_CCtx*) customMem.customAlloc(customMem.opaque, sizeof(ZSTD_CCtx));
     if (!cctx) return NULL;
     memset(cctx, 0, sizeof(ZSTD_CCtx));
     memcpy(&(cctx->customMem), &customMem, sizeof(ZSTD_customMem));
@@ -149,12 +123,12 @@ ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem)
 size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx)
 {
     if (cctx==NULL) return 0;   /* support free on NULL */
-    if (cctx->workSpace) cctx->customMem.customFree(cctx->customMem.opaque, cctx->workSpace);
+    ZSTD_free(cctx->workSpace, cctx->customMem);
-    cctx->customMem.customFree(cctx->customMem.opaque, cctx);
+    ZSTD_free(cctx, cctx->customMem);
     return 0;   /* reserved as a potential error code in the future */
 }
 
-size_t ZSTD_sizeofCCtx(const ZSTD_CCtx* cctx)
+size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx)
 {
     return sizeof(*cctx) + cctx->workSpaceSize;
 }
@@ -272,8 +246,8 @@ static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc,
         size_t const neededSpace = tableSpace + (256*sizeof(U32)) /* huffTable */ + tokenSpace
                               + ((params.cParams.strategy == ZSTD_btopt) ? optSpace : 0);
         if (zc->workSpaceSize < neededSpace) {
-            zc->customMem.customFree(zc->customMem.opaque, zc->workSpace);
+            ZSTD_free(zc->workSpace, zc->customMem);
-            zc->workSpace = zc->customMem.customAlloc(zc->customMem.opaque, neededSpace);
+            zc->workSpace = ZSTD_malloc(neededSpace, zc->customMem);
             if (zc->workSpace == NULL) return ERROR(memory_allocation);
             zc->workSpaceSize = neededSpace;
     }   }
@@ -765,8 +739,6 @@ MEM_STATIC void ZSTD_storeSeq(seqStore_t* seqStorePtr, size_t litLength, const v
         printf("Cpos %6u :%5u literals & match %3u bytes at distance %6u \n",
                pos, (U32)litLength, (U32)matchCode+MINMATCH, (U32)offsetCode);
 #endif
-    ZSTD_statsUpdatePrices(&seqStorePtr->stats, litLength, (const BYTE*)literals, offsetCode, matchCode);   /* debug only */
-
     /* copy Literals */
     ZSTD_wildcopy(seqStorePtr->lit, literals, litLength);
     seqStorePtr->lit += litLength;
@@ -1232,9 +1204,20 @@ void ZSTD_compressBlock_doubleFast_generic(ZSTD_CCtx* cctx,
                 offset = (U32)(ip-matchLong);
                 while (((ip>anchor) & (matchLong>lowest)) && (ip[-1] == matchLong[-1])) { ip--; matchLong--; mLength++; } /* catch up */
             } else if ( (matchIndexS > lowestIndex) && (MEM_read32(match) == MEM_read32(ip)) ) {
-                mLength = ZSTD_count(ip+4, match+4, iend) + 4;
+                size_t const h3 = ZSTD_hashPtr(ip+1, hBitsL, 8);
-                offset = (U32)(ip-match);
+                U32 const matchIndex3 = hashLong[h3];
-                while (((ip>anchor) & (match>lowest)) && (ip[-1] == match[-1])) { ip--; match--; mLength++; } /* catch up */
+                const BYTE* match3 = base + matchIndex3;
+                hashLong[h3] = current + 1;
+                if ( (matchIndex3 > lowestIndex) && (MEM_read64(match3) == MEM_read64(ip+1)) ) {
+                    mLength = ZSTD_count(ip+9, match3+8, iend) + 8;
+                    ip++;
+                    offset = (U32)(ip-match3);
+                    while (((ip>anchor) & (match3>lowest)) && (ip[-1] == match3[-1])) { ip--; match3--; mLength++; } /* catch up */
+                } else {
+                    mLength = ZSTD_count(ip+4, match+4, iend) + 4;
+                    offset = (U32)(ip-match);
+                    while (((ip>anchor) & (match>lowest)) && (ip[-1] == match[-1])) { ip--; match--; mLength++; } /* catch up */
+                }
             } else {
                 ip += ((ip-anchor) >> g_searchStrength) + 1;
                 continue;
@@ -1361,16 +1344,32 @@ static void ZSTD_compressBlock_doubleFast_extDict_generic(ZSTD_CCtx* ctx,
                 offset_2 = offset_1;
                 offset_1 = offset;
                 ZSTD_storeSeq(seqStorePtr, ip-anchor, anchor, offset + ZSTD_REP_MOVE, mLength-MINMATCH);
+
             } else if ((matchIndex > lowestIndex) && (MEM_read32(match) == MEM_read32(ip))) {
-                const BYTE* matchEnd = matchIndex < dictLimit ? dictEnd : iend;
+                size_t const h3 = ZSTD_hashPtr(ip+1, hBitsL, 8);
-                const BYTE* lowMatchPtr = matchIndex < dictLimit ? dictStart : lowPrefixPtr;
+                U32 const matchIndex3 = hashLong[h3];
+                const BYTE* const match3Base = matchIndex3 < dictLimit ? dictBase : base;
+                const BYTE* match3 = match3Base + matchIndex3;
                 U32 offset;
-                mLength = ZSTD_count_2segments(ip+4, match+4, iend, matchEnd, lowPrefixPtr) + 4;
+                hashLong[h3] = current + 1;
-                while (((ip>anchor) & (match>lowMatchPtr)) && (ip[-1] == match[-1])) { ip--; match--; mLength++; }   /* catch up */
+                if ( (matchIndex3 > lowestIndex) && (MEM_read64(match3) == MEM_read64(ip+1)) ) {
-                offset = current - matchIndex;
+                    const BYTE* matchEnd = matchIndex3 < dictLimit ? dictEnd : iend;
+                    const BYTE* lowMatchPtr = matchIndex3 < dictLimit ? dictStart : lowPrefixPtr;
+                    mLength = ZSTD_count_2segments(ip+9, match3+8, iend, matchEnd, lowPrefixPtr) + 8;
+                    ip++;
+                    offset = current+1 - matchIndex3;
+                    while (((ip>anchor) & (match3>lowMatchPtr)) && (ip[-1] == match3[-1])) { ip--; match3--; mLength++; } /* catch up */
+                } else {
+                    const BYTE* matchEnd = matchIndex < dictLimit ? dictEnd : iend;
+                    const BYTE* lowMatchPtr = matchIndex < dictLimit ? dictStart : lowPrefixPtr;
+                    mLength = ZSTD_count_2segments(ip+4, match+4, iend, matchEnd, lowPrefixPtr) + 4;
+                    offset = current - matchIndex;
+                    while (((ip>anchor) & (match>lowMatchPtr)) && (ip[-1] == match[-1])) { ip--; match--; mLength++; }   /* catch up */
+                }
                 offset_2 = offset_1;
                 offset_1 = offset;
                 ZSTD_storeSeq(seqStorePtr, ip-anchor, anchor, offset + ZSTD_REP_MOVE, mLength-MINMATCH);
+
             } else {
                 ip += ((ip-anchor) >> g_searchStrength) + 1;
                 continue;
@@ -1945,7 +1944,6 @@ _storeSequence:
     {   size_t const lastLLSize = iend - anchor;
         memcpy(seqStorePtr->lit, anchor, lastLLSize);
         seqStorePtr->lit += lastLLSize;
-        ZSTD_statsUpdatePrices(&seqStorePtr->stats, lastLLSize, anchor, 0, 0);
     }
 }
 
@@ -2235,8 +2233,6 @@ static size_t ZSTD_compress_generic (ZSTD_CCtx* cctx,
     BYTE* const ostart = (BYTE*)dst;
     BYTE* op = ostart;
     U32 const maxDist = 1 << cctx->params.cParams.windowLog;
-    ZSTD_stats_t* stats = &cctx->seqStore.stats;
-    ZSTD_statsInit(stats);   /* debug only */
 
     if (cctx->params.fParams.checksumFlag)
         XXH64_update(&cctx->xxhState, src, srcSize);
@@ -2244,7 +2240,6 @@ static size_t ZSTD_compress_generic (ZSTD_CCtx* cctx,
     while (remaining) {
         U32 const lastBlock = lastFrameChunk & (blockSize >= remaining);
         size_t cSize;
-        ZSTD_statsResetFreqs(stats);   /* debug only */
 
         if (dstCapacity < ZSTD_blockHeaderSize + MIN_CBLOCK_SIZE) return ERROR(dstSize_tooSmall);   /* not enough space to store compressed block */
         if (remaining < blockSize) blockSize = remaining;
@@ -2253,7 +2248,8 @@ static size_t ZSTD_compress_generic (ZSTD_CCtx* cctx,
         if (cctx->lowLimit > (1<<30)) {
             U32 const btplus = (cctx->params.cParams.strategy == ZSTD_btlazy2) | (cctx->params.cParams.strategy == ZSTD_btopt);
             U32 const chainMask = (1 << (cctx->params.cParams.chainLog - btplus)) - 1;
-            U32 const newLowLimit = cctx->lowLimit & chainMask;   /* preserve position % chainSize */
+            U32 const supLog = MAX(cctx->params.cParams.chainLog, 17 /* blockSize */);
+            U32 const newLowLimit = (cctx->lowLimit & chainMask) + (1 << supLog);   /* preserve position % chainSize, ensure current-repcode doesn't underflow */
             U32 const correction = cctx->lowLimit - newLowLimit;
             ZSTD_reduceIndex(cctx, correction);
             cctx->base += correction;
@@ -2293,7 +2289,6 @@ static size_t ZSTD_compress_generic (ZSTD_CCtx* cctx,
     }
 
     if (lastFrameChunk && (op>ostart)) cctx->stage = ZSTDcs_ending;
-    ZSTD_statsPrint(stats, cctx->params.cParams.searchLength);   /* debug only */
     return op-ostart;
 }
 
@@ -2402,7 +2397,6 @@ size_t ZSTD_compressBlock(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const
 {
     size_t const blockSizeMax = ZSTD_getBlockSizeMax(cctx);
     if (srcSize > blockSizeMax) return ERROR(srcSize_wrong);
-    ZSTD_LOG_BLOCK("%p: ZSTD_compressBlock searchLength=%d\n", cctx->base, cctx->params.cParams.searchLength);
     return ZSTD_compressContinue_internal(cctx, dst, dstCapacity, src, srcSize, 0, 0);
 }
 
@@ -2523,8 +2517,9 @@ static size_t ZSTD_compress_insertDictionary(ZSTD_CCtx* zc, const void* dict, si
     zc->dictID = zc->params.fParams.noDictIDFlag ? 0 :  MEM_readLE32((const char*)dict+4);
 
     /* known magic number : dict is parsed for entropy stats and content */
-    {   size_t const eSize = ZSTD_loadDictEntropyStats(zc, (const char*)dict+8 /* skip dictHeader */, dictSize-8) + 8;
+    {   size_t const eSize_8 = ZSTD_loadDictEntropyStats(zc, (const char*)dict+8 /* skip dictHeader */, dictSize-8);
-        if (ZSTD_isError(eSize)) return eSize;
+        size_t const eSize = eSize_8 + 8;
+        if (ZSTD_isError(eSize_8)) return eSize_8;
         return ZSTD_loadDictionaryContent(zc, (const char*)dict+eSize, dictSize-eSize);
     }
 }
@@ -2560,14 +2555,12 @@ size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx,
 size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel)
 {
     ZSTD_parameters const params = ZSTD_getParams(compressionLevel, 0, dictSize);
-    ZSTD_LOG_BLOCK("%p: ZSTD_compressBegin_usingDict compressionLevel=%d\n", cctx->base, compressionLevel);
     return ZSTD_compressBegin_internal(cctx, dict, dictSize, params, 0);
 }
 
 
 size_t ZSTD_compressBegin(ZSTD_CCtx* zc, int compressionLevel)
 {
-    ZSTD_LOG_BLOCK("%p: ZSTD_compressBegin compressionLevel=%d\n", zc->base, compressionLevel);
     return ZSTD_compressBegin_usingDict(zc, NULL, 0, compressionLevel);
 }
 
@@ -2581,7 +2574,7 @@ static size_t ZSTD_writeEpilogue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity)
     BYTE* op = ostart;
     size_t fhSize = 0;
 
-    if (cctx->stage == ZSTDcs_created) return ERROR(stage_wrong);  /*< not even init ! */
+    if (cctx->stage == ZSTDcs_created) return ERROR(stage_wrong);  /* init missing */
 
     /* special case : empty frame */
     if (cctx->stage == ZSTDcs_init) {
@@ -2653,13 +2646,11 @@ size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx, void* dst, size_t dstCapacity, co
 {
     ZSTD_parameters params = ZSTD_getParams(compressionLevel, srcSize, dictSize);
     params.fParams.contentSizeFlag = 1;
-    ZSTD_LOG_BLOCK("%p: ZSTD_compress_usingDict srcSize=%d dictSize=%d compressionLevel=%d\n", ctx->base, (int)srcSize, (int)dictSize, compressionLevel);
     return ZSTD_compress_internal(ctx, dst, dstCapacity, src, srcSize, dict, dictSize, params);
 }
 
 size_t ZSTD_compressCCtx (ZSTD_CCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, int compressionLevel)
 {
-    ZSTD_LOG_BLOCK("%p: ZSTD_compressCCtx srcSize=%d compressionLevel=%d\n", ctx->base, (int)srcSize, compressionLevel);
     return ZSTD_compress_usingDict(ctx, dst, dstCapacity, src, srcSize, NULL, 0, compressionLevel);
 }
 
@@ -2670,7 +2661,7 @@ size_t ZSTD_compress(void* dst, size_t dstCapacity, const void* src, size_t srcS
     memset(&ctxBody, 0, sizeof(ctxBody));
     memcpy(&ctxBody.customMem, &defaultCustomMem, sizeof(ZSTD_customMem));
     result = ZSTD_compressCCtx(&ctxBody, dst, dstCapacity, src, srcSize, compressionLevel);
-    ctxBody.customMem.customFree(ctxBody.customMem.opaque, ctxBody.workSpace);   /* can't free ctxBody, since it's on stack; just free heap content */
+    ZSTD_free(ctxBody.workSpace, defaultCustomMem);  /* can't free ctxBody itself, as it's on stack; free only heap content */
     return result;
 }
 
@@ -2681,33 +2672,30 @@ struct ZSTD_CDict_s {
     void* dictContent;
     size_t dictContentSize;
     ZSTD_CCtx* refContext;
-};  /* typedef'd tp ZSTD_CDict within zstd.h */
+};  /* typedef'd tp ZSTD_CDict within "zstd.h" */
 
 ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictSize, ZSTD_parameters params, ZSTD_customMem customMem)
 {
-    if (!customMem.customAlloc && !customMem.customFree)
+    if (!customMem.customAlloc && !customMem.customFree) customMem = defaultCustomMem;
-        customMem = defaultCustomMem;
+    if (!customMem.customAlloc || !customMem.customFree) return NULL;
 
-    if (!customMem.customAlloc || !customMem.customFree)  /* can't have 1/2 custom alloc/free as NULL */
+    {   ZSTD_CDict* const cdict = (ZSTD_CDict*) ZSTD_malloc(sizeof(ZSTD_CDict), customMem);
-        return NULL;
+        void* const dictContent = ZSTD_malloc(dictSize, customMem);
-
-    {   ZSTD_CDict* const cdict = (ZSTD_CDict*) customMem.customAlloc(customMem.opaque, sizeof(*cdict));
-        void* const dictContent = customMem.customAlloc(customMem.opaque, dictSize);
         ZSTD_CCtx* const cctx = ZSTD_createCCtx_advanced(customMem);
 
         if (!dictContent || !cdict || !cctx) {
-            customMem.customFree(customMem.opaque, dictContent);
+            ZSTD_free(dictContent, customMem);
-            customMem.customFree(customMem.opaque, cdict);
+            ZSTD_free(cdict, customMem);
-            customMem.customFree(customMem.opaque, cctx);
+            ZSTD_free(cctx, customMem);
             return NULL;
         }
 
         memcpy(dictContent, dict, dictSize);
         {   size_t const errorCode = ZSTD_compressBegin_advanced(cctx, dictContent, dictSize, params, 0);
             if (ZSTD_isError(errorCode)) {
-                customMem.customFree(customMem.opaque, dictContent);
+                ZSTD_free(dictContent, customMem);
-                customMem.customFree(customMem.opaque, cdict);
+                ZSTD_free(cdict, customMem);
-                customMem.customFree(customMem.opaque, cctx);
+                ZSTD_free(cctx, customMem);
                 return NULL;
         }   }
 
@@ -2728,12 +2716,13 @@ ZSTD_CDict* ZSTD_createCDict(const void* dict, size_t dictSize, int compressionL
 
 size_t ZSTD_freeCDict(ZSTD_CDict* cdict)
 {
-    ZSTD_freeFunction const cFree = cdict->refContext->customMem.customFree;
+    if (cdict==NULL) return 0;   /* support free on NULL */
-    void* const opaque = cdict->refContext->customMem.opaque;
+    {   ZSTD_customMem cMem = cdict->refContext->customMem;
-    ZSTD_freeCCtx(cdict->refContext);
+        ZSTD_freeCCtx(cdict->refContext);
-    cFree(opaque, cdict->dictContent);
+        ZSTD_free(cdict->dictContent, cMem);
-    cFree(opaque, cdict);
+        ZSTD_free(cdict, cMem);
-    return 0;
+        return 0;
+    }
 }
 
 /*! ZSTD_compress_usingCDict() :
@@ -2791,13 +2780,10 @@ ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem)
 {
     ZSTD_CStream* zcs;
 
-    if (!customMem.customAlloc && !customMem.customFree)
+    if (!customMem.customAlloc && !customMem.customFree) customMem = defaultCustomMem;
-        customMem = defaultCustomMem;
+    if (!customMem.customAlloc || !customMem.customFree) return NULL;
 
-    if (!customMem.customAlloc || !customMem.customFree)
+    zcs = (ZSTD_CStream*)ZSTD_malloc(sizeof(ZSTD_CStream), customMem);
-        return NULL;
-
-    zcs = (ZSTD_CStream*)customMem.customAlloc(customMem.opaque, sizeof(ZSTD_CStream));
     if (zcs==NULL) return NULL;
     memset(zcs, 0, sizeof(ZSTD_CStream));
     memcpy(&zcs->customMem, &customMem, sizeof(ZSTD_customMem));
@@ -2809,11 +2795,13 @@ ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem)
 size_t ZSTD_freeCStream(ZSTD_CStream* zcs)
 {
     if (zcs==NULL) return 0;   /* support free on NULL */
-    ZSTD_freeCCtx(zcs->zc);
+    {   ZSTD_customMem const cMem = zcs->customMem;
-    if (zcs->inBuff) zcs->customMem.customFree(zcs->customMem.opaque, zcs->inBuff);
+        ZSTD_freeCCtx(zcs->zc);
-    if (zcs->outBuff) zcs->customMem.customFree(zcs->customMem.opaque, zcs->outBuff);
+        ZSTD_free(zcs->inBuff, cMem);
-    zcs->customMem.customFree(zcs->customMem.opaque, zcs);
+        ZSTD_free(zcs->outBuff, cMem);
-    return 0;
+        ZSTD_free(zcs, cMem);
+        return 0;
+    }
 }
 
 
@@ -2830,16 +2818,16 @@ size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs,
     {   size_t const neededInBuffSize = (size_t)1 << params.cParams.windowLog;
         if (zcs->inBuffSize < neededInBuffSize) {
             zcs->inBuffSize = neededInBuffSize;
-            zcs->customMem.customFree(zcs->customMem.opaque, zcs->inBuff);   /* should not be necessary */
+            ZSTD_free(zcs->inBuff, zcs->customMem);  /* should not be necessary */
-            zcs->inBuff = (char*)zcs->customMem.customAlloc(zcs->customMem.opaque, neededInBuffSize);
+            zcs->inBuff = (char*) ZSTD_malloc(neededInBuffSize, zcs->customMem);
             if (zcs->inBuff == NULL) return ERROR(memory_allocation);
         }
         zcs->blockSize = MIN(ZSTD_BLOCKSIZE_ABSOLUTEMAX, neededInBuffSize);
     }
     if (zcs->outBuffSize < ZSTD_compressBound(zcs->blockSize)+1) {
         zcs->outBuffSize = ZSTD_compressBound(zcs->blockSize)+1;
-        zcs->customMem.customFree(zcs->customMem.opaque, zcs->outBuff);   /* should not be necessary */
+        ZSTD_free(zcs->outBuff, zcs->customMem);   /* should not be necessary */
-        zcs->outBuff = (char*)zcs->customMem.customAlloc(zcs->customMem.opaque, zcs->outBuffSize);
+        zcs->outBuff = (char*) ZSTD_malloc(zcs->outBuffSize, zcs->customMem);
         if (zcs->outBuff == NULL) return ERROR(memory_allocation);
     }
 
@@ -2867,6 +2855,10 @@ size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel)
     return ZSTD_initCStream_usingDict(zcs, NULL, 0, compressionLevel);
 }
 
+size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs)
+{
+    return sizeof(zcs) + ZSTD_sizeof_CCtx(zcs->zc) + zcs->outBuffSize + zcs->inBuffSize;
+}
 
 /*======   Compression   ======*/
 
@@ -3036,15 +3028,15 @@ int ZSTD_maxCLevel(void) { return ZSTD_MAX_CLEVEL; }
 static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEVEL+1] = {
 {   /* "default" */
     /* W,  C,  H,  S,  L, TL, strat */
-    { 18, 12, 12,  1,  7, 16, ZSTD_fast    },  /* level  0 - not used */
+    { 18, 12, 12,  1,  7, 16, ZSTD_fast    },  /* level  0 - never used */
     { 19, 13, 14,  1,  7, 16, ZSTD_fast    },  /* level  1 */
     { 19, 15, 16,  1,  6, 16, ZSTD_fast    },  /* level  2 */
-    { 20, 16, 18,  1,  5, 16, ZSTD_dfast   },  /* level  3 */
+    { 20, 16, 17,  1,  5, 16, ZSTD_dfast   },  /* level  3.*/
-    { 20, 13, 17,  2,  5, 16, ZSTD_greedy  },  /* level  4.*/
+    { 20, 18, 18,  1,  5, 16, ZSTD_dfast   },  /* level  4.*/
     { 20, 15, 18,  3,  5, 16, ZSTD_greedy  },  /* level  5 */
     { 21, 16, 19,  2,  5, 16, ZSTD_lazy    },  /* level  6 */
     { 21, 17, 20,  3,  5, 16, ZSTD_lazy    },  /* level  7 */
-    { 21, 18, 20,  3,  5, 16, ZSTD_lazy2   },  /* level  8.*/
+    { 21, 18, 20,  3,  5, 16, ZSTD_lazy2   },  /* level  8 */
     { 21, 20, 20,  3,  5, 16, ZSTD_lazy2   },  /* level  9 */
     { 21, 19, 21,  4,  5, 16, ZSTD_lazy2   },  /* level 10 */
     { 22, 20, 22,  4,  5, 16, ZSTD_lazy2   },  /* level 11 */
@@ -3053,33 +3045,33 @@ static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEV
     { 22, 21, 22,  6,  5, 16, ZSTD_lazy2   },  /* level 14 */
     { 22, 21, 21,  5,  5, 16, ZSTD_btlazy2 },  /* level 15 */
     { 23, 22, 22,  5,  5, 16, ZSTD_btlazy2 },  /* level 16 */
-    { 23, 23, 22,  5,  5, 16, ZSTD_btlazy2 },  /* level 17.*/
+    { 23, 21, 22,  4,  5, 24, ZSTD_btopt   },  /* level 17 */
-    { 23, 23, 22,  6,  5, 24, ZSTD_btopt   },  /* level 18.*/
+    { 23, 23, 22,  6,  5, 32, ZSTD_btopt   },  /* level 18 */
-    { 23, 23, 22,  6,  3, 48, ZSTD_btopt   },  /* level 19.*/
+    { 23, 23, 22,  6,  3, 48, ZSTD_btopt   },  /* level 19 */
-    { 25, 26, 23,  7,  3, 64, ZSTD_btopt   },  /* level 20.*/
+    { 25, 25, 23,  7,  3, 64, ZSTD_btopt   },  /* level 20 */
-    { 26, 26, 23,  7,  3,256, ZSTD_btopt   },  /* level 21.*/
+    { 26, 26, 23,  7,  3,256, ZSTD_btopt   },  /* level 21 */
-    { 27, 27, 25,  9,  3,512, ZSTD_btopt   },  /* level 22.*/
+    { 27, 27, 25,  9,  3,512, ZSTD_btopt   },  /* level 22 */
 },
 {   /* for srcSize <= 256 KB */
     /* W,  C,  H,  S,  L,  T, strat */
-    { 18, 12, 12,  1,  7,  4, ZSTD_fast    },  /* level  0 - not used */
+    {  0,  0,  0,  0,  0,  0, ZSTD_fast    },  /* level  0 - not used */
-    { 18, 13, 14,  1,  6,  4, ZSTD_fast    },  /* level  1 */
+    { 18, 13, 14,  1,  6,  8, ZSTD_fast    },  /* level  1 */
-    { 18, 15, 17,  1,  5,  4, ZSTD_fast    },  /* level  2 */
+    { 18, 14, 13,  1,  5,  8, ZSTD_dfast   },  /* level  2 */
-    { 18, 13, 15,  1,  5,  4, ZSTD_greedy  },  /* level  3.*/
+    { 18, 16, 15,  1,  5,  8, ZSTD_dfast   },  /* level  3 */
-    { 18, 15, 17,  1,  5,  4, ZSTD_greedy  },  /* level  4.*/
+    { 18, 15, 17,  1,  5,  8, ZSTD_greedy  },  /* level  4.*/
-    { 18, 16, 17,  4,  5,  4, ZSTD_greedy  },  /* level  5 */
+    { 18, 16, 17,  4,  5,  8, ZSTD_greedy  },  /* level  5.*/
-    { 18, 17, 17,  5,  5,  4, ZSTD_greedy  },  /* level  6 */
+    { 18, 16, 17,  3,  5,  8, ZSTD_lazy    },  /* level  6.*/
-    { 18, 17, 17,  4,  4,  4, ZSTD_lazy    },  /* level  7 */
+    { 18, 17, 17,  4,  4,  8, ZSTD_lazy    },  /* level  7 */
-    { 18, 17, 17,  4,  4,  4, ZSTD_lazy2   },  /* level  8 */
+    { 18, 17, 17,  4,  4,  8, ZSTD_lazy2   },  /* level  8 */
-    { 18, 17, 17,  5,  4,  4, ZSTD_lazy2   },  /* level  9 */
+    { 18, 17, 17,  5,  4,  8, ZSTD_lazy2   },  /* level  9 */
-    { 18, 17, 17,  6,  4,  4, ZSTD_lazy2   },  /* level 10 */
+    { 18, 17, 17,  6,  4,  8, ZSTD_lazy2   },  /* level 10 */
-    { 18, 18, 17,  6,  4,  4, ZSTD_lazy2   },  /* level 11.*/
+    { 18, 18, 17,  6,  4,  8, ZSTD_lazy2   },  /* level 11.*/
-    { 18, 18, 17,  7,  4,  4, ZSTD_lazy2   },  /* level 12.*/
+    { 18, 18, 17,  7,  4,  8, ZSTD_lazy2   },  /* level 12.*/
-    { 18, 19, 17,  7,  4,  4, ZSTD_btlazy2 },  /* level 13 */
+    { 18, 19, 17,  6,  4,  8, ZSTD_btlazy2 },  /* level 13 */
     { 18, 18, 18,  4,  4, 16, ZSTD_btopt   },  /* level 14.*/
-    { 18, 18, 18,  8,  4, 24, ZSTD_btopt   },  /* level 15.*/
+    { 18, 18, 18,  4,  3, 16, ZSTD_btopt   },  /* level 15.*/
-    { 18, 19, 18,  8,  3, 48, ZSTD_btopt   },  /* level 16.*/
+    { 18, 19, 18,  6,  3, 32, ZSTD_btopt   },  /* level 16.*/
-    { 18, 19, 18,  8,  3, 96, ZSTD_btopt   },  /* level 17.*/
+    { 18, 19, 18,  8,  3, 64, ZSTD_btopt   },  /* level 17.*/
     { 18, 19, 18,  9,  3,128, ZSTD_btopt   },  /* level 18.*/
     { 18, 19, 18, 10,  3,256, ZSTD_btopt   },  /* level 19.*/
     { 18, 19, 18, 11,  3,512, ZSTD_btopt   },  /* level 20.*/

lib/compress/zstd_opt.h

@@ -1,35 +1,12 @@
-/*
+/**
-    ZSTD Optimal mode
+ * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
-    Copyright (C) 2016, Przemyslaw Skibinski, Yann Collet.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
 
-    BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
-
-    Redistribution and use in source and binary forms, with or without
-    modification, are permitted provided that the following conditions are
-    met:
-
-    * Redistributions of source code must retain the above copyright
-    notice, this list of conditions and the following disclaimer.
-    * Redistributions in binary form must reproduce the above
-    copyright notice, this list of conditions and the following disclaimer
-    in the documentation and/or other materials provided with the
-    distribution.
-
-    THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-    "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-    LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-    A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-    OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-    SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-    LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-    DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-    THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-    (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-    OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-    You can contact the author at :
-       - Zstd source repository : https://www.zstd.net
-*/
 
 /* Note : this file is intended to be included within zstd_compress.c */
 
@@ -199,7 +176,6 @@ MEM_STATIC void ZSTD_updatePrice(seqStore_t* seqStorePtr, U32 litLength, const B
         opt[pos].off = offset_;                        \
         opt[pos].litlen = litlen_;                     \
         opt[pos].price = price_;                       \
-        ZSTD_LOG_PARSER("%d: SET price[%d/%d]=%d litlen=%d len=%d off=%d\n", (int)(inr-base), (int)pos, (int)last_pos, opt[pos].price, opt[pos].litlen, opt[pos].mlen, opt[pos].off); \
     }
 
 
@@ -295,25 +271,11 @@ static U32 ZSTD_insertBtAndGetAllMatches (
         if ((!extDict) || (matchIndex+matchLength >= dictLimit)) {
             match = base + matchIndex;
             if (match[matchLength] == ip[matchLength]) {
-#if ZSTD_OPT_DEBUG >= 5
-            size_t ml;
-            if (matchIndex < dictLimit)
-                ml = ZSTD_count_2segments(ip, dictBase + matchIndex, iLimit, dictEnd, prefixStart);
-            else
-                ml = ZSTD_count(ip, match, ip+matchLength);
-            if (ml < matchLength)
-                printf("%d: ERROR_NOEXT: offset=%d matchLength=%d matchIndex=%d dictLimit=%d ml=%d\n", current, (int)(current - matchIndex), (int)matchLength, (int)matchIndex, (int)dictLimit, (int)ml), exit(0);
-#endif
                 matchLength += ZSTD_count(ip+matchLength+1, match+matchLength+1, iLimit) +1;
             }
         } else {
             match = dictBase + matchIndex;
-#if ZSTD_OPT_DEBUG >= 5
-            if (memcmp(match, ip, matchLength) != 0)
-                 printf("%d: ERROR_EXT: matchLength=%d ZSTD_count=%d\n", current, (int)matchLength, (int)ZSTD_count_2segments(ip+matchLength, match+matchLength, iLimit, dictEnd, prefixStart)), exit(0);
-#endif
             matchLength += ZSTD_count_2segments(ip+matchLength, match+matchLength, iLimit, dictEnd, prefixStart);
-            ZSTD_LOG_PARSER("%d: ZSTD_INSERTBTANDGETALLMATCHES=%d offset=%d dictBase=%p dictEnd=%p prefixStart=%p ip=%p match=%p\n", (int)current, (int)matchLength, (int)(current - matchIndex), dictBase, dictEnd, prefixStart, ip, match);
             if (matchIndex+matchLength >= dictLimit)
                 match = base + matchIndex;   /* to prepare for next usage of match[matchLength] */
         }
@@ -441,8 +403,6 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx,
     { U32 i; for (i=0; i<ZSTD_REP_NUM; i++) rep[i]=ctx->rep[i]; }
     inr = ip;
 
-    ZSTD_LOG_BLOCK("%d: COMPBLOCK_OPT_GENERIC srcSz=%d maxSrch=%d mls=%d sufLen=%d\n", (int)(ip-base), (int)srcSize, maxSearches, mls, sufficient_len);
-
     /* Match Loop */
     while (ip < ilimit) {
         U32 cur, match_num, last_pos, litlen, price;
@@ -458,7 +418,6 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx,
                 if ( (repCur > 0) && (repCur < (S32)(ip-prefixStart))
                     && (MEM_readMINMATCH(ip, minMatch) == MEM_readMINMATCH(ip - repCur, minMatch))) {
                     mlen = (U32)ZSTD_count(ip+minMatch, ip+minMatch-repCur, iend) + minMatch;
-                    ZSTD_LOG_PARSER("%d: start try REP rep[%d]=%d mlen=%d\n", (int)(ip-base), i, (int)rep[i], (int)mlen);
                     if (mlen > sufficient_len || mlen >= ZSTD_OPT_NUM) {
                         best_mlen = mlen; best_off = i; cur = 0; last_pos = 1;
                         goto _storeSequence;
@@ -474,7 +433,6 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx,
 
         match_num = ZSTD_BtGetAllMatches_selectMLS(ctx, ip, iend, maxSearches, mls, matches, minMatch);
 
-        ZSTD_LOG_PARSER("%d: match_num=%d last_pos=%d\n", (int)(ip-base), match_num, last_pos);
         if (!last_pos && !match_num) { ip++; continue; }
 
         if (match_num && (matches[match_num-1].len > sufficient_len || matches[match_num-1].len >= ZSTD_OPT_NUM)) {
@@ -490,7 +448,6 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx,
         for (u = 0; u < match_num; u++) {
             mlen = (u>0) ? matches[u-1].len+1 : best_mlen;
             best_mlen = matches[u].len;
-            ZSTD_LOG_PARSER("%d: start Found mlen=%d off=%d best_mlen=%d last_pos=%d\n", (int)(ip-base), matches[u].len, matches[u].off, (int)best_mlen, (int)last_pos);
             while (mlen <= best_mlen) {
                 price = ZSTD_getPrice(seqStorePtr, litlen, anchor, matches[u].off-1, mlen - MINMATCH);
                 if (mlen > last_pos || price < opt[mlen].price)
@@ -520,7 +477,7 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx,
                 price = opt[cur - 1].price + ZSTD_getLiteralPrice(seqStorePtr, litlen, inr-1);
            }
 
-           if (cur > last_pos || price <= opt[cur].price) // || ((price == opt[cur].price) && (opt[cur-1].mlen == 1) && (cur != litlen)))
+           if (cur > last_pos || price <= opt[cur].price)
                 SET_PRICE(cur, 1, 0, litlen, price);
 
            if (cur == last_pos) break;
@@ -529,20 +486,16 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx,
                continue;
 
            mlen = opt[cur].mlen;
-            if (opt[cur].off > ZSTD_REP_MOVE_OPT) {
+           if (opt[cur].off > ZSTD_REP_MOVE_OPT) {
                 opt[cur].rep[2] = opt[cur-mlen].rep[1];
                 opt[cur].rep[1] = opt[cur-mlen].rep[0];
                 opt[cur].rep[0] = opt[cur].off - ZSTD_REP_MOVE_OPT;
-                ZSTD_LOG_ENCODE("%d: COPYREP_OFF cur=%d mlen=%d rep[0]=%d rep[1]=%d\n", (int)(inr-base), cur, mlen, opt[cur].rep[0], opt[cur].rep[1]);
            } else {
                 opt[cur].rep[2] = (opt[cur].off > 1) ? opt[cur-mlen].rep[1] : opt[cur-mlen].rep[2];
                 opt[cur].rep[1] = (opt[cur].off > 0) ? opt[cur-mlen].rep[0] : opt[cur-mlen].rep[1];
                 opt[cur].rep[0] = ((opt[cur].off==ZSTD_REP_MOVE_OPT) && (mlen != 1)) ? (opt[cur-mlen].rep[0] - 1) : (opt[cur-mlen].rep[opt[cur].off]);
-                ZSTD_LOG_ENCODE("%d: COPYREP_NOR cur=%d mlen=%d rep[0]=%d rep[1]=%d\n", (int)(inr-base), cur, mlen, opt[cur].rep[0], opt[cur].rep[1]);
            }
 
-           ZSTD_LOG_PARSER("%d: CURRENT_NoExt price[%d/%d]=%d off=%d mlen=%d litlen=%d rep[0]=%d rep[1]=%d\n", (int)(inr-base), cur, last_pos, opt[cur].price, opt[cur].off, opt[cur].mlen, opt[cur].litlen, opt[cur].rep[0], opt[cur].rep[1]);
-
            best_mlen = minMatch;
             {   U32 i, last_i = ZSTD_REP_CHECK + (mlen != 1);
                 for (i=(opt[cur].mlen != 1); i<last_i; i++) {  /* check rep */
@@ -550,15 +503,12 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx,
                     if ( (repCur > 0) && (repCur < (S32)(inr-prefixStart))
                        && (MEM_readMINMATCH(inr, minMatch) == MEM_readMINMATCH(inr - repCur, minMatch))) {
                        mlen = (U32)ZSTD_count(inr+minMatch, inr+minMatch - repCur, iend) + minMatch;
-                       ZSTD_LOG_PARSER("%d: Found REP %d/%d mlen=%d off=%d rep=%d opt[%d].off=%d\n", (int)(inr-base), i, ZSTD_REP_NUM, mlen, i, opt[cur].rep[i], cur, opt[cur].off);
 
                        if (mlen > sufficient_len || cur + mlen >= ZSTD_OPT_NUM) {
                             best_mlen = mlen; best_off = i; last_pos = cur + 1;
-                            ZSTD_LOG_PARSER("%d: REP sufficient_len=%d best_mlen=%d best_off=%d last_pos=%d\n", (int)(inr-base), sufficient_len, best_mlen, best_off, last_pos);
                             goto _storeSequence;
                        }
 
-                       //best_off = ((i<=1) & (opt[cur].mlen != 1)) ? 1-i : i;
                        best_off = i - (opt[cur].mlen != 1);
 
                        if (opt[cur].mlen == 1) {
@@ -573,7 +523,6 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx,
                         }
 
                         if (mlen > best_mlen) best_mlen = mlen;
-                        ZSTD_LOG_PARSER("%d: Found REP mlen=%d off=%d price=%d litlen=%d\n", (int)(inr-base), mlen, best_off, price, litlen);
 
                         do {
                             if (cur + mlen > last_pos || price <= opt[cur + mlen].price)
@@ -583,7 +532,6 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx,
             }   }   }
 
             match_num = ZSTD_BtGetAllMatches_selectMLS(ctx, inr, iend, maxSearches, mls, matches, best_mlen);
-            ZSTD_LOG_PARSER("%d: ZSTD_GetAllMatches match_num=%d\n", (int)(inr-base), match_num);
 
             if (match_num > 0 && (matches[match_num-1].len > sufficient_len || cur + matches[match_num-1].len >= ZSTD_OPT_NUM)) {
                 best_mlen = matches[match_num-1].len;
@@ -597,7 +545,6 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx,
                 mlen = (u>0) ? matches[u-1].len+1 : best_mlen;
                 best_mlen = matches[u].len;
 
-              //  ZSTD_LOG_PARSER("%d: Found1 cur=%d mlen=%d off=%d best_mlen=%d last_pos=%d\n", (int)(inr-base), cur, matches[u].len, matches[u].off, best_mlen, last_pos);
                 while (mlen <= best_mlen) {
                     if (opt[cur].mlen == 1) {
                         litlen = opt[cur].litlen;
@@ -610,12 +557,11 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx,
                         price = opt[cur].price + ZSTD_getPrice(seqStorePtr, 0, NULL, matches[u].off-1, mlen - MINMATCH);
                     }
 
-                  //  ZSTD_LOG_PARSER("%d: Found2 mlen=%d best_mlen=%d off=%d price=%d litlen=%d\n", (int)(inr-base), mlen, best_mlen, matches[u].off, price, litlen);
                     if (cur + mlen > last_pos || (price < opt[cur + mlen].price))
                         SET_PRICE(cur + mlen, mlen, matches[u].off, litlen, price);
 
                     mlen++;
-        }   }   }   //  for (cur = 1; cur <= last_pos; cur++)
+        }   }   }
 
         best_mlen = opt[last_pos].mlen;
         best_off = opt[last_pos].off;
@@ -623,10 +569,6 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx,
 
         /* store sequence */
 _storeSequence:   /* cur, last_pos, best_mlen, best_off have to be set */
-        for (u = 1; u <= last_pos; u++)
-            ZSTD_LOG_PARSER("%d: price[%d/%d]=%d off=%d mlen=%d litlen=%d rep[0]=%d rep[1]=%d\n", (int)(ip-base+u), u, last_pos, opt[u].price, opt[u].off, opt[u].mlen, opt[u].litlen, opt[u].rep[0], opt[u].rep[1]);
-        ZSTD_LOG_PARSER("%d: cur=%d/%d best_mlen=%d best_off=%d rep[0]=%d\n", (int)(ip-base+cur), (int)cur, (int)last_pos, (int)best_mlen, (int)best_off, opt[cur].rep[0]);
-
         opt[0].mlen = 1;
 
         while (1) {
@@ -641,18 +583,15 @@ _storeSequence:   /* cur, last_pos, best_mlen, best_off have to be set */
         }
 
         for (u = 0; u <= last_pos;) {
-            ZSTD_LOG_PARSER("%d: price2[%d/%d]=%d off=%d mlen=%d litlen=%d rep[0]=%d rep[1]=%d\n", (int)(ip-base+u), u, last_pos, opt[u].price, opt[u].off, opt[u].mlen, opt[u].litlen, opt[u].rep[0], opt[u].rep[1]);
             u += opt[u].mlen;
         }
 
         for (cur=0; cur < last_pos; ) {
-            ZSTD_LOG_PARSER("%d: price3[%d/%d]=%d off=%d mlen=%d litlen=%d rep[0]=%d rep[1]=%d\n", (int)(ip-base+cur), cur, last_pos, opt[cur].price, opt[cur].off, opt[cur].mlen, opt[cur].litlen, opt[cur].rep[0], opt[cur].rep[1]);
             mlen = opt[cur].mlen;
             if (mlen == 1) { ip++; cur++; continue; }
             offset = opt[cur].off;
             cur += mlen;
             litLength = (U32)(ip - anchor);
-           // ZSTD_LOG_ENCODE("%d/%d: ENCODE literals=%d mlen=%d off=%d rep[0]=%d rep[1]=%d\n", (int)(ip-base), (int)(iend-base), (int)(litLength), (int)mlen, (int)(offset), (int)rep[0], (int)rep[1]);
 
             if (offset > ZSTD_REP_MOVE_OPT) {
                 rep[2] = rep[1];
@@ -666,26 +605,9 @@ _storeSequence:   /* cur, last_pos, best_mlen, best_off have to be set */
                     rep[1] = rep[0];
                     rep[0] = best_off;
                 }
-                if ((litLength==0) & (offset==0)) { ZSTD_LOG_ENCODE("ERROR (litLength==0) & (offset==0)\n"); };
                 if (litLength==0) offset--;
             }
 
-            ZSTD_LOG_ENCODE("%d/%d: ENCODE literals=%d mlen=%d off=%d rep[0]=%d rep[1]=%d\n", (int)(ip-base), (int)(iend-base), (int)(litLength), (int)mlen, (int)(offset), (int)rep[0], (int)rep[1]);
-
-#if ZSTD_OPT_DEBUG >= 5
-            U32 ml2;
-            if (offset+1 > ZSTD_REP_MOVE_OPT)
-                ml2 = (U32)ZSTD_count(ip, ip-(offset+1-ZSTD_REP_MOVE_OPT), iend);
-            else
-                ml2 = (U32)ZSTD_count(ip, ip-rep[0], iend);
-            if ((offset >= 8) && (ml2 < mlen || ml2 < minMatch)) {
-                printf("%d: ERROR_NoExt iend=%d mlen=%d offset=%d ml2=%d\n", (int)(ip - base), (int)(iend - ip), (int)mlen, (int)offset, (int)ml2); exit(0); }
-            if (ip < anchor) {
-                printf("%d: ERROR_NoExt ip < anchor iend=%d mlen=%d offset=%d\n", (int)(ip - base), (int)(iend - ip), (int)mlen, (int)offset); exit(0); }
-            if (ip + mlen > iend) {
-                printf("%d: ERROR_NoExt ip + mlen >= iend iend=%d mlen=%d offset=%d\n", (int)(ip - base), (int)(iend - ip), (int)mlen, (int)offset); exit(0); }
-#endif
-
             ZSTD_updatePrice(seqStorePtr, litLength, anchor, offset, mlen-MINMATCH);
             ZSTD_storeSeq(seqStorePtr, litLength, anchor, offset, mlen-MINMATCH);
             anchor = ip = ip + mlen;
@@ -696,7 +618,6 @@ _storeSequence:   /* cur, last_pos, best_mlen, best_off have to be set */
 
     /* Last Literals */
     {   size_t const lastLLSize = iend - anchor;
-        ZSTD_LOG_ENCODE("%d: lastLLSize literals=%u\n", (int)(ip-base), (U32)lastLLSize);
         memcpy(seqStorePtr->lit, anchor, lastLLSize);
         seqStorePtr->lit += lastLLSize;
     }
@@ -738,8 +659,6 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx,
     ip += (ip==prefixStart);
     inr = ip;
 
-    ZSTD_LOG_BLOCK("%d: COMPBLOCK_OPT_EXTDICT srcSz=%d maxSrch=%d mls=%d sufLen=%d\n", (int)(ip-base), (int)srcSize, maxSearches, mls, sufficient_len);
-
     /* Match Loop */
     while (ip < ilimit) {
         U32 cur, match_num, last_pos, litlen, price;
@@ -764,7 +683,6 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx,
                     const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend;
                     mlen = (U32)ZSTD_count_2segments(ip+minMatch, repMatch+minMatch, iend, repEnd, prefixStart) + minMatch;
 
-                    ZSTD_LOG_PARSER("%d: start try REP rep[%d]=%d mlen=%d\n", (int)(ip-base), i, (int)rep[i], (int)mlen);
                     if (mlen > sufficient_len || mlen >= ZSTD_OPT_NUM) {
                         best_mlen = mlen; best_off = i; cur = 0; last_pos = 1;
                         goto _storeSequence;
@@ -782,7 +700,6 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx,
 
         match_num = ZSTD_BtGetAllMatches_selectMLS_extDict(ctx, ip, iend, maxSearches, mls, matches, minMatch);  /* first search (depth 0) */
 
-        ZSTD_LOG_PARSER("%d: match_num=%d last_pos=%d\n", (int)(ip-base), match_num, last_pos);
         if (!last_pos && !match_num) { ip++; continue; }
 
         { U32 i; for (i=0; i<ZSTD_REP_NUM; i++) opt[0].rep[i] = rep[i]; }
@@ -798,11 +715,10 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx,
 
         best_mlen = (last_pos) ? last_pos : minMatch;
 
-        // set prices using matches at position = 0
+        /* set prices using matches at position = 0 */
         for (u = 0; u < match_num; u++) {
             mlen = (u>0) ? matches[u-1].len+1 : best_mlen;
             best_mlen = matches[u].len;
-            ZSTD_LOG_PARSER("%d: start Found mlen=%d off=%d best_mlen=%d last_pos=%d\n", (int)(ip-base), matches[u].len, matches[u].off, (int)best_mlen, (int)last_pos);
             litlen = opt[0].litlen;
             while (mlen <= best_mlen) {
                 price = ZSTD_getPrice(seqStorePtr, litlen, anchor, matches[u].off-1, mlen - MINMATCH);
@@ -812,7 +728,6 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx,
         }   }
 
         if (last_pos < minMatch) {
-            // ip += ((ip-anchor) >> g_searchStrength) + 1;   /* jump faster over incompressible sections */
             ip++; continue;
         }
 
@@ -831,7 +746,7 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx,
                 price = opt[cur - 1].price + ZSTD_getLiteralPrice(seqStorePtr, litlen, inr-1);
             }
 
-            if (cur > last_pos || price <= opt[cur].price) // || ((price == opt[cur].price) && (opt[cur-1].mlen == 1) && (cur != litlen)))
+            if (cur > last_pos || price <= opt[cur].price)
                 SET_PRICE(cur, 1, 0, litlen, price);
 
             if (cur == last_pos) break;
@@ -844,15 +759,12 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx,
                 opt[cur].rep[2] = opt[cur-mlen].rep[1];
                 opt[cur].rep[1] = opt[cur-mlen].rep[0];
                 opt[cur].rep[0] = opt[cur].off - ZSTD_REP_MOVE_OPT;
-                ZSTD_LOG_ENCODE("%d: COPYREP_OFF cur=%d mlen=%d rep[0]=%d rep[1]=%d\n", (int)(inr-base), cur, mlen, opt[cur].rep[0], opt[cur].rep[1]);
             } else {
                 opt[cur].rep[2] = (opt[cur].off > 1) ? opt[cur-mlen].rep[1] : opt[cur-mlen].rep[2];
                 opt[cur].rep[1] = (opt[cur].off > 0) ? opt[cur-mlen].rep[0] : opt[cur-mlen].rep[1];
                 opt[cur].rep[0] = ((opt[cur].off==ZSTD_REP_MOVE_OPT) && (mlen != 1)) ? (opt[cur-mlen].rep[0] - 1) : (opt[cur-mlen].rep[opt[cur].off]);
-                ZSTD_LOG_ENCODE("%d: COPYREP_NOR cur=%d mlen=%d rep[0]=%d rep[1]=%d\n", (int)(inr-base), cur, mlen, opt[cur].rep[0], opt[cur].rep[1]);
             }
 
-            ZSTD_LOG_PARSER("%d: CURRENT_Ext price[%d/%d]=%d off=%d mlen=%d litlen=%d rep[0]=%d rep[1]=%d\n", (int)(inr-base), cur, last_pos, opt[cur].price, opt[cur].off, opt[cur].mlen, opt[cur].litlen, opt[cur].rep[0], opt[cur].rep[1]);
             best_mlen = 0;
 
             {   U32 i, last_i = ZSTD_REP_CHECK + (mlen != 1);
@@ -867,11 +779,9 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx,
                         /* repcode detected */
                         const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend;
                         mlen = (U32)ZSTD_count_2segments(inr+minMatch, repMatch+minMatch, iend, repEnd, prefixStart) + minMatch;
-                        ZSTD_LOG_PARSER("%d: Found REP %d/%d mlen=%d off=%d rep=%d opt[%d].off=%d\n", (int)(inr-base), i, ZSTD_REP_NUM, mlen, i, opt[cur].rep[i], cur, opt[cur].off);
 
                         if (mlen > sufficient_len || cur + mlen >= ZSTD_OPT_NUM) {
                             best_mlen = mlen; best_off = i; last_pos = cur + 1;
-                            ZSTD_LOG_PARSER("%d: REP sufficient_len=%d best_mlen=%d best_off=%d last_pos=%d\n", (int)(inr-base), sufficient_len, best_mlen, best_off, last_pos);
                             goto _storeSequence;
                         }
 
@@ -888,7 +798,6 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx,
                         }
 
                         best_mlen = mlen;
-                        ZSTD_LOG_PARSER("%d: Found REP mlen=%d off=%d price=%d litlen=%d\n", (int)(inr-base), mlen, best_off, price, litlen);
 
                         do {
                             if (cur + mlen > last_pos || price <= opt[cur + mlen].price)
@@ -898,7 +807,6 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx,
             }   }   }
 
             match_num = ZSTD_BtGetAllMatches_selectMLS_extDict(ctx, inr, iend, maxSearches, mls, matches, minMatch);
-            ZSTD_LOG_PARSER("%d: ZSTD_GetAllMatches match_num=%d\n", (int)(inr-base), match_num);
 
             if (match_num > 0 && matches[match_num-1].len > sufficient_len) {
                 best_mlen = matches[match_num-1].len;
@@ -914,7 +822,6 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx,
                 mlen = (u>0) ? matches[u-1].len+1 : best_mlen;
                 best_mlen = (cur + matches[u].len < ZSTD_OPT_NUM) ? matches[u].len : ZSTD_OPT_NUM - cur;
 
-            //    ZSTD_LOG_PARSER("%d: Found1 cur=%d mlen=%d off=%d best_mlen=%d last_pos=%d\n", (int)(inr-base), cur, matches[u].len, matches[u].off, best_mlen, last_pos);
                 while (mlen <= best_mlen) {
                     if (opt[cur].mlen == 1) {
                         litlen = opt[cur].litlen;
@@ -927,7 +834,6 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx,
                         price = opt[cur].price + ZSTD_getPrice(seqStorePtr, 0, NULL, matches[u].off-1, mlen - MINMATCH);
                     }
 
-                //    ZSTD_LOG_PARSER("%d: Found2 mlen=%d best_mlen=%d off=%d price=%d litlen=%d\n", (int)(inr-base), mlen, best_mlen, matches[u].off, price, litlen);
                     if (cur + mlen > last_pos || (price < opt[cur + mlen].price))
                         SET_PRICE(cur + mlen, mlen, matches[u].off, litlen, price);
 
@@ -940,10 +846,6 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx,
 
         /* store sequence */
 _storeSequence:   /* cur, last_pos, best_mlen, best_off have to be set */
-        for (u = 1; u <= last_pos; u++)
-            ZSTD_LOG_PARSER("%d: price[%u/%d]=%d off=%d mlen=%d litlen=%d rep[0]=%d rep[1]=%d\n", (int)(ip-base+u), u, last_pos, opt[u].price, opt[u].off, opt[u].mlen, opt[u].litlen, opt[u].rep[0], opt[u].rep[1]);
-        ZSTD_LOG_PARSER("%d: cur=%d/%d best_mlen=%d best_off=%d rep[0]=%d\n", (int)(ip-base+cur), (int)cur, (int)last_pos, (int)best_mlen, (int)best_off, opt[cur].rep[0]);
-
         opt[0].mlen = 1;
 
         while (1) {
@@ -958,18 +860,15 @@ _storeSequence:   /* cur, last_pos, best_mlen, best_off have to be set */
         }
 
         for (u = 0; u <= last_pos; ) {
-            ZSTD_LOG_PARSER("%d: price2[%d/%d]=%d off=%d mlen=%d litlen=%d rep[0]=%d rep[1]=%d\n", (int)(ip-base+u), u, last_pos, opt[u].price, opt[u].off, opt[u].mlen, opt[u].litlen, opt[u].rep[0], opt[u].rep[1]);
             u += opt[u].mlen;
         }
 
         for (cur=0; cur < last_pos; ) {
-            ZSTD_LOG_PARSER("%d: price3[%d/%d]=%d off=%d mlen=%d litlen=%d rep[0]=%d rep[1]=%d\n", (int)(ip-base+cur), cur, last_pos, opt[cur].price, opt[cur].off, opt[cur].mlen, opt[cur].litlen, opt[cur].rep[0], opt[cur].rep[1]);
             mlen = opt[cur].mlen;
             if (mlen == 1) { ip++; cur++; continue; }
             offset = opt[cur].off;
             cur += mlen;
             litLength = (U32)(ip - anchor);
-         //   ZSTD_LOG_ENCODE("%d/%d: ENCODE1 literals=%d mlen=%d off=%d rep[0]=%d rep[1]=%d\n", (int)(ip-base), (int)(iend-base), (int)(litLength), (int)mlen, (int)(offset), (int)rep[0], (int)rep[1]);
 
             if (offset > ZSTD_REP_MOVE_OPT) {
                 rep[2] = rep[1];
@@ -984,32 +883,9 @@ _storeSequence:   /* cur, last_pos, best_mlen, best_off have to be set */
                     rep[0] = best_off;
                 }
 
-                if ((litLength==0) & (offset==0)) { ZSTD_LOG_ENCODE("ERROR (litLength==0) & (offset==0)\n"); };
                 if (litLength==0) offset--;
             }
 
-            ZSTD_LOG_ENCODE("%d/%d: ENCODE literals=%d mlen=%d off=%d rep[0]=%d rep[1]=%d\n", (int)(ip-base), (int)(iend-base), (int)(litLength), (int)mlen, (int)(offset), (int)rep[0], (int)rep[1]);
-
-#if ZSTD_OPT_DEBUG >= 5
-            U32 ml2;
-            if (offset+1 > ZSTD_REP_MOVE_OPT) {
-                best_off = offset+1 - ZSTD_REP_MOVE_OPT;
-                if (best_off > (size_t)(ip - prefixStart))  {
-                    const BYTE* match = dictEnd - (best_off - (ip - prefixStart));
-                    ml2 = ZSTD_count_2segments(ip, match, iend, dictEnd, prefixStart);
-                    ZSTD_LOG_PARSER("%d: ZSTD_count_2segments=%d offset=%d dictBase=%p dictEnd=%p prefixStart=%p ip=%p match=%p\n", (int)current, (int)ml2, (int)best_off, dictBase, dictEnd, prefixStart, ip, match);
-                }
-                else ml2 = (U32)ZSTD_count(ip, ip-best_off, iend);
-            }
-            else ml2 = (U32)ZSTD_count(ip, ip-rep[0], iend);
-            if ((offset >= 8) && (ml2 < mlen || ml2 < minMatch)) {
-                printf("%d: ERROR_Ext iend=%d mlen=%d offset=%d ml2=%d\n", (int)(ip - base), (int)(iend - ip), (int)mlen, (int)offset, (int)ml2); exit(0); }
-            if (ip < anchor) {
-                printf("%d: ERROR_Ext ip < anchor iend=%d mlen=%d offset=%d\n", (int)(ip - base), (int)(iend - ip), (int)mlen, (int)offset); exit(0); }
-            if (ip + mlen > iend) {
-                printf("%d: ERROR_Ext ip + mlen >= iend iend=%d mlen=%d offset=%d\n", (int)(ip - base), (int)(iend - ip), (int)mlen, (int)offset); exit(0); }
-#endif
-
             ZSTD_updatePrice(seqStorePtr, litLength, anchor, offset, mlen-MINMATCH);
             ZSTD_storeSeq(seqStorePtr, litLength, anchor, offset, mlen-MINMATCH);
             anchor = ip = ip + mlen;
@@ -1020,7 +896,6 @@ _storeSequence:   /* cur, last_pos, best_mlen, best_off have to be set */
 
     /* Last Literals */
     {   size_t lastLLSize = iend - anchor;
-        ZSTD_LOG_ENCODE("%d: lastLLSize literals=%u\n", (int)(ip-base), (U32)(lastLLSize));
         memcpy(seqStorePtr->lit, anchor, lastLLSize);
         seqStorePtr->lit += lastLLSize;
     }

lib/decompress/huf_decompress.c

@@ -37,7 +37,7 @@
 ****************************************************************/
 #if defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
 /* inline is defined */
-#elif defined(_MSC_VER)
+#elif defined(_MSC_VER) || defined(__GNUC__)
 #  define inline __inline
 #else
 #  define inline /* disable inline */
@@ -57,10 +57,10 @@
 
 
 /* **************************************************************
-*  Includes
+*  Dependencies
 ****************************************************************/
 #include <string.h>     /* memcpy, memset */
-#include "bitstream.h"
+#include "bitstream.h"  /* BIT_* */
 #include "fse.h"        /* header compression */
 #define HUF_STATIC_LINKING_ONLY
 #include "huf.h"
@@ -103,7 +103,7 @@ size_t HUF_readDTableX2 (HUF_DTable* DTable, const void* src, size_t srcSize)
     HUF_DEltX2* const dt = (HUF_DEltX2*)dtPtr;
 
     HUF_STATIC_ASSERT(sizeof(DTableDesc) == sizeof(HUF_DTable));
-    //memset(huffWeight, 0, sizeof(huffWeight));   /* is not necessary, even though some analyzer complain ... */
+    /* memset(huffWeight, 0, sizeof(huffWeight)); */   /* is not necessary, even though some analyzer complain ... */
 
     iSize = HUF_readStats(huffWeight, HUF_SYMBOLVALUE_MAX + 1, rankVal, &nbSymbols, &tableLog, src, srcSize);
     if (HUF_isError(iSize)) return iSize;
@@ -388,22 +388,22 @@ static void HUF_fillDTableX4Level2(HUF_DEltX4* DTable, U32 sizeLog, const U32 co
     }
 
     /* fill DTable */
-    { U32 s; for (s=0; s<sortedListSize; s++) {   /* note : sortedSymbols already skipped */
+    {   U32 s; for (s=0; s<sortedListSize; s++) {   /* note : sortedSymbols already skipped */
-        const U32 symbol = sortedSymbols[s].symbol;
+            const U32 symbol = sortedSymbols[s].symbol;
-        const U32 weight = sortedSymbols[s].weight;
+            const U32 weight = sortedSymbols[s].weight;
-        const U32 nbBits = nbBitsBaseline - weight;
+            const U32 nbBits = nbBitsBaseline - weight;
-        const U32 length = 1 << (sizeLog-nbBits);
+            const U32 length = 1 << (sizeLog-nbBits);
-        const U32 start = rankVal[weight];
+            const U32 start = rankVal[weight];
-        U32 i = start;
+            U32 i = start;
-        const U32 end = start + length;
+            const U32 end = start + length;
 
-        MEM_writeLE16(&(DElt.sequence), (U16)(baseSeq + (symbol << 8)));
+            MEM_writeLE16(&(DElt.sequence), (U16)(baseSeq + (symbol << 8)));
-        DElt.nbBits = (BYTE)(nbBits + consumed);
+            DElt.nbBits = (BYTE)(nbBits + consumed);
-        DElt.length = 2;
+            DElt.length = 2;
-        do { DTable[i++] = DElt; } while (i<end);   /* since length >= 1 */
+            do { DTable[i++] = DElt; } while (i<end);   /* since length >= 1 */
 
-        rankVal[weight] += length;
+            rankVal[weight] += length;
-    }}
+    }   }
 }
 
 typedef U32 rankVal_t[HUF_TABLELOG_ABSOLUTEMAX][HUF_TABLELOG_ABSOLUTEMAX + 1];
@@ -442,8 +442,8 @@ static void HUF_fillDTableX4(HUF_DEltX4* DTable, const U32 targetLog,
             MEM_writeLE16(&(DElt.sequence), symbol);
             DElt.nbBits = (BYTE)(nbBits);
             DElt.length = 1;
-            {   U32 u;
+            {   U32 const end = start + length;
-                const U32 end = start + length;
+                U32 u;
                 for (u = start; u < end; u++) DTable[u] = DElt;
         }   }
         rankVal[weight] += length;
@@ -467,7 +467,7 @@ size_t HUF_readDTableX4 (HUF_DTable* DTable, const void* src, size_t srcSize)
 
     HUF_STATIC_ASSERT(sizeof(HUF_DEltX4) == sizeof(HUF_DTable));   /* if compilation fails here, assertion is false */
     if (maxTableLog > HUF_TABLELOG_ABSOLUTEMAX) return ERROR(tableLog_tooLarge);
-    //memset(weightList, 0, sizeof(weightList));   /* is not necessary, even though some analyzer complain ... */
+    /* memset(weightList, 0, sizeof(weightList)); */  /* is not necessary, even though some analyzer complain ... */
 
     iSize = HUF_readStats(weightList, HUF_SYMBOLVALUE_MAX + 1, rankStats, &nbSymbols, &tableLog, src, srcSize);
     if (HUF_isError(iSize)) return iSize;
@@ -533,7 +533,7 @@ size_t HUF_readDTableX4 (HUF_DTable* DTable, const void* src, size_t srcSize)
 
 static U32 HUF_decodeSymbolX4(void* op, BIT_DStream_t* DStream, const HUF_DEltX4* dt, const U32 dtLog)
 {
-    const size_t val = BIT_lookBitsFast(DStream, dtLog);   /* note : dtLog >= 1 */
+    size_t const val = BIT_lookBitsFast(DStream, dtLog);   /* note : dtLog >= 1 */
     memcpy(op, dt+val, 2);
     BIT_skipBits(DStream, dt[val].nbBits);
     return dt[val].length;
@@ -541,7 +541,7 @@ static U32 HUF_decodeSymbolX4(void* op, BIT_DStream_t* DStream, const HUF_DEltX4
 
 static U32 HUF_decodeLastSymbolX4(void* op, BIT_DStream_t* DStream, const HUF_DEltX4* dt, const U32 dtLog)
 {
-    const size_t val = BIT_lookBitsFast(DStream, dtLog);   /* note : dtLog >= 1 */
+    size_t const val = BIT_lookBitsFast(DStream, dtLog);   /* note : dtLog >= 1 */
     memcpy(op, dt+val, 1);
     if (dt[val].length==1) BIT_skipBits(DStream, dt[val].nbBits);
     else {
@@ -570,7 +570,7 @@ static inline size_t HUF_decodeStreamX4(BYTE* p, BIT_DStream_t* bitDPtr, BYTE* c
     BYTE* const pStart = p;
 
     /* up to 8 symbols at a time */
-    while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) && (p < pEnd-7)) {
+    while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) & (p < pEnd-(sizeof(bitDPtr->bitContainer)-1))) {
         HUF_DECODE_SYMBOLX4_2(p, bitDPtr);
         HUF_DECODE_SYMBOLX4_1(p, bitDPtr);
         HUF_DECODE_SYMBOLX4_2(p, bitDPtr);
@@ -578,7 +578,7 @@ static inline size_t HUF_decodeStreamX4(BYTE* p, BIT_DStream_t* bitDPtr, BYTE* c
     }
 
     /* closer to end : up to 2 symbols at a time */
-    while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) && (p <= pEnd-2))
+    while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) & (p <= pEnd-2))
         HUF_DECODE_SYMBOLX4_0(p, bitDPtr);
 
     while (p <= pEnd-2)
@@ -697,7 +697,7 @@ static size_t HUF_decompress4X4_usingDTable_internal(
 
         /* 16-32 symbols per loop (4-8 symbols per stream) */
         endSignal = BIT_reloadDStream(&bitD1) | BIT_reloadDStream(&bitD2) | BIT_reloadDStream(&bitD3) | BIT_reloadDStream(&bitD4);
-        for ( ; (endSignal==BIT_DStream_unfinished) && (op4<(oend-7)) ; ) {
+        for ( ; (endSignal==BIT_DStream_unfinished) & (op4<(oend-(sizeof(bitD4.bitContainer)-1))) ; ) {
             HUF_DECODE_SYMBOLX4_2(op1, &bitD1);
             HUF_DECODE_SYMBOLX4_2(op2, &bitD2);
             HUF_DECODE_SYMBOLX4_2(op3, &bitD3);
@@ -722,7 +722,7 @@ static size_t HUF_decompress4X4_usingDTable_internal(
         if (op1 > opStart2) return ERROR(corruption_detected);
         if (op2 > opStart3) return ERROR(corruption_detected);
         if (op3 > opStart4) return ERROR(corruption_detected);
-        /* note : op4 supposed already verified within main loop */
+        /* note : op4 already verified within main loop */
 
         /* finish bitStreams one by one */
         HUF_decodeStreamX4(op1, &bitD1, opStart2, dt, dtLog);
@@ -848,9 +848,6 @@ size_t HUF_decompress (void* dst, size_t dstSize, const void* cSrc, size_t cSrcS
     {   U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize);
         return decompress[algoNb](dst, dstSize, cSrc, cSrcSize);
     }
-
-    //return HUF_decompress4X2(dst, dstSize, cSrc, cSrcSize);   /* multi-streams single-symbol decoding */
-    //return HUF_decompress4X4(dst, dstSize, cSrc, cSrcSize);   /* multi-streams double-symbols decoding */
 }
 
 size_t HUF_decompress4X_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)

lib/decompress/zbuff_decompress.c

@@ -1,33 +1,12 @@
-/*
+/**
-    Buffered version of Zstd compression library
+ * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
-    Copyright (C) 2015-2016, Yann Collet.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
 
-    BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
-
-    Redistribution and use in source and binary forms, with or without
-    modification, are permitted provided that the following conditions are
-    met:
-    * Redistributions of source code must retain the above copyright
-    notice, this list of conditions and the following disclaimer.
-    * Redistributions in binary form must reproduce the above
-    copyright notice, this list of conditions and the following disclaimer
-    in the documentation and/or other materials provided with the
-    distribution.
-    THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-    "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-    LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-    A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-    OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-    SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-    LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-    DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-    THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-    (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-    OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-    You can contact the author at :
-    - zstd homepage : http://www.zstd.net/
-*/
 
 
 /* *************************************
@@ -40,30 +19,6 @@
 #include "zbuff.h"
 
 
-/*-***************************************************************************
-*  Streaming decompression howto
-*
-*  A ZBUFF_DCtx object is required to track streaming operations.
-*  Use ZBUFF_createDCtx() and ZBUFF_freeDCtx() to create/release resources.
-*  Use ZBUFF_decompressInit() to start a new decompression operation,
-*   or ZBUFF_decompressInitDictionary() if decompression requires a dictionary.
-*  Note that ZBUFF_DCtx objects can be re-init multiple times.
-*
-*  Use ZBUFF_decompressContinue() repetitively to consume your input.
-*  *srcSizePtr and *dstCapacityPtr can be any size.
-*  The function will report how many bytes were read or written by modifying *srcSizePtr and *dstCapacityPtr.
-*  Note that it may not consume the entire input, in which case it's up to the caller to present remaining input again.
-*  The content of @dst will be overwritten (up to *dstCapacityPtr) at each function call, so save its content if it matters, or change @dst.
-*  @return : a hint to preferred nb of bytes to use as input for next function call (it's only a hint, to help latency),
-*            or 0 when a frame is completely decoded,
-*            or an error code, which can be tested using ZBUFF_isError().
-*
-*  Hint : recommended buffer sizes (not compulsory) : ZBUFF_recommendedDInSize() and ZBUFF_recommendedDOutSize()
-*  output : ZBUFF_recommendedDOutSize==128 KB block size is the internal unit, it ensures it's always possible to write a full block when decoded.
-*  input  : ZBUFF_recommendedDInSize == 128KB + 3;
-*           just follow indications from ZBUFF_decompressContinue() to minimize latency. It should always be <= 128 KB + 3 .
-* *******************************************************************************/
-
 typedef enum { ZBUFFds_init, ZBUFFds_loadHeader,
                ZBUFFds_read, ZBUFFds_load, ZBUFFds_flush } ZBUFF_dStage;
 
@@ -83,7 +38,7 @@ struct ZBUFF_DCtx_s {
     BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX];
     size_t lhSize;
     ZSTD_customMem customMem;
-};   /* typedef'd to ZBUFF_DCtx within "zstd_buffered.h" */
+};   /* typedef'd to ZBUFF_DCtx within "zbuff.h" */
 
 
 ZBUFF_DCtx* ZBUFF_createDCtx(void)

lib/decompress/zstd_decompress.c

@@ -1,33 +1,12 @@
-/*
+/**
-    zstd - standard compression library
+ * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
-    Copyright (C) 2014-2016, Yann Collet.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
 
-    BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
-
-    Redistribution and use in source and binary forms, with or without
-    modification, are permitted provided that the following conditions are
-    met:
-    * Redistributions of source code must retain the above copyright
-    notice, this list of conditions and the following disclaimer.
-    * Redistributions in binary form must reproduce the above
-    copyright notice, this list of conditions and the following disclaimer
-    in the documentation and/or other materials provided with the
-    distribution.
-    THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-    "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-    LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-    A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-    OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-    SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-    LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-    DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-    THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-    (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-    OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-    You can contact the author at :
-    - zstd homepage : http://www.zstd.net
-*/
 
 /* ***************************************************************
 *  Tuning parameters
@@ -50,11 +29,20 @@
 #endif
 
 
+/*!
+*  MAXWINDOWSIZE_DEFAULT :
+*  maximum window size accepted by DStream, by default.
+*  Frames requiring more memory will be rejected.
+*/
+#ifndef ZSTD_MAXWINDOWSIZE_DEFAULT
+#  define ZSTD_MAXWINDOWSIZE_DEFAULT (257 << 20)   /* 257 MB */
+#endif
+
+
 /*-*******************************************************
 *  Dependencies
 *********************************************************/
 #include <string.h>      /* memcpy, memmove, memset */
-#include <stdio.h>       /* debug only : printf */
 #include "mem.h"         /* low level memory routines */
 #define XXH_STATIC_LINKING_ONLY   /* XXH64_state_t */
 #include "xxhash.h"      /* XXH64_* */
@@ -136,9 +124,9 @@ struct ZSTD_DCtx_s
     size_t rleSize;
     BYTE litBuffer[ZSTD_BLOCKSIZE_ABSOLUTEMAX + WILDCOPY_OVERLENGTH];
     BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX];
-};  /* typedef'd to ZSTD_DCtx within "zstd_static.h" */
+};  /* typedef'd to ZSTD_DCtx within "zstd.h" */
 
-size_t ZSTD_sizeofDCtx (const ZSTD_DCtx* dctx) { return sizeof(*dctx); }
+size_t ZSTD_sizeof_DCtx (const ZSTD_DCtx* dctx) { return sizeof(*dctx); }
 
 size_t ZSTD_estimateDCtxSize(void) { return sizeof(ZSTD_DCtx); }
 
@@ -165,7 +153,7 @@ ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem)
     if (!customMem.customAlloc && !customMem.customFree) customMem = defaultCustomMem;
     if (!customMem.customAlloc || !customMem.customFree) return NULL;
 
-    dctx = (ZSTD_DCtx*) customMem.customAlloc(customMem.opaque, sizeof(ZSTD_DCtx));
+    dctx = (ZSTD_DCtx*) ZSTD_malloc(sizeof(ZSTD_DCtx), customMem);
     if (!dctx) return NULL;
     memcpy(&dctx->customMem, &customMem, sizeof(customMem));
     ZSTD_decompressBegin(dctx);
@@ -180,7 +168,7 @@ ZSTD_DCtx* ZSTD_createDCtx(void)
 size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx)
 {
     if (dctx==NULL) return 0;   /* support free on NULL */
-    dctx->customMem.customFree(dctx->customMem.opaque, dctx);
+    ZSTD_free(dctx, dctx->customMem);
     return 0;   /* reserved as a potential error code in the future */
 }
 
@@ -590,9 +578,9 @@ static seq_t ZSTD_decodeSequence(seqState_t* seqState)
 {
     seq_t seq;
 
-    U32 const llCode = FSE_peekSymbol(&(seqState->stateLL));
+    U32 const llCode = FSE_peekSymbol(&seqState->stateLL);
-    U32 const mlCode = FSE_peekSymbol(&(seqState->stateML));
+    U32 const mlCode = FSE_peekSymbol(&seqState->stateML);
-    U32 const ofCode = FSE_peekSymbol(&(seqState->stateOffb));   /* <= maxOff, by table construction */
+    U32 const ofCode = FSE_peekSymbol(&seqState->stateOffb);   /* <= maxOff, by table construction */
 
     U32 const llBits = LL_bits[llCode];
     U32 const mlBits = ML_bits[mlCode];
@@ -621,8 +609,8 @@ static seq_t ZSTD_decodeSequence(seqState_t* seqState)
         if (!ofCode)
             offset = 0;
         else {
-            offset = OF_base[ofCode] + BIT_readBits(&(seqState->DStream), ofBits);   /* <=  (ZSTD_WINDOWLOG_MAX-1) bits */
+            offset = OF_base[ofCode] + BIT_readBits(&seqState->DStream, ofBits);   /* <=  (ZSTD_WINDOWLOG_MAX-1) bits */
-            if (MEM_32bits()) BIT_reloadDStream(&(seqState->DStream));
+            if (MEM_32bits()) BIT_reloadDStream(&seqState->DStream);
         }
 
         if (ofCode <= 1) {
@@ -643,18 +631,18 @@ static seq_t ZSTD_decodeSequence(seqState_t* seqState)
         seq.offset = offset;
     }
 
-    seq.matchLength = ML_base[mlCode] + ((mlCode>31) ? BIT_readBits(&(seqState->DStream), mlBits) : 0);   /* <=  16 bits */
+    seq.matchLength = ML_base[mlCode] + ((mlCode>31) ? BIT_readBits(&seqState->DStream, mlBits) : 0);  /* <=  16 bits */
-    if (MEM_32bits() && (mlBits+llBits>24)) BIT_reloadDStream(&(seqState->DStream));
+    if (MEM_32bits() && (mlBits+llBits>24)) BIT_reloadDStream(&seqState->DStream);
 
-    seq.litLength = LL_base[llCode] + ((llCode>15) ? BIT_readBits(&(seqState->DStream), llBits) : 0);   /* <=  16 bits */
+    seq.litLength = LL_base[llCode] + ((llCode>15) ? BIT_readBits(&seqState->DStream, llBits) : 0);    /* <=  16 bits */
     if (MEM_32bits() ||
-       (totalBits > 64 - 7 - (LLFSELog+MLFSELog+OffFSELog)) ) BIT_reloadDStream(&(seqState->DStream));
+       (totalBits > 64 - 7 - (LLFSELog+MLFSELog+OffFSELog)) ) BIT_reloadDStream(&seqState->DStream);
 
     /* ANS state update */
-    FSE_updateState(&(seqState->stateLL), &(seqState->DStream));   /* <=  9 bits */
+    FSE_updateState(&seqState->stateLL, &seqState->DStream);    /* <=  9 bits */
-    FSE_updateState(&(seqState->stateML), &(seqState->DStream));   /* <=  9 bits */
+    FSE_updateState(&seqState->stateML, &seqState->DStream);    /* <=  9 bits */
-    if (MEM_32bits()) BIT_reloadDStream(&(seqState->DStream));     /* <= 18 bits */
+    if (MEM_32bits()) BIT_reloadDStream(&seqState->DStream);    /* <= 18 bits */
-    FSE_updateState(&(seqState->stateOffb), &(seqState->DStream)); /* <=  8 bits */
+    FSE_updateState(&seqState->stateOffb, &seqState->DStream);  /* <=  8 bits */
 
     return seq;
 }
@@ -678,7 +666,9 @@ size_t ZSTD_execSequence(BYTE* op,
     if (iLitEnd > litLimit_w) return ERROR(corruption_detected);   /* over-read beyond lit buffer */
 
     /* copy Literals */
-    ZSTD_wildcopy(op, *litPtr, sequence.litLength);   /* note : since oLitEnd <= oend-WILDCOPY_OVERLENGTH, no risk of overwrite beyond oend */
+    ZSTD_copy8(op, *litPtr);
+    if (sequence.litLength > 8)
+        ZSTD_wildcopy(op+8, (*litPtr)+8, sequence.litLength - 8);   /* note : since oLitEnd <= oend-WILDCOPY_OVERLENGTH, no risk of overwrite beyond oend */
     op = oLitEnd;
     *litPtr = iLitEnd;   /* update for next sequence */
 
@@ -978,9 +968,10 @@ size_t ZSTD_decompress(void* dst, size_t dstCapacity, const void* src, size_t sr
 }
 
 
-/*-**********************************
+/*-**************************************
-*   Streaming Decompression API
+*   Advanced Streaming Decompression API
-************************************/
+*   Bufferless and synchronous
+****************************************/
 size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx) { return dctx->expected; }
 
 ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx) {
@@ -1228,33 +1219,30 @@ struct ZSTD_DDict_s {
     void* dict;
     size_t dictSize;
     ZSTD_DCtx* refContext;
-};  /* typedef'd tp ZSTD_CDict within zstd.h */
+};  /* typedef'd to ZSTD_DDict within "zstd.h" */
 
 ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictSize, ZSTD_customMem customMem)
 {
-    if (!customMem.customAlloc && !customMem.customFree)
+    if (!customMem.customAlloc && !customMem.customFree) customMem = defaultCustomMem;
-        customMem = defaultCustomMem;
+    if (!customMem.customAlloc || !customMem.customFree) return NULL;
 
-    if (!customMem.customAlloc || !customMem.customFree)
+    {   ZSTD_DDict* const ddict = (ZSTD_DDict*) ZSTD_malloc(sizeof(ZSTD_DDict), customMem);
-        return NULL;
+        void* const dictContent = ZSTD_malloc(dictSize, customMem);
-
-    {   ZSTD_DDict* const ddict = (ZSTD_DDict*) customMem.customAlloc(customMem.opaque, sizeof(*ddict));
-        void* const dictContent = customMem.customAlloc(customMem.opaque, dictSize);
         ZSTD_DCtx* const dctx = ZSTD_createDCtx_advanced(customMem);
 
         if (!dictContent || !ddict || !dctx) {
-            customMem.customFree(customMem.opaque, dictContent);
+            ZSTD_free(dictContent, customMem);
-            customMem.customFree(customMem.opaque, ddict);
+            ZSTD_free(ddict, customMem);
-            customMem.customFree(customMem.opaque, dctx);
+            ZSTD_free(dctx, customMem);
             return NULL;
         }
 
         memcpy(dictContent, dict, dictSize);
         {   size_t const errorCode = ZSTD_decompressBegin_usingDict(dctx, dictContent, dictSize);
             if (ZSTD_isError(errorCode)) {
-                customMem.customFree(customMem.opaque, dictContent);
+                ZSTD_free(dictContent, customMem);
-                customMem.customFree(customMem.opaque, ddict);
+                ZSTD_free(ddict, customMem);
-                customMem.customFree(customMem.opaque, dctx);
+                ZSTD_free(dctx, customMem);
                 return NULL;
         }   }
 
@@ -1276,12 +1264,13 @@ ZSTD_DDict* ZSTD_createDDict(const void* dict, size_t dictSize)
 
 size_t ZSTD_freeDDict(ZSTD_DDict* ddict)
 {
-    ZSTD_freeFunction const cFree = ddict->refContext->customMem.customFree;
+    if (ddict==NULL) return 0;   /* support free on NULL */
-    void* const opaque = ddict->refContext->customMem.opaque;
+    {   ZSTD_customMem const cMem = ddict->refContext->customMem;
-    ZSTD_freeDCtx(ddict->refContext);
+        ZSTD_freeDCtx(ddict->refContext);
-    cFree(opaque, ddict->dict);
+        ZSTD_free(ddict->dict, cMem);
-    cFree(opaque, ddict);
+        ZSTD_free(ddict, cMem);
-    return 0;
+        return 0;
+    }
 }
 
 /*! ZSTD_decompress_usingDDict() :
@@ -1296,8 +1285,8 @@ ZSTDLIB_API size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx,
     if (ZSTD_isLegacy(src, srcSize)) return ZSTD_decompressLegacy(dst, dstCapacity, src, srcSize, ddict->dict, ddict->dictSize);
 #endif
     return ZSTD_decompress_usingPreparedDCtx(dctx, ddict->refContext,
-                                           dst, dstCapacity,
+                                             dst, dstCapacity,
-                                           src, srcSize);
+                                             src, srcSize);
 }
 
 
@@ -1316,6 +1305,7 @@ struct ZSTD_DStream_s {
     char*  inBuff;
     size_t inBuffSize;
     size_t inPos;
+    size_t maxWindowSize;
     char*  outBuff;
     size_t outBuffSize;
     size_t outStart;
@@ -1324,6 +1314,12 @@ struct ZSTD_DStream_s {
     BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX];
     size_t lhSize;
     ZSTD_customMem customMem;
+    void* dictContent;
+    size_t dictSize;
+    const void* dictSource;
+    void* legacyContext;
+    U32 previousLegacyVersion;
+    U32 legacyVersion;
 };   /* typedef'd to ZSTD_DStream within "zstd.h" */
 
 
@@ -1336,30 +1332,35 @@ ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem)
 {
     ZSTD_DStream* zds;
 
-    if (!customMem.customAlloc && !customMem.customFree)
+    if (!customMem.customAlloc && !customMem.customFree) customMem = defaultCustomMem;
-        customMem = defaultCustomMem;
+    if (!customMem.customAlloc || !customMem.customFree) return NULL;
 
-    if (!customMem.customAlloc || !customMem.customFree)
+    zds = (ZSTD_DStream*) ZSTD_malloc(sizeof(ZSTD_DStream), customMem);
-        return NULL;
-
-    zds = (ZSTD_DStream*)customMem.customAlloc(customMem.opaque, sizeof(ZSTD_DStream));
     if (zds==NULL) return NULL;
     memset(zds, 0, sizeof(ZSTD_DStream));
     memcpy(&zds->customMem, &customMem, sizeof(ZSTD_customMem));
     zds->zd = ZSTD_createDCtx_advanced(customMem);
     if (zds->zd == NULL) { ZSTD_freeDStream(zds); return NULL; }
     zds->stage = zdss_init;
+    zds->maxWindowSize = ZSTD_MAXWINDOWSIZE_DEFAULT;
     return zds;
 }
 
 size_t ZSTD_freeDStream(ZSTD_DStream* zds)
 {
     if (zds==NULL) return 0;   /* support free on null */
-    ZSTD_freeDCtx(zds->zd);
+    {   ZSTD_customMem const cMem = zds->customMem;
-    if (zds->inBuff) zds->customMem.customFree(zds->customMem.opaque, zds->inBuff);
+        ZSTD_freeDCtx(zds->zd);
-    if (zds->outBuff) zds->customMem.customFree(zds->customMem.opaque, zds->outBuff);
+        ZSTD_free(zds->inBuff, cMem);
-    zds->customMem.customFree(zds->customMem.opaque, zds);
+        ZSTD_free(zds->outBuff, cMem);
-    return 0;
+        ZSTD_free(zds->dictContent, cMem);
+#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1)
+        if (zds->legacyContext)
+            ZSTD_freeLegacyStreamContext(zds->legacyContext, zds->previousLegacyVersion);
+#endif
+        ZSTD_free(zds, cMem);
+        return 0;
+    }
 }
 
 
@@ -1372,7 +1373,17 @@ size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t di
 {
     zds->stage = zdss_loadHeader;
     zds->lhSize = zds->inPos = zds->outStart = zds->outEnd = 0;
-    return ZSTD_decompressBegin_usingDict(zds->zd, dict, dictSize);
+    if ((dict != zds->dictSource) | (dictSize != zds->dictSize)) {   /* new dictionary */
+        if (dictSize > zds->dictSize) {
+            ZSTD_free(zds->dictContent, zds->customMem);
+            zds->dictContent = ZSTD_malloc(dictSize, zds->customMem);
+            if (zds->dictContent == NULL) return ERROR(memory_allocation);
+        }
+        memcpy(zds->dictContent, dict, dictSize);
+        zds->dictSize = dictSize;
+    }
+    zds->legacyVersion = 0;
+    return 0;
 }
 
 size_t ZSTD_initDStream(ZSTD_DStream* zds)
@@ -1380,6 +1391,23 @@ size_t ZSTD_initDStream(ZSTD_DStream* zds)
     return ZSTD_initDStream_usingDict(zds, NULL, 0);
 }
 
+size_t ZSTD_setDStreamParameter(ZSTD_DStream* zds,
+                                ZSTD_DStreamParameter_e paramType, unsigned paramValue)
+{
+    switch(paramType)
+    {
+        default : return ERROR(parameter_unknown);
+        case ZSTDdsp_maxWindowSize : zds->maxWindowSize = paramValue; break;
+    }
+    return 0;
+}
+
+
+size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds)
+{
+    return sizeof(*zds) + ZSTD_sizeof_DCtx(zds->zd) + zds->inBuffSize + zds->outBuffSize;
+}
+
 
 /* *** Decompression *** */
 
@@ -1401,6 +1429,11 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
     char* op = ostart;
     U32 someMoreWork = 1;
 
+#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1)
+    if (zds->legacyVersion)
+        return ZSTD_decompressLegacyStream(zds->legacyContext, zds->legacyVersion, output, input);
+#endif
+
     while (someMoreWork) {
         switch(zds->stage)
         {
@@ -1408,8 +1441,23 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
             return ERROR(init_missing);
 
         case zdss_loadHeader :
-            {   size_t const hSize = ZSTD_getFrameParams(&(zds->fParams), zds->headerBuffer, zds->lhSize);
+            {   size_t const hSize = ZSTD_getFrameParams(&zds->fParams, zds->headerBuffer, zds->lhSize);
-                if (ZSTD_isError(hSize)) return hSize;
+                if (ZSTD_isError(hSize))
+#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1)
+                {   U32 const legacyVersion = ZSTD_isLegacy(istart, iend-istart);
+                    if (legacyVersion) {
+                        size_t initResult;
+                        initResult = ZSTD_initLegacyStream(&zds->legacyContext, zds->previousLegacyVersion, legacyVersion,
+                                                           zds->dictContent, zds->dictSize);
+                        if (ZSTD_isError(initResult)) return initResult;
+                        zds->legacyVersion = zds->previousLegacyVersion = legacyVersion;
+                        return ZSTD_decompressLegacyStream(zds->legacyContext, zds->legacyVersion, output, input);
+                    } else {
+                        return hSize; /* error */
+                }   }
+#else
+                    return hSize;
+#endif
                 if (hSize != 0) {   /* need more input */
                     size_t const toLoad = hSize - zds->lhSize;   /* if hSize!=0, hSize > zds->lhSize */
                     if (toLoad > (size_t)(iend-ip)) {   /* not enough input to load full header */
@@ -1423,6 +1471,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
             }   }
 
             /* Consume header */
+            ZSTD_decompressBegin_usingDict(zds->zd, zds->dictContent, zds->dictSize);
             {   size_t const h1Size = ZSTD_nextSrcSizeToDecompress(zds->zd);  /* == ZSTD_frameHeaderSize_min */
                 size_t const h1Result = ZSTD_decompressContinue(zds->zd, NULL, 0, zds->headerBuffer, h1Size);
                 if (ZSTD_isError(h1Result)) return h1Result;   /* should not happen : already checked */
@@ -1433,21 +1482,22 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
             }   }
 
             zds->fParams.windowSize = MAX(zds->fParams.windowSize, 1U << ZSTD_WINDOWLOG_ABSOLUTEMIN);
+            if (zds->fParams.windowSize > zds->maxWindowSize) return ERROR(frameParameter_unsupported);
 
             /* Frame header instruct buffer sizes */
             {   size_t const blockSize = MIN(zds->fParams.windowSize, ZSTD_BLOCKSIZE_ABSOLUTEMAX);
                 size_t const neededOutSize = zds->fParams.windowSize + blockSize;
                 zds->blockSize = blockSize;
                 if (zds->inBuffSize < blockSize) {
-                    zds->customMem.customFree(zds->customMem.opaque, zds->inBuff);
+                    ZSTD_free(zds->inBuff, zds->customMem);
                     zds->inBuffSize = blockSize;
-                    zds->inBuff = (char*)zds->customMem.customAlloc(zds->customMem.opaque, blockSize);
+                    zds->inBuff = (char*)ZSTD_malloc(blockSize, zds->customMem);
                     if (zds->inBuff == NULL) return ERROR(memory_allocation);
                 }
                 if (zds->outBuffSize < neededOutSize) {
-                    zds->customMem.customFree(zds->customMem.opaque, zds->outBuff);
+                    ZSTD_free(zds->outBuff, zds->customMem);
                     zds->outBuffSize = neededOutSize;
-                    zds->outBuff = (char*)zds->customMem.customAlloc(zds->customMem.opaque, neededOutSize);
+                    zds->outBuff = (char*)ZSTD_malloc(neededOutSize, zds->customMem);
                     if (zds->outBuff == NULL) return ERROR(memory_allocation);
             }   }
             zds->stage = zdss_read;
@@ -1529,9 +1579,3 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
         return nextSrcSizeHint;
     }
 }
-
-
-
-
-
-

lib/dictBuilder/divsufsort.h

@@ -36,8 +36,8 @@ extern "C" {
 
 /**
  * Constructs the suffix array of a given string.
- * @param T[0..n-1] The input string.
+ * @param T [0..n-1] The input string.
- * @param SA[0..n-1] The output array of suffixes.
+ * @param SA [0..n-1] The output array of suffixes.
  * @param n The length of the given string.
  * @param openMP enables OpenMP optimization.
  * @return 0 if no error occurred, -1 or -2 otherwise.
@@ -47,9 +47,9 @@ divsufsort(const unsigned char *T, int *SA, int n, int openMP);
 
 /**
  * Constructs the burrows-wheeler transformed string of a given string.
- * @param T[0..n-1] The input string.
+ * @param T [0..n-1] The input string.
- * @param U[0..n-1] The output string. (can be T)
+ * @param U [0..n-1] The output string. (can be T)
- * @param A[0..n-1] The temporary array. (can be NULL)
+ * @param A [0..n-1] The temporary array. (can be NULL)
  * @param n The length of the given string.
  * @param num_indexes The length of secondary indexes array. (can be NULL)
  * @param indexes The secondary indexes array. (can be NULL)

lib/dictBuilder/zdict.c

@@ -1,40 +1,18 @@
-/*
+/**
-    dictBuilder - dictionary builder for zstd
+ * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
-    Copyright (C) Yann Collet 2016
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
 
-    BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
-
-    Redistribution and use in source and binary forms, with or without
-    modification, are permitted provided that the following conditions are
-    met:
-
-    * Redistributions of source code must retain the above copyright
-    notice, this list of conditions and the following disclaimer.
-    * Redistributions in binary form must reproduce the above
-    copyright notice, this list of conditions and the following disclaimer
-    in the documentation and/or other materials provided with the
-    distribution.
-
-    THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-    "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-    LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-    A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-    OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-    SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-    LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-    DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-    THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-    (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-    OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-    You can contact the author at :
-    - Zstd homepage : https://www.zstd.net
-*/
 
 /*-**************************************
 *  Tuning parameters
 ****************************************/
 #define ZDICT_MAX_SAMPLES_SIZE (2000U << 20)
+#define ZDICT_MIN_SAMPLES_SIZE 512
 
 
 /*-**************************************
@@ -78,11 +56,9 @@
 #define MB *(1 <<20)
 #define GB *(1U<<30)
 
-#define DICTLISTSIZE 10000
+#define DICTLISTSIZE_DEFAULT 10000
 
 #define NOISELENGTH 32
-#define PRIME1   2654435761U
-#define PRIME2   2246822519U
 
 #define MINRATIO 4
 static const int g_compressionLevel_default = 5;
@@ -126,6 +102,13 @@ unsigned ZDICT_isError(size_t errorCode) { return ERR_isError(errorCode); }
 
 const char* ZDICT_getErrorName(size_t errorCode) { return ERR_getErrorName(errorCode); }
 
+unsigned ZDICT_getDictID(const void* dictBuffer, size_t dictSize)
+{
+    if (dictSize < 8) return 0;
+    if (MEM_readLE32(dictBuffer) != ZSTD_DICT_MAGIC) return 0;
+    return MEM_readLE32((const char*)dictBuffer + 4);
+}
+
 
 /*-********************************************************
 *  Dictionary training functions
@@ -553,10 +536,12 @@ _cleanup:
 
 static void ZDICT_fillNoise(void* buffer, size_t length)
 {
-    unsigned acc = PRIME1;
+    unsigned const prime1 = 2654435761U;
+    unsigned const prime2 = 2246822519U;
+    unsigned acc = prime1;
     size_t p=0;;
     for (p=0; p<length; p++) {
-        acc *= PRIME2;
+        acc *= prime2;
         ((unsigned char*)buffer)[p] = (unsigned char)(acc >> 21);
     }
 }
@@ -871,7 +856,6 @@ size_t ZDICT_addEntropyTablesFromBuffer_advanced(void* dictBuffer, size_t dictCo
 }
 
 
-#define DIB_MINSAMPLESSIZE 512
 /*! ZDICT_trainFromBuffer_unsafe() :
 *   Warning : `samplesBuffer` must be followed by noisy guard band.
 *   @return : size of dictionary, or an error code which can be tested with ZDICT_isError()
@@ -881,7 +865,7 @@ size_t ZDICT_trainFromBuffer_unsafe(
                             const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples,
                             ZDICT_params_t params)
 {
-    U32 const dictListSize = MAX(MAX(DICTLISTSIZE, nbSamples), (U32)(maxDictSize/16));
+    U32 const dictListSize = MAX(MAX(DICTLISTSIZE_DEFAULT, nbSamples), (U32)(maxDictSize/16));
     dictItem* const dictList = (dictItem*)malloc(dictListSize * sizeof(*dictList));
     unsigned const selectivity = params.selectivityLevel == 0 ? g_selectivity_default : params.selectivityLevel;
     unsigned const minRep = (selectivity > 30) ? MINRATIO : nbSamples >> selectivity;
@@ -892,7 +876,7 @@ size_t ZDICT_trainFromBuffer_unsafe(
     /* checks */
     if (!dictList) return ERROR(memory_allocation);
     if (maxDictSize <= g_provision_entropySize + g_min_fast_dictContent) { free(dictList); return ERROR(dstSize_tooSmall); }
-    if (samplesBuffSize < DIB_MINSAMPLESSIZE) { free(dictList); return 0; }   /* not enough source to create dictionary */
+    if (samplesBuffSize < ZDICT_MIN_SAMPLES_SIZE) { free(dictList); return 0; }   /* not enough source to create dictionary */
 
     /* init */
     ZDICT_initDictItem(dictList);
@@ -983,7 +967,7 @@ size_t ZDICT_trainFromBuffer_advanced(void* dictBuffer, size_t dictBufferCapacit
     size_t result;
     void* newBuff;
     size_t const sBuffSize = ZDICT_totalSampleSize(samplesSizes, nbSamples);
-    if (sBuffSize < DIB_MINSAMPLESSIZE) return 0;   /* not enough content => no dictionary */
+    if (sBuffSize < ZDICT_MIN_SAMPLES_SIZE) return 0;   /* not enough content => no dictionary */
 
     newBuff = malloc(sBuffSize + NOISELENGTH);
     if (!newBuff) return ERROR(memory_allocation);

lib/dictBuilder/zdict.h

@@ -1,35 +1,11 @@
-/*
+/**
-    dictBuilder header file
+ * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
-    Copyright (C) Yann Collet 2016
+ * All rights reserved.
-
+ *
-    BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+ * This source code is licensed under the BSD-style license found in the
-
+ * LICENSE file in the root directory of this source tree. An additional grant
-    Redistribution and use in source and binary forms, with or without
+ * of patent rights can be found in the PATENTS file in the same directory.
-    modification, are permitted provided that the following conditions are
+ */
-    met:
-
-    * Redistributions of source code must retain the above copyright
-    notice, this list of conditions and the following disclaimer.
-    * Redistributions in binary form must reproduce the above
-    copyright notice, this list of conditions and the following disclaimer
-    in the documentation and/or other materials provided with the
-    distribution.
-
-    THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-    "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-    LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-    A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-    OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-    SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-    LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-    DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-    THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-    (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-    OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-    You can contact the author at :
-       - Zstd source repository : https://www.zstd.net
-*/
 
 #ifndef DICTBUILDER_H_001
 #define DICTBUILDER_H_001
@@ -39,6 +15,10 @@ extern "C" {
 #endif
 
 
+/*======  Dependencies  ======*/
+#include <stddef.h>  /* size_t */
+
+
 /*======  Export for Windows  ======*/
 /*!
 *  ZSTD_DLL_EXPORT :
@@ -68,6 +48,7 @@ ZDICTLIB_API size_t ZDICT_trainFromBuffer(void* dictBuffer, size_t dictBufferCap
 
 
 /*======   Helper functions   ======*/
+ZDICTLIB_API unsigned ZDICT_getDictID(const void* dictBuffer, size_t dictSize);  /**< extracts dictID; @return zero if error (not a valid dictionary) */
 ZDICTLIB_API unsigned ZDICT_isError(size_t errorCode);
 ZDICTLIB_API const char* ZDICT_getErrorName(size_t errorCode);
 

lib/legacy/zstd_legacy.h

@@ -1,35 +1,12 @@
-/*
+/**
-    zstd_legacy - decoder for legacy format
+ * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
-    Header File
+ * All rights reserved.
-    Copyright (C) 2015-2016, Yann Collet.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
 
-    BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
-
-    Redistribution and use in source and binary forms, with or without
-    modification, are permitted provided that the following conditions are
-    met:
-    * Redistributions of source code must retain the above copyright
-    notice, this list of conditions and the following disclaimer.
-    * Redistributions in binary form must reproduce the above
-    copyright notice, this list of conditions and the following disclaimer
-    in the documentation and/or other materials provided with the
-    distribution.
-    THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-    "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-    LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-    A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-    OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-    SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-    LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-    DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-    THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-    (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-    OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-    You can contact the author at :
-    - zstd source repository : https://github.com/Cyan4973/zstd
-    - ztsd public forum : https://groups.google.com/forum/#!forum/lz4c
-*/
 #ifndef ZSTD_LEGACY_H
 #define ZSTD_LEGACY_H
 
@@ -42,6 +19,7 @@ extern "C" {
 ***************************************/
 #include "mem.h"            /* MEM_STATIC */
 #include "error_private.h"  /* ERROR */
+#include "zstd.h"           /* ZSTD_inBuffer, ZSTD_outBuffer */
 #include "zstd_v01.h"
 #include "zstd_v02.h"
 #include "zstd_v03.h"
@@ -76,32 +54,30 @@ MEM_STATIC unsigned ZSTD_isLegacy(const void* src, size_t srcSize)
 
 MEM_STATIC unsigned long long ZSTD_getDecompressedSize_legacy(const void* src, size_t srcSize)
 {
-    if (srcSize < 4) return 0;
+    U32 const version = ZSTD_isLegacy(src, srcSize);
-
+    if (version < 5) return 0;  /* no decompressed size in frame header, or not a legacy format */
-    {   U32 const version = ZSTD_isLegacy(src, srcSize);
+    if (version==5) {
-        if (version < 5) return 0;  /* no decompressed size in frame header, or not a legacy format */
+        ZSTDv05_parameters fParams;
-        if (version==5) {
+        size_t const frResult = ZSTDv05_getFrameParams(&fParams, src, srcSize);
-            ZSTDv05_parameters fParams;
+        if (frResult != 0) return 0;
-            size_t const frResult = ZSTDv05_getFrameParams(&fParams, src, srcSize);
+        return fParams.srcSize;
-            if (frResult != 0) return 0;
-            return fParams.srcSize;
-        }
-        if (version==6) {
-            ZSTDv06_frameParams fParams;
-            size_t const frResult = ZSTDv06_getFrameParams(&fParams, src, srcSize);
-            if (frResult != 0) return 0;
-            return fParams.frameContentSize;
-        }
-        if (version==7) {
-            ZSTDv07_frameParams fParams;
-            size_t const frResult = ZSTDv07_getFrameParams(&fParams, src, srcSize);
-            if (frResult != 0) return 0;
-            return fParams.frameContentSize;
-        }
-        return 0;   /* should not be possible */
     }
+    if (version==6) {
+        ZSTDv06_frameParams fParams;
+        size_t const frResult = ZSTDv06_getFrameParams(&fParams, src, srcSize);
+        if (frResult != 0) return 0;
+        return fParams.frameContentSize;
+    }
+    if (version==7) {
+        ZSTDv07_frameParams fParams;
+        size_t const frResult = ZSTDv07_getFrameParams(&fParams, src, srcSize);
+        if (frResult != 0) return 0;
+        return fParams.frameContentSize;
+    }
+    return 0;   /* should not be possible */
 }
 
+
 MEM_STATIC size_t ZSTD_decompressLegacy(
                      void* dst, size_t dstCapacity,
                const void* src, size_t compressedSize,
@@ -148,6 +124,133 @@ MEM_STATIC size_t ZSTD_decompressLegacy(
 }
 
 
+MEM_STATIC size_t ZSTD_freeLegacyStreamContext(void* legacyContext, U32 version)
+{
+    switch(version)
+    {
+        default :
+        case 1 :
+        case 2 :
+        case 3 :
+            return ERROR(version_unsupported);
+        case 4 : return ZBUFFv04_freeDCtx((ZBUFFv04_DCtx*)legacyContext);
+        case 5 : return ZBUFFv05_freeDCtx((ZBUFFv05_DCtx*)legacyContext);
+        case 6 : return ZBUFFv06_freeDCtx((ZBUFFv06_DCtx*)legacyContext);
+        case 7 : return ZBUFFv07_freeDCtx((ZBUFFv07_DCtx*)legacyContext);
+    }
+}
+
+
+MEM_STATIC size_t ZSTD_initLegacyStream(void** legacyContext, U32 prevVersion, U32 newVersion,
+                                        const void* dict, size_t dictSize)
+{
+    if (prevVersion != newVersion) ZSTD_freeLegacyStreamContext(*legacyContext, prevVersion);
+    switch(newVersion)
+    {
+        default :
+        case 1 :
+        case 2 :
+        case 3 :
+            return 0;
+        case 4 :
+        {
+            ZBUFFv04_DCtx* dctx = (prevVersion != newVersion) ? ZBUFFv04_createDCtx() : (ZBUFFv04_DCtx*)*legacyContext;
+            if (dctx==NULL) return ERROR(memory_allocation);
+            ZBUFFv04_decompressInit(dctx);
+            ZBUFFv04_decompressWithDictionary(dctx, dict, dictSize);
+            *legacyContext = dctx;
+            return 0;
+        }
+        case 5 :
+        {
+            ZBUFFv05_DCtx* dctx = (prevVersion != newVersion) ? ZBUFFv05_createDCtx() : (ZBUFFv05_DCtx*)*legacyContext;
+            if (dctx==NULL) return ERROR(memory_allocation);
+            ZBUFFv05_decompressInitDictionary(dctx, dict, dictSize);
+            *legacyContext = dctx;
+            return 0;
+        }
+        case 6 :
+        {
+            ZBUFFv06_DCtx* dctx = (prevVersion != newVersion) ? ZBUFFv06_createDCtx() : (ZBUFFv06_DCtx*)*legacyContext;
+            if (dctx==NULL) return ERROR(memory_allocation);
+            ZBUFFv06_decompressInitDictionary(dctx, dict, dictSize);
+            *legacyContext = dctx;
+            return 0;
+        }
+        case 7 :
+        {
+            ZBUFFv07_DCtx* dctx = (prevVersion != newVersion) ? ZBUFFv07_createDCtx() : (ZBUFFv07_DCtx*)*legacyContext;
+            if (dctx==NULL) return ERROR(memory_allocation);
+            ZBUFFv07_decompressInitDictionary(dctx, dict, dictSize);
+            *legacyContext = dctx;
+            return 0;
+        }
+    }
+}
+
+
+
+MEM_STATIC size_t ZSTD_decompressLegacyStream(void* legacyContext, U32 version,
+                                              ZSTD_outBuffer* output, ZSTD_inBuffer* input)
+{
+    switch(version)
+    {
+        default :
+        case 1 :
+        case 2 :
+        case 3 :
+            return ERROR(version_unsupported);
+        case 4 :
+            {
+                ZBUFFv04_DCtx* dctx = (ZBUFFv04_DCtx*) legacyContext;
+                const void* src = (const char*)input->src + input->pos;
+                size_t readSize = input->size - input->pos;
+                void* dst = (char*)output->dst + output->pos;
+                size_t decodedSize = output->size - output->pos;
+                size_t const hintSize = ZBUFFv04_decompressContinue(dctx, dst, &decodedSize, src, &readSize);
+                output->pos += decodedSize;
+                input->pos += readSize;
+                return hintSize;
+            }
+        case 5 :
+            {
+                ZBUFFv05_DCtx* dctx = (ZBUFFv05_DCtx*) legacyContext;
+                const void* src = (const char*)input->src + input->pos;
+                size_t readSize = input->size - input->pos;
+                void* dst = (char*)output->dst + output->pos;
+                size_t decodedSize = output->size - output->pos;
+                size_t const hintSize = ZBUFFv05_decompressContinue(dctx, dst, &decodedSize, src, &readSize);
+                output->pos += decodedSize;
+                input->pos += readSize;
+                return hintSize;
+            }
+        case 6 :
+            {
+                ZBUFFv06_DCtx* dctx = (ZBUFFv06_DCtx*) legacyContext;
+                const void* src = (const char*)input->src + input->pos;
+                size_t readSize = input->size - input->pos;
+                void* dst = (char*)output->dst + output->pos;
+                size_t decodedSize = output->size - output->pos;
+                size_t const hintSize = ZBUFFv06_decompressContinue(dctx, dst, &decodedSize, src, &readSize);
+                output->pos += decodedSize;
+                input->pos += readSize;
+                return hintSize;
+            }
+        case 7 :
+            {
+                ZBUFFv07_DCtx* dctx = (ZBUFFv07_DCtx*) legacyContext;
+                const void* src = (const char*)input->src + input->pos;
+                size_t readSize = input->size - input->pos;
+                void* dst = (char*)output->dst + output->pos;
+                size_t decodedSize = output->size - output->pos;
+                size_t const hintSize = ZBUFFv07_decompressContinue(dctx, dst, &decodedSize, src, &readSize);
+                output->pos += decodedSize;
+                input->pos += readSize;
+                return hintSize;
+            }
+    }
+}
+
 
 #if defined (__cplusplus)
 }

lib/legacy/zstd_v01.c

@@ -1,37 +1,12 @@
-/* ******************************************************************
+/**
-   ZSTD_v01
+ * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
-   Zstandard decoder, compatible with v0.1.x format
+ * All rights reserved.
-   Copyright (C) 2013-2015, Yann Collet.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
 
-   BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
-
-   Redistribution and use in source and binary forms, with or without
-   modification, are permitted provided that the following conditions are
-   met:
-
-       * Redistributions of source code must retain the above copyright
-   notice, this list of conditions and the following disclaimer.
-       * Redistributions in binary form must reproduce the above
-   copyright notice, this list of conditions and the following disclaimer
-   in the documentation and/or other materials provided with the
-   distribution.
-
-   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-   You can contact the author at :
-   - Source repository : https://github.com/Cyan4973/FiniteStateEntropy
-   - Public forum : https://groups.google.com/forum/#!forum/lz4c
-****************************************************************** */
 
 /******************************************
 *  Includes
@@ -1685,7 +1660,6 @@ static size_t ZSTD_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* d
         /* Build DTables */
         switch(LLtype)
         {
-        U32 max;
         case bt_rle :
             LLlog = 0;
             FSE_buildDTable_rle(DTableLL, *ip++); break;
@@ -1693,17 +1667,16 @@ static size_t ZSTD_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* d
             LLlog = LLbits;
             FSE_buildDTable_raw(DTableLL, LLbits); break;
         default :
-            max = MaxLL;
+            {   U32 max = MaxLL;
-            headerSize = FSE_readNCount(norm, &max, &LLlog, ip, iend-ip);
+                headerSize = FSE_readNCount(norm, &max, &LLlog, ip, iend-ip);
-            if (FSE_isError(headerSize)) return (size_t)-ZSTD_ERROR_GENERIC;
+                if (FSE_isError(headerSize)) return (size_t)-ZSTD_ERROR_GENERIC;
-            if (LLlog > LLFSELog) return (size_t)-ZSTD_ERROR_corruption;
+                if (LLlog > LLFSELog) return (size_t)-ZSTD_ERROR_corruption;
-            ip += headerSize;
+                ip += headerSize;
-            FSE_buildDTable(DTableLL, norm, max, LLlog);
+                FSE_buildDTable(DTableLL, norm, max, LLlog);
-        }
+        }   }
 
         switch(Offtype)
         {
-        U32 max;
         case bt_rle :
             Offlog = 0;
             if (ip > iend-2) return (size_t)-ZSTD_ERROR_SrcSize; /* min : "raw", hence no header, but at least xxLog bits */
@@ -1712,17 +1685,16 @@ static size_t ZSTD_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* d
             Offlog = Offbits;
             FSE_buildDTable_raw(DTableOffb, Offbits); break;
         default :
-            max = MaxOff;
+            {   U32 max = MaxOff;
-            headerSize = FSE_readNCount(norm, &max, &Offlog, ip, iend-ip);
+                headerSize = FSE_readNCount(norm, &max, &Offlog, ip, iend-ip);
-            if (FSE_isError(headerSize)) return (size_t)-ZSTD_ERROR_GENERIC;
+                if (FSE_isError(headerSize)) return (size_t)-ZSTD_ERROR_GENERIC;
-            if (Offlog > OffFSELog) return (size_t)-ZSTD_ERROR_corruption;
+                if (Offlog > OffFSELog) return (size_t)-ZSTD_ERROR_corruption;
-            ip += headerSize;
+                ip += headerSize;
-            FSE_buildDTable(DTableOffb, norm, max, Offlog);
+                FSE_buildDTable(DTableOffb, norm, max, Offlog);
-        }
+        }   }
 
         switch(MLtype)
         {
-        U32 max;
         case bt_rle :
             MLlog = 0;
             if (ip > iend-2) return (size_t)-ZSTD_ERROR_SrcSize; /* min : "raw", hence no header, but at least xxLog bits */
@@ -1731,14 +1703,13 @@ static size_t ZSTD_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* d
             MLlog = MLbits;
             FSE_buildDTable_raw(DTableML, MLbits); break;
         default :
-            max = MaxML;
+            {   U32 max = MaxML;
-            headerSize = FSE_readNCount(norm, &max, &MLlog, ip, iend-ip);
+                headerSize = FSE_readNCount(norm, &max, &MLlog, ip, iend-ip);
-            if (FSE_isError(headerSize)) return (size_t)-ZSTD_ERROR_GENERIC;
+                if (FSE_isError(headerSize)) return (size_t)-ZSTD_ERROR_GENERIC;
-            if (MLlog > MLFSELog) return (size_t)-ZSTD_ERROR_corruption;
+                if (MLlog > MLFSELog) return (size_t)-ZSTD_ERROR_corruption;
-            ip += headerSize;
+                ip += headerSize;
-            FSE_buildDTable(DTableML, norm, max, MLlog);
+                FSE_buildDTable(DTableML, norm, max, MLlog);
-        }
+    }   }   }
-    }
 
     return ip-istart;
 }
@@ -2174,5 +2145,3 @@ size_t ZSTDv01_decompressContinue(ZSTDv01_Dctx* dctx, void* dst, size_t maxDstSi
     }
 
 }
-
-

lib/legacy/zstd_v01.h

@@ -1,36 +1,14 @@
-/*
+/**
-    zstd - standard compression library
+ * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
-    Header File
+ * All rights reserved.
-    Copyright (C) 2014-2015, Yann Collet.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
 
-    BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+#ifndef ZSTD_V01_H_28739879432
-
+#define ZSTD_V01_H_28739879432
-    Redistribution and use in source and binary forms, with or without
-    modification, are permitted provided that the following conditions are
-    met:
-    * Redistributions of source code must retain the above copyright
-    notice, this list of conditions and the following disclaimer.
-    * Redistributions in binary form must reproduce the above
-    copyright notice, this list of conditions and the following disclaimer
-    in the documentation and/or other materials provided with the
-    distribution.
-    THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-    "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-    LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-    A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-    OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-    SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-    LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-    DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-    THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-    (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-    OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-    You can contact the author at :
-    - zstd source repository : https://github.com/Cyan4973/zstd
-    - ztsd public forum : https://groups.google.com/forum/#!forum/lz4c
-*/
-#pragma once
 
 #if defined (__cplusplus)
 extern "C" {
@@ -98,3 +76,5 @@ size_t ZSTDv01_decompressContinue(ZSTDv01_Dctx* dctx, void* dst, size_t maxDstSi
 #if defined (__cplusplus)
 }
 #endif
+
+#endif /* ZSTD_V01_H_28739879432 */

lib/legacy/zstd_v02.c

@@ -1,36 +1,12 @@
-/* ******************************************************************
+/**
-   Error codes and messages
+ * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
-   Copyright (C) 2013-2015, Yann Collet
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
 
-   BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
-
-   Redistribution and use in source and binary forms, with or without
-   modification, are permitted provided that the following conditions are
-   met:
-
-       * Redistributions of source code must retain the above copyright
-   notice, this list of conditions and the following disclaimer.
-       * Redistributions in binary form must reproduce the above
-   copyright notice, this list of conditions and the following disclaimer
-   in the documentation and/or other materials provided with the
-   distribution.
-
-   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-   You can contact the author at :
-   - Source repository : https://github.com/Cyan4973/FiniteStateEntropy
-   - Public forum : https://groups.google.com/forum/#!forum/lz4c
-****************************************************************** */
 #ifndef ERROR_H_MODULE
 #define ERROR_H_MODULE
 
@@ -3100,7 +3076,6 @@ static size_t ZSTD_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* d
         /* Build DTables */
         switch(LLtype)
         {
-        U32 max;
         case bt_rle :
             LLlog = 0;
             FSE_buildDTable_rle(DTableLL, *ip++); break;
@@ -3108,17 +3083,16 @@ static size_t ZSTD_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* d
             LLlog = LLbits;
             FSE_buildDTable_raw(DTableLL, LLbits); break;
         default :
-            max = MaxLL;
+            {   U32 max = MaxLL;
-            headerSize = FSE_readNCount(norm, &max, &LLlog, ip, iend-ip);
+                headerSize = FSE_readNCount(norm, &max, &LLlog, ip, iend-ip);
-            if (FSE_isError(headerSize)) return ERROR(GENERIC);
+                if (FSE_isError(headerSize)) return ERROR(GENERIC);
-            if (LLlog > LLFSELog) return ERROR(corruption_detected);
+                if (LLlog > LLFSELog) return ERROR(corruption_detected);
-            ip += headerSize;
+                ip += headerSize;
-            FSE_buildDTable(DTableLL, norm, max, LLlog);
+                FSE_buildDTable(DTableLL, norm, max, LLlog);
-        }
+        }   }
 
         switch(Offtype)
         {
-        U32 max;
         case bt_rle :
             Offlog = 0;
             if (ip > iend-2) return ERROR(srcSize_wrong);   /* min : "raw", hence no header, but at least xxLog bits */
@@ -3128,17 +3102,16 @@ static size_t ZSTD_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* d
             Offlog = Offbits;
             FSE_buildDTable_raw(DTableOffb, Offbits); break;
         default :
-            max = MaxOff;
+            {   U32 max = MaxOff;
-            headerSize = FSE_readNCount(norm, &max, &Offlog, ip, iend-ip);
+                headerSize = FSE_readNCount(norm, &max, &Offlog, ip, iend-ip);
-            if (FSE_isError(headerSize)) return ERROR(GENERIC);
+                if (FSE_isError(headerSize)) return ERROR(GENERIC);
-            if (Offlog > OffFSELog) return ERROR(corruption_detected);
+                if (Offlog > OffFSELog) return ERROR(corruption_detected);
-            ip += headerSize;
+                ip += headerSize;
-            FSE_buildDTable(DTableOffb, norm, max, Offlog);
+                FSE_buildDTable(DTableOffb, norm, max, Offlog);
-        }
+        }   }
 
         switch(MLtype)
         {
-        U32 max;
         case bt_rle :
             MLlog = 0;
             if (ip > iend-2) return ERROR(srcSize_wrong); /* min : "raw", hence no header, but at least xxLog bits */
@@ -3147,14 +3120,13 @@ static size_t ZSTD_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* d
             MLlog = MLbits;
             FSE_buildDTable_raw(DTableML, MLbits); break;
         default :
-            max = MaxML;
+            {   U32 max = MaxML;
-            headerSize = FSE_readNCount(norm, &max, &MLlog, ip, iend-ip);
+                headerSize = FSE_readNCount(norm, &max, &MLlog, ip, iend-ip);
-            if (FSE_isError(headerSize)) return ERROR(GENERIC);
+                if (FSE_isError(headerSize)) return ERROR(GENERIC);
-            if (MLlog > MLFSELog) return ERROR(corruption_detected);
+                if (MLlog > MLFSELog) return ERROR(corruption_detected);
-            ip += headerSize;
+                ip += headerSize;
-            FSE_buildDTable(DTableML, norm, max, MLlog);
+                FSE_buildDTable(DTableML, norm, max, MLlog);
-        }
+    }   }   }
-    }
 
     return ip-istart;
 }

lib/legacy/zstd_v02.h

@@ -1,36 +1,14 @@
-/*
+/**
-    zstd_v02 - decoder for 0.2 format
+ * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
-    Header File
+ * All rights reserved.
-    Copyright (C) 2015, Yann Collet.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
 
-    BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+#ifndef ZSTD_V02_H_4174539423
-
+#define ZSTD_V02_H_4174539423
-    Redistribution and use in source and binary forms, with or without
-    modification, are permitted provided that the following conditions are
-    met:
-    * Redistributions of source code must retain the above copyright
-    notice, this list of conditions and the following disclaimer.
-    * Redistributions in binary form must reproduce the above
-    copyright notice, this list of conditions and the following disclaimer
-    in the documentation and/or other materials provided with the
-    distribution.
-    THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-    "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-    LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-    A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-    OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-    SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-    LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-    DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-    THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-    (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-    OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-    You can contact the author at :
-    - zstd source repository : https://github.com/Cyan4973/zstd
-    - ztsd public forum : https://groups.google.com/forum/#!forum/lz4c
-*/
-#pragma once
 
 #if defined (__cplusplus)
 extern "C" {
@@ -97,3 +75,5 @@ size_t ZSTDv02_decompressContinue(ZSTDv02_Dctx* dctx, void* dst, size_t maxDstSi
 #if defined (__cplusplus)
 }
 #endif
+
+#endif /* ZSTD_V02_H_4174539423 */

lib/legacy/zstd_v03.c

@@ -1,36 +1,12 @@
-/* ******************************************************************
+/**
-   Error codes and messages
+ * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
-   Copyright (C) 2013-2015, Yann Collet
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
 
-   BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
-
-   Redistribution and use in source and binary forms, with or without
-   modification, are permitted provided that the following conditions are
-   met:
-
-       * Redistributions of source code must retain the above copyright
-   notice, this list of conditions and the following disclaimer.
-       * Redistributions in binary form must reproduce the above
-   copyright notice, this list of conditions and the following disclaimer
-   in the documentation and/or other materials provided with the
-   distribution.
-
-   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-   You can contact the author at :
-   - Source repository : https://github.com/Cyan4973/FiniteStateEntropy
-   - Public forum : https://groups.google.com/forum/#!forum/lz4c
-****************************************************************** */
 #ifndef ERROR_H_MODULE
 #define ERROR_H_MODULE
 
@@ -2742,7 +2718,6 @@ static size_t ZSTD_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* d
         /* Build DTables */
         switch(LLtype)
         {
-        U32 max;
         case bt_rle :
             LLlog = 0;
             FSE_buildDTable_rle(DTableLL, *ip++); break;
@@ -2750,17 +2725,16 @@ static size_t ZSTD_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* d
             LLlog = LLbits;
             FSE_buildDTable_raw(DTableLL, LLbits); break;
         default :
-            max = MaxLL;
+            {   U32 max = MaxLL;
-            headerSize = FSE_readNCount(norm, &max, &LLlog, ip, iend-ip);
+                headerSize = FSE_readNCount(norm, &max, &LLlog, ip, iend-ip);
-            if (FSE_isError(headerSize)) return ERROR(GENERIC);
+                if (FSE_isError(headerSize)) return ERROR(GENERIC);
-            if (LLlog > LLFSELog) return ERROR(corruption_detected);
+                if (LLlog > LLFSELog) return ERROR(corruption_detected);
-            ip += headerSize;
+                ip += headerSize;
-            FSE_buildDTable(DTableLL, norm, max, LLlog);
+                FSE_buildDTable(DTableLL, norm, max, LLlog);
-        }
+        }   }
 
         switch(Offtype)
         {
-        U32 max;
         case bt_rle :
             Offlog = 0;
             if (ip > iend-2) return ERROR(srcSize_wrong);   /* min : "raw", hence no header, but at least xxLog bits */
@@ -2770,17 +2744,16 @@ static size_t ZSTD_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* d
             Offlog = Offbits;
             FSE_buildDTable_raw(DTableOffb, Offbits); break;
         default :
-            max = MaxOff;
+            {   U32 max = MaxOff;
-            headerSize = FSE_readNCount(norm, &max, &Offlog, ip, iend-ip);
+                headerSize = FSE_readNCount(norm, &max, &Offlog, ip, iend-ip);
-            if (FSE_isError(headerSize)) return ERROR(GENERIC);
+                if (FSE_isError(headerSize)) return ERROR(GENERIC);
-            if (Offlog > OffFSELog) return ERROR(corruption_detected);
+                if (Offlog > OffFSELog) return ERROR(corruption_detected);
-            ip += headerSize;
+                ip += headerSize;
-            FSE_buildDTable(DTableOffb, norm, max, Offlog);
+                FSE_buildDTable(DTableOffb, norm, max, Offlog);
-        }
+        }   }
 
         switch(MLtype)
         {
-        U32 max;
         case bt_rle :
             MLlog = 0;
             if (ip > iend-2) return ERROR(srcSize_wrong); /* min : "raw", hence no header, but at least xxLog bits */
@@ -2789,14 +2762,13 @@ static size_t ZSTD_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* d
             MLlog = MLbits;
             FSE_buildDTable_raw(DTableML, MLbits); break;
         default :
-            max = MaxML;
+            {   U32 max = MaxML;
-            headerSize = FSE_readNCount(norm, &max, &MLlog, ip, iend-ip);
+                headerSize = FSE_readNCount(norm, &max, &MLlog, ip, iend-ip);
-            if (FSE_isError(headerSize)) return ERROR(GENERIC);
+                if (FSE_isError(headerSize)) return ERROR(GENERIC);
-            if (MLlog > MLFSELog) return ERROR(corruption_detected);
+                if (MLlog > MLFSELog) return ERROR(corruption_detected);
-            ip += headerSize;
+                ip += headerSize;
-            FSE_buildDTable(DTableML, norm, max, MLlog);
+                FSE_buildDTable(DTableML, norm, max, MLlog);
-        }
+    }   }   }
-    }
 
     return ip-istart;
 }

lib/legacy/zstd_v03.h

@@ -1,36 +1,14 @@
-/*
+/**
-    zstd_v03 - decoder for 0.3 format
+ * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
-    Header File
+ * All rights reserved.
-    Copyright (C) 2015, Yann Collet.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
 
-    BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+#ifndef ZSTD_V03_H_298734209782
-
+#define ZSTD_V03_H_298734209782
-    Redistribution and use in source and binary forms, with or without
-    modification, are permitted provided that the following conditions are
-    met:
-    * Redistributions of source code must retain the above copyright
-    notice, this list of conditions and the following disclaimer.
-    * Redistributions in binary form must reproduce the above
-    copyright notice, this list of conditions and the following disclaimer
-    in the documentation and/or other materials provided with the
-    distribution.
-    THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-    "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-    LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-    A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-    OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-    SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-    LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-    DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-    THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-    (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-    OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-    You can contact the author at :
-    - zstd source repository : https://github.com/Cyan4973/zstd
-    - ztsd public forum : https://groups.google.com/forum/#!forum/lz4c
-*/
-#pragma once
 
 #if defined (__cplusplus)
 extern "C" {
@@ -97,3 +75,5 @@ size_t ZSTDv03_decompressContinue(ZSTDv03_Dctx* dctx, void* dst, size_t maxDstSi
 #if defined (__cplusplus)
 }
 #endif
+
+#endif /* ZSTD_V03_H_298734209782 */

lib/legacy/zstd_v04.c

@@ -1,36 +1,12 @@
-/* ******************************************************************
+/**
-   zstd_v04.c
+ * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
-   Decompression module for ZSTD v0.4 legacy format
+ * All rights reserved.
-   Copyright (C) 2016, Yann Collet.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
 
-   BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
-
-   Redistribution and use in source and binary forms, with or without
-   modification, are permitted provided that the following conditions are
-   met:
-
-       * Redistributions of source code must retain the above copyright
-   notice, this list of conditions and the following disclaimer.
-       * Redistributions in binary form must reproduce the above
-   copyright notice, this list of conditions and the following disclaimer
-   in the documentation and/or other materials provided with the
-   distribution.
-
-   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-    You can contact the author at :
-    - Homepage : http://www.zstd.net/
-****************************************************************** */
 
 /*- Dependencies -*/
 #include "zstd_v04.h"
@@ -3145,7 +3121,6 @@ static size_t ZSTD_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* d
         /* Build DTables */
         switch(LLtype)
         {
-        U32 max;
         case bt_rle :
             LLlog = 0;
             FSE_buildDTable_rle(DTableLL, *ip++); break;
@@ -3153,17 +3128,16 @@ static size_t ZSTD_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* d
             LLlog = LLbits;
             FSE_buildDTable_raw(DTableLL, LLbits); break;
         default :
-            max = MaxLL;
+            {   U32 max = MaxLL;
-            headerSize = FSE_readNCount(norm, &max, &LLlog, ip, iend-ip);
+                headerSize = FSE_readNCount(norm, &max, &LLlog, ip, iend-ip);
-            if (FSE_isError(headerSize)) return ERROR(GENERIC);
+                if (FSE_isError(headerSize)) return ERROR(GENERIC);
-            if (LLlog > LLFSELog) return ERROR(corruption_detected);
+                if (LLlog > LLFSELog) return ERROR(corruption_detected);
-            ip += headerSize;
+                ip += headerSize;
-            FSE_buildDTable(DTableLL, norm, max, LLlog);
+                FSE_buildDTable(DTableLL, norm, max, LLlog);
-        }
+        }   }
 
         switch(Offtype)
         {
-        U32 max;
         case bt_rle :
             Offlog = 0;
             if (ip > iend-2) return ERROR(srcSize_wrong);   /* min : "raw", hence no header, but at least xxLog bits */
@@ -3173,17 +3147,16 @@ static size_t ZSTD_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* d
             Offlog = Offbits;
             FSE_buildDTable_raw(DTableOffb, Offbits); break;
         default :
-            max = MaxOff;
+            {   U32 max = MaxOff;
-            headerSize = FSE_readNCount(norm, &max, &Offlog, ip, iend-ip);
+                headerSize = FSE_readNCount(norm, &max, &Offlog, ip, iend-ip);
-            if (FSE_isError(headerSize)) return ERROR(GENERIC);
+                if (FSE_isError(headerSize)) return ERROR(GENERIC);
-            if (Offlog > OffFSELog) return ERROR(corruption_detected);
+                if (Offlog > OffFSELog) return ERROR(corruption_detected);
-            ip += headerSize;
+                ip += headerSize;
-            FSE_buildDTable(DTableOffb, norm, max, Offlog);
+                FSE_buildDTable(DTableOffb, norm, max, Offlog);
-        }
+        }   }
 
         switch(MLtype)
         {
-        U32 max;
         case bt_rle :
             MLlog = 0;
             if (ip > iend-2) return ERROR(srcSize_wrong); /* min : "raw", hence no header, but at least xxLog bits */
@@ -3192,14 +3165,13 @@ static size_t ZSTD_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* d
             MLlog = MLbits;
             FSE_buildDTable_raw(DTableML, MLbits); break;
         default :
-            max = MaxML;
+            {   U32 max = MaxML;
-            headerSize = FSE_readNCount(norm, &max, &MLlog, ip, iend-ip);
+                headerSize = FSE_readNCount(norm, &max, &MLlog, ip, iend-ip);
-            if (FSE_isError(headerSize)) return ERROR(GENERIC);
+                if (FSE_isError(headerSize)) return ERROR(GENERIC);
-            if (MLlog > MLFSELog) return ERROR(corruption_detected);
+                if (MLlog > MLFSELog) return ERROR(corruption_detected);
-            ip += headerSize;
+                ip += headerSize;
-            FSE_buildDTable(DTableML, norm, max, MLlog);
+                FSE_buildDTable(DTableML, norm, max, MLlog);
-        }
+    }   }   }
-    }
 
     return ip-istart;
 }
@@ -3701,7 +3673,7 @@ static void ZSTD_decompress_insertDictionary(ZSTD_DCtx* ctx, const void* dict, s
 *  The function will report how many bytes were read or written by modifying *srcSizePtr and *maxDstSizePtr.
 *  Note that it may not consume the entire input, in which case it's up to the caller to call again the function with remaining input.
 *  The content of dst will be overwritten (up to *maxDstSizePtr) at each function call, so save its content if it matters or change dst .
-*  @return : a hint to preferred nb of bytes to use as input for next function call (it's only a hint, to improve latency)
+*  return : a hint to preferred nb of bytes to use as input for next function call (it's only a hint, to improve latency)
 *            or 0 when a frame is completely decoded
 *            or an error code, which can be tested using ZBUFF_isError().
 *

lib/legacy/zstd_v04.h

@@ -1,36 +1,14 @@
-/*
+/**
-    zstd_v04 - decoder for 0.4 format
+ * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
-    Header File
+ * All rights reserved.
-    Copyright (C) 2016, Yann Collet.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
 
-    BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+#ifndef ZSTD_V04_H_91868324769238
-
+#define ZSTD_V04_H_91868324769238
-    Redistribution and use in source and binary forms, with or without
-    modification, are permitted provided that the following conditions are
-    met:
-    * Redistributions of source code must retain the above copyright
-    notice, this list of conditions and the following disclaimer.
-    * Redistributions in binary form must reproduce the above
-    copyright notice, this list of conditions and the following disclaimer
-    in the documentation and/or other materials provided with the
-    distribution.
-    THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-    "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-    LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-    A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-    OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-    SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-    LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-    DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-    THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-    (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-    OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-    You can contact the author at :
-    - zstd source repository : https://github.com/Cyan4973/zstd
-    - ztsd public forum : https://groups.google.com/forum/#!forum/lz4c
-*/
-#pragma once
 
 #if defined (__cplusplus)
 extern "C" {
@@ -146,3 +124,5 @@ size_t ZBUFFv04_recommendedDOutSize(void);
 #if defined (__cplusplus)
 }
 #endif
+
+#endif /* ZSTD_V04_H_91868324769238 */

lib/legacy/zstd_v05.c

@@ -1,36 +1,12 @@
-/* ******************************************************************
+/**
-   zstd_v05.c
+ * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
-   Decompression module for ZSTD v0.5 legacy format
+ * All rights reserved.
-   Copyright (C) 2016, Yann Collet.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
 
-   BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
-
-   Redistribution and use in source and binary forms, with or without
-   modification, are permitted provided that the following conditions are
-   met:
-
-       * Redistributions of source code must retain the above copyright
-   notice, this list of conditions and the following disclaimer.
-       * Redistributions in binary form must reproduce the above
-   copyright notice, this list of conditions and the following disclaimer
-   in the documentation and/or other materials provided with the
-   distribution.
-
-   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-    You can contact the author at :
-    - Homepage : http://www.zstd.net/
-****************************************************************** */
 
 /*- Dependencies -*/
 #include "zstd_v05.h"
@@ -859,16 +835,16 @@ void        FSEv05_freeDTable(FSEv05_DTable* dt);
 /*!
 FSEv05_buildDTable():
    Builds 'dt', which must be already allocated, using FSEv05_createDTable()
-   return : 0,
+   @return : 0,
-            or an errorCode, which can be tested using FSEv05_isError() */
+             or an errorCode, which can be tested using FSEv05_isError() */
 size_t FSEv05_buildDTable (FSEv05_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog);
 
 /*!
 FSEv05_decompress_usingDTable():
-   Decompress compressed source @cSrc of size @cSrcSize using @dt
+   Decompress compressed source @cSrc of size @cSrcSize using `dt`
-   into @dst which must be already allocated.
+   into `dst` which must be already allocated.
-   return : size of regenerated data (necessarily <= @dstCapacity)
+   @return : size of regenerated data (necessarily <= @dstCapacity)
-            or an errorCode, which can be tested using FSEv05_isError() */
+             or an errorCode, which can be tested using FSEv05_isError() */
 size_t FSEv05_decompress_usingDTable(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, const FSEv05_DTable* dt);
 
 
@@ -3333,7 +3309,6 @@ size_t ZSTDv05_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* dumps
         /* Build DTables */
         switch(LLtype)
         {
-        U32 max;
         case FSEv05_ENCODING_RLE :
             LLlog = 0;
             FSEv05_buildDTable_rle(DTableLL, *ip++);
@@ -3346,17 +3321,16 @@ size_t ZSTDv05_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* dumps
             break;
         case FSEv05_ENCODING_DYNAMIC :
         default :   /* impossible */
-            max = MaxLL;
+            {   U32 max = MaxLL;
-            headerSize = FSEv05_readNCount(norm, &max, &LLlog, ip, iend-ip);
+                headerSize = FSEv05_readNCount(norm, &max, &LLlog, ip, iend-ip);
-            if (FSEv05_isError(headerSize)) return ERROR(GENERIC);
+                if (FSEv05_isError(headerSize)) return ERROR(GENERIC);
-            if (LLlog > LLFSEv05Log) return ERROR(corruption_detected);
+                if (LLlog > LLFSEv05Log) return ERROR(corruption_detected);
-            ip += headerSize;
+                ip += headerSize;
-            FSEv05_buildDTable(DTableLL, norm, max, LLlog);
+                FSEv05_buildDTable(DTableLL, norm, max, LLlog);
-        }
+        }   }
 
         switch(Offtype)
         {
-        U32 max;
         case FSEv05_ENCODING_RLE :
             Offlog = 0;
             if (ip > iend-2) return ERROR(srcSize_wrong);   /* min : "raw", hence no header, but at least xxLog bits */
@@ -3370,17 +3344,16 @@ size_t ZSTDv05_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* dumps
             break;
         case FSEv05_ENCODING_DYNAMIC :
         default :   /* impossible */
-            max = MaxOff;
+            {   U32 max = MaxOff;
-            headerSize = FSEv05_readNCount(norm, &max, &Offlog, ip, iend-ip);
+                headerSize = FSEv05_readNCount(norm, &max, &Offlog, ip, iend-ip);
-            if (FSEv05_isError(headerSize)) return ERROR(GENERIC);
+                if (FSEv05_isError(headerSize)) return ERROR(GENERIC);
-            if (Offlog > OffFSEv05Log) return ERROR(corruption_detected);
+                if (Offlog > OffFSEv05Log) return ERROR(corruption_detected);
-            ip += headerSize;
+                ip += headerSize;
-            FSEv05_buildDTable(DTableOffb, norm, max, Offlog);
+                FSEv05_buildDTable(DTableOffb, norm, max, Offlog);
-        }
+        }   }
 
         switch(MLtype)
         {
-        U32 max;
         case FSEv05_ENCODING_RLE :
             MLlog = 0;
             if (ip > iend-2) return ERROR(srcSize_wrong); /* min : "raw", hence no header, but at least xxLog bits */
@@ -3394,13 +3367,13 @@ size_t ZSTDv05_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* dumps
             break;
         case FSEv05_ENCODING_DYNAMIC :
         default :   /* impossible */
-            max = MaxML;
+            {   U32 max = MaxML;
-            headerSize = FSEv05_readNCount(norm, &max, &MLlog, ip, iend-ip);
+                headerSize = FSEv05_readNCount(norm, &max, &MLlog, ip, iend-ip);
-            if (FSEv05_isError(headerSize)) return ERROR(GENERIC);
+                if (FSEv05_isError(headerSize)) return ERROR(GENERIC);
-            if (MLlog > MLFSEv05Log) return ERROR(corruption_detected);
+                if (MLlog > MLFSEv05Log) return ERROR(corruption_detected);
-            ip += headerSize;
+                ip += headerSize;
-            FSEv05_buildDTable(DTableML, norm, max, MLlog);
+                FSEv05_buildDTable(DTableML, norm, max, MLlog);
-    }   }
+    }   }   }
 
     return ip-istart;
 }
@@ -4030,7 +4003,7 @@ static size_t ZBUFFv05_limitCopy(void* dst, size_t maxDstSize, const void* src,
 *  The function will report how many bytes were read or written by modifying *srcSizePtr and *maxDstSizePtr.
 *  Note that it may not consume the entire input, in which case it's up to the caller to call again the function with remaining input.
 *  The content of dst will be overwritten (up to *maxDstSizePtr) at each function call, so save its content if it matters or change dst .
-*  @return : a hint to preferred nb of bytes to use as input for next function call (it's only a hint, to improve latency)
+*  return : a hint to preferred nb of bytes to use as input for next function call (it's only a hint, to improve latency)
 *            or 0 when a frame is completely decoded
 *            or an error code, which can be tested using ZBUFFv05_isError().
 *

lib/legacy/zstd_v05.h

@@ -1,34 +1,12 @@
-/*
+/**
-    zstd_v05 - decoder for 0.5 format
+ * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
-    Header File
+ * All rights reserved.
-    Copyright (C) 2014-2016, Yann Collet.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
 
-    BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
-
-    Redistribution and use in source and binary forms, with or without
-    modification, are permitted provided that the following conditions are
-    met:
-    * Redistributions of source code must retain the above copyright
-    notice, this list of conditions and the following disclaimer.
-    * Redistributions in binary form must reproduce the above
-    copyright notice, this list of conditions and the following disclaimer
-    in the documentation and/or other materials provided with the
-    distribution.
-    THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-    "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-    LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-    A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-    OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-    SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-    LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-    DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-    THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-    (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-    OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-    You can contact the author at :
-    - zstd source repository : https://github.com/Cyan4973/zstd
-*/
 #ifndef ZSTDv05_H
 #define ZSTDv05_H
 

lib/legacy/zstd_v06.c

@@ -1,36 +1,12 @@
-/* ******************************************************************
+/**
-   zstd_v06.c
+ * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
-   Decompression module for ZSTD v0.6 legacy format
+ * All rights reserved.
-   Copyright (C) 2016, Yann Collet.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
 
-   BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
-
-   Redistribution and use in source and binary forms, with or without
-   modification, are permitted provided that the following conditions are
-   met:
-
-       * Redistributions of source code must retain the above copyright
-   notice, this list of conditions and the following disclaimer.
-       * Redistributions in binary form must reproduce the above
-   copyright notice, this list of conditions and the following disclaimer
-   in the documentation and/or other materials provided with the
-   distribution.
-
-   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-    You can contact the author at :
-    - Homepage : http://www.zstd.net/
-****************************************************************** */
 
 /*- Dependencies -*/
 #include "zstd_v06.h"
@@ -669,19 +645,6 @@ ERR_STATIC const char* ERR_getErrorName(size_t code)
 /*-*************************************
 *  Common constants
 ***************************************/
-#define ZSTDv06_OPT_DEBUG 0     // 3 = compression stats;  5 = check encoded sequences;  9 = full logs
-#include <stdio.h>
-#if defined(ZSTDv06_OPT_DEBUG) && ZSTDv06_OPT_DEBUG>=9
-    #define ZSTDv06_LOG_PARSER(...) printf(__VA_ARGS__)
-    #define ZSTDv06_LOG_ENCODE(...) printf(__VA_ARGS__)
-    #define ZSTDv06_LOG_BLOCK(...) printf(__VA_ARGS__)
-#else
-    #define ZSTDv06_LOG_PARSER(...)
-    #define ZSTDv06_LOG_ENCODE(...)
-    #define ZSTDv06_LOG_BLOCK(...)
-#endif
-
-#define ZSTDv06_OPT_NUM    (1<<12)
 #define ZSTDv06_DICT_MAGIC  0xEC30A436
 
 #define ZSTDv06_REP_NUM    3
@@ -1396,7 +1359,7 @@ static unsigned char FSEv06_decodeSymbolFast(FSEv06_DState_t* DStatePtr, BITv06_
 *******************************************/
 
 
-/*<=====    Decompression    =====>*/
+/* ======    Decompression    ====== */
 
 typedef struct {
     U16 tableLog;

lib/legacy/zstd_v06.h

@@ -1,34 +1,12 @@
-/*
+/**
-    zstd_v06 - decoder for 0.6 format
+ * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
-    Header File
+ * All rights reserved.
-    Copyright (C) 2014-2016, Yann Collet.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
 
-    BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
-
-    Redistribution and use in source and binary forms, with or without
-    modification, are permitted provided that the following conditions are
-    met:
-    * Redistributions of source code must retain the above copyright
-    notice, this list of conditions and the following disclaimer.
-    * Redistributions in binary form must reproduce the above
-    copyright notice, this list of conditions and the following disclaimer
-    in the documentation and/or other materials provided with the
-    distribution.
-    THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-    "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-    LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-    A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-    OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-    SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-    LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-    DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-    THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-    (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-    OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-    You can contact the author at :
-    - zstd source repository : https://github.com/Cyan4973/zstd
-*/
 #ifndef ZSTDv06_H
 #define ZSTDv06_H
 

lib/legacy/zstd_v07.c

@@ -1,36 +1,12 @@
-/* ******************************************************************
+/**
-   zstd_v07.c
+ * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
-   Decompression module for ZSTD v0.7 legacy format
+ * All rights reserved.
-   Copyright (C) 2016, Yann Collet.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
 
-   BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
-
-   Redistribution and use in source and binary forms, with or without
-   modification, are permitted provided that the following conditions are
-   met:
-
-       * Redistributions of source code must retain the above copyright
-   notice, this list of conditions and the following disclaimer.
-       * Redistributions in binary form must reproduce the above
-   copyright notice, this list of conditions and the following disclaimer
-   in the documentation and/or other materials provided with the
-   distribution.
-
-   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-    You can contact the author at :
-    - Homepage : http://www.zstd.net/
-****************************************************************** */
 
 /*- Dependencies -*/
 #include <stddef.h>     /* size_t, ptrdiff_t */
@@ -1147,7 +1123,7 @@ static unsigned char FSEv07_decodeSymbolFast(FSEv07_DState_t* DStatePtr, BITv07_
 /* faster, but works only if nbBits is always >= 1 (otherwise, result will be corrupted) */
 
 
-/*<=====    Decompression    =====>*/
+/* ======    Decompression    ====== */
 
 typedef struct {
     U16 tableLog;
@@ -2989,18 +2965,6 @@ void ZSTDv07_defaultFreeFunction(void* opaque, void* address)
 /*-*************************************
 *  Common constants
 ***************************************/
-#define ZSTDv07_OPT_DEBUG 0     /* 3 = compression stats;  5 = check encoded sequences;  9 = full logs */
-#include <stdio.h>
-#if defined(ZSTDv07_OPT_DEBUG) && ZSTDv07_OPT_DEBUG>=9
-    #define ZSTDv07_LOG_PARSER(...) printf(__VA_ARGS__)
-    #define ZSTDv07_LOG_ENCODE(...) printf(__VA_ARGS__)
-    #define ZSTDv07_LOG_BLOCK(...) printf(__VA_ARGS__)
-#else
-    #define ZSTDv07_LOG_PARSER(...)
-    #define ZSTDv07_LOG_ENCODE(...)
-    #define ZSTDv07_LOG_BLOCK(...)
-#endif
-
 #define ZSTDv07_OPT_NUM    (1<<12)
 #define ZSTDv07_DICT_MAGIC  0xEC30A437   /* v0.7 */
 
@@ -4655,9 +4619,9 @@ size_t ZBUFFv07_decompressContinue(ZBUFFv07_DCtx* zbd,
 
         case ZBUFFds_loadHeader :
             {   size_t const hSize = ZSTDv07_getFrameParams(&(zbd->fParams), zbd->headerBuffer, zbd->lhSize);
+                if (ZSTDv07_isError(hSize)) return hSize;
                 if (hSize != 0) {
                     size_t const toLoad = hSize - zbd->lhSize;   /* if hSize!=0, hSize > zbd->lhSize */
-                    if (ZSTDv07_isError(hSize)) return hSize;
                     if (toLoad > (size_t)(iend-ip)) {   /* not enough input to load full header */
                         memcpy(zbd->headerBuffer + zbd->lhSize, ip, iend-ip);
                         zbd->lhSize += iend-ip;
@@ -4697,6 +4661,7 @@ size_t ZBUFFv07_decompressContinue(ZBUFFv07_DCtx* zbd,
                         if (zbd->outBuff == NULL) return ERROR(memory_allocation);
             }   }   }
             zbd->stage = ZBUFFds_read;
+            /* pass-through */
 
         case ZBUFFds_read:
             {   size_t const neededInSize = ZSTDv07_nextSrcSizeToDecompress(zbd->zd);

lib/legacy/zstd_v07.h

@@ -1,34 +1,12 @@
-/*
+/**
-    zstd_v07 - decoder for 0.7 format
+ * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
-    Header File
+ * All rights reserved.
-    Copyright (C) 2014-2016, Yann Collet.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
 
-    BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
-
-    Redistribution and use in source and binary forms, with or without
-    modification, are permitted provided that the following conditions are
-    met:
-    * Redistributions of source code must retain the above copyright
-    notice, this list of conditions and the following disclaimer.
-    * Redistributions in binary form must reproduce the above
-    copyright notice, this list of conditions and the following disclaimer
-    in the documentation and/or other materials provided with the
-    distribution.
-    THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-    "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-    LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-    A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-    OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-    SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-    LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-    DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-    THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-    (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-    OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-    You can contact the author at :
-    - zstd source repository : https://github.com/Cyan4973/zstd
-*/
 #ifndef ZSTDv07_H_235446
 #define ZSTDv07_H_235446
 

lib/zstd.h

@@ -1,34 +1,12 @@
-/*
+/**
-    zstd - standard compression library
+ * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
-    Header File
+ * All rights reserved.
-    Copyright (C) 2014-2016, Yann Collet.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
 
-    BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
-
-    Redistribution and use in source and binary forms, with or without
-    modification, are permitted provided that the following conditions are
-    met:
-    * Redistributions of source code must retain the above copyright
-    notice, this list of conditions and the following disclaimer.
-    * Redistributions in binary form must reproduce the above
-    copyright notice, this list of conditions and the following disclaimer
-    in the documentation and/or other materials provided with the
-    distribution.
-    THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-    "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-    LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-    A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-    OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-    SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-    LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-    DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-    THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-    (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-    OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-    You can contact the author at :
-    - zstd source repository : https://github.com/Cyan4973/zstd
-*/
 #ifndef ZSTD_H_235446
 #define ZSTD_H_235446
 
@@ -53,9 +31,9 @@ extern "C" {
 
 
 /*=======   Version   =======*/
-#define ZSTD_VERSION_MAJOR    0
+#define ZSTD_VERSION_MAJOR    1
-#define ZSTD_VERSION_MINOR    8
+#define ZSTD_VERSION_MINOR    0
-#define ZSTD_VERSION_RELEASE  1
+#define ZSTD_VERSION_RELEASE  0
 
 #define ZSTD_LIB_VERSION ZSTD_VERSION_MAJOR.ZSTD_VERSION_MINOR.ZSTD_VERSION_RELEASE
 #define ZSTD_QUOTE(str) #str
@@ -87,7 +65,7 @@ ZSTDLIB_API size_t ZSTD_compress( void* dst, size_t dstCapacity,
 *             When `return==0`, data to decompress can have any size.
 *             In which case, it's necessary to use streaming mode to decompress data.
 *             Optionally, application may rely on its own implied limits.
-*             (For example, application own data could be necessarily cut into blocks <= 16 KB).
+*             (For example, application data could be necessarily cut into blocks <= 16 KB).
 *    note 3 : decompressed size could be wrong or intentionally modified !
 *             Always ensure result fits within application's authorized limits !
 *             Each application can set its own limits.
@@ -116,7 +94,7 @@ ZSTDLIB_API const char* ZSTD_getErrorName(size_t code);     /*!< provides readab
 *  Explicit memory management
 ***************************************/
 /** Compression context */
-typedef struct ZSTD_CCtx_s ZSTD_CCtx;                       /*< incomplete type */
+typedef struct ZSTD_CCtx_s ZSTD_CCtx;
 ZSTDLIB_API ZSTD_CCtx* ZSTD_createCCtx(void);
 ZSTDLIB_API size_t     ZSTD_freeCCtx(ZSTD_CCtx* cctx);
 
@@ -125,7 +103,7 @@ ZSTDLIB_API size_t     ZSTD_freeCCtx(ZSTD_CCtx* cctx);
 ZSTDLIB_API size_t ZSTD_compressCCtx(ZSTD_CCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, int compressionLevel);
 
 /** Decompression context */
-typedef struct ZSTD_DCtx_s ZSTD_DCtx;                       /*< incomplete type */
+typedef struct ZSTD_DCtx_s ZSTD_DCtx;
 ZSTDLIB_API ZSTD_DCtx* ZSTD_createDCtx(void);
 ZSTDLIB_API size_t     ZSTD_freeDCtx(ZSTD_DCtx* dctx);
 
@@ -191,6 +169,109 @@ ZSTDLIB_API size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx,
                                         const ZSTD_DDict* ddict);
 
 
+/*-**************************
+*  Streaming
+****************************/
+
+typedef struct ZSTD_inBuffer_s {
+  const void* src;    /**< start of input buffer */
+  size_t size;        /**< size of input buffer */
+  size_t pos;         /**< position where reading stopped. Will be updated. Necessarily 0 <= pos <= size */
+} ZSTD_inBuffer;
+
+typedef struct ZSTD_outBuffer_s {
+  void*  dst;         /**< start of output buffer */
+  size_t size;        /**< size of output buffer */
+  size_t pos;         /**< position where writing stopped. Will be updated. Necessarily 0 <= pos <= size */
+} ZSTD_outBuffer;
+
+
+/*======   streaming compression   ======*/
+
+/*-***********************************************************************
+*  Streaming compression - howto
+*
+*  A ZSTD_CStream object is required to track streaming operation.
+*  Use ZSTD_createCStream() and ZSTD_freeCStream() to create/release resources.
+*  ZSTD_CStream objects can be reused multiple times on consecutive compression operations.
+*
+*  Start by initializing ZSTD_CStream.
+*  Use ZSTD_initCStream() to start a new compression operation.
+*  Use ZSTD_initCStream_usingDict() for a compression which requires a dictionary.
+*
+*  Use ZSTD_compressStream() repetitively to consume input stream.
+*  The function will automatically update both `pos`.
+*  Note that it may not consume the entire input, in which case `pos < size`,
+*  and it's up to the caller to present again remaining data.
+*  @return : a size hint, preferred nb of bytes to use as input for next function call
+*           (it's just a hint, to help latency a little, any other value will work fine)
+*           (note : the size hint is guaranteed to be <= ZSTD_CStreamInSize() )
+*            or an error code, which can be tested using ZSTD_isError().
+*
+*  At any moment, it's possible to flush whatever data remains within buffer, using ZSTD_flushStream().
+*  `output->pos` will be updated.
+*  Note some content might still be left within internal buffer if `output->size` is too small.
+*  @return : nb of bytes still present within internal buffer (0 if it's empty)
+*            or an error code, which can be tested using ZSTD_isError().
+*
+*  ZSTD_endStream() instructs to finish a frame.
+*  It will perform a flush and write frame epilogue.
+*  The epilogue is required for decoders to consider a frame completed.
+*  Similar to ZSTD_flushStream(), it may not be able to flush the full content if `output->size` is too small.
+*  In which case, call again ZSTD_endStream() to complete the flush.
+*  @return : nb of bytes still present within internal buffer (0 if it's empty)
+*            or an error code, which can be tested using ZSTD_isError().
+*
+* *******************************************************************/
+
+typedef struct ZSTD_CStream_s ZSTD_CStream;
+ZSTDLIB_API ZSTD_CStream* ZSTD_createCStream(void);
+ZSTDLIB_API size_t ZSTD_freeCStream(ZSTD_CStream* zcs);
+
+ZSTDLIB_API size_t ZSTD_CStreamInSize(void);    /**< recommended size for input buffer */
+ZSTDLIB_API size_t ZSTD_CStreamOutSize(void);   /**< recommended size for output buffer */
+
+ZSTDLIB_API size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel);
+ZSTDLIB_API size_t ZSTD_compressStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output, ZSTD_inBuffer* input);
+ZSTDLIB_API size_t ZSTD_flushStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output);
+ZSTDLIB_API size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output);
+
+
+/*======   decompression   ======*/
+
+/*-***************************************************************************
+*  Streaming decompression howto
+*
+*  A ZSTD_DStream object is required to track streaming operations.
+*  Use ZSTD_createDStream() and ZSTD_freeDStream() to create/release resources.
+*  ZSTD_DStream objects can be re-used multiple times.
+*
+*  Use ZSTD_initDStream() to start a new decompression operation,
+*   or ZSTD_initDStream_usingDict() if decompression requires a dictionary.
+*
+*  Use ZSTD_decompressStream() repetitively to consume your input.
+*  The function will update both `pos`.
+*  Note that it may not consume the entire input (pos < size),
+*  in which case it's up to the caller to present remaining input again.
+*  @return : 0 when a frame is completely decoded and fully flushed,
+*            1 when there is still some data left within internal buffer to flush,
+*            >1 when more data is expected, with value being a suggested next input size (it's just a hint, which helps latency, any size is accepted),
+*            or an error code, which can be tested using ZSTD_isError().
+*
+* *******************************************************************************/
+
+typedef struct ZSTD_DStream_s ZSTD_DStream;
+ZSTDLIB_API ZSTD_DStream* ZSTD_createDStream(void);
+ZSTDLIB_API size_t ZSTD_freeDStream(ZSTD_DStream* zds);
+
+ZSTDLIB_API size_t ZSTD_DStreamInSize(void);    /*!< recommended size for input buffer */
+ZSTDLIB_API size_t ZSTD_DStreamOutSize(void);   /*!< recommended size for output buffer */
+
+ZSTDLIB_API size_t ZSTD_initDStream(ZSTD_DStream* zds);
+ZSTDLIB_API size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inBuffer* input);
+
+
+
 #ifdef ZSTD_STATIC_LINKING_ONLY
 
 /* ====================================================================================
@@ -227,7 +308,7 @@ static const size_t ZSTD_skippableHeaderSize = 8;  /* magic number + skippable f
 
 
 /*--- Types ---*/
-typedef enum { ZSTD_fast, ZSTD_dfast, ZSTD_greedy, ZSTD_lazy, ZSTD_lazy2, ZSTD_btlazy2, ZSTD_btopt } ZSTD_strategy;   /*< from faster to stronger */
+typedef enum { ZSTD_fast, ZSTD_dfast, ZSTD_greedy, ZSTD_lazy, ZSTD_lazy2, ZSTD_btlazy2, ZSTD_btopt } ZSTD_strategy;   /* from faster to stronger */
 
 typedef struct {
     unsigned windowLog;      /**< largest match distance : larger == more compression, more memory needed during decompression */
@@ -275,12 +356,12 @@ ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictS
 
 /*! ZSTD_sizeofCCtx() :
  *  Gives the amount of memory used by a given ZSTD_CCtx */
-ZSTDLIB_API size_t ZSTD_sizeofCCtx(const ZSTD_CCtx* cctx);
+ZSTDLIB_API size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx);
 
 /*! ZSTD_getParams() :
 *   same as ZSTD_getCParams(), but @return a full `ZSTD_parameters` object instead of a `ZSTD_compressionParameters`.
 *   All fields of `ZSTD_frameParameters` are set to default (0) */
-ZSTD_parameters ZSTD_getParams(int compressionLevel, unsigned long long srcSize, size_t dictSize);
+ZSTDLIB_API ZSTD_parameters ZSTD_getParams(int compressionLevel, unsigned long long srcSize, size_t dictSize);
 
 /*! ZSTD_getCParams() :
 *   @return ZSTD_compressionParameters structure for a selected compression level and srcSize.
@@ -317,118 +398,30 @@ ZSTDLIB_API ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem);
 
 /*! ZSTD_sizeofDCtx() :
  *  Gives the amount of memory used by a given ZSTD_DCtx */
-ZSTDLIB_API size_t ZSTD_sizeofDCtx(const ZSTD_DCtx* dctx);
+ZSTDLIB_API size_t ZSTD_sizeof_DCtx(const ZSTD_DCtx* dctx);
 
 
 /* ******************************************************************
-*  Streaming
+*  Advanced Streaming functions
 ********************************************************************/
 
-typedef struct ZSTD_inBuffer_s {
-  const void* src;    /**< start of input buffer */
-  size_t size;        /**< size of input buffer */
-  size_t pos;         /**< position where reading stopped. Will be updated. Necessarily 0 <= pos <= size */
-} ZSTD_inBuffer;
-
-typedef struct ZSTD_outBuffer_s {
-  void*  dst;         /**< start of output buffer */
-  size_t size;        /**< size of output buffer */
-  size_t pos;         /**< position where writing stopped. Will be updated. Necessarily 0 <= pos <= size */
-} ZSTD_outBuffer;
-
-
 /*======   compression   ======*/
 
-/*-***********************************************************************
+ZSTDLIB_API ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem);
-*  Streaming compression - howto
+ZSTDLIB_API size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel);
-*
+ZSTDLIB_API size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize,
-*  A ZSTD_CStream object is required to track streaming operation.
-*  Use ZSTD_createCStream() and ZSTD_freeCStream() to create/release resources.
-*  ZSTD_CStream objects can be reused multiple times on consecutive compression operations.
-*
-*  Start by initializing ZSTD_CStream.
-*  Use ZSTD_initCStream() to start a new compression operation.
-*  Use ZSTD_initCStream_usingDict() for a compression which requires a dictionary.
-*
-*  Use ZSTD_compressStream() repetitively to consume input stream.
-*  The function will automatically update both `pos`.
-*  Note that it may not consume the entire input, in which case `pos < size`,
-*  and it's up to the caller to present again remaining data.
-*  @return : a hint to preferred nb of bytes to use as input for next function call (it's just a hint, to improve latency)
-*            or an error code, which can be tested using ZSTD_isError().
-*
-*  At any moment, it's possible to flush whatever data remains within buffer, using ZSTD_flushStream().
-*  `output->pos` will be updated.
-*  Note some content might still be left within internal buffer if `output->size` is too small.
-*  @return : nb of bytes still present within internal buffer (0 if it's empty)
-*            or an error code, which can be tested using ZSTD_isError().
-*
-*  ZSTD_endStream() instructs to finish a frame.
-*  It will perform a flush and write frame epilogue.
-*  The epilogue is required for decoders to consider a frame completed.
-*  Similar to ZSTD_flushStream(), it may not be able to flush the full content if `output->size` is too small.
-*  In which case, call again ZSTD_endStream() to complete the flush.
-*  @return : nb of bytes still present within internal buffer (0 if it's empty)
-*            or an error code, which can be tested using ZSTD_isError().
-*
-* *******************************************************************/
-
-typedef struct ZSTD_CStream_s ZSTD_CStream;
-ZSTD_CStream* ZSTD_createCStream(void);
-size_t ZSTD_freeCStream(ZSTD_CStream* zcs);
-
-size_t ZSTD_CStreamInSize(void);    /**< recommended size for input buffer */
-size_t ZSTD_CStreamOutSize(void);   /**< recommended size for output buffer */
-
-size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel);
-size_t ZSTD_compressStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output, ZSTD_inBuffer* input);
-size_t ZSTD_flushStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output);
-size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output);
-
-/* advanced */
-ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem);
-size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel);
-size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize,
                                  ZSTD_parameters params, unsigned long long pledgedSrcSize);
+ZSTDLIB_API size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs);
 
 
 /*======   decompression   ======*/
 
-/*-***************************************************************************
+typedef enum { ZSTDdsp_maxWindowSize } ZSTD_DStreamParameter_e;
-*  Streaming decompression howto
-*
-*  A ZSTD_DStream object is required to track streaming operations.
-*  Use ZSTD_createDStream() and ZSTD_freeDStream() to create/release resources.
-*  ZSTD_DStream objects can be re-init multiple times.
-*
-*  Use ZSTD_initDStream() to start a new decompression operation,
-*   or ZSTD_initDStream_usingDict() if decompression requires a dictionary.
-*
-*  Use ZSTD_decompressStream() repetitively to consume your input.
-*  The function will update both `pos`.
-*  Note that it may not consume the entire input (pos < size),
-*  in which case it's up to the caller to present remaining input again.
-*  @return : 0 when a frame is completely decoded and fully flushed,
-*            1 when there is still some data left within internal buffer to flush,
-*            >1 when more data is expected, with value being a suggested next input size (it's just a hint, which helps latency, any size is accepted),
-*            or an error code, which can be tested using ZSTD_isError().
-*
-* *******************************************************************************/
-
-typedef struct ZSTD_DStream_s ZSTD_DStream;
-ZSTD_DStream* ZSTD_createDStream(void);
-size_t ZSTD_freeDStream(ZSTD_DStream* zds);
-
-size_t ZSTD_DStreamInSize(void);    /*!< recommended size for input buffer */
-size_t ZSTD_DStreamOutSize(void);   /*!< recommended size for output buffer */
-
-size_t ZSTD_initDStream(ZSTD_DStream* zds);
-size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inBuffer* input);
-
-/* advanced */
-ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem);
-size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize);
 
+ZSTDLIB_API ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem);
+ZSTDLIB_API size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize);
+ZSTDLIB_API size_t ZSTD_setDStreamParameter(ZSTD_DStream* zds, ZSTD_DStreamParameter_e paramType, unsigned paramValue);
+ZSTDLIB_API size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds);
 
 
 /* ******************************************************************

programs/.gitignore

@@ -3,42 +3,16 @@ zstd
 zstd32
 zstd-compress
 zstd-decompress
-fullbench
-fullbench32
-fuzzer
-fuzzer32
-zbufftest
-zbufftest32
-zstreamtest
-zstreamtest32
-datagen
-paramgrill
-paramgrill32
-roundTripCrash
 
 # Object files
 *.o
 *.ko
 
-# Libraries
-*.lib
-*.a
-
-# Shared objects (inc. Windows DLLs)
-*.dll
-*.so
-*.so.*
-*.dylib
-
 # Executables
 *.exe
 *.out
 *.app
 
-# Visual solution files
-*.suo
-*.user
-
 # Default result files
 dictionary
 grillResults.txt
@@ -53,5 +27,4 @@ afl
 
 # Misc files
 *.bat
-fileTests.sh
 dirTest*

programs/COPYING

@@ -1,339 +0,0 @@
-                    GNU GENERAL PUBLIC LICENSE
-                       Version 2, June 1991
-
- Copyright (C) 1989, 1991 Free Software Foundation, Inc.,
- 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
- Everyone is permitted to copy and distribute verbatim copies
- of this license document, but changing it is not allowed.
-
-                            Preamble
-
-  The licenses for most software are designed to take away your
-freedom to share and change it.  By contrast, the GNU General Public
-License is intended to guarantee your freedom to share and change free
-software--to make sure the software is free for all its users.  This
-General Public License applies to most of the Free Software
-Foundation's software and to any other program whose authors commit to
-using it.  (Some other Free Software Foundation software is covered by
-the GNU Lesser General Public License instead.)  You can apply it to
-your programs, too.
-
-  When we speak of free software, we are referring to freedom, not
-price.  Our General Public Licenses are designed to make sure that you
-have the freedom to distribute copies of free software (and charge for
-this service if you wish), that you receive source code or can get it
-if you want it, that you can change the software or use pieces of it
-in new free programs; and that you know you can do these things.
-
-  To protect your rights, we need to make restrictions that forbid
-anyone to deny you these rights or to ask you to surrender the rights.
-These restrictions translate to certain responsibilities for you if you
-distribute copies of the software, or if you modify it.
-
-  For example, if you distribute copies of such a program, whether
-gratis or for a fee, you must give the recipients all the rights that
-you have.  You must make sure that they, too, receive or can get the
-source code.  And you must show them these terms so they know their
-rights.
-
-  We protect your rights with two steps: (1) copyright the software, and
-(2) offer you this license which gives you legal permission to copy,
-distribute and/or modify the software.
-
-  Also, for each author's protection and ours, we want to make certain
-that everyone understands that there is no warranty for this free
-software.  If the software is modified by someone else and passed on, we
-want its recipients to know that what they have is not the original, so
-that any problems introduced by others will not reflect on the original
-authors' reputations.
-
-  Finally, any free program is threatened constantly by software
-patents.  We wish to avoid the danger that redistributors of a free
-program will individually obtain patent licenses, in effect making the
-program proprietary.  To prevent this, we have made it clear that any
-patent must be licensed for everyone's free use or not licensed at all.
-
-  The precise terms and conditions for copying, distribution and
-modification follow.
-
-                    GNU GENERAL PUBLIC LICENSE
-   TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
-
-  0. This License applies to any program or other work which contains
-a notice placed by the copyright holder saying it may be distributed
-under the terms of this General Public License.  The "Program", below,
-refers to any such program or work, and a "work based on the Program"
-means either the Program or any derivative work under copyright law:
-that is to say, a work containing the Program or a portion of it,
-either verbatim or with modifications and/or translated into another
-language.  (Hereinafter, translation is included without limitation in
-the term "modification".)  Each licensee is addressed as "you".
-
-Activities other than copying, distribution and modification are not
-covered by this License; they are outside its scope.  The act of
-running the Program is not restricted, and the output from the Program
-is covered only if its contents constitute a work based on the
-Program (independent of having been made by running the Program).
-Whether that is true depends on what the Program does.
-
-  1. You may copy and distribute verbatim copies of the Program's
-source code as you receive it, in any medium, provided that you
-conspicuously and appropriately publish on each copy an appropriate
-copyright notice and disclaimer of warranty; keep intact all the
-notices that refer to this License and to the absence of any warranty;
-and give any other recipients of the Program a copy of this License
-along with the Program.
-
-You may charge a fee for the physical act of transferring a copy, and
-you may at your option offer warranty protection in exchange for a fee.
-
-  2. You may modify your copy or copies of the Program or any portion
-of it, thus forming a work based on the Program, and copy and
-distribute such modifications or work under the terms of Section 1
-above, provided that you also meet all of these conditions:
-
-    a) You must cause the modified files to carry prominent notices
-    stating that you changed the files and the date of any change.
-
-    b) You must cause any work that you distribute or publish, that in
-    whole or in part contains or is derived from the Program or any
-    part thereof, to be licensed as a whole at no charge to all third
-    parties under the terms of this License.
-
-    c) If the modified program normally reads commands interactively
-    when run, you must cause it, when started running for such
-    interactive use in the most ordinary way, to print or display an
-    announcement including an appropriate copyright notice and a
-    notice that there is no warranty (or else, saying that you provide
-    a warranty) and that users may redistribute the program under
-    these conditions, and telling the user how to view a copy of this
-    License.  (Exception: if the Program itself is interactive but
-    does not normally print such an announcement, your work based on
-    the Program is not required to print an announcement.)
-
-These requirements apply to the modified work as a whole.  If
-identifiable sections of that work are not derived from the Program,
-and can be reasonably considered independent and separate works in
-themselves, then this License, and its terms, do not apply to those
-sections when you distribute them as separate works.  But when you
-distribute the same sections as part of a whole which is a work based
-on the Program, the distribution of the whole must be on the terms of
-this License, whose permissions for other licensees extend to the
-entire whole, and thus to each and every part regardless of who wrote it.
-
-Thus, it is not the intent of this section to claim rights or contest
-your rights to work written entirely by you; rather, the intent is to
-exercise the right to control the distribution of derivative or
-collective works based on the Program.
-
-In addition, mere aggregation of another work not based on the Program
-with the Program (or with a work based on the Program) on a volume of
-a storage or distribution medium does not bring the other work under
-the scope of this License.
-
-  3. You may copy and distribute the Program (or a work based on it,
-under Section 2) in object code or executable form under the terms of
-Sections 1 and 2 above provided that you also do one of the following:
-
-    a) Accompany it with the complete corresponding machine-readable
-    source code, which must be distributed under the terms of Sections
-    1 and 2 above on a medium customarily used for software interchange; or,
-
-    b) Accompany it with a written offer, valid for at least three
-    years, to give any third party, for a charge no more than your
-    cost of physically performing source distribution, a complete
-    machine-readable copy of the corresponding source code, to be
-    distributed under the terms of Sections 1 and 2 above on a medium
-    customarily used for software interchange; or,
-
-    c) Accompany it with the information you received as to the offer
-    to distribute corresponding source code.  (This alternative is
-    allowed only for noncommercial distribution and only if you
-    received the program in object code or executable form with such
-    an offer, in accord with Subsection b above.)
-
-The source code for a work means the preferred form of the work for
-making modifications to it.  For an executable work, complete source
-code means all the source code for all modules it contains, plus any
-associated interface definition files, plus the scripts used to
-control compilation and installation of the executable.  However, as a
-special exception, the source code distributed need not include
-anything that is normally distributed (in either source or binary
-form) with the major components (compiler, kernel, and so on) of the
-operating system on which the executable runs, unless that component
-itself accompanies the executable.
-
-If distribution of executable or object code is made by offering
-access to copy from a designated place, then offering equivalent
-access to copy the source code from the same place counts as
-distribution of the source code, even though third parties are not
-compelled to copy the source along with the object code.
-
-  4. You may not copy, modify, sublicense, or distribute the Program
-except as expressly provided under this License.  Any attempt
-otherwise to copy, modify, sublicense or distribute the Program is
-void, and will automatically terminate your rights under this License.
-However, parties who have received copies, or rights, from you under
-this License will not have their licenses terminated so long as such
-parties remain in full compliance.
-
-  5. You are not required to accept this License, since you have not
-signed it.  However, nothing else grants you permission to modify or
-distribute the Program or its derivative works.  These actions are
-prohibited by law if you do not accept this License.  Therefore, by
-modifying or distributing the Program (or any work based on the
-Program), you indicate your acceptance of this License to do so, and
-all its terms and conditions for copying, distributing or modifying
-the Program or works based on it.
-
-  6. Each time you redistribute the Program (or any work based on the
-Program), the recipient automatically receives a license from the
-original licensor to copy, distribute or modify the Program subject to
-these terms and conditions.  You may not impose any further
-restrictions on the recipients' exercise of the rights granted herein.
-You are not responsible for enforcing compliance by third parties to
-this License.
-
-  7. If, as a consequence of a court judgment or allegation of patent
-infringement or for any other reason (not limited to patent issues),
-conditions are imposed on you (whether by court order, agreement or
-otherwise) that contradict the conditions of this License, they do not
-excuse you from the conditions of this License.  If you cannot
-distribute so as to satisfy simultaneously your obligations under this
-License and any other pertinent obligations, then as a consequence you
-may not distribute the Program at all.  For example, if a patent
-license would not permit royalty-free redistribution of the Program by
-all those who receive copies directly or indirectly through you, then
-the only way you could satisfy both it and this License would be to
-refrain entirely from distribution of the Program.
-
-If any portion of this section is held invalid or unenforceable under
-any particular circumstance, the balance of the section is intended to
-apply and the section as a whole is intended to apply in other
-circumstances.
-
-It is not the purpose of this section to induce you to infringe any
-patents or other property right claims or to contest validity of any
-such claims; this section has the sole purpose of protecting the
-integrity of the free software distribution system, which is
-implemented by public license practices.  Many people have made
-generous contributions to the wide range of software distributed
-through that system in reliance on consistent application of that
-system; it is up to the author/donor to decide if he or she is willing
-to distribute software through any other system and a licensee cannot
-impose that choice.
-
-This section is intended to make thoroughly clear what is believed to
-be a consequence of the rest of this License.
-
-  8. If the distribution and/or use of the Program is restricted in
-certain countries either by patents or by copyrighted interfaces, the
-original copyright holder who places the Program under this License
-may add an explicit geographical distribution limitation excluding
-those countries, so that distribution is permitted only in or among
-countries not thus excluded.  In such case, this License incorporates
-the limitation as if written in the body of this License.
-
-  9. The Free Software Foundation may publish revised and/or new versions
-of the General Public License from time to time.  Such new versions will
-be similar in spirit to the present version, but may differ in detail to
-address new problems or concerns.
-
-Each version is given a distinguishing version number.  If the Program
-specifies a version number of this License which applies to it and "any
-later version", you have the option of following the terms and conditions
-either of that version or of any later version published by the Free
-Software Foundation.  If the Program does not specify a version number of
-this License, you may choose any version ever published by the Free Software
-Foundation.
-
-  10. If you wish to incorporate parts of the Program into other free
-programs whose distribution conditions are different, write to the author
-to ask for permission.  For software which is copyrighted by the Free
-Software Foundation, write to the Free Software Foundation; we sometimes
-make exceptions for this.  Our decision will be guided by the two goals
-of preserving the free status of all derivatives of our free software and
-of promoting the sharing and reuse of software generally.
-
-                            NO WARRANTY
-
-  11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY
-FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW.  EXCEPT WHEN
-OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES
-PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED
-OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
-MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.  THE ENTIRE RISK AS
-TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU.  SHOULD THE
-PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING,
-REPAIR OR CORRECTION.
-
-  12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
-WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR
-REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES,
-INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING
-OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED
-TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY
-YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER
-PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE
-POSSIBILITY OF SUCH DAMAGES.
-
-                     END OF TERMS AND CONDITIONS
-
-            How to Apply These Terms to Your New Programs
-
-  If you develop a new program, and you want it to be of the greatest
-possible use to the public, the best way to achieve this is to make it
-free software which everyone can redistribute and change under these terms.
-
-  To do so, attach the following notices to the program.  It is safest
-to attach them to the start of each source file to most effectively
-convey the exclusion of warranty; and each file should have at least
-the "copyright" line and a pointer to where the full notice is found.
-
-    <one line to give the program's name and a brief idea of what it does.>
-    Copyright (C) <year>  <name of author>
-
-    This program is free software; you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
-    the Free Software Foundation; either version 2 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
-
-    You should have received a copy of the GNU General Public License along
-    with this program; if not, write to the Free Software Foundation, Inc.,
-    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-Also add information on how to contact you by electronic and paper mail.
-
-If the program is interactive, make it output a short notice like this
-when it starts in an interactive mode:
-
-    Gnomovision version 69, Copyright (C) year name of author
-    Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
-    This is free software, and you are welcome to redistribute it
-    under certain conditions; type `show c' for details.
-
-The hypothetical commands `show w' and `show c' should show the appropriate
-parts of the General Public License.  Of course, the commands you use may
-be called something other than `show w' and `show c'; they could even be
-mouse-clicks or menu items--whatever suits your program.
-
-You should also get your employer (if you work as a programmer) or your
-school, if any, to sign a "copyright disclaimer" for the program, if
-necessary.  Here is a sample; alter the names:
-
-  Yoyodyne, Inc., hereby disclaims all copyright interest in the program
-  `Gnomovision' (which makes passes at compilers) written by James Hacker.
-
-  <signature of Ty Coon>, 1 April 1989
-  Ty Coon, President of Vice
-
-This General Public License does not permit incorporating your program into
-proprietary programs.  If your program is a subroutine library, you may
-consider it more useful to permit linking proprietary applications with the
-library.  If this is what you want to do, use the GNU Lesser General
-Public License instead of this License.

programs/Makefile

@@ -1,34 +1,17 @@
 # ##########################################################################
-# ZSTD programs - Makefile
+# Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
-# Copyright (C) Yann Collet 2015-2016
+# All rights reserved.
 #
-# GPL v2 License
+# This source code is licensed under the BSD-style license found in the
-#
+# LICENSE file in the root directory of this source tree. An additional grant
-# This program is free software; you can redistribute it and/or modify
+# of patent rights can be found in the PATENTS file in the same directory.
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-#
-# You can contact the author at :
-#  - zstd homepage : http://www.zstd.net/
 # ##########################################################################
 # zstd : Command Line Utility, supporting gzip-like arguments
-# datagen : Synthetic and parametrable data generator, for tests
+# zstd32 : Same as zstd, but forced to compile in 32-bits mode
-# fuzzer  : Test tool, to check zstd integrity on target platform
+# zstd_nolegacy : zstd without support of decompression of legacy versions
-# fuzzer32: Same as fuzzer, but forced to compile in 32-bits mode
+# zstd-small : minimal zstd without dictionary builder and benchmark
-# zbufftest  : Test tool, to check ZBUFF integrity on target platform
+# zstd-compress : compressor-only version of zstd
-# zbufftest32: Same as zbufftest, but forced to compile in 32-bits mode
+# zstd-decompress : decompressor-only version of zstd
-# fullbench  : Precisely measure speed for each zstd inner function
-# fullbench32: Same as fullbench, but forced to compile in 32-bits mode
 # ##########################################################################
 
 DESTDIR?=
@@ -56,7 +39,6 @@ ZSTDCOMMON_FILES := $(ZSTDDIR)/common/*.c
 ZSTDCOMP_FILES := $(ZSTDDIR)/compress/zstd_compress.c $(ZSTDDIR)/compress/fse_compress.c $(ZSTDDIR)/compress/huf_compress.c
 ZSTDDECOMP_FILES := $(ZSTDDIR)/decompress/huf_decompress.c
 ZSTD_FILES := $(ZSTDDECOMP_FILES) $(ZSTDCOMMON_FILES) $(ZSTDCOMP_FILES)
-ZBUFF_FILES := $(ZSTDDIR)/compress/zbuff_compress.c $(ZSTDDIR)/decompress/zbuff_decompress.c
 ZDICT_FILES := $(ZSTDDIR)/dictBuilder/*.c
 ZSTDDECOMP_O = $(ZSTDDIR)/decompress/zstd_decompress.o
 ZSTDDECOMP32_O = $(ZSTDDIR)/decompress/zstd_decompress32.o
@@ -66,8 +48,8 @@ CPPFLAGS  += -DZSTD_LEGACY_SUPPORT=0
 ZSTDLEGACY_FILES:=
 else
 ZSTD_LEGACY_SUPPORT:=1
-CPPFLAGS  += -I$(ZSTDDIR)/legacy -I./legacy
+CPPFLAGS  += -I$(ZSTDDIR)/legacy
-ZSTDLEGACY_FILES:= $(ZSTDDIR)/legacy/*.c legacy/fileio_legacy.c
+ZSTDLEGACY_FILES:= $(ZSTDDIR)/legacy/*.c
 endif
 
 
@@ -80,24 +62,19 @@ EXT =
 VOID = /dev/null
 endif
 
-ZBUFFTEST = -T2mn
-FUZZERTEST= -T5mn
-ZSTDRTTEST= --test-large-data
 
-.PHONY: default all all32 clean install uninstall test test32 test-all
+.PHONY: default all clean install uninstall
 
 default: zstd
 
-all: zstd fullbench fuzzer zbufftest zstreamtest paramgrill datagen
+all: zstd
-
-all32: cleano32 zstd32 fullbench32 fuzzer32 zbufftest32 zstreamtest32
 
 
 $(ZSTDDECOMP_O): $(ZSTDDIR)/decompress/zstd_decompress.c
-	$(CC)    $(ALIGN_LOOP) $(FLAGS) $^ -c -o $@
+	$(CC)    $(ALIGN_LOOP) $(FLAGS) -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT) $^ -c -o $@
 
 $(ZSTDDECOMP32_O): $(ZSTDDIR)/decompress/zstd_decompress.c
-	$(CC)  -m32 $(ALIGN_LOOP) $(FLAGS) $^ -c -o $@
+	$(CC)  -m32 $(ALIGN_LOOP) $(FLAGS) -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT) $^ -c -o $@
 
 zstd  : $(ZSTDDECOMP_O) $(ZSTD_FILES) $(ZSTDLEGACY_FILES) $(ZDICT_FILES) \
         zstdcli.c fileio.c bench.c datagen.c dibio.c
@@ -119,7 +96,7 @@ zstd-pgo : clean zstd
 	./zstd -b $(PROFILE_WITH)
 	./zstd -b7i2 $(PROFILE_WITH)
 	./zstd -b5 $(PROFILE_WITH)
-	rm zstd
+	$(RM) zstd
 	$(MAKE) zstd MOREFLAGS=-fprofile-use
 
 zstd-frugal: $(ZSTDDECOMP_O) $(ZSTD_FILES) zstdcli.c fileio.c
@@ -136,58 +113,19 @@ zstd-decompress: $(ZSTDCOMMON_FILES) $(ZSTDDECOMP_FILES) \
 zstd-small: clean
 	CFLAGS="-Os -s" $(MAKE) zstd-frugal
 
-fullbench  : $(ZSTDDECOMP_O) $(ZSTD_FILES) $(ZBUFF_FILES) datagen.c fullbench.c
-	$(CC)      $(FLAGS) $^ -o $@$(EXT)
-
-fullbench32 : $(ZSTDDECOMP32_O) $(ZSTD_FILES) $(ZBUFF_FILES) datagen.c fullbench.c
-	$(CC)  -m32  $(FLAGS) $^ -o $@$(EXT)
-
-fuzzer  : CPPFLAGS += -I$(ZSTDDIR)/dictBuilder
-fuzzer  : $(ZSTDDECOMP_O) $(ZSTD_FILES) $(ZDICT_FILES) datagen.c fuzzer.c
-	$(CC)      $(FLAGS) $^ -o $@$(EXT)
-
-fuzzer32 : CPPFLAGS += -I$(ZSTDDIR)/dictBuilder
-fuzzer32 : $(ZSTDDECOMP32_O) $(ZSTD_FILES) $(ZDICT_FILES) datagen.c fuzzer.c
-	$(CC)  -m32  $(FLAGS) $^ -o $@$(EXT)
-
-zbufftest  : $(ZSTDDECOMP_O) $(ZSTD_FILES) $(ZBUFF_FILES) datagen.c zbufftest.c
-	$(CC)      $(FLAGS) $^ -o $@$(EXT)
-
-zbufftest32  : $(ZSTDDECOMP32_O) $(ZSTD_FILES) $(ZBUFF_FILES) datagen.c zbufftest.c
-	$(CC)  -m32  $(FLAGS) $^ -o $@$(EXT)
-
-zstreamtest  : $(ZSTDDECOMP_O) $(ZSTD_FILES) datagen.c zstreamtest.c
-	$(CC)      $(FLAGS) $^ -o $@$(EXT)
-
-zstreamtest32 : $(ZSTDDECOMP32_O) $(ZSTD_FILES) datagen.c zstreamtest.c
-	$(CC) -m32  $(FLAGS) $^ -o $@$(EXT)
-
-paramgrill : $(ZSTDDECOMP_O) $(ZSTD_FILES) datagen.c paramgrill.c
-	$(CC)      $(FLAGS) $^ -lm -o $@$(EXT)
-
-datagen : datagen.c datagencli.c
-	$(CC)      $(FLAGS) $^ -o $@$(EXT)
-
-roundTripCrash : $(ZSTDDECOMP_O) $(ZSTD_FILES) roundTripCrash.c
-	$(CC)      $(FLAGS) $^ -o $@$(EXT)
 
 clean:
 	$(MAKE) -C ../lib clean
-	@rm -f core *.o tmp* result* *.gcda dictionary *.zst \
+	@$(RM) ../lib/decompress/*.o
-        zstd$(EXT) zstd32$(EXT) zstd-compress$(EXT) zstd-decompress$(EXT) \
+	@$(RM) core *.o tmp* result* *.gcda dictionary *.zst \
-        fullbench$(EXT) fullbench32$(EXT) \
+        zstd$(EXT) zstd32$(EXT) zstd-compress$(EXT) zstd-decompress$(EXT)
-        fuzzer$(EXT) fuzzer32$(EXT) zbufftest$(EXT) zbufftest32$(EXT) \
-        datagen$(EXT) paramgrill$(EXT) roundTripCrash$(EXT)
 	@echo Cleaning completed
 
-cleano32:
-	@rm -f ../lib/decompress/*.o
 
 #----------------------------------------------------------------------------------
 #make install is validated only for Linux, OSX, kFreeBSD, Hurd and some BSD targets
 #----------------------------------------------------------------------------------
 ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD DragonFly))
-HOST_OS = POSIX
 install: zstd
 	@echo Installing binaries
 	@install -d -m 755 $(DESTDIR)$(BINDIR)/ $(DESTDIR)$(MANDIR)/
@@ -201,83 +139,11 @@ install: zstd
 	@echo zstd installation completed
 
 uninstall:
-	rm -f $(DESTDIR)$(BINDIR)/zstdcat
+	$(RM) $(DESTDIR)$(BINDIR)/zstdcat
-	rm -f $(DESTDIR)$(BINDIR)/unzstd
+	$(RM) $(DESTDIR)$(BINDIR)/unzstd
-	[ -x $(DESTDIR)$(BINDIR)/zstd$(EXT) ] && rm -f $(DESTDIR)$(BINDIR)/zstd$(EXT)
+	[ -x $(DESTDIR)$(BINDIR)/zstd$(EXT) ] && $(RM) $(DESTDIR)$(BINDIR)/zstd$(EXT)
-	rm -f $(DESTDIR)$(MANDIR)/zstdcat.1
+	$(RM) $(DESTDIR)$(MANDIR)/zstdcat.1
-	rm -f $(DESTDIR)$(MANDIR)/unzstd.1
+	$(RM) $(DESTDIR)$(MANDIR)/unzstd.1
-	[ -f $(DESTDIR)$(MANDIR)/zstd.1 ] && rm -f $(DESTDIR)$(MANDIR)/zstd.1
+	[ -f $(DESTDIR)$(MANDIR)/zstd.1 ] && $(RM) $(DESTDIR)$(MANDIR)/zstd.1
 	@echo zstd programs successfully uninstalled
-
-valgrindTest: VALGRIND = valgrind --leak-check=full --error-exitcode=1
-valgrindTest: zstd datagen fuzzer fullbench zbufftest
-	@echo "\n ---- valgrind tests : memory analyzer ----"
-	$(VALGRIND) ./datagen -g50M > $(VOID)
-	$(VALGRIND) ./zstd ; if [ $$? -eq 0 ] ; then echo "zstd without argument should have failed"; false; fi
-	./datagen -g80 | $(VALGRIND) ./zstd - -c > $(VOID)
-	./datagen -g16KB | $(VALGRIND) ./zstd -vf - -o $(VOID)
-	./datagen -g2930KB | $(VALGRIND) ./zstd -5 -vf - -o tmp
-	$(VALGRIND) ./zstd -vdf tmp -o $(VOID)
-	./datagen -g64MB | $(VALGRIND) ./zstd -vf - -o $(VOID)
-	@rm tmp
-	$(VALGRIND) ./fuzzer -T1mn -t1
-	$(VALGRIND) ./fullbench -i1
-	$(VALGRIND) ./zbufftest -T1mn
-
-endif
-
-
-ifneq (,$(filter MSYS%,$(shell uname)))
-HOST_OS = MSYS
-endif
-
-
-#------------------------------------------------------------------------
-#make tests validated only for MSYS, Linux, OSX, kFreeBSD and Hurd targets
-#------------------------------------------------------------------------
-ifneq (,$(filter $(HOST_OS),MSYS POSIX))
-zstd-playTests: datagen
-	ZSTD=$(ZSTD) ./playTests.sh $(ZSTDRTTEST)
-
-test: test-zstd test-fullbench test-fuzzer test-zbuff test-zstream
-
-test32: test-zstd32 test-fullbench32 test-fuzzer32 test-zbuff32 test-zstream32
-
-test-all: test test32 valgrindTest
-
-test-zstd: ZSTD = ./zstd
-test-zstd: zstd zstd-playTests
-
-test-zstd32: ZSTD = ./zstd32
-test-zstd32: zstd32 zstd-playTests
-
-test-zstd_nolegacy: ZSTD = ./zstd
-test-zstd_nolegacy: zstd_nolegacy zstd-playTests
-
-test-fullbench: fullbench datagen
-	./fullbench -i1
-	./fullbench -i1 -P0
-
-test-fullbench32: fullbench32 datagen
-	./fullbench32 -i1
-	./fullbench32 -i1 -P0
-
-test-fuzzer: fuzzer
-	./fuzzer $(FUZZERTEST)
-
-test-fuzzer32: fuzzer32
-	./fuzzer32 $(FUZZERTEST)
-
-test-zbuff: zbufftest
-	./zbufftest $(ZBUFFTEST)
-
-test-zbuff32: zbufftest32
-	./zbufftest32 $(ZBUFFTEST)
-
-test-zstream: zstreamtest
-	./zstreamtest $(ZBUFFTEST)
-
-test-zstream32: zstreamtest32
-	./zstreamtest32 $(ZBUFFTEST)
-
 endif

programs/README.md

@@ -0,0 +1,94 @@
+Command Line Interface for Zstandard library
+============================================
+
+Command Line Interface (CLI) can be created using the `make` command without any additional parameters.
+There are however other Makefile targets that create different variations of CLI:
+- `zstd` : default CLI supporting gzip-like arguments; includes dictionary builder, benchmark, and support for decompression of legacy zstd versions
+- `zstd32` : Same as `zstd`, but forced to compile in 32-bits mode
+- `zstd_nolegacy` : Same as `zstd` except of support for decompression of legacy zstd versions
+- `zstd-small` : CLI optimized for minimal size; without dictionary builder, benchmark, and support for decompression of legacy zstd versions
+- `zstd-compress` : compressor-only version of CLI; without dictionary builder, benchmark, and support for decompression of legacy zstd versions
+- `zstd-decompress` : decompressor-only version of CLI; without dictionary builder, benchmark, and support for decompression of legacy zstd versions
+
+
+#### Aggregation of parameters
+CLI supports aggregation of parameters i.e. `-b1`, `-e18`, and `-i1` can be joined into `-b1e18i1`. 
+
+
+#### Dictionary builder in Command Line Interface
+Zstd offers a training mode, which can be used to tune the algorithm for a selected
+type of data, by providing it with a few samples. The result of the training is stored
+in a file selected with the `-o` option (default name is `dictionary`),
+which can be loaded before compression and decompression.
+
+Using a dictionary, the compression ratio achievable on small data improves dramatically.
+These compression gains are achieved while simultaneously providing faster compression and decompression speeds.
+Dictionary work if there is some correlation in a family of small data (there is no universal dictionary). 
+Hence, deploying one dictionary per type of data will provide the greater benefits.
+Dictionary gains are mostly effective in the first few KB. Then, the compression algorithm
+will rely more and more on previously decoded content to compress the rest of the file.
+
+Usage of the dictionary builder and created dictionaries with CLI:
+
+1. Create the dictionary : `zstd --train FullPathToTrainingSet/* -o dictionaryName`
+2. Compress with the dictionary: `zstd FILE -D dictionaryName`
+3. Decompress with the dictionary: `zstd --decompress FILE.zst -D dictionaryName`
+
+
+
+#### Benchmark in Command Line Interface
+CLI includes in-memory compression benchmark module for zstd.
+The benchmark is conducted using given filenames. The files are read into memory and joined together.
+It makes benchmark more precise as it eliminates I/O overhead.
+Many filenames can be supplied as multiple parameters, parameters with wildcards or
+names of directories can be used as parameters with the `-r` option.
+
+The benchmark measures ratio, compressed size, compression and decompression speed.
+One can select compression levels starting from `-b` and ending with `-e`.
+The `-i` parameter selects minimal time used for each of tested levels.
+
+
+
+#### Usage of Command Line Interface
+The full list of options can be obtained with `-h` or `-H` parameter:
+```
+Usage :
+      zstd [args] [FILE(s)] [-o file]
+
+FILE    : a filename
+          with no FILE, or when FILE is - , read standard input
+Arguments :
+ -#     : # compression level (1-19, default:3)
+ -d     : decompression
+ -D file: use `file` as Dictionary
+ -o file: result stored into `file` (only if 1 input file)
+ -f     : overwrite output without prompting
+--rm    : remove source file(s) after successful de/compression
+ -k     : preserve source file(s) (default)
+ -h/-H  : display help/long help and exit
+
+Advanced arguments :
+ -V     : display Version number and exit
+ -v     : verbose mode; specify multiple times to increase log level (default:2)
+ -q     : suppress warnings; specify twice to suppress errors too
+ -c     : force write to standard output, even if it is the console
+ -r     : operate recursively on directories
+--ultra : enable levels beyond 19, up to 22 (requires more memory)
+--no-dictID : don't write dictID into header (dictionary compression)
+--[no-]check : integrity check (default:enabled)
+--test  : test compressed file integrity
+--[no-]sparse : sparse mode (default:enabled on file, disabled on stdout)
+
+Dictionary builder :
+--train ## : create a dictionary from a training set of files
+ -o file : `file` is dictionary name (default: dictionary)
+--maxdict ## : limit dictionary to specified size (default : 112640)
+ -s#    : dictionary selectivity level (default: 9)
+--dictID ## : force dictionary ID to specified value (default: random)
+
+Benchmark arguments :
+ -b#    : benchmark file(s), using # compression level (default : 1)
+ -e#    : test all compression levels from -bX to # (default: 1)
+ -i#    : minimum evaluation time in seconds (default : 3s)
+ -B#    : cut file into independent blocks of size # (default: no block)
+ ```

programs/bench.c

@@ -1,27 +1,12 @@
-/*
+/**
-    bench.c - open-source compression benchmark module
+ * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
-    Copyright (C) Yann Collet 2012-2016
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
 
-    GPL v2 License
-
-    This program is free software; you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
-    the Free Software Foundation; either version 2 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
-
-    You should have received a copy of the GNU General Public License along
-    with this program; if not, write to the Free Software Foundation, Inc.,
-    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-    You can contact the author at :
-    - zstd homepage : http://www.zstd.net
-    - zstd source repository : https://github.com/Cyan4973/zstd
-*/
 
 /* *************************************
 *  Includes
@@ -257,7 +242,7 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
             (void)fastestD; (void)crcOrig;   /*  unused when decompression disabled */
 #if 1
             /* Decompression */
-            memset(resultBuffer, 0xD6, srcSize);  /* warm result buffer */
+            if (!dCompleted) memset(resultBuffer, 0xD6, srcSize);  /* warm result buffer */
 
             UTIL_sleepMilli(1); /* give processor time to other processes */
             UTIL_waitForNextTick(ticksPerSecond);

programs/bench.h

@@ -1,26 +1,13 @@
-/*
+/**
-    bench.h - Demo program to benchmark open-source compression algorithm
+ * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
-    Copyright (C) Yann Collet 2012-2016
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
 
-    GPL v2 License
 
-    This program is free software; you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
-    the Free Software Foundation; either version 2 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
-
-    You should have received a copy of the GNU General Public License along
-    with this program; if not, write to the Free Software Foundation, Inc.,
-    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-    You can contact the author at :
-    - ZSTD homepage : http://www.zstd.net/
-*/
 #ifndef BENCH_H_121279284357
 #define BENCH_H_121279284357
 

programs/datagen.c

@@ -1,27 +1,13 @@
-/*
+/**
-    datagen.c - compressible data generator test tool
+ * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
-    Copyright (C) Yann Collet 2012-2016
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
 
-    GPL v2 License
 
-    This program is free software; you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
-    the Free Software Foundation; either version 2 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
-
-    You should have received a copy of the GNU General Public License along
-    with this program; if not, write to the Free Software Foundation, Inc.,
-    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-    You can contact the author at :
-    - zstd homepage : http://www.zstd.net/
-    - source repository : https://github.com/Cyan4973/zstd
-*/
 
 /* *************************************
 *  Compiler Options

programs/datagen.h

@@ -1,27 +1,11 @@
-/*
+/**
-    datagen.h - compressible data generator header
+ * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
-    Copyright (C) Yann Collet 2012-2015
+ * All rights reserved.
-
+ *
-    GPL v2 License
+ * This source code is licensed under the BSD-style license found in the
-
+ * LICENSE file in the root directory of this source tree. An additional grant
-    This program is free software; you can redistribute it and/or modify
+ * of patent rights can be found in the PATENTS file in the same directory.
-    it under the terms of the GNU General Public License as published by
+ */
-    the Free Software Foundation; either version 2 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
-
-    You should have received a copy of the GNU General Public License along
-    with this program; if not, write to the Free Software Foundation, Inc.,
-    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-    You can contact the author at :
-   - ZSTD source repository : https://github.com/Cyan4973/zstd
-   - Public forum : https://groups.google.com/forum/#!forum/lz4c
-*/
 
 
 #include <stddef.h>   /* size_t */

programs/dibio.c

@@ -1,26 +1,13 @@
-/*
+/**
-    dibio - I/O API for dictionary builder
+ * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
-    Copyright (C) Yann Collet 2016
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
 
-    GPL v2 License
 
-    This program is free software; you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
-    the Free Software Foundation; either version 2 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
-
-    You should have received a copy of the GNU General Public License along
-    with this program; if not, write to the Free Software Foundation, Inc.,
-    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-    You can contact the author at :
-    - zstd homepage : http://www.zstd.net/
-*/
 
 /*-*************************************
 *  Includes

programs/dibio.h

@@ -1,26 +1,11 @@
-/*
+/**
-    dibio.h - I/O API for dictionary builder
+ * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
-    Copyright (C) Yann Collet 2016
+ * All rights reserved.
-
+ *
-    GPL v2 License
+ * This source code is licensed under the BSD-style license found in the
-
+ * LICENSE file in the root directory of this source tree. An additional grant
-    This program is free software; you can redistribute it and/or modify
+ * of patent rights can be found in the PATENTS file in the same directory.
-    it under the terms of the GNU General Public License as published by
+ */
-    the Free Software Foundation; either version 2 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
-
-    You should have received a copy of the GNU General Public License along
-    with this program; if not, write to the Free Software Foundation, Inc.,
-    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-    You can contact the author at :
-    - zstd homepage : http://www.zstd.net/
-*/
 
 /* This library is designed for a single-threaded console application.
 *  It exit() and printf() into stderr when it encounters an error condition. */

programs/fileio.c

@@ -1,38 +1,26 @@
-/*
+/**
-  fileio.c - File i/o handler for zstd
+ * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
-  Copyright (C) Yann Collet 2013-2016
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
 
-  GPL v2 License
 
-  This program is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published by
-  the Free Software Foundation; either version 2 of the License, or
-  (at your option) any later version.
-
-  This program is distributed in the hope that it will be useful,
-  but WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-  GNU General Public License for more details.
-
-  You should have received a copy of the GNU General Public License along
-  with this program; if not, write to the Free Software Foundation, Inc.,
-  51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-  You can contact the author at :
-  - zstd homepage : http://www.zstd.net
-*/
 /*
   Note : this file is part of zstd command line, which is not library.
   The license of ZSTD library is BSD.
   The license of this file is GPLv2.
 */
 
+
 /* *************************************
-*  Tuning options
+ *  Tuning options
-***************************************/
+ ***************************************/
 #ifndef ZSTD_LEGACY_SUPPORT
 /* LEGACY_SUPPORT :
-*  decompressor can decode older formats (starting from Zstd 0.1+) */
+ *  decompressor can decode older formats (starting from Zstd 0.1+) */
 #  define ZSTD_LEGACY_SUPPORT 1
 #endif
 
@@ -65,7 +53,6 @@
 
 #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1)
 #  include "zstd_legacy.h"    /* ZSTD_isLegacy */
-#  include "fileio_legacy.h"  /* FIO_decompressLegacyFrame */
 #endif
 
 
@@ -610,7 +597,7 @@ static void FIO_fwriteSparseEnd(FILE* file, unsigned storedSkips)
 unsigned long long FIO_decompressFrame(dRess_t ress,
                                        FILE* foutput, FILE* finput, size_t alreadyLoaded)
 {
-    U64    frameSize = 0;
+    U64 frameSize = 0;
     size_t readSize;
     U32 storedSkips = 0;
 
@@ -702,13 +689,11 @@ static int FIO_decompressSrcFile(dRess_t ress, const char* srcFileName)
         readSomething = 1;
         if (sizeCheck != toRead) { DISPLAY("zstd: %s: unknown header \n", srcFileName); fclose(srcFile); return 1; }  /* srcFileName is empty */
         {   U32 const magic = MEM_readLE32(ress.srcBuffer);
-#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1)
+            if (((magic & 0xFFFFFFF0U) != ZSTD_MAGIC_SKIPPABLE_START) & (magic != ZSTD_MAGICNUMBER)
-            if (ZSTD_isLegacy(ress.srcBuffer, 4)) {
+#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1)
-                filesize += FIO_decompressLegacyFrame(dstFile, srcFile, ress.dictBuffer, ress.dictBufferSize, magic);
+                  & (!ZSTD_isLegacy(ress.srcBuffer, toRead))
-                continue;
-            }
 #endif
-            if (((magic & 0xFFFFFFF0U) != ZSTD_MAGIC_SKIPPABLE_START) & (magic != ZSTD_MAGICNUMBER)) {
+                ) {
                 if ((g_overwrite) && !strcmp (srcFileName, stdinmark)) {  /* pass-through mode */
                     unsigned const result = FIO_passThrough(dstFile, srcFile, ress.srcBuffer, ress.srcBufferSize);
                     if (fclose(srcFile)) EXM_THROW(32, "zstd: %s close error", srcFileName);  /* error should never happen */
@@ -723,7 +708,7 @@ static int FIO_decompressSrcFile(dRess_t ress, const char* srcFileName)
 
     /* Final Status */
     DISPLAYLEVEL(2, "\r%79s\r", "");
-    DISPLAYLEVEL(2, "%-20.20s: %llu bytes \n", srcFileName, filesize);
+    DISPLAYLEVEL(2, "%-20s: %llu bytes \n", srcFileName, filesize);
 
     /* Close */
     if (fclose(srcFile)) EXM_THROW(33, "zstd: %s close error", srcFileName);  /* error should never happen */

programs/fileio.h

@@ -1,26 +1,13 @@
-/*
+/**
-  fileio.h - file i/o handler
+ * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
-  Copyright (C) Yann Collet 2013-2016
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
 
-  GPL v2 License
 
-  This program is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published by
-  the Free Software Foundation; either version 2 of the License, or
-  (at your option) any later version.
-
-  This program is distributed in the hope that it will be useful,
-  but WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-  GNU General Public License for more details.
-
-  You should have received a copy of the GNU General Public License along
-  with this program; if not, write to the Free Software Foundation, Inc.,
-  51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-  You can contact the author at :
-  - ZSTD homepage : http://www.zstd.net/
-*/
 #pragma once
 
 #if defined (__cplusplus)

programs/legacy/fileio_legacy.c

@@ -1,673 +0,0 @@
-/*
-  fileio_legacy.c - File i/o handler for legacy format
-  Copyright (C) Yann Collet 2015-2016
-
-  GPL v2 License
-
-  This program is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published by
-  the Free Software Foundation; either version 2 of the License, or
-  (at your option) any later version.
-
-  This program is distributed in the hope that it will be useful,
-  but WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-  GNU General Public License for more details.
-
-  You should have received a copy of the GNU General Public License along
-  with this program; if not, write to the Free Software Foundation, Inc.,
-  51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-  You can contact the author at :
-  - zstd homepage : http://www.zstd.net
-  - zstd source repository : https://github.com/Cyan4973/zstd
-*/
-/*
-  Note : this file is not part of ZSTD compression library.
-  The license of ZSTD library is BSD.
-  The license of this file is GPLv2.
-*/
-
-/* *************************************
-*  Compiler Options
-***************************************/
-/* Disable some Visual warning messages */
-#ifdef _MSC_VER
-#  define _CRT_SECURE_NO_WARNINGS
-#  define _CRT_SECURE_NO_DEPRECATE     /* VS2005 */
-#  pragma warning(disable : 4127)      /* disable: C4127: conditional expression is constant */
-#endif
-
-#define _FILE_OFFSET_BITS 64   /* Large file support on 32-bits unix */
-#define _POSIX_SOURCE 1        /* enable fileno() within <stdio.h> on unix */
-
-
-/* *************************************
-*  Includes
-***************************************/
-#include <stdio.h>     /* fprintf, fopen, fread, _fileno, stdin, stdout */
-#include <stdlib.h>    /* malloc, free */
-#include <string.h>    /* strcmp, strlen */
-#include <time.h>      /* clock */
-#include <errno.h>     /* errno */
-#include "mem.h"
-#include "fileio_legacy.h"
-#include "zstd_legacy.h"  /* legacy support */
-
-
-/* *************************************
-*  OS-specific Includes
-***************************************/
-#if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) || defined(__CYGWIN__)
-#  include <fcntl.h>    /* _O_BINARY */
-#  include <io.h>       /* _setmode, _isatty */
-#  ifdef __MINGW32__
-   /* int _fileno(FILE *stream);   // seems no longer useful // MINGW somehow forgets to include this windows declaration into <stdio.h> */
-#  endif
-#  define SET_BINARY_MODE(file) { int unused = _setmode(_fileno(file), _O_BINARY); (void)unused; }
-#  define IS_CONSOLE(stdStream) _isatty(_fileno(stdStream))
-#else
-#  include <unistd.h>   /* isatty */
-#  define SET_BINARY_MODE(file)
-#  define IS_CONSOLE(stdStream) isatty(fileno(stdStream))
-#endif
-
-
-/* *************************************
-*  Constants
-***************************************/
-#define KB *(1U<<10)
-#define MB *(1U<<20)
-#define GB *(1U<<30)
-
-#define _1BIT  0x01
-#define _2BITS 0x03
-#define _3BITS 0x07
-#define _4BITS 0x0F
-#define _6BITS 0x3F
-#define _8BITS 0xFF
-
-#define BIT6  0x40
-#define BIT7  0x80
-
-#define FIO_FRAMEHEADERSIZE 5     /* as a define, because needed to allocated table on stack */
-#define FSE_CHECKSUM_SEED   0
-
-#define CACHELINE 64
-
-
-/* *************************************
-*  Macros
-***************************************/
-#define DISPLAY(...)         fprintf(stderr, __VA_ARGS__)
-#define DISPLAYLEVEL(l, ...) if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); }
-static U32 g_displayLevel = 1;   /* 0 : no display;   1: errors;   2 : + result + interaction + warnings;   3 : + progression;   4 : + information */
-
-#define DISPLAYUPDATE(l, ...) if (g_displayLevel>=l) { \
-            if ((FIO_GetMilliSpan(g_time) > refreshRate) || (g_displayLevel>=4)) \
-            { g_time = clock(); DISPLAY(__VA_ARGS__); \
-            if (g_displayLevel>=4) fflush(stdout); } }
-static const unsigned refreshRate = 150;
-static clock_t g_time = 0;
-
-
-/* *************************************
-*  Local Parameters
-***************************************/
-void FIO_legacy_setNotificationLevel(unsigned level) { g_displayLevel=level; }
-
-
-/* *************************************
-*  Exceptions
-***************************************/
-#ifndef DEBUG
-#  define DEBUG 0
-#endif
-#define DEBUGOUTPUT(...) if (DEBUG) DISPLAY(__VA_ARGS__);
-#define EXM_THROW(error, ...)                                             \
-{                                                                         \
-    DEBUGOUTPUT("Error defined at %s, line %i : \n", __FILE__, __LINE__); \
-    DISPLAYLEVEL(1, "Error %i : ", error);                                \
-    DISPLAYLEVEL(1, __VA_ARGS__);                                         \
-    DISPLAYLEVEL(1, "\n");                                                \
-    exit(error);                                                          \
-}
-
-
-/* *************************************
-*  Functions
-***************************************/
-static unsigned FIO_GetMilliSpan(clock_t nPrevious)
-{
-    clock_t nCurrent = clock();
-    unsigned nSpan = (unsigned)(((nCurrent - nPrevious) * 1000) / CLOCKS_PER_SEC);
-    return nSpan;
-}
-
-
-unsigned long long FIOv01_decompressFrame(FILE* foutput, FILE* finput)
-{
-    size_t const outBuffSize = 512 KB;
-    BYTE* outBuff = (BYTE*)malloc(outBuffSize);
-    size_t inBuffSize = 128 KB + 8;
-    BYTE inBuff[128 KB + 8];
-    BYTE* op = outBuff;
-    BYTE* const oend = outBuff + outBuffSize;
-    U64   filesize = 0;
-    size_t toRead;
-    size_t sizeCheck;
-    ZSTDv01_Dctx* dctx = ZSTDv01_createDCtx();
-
-
-    /* init */
-    if (outBuff==NULL) EXM_THROW(41, "Error : not enough memory to decode legacy frame");
-
-    /* restore header, already read from input */
-    MEM_writeLE32(inBuff, ZSTDv01_magicNumberLE);
-    sizeCheck = ZSTDv01_decompressContinue(dctx, NULL, 0, inBuff, sizeof(ZSTDv01_magicNumberLE));   /* Decode frame header */
-    if (ZSTDv01_isError(sizeCheck)) EXM_THROW(42, "Error decoding legacy header");
-
-    /* Main decompression Loop */
-    toRead = ZSTDv01_nextSrcSizeToDecompress(dctx);
-    while (toRead){
-        size_t readSize, decodedSize;
-
-        /* Fill input buffer */
-        if (toRead > inBuffSize)
-            EXM_THROW(43, "too large block");
-        readSize = fread(inBuff, 1, toRead, finput);
-        if (readSize != toRead)
-            EXM_THROW(44, "Read error");
-
-        /* Decode block */
-        decodedSize = ZSTDv01_decompressContinue(dctx, op, oend-op, inBuff, readSize);
-        if (ZSTDv01_isError(decodedSize)) EXM_THROW(45, "Decoding error : input corrupted");
-
-        if (decodedSize) {  /* not a header */
-            /* Write block */
-            sizeCheck = fwrite(op, 1, decodedSize, foutput);
-            if (sizeCheck != decodedSize) EXM_THROW(46, "Write error : unable to write data block to destination file");
-            filesize += decodedSize;
-            op += decodedSize;
-            if (op==oend) op = outBuff;
-            DISPLAYUPDATE(2, "\rDecoded : %u MB...     ", (U32)(filesize>>20) );
-        }
-
-        /* prepare for next Block */
-        toRead = ZSTDv01_nextSrcSizeToDecompress(dctx);
-    }
-
-    /* release resources */
-    free(outBuff);
-    free(dctx);
-    return filesize;
-}
-
-
-unsigned long long FIOv02_decompressFrame(FILE* foutput, FILE* finput)
-{
-    size_t const outBuffSize = 512 KB;
-    BYTE* outBuff = (BYTE*)malloc(outBuffSize);
-    size_t inBuffSize = 128 KB + 8;
-    BYTE inBuff[128 KB + 8];
-    BYTE* op = outBuff;
-    BYTE* const oend = outBuff + outBuffSize;
-    U64   filesize = 0;
-    size_t toRead;
-    size_t sizeCheck;
-    ZSTDv02_Dctx* dctx = ZSTDv02_createDCtx();
-
-    /* init */
-    if (outBuff==NULL) EXM_THROW(41, "Error : not enough memory to decode legacy frame");
-
-    /* restore header, already read from input */
-    MEM_writeLE32(inBuff, ZSTDv02_magicNumber);
-    sizeCheck = ZSTDv02_decompressContinue(dctx, NULL, 0, inBuff, sizeof(ZSTDv02_magicNumber));   /* Decode frame header */
-    if (ZSTDv02_isError(sizeCheck)) EXM_THROW(42, "Error decoding legacy header");
-
-    /* Main decompression Loop */
-    toRead = ZSTDv02_nextSrcSizeToDecompress(dctx);
-    while (toRead) {
-        size_t readSize, decodedSize;
-
-        /* Fill input buffer */
-        if (toRead > inBuffSize)
-            EXM_THROW(43, "too large block");
-        readSize = fread(inBuff, 1, toRead, finput);
-        if (readSize != toRead)
-            EXM_THROW(44, "Read error");
-
-        /* Decode block */
-        decodedSize = ZSTDv02_decompressContinue(dctx, op, oend-op, inBuff, readSize);
-        if (ZSTDv02_isError(decodedSize)) EXM_THROW(45, "Decoding error : input corrupted");
-
-        if (decodedSize) {   /* not a header */
-            /* Write block */
-            sizeCheck = fwrite(op, 1, decodedSize, foutput);
-            if (sizeCheck != decodedSize) EXM_THROW(46, "Write error : unable to write data block to destination file");
-            filesize += decodedSize;
-            op += decodedSize;
-            if (op==oend) op = outBuff;
-            DISPLAYUPDATE(2, "\rDecoded : %u MB...     ", (U32)(filesize>>20) );
-        }
-
-        /* prepare for next Block */
-        toRead = ZSTDv02_nextSrcSizeToDecompress(dctx);
-    }
-
-    /* release resources */
-    free(outBuff);
-    free(dctx);
-    return filesize;
-}
-
-
-unsigned long long FIOv03_decompressFrame(FILE* foutput, FILE* finput)
-{
-    size_t const outBuffSize = 512 KB;
-    BYTE* outBuff = (BYTE*)malloc(outBuffSize);
-    size_t inBuffSize = 128 KB + 8;
-    BYTE inBuff[128 KB + 8];
-    BYTE* op = outBuff;
-    BYTE* const oend = outBuff + outBuffSize;
-    U64   filesize = 0;
-    size_t toRead;
-    size_t sizeCheck;
-    ZSTDv03_Dctx* dctx = ZSTDv03_createDCtx();
-
-    /* init */
-    if (outBuff==NULL) EXM_THROW(41, "Error : not enough memory to decode legacy frame");
-
-    /* restore header, already read from input */
-    MEM_writeLE32(inBuff, ZSTDv03_magicNumber);
-    sizeCheck = ZSTDv03_decompressContinue(dctx, NULL, 0, inBuff, sizeof(ZSTDv03_magicNumber));   /* Decode frame header */
-    if (ZSTDv03_isError(sizeCheck)) EXM_THROW(42, "Error decoding legacy header");
-
-    /* Main decompression Loop */
-    toRead = ZSTDv03_nextSrcSizeToDecompress(dctx);
-    while (toRead) {
-        size_t readSize, decodedSize;
-
-        /* Fill input buffer */
-        if (toRead > inBuffSize)
-            EXM_THROW(43, "too large block");
-        readSize = fread(inBuff, 1, toRead, finput);
-        if (readSize != toRead)
-            EXM_THROW(44, "Read error");
-
-        /* Decode block */
-        decodedSize = ZSTDv03_decompressContinue(dctx, op, oend-op, inBuff, readSize);
-        if (ZSTDv03_isError(decodedSize)) EXM_THROW(45, "Decoding error : input corrupted");
-
-        if (decodedSize) {   /* not a header */
-            /* Write block */
-            sizeCheck = fwrite(op, 1, decodedSize, foutput);
-            if (sizeCheck != decodedSize) EXM_THROW(46, "Write error : unable to write data block to destination file");
-            filesize += decodedSize;
-            op += decodedSize;
-            if (op==oend) op = outBuff;
-            DISPLAYUPDATE(2, "\rDecoded : %u MB...     ", (U32)(filesize>>20) );
-        }
-
-        /* prepare for next Block */
-        toRead = ZSTDv03_nextSrcSizeToDecompress(dctx);
-    }
-
-    /* release resources */
-    free(outBuff);
-    free(dctx);
-    return filesize;
-}
-
-
-/*=====    v0.4.x    =====*/
-
-typedef struct {
-    void*  srcBuffer;
-    size_t srcBufferSize;
-    void*  dstBuffer;
-    size_t dstBufferSize;
-    void*  dictBuffer;
-    size_t dictBufferSize;
-    ZBUFFv04_DCtx* dctx;
-} dRessv04_t;
-
-static dRessv04_t FIOv04_createDResources(void)
-{
-    dRessv04_t ress;
-
-    /* init */
-    ress.dctx = ZBUFFv04_createDCtx();
-    if (ress.dctx==NULL) EXM_THROW(60, "Can't create ZBUFF decompression context");
-    ress.dictBuffer = NULL; ress.dictBufferSize=0;
-
-    /* Allocate Memory */
-    ress.srcBufferSize = ZBUFFv04_recommendedDInSize();
-    ress.srcBuffer = malloc(ress.srcBufferSize);
-    ress.dstBufferSize = ZBUFFv04_recommendedDOutSize();
-    ress.dstBuffer = malloc(ress.dstBufferSize);
-    if (!ress.srcBuffer || !ress.dstBuffer) EXM_THROW(61, "Allocation error : not enough memory");
-
-    return ress;
-}
-
-static void FIOv04_freeDResources(dRessv04_t ress)
-{
-    size_t errorCode = ZBUFFv04_freeDCtx(ress.dctx);
-    if (ZBUFFv04_isError(errorCode)) EXM_THROW(69, "Error : can't free ZBUFF context resource : %s", ZBUFFv04_getErrorName(errorCode));
-    free(ress.srcBuffer);
-    free(ress.dstBuffer);
-    free(ress.dictBuffer);
-}
-
-
-unsigned long long FIOv04_decompressFrame(dRessv04_t ress,
-                                          FILE* foutput, FILE* finput)
-{
-    U64    frameSize = 0;
-    size_t readSize = 4;
-
-    MEM_writeLE32(ress.srcBuffer, ZSTDv04_magicNumber);
-    ZBUFFv04_decompressInit(ress.dctx);
-    ZBUFFv04_decompressWithDictionary(ress.dctx, ress.dictBuffer, ress.dictBufferSize);
-
-    while (1) {
-        /* Decode */
-        size_t sizeCheck;
-        size_t inSize=readSize, decodedSize=ress.dstBufferSize;
-        size_t toRead = ZBUFFv04_decompressContinue(ress.dctx, ress.dstBuffer, &decodedSize, ress.srcBuffer, &inSize);
-        if (ZBUFFv04_isError(toRead)) EXM_THROW(36, "Decoding error : %s", ZBUFFv04_getErrorName(toRead));
-        readSize -= inSize;
-
-        /* Write block */
-        sizeCheck = fwrite(ress.dstBuffer, 1, decodedSize, foutput);
-        if (sizeCheck != decodedSize) EXM_THROW(37, "Write error : unable to write data block to destination file");
-        frameSize += decodedSize;
-        DISPLAYUPDATE(2, "\rDecoded : %u MB...     ", (U32)(frameSize>>20) );
-
-        if (toRead == 0) break;
-        if (readSize) EXM_THROW(38, "Decoding error : should consume entire input");
-
-        /* Fill input buffer */
-        if (toRead > ress.srcBufferSize) EXM_THROW(34, "too large block");
-        readSize = fread(ress.srcBuffer, 1, toRead, finput);
-        if (readSize != toRead) EXM_THROW(35, "Read error");
-    }
-
-    return frameSize;
-}
-
-
-/*=====    v0.5.x    =====*/
-
-typedef struct {
-    void*  srcBuffer;
-    size_t srcBufferSize;
-    void*  dstBuffer;
-    size_t dstBufferSize;
-    const void*  dictBuffer;
-    size_t dictBufferSize;
-    ZBUFFv05_DCtx* dctx;
-} dRessv05_t;
-
-static dRessv05_t FIOv05_createDResources(void)
-{
-    dRessv05_t ress;
-
-    /* init */
-    ress.dctx = ZBUFFv05_createDCtx();
-    if (ress.dctx==NULL) EXM_THROW(60, "Can't create ZBUFF decompression context");
-    ress.dictBuffer = NULL; ress.dictBufferSize=0;
-
-    /* Allocate Memory */
-    ress.srcBufferSize = ZBUFFv05_recommendedDInSize();
-    ress.srcBuffer = malloc(ress.srcBufferSize);
-    ress.dstBufferSize = ZBUFFv05_recommendedDOutSize();
-    ress.dstBuffer = malloc(ress.dstBufferSize);
-    if (!ress.srcBuffer || !ress.dstBuffer) EXM_THROW(61, "Allocation error : not enough memory");
-
-    return ress;
-}
-
-static void FIOv05_freeDResources(dRessv05_t ress)
-{
-    size_t const errorCode = ZBUFFv05_freeDCtx(ress.dctx);
-    if (ZBUFFv05_isError(errorCode)) EXM_THROW(69, "Error : can't free ZBUFF context resource : %s", ZBUFFv05_getErrorName(errorCode));
-    free(ress.srcBuffer);
-    free(ress.dstBuffer);
-}
-
-
-unsigned long long FIOv05_decompressFrame(dRessv05_t ress,
-                                          FILE* foutput, FILE* finput)
-{
-    U64    frameSize = 0;
-    size_t readSize  = 4;
-
-    MEM_writeLE32(ress.srcBuffer, ZSTDv05_MAGICNUMBER);
-    ZBUFFv05_decompressInitDictionary(ress.dctx, ress.dictBuffer, ress.dictBufferSize);
-
-    while (1) {
-        /* Decode */
-        size_t inSize=readSize, decodedSize=ress.dstBufferSize;
-        size_t toRead = ZBUFFv05_decompressContinue(ress.dctx, ress.dstBuffer, &decodedSize, ress.srcBuffer, &inSize);
-        if (ZBUFFv05_isError(toRead)) EXM_THROW(36, "Decoding error : %s", ZBUFFv05_getErrorName(toRead));
-        readSize -= inSize;
-
-        /* Write block */
-        { size_t const sizeCheck = fwrite(ress.dstBuffer, 1, decodedSize, foutput);
-          if (sizeCheck != decodedSize) EXM_THROW(37, "Write error : unable to write data block to destination file"); }
-        frameSize += decodedSize;
-        DISPLAYUPDATE(2, "\rDecoded : %u MB...     ", (U32)(frameSize>>20) );
-
-        if (toRead == 0) break;
-        if (readSize) EXM_THROW(38, "Decoding error : should consume entire input");
-
-        /* Fill input buffer */
-        if (toRead > ress.srcBufferSize) EXM_THROW(34, "too large block");
-        readSize = fread(ress.srcBuffer, 1, toRead, finput);
-        if (readSize != toRead) EXM_THROW(35, "Read error");
-    }
-
-    return frameSize;
-}
-
-
-/*=====    v0.6.x    =====*/
-
-typedef struct {
-    void*  srcBuffer;
-    size_t srcBufferSize;
-    void*  dstBuffer;
-    size_t dstBufferSize;
-    const void*  dictBuffer;
-    size_t dictBufferSize;
-    ZBUFFv06_DCtx* dctx;
-} dRessv06_t;
-
-static dRessv06_t FIOv06_createDResources(void)
-{
-    dRessv06_t ress;
-
-    /* init */
-    ress.dctx = ZBUFFv06_createDCtx();
-    if (ress.dctx==NULL) EXM_THROW(60, "Can't create ZBUFF decompression context");
-    ress.dictBuffer = NULL; ress.dictBufferSize=0;
-
-    /* Allocate Memory */
-    ress.srcBufferSize = ZBUFFv06_recommendedDInSize();
-    ress.srcBuffer = malloc(ress.srcBufferSize);
-    ress.dstBufferSize = ZBUFFv06_recommendedDOutSize();
-    ress.dstBuffer = malloc(ress.dstBufferSize);
-    if (!ress.srcBuffer || !ress.dstBuffer) EXM_THROW(61, "Allocation error : not enough memory");
-
-    return ress;
-}
-
-static void FIOv06_freeDResources(dRessv06_t ress)
-{
-    size_t const errorCode = ZBUFFv06_freeDCtx(ress.dctx);
-    if (ZBUFFv06_isError(errorCode)) EXM_THROW(69, "Error : can't free ZBUFF context resource : %s", ZBUFFv06_getErrorName(errorCode));
-    free(ress.srcBuffer);
-    free(ress.dstBuffer);
-}
-
-
-unsigned long long FIOv06_decompressFrame(dRessv06_t ress,
-                                          FILE* foutput, FILE* finput)
-{
-    U64    frameSize = 0;
-    size_t readSize  = 4;
-
-    MEM_writeLE32(ress.srcBuffer, ZSTDv06_MAGICNUMBER);
-    ZBUFFv06_decompressInitDictionary(ress.dctx, ress.dictBuffer, ress.dictBufferSize);
-
-    while (1) {
-        /* Decode */
-        size_t inSize=readSize, decodedSize=ress.dstBufferSize;
-        size_t toRead = ZBUFFv06_decompressContinue(ress.dctx, ress.dstBuffer, &decodedSize, ress.srcBuffer, &inSize);
-        if (ZBUFFv06_isError(toRead)) EXM_THROW(36, "Decoding error : %s", ZBUFFv06_getErrorName(toRead));
-        readSize -= inSize;
-
-        /* Write block */
-        { size_t const sizeCheck = fwrite(ress.dstBuffer, 1, decodedSize, foutput);
-          if (sizeCheck != decodedSize) EXM_THROW(37, "Write error : unable to write data block to destination file"); }
-        frameSize += decodedSize;
-        DISPLAYUPDATE(2, "\rDecoded : %u MB...     ", (U32)(frameSize>>20) );
-
-        if (toRead == 0) break;
-        if (readSize) EXM_THROW(38, "Decoding error : should consume entire input");
-
-        /* Fill input buffer */
-        if (toRead > ress.srcBufferSize) EXM_THROW(34, "too large block");
-        readSize = fread(ress.srcBuffer, 1, toRead, finput);
-        if (readSize != toRead) EXM_THROW(35, "Read error");
-    }
-
-    return frameSize;
-}
-
-
-/*=====    v0.7.x    =====*/
-
-typedef struct {
-    void*  srcBuffer;
-    size_t srcBufferSize;
-    void*  dstBuffer;
-    size_t dstBufferSize;
-    const void*  dictBuffer;
-    size_t dictBufferSize;
-    ZBUFFv07_DCtx* dctx;
-} dRessv07_t;
-
-static dRessv07_t FIOv07_createDResources(void)
-{
-    dRessv07_t ress;
-
-    /* init */
-    ress.dctx = ZBUFFv07_createDCtx();
-    if (ress.dctx==NULL) EXM_THROW(60, "Can't create ZBUFF decompression context");
-    ress.dictBuffer = NULL; ress.dictBufferSize=0;
-
-    /* Allocate Memory */
-    ress.srcBufferSize = ZBUFFv07_recommendedDInSize();
-    ress.srcBuffer = malloc(ress.srcBufferSize);
-    ress.dstBufferSize = ZBUFFv07_recommendedDOutSize();
-    ress.dstBuffer = malloc(ress.dstBufferSize);
-    if (!ress.srcBuffer || !ress.dstBuffer) EXM_THROW(61, "Allocation error : not enough memory");
-
-    return ress;
-}
-
-static void FIOv07_freeDResources(dRessv07_t ress)
-{
-    size_t const errorCode = ZBUFFv07_freeDCtx(ress.dctx);
-    if (ZBUFFv07_isError(errorCode)) EXM_THROW(69, "Error : can't free ZBUFF context resource : %s", ZBUFFv07_getErrorName(errorCode));
-    free(ress.srcBuffer);
-    free(ress.dstBuffer);
-}
-
-
-unsigned long long FIOv07_decompressFrame(dRessv07_t ress,
-                                          FILE* foutput, FILE* finput)
-{
-    U64    frameSize = 0;
-    size_t readSize  = 4;
-
-    MEM_writeLE32(ress.srcBuffer, ZSTDv07_MAGICNUMBER);
-    ZBUFFv07_decompressInitDictionary(ress.dctx, ress.dictBuffer, ress.dictBufferSize);
-
-    while (1) {
-        /* Decode */
-        size_t inSize=readSize, decodedSize=ress.dstBufferSize;
-        size_t toRead = ZBUFFv07_decompressContinue(ress.dctx, ress.dstBuffer, &decodedSize, ress.srcBuffer, &inSize);
-        if (ZBUFFv07_isError(toRead)) EXM_THROW(36, "Decoding error : %s", ZBUFFv07_getErrorName(toRead));
-        readSize -= inSize;
-
-        /* Write block */
-        { size_t const sizeCheck = fwrite(ress.dstBuffer, 1, decodedSize, foutput);
-          if (sizeCheck != decodedSize) EXM_THROW(37, "Write error : unable to write data block to destination file"); }
-        frameSize += decodedSize;
-        DISPLAYUPDATE(2, "\rDecoded : %u MB...     ", (U32)(frameSize>>20) );
-
-        if (toRead == 0) break;
-        if (readSize) EXM_THROW(38, "Decoding error : should consume entire input");
-
-        /* Fill input buffer */
-        if (toRead > ress.srcBufferSize) EXM_THROW(34, "too large block");
-        readSize = fread(ress.srcBuffer, 1, toRead, finput);
-        if (readSize != toRead) EXM_THROW(35, "Read error");
-    }
-
-    return frameSize;
-}
-
-
-/*=====   General legacy dispatcher   =====*/
-
-unsigned long long FIO_decompressLegacyFrame(FILE* foutput, FILE* finput,
-                                             const void* dictBuffer, size_t dictSize,
-                                             U32 magicNumberLE)
-{
-    switch(magicNumberLE)
-    {
-        case ZSTDv01_magicNumberLE :
-            return FIOv01_decompressFrame(foutput, finput);
-        case ZSTDv02_magicNumber :
-            return FIOv02_decompressFrame(foutput, finput);
-        case ZSTDv03_magicNumber :
-            return FIOv03_decompressFrame(foutput, finput);
-        case ZSTDv04_magicNumber :
-            {   dRessv04_t r = FIOv04_createDResources();
-                unsigned long long const s = FIOv04_decompressFrame(r, foutput, finput);
-                FIOv04_freeDResources(r);
-                return s;
-            }
-        case ZSTDv05_MAGICNUMBER :
-            {   dRessv05_t r = FIOv05_createDResources();
-                r.dictBuffer = dictBuffer;
-                r.dictBufferSize = dictSize;
-                {   unsigned long long const s = FIOv05_decompressFrame(r, foutput, finput);
-                    FIOv05_freeDResources(r);
-                    return s;
-            }   }
-        case ZSTDv06_MAGICNUMBER :
-            {   dRessv06_t r = FIOv06_createDResources();
-                r.dictBuffer = dictBuffer;
-                r.dictBufferSize = dictSize;
-                {   unsigned long long const s = FIOv06_decompressFrame(r, foutput, finput);
-                    FIOv06_freeDResources(r);
-                    return s;
-            }   }
-        case ZSTDv07_MAGICNUMBER :
-            {   dRessv07_t r = FIOv07_createDResources();
-                r.dictBuffer = dictBuffer;
-                r.dictBufferSize = dictSize;
-                {   unsigned long long const s = FIOv07_decompressFrame(r, foutput, finput);
-                    FIOv07_freeDResources(r);
-                    return s;
-            }   }
-        default :
-            return ERROR(prefix_unknown);
-    }
-}

programs/legacy/fileio_legacy.h

@@ -1,48 +0,0 @@
-/*
-  fileio.h - file i/o handler
-  Copyright (C) Yann Collet 2013-2015
-
-  GPL v2 License
-
-  This program is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published by
-  the Free Software Foundation; either version 2 of the License, or
-  (at your option) any later version.
-
-  This program is distributed in the hope that it will be useful,
-  but WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-  GNU General Public License for more details.
-
-  You should have received a copy of the GNU General Public License along
-  with this program; if not, write to the Free Software Foundation, Inc.,
-  51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-  You can contact the author at :
-  - ZSTD source repository : https://github.com/Cyan4973/zstd
-  - Public forum : https://groups.google.com/forum/#!forum/lz4c
-*/
-#pragma once
-
-#if defined (__cplusplus)
-extern "C" {
-#endif
-
-
-/* *************************************
-*  Parameters
-***************************************/
-void FIO_legacy_setNotificationLevel(unsigned level);
-
-
-/* *************************************
-*  Stream/File functions
-***************************************/
-unsigned long long FIO_decompressLegacyFrame(FILE* foutput, FILE* finput,
-                                             const void* dictBuffer, size_t dictSize,
-                                             U32 magicNumberLE);
-
-
-#if defined (__cplusplus)
-}
-#endif

programs/util.h

@@ -1,26 +1,12 @@
-/* ******************************************************************
+/**
-  util.h - utility functions
+ * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
-  Copyright (C) 2016, Przemyslaw Skibinski, Yann Collet.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
 
-  GPL v2 License
-
-  This program is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published by
-  the Free Software Foundation; either version 2 of the License, or
-  (at your option) any later version.
-
-  This program is distributed in the hope that it will be useful,
-  but WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-  GNU General Public License for more details.
-
-  You should have received a copy of the GNU General Public License along
-  with this program; if not, write to the Free Software Foundation, Inc.,
-  51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-  You can contact the author at :
-  - ZSTD homepage : http://www.zstd.net/
-*/
 #ifndef UTIL_H_MODULE
 #define UTIL_H_MODULE
 
@@ -245,7 +231,7 @@ UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_
         if (cFile.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) {
             if (strcmp (cFile.cFileName, "..") == 0 ||
                 strcmp (cFile.cFileName, ".") == 0) continue;
-         //   printf ("[%s]\n", path);
+
             nbFiles += UTIL_prepareFileList(path, bufStart, pos, bufEnd);  /* Recursively call "UTIL_prepareFileList" with the new path. */
             if (*bufStart == NULL) { FindClose(hFile); return 0; }
         }
@@ -261,7 +247,6 @@ UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_
                 *pos += pathLength + 1;
                 nbFiles++;
             }
-         //   printf ("%s\\%s nbFiles=%d left=%d\n", dirName, cFile.cFileName, nbFiles, (int)(bufEnd - *bufStart));
         }
     } while (FindNextFile(hFile, &cFile));
 
@@ -297,7 +282,6 @@ UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_
             continue;
         }
         if (UTIL_isDirectory(path)) {
-         //   printf ("[%s]\n", path);
             nbFiles += UTIL_prepareFileList(path, bufStart, pos, bufEnd);  /* Recursively call "UTIL_prepareFileList" with the new path. */
             if (*bufStart == NULL) { closedir(dir); return 0; }
         } else {
@@ -312,9 +296,8 @@ UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_
                 *pos += pathLength + 1;
                 nbFiles++;
             }
-         //   printf ("%s/%s nbFiles=%d left=%d\n", dirName, entry->d_name, nbFiles, (int)(bufEnd - *bufStart));
         }
-        errno = 0; // clear errno after UTIL_isDirectory, UTIL_prepareFileList
+        errno = 0; /* clear errno after UTIL_isDirectory, UTIL_prepareFileList */
     }
 
     if (errno != 0) {
@@ -335,7 +318,7 @@ UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_
     return 0;
 }
 
-#endif // #ifdef _WIN32
+#endif /* #ifdef _WIN32 */
 
 /*
  * UTIL_createFileList - takes a list of files and directories (params: inputNames, inputNamesNb), scans directories,
@@ -405,4 +388,3 @@ UTIL_STATIC void UTIL_freeFileList(const char** filenameTable, char* allocatedBu
 #endif
 
 #endif /* UTIL_H_MODULE */
-

programs/zstdcli.c

@@ -1,26 +1,13 @@
-/*
+/**
-  zstdcli - Command Line Interface (cli) for zstd
+ * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
-  Copyright (C) Yann Collet 2014-2016
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
 
-  GPL v2 License
 
-  This program is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published by
-  the Free Software Foundation; either version 2 of the License, or
-  (at your option) any later version.
-
-  This program is distributed in the hope that it will be useful,
-  but WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-  GNU General Public License for more details.
-
-  You should have received a copy of the GNU General Public License along
-  with this program; if not, write to the Free Software Foundation, Inc.,
-  51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-  You can contact the author at :
-  - zstd homepage : http://www.zstd.net/
-*/
 /*
   Note : this is a user program, not part of libzstd.
   The license of libzstd is BSD.
@@ -478,8 +465,8 @@ int main(int argCount, char** argv)
     }
 
     /* No input filename ==> use stdin and stdout */
-    filenameIdx += !filenameIdx;   /*< default input is stdin */
+    filenameIdx += !filenameIdx;   /* filenameTable[0] is stdin by default */
-    if (!strcmp(filenameTable[0], stdinmark) && !outFileName) outFileName = stdoutmark;   /*< when input is stdin, default output is stdout */
+    if (!strcmp(filenameTable[0], stdinmark) && !outFileName) outFileName = stdoutmark;   /* when input is stdin, default output is stdout */
 
     /* Check if input/output defined as console; trigger an error in this case */
     if (!strcmp(filenameTable[0], stdinmark) && IS_CONSOLE(stdin) ) CLEAN_RETURN(badusage(programName));

projects/README.md

@@ -1,10 +1,23 @@
-projects for various integrated development environments (IDE)
+Projects for various integrated development environments (IDE)
-================================
+==============================================================
 
 #### Included projects
 
 The following projects are included with the zstd distribution:
-- cmake - CMake project contributed by Artyom Dymchenko
+- `cmake` - CMake project contributed by Artyom Dymchenko
-- VS2008 - Visual Studio 2008 project
+- `VS2005` - Visual Studio 2005 project
-- VS2010 - Visual Studio 2010 project (which also works well with Visual Studio 2012, 2013, 2015)
+- `VS2008` - Visual Studio 2008 project
-- build - command line scripts prepared for Visual Studio compilation without IDE
+- `VS2010` - Visual Studio 2010 project (which also works well with Visual Studio 2012, 2013, 2015)
+- `build` - command line scripts prepared for Visual Studio compilation without IDE
+
+
+#### How to compile zstd with Visual Studio
+
+1. Install Visual Studio e.g. VS 2015 Community Edition (it's free).
+2. Download the latest version of zstd from https://github.com/Cyan4973/zstd/releases
+3. Decompress ZIP archive.
+4. Go to decompressed directory then to `projects` then `VS2010` and open `zstd.sln`
+5. Visual Studio will ask about converting VS2010 project to VS2015 and you should agree.
+6. Change `Debug` to `Release` and if you have 64-bit Windows change also `Win32` to `x64`.
+7. Press F7 on keyboard or select `BUILD` from the menu bar and choose `Build Solution`.
+8. If compilation will be fine a compiled executable will be in `projects\VS2010\bin\x64\Release\zstd.exe`

projects/VS2005/fullbench/fullbench.vcproj

@@ -43,7 +43,7 @@
 			<Tool
 				Name="VCCLCompilerTool"
 				Optimization="0"
-				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy"
+				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs"
 				PreprocessorDefinitions="WIN32;_DEBUG;_CONSOLE"
 				MinimalRebuild="true"
 				BasicRuntimeChecks="3"
@@ -119,7 +119,7 @@
 				Optimization="2"
 				EnableIntrinsicFunctions="true"
 				OmitFramePointers="true"
-				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy"
+				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs"
 				PreprocessorDefinitions="WIN32;NDEBUG;_CONSOLE"
 				RuntimeLibrary="0"
 				EnableFunctionLevelLinking="true"
@@ -193,7 +193,7 @@
 			<Tool
 				Name="VCCLCompilerTool"
 				Optimization="0"
-				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy"
+				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs"
 				PreprocessorDefinitions="WIN32;_DEBUG;_CONSOLE"
 				MinimalRebuild="true"
 				BasicRuntimeChecks="3"
@@ -270,7 +270,7 @@
 				Optimization="2"
 				EnableIntrinsicFunctions="true"
 				OmitFramePointers="true"
-				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy"
+				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs"
 				PreprocessorDefinitions="WIN32;NDEBUG;_CONSOLE"
 				RuntimeLibrary="0"
 				EnableFunctionLevelLinking="true"
@@ -348,7 +348,7 @@
 				>
 			</File>
 			<File
-				RelativePath="..\..\..\programs\fullbench.c"
+				RelativePath="..\..\..\tests\fullbench.c"
 				>
 			</File>
 			<File

projects/VS2005/fuzzer/fuzzer.vcproj

@@ -43,7 +43,7 @@
 			<Tool
 				Name="VCCLCompilerTool"
 				Optimization="0"
-				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy"
+				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(SolutionDir)..\..\programs"
 				PreprocessorDefinitions="WIN32;_DEBUG;_CONSOLE"
 				MinimalRebuild="true"
 				BasicRuntimeChecks="3"
@@ -119,7 +119,7 @@
 				Optimization="2"
 				EnableIntrinsicFunctions="true"
 				OmitFramePointers="true"
-				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy"
+				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(SolutionDir)..\..\programs"
 				PreprocessorDefinitions="WIN32;NDEBUG;_CONSOLE"
 				RuntimeLibrary="0"
 				EnableFunctionLevelLinking="true"
@@ -193,7 +193,7 @@
 			<Tool
 				Name="VCCLCompilerTool"
 				Optimization="0"
-				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy"
+				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(SolutionDir)..\..\programs"
 				PreprocessorDefinitions="WIN32;_DEBUG;_CONSOLE"
 				MinimalRebuild="true"
 				BasicRuntimeChecks="3"
@@ -270,7 +270,7 @@
 				Optimization="2"
 				EnableIntrinsicFunctions="true"
 				OmitFramePointers="true"
-				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy"
+				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(SolutionDir)..\..\programs"
 				PreprocessorDefinitions="WIN32;NDEBUG;_CONSOLE"
 				RuntimeLibrary="0"
 				EnableFunctionLevelLinking="true"
@@ -348,7 +348,7 @@
 				>
 			</File>
 			<File
-				RelativePath="..\..\..\programs\fuzzer.c"
+				RelativePath="..\..\..\tests\fuzzer.c"
 				>
 			</File>
 			<File

projects/VS2005/zstd/zstd.vcproj

@@ -43,7 +43,7 @@
 			<Tool
 				Name="VCCLCompilerTool"
 				Optimization="0"
-				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\dictBuilder"
+				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\dictBuilder"
 				PreprocessorDefinitions="WIN32;_DEBUG;_CONSOLE"
 				MinimalRebuild="true"
 				BasicRuntimeChecks="3"
@@ -120,7 +120,7 @@
 				Optimization="2"
 				EnableIntrinsicFunctions="true"
 				OmitFramePointers="true"
-				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\dictBuilder"
+				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\dictBuilder"
 				PreprocessorDefinitions="WIN32;NDEBUG;_CONSOLE"
 				RuntimeLibrary="0"
 				EnableFunctionLevelLinking="true"
@@ -195,7 +195,7 @@
 			<Tool
 				Name="VCCLCompilerTool"
 				Optimization="0"
-				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\dictBuilder"
+				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\dictBuilder"
 				PreprocessorDefinitions="WIN32;_DEBUG;_CONSOLE"
 				MinimalRebuild="true"
 				BasicRuntimeChecks="3"
@@ -273,7 +273,7 @@
 				Optimization="2"
 				EnableIntrinsicFunctions="true"
 				OmitFramePointers="true"
-				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\dictBuilder"
+				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\dictBuilder"
 				PreprocessorDefinitions="WIN32;NDEBUG;_CONSOLE"
 				RuntimeLibrary="0"
 				EnableFunctionLevelLinking="true"
@@ -355,10 +355,6 @@
 				RelativePath="..\..\..\programs\fileio.c"
 				>
 			</File>
-			<File
-				RelativePath="..\..\..\programs\legacy\fileio_legacy.c"
-				>
-			</File>
 			<File
 				RelativePath="..\..\..\lib\compress\fse_compress.c"
 				>

projects/VS2008/fullbench/fullbench.vcproj

@@ -44,7 +44,7 @@
 			<Tool
 				Name="VCCLCompilerTool"
 				Optimization="0"
-				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy"
+				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs;$(SolutionDir)..\..\programs\legacy"
 				PreprocessorDefinitions="WIN32;_DEBUG;_CONSOLE"
 				MinimalRebuild="true"
 				BasicRuntimeChecks="3"
@@ -120,7 +120,7 @@
 				Optimization="2"
 				EnableIntrinsicFunctions="true"
 				OmitFramePointers="true"
-				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy"
+				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs;$(SolutionDir)..\..\programs\legacy"
 				PreprocessorDefinitions="WIN32;NDEBUG;_CONSOLE"
 				RuntimeLibrary="0"
 				EnableFunctionLevelLinking="true"
@@ -194,7 +194,7 @@
 			<Tool
 				Name="VCCLCompilerTool"
 				Optimization="0"
-				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy"
+				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs;$(SolutionDir)..\..\programs\legacy"
 				PreprocessorDefinitions="WIN32;_DEBUG;_CONSOLE"
 				MinimalRebuild="true"
 				BasicRuntimeChecks="3"
@@ -271,7 +271,7 @@
 				Optimization="2"
 				EnableIntrinsicFunctions="true"
 				OmitFramePointers="true"
-				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy"
+				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs;$(SolutionDir)..\..\programs\legacy"
 				PreprocessorDefinitions="WIN32;NDEBUG;_CONSOLE"
 				RuntimeLibrary="0"
 				EnableFunctionLevelLinking="true"
@@ -349,7 +349,7 @@
 				>
 			</File>
 			<File
-				RelativePath="..\..\..\programs\fullbench.c"
+				RelativePath="..\..\..\tests\fullbench.c"
 				>
 			</File>
 			<File

projects/VS2008/fuzzer/fuzzer.vcproj

@@ -44,7 +44,7 @@
 			<Tool
 				Name="VCCLCompilerTool"
 				Optimization="0"
-				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy"
+				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs"
 				PreprocessorDefinitions="WIN32;_DEBUG;_CONSOLE"
 				MinimalRebuild="true"
 				BasicRuntimeChecks="3"
@@ -120,7 +120,7 @@
 				Optimization="2"
 				EnableIntrinsicFunctions="true"
 				OmitFramePointers="true"
-				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy"
+				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs"
 				PreprocessorDefinitions="WIN32;NDEBUG;_CONSOLE"
 				RuntimeLibrary="0"
 				EnableFunctionLevelLinking="true"
@@ -194,7 +194,7 @@
 			<Tool
 				Name="VCCLCompilerTool"
 				Optimization="0"
-				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy"
+				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs"
 				PreprocessorDefinitions="WIN32;_DEBUG;_CONSOLE"
 				MinimalRebuild="true"
 				BasicRuntimeChecks="3"
@@ -271,7 +271,7 @@
 				Optimization="2"
 				EnableIntrinsicFunctions="true"
 				OmitFramePointers="true"
-				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy"
+				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs"
 				PreprocessorDefinitions="WIN32;NDEBUG;_CONSOLE"
 				RuntimeLibrary="0"
 				EnableFunctionLevelLinking="true"
@@ -349,7 +349,7 @@
 				>
 			</File>
 			<File
-				RelativePath="..\..\..\programs\fuzzer.c"
+				RelativePath="..\..\..\tests\fuzzer.c"
 				>
 			</File>
 			<File

projects/VS2008/zstd/zstd.vcproj

@@ -44,7 +44,7 @@
 			<Tool
 				Name="VCCLCompilerTool"
 				Optimization="0"
-				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\dictBuilder"
+				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\dictBuilder"
 				PreprocessorDefinitions="WIN32;_DEBUG;_CONSOLE"
 				MinimalRebuild="true"
 				BasicRuntimeChecks="3"
@@ -121,7 +121,7 @@
 				Optimization="2"
 				EnableIntrinsicFunctions="true"
 				OmitFramePointers="true"
-				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\dictBuilder"
+				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\dictBuilder"
 				PreprocessorDefinitions="WIN32;NDEBUG;_CONSOLE"
 				RuntimeLibrary="0"
 				EnableFunctionLevelLinking="true"
@@ -196,7 +196,7 @@
 			<Tool
 				Name="VCCLCompilerTool"
 				Optimization="0"
-				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\dictBuilder"
+				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\dictBuilder"
 				PreprocessorDefinitions="WIN32;_DEBUG;_CONSOLE"
 				MinimalRebuild="true"
 				BasicRuntimeChecks="3"
@@ -274,7 +274,7 @@
 				Optimization="2"
 				EnableIntrinsicFunctions="true"
 				OmitFramePointers="true"
-				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\dictBuilder"
+				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\dictBuilder"
 				PreprocessorDefinitions="WIN32;NDEBUG;_CONSOLE"
 				RuntimeLibrary="0"
 				EnableFunctionLevelLinking="true"
@@ -356,10 +356,6 @@
 				RelativePath="..\..\..\programs\fileio.c"
 				>
 			</File>
-			<File
-				RelativePath="..\..\..\programs\legacy\fileio_legacy.c"
-				>
-			</File>
 			<File
 				RelativePath="..\..\..\lib\compress\fse_compress.c"
 				>

projects/VS2010/datagen/datagen.vcxproj

@@ -1,4 +1,4 @@
-<?xml version="1.0" encoding="utf-8"?>
+<?xml version="1.0" encoding="utf-8"?>
 <Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
   <ItemGroup Label="ProjectConfigurations">
     <ProjectConfiguration Include="Debug|Win32">
@@ -66,24 +66,24 @@
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
     <LinkIncremental>true</LinkIncremental>
     <RunCodeAnalysis>false</RunCodeAnalysis>
-    <IncludePath>$(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath);</IncludePath>
+    <IncludePath>$(IncludePath);$(SolutionDir)..\..\programs;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath);</IncludePath>
     <IntDir>$(Platform)\$(Configuration)\</IntDir>
   </PropertyGroup>
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
     <LinkIncremental>true</LinkIncremental>
     <RunCodeAnalysis>false</RunCodeAnalysis>
-    <IncludePath>$(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath);</IncludePath>
+    <IncludePath>$(IncludePath);$(SolutionDir)..\..\programs;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath);</IncludePath>
   </PropertyGroup>
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
     <LinkIncremental>false</LinkIncremental>
     <RunCodeAnalysis>false</RunCodeAnalysis>
-    <IncludePath>$(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath);</IncludePath>
+    <IncludePath>$(IncludePath);$(SolutionDir)..\..\programs;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath);</IncludePath>
     <IntDir>$(Platform)\$(Configuration)\</IntDir>
   </PropertyGroup>
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
     <LinkIncremental>false</LinkIncremental>
     <RunCodeAnalysis>false</RunCodeAnalysis>
-    <IncludePath>$(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath);</IncludePath>
+    <IncludePath>$(IncludePath);$(SolutionDir)..\..\programs;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath);</IncludePath>
   </PropertyGroup>
   <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
     <ClCompile>
@@ -157,7 +157,7 @@
   </ItemDefinitionGroup>
   <ItemGroup>
     <ClCompile Include="..\..\..\programs\datagen.c" />
-    <ClCompile Include="..\..\..\programs\datagencli.c" />
+    <ClCompile Include="..\..\..\tests\datagencli.c" />
   </ItemGroup>
   <ItemGroup>
     <ClInclude Include="..\..\..\programs\datagen.h" />

projects/VS2010/fullbench/fullbench.vcxproj

@@ -1,4 +1,4 @@
-<?xml version="1.0" encoding="utf-8"?>
+<?xml version="1.0" encoding="utf-8"?>
 <Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
   <ItemGroup Label="ProjectConfigurations">
     <ProjectConfiguration Include="Debug|Win32">
@@ -65,24 +65,24 @@
   <PropertyGroup Label="UserMacros" />
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
     <LinkIncremental>true</LinkIncremental>
-    <IncludePath>$(IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath);</IncludePath>
+    <IncludePath>$(IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath);</IncludePath>
     <RunCodeAnalysis>false</RunCodeAnalysis>
     <IntDir>$(Platform)\$(Configuration)\</IntDir>
   </PropertyGroup>
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
     <LinkIncremental>true</LinkIncremental>
-    <IncludePath>$(IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath);</IncludePath>
+    <IncludePath>$(IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath);</IncludePath>
     <RunCodeAnalysis>false</RunCodeAnalysis>
   </PropertyGroup>
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
     <LinkIncremental>false</LinkIncremental>
-    <IncludePath>$(IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath);</IncludePath>
+    <IncludePath>$(IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath);</IncludePath>
     <RunCodeAnalysis>false</RunCodeAnalysis>
     <IntDir>$(Platform)\$(Configuration)\</IntDir>
   </PropertyGroup>
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
     <LinkIncremental>false</LinkIncremental>
-    <IncludePath>$(IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath);</IncludePath>
+    <IncludePath>$(IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath);</IncludePath>
     <RunCodeAnalysis>false</RunCodeAnalysis>
   </PropertyGroup>
   <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
@@ -168,7 +168,7 @@
     <ClCompile Include="..\..\..\lib\decompress\zbuff_decompress.c" />
     <ClCompile Include="..\..\..\lib\decompress\zstd_decompress.c" />
     <ClCompile Include="..\..\..\programs\datagen.c" />
-    <ClCompile Include="..\..\..\programs\fullbench.c" />
+    <ClCompile Include="..\..\..\tests\fullbench.c" />
   </ItemGroup>
   <ItemGroup>
     <ClInclude Include="..\..\..\lib\common\fse.h" />

projects/VS2010/fuzzer/fuzzer.vcxproj

@@ -1,4 +1,4 @@
-<?xml version="1.0" encoding="utf-8"?>
+<?xml version="1.0" encoding="utf-8"?>
 <Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
   <ItemGroup Label="ProjectConfigurations">
     <ProjectConfiguration Include="Debug|Win32">
@@ -66,24 +66,24 @@
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
     <LinkIncremental>true</LinkIncremental>
     <RunCodeAnalysis>false</RunCodeAnalysis>
-    <IncludePath>$(IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(UniversalCRT_IncludePath);</IncludePath>
+    <IncludePath>$(IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(UniversalCRT_IncludePath);</IncludePath>
     <IntDir>$(Platform)\$(Configuration)\</IntDir>
   </PropertyGroup>
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
     <LinkIncremental>true</LinkIncremental>
     <RunCodeAnalysis>false</RunCodeAnalysis>
-    <IncludePath>$(IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(UniversalCRT_IncludePath);</IncludePath>
+    <IncludePath>$(IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(UniversalCRT_IncludePath);</IncludePath>
   </PropertyGroup>
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
     <LinkIncremental>false</LinkIncremental>
     <RunCodeAnalysis>false</RunCodeAnalysis>
-    <IncludePath>$(IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(UniversalCRT_IncludePath);</IncludePath>
+    <IncludePath>$(IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(UniversalCRT_IncludePath);</IncludePath>
     <IntDir>$(Platform)\$(Configuration)\</IntDir>
   </PropertyGroup>
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
     <LinkIncremental>false</LinkIncremental>
     <RunCodeAnalysis>false</RunCodeAnalysis>
-    <IncludePath>$(IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(UniversalCRT_IncludePath);</IncludePath>
+    <IncludePath>$(IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(UniversalCRT_IncludePath);</IncludePath>
   </PropertyGroup>
   <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
     <ClCompile>
@@ -168,7 +168,7 @@
     <ClCompile Include="..\..\..\lib\dictBuilder\divsufsort.c" />
     <ClCompile Include="..\..\..\lib\dictBuilder\zdict.c" />
     <ClCompile Include="..\..\..\programs\datagen.c" />
-    <ClCompile Include="..\..\..\programs\fuzzer.c" />
+    <ClCompile Include="..\..\..\tests\fuzzer.c" />
   </ItemGroup>
   <ItemGroup>
     <ClInclude Include="..\..\..\lib\common\fse.h" />

projects/VS2010/zstd/zstd.vcxproj

@@ -43,7 +43,6 @@
     <ClCompile Include="..\..\..\programs\datagen.c" />
     <ClCompile Include="..\..\..\programs\dibio.c" />
     <ClCompile Include="..\..\..\programs\fileio.c" />
-    <ClCompile Include="..\..\..\programs\legacy\fileio_legacy.c" />
     <ClCompile Include="..\..\..\programs\zstdcli.c" />
   </ItemGroup>
   <ItemGroup>
@@ -68,7 +67,6 @@
     <ClInclude Include="..\..\..\programs\datagen.h" />
     <ClInclude Include="..\..\..\programs\dibio.h" />
     <ClInclude Include="..\..\..\programs\fileio.h" />
-    <ClInclude Include="..\..\..\programs\legacy\fileio_legacy.h" />
     <ClInclude Include="..\..\..\programs\util.h" />
   </ItemGroup>
   <PropertyGroup Label="Globals">
@@ -118,27 +116,27 @@
   <PropertyGroup Label="UserMacros" />
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
     <LinkIncremental>true</LinkIncremental>
-    <IncludePath>$(IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(UniversalCRT_IncludePath);</IncludePath>
+    <IncludePath>$(IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(UniversalCRT_IncludePath);</IncludePath>
     <RunCodeAnalysis>false</RunCodeAnalysis>
     <LibraryPath>$(LibraryPath)</LibraryPath>
     <IntDir>$(Platform)\$(Configuration)\</IntDir>
   </PropertyGroup>
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
     <LinkIncremental>true</LinkIncremental>
-    <IncludePath>$(IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(UniversalCRT_IncludePath);</IncludePath>
+    <IncludePath>$(IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(UniversalCRT_IncludePath);</IncludePath>
     <RunCodeAnalysis>false</RunCodeAnalysis>
     <LibraryPath>$(LibraryPath);</LibraryPath>
   </PropertyGroup>
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
     <LinkIncremental>false</LinkIncremental>
-    <IncludePath>$(IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(UniversalCRT_IncludePath);</IncludePath>
+    <IncludePath>$(IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(UniversalCRT_IncludePath);</IncludePath>
     <RunCodeAnalysis>false</RunCodeAnalysis>
     <LibraryPath>$(LibraryPath)</LibraryPath>
     <IntDir>$(Platform)\$(Configuration)\</IntDir>
   </PropertyGroup>
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
     <LinkIncremental>false</LinkIncremental>
-    <IncludePath>$(IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(UniversalCRT_IncludePath);</IncludePath>
+    <IncludePath>$(IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(UniversalCRT_IncludePath);</IncludePath>
     <RunCodeAnalysis>false</RunCodeAnalysis>
     <LibraryPath>$(LibraryPath);</LibraryPath>
   </PropertyGroup>

projects/VS2010/zstdlib/zstdlib.vcxproj

@@ -1,4 +1,4 @@
-<?xml version="1.0" encoding="utf-8"?>
+<?xml version="1.0" encoding="utf-8"?>
 <Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
   <ItemGroup Label="ProjectConfigurations">
     <ProjectConfiguration Include="Debug|Win32">
@@ -43,7 +43,6 @@
     <ClInclude Include="..\..\..\lib\zstd.h" />
     <ClInclude Include="..\..\..\lib\common\zstd_internal.h" />
     <ClInclude Include="..\..\..\lib\compress\zstd_opt.h" />
-    <ClInclude Include="..\..\..\programs\util.h" />
   </ItemGroup>
   <ItemGroup>
     <ResourceCompile Include="zstdlib.rc" />

projects/build/README.md

@@ -1,3 +1,6 @@
+Command line scripts for Visual Studio compilation without IDE
+==============================================================
+
 Here are a few command lines for reference :
 
 ### Build with Visual Studio 2013 for msvcr120.dll

projects/cmake/CMakeLists.txt

@@ -46,6 +46,7 @@ ENDIF (ZSTD_LEGACY_SUPPORT)
 
 ADD_SUBDIRECTORY(lib)
 ADD_SUBDIRECTORY(programs)
+ADD_SUBDIRECTORY(tests)
 
 #-----------------------------------------------------------------------------
 # Add extra compilation flags

projects/cmake/programs/.gitignore

@@ -1,8 +1,3 @@
 # produced by make
-datagen
-fullbench
-fuzzer
-paramgrill
-zbufftest
 zstd
 zstd-frugal

projects/cmake/programs/CMakeLists.txt

@@ -45,29 +45,13 @@ INCLUDE_DIRECTORIES(${PROGRAMS_DIR} ${LIBRARY_DIR} ${LIBRARY_DIR}/common ${LIBRA
 IF (ZSTD_LEGACY_SUPPORT)
     SET(PROGRAMS_LEGACY_DIR ${PROGRAMS_DIR}/legacy)
     INCLUDE_DIRECTORIES(${PROGRAMS_LEGACY_DIR} ${LIBRARY_DIR}/legacy)
-    SET(ZSTD_FILEIO_LEGACY ${PROGRAMS_LEGACY_DIR}/fileio_legacy.c)
 ENDIF (ZSTD_LEGACY_SUPPORT)
 
-ADD_EXECUTABLE(zstd ${PROGRAMS_DIR}/zstdcli.c ${PROGRAMS_DIR}/fileio.c ${PROGRAMS_DIR}/bench.c ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/dibio.c ${ZSTD_FILEIO_LEGACY})
+ADD_EXECUTABLE(zstd ${PROGRAMS_DIR}/zstdcli.c ${PROGRAMS_DIR}/fileio.c ${PROGRAMS_DIR}/bench.c ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/dibio.c)
 TARGET_LINK_LIBRARIES(zstd libzstd_static)
 
-ADD_EXECUTABLE(fullbench ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/fullbench.c)
-TARGET_LINK_LIBRARIES(fullbench libzstd_static)
-
-ADD_EXECUTABLE(fuzzer ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/fuzzer.c)
-TARGET_LINK_LIBRARIES(fuzzer libzstd_static)
-
 IF (UNIX)
     ADD_EXECUTABLE(zstd-frugal ${PROGRAMS_DIR}/zstdcli.c ${PROGRAMS_DIR}/fileio.c)
     TARGET_LINK_LIBRARIES(zstd-frugal libzstd_static)
     SET_TARGET_PROPERTIES(zstd-frugal PROPERTIES COMPILE_DEFINITIONS "ZSTD_NOBENCH;ZSTD_NODICT")
-
-    ADD_EXECUTABLE(zbufftest ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/zbufftest.c)
-    TARGET_LINK_LIBRARIES(zbufftest libzstd_static)
-
-    ADD_EXECUTABLE(paramgrill ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/paramgrill.c)
-    TARGET_LINK_LIBRARIES(paramgrill libzstd_static m) #m is math library
-
-    ADD_EXECUTABLE(datagen ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/datagencli.c)
-    TARGET_LINK_LIBRARIES(datagen libzstd_static)
 ENDIF (UNIX)

projects/cmake/tests/.gitignore

@@ -0,0 +1,7 @@
+# produced by make
+datagen
+fullbench
+fuzzer
+paramgrill
+zbufftest
+

projects/cmake/tests/CMakeLists.txt

@@ -0,0 +1,61 @@
+# ################################################################
+# zstd - Makefile
+# Copyright (C) Yann Collet 2014-2016
+# All rights reserved.
+#
+# BSD license
+#
+# Redistribution and use in source and binary forms, with or without modification,
+# are permitted provided that the following conditions are met:
+#
+# * Redistributions of source code must retain the above copyright notice, this
+#   list of conditions and the following disclaimer.
+#
+# * Redistributions in binary form must reproduce the above copyright notice, this
+#   list of conditions and the following disclaimer in the documentation and/or
+#   other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
+# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+# You can contact the author at :
+#  - zstd homepage : http://www.zstd.net/
+# ################################################################
+
+PROJECT(tests)
+
+SET(CMAKE_INCLUDE_CURRENT_DIR TRUE)
+
+# Define project root directory
+SET(ROOT_DIR ../../..)
+
+# Define programs directory, where sources and header files are located
+SET(LIBRARY_DIR ${ROOT_DIR}/lib)
+SET(PROGRAMS_DIR ${ROOT_DIR}/programs)
+SET(TESTS_DIR ${ROOT_DIR}/tests)
+INCLUDE_DIRECTORIES(${TESTS_DIR} ${PROGRAMS_DIR} ${LIBRARY_DIR} ${LIBRARY_DIR}/common ${LIBRARY_DIR}/dictBuilder)
+
+ADD_EXECUTABLE(fullbench ${PROGRAMS_DIR}/datagen.c ${TESTS_DIR}/fullbench.c)
+TARGET_LINK_LIBRARIES(fullbench libzstd_static)
+
+ADD_EXECUTABLE(fuzzer ${PROGRAMS_DIR}/datagen.c ${TESTS_DIR}/fuzzer.c)
+TARGET_LINK_LIBRARIES(fuzzer libzstd_static)
+
+IF (UNIX)
+    ADD_EXECUTABLE(zbufftest ${PROGRAMS_DIR}/datagen.c ${TESTS_DIR}/zbufftest.c)
+    TARGET_LINK_LIBRARIES(zbufftest libzstd_static)
+
+    ADD_EXECUTABLE(paramgrill ${PROGRAMS_DIR}/datagen.c ${TESTS_DIR}/paramgrill.c)
+    TARGET_LINK_LIBRARIES(paramgrill libzstd_static m) #m is math library
+
+    ADD_EXECUTABLE(datagen ${PROGRAMS_DIR}/datagen.c ${TESTS_DIR}/datagencli.c)
+    TARGET_LINK_LIBRARIES(datagen libzstd_static)
+ENDIF (UNIX)

tests/.gitignore

@@ -1,3 +1,17 @@
+# local binary (Makefile)
+fullbench
+fullbench32
+fuzzer
+fuzzer32
+zbufftest
+zbufftest32
+zstreamtest
+zstreamtest32
+datagen
+paramgrill
+paramgrill32
+roundTripCrash
+
 # Tmp test directory
 zstdtest
 speedTest
@@ -7,3 +21,28 @@ namespaceTest
 # Local script
 startSpeedTest
 speedTest.pid
+
+# Object files
+*.o
+*.ko
+
+# Executables
+*.exe
+*.out
+*.app
+
+# Default result files
+dictionary
+grillResults.txt
+_*
+tmp*
+*.zst
+result
+out
+
+# fuzzer
+afl
+
+# Misc files
+*.bat
+dirTest*

tests/Makefile

@@ -1,7 +1,6 @@
 # ##########################################################################
-# Zstd tests - Makefile
+# ZSTD tests - Makefile
 # Copyright (C) Yann Collet 2015-2016
-# All rights reserved.
 #
 # GPL v2 License
 #
@@ -20,19 +19,115 @@
 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 #
 # You can contact the author at :
-#  - ZSTD homepage : http://www.zstd.net/
+#  - zstd homepage : http://www.zstd.net/
 # ##########################################################################
-# versionstest : Compatibility test between zstd versions stored on Github (v0.1+)
+# datagen : Synthetic and parametrable data generator, for tests
+# fullbench  : Precisely measure speed for each zstd inner functions
+# fullbench32: Same as fullbench, but forced to compile in 32-bits mode
+# fuzzer  : Test tool, to check zstd integrity on target platform
+# fuzzer32: Same as fuzzer, but forced to compile in 32-bits mode
+# paramgrill : parameter tester for zstd
+# test-zstd-speed.py : script for testing zstd speed difference between commits
+# versionsTest : compatibility test between zstd versions stored on Github (v0.1+)
+# zbufftest  : Test tool, to check ZBUFF integrity on target platform
+# zbufftest32: Same as zbufftest, but forced to compile in 32-bits mode
+# zstreamtest : Fuzzer test tool for zstd streaming API
+# zbufftest32: Same as zstreamtest, but forced to compile in 32-bits mode
 # ##########################################################################
 
+DESTDIR?=
+PREFIX ?= /usr/local
+BINDIR  = $(PREFIX)/bin
+MANDIR  = $(PREFIX)/share/man/man1
+ZSTDDIR = ../lib
+PRGDIR  = ../programs
 PYTHON ?= python3
 TESTARTEFACT := versionsTest namespaceTest
 
-.PHONY: default all clean namespaceTest versionsTest
 
-default: all
+CPPFLAGS= -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/dictBuilder -I$(PRGDIR)
+CFLAGS ?= -O3
+CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wstrict-aliasing=1 \
+          -Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes -Wundef
+CFLAGS += $(MOREFLAGS)
+FLAGS   = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS)
 
-all: namespaceTest versionsTest
+
+ZSTDCOMMON_FILES := $(ZSTDDIR)/common/*.c
+ZSTDCOMP_FILES := $(ZSTDDIR)/compress/zstd_compress.c $(ZSTDDIR)/compress/fse_compress.c $(ZSTDDIR)/compress/huf_compress.c
+ZSTDDECOMP_FILES := $(ZSTDDIR)/decompress/zstd_decompress.c $(ZSTDDIR)/decompress/huf_decompress.c
+ZSTD_FILES := $(ZSTDDECOMP_FILES) $(ZSTDCOMMON_FILES) $(ZSTDCOMP_FILES)
+ZBUFF_FILES := $(ZSTDDIR)/compress/zbuff_compress.c $(ZSTDDIR)/decompress/zbuff_decompress.c
+ZDICT_FILES := $(ZSTDDIR)/dictBuilder/*.c
+
+
+
+# Define *.exe as extension for Windows systems
+ifneq (,$(filter Windows%,$(OS)))
+EXT =.exe
+VOID = nul
+else
+EXT =
+VOID = /dev/null
+endif
+
+ZBUFFTEST = -T2mn
+FUZZERTEST= -T5mn
+ZSTDRTTEST= --test-large-data
+
+.PHONY: default all all32 clean test test32 test-all namespaceTest versionsTest
+
+default: fullbench
+
+all: fullbench fuzzer zbufftest zstreamtest paramgrill datagen
+
+all32: fullbench32 fuzzer32 zbufftest32 zstreamtest32
+
+
+
+zstd:
+	$(MAKE) -C $(PRGDIR) $@
+
+zstd32:
+	$(MAKE) -C $(PRGDIR) $@
+
+zstd_nolegacy:
+	$(MAKE) -C $(PRGDIR) $@
+
+fullbench  : $(ZSTD_FILES) $(ZBUFF_FILES) $(PRGDIR)/datagen.c fullbench.c
+	$(CC)      $(FLAGS) $^ -o $@$(EXT)
+
+fullbench32 : $(ZSTD_FILES) $(ZBUFF_FILES) $(PRGDIR)/datagen.c fullbench.c
+	$(CC)  -m32  $(FLAGS) $^ -o $@$(EXT)
+
+fuzzer  : CPPFLAGS += -I$(ZSTDDIR)/dictBuilder
+fuzzer  : $(ZSTD_FILES) $(ZDICT_FILES) $(PRGDIR)/datagen.c fuzzer.c
+	$(CC)      $(FLAGS) $^ -o $@$(EXT)
+
+fuzzer32 : CPPFLAGS += -I$(ZSTDDIR)/dictBuilder
+fuzzer32 : $(ZSTD_FILES) $(ZDICT_FILES) $(PRGDIR)/datagen.c fuzzer.c
+	$(CC)  -m32  $(FLAGS) $^ -o $@$(EXT)
+
+zbufftest  : $(ZSTD_FILES) $(ZBUFF_FILES) $(PRGDIR)/datagen.c zbufftest.c
+	$(CC)      $(FLAGS) $^ -o $@$(EXT)
+
+zbufftest32  : $(ZSTD_FILES) $(ZBUFF_FILES) $(PRGDIR)/datagen.c zbufftest.c
+	$(CC)  -m32  $(FLAGS) $^ -o $@$(EXT)
+
+zstreamtest  : $(ZSTD_FILES) $(PRGDIR)/datagen.c zstreamtest.c
+	$(CC)      $(FLAGS) $^ -o $@$(EXT)
+
+zstreamtest32 : $(ZSTD_FILES) $(PRGDIR)/datagen.c zstreamtest.c
+	$(CC) -m32  $(FLAGS) $^ -o $@$(EXT)
+
+paramgrill : $(ZSTD_FILES) $(PRGDIR)/datagen.c paramgrill.c
+	$(CC)      $(FLAGS) $^ -lm -o $@$(EXT)
+
+datagen : $(PRGDIR)/datagen.c datagencli.c
+	$(CC)      $(FLAGS) $^ -o $@$(EXT)
+
+roundTripCrash : $(ZSTD_FILES) roundTripCrash.c
+	$(CC)      $(FLAGS) $^ -o $@$(EXT)
 
 namespaceTest:
 	if $(CC) namespaceTest.c ../lib/common/xxhash.c -o $@ ; then echo compilation should fail; exit 1 ; fi
@@ -42,5 +137,92 @@ versionsTest:
 	$(PYTHON) test-zstd-versions.py
 
 clean:
+	$(MAKE) -C ../lib clean
 	@$(RM) -fR $(TESTARTEFACT)
+	@$(RM) -f core *.o tmp* result* *.gcda dictionary *.zst \
+        $(PRGDIR)/zstd$(EXT) $(PRGDIR)/zstd32$(EXT) \
+        fullbench$(EXT) fullbench32$(EXT) \
+        fuzzer$(EXT) fuzzer32$(EXT) zbufftest$(EXT) zbufftest32$(EXT) \
+		zstreamtest$(EXT) zstreamtest32$(EXT) \
+        datagen$(EXT) paramgrill$(EXT) roundTripCrash$(EXT)
 	@echo Cleaning completed
+
+
+#----------------------------------------------------------------------------------
+#make valgrindTest is validated only for Linux, OSX, kFreeBSD, Hurd and some BSD targets
+#----------------------------------------------------------------------------------
+ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD DragonFly))
+HOST_OS = POSIX
+
+valgrindTest: VALGRIND = valgrind --leak-check=full --error-exitcode=1
+valgrindTest: zstd datagen fuzzer fullbench zbufftest
+	@echo "\n ---- valgrind tests : memory analyzer ----"
+	$(VALGRIND) ./datagen -g50M > $(VOID)
+	$(VALGRIND) $(PRGDIR)/zstd ; if [ $$? -eq 0 ] ; then echo "zstd without argument should have failed"; false; fi
+	./datagen -g80 | $(VALGRIND) $(PRGDIR)/zstd - -c > $(VOID)
+	./datagen -g16KB | $(VALGRIND) $(PRGDIR)/zstd -vf - -o $(VOID)
+	./datagen -g2930KB | $(VALGRIND) $(PRGDIR)/zstd -5 -vf - -o tmp
+	$(VALGRIND) $(PRGDIR)/zstd -vdf tmp -o $(VOID)
+	./datagen -g64MB | $(VALGRIND) $(PRGDIR)/zstd -vf - -o $(VOID)
+	@rm tmp
+	$(VALGRIND) ./fuzzer -T1mn -t1
+	$(VALGRIND) ./fullbench -i1
+	$(VALGRIND) ./zbufftest -T1mn
+
+endif
+
+
+ifneq (,$(filter MSYS%,$(shell uname)))
+HOST_OS = MSYS
+endif
+
+
+#------------------------------------------------------------------------
+#make tests validated only for MSYS, Linux, OSX, kFreeBSD and Hurd targets
+#------------------------------------------------------------------------
+ifneq (,$(filter $(HOST_OS),MSYS POSIX))
+zstd-playTests: datagen
+	ZSTD=$(ZSTD) ./playTests.sh $(ZSTDRTTEST)
+
+test: test-zstd test-fullbench test-fuzzer test-zbuff test-zstream
+
+test32: test-zstd32 test-fullbench32 test-fuzzer32 test-zbuff32 test-zstream32
+
+test-all: test test32 valgrindTest
+
+test-zstd: ZSTD = $(PRGDIR)/zstd
+test-zstd: zstd zstd-playTests
+
+test-zstd32: ZSTD = $(PRGDIR)/zstd32
+test-zstd32: zstd32 zstd-playTests
+
+test-zstd_nolegacy: ZSTD = $(PRGDIR)/zstd
+test-zstd_nolegacy: zstd_nolegacy zstd-playTests
+
+test-fullbench: fullbench datagen
+	./fullbench -i1
+	./fullbench -i1 -P0
+
+test-fullbench32: fullbench32 datagen
+	./fullbench32 -i1
+	./fullbench32 -i1 -P0
+
+test-fuzzer: fuzzer
+	./fuzzer $(FUZZERTEST)
+
+test-fuzzer32: fuzzer32
+	./fuzzer32 $(FUZZERTEST)
+
+test-zbuff: zbufftest
+	./zbufftest $(ZBUFFTEST)
+
+test-zbuff32: zbufftest32
+	./zbufftest32 $(ZBUFFTEST)
+
+test-zstream: zstreamtest
+	./zstreamtest $(ZBUFFTEST)
+
+test-zstream32: zstreamtest32
+	./zstreamtest32 $(ZBUFFTEST)
+
+endif

tests/README.md

@@ -1,14 +1,25 @@
-scripts for automated testing of zstd
+Programs and scripts for automated testing of Zstandard
-================================
+=======================================================
 
-#### test-zstd-versions.py - script for testing zstd interoperability between versions
+This directory contains the following programs and scripts:
+- `datagen` : Synthetic and parametrable data generator, for tests
+- `fullbench`  : Precisely measure speed for each zstd inner functions
+- `fuzzer`  : Test tool, to check zstd integrity on target platform
+- `paramgrill` : parameter tester for zstd
+- `test-zstd-speed.py` : script for testing zstd speed difference between commits
+- `test-zstd-versions.py` : compatibility test between zstd versions stored on Github (v0.1+)
+- `zbufftest`  : Test tool to check ZBUFF (a buffered streaming API) integrity
+- `zstreamtest` : Fuzzer test tool for zstd streaming API
+
+
+#### `test-zstd-versions.py` - script for testing zstd interoperability between versions
 
 This script creates `versionsTest` directory to which zstd repository is cloned.
 Then all taged (released) versions of zstd are compiled.
 In the following step interoperability between zstd versions is checked.
 
 
-#### test-zstd-speed.py - script for testing zstd speed difference between commits
+#### `test-zstd-speed.py` - script for testing zstd speed difference between commits
 
 This script creates `speedTest` directory to which zstd repository is cloned.
 Then it compiles all branches of zstd and performs a speed benchmark for a given list of files (the `testFileNames` parameter).

tests/datagencli.c

@@ -1,28 +1,12 @@
-/*
+/**
-    datagencli.c
+ * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
-    compressible data command line generator
+ * All rights reserved.
-    Copyright (C) Yann Collet 2012-2015
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
 
-    GPL v2 License
-
-    This program is free software; you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
-    the Free Software Foundation; either version 2 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
-
-    You should have received a copy of the GNU General Public License along
-    with this program; if not, write to the Free Software Foundation, Inc.,
-    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-    You can contact the author at :
-   - ZSTD source repository : https://github.com/Cyan4973/zstd
-   - Public forum : https://groups.google.com/forum/#!forum/lz4c
-*/
 
 /*-************************************
 *  Includes

tests/fullbench.c

@@ -1,26 +1,12 @@
-/*
+/**
-    fullbench.c - Detailed bench program for zstd
+ * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
-    Copyright (C) Yann Collet 2014-2016
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
 
-    GPL v2 License
-
-    This program is free software; you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
-    the Free Software Foundation; either version 2 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
-
-    You should have received a copy of the GNU General Public License along
-    with this program; if not, write to the Free Software Foundation, Inc.,
-    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-    You can contact the author at :
-    - zstd homepage : http://www.zstd.net
-*/
 
 /*_************************************
 *  Includes

tests/fuzzer.c

@@ -1,26 +1,12 @@
-/*
+/**
-    Fuzzer test tool for zstd
+ * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
-    Copyright (C) Yann Collet 2014-2016
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
 
-    GPL v2 License
-
-    This program is free software; you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
-    the Free Software Foundation; either version 2 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
-
-    You should have received a copy of the GNU General Public License along
-    with this program; if not, write to the Free Software Foundation, Inc.,
-    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-    You can contact the author at :
-    - ZSTD homepage : http://www.zstd.net
-*/
 
 /*-************************************
 *  Compiler specific
@@ -100,7 +86,6 @@ static unsigned FUZ_rand(unsigned* src)
     return rand32 >> 5;
 }
 
-
 static unsigned FUZ_highbit32(U32 v32)
 {
     unsigned nbBits = 0;
@@ -110,6 +95,10 @@ static unsigned FUZ_highbit32(U32 v32)
 }
 
 
+/*=============================================
+*   Basic Unit tests
+=============================================*/
+
 #define CHECK_V(var, fn)  size_t const var = fn; if (ZSTD_isError(var)) goto _output_error
 #define CHECK(fn)  { CHECK_V(err, fn); }
 #define CHECKPLUS(var, fn, more)  { CHECK_V(var, fn); more; }
@@ -272,6 +261,12 @@ static int basicUnitTests(U32 seed, double compressibility)
         if (ZDICT_isError(dictSize)) goto _output_error;
         DISPLAYLEVEL(4, "OK, created dictionary of size %u \n", (U32)dictSize);
 
+        DISPLAYLEVEL(4, "test%3i : check dictID : ", testNb++);
+        {   U32 const dictID = ZDICT_getDictID(dictBuffer, dictSize);
+            if (dictID==0) goto _output_error;
+            DISPLAYLEVEL(4, "OK : %u \n", dictID);
+        }
+
         DISPLAYLEVEL(4, "test%3i : compress with dictionary : ", testNb++);
         cSize = ZSTD_compress_usingDict(cctx, compressedBuffer, ZSTD_compressBound(CNBuffSize),
                                         CNBuffer, CNBuffSize,

tests/namespaceTest.c

@@ -1,27 +1,12 @@
-/* ##########################################################################
+/**
-# namespaceTest
+ * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
-# ensure xxhash namespace emulation is properly triggered
+ * All rights reserved.
-# Copyright (C) Yann Collet 2016
+ *
-#
+ * This source code is licensed under the BSD-style license found in the
-# GPL v2 License
+ * LICENSE file in the root directory of this source tree. An additional grant
-#
+ * of patent rights can be found in the PATENTS file in the same directory.
-# This program is free software; you can redistribute it and/or modify
+ */
-# it under the terms of the GNU General Public License as published by
+
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-#
-# You can contact the author at :
-#  - zstd homepage : http://www.zstd.net/
-# ########################################################################*/
 
 
 #include <stddef.h>  /* size_t */

tests/paramgrill.c

@@ -1,26 +1,11 @@
-/*
+/**
-    paramgrill.c - parameter tester for zstd
+ * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
-    Copyright (C) Yann Collet 2015-2016
+ * All rights reserved.
-
+ *
-    GPL v2 License
+ * This source code is licensed under the BSD-style license found in the
-
+ * LICENSE file in the root directory of this source tree. An additional grant
-    This program is free software; you can redistribute it and/or modify
+ * of patent rights can be found in the PATENTS file in the same directory.
-    it under the terms of the GNU General Public License as published by
+ */
-    the Free Software Foundation; either version 2 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
-
-    You should have received a copy of the GNU General Public License along
-    with this program; if not, write to the Free Software Foundation, Inc.,
-    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-    You can contact the author at :
-    - zstd homepage : http://www.zstd.net/
-*/
 
 
 /*-************************************

tests/playTests.sh

@@ -32,11 +32,11 @@ case "$OS" in
 esac
 
 MD5SUM="md5sum"
-if [ "$TRAVIS_OS_NAME" = "osx" ]; then
+if [[ "$OSTYPE" == "darwin"* ]]; then
     MD5SUM="md5 -r"
 fi
 
-$ECHO "\nStarting playTests.sh isWindows=$isWindows TRAVIS_OS_NAME=$TRAVIS_OS_NAME"
+$ECHO "\nStarting playTests.sh isWindows=$isWindows"
 
 [ -n "$ZSTD" ] || die "ZSTD variable must be defined!"
 
@@ -169,35 +169,41 @@ $ZSTD -f tmp1 notHere tmp2 && die "missing file not detected!"
 
 $ECHO "\n**** dictionary tests **** "
 
+TESTFILE=../programs/zstdcli.c
 ./datagen > tmpDict
 ./datagen -g1M | $MD5SUM > tmp1
 ./datagen -g1M | $ZSTD -D tmpDict | $ZSTD -D tmpDict -dvq | $MD5SUM > tmp2
 diff -q tmp1 tmp2
 $ECHO "- Create first dictionary"
-$ZSTD --train *.c -o tmpDict
+$ZSTD --train *.c ../programs/*.c -o tmpDict
-cp zstdcli.c tmp
+cp $TESTFILE tmp
 $ZSTD -f tmp -D tmpDict
 $ZSTD -d tmp.zst -D tmpDict -of result
-diff zstdcli.c result
+diff $TESTFILE result
 $ECHO "- Create second (different) dictionary"
-$ZSTD --train *.c *.h -o tmpDictC
+$ZSTD --train *.c ../programs/*.c ../programs/*.h -o tmpDictC
 $ZSTD -d tmp.zst -D tmpDictC -of result && die "wrong dictionary not detected!"
 $ECHO "- Create dictionary with short dictID"
-$ZSTD --train *.c --dictID 1 -o tmpDict1
+$ZSTD --train *.c ../programs/*.c --dictID 1 -o tmpDict1
 cmp tmpDict tmpDict1 && die "dictionaries should have different ID !"
 $ECHO "- Compress without dictID"
 $ZSTD -f tmp -D tmpDict1 --no-dictID
 $ZSTD -d tmp.zst -D tmpDict -of result
-diff zstdcli.c result
+diff $TESTFILE result
 $ECHO "- Compress multiple files with dictionary"
 rm -rf dirTestDict
 mkdir dirTestDict
 cp *.c dirTestDict
-cp *.h dirTestDict
+cp ../programs/*.c dirTestDict
-cat dirTestDict/* | $MD5SUM > tmph1  # note : we expect same file order to generate same hash
+cp ../programs/*.h dirTestDict
-$ZSTD -f dirTestDict/* -D tmpDictC
+$MD5SUM dirTestDict/* > tmph1
-$ZSTD -d dirTestDict/*.zst -D tmpDictC -c | $MD5SUM > tmph2
+$ZSTD -f --rm dirTestDict/* -D tmpDictC
-diff -q tmph1 tmph2
+$ZSTD -d --rm dirTestDict/*.zst -D tmpDictC  # note : use internal checksum by default
+if [[ "$OSTYPE" == "darwin"* ]]; then
+  $ECHO "test skipped on OS-X"  # not compatible with OS-X's md5
+else
+  $MD5SUM -c tmph1
+fi
 rm -rf dirTestDict
 rm tmp*
 

tests/roundTripCrash.c

@@ -1,26 +1,12 @@
-/*
+/**
-  roundTripCrash
+ * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
-  Copyright (C) Yann Collet 2013-2016
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
 
-  GPL v2 License
-
-  This program is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published by
-  the Free Software Foundation; either version 2 of the License, or
-  (at your option) any later version.
-
-  This program is distributed in the hope that it will be useful,
-  but WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-  GNU General Public License for more details.
-
-  You should have received a copy of the GNU General Public License along
-  with this program; if not, write to the Free Software Foundation, Inc.,
-  51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-  You can contact the author at :
-  - zstd homepage : http://www.zstd.net
-*/
 /*
   This program takes a file in input,
   performs a zstd round-trip test (compression - decompress)

tests/test-zstd-speed.py

@@ -1,5 +1,14 @@
 #! /usr/bin/env python
 
+#
+# Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree. An additional grant
+# of patent rights can be found in the PATENTS file in the same directory.
+#
+
 import argparse
 import os
 import string
@@ -192,7 +201,7 @@ def test_commit(branch, commit, last_commit, args, testFilePaths, have_mutt, hav
     md5_zstd = hashfile(hashlib.md5(), clone_path + '/programs/zstd')
     md5_zstd32 = hashfile(hashlib.md5(), clone_path + '/programs/zstd32')
     md5_zstd_clang = hashfile(hashlib.md5(), clone_path + '/programs/zstd_clang')
-    print("md5(zstd)=%s\nmd5(zstd32)=%s\nmd5(zstd32_clang)=%s" % (md5_zstd, md5_zstd32, md5_zstd_clang))
+    print("md5(zstd)=%s\nmd5(zstd32)=%s\nmd5(zstd_clang)=%s" % (md5_zstd, md5_zstd32, md5_zstd_clang))
     logFileName = working_path + "/log_" + branch.replace("/", "_") + ".txt"
     text_to_send = []
     results_files = ""

tests/test-zstd-versions.py

@@ -1,6 +1,14 @@
 #!/usr/bin/env python3
 """Test zstd interoperability between versions"""
-# Based on LZ4 version test script, by Takayuki Matsuoka
+
+#
+# Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree. An additional grant
+# of patent rights can be found in the PATENTS file in the same directory.
+#
 
 import filecmp
 import glob
@@ -152,6 +160,7 @@ def decompress_zst(tag):
                 print('OK     ')
         else:
             print('command does not work')
+            dec_error = 1
     return dec_error
 
 
@@ -203,7 +212,8 @@ if __name__ == '__main__':
     # Retrieve all release tags
     print('Retrieve all release tags :')
     os.chdir(clone_dir)
-    tags = get_git_tags() + [head]
+    alltags = get_git_tags() + [head]
+    tags = [t for t in alltags if t >= 'v0.4.0']
     print(tags)
 
     # Build all release zstd
@@ -261,6 +271,6 @@ if __name__ == '__main__':
         print(zstd + ' : ' + repr(os.path.getsize(zstd)) + ', ' + sha1_of_file(zstd))
 
     if error_code != 0:
-        print('==== ERROR !!! =====')
+        print('======  ERROR !!!  =======')
 
     sys.exit(error_code)

tests/zbufftest.c

@@ -1,26 +1,12 @@
-/*
+/**
-    Fuzzer test tool for zstd_buffered
+ * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
-    Copyright (C) Yann Collet 2015-2016
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
 
-    GPL v2 License
-
-    This program is free software; you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
-    the Free Software Foundation; either version 2 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
-
-    You should have received a copy of the GNU General Public License along
-    with this program; if not, write to the Free Software Foundation, Inc.,
-    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-    You can contact the author at :
-    - ZSTD homepage : https://www.zstd.net/
-*/
 
 /*-************************************
 *  Compiler specific
@@ -183,7 +169,7 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
     cSize += genSize;
     genSize = compressedBufferSize - cSize;
     { size_t const r = ZBUFF_compressEnd(zc, ((char*)compressedBuffer)+cSize, &genSize);
-      if (r != 0) goto _output_error; }  /*< error, or some data not flushed */
+      if (r != 0) goto _output_error; }  /* error, or some data not flushed */
     cSize += genSize;
     DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/COMPRESSIBLE_NOISE_LENGTH*100);
 

tests/zstreamtest.c

@@ -1,26 +1,12 @@
-/*
+/**
-    Fuzzer test tool for zstd streaming API
+ * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
-    Copyright (C) Yann Collet 2016
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
 
-    GPL v2 License
-
-    This program is free software; you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
-    the Free Software Foundation; either version 2 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
-
-    You should have received a copy of the GNU General Public License along
-    with this program; if not, write to the Free Software Foundation, Inc.,
-    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-    You can contact the author at :
-    - ZSTD homepage : https://www.zstd.net/
-*/
 
 /*-************************************
 *  Compiler specific
@@ -61,7 +47,6 @@ static const U32 prime1 = 2654435761U;
 static const U32 prime2 = 2246822519U;
 
 
-
 /*-************************************
 *  Display Macros
 **************************************/
@@ -93,7 +78,6 @@ static U32 FUZ_GetMilliStart(void)
     return nCount;
 }
 
-
 static U32 FUZ_GetMilliSpan(U32 nTimeStart)
 {
     U32 const nCurrent = FUZ_GetMilliStart();
@@ -117,7 +101,6 @@ unsigned int FUZ_rand(unsigned int* seedPtr)
     return rand32 >> 5;
 }
 
-
 /*
 static unsigned FUZ_highbit32(U32 v32)
 {
@@ -141,6 +124,11 @@ static void freeFunction(void* opaque, void* address)
     free(address);
 }
 
+
+/*======================================================
+*   Basic Unit tests
+======================================================*/
+
 static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem customMem)
 {
     int testResult = 0;
@@ -183,10 +171,16 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
       if (ZSTD_isError(r)) goto _output_error; }
     if (inBuff.pos != inBuff.size) goto _output_error;   /* entire input should be consumed */
     { size_t const r = ZSTD_endStream(zc, &outBuff);
-      if (r != 0) goto _output_error; }  /*< error, or some data not flushed */
+      if (r != 0) goto _output_error; }  /* error, or some data not flushed */
     cSize += outBuff.pos;
     DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/COMPRESSIBLE_NOISE_LENGTH*100);
 
+    DISPLAYLEVEL(4, "test%3i : check CStream size : ", testNb++);
+    { size_t const s = ZSTD_sizeof_CStream(zc);
+      if (ZSTD_isError(s)) goto _output_error;
+      DISPLAYLEVEL(4, "OK (%u bytes) \n", (U32)s);
+    }
+
     /* skippable frame test */
     DISPLAYLEVEL(4, "test%3i : decompress skippable frame : ", testNb++);
     ZSTD_initDStream_usingDict(zd, CNBuffer, 128 KB);
@@ -204,6 +198,8 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
     /* Basic decompression test */
     DISPLAYLEVEL(4, "test%3i : decompress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH);
     ZSTD_initDStream_usingDict(zd, CNBuffer, 128 KB);
+    { size_t const r = ZSTD_setDStreamParameter(zd, ZSTDdsp_maxWindowSize, 1000000000);  /* large limit */
+      if (ZSTD_isError(r)) goto _output_error; }
     { size_t const r = ZSTD_decompressStream(zd, &outBuff, &inBuff);
       if (r != 0) goto _output_error; }  /* should reach end of frame == 0; otherwise, some data left, or an error */
     if (outBuff.pos != CNBufferSize) goto _output_error;   /* should regenerate the same amount */
@@ -218,6 +214,12 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
     }   }
     DISPLAYLEVEL(4, "OK \n");
 
+    DISPLAYLEVEL(4, "test%3i : check DStream size : ", testNb++);
+    { size_t const s = ZSTD_sizeof_DStream(zd);
+      if (ZSTD_isError(s)) goto _output_error;
+      DISPLAYLEVEL(4, "OK (%u bytes) \n", (U32)s);
+    }
+
     /* Byte-by-byte decompression test */
     DISPLAYLEVEL(4, "test%3i : decompress byte-by-byte : ", testNb++);
     {   size_t r = 1;
@@ -253,6 +255,27 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
     }   }
     DISPLAYLEVEL(4, "OK \n");
 
+    DISPLAYLEVEL(4, "test%3i : wrong parameter for ZSTD_setDStreamParameter(): ", testNb++);
+    { size_t const r = ZSTD_setDStreamParameter(zd, (ZSTD_DStreamParameter_e)999, 1);  /* large limit */
+      if (!ZSTD_isError(r)) goto _output_error; }
+    DISPLAYLEVEL(4, "OK \n");
+
+    /* Memory restriction */
+    DISPLAYLEVEL(4, "test%3i : maxWindowSize < frame requirement : ", testNb++);
+    ZSTD_initDStream_usingDict(zd, CNBuffer, 128 KB);
+    { size_t const r = ZSTD_setDStreamParameter(zd, ZSTDdsp_maxWindowSize, 1000);  /* too small limit */
+      if (ZSTD_isError(r)) goto _output_error; }
+    inBuff.src = compressedBuffer;
+    inBuff.size = cSize;
+    inBuff.pos = 0;
+    outBuff.dst = decodedBuffer;
+    outBuff.size = CNBufferSize;
+    outBuff.pos = 0;
+    { size_t const r = ZSTD_decompressStream(zd, &outBuff, &inBuff);
+      if (!ZSTD_isError(r)) goto _output_error;  /* must fail : frame requires > 100 bytes */
+      DISPLAYLEVEL(4, "OK (%s)\n", ZSTD_getErrorName(r)); }
+
+
 _end:
     ZSTD_freeCStream(zc);
     ZSTD_freeDStream(zd);
@@ -631,7 +654,7 @@ int main(int argc, const char** argv)
     }   }   }   /* for(argNb=1; argNb<argc; argNb++) */
 
     /* Get Seed */
-    DISPLAY("Starting zstd_buffered tester (%i-bits, %s)\n", (int)(sizeof(size_t)*8), ZSTD_VERSION_STRING);
+    DISPLAY("Starting zstream tester (%i-bits, %s)\n", (int)(sizeof(size_t)*8), ZSTD_VERSION_STRING);
 
     if (!seedset) seed = FUZ_GetMilliStart() % 10000;
     DISPLAY("Seed = %u\n", seed);