Merge pull request #2885 from yoniko/limit-level-32bit-systems

Limit `ZSTD_maxCLevel` to 21 for 32-bit binaries.

.github/workflows/dev-short-tests.yml

@@ -30,6 +30,8 @@ jobs:
     steps:
     - uses: actions/checkout@v2
     - name: make check on 32-bit
+      env:
+        CHECK_CONSTRAINED_MEM: true
       run: |
         sudo apt update
         APT_PACKAGES="gcc-multilib" make apt-install

doc/zstd_manual.html

@@ -40,7 +40,7 @@
   functions.
 
   The library supports regular compression levels from 1 up to ZSTD_maxCLevel(),
-  which is currently 22. Levels >= 20, labeled `--ultra`, should be used with
+  which is 22 in most cases. Levels >= 20, labeled `--ultra`, should be used with
   caution, as they require more memory. The library also offers negative
   compression levels, which extend the range of speed vs. ratio preferences.
   The lower the level, the faster the speed (at the cost of compression).

lib/compress/clevels.h

@@ -17,6 +17,7 @@
 /*-=====  Pre-defined compression levels  =====-*/
 
 #define ZSTD_MAX_CLEVEL           22
+#define ZSTD_MAX_32BIT_CLEVEL     21
 
 #ifdef __GNUC__
 __attribute__((__unused__))

lib/compress/zstd_compress.c

@@ -6165,7 +6165,7 @@ size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output)
 /*-=====  Pre-defined compression levels  =====-*/
 #include "clevels.h"
 
-int ZSTD_maxCLevel(void) { return ZSTD_MAX_CLEVEL; }
+int ZSTD_maxCLevel(void) { return MEM_32bits() ? ZSTD_MAX_32BIT_CLEVEL : ZSTD_MAX_CLEVEL; }
 int ZSTD_minCLevel(void) { return (int)-ZSTD_TARGETLENGTH_MAX; }
 int ZSTD_defaultCLevel(void) { return ZSTD_CLEVEL_DEFAULT; }
 
@@ -6261,7 +6261,7 @@ static ZSTD_compressionParameters ZSTD_getCParams_internal(int compressionLevel,
     /* row */
     if (compressionLevel == 0) row = ZSTD_CLEVEL_DEFAULT;   /* 0 == default */
     else if (compressionLevel < 0) row = 0;   /* entry 0 is baseline for fast mode */
-    else if (compressionLevel > ZSTD_MAX_CLEVEL) row = ZSTD_MAX_CLEVEL;
+    else if (compressionLevel > ZSTD_maxCLevel()) row = ZSTD_maxCLevel();
     else row = compressionLevel;
 
     {   ZSTD_compressionParameters cp = ZSTD_defaultCParameters[tableID][row];

tests/fuzzer.c

@@ -1982,7 +1982,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
             int const rowLevelEnd = 8;
 
             DISPLAYLEVEL(3, "test%3i : flat-dictionary efficiency test : \n", testNb++);
-            assert(maxLevel == 22);
+            assert(maxLevel == (MEM_32bits() ? 21 : 22));
             RDG_genBuffer(CNBuffer, flatdictSize + contentSize, compressibility, 0., seed);
             DISPLAYLEVEL(4, "content hash : %016llx;  dict hash : %016llx \n", XXH64(contentStart, contentSize, 0), XXH64(dict, flatdictSize, 0));
 

tests/playTests.sh

@@ -1538,6 +1538,11 @@ elif [ "$longCSize19wlog23" -gt "$optCSize19wlog23" ]; then
     exit 1
 fi
 
+if [ -n "$CHECK_CONSTRAINED_MEM" ]; then
+    println "\n===>  zsdt constrained memory tests "
+    # shellcheck disable=SC2039
+    (ulimit -Sv 500000 ; datagen -g2M | zstd -22 --single-thread --ultra > /dev/null)
+fi
 
 if [ "$1" != "--test-large-data" ]; then
     println "Skipping large data tests"