Merge pull request #2432 from facebook/check32

added runtime test in CI for 32-bit binaries

.circleci/config.yml

@@ -143,8 +143,9 @@ workflows:
           filters:
             branches:
               only:
-                - master
+                - release
                 - dev
+                - master
     jobs:
       # Run daily long regression tests
       - regression-test

.github/workflows/generic-dev.yml

@@ -2,14 +2,13 @@ name: generic-dev
 
 on:
   pull_request:
-    branches: [ dev, master, actionsTest ]
+    branches: [ dev, release, actionsTest ]
 
 jobs:
 
 # Dev PR jobs that still have to be migrated from travis
 #
-# icc (need self-hosted)
-# versionTag
+# versionTag (only on release tags)
 # valgrindTest (keeps failing for some reason. need investigation)
 # staticAnalyze (need trusty so need self-hosted)
 # pcc-fuzz: (need trusty so need self-hosted)
@@ -19,7 +18,7 @@ jobs:
 # I need admins permissions to the repo for that it looks like
 # So I'm tabling that for now
 #
-# The master branch exclusive jobs will be in a separate
+# The release branch exclusive jobs will be in a separate
 # workflow file (the osx tests and meson build that is)
 
   benchmarking:
@@ -36,6 +35,15 @@ jobs:
     - name: make test
       run: make test
 
+  check-32bit: # designed to catch https://github.com/facebook/zstd/issues/2428
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: make check on 32-bit
+      run: |
+        make libc6install
+        CFLAGS="-m32 -O1 -fstack-protector" make check V=1
+
   gcc-6-7-libzstd:
     runs-on: ubuntu-latest
     steps:

.github/workflows/generic-release.yml

@@ -2,10 +2,10 @@ name: generic-release
 
 on:
   pull_request:
-    # This will eventually only be for pushes to master
+    # This will eventually only be for pushes to release
     # but for dogfooding purposes, I'm running it even
     # on dev pushes
-    branches: [ dev, master, actionsTest ]
+    branches: [ dev, release, actionsTest ]
 
 jobs:
   # missing jobs

.github/workflows/linux-kernel.yml

@@ -2,7 +2,7 @@ name: linux-kernel
 
 on:
   pull_request:
-    branches: [ dev, master, actionsTest ]
+    branches: [ dev, release, actionsTest ]
 
 jobs:
   test:

.travis.yml

@@ -8,6 +8,7 @@ git:
 branches:
   only:
   - dev
+  - release
   - master
   - travisTest
 
@@ -162,33 +163,33 @@ matrix:
         - make -C tests checkTag
         - tests/checkTag "$TRAVIS_BRANCH"
 
-    # tests for master branch and cron job only
+    # tests for release branch and cron job only
     - name: OS-X    # ~13mn
-      if: branch = master
+      if: branch = release
       os: osx
       script:
         - make test
         - make -C lib all
 
     - name: zbuff test
-      if: branch = master
+      if: branch = release
       script:
         - make -C tests test-zbuff
 
     - name: Versions Compatibility Test   # 11.5mn
-      if: branch = master
+      if: branch = release
       script:
         - make -C tests versionsTest
 
     - name: thread sanitizer   # ~29mn
-      if: branch = master
+      if: branch = release
       script:
         - make clang38install
         - CC=clang-3.8 make tsan-test-zstream
         - CC=clang-3.8 make tsan-fuzztest
 
     - name: PPC64LE + Fuzz test  # ~13mn
-      if: branch = master
+      if: branch = release
       arch: ppc64le
       script:
         - cat /proc/cpuinfo
@@ -196,28 +197,28 @@ matrix:
 
     - name: Qemu PPC64 + Fuzz test  # ~13mn, presumed Big-Endian (?)
       dist: trusty  # note : PPC64 cross-compilation for Qemu tests seems broken on Xenial
-      if: branch = master
+      if: branch = release
       script:
         - make ppcinstall
         - make ppc64fuzz
 
     # note : we already have aarch64 tests on hardware
     - name: Qemu aarch64 + Fuzz Test (on Xenial)    # ~14mn
-      if: branch = master
+      if: branch = release
       dist: xenial
       script:
         - make arminstall
         - make aarch64fuzz
 
     - name: zlib wrapper test    # ~7.5mn
-      if: branch = master
+      if: branch = release
       script:
         - make gpp6install valgrindinstall
         - make -C zlibWrapper test
         - make -C zlibWrapper valgrindTest
 
     - name: LZ4, thread pool, and partial libs tests    # ~4mn
-      if: branch = master
+      if: branch = release
       script:
         - make lz4install
         - make -C tests test-lz4
@@ -229,7 +230,7 @@ matrix:
 
     # meson dedicated test
     - name: Xenial (Meson + clang)    # ~15mn
-      if: branch = master
+      if: branch = release
       dist: xenial
       language: cpp
       compiler: clang

lib/common/fse.h

@@ -335,9 +335,10 @@ size_t FSE_buildCTable_rle (FSE_CTable* ct, unsigned char symbolValue);
 
 /* FSE_buildCTable_wksp() :
  * Same as FSE_buildCTable(), but using an externally allocated scratch buffer (`workSpace`).
- * `wkspSize` must be >= `FSE_BUILD_CTABLE_WORKSPACE_SIZE(maxSymbolValue, tableLog)`.
+ * `wkspSize` must be >= `FSE_BUILD_CTABLE_WORKSPACE_SIZE_U32(maxSymbolValue, tableLog)` of `unsigned`.
  */
-#define FSE_BUILD_CTABLE_WORKSPACE_SIZE(maxSymbolValue, tableLog) (sizeof(unsigned) * (maxSymbolValue + 2) + (1ull << tableLog))
+#define FSE_BUILD_CTABLE_WORKSPACE_SIZE_U32(maxSymbolValue, tableLog) (maxSymbolValue + 2 + (1ull << (tableLog - 2)))
+#define FSE_BUILD_CTABLE_WORKSPACE_SIZE(maxSymbolValue, tableLog) (sizeof(unsigned) * FSE_BUILD_CTABLE_WORKSPACE_SIZE_U32(maxSymbolValue, tableLog))
 size_t FSE_buildCTable_wksp(FSE_CTable* ct, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize);
 
 #define FSE_BUILD_DTABLE_WKSP_SIZE(maxTableLog, maxSymbolValue) (sizeof(short) * (maxSymbolValue + 1) + (1ULL << maxTableLog) + 8)

lib/compress/huf_compress.c

@@ -69,7 +69,7 @@ static size_t HUF_compressWeights (void* dst, size_t dstSize, const void* weight
     U32 tableLog = MAX_FSE_TABLELOG_FOR_HUFF_HEADER;
 
     FSE_CTable CTable[FSE_CTABLE_SIZE_U32(MAX_FSE_TABLELOG_FOR_HUFF_HEADER, HUF_TABLELOG_MAX)];
-    BYTE scratchBuffer[FSE_BUILD_CTABLE_WORKSPACE_SIZE(HUF_TABLELOG_MAX, MAX_FSE_TABLELOG_FOR_HUFF_HEADER)];
+    U32 scratchBuffer[FSE_BUILD_CTABLE_WORKSPACE_SIZE_U32(HUF_TABLELOG_MAX, MAX_FSE_TABLELOG_FOR_HUFF_HEADER)];
 
     unsigned count[HUF_TABLELOG_MAX+1];
     S16 norm[HUF_TABLELOG_MAX+1];

tests/playTests.sh

@@ -1342,8 +1342,6 @@ optCSize19=$(datagen -g2M | zstd -19 -c | wc -c)
 longCSize19=$(datagen -g2M | zstd -19 --long -c | wc -c)
 optCSize19wlog23=$(datagen -g2M | zstd -19 -c  --zstd=wlog=23 | wc -c)
 longCSize19wlog23=$(datagen -g2M | zstd -19 -c --long=23 | wc -c)
-optCSize22=$(datagen -g900K | zstd -22 --ultra -c | wc -c)
-longCSize22=$(datagen -g900K | zstd -22 --ultra --long -c | wc -c)
 if [ "$longCSize16" -gt "$optCSize16" ]; then
     echo using --long on compression level 16 should not cause compressed size regression
     exit 1
@@ -1353,9 +1351,6 @@ elif [ "$longCSize19" -gt "$optCSize19" ]; then
 elif [ "$longCSize19wlog23" -gt "$optCSize19wlog23" ]; then
     echo using --long on compression level 19 with wLog=23 should not cause compressed size regression
     exit 1
-elif [ "$longCSize22" -gt "$optCSize22" ]; then
-    echo using --long on compression level 22 should not cause compressed size regression
-    exit 1
 fi