facebook
/
zstd
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fixed multiple conversions 

from 64-bit to 32-bit
dev
Yann Collet 2018-12-13 12:14:36 -08:00
parent 1993f5d412
commit 34f01e600f
8 changed files with 50 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
contrib/largeNbDicts/largeNbDicts.c
View File

@ -21,6 +21,7 @@
#include <stddef.h> /* size_t */
#include <stdlib.h> /* malloc, free, abort */
#include <stdio.h> /* fprintf */
#include <limits.h> /* UINT_MAX */
#include <assert.h> /* assert */
#include "util.h"
@ -127,7 +128,7 @@ static buffer_t createBuffer_fromFile(const char* fileName)
static buffer_t
createDictionaryBuffer(const char* dictionaryName,
const void* srcBuffer,
const size_t* srcBlockSizes, unsigned nbBlocks,
const size_t* srcBlockSizes, size_t nbBlocks,
size_t requestedDictSize)
{
if (dictionaryName) {
@ -141,9 +142,10 @@ createDictionaryBuffer(const char* dictionaryName,
void* const dictBuffer = malloc(requestedDictSize);
CONTROL(dictBuffer != NULL);
assert(nbBlocks <= UINT_MAX);
size_t const dictSize = ZDICT_trainFromBuffer(dictBuffer, requestedDictSize,
srcBuffer,
srcBlockSizes, nbBlocks);
srcBlockSizes, (unsigned)nbBlocks);
CONTROL(!ZSTD_isError(dictSize));
buffer_t result;

16
contrib/seekable_format/examples/parallel_processing.c
View File

@ -148,20 +148,20 @@ static void sumFile_orDie(const char* fname, int nbThreads)
size_t const initResult = ZSTD_seekable_initFile(seekable, fin);
if (ZSTD_isError(initResult)) { fprintf(stderr, "ZSTD_seekable_init() error : %s \n", ZSTD_getErrorName(initResult)); exit(11); }
size_t const numFrames = ZSTD_seekable_getNumFrames(seekable);
unsigned const numFrames = ZSTD_seekable_getNumFrames(seekable);
struct sum_job* jobs = (struct sum_job*)malloc(numFrames * sizeof(struct sum_job));
size_t i;
for (i = 0; i < numFrames; i++) {
jobs[i] = (struct sum_job){ fname, 0, i, 0 };
POOL_add(pool, sumFrame, &jobs[i]);
unsigned fnb;
for (fnb = 0; fnb < numFrames; fnb++) {
jobs[fnb] = (struct sum_job){ fname, 0, fnb, 0 };
POOL_add(pool, sumFrame, &jobs[fnb]);
}
unsigned long long total = 0;
for (i = 0; i < numFrames; i++) {
while (!jobs[i].done) SLEEP(5); /* wake up every 5 milliseconds to check */
total += jobs[i].sum;
for (fnb = 0; fnb < numFrames; fnb++) {
while (!jobs[fnb].done) SLEEP(5); /* wake up every 5 milliseconds to check */
total += jobs[fnb].sum;
}
printf("Sum: %llu\n", total);

31
contrib/seekable_format/zstdseek_compress.c
View File

@ -8,6 +8,8 @@
*/
#include <stdlib.h> /* malloc, free */
#include <limits.h> /* UINT_MAX */
#include <assert.h>
#define XXH_STATIC_LINKING_ONLY
#define XXH_NAMESPACE ZSTD_
@ -167,9 +169,9 @@ size_t ZSTD_seekable_initCStream(ZSTD_seekable_CStream* zcs,
}
size_t ZSTD_seekable_logFrame(ZSTD_frameLog* fl,
unsigned compressedSize,
unsigned decompressedSize,
unsigned checksum)
unsigned compressedSize,
unsigned decompressedSize,
unsigned checksum)
{
if (fl->size == ZSTD_SEEKABLE_MAXFRAMES)
return ERROR(frameIndex_tooLarge);
@ -184,7 +186,8 @@ size_t ZSTD_seekable_logFrame(ZSTD_frameLog* fl,
if (newEntries == NULL) return ERROR(memory_allocation);
fl->entries = newEntries;
fl->capacity = newCapacity;
assert(newCapacity <= UINT_MAX);
fl->capacity = (U32)newCapacity;
}
fl->entries[fl->size] = (framelogEntry_t){
@ -307,32 +310,32 @@ size_t ZSTD_seekable_writeSeekTable(ZSTD_frameLog* fl, ZSTD_outBuffer* output)
size_t const seekTableLen = ZSTD_seekable_seekTableSize(fl);
CHECK_Z(ZSTD_stwrite32(fl, output, ZSTD_MAGIC_SKIPPABLE_START | 0xE, 0));
CHECK_Z(ZSTD_stwrite32(fl, output, seekTableLen - ZSTD_SKIPPABLEHEADERSIZE,
4));
assert(seekTableLen <= (size_t)UINT_MAX);
CHECK_Z(ZSTD_stwrite32(fl, output, (U32)seekTableLen - ZSTD_SKIPPABLEHEADERSIZE, 4));
while (fl->seekTableIndex < fl->size) {
unsigned long long const start = ZSTD_SKIPPABLEHEADERSIZE + sizePerFrame * fl->seekTableIndex;
assert(start + 8 <= UINT_MAX);
CHECK_Z(ZSTD_stwrite32(fl, output,
fl->entries[fl->seekTableIndex].cSize,
ZSTD_SKIPPABLEHEADERSIZE +
sizePerFrame * fl->seekTableIndex + 0));
(U32)start + 0));
CHECK_Z(ZSTD_stwrite32(fl, output,
fl->entries[fl->seekTableIndex].dSize,
ZSTD_SKIPPABLEHEADERSIZE +
sizePerFrame * fl->seekTableIndex + 4));
(U32)start + 4));
if (fl->checksumFlag) {
CHECK_Z(ZSTD_stwrite32(
fl, output, fl->entries[fl->seekTableIndex].checksum,
ZSTD_SKIPPABLEHEADERSIZE +
sizePerFrame * fl->seekTableIndex + 8));
(U32)start + 8));
}
fl->seekTableIndex++;
}
assert(seekTableLen <= UINT_MAX);
CHECK_Z(ZSTD_stwrite32(fl, output, fl->size,
seekTableLen - ZSTD_seekTableFooterSize));
(U32)seekTableLen - ZSTD_seekTableFooterSize));
if (output->size - output->pos < 1) return seekTableLen - fl->seekTablePos;
if (fl->seekTablePos < seekTableLen - 4) {
@ -345,7 +348,7 @@ size_t ZSTD_seekable_writeSeekTable(ZSTD_frameLog* fl, ZSTD_outBuffer* output)
}
CHECK_Z(ZSTD_stwrite32(fl, output, ZSTD_SEEKABLE_MAGICNUMBER,
seekTableLen - 4));
(U32)seekTableLen - 4));
if (fl->seekTablePos != seekTableLen) return ERROR(GENERIC);
return 0;

13
contrib/seekable_format/zstdseek_decompress.c
View File

@ -54,8 +54,9 @@
# define LONG_SEEK fseek
#endif
#include <stdlib.h> /* malloc, free */
#include <stdio.h> /* FILE* */
#include <stdlib.h> /* malloc, free */
#include <stdio.h> /* FILE* */
#include <limits.h> /* UNIT_MAX */
#include <assert.h>
#define XXH_STATIC_LINKING_ONLY
@ -203,10 +204,11 @@ size_t ZSTD_seekable_free(ZSTD_seekable* zs)
U32 ZSTD_seekable_offsetToFrameIndex(ZSTD_seekable* const zs, unsigned long long pos)
{
U32 lo = 0;
U32 hi = zs->seekTable.tableLen;
U32 hi = (U32)zs->seekTable.tableLen;
assert(zs->seekTable.tableLen <= UINT_MAX);
if (pos >= zs->seekTable.entries[zs->seekTable.tableLen].dOffset) {
return zs->seekTable.tableLen;
return (U32)zs->seekTable.tableLen;
}
while (lo + 1 < hi) {
@ -222,7 +224,8 @@ U32 ZSTD_seekable_offsetToFrameIndex(ZSTD_seekable* const zs, unsigned long long
U32 ZSTD_seekable_getNumFrames(ZSTD_seekable* const zs)
{
return zs->seekTable.tableLen;
assert(zs->seekTable.tableLen <= UINT_MAX);
return (U32)zs->seekTable.tableLen;
}
unsigned long long ZSTD_seekable_getFrameCompressedOffset(ZSTD_seekable* const zs, U32 frameIndex)

6
doc/zstd_manual.html
View File

@ -514,12 +514,12 @@ size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
* It helps preserve compression ratio, while each job is compressed in parallel.
* This value is enforced only when nbWorkers >= 1.
* Larger values increase compression ratio, but decrease speed.
* Values range from 0 to 9 (overlap a full windowSize).
* - 0 means "auto" : value will be determined by the library, depending on strategy
* Possible values range from 0 to 9 :
* - 0 means "default" : value will be determined by the library, depending on strategy
* - 1 means "no overlap"
* - 9 means "full overlap", using a full window size.
* Each intermediate rank increases/decreases load size by a factor 2 :
* 9: full window; 8: w/2; 7: w/4; 6: w/8; 5:w/16; 4: w/32; 3:w/64; 2:w/128; 1:no overlap; 0:auto
* 9: full window; 8: w/2; 7: w/4; 6: w/8; 5:w/16; 4: w/32; 3:w/64; 2:w/128; 1:no overlap; 0:default
* default value varies between 6 and 9, depending on strategy */
</b>/* note : additional experimental parameters are also available<b>

4
lib/compress/zstd_compress.c
View File

@ -11,6 +11,7 @@
/*-*************************************
* Dependencies
***************************************/
#include <limits.h> /* INT_MAX */
#include <string.h> /* memset */
#include "cpu.h"
#include "mem.h"
@ -695,7 +696,8 @@ size_t ZSTD_CCtxParam_getParameter(
#ifndef ZSTD_MULTITHREAD
return ERROR(parameter_unsupported);
#else
*value = CCtxParams->jobSize;
assert(CCtxParams->jobSize <= INT_MAX);
*value = (int)CCtxParams->jobSize;
break;
#endif
case ZSTD_c_overlapLog :

11
lib/compress/zstdmt_compress.c
View File

@ -21,7 +21,7 @@
/* ====== Dependencies ====== */
#include <string.h> /* memcpy, memset */
#include <limits.h> /* INT_MAX */
#include <limits.h> /* INT_MAX, UINT_MAX */
#include "pool.h" /* threadpool */
#include "threading.h" /* mutex */
#include "zstd_compress_internal.h" /* MIN, ERROR, ZSTD_*, ZSTD_highbit32 */
@ -989,7 +989,7 @@ ZSTDMT_CCtxParam_setMTCtxParameter(ZSTD_CCtx_params* params,
if (value < ZSTDMT_JOBSIZE_MIN) value = ZSTDMT_JOBSIZE_MIN;
assert(value >= 0);
{ size_t jobSize = value;
if (jobSize > ZSTDMT_JOBSIZE_MAX) jobSize = ZSTDMT_JOBSIZE_MAX;
if (jobSize > (size_t)ZSTDMT_JOBSIZE_MAX) jobSize = ZSTDMT_JOBSIZE_MAX;
params->jobSize = jobSize;
return jobSize;
}
@ -1021,7 +1021,8 @@ size_t ZSTDMT_getMTCtxParameter(ZSTDMT_CCtx* mtctx, ZSTDMT_parameter parameter,
{
switch (parameter) {
case ZSTDMT_p_jobSize:
*value = mtctx->params.jobSize;
assert(mtctx->params.jobSize <= UINT_MAX);
*value = (unsigned)(mtctx->params.jobSize);
break;
case ZSTDMT_p_overlapLog:
*value = mtctx->params.overlapLog;
@ -1153,7 +1154,7 @@ size_t ZSTDMT_toFlushNow(ZSTDMT_CCtx* mtctx)
/* ===== Multi-threaded compression ===== */
/* ------------------------------------------ */
static size_t ZSTDMT_computeTargetJobLog(ZSTD_CCtx_params const params)
static unsigned ZSTDMT_computeTargetJobLog(ZSTD_CCtx_params const params)
{
if (params.ldmParams.enableLdm)
/* In Long Range Mode, the windowLog is typically oversized.
@ -1396,7 +1397,7 @@ size_t ZSTDMT_initCStream_internal(
CHECK_F( ZSTDMT_resize(mtctx, params.nbWorkers) );
if (params.jobSize > 0 && params.jobSize < ZSTDMT_JOBSIZE_MIN) params.jobSize = ZSTDMT_JOBSIZE_MIN;
if (params.jobSize > ZSTDMT_JOBSIZE_MAX) params.jobSize = ZSTDMT_JOBSIZE_MAX;
if (params.jobSize > (size_t)ZSTDMT_JOBSIZE_MAX) params.jobSize = ZSTDMT_JOBSIZE_MAX;
mtctx->singleBlockingThread = (pledgedSrcSize <= ZSTDMT_JOBSIZE_MIN); /* do not trigger multi-threading when srcSize is too small */
if (mtctx->singleBlockingThread) {

2
lib/compress/zstdmt_compress.h
View File

@ -35,7 +35,7 @@
#ifndef ZSTDMT_JOBSIZE_MIN
# define ZSTDMT_JOBSIZE_MIN (1 MB)
#endif
#define ZSTDMT_JOBSIZE_MAX ((size_t)(MEM_32bits() ? (512 MB) : (1 GB)))
#define ZSTDMT_JOBSIZE_MAX (MEM_32bits() ? (512 MB) : (1024 MB))
/* === Memory management === */