facebook
/
zstd
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #1581 from facebook/benchfn 

benchfn's reduced dependencies
dev
Yann Collet 2019-04-11 14:23:04 -07:00 committed by GitHub
parent 9563fb4bfc 1e01560b83
commit 8ac2831f3d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
35 changed files with 1034 additions and 766 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
build/VS2008/fullbench/fullbench.vcproj
View File

@ -408,6 +408,10 @@
RelativePath="..\..\..\programs\util.c"
>
</File>
<File
RelativePath="..\..\..\programs\timefn.c"
>
</File>
<File
RelativePath="..\..\..\lib\compress\zstd_fast.c"
>

4
build/VS2008/fuzzer/fuzzer.vcproj
View File

@ -332,6 +332,10 @@
RelativePath="..\..\..\programs\util.c"
>
</File>
<File
RelativePath="..\..\..\programs\timefn.c"
>
</File>
<File
RelativePath="..\..\..\programs\datagen.c"
>

4
build/VS2008/zstd/zstd.vcproj
View File

@ -336,6 +336,10 @@
RelativePath="..\..\..\programs\util.c"
>
</File>
<File
RelativePath="..\..\..\programs\timefn.c"
>
</File>
<File
RelativePath="..\..\..\programs\benchfn.c"
>

1
build/VS2010/fullbench-dll/fullbench-dll.vcxproj
View File

@ -167,6 +167,7 @@
<ItemGroup>
<ClCompile Include="..\..\..\lib\common\xxhash.c" />
<ClCompile Include="..\..\..\programs\util.c" />
<ClCompile Include="..\..\..\programs\timefn.c" />
<ClCompile Include="..\..\..\programs\datagen.c" />
<ClCompile Include="..\..\..\programs\benchfn.c" />
<ClCompile Include="..\..\..\tests\fullbench.c" />

3
build/VS2010/fullbench/fullbench.vcxproj
View File

@ -156,7 +156,6 @@
</ItemDefinitionGroup>
<ItemGroup>
<ClCompile Include="..\..\..\lib\common\entropy_common.c" />
<ClCompile Include="..\..\..\programs\util.c" />
<ClCompile Include="..\..\..\lib\common\debug.c" />
<ClCompile Include="..\..\..\lib\common\fse_decompress.c" />
<ClCompile Include="..\..\..\lib\common\zstd_common.c" />
@ -178,6 +177,8 @@
<ClCompile Include="..\..\..\lib\decompress\zstd_decompress.c" />
<ClCompile Include="..\..\..\lib\decompress\zstd_decompress_block.c" />
<ClCompile Include="..\..\..\lib\decompress\zstd_ddict.c" />
<ClCompile Include="..\..\..\programs\util.c" />
<ClCompile Include="..\..\..\programs\timefn.c" />
<ClCompile Include="..\..\..\programs\datagen.c" />
<ClCompile Include="..\..\..\programs\benchfn.c" />
<ClCompile Include="..\..\..\tests\fullbench.c" />

1
build/VS2010/fuzzer/fuzzer.vcxproj
View File

@ -182,6 +182,7 @@
<ClCompile Include="..\..\..\lib\dictBuilder\divsufsort.c" />
<ClCompile Include="..\..\..\lib\dictBuilder\zdict.c" />
<ClCompile Include="..\..\..\programs\util.c" />
<ClCompile Include="..\..\..\programs\timefn.c" />
<ClCompile Include="..\..\..\programs\datagen.c" />
<ClCompile Include="..\..\..\tests\fuzzer.c" />
</ItemGroup>

1
build/VS2010/zstd/zstd.vcxproj
View File

@ -53,6 +53,7 @@
<ClCompile Include="..\..\..\lib\legacy\zstd_v06.c" />
<ClCompile Include="..\..\..\lib\legacy\zstd_v07.c" />
<ClCompile Include="..\..\..\programs\util.c" />
<ClCompile Include="..\..\..\programs\timefn.c" />
<ClCompile Include="..\..\..\programs\benchfn.c" />
<ClCompile Include="..\..\..\programs\benchzstd.c" />
<ClCompile Include="..\..\..\programs\datagen.c" />

4
build/cmake/programs/CMakeLists.txt
View File

@ -26,7 +26,7 @@ if (MSVC)
set(PlatformDependResources ${MSVC_RESOURCE_DIR}/zstd.rc)
endif ()
add_executable(zstd ${PROGRAMS_DIR}/zstdcli.c ${PROGRAMS_DIR}/util.c ${PROGRAMS_DIR}/fileio.c ${PROGRAMS_DIR}/benchfn.c ${PROGRAMS_DIR}/benchzstd.c ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/dibio.c ${PlatformDependResources})
add_executable(zstd ${PROGRAMS_DIR}/zstdcli.c ${PROGRAMS_DIR}/util.c ${PROGRAMS_DIR}/timefn.c ${PROGRAMS_DIR}/fileio.c ${PROGRAMS_DIR}/benchfn.c ${PROGRAMS_DIR}/benchzstd.c ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/dibio.c ${PlatformDependResources})
target_link_libraries(zstd libzstd_static)
if (CMAKE_SYSTEM_NAME MATCHES "(Solaris|SunOS)")
target_link_libraries(zstd rt)
@ -63,7 +63,7 @@ if (UNIX)
install(FILES ${CMAKE_CURRENT_BINARY_DIR}/zstdgrep.1 DESTINATION "${MAN_INSTALL_DIR}")
install(FILES ${CMAKE_CURRENT_BINARY_DIR}/zstdless.1 DESTINATION "${MAN_INSTALL_DIR}")
add_executable(zstd-frugal ${PROGRAMS_DIR}/zstdcli.c ${PROGRAMS_DIR}/util.c ${PROGRAMS_DIR}/fileio.c)
add_executable(zstd-frugal ${PROGRAMS_DIR}/zstdcli.c ${PROGRAMS_DIR}/util.c ${PROGRAMS_DIR}/timefn.c ${PROGRAMS_DIR}/fileio.c)
target_link_libraries(zstd-frugal libzstd_static)
set_property(TARGET zstd-frugal APPEND PROPERTY COMPILE_DEFINITIONS "ZSTD_NOBENCH;ZSTD_NODICT")
endif ()

6
build/cmake/tests/CMakeLists.txt
View File

@ -43,13 +43,13 @@ include_directories(${TESTS_DIR} ${PROGRAMS_DIR} ${LIBRARY_DIR} ${LIBRARY_DIR}/c
add_executable(datagen ${PROGRAMS_DIR}/datagen.c ${TESTS_DIR}/datagencli.c)
target_link_libraries(datagen libzstd_static)
add_executable(fullbench ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/util.c ${PROGRAMS_DIR}/benchfn.c ${PROGRAMS_DIR}/benchzstd.c ${TESTS_DIR}/fullbench.c)
add_executable(fullbench ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/util.c ${PROGRAMS_DIR}/timefn.c ${PROGRAMS_DIR}/benchfn.c ${PROGRAMS_DIR}/benchzstd.c ${TESTS_DIR}/fullbench.c)
target_link_libraries(fullbench libzstd_static)
add_executable(fuzzer ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/util.c ${TESTS_DIR}/fuzzer.c)
add_executable(fuzzer ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/util.c ${PROGRAMS_DIR}/timefn.c ${TESTS_DIR}/fuzzer.c)
target_link_libraries(fuzzer libzstd_static)
if (UNIX)
add_executable(paramgrill ${PROGRAMS_DIR}/benchfn.c ${PROGRAMS_DIR}/benchzstd.c ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/util.c ${TESTS_DIR}/paramgrill.c)
add_executable(paramgrill ${PROGRAMS_DIR}/benchfn.c ${PROGRAMS_DIR}/benchzstd.c ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/util.c ${PROGRAMS_DIR}/timefn.c ${TESTS_DIR}/paramgrill.c)
target_link_libraries(paramgrill libzstd_static m) #m is math library
endif ()

4
contrib/adaptive-compression/Makefile
View File

@ -22,10 +22,10 @@ FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) $(MULTITHREAD_LDFLAGS)
all: adapt datagen
adapt: $(ZSTD_FILES) $(PRGDIR)/util.c adapt.c
adapt: $(ZSTD_FILES) $(PRGDIR)/util.c $(PRGDIR)/timefn.c adapt.c
$(CC) $(FLAGS) $^ -o $@
adapt-debug: $(ZSTD_FILES) $(PRGDIR)/util.c adapt.c
adapt-debug: $(ZSTD_FILES) $(PRGDIR)/util.c $(PRGDIR)/timefn.c adapt.c
$(CC) $(FLAGS) -DDEBUG_MODE=2 $^ -o adapt
datagen : $(PRGDIR)/datagen.c datagencli.c

1
contrib/adaptive-compression/adapt.c
View File

@ -13,6 +13,7 @@
#include <string.h> /* memset */
#include "zstd_internal.h"
#include "util.h"
#include "timefn.h" /* UTIL_time_t, UTIL_getTime, UTIL_getSpanTimeMicro */
#define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
#define PRINT(...) fprintf(stdout, __VA_ARGS__)

8
contrib/largeNbDicts/Makefile
View File

@ -28,14 +28,17 @@ default: largeNbDicts
all : largeNbDicts
largeNbDicts: util.o benchfn.o datagen.o xxhash.o largeNbDicts.c $(LIBZSTD)
largeNbDicts: util.o timefn.o benchfn.o datagen.o xxhash.o largeNbDicts.c $(LIBZSTD)
$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
.PHONY: $(LIBZSTD)
$(LIBZSTD):
$(MAKE) -C $(LIBDIR) libzstd.a CFLAGS="$(CFLAGS)"
benchfn.o : $(PROGDIR)/benchfn.c
benchfn.o: $(PROGDIR)/benchfn.c
$(CC) $(CPPFLAGS) $(CFLAGS) $^ -c
timefn.o: $(PROGDIR)/timefn.c
$(CC) $(CPPFLAGS) $(CFLAGS) $^ -c
datagen.o: $(PROGDIR)/datagen.c
@ -48,6 +51,7 @@ util.o: $(PROGDIR)/util.c
xxhash.o : $(LIBDIR)/common/xxhash.c
$(CC) $(CPPFLAGS) $(CFLAGS) $^ -c
clean:
$(RM) *.o
$(MAKE) -C $(LIBDIR) clean > /dev/null

917
doc/zstd_manual.html
View File

File diff suppressed because it is too large Load Diff

18
programs/Makefile
View File

@ -165,7 +165,7 @@ $(ZSTDDECOMP_O): CFLAGS += $(ALIGN_LOOP)
zstd : CPPFLAGS += $(THREAD_CPP) $(ZLIBCPP) $(LZMACPP) $(LZ4CPP)
zstd : LDFLAGS += $(THREAD_LD) $(ZLIBLD) $(LZMALD) $(LZ4LD) $(DEBUGFLAGS_LD)
zstd : CPPFLAGS += -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT)
zstd : $(ZSTDLIB_FILES) zstdcli.o util.o fileio.o benchfn.o benchzstd.o datagen.o dibio.o
zstd : $(ZSTDLIB_FILES) zstdcli.o util.o timefn.o fileio.o benchfn.o benchzstd.o datagen.o dibio.o
@echo "$(THREAD_MSG)"
@echo "$(ZLIB_MSG)"
@echo "$(LZMA_MSG)"
@ -183,13 +183,13 @@ zstd-release: zstd
zstd32 : CPPFLAGS += $(THREAD_CPP)
zstd32 : LDFLAGS += $(THREAD_LD)
zstd32 : CPPFLAGS += -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT)
zstd32 : $(ZSTDLIB_FILES) zstdcli.c util.c fileio.c benchfn.c benchzstd.c datagen.c dibio.c
zstd32 : $(ZSTDLIB_FILES) zstdcli.c util.c timefn.c fileio.c benchfn.c benchzstd.c datagen.c dibio.c
ifneq (,$(filter Windows%,$(OS)))
windres/generate_res.bat
endif
$(CC) -m32 $(FLAGS) $^ $(RES32_FILE) -o $@$(EXT)
zstd-nolegacy : $(ZSTD_FILES) $(ZDICT_FILES) zstdcli.o util.o fileio.c benchfn.o benchzstd.o datagen.o dibio.o
zstd-nolegacy : $(ZSTD_FILES) $(ZDICT_FILES) zstdcli.o util.o fileio.c benchfn.o benchzstd.o timefn.o datagen.o dibio.o
$(CC) $(FLAGS) $^ -o $@$(EXT) $(LDFLAGS)
zstd-nomt : THREAD_CPP :=
@ -222,13 +222,13 @@ zstd-pgo :
# minimal target, with only zstd compression and decompression. no bench. no legacy.
zstd-small: CFLAGS = -Os -s
zstd-frugal zstd-small: $(ZSTD_FILES) zstdcli.c util.c fileio.c
zstd-frugal zstd-small: $(ZSTD_FILES) zstdcli.c util.c timefn.c fileio.c
$(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT $^ -o $@$(EXT)
zstd-decompress: $(ZSTDCOMMON_FILES) $(ZSTDDECOMP_FILES) zstdcli.c util.c fileio.c
zstd-decompress: $(ZSTDCOMMON_FILES) $(ZSTDDECOMP_FILES) zstdcli.c util.c timefn.c fileio.c
$(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT -DZSTD_NOCOMPRESS $^ -o $@$(EXT)
zstd-compress: $(ZSTDCOMMON_FILES) $(ZSTDCOMP_FILES) zstdcli.c util.c fileio.c
zstd-compress: $(ZSTDCOMMON_FILES) $(ZSTDCOMP_FILES) zstdcli.c util.c timefn.c fileio.c
$(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT -DZSTD_NODECOMPRESS $^ -o $@$(EXT)
zstdmt: zstd
@ -265,9 +265,9 @@ man: zstd.1 zstdgrep.1 zstdless.1
.PHONY: clean-man
clean-man:
rm zstd.1
rm zstdgrep.1
rm zstdless.1
$(RM) zstd.1
$(RM) zstdgrep.1
$(RM) zstdless.1
.PHONY: preview-man
preview-man: clean-man man

87
programs/benchfn.c
View File

@ -13,25 +13,20 @@
/* *************************************
* Includes
***************************************/
#include "platform.h" /* Large Files support */
#include "util.h" /* UTIL_getFileSize, UTIL_sleep */
#include <stdlib.h> /* malloc, free */
#include <string.h> /* memset */
#include <stdio.h> /* fprintf, fopen */
#undef NDEBUG /* assert must not be disabled */
#include <assert.h> /* assert */
#include "mem.h"
#include "timefn.h" /* UTIL_time_t, UTIL_getTime */
#include "benchfn.h"
/* *************************************
* Constants
***************************************/
#define TIMELOOP_MICROSEC (1*1000000ULL) /* 1 second */
#define TIMELOOP_MICROSEC SEC_TO_MICRO /* 1 second */
#define TIMELOOP_NANOSEC (1*1000000000ULL) /* 1 second */
#define ACTIVEPERIOD_MICROSEC (70*TIMELOOP_MICROSEC) /* 70 seconds */
#define COOLPERIOD_SEC 10
#define KB *(1 <<10)
#define MB *(1 <<20)
@ -39,14 +34,16 @@
/* *************************************
* Errors
* Debug errors
***************************************/
#ifndef DEBUG
# define DEBUG 0
#if defined(DEBUG) && (DEBUG >= 1)
# include <stdio.h> /* fprintf */
# define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
# define DEBUGOUTPUT(...) { if (DEBUG) DISPLAY(__VA_ARGS__); }
#else
# define DEBUGOUTPUT(...)
#endif
#define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
#define DEBUGOUTPUT(...) { if (DEBUG) DISPLAY(__VA_ARGS__); }
/* error without displaying */
#define RETURN_QUIET_ERROR(retValue, ...) { \
@ -116,15 +113,7 @@ BMK_runOutcome_t BMK_benchFunction(BMK_benchParams_t p,
{ size_t i;
for(i = 0; i < p.blockCount; i++) {
memset(p.dstBuffers[i], 0xE5, p.dstCapacities[i]); /* warm up and erase result buffer */
}
#if 0
/* based on testing these seem to lower accuracy of multiple calls of 1 nbLoops vs 1 call of multiple nbLoops
* (Makes former slower)
*/
UTIL_sleepMilli(5); /* give processor time to other processes */
UTIL_waitForNextTick();
#endif
}
} }
/* benchmark */
{ UTIL_time_t const clockStart = UTIL_getTime();
@ -146,9 +135,9 @@ BMK_runOutcome_t BMK_benchFunction(BMK_benchParams_t p,
} }
} /* for (loopNb = 0; loopNb < nbLoops; loopNb++) */
{ U64 const totalTime = UTIL_clockSpanNano(clockStart);
{ PTime const totalTime = UTIL_clockSpanNano(clockStart);
BMK_runTime_t rt;
rt.nanoSecPerRun = totalTime / nbLoops;
rt.nanoSecPerRun = (double)totalTime / nbLoops;
rt.sumOfReturn = dstSize;
return BMK_setValid_runTime(rt);
} }
@ -158,9 +147,9 @@ BMK_runOutcome_t BMK_benchFunction(BMK_benchParams_t p,
/* ==== Benchmarking any function, providing intermediate results ==== */
struct BMK_timedFnState_s {
U64 timeSpent_ns;
U64 timeBudget_ns;
U64 runBudget_ns;
PTime timeSpent_ns;
PTime timeBudget_ns;
PTime runBudget_ns;
BMK_runTime_t fastestRun;
unsigned nbLoops;
UTIL_time_t coolTime;
@ -174,8 +163,20 @@ BMK_timedFnState_t* BMK_createTimedFnState(unsigned total_ms, unsigned run_ms)
return r;
}
void BMK_freeTimedFnState(BMK_timedFnState_t* state) {
free(state);
void BMK_freeTimedFnState(BMK_timedFnState_t* state) { free(state); }
BMK_timedFnState_t*
BMK_initStatic_timedFnState(void* buffer, size_t size, unsigned total_ms, unsigned run_ms)
{
typedef char check_size[ 2 * (sizeof(BMK_timedFnState_shell) >= sizeof(struct BMK_timedFnState_s)) - 1]; /* static assert : a compilation failure indicates that BMK_timedFnState_shell is not large enough */
typedef struct { check_size c; BMK_timedFnState_t tfs; } tfs_align; /* force tfs to be aligned at its next best position */
size_t const tfs_alignment = offsetof(tfs_align, tfs); /* provides the minimal alignment restriction for BMK_timedFnState_t */
BMK_timedFnState_t* const r = (BMK_timedFnState_t*)buffer;
if (buffer == NULL) return NULL;
if (size < sizeof(struct BMK_timedFnState_s)) return NULL;
if ((size_t)buffer % tfs_alignment) return NULL; /* buffer must be properly aligned */
BMK_resetTimedFnState(r, total_ms, run_ms);
return r;
}
void BMK_resetTimedFnState(BMK_timedFnState_t* timedFnState, unsigned total_ms, unsigned run_ms)
@ -184,9 +185,9 @@ void BMK_resetTimedFnState(BMK_timedFnState_t* timedFnState, unsigned total_ms,
if (!run_ms) run_ms = 1;
if (run_ms > total_ms) run_ms = total_ms;
timedFnState->timeSpent_ns = 0;
timedFnState->timeBudget_ns = (U64)total_ms * TIMELOOP_NANOSEC / 1000;
timedFnState->runBudget_ns = (U64)run_ms * TIMELOOP_NANOSEC / 1000;
timedFnState->fastestRun.nanoSecPerRun = (U64)(-1LL);
timedFnState->timeBudget_ns = (PTime)total_ms * TIMELOOP_NANOSEC / 1000;
timedFnState->runBudget_ns = (PTime)run_ms * TIMELOOP_NANOSEC / 1000;
timedFnState->fastestRun.nanoSecPerRun = (double)TIMELOOP_NANOSEC * 2000000000; /* hopefully large enough : must be larger than any potential measurement */
timedFnState->fastestRun.sumOfReturn = (size_t)(-1LL);
timedFnState->nbLoops = 1;
timedFnState->coolTime = UTIL_getTime();
@ -208,37 +209,27 @@ int BMK_isCompleted_TimedFn(const BMK_timedFnState_t* timedFnState)
BMK_runOutcome_t BMK_benchTimedFn(BMK_timedFnState_t* cont,
BMK_benchParams_t p)
{
U64 const runBudget_ns = cont->runBudget_ns;
U64 const runTimeMin_ns = runBudget_ns / 2;
PTime const runBudget_ns = cont->runBudget_ns;
PTime const runTimeMin_ns = runBudget_ns / 2;
int completed = 0;
BMK_runTime_t bestRunTime = cont->fastestRun;
while (!completed) {
BMK_runOutcome_t runResult;
/* Overheat protection */
if (UTIL_clockSpanMicro(cont->coolTime) > ACTIVEPERIOD_MICROSEC) {
DEBUGOUTPUT("\rcooling down ... \r");
UTIL_sleep(COOLPERIOD_SEC);
cont->coolTime = UTIL_getTime();
}
/* reinitialize capacity */
runResult = BMK_benchFunction(p, cont->nbLoops);
BMK_runOutcome_t const runResult = BMK_benchFunction(p, cont->nbLoops);
if(!BMK_isSuccessful_runOutcome(runResult)) { /* error : move out */
return runResult;
}
{ BMK_runTime_t const newRunTime = BMK_extract_runTime(runResult);
U64 const loopDuration_ns = newRunTime.nanoSecPerRun * cont->nbLoops;
double const loopDuration_ns = newRunTime.nanoSecPerRun * cont->nbLoops;
cont->timeSpent_ns += loopDuration_ns;
cont->timeSpent_ns += (unsigned long long)loopDuration_ns;
/* estimate nbLoops for next run to last approximately 1 second */
if (loopDuration_ns > (runBudget_ns / 50)) {
U64 const fastestRun_ns = MIN(bestRunTime.nanoSecPerRun, newRunTime.nanoSecPerRun);
cont->nbLoops = (U32)(runBudget_ns / fastestRun_ns) + 1;
double const fastestRun_ns = MIN(bestRunTime.nanoSecPerRun, newRunTime.nanoSecPerRun);
cont->nbLoops = (unsigned)(runBudget_ns / fastestRun_ns) + 1;
} else {
/* previous run was too short : blindly increase workload by x multiplier */
const unsigned multiplier = 10;

62
programs/benchfn.h
View File

@ -31,7 +31,7 @@ extern "C" {
/* BMK_runTime_t: valid result return type */
typedef struct {
unsigned long long nanoSecPerRun; /* time per iteration (over all blocks) */
double nanoSecPerRun; /* time per iteration (over all blocks) */
size_t sumOfReturn; /* sum of return values */
} BMK_runTime_t;
@ -58,30 +58,31 @@ typedef size_t (*BMK_initFn_t)(void* initPayload);
typedef unsigned (*BMK_errorFn_t)(size_t);
/* BMK_benchFunction() parameters are provided through following structure.
* This is preferable for readability,
* as the number of parameters required is pretty large.
/* BMK_benchFunction() parameters are provided via the following structure.
* A structure is preferable for readability,
* as the number of parameters required is fairly large.
* No initializer is provided, because it doesn't make sense to provide some "default" :
* all parameters should be specified by the caller */
* all parameters must be specified by the caller.
* optional parameters are labelled explicitly, and accept value NULL when not used */
typedef struct {
BMK_benchFn_t benchFn; /* the function to benchmark, over the set of blocks */
void* benchPayload; /* pass custom parameters to benchFn :
* (*benchFn)(srcBuffers[i], srcSizes[i], dstBuffers[i], dstCapacities[i], benchPayload) */
BMK_initFn_t initFn; /* (*initFn)(initPayload) is run once per run, at the beginning. */
void* initPayload; /* Both arguments can be NULL, in which case nothing is run. */
BMK_errorFn_t errorFn; /* errorFn will check each return value of benchFn over each block, to determine if it failed or not.
* errorFn can be NULL, in which case no check is performed.
* errorFn must return 0 when benchFn was successful, and >= 1 if it detects an error.
* Execution is stopped as soon as an error is detected.
* the triggering return value can be retrieved using BMK_extract_errorResult(). */
size_t blockCount; /* number of blocks to operate benchFn on.
* It's also the size of all array parameters :
* srcBuffers, srcSizes, dstBuffers, dstCapacities, blockResults */
const void *const * srcBuffers; /* array of buffers to be operated on by benchFn */
const size_t* srcSizes; /* array of the sizes of srcBuffers buffers */
void *const * dstBuffers;/* array of buffers to be written into by benchFn */
const size_t* dstCapacities; /* array of the capacities of dstBuffers buffers */
size_t* blockResults; /* Optional: store the return value of benchFn for each block. Use NULL if this result is not requested. */
BMK_benchFn_t benchFn; /* the function to benchmark, over the set of blocks */
void* benchPayload; /* pass custom parameters to benchFn :
* (*benchFn)(srcBuffers[i], srcSizes[i], dstBuffers[i], dstCapacities[i], benchPayload) */
BMK_initFn_t initFn; /* (*initFn)(initPayload) is run once per run, at the beginning. */
void* initPayload; /* Both arguments can be NULL, in which case nothing is run. */
BMK_errorFn_t errorFn; /* errorFn will check each return value of benchFn over each block, to determine if it failed or not.
* errorFn can be NULL, in which case no check is performed.
* errorFn must return 0 when benchFn was successful, and >= 1 if it detects an error.
* Execution is stopped as soon as an error is detected.
* the triggering return value can be retrieved using BMK_extract_errorResult(). */
size_t blockCount; /* number of blocks to operate benchFn on.
* It's also the size of all array parameters :
* srcBuffers, srcSizes, dstBuffers, dstCapacities, blockResults */
const void *const * srcBuffers; /* read-only array of buffers to be operated on by benchFn */
const size_t* srcSizes; /* read-only array containing sizes of srcBuffers */
void *const * dstBuffers; /* array of buffers to be written into by benchFn. This array is not optional, it must be provided even if unused by benchfn. */
const size_t* dstCapacities; /* read-only array containing capacities of dstBuffers. This array must be present. */
size_t* blockResults; /* Optional: store the return value of benchFn for each block. Use NULL if this result is not requested. */
} BMK_benchParams_t;
@ -159,6 +160,21 @@ void BMK_resetTimedFnState(BMK_timedFnState_t* timedFnState, unsigned total_ms,
void BMK_freeTimedFnState(BMK_timedFnState_t* state);
/* BMK_timedFnState_shell and BMK_initStatic_timedFnState() :
* Makes it possible to statically allocate a BMK_timedFnState_t on stack.
* BMK_timedFnState_shell is only there to allocate space,
* never ever access its members.
* BMK_timedFnState_t() actually accepts any buffer.
* It will check if provided buffer is large enough and is correctly aligned,
* and will return NULL if conditions are not respected.
*/
#define BMK_TIMEDFNSTATE_SIZE 64
typedef union {
char never_access_space[BMK_TIMEDFNSTATE_SIZE];
long long alignment_enforcer; /* must be aligned on 8-bytes boundaries */
} BMK_timedFnState_shell;
BMK_timedFnState_t* BMK_initStatic_timedFnState(void* buffer, size_t size, unsigned total_ms, unsigned run_ms);
#endif /* BENCH_FN_H_23876 */

43
programs/benchzstd.c
View File

@ -28,6 +28,7 @@
#include <errno.h>
#include <assert.h> /* assert */
#include "timefn.h" /* UTIL_time_t */
#include "benchfn.h"
#include "mem.h"
#define ZSTD_STATIC_LINKING_ONLY
@ -160,9 +161,13 @@ typedef struct {
#define MIN(a,b) ((a) < (b) ? (a) : (b))
#define MAX(a,b) ((a) > (b) ? (a) : (b))
static void BMK_initCCtx(ZSTD_CCtx* ctx,
const void* dictBuffer, size_t dictBufferSize, int cLevel,
const ZSTD_compressionParameters* comprParams, const BMK_advancedParams_t* adv) {
static void
BMK_initCCtx(ZSTD_CCtx* ctx,
const void* dictBuffer, size_t dictBufferSize,
int cLevel,
const ZSTD_compressionParameters* comprParams,
const BMK_advancedParams_t* adv)
{
ZSTD_CCtx_reset(ctx, ZSTD_reset_session_and_parameters);
if (adv->nbWorkers==1) {
CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_nbWorkers, 0));
@ -175,13 +180,13 @@ static void BMK_initCCtx(ZSTD_CCtx* ctx,
CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_ldmHashLog, adv->ldmHashLog));
CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_ldmBucketSizeLog, adv->ldmBucketSizeLog));
CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_ldmHashRateLog, adv->ldmHashRateLog));
CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_windowLog, (int)comprParams->windowLog));
CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_hashLog, (int)comprParams->hashLog));
CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_chainLog, (int)comprParams->chainLog));
CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_searchLog, (int)comprParams->searchLog));
CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_minMatch, (int)comprParams->minMatch));
CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_targetLength, (int)comprParams->targetLength));
CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_literalCompressionMode, (int)adv->literalCompressionMode));
CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_windowLog, comprParams->windowLog));
CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_hashLog, comprParams->hashLog));
CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_chainLog, comprParams->chainLog));
CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_searchLog, comprParams->searchLog));
CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_minMatch, comprParams->minMatch));
CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_targetLength, comprParams->targetLength));
CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_strategy, comprParams->strategy));
CHECK_Z(ZSTD_CCtx_loadDictionary(ctx, dictBuffer, dictBufferSize));
}
@ -446,7 +451,7 @@ BMK_benchMemAdvancedNoAlloc(
cSize = cResult.sumOfReturn;
ratio = (double)srcSize / cSize;
{ BMK_benchResult_t newResult;
newResult.cSpeed = ((U64)srcSize * TIMELOOP_NANOSEC / cResult.nanoSecPerRun);
newResult.cSpeed = (U64)((double)srcSize * TIMELOOP_NANOSEC / cResult.nanoSecPerRun);
benchResult.cSize = cSize;
if (newResult.cSpeed > benchResult.cSpeed)
benchResult.cSpeed = newResult.cSpeed;
@ -470,7 +475,7 @@ BMK_benchMemAdvancedNoAlloc(
}
{ BMK_runTime_t const dResult = BMK_extract_runTime(dOutcome);
U64 const newDSpeed = (srcSize * TIMELOOP_NANOSEC / dResult.nanoSecPerRun);
U64 const newDSpeed = (U64)((double)srcSize * TIMELOOP_NANOSEC / dResult.nanoSecPerRun);
if (newDSpeed > benchResult.dSpeed)
benchResult.dSpeed = newDSpeed;
}
@ -507,17 +512,21 @@ BMK_benchMemAdvancedNoAlloc(
pos = (U32)(u - bacc);
bNb = pos / (128 KB);
DISPLAY("(sample %u, block %u, pos %u) \n", segNb, bNb, pos);
if (u>5) {
int n;
{ size_t const lowest = (u>5) ? 5 : u;
size_t n;
DISPLAY("origin: ");
for (n=-5; n<0; n++) DISPLAY("%02X ", ((const BYTE*)srcBuffer)[u+n]);
for (n=lowest; n>0; n--)
DISPLAY("%02X ", ((const BYTE*)srcBuffer)[u-n]);
DISPLAY(" :%02X: ", ((const BYTE*)srcBuffer)[u]);
for (n=1; n<3; n++) DISPLAY("%02X ", ((const BYTE*)srcBuffer)[u+n]);
for (n=1; n<3; n++)
DISPLAY("%02X ", ((const BYTE*)srcBuffer)[u+n]);
DISPLAY(" \n");
DISPLAY("decode: ");
for (n=-5; n<0; n++) DISPLAY("%02X ", resultBuffer[u+n]);
for (n=lowest; n>0; n++)
DISPLAY("%02X ", resultBuffer[u-n]);
DISPLAY(" :%02X: ", resultBuffer[u]);
for (n=1; n<3; n++) DISPLAY("%02X ", resultBuffer[u+n]);
for (n=1; n<3; n++)
DISPLAY("%02X ", resultBuffer[u+n]);
DISPLAY(" \n");
}
break;

22
programs/benchzstd.h
View File

@ -105,17 +105,17 @@ typedef enum {
} BMK_mode_t;
typedef struct {
BMK_mode_t mode; /* 0: all, 1: compress only 2: decode only */
unsigned nbSeconds; /* default timing is in nbSeconds */
size_t blockSize; /* Maximum size of each block*/
unsigned nbWorkers; /* multithreading */
unsigned realTime; /* real time priority */
int additionalParam; /* used by python speed benchmark */
unsigned ldmFlag; /* enables long distance matching */
unsigned ldmMinMatch; /* below: parameters for long distance matching, see zstd.1.md */
unsigned ldmHashLog;
unsigned ldmBucketSizeLog;
unsigned ldmHashRateLog;
BMK_mode_t mode; /* 0: all, 1: compress only 2: decode only */
unsigned nbSeconds; /* default timing is in nbSeconds */
size_t blockSize; /* Maximum size of each block*/
int nbWorkers; /* multithreading */
unsigned realTime; /* real time priority */
int additionalParam; /* used by python speed benchmark */
int ldmFlag; /* enables long distance matching */
int ldmMinMatch; /* below: parameters for long distance matching, see zstd.1.md */
int ldmHashLog;
int ldmBucketSizeLog;
int ldmHashRateLog;
ZSTD_literalCompressionMode_e literalCompressionMode;
} BMK_advancedParams_t;

1
programs/dibio.c
View File

@ -29,6 +29,7 @@
#include <errno.h> /* errno */
#include <assert.h>
#include "timefn.h" /* UTIL_time_t, UTIL_clockSpanMicro, UTIL_getTime */
#include "mem.h" /* read */
#include "error_private.h"
#include "dibio.h"

5
programs/fileio.c
View File

@ -31,6 +31,7 @@
#include <assert.h>
#include <errno.h> /* errno */
#include <signal.h>
#include "timefn.h" /* UTIL_getTime, UTIL_clockSpanMicro */
#if defined (_MSC_VER)
# include <sys/stat.h>
@ -1546,10 +1547,12 @@ static unsigned FIO_fwriteSparse(FIO_prefs_t* const prefs, FILE* file, const voi
return storedSkips;
}
static void FIO_fwriteSparseEnd(FIO_prefs_t* const prefs, FILE* file, unsigned storedSkips)
static void
FIO_fwriteSparseEnd(FIO_prefs_t* const prefs, FILE* file, unsigned storedSkips)
{
if (storedSkips>0) {
assert(prefs->sparseFileSupport > 0); /* storedSkips>0 implies sparse support is enabled */
(void)prefs; /* assert can be disabled, in which case prefs becomes unused */
if (LONG_SEEK(file, storedSkips-1, SEEK_CUR) != 0)
EXM_THROW(69, "Final skip error (sparse file support)");
/* last zero must be explicitly written,

5
programs/platform.h
View File

@ -87,8 +87,8 @@ extern "C" {
* The following list of build macros tries to "guess" if target OS is likely unix-like, and therefore can #include <unistd.h>
*/
# elif !defined(_WIN32) \
&& (defined(__unix__) || defined(__unix) \
|| defined(__midipix__) || defined(__VMS) || defined(__HAIKU__))
&& ( defined(__unix__) || defined(__unix) \
|| defined(__midipix__) || defined(__VMS) || defined(__HAIKU__) )
# if defined(__linux__) || defined(__linux)
# ifndef _POSIX_C_SOURCE
@ -108,6 +108,7 @@ extern "C" {
#endif /* PLATFORM_POSIX_VERSION */
/*-*********************************************
* Detect if isatty() and fileno() are available
************************************************/

168
programs/timefn.c Normal file
View File

@ -0,0 +1,168 @@
/*
* Copyright (c) 2019-present, Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
* in the COPYING file in the root directory of this source tree).
* You may select, at your option, one of the above-listed licenses.
*/
/* === Dependencies === */
#include "timefn.h"
/*-****************************************
* Time functions
******************************************/
#if defined(_WIN32) /* Windows */
#include <stdlib.h> /* abort */
#include <stdio.h> /* perror */
UTIL_time_t UTIL_getTime(void) { UTIL_time_t x; QueryPerformanceCounter(&x); return x; }
PTime UTIL_getSpanTimeMicro(UTIL_time_t clockStart, UTIL_time_t clockEnd)
{
static LARGE_INTEGER ticksPerSecond;
static int init = 0;
if (!init) {
if (!QueryPerformanceFrequency(&ticksPerSecond)) {
perror("timefn::QueryPerformanceFrequency");
abort();
}
init = 1;
}
return 1000000ULL*(clockEnd.QuadPart - clockStart.QuadPart)/ticksPerSecond.QuadPart;
}
PTime UTIL_getSpanTimeNano(UTIL_time_t clockStart, UTIL_time_t clockEnd)
{
static LARGE_INTEGER ticksPerSecond;
static int init = 0;
if (!init) {
if (!QueryPerformanceFrequency(&ticksPerSecond)) {
perror("timefn::QueryPerformanceFrequency");
abort();
}
init = 1;
}
return 1000000000ULL*(clockEnd.QuadPart - clockStart.QuadPart)/ticksPerSecond.QuadPart;
}
#elif defined(__APPLE__) && defined(__MACH__)
UTIL_time_t UTIL_getTime(void) { return mach_absolute_time(); }
PTime UTIL_getSpanTimeMicro(UTIL_time_t clockStart, UTIL_time_t clockEnd)
{
static mach_timebase_info_data_t rate;
static int init = 0;
if (!init) {
mach_timebase_info(&rate);
init = 1;
}
return (((clockEnd - clockStart) * (PTime)rate.numer) / ((PTime)rate.denom))/1000ULL;
}
PTime UTIL_getSpanTimeNano(UTIL_time_t clockStart, UTIL_time_t clockEnd)
{
static mach_timebase_info_data_t rate;
static int init = 0;
if (!init) {
mach_timebase_info(&rate);
init = 1;
}
return ((clockEnd - clockStart) * (PTime)rate.numer) / ((PTime)rate.denom);
}
#elif (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) /* C11 */) \
&& defined(TIME_UTC) /* C11 requires timespec_get, but FreeBSD 11 lacks it, while still claiming C11 compliance */
#include <stdlib.h> /* abort */
#include <stdio.h> /* perror */
UTIL_time_t UTIL_getTime(void)
{
/* time must be initialized, othersize it may fail msan test.
* No good reason, likely a limitation of timespec_get() for some target */
UTIL_time_t time = UTIL_TIME_INITIALIZER;
if (timespec_get(&time, TIME_UTC) != TIME_UTC) {
perror("timefn::timespec_get");
abort();
}
return time;
}
static UTIL_time_t UTIL_getSpanTime(UTIL_time_t begin, UTIL_time_t end)
{
UTIL_time_t diff;
if (end.tv_nsec < begin.tv_nsec) {
diff.tv_sec = (end.tv_sec - 1) - begin.tv_sec;
diff.tv_nsec = (end.tv_nsec + 1000000000ULL) - begin.tv_nsec;
} else {
diff.tv_sec = end.tv_sec - begin.tv_sec;
diff.tv_nsec = end.tv_nsec - begin.tv_nsec;
}
return diff;
}
PTime UTIL_getSpanTimeMicro(UTIL_time_t begin, UTIL_time_t end)
{
UTIL_time_t const diff = UTIL_getSpanTime(begin, end);
PTime micro = 0;
micro += 1000000ULL * diff.tv_sec;
micro += diff.tv_nsec / 1000ULL;
return micro;
}
PTime UTIL_getSpanTimeNano(UTIL_time_t begin, UTIL_time_t end)
{
UTIL_time_t const diff = UTIL_getSpanTime(begin, end);
PTime nano = 0;
nano += 1000000000ULL * diff.tv_sec;
nano += diff.tv_nsec;
return nano;
}
#else /* relies on standard C90 (note : clock_t measurements can be wrong when using multi-threading) */
UTIL_time_t UTIL_getTime(void) { return clock(); }
PTime UTIL_getSpanTimeMicro(UTIL_time_t clockStart, UTIL_time_t clockEnd) { return 1000000ULL * (clockEnd - clockStart) / CLOCKS_PER_SEC; }
PTime UTIL_getSpanTimeNano(UTIL_time_t clockStart, UTIL_time_t clockEnd) { return 1000000000ULL * (clockEnd - clockStart) / CLOCKS_PER_SEC; }
#endif
/* returns time span in microseconds */
PTime UTIL_clockSpanMicro(UTIL_time_t clockStart )
{
UTIL_time_t const clockEnd = UTIL_getTime();
return UTIL_getSpanTimeMicro(clockStart, clockEnd);
}
/* returns time span in microseconds */
PTime UTIL_clockSpanNano(UTIL_time_t clockStart )
{
UTIL_time_t const clockEnd = UTIL_getTime();
return UTIL_getSpanTimeNano(clockStart, clockEnd);
}
void UTIL_waitForNextTick(void)
{
UTIL_time_t const clockStart = UTIL_getTime();
UTIL_time_t clockEnd;
do {
clockEnd = UTIL_getTime();
} while (UTIL_getSpanTimeNano(clockStart, clockEnd) == 0);
}

89
programs/timefn.h Normal file
View File

@ -0,0 +1,89 @@
/*
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
* in the COPYING file in the root directory of this source tree).
* You may select, at your option, one of the above-listed licenses.
*/
#ifndef TIME_FN_H_MODULE_287987
#define TIME_FN_H_MODULE_287987
#if defined (__cplusplus)
extern "C" {
#endif
/*-****************************************
* Dependencies
******************************************/
#include <sys/types.h> /* utime */
#if defined(_MSC_VER)
# include <sys/utime.h> /* utime */
#else
# include <utime.h> /* utime */
#endif
#include <time.h> /* clock_t, clock, CLOCKS_PER_SEC */
/*-****************************************
* Local Types
******************************************/
#if !defined (__VMS) && (defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) )
# include <stdint.h>
typedef uint64_t PTime; /* Precise Time */
#else
typedef unsigned long long PTime; /* does not support compilers without long long support */
#endif
/*-****************************************
* Time functions
******************************************/
#if defined(_WIN32) /* Windows */
#include <Windows.h> /* LARGE_INTEGER */
typedef LARGE_INTEGER UTIL_time_t;
#define UTIL_TIME_INITIALIZER { { 0, 0 } }
#elif defined(__APPLE__) && defined(__MACH__)
#include <mach/mach_time.h>
typedef PTime UTIL_time_t;
#define UTIL_TIME_INITIALIZER 0
#elif (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) /* C11 */) \
&& defined(TIME_UTC) /* C11 requires timespec_get, but FreeBSD 11 lacks it, while still claiming C11 compliance */
typedef struct timespec UTIL_time_t;
#define UTIL_TIME_INITIALIZER { 0, 0 }
#else /* relies on standard C90 (note : clock_t measurements can be wrong when using multi-threading) */
typedef clock_t UTIL_time_t;
#define UTIL_TIME_INITIALIZER 0
#endif
UTIL_time_t UTIL_getTime(void);
PTime UTIL_getSpanTimeMicro(UTIL_time_t clockStart, UTIL_time_t clockEnd);
PTime UTIL_getSpanTimeNano(UTIL_time_t clockStart, UTIL_time_t clockEnd);
#define SEC_TO_MICRO ((PTime)1000000)
PTime UTIL_clockSpanMicro(UTIL_time_t clockStart);
PTime UTIL_clockSpanNano(UTIL_time_t clockStart);
void UTIL_waitForNextTick(void);
#if defined (__cplusplus)
}
#endif
#endif /* TIME_FN_H_MODULE_287987 */

134
programs/util.c
View File

@ -352,146 +352,18 @@ UTIL_createFileList(const char **inputNames, unsigned inputNamesNb,
return fileTable;
}
/*-****************************************
* Console log
******************************************/
int g_utilDisplayLevel;
/*-****************************************
* Time functions
* count the number of physical cores
******************************************/
#if defined(_WIN32) /* Windows */
UTIL_time_t UTIL_getTime(void) { UTIL_time_t x; QueryPerformanceCounter(&x); return x; }
U64 UTIL_getSpanTimeMicro(UTIL_time_t clockStart, UTIL_time_t clockEnd)
{
static LARGE_INTEGER ticksPerSecond;
static int init = 0;
if (!init) {
if (!QueryPerformanceFrequency(&ticksPerSecond))
UTIL_DISPLAYLEVEL(1, "ERROR: QueryPerformanceFrequency() failure\n");
init = 1;
}
return 1000000ULL*(clockEnd.QuadPart - clockStart.QuadPart)/ticksPerSecond.QuadPart;
}
U64 UTIL_getSpanTimeNano(UTIL_time_t clockStart, UTIL_time_t clockEnd)
{
static LARGE_INTEGER ticksPerSecond;
static int init = 0;
if (!init) {
if (!QueryPerformanceFrequency(&ticksPerSecond))
UTIL_DISPLAYLEVEL(1, "ERROR: QueryPerformanceFrequency() failure\n");
init = 1;
}
return 1000000000ULL*(clockEnd.QuadPart - clockStart.QuadPart)/ticksPerSecond.QuadPart;
}
#elif defined(__APPLE__) && defined(__MACH__)
UTIL_time_t UTIL_getTime(void) { return mach_absolute_time(); }
U64 UTIL_getSpanTimeMicro(UTIL_time_t clockStart, UTIL_time_t clockEnd)
{
static mach_timebase_info_data_t rate;
static int init = 0;
if (!init) {
mach_timebase_info(&rate);
init = 1;
}
return (((clockEnd - clockStart) * (U64)rate.numer) / ((U64)rate.denom))/1000ULL;
}
U64 UTIL_getSpanTimeNano(UTIL_time_t clockStart, UTIL_time_t clockEnd)
{
static mach_timebase_info_data_t rate;
static int init = 0;
if (!init) {
mach_timebase_info(&rate);
init = 1;
}
return ((clockEnd - clockStart) * (U64)rate.numer) / ((U64)rate.denom);
}
#elif (PLATFORM_POSIX_VERSION >= 200112L) \
&& (defined(__UCLIBC__) \
|| (defined(__GLIBC__) \
&& ((__GLIBC__ == 2 && __GLIBC_MINOR__ >= 17) \
|| (__GLIBC__ > 2))))
UTIL_time_t UTIL_getTime(void)
{
UTIL_time_t time;
if (clock_gettime(CLOCK_MONOTONIC, &time))
UTIL_DISPLAYLEVEL(1, "ERROR: Failed to get time\n"); /* we could also exit() */
return time;
}
UTIL_time_t UTIL_getSpanTime(UTIL_time_t begin, UTIL_time_t end)
{
UTIL_time_t diff;
if (end.tv_nsec < begin.tv_nsec) {
diff.tv_sec = (end.tv_sec - 1) - begin.tv_sec;
diff.tv_nsec = (end.tv_nsec + 1000000000ULL) - begin.tv_nsec;
} else {
diff.tv_sec = end.tv_sec - begin.tv_sec;
diff.tv_nsec = end.tv_nsec - begin.tv_nsec;
}
return diff;
}
U64 UTIL_getSpanTimeMicro(UTIL_time_t begin, UTIL_time_t end)
{
UTIL_time_t const diff = UTIL_getSpanTime(begin, end);
U64 micro = 0;
micro += 1000000ULL * diff.tv_sec;
micro += diff.tv_nsec / 1000ULL;
return micro;
}
U64 UTIL_getSpanTimeNano(UTIL_time_t begin, UTIL_time_t end)
{
UTIL_time_t const diff = UTIL_getSpanTime(begin, end);
U64 nano = 0;
nano += 1000000000ULL * diff.tv_sec;
nano += diff.tv_nsec;
return nano;
}
#else /* relies on standard C (note : clock_t measurements can be wrong when using multi-threading) */
UTIL_time_t UTIL_getTime(void) { return clock(); }
U64 UTIL_getSpanTimeMicro(UTIL_time_t clockStart, UTIL_time_t clockEnd) { return 1000000ULL * (clockEnd - clockStart) / CLOCKS_PER_SEC; }
U64 UTIL_getSpanTimeNano(UTIL_time_t clockStart, UTIL_time_t clockEnd) { return 1000000000ULL * (clockEnd - clockStart) / CLOCKS_PER_SEC; }
#endif
/* returns time span in microseconds */
U64 UTIL_clockSpanMicro(UTIL_time_t clockStart )
{
UTIL_time_t const clockEnd = UTIL_getTime();
return UTIL_getSpanTimeMicro(clockStart, clockEnd);
}
/* returns time span in microseconds */
U64 UTIL_clockSpanNano(UTIL_time_t clockStart )
{
UTIL_time_t const clockEnd = UTIL_getTime();
return UTIL_getSpanTimeNano(clockStart, clockEnd);
}
void UTIL_waitForNextTick(void)
{
UTIL_time_t const clockStart = UTIL_getTime();
UTIL_time_t clockEnd;
do {
clockEnd = UTIL_getTime();
} while (UTIL_getSpanTimeNano(clockStart, clockEnd) == 0);
}
/* count the number of physical cores */
#if defined(_WIN32) || defined(WIN32)
#include <windows.h>

46
programs/util.h
View File

@ -112,52 +112,6 @@ extern int g_utilDisplayLevel;
#define UTIL_DISPLAYLEVEL(l, ...) { if (g_utilDisplayLevel>=l) { UTIL_DISPLAY(__VA_ARGS__); } }
/*-****************************************
* Time functions
******************************************/
#if defined(_WIN32) /* Windows */
#define UTIL_TIME_INITIALIZER { { 0, 0 } }
typedef LARGE_INTEGER UTIL_time_t;
#elif defined(__APPLE__) && defined(__MACH__)
#include <mach/mach_time.h>
#define UTIL_TIME_INITIALIZER 0
typedef U64 UTIL_time_t;
#elif (PLATFORM_POSIX_VERSION >= 200112L) \
&& (defined(__UCLIBC__) \
|| (defined(__GLIBC__) \
&& ((__GLIBC__ == 2 && __GLIBC_MINOR__ >= 17) \
|| (__GLIBC__ > 2))))
#define UTIL_TIME_INITIALIZER { 0, 0 }
typedef struct timespec UTIL_freq_t;
typedef struct timespec UTIL_time_t;
UTIL_time_t UTIL_getSpanTime(UTIL_time_t begin, UTIL_time_t end);
#else /* relies on standard C (note : clock_t measurements can be wrong when using multi-threading) */
typedef clock_t UTIL_time_t;
#define UTIL_TIME_INITIALIZER 0
#endif
UTIL_time_t UTIL_getTime(void);
U64 UTIL_getSpanTimeMicro(UTIL_time_t clockStart, UTIL_time_t clockEnd);
U64 UTIL_getSpanTimeNano(UTIL_time_t clockStart, UTIL_time_t clockEnd);
#define SEC_TO_MICRO 1000000
/* returns time span in microseconds */
U64 UTIL_clockSpanMicro(UTIL_time_t clockStart);
/* returns time span in microseconds */
U64 UTIL_clockSpanNano(UTIL_time_t clockStart);
void UTIL_waitForNextTick(void);
/*-****************************************
* File functions
******************************************/

12
programs/zstdcli.c
View File

@ -242,18 +242,20 @@ static void errorOut(const char* msg)
* @return 1 if an overflow error occurs */
static int readU32FromCharChecked(const char** stringPtr, unsigned* value)
{
static unsigned const max = (((unsigned)(-1)) / 10) - 1;
unsigned result = 0;
while ((**stringPtr >='0') && (**stringPtr <='9')) {
if (result > max) return 1; // overflow error
result *= 10, result += **stringPtr - '0', (*stringPtr)++ ;
unsigned const max = (((unsigned)(-1)) / 10) - 1;
if (result > max) return 1; /* overflow error */
result *= 10;
result += (unsigned)(**stringPtr - '0');
(*stringPtr)++ ;
}
if ((**stringPtr=='K') || (**stringPtr=='M')) {
unsigned const maxK = ((unsigned)(-1)) >> 10;
if (result > maxK) return 1; // overflow error
if (result > maxK) return 1; /* overflow error */
result <<= 10;
if (**stringPtr=='M') {
if (result > maxK) return 1; // overflow error
if (result > maxK) return 1; /* overflow error */
result <<= 10;
}
(*stringPtr)++; /* skip `K` or `M` */

26
tests/Makefile
View File

@ -82,7 +82,7 @@ default: fullbench
@echo $(ZSTDMT_OBJECTS)
all: fullbench fuzzer zstreamtest paramgrill datagen decodecorpus roundTripCrash \
fullbench-lib
fullbench-lib poolTests
all32: fullbench32 fuzzer32 zstreamtest32
@ -132,18 +132,18 @@ fullbench fullbench32 : CPPFLAGS += $(MULTITHREAD_CPP)
fullbench fullbench32 : LDFLAGS += $(MULTITHREAD_LD)
fullbench fullbench32 : DEBUGFLAGS = -DNDEBUG # turn off assert() for speed measurements
fullbench fullbench32 : $(ZSTD_FILES)
fullbench fullbench32 : $(PRGDIR)/datagen.c $(PRGDIR)/util.c $(PRGDIR)/benchfn.c fullbench.c
fullbench fullbench32 : $(PRGDIR)/datagen.c $(PRGDIR)/util.c $(PRGDIR)/timefn.c $(PRGDIR)/benchfn.c fullbench.c
$(CC) $(FLAGS) $^ -o $@$(EXT)
fullbench-lib : CPPFLAGS += -DXXH_NAMESPACE=ZSTD_
fullbench-lib : zstd-staticLib
fullbench-lib : $(PRGDIR)/datagen.c $(PRGDIR)/util.c $(PRGDIR)/benchfn.c fullbench.c
fullbench-lib : $(PRGDIR)/datagen.c $(PRGDIR)/util.c $(PRGDIR)/timefn.c $(PRGDIR)/benchfn.c fullbench.c
$(CC) $(FLAGS) $(filter %.c,$^) -o $@$(EXT) $(ZSTDDIR)/libzstd.a
# note : broken : requires unavailable symbols
fullbench-dll : zstd-dll
fullbench-dll : LDFLAGS+= -L$(ZSTDDIR) -lzstd
fullbench-dll: $(PRGDIR)/datagen.c $(PRGDIR)/util.c $(PRGDIR)/benchfn.c fullbench.c
fullbench-dll: $(PRGDIR)/datagen.c $(PRGDIR)/util.c $(PRGDIR)/benchfn.c $(PRGDIR)/timefn.c fullbench.c
# $(CC) $(FLAGS) $(filter %.c,$^) -o $@$(EXT) -DZSTD_DLL_IMPORT=1 $(ZSTDDIR)/dll/libzstd.dll
$(CC) $(FLAGS) $(filter %.c,$^) -o $@$(EXT)
@ -152,32 +152,32 @@ fuzzer : LDFLAGS += $(MULTITHREAD_LD)
fuzzer32: CFLAGS += -m32
fuzzer : $(ZSTDMT_OBJECTS)
fuzzer32: $(ZSTD_FILES)
fuzzer fuzzer32 : $(ZDICT_FILES) $(PRGDIR)/util.c $(PRGDIR)/datagen.c fuzzer.c
fuzzer fuzzer32 : $(ZDICT_FILES) $(PRGDIR)/util.c $(PRGDIR)/timefn.c $(PRGDIR)/datagen.c fuzzer.c
$(CC) $(FLAGS) $^ -o $@$(EXT)
fuzzer-dll : zstd-dll
fuzzer-dll : LDFLAGS+= -L$(ZSTDDIR) -lzstd
fuzzer-dll : $(ZSTDDIR)/common/xxhash.c $(PRGDIR)/util.c $(PRGDIR)/datagen.c fuzzer.c
fuzzer-dll : $(ZSTDDIR)/common/xxhash.c $(PRGDIR)/util.c $(PRGDIR)/timefn.c $(PRGDIR)/datagen.c fuzzer.c
$(CC) $(CPPFLAGS) $(CFLAGS) $(filter %.c,$^) $(LDFLAGS) -o $@$(EXT)
zbufftest : CPPFLAGS += -I$(ZSTDDIR)/deprecated
zbufftest : CFLAGS += -Wno-deprecated-declarations # required to silence deprecation warnings
zbufftest : $(ZSTD_OBJECTS) $(ZBUFF_FILES) $(PRGDIR)/util.c $(PRGDIR)/datagen.c zbufftest.c
zbufftest : $(ZSTD_OBJECTS) $(ZBUFF_FILES) $(PRGDIR)/util.c $(PRGDIR)/timefn.c $(PRGDIR)/datagen.c zbufftest.c
$(CC) $(FLAGS) $^ -o $@$(EXT)
zbufftest32 : CPPFLAGS += -I$(ZSTDDIR)/deprecated
zbufftest32 : CFLAGS += -Wno-deprecated-declarations -m32
zbufftest32 : $(ZSTD_FILES) $(ZBUFF_FILES) $(PRGDIR)/util.c $(PRGDIR)/datagen.c zbufftest.c
zbufftest32 : $(ZSTD_FILES) $(ZBUFF_FILES) $(PRGDIR)/util.c $(PRGDIR)/timefn.c $(PRGDIR)/datagen.c zbufftest.c
$(CC) $(FLAGS) $^ -o $@$(EXT)
zbufftest-dll : zstd-dll
zbufftest-dll : CPPFLAGS += -I$(ZSTDDIR)/deprecated
zbufftest-dll : CFLAGS += -Wno-deprecated-declarations # required to silence deprecation warnings
zbufftest-dll : LDFLAGS+= -L$(ZSTDDIR) -lzstd
zbufftest-dll : $(ZSTDDIR)/common/xxhash.c $(PRGDIR)/util.c $(PRGDIR)/datagen.c zbufftest.c
zbufftest-dll : $(ZSTDDIR)/common/xxhash.c $(PRGDIR)/util.c $(PRGDIR)/timefn.c $(PRGDIR)/datagen.c zbufftest.c
$(CC) $(CPPFLAGS) $(CFLAGS) $(filter %.c,$^) $(LDFLAGS) -o $@$(EXT)
ZSTREAM_LOCAL_FILES := $(PRGDIR)/datagen.c $(PRGDIR)/util.c seqgen.c zstreamtest.c
ZSTREAM_LOCAL_FILES := $(PRGDIR)/datagen.c $(PRGDIR)/util.c $(PRGDIR)/timefn.c seqgen.c zstreamtest.c
ZSTREAM_PROPER_FILES := $(ZDICT_FILES) $(ZSTREAM_LOCAL_FILES)
ZSTREAMFILES := $(ZSTD_FILES) $(ZSTREAM_PROPER_FILES)
zstreamtest32 : CFLAGS += -m32
@ -203,7 +203,7 @@ zstreamtest-dll : $(ZSTREAM_LOCAL_FILES)
$(CC) $(CPPFLAGS) $(CFLAGS) $(filter %.c,$^) $(LDFLAGS) -o $@$(EXT)
paramgrill : DEBUGFLAGS = # turn off assert() by default for speed measurements
paramgrill : $(ZSTD_FILES) $(PRGDIR)/util.c $(PRGDIR)/benchfn.c $(PRGDIR)/benchzstd.c $(PRGDIR)/datagen.c paramgrill.c
paramgrill : $(ZSTD_FILES) $(PRGDIR)/util.c $(PRGDIR)/timefn.c $(PRGDIR)/benchfn.c $(PRGDIR)/benchzstd.c $(PRGDIR)/datagen.c paramgrill.c
$(CC) $(FLAGS) $^ -lm -o $@$(EXT)
datagen : $(PRGDIR)/datagen.c datagencli.c
@ -222,7 +222,7 @@ legacy : CPPFLAGS += -I$(ZSTDDIR)/legacy -DZSTD_LEGACY_SUPPORT=4
legacy : $(ZSTD_FILES) $(wildcard $(ZSTDDIR)/legacy/*.c) legacy.c
$(CC) $(FLAGS) $^ -o $@$(EXT)
decodecorpus : $(filter-out zstdc_zstd_compress.o, $(ZSTD_OBJECTS)) $(ZDICT_FILES) $(PRGDIR)/util.c decodecorpus.c
decodecorpus : $(filter-out zstdc_zstd_compress.o, $(ZSTD_OBJECTS)) $(ZDICT_FILES) $(PRGDIR)/util.c $(PRGDIR)/timefn.c decodecorpus.c
$(CC) $(FLAGS) $^ -o $@$(EXT) -lm
symbols : symbols.c zstd-dll
@ -233,7 +233,7 @@ else
$(CC) $(FLAGS) $< -o $@$(EXT) -Wl,-rpath=$(ZSTDDIR) $(ZSTDDIR)/libzstd.so # broken on Mac
endif
poolTests : $(PRGDIR)/util.c poolTests.c $(ZSTDDIR)/common/pool.c $(ZSTDDIR)/common/threading.c $(ZSTDDIR)/common/zstd_common.c $(ZSTDDIR)/common/error_private.c
poolTests : $(PRGDIR)/util.c $(PRGDIR)/timefn.c poolTests.c $(ZSTDDIR)/common/pool.c $(ZSTDDIR)/common/threading.c $(ZSTDDIR)/common/zstd_common.c $(ZSTDDIR)/common/error_private.c
$(CC) $(FLAGS) $(MULTITHREAD) $^ -o $@$(EXT)
.PHONY: versionsTest

1
tests/decodecorpus.c
View File

@ -16,6 +16,7 @@
#include <string.h>
#include "util.h"
#include "timefn.h" /* UTIL_clockSpanMicro, SEC_TO_MICRO, UTIL_TIME_INITIALIZER */
#include "zstd.h"
#include "zstd_internal.h"
#include "mem.h"

54
tests/fullbench.c
View File

@ -17,6 +17,7 @@
#include <stdio.h> /* fprintf, fopen, ftello64 */
#include <assert.h> /* assert */
#include "timefn.h" /* UTIL_clockSpanNano, UTIL_getTime */
#include "mem.h" /* U32 */
#ifndef ZSTD_DLL_IMPORT
#include "zstd_internal.h" /* ZSTD_decodeSeqHeaders, ZSTD_blockHeaderSize, blockType_e, KB, MB */
@ -67,12 +68,6 @@ static const size_t g_sampleSize = 10000000;
static unsigned g_nbIterations = NBLOOPS;
static double g_compressibility = COMPRESSIBILITY_DEFAULT;
static void BMK_SetNbIterations(unsigned nbLoops)
{
g_nbIterations = nbLoops;
DISPLAY("- %i iterations -\n", g_nbIterations);
}
/*_*******************************************************
* Private functions
@ -316,9 +311,9 @@ static size_t local_ZSTD_decompressContinue(const void* src, size_t srcSize,
/*_*******************************************************
* Bench functions
*********************************************************/
static size_t benchMem(unsigned benchNb,
const void* src, size_t srcSize,
int cLevel, ZSTD_compressionParameters cparams)
static int benchMem(unsigned benchNb,
const void* src, size_t srcSize,
int cLevel, ZSTD_compressionParameters cparams)
{
size_t dstBuffSize = ZSTD_compressBound(srcSize);
BYTE* dstBuff;
@ -395,22 +390,22 @@ static size_t benchMem(unsigned benchNb,
cparams->minMatch, cparams->targetLength, cparams->strategy); */
ZSTD_CCtx_setParameter(g_zcc, ZSTD_c_compressionLevel, cLevel);
ZSTD_CCtx_setParameter(g_zcc, ZSTD_c_windowLog, cparams.windowLog);
ZSTD_CCtx_setParameter(g_zcc, ZSTD_c_hashLog, cparams.hashLog);
ZSTD_CCtx_setParameter(g_zcc, ZSTD_c_chainLog, cparams.chainLog);
ZSTD_CCtx_setParameter(g_zcc, ZSTD_c_searchLog, cparams.searchLog);
ZSTD_CCtx_setParameter(g_zcc, ZSTD_c_minMatch, cparams.minMatch);
ZSTD_CCtx_setParameter(g_zcc, ZSTD_c_targetLength, cparams.targetLength);
ZSTD_CCtx_setParameter(g_zcc, ZSTD_c_windowLog, (int)cparams.windowLog);
ZSTD_CCtx_setParameter(g_zcc, ZSTD_c_hashLog, (int)cparams.hashLog);
ZSTD_CCtx_setParameter(g_zcc, ZSTD_c_chainLog, (int)cparams.chainLog);
ZSTD_CCtx_setParameter(g_zcc, ZSTD_c_searchLog, (int)cparams.searchLog);
ZSTD_CCtx_setParameter(g_zcc, ZSTD_c_minMatch, (int)cparams.minMatch);
ZSTD_CCtx_setParameter(g_zcc, ZSTD_c_targetLength, (int)cparams.targetLength);
ZSTD_CCtx_setParameter(g_zcc, ZSTD_c_strategy, cparams.strategy);
ZSTD_CCtx_setParameter(g_cstream, ZSTD_c_compressionLevel, cLevel);
ZSTD_CCtx_setParameter(g_cstream, ZSTD_c_windowLog, cparams.windowLog);
ZSTD_CCtx_setParameter(g_cstream, ZSTD_c_hashLog, cparams.hashLog);
ZSTD_CCtx_setParameter(g_cstream, ZSTD_c_chainLog, cparams.chainLog);
ZSTD_CCtx_setParameter(g_cstream, ZSTD_c_searchLog, cparams.searchLog);
ZSTD_CCtx_setParameter(g_cstream, ZSTD_c_minMatch, cparams.minMatch);
ZSTD_CCtx_setParameter(g_cstream, ZSTD_c_targetLength, cparams.targetLength);
ZSTD_CCtx_setParameter(g_cstream, ZSTD_c_windowLog, (int)cparams.windowLog);
ZSTD_CCtx_setParameter(g_cstream, ZSTD_c_hashLog, (int)cparams.hashLog);
ZSTD_CCtx_setParameter(g_cstream, ZSTD_c_chainLog, (int)cparams.chainLog);
ZSTD_CCtx_setParameter(g_cstream, ZSTD_c_searchLog, (int)cparams.searchLog);
ZSTD_CCtx_setParameter(g_cstream, ZSTD_c_minMatch, (int)cparams.minMatch);
ZSTD_CCtx_setParameter(g_cstream, ZSTD_c_targetLength, (int)cparams.targetLength);
ZSTD_CCtx_setParameter(g_cstream, ZSTD_c_strategy, cparams.strategy);
/* Preparation */
@ -469,8 +464,9 @@ static size_t benchMem(unsigned benchNb,
iend = ip + ZSTD_blockHeaderSize + cBlockSize; /* End of first block */
ip += ZSTD_blockHeaderSize; /* skip block header */
ZSTD_decompressBegin(g_zdc);
ip += ZSTD_decodeLiteralsBlock(g_zdc, ip, iend-ip); /* skip literal segment */
g_cSize = iend-ip;
assert(iend > ip);
ip += ZSTD_decodeLiteralsBlock(g_zdc, ip, (size_t)(iend-ip)); /* skip literal segment */
g_cSize = (size_t)(iend-ip);
memcpy(buff2, ip, g_cSize); /* copy rest of block (it starts by SeqHeader) */
srcSize = srcSize > 128 KB ? 128 KB : srcSize; /* speed relative to block */
break;
@ -501,7 +497,7 @@ static size_t benchMem(unsigned benchNb,
BMK_benchParams_t bp;
BMK_runTime_t bestResult;
bestResult.sumOfReturn = 0;
bestResult.nanoSecPerRun = (unsigned long long)(-1LL);
bestResult.nanoSecPerRun = (double)(-1);
assert(tfs != NULL);
bp.benchFn = benchFunction;
@ -654,7 +650,9 @@ static unsigned readU32FromChar(const char** stringPtr)
while ((**stringPtr >='0') && (**stringPtr <='9')) {
unsigned const max = (((unsigned)(-1)) / 10) - 1;
if (result > max) ERROR_OUT(errorMsg);
result *= 10, result += **stringPtr - '0', (*stringPtr)++ ;
result *= 10;
result += (unsigned)(**stringPtr - '0');
(*stringPtr)++ ;
}
if ((**stringPtr=='K') || (**stringPtr=='M')) {
unsigned const maxK = ((unsigned)(-1)) >> 10;
@ -671,7 +669,7 @@ static unsigned readU32FromChar(const char** stringPtr)
return result;
}
static unsigned longCommandWArg(const char** stringPtr, const char* longCommand)
static int longCommandWArg(const char** stringPtr, const char* longCommand)
{
size_t const comSize = strlen(longCommand);
int const result = !strncmp(*stringPtr, longCommand, comSize);
@ -772,7 +770,7 @@ int main(int argc, const char** argv)
/* Modify Nb Iterations */
case 'i':
argument++;
BMK_SetNbIterations((int)readU32FromChar(&argument));
g_nbIterations = readU32FromChar(&argument);
break;
/* Select compressibility of synthetic sample */
@ -782,7 +780,7 @@ int main(int argc, const char** argv)
break;
case 'l':
argument++;
cLevel = readU32FromChar(&argument);
cLevel = (int)readU32FromChar(&argument);
cparams = ZSTD_getCParams(cLevel, 0, 0);
break;

1
tests/fuzzer.c
View File

@ -38,6 +38,7 @@
#define XXH_STATIC_LINKING_ONLY /* XXH64_state_t */
#include "xxhash.h" /* XXH64 */
#include "util.h"
#include "timefn.h" /* SEC_TO_MICRO, UTIL_time_t, UTIL_TIME_INITIALIZER, UTIL_clockSpanMicro, UTIL_getTime */
/*-************************************

45
tests/paramgrill.c
View File

@ -19,6 +19,7 @@
#include <math.h> /* log */
#include <assert.h>
#include "timefn.h" /* SEC_TO_MICRO, UTIL_time_t, UTIL_clockSpanMicro, UTIL_clockSpanNano, UTIL_getTime */
#include "mem.h"
#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_parameters, ZSTD_estimateCCtxSize */
#include "zstd.h"
@ -141,7 +142,7 @@ static const char* g_shortParamNames[NUM_PARAMS] =
/* maps value from { 0 to rangetable[param] - 1 } to valid paramvalues */
static U32 rangeMap(varInds_t param, int ind)
{
ind = MAX(MIN(ind, (int)rangetable[param] - 1), 0);
U32 const uind = (U32)MAX(MIN(ind, (int)rangetable[param] - 1), 0);
switch(param) {
case wlog_ind: /* using default: triggers -Wswitch-enum */
case clog_ind:
@ -149,11 +150,11 @@ static U32 rangeMap(varInds_t param, int ind)
case slog_ind:
case mml_ind:
case strt_ind:
return mintable[param] + ind;
return mintable[param] + uind;
case tlen_ind:
return tlen_table[ind];
return tlen_table[uind];
case fadt_ind: /* 0, 1, 2 -> -1, 0, 1 */
return ind - 1;
return uind - 1;
case NUM_PARAMS:
default:;
}
@ -173,7 +174,7 @@ static int invRangeMap(varInds_t param, U32 value)
case slog_ind:
case mml_ind:
case strt_ind:
return value - mintable[param];
return (int)(value - mintable[param]);
case tlen_ind: /* bin search */
{
int lo = 0;
@ -493,13 +494,15 @@ static void
paramVariation(paramValues_t* ptr, memoTable_t* mtAll, const U32 nbChanges)
{
paramValues_t p;
U32 validated = 0;
int validated = 0;
while (!validated) {
U32 i;
p = *ptr;
for (i = 0 ; i < nbChanges ; i++) {
const U32 changeID = (U32)FUZ_rand(&g_rand) % (mtAll[p.vals[strt_ind]].varLen << 1);
paramVaryOnce(mtAll[p.vals[strt_ind]].varArray[changeID >> 1], ((changeID & 1) << 1) - 1, &p);
paramVaryOnce(mtAll[p.vals[strt_ind]].varArray[changeID >> 1],
(int)((changeID & 1) << 1) - 1,
&p);
}
validated = paramValid(p);
}
@ -511,7 +514,7 @@ static paramValues_t randomParams(void)
{
varInds_t v; paramValues_t p;
for(v = 0; v < NUM_PARAMS; v++) {
p.vals[v] = rangeMap(v, FUZ_rand(&g_rand) % rangetable[v]);
p.vals[v] = rangeMap(v, (int)(FUZ_rand(&g_rand) % rangetable[v]));
}
return p;
}
@ -1638,7 +1641,7 @@ BMK_benchMemInvertible( buffers_t buf, contexts_t ctx,
return bOut;
}
{ BMK_runTime_t const rResult = BMK_extract_runTime(cOutcome);
bResult.cSpeed = (srcSize * TIMELOOP_NANOSEC) / rResult.nanoSecPerRun;
bResult.cSpeed = (unsigned long long)((double)srcSize * TIMELOOP_NANOSEC / rResult.nanoSecPerRun);
bResult.cSize = rResult.sumOfReturn;
}
compressionCompleted = BMK_isCompleted_TimedFn(timeStateCompress);
@ -1656,7 +1659,7 @@ BMK_benchMemInvertible( buffers_t buf, contexts_t ctx,
return bOut;
}
{ BMK_runTime_t const rResult = BMK_extract_runTime(dOutcome);
bResult.dSpeed = (srcSize * TIMELOOP_NANOSEC) / rResult.nanoSecPerRun;
bResult.dSpeed = (unsigned long long)((double)srcSize * TIMELOOP_NANOSEC / rResult.nanoSecPerRun);
}
decompressionCompleted = BMK_isCompleted_TimedFn(timeStateDecompress);
}
@ -2199,7 +2202,9 @@ static winnerInfo_t climbOnce(const constraint_t target,
for (offset = -1; offset <= 1; offset += 2) {
CHECKTIME(winnerInfo);
candidateInfo.params = cparam;
paramVaryOnce(mtAll[cparam.vals[strt_ind]].varArray[i], offset, &candidateInfo.params);
paramVaryOnce(mtAll[cparam.vals[strt_ind]].varArray[i],
offset,
&candidateInfo.params);
if(paramValid(candidateInfo.params)) {
int res;
@ -2351,7 +2356,7 @@ static int nextStrategy(const int currentStrategy, const int bestStrategy)
* cLevel - compression level to exceed (all solutions must be > lvl in cSpeed + ratio)
*/
static int g_maxTries = 5;
static unsigned g_maxTries = 5;
#define TRY_DECAY 1
static int
@ -2561,7 +2566,7 @@ _cleanUp:
* @return 0 and doesn't modify *stringPtr otherwise.
* from zstdcli.c
*/
static unsigned longCommandWArg(const char** stringPtr, const char* longCommand)
static int longCommandWArg(const char** stringPtr, const char* longCommand)
{
size_t const comSize = strlen(longCommand);
int const result = !strncmp(*stringPtr, longCommand, comSize);
@ -2588,7 +2593,10 @@ static unsigned readU32FromChar(const char** stringPtr)
while ((**stringPtr >='0') && (**stringPtr <='9')) {
unsigned const max = (((unsigned)(-1)) / 10) - 1;
if (result > max) errorOut(errorMsg);
result *= 10, result += **stringPtr - '0', (*stringPtr)++ ;
result *= 10;
assert(**stringPtr >= '0');
result += (unsigned)(**stringPtr - '0');
(*stringPtr)++ ;
}
if ((**stringPtr=='K') || (**stringPtr=='M')) {
unsigned const maxK = ((unsigned)(-1)) >> 10;
@ -2726,7 +2734,7 @@ int main(int argc, const char** argv)
PARSE_SUB_ARGS("strict=", "stc=", g_strictness);
PARSE_SUB_ARGS("maxTries=", "tries=", g_maxTries);
PARSE_SUB_ARGS("memoLimitLog=", "memLog=", memoTableLog);
if (longCommandWArg(&argument, "level=") || longCommandWArg(&argument, "lvl=")) { cLevelOpt = readU32FromChar(&argument); g_optmode = 1; if (argument[0]==',') { argument++; continue; } else break; }
if (longCommandWArg(&argument, "level=") || longCommandWArg(&argument, "lvl=")) { cLevelOpt = (int)readU32FromChar(&argument); g_optmode = 1; if (argument[0]==',') { argument++; continue; } else break; }
if (longCommandWArg(&argument, "speedForRatio=") || longCommandWArg(&argument, "speedRatio=")) { g_ratioMultiplier = readDoubleFromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; }
DISPLAY("invalid optimization parameter \n");
@ -2743,7 +2751,7 @@ int main(int argc, const char** argv)
g_singleRun = 1;
for ( ; ;) {
if(parse_params(&argument, &g_params)) { if(argument[0] == ',') { argument++; continue; } else break; }
if (longCommandWArg(&argument, "level=") || longCommandWArg(&argument, "lvl=")) { cLevelRun = readU32FromChar(&argument); g_params = emptyParams(); if (argument[0]==',') { argument++; continue; } else break; }
if (longCommandWArg(&argument, "level=") || longCommandWArg(&argument, "lvl=")) { cLevelRun = (int)readU32FromChar(&argument); g_params = emptyParams(); if (argument[0]==',') { argument++; continue; } else break; }
DISPLAY("invalid compression parameter \n");
return 1;
@ -2855,7 +2863,7 @@ int main(int argc, const char** argv)
continue;
case 'L':
{ argument++;
cLevelRun = readU32FromChar(&argument);
cLevelRun = (int)readU32FromChar(&argument);
g_params = emptyParams();
continue;
}
@ -2944,7 +2952,8 @@ int main(int argc, const char** argv)
}
} else {
if (g_optimizer) {
result = optimizeForSize(argv+filenamesStart, argc-filenamesStart, dictFileName, target, paramTarget, cLevelOpt, cLevelRun, memoTableLog);
assert(filenamesStart < argc);
result = optimizeForSize(argv+filenamesStart, (size_t)(argc-filenamesStart), dictFileName, target, paramTarget, cLevelOpt, cLevelRun, memoTableLog);
} else {
result = benchFiles(argv+filenamesStart, argc-filenamesStart, dictFileName, cLevelRun);
}

19
tests/poolTests.c
View File

@ -12,6 +12,7 @@
#include "pool.h"
#include "threading.h"
#include "util.h"
#include "timefn.h"
#include <stddef.h>
#include <stdio.h>
@ -25,25 +26,27 @@
#define ASSERT_EQ(lhs, rhs) ASSERT_TRUE((lhs) == (rhs))
struct data {
pthread_mutex_t mutex;
ZSTD_pthread_mutex_t mutex;
unsigned data[16];
size_t i;
};
static void fn(void *opaque) {
static void fn(void *opaque)
{
struct data *data = (struct data *)opaque;
ZSTD_pthread_mutex_lock(&data->mutex);
data->data[data->i] = data->i;
data->data[data->i] = (unsigned)(data->i);
++data->i;
ZSTD_pthread_mutex_unlock(&data->mutex);
}
static int testOrder(size_t numThreads, size_t queueSize) {
static int testOrder(size_t numThreads, size_t queueSize)
{
struct data data;
POOL_ctx *ctx = POOL_create(numThreads, queueSize);
POOL_ctx* const ctx = POOL_create(numThreads, queueSize);
ASSERT_TRUE(ctx);
data.i = 0;
ZSTD_pthread_mutex_init(&data.mutex, NULL);
(void)ZSTD_pthread_mutex_init(&data.mutex, NULL);
{ size_t i;
for (i = 0; i < 16; ++i) {
POOL_add(ctx, &fn, &data);
@ -71,7 +74,7 @@ static void waitFn(void *opaque) {
/* Tests for deadlock */
static int testWait(size_t numThreads, size_t queueSize) {
struct data data;
POOL_ctx *ctx = POOL_create(numThreads, queueSize);
POOL_ctx* const ctx = POOL_create(numThreads, queueSize);
ASSERT_TRUE(ctx);
{ size_t i;
for (i = 0; i < 16; ++i) {
@ -93,7 +96,7 @@ typedef struct {
} poolTest_t;
static void waitLongFn(void *opaque) {
poolTest_t* test = (poolTest_t*) opaque;
poolTest_t* const test = (poolTest_t*) opaque;
UTIL_sleepMilli(10);
ZSTD_pthread_mutex_lock(&test->mut);
test->val = test->val + 1;

1
tests/zstreamtest.c
View File

@ -37,6 +37,7 @@
#include "xxhash.h" /* XXH64_* */
#include "seqgen.h"
#include "util.h"
#include "timefn.h" /* UTIL_time_t, UTIL_clockSpanMicro, UTIL_getTime */
/*-************************************

2
zlibWrapper/Makefile
View File

@ -88,7 +88,7 @@ fitblk: $(EXAMPLE_PATH)/fitblk.o $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.o $(ZSTDLI
fitblk_zstd: $(EXAMPLE_PATH)/fitblk.o $(ZLIBWRAPPER_PATH)/zstdTurnedOn_zlibwrapper.o $(ZSTDLIBRARY)
$(CC) $(CFLAGS) $(CPPFLAGS) $(LDFLAGS) $^ $(ZLIB_LIBRARY) -o $@
zwrapbench: $(EXAMPLE_PATH)/zwrapbench.o $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.o $(PROGRAMS_PATH)/util.o $(PROGRAMS_PATH)/datagen.o $(ZSTDLIBRARY)
zwrapbench: $(EXAMPLE_PATH)/zwrapbench.o $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.o $(PROGRAMS_PATH)/util.o $(PROGRAMS_PATH)/timefn.o $(PROGRAMS_PATH)/datagen.o $(ZSTDLIBRARY)
$(CC) $(CFLAGS) $(CPPFLAGS) $(LDFLAGS) $^ $(ZLIB_LIBRARY) -o $@

1
zlibWrapper/examples/zwrapbench.c
View File

@ -19,6 +19,7 @@
#include <ctype.h> /* toupper */
#include <errno.h> /* errno */
#include "timefn.h" /* UTIL_time_t, UTIL_getTime, UTIL_clockSpanMicro, UTIL_waitForNextTick */
#include "mem.h"
#define ZSTD_STATIC_LINKING_ONLY
#include "zstd.h"