facebook
/
zstd
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

nbThreads instead of numThreads

dev
senhuang42 2020-09-09 12:35:40 -04:00
parent 0a170b20a8
commit a71963c7b8
4 changed files with 29 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
programs/README.md
View File

@ -204,19 +204,19 @@ Benchmark arguments :
### Passing parameters through Environment Variables
`ZSTD_CLEVEL` can be used to modify the default compression level of `zstd`
(usually set to `3`) to another value between 1 and 19 (the "normal" range).
`ZSTD_NUMTHREADS` can be used to specify number of threads that `zstd` will use during compression, which by default is `1`.
`ZSTD_NBTHREADS` can be used to specify number of threads that `zstd` will use during compression, which by default is `1`.
This functionality only exists when `zstd` is compiled with multithread support.
The max # of threads is capped at: `ZSTDMT_NBWORKERS_MAX==200`.
This functionality can be useful when `zstd` CLI is invoked in a way that doesn't allow passing arguments.
One such scenario is `tar --zstd`.
As `ZSTD_CLEVEL` and `ZSTD_NUMTHREADS` only replace the default compression level
As `ZSTD_CLEVEL` and `ZSTD_NBTHREADS` only replace the default compression level
and number of threads, respectively, they can both be overridden by corresponding command line arguments:
`-#` for compression level and `-T#` for number of threads.
There is no "generic" way to pass "any kind of parameter" to `zstd` in a pass-through manner.
Using environment variables for this purpose has security implications.
Therefore, this avenue is intentionally restricted and only supports `ZSTD_CLEVEL` and `ZSTD_NUMTHREADS`.
Therefore, this avenue is intentionally restricted and only supports `ZSTD_CLEVEL` and `ZSTD_NBTHREADS`.
### Long distance matching mode
The long distance matching mode, enabled with `--long`, is designed to improve

8
programs/zstd.1.md
View File

@ -270,16 +270,16 @@ the last one takes effect.
Using environment variables to set parameters has security implications.
Therefore, this avenue is intentionally restricted.
Only `ZSTD_CLEVEL` and `ZSTD_NUMTHREADS` are currently supported.
Only `ZSTD_CLEVEL` and `ZSTD_NBTHREADS` are currently supported.
They set the compression level and number of threads to use during compression, respectively.
`ZSTD_CLEVEL` can be used to set the level between 1 and 19 (the "normal" range).
If the value of `ZSTD_CLEVEL` is not a valid integer, it will be ignored with a warning message.
`ZSTD_CLEVEL` just replaces the default compression level (`3`).
`ZSTD_NUMTHREADS` can be used to set the number of threads `zstd` will attempt to use during compression.
If the value of `ZSTD_NUMTHREADS` is not a valid unsigned integer, it will be ignored with a warning message.
'ZSTD_NUMTHREADS` has a default value of (`1`), and is capped at ZSTDMT_NBWORKERS_MAX==200. `zstd` must be
`ZSTD_NBTHREADS` can be used to set the number of threads `zstd` will attempt to use during compression.
If the value of `ZSTD_NBTHREADS` is not a valid unsigned integer, it will be ignored with a warning message.
'ZSTD_NBTHREADS` has a default value of (`1`), and is capped at ZSTDMT_NBWORKERS_MAX==200. `zstd` must be
compiled with multithread support for this to have any effect.
They can both be overridden by corresponding command line arguments:

26
programs/zstdcli.c
View File

@ -20,8 +20,8 @@
# define ZSTDCLI_CLEVEL_MAX 19 /* without using --ultra */
#endif
#ifndef ZSTDCLI_NUMTHREADS_DEFAULT
# define ZSTDCLI_NUMTHREADS_DEFAULT 1
#ifndef ZSTDCLI_NBTHREADS_DEFAULT
# define ZSTDCLI_NBTHREADS_DEFAULT 1
#endif
/*-************************************
@ -588,7 +588,7 @@ static void printVersion(void)
/* Environment variables for parameter setting */
#define ENV_CLEVEL "ZSTD_CLEVEL"
#define ENV_NUMTHREADS "ZSTD_NUMTHREADS" /* takes lower precedence than directly specifying -T# in the CLI */
#define ENV_NBTHREADS "ZSTD_NBTHREADS" /* takes lower precedence than directly specifying -T# in the CLI */
/* pick up environment variable */
static int init_cLevel(void) {
@ -619,23 +619,23 @@ static int init_cLevel(void) {
}
#ifdef ZSTD_MULTITHREAD
static unsigned init_numThreads(void) {
const char* const env = getenv(ENV_NUMTHREADS);
static unsigned init_nbThreads(void) {
const char* const env = getenv(ENV_NBTHREADS);
if (env != NULL) {
const char* ptr = env;
if ((*ptr>='0') && (*ptr<='9')) {
unsigned numThreads;
if (readU32FromCharChecked(&ptr, &numThreads)) {
DISPLAYLEVEL(2, "Ignore environment variable setting %s=%s: numeric value too large \n", ENV_NUMTHREADS, env);
return ZSTDCLI_NUMTHREADS_DEFAULT;
unsigned nbThreads;
if (readU32FromCharChecked(&ptr, &nbThreads)) {
DISPLAYLEVEL(2, "Ignore environment variable setting %s=%s: numeric value too large \n", ENV_NBTHREADS, env);
return ZSTDCLI_NBTHREADS_DEFAULT;
} else if (*ptr == 0) {
return numThreads;
return nbThreads;
}
}
DISPLAYLEVEL(2, "Ignore environment variable setting %s=%s: not a valid unsigned value \n", ENV_NUMTHREADS, env);
DISPLAYLEVEL(2, "Ignore environment variable setting %s=%s: not a valid unsigned value \n", ENV_NBTHREADS, env);
}
return ZSTDCLI_NUMTHREADS_DEFAULT;
return ZSTDCLI_NBTHREADS_DEFAULT;
}
#endif
@ -745,7 +745,7 @@ int main(int const argCount, const char* argv[])
if ((filenames==NULL) || (file_of_names==NULL)) { DISPLAY("zstd: allocation error \n"); exit(1); }
programName = lastNameFromPath(programName);
#ifdef ZSTD_MULTITHREAD
nbWorkers = init_numThreads();
nbWorkers = init_nbThreads();
#endif
/* preset behaviors */

18
tests/playTests.sh
View File

@ -1181,15 +1181,15 @@ then
println "\n===> zstdmt environment variable tests "
echo "multifoo" >> mt_tmp
ZSTD_NUMTHREADS=-3 zstd -f mt_tmp # negative value, warn and revert to default setting
ZSTD_NUMTHREADS='' zstd -f mt_tmp # empty env var, warn and revert to default setting
ZSTD_NUMTHREADS=- zstd -f mt_tmp # malformed env var, warn and revert to default setting
ZSTD_NUMTHREADS=a zstd -f mt_tmp # malformed env var, warn and revert to default setting
ZSTD_NUMTHREADS=+a zstd -f mt_tmp # malformed env var, warn and revert to default setting
ZSTD_NUMTHREADS=3a7 zstd -f mt_tmp # malformed env var, warn and revert to default setting
ZSTD_NUMTHREADS=50000000000 zstd -f mt_tmp # numeric value too large, warn and revert to default setting=
ZSTD_NUMTHREADS=2 zstd -f mt_tmp # correct usage
ZSTD_NUMTHREADS=1 zstd -f mt_tmp # correct usage: single thread
ZSTD_NBTHREADS=-3 zstd -f mt_tmp # negative value, warn and revert to default setting
ZSTD_NBTHREADS='' zstd -f mt_tmp # empty env var, warn and revert to default setting
ZSTD_NBTHREADS=- zstd -f mt_tmp # malformed env var, warn and revert to default setting
ZSTD_NBTHREADS=a zstd -f mt_tmp # malformed env var, warn and revert to default setting
ZSTD_NBTHREADS=+a zstd -f mt_tmp # malformed env var, warn and revert to default setting
ZSTD_NBTHREADS=3a7 zstd -f mt_tmp # malformed env var, warn and revert to default setting
ZSTD_NBTHREADS=50000000000 zstd -f mt_tmp # numeric value too large, warn and revert to default setting=
ZSTD_NBTHREADS=2 zstd -f mt_tmp # correct usage
ZSTD_NBTHREADS=1 zstd -f mt_tmp # correct usage: single thread
rm mt_tmp*
println "\n===> ovLog tests "