Skip to content

Commit

Permalink
Document changes to benchmarking and print number of threads for disp…
Browse files Browse the repository at this point in the history
…lay level >= 4
  • Loading branch information
daniellerozenblit committed Dec 11, 2024
1 parent 20f9e1e commit 180a66c
Show file tree
Hide file tree
Showing 2 changed files with 17 additions and 10 deletions.
5 changes: 4 additions & 1 deletion programs/zstd.1.md
Original file line number Diff line number Diff line change
Expand Up @@ -343,7 +343,7 @@ If the value of `ZSTD_CLEVEL` is not a valid integer, it will be ignored with a

`ZSTD_NBTHREADS` can be used to set the number of threads `zstd` will attempt to use during compression.
If the value of `ZSTD_NBTHREADS` is not a valid unsigned integer, it will be ignored with a warning message.
`ZSTD_NBTHREADS` has a default value of (`min(4, nbCores/4`), and is capped at ZSTDMT_NBWORKERS_MAX==200.
`ZSTD_NBTHREADS` has a default value of (`min(4, nbCores/4)`), and is capped at ZSTDMT_NBWORKERS_MAX==200.
`zstd` must be compiled with multithread support for this variable to have any effect.

They can both be overridden by corresponding command line arguments:
Expand Down Expand Up @@ -664,9 +664,12 @@ BENCHMARK
The `zstd` CLI provides a benchmarking mode that can be used to easily find suitable compression parameters, or alternatively to benchmark a computer's performance.
`zstd -b [FILE(s)]` will benchmark `zstd` for both compression and decompression using default compression level.
Note that results are very dependent on the content being compressed.

It's possible to pass multiple files to the benchmark, and even a directory with `-r DIRECTORY`.
When no `FILE` is provided, the benchmark will use a procedurally generated `lorem ipsum` text.

Benchmarking will employ `min(4, nbCores/4)` worker threads by default in order to match the behavior of the normal CLI I/O.

* `-b#`:
benchmark file(s) using compression level #
* `-e#`:
Expand Down
22 changes: 13 additions & 9 deletions programs/zstdcli.c
Original file line number Diff line number Diff line change
Expand Up @@ -21,8 +21,7 @@
#endif

#ifndef ZSTDCLI_NBTHREADS_DEFAULT
#define ZSTDCLI_NBTHREADS_DEFAULT \
UTIL_countLogicalCores() / 4 > 4 ? 4 : UTIL_countLogicalCores() / 4
#define ZSTDCLI_NBTHREADS_DEFAULT (UTIL_countLogicalCores() / 4 > 4 ? 4 : UTIL_countLogicalCores() / 4)
#endif

/*-************************************
Expand Down Expand Up @@ -761,7 +760,7 @@ static int init_cLevel(void) {
}

#ifdef ZSTD_MULTITHREAD
static unsigned init_nbThreads(void) {
static unsigned default_nbThreads(void) {
const char* const env = getenv(ENV_NBTHREADS);
if (env != NULL) {
const char* ptr = env;
Expand Down Expand Up @@ -856,7 +855,7 @@ int main(int argCount, const char* argv[])
ZSTD_paramSwitch_e mmapDict=ZSTD_ps_auto;
ZSTD_paramSwitch_e useRowMatchFinder = ZSTD_ps_auto;
FIO_compressionType_t cType = FIO_zstdCompression;
unsigned nbWorkers = 0;
int nbWorkers = -1; /* -1 means unset */
double compressibility = -1.0; /* lorem ipsum generator */
unsigned bench_nbSeconds = 3; /* would be better if this value was synchronized from bench */
size_t blockSize = 0;
Expand Down Expand Up @@ -897,17 +896,13 @@ int main(int argCount, const char* argv[])
#endif
ZSTD_paramSwitch_e literalCompressionMode = ZSTD_ps_auto;


/* init */
checkLibVersion();
(void)recursive; (void)cLevelLast; /* not used when ZSTD_NOBENCH set */
(void)memLimit;
assert(argCount >= 1);
if ((filenames==NULL) || (file_of_names==NULL)) { DISPLAYLEVEL(1, "zstd: allocation error \n"); exit(1); }
programName = lastNameFromPath(programName);
#ifdef ZSTD_MULTITHREAD
nbWorkers = init_nbThreads();
#endif

/* preset behaviors */
if (exeNameMatch(programName, ZSTD_ZSTDMT)) nbWorkers=0, singleThread=0;
Expand Down Expand Up @@ -1299,7 +1294,7 @@ int main(int argCount, const char* argv[])
DISPLAYLEVEL(3, WELCOME_MESSAGE);

#ifdef ZSTD_MULTITHREAD
if ((operation==zom_decompress) && (!singleThread) && (nbWorkers > 1)) {
if ((operation==zom_decompress) && (nbWorkers > 1)) {
DISPLAYLEVEL(2, "Warning : decompression does not support multi-threading\n");
}
if ((nbWorkers==0) && (!singleThread)) {
Expand All @@ -1312,6 +1307,15 @@ int main(int argCount, const char* argv[])
DISPLAYLEVEL(3, "Note: %d physical core(s) detected \n", nbWorkers);
}
}
// Resolve to default if nbWorkers is still unset
if (nbWorkers == -1) {
if (operation == zom_decompress) {
nbWorkers = 1;
} else {
nbWorkers = default_nbThreads();
}
}
DISPLAYLEVEL(4, "Compressing with %u worker threads \n", nbWorkers);
#else
(void)singleThread; (void)nbWorkers; (void)defaultLogicalCores;
#endif
Expand Down

0 comments on commit 180a66c

Please sign in to comment.