Lines Matching refs:prefs
329 void FIO_freePreferences(FIO_prefs_t* const prefs)
331 free(prefs);
355 void FIO_setCompressionType(FIO_prefs_t* const prefs, FIO_compressionType_t compressionType) { prefs->compressionType = compressionType; }
357 void FIO_overwriteMode(FIO_prefs_t* const prefs) { prefs->overwrite = 1; }
359 void FIO_setSparseWrite(FIO_prefs_t* const prefs, int sparse) { prefs->sparseFileSupport = sparse; }
361 void FIO_setDictIDFlag(FIO_prefs_t* const prefs, int dictIDFlag) { prefs->dictIDFlag = dictIDFlag; }
363 void FIO_setChecksumFlag(FIO_prefs_t* const prefs, int checksumFlag) { prefs->checksumFlag = checksumFlag; }
365 void FIO_setRemoveSrcFile(FIO_prefs_t* const prefs, int flag) { prefs->removeSrcFile = (flag!=0); }
367 void FIO_setMemLimit(FIO_prefs_t* const prefs, unsigned memLimit) { prefs->memLimit = memLimit; }
369 void FIO_setNbWorkers(FIO_prefs_t* const prefs, int nbWorkers) {
373 prefs->nbWorkers = nbWorkers;
376 void FIO_setExcludeCompressedFile(FIO_prefs_t* const prefs, int excludeCompressedFiles) { prefs->excludeCompressedFiles = excludeCompressedFiles; }
378 void FIO_setAllowBlockDevices(FIO_prefs_t* const prefs, int allowBlockDevices) { prefs->allowBlockDevices = allowBlockDevices; }
380 void FIO_setBlockSize(FIO_prefs_t* const prefs, int blockSize) {
381 if (blockSize && prefs->nbWorkers==0)
383 prefs->blockSize = blockSize;
386 void FIO_setOverlapLog(FIO_prefs_t* const prefs, int overlapLog){
387 if (overlapLog && prefs->nbWorkers==0)
389 prefs->overlapLog = overlapLog;
392 void FIO_setAdaptiveMode(FIO_prefs_t* const prefs, int adapt) {
393 if ((adapt>0) && (prefs->nbWorkers==0))
395 prefs->adaptiveMode = adapt;
398 void FIO_setUseRowMatchFinder(FIO_prefs_t* const prefs, int useRowMatchFinder) {
399 prefs->useRowMatchFinder = useRowMatchFinder;
402 void FIO_setRsyncable(FIO_prefs_t* const prefs, int rsyncable) {
403 if ((rsyncable>0) && (prefs->nbWorkers==0))
405 prefs->rsyncable = rsyncable;
408 void FIO_setStreamSrcSize(FIO_prefs_t* const prefs, size_t streamSrcSize) {
409 prefs->streamSrcSize = streamSrcSize;
412 void FIO_setTargetCBlockSize(FIO_prefs_t* const prefs, size_t targetCBlockSize) {
413 prefs->targetCBlockSize = targetCBlockSize;
416 void FIO_setSrcSizeHint(FIO_prefs_t* const prefs, size_t srcSizeHint) {
417 prefs->srcSizeHint = (int)MIN((size_t)INT_MAX, srcSizeHint);
420 void FIO_setTestMode(FIO_prefs_t* const prefs, int testMode) {
421 prefs->testMode = (testMode!=0);
425 FIO_prefs_t* const prefs,
427 prefs->literalCompressionMode = mode;
430 void FIO_setAdaptMin(FIO_prefs_t* const prefs, int minCLevel)
435 prefs->minAdaptLevel = minCLevel;
438 void FIO_setAdaptMax(FIO_prefs_t* const prefs, int maxCLevel)
440 prefs->maxAdaptLevel = maxCLevel;
443 void FIO_setLdmFlag(FIO_prefs_t* const prefs, unsigned ldmFlag) {
444 prefs->ldmFlag = (ldmFlag>0);
447 void FIO_setLdmHashLog(FIO_prefs_t* const prefs, int ldmHashLog) {
448 prefs->ldmHashLog = ldmHashLog;
451 void FIO_setLdmMinMatch(FIO_prefs_t* const prefs, int ldmMinMatch) {
452 prefs->ldmMinMatch = ldmMinMatch;
455 void FIO_setLdmBucketSizeLog(FIO_prefs_t* const prefs, int ldmBucketSizeLog) {
456 prefs->ldmBucketSizeLog = ldmBucketSizeLog;
460 void FIO_setLdmHashRateLog(FIO_prefs_t* const prefs, int ldmHashRateLog) {
461 prefs->ldmHashRateLog = ldmHashRateLog;
464 void FIO_setPatchFromMode(FIO_prefs_t* const prefs, int value)
466 prefs->patchFromMode = value != 0;
469 void FIO_setContentSize(FIO_prefs_t* const prefs, int value)
471 prefs->contentSize = value != 0;
474 void FIO_setAsyncIOFlag(FIO_prefs_t* const prefs, int value) {
476 prefs->asyncIO = value;
478 (void) prefs;
484 void FIO_setPassThroughFlag(FIO_prefs_t* const prefs, int value) {
485 prefs->passThrough = (value != 0);
488 void FIO_setMMapDict(FIO_prefs_t* const prefs, ZSTD_ParamSwitch_e value)
490 prefs->mmapDict = value;
541 * condition : `srcFileName` must be non-NULL. `prefs` may be NULL.
543 static FILE* FIO_openSrcFile(const FIO_prefs_t* const prefs, const char* srcFileName, stat_t* statbuf)
545 int allowBlockDevices = prefsprefs->allowBlockDevices : 0;
580 FIO_openDstFile(FIO_ctx_t* fCtx, FIO_prefs_t* const prefs,
586 if (prefs->testMode) return NULL; /* do not open file in test mode */
592 if (prefs->sparseFileSupport == 1) {
593 prefs->sparseFileSupport = 0;
606 if (prefs->sparseFileSupport == 1) {
607 prefs->sparseFileSupport = ZSTD_SPARSE_DEFAULT;
609 prefs->sparseFileSupport = 0;
624 if (!prefs->overwrite) {
703 static size_t FIO_setDictBufferMalloc(FIO_Dict_t* dict, const char* fileName, FIO_prefs_t* const prefs, stat_t* dictFileStat)
724 size_t const dictSizeMax = prefs->patchFromMode ? prefs->memLimit : DICTSIZE_MAX;
750 static size_t FIO_setDictBufferMMap(FIO_Dict_t* dict, const char* fileName, FIO_prefs_t* const prefs, stat_t* dictFileStat)
771 size_t const dictSizeMax = prefs->patchFromMode ? prefs->memLimit : DICTSIZE_MAX;
793 static size_t FIO_setDictBufferMMap(FIO_Dict_t* dict, const char* fileName, FIO_prefs_t* const prefs, stat_t* dictFileStat)
814 size_t const dictSizeMax = prefs->patchFromMode ? prefs->memLimit : DICTSIZE_MAX;
833 static size_t FIO_setDictBufferMMap(FIO_Dict_t* dict, const char* fileName, FIO_prefs_t* const prefs, stat_t* dictFileStat)
835 return FIO_setDictBufferMalloc(dict, fileName, prefs, dictFileStat);
856 static void FIO_initDict(FIO_Dict_t* dict, const char* fileName, FIO_prefs_t* const prefs, stat_t* dictFileStat, FIO_dictBufferType_t dictBufferType) {
859 dict->dictBufferSize = FIO_setDictBufferMalloc(dict, fileName, prefs, dictFileStat);
861 dict->dictBufferSize = FIO_setDictBufferMMap(dict, fileName, prefs, dictFileStat);
963 static void FIO_adjustMemLimitForPatchFromMode(FIO_prefs_t* const prefs,
967 unsigned long long maxSize = MAX(prefs->memLimit, MAX(dictSize, maxSrcFileSize));
974 FIO_setMemLimit(prefs, (unsigned)maxSize);
992 static int FIO_multiFilesConcatWarning(const FIO_ctx_t* fCtx, FIO_prefs_t* prefs, const char* outFileName, int displayLevelCutoff)
995 if (prefs->removeSrcFile)
1002 if (prefs->testMode) {
1003 if (prefs->removeSrcFile)
1025 if (prefs->removeSrcFile) {
1027 prefs->removeSrcFile = 0;
1031 if (prefs->overwrite) return 0;
1085 static void FIO_adjustParamsForPatchFromMode(FIO_prefs_t* const prefs,
1093 FIO_adjustMemLimitForPatchFromMode(prefs, dictSize, maxSrcFileSize);
1098 if (!prefs->ldmFlag)
1100 FIO_setLdmFlag(prefs, 1);
1112 static cRess_t FIO_createCResources(FIO_prefs_t* const prefs,
1115 int useMMap = prefs->mmapDict == ZSTD_ps_enable;
1116 int forceNoUseMMap = prefs->mmapDict == ZSTD_ps_disable;
1131 if (prefs->patchFromMode) {
1133 unsigned long long const ssSize = (unsigned long long)prefs->streamSrcSize;
1134 useMMap |= dictSize > prefs->memLimit;
1135 FIO_adjustParamsForPatchFromMode(prefs, &comprParams, dictSize, ssSize > 0 ? ssSize : maxSrcFileSize, cLevel);
1139 FIO_initDict(&ress.dict, dictFileName, prefs, &ress.dictFileStat, dictBufferType); /* works with dictFileName==NULL */
1141 ress.writeCtx = AIO_WritePool_create(prefs, ZSTD_CStreamOutSize());
1142 ress.readCtx = AIO_ReadPool_create(prefs, ZSTD_CStreamInSize());
1149 if (prefs->adaptiveMode && !prefs->ldmFlag && !comprParams.windowLog)
1152 CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_contentSizeFlag, prefs->contentSize) ); /* always enable content size when available (note: supposed to be default) */
1153 CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_dictIDFlag, prefs->dictIDFlag) );
1154 CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_checksumFlag, prefs->checksumFlag) );
1158 CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_targetCBlockSize, (int)prefs->targetCBlockSize) );
1160 CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_srcSizeHint, (int)prefs->srcSizeHint) );
1162 CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_enableLongDistanceMatching, prefs->ldmFlag) );
1163 CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_ldmHashLog, prefs->ldmHashLog) );
1164 CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_ldmMinMatch, prefs->ldmMinMatch) );
1165 if (prefs->ldmBucketSizeLog != FIO_LDM_PARAM_NOTSET) {
1166 CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_ldmBucketSizeLog, prefs->ldmBucketSizeLog) );
1168 if (prefs->ldmHashRateLog != FIO_LDM_PARAM_NOTSET) {
1169 CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_ldmHashRateLog, prefs->ldmHashRateLog) );
1171 CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_useRowMatchFinder, prefs->useRowMatchFinder));
1180 CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_literalCompressionMode, (int)prefs->literalCompressionMode) );
1184 DISPLAYLEVEL(5,"set nb workers = %u \n", prefs->nbWorkers);
1185 CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_nbWorkers, prefs->nbWorkers) );
1186 CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_jobSize, prefs->blockSize) );
1187 if (prefs->overlapLog != FIO_OVERLAP_LOG_NOTSET) {
1188 DISPLAYLEVEL(3,"set overlapLog = %u \n", prefs->overlapLog);
1189 CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_overlapLog, prefs->overlapLog) );
1191 CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_rsyncable, prefs->rsyncable) );
1194 if (prefs->patchFromMode) {
1406 LZ4F_preferences_t prefs;
1415 memset(&prefs, 0, sizeof(prefs));
1420 prefs.autoFlush = 0;
1421 prefs.compressionLevel = compressionLevel;
1422 prefs.frameInfo.blockMode = LZ4F_blockLinked;
1423 prefs.frameInfo.blockSizeID = LZ4F_max64KB;
1424 prefs.frameInfo.contentChecksumFlag = (contentChecksum_t)checksumFlag;
1426 prefs.frameInfo.contentSize = (srcFileSize==UTIL_FILESIZE_UNKNOWN) ? 0 : srcFileSize;
1428 assert(LZ4F_compressBound(blockSize, &prefs) <= writeJob->bufferSize);
1431 size_t headerSize = LZ4F_compressBegin(ctx, writeJob->buffer, writeJob->bufferSize, &prefs);
1492 FIO_prefs_t* const prefs,
1524 } else if (prefs->streamSrcSize > 0) {
1526 pledgedSrcSize = prefs->streamSrcSize;
1527 CHECK( ZSTD_CCtx_setPledgedSrcSize(ress.cctx, prefs->streamSrcSize) );
1534 if (prefs->ldmFlag) {
1583 if (prefs->adaptiveMode && UTIL_clockSpanMicro(lastAdaptTime) > adaptEveryMicro) {
1594 assert(prefs->nbWorkers >= 1);
1623 if (zfp.currentJobID > (unsigned)(prefs->nbWorkers+1)) { /* warm up period, to fill all workers */
1655 if (compressionLevel > prefs->maxAdaptLevel) compressionLevel = prefs->maxAdaptLevel;
1662 if (compressionLevel < prefs->minAdaptLevel) compressionLevel = prefs->minAdaptLevel;
1670 } /* if (prefs->adaptiveMode && UTIL_clockSpanMicro(lastAdaptTime) > adaptEveryMicro) */
1734 FIO_prefs_t* const prefs,
1747 switch (prefs->compressionType) {
1750 compressedfilesize = FIO_compressZstdFrame(fCtx, prefs, &ress, srcFileName, fileSize, compressionLevel, &readsize);
1766 compressedfilesize = FIO_compressLzmaFrame(&ress, srcFileName, fileSize, compressionLevel, &readsize, prefs->compressionType==FIO_lzmaCompression);
1776 compressedfilesize = FIO_compressLz4Frame(&ress, srcFileName, fileSize, compressionLevel, prefs->checksumFlag, &readsize);
1831 FIO_prefs_t* const prefs,
1855 { FILE *dstFile = FIO_openDstFile(fCtx, prefs, srcFileName, dstFileName, dstFileInitialPermissions);
1867 result = FIO_compressFilename_internal(fCtx, prefs, ress, dstFileName, srcFileName, compressionLevel);
2022 FIO_prefs_t* const prefs,
2056 if (prefs->excludeCompressedFiles == 1 && UTIL_isCompressedFile(srcFileName, compressedFileExtensions)) {
2061 srcFile = FIO_openSrcFile(prefs, srcFileName, &srcFileStat);
2077 fCtx, prefs, ress,
2082 if ( prefs->removeSrcFile /* --rm */
2106 void FIO_displayCompressionParameters(const FIO_prefs_t* prefs)
2117 DISPLAY("--format=%s", formatOptions[prefs->compressionType]);
2118 DISPLAY("%s", INDEX(sparseOptions, prefs->sparseFileSupport));
2119 DISPLAY("%s", prefs->dictIDFlag ? "" : " --no-dictID");
2120 DISPLAY("%s", INDEX(checkSumOptions, prefs->checksumFlag));
2121 DISPLAY(" --block-size=%d", prefs->blockSize);
2122 if (prefs->adaptiveMode)
2123 DISPLAY(" --adapt=min=%d,max=%d", prefs->minAdaptLevel, prefs->maxAdaptLevel);
2124 DISPLAY("%s", INDEX(rowMatchFinderOptions, prefs->useRowMatchFinder));
2125 DISPLAY("%s", prefs->rsyncable ? " --rsyncable" : "");
2126 if (prefs->streamSrcSize)
2127 DISPLAY(" --stream-size=%u", (unsigned) prefs->streamSrcSize);
2128 if (prefs->srcSizeHint)
2129 DISPLAY(" --size-hint=%d", prefs->srcSizeHint);
2130 if (prefs->targetCBlockSize)
2131 DISPLAY(" --target-compressed-block-size=%u", (unsigned) prefs->targetCBlockSize);
2132 DISPLAY("%s", INDEX(compressLiteralsOptions, prefs->literalCompressionMode));
2133 DISPLAY(" --memory=%u", prefs->memLimit ? prefs->memLimit : 128 MB);
2134 DISPLAY(" --threads=%d", prefs->nbWorkers);
2135 DISPLAY("%s", prefs->excludeCompressedFiles ? " --exclude-compressed" : "");
2136 DISPLAY(" --%scontent-size", prefs->contentSize ? "" : "no-");
2142 int FIO_compressFilename(FIO_ctx_t* const fCtx, FIO_prefs_t* const prefs, const char* dstFileName,
2146 cRess_t ress = FIO_createCResources(prefs, dictFileName, UTIL_getFileSize(srcFileName), compressionLevel, comprParams);
2147 int const result = FIO_compressFilename_srcFile(fCtx, prefs, ress, dstFileName, srcFileName, compressionLevel);
2218 FIO_prefs_t* const prefs,
2228 cRess_t ress = FIO_createCResources(prefs, dictFileName,
2236 if (FIO_multiFilesConcatWarning(fCtx, prefs, outFileName, 1 /* displayLevelCutoff */)) {
2240 dstFile = FIO_openDstFile(fCtx, prefs, NULL, outFileName, DEFAULT_FILE_PERMISSIONS);
2246 status = FIO_compressFilename_srcFile(fCtx, prefs, ress, outFileName, inFileNamesTable[fCtx->currFileIdx], compressionLevel);
2274 status = FIO_compressFilename_srcFile(fCtx, prefs, ress, dstFileName, srcFileName, compressionLevel);
2322 static dRess_t FIO_createDResources(FIO_prefs_t* const prefs, const char* dictFileName)
2324 int useMMap = prefs->mmapDict == ZSTD_ps_enable;
2325 int forceNoUseMMap = prefs->mmapDict == ZSTD_ps_disable;
2333 if (prefs->patchFromMode){
2335 useMMap |= dictSize > prefs->memLimit;
2336 FIO_adjustMemLimitForPatchFromMode(prefs, dictSize, 0 /* just use the dict size */);
2343 CHECK( ZSTD_DCtx_setMaxWindowSize(ress.dctx, prefs->memLimit) );
2344 CHECK( ZSTD_DCtx_setParameter(ress.dctx, ZSTD_d_forceIgnoreChecksum, !prefs->checksumFlag));
2349 FIO_initDict(&ress.dict, dictFileName, prefs, &statbuf, dictBufferType);
2353 if (prefs->patchFromMode){
2360 ress.writeCtx = AIO_WritePool_create(prefs, ZSTD_DStreamOutSize());
2361 ress.readCtx = AIO_ReadPool_create(prefs, ZSTD_DStreamInSize());
2400 FIO_zstdErrorHelp(const FIO_prefs_t* const prefs,
2416 assert(prefs->memLimit > 0);
2418 srcFileName, windowSize, prefs->memLimit);
2436 const FIO_prefs_t* const prefs,
2465 FIO_zstdErrorHelp(prefs, ress, readSizeHint, srcFileName);
2723 dRess_t ress, const FIO_prefs_t* const prefs,
2728 int passThrough = prefs->passThrough;
2735 passThrough = prefs->overwrite && !strcmp(dstFileName, stdoutmark);
2762 unsigned long long const frameSize = FIO_decompressZstdFrame(fCtx, &ress, prefs, srcFileName, filesize);
2816 FIO_prefs_t* const prefs,
2827 if ((AIO_WritePool_getFile(ress.writeCtx) == NULL) && (prefs->testMode == 0)) {
2839 dstFile = FIO_openDstFile(fCtx, prefs, srcFileName, dstFileName, dstFilePermissions);
2851 result = FIO_decompressFrames(fCtx, ress, prefs, dstFileName, srcFileName);
2885 static int FIO_decompressSrcFile(FIO_ctx_t* const fCtx, FIO_prefs_t* const prefs, dRess_t ress, const char* dstFileName, const char* srcFileName)
2897 srcFile = FIO_openSrcFile(prefs, srcFileName, &srcFileStat);
2913 result = FIO_decompressDstFile(fCtx, prefs, ress, dstFileName, srcFileName, &srcFileStat);
2922 if ( prefs->removeSrcFile /* --rm */
2939 int FIO_decompressFilename(FIO_ctx_t* const fCtx, FIO_prefs_t* const prefs,
2943 dRess_t const ress = FIO_createDResources(prefs, dictFileName);
2945 int const decodingError = FIO_decompressSrcFile(fCtx, prefs, ress, dstFileName, srcFileName);
3081 FIO_prefs_t* const prefs,
3089 dRess_t ress = FIO_createDResources(prefs, dictFileName);
3092 if (FIO_multiFilesConcatWarning(fCtx, prefs, outFileName, 1 /* displayLevelCutoff */)) {
3096 if (!prefs->testMode) {
3097 FILE* dstFile = FIO_openDstFile(fCtx, prefs, NULL, outFileName, DEFAULT_FILE_PERMISSIONS);
3102 status = FIO_decompressSrcFile(fCtx, prefs, ress, outFileName, srcNamesTable[fCtx->currFileIdx]);
3106 if ((!prefs->testMode) && (AIO_WritePool_closeFile(ress.writeCtx)))
3128 status = FIO_decompressSrcFile(fCtx, prefs, ress, dstFileName, srcFileName);
Indexes created Fri Jun 12 00:25:51 UTC 2026