1 /* 2 * Copyright (c) Meta Platforms, Inc. and affiliates. 3 * All rights reserved. 4 * 5 * This source code is licensed under both the BSD-style license (found in the 6 * LICENSE file in the root directory of this source tree) and the GPLv2 (found 7 * in the COPYING file in the root directory of this source tree). 8 * You may select, at your option, one of the above-listed licenses. 9 */ 10 11 /*-************************************* 12 * Dependencies 13 ***************************************/ 14 #include "../common/allocations.h" /* ZSTD_customMalloc, ZSTD_customCalloc, ZSTD_customFree */ 15 #include "../common/zstd_deps.h" /* INT_MAX, ZSTD_memset, ZSTD_memcpy */ 16 #include "../common/mem.h" 17 #include "hist.h" /* HIST_countFast_wksp */ 18 #define FSE_STATIC_LINKING_ONLY /* FSE_encodeSymbol */ 19 #include "../common/fse.h" 20 #include "../common/huf.h" 21 #include "zstd_compress_internal.h" 22 #include "zstd_compress_sequences.h" 23 #include "zstd_compress_literals.h" 24 #include "zstd_fast.h" 25 #include "zstd_double_fast.h" 26 #include "zstd_lazy.h" 27 #include "zstd_opt.h" 28 #include "zstd_ldm.h" 29 #include "zstd_compress_superblock.h" 30 #include "../common/bits.h" /* ZSTD_highbit32, ZSTD_rotateRight_U64 */ 31 32 /* *************************************************************** 33 * Tuning parameters 34 *****************************************************************/ 35 /*! 36 * COMPRESS_HEAPMODE : 37 * Select how default decompression function ZSTD_compress() allocates its context, 38 * on stack (0, default), or into heap (1). 39 * Note that functions with explicit context such as ZSTD_compressCCtx() are unaffected. 40 */ 41 #ifndef ZSTD_COMPRESS_HEAPMODE 42 # define ZSTD_COMPRESS_HEAPMODE 0 43 #endif 44 45 /*! 46 * ZSTD_HASHLOG3_MAX : 47 * Maximum size of the hash table dedicated to find 3-bytes matches, 48 * in log format, aka 17 => 1 << 17 == 128Ki positions. 49 * This structure is only used in zstd_opt. 50 * Since allocation is centralized for all strategies, it has to be known here. 51 * The actual (selected) size of the hash table is then stored in ZSTD_matchState_t.hashLog3, 52 * so that zstd_opt.c doesn't need to know about this constant. 53 */ 54 #ifndef ZSTD_HASHLOG3_MAX 55 # define ZSTD_HASHLOG3_MAX 17 56 #endif 57 58 /*-************************************* 59 * Helper functions 60 ***************************************/ 61 /* ZSTD_compressBound() 62 * Note that the result from this function is only valid for 63 * the one-pass compression functions. 64 * When employing the streaming mode, 65 * if flushes are frequently altering the size of blocks, 66 * the overhead from block headers can make the compressed data larger 67 * than the return value of ZSTD_compressBound(). 68 */ 69 size_t ZSTD_compressBound(size_t srcSize) { 70 size_t const r = ZSTD_COMPRESSBOUND(srcSize); 71 if (r==0) return ERROR(srcSize_wrong); 72 return r; 73 } 74 75 76 /*-************************************* 77 * Context memory management 78 ***************************************/ 79 struct ZSTD_CDict_s { 80 const void* dictContent; 81 size_t dictContentSize; 82 ZSTD_dictContentType_e dictContentType; /* The dictContentType the CDict was created with */ 83 U32* entropyWorkspace; /* entropy workspace of HUF_WORKSPACE_SIZE bytes */ 84 ZSTD_cwksp workspace; 85 ZSTD_matchState_t matchState; 86 ZSTD_compressedBlockState_t cBlockState; 87 ZSTD_customMem customMem; 88 U32 dictID; 89 int compressionLevel; /* 0 indicates that advanced API was used to select CDict params */ 90 ZSTD_paramSwitch_e useRowMatchFinder; /* Indicates whether the CDict was created with params that would use 91 * row-based matchfinder. Unless the cdict is reloaded, we will use 92 * the same greedy/lazy matchfinder at compression time. 93 */ 94 }; /* typedef'd to ZSTD_CDict within "zstd.h" */ 95 96 ZSTD_CCtx* ZSTD_createCCtx(void) 97 { 98 return ZSTD_createCCtx_advanced(ZSTD_defaultCMem); 99 } 100 101 static void ZSTD_initCCtx(ZSTD_CCtx* cctx, ZSTD_customMem memManager) 102 { 103 assert(cctx != NULL); 104 ZSTD_memset(cctx, 0, sizeof(*cctx)); 105 cctx->customMem = memManager; 106 cctx->bmi2 = ZSTD_cpuSupportsBmi2(); 107 { size_t const err = ZSTD_CCtx_reset(cctx, ZSTD_reset_parameters); 108 assert(!ZSTD_isError(err)); 109 (void)err; 110 } 111 } 112 113 ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem) 114 { 115 ZSTD_STATIC_ASSERT(zcss_init==0); 116 ZSTD_STATIC_ASSERT(ZSTD_CONTENTSIZE_UNKNOWN==(0ULL - 1)); 117 if ((!customMem.customAlloc) ^ (!customMem.customFree)) return NULL; 118 { ZSTD_CCtx* const cctx = (ZSTD_CCtx*)ZSTD_customMalloc(sizeof(ZSTD_CCtx), customMem); 119 if (!cctx) return NULL; 120 ZSTD_initCCtx(cctx, customMem); 121 return cctx; 122 } 123 } 124 125 ZSTD_CCtx* ZSTD_initStaticCCtx(void* workspace, size_t workspaceSize) 126 { 127 ZSTD_cwksp ws; 128 ZSTD_CCtx* cctx; 129 if (workspaceSize <= sizeof(ZSTD_CCtx)) return NULL; /* minimum size */ 130 if ((size_t)workspace & 7) return NULL; /* must be 8-aligned */ 131 ZSTD_cwksp_init(&ws, workspace, workspaceSize, ZSTD_cwksp_static_alloc); 132 133 cctx = (ZSTD_CCtx*)ZSTD_cwksp_reserve_object(&ws, sizeof(ZSTD_CCtx)); 134 if (cctx == NULL) return NULL; 135 136 ZSTD_memset(cctx, 0, sizeof(ZSTD_CCtx)); 137 ZSTD_cwksp_move(&cctx->workspace, &ws); 138 cctx->staticSize = workspaceSize; 139 140 /* statically sized space. entropyWorkspace never moves (but prev/next block swap places) */ 141 if (!ZSTD_cwksp_check_available(&cctx->workspace, ENTROPY_WORKSPACE_SIZE + 2 * sizeof(ZSTD_compressedBlockState_t))) return NULL; 142 cctx->blockState.prevCBlock = (ZSTD_compressedBlockState_t*)ZSTD_cwksp_reserve_object(&cctx->workspace, sizeof(ZSTD_compressedBlockState_t)); 143 cctx->blockState.nextCBlock = (ZSTD_compressedBlockState_t*)ZSTD_cwksp_reserve_object(&cctx->workspace, sizeof(ZSTD_compressedBlockState_t)); 144 cctx->entropyWorkspace = (U32*)ZSTD_cwksp_reserve_object(&cctx->workspace, ENTROPY_WORKSPACE_SIZE); 145 cctx->bmi2 = ZSTD_cpuid_bmi2(ZSTD_cpuid()); 146 return cctx; 147 } 148 149 /** 150 * Clears and frees all of the dictionaries in the CCtx. 151 */ 152 static void ZSTD_clearAllDicts(ZSTD_CCtx* cctx) 153 { 154 ZSTD_customFree(cctx->localDict.dictBuffer, cctx->customMem); 155 ZSTD_freeCDict(cctx->localDict.cdict); 156 ZSTD_memset(&cctx->localDict, 0, sizeof(cctx->localDict)); 157 ZSTD_memset(&cctx->prefixDict, 0, sizeof(cctx->prefixDict)); 158 cctx->cdict = NULL; 159 } 160 161 static size_t ZSTD_sizeof_localDict(ZSTD_localDict dict) 162 { 163 size_t const bufferSize = dict.dictBuffer != NULL ? dict.dictSize : 0; 164 size_t const cdictSize = ZSTD_sizeof_CDict(dict.cdict); 165 return bufferSize + cdictSize; 166 } 167 168 static void ZSTD_freeCCtxContent(ZSTD_CCtx* cctx) 169 { 170 assert(cctx != NULL); 171 assert(cctx->staticSize == 0); 172 ZSTD_clearAllDicts(cctx); 173 #ifdef ZSTD_MULTITHREAD 174 ZSTDMT_freeCCtx(cctx->mtctx); cctx->mtctx = NULL; 175 #endif 176 ZSTD_cwksp_free(&cctx->workspace, cctx->customMem); 177 } 178 179 size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx) 180 { 181 DEBUGLOG(3, "ZSTD_freeCCtx (address: %p)", (void*)cctx); 182 if (cctx==NULL) return 0; /* support free on NULL */ 183 RETURN_ERROR_IF(cctx->staticSize, memory_allocation, 184 "not compatible with static CCtx"); 185 { int cctxInWorkspace = ZSTD_cwksp_owns_buffer(&cctx->workspace, cctx); 186 ZSTD_freeCCtxContent(cctx); 187 if (!cctxInWorkspace) ZSTD_customFree(cctx, cctx->customMem); 188 } 189 return 0; 190 } 191 192 193 static size_t ZSTD_sizeof_mtctx(const ZSTD_CCtx* cctx) 194 { 195 #ifdef ZSTD_MULTITHREAD 196 return ZSTDMT_sizeof_CCtx(cctx->mtctx); 197 #else 198 (void)cctx; 199 return 0; 200 #endif 201 } 202 203 204 size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx) 205 { 206 if (cctx==NULL) return 0; /* support sizeof on NULL */ 207 /* cctx may be in the workspace */ 208 return (cctx->workspace.workspace == cctx ? 0 : sizeof(*cctx)) 209 + ZSTD_cwksp_sizeof(&cctx->workspace) 210 + ZSTD_sizeof_localDict(cctx->localDict) 211 + ZSTD_sizeof_mtctx(cctx); 212 } 213 214 size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs) 215 { 216 return ZSTD_sizeof_CCtx(zcs); /* same object */ 217 } 218 219 /* private API call, for dictBuilder only */ 220 const seqStore_t* ZSTD_getSeqStore(const ZSTD_CCtx* ctx) { return &(ctx->seqStore); } 221 222 /* Returns true if the strategy supports using a row based matchfinder */ 223 static int ZSTD_rowMatchFinderSupported(const ZSTD_strategy strategy) { 224 return (strategy >= ZSTD_greedy && strategy <= ZSTD_lazy2); 225 } 226 227 /* Returns true if the strategy and useRowMatchFinder mode indicate that we will use the row based matchfinder 228 * for this compression. 229 */ 230 static int ZSTD_rowMatchFinderUsed(const ZSTD_strategy strategy, const ZSTD_paramSwitch_e mode) { 231 assert(mode != ZSTD_ps_auto); 232 return ZSTD_rowMatchFinderSupported(strategy) && (mode == ZSTD_ps_enable); 233 } 234 235 /* Returns row matchfinder usage given an initial mode and cParams */ 236 static ZSTD_paramSwitch_e ZSTD_resolveRowMatchFinderMode(ZSTD_paramSwitch_e mode, 237 const ZSTD_compressionParameters* const cParams) { 238 #if defined(ZSTD_ARCH_X86_SSE2) || defined(ZSTD_ARCH_ARM_NEON) 239 int const kHasSIMD128 = 1; 240 #else 241 int const kHasSIMD128 = 0; 242 #endif 243 if (mode != ZSTD_ps_auto) return mode; /* if requested enabled, but no SIMD, we still will use row matchfinder */ 244 mode = ZSTD_ps_disable; 245 if (!ZSTD_rowMatchFinderSupported(cParams->strategy)) return mode; 246 if (kHasSIMD128) { 247 if (cParams->windowLog > 14) mode = ZSTD_ps_enable; 248 } else { 249 if (cParams->windowLog > 17) mode = ZSTD_ps_enable; 250 } 251 return mode; 252 } 253 254 /* Returns block splitter usage (generally speaking, when using slower/stronger compression modes) */ 255 static ZSTD_paramSwitch_e ZSTD_resolveBlockSplitterMode(ZSTD_paramSwitch_e mode, 256 const ZSTD_compressionParameters* const cParams) { 257 if (mode != ZSTD_ps_auto) return mode; 258 return (cParams->strategy >= ZSTD_btopt && cParams->windowLog >= 17) ? ZSTD_ps_enable : ZSTD_ps_disable; 259 } 260 261 /* Returns 1 if the arguments indicate that we should allocate a chainTable, 0 otherwise */ 262 static int ZSTD_allocateChainTable(const ZSTD_strategy strategy, 263 const ZSTD_paramSwitch_e useRowMatchFinder, 264 const U32 forDDSDict) { 265 assert(useRowMatchFinder != ZSTD_ps_auto); 266 /* We always should allocate a chaintable if we are allocating a matchstate for a DDS dictionary matchstate. 267 * We do not allocate a chaintable if we are using ZSTD_fast, or are using the row-based matchfinder. 268 */ 269 return forDDSDict || ((strategy != ZSTD_fast) && !ZSTD_rowMatchFinderUsed(strategy, useRowMatchFinder)); 270 } 271 272 /* Returns ZSTD_ps_enable if compression parameters are such that we should 273 * enable long distance matching (wlog >= 27, strategy >= btopt). 274 * Returns ZSTD_ps_disable otherwise. 275 */ 276 static ZSTD_paramSwitch_e ZSTD_resolveEnableLdm(ZSTD_paramSwitch_e mode, 277 const ZSTD_compressionParameters* const cParams) { 278 if (mode != ZSTD_ps_auto) return mode; 279 return (cParams->strategy >= ZSTD_btopt && cParams->windowLog >= 27) ? ZSTD_ps_enable : ZSTD_ps_disable; 280 } 281 282 static int ZSTD_resolveExternalSequenceValidation(int mode) { 283 return mode; 284 } 285 286 /* Resolves maxBlockSize to the default if no value is present. */ 287 static size_t ZSTD_resolveMaxBlockSize(size_t maxBlockSize) { 288 if (maxBlockSize == 0) { 289 return ZSTD_BLOCKSIZE_MAX; 290 } else { 291 return maxBlockSize; 292 } 293 } 294 295 static ZSTD_paramSwitch_e ZSTD_resolveExternalRepcodeSearch(ZSTD_paramSwitch_e value, int cLevel) { 296 if (value != ZSTD_ps_auto) return value; 297 if (cLevel < 10) { 298 return ZSTD_ps_disable; 299 } else { 300 return ZSTD_ps_enable; 301 } 302 } 303 304 /* Returns 1 if compression parameters are such that CDict hashtable and chaintable indices are tagged. 305 * If so, the tags need to be removed in ZSTD_resetCCtx_byCopyingCDict. */ 306 static int ZSTD_CDictIndicesAreTagged(const ZSTD_compressionParameters* const cParams) { 307 return cParams->strategy == ZSTD_fast || cParams->strategy == ZSTD_dfast; 308 } 309 310 static ZSTD_CCtx_params ZSTD_makeCCtxParamsFromCParams( 311 ZSTD_compressionParameters cParams) 312 { 313 ZSTD_CCtx_params cctxParams; 314 /* should not matter, as all cParams are presumed properly defined */ 315 ZSTD_CCtxParams_init(&cctxParams, ZSTD_CLEVEL_DEFAULT); 316 cctxParams.cParams = cParams; 317 318 /* Adjust advanced params according to cParams */ 319 cctxParams.ldmParams.enableLdm = ZSTD_resolveEnableLdm(cctxParams.ldmParams.enableLdm, &cParams); 320 if (cctxParams.ldmParams.enableLdm == ZSTD_ps_enable) { 321 ZSTD_ldm_adjustParameters(&cctxParams.ldmParams, &cParams); 322 assert(cctxParams.ldmParams.hashLog >= cctxParams.ldmParams.bucketSizeLog); 323 assert(cctxParams.ldmParams.hashRateLog < 32); 324 } 325 cctxParams.useBlockSplitter = ZSTD_resolveBlockSplitterMode(cctxParams.useBlockSplitter, &cParams); 326 cctxParams.useRowMatchFinder = ZSTD_resolveRowMatchFinderMode(cctxParams.useRowMatchFinder, &cParams); 327 cctxParams.validateSequences = ZSTD_resolveExternalSequenceValidation(cctxParams.validateSequences); 328 cctxParams.maxBlockSize = ZSTD_resolveMaxBlockSize(cctxParams.maxBlockSize); 329 cctxParams.searchForExternalRepcodes = ZSTD_resolveExternalRepcodeSearch(cctxParams.searchForExternalRepcodes, 330 cctxParams.compressionLevel); 331 assert(!ZSTD_checkCParams(cParams)); 332 return cctxParams; 333 } 334 335 static ZSTD_CCtx_params* ZSTD_createCCtxParams_advanced( 336 ZSTD_customMem customMem) 337 { 338 ZSTD_CCtx_params* params; 339 if ((!customMem.customAlloc) ^ (!customMem.customFree)) return NULL; 340 params = (ZSTD_CCtx_params*)ZSTD_customCalloc( 341 sizeof(ZSTD_CCtx_params), customMem); 342 if (!params) { return NULL; } 343 ZSTD_CCtxParams_init(params, ZSTD_CLEVEL_DEFAULT); 344 params->customMem = customMem; 345 return params; 346 } 347 348 ZSTD_CCtx_params* ZSTD_createCCtxParams(void) 349 { 350 return ZSTD_createCCtxParams_advanced(ZSTD_defaultCMem); 351 } 352 353 size_t ZSTD_freeCCtxParams(ZSTD_CCtx_params* params) 354 { 355 if (params == NULL) { return 0; } 356 ZSTD_customFree(params, params->customMem); 357 return 0; 358 } 359 360 size_t ZSTD_CCtxParams_reset(ZSTD_CCtx_params* params) 361 { 362 return ZSTD_CCtxParams_init(params, ZSTD_CLEVEL_DEFAULT); 363 } 364 365 size_t ZSTD_CCtxParams_init(ZSTD_CCtx_params* cctxParams, int compressionLevel) { 366 RETURN_ERROR_IF(!cctxParams, GENERIC, "NULL pointer!"); 367 ZSTD_memset(cctxParams, 0, sizeof(*cctxParams)); 368 cctxParams->compressionLevel = compressionLevel; 369 cctxParams->fParams.contentSizeFlag = 1; 370 return 0; 371 } 372 373 #define ZSTD_NO_CLEVEL 0 374 375 /** 376 * Initializes `cctxParams` from `params` and `compressionLevel`. 377 * @param compressionLevel If params are derived from a compression level then that compression level, otherwise ZSTD_NO_CLEVEL. 378 */ 379 static void 380 ZSTD_CCtxParams_init_internal(ZSTD_CCtx_params* cctxParams, 381 const ZSTD_parameters* params, 382 int compressionLevel) 383 { 384 assert(!ZSTD_checkCParams(params->cParams)); 385 ZSTD_memset(cctxParams, 0, sizeof(*cctxParams)); 386 cctxParams->cParams = params->cParams; 387 cctxParams->fParams = params->fParams; 388 /* Should not matter, as all cParams are presumed properly defined. 389 * But, set it for tracing anyway. 390 */ 391 cctxParams->compressionLevel = compressionLevel; 392 cctxParams->useRowMatchFinder = ZSTD_resolveRowMatchFinderMode(cctxParams->useRowMatchFinder, ¶ms->cParams); 393 cctxParams->useBlockSplitter = ZSTD_resolveBlockSplitterMode(cctxParams->useBlockSplitter, ¶ms->cParams); 394 cctxParams->ldmParams.enableLdm = ZSTD_resolveEnableLdm(cctxParams->ldmParams.enableLdm, ¶ms->cParams); 395 cctxParams->validateSequences = ZSTD_resolveExternalSequenceValidation(cctxParams->validateSequences); 396 cctxParams->maxBlockSize = ZSTD_resolveMaxBlockSize(cctxParams->maxBlockSize); 397 cctxParams->searchForExternalRepcodes = ZSTD_resolveExternalRepcodeSearch(cctxParams->searchForExternalRepcodes, compressionLevel); 398 DEBUGLOG(4, "ZSTD_CCtxParams_init_internal: useRowMatchFinder=%d, useBlockSplitter=%d ldm=%d", 399 cctxParams->useRowMatchFinder, cctxParams->useBlockSplitter, cctxParams->ldmParams.enableLdm); 400 } 401 402 size_t ZSTD_CCtxParams_init_advanced(ZSTD_CCtx_params* cctxParams, ZSTD_parameters params) 403 { 404 RETURN_ERROR_IF(!cctxParams, GENERIC, "NULL pointer!"); 405 FORWARD_IF_ERROR( ZSTD_checkCParams(params.cParams) , ""); 406 ZSTD_CCtxParams_init_internal(cctxParams, ¶ms, ZSTD_NO_CLEVEL); 407 return 0; 408 } 409 410 /** 411 * Sets cctxParams' cParams and fParams from params, but otherwise leaves them alone. 412 * @param params Validated zstd parameters. 413 */ 414 static void ZSTD_CCtxParams_setZstdParams( 415 ZSTD_CCtx_params* cctxParams, const ZSTD_parameters* params) 416 { 417 assert(!ZSTD_checkCParams(params->cParams)); 418 cctxParams->cParams = params->cParams; 419 cctxParams->fParams = params->fParams; 420 /* Should not matter, as all cParams are presumed properly defined. 421 * But, set it for tracing anyway. 422 */ 423 cctxParams->compressionLevel = ZSTD_NO_CLEVEL; 424 } 425 426 ZSTD_bounds ZSTD_cParam_getBounds(ZSTD_cParameter param) 427 { 428 ZSTD_bounds bounds = { 0, 0, 0 }; 429 430 switch(param) 431 { 432 case ZSTD_c_compressionLevel: 433 bounds.lowerBound = ZSTD_minCLevel(); 434 bounds.upperBound = ZSTD_maxCLevel(); 435 return bounds; 436 437 case ZSTD_c_windowLog: 438 bounds.lowerBound = ZSTD_WINDOWLOG_MIN; 439 bounds.upperBound = ZSTD_WINDOWLOG_MAX; 440 return bounds; 441 442 case ZSTD_c_hashLog: 443 bounds.lowerBound = ZSTD_HASHLOG_MIN; 444 bounds.upperBound = ZSTD_HASHLOG_MAX; 445 return bounds; 446 447 case ZSTD_c_chainLog: 448 bounds.lowerBound = ZSTD_CHAINLOG_MIN; 449 bounds.upperBound = ZSTD_CHAINLOG_MAX; 450 return bounds; 451 452 case ZSTD_c_searchLog: 453 bounds.lowerBound = ZSTD_SEARCHLOG_MIN; 454 bounds.upperBound = ZSTD_SEARCHLOG_MAX; 455 return bounds; 456 457 case ZSTD_c_minMatch: 458 bounds.lowerBound = ZSTD_MINMATCH_MIN; 459 bounds.upperBound = ZSTD_MINMATCH_MAX; 460 return bounds; 461 462 case ZSTD_c_targetLength: 463 bounds.lowerBound = ZSTD_TARGETLENGTH_MIN; 464 bounds.upperBound = ZSTD_TARGETLENGTH_MAX; 465 return bounds; 466 467 case ZSTD_c_strategy: 468 bounds.lowerBound = ZSTD_STRATEGY_MIN; 469 bounds.upperBound = ZSTD_STRATEGY_MAX; 470 return bounds; 471 472 case ZSTD_c_contentSizeFlag: 473 bounds.lowerBound = 0; 474 bounds.upperBound = 1; 475 return bounds; 476 477 case ZSTD_c_checksumFlag: 478 bounds.lowerBound = 0; 479 bounds.upperBound = 1; 480 return bounds; 481 482 case ZSTD_c_dictIDFlag: 483 bounds.lowerBound = 0; 484 bounds.upperBound = 1; 485 return bounds; 486 487 case ZSTD_c_nbWorkers: 488 bounds.lowerBound = 0; 489 #ifdef ZSTD_MULTITHREAD 490 bounds.upperBound = ZSTDMT_NBWORKERS_MAX; 491 #else 492 bounds.upperBound = 0; 493 #endif 494 return bounds; 495 496 case ZSTD_c_jobSize: 497 bounds.lowerBound = 0; 498 #ifdef ZSTD_MULTITHREAD 499 bounds.upperBound = ZSTDMT_JOBSIZE_MAX; 500 #else 501 bounds.upperBound = 0; 502 #endif 503 return bounds; 504 505 case ZSTD_c_overlapLog: 506 #ifdef ZSTD_MULTITHREAD 507 bounds.lowerBound = ZSTD_OVERLAPLOG_MIN; 508 bounds.upperBound = ZSTD_OVERLAPLOG_MAX; 509 #else 510 bounds.lowerBound = 0; 511 bounds.upperBound = 0; 512 #endif 513 return bounds; 514 515 case ZSTD_c_enableDedicatedDictSearch: 516 bounds.lowerBound = 0; 517 bounds.upperBound = 1; 518 return bounds; 519 520 case ZSTD_c_enableLongDistanceMatching: 521 bounds.lowerBound = (int)ZSTD_ps_auto; 522 bounds.upperBound = (int)ZSTD_ps_disable; 523 return bounds; 524 525 case ZSTD_c_ldmHashLog: 526 bounds.lowerBound = ZSTD_LDM_HASHLOG_MIN; 527 bounds.upperBound = ZSTD_LDM_HASHLOG_MAX; 528 return bounds; 529 530 case ZSTD_c_ldmMinMatch: 531 bounds.lowerBound = ZSTD_LDM_MINMATCH_MIN; 532 bounds.upperBound = ZSTD_LDM_MINMATCH_MAX; 533 return bounds; 534 535 case ZSTD_c_ldmBucketSizeLog: 536 bounds.lowerBound = ZSTD_LDM_BUCKETSIZELOG_MIN; 537 bounds.upperBound = ZSTD_LDM_BUCKETSIZELOG_MAX; 538 return bounds; 539 540 case ZSTD_c_ldmHashRateLog: 541 bounds.lowerBound = ZSTD_LDM_HASHRATELOG_MIN; 542 bounds.upperBound = ZSTD_LDM_HASHRATELOG_MAX; 543 return bounds; 544 545 /* experimental parameters */ 546 case ZSTD_c_rsyncable: 547 bounds.lowerBound = 0; 548 bounds.upperBound = 1; 549 return bounds; 550 551 case ZSTD_c_forceMaxWindow : 552 bounds.lowerBound = 0; 553 bounds.upperBound = 1; 554 return bounds; 555 556 case ZSTD_c_format: 557 ZSTD_STATIC_ASSERT(ZSTD_f_zstd1 < ZSTD_f_zstd1_magicless); 558 bounds.lowerBound = ZSTD_f_zstd1; 559 bounds.upperBound = ZSTD_f_zstd1_magicless; /* note : how to ensure at compile time that this is the highest value enum ? */ 560 return bounds; 561 562 case ZSTD_c_forceAttachDict: 563 ZSTD_STATIC_ASSERT(ZSTD_dictDefaultAttach < ZSTD_dictForceLoad); 564 bounds.lowerBound = ZSTD_dictDefaultAttach; 565 bounds.upperBound = ZSTD_dictForceLoad; /* note : how to ensure at compile time that this is the highest value enum ? */ 566 return bounds; 567 568 case ZSTD_c_literalCompressionMode: 569 ZSTD_STATIC_ASSERT(ZSTD_ps_auto < ZSTD_ps_enable && ZSTD_ps_enable < ZSTD_ps_disable); 570 bounds.lowerBound = (int)ZSTD_ps_auto; 571 bounds.upperBound = (int)ZSTD_ps_disable; 572 return bounds; 573 574 case ZSTD_c_targetCBlockSize: 575 bounds.lowerBound = ZSTD_TARGETCBLOCKSIZE_MIN; 576 bounds.upperBound = ZSTD_TARGETCBLOCKSIZE_MAX; 577 return bounds; 578 579 case ZSTD_c_srcSizeHint: 580 bounds.lowerBound = ZSTD_SRCSIZEHINT_MIN; 581 bounds.upperBound = ZSTD_SRCSIZEHINT_MAX; 582 return bounds; 583 584 case ZSTD_c_stableInBuffer: 585 case ZSTD_c_stableOutBuffer: 586 bounds.lowerBound = (int)ZSTD_bm_buffered; 587 bounds.upperBound = (int)ZSTD_bm_stable; 588 return bounds; 589 590 case ZSTD_c_blockDelimiters: 591 bounds.lowerBound = (int)ZSTD_sf_noBlockDelimiters; 592 bounds.upperBound = (int)ZSTD_sf_explicitBlockDelimiters; 593 return bounds; 594 595 case ZSTD_c_validateSequences: 596 bounds.lowerBound = 0; 597 bounds.upperBound = 1; 598 return bounds; 599 600 case ZSTD_c_useBlockSplitter: 601 bounds.lowerBound = (int)ZSTD_ps_auto; 602 bounds.upperBound = (int)ZSTD_ps_disable; 603 return bounds; 604 605 case ZSTD_c_useRowMatchFinder: 606 bounds.lowerBound = (int)ZSTD_ps_auto; 607 bounds.upperBound = (int)ZSTD_ps_disable; 608 return bounds; 609 610 case ZSTD_c_deterministicRefPrefix: 611 bounds.lowerBound = 0; 612 bounds.upperBound = 1; 613 return bounds; 614 615 case ZSTD_c_prefetchCDictTables: 616 bounds.lowerBound = (int)ZSTD_ps_auto; 617 bounds.upperBound = (int)ZSTD_ps_disable; 618 return bounds; 619 620 case ZSTD_c_enableSeqProducerFallback: 621 bounds.lowerBound = 0; 622 bounds.upperBound = 1; 623 return bounds; 624 625 case ZSTD_c_maxBlockSize: 626 bounds.lowerBound = ZSTD_BLOCKSIZE_MAX_MIN; 627 bounds.upperBound = ZSTD_BLOCKSIZE_MAX; 628 return bounds; 629 630 case ZSTD_c_searchForExternalRepcodes: 631 bounds.lowerBound = (int)ZSTD_ps_auto; 632 bounds.upperBound = (int)ZSTD_ps_disable; 633 return bounds; 634 635 default: 636 bounds.error = ERROR(parameter_unsupported); 637 return bounds; 638 } 639 } 640 641 /* ZSTD_cParam_clampBounds: 642 * Clamps the value into the bounded range. 643 */ 644 static size_t ZSTD_cParam_clampBounds(ZSTD_cParameter cParam, int* value) 645 { 646 ZSTD_bounds const bounds = ZSTD_cParam_getBounds(cParam); 647 if (ZSTD_isError(bounds.error)) return bounds.error; 648 if (*value < bounds.lowerBound) *value = bounds.lowerBound; 649 if (*value > bounds.upperBound) *value = bounds.upperBound; 650 return 0; 651 } 652 653 #define BOUNDCHECK(cParam, val) \ 654 do { \ 655 RETURN_ERROR_IF(!ZSTD_cParam_withinBounds(cParam,val), \ 656 parameter_outOfBound, "Param out of bounds"); \ 657 } while (0) 658 659 660 static int ZSTD_isUpdateAuthorized(ZSTD_cParameter param) 661 { 662 switch(param) 663 { 664 case ZSTD_c_compressionLevel: 665 case ZSTD_c_hashLog: 666 case ZSTD_c_chainLog: 667 case ZSTD_c_searchLog: 668 case ZSTD_c_minMatch: 669 case ZSTD_c_targetLength: 670 case ZSTD_c_strategy: 671 return 1; 672 673 case ZSTD_c_format: 674 case ZSTD_c_windowLog: 675 case ZSTD_c_contentSizeFlag: 676 case ZSTD_c_checksumFlag: 677 case ZSTD_c_dictIDFlag: 678 case ZSTD_c_forceMaxWindow : 679 case ZSTD_c_nbWorkers: 680 case ZSTD_c_jobSize: 681 case ZSTD_c_overlapLog: 682 case ZSTD_c_rsyncable: 683 case ZSTD_c_enableDedicatedDictSearch: 684 case ZSTD_c_enableLongDistanceMatching: 685 case ZSTD_c_ldmHashLog: 686 case ZSTD_c_ldmMinMatch: 687 case ZSTD_c_ldmBucketSizeLog: 688 case ZSTD_c_ldmHashRateLog: 689 case ZSTD_c_forceAttachDict: 690 case ZSTD_c_literalCompressionMode: 691 case ZSTD_c_targetCBlockSize: 692 case ZSTD_c_srcSizeHint: 693 case ZSTD_c_stableInBuffer: 694 case ZSTD_c_stableOutBuffer: 695 case ZSTD_c_blockDelimiters: 696 case ZSTD_c_validateSequences: 697 case ZSTD_c_useBlockSplitter: 698 case ZSTD_c_useRowMatchFinder: 699 case ZSTD_c_deterministicRefPrefix: 700 case ZSTD_c_prefetchCDictTables: 701 case ZSTD_c_enableSeqProducerFallback: 702 case ZSTD_c_maxBlockSize: 703 case ZSTD_c_searchForExternalRepcodes: 704 default: 705 return 0; 706 } 707 } 708 709 size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, int value) 710 { 711 DEBUGLOG(4, "ZSTD_CCtx_setParameter (%i, %i)", (int)param, value); 712 if (cctx->streamStage != zcss_init) { 713 if (ZSTD_isUpdateAuthorized(param)) { 714 cctx->cParamsChanged = 1; 715 } else { 716 RETURN_ERROR(stage_wrong, "can only set params in cctx init stage"); 717 } } 718 719 switch(param) 720 { 721 case ZSTD_c_nbWorkers: 722 RETURN_ERROR_IF((value!=0) && cctx->staticSize, parameter_unsupported, 723 "MT not compatible with static alloc"); 724 break; 725 726 case ZSTD_c_compressionLevel: 727 case ZSTD_c_windowLog: 728 case ZSTD_c_hashLog: 729 case ZSTD_c_chainLog: 730 case ZSTD_c_searchLog: 731 case ZSTD_c_minMatch: 732 case ZSTD_c_targetLength: 733 case ZSTD_c_strategy: 734 case ZSTD_c_ldmHashRateLog: 735 case ZSTD_c_format: 736 case ZSTD_c_contentSizeFlag: 737 case ZSTD_c_checksumFlag: 738 case ZSTD_c_dictIDFlag: 739 case ZSTD_c_forceMaxWindow: 740 case ZSTD_c_forceAttachDict: 741 case ZSTD_c_literalCompressionMode: 742 case ZSTD_c_jobSize: 743 case ZSTD_c_overlapLog: 744 case ZSTD_c_rsyncable: 745 case ZSTD_c_enableDedicatedDictSearch: 746 case ZSTD_c_enableLongDistanceMatching: 747 case ZSTD_c_ldmHashLog: 748 case ZSTD_c_ldmMinMatch: 749 case ZSTD_c_ldmBucketSizeLog: 750 case ZSTD_c_targetCBlockSize: 751 case ZSTD_c_srcSizeHint: 752 case ZSTD_c_stableInBuffer: 753 case ZSTD_c_stableOutBuffer: 754 case ZSTD_c_blockDelimiters: 755 case ZSTD_c_validateSequences: 756 case ZSTD_c_useBlockSplitter: 757 case ZSTD_c_useRowMatchFinder: 758 case ZSTD_c_deterministicRefPrefix: 759 case ZSTD_c_prefetchCDictTables: 760 case ZSTD_c_enableSeqProducerFallback: 761 case ZSTD_c_maxBlockSize: 762 case ZSTD_c_searchForExternalRepcodes: 763 break; 764 765 default: RETURN_ERROR(parameter_unsupported, "unknown parameter"); 766 } 767 return ZSTD_CCtxParams_setParameter(&cctx->requestedParams, param, value); 768 } 769 770 size_t ZSTD_CCtxParams_setParameter(ZSTD_CCtx_params* CCtxParams, 771 ZSTD_cParameter param, int value) 772 { 773 DEBUGLOG(4, "ZSTD_CCtxParams_setParameter (%i, %i)", (int)param, value); 774 switch(param) 775 { 776 case ZSTD_c_format : 777 BOUNDCHECK(ZSTD_c_format, value); 778 CCtxParams->format = (ZSTD_format_e)value; 779 return (size_t)CCtxParams->format; 780 781 case ZSTD_c_compressionLevel : { 782 FORWARD_IF_ERROR(ZSTD_cParam_clampBounds(param, &value), ""); 783 if (value == 0) 784 CCtxParams->compressionLevel = ZSTD_CLEVEL_DEFAULT; /* 0 == default */ 785 else 786 CCtxParams->compressionLevel = value; 787 if (CCtxParams->compressionLevel >= 0) return (size_t)CCtxParams->compressionLevel; 788 return 0; /* return type (size_t) cannot represent negative values */ 789 } 790 791 case ZSTD_c_windowLog : 792 if (value!=0) /* 0 => use default */ 793 BOUNDCHECK(ZSTD_c_windowLog, value); 794 CCtxParams->cParams.windowLog = (U32)value; 795 return CCtxParams->cParams.windowLog; 796 797 case ZSTD_c_hashLog : 798 if (value!=0) /* 0 => use default */ 799 BOUNDCHECK(ZSTD_c_hashLog, value); 800 CCtxParams->cParams.hashLog = (U32)value; 801 return CCtxParams->cParams.hashLog; 802 803 case ZSTD_c_chainLog : 804 if (value!=0) /* 0 => use default */ 805 BOUNDCHECK(ZSTD_c_chainLog, value); 806 CCtxParams->cParams.chainLog = (U32)value; 807 return CCtxParams->cParams.chainLog; 808 809 case ZSTD_c_searchLog : 810 if (value!=0) /* 0 => use default */ 811 BOUNDCHECK(ZSTD_c_searchLog, value); 812 CCtxParams->cParams.searchLog = (U32)value; 813 return (size_t)value; 814 815 case ZSTD_c_minMatch : 816 if (value!=0) /* 0 => use default */ 817 BOUNDCHECK(ZSTD_c_minMatch, value); 818 CCtxParams->cParams.minMatch = (U32)value; 819 return CCtxParams->cParams.minMatch; 820 821 case ZSTD_c_targetLength : 822 BOUNDCHECK(ZSTD_c_targetLength, value); 823 CCtxParams->cParams.targetLength = (U32)value; 824 return CCtxParams->cParams.targetLength; 825 826 case ZSTD_c_strategy : 827 if (value!=0) /* 0 => use default */ 828 BOUNDCHECK(ZSTD_c_strategy, value); 829 CCtxParams->cParams.strategy = (ZSTD_strategy)value; 830 return (size_t)CCtxParams->cParams.strategy; 831 832 case ZSTD_c_contentSizeFlag : 833 /* Content size written in frame header _when known_ (default:1) */ 834 DEBUGLOG(4, "set content size flag = %u", (value!=0)); 835 CCtxParams->fParams.contentSizeFlag = value != 0; 836 return (size_t)CCtxParams->fParams.contentSizeFlag; 837 838 case ZSTD_c_checksumFlag : 839 /* A 32-bits content checksum will be calculated and written at end of frame (default:0) */ 840 CCtxParams->fParams.checksumFlag = value != 0; 841 return (size_t)CCtxParams->fParams.checksumFlag; 842 843 case ZSTD_c_dictIDFlag : /* When applicable, dictionary's dictID is provided in frame header (default:1) */ 844 DEBUGLOG(4, "set dictIDFlag = %u", (value!=0)); 845 CCtxParams->fParams.noDictIDFlag = !value; 846 return !CCtxParams->fParams.noDictIDFlag; 847 848 case ZSTD_c_forceMaxWindow : 849 CCtxParams->forceWindow = (value != 0); 850 return (size_t)CCtxParams->forceWindow; 851 852 case ZSTD_c_forceAttachDict : { 853 const ZSTD_dictAttachPref_e pref = (ZSTD_dictAttachPref_e)value; 854 BOUNDCHECK(ZSTD_c_forceAttachDict, (int)pref); 855 CCtxParams->attachDictPref = pref; 856 return CCtxParams->attachDictPref; 857 } 858 859 case ZSTD_c_literalCompressionMode : { 860 const ZSTD_paramSwitch_e lcm = (ZSTD_paramSwitch_e)value; 861 BOUNDCHECK(ZSTD_c_literalCompressionMode, (int)lcm); 862 CCtxParams->literalCompressionMode = lcm; 863 return CCtxParams->literalCompressionMode; 864 } 865 866 case ZSTD_c_nbWorkers : 867 #ifndef ZSTD_MULTITHREAD 868 RETURN_ERROR_IF(value!=0, parameter_unsupported, "not compiled with multithreading"); 869 return 0; 870 #else 871 FORWARD_IF_ERROR(ZSTD_cParam_clampBounds(param, &value), ""); 872 CCtxParams->nbWorkers = value; 873 return (size_t)(CCtxParams->nbWorkers); 874 #endif 875 876 case ZSTD_c_jobSize : 877 #ifndef ZSTD_MULTITHREAD 878 RETURN_ERROR_IF(value!=0, parameter_unsupported, "not compiled with multithreading"); 879 return 0; 880 #else 881 /* Adjust to the minimum non-default value. */ 882 if (value != 0 && value < ZSTDMT_JOBSIZE_MIN) 883 value = ZSTDMT_JOBSIZE_MIN; 884 FORWARD_IF_ERROR(ZSTD_cParam_clampBounds(param, &value), ""); 885 assert(value >= 0); 886 CCtxParams->jobSize = value; 887 return CCtxParams->jobSize; 888 #endif 889 890 case ZSTD_c_overlapLog : 891 #ifndef ZSTD_MULTITHREAD 892 RETURN_ERROR_IF(value!=0, parameter_unsupported, "not compiled with multithreading"); 893 return 0; 894 #else 895 FORWARD_IF_ERROR(ZSTD_cParam_clampBounds(ZSTD_c_overlapLog, &value), ""); 896 CCtxParams->overlapLog = value; 897 return (size_t)CCtxParams->overlapLog; 898 #endif 899 900 case ZSTD_c_rsyncable : 901 #ifndef ZSTD_MULTITHREAD 902 RETURN_ERROR_IF(value!=0, parameter_unsupported, "not compiled with multithreading"); 903 return 0; 904 #else 905 FORWARD_IF_ERROR(ZSTD_cParam_clampBounds(ZSTD_c_overlapLog, &value), ""); 906 CCtxParams->rsyncable = value; 907 return (size_t)CCtxParams->rsyncable; 908 #endif 909 910 case ZSTD_c_enableDedicatedDictSearch : 911 CCtxParams->enableDedicatedDictSearch = (value!=0); 912 return (size_t)CCtxParams->enableDedicatedDictSearch; 913 914 case ZSTD_c_enableLongDistanceMatching : 915 BOUNDCHECK(ZSTD_c_enableLongDistanceMatching, value); 916 CCtxParams->ldmParams.enableLdm = (ZSTD_paramSwitch_e)value; 917 return CCtxParams->ldmParams.enableLdm; 918 919 case ZSTD_c_ldmHashLog : 920 if (value!=0) /* 0 ==> auto */ 921 BOUNDCHECK(ZSTD_c_ldmHashLog, value); 922 CCtxParams->ldmParams.hashLog = (U32)value; 923 return CCtxParams->ldmParams.hashLog; 924 925 case ZSTD_c_ldmMinMatch : 926 if (value!=0) /* 0 ==> default */ 927 BOUNDCHECK(ZSTD_c_ldmMinMatch, value); 928 CCtxParams->ldmParams.minMatchLength = (U32)value; 929 return CCtxParams->ldmParams.minMatchLength; 930 931 case ZSTD_c_ldmBucketSizeLog : 932 if (value!=0) /* 0 ==> default */ 933 BOUNDCHECK(ZSTD_c_ldmBucketSizeLog, value); 934 CCtxParams->ldmParams.bucketSizeLog = (U32)value; 935 return CCtxParams->ldmParams.bucketSizeLog; 936 937 case ZSTD_c_ldmHashRateLog : 938 if (value!=0) /* 0 ==> default */ 939 BOUNDCHECK(ZSTD_c_ldmHashRateLog, value); 940 CCtxParams->ldmParams.hashRateLog = (U32)value; 941 return CCtxParams->ldmParams.hashRateLog; 942 943 case ZSTD_c_targetCBlockSize : 944 if (value!=0) { /* 0 ==> default */ 945 value = MAX(value, ZSTD_TARGETCBLOCKSIZE_MIN); 946 BOUNDCHECK(ZSTD_c_targetCBlockSize, value); 947 } 948 CCtxParams->targetCBlockSize = (U32)value; 949 return CCtxParams->targetCBlockSize; 950 951 case ZSTD_c_srcSizeHint : 952 if (value!=0) /* 0 ==> default */ 953 BOUNDCHECK(ZSTD_c_srcSizeHint, value); 954 CCtxParams->srcSizeHint = value; 955 return (size_t)CCtxParams->srcSizeHint; 956 957 case ZSTD_c_stableInBuffer: 958 BOUNDCHECK(ZSTD_c_stableInBuffer, value); 959 CCtxParams->inBufferMode = (ZSTD_bufferMode_e)value; 960 return CCtxParams->inBufferMode; 961 962 case ZSTD_c_stableOutBuffer: 963 BOUNDCHECK(ZSTD_c_stableOutBuffer, value); 964 CCtxParams->outBufferMode = (ZSTD_bufferMode_e)value; 965 return CCtxParams->outBufferMode; 966 967 case ZSTD_c_blockDelimiters: 968 BOUNDCHECK(ZSTD_c_blockDelimiters, value); 969 CCtxParams->blockDelimiters = (ZSTD_sequenceFormat_e)value; 970 return CCtxParams->blockDelimiters; 971 972 case ZSTD_c_validateSequences: 973 BOUNDCHECK(ZSTD_c_validateSequences, value); 974 CCtxParams->validateSequences = value; 975 return (size_t)CCtxParams->validateSequences; 976 977 case ZSTD_c_useBlockSplitter: 978 BOUNDCHECK(ZSTD_c_useBlockSplitter, value); 979 CCtxParams->useBlockSplitter = (ZSTD_paramSwitch_e)value; 980 return CCtxParams->useBlockSplitter; 981 982 case ZSTD_c_useRowMatchFinder: 983 BOUNDCHECK(ZSTD_c_useRowMatchFinder, value); 984 CCtxParams->useRowMatchFinder = (ZSTD_paramSwitch_e)value; 985 return CCtxParams->useRowMatchFinder; 986 987 case ZSTD_c_deterministicRefPrefix: 988 BOUNDCHECK(ZSTD_c_deterministicRefPrefix, value); 989 CCtxParams->deterministicRefPrefix = !!value; 990 return (size_t)CCtxParams->deterministicRefPrefix; 991 992 case ZSTD_c_prefetchCDictTables: 993 BOUNDCHECK(ZSTD_c_prefetchCDictTables, value); 994 CCtxParams->prefetchCDictTables = (ZSTD_paramSwitch_e)value; 995 return CCtxParams->prefetchCDictTables; 996 997 case ZSTD_c_enableSeqProducerFallback: 998 BOUNDCHECK(ZSTD_c_enableSeqProducerFallback, value); 999 CCtxParams->enableMatchFinderFallback = value; 1000 return (size_t)CCtxParams->enableMatchFinderFallback; 1001 1002 case ZSTD_c_maxBlockSize: 1003 if (value!=0) /* 0 ==> default */ 1004 BOUNDCHECK(ZSTD_c_maxBlockSize, value); 1005 CCtxParams->maxBlockSize = value; 1006 return CCtxParams->maxBlockSize; 1007 1008 case ZSTD_c_searchForExternalRepcodes: 1009 BOUNDCHECK(ZSTD_c_searchForExternalRepcodes, value); 1010 CCtxParams->searchForExternalRepcodes = (ZSTD_paramSwitch_e)value; 1011 return CCtxParams->searchForExternalRepcodes; 1012 1013 default: RETURN_ERROR(parameter_unsupported, "unknown parameter"); 1014 } 1015 } 1016 1017 size_t ZSTD_CCtx_getParameter(ZSTD_CCtx const* cctx, ZSTD_cParameter param, int* value) 1018 { 1019 return ZSTD_CCtxParams_getParameter(&cctx->requestedParams, param, value); 1020 } 1021 1022 size_t ZSTD_CCtxParams_getParameter( 1023 ZSTD_CCtx_params const* CCtxParams, ZSTD_cParameter param, int* value) 1024 { 1025 switch(param) 1026 { 1027 case ZSTD_c_format : 1028 *value = CCtxParams->format; 1029 break; 1030 case ZSTD_c_compressionLevel : 1031 *value = CCtxParams->compressionLevel; 1032 break; 1033 case ZSTD_c_windowLog : 1034 *value = (int)CCtxParams->cParams.windowLog; 1035 break; 1036 case ZSTD_c_hashLog : 1037 *value = (int)CCtxParams->cParams.hashLog; 1038 break; 1039 case ZSTD_c_chainLog : 1040 *value = (int)CCtxParams->cParams.chainLog; 1041 break; 1042 case ZSTD_c_searchLog : 1043 *value = CCtxParams->cParams.searchLog; 1044 break; 1045 case ZSTD_c_minMatch : 1046 *value = CCtxParams->cParams.minMatch; 1047 break; 1048 case ZSTD_c_targetLength : 1049 *value = CCtxParams->cParams.targetLength; 1050 break; 1051 case ZSTD_c_strategy : 1052 *value = (unsigned)CCtxParams->cParams.strategy; 1053 break; 1054 case ZSTD_c_contentSizeFlag : 1055 *value = CCtxParams->fParams.contentSizeFlag; 1056 break; 1057 case ZSTD_c_checksumFlag : 1058 *value = CCtxParams->fParams.checksumFlag; 1059 break; 1060 case ZSTD_c_dictIDFlag : 1061 *value = !CCtxParams->fParams.noDictIDFlag; 1062 break; 1063 case ZSTD_c_forceMaxWindow : 1064 *value = CCtxParams->forceWindow; 1065 break; 1066 case ZSTD_c_forceAttachDict : 1067 *value = CCtxParams->attachDictPref; 1068 break; 1069 case ZSTD_c_literalCompressionMode : 1070 *value = CCtxParams->literalCompressionMode; 1071 break; 1072 case ZSTD_c_nbWorkers : 1073 #ifndef ZSTD_MULTITHREAD 1074 assert(CCtxParams->nbWorkers == 0); 1075 #endif 1076 *value = CCtxParams->nbWorkers; 1077 break; 1078 case ZSTD_c_jobSize : 1079 #ifndef ZSTD_MULTITHREAD 1080 RETURN_ERROR(parameter_unsupported, "not compiled with multithreading"); 1081 #else 1082 assert(CCtxParams->jobSize <= INT_MAX); 1083 *value = (int)CCtxParams->jobSize; 1084 break; 1085 #endif 1086 case ZSTD_c_overlapLog : 1087 #ifndef ZSTD_MULTITHREAD 1088 RETURN_ERROR(parameter_unsupported, "not compiled with multithreading"); 1089 #else 1090 *value = CCtxParams->overlapLog; 1091 break; 1092 #endif 1093 case ZSTD_c_rsyncable : 1094 #ifndef ZSTD_MULTITHREAD 1095 RETURN_ERROR(parameter_unsupported, "not compiled with multithreading"); 1096 #else 1097 *value = CCtxParams->rsyncable; 1098 break; 1099 #endif 1100 case ZSTD_c_enableDedicatedDictSearch : 1101 *value = CCtxParams->enableDedicatedDictSearch; 1102 break; 1103 case ZSTD_c_enableLongDistanceMatching : 1104 *value = CCtxParams->ldmParams.enableLdm; 1105 break; 1106 case ZSTD_c_ldmHashLog : 1107 *value = CCtxParams->ldmParams.hashLog; 1108 break; 1109 case ZSTD_c_ldmMinMatch : 1110 *value = CCtxParams->ldmParams.minMatchLength; 1111 break; 1112 case ZSTD_c_ldmBucketSizeLog : 1113 *value = CCtxParams->ldmParams.bucketSizeLog; 1114 break; 1115 case ZSTD_c_ldmHashRateLog : 1116 *value = CCtxParams->ldmParams.hashRateLog; 1117 break; 1118 case ZSTD_c_targetCBlockSize : 1119 *value = (int)CCtxParams->targetCBlockSize; 1120 break; 1121 case ZSTD_c_srcSizeHint : 1122 *value = (int)CCtxParams->srcSizeHint; 1123 break; 1124 case ZSTD_c_stableInBuffer : 1125 *value = (int)CCtxParams->inBufferMode; 1126 break; 1127 case ZSTD_c_stableOutBuffer : 1128 *value = (int)CCtxParams->outBufferMode; 1129 break; 1130 case ZSTD_c_blockDelimiters : 1131 *value = (int)CCtxParams->blockDelimiters; 1132 break; 1133 case ZSTD_c_validateSequences : 1134 *value = (int)CCtxParams->validateSequences; 1135 break; 1136 case ZSTD_c_useBlockSplitter : 1137 *value = (int)CCtxParams->useBlockSplitter; 1138 break; 1139 case ZSTD_c_useRowMatchFinder : 1140 *value = (int)CCtxParams->useRowMatchFinder; 1141 break; 1142 case ZSTD_c_deterministicRefPrefix: 1143 *value = (int)CCtxParams->deterministicRefPrefix; 1144 break; 1145 case ZSTD_c_prefetchCDictTables: 1146 *value = (int)CCtxParams->prefetchCDictTables; 1147 break; 1148 case ZSTD_c_enableSeqProducerFallback: 1149 *value = CCtxParams->enableMatchFinderFallback; 1150 break; 1151 case ZSTD_c_maxBlockSize: 1152 *value = (int)CCtxParams->maxBlockSize; 1153 break; 1154 case ZSTD_c_searchForExternalRepcodes: 1155 *value = (int)CCtxParams->searchForExternalRepcodes; 1156 break; 1157 default: RETURN_ERROR(parameter_unsupported, "unknown parameter"); 1158 } 1159 return 0; 1160 } 1161 1162 /** ZSTD_CCtx_setParametersUsingCCtxParams() : 1163 * just applies `params` into `cctx` 1164 * no action is performed, parameters are merely stored. 1165 * If ZSTDMT is enabled, parameters are pushed to cctx->mtctx. 1166 * This is possible even if a compression is ongoing. 1167 * In which case, new parameters will be applied on the fly, starting with next compression job. 1168 */ 1169 size_t ZSTD_CCtx_setParametersUsingCCtxParams( 1170 ZSTD_CCtx* cctx, const ZSTD_CCtx_params* params) 1171 { 1172 DEBUGLOG(4, "ZSTD_CCtx_setParametersUsingCCtxParams"); 1173 RETURN_ERROR_IF(cctx->streamStage != zcss_init, stage_wrong, 1174 "The context is in the wrong stage!"); 1175 RETURN_ERROR_IF(cctx->cdict, stage_wrong, 1176 "Can't override parameters with cdict attached (some must " 1177 "be inherited from the cdict)."); 1178 1179 cctx->requestedParams = *params; 1180 return 0; 1181 } 1182 1183 size_t ZSTD_CCtx_setCParams(ZSTD_CCtx* cctx, ZSTD_compressionParameters cparams) 1184 { 1185 ZSTD_STATIC_ASSERT(sizeof(cparams) == 7 * 4 /* all params are listed below */); 1186 DEBUGLOG(4, "ZSTD_CCtx_setCParams"); 1187 /* only update if all parameters are valid */ 1188 FORWARD_IF_ERROR(ZSTD_checkCParams(cparams), ""); 1189 FORWARD_IF_ERROR(ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, cparams.windowLog), ""); 1190 FORWARD_IF_ERROR(ZSTD_CCtx_setParameter(cctx, ZSTD_c_chainLog, cparams.chainLog), ""); 1191 FORWARD_IF_ERROR(ZSTD_CCtx_setParameter(cctx, ZSTD_c_hashLog, cparams.hashLog), ""); 1192 FORWARD_IF_ERROR(ZSTD_CCtx_setParameter(cctx, ZSTD_c_searchLog, cparams.searchLog), ""); 1193 FORWARD_IF_ERROR(ZSTD_CCtx_setParameter(cctx, ZSTD_c_minMatch, cparams.minMatch), ""); 1194 FORWARD_IF_ERROR(ZSTD_CCtx_setParameter(cctx, ZSTD_c_targetLength, cparams.targetLength), ""); 1195 FORWARD_IF_ERROR(ZSTD_CCtx_setParameter(cctx, ZSTD_c_strategy, cparams.strategy), ""); 1196 return 0; 1197 } 1198 1199 size_t ZSTD_CCtx_setFParams(ZSTD_CCtx* cctx, ZSTD_frameParameters fparams) 1200 { 1201 ZSTD_STATIC_ASSERT(sizeof(fparams) == 3 * 4 /* all params are listed below */); 1202 DEBUGLOG(4, "ZSTD_CCtx_setFParams"); 1203 FORWARD_IF_ERROR(ZSTD_CCtx_setParameter(cctx, ZSTD_c_contentSizeFlag, fparams.contentSizeFlag != 0), ""); 1204 FORWARD_IF_ERROR(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, fparams.checksumFlag != 0), ""); 1205 FORWARD_IF_ERROR(ZSTD_CCtx_setParameter(cctx, ZSTD_c_dictIDFlag, fparams.noDictIDFlag == 0), ""); 1206 return 0; 1207 } 1208 1209 size_t ZSTD_CCtx_setParams(ZSTD_CCtx* cctx, ZSTD_parameters params) 1210 { 1211 DEBUGLOG(4, "ZSTD_CCtx_setParams"); 1212 /* First check cParams, because we want to update all or none. */ 1213 FORWARD_IF_ERROR(ZSTD_checkCParams(params.cParams), ""); 1214 /* Next set fParams, because this could fail if the cctx isn't in init stage. */ 1215 FORWARD_IF_ERROR(ZSTD_CCtx_setFParams(cctx, params.fParams), ""); 1216 /* Finally set cParams, which should succeed. */ 1217 FORWARD_IF_ERROR(ZSTD_CCtx_setCParams(cctx, params.cParams), ""); 1218 return 0; 1219 } 1220 1221 size_t ZSTD_CCtx_setPledgedSrcSize(ZSTD_CCtx* cctx, unsigned long long pledgedSrcSize) 1222 { 1223 DEBUGLOG(4, "ZSTD_CCtx_setPledgedSrcSize to %llu bytes", pledgedSrcSize); 1224 RETURN_ERROR_IF(cctx->streamStage != zcss_init, stage_wrong, 1225 "Can't set pledgedSrcSize when not in init stage."); 1226 cctx->pledgedSrcSizePlusOne = pledgedSrcSize+1; 1227 return 0; 1228 } 1229 1230 static ZSTD_compressionParameters ZSTD_dedicatedDictSearch_getCParams( 1231 int const compressionLevel, 1232 size_t const dictSize); 1233 static int ZSTD_dedicatedDictSearch_isSupported( 1234 const ZSTD_compressionParameters* cParams); 1235 static void ZSTD_dedicatedDictSearch_revertCParams( 1236 ZSTD_compressionParameters* cParams); 1237 1238 /** 1239 * Initializes the local dictionary using requested parameters. 1240 * NOTE: Initialization does not employ the pledged src size, 1241 * because the dictionary may be used for multiple compressions. 1242 */ 1243 static size_t ZSTD_initLocalDict(ZSTD_CCtx* cctx) 1244 { 1245 ZSTD_localDict* const dl = &cctx->localDict; 1246 if (dl->dict == NULL) { 1247 /* No local dictionary. */ 1248 assert(dl->dictBuffer == NULL); 1249 assert(dl->cdict == NULL); 1250 assert(dl->dictSize == 0); 1251 return 0; 1252 } 1253 if (dl->cdict != NULL) { 1254 /* Local dictionary already initialized. */ 1255 assert(cctx->cdict == dl->cdict); 1256 return 0; 1257 } 1258 assert(dl->dictSize > 0); 1259 assert(cctx->cdict == NULL); 1260 assert(cctx->prefixDict.dict == NULL); 1261 1262 dl->cdict = ZSTD_createCDict_advanced2( 1263 dl->dict, 1264 dl->dictSize, 1265 ZSTD_dlm_byRef, 1266 dl->dictContentType, 1267 &cctx->requestedParams, 1268 cctx->customMem); 1269 RETURN_ERROR_IF(!dl->cdict, memory_allocation, "ZSTD_createCDict_advanced failed"); 1270 cctx->cdict = dl->cdict; 1271 return 0; 1272 } 1273 1274 size_t ZSTD_CCtx_loadDictionary_advanced( 1275 ZSTD_CCtx* cctx, 1276 const void* dict, size_t dictSize, 1277 ZSTD_dictLoadMethod_e dictLoadMethod, 1278 ZSTD_dictContentType_e dictContentType) 1279 { 1280 DEBUGLOG(4, "ZSTD_CCtx_loadDictionary_advanced (size: %u)", (U32)dictSize); 1281 RETURN_ERROR_IF(cctx->streamStage != zcss_init, stage_wrong, 1282 "Can't load a dictionary when cctx is not in init stage."); 1283 ZSTD_clearAllDicts(cctx); /* erase any previously set dictionary */ 1284 if (dict == NULL || dictSize == 0) /* no dictionary */ 1285 return 0; 1286 if (dictLoadMethod == ZSTD_dlm_byRef) { 1287 cctx->localDict.dict = dict; 1288 } else { 1289 /* copy dictionary content inside CCtx to own its lifetime */ 1290 void* dictBuffer; 1291 RETURN_ERROR_IF(cctx->staticSize, memory_allocation, 1292 "static CCtx can't allocate for an internal copy of dictionary"); 1293 dictBuffer = ZSTD_customMalloc(dictSize, cctx->customMem); 1294 RETURN_ERROR_IF(dictBuffer==NULL, memory_allocation, 1295 "allocation failed for dictionary content"); 1296 ZSTD_memcpy(dictBuffer, dict, dictSize); 1297 cctx->localDict.dictBuffer = dictBuffer; /* owned ptr to free */ 1298 cctx->localDict.dict = dictBuffer; /* read-only reference */ 1299 } 1300 cctx->localDict.dictSize = dictSize; 1301 cctx->localDict.dictContentType = dictContentType; 1302 return 0; 1303 } 1304 1305 size_t ZSTD_CCtx_loadDictionary_byReference( 1306 ZSTD_CCtx* cctx, const void* dict, size_t dictSize) 1307 { 1308 return ZSTD_CCtx_loadDictionary_advanced( 1309 cctx, dict, dictSize, ZSTD_dlm_byRef, ZSTD_dct_auto); 1310 } 1311 1312 size_t ZSTD_CCtx_loadDictionary(ZSTD_CCtx* cctx, const void* dict, size_t dictSize) 1313 { 1314 return ZSTD_CCtx_loadDictionary_advanced( 1315 cctx, dict, dictSize, ZSTD_dlm_byCopy, ZSTD_dct_auto); 1316 } 1317 1318 1319 size_t ZSTD_CCtx_refCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict) 1320 { 1321 RETURN_ERROR_IF(cctx->streamStage != zcss_init, stage_wrong, 1322 "Can't ref a dict when ctx not in init stage."); 1323 /* Free the existing local cdict (if any) to save memory. */ 1324 ZSTD_clearAllDicts(cctx); 1325 cctx->cdict = cdict; 1326 return 0; 1327 } 1328 1329 size_t ZSTD_CCtx_refThreadPool(ZSTD_CCtx* cctx, ZSTD_threadPool* pool) 1330 { 1331 RETURN_ERROR_IF(cctx->streamStage != zcss_init, stage_wrong, 1332 "Can't ref a pool when ctx not in init stage."); 1333 cctx->pool = pool; 1334 return 0; 1335 } 1336 1337 size_t ZSTD_CCtx_refPrefix(ZSTD_CCtx* cctx, const void* prefix, size_t prefixSize) 1338 { 1339 return ZSTD_CCtx_refPrefix_advanced(cctx, prefix, prefixSize, ZSTD_dct_rawContent); 1340 } 1341 1342 size_t ZSTD_CCtx_refPrefix_advanced( 1343 ZSTD_CCtx* cctx, const void* prefix, size_t prefixSize, ZSTD_dictContentType_e dictContentType) 1344 { 1345 RETURN_ERROR_IF(cctx->streamStage != zcss_init, stage_wrong, 1346 "Can't ref a prefix when ctx not in init stage."); 1347 ZSTD_clearAllDicts(cctx); 1348 if (prefix != NULL && prefixSize > 0) { 1349 cctx->prefixDict.dict = prefix; 1350 cctx->prefixDict.dictSize = prefixSize; 1351 cctx->prefixDict.dictContentType = dictContentType; 1352 } 1353 return 0; 1354 } 1355 1356 /*! ZSTD_CCtx_reset() : 1357 * Also dumps dictionary */ 1358 size_t ZSTD_CCtx_reset(ZSTD_CCtx* cctx, ZSTD_ResetDirective reset) 1359 { 1360 if ( (reset == ZSTD_reset_session_only) 1361 || (reset == ZSTD_reset_session_and_parameters) ) { 1362 cctx->streamStage = zcss_init; 1363 cctx->pledgedSrcSizePlusOne = 0; 1364 } 1365 if ( (reset == ZSTD_reset_parameters) 1366 || (reset == ZSTD_reset_session_and_parameters) ) { 1367 RETURN_ERROR_IF(cctx->streamStage != zcss_init, stage_wrong, 1368 "Reset parameters is only possible during init stage."); 1369 ZSTD_clearAllDicts(cctx); 1370 return ZSTD_CCtxParams_reset(&cctx->requestedParams); 1371 } 1372 return 0; 1373 } 1374 1375 1376 /** ZSTD_checkCParams() : 1377 control CParam values remain within authorized range. 1378 @return : 0, or an error code if one value is beyond authorized range */ 1379 size_t ZSTD_checkCParams(ZSTD_compressionParameters cParams) 1380 { 1381 BOUNDCHECK(ZSTD_c_windowLog, (int)cParams.windowLog); 1382 BOUNDCHECK(ZSTD_c_chainLog, (int)cParams.chainLog); 1383 BOUNDCHECK(ZSTD_c_hashLog, (int)cParams.hashLog); 1384 BOUNDCHECK(ZSTD_c_searchLog, (int)cParams.searchLog); 1385 BOUNDCHECK(ZSTD_c_minMatch, (int)cParams.minMatch); 1386 BOUNDCHECK(ZSTD_c_targetLength,(int)cParams.targetLength); 1387 BOUNDCHECK(ZSTD_c_strategy, cParams.strategy); 1388 return 0; 1389 } 1390 1391 /** ZSTD_clampCParams() : 1392 * make CParam values within valid range. 1393 * @return : valid CParams */ 1394 static ZSTD_compressionParameters 1395 ZSTD_clampCParams(ZSTD_compressionParameters cParams) 1396 { 1397 # define CLAMP_TYPE(cParam, val, type) \ 1398 do { \ 1399 ZSTD_bounds const bounds = ZSTD_cParam_getBounds(cParam); \ 1400 if ((int)val<bounds.lowerBound) val=(type)bounds.lowerBound; \ 1401 else if ((int)val>bounds.upperBound) val=(type)bounds.upperBound; \ 1402 } while (0) 1403 # define CLAMP(cParam, val) CLAMP_TYPE(cParam, val, unsigned) 1404 CLAMP(ZSTD_c_windowLog, cParams.windowLog); 1405 CLAMP(ZSTD_c_chainLog, cParams.chainLog); 1406 CLAMP(ZSTD_c_hashLog, cParams.hashLog); 1407 CLAMP(ZSTD_c_searchLog, cParams.searchLog); 1408 CLAMP(ZSTD_c_minMatch, cParams.minMatch); 1409 CLAMP(ZSTD_c_targetLength,cParams.targetLength); 1410 CLAMP_TYPE(ZSTD_c_strategy,cParams.strategy, ZSTD_strategy); 1411 return cParams; 1412 } 1413 1414 /** ZSTD_cycleLog() : 1415 * condition for correct operation : hashLog > 1 */ 1416 U32 ZSTD_cycleLog(U32 hashLog, ZSTD_strategy strat) 1417 { 1418 U32 const btScale = ((U32)strat >= (U32)ZSTD_btlazy2); 1419 return hashLog - btScale; 1420 } 1421 1422 /** ZSTD_dictAndWindowLog() : 1423 * Returns an adjusted window log that is large enough to fit the source and the dictionary. 1424 * The zstd format says that the entire dictionary is valid if one byte of the dictionary 1425 * is within the window. So the hashLog and chainLog should be large enough to reference both 1426 * the dictionary and the window. So we must use this adjusted dictAndWindowLog when downsizing 1427 * the hashLog and windowLog. 1428 * NOTE: srcSize must not be ZSTD_CONTENTSIZE_UNKNOWN. 1429 */ 1430 static U32 ZSTD_dictAndWindowLog(U32 windowLog, U64 srcSize, U64 dictSize) 1431 { 1432 const U64 maxWindowSize = 1ULL << ZSTD_WINDOWLOG_MAX; 1433 /* No dictionary ==> No change */ 1434 if (dictSize == 0) { 1435 return windowLog; 1436 } 1437 assert(windowLog <= ZSTD_WINDOWLOG_MAX); 1438 assert(srcSize != ZSTD_CONTENTSIZE_UNKNOWN); /* Handled in ZSTD_adjustCParams_internal() */ 1439 { 1440 U64 const windowSize = 1ULL << windowLog; 1441 U64 const dictAndWindowSize = dictSize + windowSize; 1442 /* If the window size is already large enough to fit both the source and the dictionary 1443 * then just use the window size. Otherwise adjust so that it fits the dictionary and 1444 * the window. 1445 */ 1446 if (windowSize >= dictSize + srcSize) { 1447 return windowLog; /* Window size large enough already */ 1448 } else if (dictAndWindowSize >= maxWindowSize) { 1449 return ZSTD_WINDOWLOG_MAX; /* Larger than max window log */ 1450 } else { 1451 return ZSTD_highbit32((U32)dictAndWindowSize - 1) + 1; 1452 } 1453 } 1454 } 1455 1456 /** ZSTD_adjustCParams_internal() : 1457 * optimize `cPar` for a specified input (`srcSize` and `dictSize`). 1458 * mostly downsize to reduce memory consumption and initialization latency. 1459 * `srcSize` can be ZSTD_CONTENTSIZE_UNKNOWN when not known. 1460 * `mode` is the mode for parameter adjustment. See docs for `ZSTD_cParamMode_e`. 1461 * note : `srcSize==0` means 0! 1462 * condition : cPar is presumed validated (can be checked using ZSTD_checkCParams()). */ 1463 static ZSTD_compressionParameters 1464 ZSTD_adjustCParams_internal(ZSTD_compressionParameters cPar, 1465 unsigned long long srcSize, 1466 size_t dictSize, 1467 ZSTD_cParamMode_e mode, 1468 ZSTD_paramSwitch_e useRowMatchFinder) 1469 { 1470 const U64 minSrcSize = 513; /* (1<<9) + 1 */ 1471 const U64 maxWindowResize = 1ULL << (ZSTD_WINDOWLOG_MAX-1); 1472 assert(ZSTD_checkCParams(cPar)==0); 1473 1474 /* Cascade the selected strategy down to the next-highest one built into 1475 * this binary. */ 1476 #ifdef ZSTD_EXCLUDE_BTULTRA_BLOCK_COMPRESSOR 1477 if (cPar.strategy == ZSTD_btultra2) { 1478 cPar.strategy = ZSTD_btultra; 1479 } 1480 if (cPar.strategy == ZSTD_btultra) { 1481 cPar.strategy = ZSTD_btopt; 1482 } 1483 #endif 1484 #ifdef ZSTD_EXCLUDE_BTOPT_BLOCK_COMPRESSOR 1485 if (cPar.strategy == ZSTD_btopt) { 1486 cPar.strategy = ZSTD_btlazy2; 1487 } 1488 #endif 1489 #ifdef ZSTD_EXCLUDE_BTLAZY2_BLOCK_COMPRESSOR 1490 if (cPar.strategy == ZSTD_btlazy2) { 1491 cPar.strategy = ZSTD_lazy2; 1492 } 1493 #endif 1494 #ifdef ZSTD_EXCLUDE_LAZY2_BLOCK_COMPRESSOR 1495 if (cPar.strategy == ZSTD_lazy2) { 1496 cPar.strategy = ZSTD_lazy; 1497 } 1498 #endif 1499 #ifdef ZSTD_EXCLUDE_LAZY_BLOCK_COMPRESSOR 1500 if (cPar.strategy == ZSTD_lazy) { 1501 cPar.strategy = ZSTD_greedy; 1502 } 1503 #endif 1504 #ifdef ZSTD_EXCLUDE_GREEDY_BLOCK_COMPRESSOR 1505 if (cPar.strategy == ZSTD_greedy) { 1506 cPar.strategy = ZSTD_dfast; 1507 } 1508 #endif 1509 #ifdef ZSTD_EXCLUDE_DFAST_BLOCK_COMPRESSOR 1510 if (cPar.strategy == ZSTD_dfast) { 1511 cPar.strategy = ZSTD_fast; 1512 cPar.targetLength = 0; 1513 } 1514 #endif 1515 1516 switch (mode) { 1517 case ZSTD_cpm_unknown: 1518 case ZSTD_cpm_noAttachDict: 1519 /* If we don't know the source size, don't make any 1520 * assumptions about it. We will already have selected 1521 * smaller parameters if a dictionary is in use. 1522 */ 1523 break; 1524 case ZSTD_cpm_createCDict: 1525 /* Assume a small source size when creating a dictionary 1526 * with an unknown source size. 1527 */ 1528 if (dictSize && srcSize == ZSTD_CONTENTSIZE_UNKNOWN) 1529 srcSize = minSrcSize; 1530 break; 1531 case ZSTD_cpm_attachDict: 1532 /* Dictionary has its own dedicated parameters which have 1533 * already been selected. We are selecting parameters 1534 * for only the source. 1535 */ 1536 dictSize = 0; 1537 break; 1538 default: 1539 assert(0); 1540 break; 1541 } 1542 1543 /* resize windowLog if input is small enough, to use less memory */ 1544 if ( (srcSize <= maxWindowResize) 1545 && (dictSize <= maxWindowResize) ) { 1546 U32 const tSize = (U32)(srcSize + dictSize); 1547 static U32 const hashSizeMin = 1 << ZSTD_HASHLOG_MIN; 1548 U32 const srcLog = (tSize < hashSizeMin) ? ZSTD_HASHLOG_MIN : 1549 ZSTD_highbit32(tSize-1) + 1; 1550 if (cPar.windowLog > srcLog) cPar.windowLog = srcLog; 1551 } 1552 if (srcSize != ZSTD_CONTENTSIZE_UNKNOWN) { 1553 U32 const dictAndWindowLog = ZSTD_dictAndWindowLog(cPar.windowLog, (U64)srcSize, (U64)dictSize); 1554 U32 const cycleLog = ZSTD_cycleLog(cPar.chainLog, cPar.strategy); 1555 if (cPar.hashLog > dictAndWindowLog+1) cPar.hashLog = dictAndWindowLog+1; 1556 if (cycleLog > dictAndWindowLog) 1557 cPar.chainLog -= (cycleLog - dictAndWindowLog); 1558 } 1559 1560 if (cPar.windowLog < ZSTD_WINDOWLOG_ABSOLUTEMIN) 1561 cPar.windowLog = ZSTD_WINDOWLOG_ABSOLUTEMIN; /* minimum wlog required for valid frame header */ 1562 1563 /* We can't use more than 32 bits of hash in total, so that means that we require: 1564 * (hashLog + 8) <= 32 && (chainLog + 8) <= 32 1565 */ 1566 if (mode == ZSTD_cpm_createCDict && ZSTD_CDictIndicesAreTagged(&cPar)) { 1567 U32 const maxShortCacheHashLog = 32 - ZSTD_SHORT_CACHE_TAG_BITS; 1568 if (cPar.hashLog > maxShortCacheHashLog) { 1569 cPar.hashLog = maxShortCacheHashLog; 1570 } 1571 if (cPar.chainLog > maxShortCacheHashLog) { 1572 cPar.chainLog = maxShortCacheHashLog; 1573 } 1574 } 1575 1576 1577 /* At this point, we aren't 100% sure if we are using the row match finder. 1578 * Unless it is explicitly disabled, conservatively assume that it is enabled. 1579 * In this case it will only be disabled for small sources, so shrinking the 1580 * hash log a little bit shouldn't result in any ratio loss. 1581 */ 1582 if (useRowMatchFinder == ZSTD_ps_auto) 1583 useRowMatchFinder = ZSTD_ps_enable; 1584 1585 /* We can't hash more than 32-bits in total. So that means that we require: 1586 * (hashLog - rowLog + 8) <= 32 1587 */ 1588 if (ZSTD_rowMatchFinderUsed(cPar.strategy, useRowMatchFinder)) { 1589 /* Switch to 32-entry rows if searchLog is 5 (or more) */ 1590 U32 const rowLog = BOUNDED(4, cPar.searchLog, 6); 1591 U32 const maxRowHashLog = 32 - ZSTD_ROW_HASH_TAG_BITS; 1592 U32 const maxHashLog = maxRowHashLog + rowLog; 1593 assert(cPar.hashLog >= rowLog); 1594 if (cPar.hashLog > maxHashLog) { 1595 cPar.hashLog = maxHashLog; 1596 } 1597 } 1598 1599 return cPar; 1600 } 1601 1602 ZSTD_compressionParameters 1603 ZSTD_adjustCParams(ZSTD_compressionParameters cPar, 1604 unsigned long long srcSize, 1605 size_t dictSize) 1606 { 1607 cPar = ZSTD_clampCParams(cPar); /* resulting cPar is necessarily valid (all parameters within range) */ 1608 if (srcSize == 0) srcSize = ZSTD_CONTENTSIZE_UNKNOWN; 1609 return ZSTD_adjustCParams_internal(cPar, srcSize, dictSize, ZSTD_cpm_unknown, ZSTD_ps_auto); 1610 } 1611 1612 static ZSTD_compressionParameters ZSTD_getCParams_internal(int compressionLevel, unsigned long long srcSizeHint, size_t dictSize, ZSTD_cParamMode_e mode); 1613 static ZSTD_parameters ZSTD_getParams_internal(int compressionLevel, unsigned long long srcSizeHint, size_t dictSize, ZSTD_cParamMode_e mode); 1614 1615 static void ZSTD_overrideCParams( 1616 ZSTD_compressionParameters* cParams, 1617 const ZSTD_compressionParameters* overrides) 1618 { 1619 if (overrides->windowLog) cParams->windowLog = overrides->windowLog; 1620 if (overrides->hashLog) cParams->hashLog = overrides->hashLog; 1621 if (overrides->chainLog) cParams->chainLog = overrides->chainLog; 1622 if (overrides->searchLog) cParams->searchLog = overrides->searchLog; 1623 if (overrides->minMatch) cParams->minMatch = overrides->minMatch; 1624 if (overrides->targetLength) cParams->targetLength = overrides->targetLength; 1625 if (overrides->strategy) cParams->strategy = overrides->strategy; 1626 } 1627 1628 ZSTD_compressionParameters ZSTD_getCParamsFromCCtxParams( 1629 const ZSTD_CCtx_params* CCtxParams, U64 srcSizeHint, size_t dictSize, ZSTD_cParamMode_e mode) 1630 { 1631 ZSTD_compressionParameters cParams; 1632 if (srcSizeHint == ZSTD_CONTENTSIZE_UNKNOWN && CCtxParams->srcSizeHint > 0) { 1633 srcSizeHint = CCtxParams->srcSizeHint; 1634 } 1635 cParams = ZSTD_getCParams_internal(CCtxParams->compressionLevel, srcSizeHint, dictSize, mode); 1636 if (CCtxParams->ldmParams.enableLdm == ZSTD_ps_enable) cParams.windowLog = ZSTD_LDM_DEFAULT_WINDOW_LOG; 1637 ZSTD_overrideCParams(&cParams, &CCtxParams->cParams); 1638 assert(!ZSTD_checkCParams(cParams)); 1639 /* srcSizeHint == 0 means 0 */ 1640 return ZSTD_adjustCParams_internal(cParams, srcSizeHint, dictSize, mode, CCtxParams->useRowMatchFinder); 1641 } 1642 1643 static size_t 1644 ZSTD_sizeof_matchState(const ZSTD_compressionParameters* const cParams, 1645 const ZSTD_paramSwitch_e useRowMatchFinder, 1646 const U32 enableDedicatedDictSearch, 1647 const U32 forCCtx) 1648 { 1649 /* chain table size should be 0 for fast or row-hash strategies */ 1650 size_t const chainSize = ZSTD_allocateChainTable(cParams->strategy, useRowMatchFinder, enableDedicatedDictSearch && !forCCtx) 1651 ? ((size_t)1 << cParams->chainLog) 1652 : 0; 1653 size_t const hSize = ((size_t)1) << cParams->hashLog; 1654 U32 const hashLog3 = (forCCtx && cParams->minMatch==3) ? MIN(ZSTD_HASHLOG3_MAX, cParams->windowLog) : 0; 1655 size_t const h3Size = hashLog3 ? ((size_t)1) << hashLog3 : 0; 1656 /* We don't use ZSTD_cwksp_alloc_size() here because the tables aren't 1657 * surrounded by redzones in ASAN. */ 1658 size_t const tableSpace = chainSize * sizeof(U32) 1659 + hSize * sizeof(U32) 1660 + h3Size * sizeof(U32); 1661 size_t const optPotentialSpace = 1662 ZSTD_cwksp_aligned_alloc_size((MaxML+1) * sizeof(U32)) 1663 + ZSTD_cwksp_aligned_alloc_size((MaxLL+1) * sizeof(U32)) 1664 + ZSTD_cwksp_aligned_alloc_size((MaxOff+1) * sizeof(U32)) 1665 + ZSTD_cwksp_aligned_alloc_size((1<<Litbits) * sizeof(U32)) 1666 + ZSTD_cwksp_aligned_alloc_size(ZSTD_OPT_SIZE * sizeof(ZSTD_match_t)) 1667 + ZSTD_cwksp_aligned_alloc_size(ZSTD_OPT_SIZE * sizeof(ZSTD_optimal_t)); 1668 size_t const lazyAdditionalSpace = ZSTD_rowMatchFinderUsed(cParams->strategy, useRowMatchFinder) 1669 ? ZSTD_cwksp_aligned_alloc_size(hSize) 1670 : 0; 1671 size_t const optSpace = (forCCtx && (cParams->strategy >= ZSTD_btopt)) 1672 ? optPotentialSpace 1673 : 0; 1674 size_t const slackSpace = ZSTD_cwksp_slack_space_required(); 1675 1676 /* tables are guaranteed to be sized in multiples of 64 bytes (or 16 uint32_t) */ 1677 ZSTD_STATIC_ASSERT(ZSTD_HASHLOG_MIN >= 4 && ZSTD_WINDOWLOG_MIN >= 4 && ZSTD_CHAINLOG_MIN >= 4); 1678 assert(useRowMatchFinder != ZSTD_ps_auto); 1679 1680 DEBUGLOG(4, "chainSize: %u - hSize: %u - h3Size: %u", 1681 (U32)chainSize, (U32)hSize, (U32)h3Size); 1682 return tableSpace + optSpace + slackSpace + lazyAdditionalSpace; 1683 } 1684 1685 /* Helper function for calculating memory requirements. 1686 * Gives a tighter bound than ZSTD_sequenceBound() by taking minMatch into account. */ 1687 static size_t ZSTD_maxNbSeq(size_t blockSize, unsigned minMatch, int useSequenceProducer) { 1688 U32 const divider = (minMatch==3 || useSequenceProducer) ? 3 : 4; 1689 return blockSize / divider; 1690 } 1691 1692 static size_t ZSTD_estimateCCtxSize_usingCCtxParams_internal( 1693 const ZSTD_compressionParameters* cParams, 1694 const ldmParams_t* ldmParams, 1695 const int isStatic, 1696 const ZSTD_paramSwitch_e useRowMatchFinder, 1697 const size_t buffInSize, 1698 const size_t buffOutSize, 1699 const U64 pledgedSrcSize, 1700 int useSequenceProducer, 1701 size_t maxBlockSize) 1702 { 1703 size_t const windowSize = (size_t) BOUNDED(1ULL, 1ULL << cParams->windowLog, pledgedSrcSize); 1704 size_t const blockSize = MIN(ZSTD_resolveMaxBlockSize(maxBlockSize), windowSize); 1705 size_t const maxNbSeq = ZSTD_maxNbSeq(blockSize, cParams->minMatch, useSequenceProducer); 1706 size_t const tokenSpace = ZSTD_cwksp_alloc_size(WILDCOPY_OVERLENGTH + blockSize) 1707 + ZSTD_cwksp_aligned_alloc_size(maxNbSeq * sizeof(seqDef)) 1708 + 3 * ZSTD_cwksp_alloc_size(maxNbSeq * sizeof(BYTE)); 1709 size_t const entropySpace = ZSTD_cwksp_alloc_size(ENTROPY_WORKSPACE_SIZE); 1710 size_t const blockStateSpace = 2 * ZSTD_cwksp_alloc_size(sizeof(ZSTD_compressedBlockState_t)); 1711 size_t const matchStateSize = ZSTD_sizeof_matchState(cParams, useRowMatchFinder, /* enableDedicatedDictSearch */ 0, /* forCCtx */ 1); 1712 1713 size_t const ldmSpace = ZSTD_ldm_getTableSize(*ldmParams); 1714 size_t const maxNbLdmSeq = ZSTD_ldm_getMaxNbSeq(*ldmParams, blockSize); 1715 size_t const ldmSeqSpace = ldmParams->enableLdm == ZSTD_ps_enable ? 1716 ZSTD_cwksp_aligned_alloc_size(maxNbLdmSeq * sizeof(rawSeq)) : 0; 1717 1718 1719 size_t const bufferSpace = ZSTD_cwksp_alloc_size(buffInSize) 1720 + ZSTD_cwksp_alloc_size(buffOutSize); 1721 1722 size_t const cctxSpace = isStatic ? ZSTD_cwksp_alloc_size(sizeof(ZSTD_CCtx)) : 0; 1723 1724 size_t const maxNbExternalSeq = ZSTD_sequenceBound(blockSize); 1725 size_t const externalSeqSpace = useSequenceProducer 1726 ? ZSTD_cwksp_aligned_alloc_size(maxNbExternalSeq * sizeof(ZSTD_Sequence)) 1727 : 0; 1728 1729 size_t const neededSpace = 1730 cctxSpace + 1731 entropySpace + 1732 blockStateSpace + 1733 ldmSpace + 1734 ldmSeqSpace + 1735 matchStateSize + 1736 tokenSpace + 1737 bufferSpace + 1738 externalSeqSpace; 1739 1740 DEBUGLOG(5, "estimate workspace : %u", (U32)neededSpace); 1741 return neededSpace; 1742 } 1743 1744 size_t ZSTD_estimateCCtxSize_usingCCtxParams(const ZSTD_CCtx_params* params) 1745 { 1746 ZSTD_compressionParameters const cParams = 1747 ZSTD_getCParamsFromCCtxParams(params, ZSTD_CONTENTSIZE_UNKNOWN, 0, ZSTD_cpm_noAttachDict); 1748 ZSTD_paramSwitch_e const useRowMatchFinder = ZSTD_resolveRowMatchFinderMode(params->useRowMatchFinder, 1749 &cParams); 1750 1751 RETURN_ERROR_IF(params->nbWorkers > 0, GENERIC, "Estimate CCtx size is supported for single-threaded compression only."); 1752 /* estimateCCtxSize is for one-shot compression. So no buffers should 1753 * be needed. However, we still allocate two 0-sized buffers, which can 1754 * take space under ASAN. */ 1755 return ZSTD_estimateCCtxSize_usingCCtxParams_internal( 1756 &cParams, ¶ms->ldmParams, 1, useRowMatchFinder, 0, 0, ZSTD_CONTENTSIZE_UNKNOWN, ZSTD_hasExtSeqProd(params), params->maxBlockSize); 1757 } 1758 1759 size_t ZSTD_estimateCCtxSize_usingCParams(ZSTD_compressionParameters cParams) 1760 { 1761 ZSTD_CCtx_params initialParams = ZSTD_makeCCtxParamsFromCParams(cParams); 1762 if (ZSTD_rowMatchFinderSupported(cParams.strategy)) { 1763 /* Pick bigger of not using and using row-based matchfinder for greedy and lazy strategies */ 1764 size_t noRowCCtxSize; 1765 size_t rowCCtxSize; 1766 initialParams.useRowMatchFinder = ZSTD_ps_disable; 1767 noRowCCtxSize = ZSTD_estimateCCtxSize_usingCCtxParams(&initialParams); 1768 initialParams.useRowMatchFinder = ZSTD_ps_enable; 1769 rowCCtxSize = ZSTD_estimateCCtxSize_usingCCtxParams(&initialParams); 1770 return MAX(noRowCCtxSize, rowCCtxSize); 1771 } else { 1772 return ZSTD_estimateCCtxSize_usingCCtxParams(&initialParams); 1773 } 1774 } 1775 1776 static size_t ZSTD_estimateCCtxSize_internal(int compressionLevel) 1777 { 1778 int tier = 0; 1779 size_t largestSize = 0; 1780 static const unsigned long long srcSizeTiers[4] = {16 KB, 128 KB, 256 KB, ZSTD_CONTENTSIZE_UNKNOWN}; 1781 for (; tier < 4; ++tier) { 1782 /* Choose the set of cParams for a given level across all srcSizes that give the largest cctxSize */ 1783 ZSTD_compressionParameters const cParams = ZSTD_getCParams_internal(compressionLevel, srcSizeTiers[tier], 0, ZSTD_cpm_noAttachDict); 1784 largestSize = MAX(ZSTD_estimateCCtxSize_usingCParams(cParams), largestSize); 1785 } 1786 return largestSize; 1787 } 1788 1789 size_t ZSTD_estimateCCtxSize(int compressionLevel) 1790 { 1791 int level; 1792 size_t memBudget = 0; 1793 for (level=MIN(compressionLevel, 1); level<=compressionLevel; level++) { 1794 /* Ensure monotonically increasing memory usage as compression level increases */ 1795 size_t const newMB = ZSTD_estimateCCtxSize_internal(level); 1796 if (newMB > memBudget) memBudget = newMB; 1797 } 1798 return memBudget; 1799 } 1800 1801 size_t ZSTD_estimateCStreamSize_usingCCtxParams(const ZSTD_CCtx_params* params) 1802 { 1803 RETURN_ERROR_IF(params->nbWorkers > 0, GENERIC, "Estimate CCtx size is supported for single-threaded compression only."); 1804 { ZSTD_compressionParameters const cParams = 1805 ZSTD_getCParamsFromCCtxParams(params, ZSTD_CONTENTSIZE_UNKNOWN, 0, ZSTD_cpm_noAttachDict); 1806 size_t const blockSize = MIN(ZSTD_resolveMaxBlockSize(params->maxBlockSize), (size_t)1 << cParams.windowLog); 1807 size_t const inBuffSize = (params->inBufferMode == ZSTD_bm_buffered) 1808 ? ((size_t)1 << cParams.windowLog) + blockSize 1809 : 0; 1810 size_t const outBuffSize = (params->outBufferMode == ZSTD_bm_buffered) 1811 ? ZSTD_compressBound(blockSize) + 1 1812 : 0; 1813 ZSTD_paramSwitch_e const useRowMatchFinder = ZSTD_resolveRowMatchFinderMode(params->useRowMatchFinder, ¶ms->cParams); 1814 1815 return ZSTD_estimateCCtxSize_usingCCtxParams_internal( 1816 &cParams, ¶ms->ldmParams, 1, useRowMatchFinder, inBuffSize, outBuffSize, 1817 ZSTD_CONTENTSIZE_UNKNOWN, ZSTD_hasExtSeqProd(params), params->maxBlockSize); 1818 } 1819 } 1820 1821 size_t ZSTD_estimateCStreamSize_usingCParams(ZSTD_compressionParameters cParams) 1822 { 1823 ZSTD_CCtx_params initialParams = ZSTD_makeCCtxParamsFromCParams(cParams); 1824 if (ZSTD_rowMatchFinderSupported(cParams.strategy)) { 1825 /* Pick bigger of not using and using row-based matchfinder for greedy and lazy strategies */ 1826 size_t noRowCCtxSize; 1827 size_t rowCCtxSize; 1828 initialParams.useRowMatchFinder = ZSTD_ps_disable; 1829 noRowCCtxSize = ZSTD_estimateCStreamSize_usingCCtxParams(&initialParams); 1830 initialParams.useRowMatchFinder = ZSTD_ps_enable; 1831 rowCCtxSize = ZSTD_estimateCStreamSize_usingCCtxParams(&initialParams); 1832 return MAX(noRowCCtxSize, rowCCtxSize); 1833 } else { 1834 return ZSTD_estimateCStreamSize_usingCCtxParams(&initialParams); 1835 } 1836 } 1837 1838 static size_t ZSTD_estimateCStreamSize_internal(int compressionLevel) 1839 { 1840 ZSTD_compressionParameters const cParams = ZSTD_getCParams_internal(compressionLevel, ZSTD_CONTENTSIZE_UNKNOWN, 0, ZSTD_cpm_noAttachDict); 1841 return ZSTD_estimateCStreamSize_usingCParams(cParams); 1842 } 1843 1844 size_t ZSTD_estimateCStreamSize(int compressionLevel) 1845 { 1846 int level; 1847 size_t memBudget = 0; 1848 for (level=MIN(compressionLevel, 1); level<=compressionLevel; level++) { 1849 size_t const newMB = ZSTD_estimateCStreamSize_internal(level); 1850 if (newMB > memBudget) memBudget = newMB; 1851 } 1852 return memBudget; 1853 } 1854 1855 /* ZSTD_getFrameProgression(): 1856 * tells how much data has been consumed (input) and produced (output) for current frame. 1857 * able to count progression inside worker threads (non-blocking mode). 1858 */ 1859 ZSTD_frameProgression ZSTD_getFrameProgression(const ZSTD_CCtx* cctx) 1860 { 1861 #ifdef ZSTD_MULTITHREAD 1862 if (cctx->appliedParams.nbWorkers > 0) { 1863 return ZSTDMT_getFrameProgression(cctx->mtctx); 1864 } 1865 #endif 1866 { ZSTD_frameProgression fp; 1867 size_t const buffered = (cctx->inBuff == NULL) ? 0 : 1868 cctx->inBuffPos - cctx->inToCompress; 1869 if (buffered) assert(cctx->inBuffPos >= cctx->inToCompress); 1870 assert(buffered <= ZSTD_BLOCKSIZE_MAX); 1871 fp.ingested = cctx->consumedSrcSize + buffered; 1872 fp.consumed = cctx->consumedSrcSize; 1873 fp.produced = cctx->producedCSize; 1874 fp.flushed = cctx->producedCSize; /* simplified; some data might still be left within streaming output buffer */ 1875 fp.currentJobID = 0; 1876 fp.nbActiveWorkers = 0; 1877 return fp; 1878 } } 1879 1880 /*! ZSTD_toFlushNow() 1881 * Only useful for multithreading scenarios currently (nbWorkers >= 1). 1882 */ 1883 size_t ZSTD_toFlushNow(ZSTD_CCtx* cctx) 1884 { 1885 #ifdef ZSTD_MULTITHREAD 1886 if (cctx->appliedParams.nbWorkers > 0) { 1887 return ZSTDMT_toFlushNow(cctx->mtctx); 1888 } 1889 #endif 1890 (void)cctx; 1891 return 0; /* over-simplification; could also check if context is currently running in streaming mode, and in which case, report how many bytes are left to be flushed within output buffer */ 1892 } 1893 1894 static void ZSTD_assertEqualCParams(ZSTD_compressionParameters cParams1, 1895 ZSTD_compressionParameters cParams2) 1896 { 1897 (void)cParams1; 1898 (void)cParams2; 1899 assert(cParams1.windowLog == cParams2.windowLog); 1900 assert(cParams1.chainLog == cParams2.chainLog); 1901 assert(cParams1.hashLog == cParams2.hashLog); 1902 assert(cParams1.searchLog == cParams2.searchLog); 1903 assert(cParams1.minMatch == cParams2.minMatch); 1904 assert(cParams1.targetLength == cParams2.targetLength); 1905 assert(cParams1.strategy == cParams2.strategy); 1906 } 1907 1908 void ZSTD_reset_compressedBlockState(ZSTD_compressedBlockState_t* bs) 1909 { 1910 int i; 1911 for (i = 0; i < ZSTD_REP_NUM; ++i) 1912 bs->rep[i] = repStartValue[i]; 1913 bs->entropy.huf.repeatMode = HUF_repeat_none; 1914 bs->entropy.fse.offcode_repeatMode = FSE_repeat_none; 1915 bs->entropy.fse.matchlength_repeatMode = FSE_repeat_none; 1916 bs->entropy.fse.litlength_repeatMode = FSE_repeat_none; 1917 } 1918 1919 /*! ZSTD_invalidateMatchState() 1920 * Invalidate all the matches in the match finder tables. 1921 * Requires nextSrc and base to be set (can be NULL). 1922 */ 1923 static void ZSTD_invalidateMatchState(ZSTD_matchState_t* ms) 1924 { 1925 ZSTD_window_clear(&ms->window); 1926 1927 ms->nextToUpdate = ms->window.dictLimit; 1928 ms->loadedDictEnd = 0; 1929 ms->opt.litLengthSum = 0; /* force reset of btopt stats */ 1930 ms->dictMatchState = NULL; 1931 } 1932 1933 /** 1934 * Controls, for this matchState reset, whether the tables need to be cleared / 1935 * prepared for the coming compression (ZSTDcrp_makeClean), or whether the 1936 * tables can be left unclean (ZSTDcrp_leaveDirty), because we know that a 1937 * subsequent operation will overwrite the table space anyways (e.g., copying 1938 * the matchState contents in from a CDict). 1939 */ 1940 typedef enum { 1941 ZSTDcrp_makeClean, 1942 ZSTDcrp_leaveDirty 1943 } ZSTD_compResetPolicy_e; 1944 1945 /** 1946 * Controls, for this matchState reset, whether indexing can continue where it 1947 * left off (ZSTDirp_continue), or whether it needs to be restarted from zero 1948 * (ZSTDirp_reset). 1949 */ 1950 typedef enum { 1951 ZSTDirp_continue, 1952 ZSTDirp_reset 1953 } ZSTD_indexResetPolicy_e; 1954 1955 typedef enum { 1956 ZSTD_resetTarget_CDict, 1957 ZSTD_resetTarget_CCtx 1958 } ZSTD_resetTarget_e; 1959 1960 /* Mixes bits in a 64 bits in a value, based on XXH3_rrmxmx */ 1961 static U64 ZSTD_bitmix(U64 val, U64 len) { 1962 val ^= ZSTD_rotateRight_U64(val, 49) ^ ZSTD_rotateRight_U64(val, 24); 1963 val *= 0x9FB21C651E98DF25ULL; 1964 val ^= (val >> 35) + len ; 1965 val *= 0x9FB21C651E98DF25ULL; 1966 return val ^ (val >> 28); 1967 } 1968 1969 /* Mixes in the hashSalt and hashSaltEntropy to create a new hashSalt */ 1970 static void ZSTD_advanceHashSalt(ZSTD_matchState_t* ms) { 1971 ms->hashSalt = ZSTD_bitmix(ms->hashSalt, 8) ^ ZSTD_bitmix((U64) ms->hashSaltEntropy, 4); 1972 } 1973 1974 static size_t 1975 ZSTD_reset_matchState(ZSTD_matchState_t* ms, 1976 ZSTD_cwksp* ws, 1977 const ZSTD_compressionParameters* cParams, 1978 const ZSTD_paramSwitch_e useRowMatchFinder, 1979 const ZSTD_compResetPolicy_e crp, 1980 const ZSTD_indexResetPolicy_e forceResetIndex, 1981 const ZSTD_resetTarget_e forWho) 1982 { 1983 /* disable chain table allocation for fast or row-based strategies */ 1984 size_t const chainSize = ZSTD_allocateChainTable(cParams->strategy, useRowMatchFinder, 1985 ms->dedicatedDictSearch && (forWho == ZSTD_resetTarget_CDict)) 1986 ? ((size_t)1 << cParams->chainLog) 1987 : 0; 1988 size_t const hSize = ((size_t)1) << cParams->hashLog; 1989 U32 const hashLog3 = ((forWho == ZSTD_resetTarget_CCtx) && cParams->minMatch==3) ? MIN(ZSTD_HASHLOG3_MAX, cParams->windowLog) : 0; 1990 size_t const h3Size = hashLog3 ? ((size_t)1) << hashLog3 : 0; 1991 1992 DEBUGLOG(4, "reset indices : %u", forceResetIndex == ZSTDirp_reset); 1993 assert(useRowMatchFinder != ZSTD_ps_auto); 1994 if (forceResetIndex == ZSTDirp_reset) { 1995 ZSTD_window_init(&ms->window); 1996 ZSTD_cwksp_mark_tables_dirty(ws); 1997 } 1998 1999 ms->hashLog3 = hashLog3; 2000 ms->lazySkipping = 0; 2001 2002 ZSTD_invalidateMatchState(ms); 2003 2004 assert(!ZSTD_cwksp_reserve_failed(ws)); /* check that allocation hasn't already failed */ 2005 2006 ZSTD_cwksp_clear_tables(ws); 2007 2008 DEBUGLOG(5, "reserving table space"); 2009 /* table Space */ 2010 ms->hashTable = (U32*)ZSTD_cwksp_reserve_table(ws, hSize * sizeof(U32)); 2011 ms->chainTable = (U32*)ZSTD_cwksp_reserve_table(ws, chainSize * sizeof(U32)); 2012 ms->hashTable3 = (U32*)ZSTD_cwksp_reserve_table(ws, h3Size * sizeof(U32)); 2013 RETURN_ERROR_IF(ZSTD_cwksp_reserve_failed(ws), memory_allocation, 2014 "failed a workspace allocation in ZSTD_reset_matchState"); 2015 2016 DEBUGLOG(4, "reset table : %u", crp!=ZSTDcrp_leaveDirty); 2017 if (crp!=ZSTDcrp_leaveDirty) { 2018 /* reset tables only */ 2019 ZSTD_cwksp_clean_tables(ws); 2020 } 2021 2022 if (ZSTD_rowMatchFinderUsed(cParams->strategy, useRowMatchFinder)) { 2023 /* Row match finder needs an additional table of hashes ("tags") */ 2024 size_t const tagTableSize = hSize; 2025 /* We want to generate a new salt in case we reset a Cctx, but we always want to use 2026 * 0 when we reset a Cdict */ 2027 if(forWho == ZSTD_resetTarget_CCtx) { 2028 ms->tagTable = (BYTE*) ZSTD_cwksp_reserve_aligned_init_once(ws, tagTableSize); 2029 ZSTD_advanceHashSalt(ms); 2030 } else { 2031 /* When we are not salting we want to always memset the memory */ 2032 ms->tagTable = (BYTE*) ZSTD_cwksp_reserve_aligned(ws, tagTableSize); 2033 ZSTD_memset(ms->tagTable, 0, tagTableSize); 2034 ms->hashSalt = 0; 2035 } 2036 { /* Switch to 32-entry rows if searchLog is 5 (or more) */ 2037 U32 const rowLog = BOUNDED(4, cParams->searchLog, 6); 2038 assert(cParams->hashLog >= rowLog); 2039 ms->rowHashLog = cParams->hashLog - rowLog; 2040 } 2041 } 2042 2043 /* opt parser space */ 2044 if ((forWho == ZSTD_resetTarget_CCtx) && (cParams->strategy >= ZSTD_btopt)) { 2045 DEBUGLOG(4, "reserving optimal parser space"); 2046 ms->opt.litFreq = (unsigned*)ZSTD_cwksp_reserve_aligned(ws, (1<<Litbits) * sizeof(unsigned)); 2047 ms->opt.litLengthFreq = (unsigned*)ZSTD_cwksp_reserve_aligned(ws, (MaxLL+1) * sizeof(unsigned)); 2048 ms->opt.matchLengthFreq = (unsigned*)ZSTD_cwksp_reserve_aligned(ws, (MaxML+1) * sizeof(unsigned)); 2049 ms->opt.offCodeFreq = (unsigned*)ZSTD_cwksp_reserve_aligned(ws, (MaxOff+1) * sizeof(unsigned)); 2050 ms->opt.matchTable = (ZSTD_match_t*)ZSTD_cwksp_reserve_aligned(ws, ZSTD_OPT_SIZE * sizeof(ZSTD_match_t)); 2051 ms->opt.priceTable = (ZSTD_optimal_t*)ZSTD_cwksp_reserve_aligned(ws, ZSTD_OPT_SIZE * sizeof(ZSTD_optimal_t)); 2052 } 2053 2054 ms->cParams = *cParams; 2055 2056 RETURN_ERROR_IF(ZSTD_cwksp_reserve_failed(ws), memory_allocation, 2057 "failed a workspace allocation in ZSTD_reset_matchState"); 2058 return 0; 2059 } 2060 2061 /* ZSTD_indexTooCloseToMax() : 2062 * minor optimization : prefer memset() rather than reduceIndex() 2063 * which is measurably slow in some circumstances (reported for Visual Studio). 2064 * Works when re-using a context for a lot of smallish inputs : 2065 * if all inputs are smaller than ZSTD_INDEXOVERFLOW_MARGIN, 2066 * memset() will be triggered before reduceIndex(). 2067 */ 2068 #define ZSTD_INDEXOVERFLOW_MARGIN (16 MB) 2069 static int ZSTD_indexTooCloseToMax(ZSTD_window_t w) 2070 { 2071 return (size_t)(w.nextSrc - w.base) > (ZSTD_CURRENT_MAX - ZSTD_INDEXOVERFLOW_MARGIN); 2072 } 2073 2074 /** ZSTD_dictTooBig(): 2075 * When dictionaries are larger than ZSTD_CHUNKSIZE_MAX they can't be loaded in 2076 * one go generically. So we ensure that in that case we reset the tables to zero, 2077 * so that we can load as much of the dictionary as possible. 2078 */ 2079 static int ZSTD_dictTooBig(size_t const loadedDictSize) 2080 { 2081 return loadedDictSize > ZSTD_CHUNKSIZE_MAX; 2082 } 2083 2084 /*! ZSTD_resetCCtx_internal() : 2085 * @param loadedDictSize The size of the dictionary to be loaded 2086 * into the context, if any. If no dictionary is used, or the 2087 * dictionary is being attached / copied, then pass 0. 2088 * note : `params` are assumed fully validated at this stage. 2089 */ 2090 static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, 2091 ZSTD_CCtx_params const* params, 2092 U64 const pledgedSrcSize, 2093 size_t const loadedDictSize, 2094 ZSTD_compResetPolicy_e const crp, 2095 ZSTD_buffered_policy_e const zbuff) 2096 { 2097 ZSTD_cwksp* const ws = &zc->workspace; 2098 DEBUGLOG(4, "ZSTD_resetCCtx_internal: pledgedSrcSize=%u, wlog=%u, useRowMatchFinder=%d useBlockSplitter=%d", 2099 (U32)pledgedSrcSize, params->cParams.windowLog, (int)params->useRowMatchFinder, (int)params->useBlockSplitter); 2100 assert(!ZSTD_isError(ZSTD_checkCParams(params->cParams))); 2101 2102 zc->isFirstBlock = 1; 2103 2104 /* Set applied params early so we can modify them for LDM, 2105 * and point params at the applied params. 2106 */ 2107 zc->appliedParams = *params; 2108 params = &zc->appliedParams; 2109 2110 assert(params->useRowMatchFinder != ZSTD_ps_auto); 2111 assert(params->useBlockSplitter != ZSTD_ps_auto); 2112 assert(params->ldmParams.enableLdm != ZSTD_ps_auto); 2113 assert(params->maxBlockSize != 0); 2114 if (params->ldmParams.enableLdm == ZSTD_ps_enable) { 2115 /* Adjust long distance matching parameters */ 2116 ZSTD_ldm_adjustParameters(&zc->appliedParams.ldmParams, ¶ms->cParams); 2117 assert(params->ldmParams.hashLog >= params->ldmParams.bucketSizeLog); 2118 assert(params->ldmParams.hashRateLog < 32); 2119 } 2120 2121 { size_t const windowSize = MAX(1, (size_t)MIN(((U64)1 << params->cParams.windowLog), pledgedSrcSize)); 2122 size_t const blockSize = MIN(params->maxBlockSize, windowSize); 2123 size_t const maxNbSeq = ZSTD_maxNbSeq(blockSize, params->cParams.minMatch, ZSTD_hasExtSeqProd(params)); 2124 size_t const buffOutSize = (zbuff == ZSTDb_buffered && params->outBufferMode == ZSTD_bm_buffered) 2125 ? ZSTD_compressBound(blockSize) + 1 2126 : 0; 2127 size_t const buffInSize = (zbuff == ZSTDb_buffered && params->inBufferMode == ZSTD_bm_buffered) 2128 ? windowSize + blockSize 2129 : 0; 2130 size_t const maxNbLdmSeq = ZSTD_ldm_getMaxNbSeq(params->ldmParams, blockSize); 2131 2132 int const indexTooClose = ZSTD_indexTooCloseToMax(zc->blockState.matchState.window); 2133 int const dictTooBig = ZSTD_dictTooBig(loadedDictSize); 2134 ZSTD_indexResetPolicy_e needsIndexReset = 2135 (indexTooClose || dictTooBig || !zc->initialized) ? ZSTDirp_reset : ZSTDirp_continue; 2136 2137 size_t const neededSpace = 2138 ZSTD_estimateCCtxSize_usingCCtxParams_internal( 2139 ¶ms->cParams, ¶ms->ldmParams, zc->staticSize != 0, params->useRowMatchFinder, 2140 buffInSize, buffOutSize, pledgedSrcSize, ZSTD_hasExtSeqProd(params), params->maxBlockSize); 2141 2142 FORWARD_IF_ERROR(neededSpace, "cctx size estimate failed!"); 2143 2144 if (!zc->staticSize) ZSTD_cwksp_bump_oversized_duration(ws, 0); 2145 2146 { /* Check if workspace is large enough, alloc a new one if needed */ 2147 int const workspaceTooSmall = ZSTD_cwksp_sizeof(ws) < neededSpace; 2148 int const workspaceWasteful = ZSTD_cwksp_check_wasteful(ws, neededSpace); 2149 int resizeWorkspace = workspaceTooSmall || workspaceWasteful; 2150 DEBUGLOG(4, "Need %zu B workspace", neededSpace); 2151 DEBUGLOG(4, "windowSize: %zu - blockSize: %zu", windowSize, blockSize); 2152 2153 if (resizeWorkspace) { 2154 DEBUGLOG(4, "Resize workspaceSize from %zuKB to %zuKB", 2155 ZSTD_cwksp_sizeof(ws) >> 10, 2156 neededSpace >> 10); 2157 2158 RETURN_ERROR_IF(zc->staticSize, memory_allocation, "static cctx : no resize"); 2159 2160 needsIndexReset = ZSTDirp_reset; 2161 2162 ZSTD_cwksp_free(ws, zc->customMem); 2163 FORWARD_IF_ERROR(ZSTD_cwksp_create(ws, neededSpace, zc->customMem), ""); 2164 2165 DEBUGLOG(5, "reserving object space"); 2166 /* Statically sized space. 2167 * entropyWorkspace never moves, 2168 * though prev/next block swap places */ 2169 assert(ZSTD_cwksp_check_available(ws, 2 * sizeof(ZSTD_compressedBlockState_t))); 2170 zc->blockState.prevCBlock = (ZSTD_compressedBlockState_t*) ZSTD_cwksp_reserve_object(ws, sizeof(ZSTD_compressedBlockState_t)); 2171 RETURN_ERROR_IF(zc->blockState.prevCBlock == NULL, memory_allocation, "couldn't allocate prevCBlock"); 2172 zc->blockState.nextCBlock = (ZSTD_compressedBlockState_t*) ZSTD_cwksp_reserve_object(ws, sizeof(ZSTD_compressedBlockState_t)); 2173 RETURN_ERROR_IF(zc->blockState.nextCBlock == NULL, memory_allocation, "couldn't allocate nextCBlock"); 2174 zc->entropyWorkspace = (U32*) ZSTD_cwksp_reserve_object(ws, ENTROPY_WORKSPACE_SIZE); 2175 RETURN_ERROR_IF(zc->entropyWorkspace == NULL, memory_allocation, "couldn't allocate entropyWorkspace"); 2176 } } 2177 2178 ZSTD_cwksp_clear(ws); 2179 2180 /* init params */ 2181 zc->blockState.matchState.cParams = params->cParams; 2182 zc->blockState.matchState.prefetchCDictTables = params->prefetchCDictTables == ZSTD_ps_enable; 2183 zc->pledgedSrcSizePlusOne = pledgedSrcSize+1; 2184 zc->consumedSrcSize = 0; 2185 zc->producedCSize = 0; 2186 if (pledgedSrcSize == ZSTD_CONTENTSIZE_UNKNOWN) 2187 zc->appliedParams.fParams.contentSizeFlag = 0; 2188 DEBUGLOG(4, "pledged content size : %u ; flag : %u", 2189 (unsigned)pledgedSrcSize, zc->appliedParams.fParams.contentSizeFlag); 2190 zc->blockSize = blockSize; 2191 2192 XXH64_reset(&zc->xxhState, 0); 2193 zc->stage = ZSTDcs_init; 2194 zc->dictID = 0; 2195 zc->dictContentSize = 0; 2196 2197 ZSTD_reset_compressedBlockState(zc->blockState.prevCBlock); 2198 2199 FORWARD_IF_ERROR(ZSTD_reset_matchState( 2200 &zc->blockState.matchState, 2201 ws, 2202 ¶ms->cParams, 2203 params->useRowMatchFinder, 2204 crp, 2205 needsIndexReset, 2206 ZSTD_resetTarget_CCtx), ""); 2207 2208 zc->seqStore.sequencesStart = (seqDef*)ZSTD_cwksp_reserve_aligned(ws, maxNbSeq * sizeof(seqDef)); 2209 2210 /* ldm hash table */ 2211 if (params->ldmParams.enableLdm == ZSTD_ps_enable) { 2212 /* TODO: avoid memset? */ 2213 size_t const ldmHSize = ((size_t)1) << params->ldmParams.hashLog; 2214 zc->ldmState.hashTable = (ldmEntry_t*)ZSTD_cwksp_reserve_aligned(ws, ldmHSize * sizeof(ldmEntry_t)); 2215 ZSTD_memset(zc->ldmState.hashTable, 0, ldmHSize * sizeof(ldmEntry_t)); 2216 zc->ldmSequences = (rawSeq*)ZSTD_cwksp_reserve_aligned(ws, maxNbLdmSeq * sizeof(rawSeq)); 2217 zc->maxNbLdmSequences = maxNbLdmSeq; 2218 2219 ZSTD_window_init(&zc->ldmState.window); 2220 zc->ldmState.loadedDictEnd = 0; 2221 } 2222 2223 /* reserve space for block-level external sequences */ 2224 if (ZSTD_hasExtSeqProd(params)) { 2225 size_t const maxNbExternalSeq = ZSTD_sequenceBound(blockSize); 2226 zc->extSeqBufCapacity = maxNbExternalSeq; 2227 zc->extSeqBuf = 2228 (ZSTD_Sequence*)ZSTD_cwksp_reserve_aligned(ws, maxNbExternalSeq * sizeof(ZSTD_Sequence)); 2229 } 2230 2231 /* buffers */ 2232 2233 /* ZSTD_wildcopy() is used to copy into the literals buffer, 2234 * so we have to oversize the buffer by WILDCOPY_OVERLENGTH bytes. 2235 */ 2236 zc->seqStore.litStart = ZSTD_cwksp_reserve_buffer(ws, blockSize + WILDCOPY_OVERLENGTH); 2237 zc->seqStore.maxNbLit = blockSize; 2238 2239 zc->bufferedPolicy = zbuff; 2240 zc->inBuffSize = buffInSize; 2241 zc->inBuff = (char*)ZSTD_cwksp_reserve_buffer(ws, buffInSize); 2242 zc->outBuffSize = buffOutSize; 2243 zc->outBuff = (char*)ZSTD_cwksp_reserve_buffer(ws, buffOutSize); 2244 2245 /* ldm bucketOffsets table */ 2246 if (params->ldmParams.enableLdm == ZSTD_ps_enable) { 2247 /* TODO: avoid memset? */ 2248 size_t const numBuckets = 2249 ((size_t)1) << (params->ldmParams.hashLog - 2250 params->ldmParams.bucketSizeLog); 2251 zc->ldmState.bucketOffsets = ZSTD_cwksp_reserve_buffer(ws, numBuckets); 2252 ZSTD_memset(zc->ldmState.bucketOffsets, 0, numBuckets); 2253 } 2254 2255 /* sequences storage */ 2256 ZSTD_referenceExternalSequences(zc, NULL, 0); 2257 zc->seqStore.maxNbSeq = maxNbSeq; 2258 zc->seqStore.llCode = ZSTD_cwksp_reserve_buffer(ws, maxNbSeq * sizeof(BYTE)); 2259 zc->seqStore.mlCode = ZSTD_cwksp_reserve_buffer(ws, maxNbSeq * sizeof(BYTE)); 2260 zc->seqStore.ofCode = ZSTD_cwksp_reserve_buffer(ws, maxNbSeq * sizeof(BYTE)); 2261 2262 DEBUGLOG(3, "wksp: finished allocating, %zd bytes remain available", ZSTD_cwksp_available_space(ws)); 2263 assert(ZSTD_cwksp_estimated_space_within_bounds(ws, neededSpace)); 2264 2265 zc->initialized = 1; 2266 2267 return 0; 2268 } 2269 } 2270 2271 /* ZSTD_invalidateRepCodes() : 2272 * ensures next compression will not use repcodes from previous block. 2273 * Note : only works with regular variant; 2274 * do not use with extDict variant ! */ 2275 void ZSTD_invalidateRepCodes(ZSTD_CCtx* cctx) { 2276 int i; 2277 for (i=0; i<ZSTD_REP_NUM; i++) cctx->blockState.prevCBlock->rep[i] = 0; 2278 assert(!ZSTD_window_hasExtDict(cctx->blockState.matchState.window)); 2279 } 2280 2281 /* These are the approximate sizes for each strategy past which copying the 2282 * dictionary tables into the working context is faster than using them 2283 * in-place. 2284 */ 2285 static const size_t attachDictSizeCutoffs[ZSTD_STRATEGY_MAX+1] = { 2286 8 KB, /* unused */ 2287 8 KB, /* ZSTD_fast */ 2288 16 KB, /* ZSTD_dfast */ 2289 32 KB, /* ZSTD_greedy */ 2290 32 KB, /* ZSTD_lazy */ 2291 32 KB, /* ZSTD_lazy2 */ 2292 32 KB, /* ZSTD_btlazy2 */ 2293 32 KB, /* ZSTD_btopt */ 2294 8 KB, /* ZSTD_btultra */ 2295 8 KB /* ZSTD_btultra2 */ 2296 }; 2297 2298 static int ZSTD_shouldAttachDict(const ZSTD_CDict* cdict, 2299 const ZSTD_CCtx_params* params, 2300 U64 pledgedSrcSize) 2301 { 2302 size_t cutoff = attachDictSizeCutoffs[cdict->matchState.cParams.strategy]; 2303 int const dedicatedDictSearch = cdict->matchState.dedicatedDictSearch; 2304 return dedicatedDictSearch 2305 || ( ( pledgedSrcSize <= cutoff 2306 || pledgedSrcSize == ZSTD_CONTENTSIZE_UNKNOWN 2307 || params->attachDictPref == ZSTD_dictForceAttach ) 2308 && params->attachDictPref != ZSTD_dictForceCopy 2309 && !params->forceWindow ); /* dictMatchState isn't correctly 2310 * handled in _enforceMaxDist */ 2311 } 2312 2313 static size_t 2314 ZSTD_resetCCtx_byAttachingCDict(ZSTD_CCtx* cctx, 2315 const ZSTD_CDict* cdict, 2316 ZSTD_CCtx_params params, 2317 U64 pledgedSrcSize, 2318 ZSTD_buffered_policy_e zbuff) 2319 { 2320 DEBUGLOG(4, "ZSTD_resetCCtx_byAttachingCDict() pledgedSrcSize=%llu", 2321 (unsigned long long)pledgedSrcSize); 2322 { 2323 ZSTD_compressionParameters adjusted_cdict_cParams = cdict->matchState.cParams; 2324 unsigned const windowLog = params.cParams.windowLog; 2325 assert(windowLog != 0); 2326 /* Resize working context table params for input only, since the dict 2327 * has its own tables. */ 2328 /* pledgedSrcSize == 0 means 0! */ 2329 2330 if (cdict->matchState.dedicatedDictSearch) { 2331 ZSTD_dedicatedDictSearch_revertCParams(&adjusted_cdict_cParams); 2332 } 2333 2334 params.cParams = ZSTD_adjustCParams_internal(adjusted_cdict_cParams, pledgedSrcSize, 2335 cdict->dictContentSize, ZSTD_cpm_attachDict, 2336 params.useRowMatchFinder); 2337 params.cParams.windowLog = windowLog; 2338 params.useRowMatchFinder = cdict->useRowMatchFinder; /* cdict overrides */ 2339 FORWARD_IF_ERROR(ZSTD_resetCCtx_internal(cctx, ¶ms, pledgedSrcSize, 2340 /* loadedDictSize */ 0, 2341 ZSTDcrp_makeClean, zbuff), ""); 2342 assert(cctx->appliedParams.cParams.strategy == adjusted_cdict_cParams.strategy); 2343 } 2344 2345 { const U32 cdictEnd = (U32)( cdict->matchState.window.nextSrc 2346 - cdict->matchState.window.base); 2347 const U32 cdictLen = cdictEnd - cdict->matchState.window.dictLimit; 2348 if (cdictLen == 0) { 2349 /* don't even attach dictionaries with no contents */ 2350 DEBUGLOG(4, "skipping attaching empty dictionary"); 2351 } else { 2352 DEBUGLOG(4, "attaching dictionary into context"); 2353 cctx->blockState.matchState.dictMatchState = &cdict->matchState; 2354 2355 /* prep working match state so dict matches never have negative indices 2356 * when they are translated to the working context's index space. */ 2357 if (cctx->blockState.matchState.window.dictLimit < cdictEnd) { 2358 cctx->blockState.matchState.window.nextSrc = 2359 cctx->blockState.matchState.window.base + cdictEnd; 2360 ZSTD_window_clear(&cctx->blockState.matchState.window); 2361 } 2362 /* loadedDictEnd is expressed within the referential of the active context */ 2363 cctx->blockState.matchState.loadedDictEnd = cctx->blockState.matchState.window.dictLimit; 2364 } } 2365 2366 cctx->dictID = cdict->dictID; 2367 cctx->dictContentSize = cdict->dictContentSize; 2368 2369 /* copy block state */ 2370 ZSTD_memcpy(cctx->blockState.prevCBlock, &cdict->cBlockState, sizeof(cdict->cBlockState)); 2371 2372 return 0; 2373 } 2374 2375 static void ZSTD_copyCDictTableIntoCCtx(U32* dst, U32 const* src, size_t tableSize, 2376 ZSTD_compressionParameters const* cParams) { 2377 if (ZSTD_CDictIndicesAreTagged(cParams)){ 2378 /* Remove tags from the CDict table if they are present. 2379 * See docs on "short cache" in zstd_compress_internal.h for context. */ 2380 size_t i; 2381 for (i = 0; i < tableSize; i++) { 2382 U32 const taggedIndex = src[i]; 2383 U32 const index = taggedIndex >> ZSTD_SHORT_CACHE_TAG_BITS; 2384 dst[i] = index; 2385 } 2386 } else { 2387 ZSTD_memcpy(dst, src, tableSize * sizeof(U32)); 2388 } 2389 } 2390 2391 static size_t ZSTD_resetCCtx_byCopyingCDict(ZSTD_CCtx* cctx, 2392 const ZSTD_CDict* cdict, 2393 ZSTD_CCtx_params params, 2394 U64 pledgedSrcSize, 2395 ZSTD_buffered_policy_e zbuff) 2396 { 2397 const ZSTD_compressionParameters *cdict_cParams = &cdict->matchState.cParams; 2398 2399 assert(!cdict->matchState.dedicatedDictSearch); 2400 DEBUGLOG(4, "ZSTD_resetCCtx_byCopyingCDict() pledgedSrcSize=%llu", 2401 (unsigned long long)pledgedSrcSize); 2402 2403 { unsigned const windowLog = params.cParams.windowLog; 2404 assert(windowLog != 0); 2405 /* Copy only compression parameters related to tables. */ 2406 params.cParams = *cdict_cParams; 2407 params.cParams.windowLog = windowLog; 2408 params.useRowMatchFinder = cdict->useRowMatchFinder; 2409 FORWARD_IF_ERROR(ZSTD_resetCCtx_internal(cctx, ¶ms, pledgedSrcSize, 2410 /* loadedDictSize */ 0, 2411 ZSTDcrp_leaveDirty, zbuff), ""); 2412 assert(cctx->appliedParams.cParams.strategy == cdict_cParams->strategy); 2413 assert(cctx->appliedParams.cParams.hashLog == cdict_cParams->hashLog); 2414 assert(cctx->appliedParams.cParams.chainLog == cdict_cParams->chainLog); 2415 } 2416 2417 ZSTD_cwksp_mark_tables_dirty(&cctx->workspace); 2418 assert(params.useRowMatchFinder != ZSTD_ps_auto); 2419 2420 /* copy tables */ 2421 { size_t const chainSize = ZSTD_allocateChainTable(cdict_cParams->strategy, cdict->useRowMatchFinder, 0 /* DDS guaranteed disabled */) 2422 ? ((size_t)1 << cdict_cParams->chainLog) 2423 : 0; 2424 size_t const hSize = (size_t)1 << cdict_cParams->hashLog; 2425 2426 ZSTD_copyCDictTableIntoCCtx(cctx->blockState.matchState.hashTable, 2427 cdict->matchState.hashTable, 2428 hSize, cdict_cParams); 2429 2430 /* Do not copy cdict's chainTable if cctx has parameters such that it would not use chainTable */ 2431 if (ZSTD_allocateChainTable(cctx->appliedParams.cParams.strategy, cctx->appliedParams.useRowMatchFinder, 0 /* forDDSDict */)) { 2432 ZSTD_copyCDictTableIntoCCtx(cctx->blockState.matchState.chainTable, 2433 cdict->matchState.chainTable, 2434 chainSize, cdict_cParams); 2435 } 2436 /* copy tag table */ 2437 if (ZSTD_rowMatchFinderUsed(cdict_cParams->strategy, cdict->useRowMatchFinder)) { 2438 size_t const tagTableSize = hSize; 2439 ZSTD_memcpy(cctx->blockState.matchState.tagTable, 2440 cdict->matchState.tagTable, 2441 tagTableSize); 2442 cctx->blockState.matchState.hashSalt = cdict->matchState.hashSalt; 2443 } 2444 } 2445 2446 /* Zero the hashTable3, since the cdict never fills it */ 2447 { int const h3log = cctx->blockState.matchState.hashLog3; 2448 size_t const h3Size = h3log ? ((size_t)1 << h3log) : 0; 2449 assert(cdict->matchState.hashLog3 == 0); 2450 ZSTD_memset(cctx->blockState.matchState.hashTable3, 0, h3Size * sizeof(U32)); 2451 } 2452 2453 ZSTD_cwksp_mark_tables_clean(&cctx->workspace); 2454 2455 /* copy dictionary offsets */ 2456 { ZSTD_matchState_t const* srcMatchState = &cdict->matchState; 2457 ZSTD_matchState_t* dstMatchState = &cctx->blockState.matchState; 2458 dstMatchState->window = srcMatchState->window; 2459 dstMatchState->nextToUpdate = srcMatchState->nextToUpdate; 2460 dstMatchState->loadedDictEnd= srcMatchState->loadedDictEnd; 2461 } 2462 2463 cctx->dictID = cdict->dictID; 2464 cctx->dictContentSize = cdict->dictContentSize; 2465 2466 /* copy block state */ 2467 ZSTD_memcpy(cctx->blockState.prevCBlock, &cdict->cBlockState, sizeof(cdict->cBlockState)); 2468 2469 return 0; 2470 } 2471 2472 /* We have a choice between copying the dictionary context into the working 2473 * context, or referencing the dictionary context from the working context 2474 * in-place. We decide here which strategy to use. */ 2475 static size_t ZSTD_resetCCtx_usingCDict(ZSTD_CCtx* cctx, 2476 const ZSTD_CDict* cdict, 2477 const ZSTD_CCtx_params* params, 2478 U64 pledgedSrcSize, 2479 ZSTD_buffered_policy_e zbuff) 2480 { 2481 2482 DEBUGLOG(4, "ZSTD_resetCCtx_usingCDict (pledgedSrcSize=%u)", 2483 (unsigned)pledgedSrcSize); 2484 2485 if (ZSTD_shouldAttachDict(cdict, params, pledgedSrcSize)) { 2486 return ZSTD_resetCCtx_byAttachingCDict( 2487 cctx, cdict, *params, pledgedSrcSize, zbuff); 2488 } else { 2489 return ZSTD_resetCCtx_byCopyingCDict( 2490 cctx, cdict, *params, pledgedSrcSize, zbuff); 2491 } 2492 } 2493 2494 /*! ZSTD_copyCCtx_internal() : 2495 * Duplicate an existing context `srcCCtx` into another one `dstCCtx`. 2496 * Only works during stage ZSTDcs_init (i.e. after creation, but before first call to ZSTD_compressContinue()). 2497 * The "context", in this case, refers to the hash and chain tables, 2498 * entropy tables, and dictionary references. 2499 * `windowLog` value is enforced if != 0, otherwise value is copied from srcCCtx. 2500 * @return : 0, or an error code */ 2501 static size_t ZSTD_copyCCtx_internal(ZSTD_CCtx* dstCCtx, 2502 const ZSTD_CCtx* srcCCtx, 2503 ZSTD_frameParameters fParams, 2504 U64 pledgedSrcSize, 2505 ZSTD_buffered_policy_e zbuff) 2506 { 2507 RETURN_ERROR_IF(srcCCtx->stage!=ZSTDcs_init, stage_wrong, 2508 "Can't copy a ctx that's not in init stage."); 2509 DEBUGLOG(5, "ZSTD_copyCCtx_internal"); 2510 ZSTD_memcpy(&dstCCtx->customMem, &srcCCtx->customMem, sizeof(ZSTD_customMem)); 2511 { ZSTD_CCtx_params params = dstCCtx->requestedParams; 2512 /* Copy only compression parameters related to tables. */ 2513 params.cParams = srcCCtx->appliedParams.cParams; 2514 assert(srcCCtx->appliedParams.useRowMatchFinder != ZSTD_ps_auto); 2515 assert(srcCCtx->appliedParams.useBlockSplitter != ZSTD_ps_auto); 2516 assert(srcCCtx->appliedParams.ldmParams.enableLdm != ZSTD_ps_auto); 2517 params.useRowMatchFinder = srcCCtx->appliedParams.useRowMatchFinder; 2518 params.useBlockSplitter = srcCCtx->appliedParams.useBlockSplitter; 2519 params.ldmParams = srcCCtx->appliedParams.ldmParams; 2520 params.fParams = fParams; 2521 params.maxBlockSize = srcCCtx->appliedParams.maxBlockSize; 2522 ZSTD_resetCCtx_internal(dstCCtx, ¶ms, pledgedSrcSize, 2523 /* loadedDictSize */ 0, 2524 ZSTDcrp_leaveDirty, zbuff); 2525 assert(dstCCtx->appliedParams.cParams.windowLog == srcCCtx->appliedParams.cParams.windowLog); 2526 assert(dstCCtx->appliedParams.cParams.strategy == srcCCtx->appliedParams.cParams.strategy); 2527 assert(dstCCtx->appliedParams.cParams.hashLog == srcCCtx->appliedParams.cParams.hashLog); 2528 assert(dstCCtx->appliedParams.cParams.chainLog == srcCCtx->appliedParams.cParams.chainLog); 2529 assert(dstCCtx->blockState.matchState.hashLog3 == srcCCtx->blockState.matchState.hashLog3); 2530 } 2531 2532 ZSTD_cwksp_mark_tables_dirty(&dstCCtx->workspace); 2533 2534 /* copy tables */ 2535 { size_t const chainSize = ZSTD_allocateChainTable(srcCCtx->appliedParams.cParams.strategy, 2536 srcCCtx->appliedParams.useRowMatchFinder, 2537 0 /* forDDSDict */) 2538 ? ((size_t)1 << srcCCtx->appliedParams.cParams.chainLog) 2539 : 0; 2540 size_t const hSize = (size_t)1 << srcCCtx->appliedParams.cParams.hashLog; 2541 int const h3log = srcCCtx->blockState.matchState.hashLog3; 2542 size_t const h3Size = h3log ? ((size_t)1 << h3log) : 0; 2543 2544 ZSTD_memcpy(dstCCtx->blockState.matchState.hashTable, 2545 srcCCtx->blockState.matchState.hashTable, 2546 hSize * sizeof(U32)); 2547 ZSTD_memcpy(dstCCtx->blockState.matchState.chainTable, 2548 srcCCtx->blockState.matchState.chainTable, 2549 chainSize * sizeof(U32)); 2550 ZSTD_memcpy(dstCCtx->blockState.matchState.hashTable3, 2551 srcCCtx->blockState.matchState.hashTable3, 2552 h3Size * sizeof(U32)); 2553 } 2554 2555 ZSTD_cwksp_mark_tables_clean(&dstCCtx->workspace); 2556 2557 /* copy dictionary offsets */ 2558 { 2559 const ZSTD_matchState_t* srcMatchState = &srcCCtx->blockState.matchState; 2560 ZSTD_matchState_t* dstMatchState = &dstCCtx->blockState.matchState; 2561 dstMatchState->window = srcMatchState->window; 2562 dstMatchState->nextToUpdate = srcMatchState->nextToUpdate; 2563 dstMatchState->loadedDictEnd= srcMatchState->loadedDictEnd; 2564 } 2565 dstCCtx->dictID = srcCCtx->dictID; 2566 dstCCtx->dictContentSize = srcCCtx->dictContentSize; 2567 2568 /* copy block state */ 2569 ZSTD_memcpy(dstCCtx->blockState.prevCBlock, srcCCtx->blockState.prevCBlock, sizeof(*srcCCtx->blockState.prevCBlock)); 2570 2571 return 0; 2572 } 2573 2574 /*! ZSTD_copyCCtx() : 2575 * Duplicate an existing context `srcCCtx` into another one `dstCCtx`. 2576 * Only works during stage ZSTDcs_init (i.e. after creation, but before first call to ZSTD_compressContinue()). 2577 * pledgedSrcSize==0 means "unknown". 2578 * @return : 0, or an error code */ 2579 size_t ZSTD_copyCCtx(ZSTD_CCtx* dstCCtx, const ZSTD_CCtx* srcCCtx, unsigned long long pledgedSrcSize) 2580 { 2581 ZSTD_frameParameters fParams = { 1 /*content*/, 0 /*checksum*/, 0 /*noDictID*/ }; 2582 ZSTD_buffered_policy_e const zbuff = srcCCtx->bufferedPolicy; 2583 ZSTD_STATIC_ASSERT((U32)ZSTDb_buffered==1); 2584 if (pledgedSrcSize==0) pledgedSrcSize = ZSTD_CONTENTSIZE_UNKNOWN; 2585 fParams.contentSizeFlag = (pledgedSrcSize != ZSTD_CONTENTSIZE_UNKNOWN); 2586 2587 return ZSTD_copyCCtx_internal(dstCCtx, srcCCtx, 2588 fParams, pledgedSrcSize, 2589 zbuff); 2590 } 2591 2592 2593 #define ZSTD_ROWSIZE 16 2594 /*! ZSTD_reduceTable() : 2595 * reduce table indexes by `reducerValue`, or squash to zero. 2596 * PreserveMark preserves "unsorted mark" for btlazy2 strategy. 2597 * It must be set to a clear 0/1 value, to remove branch during inlining. 2598 * Presume table size is a multiple of ZSTD_ROWSIZE 2599 * to help auto-vectorization */ 2600 FORCE_INLINE_TEMPLATE void 2601 ZSTD_reduceTable_internal (U32* const table, U32 const size, U32 const reducerValue, int const preserveMark) 2602 { 2603 int const nbRows = (int)size / ZSTD_ROWSIZE; 2604 int cellNb = 0; 2605 int rowNb; 2606 /* Protect special index values < ZSTD_WINDOW_START_INDEX. */ 2607 U32 const reducerThreshold = reducerValue + ZSTD_WINDOW_START_INDEX; 2608 assert((size & (ZSTD_ROWSIZE-1)) == 0); /* multiple of ZSTD_ROWSIZE */ 2609 assert(size < (1U<<31)); /* can be casted to int */ 2610 2611 #if ZSTD_MEMORY_SANITIZER && !defined (ZSTD_MSAN_DONT_POISON_WORKSPACE) 2612 /* To validate that the table reuse logic is sound, and that we don't 2613 * access table space that we haven't cleaned, we re-"poison" the table 2614 * space every time we mark it dirty. 2615 * 2616 * This function however is intended to operate on those dirty tables and 2617 * re-clean them. So when this function is used correctly, we can unpoison 2618 * the memory it operated on. This introduces a blind spot though, since 2619 * if we now try to operate on __actually__ poisoned memory, we will not 2620 * detect that. */ 2621 __msan_unpoison(table, size * sizeof(U32)); 2622 #endif 2623 2624 for (rowNb=0 ; rowNb < nbRows ; rowNb++) { 2625 int column; 2626 for (column=0; column<ZSTD_ROWSIZE; column++) { 2627 U32 newVal; 2628 if (preserveMark && table[cellNb] == ZSTD_DUBT_UNSORTED_MARK) { 2629 /* This write is pointless, but is required(?) for the compiler 2630 * to auto-vectorize the loop. */ 2631 newVal = ZSTD_DUBT_UNSORTED_MARK; 2632 } else if (table[cellNb] < reducerThreshold) { 2633 newVal = 0; 2634 } else { 2635 newVal = table[cellNb] - reducerValue; 2636 } 2637 table[cellNb] = newVal; 2638 cellNb++; 2639 } } 2640 } 2641 2642 static void ZSTD_reduceTable(U32* const table, U32 const size, U32 const reducerValue) 2643 { 2644 ZSTD_reduceTable_internal(table, size, reducerValue, 0); 2645 } 2646 2647 static void ZSTD_reduceTable_btlazy2(U32* const table, U32 const size, U32 const reducerValue) 2648 { 2649 ZSTD_reduceTable_internal(table, size, reducerValue, 1); 2650 } 2651 2652 /*! ZSTD_reduceIndex() : 2653 * rescale all indexes to avoid future overflow (indexes are U32) */ 2654 static void ZSTD_reduceIndex (ZSTD_matchState_t* ms, ZSTD_CCtx_params const* params, const U32 reducerValue) 2655 { 2656 { U32 const hSize = (U32)1 << params->cParams.hashLog; 2657 ZSTD_reduceTable(ms->hashTable, hSize, reducerValue); 2658 } 2659 2660 if (ZSTD_allocateChainTable(params->cParams.strategy, params->useRowMatchFinder, (U32)ms->dedicatedDictSearch)) { 2661 U32 const chainSize = (U32)1 << params->cParams.chainLog; 2662 if (params->cParams.strategy == ZSTD_btlazy2) 2663 ZSTD_reduceTable_btlazy2(ms->chainTable, chainSize, reducerValue); 2664 else 2665 ZSTD_reduceTable(ms->chainTable, chainSize, reducerValue); 2666 } 2667 2668 if (ms->hashLog3) { 2669 U32 const h3Size = (U32)1 << ms->hashLog3; 2670 ZSTD_reduceTable(ms->hashTable3, h3Size, reducerValue); 2671 } 2672 } 2673 2674 2675 /*-******************************************************* 2676 * Block entropic compression 2677 *********************************************************/ 2678 2679 /* See doc/zstd_compression_format.md for detailed format description */ 2680 2681 int ZSTD_seqToCodes(const seqStore_t* seqStorePtr) 2682 { 2683 const seqDef* const sequences = seqStorePtr->sequencesStart; 2684 BYTE* const llCodeTable = seqStorePtr->llCode; 2685 BYTE* const ofCodeTable = seqStorePtr->ofCode; 2686 BYTE* const mlCodeTable = seqStorePtr->mlCode; 2687 U32 const nbSeq = (U32)(seqStorePtr->sequences - seqStorePtr->sequencesStart); 2688 U32 u; 2689 int longOffsets = 0; 2690 assert(nbSeq <= seqStorePtr->maxNbSeq); 2691 for (u=0; u<nbSeq; u++) { 2692 U32 const llv = sequences[u].litLength; 2693 U32 const ofCode = ZSTD_highbit32(sequences[u].offBase); 2694 U32 const mlv = sequences[u].mlBase; 2695 llCodeTable[u] = (BYTE)ZSTD_LLcode(llv); 2696 ofCodeTable[u] = (BYTE)ofCode; 2697 mlCodeTable[u] = (BYTE)ZSTD_MLcode(mlv); 2698 assert(!(MEM_64bits() && ofCode >= STREAM_ACCUMULATOR_MIN)); 2699 if (MEM_32bits() && ofCode >= STREAM_ACCUMULATOR_MIN) 2700 longOffsets = 1; 2701 } 2702 if (seqStorePtr->longLengthType==ZSTD_llt_literalLength) 2703 llCodeTable[seqStorePtr->longLengthPos] = MaxLL; 2704 if (seqStorePtr->longLengthType==ZSTD_llt_matchLength) 2705 mlCodeTable[seqStorePtr->longLengthPos] = MaxML; 2706 return longOffsets; 2707 } 2708 2709 /* ZSTD_useTargetCBlockSize(): 2710 * Returns if target compressed block size param is being used. 2711 * If used, compression will do best effort to make a compressed block size to be around targetCBlockSize. 2712 * Returns 1 if true, 0 otherwise. */ 2713 static int ZSTD_useTargetCBlockSize(const ZSTD_CCtx_params* cctxParams) 2714 { 2715 DEBUGLOG(5, "ZSTD_useTargetCBlockSize (targetCBlockSize=%zu)", cctxParams->targetCBlockSize); 2716 return (cctxParams->targetCBlockSize != 0); 2717 } 2718 2719 /* ZSTD_blockSplitterEnabled(): 2720 * Returns if block splitting param is being used 2721 * If used, compression will do best effort to split a block in order to improve compression ratio. 2722 * At the time this function is called, the parameter must be finalized. 2723 * Returns 1 if true, 0 otherwise. */ 2724 static int ZSTD_blockSplitterEnabled(ZSTD_CCtx_params* cctxParams) 2725 { 2726 DEBUGLOG(5, "ZSTD_blockSplitterEnabled (useBlockSplitter=%d)", cctxParams->useBlockSplitter); 2727 assert(cctxParams->useBlockSplitter != ZSTD_ps_auto); 2728 return (cctxParams->useBlockSplitter == ZSTD_ps_enable); 2729 } 2730 2731 /* Type returned by ZSTD_buildSequencesStatistics containing finalized symbol encoding types 2732 * and size of the sequences statistics 2733 */ 2734 typedef struct { 2735 U32 LLtype; 2736 U32 Offtype; 2737 U32 MLtype; 2738 size_t size; 2739 size_t lastCountSize; /* Accounts for bug in 1.3.4. More detail in ZSTD_entropyCompressSeqStore_internal() */ 2740 int longOffsets; 2741 } ZSTD_symbolEncodingTypeStats_t; 2742 2743 /* ZSTD_buildSequencesStatistics(): 2744 * Returns a ZSTD_symbolEncodingTypeStats_t, or a zstd error code in the `size` field. 2745 * Modifies `nextEntropy` to have the appropriate values as a side effect. 2746 * nbSeq must be greater than 0. 2747 * 2748 * entropyWkspSize must be of size at least ENTROPY_WORKSPACE_SIZE - (MaxSeq + 1)*sizeof(U32) 2749 */ 2750 static ZSTD_symbolEncodingTypeStats_t 2751 ZSTD_buildSequencesStatistics( 2752 const seqStore_t* seqStorePtr, size_t nbSeq, 2753 const ZSTD_fseCTables_t* prevEntropy, ZSTD_fseCTables_t* nextEntropy, 2754 BYTE* dst, const BYTE* const dstEnd, 2755 ZSTD_strategy strategy, unsigned* countWorkspace, 2756 void* entropyWorkspace, size_t entropyWkspSize) 2757 { 2758 BYTE* const ostart = dst; 2759 const BYTE* const oend = dstEnd; 2760 BYTE* op = ostart; 2761 FSE_CTable* CTable_LitLength = nextEntropy->litlengthCTable; 2762 FSE_CTable* CTable_OffsetBits = nextEntropy->offcodeCTable; 2763 FSE_CTable* CTable_MatchLength = nextEntropy->matchlengthCTable; 2764 const BYTE* const ofCodeTable = seqStorePtr->ofCode; 2765 const BYTE* const llCodeTable = seqStorePtr->llCode; 2766 const BYTE* const mlCodeTable = seqStorePtr->mlCode; 2767 ZSTD_symbolEncodingTypeStats_t stats; 2768 2769 stats.lastCountSize = 0; 2770 /* convert length/distances into codes */ 2771 stats.longOffsets = ZSTD_seqToCodes(seqStorePtr); 2772 assert(op <= oend); 2773 assert(nbSeq != 0); /* ZSTD_selectEncodingType() divides by nbSeq */ 2774 /* build CTable for Literal Lengths */ 2775 { unsigned max = MaxLL; 2776 size_t const mostFrequent = HIST_countFast_wksp(countWorkspace, &max, llCodeTable, nbSeq, entropyWorkspace, entropyWkspSize); /* can't fail */ 2777 DEBUGLOG(5, "Building LL table"); 2778 nextEntropy->litlength_repeatMode = prevEntropy->litlength_repeatMode; 2779 stats.LLtype = ZSTD_selectEncodingType(&nextEntropy->litlength_repeatMode, 2780 countWorkspace, max, mostFrequent, nbSeq, 2781 LLFSELog, prevEntropy->litlengthCTable, 2782 LL_defaultNorm, LL_defaultNormLog, 2783 ZSTD_defaultAllowed, strategy); 2784 assert(set_basic < set_compressed && set_rle < set_compressed); 2785 assert(!(stats.LLtype < set_compressed && nextEntropy->litlength_repeatMode != FSE_repeat_none)); /* We don't copy tables */ 2786 { size_t const countSize = ZSTD_buildCTable( 2787 op, (size_t)(oend - op), 2788 CTable_LitLength, LLFSELog, (symbolEncodingType_e)stats.LLtype, 2789 countWorkspace, max, llCodeTable, nbSeq, 2790 LL_defaultNorm, LL_defaultNormLog, MaxLL, 2791 prevEntropy->litlengthCTable, 2792 sizeof(prevEntropy->litlengthCTable), 2793 entropyWorkspace, entropyWkspSize); 2794 if (ZSTD_isError(countSize)) { 2795 DEBUGLOG(3, "ZSTD_buildCTable for LitLens failed"); 2796 stats.size = countSize; 2797 return stats; 2798 } 2799 if (stats.LLtype == set_compressed) 2800 stats.lastCountSize = countSize; 2801 op += countSize; 2802 assert(op <= oend); 2803 } } 2804 /* build CTable for Offsets */ 2805 { unsigned max = MaxOff; 2806 size_t const mostFrequent = HIST_countFast_wksp( 2807 countWorkspace, &max, ofCodeTable, nbSeq, entropyWorkspace, entropyWkspSize); /* can't fail */ 2808 /* We can only use the basic table if max <= DefaultMaxOff, otherwise the offsets are too large */ 2809 ZSTD_defaultPolicy_e const defaultPolicy = (max <= DefaultMaxOff) ? ZSTD_defaultAllowed : ZSTD_defaultDisallowed; 2810 DEBUGLOG(5, "Building OF table"); 2811 nextEntropy->offcode_repeatMode = prevEntropy->offcode_repeatMode; 2812 stats.Offtype = ZSTD_selectEncodingType(&nextEntropy->offcode_repeatMode, 2813 countWorkspace, max, mostFrequent, nbSeq, 2814 OffFSELog, prevEntropy->offcodeCTable, 2815 OF_defaultNorm, OF_defaultNormLog, 2816 defaultPolicy, strategy); 2817 assert(!(stats.Offtype < set_compressed && nextEntropy->offcode_repeatMode != FSE_repeat_none)); /* We don't copy tables */ 2818 { size_t const countSize = ZSTD_buildCTable( 2819 op, (size_t)(oend - op), 2820 CTable_OffsetBits, OffFSELog, (symbolEncodingType_e)stats.Offtype, 2821 countWorkspace, max, ofCodeTable, nbSeq, 2822 OF_defaultNorm, OF_defaultNormLog, DefaultMaxOff, 2823 prevEntropy->offcodeCTable, 2824 sizeof(prevEntropy->offcodeCTable), 2825 entropyWorkspace, entropyWkspSize); 2826 if (ZSTD_isError(countSize)) { 2827 DEBUGLOG(3, "ZSTD_buildCTable for Offsets failed"); 2828 stats.size = countSize; 2829 return stats; 2830 } 2831 if (stats.Offtype == set_compressed) 2832 stats.lastCountSize = countSize; 2833 op += countSize; 2834 assert(op <= oend); 2835 } } 2836 /* build CTable for MatchLengths */ 2837 { unsigned max = MaxML; 2838 size_t const mostFrequent = HIST_countFast_wksp( 2839 countWorkspace, &max, mlCodeTable, nbSeq, entropyWorkspace, entropyWkspSize); /* can't fail */ 2840 DEBUGLOG(5, "Building ML table (remaining space : %i)", (int)(oend-op)); 2841 nextEntropy->matchlength_repeatMode = prevEntropy->matchlength_repeatMode; 2842 stats.MLtype = ZSTD_selectEncodingType(&nextEntropy->matchlength_repeatMode, 2843 countWorkspace, max, mostFrequent, nbSeq, 2844 MLFSELog, prevEntropy->matchlengthCTable, 2845 ML_defaultNorm, ML_defaultNormLog, 2846 ZSTD_defaultAllowed, strategy); 2847 assert(!(stats.MLtype < set_compressed && nextEntropy->matchlength_repeatMode != FSE_repeat_none)); /* We don't copy tables */ 2848 { size_t const countSize = ZSTD_buildCTable( 2849 op, (size_t)(oend - op), 2850 CTable_MatchLength, MLFSELog, (symbolEncodingType_e)stats.MLtype, 2851 countWorkspace, max, mlCodeTable, nbSeq, 2852 ML_defaultNorm, ML_defaultNormLog, MaxML, 2853 prevEntropy->matchlengthCTable, 2854 sizeof(prevEntropy->matchlengthCTable), 2855 entropyWorkspace, entropyWkspSize); 2856 if (ZSTD_isError(countSize)) { 2857 DEBUGLOG(3, "ZSTD_buildCTable for MatchLengths failed"); 2858 stats.size = countSize; 2859 return stats; 2860 } 2861 if (stats.MLtype == set_compressed) 2862 stats.lastCountSize = countSize; 2863 op += countSize; 2864 assert(op <= oend); 2865 } } 2866 stats.size = (size_t)(op-ostart); 2867 return stats; 2868 } 2869 2870 /* ZSTD_entropyCompressSeqStore_internal(): 2871 * compresses both literals and sequences 2872 * Returns compressed size of block, or a zstd error. 2873 */ 2874 #define SUSPECT_UNCOMPRESSIBLE_LITERAL_RATIO 20 2875 MEM_STATIC size_t 2876 ZSTD_entropyCompressSeqStore_internal( 2877 const seqStore_t* seqStorePtr, 2878 const ZSTD_entropyCTables_t* prevEntropy, 2879 ZSTD_entropyCTables_t* nextEntropy, 2880 const ZSTD_CCtx_params* cctxParams, 2881 void* dst, size_t dstCapacity, 2882 void* entropyWorkspace, size_t entropyWkspSize, 2883 const int bmi2) 2884 { 2885 ZSTD_strategy const strategy = cctxParams->cParams.strategy; 2886 unsigned* count = (unsigned*)entropyWorkspace; 2887 FSE_CTable* CTable_LitLength = nextEntropy->fse.litlengthCTable; 2888 FSE_CTable* CTable_OffsetBits = nextEntropy->fse.offcodeCTable; 2889 FSE_CTable* CTable_MatchLength = nextEntropy->fse.matchlengthCTable; 2890 const seqDef* const sequences = seqStorePtr->sequencesStart; 2891 const size_t nbSeq = (size_t)(seqStorePtr->sequences - seqStorePtr->sequencesStart); 2892 const BYTE* const ofCodeTable = seqStorePtr->ofCode; 2893 const BYTE* const llCodeTable = seqStorePtr->llCode; 2894 const BYTE* const mlCodeTable = seqStorePtr->mlCode; 2895 BYTE* const ostart = (BYTE*)dst; 2896 BYTE* const oend = ostart + dstCapacity; 2897 BYTE* op = ostart; 2898 size_t lastCountSize; 2899 int longOffsets = 0; 2900 2901 entropyWorkspace = count + (MaxSeq + 1); 2902 entropyWkspSize -= (MaxSeq + 1) * sizeof(*count); 2903 2904 DEBUGLOG(5, "ZSTD_entropyCompressSeqStore_internal (nbSeq=%zu, dstCapacity=%zu)", nbSeq, dstCapacity); 2905 ZSTD_STATIC_ASSERT(HUF_WORKSPACE_SIZE >= (1<<MAX(MLFSELog,LLFSELog))); 2906 assert(entropyWkspSize >= HUF_WORKSPACE_SIZE); 2907 2908 /* Compress literals */ 2909 { const BYTE* const literals = seqStorePtr->litStart; 2910 size_t const numSequences = (size_t)(seqStorePtr->sequences - seqStorePtr->sequencesStart); 2911 size_t const numLiterals = (size_t)(seqStorePtr->lit - seqStorePtr->litStart); 2912 /* Base suspicion of uncompressibility on ratio of literals to sequences */ 2913 unsigned const suspectUncompressible = (numSequences == 0) || (numLiterals / numSequences >= SUSPECT_UNCOMPRESSIBLE_LITERAL_RATIO); 2914 size_t const litSize = (size_t)(seqStorePtr->lit - literals); 2915 2916 size_t const cSize = ZSTD_compressLiterals( 2917 op, dstCapacity, 2918 literals, litSize, 2919 entropyWorkspace, entropyWkspSize, 2920 &prevEntropy->huf, &nextEntropy->huf, 2921 cctxParams->cParams.strategy, 2922 ZSTD_literalsCompressionIsDisabled(cctxParams), 2923 suspectUncompressible, bmi2); 2924 FORWARD_IF_ERROR(cSize, "ZSTD_compressLiterals failed"); 2925 assert(cSize <= dstCapacity); 2926 op += cSize; 2927 } 2928 2929 /* Sequences Header */ 2930 RETURN_ERROR_IF((oend-op) < 3 /*max nbSeq Size*/ + 1 /*seqHead*/, 2931 dstSize_tooSmall, "Can't fit seq hdr in output buf!"); 2932 if (nbSeq < 128) { 2933 *op++ = (BYTE)nbSeq; 2934 } else if (nbSeq < LONGNBSEQ) { 2935 op[0] = (BYTE)((nbSeq>>8) + 0x80); 2936 op[1] = (BYTE)nbSeq; 2937 op+=2; 2938 } else { 2939 op[0]=0xFF; 2940 MEM_writeLE16(op+1, (U16)(nbSeq - LONGNBSEQ)); 2941 op+=3; 2942 } 2943 assert(op <= oend); 2944 if (nbSeq==0) { 2945 /* Copy the old tables over as if we repeated them */ 2946 ZSTD_memcpy(&nextEntropy->fse, &prevEntropy->fse, sizeof(prevEntropy->fse)); 2947 return (size_t)(op - ostart); 2948 } 2949 { BYTE* const seqHead = op++; 2950 /* build stats for sequences */ 2951 const ZSTD_symbolEncodingTypeStats_t stats = 2952 ZSTD_buildSequencesStatistics(seqStorePtr, nbSeq, 2953 &prevEntropy->fse, &nextEntropy->fse, 2954 op, oend, 2955 strategy, count, 2956 entropyWorkspace, entropyWkspSize); 2957 FORWARD_IF_ERROR(stats.size, "ZSTD_buildSequencesStatistics failed!"); 2958 *seqHead = (BYTE)((stats.LLtype<<6) + (stats.Offtype<<4) + (stats.MLtype<<2)); 2959 lastCountSize = stats.lastCountSize; 2960 op += stats.size; 2961 longOffsets = stats.longOffsets; 2962 } 2963 2964 { size_t const bitstreamSize = ZSTD_encodeSequences( 2965 op, (size_t)(oend - op), 2966 CTable_MatchLength, mlCodeTable, 2967 CTable_OffsetBits, ofCodeTable, 2968 CTable_LitLength, llCodeTable, 2969 sequences, nbSeq, 2970 longOffsets, bmi2); 2971 FORWARD_IF_ERROR(bitstreamSize, "ZSTD_encodeSequences failed"); 2972 op += bitstreamSize; 2973 assert(op <= oend); 2974 /* zstd versions <= 1.3.4 mistakenly report corruption when 2975 * FSE_readNCount() receives a buffer < 4 bytes. 2976 * Fixed by https://github.com/facebook/zstd/pull/1146. 2977 * This can happen when the last set_compressed table present is 2 2978 * bytes and the bitstream is only one byte. 2979 * In this exceedingly rare case, we will simply emit an uncompressed 2980 * block, since it isn't worth optimizing. 2981 */ 2982 if (lastCountSize && (lastCountSize + bitstreamSize) < 4) { 2983 /* lastCountSize >= 2 && bitstreamSize > 0 ==> lastCountSize == 3 */ 2984 assert(lastCountSize + bitstreamSize == 3); 2985 DEBUGLOG(5, "Avoiding bug in zstd decoder in versions <= 1.3.4 by " 2986 "emitting an uncompressed block."); 2987 return 0; 2988 } 2989 } 2990 2991 DEBUGLOG(5, "compressed block size : %u", (unsigned)(op - ostart)); 2992 return (size_t)(op - ostart); 2993 } 2994 2995 MEM_STATIC size_t 2996 ZSTD_entropyCompressSeqStore( 2997 const seqStore_t* seqStorePtr, 2998 const ZSTD_entropyCTables_t* prevEntropy, 2999 ZSTD_entropyCTables_t* nextEntropy, 3000 const ZSTD_CCtx_params* cctxParams, 3001 void* dst, size_t dstCapacity, 3002 size_t srcSize, 3003 void* entropyWorkspace, size_t entropyWkspSize, 3004 int bmi2) 3005 { 3006 size_t const cSize = ZSTD_entropyCompressSeqStore_internal( 3007 seqStorePtr, prevEntropy, nextEntropy, cctxParams, 3008 dst, dstCapacity, 3009 entropyWorkspace, entropyWkspSize, bmi2); 3010 if (cSize == 0) return 0; 3011 /* When srcSize <= dstCapacity, there is enough space to write a raw uncompressed block. 3012 * Since we ran out of space, block must be not compressible, so fall back to raw uncompressed block. 3013 */ 3014 if ((cSize == ERROR(dstSize_tooSmall)) & (srcSize <= dstCapacity)) { 3015 DEBUGLOG(4, "not enough dstCapacity (%zu) for ZSTD_entropyCompressSeqStore_internal()=> do not compress block", dstCapacity); 3016 return 0; /* block not compressed */ 3017 } 3018 FORWARD_IF_ERROR(cSize, "ZSTD_entropyCompressSeqStore_internal failed"); 3019 3020 /* Check compressibility */ 3021 { size_t const maxCSize = srcSize - ZSTD_minGain(srcSize, cctxParams->cParams.strategy); 3022 if (cSize >= maxCSize) return 0; /* block not compressed */ 3023 } 3024 DEBUGLOG(5, "ZSTD_entropyCompressSeqStore() cSize: %zu", cSize); 3025 /* libzstd decoder before > v1.5.4 is not compatible with compressed blocks of size ZSTD_BLOCKSIZE_MAX exactly. 3026 * This restriction is indirectly already fulfilled by respecting ZSTD_minGain() condition above. 3027 */ 3028 assert(cSize < ZSTD_BLOCKSIZE_MAX); 3029 return cSize; 3030 } 3031 3032 /* ZSTD_selectBlockCompressor() : 3033 * Not static, but internal use only (used by long distance matcher) 3034 * assumption : strat is a valid strategy */ 3035 ZSTD_blockCompressor ZSTD_selectBlockCompressor(ZSTD_strategy strat, ZSTD_paramSwitch_e useRowMatchFinder, ZSTD_dictMode_e dictMode) 3036 { 3037 static const ZSTD_blockCompressor blockCompressor[4][ZSTD_STRATEGY_MAX+1] = { 3038 { ZSTD_compressBlock_fast /* default for 0 */, 3039 ZSTD_compressBlock_fast, 3040 ZSTD_COMPRESSBLOCK_DOUBLEFAST, 3041 ZSTD_COMPRESSBLOCK_GREEDY, 3042 ZSTD_COMPRESSBLOCK_LAZY, 3043 ZSTD_COMPRESSBLOCK_LAZY2, 3044 ZSTD_COMPRESSBLOCK_BTLAZY2, 3045 ZSTD_COMPRESSBLOCK_BTOPT, 3046 ZSTD_COMPRESSBLOCK_BTULTRA, 3047 ZSTD_COMPRESSBLOCK_BTULTRA2 3048 }, 3049 { ZSTD_compressBlock_fast_extDict /* default for 0 */, 3050 ZSTD_compressBlock_fast_extDict, 3051 ZSTD_COMPRESSBLOCK_DOUBLEFAST_EXTDICT, 3052 ZSTD_COMPRESSBLOCK_GREEDY_EXTDICT, 3053 ZSTD_COMPRESSBLOCK_LAZY_EXTDICT, 3054 ZSTD_COMPRESSBLOCK_LAZY2_EXTDICT, 3055 ZSTD_COMPRESSBLOCK_BTLAZY2_EXTDICT, 3056 ZSTD_COMPRESSBLOCK_BTOPT_EXTDICT, 3057 ZSTD_COMPRESSBLOCK_BTULTRA_EXTDICT, 3058 ZSTD_COMPRESSBLOCK_BTULTRA_EXTDICT 3059 }, 3060 { ZSTD_compressBlock_fast_dictMatchState /* default for 0 */, 3061 ZSTD_compressBlock_fast_dictMatchState, 3062 ZSTD_COMPRESSBLOCK_DOUBLEFAST_DICTMATCHSTATE, 3063 ZSTD_COMPRESSBLOCK_GREEDY_DICTMATCHSTATE, 3064 ZSTD_COMPRESSBLOCK_LAZY_DICTMATCHSTATE, 3065 ZSTD_COMPRESSBLOCK_LAZY2_DICTMATCHSTATE, 3066 ZSTD_COMPRESSBLOCK_BTLAZY2_DICTMATCHSTATE, 3067 ZSTD_COMPRESSBLOCK_BTOPT_DICTMATCHSTATE, 3068 ZSTD_COMPRESSBLOCK_BTULTRA_DICTMATCHSTATE, 3069 ZSTD_COMPRESSBLOCK_BTULTRA_DICTMATCHSTATE 3070 }, 3071 { NULL /* default for 0 */, 3072 NULL, 3073 NULL, 3074 ZSTD_COMPRESSBLOCK_GREEDY_DEDICATEDDICTSEARCH, 3075 ZSTD_COMPRESSBLOCK_LAZY_DEDICATEDDICTSEARCH, 3076 ZSTD_COMPRESSBLOCK_LAZY2_DEDICATEDDICTSEARCH, 3077 NULL, 3078 NULL, 3079 NULL, 3080 NULL } 3081 }; 3082 ZSTD_blockCompressor selectedCompressor; 3083 ZSTD_STATIC_ASSERT((unsigned)ZSTD_fast == 1); 3084 3085 assert(ZSTD_cParam_withinBounds(ZSTD_c_strategy, strat)); 3086 DEBUGLOG(4, "Selected block compressor: dictMode=%d strat=%d rowMatchfinder=%d", (int)dictMode, (int)strat, (int)useRowMatchFinder); 3087 if (ZSTD_rowMatchFinderUsed(strat, useRowMatchFinder)) { 3088 static const ZSTD_blockCompressor rowBasedBlockCompressors[4][3] = { 3089 { 3090 ZSTD_COMPRESSBLOCK_GREEDY_ROW, 3091 ZSTD_COMPRESSBLOCK_LAZY_ROW, 3092 ZSTD_COMPRESSBLOCK_LAZY2_ROW 3093 }, 3094 { 3095 ZSTD_COMPRESSBLOCK_GREEDY_EXTDICT_ROW, 3096 ZSTD_COMPRESSBLOCK_LAZY_EXTDICT_ROW, 3097 ZSTD_COMPRESSBLOCK_LAZY2_EXTDICT_ROW 3098 }, 3099 { 3100 ZSTD_COMPRESSBLOCK_GREEDY_DICTMATCHSTATE_ROW, 3101 ZSTD_COMPRESSBLOCK_LAZY_DICTMATCHSTATE_ROW, 3102 ZSTD_COMPRESSBLOCK_LAZY2_DICTMATCHSTATE_ROW 3103 }, 3104 { 3105 ZSTD_COMPRESSBLOCK_GREEDY_DEDICATEDDICTSEARCH_ROW, 3106 ZSTD_COMPRESSBLOCK_LAZY_DEDICATEDDICTSEARCH_ROW, 3107 ZSTD_COMPRESSBLOCK_LAZY2_DEDICATEDDICTSEARCH_ROW 3108 } 3109 }; 3110 DEBUGLOG(4, "Selecting a row-based matchfinder"); 3111 assert(useRowMatchFinder != ZSTD_ps_auto); 3112 selectedCompressor = rowBasedBlockCompressors[(int)dictMode][(int)strat - (int)ZSTD_greedy]; 3113 } else { 3114 selectedCompressor = blockCompressor[(int)dictMode][(int)strat]; 3115 } 3116 assert(selectedCompressor != NULL); 3117 return selectedCompressor; 3118 } 3119 3120 static void ZSTD_storeLastLiterals(seqStore_t* seqStorePtr, 3121 const BYTE* anchor, size_t lastLLSize) 3122 { 3123 ZSTD_memcpy(seqStorePtr->lit, anchor, lastLLSize); 3124 seqStorePtr->lit += lastLLSize; 3125 } 3126 3127 void ZSTD_resetSeqStore(seqStore_t* ssPtr) 3128 { 3129 ssPtr->lit = ssPtr->litStart; 3130 ssPtr->sequences = ssPtr->sequencesStart; 3131 ssPtr->longLengthType = ZSTD_llt_none; 3132 } 3133 3134 /* ZSTD_postProcessSequenceProducerResult() : 3135 * Validates and post-processes sequences obtained through the external matchfinder API: 3136 * - Checks whether nbExternalSeqs represents an error condition. 3137 * - Appends a block delimiter to outSeqs if one is not already present. 3138 * See zstd.h for context regarding block delimiters. 3139 * Returns the number of sequences after post-processing, or an error code. */ 3140 static size_t ZSTD_postProcessSequenceProducerResult( 3141 ZSTD_Sequence* outSeqs, size_t nbExternalSeqs, size_t outSeqsCapacity, size_t srcSize 3142 ) { 3143 RETURN_ERROR_IF( 3144 nbExternalSeqs > outSeqsCapacity, 3145 sequenceProducer_failed, 3146 "External sequence producer returned error code %lu", 3147 (unsigned long)nbExternalSeqs 3148 ); 3149 3150 RETURN_ERROR_IF( 3151 nbExternalSeqs == 0 && srcSize > 0, 3152 sequenceProducer_failed, 3153 "Got zero sequences from external sequence producer for a non-empty src buffer!" 3154 ); 3155 3156 if (srcSize == 0) { 3157 ZSTD_memset(&outSeqs[0], 0, sizeof(ZSTD_Sequence)); 3158 return 1; 3159 } 3160 3161 { 3162 ZSTD_Sequence const lastSeq = outSeqs[nbExternalSeqs - 1]; 3163 3164 /* We can return early if lastSeq is already a block delimiter. */ 3165 if (lastSeq.offset == 0 && lastSeq.matchLength == 0) { 3166 return nbExternalSeqs; 3167 } 3168 3169 /* This error condition is only possible if the external matchfinder 3170 * produced an invalid parse, by definition of ZSTD_sequenceBound(). */ 3171 RETURN_ERROR_IF( 3172 nbExternalSeqs == outSeqsCapacity, 3173 sequenceProducer_failed, 3174 "nbExternalSeqs == outSeqsCapacity but lastSeq is not a block delimiter!" 3175 ); 3176 3177 /* lastSeq is not a block delimiter, so we need to append one. */ 3178 ZSTD_memset(&outSeqs[nbExternalSeqs], 0, sizeof(ZSTD_Sequence)); 3179 return nbExternalSeqs + 1; 3180 } 3181 } 3182 3183 /* ZSTD_fastSequenceLengthSum() : 3184 * Returns sum(litLen) + sum(matchLen) + lastLits for *seqBuf*. 3185 * Similar to another function in zstd_compress.c (determine_blockSize), 3186 * except it doesn't check for a block delimiter to end summation. 3187 * Removing the early exit allows the compiler to auto-vectorize (https://godbolt.org/z/cY1cajz9P). 3188 * This function can be deleted and replaced by determine_blockSize after we resolve issue #3456. */ 3189 static size_t ZSTD_fastSequenceLengthSum(ZSTD_Sequence const* seqBuf, size_t seqBufSize) { 3190 size_t matchLenSum, litLenSum, i; 3191 matchLenSum = 0; 3192 litLenSum = 0; 3193 for (i = 0; i < seqBufSize; i++) { 3194 litLenSum += seqBuf[i].litLength; 3195 matchLenSum += seqBuf[i].matchLength; 3196 } 3197 return litLenSum + matchLenSum; 3198 } 3199 3200 typedef enum { ZSTDbss_compress, ZSTDbss_noCompress } ZSTD_buildSeqStore_e; 3201 3202 static size_t ZSTD_buildSeqStore(ZSTD_CCtx* zc, const void* src, size_t srcSize) 3203 { 3204 ZSTD_matchState_t* const ms = &zc->blockState.matchState; 3205 DEBUGLOG(5, "ZSTD_buildSeqStore (srcSize=%zu)", srcSize); 3206 assert(srcSize <= ZSTD_BLOCKSIZE_MAX); 3207 /* Assert that we have correctly flushed the ctx params into the ms's copy */ 3208 ZSTD_assertEqualCParams(zc->appliedParams.cParams, ms->cParams); 3209 /* TODO: See 3090. We reduced MIN_CBLOCK_SIZE from 3 to 2 so to compensate we are adding 3210 * additional 1. We need to revisit and change this logic to be more consistent */ 3211 if (srcSize < MIN_CBLOCK_SIZE+ZSTD_blockHeaderSize+1+1) { 3212 if (zc->appliedParams.cParams.strategy >= ZSTD_btopt) { 3213 ZSTD_ldm_skipRawSeqStoreBytes(&zc->externSeqStore, srcSize); 3214 } else { 3215 ZSTD_ldm_skipSequences(&zc->externSeqStore, srcSize, zc->appliedParams.cParams.minMatch); 3216 } 3217 return ZSTDbss_noCompress; /* don't even attempt compression below a certain srcSize */ 3218 } 3219 ZSTD_resetSeqStore(&(zc->seqStore)); 3220 /* required for optimal parser to read stats from dictionary */ 3221 ms->opt.symbolCosts = &zc->blockState.prevCBlock->entropy; 3222 /* tell the optimal parser how we expect to compress literals */ 3223 ms->opt.literalCompressionMode = zc->appliedParams.literalCompressionMode; 3224 /* a gap between an attached dict and the current window is not safe, 3225 * they must remain adjacent, 3226 * and when that stops being the case, the dict must be unset */ 3227 assert(ms->dictMatchState == NULL || ms->loadedDictEnd == ms->window.dictLimit); 3228 3229 /* limited update after a very long match */ 3230 { const BYTE* const base = ms->window.base; 3231 const BYTE* const istart = (const BYTE*)src; 3232 const U32 curr = (U32)(istart-base); 3233 if (sizeof(ptrdiff_t)==8) assert(istart - base < (ptrdiff_t)(U32)(-1)); /* ensure no overflow */ 3234 if (curr > ms->nextToUpdate + 384) 3235 ms->nextToUpdate = curr - MIN(192, (U32)(curr - ms->nextToUpdate - 384)); 3236 } 3237 3238 /* select and store sequences */ 3239 { ZSTD_dictMode_e const dictMode = ZSTD_matchState_dictMode(ms); 3240 size_t lastLLSize; 3241 { int i; 3242 for (i = 0; i < ZSTD_REP_NUM; ++i) 3243 zc->blockState.nextCBlock->rep[i] = zc->blockState.prevCBlock->rep[i]; 3244 } 3245 if (zc->externSeqStore.pos < zc->externSeqStore.size) { 3246 assert(zc->appliedParams.ldmParams.enableLdm == ZSTD_ps_disable); 3247 3248 /* External matchfinder + LDM is technically possible, just not implemented yet. 3249 * We need to revisit soon and implement it. */ 3250 RETURN_ERROR_IF( 3251 ZSTD_hasExtSeqProd(&zc->appliedParams), 3252 parameter_combination_unsupported, 3253 "Long-distance matching with external sequence producer enabled is not currently supported." 3254 ); 3255 3256 /* Updates ldmSeqStore.pos */ 3257 lastLLSize = 3258 ZSTD_ldm_blockCompress(&zc->externSeqStore, 3259 ms, &zc->seqStore, 3260 zc->blockState.nextCBlock->rep, 3261 zc->appliedParams.useRowMatchFinder, 3262 src, srcSize); 3263 assert(zc->externSeqStore.pos <= zc->externSeqStore.size); 3264 } else if (zc->appliedParams.ldmParams.enableLdm == ZSTD_ps_enable) { 3265 rawSeqStore_t ldmSeqStore = kNullRawSeqStore; 3266 3267 /* External matchfinder + LDM is technically possible, just not implemented yet. 3268 * We need to revisit soon and implement it. */ 3269 RETURN_ERROR_IF( 3270 ZSTD_hasExtSeqProd(&zc->appliedParams), 3271 parameter_combination_unsupported, 3272 "Long-distance matching with external sequence producer enabled is not currently supported." 3273 ); 3274 3275 ldmSeqStore.seq = zc->ldmSequences; 3276 ldmSeqStore.capacity = zc->maxNbLdmSequences; 3277 /* Updates ldmSeqStore.size */ 3278 FORWARD_IF_ERROR(ZSTD_ldm_generateSequences(&zc->ldmState, &ldmSeqStore, 3279 &zc->appliedParams.ldmParams, 3280 src, srcSize), ""); 3281 /* Updates ldmSeqStore.pos */ 3282 lastLLSize = 3283 ZSTD_ldm_blockCompress(&ldmSeqStore, 3284 ms, &zc->seqStore, 3285 zc->blockState.nextCBlock->rep, 3286 zc->appliedParams.useRowMatchFinder, 3287 src, srcSize); 3288 assert(ldmSeqStore.pos == ldmSeqStore.size); 3289 } else if (ZSTD_hasExtSeqProd(&zc->appliedParams)) { 3290 assert( 3291 zc->extSeqBufCapacity >= ZSTD_sequenceBound(srcSize) 3292 ); 3293 assert(zc->appliedParams.extSeqProdFunc != NULL); 3294 3295 { U32 const windowSize = (U32)1 << zc->appliedParams.cParams.windowLog; 3296 3297 size_t const nbExternalSeqs = (zc->appliedParams.extSeqProdFunc)( 3298 zc->appliedParams.extSeqProdState, 3299 zc->extSeqBuf, 3300 zc->extSeqBufCapacity, 3301 src, srcSize, 3302 NULL, 0, /* dict and dictSize, currently not supported */ 3303 zc->appliedParams.compressionLevel, 3304 windowSize 3305 ); 3306 3307 size_t const nbPostProcessedSeqs = ZSTD_postProcessSequenceProducerResult( 3308 zc->extSeqBuf, 3309 nbExternalSeqs, 3310 zc->extSeqBufCapacity, 3311 srcSize 3312 ); 3313 3314 /* Return early if there is no error, since we don't need to worry about last literals */ 3315 if (!ZSTD_isError(nbPostProcessedSeqs)) { 3316 ZSTD_sequencePosition seqPos = {0,0,0}; 3317 size_t const seqLenSum = ZSTD_fastSequenceLengthSum(zc->extSeqBuf, nbPostProcessedSeqs); 3318 RETURN_ERROR_IF(seqLenSum > srcSize, externalSequences_invalid, "External sequences imply too large a block!"); 3319 FORWARD_IF_ERROR( 3320 ZSTD_copySequencesToSeqStoreExplicitBlockDelim( 3321 zc, &seqPos, 3322 zc->extSeqBuf, nbPostProcessedSeqs, 3323 src, srcSize, 3324 zc->appliedParams.searchForExternalRepcodes 3325 ), 3326 "Failed to copy external sequences to seqStore!" 3327 ); 3328 ms->ldmSeqStore = NULL; 3329 DEBUGLOG(5, "Copied %lu sequences from external sequence producer to internal seqStore.", (unsigned long)nbExternalSeqs); 3330 return ZSTDbss_compress; 3331 } 3332 3333 /* Propagate the error if fallback is disabled */ 3334 if (!zc->appliedParams.enableMatchFinderFallback) { 3335 return nbPostProcessedSeqs; 3336 } 3337 3338 /* Fallback to software matchfinder */ 3339 { ZSTD_blockCompressor const blockCompressor = 3340 ZSTD_selectBlockCompressor( 3341 zc->appliedParams.cParams.strategy, 3342 zc->appliedParams.useRowMatchFinder, 3343 dictMode); 3344 ms->ldmSeqStore = NULL; 3345 DEBUGLOG( 3346 5, 3347 "External sequence producer returned error code %lu. Falling back to internal parser.", 3348 (unsigned long)nbExternalSeqs 3349 ); 3350 lastLLSize = blockCompressor(ms, &zc->seqStore, zc->blockState.nextCBlock->rep, src, srcSize); 3351 } } 3352 } else { /* not long range mode and no external matchfinder */ 3353 ZSTD_blockCompressor const blockCompressor = ZSTD_selectBlockCompressor( 3354 zc->appliedParams.cParams.strategy, 3355 zc->appliedParams.useRowMatchFinder, 3356 dictMode); 3357 ms->ldmSeqStore = NULL; 3358 lastLLSize = blockCompressor(ms, &zc->seqStore, zc->blockState.nextCBlock->rep, src, srcSize); 3359 } 3360 { const BYTE* const lastLiterals = (const BYTE*)src + srcSize - lastLLSize; 3361 ZSTD_storeLastLiterals(&zc->seqStore, lastLiterals, lastLLSize); 3362 } } 3363 return ZSTDbss_compress; 3364 } 3365 3366 static size_t ZSTD_copyBlockSequences(SeqCollector* seqCollector, const seqStore_t* seqStore, const U32 prevRepcodes[ZSTD_REP_NUM]) 3367 { 3368 const seqDef* inSeqs = seqStore->sequencesStart; 3369 const size_t nbInSequences = seqStore->sequences - inSeqs; 3370 const size_t nbInLiterals = (size_t)(seqStore->lit - seqStore->litStart); 3371 3372 ZSTD_Sequence* outSeqs = seqCollector->seqIndex == 0 ? seqCollector->seqStart : seqCollector->seqStart + seqCollector->seqIndex; 3373 const size_t nbOutSequences = nbInSequences + 1; 3374 size_t nbOutLiterals = 0; 3375 repcodes_t repcodes; 3376 size_t i; 3377 3378 /* Bounds check that we have enough space for every input sequence 3379 * and the block delimiter 3380 */ 3381 assert(seqCollector->seqIndex <= seqCollector->maxSequences); 3382 RETURN_ERROR_IF( 3383 nbOutSequences > (size_t)(seqCollector->maxSequences - seqCollector->seqIndex), 3384 dstSize_tooSmall, 3385 "Not enough space to copy sequences"); 3386 3387 ZSTD_memcpy(&repcodes, prevRepcodes, sizeof(repcodes)); 3388 for (i = 0; i < nbInSequences; ++i) { 3389 U32 rawOffset; 3390 outSeqs[i].litLength = inSeqs[i].litLength; 3391 outSeqs[i].matchLength = inSeqs[i].mlBase + MINMATCH; 3392 outSeqs[i].rep = 0; 3393 3394 /* Handle the possible single length >= 64K 3395 * There can only be one because we add MINMATCH to every match length, 3396 * and blocks are at most 128K. 3397 */ 3398 if (i == seqStore->longLengthPos) { 3399 if (seqStore->longLengthType == ZSTD_llt_literalLength) { 3400 outSeqs[i].litLength += 0x10000; 3401 } else if (seqStore->longLengthType == ZSTD_llt_matchLength) { 3402 outSeqs[i].matchLength += 0x10000; 3403 } 3404 } 3405 3406 /* Determine the raw offset given the offBase, which may be a repcode. */ 3407 if (OFFBASE_IS_REPCODE(inSeqs[i].offBase)) { 3408 const U32 repcode = OFFBASE_TO_REPCODE(inSeqs[i].offBase); 3409 assert(repcode > 0); 3410 outSeqs[i].rep = repcode; 3411 if (outSeqs[i].litLength != 0) { 3412 rawOffset = repcodes.rep[repcode - 1]; 3413 } else { 3414 if (repcode == 3) { 3415 assert(repcodes.rep[0] > 1); 3416 rawOffset = repcodes.rep[0] - 1; 3417 } else { 3418 rawOffset = repcodes.rep[repcode]; 3419 } 3420 } 3421 } else { 3422 rawOffset = OFFBASE_TO_OFFSET(inSeqs[i].offBase); 3423 } 3424 outSeqs[i].offset = rawOffset; 3425 3426 /* Update repcode history for the sequence */ 3427 ZSTD_updateRep(repcodes.rep, 3428 inSeqs[i].offBase, 3429 inSeqs[i].litLength == 0); 3430 3431 nbOutLiterals += outSeqs[i].litLength; 3432 } 3433 /* Insert last literals (if any exist) in the block as a sequence with ml == off == 0. 3434 * If there are no last literals, then we'll emit (of: 0, ml: 0, ll: 0), which is a marker 3435 * for the block boundary, according to the API. 3436 */ 3437 assert(nbInLiterals >= nbOutLiterals); 3438 { 3439 const size_t lastLLSize = nbInLiterals - nbOutLiterals; 3440 outSeqs[nbInSequences].litLength = (U32)lastLLSize; 3441 outSeqs[nbInSequences].matchLength = 0; 3442 outSeqs[nbInSequences].offset = 0; 3443 assert(nbOutSequences == nbInSequences + 1); 3444 } 3445 seqCollector->seqIndex += nbOutSequences; 3446 assert(seqCollector->seqIndex <= seqCollector->maxSequences); 3447 3448 return 0; 3449 } 3450 3451 size_t ZSTD_sequenceBound(size_t srcSize) { 3452 const size_t maxNbSeq = (srcSize / ZSTD_MINMATCH_MIN) + 1; 3453 const size_t maxNbDelims = (srcSize / ZSTD_BLOCKSIZE_MAX_MIN) + 1; 3454 return maxNbSeq + maxNbDelims; 3455 } 3456 3457 size_t ZSTD_generateSequences(ZSTD_CCtx* zc, ZSTD_Sequence* outSeqs, 3458 size_t outSeqsSize, const void* src, size_t srcSize) 3459 { 3460 const size_t dstCapacity = ZSTD_compressBound(srcSize); 3461 void* dst = ZSTD_customMalloc(dstCapacity, ZSTD_defaultCMem); 3462 SeqCollector seqCollector; 3463 { 3464 int targetCBlockSize; 3465 FORWARD_IF_ERROR(ZSTD_CCtx_getParameter(zc, ZSTD_c_targetCBlockSize, &targetCBlockSize), ""); 3466 RETURN_ERROR_IF(targetCBlockSize != 0, parameter_unsupported, "targetCBlockSize != 0"); 3467 } 3468 { 3469 int nbWorkers; 3470 FORWARD_IF_ERROR(ZSTD_CCtx_getParameter(zc, ZSTD_c_nbWorkers, &nbWorkers), ""); 3471 RETURN_ERROR_IF(nbWorkers != 0, parameter_unsupported, "nbWorkers != 0"); 3472 } 3473 3474 RETURN_ERROR_IF(dst == NULL, memory_allocation, "NULL pointer!"); 3475 3476 seqCollector.collectSequences = 1; 3477 seqCollector.seqStart = outSeqs; 3478 seqCollector.seqIndex = 0; 3479 seqCollector.maxSequences = outSeqsSize; 3480 zc->seqCollector = seqCollector; 3481 3482 { 3483 const size_t ret = ZSTD_compress2(zc, dst, dstCapacity, src, srcSize); 3484 ZSTD_customFree(dst, ZSTD_defaultCMem); 3485 FORWARD_IF_ERROR(ret, "ZSTD_compress2 failed"); 3486 } 3487 assert(zc->seqCollector.seqIndex <= ZSTD_sequenceBound(srcSize)); 3488 return zc->seqCollector.seqIndex; 3489 } 3490 3491 size_t ZSTD_mergeBlockDelimiters(ZSTD_Sequence* sequences, size_t seqsSize) { 3492 size_t in = 0; 3493 size_t out = 0; 3494 for (; in < seqsSize; ++in) { 3495 if (sequences[in].offset == 0 && sequences[in].matchLength == 0) { 3496 if (in != seqsSize - 1) { 3497 sequences[in+1].litLength += sequences[in].litLength; 3498 } 3499 } else { 3500 sequences[out] = sequences[in]; 3501 ++out; 3502 } 3503 } 3504 return out; 3505 } 3506 3507 /* Unrolled loop to read four size_ts of input at a time. Returns 1 if is RLE, 0 if not. */ 3508 static int ZSTD_isRLE(const BYTE* src, size_t length) { 3509 const BYTE* ip = src; 3510 const BYTE value = ip[0]; 3511 const size_t valueST = (size_t)((U64)value * 0x0101010101010101ULL); 3512 const size_t unrollSize = sizeof(size_t) * 4; 3513 const size_t unrollMask = unrollSize - 1; 3514 const size_t prefixLength = length & unrollMask; 3515 size_t i; 3516 if (length == 1) return 1; 3517 /* Check if prefix is RLE first before using unrolled loop */ 3518 if (prefixLength && ZSTD_count(ip+1, ip, ip+prefixLength) != prefixLength-1) { 3519 return 0; 3520 } 3521 for (i = prefixLength; i != length; i += unrollSize) { 3522 size_t u; 3523 for (u = 0; u < unrollSize; u += sizeof(size_t)) { 3524 if (MEM_readST(ip + i + u) != valueST) { 3525 return 0; 3526 } } } 3527 return 1; 3528 } 3529 3530 /* Returns true if the given block may be RLE. 3531 * This is just a heuristic based on the compressibility. 3532 * It may return both false positives and false negatives. 3533 */ 3534 static int ZSTD_maybeRLE(seqStore_t const* seqStore) 3535 { 3536 size_t const nbSeqs = (size_t)(seqStore->sequences - seqStore->sequencesStart); 3537 size_t const nbLits = (size_t)(seqStore->lit - seqStore->litStart); 3538 3539 return nbSeqs < 4 && nbLits < 10; 3540 } 3541 3542 static void 3543 ZSTD_blockState_confirmRepcodesAndEntropyTables(ZSTD_blockState_t* const bs) 3544 { 3545 ZSTD_compressedBlockState_t* const tmp = bs->prevCBlock; 3546 bs->prevCBlock = bs->nextCBlock; 3547 bs->nextCBlock = tmp; 3548 } 3549 3550 /* Writes the block header */ 3551 static void 3552 writeBlockHeader(void* op, size_t cSize, size_t blockSize, U32 lastBlock) 3553 { 3554 U32 const cBlockHeader = cSize == 1 ? 3555 lastBlock + (((U32)bt_rle)<<1) + (U32)(blockSize << 3) : 3556 lastBlock + (((U32)bt_compressed)<<1) + (U32)(cSize << 3); 3557 MEM_writeLE24(op, cBlockHeader); 3558 DEBUGLOG(3, "writeBlockHeader: cSize: %zu blockSize: %zu lastBlock: %u", cSize, blockSize, lastBlock); 3559 } 3560 3561 /** ZSTD_buildBlockEntropyStats_literals() : 3562 * Builds entropy for the literals. 3563 * Stores literals block type (raw, rle, compressed, repeat) and 3564 * huffman description table to hufMetadata. 3565 * Requires ENTROPY_WORKSPACE_SIZE workspace 3566 * @return : size of huffman description table, or an error code 3567 */ 3568 static size_t 3569 ZSTD_buildBlockEntropyStats_literals(void* const src, size_t srcSize, 3570 const ZSTD_hufCTables_t* prevHuf, 3571 ZSTD_hufCTables_t* nextHuf, 3572 ZSTD_hufCTablesMetadata_t* hufMetadata, 3573 const int literalsCompressionIsDisabled, 3574 void* workspace, size_t wkspSize, 3575 int hufFlags) 3576 { 3577 BYTE* const wkspStart = (BYTE*)workspace; 3578 BYTE* const wkspEnd = wkspStart + wkspSize; 3579 BYTE* const countWkspStart = wkspStart; 3580 unsigned* const countWksp = (unsigned*)workspace; 3581 const size_t countWkspSize = (HUF_SYMBOLVALUE_MAX + 1) * sizeof(unsigned); 3582 BYTE* const nodeWksp = countWkspStart + countWkspSize; 3583 const size_t nodeWkspSize = (size_t)(wkspEnd - nodeWksp); 3584 unsigned maxSymbolValue = HUF_SYMBOLVALUE_MAX; 3585 unsigned huffLog = LitHufLog; 3586 HUF_repeat repeat = prevHuf->repeatMode; 3587 DEBUGLOG(5, "ZSTD_buildBlockEntropyStats_literals (srcSize=%zu)", srcSize); 3588 3589 /* Prepare nextEntropy assuming reusing the existing table */ 3590 ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf)); 3591 3592 if (literalsCompressionIsDisabled) { 3593 DEBUGLOG(5, "set_basic - disabled"); 3594 hufMetadata->hType = set_basic; 3595 return 0; 3596 } 3597 3598 /* small ? don't even attempt compression (speed opt) */ 3599 #ifndef COMPRESS_LITERALS_SIZE_MIN 3600 # define COMPRESS_LITERALS_SIZE_MIN 63 /* heuristic */ 3601 #endif 3602 { size_t const minLitSize = (prevHuf->repeatMode == HUF_repeat_valid) ? 6 : COMPRESS_LITERALS_SIZE_MIN; 3603 if (srcSize <= minLitSize) { 3604 DEBUGLOG(5, "set_basic - too small"); 3605 hufMetadata->hType = set_basic; 3606 return 0; 3607 } } 3608 3609 /* Scan input and build symbol stats */ 3610 { size_t const largest = 3611 HIST_count_wksp (countWksp, &maxSymbolValue, 3612 (const BYTE*)src, srcSize, 3613 workspace, wkspSize); 3614 FORWARD_IF_ERROR(largest, "HIST_count_wksp failed"); 3615 if (largest == srcSize) { 3616 /* only one literal symbol */ 3617 DEBUGLOG(5, "set_rle"); 3618 hufMetadata->hType = set_rle; 3619 return 0; 3620 } 3621 if (largest <= (srcSize >> 7)+4) { 3622 /* heuristic: likely not compressible */ 3623 DEBUGLOG(5, "set_basic - no gain"); 3624 hufMetadata->hType = set_basic; 3625 return 0; 3626 } } 3627 3628 /* Validate the previous Huffman table */ 3629 if (repeat == HUF_repeat_check 3630 && !HUF_validateCTable((HUF_CElt const*)prevHuf->CTable, countWksp, maxSymbolValue)) { 3631 repeat = HUF_repeat_none; 3632 } 3633 3634 /* Build Huffman Tree */ 3635 ZSTD_memset(nextHuf->CTable, 0, sizeof(nextHuf->CTable)); 3636 huffLog = HUF_optimalTableLog(huffLog, srcSize, maxSymbolValue, nodeWksp, nodeWkspSize, nextHuf->CTable, countWksp, hufFlags); 3637 assert(huffLog <= LitHufLog); 3638 { size_t const maxBits = HUF_buildCTable_wksp((HUF_CElt*)nextHuf->CTable, countWksp, 3639 maxSymbolValue, huffLog, 3640 nodeWksp, nodeWkspSize); 3641 FORWARD_IF_ERROR(maxBits, "HUF_buildCTable_wksp"); 3642 huffLog = (U32)maxBits; 3643 } 3644 { /* Build and write the CTable */ 3645 size_t const newCSize = HUF_estimateCompressedSize( 3646 (HUF_CElt*)nextHuf->CTable, countWksp, maxSymbolValue); 3647 size_t const hSize = HUF_writeCTable_wksp( 3648 hufMetadata->hufDesBuffer, sizeof(hufMetadata->hufDesBuffer), 3649 (HUF_CElt*)nextHuf->CTable, maxSymbolValue, huffLog, 3650 nodeWksp, nodeWkspSize); 3651 /* Check against repeating the previous CTable */ 3652 if (repeat != HUF_repeat_none) { 3653 size_t const oldCSize = HUF_estimateCompressedSize( 3654 (HUF_CElt const*)prevHuf->CTable, countWksp, maxSymbolValue); 3655 if (oldCSize < srcSize && (oldCSize <= hSize + newCSize || hSize + 12 >= srcSize)) { 3656 DEBUGLOG(5, "set_repeat - smaller"); 3657 ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf)); 3658 hufMetadata->hType = set_repeat; 3659 return 0; 3660 } } 3661 if (newCSize + hSize >= srcSize) { 3662 DEBUGLOG(5, "set_basic - no gains"); 3663 ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf)); 3664 hufMetadata->hType = set_basic; 3665 return 0; 3666 } 3667 DEBUGLOG(5, "set_compressed (hSize=%u)", (U32)hSize); 3668 hufMetadata->hType = set_compressed; 3669 nextHuf->repeatMode = HUF_repeat_check; 3670 return hSize; 3671 } 3672 } 3673 3674 3675 /* ZSTD_buildDummySequencesStatistics(): 3676 * Returns a ZSTD_symbolEncodingTypeStats_t with all encoding types as set_basic, 3677 * and updates nextEntropy to the appropriate repeatMode. 3678 */ 3679 static ZSTD_symbolEncodingTypeStats_t 3680 ZSTD_buildDummySequencesStatistics(ZSTD_fseCTables_t* nextEntropy) 3681 { 3682 ZSTD_symbolEncodingTypeStats_t stats = {set_basic, set_basic, set_basic, 0, 0, 0}; 3683 nextEntropy->litlength_repeatMode = FSE_repeat_none; 3684 nextEntropy->offcode_repeatMode = FSE_repeat_none; 3685 nextEntropy->matchlength_repeatMode = FSE_repeat_none; 3686 return stats; 3687 } 3688 3689 /** ZSTD_buildBlockEntropyStats_sequences() : 3690 * Builds entropy for the sequences. 3691 * Stores symbol compression modes and fse table to fseMetadata. 3692 * Requires ENTROPY_WORKSPACE_SIZE wksp. 3693 * @return : size of fse tables or error code */ 3694 static size_t 3695 ZSTD_buildBlockEntropyStats_sequences( 3696 const seqStore_t* seqStorePtr, 3697 const ZSTD_fseCTables_t* prevEntropy, 3698 ZSTD_fseCTables_t* nextEntropy, 3699 const ZSTD_CCtx_params* cctxParams, 3700 ZSTD_fseCTablesMetadata_t* fseMetadata, 3701 void* workspace, size_t wkspSize) 3702 { 3703 ZSTD_strategy const strategy = cctxParams->cParams.strategy; 3704 size_t const nbSeq = (size_t)(seqStorePtr->sequences - seqStorePtr->sequencesStart); 3705 BYTE* const ostart = fseMetadata->fseTablesBuffer; 3706 BYTE* const oend = ostart + sizeof(fseMetadata->fseTablesBuffer); 3707 BYTE* op = ostart; 3708 unsigned* countWorkspace = (unsigned*)workspace; 3709 unsigned* entropyWorkspace = countWorkspace + (MaxSeq + 1); 3710 size_t entropyWorkspaceSize = wkspSize - (MaxSeq + 1) * sizeof(*countWorkspace); 3711 ZSTD_symbolEncodingTypeStats_t stats; 3712 3713 DEBUGLOG(5, "ZSTD_buildBlockEntropyStats_sequences (nbSeq=%zu)", nbSeq); 3714 stats = nbSeq != 0 ? ZSTD_buildSequencesStatistics(seqStorePtr, nbSeq, 3715 prevEntropy, nextEntropy, op, oend, 3716 strategy, countWorkspace, 3717 entropyWorkspace, entropyWorkspaceSize) 3718 : ZSTD_buildDummySequencesStatistics(nextEntropy); 3719 FORWARD_IF_ERROR(stats.size, "ZSTD_buildSequencesStatistics failed!"); 3720 fseMetadata->llType = (symbolEncodingType_e) stats.LLtype; 3721 fseMetadata->ofType = (symbolEncodingType_e) stats.Offtype; 3722 fseMetadata->mlType = (symbolEncodingType_e) stats.MLtype; 3723 fseMetadata->lastCountSize = stats.lastCountSize; 3724 return stats.size; 3725 } 3726 3727 3728 /** ZSTD_buildBlockEntropyStats() : 3729 * Builds entropy for the block. 3730 * Requires workspace size ENTROPY_WORKSPACE_SIZE 3731 * @return : 0 on success, or an error code 3732 * Note : also employed in superblock 3733 */ 3734 size_t ZSTD_buildBlockEntropyStats( 3735 const seqStore_t* seqStorePtr, 3736 const ZSTD_entropyCTables_t* prevEntropy, 3737 ZSTD_entropyCTables_t* nextEntropy, 3738 const ZSTD_CCtx_params* cctxParams, 3739 ZSTD_entropyCTablesMetadata_t* entropyMetadata, 3740 void* workspace, size_t wkspSize) 3741 { 3742 size_t const litSize = (size_t)(seqStorePtr->lit - seqStorePtr->litStart); 3743 int const huf_useOptDepth = (cctxParams->cParams.strategy >= HUF_OPTIMAL_DEPTH_THRESHOLD); 3744 int const hufFlags = huf_useOptDepth ? HUF_flags_optimalDepth : 0; 3745 3746 entropyMetadata->hufMetadata.hufDesSize = 3747 ZSTD_buildBlockEntropyStats_literals(seqStorePtr->litStart, litSize, 3748 &prevEntropy->huf, &nextEntropy->huf, 3749 &entropyMetadata->hufMetadata, 3750 ZSTD_literalsCompressionIsDisabled(cctxParams), 3751 workspace, wkspSize, hufFlags); 3752 3753 FORWARD_IF_ERROR(entropyMetadata->hufMetadata.hufDesSize, "ZSTD_buildBlockEntropyStats_literals failed"); 3754 entropyMetadata->fseMetadata.fseTablesSize = 3755 ZSTD_buildBlockEntropyStats_sequences(seqStorePtr, 3756 &prevEntropy->fse, &nextEntropy->fse, 3757 cctxParams, 3758 &entropyMetadata->fseMetadata, 3759 workspace, wkspSize); 3760 FORWARD_IF_ERROR(entropyMetadata->fseMetadata.fseTablesSize, "ZSTD_buildBlockEntropyStats_sequences failed"); 3761 return 0; 3762 } 3763 3764 /* Returns the size estimate for the literals section (header + content) of a block */ 3765 static size_t 3766 ZSTD_estimateBlockSize_literal(const BYTE* literals, size_t litSize, 3767 const ZSTD_hufCTables_t* huf, 3768 const ZSTD_hufCTablesMetadata_t* hufMetadata, 3769 void* workspace, size_t wkspSize, 3770 int writeEntropy) 3771 { 3772 unsigned* const countWksp = (unsigned*)workspace; 3773 unsigned maxSymbolValue = HUF_SYMBOLVALUE_MAX; 3774 size_t literalSectionHeaderSize = 3 + (litSize >= 1 KB) + (litSize >= 16 KB); 3775 U32 singleStream = litSize < 256; 3776 3777 if (hufMetadata->hType == set_basic) return litSize; 3778 else if (hufMetadata->hType == set_rle) return 1; 3779 else if (hufMetadata->hType == set_compressed || hufMetadata->hType == set_repeat) { 3780 size_t const largest = HIST_count_wksp (countWksp, &maxSymbolValue, (const BYTE*)literals, litSize, workspace, wkspSize); 3781 if (ZSTD_isError(largest)) return litSize; 3782 { size_t cLitSizeEstimate = HUF_estimateCompressedSize((const HUF_CElt*)huf->CTable, countWksp, maxSymbolValue); 3783 if (writeEntropy) cLitSizeEstimate += hufMetadata->hufDesSize; 3784 if (!singleStream) cLitSizeEstimate += 6; /* multi-stream huffman uses 6-byte jump table */ 3785 return cLitSizeEstimate + literalSectionHeaderSize; 3786 } } 3787 assert(0); /* impossible */ 3788 return 0; 3789 } 3790 3791 /* Returns the size estimate for the FSE-compressed symbols (of, ml, ll) of a block */ 3792 static size_t 3793 ZSTD_estimateBlockSize_symbolType(symbolEncodingType_e type, 3794 const BYTE* codeTable, size_t nbSeq, unsigned maxCode, 3795 const FSE_CTable* fseCTable, 3796 const U8* additionalBits, 3797 short const* defaultNorm, U32 defaultNormLog, U32 defaultMax, 3798 void* workspace, size_t wkspSize) 3799 { 3800 unsigned* const countWksp = (unsigned*)workspace; 3801 const BYTE* ctp = codeTable; 3802 const BYTE* const ctStart = ctp; 3803 const BYTE* const ctEnd = ctStart + nbSeq; 3804 size_t cSymbolTypeSizeEstimateInBits = 0; 3805 unsigned max = maxCode; 3806 3807 HIST_countFast_wksp(countWksp, &max, codeTable, nbSeq, workspace, wkspSize); /* can't fail */ 3808 if (type == set_basic) { 3809 /* We selected this encoding type, so it must be valid. */ 3810 assert(max <= defaultMax); 3811 (void)defaultMax; 3812 cSymbolTypeSizeEstimateInBits = ZSTD_crossEntropyCost(defaultNorm, defaultNormLog, countWksp, max); 3813 } else if (type == set_rle) { 3814 cSymbolTypeSizeEstimateInBits = 0; 3815 } else if (type == set_compressed || type == set_repeat) { 3816 cSymbolTypeSizeEstimateInBits = ZSTD_fseBitCost(fseCTable, countWksp, max); 3817 } 3818 if (ZSTD_isError(cSymbolTypeSizeEstimateInBits)) { 3819 return nbSeq * 10; 3820 } 3821 while (ctp < ctEnd) { 3822 if (additionalBits) cSymbolTypeSizeEstimateInBits += additionalBits[*ctp]; 3823 else cSymbolTypeSizeEstimateInBits += *ctp; /* for offset, offset code is also the number of additional bits */ 3824 ctp++; 3825 } 3826 return cSymbolTypeSizeEstimateInBits >> 3; 3827 } 3828 3829 /* Returns the size estimate for the sequences section (header + content) of a block */ 3830 static size_t 3831 ZSTD_estimateBlockSize_sequences(const BYTE* ofCodeTable, 3832 const BYTE* llCodeTable, 3833 const BYTE* mlCodeTable, 3834 size_t nbSeq, 3835 const ZSTD_fseCTables_t* fseTables, 3836 const ZSTD_fseCTablesMetadata_t* fseMetadata, 3837 void* workspace, size_t wkspSize, 3838 int writeEntropy) 3839 { 3840 size_t sequencesSectionHeaderSize = 1 /* seqHead */ + 1 /* min seqSize size */ + (nbSeq >= 128) + (nbSeq >= LONGNBSEQ); 3841 size_t cSeqSizeEstimate = 0; 3842 cSeqSizeEstimate += ZSTD_estimateBlockSize_symbolType(fseMetadata->ofType, ofCodeTable, nbSeq, MaxOff, 3843 fseTables->offcodeCTable, NULL, 3844 OF_defaultNorm, OF_defaultNormLog, DefaultMaxOff, 3845 workspace, wkspSize); 3846 cSeqSizeEstimate += ZSTD_estimateBlockSize_symbolType(fseMetadata->llType, llCodeTable, nbSeq, MaxLL, 3847 fseTables->litlengthCTable, LL_bits, 3848 LL_defaultNorm, LL_defaultNormLog, MaxLL, 3849 workspace, wkspSize); 3850 cSeqSizeEstimate += ZSTD_estimateBlockSize_symbolType(fseMetadata->mlType, mlCodeTable, nbSeq, MaxML, 3851 fseTables->matchlengthCTable, ML_bits, 3852 ML_defaultNorm, ML_defaultNormLog, MaxML, 3853 workspace, wkspSize); 3854 if (writeEntropy) cSeqSizeEstimate += fseMetadata->fseTablesSize; 3855 return cSeqSizeEstimate + sequencesSectionHeaderSize; 3856 } 3857 3858 /* Returns the size estimate for a given stream of literals, of, ll, ml */ 3859 static size_t 3860 ZSTD_estimateBlockSize(const BYTE* literals, size_t litSize, 3861 const BYTE* ofCodeTable, 3862 const BYTE* llCodeTable, 3863 const BYTE* mlCodeTable, 3864 size_t nbSeq, 3865 const ZSTD_entropyCTables_t* entropy, 3866 const ZSTD_entropyCTablesMetadata_t* entropyMetadata, 3867 void* workspace, size_t wkspSize, 3868 int writeLitEntropy, int writeSeqEntropy) 3869 { 3870 size_t const literalsSize = ZSTD_estimateBlockSize_literal(literals, litSize, 3871 &entropy->huf, &entropyMetadata->hufMetadata, 3872 workspace, wkspSize, writeLitEntropy); 3873 size_t const seqSize = ZSTD_estimateBlockSize_sequences(ofCodeTable, llCodeTable, mlCodeTable, 3874 nbSeq, &entropy->fse, &entropyMetadata->fseMetadata, 3875 workspace, wkspSize, writeSeqEntropy); 3876 return seqSize + literalsSize + ZSTD_blockHeaderSize; 3877 } 3878 3879 /* Builds entropy statistics and uses them for blocksize estimation. 3880 * 3881 * @return: estimated compressed size of the seqStore, or a zstd error. 3882 */ 3883 static size_t 3884 ZSTD_buildEntropyStatisticsAndEstimateSubBlockSize(seqStore_t* seqStore, ZSTD_CCtx* zc) 3885 { 3886 ZSTD_entropyCTablesMetadata_t* const entropyMetadata = &zc->blockSplitCtx.entropyMetadata; 3887 DEBUGLOG(6, "ZSTD_buildEntropyStatisticsAndEstimateSubBlockSize()"); 3888 FORWARD_IF_ERROR(ZSTD_buildBlockEntropyStats(seqStore, 3889 &zc->blockState.prevCBlock->entropy, 3890 &zc->blockState.nextCBlock->entropy, 3891 &zc->appliedParams, 3892 entropyMetadata, 3893 zc->entropyWorkspace, ENTROPY_WORKSPACE_SIZE), ""); 3894 return ZSTD_estimateBlockSize( 3895 seqStore->litStart, (size_t)(seqStore->lit - seqStore->litStart), 3896 seqStore->ofCode, seqStore->llCode, seqStore->mlCode, 3897 (size_t)(seqStore->sequences - seqStore->sequencesStart), 3898 &zc->blockState.nextCBlock->entropy, 3899 entropyMetadata, 3900 zc->entropyWorkspace, ENTROPY_WORKSPACE_SIZE, 3901 (int)(entropyMetadata->hufMetadata.hType == set_compressed), 1); 3902 } 3903 3904 /* Returns literals bytes represented in a seqStore */ 3905 static size_t ZSTD_countSeqStoreLiteralsBytes(const seqStore_t* const seqStore) 3906 { 3907 size_t literalsBytes = 0; 3908 size_t const nbSeqs = (size_t)(seqStore->sequences - seqStore->sequencesStart); 3909 size_t i; 3910 for (i = 0; i < nbSeqs; ++i) { 3911 seqDef const seq = seqStore->sequencesStart[i]; 3912 literalsBytes += seq.litLength; 3913 if (i == seqStore->longLengthPos && seqStore->longLengthType == ZSTD_llt_literalLength) { 3914 literalsBytes += 0x10000; 3915 } } 3916 return literalsBytes; 3917 } 3918 3919 /* Returns match bytes represented in a seqStore */ 3920 static size_t ZSTD_countSeqStoreMatchBytes(const seqStore_t* const seqStore) 3921 { 3922 size_t matchBytes = 0; 3923 size_t const nbSeqs = (size_t)(seqStore->sequences - seqStore->sequencesStart); 3924 size_t i; 3925 for (i = 0; i < nbSeqs; ++i) { 3926 seqDef seq = seqStore->sequencesStart[i]; 3927 matchBytes += seq.mlBase + MINMATCH; 3928 if (i == seqStore->longLengthPos && seqStore->longLengthType == ZSTD_llt_matchLength) { 3929 matchBytes += 0x10000; 3930 } } 3931 return matchBytes; 3932 } 3933 3934 /* Derives the seqStore that is a chunk of the originalSeqStore from [startIdx, endIdx). 3935 * Stores the result in resultSeqStore. 3936 */ 3937 static void ZSTD_deriveSeqStoreChunk(seqStore_t* resultSeqStore, 3938 const seqStore_t* originalSeqStore, 3939 size_t startIdx, size_t endIdx) 3940 { 3941 *resultSeqStore = *originalSeqStore; 3942 if (startIdx > 0) { 3943 resultSeqStore->sequences = originalSeqStore->sequencesStart + startIdx; 3944 resultSeqStore->litStart += ZSTD_countSeqStoreLiteralsBytes(resultSeqStore); 3945 } 3946 3947 /* Move longLengthPos into the correct position if necessary */ 3948 if (originalSeqStore->longLengthType != ZSTD_llt_none) { 3949 if (originalSeqStore->longLengthPos < startIdx || originalSeqStore->longLengthPos > endIdx) { 3950 resultSeqStore->longLengthType = ZSTD_llt_none; 3951 } else { 3952 resultSeqStore->longLengthPos -= (U32)startIdx; 3953 } 3954 } 3955 resultSeqStore->sequencesStart = originalSeqStore->sequencesStart + startIdx; 3956 resultSeqStore->sequences = originalSeqStore->sequencesStart + endIdx; 3957 if (endIdx == (size_t)(originalSeqStore->sequences - originalSeqStore->sequencesStart)) { 3958 /* This accounts for possible last literals if the derived chunk reaches the end of the block */ 3959 assert(resultSeqStore->lit == originalSeqStore->lit); 3960 } else { 3961 size_t const literalsBytes = ZSTD_countSeqStoreLiteralsBytes(resultSeqStore); 3962 resultSeqStore->lit = resultSeqStore->litStart + literalsBytes; 3963 } 3964 resultSeqStore->llCode += startIdx; 3965 resultSeqStore->mlCode += startIdx; 3966 resultSeqStore->ofCode += startIdx; 3967 } 3968 3969 /** 3970 * Returns the raw offset represented by the combination of offBase, ll0, and repcode history. 3971 * offBase must represent a repcode in the numeric representation of ZSTD_storeSeq(). 3972 */ 3973 static U32 3974 ZSTD_resolveRepcodeToRawOffset(const U32 rep[ZSTD_REP_NUM], const U32 offBase, const U32 ll0) 3975 { 3976 U32 const adjustedRepCode = OFFBASE_TO_REPCODE(offBase) - 1 + ll0; /* [ 0 - 3 ] */ 3977 assert(OFFBASE_IS_REPCODE(offBase)); 3978 if (adjustedRepCode == ZSTD_REP_NUM) { 3979 assert(ll0); 3980 /* litlength == 0 and offCode == 2 implies selection of first repcode - 1 3981 * This is only valid if it results in a valid offset value, aka > 0. 3982 * Note : it may happen that `rep[0]==1` in exceptional circumstances. 3983 * In which case this function will return 0, which is an invalid offset. 3984 * It's not an issue though, since this value will be 3985 * compared and discarded within ZSTD_seqStore_resolveOffCodes(). 3986 */ 3987 return rep[0] - 1; 3988 } 3989 return rep[adjustedRepCode]; 3990 } 3991 3992 /** 3993 * ZSTD_seqStore_resolveOffCodes() reconciles any possible divergences in offset history that may arise 3994 * due to emission of RLE/raw blocks that disturb the offset history, 3995 * and replaces any repcodes within the seqStore that may be invalid. 3996 * 3997 * dRepcodes are updated as would be on the decompression side. 3998 * cRepcodes are updated exactly in accordance with the seqStore. 3999 * 4000 * Note : this function assumes seq->offBase respects the following numbering scheme : 4001 * 0 : invalid 4002 * 1-3 : repcode 1-3 4003 * 4+ : real_offset+3 4004 */ 4005 static void 4006 ZSTD_seqStore_resolveOffCodes(repcodes_t* const dRepcodes, repcodes_t* const cRepcodes, 4007 const seqStore_t* const seqStore, U32 const nbSeq) 4008 { 4009 U32 idx = 0; 4010 U32 const longLitLenIdx = seqStore->longLengthType == ZSTD_llt_literalLength ? seqStore->longLengthPos : nbSeq; 4011 for (; idx < nbSeq; ++idx) { 4012 seqDef* const seq = seqStore->sequencesStart + idx; 4013 U32 const ll0 = (seq->litLength == 0) && (idx != longLitLenIdx); 4014 U32 const offBase = seq->offBase; 4015 assert(offBase > 0); 4016 if (OFFBASE_IS_REPCODE(offBase)) { 4017 U32 const dRawOffset = ZSTD_resolveRepcodeToRawOffset(dRepcodes->rep, offBase, ll0); 4018 U32 const cRawOffset = ZSTD_resolveRepcodeToRawOffset(cRepcodes->rep, offBase, ll0); 4019 /* Adjust simulated decompression repcode history if we come across a mismatch. Replace 4020 * the repcode with the offset it actually references, determined by the compression 4021 * repcode history. 4022 */ 4023 if (dRawOffset != cRawOffset) { 4024 seq->offBase = OFFSET_TO_OFFBASE(cRawOffset); 4025 } 4026 } 4027 /* Compression repcode history is always updated with values directly from the unmodified seqStore. 4028 * Decompression repcode history may use modified seq->offset value taken from compression repcode history. 4029 */ 4030 ZSTD_updateRep(dRepcodes->rep, seq->offBase, ll0); 4031 ZSTD_updateRep(cRepcodes->rep, offBase, ll0); 4032 } 4033 } 4034 4035 /* ZSTD_compressSeqStore_singleBlock(): 4036 * Compresses a seqStore into a block with a block header, into the buffer dst. 4037 * 4038 * Returns the total size of that block (including header) or a ZSTD error code. 4039 */ 4040 static size_t 4041 ZSTD_compressSeqStore_singleBlock(ZSTD_CCtx* zc, 4042 const seqStore_t* const seqStore, 4043 repcodes_t* const dRep, repcodes_t* const cRep, 4044 void* dst, size_t dstCapacity, 4045 const void* src, size_t srcSize, 4046 U32 lastBlock, U32 isPartition) 4047 { 4048 const U32 rleMaxLength = 25; 4049 BYTE* op = (BYTE*)dst; 4050 const BYTE* ip = (const BYTE*)src; 4051 size_t cSize; 4052 size_t cSeqsSize; 4053 4054 /* In case of an RLE or raw block, the simulated decompression repcode history must be reset */ 4055 repcodes_t const dRepOriginal = *dRep; 4056 DEBUGLOG(5, "ZSTD_compressSeqStore_singleBlock"); 4057 if (isPartition) 4058 ZSTD_seqStore_resolveOffCodes(dRep, cRep, seqStore, (U32)(seqStore->sequences - seqStore->sequencesStart)); 4059 4060 RETURN_ERROR_IF(dstCapacity < ZSTD_blockHeaderSize, dstSize_tooSmall, "Block header doesn't fit"); 4061 cSeqsSize = ZSTD_entropyCompressSeqStore(seqStore, 4062 &zc->blockState.prevCBlock->entropy, &zc->blockState.nextCBlock->entropy, 4063 &zc->appliedParams, 4064 op + ZSTD_blockHeaderSize, dstCapacity - ZSTD_blockHeaderSize, 4065 srcSize, 4066 zc->entropyWorkspace, ENTROPY_WORKSPACE_SIZE /* statically allocated in resetCCtx */, 4067 zc->bmi2); 4068 FORWARD_IF_ERROR(cSeqsSize, "ZSTD_entropyCompressSeqStore failed!"); 4069 4070 if (!zc->isFirstBlock && 4071 cSeqsSize < rleMaxLength && 4072 ZSTD_isRLE((BYTE const*)src, srcSize)) { 4073 /* We don't want to emit our first block as a RLE even if it qualifies because 4074 * doing so will cause the decoder (cli only) to throw a "should consume all input error." 4075 * This is only an issue for zstd <= v1.4.3 4076 */ 4077 cSeqsSize = 1; 4078 } 4079 4080 /* Sequence collection not supported when block splitting */ 4081 if (zc->seqCollector.collectSequences) { 4082 FORWARD_IF_ERROR(ZSTD_copyBlockSequences(&zc->seqCollector, seqStore, dRepOriginal.rep), "copyBlockSequences failed"); 4083 ZSTD_blockState_confirmRepcodesAndEntropyTables(&zc->blockState); 4084 return 0; 4085 } 4086 4087 if (cSeqsSize == 0) { 4088 cSize = ZSTD_noCompressBlock(op, dstCapacity, ip, srcSize, lastBlock); 4089 FORWARD_IF_ERROR(cSize, "Nocompress block failed"); 4090 DEBUGLOG(4, "Writing out nocompress block, size: %zu", cSize); 4091 *dRep = dRepOriginal; /* reset simulated decompression repcode history */ 4092 } else if (cSeqsSize == 1) { 4093 cSize = ZSTD_rleCompressBlock(op, dstCapacity, *ip, srcSize, lastBlock); 4094 FORWARD_IF_ERROR(cSize, "RLE compress block failed"); 4095 DEBUGLOG(4, "Writing out RLE block, size: %zu", cSize); 4096 *dRep = dRepOriginal; /* reset simulated decompression repcode history */ 4097 } else { 4098 ZSTD_blockState_confirmRepcodesAndEntropyTables(&zc->blockState); 4099 writeBlockHeader(op, cSeqsSize, srcSize, lastBlock); 4100 cSize = ZSTD_blockHeaderSize + cSeqsSize; 4101 DEBUGLOG(4, "Writing out compressed block, size: %zu", cSize); 4102 } 4103 4104 if (zc->blockState.prevCBlock->entropy.fse.offcode_repeatMode == FSE_repeat_valid) 4105 zc->blockState.prevCBlock->entropy.fse.offcode_repeatMode = FSE_repeat_check; 4106 4107 return cSize; 4108 } 4109 4110 /* Struct to keep track of where we are in our recursive calls. */ 4111 typedef struct { 4112 U32* splitLocations; /* Array of split indices */ 4113 size_t idx; /* The current index within splitLocations being worked on */ 4114 } seqStoreSplits; 4115 4116 #define MIN_SEQUENCES_BLOCK_SPLITTING 300 4117 4118 /* Helper function to perform the recursive search for block splits. 4119 * Estimates the cost of seqStore prior to split, and estimates the cost of splitting the sequences in half. 4120 * If advantageous to split, then we recurse down the two sub-blocks. 4121 * If not, or if an error occurred in estimation, then we do not recurse. 4122 * 4123 * Note: The recursion depth is capped by a heuristic minimum number of sequences, 4124 * defined by MIN_SEQUENCES_BLOCK_SPLITTING. 4125 * In theory, this means the absolute largest recursion depth is 10 == log2(maxNbSeqInBlock/MIN_SEQUENCES_BLOCK_SPLITTING). 4126 * In practice, recursion depth usually doesn't go beyond 4. 4127 * 4128 * Furthermore, the number of splits is capped by ZSTD_MAX_NB_BLOCK_SPLITS. 4129 * At ZSTD_MAX_NB_BLOCK_SPLITS == 196 with the current existing blockSize 4130 * maximum of 128 KB, this value is actually impossible to reach. 4131 */ 4132 static void 4133 ZSTD_deriveBlockSplitsHelper(seqStoreSplits* splits, size_t startIdx, size_t endIdx, 4134 ZSTD_CCtx* zc, const seqStore_t* origSeqStore) 4135 { 4136 seqStore_t* const fullSeqStoreChunk = &zc->blockSplitCtx.fullSeqStoreChunk; 4137 seqStore_t* const firstHalfSeqStore = &zc->blockSplitCtx.firstHalfSeqStore; 4138 seqStore_t* const secondHalfSeqStore = &zc->blockSplitCtx.secondHalfSeqStore; 4139 size_t estimatedOriginalSize; 4140 size_t estimatedFirstHalfSize; 4141 size_t estimatedSecondHalfSize; 4142 size_t midIdx = (startIdx + endIdx)/2; 4143 4144 DEBUGLOG(5, "ZSTD_deriveBlockSplitsHelper: startIdx=%zu endIdx=%zu", startIdx, endIdx); 4145 assert(endIdx >= startIdx); 4146 if (endIdx - startIdx < MIN_SEQUENCES_BLOCK_SPLITTING || splits->idx >= ZSTD_MAX_NB_BLOCK_SPLITS) { 4147 DEBUGLOG(6, "ZSTD_deriveBlockSplitsHelper: Too few sequences (%zu)", endIdx - startIdx); 4148 return; 4149 } 4150 ZSTD_deriveSeqStoreChunk(fullSeqStoreChunk, origSeqStore, startIdx, endIdx); 4151 ZSTD_deriveSeqStoreChunk(firstHalfSeqStore, origSeqStore, startIdx, midIdx); 4152 ZSTD_deriveSeqStoreChunk(secondHalfSeqStore, origSeqStore, midIdx, endIdx); 4153 estimatedOriginalSize = ZSTD_buildEntropyStatisticsAndEstimateSubBlockSize(fullSeqStoreChunk, zc); 4154 estimatedFirstHalfSize = ZSTD_buildEntropyStatisticsAndEstimateSubBlockSize(firstHalfSeqStore, zc); 4155 estimatedSecondHalfSize = ZSTD_buildEntropyStatisticsAndEstimateSubBlockSize(secondHalfSeqStore, zc); 4156 DEBUGLOG(5, "Estimated original block size: %zu -- First half split: %zu -- Second half split: %zu", 4157 estimatedOriginalSize, estimatedFirstHalfSize, estimatedSecondHalfSize); 4158 if (ZSTD_isError(estimatedOriginalSize) || ZSTD_isError(estimatedFirstHalfSize) || ZSTD_isError(estimatedSecondHalfSize)) { 4159 return; 4160 } 4161 if (estimatedFirstHalfSize + estimatedSecondHalfSize < estimatedOriginalSize) { 4162 DEBUGLOG(5, "split decided at seqNb:%zu", midIdx); 4163 ZSTD_deriveBlockSplitsHelper(splits, startIdx, midIdx, zc, origSeqStore); 4164 splits->splitLocations[splits->idx] = (U32)midIdx; 4165 splits->idx++; 4166 ZSTD_deriveBlockSplitsHelper(splits, midIdx, endIdx, zc, origSeqStore); 4167 } 4168 } 4169 4170 /* Base recursive function. 4171 * Populates a table with intra-block partition indices that can improve compression ratio. 4172 * 4173 * @return: number of splits made (which equals the size of the partition table - 1). 4174 */ 4175 static size_t ZSTD_deriveBlockSplits(ZSTD_CCtx* zc, U32 partitions[], U32 nbSeq) 4176 { 4177 seqStoreSplits splits; 4178 splits.splitLocations = partitions; 4179 splits.idx = 0; 4180 if (nbSeq <= 4) { 4181 DEBUGLOG(5, "ZSTD_deriveBlockSplits: Too few sequences to split (%u <= 4)", nbSeq); 4182 /* Refuse to try and split anything with less than 4 sequences */ 4183 return 0; 4184 } 4185 ZSTD_deriveBlockSplitsHelper(&splits, 0, nbSeq, zc, &zc->seqStore); 4186 splits.splitLocations[splits.idx] = nbSeq; 4187 DEBUGLOG(5, "ZSTD_deriveBlockSplits: final nb partitions: %zu", splits.idx+1); 4188 return splits.idx; 4189 } 4190 4191 /* ZSTD_compressBlock_splitBlock(): 4192 * Attempts to split a given block into multiple blocks to improve compression ratio. 4193 * 4194 * Returns combined size of all blocks (which includes headers), or a ZSTD error code. 4195 */ 4196 static size_t 4197 ZSTD_compressBlock_splitBlock_internal(ZSTD_CCtx* zc, 4198 void* dst, size_t dstCapacity, 4199 const void* src, size_t blockSize, 4200 U32 lastBlock, U32 nbSeq) 4201 { 4202 size_t cSize = 0; 4203 const BYTE* ip = (const BYTE*)src; 4204 BYTE* op = (BYTE*)dst; 4205 size_t i = 0; 4206 size_t srcBytesTotal = 0; 4207 U32* const partitions = zc->blockSplitCtx.partitions; /* size == ZSTD_MAX_NB_BLOCK_SPLITS */ 4208 seqStore_t* const nextSeqStore = &zc->blockSplitCtx.nextSeqStore; 4209 seqStore_t* const currSeqStore = &zc->blockSplitCtx.currSeqStore; 4210 size_t const numSplits = ZSTD_deriveBlockSplits(zc, partitions, nbSeq); 4211 4212 /* If a block is split and some partitions are emitted as RLE/uncompressed, then repcode history 4213 * may become invalid. In order to reconcile potentially invalid repcodes, we keep track of two 4214 * separate repcode histories that simulate repcode history on compression and decompression side, 4215 * and use the histories to determine whether we must replace a particular repcode with its raw offset. 4216 * 4217 * 1) cRep gets updated for each partition, regardless of whether the block was emitted as uncompressed 4218 * or RLE. This allows us to retrieve the offset value that an invalid repcode references within 4219 * a nocompress/RLE block. 4220 * 2) dRep gets updated only for compressed partitions, and when a repcode gets replaced, will use 4221 * the replacement offset value rather than the original repcode to update the repcode history. 4222 * dRep also will be the final repcode history sent to the next block. 4223 * 4224 * See ZSTD_seqStore_resolveOffCodes() for more details. 4225 */ 4226 repcodes_t dRep; 4227 repcodes_t cRep; 4228 ZSTD_memcpy(dRep.rep, zc->blockState.prevCBlock->rep, sizeof(repcodes_t)); 4229 ZSTD_memcpy(cRep.rep, zc->blockState.prevCBlock->rep, sizeof(repcodes_t)); 4230 ZSTD_memset(nextSeqStore, 0, sizeof(seqStore_t)); 4231 4232 DEBUGLOG(5, "ZSTD_compressBlock_splitBlock_internal (dstCapacity=%u, dictLimit=%u, nextToUpdate=%u)", 4233 (unsigned)dstCapacity, (unsigned)zc->blockState.matchState.window.dictLimit, 4234 (unsigned)zc->blockState.matchState.nextToUpdate); 4235 4236 if (numSplits == 0) { 4237 size_t cSizeSingleBlock = 4238 ZSTD_compressSeqStore_singleBlock(zc, &zc->seqStore, 4239 &dRep, &cRep, 4240 op, dstCapacity, 4241 ip, blockSize, 4242 lastBlock, 0 /* isPartition */); 4243 FORWARD_IF_ERROR(cSizeSingleBlock, "Compressing single block from splitBlock_internal() failed!"); 4244 DEBUGLOG(5, "ZSTD_compressBlock_splitBlock_internal: No splits"); 4245 assert(zc->blockSize <= ZSTD_BLOCKSIZE_MAX); 4246 assert(cSizeSingleBlock <= zc->blockSize + ZSTD_blockHeaderSize); 4247 return cSizeSingleBlock; 4248 } 4249 4250 ZSTD_deriveSeqStoreChunk(currSeqStore, &zc->seqStore, 0, partitions[0]); 4251 for (i = 0; i <= numSplits; ++i) { 4252 size_t cSizeChunk; 4253 U32 const lastPartition = (i == numSplits); 4254 U32 lastBlockEntireSrc = 0; 4255 4256 size_t srcBytes = ZSTD_countSeqStoreLiteralsBytes(currSeqStore) + ZSTD_countSeqStoreMatchBytes(currSeqStore); 4257 srcBytesTotal += srcBytes; 4258 if (lastPartition) { 4259 /* This is the final partition, need to account for possible last literals */ 4260 srcBytes += blockSize - srcBytesTotal; 4261 lastBlockEntireSrc = lastBlock; 4262 } else { 4263 ZSTD_deriveSeqStoreChunk(nextSeqStore, &zc->seqStore, partitions[i], partitions[i+1]); 4264 } 4265 4266 cSizeChunk = ZSTD_compressSeqStore_singleBlock(zc, currSeqStore, 4267 &dRep, &cRep, 4268 op, dstCapacity, 4269 ip, srcBytes, 4270 lastBlockEntireSrc, 1 /* isPartition */); 4271 DEBUGLOG(5, "Estimated size: %zu vs %zu : actual size", 4272 ZSTD_buildEntropyStatisticsAndEstimateSubBlockSize(currSeqStore, zc), cSizeChunk); 4273 FORWARD_IF_ERROR(cSizeChunk, "Compressing chunk failed!"); 4274 4275 ip += srcBytes; 4276 op += cSizeChunk; 4277 dstCapacity -= cSizeChunk; 4278 cSize += cSizeChunk; 4279 *currSeqStore = *nextSeqStore; 4280 assert(cSizeChunk <= zc->blockSize + ZSTD_blockHeaderSize); 4281 } 4282 /* cRep and dRep may have diverged during the compression. 4283 * If so, we use the dRep repcodes for the next block. 4284 */ 4285 ZSTD_memcpy(zc->blockState.prevCBlock->rep, dRep.rep, sizeof(repcodes_t)); 4286 return cSize; 4287 } 4288 4289 static size_t 4290 ZSTD_compressBlock_splitBlock(ZSTD_CCtx* zc, 4291 void* dst, size_t dstCapacity, 4292 const void* src, size_t srcSize, U32 lastBlock) 4293 { 4294 U32 nbSeq; 4295 size_t cSize; 4296 DEBUGLOG(4, "ZSTD_compressBlock_splitBlock"); 4297 assert(zc->appliedParams.useBlockSplitter == ZSTD_ps_enable); 4298 4299 { const size_t bss = ZSTD_buildSeqStore(zc, src, srcSize); 4300 FORWARD_IF_ERROR(bss, "ZSTD_buildSeqStore failed"); 4301 if (bss == ZSTDbss_noCompress) { 4302 if (zc->blockState.prevCBlock->entropy.fse.offcode_repeatMode == FSE_repeat_valid) 4303 zc->blockState.prevCBlock->entropy.fse.offcode_repeatMode = FSE_repeat_check; 4304 RETURN_ERROR_IF(zc->seqCollector.collectSequences, sequenceProducer_failed, "Uncompressible block"); 4305 cSize = ZSTD_noCompressBlock(dst, dstCapacity, src, srcSize, lastBlock); 4306 FORWARD_IF_ERROR(cSize, "ZSTD_noCompressBlock failed"); 4307 DEBUGLOG(4, "ZSTD_compressBlock_splitBlock: Nocompress block"); 4308 return cSize; 4309 } 4310 nbSeq = (U32)(zc->seqStore.sequences - zc->seqStore.sequencesStart); 4311 } 4312 4313 cSize = ZSTD_compressBlock_splitBlock_internal(zc, dst, dstCapacity, src, srcSize, lastBlock, nbSeq); 4314 FORWARD_IF_ERROR(cSize, "Splitting blocks failed!"); 4315 return cSize; 4316 } 4317 4318 static size_t 4319 ZSTD_compressBlock_internal(ZSTD_CCtx* zc, 4320 void* dst, size_t dstCapacity, 4321 const void* src, size_t srcSize, U32 frame) 4322 { 4323 /* This is an estimated upper bound for the length of an rle block. 4324 * This isn't the actual upper bound. 4325 * Finding the real threshold needs further investigation. 4326 */ 4327 const U32 rleMaxLength = 25; 4328 size_t cSize; 4329 const BYTE* ip = (const BYTE*)src; 4330 BYTE* op = (BYTE*)dst; 4331 DEBUGLOG(5, "ZSTD_compressBlock_internal (dstCapacity=%u, dictLimit=%u, nextToUpdate=%u)", 4332 (unsigned)dstCapacity, (unsigned)zc->blockState.matchState.window.dictLimit, 4333 (unsigned)zc->blockState.matchState.nextToUpdate); 4334 4335 { const size_t bss = ZSTD_buildSeqStore(zc, src, srcSize); 4336 FORWARD_IF_ERROR(bss, "ZSTD_buildSeqStore failed"); 4337 if (bss == ZSTDbss_noCompress) { 4338 RETURN_ERROR_IF(zc->seqCollector.collectSequences, sequenceProducer_failed, "Uncompressible block"); 4339 cSize = 0; 4340 goto out; 4341 } 4342 } 4343 4344 if (zc->seqCollector.collectSequences) { 4345 FORWARD_IF_ERROR(ZSTD_copyBlockSequences(&zc->seqCollector, ZSTD_getSeqStore(zc), zc->blockState.prevCBlock->rep), "copyBlockSequences failed"); 4346 ZSTD_blockState_confirmRepcodesAndEntropyTables(&zc->blockState); 4347 return 0; 4348 } 4349 4350 /* encode sequences and literals */ 4351 cSize = ZSTD_entropyCompressSeqStore(&zc->seqStore, 4352 &zc->blockState.prevCBlock->entropy, &zc->blockState.nextCBlock->entropy, 4353 &zc->appliedParams, 4354 dst, dstCapacity, 4355 srcSize, 4356 zc->entropyWorkspace, ENTROPY_WORKSPACE_SIZE /* statically allocated in resetCCtx */, 4357 zc->bmi2); 4358 4359 if (frame && 4360 /* We don't want to emit our first block as a RLE even if it qualifies because 4361 * doing so will cause the decoder (cli only) to throw a "should consume all input error." 4362 * This is only an issue for zstd <= v1.4.3 4363 */ 4364 !zc->isFirstBlock && 4365 cSize < rleMaxLength && 4366 ZSTD_isRLE(ip, srcSize)) 4367 { 4368 cSize = 1; 4369 op[0] = ip[0]; 4370 } 4371 4372 out: 4373 if (!ZSTD_isError(cSize) && cSize > 1) { 4374 ZSTD_blockState_confirmRepcodesAndEntropyTables(&zc->blockState); 4375 } 4376 /* We check that dictionaries have offset codes available for the first 4377 * block. After the first block, the offcode table might not have large 4378 * enough codes to represent the offsets in the data. 4379 */ 4380 if (zc->blockState.prevCBlock->entropy.fse.offcode_repeatMode == FSE_repeat_valid) 4381 zc->blockState.prevCBlock->entropy.fse.offcode_repeatMode = FSE_repeat_check; 4382 4383 return cSize; 4384 } 4385 4386 static size_t ZSTD_compressBlock_targetCBlockSize_body(ZSTD_CCtx* zc, 4387 void* dst, size_t dstCapacity, 4388 const void* src, size_t srcSize, 4389 const size_t bss, U32 lastBlock) 4390 { 4391 DEBUGLOG(6, "Attempting ZSTD_compressSuperBlock()"); 4392 if (bss == ZSTDbss_compress) { 4393 if (/* We don't want to emit our first block as a RLE even if it qualifies because 4394 * doing so will cause the decoder (cli only) to throw a "should consume all input error." 4395 * This is only an issue for zstd <= v1.4.3 4396 */ 4397 !zc->isFirstBlock && 4398 ZSTD_maybeRLE(&zc->seqStore) && 4399 ZSTD_isRLE((BYTE const*)src, srcSize)) 4400 { 4401 return ZSTD_rleCompressBlock(dst, dstCapacity, *(BYTE const*)src, srcSize, lastBlock); 4402 } 4403 /* Attempt superblock compression. 4404 * 4405 * Note that compressed size of ZSTD_compressSuperBlock() is not bound by the 4406 * standard ZSTD_compressBound(). This is a problem, because even if we have 4407 * space now, taking an extra byte now could cause us to run out of space later 4408 * and violate ZSTD_compressBound(). 4409 * 4410 * Define blockBound(blockSize) = blockSize + ZSTD_blockHeaderSize. 4411 * 4412 * In order to respect ZSTD_compressBound() we must attempt to emit a raw 4413 * uncompressed block in these cases: 4414 * * cSize == 0: Return code for an uncompressed block. 4415 * * cSize == dstSize_tooSmall: We may have expanded beyond blockBound(srcSize). 4416 * ZSTD_noCompressBlock() will return dstSize_tooSmall if we are really out of 4417 * output space. 4418 * * cSize >= blockBound(srcSize): We have expanded the block too much so 4419 * emit an uncompressed block. 4420 */ 4421 { size_t const cSize = 4422 ZSTD_compressSuperBlock(zc, dst, dstCapacity, src, srcSize, lastBlock); 4423 if (cSize != ERROR(dstSize_tooSmall)) { 4424 size_t const maxCSize = 4425 srcSize - ZSTD_minGain(srcSize, zc->appliedParams.cParams.strategy); 4426 FORWARD_IF_ERROR(cSize, "ZSTD_compressSuperBlock failed"); 4427 if (cSize != 0 && cSize < maxCSize + ZSTD_blockHeaderSize) { 4428 ZSTD_blockState_confirmRepcodesAndEntropyTables(&zc->blockState); 4429 return cSize; 4430 } 4431 } 4432 } 4433 } /* if (bss == ZSTDbss_compress)*/ 4434 4435 DEBUGLOG(6, "Resorting to ZSTD_noCompressBlock()"); 4436 /* Superblock compression failed, attempt to emit a single no compress block. 4437 * The decoder will be able to stream this block since it is uncompressed. 4438 */ 4439 return ZSTD_noCompressBlock(dst, dstCapacity, src, srcSize, lastBlock); 4440 } 4441 4442 static size_t ZSTD_compressBlock_targetCBlockSize(ZSTD_CCtx* zc, 4443 void* dst, size_t dstCapacity, 4444 const void* src, size_t srcSize, 4445 U32 lastBlock) 4446 { 4447 size_t cSize = 0; 4448 const size_t bss = ZSTD_buildSeqStore(zc, src, srcSize); 4449 DEBUGLOG(5, "ZSTD_compressBlock_targetCBlockSize (dstCapacity=%u, dictLimit=%u, nextToUpdate=%u, srcSize=%zu)", 4450 (unsigned)dstCapacity, (unsigned)zc->blockState.matchState.window.dictLimit, (unsigned)zc->blockState.matchState.nextToUpdate, srcSize); 4451 FORWARD_IF_ERROR(bss, "ZSTD_buildSeqStore failed"); 4452 4453 cSize = ZSTD_compressBlock_targetCBlockSize_body(zc, dst, dstCapacity, src, srcSize, bss, lastBlock); 4454 FORWARD_IF_ERROR(cSize, "ZSTD_compressBlock_targetCBlockSize_body failed"); 4455 4456 if (zc->blockState.prevCBlock->entropy.fse.offcode_repeatMode == FSE_repeat_valid) 4457 zc->blockState.prevCBlock->entropy.fse.offcode_repeatMode = FSE_repeat_check; 4458 4459 return cSize; 4460 } 4461 4462 static void ZSTD_overflowCorrectIfNeeded(ZSTD_matchState_t* ms, 4463 ZSTD_cwksp* ws, 4464 ZSTD_CCtx_params const* params, 4465 void const* ip, 4466 void const* iend) 4467 { 4468 U32 const cycleLog = ZSTD_cycleLog(params->cParams.chainLog, params->cParams.strategy); 4469 U32 const maxDist = (U32)1 << params->cParams.windowLog; 4470 if (ZSTD_window_needOverflowCorrection(ms->window, cycleLog, maxDist, ms->loadedDictEnd, ip, iend)) { 4471 U32 const correction = ZSTD_window_correctOverflow(&ms->window, cycleLog, maxDist, ip); 4472 ZSTD_STATIC_ASSERT(ZSTD_CHAINLOG_MAX <= 30); 4473 ZSTD_STATIC_ASSERT(ZSTD_WINDOWLOG_MAX_32 <= 30); 4474 ZSTD_STATIC_ASSERT(ZSTD_WINDOWLOG_MAX <= 31); 4475 ZSTD_cwksp_mark_tables_dirty(ws); 4476 ZSTD_reduceIndex(ms, params, correction); 4477 ZSTD_cwksp_mark_tables_clean(ws); 4478 if (ms->nextToUpdate < correction) ms->nextToUpdate = 0; 4479 else ms->nextToUpdate -= correction; 4480 /* invalidate dictionaries on overflow correction */ 4481 ms->loadedDictEnd = 0; 4482 ms->dictMatchState = NULL; 4483 } 4484 } 4485 4486 /*! ZSTD_compress_frameChunk() : 4487 * Compress a chunk of data into one or multiple blocks. 4488 * All blocks will be terminated, all input will be consumed. 4489 * Function will issue an error if there is not enough `dstCapacity` to hold the compressed content. 4490 * Frame is supposed already started (header already produced) 4491 * @return : compressed size, or an error code 4492 */ 4493 static size_t ZSTD_compress_frameChunk(ZSTD_CCtx* cctx, 4494 void* dst, size_t dstCapacity, 4495 const void* src, size_t srcSize, 4496 U32 lastFrameChunk) 4497 { 4498 size_t blockSize = cctx->blockSize; 4499 size_t remaining = srcSize; 4500 const BYTE* ip = (const BYTE*)src; 4501 BYTE* const ostart = (BYTE*)dst; 4502 BYTE* op = ostart; 4503 U32 const maxDist = (U32)1 << cctx->appliedParams.cParams.windowLog; 4504 4505 assert(cctx->appliedParams.cParams.windowLog <= ZSTD_WINDOWLOG_MAX); 4506 4507 DEBUGLOG(4, "ZSTD_compress_frameChunk (blockSize=%u)", (unsigned)blockSize); 4508 if (cctx->appliedParams.fParams.checksumFlag && srcSize) 4509 XXH64_update(&cctx->xxhState, src, srcSize); 4510 4511 while (remaining) { 4512 ZSTD_matchState_t* const ms = &cctx->blockState.matchState; 4513 U32 const lastBlock = lastFrameChunk & (blockSize >= remaining); 4514 4515 /* TODO: See 3090. We reduced MIN_CBLOCK_SIZE from 3 to 2 so to compensate we are adding 4516 * additional 1. We need to revisit and change this logic to be more consistent */ 4517 RETURN_ERROR_IF(dstCapacity < ZSTD_blockHeaderSize + MIN_CBLOCK_SIZE + 1, 4518 dstSize_tooSmall, 4519 "not enough space to store compressed block"); 4520 if (remaining < blockSize) blockSize = remaining; 4521 4522 ZSTD_overflowCorrectIfNeeded( 4523 ms, &cctx->workspace, &cctx->appliedParams, ip, ip + blockSize); 4524 ZSTD_checkDictValidity(&ms->window, ip + blockSize, maxDist, &ms->loadedDictEnd, &ms->dictMatchState); 4525 ZSTD_window_enforceMaxDist(&ms->window, ip, maxDist, &ms->loadedDictEnd, &ms->dictMatchState); 4526 4527 /* Ensure hash/chain table insertion resumes no sooner than lowlimit */ 4528 if (ms->nextToUpdate < ms->window.lowLimit) ms->nextToUpdate = ms->window.lowLimit; 4529 4530 { size_t cSize; 4531 if (ZSTD_useTargetCBlockSize(&cctx->appliedParams)) { 4532 cSize = ZSTD_compressBlock_targetCBlockSize(cctx, op, dstCapacity, ip, blockSize, lastBlock); 4533 FORWARD_IF_ERROR(cSize, "ZSTD_compressBlock_targetCBlockSize failed"); 4534 assert(cSize > 0); 4535 assert(cSize <= blockSize + ZSTD_blockHeaderSize); 4536 } else if (ZSTD_blockSplitterEnabled(&cctx->appliedParams)) { 4537 cSize = ZSTD_compressBlock_splitBlock(cctx, op, dstCapacity, ip, blockSize, lastBlock); 4538 FORWARD_IF_ERROR(cSize, "ZSTD_compressBlock_splitBlock failed"); 4539 assert(cSize > 0 || cctx->seqCollector.collectSequences == 1); 4540 } else { 4541 cSize = ZSTD_compressBlock_internal(cctx, 4542 op+ZSTD_blockHeaderSize, dstCapacity-ZSTD_blockHeaderSize, 4543 ip, blockSize, 1 /* frame */); 4544 FORWARD_IF_ERROR(cSize, "ZSTD_compressBlock_internal failed"); 4545 4546 if (cSize == 0) { /* block is not compressible */ 4547 cSize = ZSTD_noCompressBlock(op, dstCapacity, ip, blockSize, lastBlock); 4548 FORWARD_IF_ERROR(cSize, "ZSTD_noCompressBlock failed"); 4549 } else { 4550 U32 const cBlockHeader = cSize == 1 ? 4551 lastBlock + (((U32)bt_rle)<<1) + (U32)(blockSize << 3) : 4552 lastBlock + (((U32)bt_compressed)<<1) + (U32)(cSize << 3); 4553 MEM_writeLE24(op, cBlockHeader); 4554 cSize += ZSTD_blockHeaderSize; 4555 } 4556 } /* if (ZSTD_useTargetCBlockSize(&cctx->appliedParams))*/ 4557 4558 4559 ip += blockSize; 4560 assert(remaining >= blockSize); 4561 remaining -= blockSize; 4562 op += cSize; 4563 assert(dstCapacity >= cSize); 4564 dstCapacity -= cSize; 4565 cctx->isFirstBlock = 0; 4566 DEBUGLOG(5, "ZSTD_compress_frameChunk: adding a block of size %u", 4567 (unsigned)cSize); 4568 } } 4569 4570 if (lastFrameChunk && (op>ostart)) cctx->stage = ZSTDcs_ending; 4571 return (size_t)(op-ostart); 4572 } 4573 4574 4575 static size_t ZSTD_writeFrameHeader(void* dst, size_t dstCapacity, 4576 const ZSTD_CCtx_params* params, U64 pledgedSrcSize, U32 dictID) 4577 { BYTE* const op = (BYTE*)dst; 4578 U32 const dictIDSizeCodeLength = (dictID>0) + (dictID>=256) + (dictID>=65536); /* 0-3 */ 4579 U32 const dictIDSizeCode = params->fParams.noDictIDFlag ? 0 : dictIDSizeCodeLength; /* 0-3 */ 4580 U32 const checksumFlag = params->fParams.checksumFlag>0; 4581 U32 const windowSize = (U32)1 << params->cParams.windowLog; 4582 U32 const singleSegment = params->fParams.contentSizeFlag && (windowSize >= pledgedSrcSize); 4583 BYTE const windowLogByte = (BYTE)((params->cParams.windowLog - ZSTD_WINDOWLOG_ABSOLUTEMIN) << 3); 4584 U32 const fcsCode = params->fParams.contentSizeFlag ? 4585 (pledgedSrcSize>=256) + (pledgedSrcSize>=65536+256) + (pledgedSrcSize>=0xFFFFFFFFU) : 0; /* 0-3 */ 4586 BYTE const frameHeaderDescriptionByte = (BYTE)(dictIDSizeCode + (checksumFlag<<2) + (singleSegment<<5) + (fcsCode<<6) ); 4587 size_t pos=0; 4588 4589 assert(!(params->fParams.contentSizeFlag && pledgedSrcSize == ZSTD_CONTENTSIZE_UNKNOWN)); 4590 RETURN_ERROR_IF(dstCapacity < ZSTD_FRAMEHEADERSIZE_MAX, dstSize_tooSmall, 4591 "dst buf is too small to fit worst-case frame header size."); 4592 DEBUGLOG(4, "ZSTD_writeFrameHeader : dictIDFlag : %u ; dictID : %u ; dictIDSizeCode : %u", 4593 !params->fParams.noDictIDFlag, (unsigned)dictID, (unsigned)dictIDSizeCode); 4594 if (params->format == ZSTD_f_zstd1) { 4595 MEM_writeLE32(dst, ZSTD_MAGICNUMBER); 4596 pos = 4; 4597 } 4598 op[pos++] = frameHeaderDescriptionByte; 4599 if (!singleSegment) op[pos++] = windowLogByte; 4600 switch(dictIDSizeCode) 4601 { 4602 default: 4603 assert(0); /* impossible */ 4604 ZSTD_FALLTHROUGH; 4605 case 0 : break; 4606 case 1 : op[pos] = (BYTE)(dictID); pos++; break; 4607 case 2 : MEM_writeLE16(op+pos, (U16)dictID); pos+=2; break; 4608 case 3 : MEM_writeLE32(op+pos, dictID); pos+=4; break; 4609 } 4610 switch(fcsCode) 4611 { 4612 default: 4613 assert(0); /* impossible */ 4614 ZSTD_FALLTHROUGH; 4615 case 0 : if (singleSegment) op[pos++] = (BYTE)(pledgedSrcSize); break; 4616 case 1 : MEM_writeLE16(op+pos, (U16)(pledgedSrcSize-256)); pos+=2; break; 4617 case 2 : MEM_writeLE32(op+pos, (U32)(pledgedSrcSize)); pos+=4; break; 4618 case 3 : MEM_writeLE64(op+pos, (U64)(pledgedSrcSize)); pos+=8; break; 4619 } 4620 return pos; 4621 } 4622 4623 /* ZSTD_writeSkippableFrame_advanced() : 4624 * Writes out a skippable frame with the specified magic number variant (16 are supported), 4625 * from ZSTD_MAGIC_SKIPPABLE_START to ZSTD_MAGIC_SKIPPABLE_START+15, and the desired source data. 4626 * 4627 * Returns the total number of bytes written, or a ZSTD error code. 4628 */ 4629 size_t ZSTD_writeSkippableFrame(void* dst, size_t dstCapacity, 4630 const void* src, size_t srcSize, unsigned magicVariant) { 4631 BYTE* op = (BYTE*)dst; 4632 RETURN_ERROR_IF(dstCapacity < srcSize + ZSTD_SKIPPABLEHEADERSIZE /* Skippable frame overhead */, 4633 dstSize_tooSmall, "Not enough room for skippable frame"); 4634 RETURN_ERROR_IF(srcSize > (unsigned)0xFFFFFFFF, srcSize_wrong, "Src size too large for skippable frame"); 4635 RETURN_ERROR_IF(magicVariant > 15, parameter_outOfBound, "Skippable frame magic number variant not supported"); 4636 4637 MEM_writeLE32(op, (U32)(ZSTD_MAGIC_SKIPPABLE_START + magicVariant)); 4638 MEM_writeLE32(op+4, (U32)srcSize); 4639 ZSTD_memcpy(op+8, src, srcSize); 4640 return srcSize + ZSTD_SKIPPABLEHEADERSIZE; 4641 } 4642 4643 /* ZSTD_writeLastEmptyBlock() : 4644 * output an empty Block with end-of-frame mark to complete a frame 4645 * @return : size of data written into `dst` (== ZSTD_blockHeaderSize (defined in zstd_internal.h)) 4646 * or an error code if `dstCapacity` is too small (<ZSTD_blockHeaderSize) 4647 */ 4648 size_t ZSTD_writeLastEmptyBlock(void* dst, size_t dstCapacity) 4649 { 4650 RETURN_ERROR_IF(dstCapacity < ZSTD_blockHeaderSize, dstSize_tooSmall, 4651 "dst buf is too small to write frame trailer empty block."); 4652 { U32 const cBlockHeader24 = 1 /*lastBlock*/ + (((U32)bt_raw)<<1); /* 0 size */ 4653 MEM_writeLE24(dst, cBlockHeader24); 4654 return ZSTD_blockHeaderSize; 4655 } 4656 } 4657 4658 void ZSTD_referenceExternalSequences(ZSTD_CCtx* cctx, rawSeq* seq, size_t nbSeq) 4659 { 4660 assert(cctx->stage == ZSTDcs_init); 4661 assert(nbSeq == 0 || cctx->appliedParams.ldmParams.enableLdm != ZSTD_ps_enable); 4662 cctx->externSeqStore.seq = seq; 4663 cctx->externSeqStore.size = nbSeq; 4664 cctx->externSeqStore.capacity = nbSeq; 4665 cctx->externSeqStore.pos = 0; 4666 cctx->externSeqStore.posInSequence = 0; 4667 } 4668 4669 4670 static size_t ZSTD_compressContinue_internal (ZSTD_CCtx* cctx, 4671 void* dst, size_t dstCapacity, 4672 const void* src, size_t srcSize, 4673 U32 frame, U32 lastFrameChunk) 4674 { 4675 ZSTD_matchState_t* const ms = &cctx->blockState.matchState; 4676 size_t fhSize = 0; 4677 4678 DEBUGLOG(5, "ZSTD_compressContinue_internal, stage: %u, srcSize: %u", 4679 cctx->stage, (unsigned)srcSize); 4680 RETURN_ERROR_IF(cctx->stage==ZSTDcs_created, stage_wrong, 4681 "missing init (ZSTD_compressBegin)"); 4682 4683 if (frame && (cctx->stage==ZSTDcs_init)) { 4684 fhSize = ZSTD_writeFrameHeader(dst, dstCapacity, &cctx->appliedParams, 4685 cctx->pledgedSrcSizePlusOne-1, cctx->dictID); 4686 FORWARD_IF_ERROR(fhSize, "ZSTD_writeFrameHeader failed"); 4687 assert(fhSize <= dstCapacity); 4688 dstCapacity -= fhSize; 4689 dst = (char*)dst + fhSize; 4690 cctx->stage = ZSTDcs_ongoing; 4691 } 4692 4693 if (!srcSize) return fhSize; /* do not generate an empty block if no input */ 4694 4695 if (!ZSTD_window_update(&ms->window, src, srcSize, ms->forceNonContiguous)) { 4696 ms->forceNonContiguous = 0; 4697 ms->nextToUpdate = ms->window.dictLimit; 4698 } 4699 if (cctx->appliedParams.ldmParams.enableLdm == ZSTD_ps_enable) { 4700 ZSTD_window_update(&cctx->ldmState.window, src, srcSize, /* forceNonContiguous */ 0); 4701 } 4702 4703 if (!frame) { 4704 /* overflow check and correction for block mode */ 4705 ZSTD_overflowCorrectIfNeeded( 4706 ms, &cctx->workspace, &cctx->appliedParams, 4707 src, (BYTE const*)src + srcSize); 4708 } 4709 4710 DEBUGLOG(5, "ZSTD_compressContinue_internal (blockSize=%u)", (unsigned)cctx->blockSize); 4711 { size_t const cSize = frame ? 4712 ZSTD_compress_frameChunk (cctx, dst, dstCapacity, src, srcSize, lastFrameChunk) : 4713 ZSTD_compressBlock_internal (cctx, dst, dstCapacity, src, srcSize, 0 /* frame */); 4714 FORWARD_IF_ERROR(cSize, "%s", frame ? "ZSTD_compress_frameChunk failed" : "ZSTD_compressBlock_internal failed"); 4715 cctx->consumedSrcSize += srcSize; 4716 cctx->producedCSize += (cSize + fhSize); 4717 assert(!(cctx->appliedParams.fParams.contentSizeFlag && cctx->pledgedSrcSizePlusOne == 0)); 4718 if (cctx->pledgedSrcSizePlusOne != 0) { /* control src size */ 4719 ZSTD_STATIC_ASSERT(ZSTD_CONTENTSIZE_UNKNOWN == (unsigned long long)-1); 4720 RETURN_ERROR_IF( 4721 cctx->consumedSrcSize+1 > cctx->pledgedSrcSizePlusOne, 4722 srcSize_wrong, 4723 "error : pledgedSrcSize = %u, while realSrcSize >= %u", 4724 (unsigned)cctx->pledgedSrcSizePlusOne-1, 4725 (unsigned)cctx->consumedSrcSize); 4726 } 4727 return cSize + fhSize; 4728 } 4729 } 4730 4731 size_t ZSTD_compressContinue_public(ZSTD_CCtx* cctx, 4732 void* dst, size_t dstCapacity, 4733 const void* src, size_t srcSize) 4734 { 4735 DEBUGLOG(5, "ZSTD_compressContinue (srcSize=%u)", (unsigned)srcSize); 4736 return ZSTD_compressContinue_internal(cctx, dst, dstCapacity, src, srcSize, 1 /* frame mode */, 0 /* last chunk */); 4737 } 4738 4739 /* NOTE: Must just wrap ZSTD_compressContinue_public() */ 4740 size_t ZSTD_compressContinue(ZSTD_CCtx* cctx, 4741 void* dst, size_t dstCapacity, 4742 const void* src, size_t srcSize) 4743 { 4744 return ZSTD_compressContinue_public(cctx, dst, dstCapacity, src, srcSize); 4745 } 4746 4747 static size_t ZSTD_getBlockSize_deprecated(const ZSTD_CCtx* cctx) 4748 { 4749 ZSTD_compressionParameters const cParams = cctx->appliedParams.cParams; 4750 assert(!ZSTD_checkCParams(cParams)); 4751 return MIN(cctx->appliedParams.maxBlockSize, (size_t)1 << cParams.windowLog); 4752 } 4753 4754 /* NOTE: Must just wrap ZSTD_getBlockSize_deprecated() */ 4755 size_t ZSTD_getBlockSize(const ZSTD_CCtx* cctx) 4756 { 4757 return ZSTD_getBlockSize_deprecated(cctx); 4758 } 4759 4760 /* NOTE: Must just wrap ZSTD_compressBlock_deprecated() */ 4761 size_t ZSTD_compressBlock_deprecated(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize) 4762 { 4763 DEBUGLOG(5, "ZSTD_compressBlock: srcSize = %u", (unsigned)srcSize); 4764 { size_t const blockSizeMax = ZSTD_getBlockSize_deprecated(cctx); 4765 RETURN_ERROR_IF(srcSize > blockSizeMax, srcSize_wrong, "input is larger than a block"); } 4766 4767 return ZSTD_compressContinue_internal(cctx, dst, dstCapacity, src, srcSize, 0 /* frame mode */, 0 /* last chunk */); 4768 } 4769 4770 /* NOTE: Must just wrap ZSTD_compressBlock_deprecated() */ 4771 size_t ZSTD_compressBlock(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize) 4772 { 4773 return ZSTD_compressBlock_deprecated(cctx, dst, dstCapacity, src, srcSize); 4774 } 4775 4776 /*! ZSTD_loadDictionaryContent() : 4777 * @return : 0, or an error code 4778 */ 4779 static size_t ZSTD_loadDictionaryContent(ZSTD_matchState_t* ms, 4780 ldmState_t* ls, 4781 ZSTD_cwksp* ws, 4782 ZSTD_CCtx_params const* params, 4783 const void* src, size_t srcSize, 4784 ZSTD_dictTableLoadMethod_e dtlm, 4785 ZSTD_tableFillPurpose_e tfp) 4786 { 4787 const BYTE* ip = (const BYTE*) src; 4788 const BYTE* const iend = ip + srcSize; 4789 int const loadLdmDict = params->ldmParams.enableLdm == ZSTD_ps_enable && ls != NULL; 4790 4791 /* Assert that the ms params match the params we're being given */ 4792 ZSTD_assertEqualCParams(params->cParams, ms->cParams); 4793 4794 { /* Ensure large dictionaries can't cause index overflow */ 4795 4796 /* Allow the dictionary to set indices up to exactly ZSTD_CURRENT_MAX. 4797 * Dictionaries right at the edge will immediately trigger overflow 4798 * correction, but I don't want to insert extra constraints here. 4799 */ 4800 U32 maxDictSize = ZSTD_CURRENT_MAX - ZSTD_WINDOW_START_INDEX; 4801 4802 int const CDictTaggedIndices = ZSTD_CDictIndicesAreTagged(¶ms->cParams); 4803 if (CDictTaggedIndices && tfp == ZSTD_tfp_forCDict) { 4804 /* Some dictionary matchfinders in zstd use "short cache", 4805 * which treats the lower ZSTD_SHORT_CACHE_TAG_BITS of each 4806 * CDict hashtable entry as a tag rather than as part of an index. 4807 * When short cache is used, we need to truncate the dictionary 4808 * so that its indices don't overlap with the tag. */ 4809 U32 const shortCacheMaxDictSize = (1u << (32 - ZSTD_SHORT_CACHE_TAG_BITS)) - ZSTD_WINDOW_START_INDEX; 4810 maxDictSize = MIN(maxDictSize, shortCacheMaxDictSize); 4811 assert(!loadLdmDict); 4812 } 4813 4814 /* If the dictionary is too large, only load the suffix of the dictionary. */ 4815 if (srcSize > maxDictSize) { 4816 ip = iend - maxDictSize; 4817 src = ip; 4818 srcSize = maxDictSize; 4819 } 4820 } 4821 4822 if (srcSize > ZSTD_CHUNKSIZE_MAX) { 4823 /* We must have cleared our windows when our source is this large. */ 4824 assert(ZSTD_window_isEmpty(ms->window)); 4825 if (loadLdmDict) assert(ZSTD_window_isEmpty(ls->window)); 4826 } 4827 ZSTD_window_update(&ms->window, src, srcSize, /* forceNonContiguous */ 0); 4828 4829 DEBUGLOG(4, "ZSTD_loadDictionaryContent(): useRowMatchFinder=%d", (int)params->useRowMatchFinder); 4830 4831 if (loadLdmDict) { /* Load the entire dict into LDM matchfinders. */ 4832 ZSTD_window_update(&ls->window, src, srcSize, /* forceNonContiguous */ 0); 4833 ls->loadedDictEnd = params->forceWindow ? 0 : (U32)(iend - ls->window.base); 4834 ZSTD_ldm_fillHashTable(ls, ip, iend, ¶ms->ldmParams); 4835 } 4836 4837 /* If the dict is larger than we can reasonably index in our tables, only load the suffix. */ 4838 if (params->cParams.strategy < ZSTD_btultra) { 4839 U32 maxDictSize = 8U << MIN(MAX(params->cParams.hashLog, params->cParams.chainLog), 28); 4840 if (srcSize > maxDictSize) { 4841 ip = iend - maxDictSize; 4842 src = ip; 4843 srcSize = maxDictSize; 4844 } 4845 } 4846 4847 ms->nextToUpdate = (U32)(ip - ms->window.base); 4848 ms->loadedDictEnd = params->forceWindow ? 0 : (U32)(iend - ms->window.base); 4849 ms->forceNonContiguous = params->deterministicRefPrefix; 4850 4851 if (srcSize <= HASH_READ_SIZE) return 0; 4852 4853 ZSTD_overflowCorrectIfNeeded(ms, ws, params, ip, iend); 4854 4855 switch(params->cParams.strategy) 4856 { 4857 case ZSTD_fast: 4858 ZSTD_fillHashTable(ms, iend, dtlm, tfp); 4859 break; 4860 case ZSTD_dfast: 4861 #ifndef ZSTD_EXCLUDE_DFAST_BLOCK_COMPRESSOR 4862 ZSTD_fillDoubleHashTable(ms, iend, dtlm, tfp); 4863 #else 4864 assert(0); /* shouldn't be called: cparams should've been adjusted. */ 4865 #endif 4866 break; 4867 4868 case ZSTD_greedy: 4869 case ZSTD_lazy: 4870 case ZSTD_lazy2: 4871 #if !defined(ZSTD_EXCLUDE_GREEDY_BLOCK_COMPRESSOR) \ 4872 || !defined(ZSTD_EXCLUDE_LAZY_BLOCK_COMPRESSOR) \ 4873 || !defined(ZSTD_EXCLUDE_LAZY2_BLOCK_COMPRESSOR) 4874 assert(srcSize >= HASH_READ_SIZE); 4875 if (ms->dedicatedDictSearch) { 4876 assert(ms->chainTable != NULL); 4877 ZSTD_dedicatedDictSearch_lazy_loadDictionary(ms, iend-HASH_READ_SIZE); 4878 } else { 4879 assert(params->useRowMatchFinder != ZSTD_ps_auto); 4880 if (params->useRowMatchFinder == ZSTD_ps_enable) { 4881 size_t const tagTableSize = ((size_t)1 << params->cParams.hashLog); 4882 ZSTD_memset(ms->tagTable, 0, tagTableSize); 4883 ZSTD_row_update(ms, iend-HASH_READ_SIZE); 4884 DEBUGLOG(4, "Using row-based hash table for lazy dict"); 4885 } else { 4886 ZSTD_insertAndFindFirstIndex(ms, iend-HASH_READ_SIZE); 4887 DEBUGLOG(4, "Using chain-based hash table for lazy dict"); 4888 } 4889 } 4890 #else 4891 assert(0); /* shouldn't be called: cparams should've been adjusted. */ 4892 #endif 4893 break; 4894 4895 case ZSTD_btlazy2: /* we want the dictionary table fully sorted */ 4896 case ZSTD_btopt: 4897 case ZSTD_btultra: 4898 case ZSTD_btultra2: 4899 #if !defined(ZSTD_EXCLUDE_BTLAZY2_BLOCK_COMPRESSOR) \ 4900 || !defined(ZSTD_EXCLUDE_BTOPT_BLOCK_COMPRESSOR) \ 4901 || !defined(ZSTD_EXCLUDE_BTULTRA_BLOCK_COMPRESSOR) 4902 assert(srcSize >= HASH_READ_SIZE); 4903 ZSTD_updateTree(ms, iend-HASH_READ_SIZE, iend); 4904 #else 4905 assert(0); /* shouldn't be called: cparams should've been adjusted. */ 4906 #endif 4907 break; 4908 4909 default: 4910 assert(0); /* not possible : not a valid strategy id */ 4911 } 4912 4913 ms->nextToUpdate = (U32)(iend - ms->window.base); 4914 return 0; 4915 } 4916 4917 4918 /* Dictionaries that assign zero probability to symbols that show up causes problems 4919 * when FSE encoding. Mark dictionaries with zero probability symbols as FSE_repeat_check 4920 * and only dictionaries with 100% valid symbols can be assumed valid. 4921 */ 4922 static FSE_repeat ZSTD_dictNCountRepeat(short* normalizedCounter, unsigned dictMaxSymbolValue, unsigned maxSymbolValue) 4923 { 4924 U32 s; 4925 if (dictMaxSymbolValue < maxSymbolValue) { 4926 return FSE_repeat_check; 4927 } 4928 for (s = 0; s <= maxSymbolValue; ++s) { 4929 if (normalizedCounter[s] == 0) { 4930 return FSE_repeat_check; 4931 } 4932 } 4933 return FSE_repeat_valid; 4934 } 4935 4936 size_t ZSTD_loadCEntropy(ZSTD_compressedBlockState_t* bs, void* workspace, 4937 const void* const dict, size_t dictSize) 4938 { 4939 short offcodeNCount[MaxOff+1]; 4940 unsigned offcodeMaxValue = MaxOff; 4941 const BYTE* dictPtr = (const BYTE*)dict; /* skip magic num and dict ID */ 4942 const BYTE* const dictEnd = dictPtr + dictSize; 4943 dictPtr += 8; 4944 bs->entropy.huf.repeatMode = HUF_repeat_check; 4945 4946 { unsigned maxSymbolValue = 255; 4947 unsigned hasZeroWeights = 1; 4948 size_t const hufHeaderSize = HUF_readCTable((HUF_CElt*)bs->entropy.huf.CTable, &maxSymbolValue, dictPtr, 4949 dictEnd-dictPtr, &hasZeroWeights); 4950 4951 /* We only set the loaded table as valid if it contains all non-zero 4952 * weights. Otherwise, we set it to check */ 4953 if (!hasZeroWeights && maxSymbolValue == 255) 4954 bs->entropy.huf.repeatMode = HUF_repeat_valid; 4955 4956 RETURN_ERROR_IF(HUF_isError(hufHeaderSize), dictionary_corrupted, ""); 4957 dictPtr += hufHeaderSize; 4958 } 4959 4960 { unsigned offcodeLog; 4961 size_t const offcodeHeaderSize = FSE_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dictPtr, dictEnd-dictPtr); 4962 RETURN_ERROR_IF(FSE_isError(offcodeHeaderSize), dictionary_corrupted, ""); 4963 RETURN_ERROR_IF(offcodeLog > OffFSELog, dictionary_corrupted, ""); 4964 /* fill all offset symbols to avoid garbage at end of table */ 4965 RETURN_ERROR_IF(FSE_isError(FSE_buildCTable_wksp( 4966 bs->entropy.fse.offcodeCTable, 4967 offcodeNCount, MaxOff, offcodeLog, 4968 workspace, HUF_WORKSPACE_SIZE)), 4969 dictionary_corrupted, ""); 4970 /* Defer checking offcodeMaxValue because we need to know the size of the dictionary content */ 4971 dictPtr += offcodeHeaderSize; 4972 } 4973 4974 { short matchlengthNCount[MaxML+1]; 4975 unsigned matchlengthMaxValue = MaxML, matchlengthLog; 4976 size_t const matchlengthHeaderSize = FSE_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dictPtr, dictEnd-dictPtr); 4977 RETURN_ERROR_IF(FSE_isError(matchlengthHeaderSize), dictionary_corrupted, ""); 4978 RETURN_ERROR_IF(matchlengthLog > MLFSELog, dictionary_corrupted, ""); 4979 RETURN_ERROR_IF(FSE_isError(FSE_buildCTable_wksp( 4980 bs->entropy.fse.matchlengthCTable, 4981 matchlengthNCount, matchlengthMaxValue, matchlengthLog, 4982 workspace, HUF_WORKSPACE_SIZE)), 4983 dictionary_corrupted, ""); 4984 bs->entropy.fse.matchlength_repeatMode = ZSTD_dictNCountRepeat(matchlengthNCount, matchlengthMaxValue, MaxML); 4985 dictPtr += matchlengthHeaderSize; 4986 } 4987 4988 { short litlengthNCount[MaxLL+1]; 4989 unsigned litlengthMaxValue = MaxLL, litlengthLog; 4990 size_t const litlengthHeaderSize = FSE_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dictPtr, dictEnd-dictPtr); 4991 RETURN_ERROR_IF(FSE_isError(litlengthHeaderSize), dictionary_corrupted, ""); 4992 RETURN_ERROR_IF(litlengthLog > LLFSELog, dictionary_corrupted, ""); 4993 RETURN_ERROR_IF(FSE_isError(FSE_buildCTable_wksp( 4994 bs->entropy.fse.litlengthCTable, 4995 litlengthNCount, litlengthMaxValue, litlengthLog, 4996 workspace, HUF_WORKSPACE_SIZE)), 4997 dictionary_corrupted, ""); 4998 bs->entropy.fse.litlength_repeatMode = ZSTD_dictNCountRepeat(litlengthNCount, litlengthMaxValue, MaxLL); 4999 dictPtr += litlengthHeaderSize; 5000 } 5001 5002 RETURN_ERROR_IF(dictPtr+12 > dictEnd, dictionary_corrupted, ""); 5003 bs->rep[0] = MEM_readLE32(dictPtr+0); 5004 bs->rep[1] = MEM_readLE32(dictPtr+4); 5005 bs->rep[2] = MEM_readLE32(dictPtr+8); 5006 dictPtr += 12; 5007 5008 { size_t const dictContentSize = (size_t)(dictEnd - dictPtr); 5009 U32 offcodeMax = MaxOff; 5010 if (dictContentSize <= ((U32)-1) - 128 KB) { 5011 U32 const maxOffset = (U32)dictContentSize + 128 KB; /* The maximum offset that must be supported */ 5012 offcodeMax = ZSTD_highbit32(maxOffset); /* Calculate minimum offset code required to represent maxOffset */ 5013 } 5014 /* All offset values <= dictContentSize + 128 KB must be representable for a valid table */ 5015 bs->entropy.fse.offcode_repeatMode = ZSTD_dictNCountRepeat(offcodeNCount, offcodeMaxValue, MIN(offcodeMax, MaxOff)); 5016 5017 /* All repCodes must be <= dictContentSize and != 0 */ 5018 { U32 u; 5019 for (u=0; u<3; u++) { 5020 RETURN_ERROR_IF(bs->rep[u] == 0, dictionary_corrupted, ""); 5021 RETURN_ERROR_IF(bs->rep[u] > dictContentSize, dictionary_corrupted, ""); 5022 } } } 5023 5024 return dictPtr - (const BYTE*)dict; 5025 } 5026 5027 /* Dictionary format : 5028 * See : 5029 * https://github.com/facebook/zstd/blob/release/doc/zstd_compression_format.md#dictionary-format 5030 */ 5031 /*! ZSTD_loadZstdDictionary() : 5032 * @return : dictID, or an error code 5033 * assumptions : magic number supposed already checked 5034 * dictSize supposed >= 8 5035 */ 5036 static size_t ZSTD_loadZstdDictionary(ZSTD_compressedBlockState_t* bs, 5037 ZSTD_matchState_t* ms, 5038 ZSTD_cwksp* ws, 5039 ZSTD_CCtx_params const* params, 5040 const void* dict, size_t dictSize, 5041 ZSTD_dictTableLoadMethod_e dtlm, 5042 ZSTD_tableFillPurpose_e tfp, 5043 void* workspace) 5044 { 5045 const BYTE* dictPtr = (const BYTE*)dict; 5046 const BYTE* const dictEnd = dictPtr + dictSize; 5047 size_t dictID; 5048 size_t eSize; 5049 ZSTD_STATIC_ASSERT(HUF_WORKSPACE_SIZE >= (1<<MAX(MLFSELog,LLFSELog))); 5050 assert(dictSize >= 8); 5051 assert(MEM_readLE32(dictPtr) == ZSTD_MAGIC_DICTIONARY); 5052 5053 dictID = params->fParams.noDictIDFlag ? 0 : MEM_readLE32(dictPtr + 4 /* skip magic number */ ); 5054 eSize = ZSTD_loadCEntropy(bs, workspace, dict, dictSize); 5055 FORWARD_IF_ERROR(eSize, "ZSTD_loadCEntropy failed"); 5056 dictPtr += eSize; 5057 5058 { 5059 size_t const dictContentSize = (size_t)(dictEnd - dictPtr); 5060 FORWARD_IF_ERROR(ZSTD_loadDictionaryContent( 5061 ms, NULL, ws, params, dictPtr, dictContentSize, dtlm, tfp), ""); 5062 } 5063 return dictID; 5064 } 5065 5066 /** ZSTD_compress_insertDictionary() : 5067 * @return : dictID, or an error code */ 5068 static size_t 5069 ZSTD_compress_insertDictionary(ZSTD_compressedBlockState_t* bs, 5070 ZSTD_matchState_t* ms, 5071 ldmState_t* ls, 5072 ZSTD_cwksp* ws, 5073 const ZSTD_CCtx_params* params, 5074 const void* dict, size_t dictSize, 5075 ZSTD_dictContentType_e dictContentType, 5076 ZSTD_dictTableLoadMethod_e dtlm, 5077 ZSTD_tableFillPurpose_e tfp, 5078 void* workspace) 5079 { 5080 DEBUGLOG(4, "ZSTD_compress_insertDictionary (dictSize=%u)", (U32)dictSize); 5081 if ((dict==NULL) || (dictSize<8)) { 5082 RETURN_ERROR_IF(dictContentType == ZSTD_dct_fullDict, dictionary_wrong, ""); 5083 return 0; 5084 } 5085 5086 ZSTD_reset_compressedBlockState(bs); 5087 5088 /* dict restricted modes */ 5089 if (dictContentType == ZSTD_dct_rawContent) 5090 return ZSTD_loadDictionaryContent(ms, ls, ws, params, dict, dictSize, dtlm, tfp); 5091 5092 if (MEM_readLE32(dict) != ZSTD_MAGIC_DICTIONARY) { 5093 if (dictContentType == ZSTD_dct_auto) { 5094 DEBUGLOG(4, "raw content dictionary detected"); 5095 return ZSTD_loadDictionaryContent( 5096 ms, ls, ws, params, dict, dictSize, dtlm, tfp); 5097 } 5098 RETURN_ERROR_IF(dictContentType == ZSTD_dct_fullDict, dictionary_wrong, ""); 5099 assert(0); /* impossible */ 5100 } 5101 5102 /* dict as full zstd dictionary */ 5103 return ZSTD_loadZstdDictionary( 5104 bs, ms, ws, params, dict, dictSize, dtlm, tfp, workspace); 5105 } 5106 5107 #define ZSTD_USE_CDICT_PARAMS_SRCSIZE_CUTOFF (128 KB) 5108 #define ZSTD_USE_CDICT_PARAMS_DICTSIZE_MULTIPLIER (6ULL) 5109 5110 /*! ZSTD_compressBegin_internal() : 5111 * Assumption : either @dict OR @cdict (or none) is non-NULL, never both 5112 * @return : 0, or an error code */ 5113 static size_t ZSTD_compressBegin_internal(ZSTD_CCtx* cctx, 5114 const void* dict, size_t dictSize, 5115 ZSTD_dictContentType_e dictContentType, 5116 ZSTD_dictTableLoadMethod_e dtlm, 5117 const ZSTD_CDict* cdict, 5118 const ZSTD_CCtx_params* params, U64 pledgedSrcSize, 5119 ZSTD_buffered_policy_e zbuff) 5120 { 5121 size_t const dictContentSize = cdict ? cdict->dictContentSize : dictSize; 5122 #if ZSTD_TRACE 5123 cctx->traceCtx = (ZSTD_trace_compress_begin != NULL) ? ZSTD_trace_compress_begin(cctx) : 0; 5124 #endif 5125 DEBUGLOG(4, "ZSTD_compressBegin_internal: wlog=%u", params->cParams.windowLog); 5126 /* params are supposed to be fully validated at this point */ 5127 assert(!ZSTD_isError(ZSTD_checkCParams(params->cParams))); 5128 assert(!((dict) && (cdict))); /* either dict or cdict, not both */ 5129 if ( (cdict) 5130 && (cdict->dictContentSize > 0) 5131 && ( pledgedSrcSize < ZSTD_USE_CDICT_PARAMS_SRCSIZE_CUTOFF 5132 || pledgedSrcSize < cdict->dictContentSize * ZSTD_USE_CDICT_PARAMS_DICTSIZE_MULTIPLIER 5133 || pledgedSrcSize == ZSTD_CONTENTSIZE_UNKNOWN 5134 || cdict->compressionLevel == 0) 5135 && (params->attachDictPref != ZSTD_dictForceLoad) ) { 5136 return ZSTD_resetCCtx_usingCDict(cctx, cdict, params, pledgedSrcSize, zbuff); 5137 } 5138 5139 FORWARD_IF_ERROR( ZSTD_resetCCtx_internal(cctx, params, pledgedSrcSize, 5140 dictContentSize, 5141 ZSTDcrp_makeClean, zbuff) , ""); 5142 { size_t const dictID = cdict ? 5143 ZSTD_compress_insertDictionary( 5144 cctx->blockState.prevCBlock, &cctx->blockState.matchState, 5145 &cctx->ldmState, &cctx->workspace, &cctx->appliedParams, cdict->dictContent, 5146 cdict->dictContentSize, cdict->dictContentType, dtlm, 5147 ZSTD_tfp_forCCtx, cctx->entropyWorkspace) 5148 : ZSTD_compress_insertDictionary( 5149 cctx->blockState.prevCBlock, &cctx->blockState.matchState, 5150 &cctx->ldmState, &cctx->workspace, &cctx->appliedParams, dict, dictSize, 5151 dictContentType, dtlm, ZSTD_tfp_forCCtx, cctx->entropyWorkspace); 5152 FORWARD_IF_ERROR(dictID, "ZSTD_compress_insertDictionary failed"); 5153 assert(dictID <= UINT_MAX); 5154 cctx->dictID = (U32)dictID; 5155 cctx->dictContentSize = dictContentSize; 5156 } 5157 return 0; 5158 } 5159 5160 size_t ZSTD_compressBegin_advanced_internal(ZSTD_CCtx* cctx, 5161 const void* dict, size_t dictSize, 5162 ZSTD_dictContentType_e dictContentType, 5163 ZSTD_dictTableLoadMethod_e dtlm, 5164 const ZSTD_CDict* cdict, 5165 const ZSTD_CCtx_params* params, 5166 unsigned long long pledgedSrcSize) 5167 { 5168 DEBUGLOG(4, "ZSTD_compressBegin_advanced_internal: wlog=%u", params->cParams.windowLog); 5169 /* compression parameters verification and optimization */ 5170 FORWARD_IF_ERROR( ZSTD_checkCParams(params->cParams) , ""); 5171 return ZSTD_compressBegin_internal(cctx, 5172 dict, dictSize, dictContentType, dtlm, 5173 cdict, 5174 params, pledgedSrcSize, 5175 ZSTDb_not_buffered); 5176 } 5177 5178 /*! ZSTD_compressBegin_advanced() : 5179 * @return : 0, or an error code */ 5180 size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, 5181 const void* dict, size_t dictSize, 5182 ZSTD_parameters params, unsigned long long pledgedSrcSize) 5183 { 5184 ZSTD_CCtx_params cctxParams; 5185 ZSTD_CCtxParams_init_internal(&cctxParams, ¶ms, ZSTD_NO_CLEVEL); 5186 return ZSTD_compressBegin_advanced_internal(cctx, 5187 dict, dictSize, ZSTD_dct_auto, ZSTD_dtlm_fast, 5188 NULL /*cdict*/, 5189 &cctxParams, pledgedSrcSize); 5190 } 5191 5192 static size_t 5193 ZSTD_compressBegin_usingDict_deprecated(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel) 5194 { 5195 ZSTD_CCtx_params cctxParams; 5196 { ZSTD_parameters const params = ZSTD_getParams_internal(compressionLevel, ZSTD_CONTENTSIZE_UNKNOWN, dictSize, ZSTD_cpm_noAttachDict); 5197 ZSTD_CCtxParams_init_internal(&cctxParams, ¶ms, (compressionLevel == 0) ? ZSTD_CLEVEL_DEFAULT : compressionLevel); 5198 } 5199 DEBUGLOG(4, "ZSTD_compressBegin_usingDict (dictSize=%u)", (unsigned)dictSize); 5200 return ZSTD_compressBegin_internal(cctx, dict, dictSize, ZSTD_dct_auto, ZSTD_dtlm_fast, NULL, 5201 &cctxParams, ZSTD_CONTENTSIZE_UNKNOWN, ZSTDb_not_buffered); 5202 } 5203 5204 size_t 5205 ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel) 5206 { 5207 return ZSTD_compressBegin_usingDict_deprecated(cctx, dict, dictSize, compressionLevel); 5208 } 5209 5210 size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel) 5211 { 5212 return ZSTD_compressBegin_usingDict_deprecated(cctx, NULL, 0, compressionLevel); 5213 } 5214 5215 5216 /*! ZSTD_writeEpilogue() : 5217 * Ends a frame. 5218 * @return : nb of bytes written into dst (or an error code) */ 5219 static size_t ZSTD_writeEpilogue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity) 5220 { 5221 BYTE* const ostart = (BYTE*)dst; 5222 BYTE* op = ostart; 5223 5224 DEBUGLOG(4, "ZSTD_writeEpilogue"); 5225 RETURN_ERROR_IF(cctx->stage == ZSTDcs_created, stage_wrong, "init missing"); 5226 5227 /* special case : empty frame */ 5228 if (cctx->stage == ZSTDcs_init) { 5229 size_t fhSize = ZSTD_writeFrameHeader(dst, dstCapacity, &cctx->appliedParams, 0, 0); 5230 FORWARD_IF_ERROR(fhSize, "ZSTD_writeFrameHeader failed"); 5231 dstCapacity -= fhSize; 5232 op += fhSize; 5233 cctx->stage = ZSTDcs_ongoing; 5234 } 5235 5236 if (cctx->stage != ZSTDcs_ending) { 5237 /* write one last empty block, make it the "last" block */ 5238 U32 const cBlockHeader24 = 1 /* last block */ + (((U32)bt_raw)<<1) + 0; 5239 ZSTD_STATIC_ASSERT(ZSTD_BLOCKHEADERSIZE == 3); 5240 RETURN_ERROR_IF(dstCapacity<3, dstSize_tooSmall, "no room for epilogue"); 5241 MEM_writeLE24(op, cBlockHeader24); 5242 op += ZSTD_blockHeaderSize; 5243 dstCapacity -= ZSTD_blockHeaderSize; 5244 } 5245 5246 if (cctx->appliedParams.fParams.checksumFlag) { 5247 U32 const checksum = (U32) XXH64_digest(&cctx->xxhState); 5248 RETURN_ERROR_IF(dstCapacity<4, dstSize_tooSmall, "no room for checksum"); 5249 DEBUGLOG(4, "ZSTD_writeEpilogue: write checksum : %08X", (unsigned)checksum); 5250 MEM_writeLE32(op, checksum); 5251 op += 4; 5252 } 5253 5254 cctx->stage = ZSTDcs_created; /* return to "created but no init" status */ 5255 return op-ostart; 5256 } 5257 5258 void ZSTD_CCtx_trace(ZSTD_CCtx* cctx, size_t extraCSize) 5259 { 5260 #if ZSTD_TRACE 5261 if (cctx->traceCtx && ZSTD_trace_compress_end != NULL) { 5262 int const streaming = cctx->inBuffSize > 0 || cctx->outBuffSize > 0 || cctx->appliedParams.nbWorkers > 0; 5263 ZSTD_Trace trace; 5264 ZSTD_memset(&trace, 0, sizeof(trace)); 5265 trace.version = ZSTD_VERSION_NUMBER; 5266 trace.streaming = streaming; 5267 trace.dictionaryID = cctx->dictID; 5268 trace.dictionarySize = cctx->dictContentSize; 5269 trace.uncompressedSize = cctx->consumedSrcSize; 5270 trace.compressedSize = cctx->producedCSize + extraCSize; 5271 trace.params = &cctx->appliedParams; 5272 trace.cctx = cctx; 5273 ZSTD_trace_compress_end(cctx->traceCtx, &trace); 5274 } 5275 cctx->traceCtx = 0; 5276 #else 5277 (void)cctx; 5278 (void)extraCSize; 5279 #endif 5280 } 5281 5282 size_t ZSTD_compressEnd_public(ZSTD_CCtx* cctx, 5283 void* dst, size_t dstCapacity, 5284 const void* src, size_t srcSize) 5285 { 5286 size_t endResult; 5287 size_t const cSize = ZSTD_compressContinue_internal(cctx, 5288 dst, dstCapacity, src, srcSize, 5289 1 /* frame mode */, 1 /* last chunk */); 5290 FORWARD_IF_ERROR(cSize, "ZSTD_compressContinue_internal failed"); 5291 endResult = ZSTD_writeEpilogue(cctx, (char*)dst + cSize, dstCapacity-cSize); 5292 FORWARD_IF_ERROR(endResult, "ZSTD_writeEpilogue failed"); 5293 assert(!(cctx->appliedParams.fParams.contentSizeFlag && cctx->pledgedSrcSizePlusOne == 0)); 5294 if (cctx->pledgedSrcSizePlusOne != 0) { /* control src size */ 5295 ZSTD_STATIC_ASSERT(ZSTD_CONTENTSIZE_UNKNOWN == (unsigned long long)-1); 5296 DEBUGLOG(4, "end of frame : controlling src size"); 5297 RETURN_ERROR_IF( 5298 cctx->pledgedSrcSizePlusOne != cctx->consumedSrcSize+1, 5299 srcSize_wrong, 5300 "error : pledgedSrcSize = %u, while realSrcSize = %u", 5301 (unsigned)cctx->pledgedSrcSizePlusOne-1, 5302 (unsigned)cctx->consumedSrcSize); 5303 } 5304 ZSTD_CCtx_trace(cctx, endResult); 5305 return cSize + endResult; 5306 } 5307 5308 /* NOTE: Must just wrap ZSTD_compressEnd_public() */ 5309 size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, 5310 void* dst, size_t dstCapacity, 5311 const void* src, size_t srcSize) 5312 { 5313 return ZSTD_compressEnd_public(cctx, dst, dstCapacity, src, srcSize); 5314 } 5315 5316 size_t ZSTD_compress_advanced (ZSTD_CCtx* cctx, 5317 void* dst, size_t dstCapacity, 5318 const void* src, size_t srcSize, 5319 const void* dict,size_t dictSize, 5320 ZSTD_parameters params) 5321 { 5322 DEBUGLOG(4, "ZSTD_compress_advanced"); 5323 FORWARD_IF_ERROR(ZSTD_checkCParams(params.cParams), ""); 5324 ZSTD_CCtxParams_init_internal(&cctx->simpleApiParams, ¶ms, ZSTD_NO_CLEVEL); 5325 return ZSTD_compress_advanced_internal(cctx, 5326 dst, dstCapacity, 5327 src, srcSize, 5328 dict, dictSize, 5329 &cctx->simpleApiParams); 5330 } 5331 5332 /* Internal */ 5333 size_t ZSTD_compress_advanced_internal( 5334 ZSTD_CCtx* cctx, 5335 void* dst, size_t dstCapacity, 5336 const void* src, size_t srcSize, 5337 const void* dict,size_t dictSize, 5338 const ZSTD_CCtx_params* params) 5339 { 5340 DEBUGLOG(4, "ZSTD_compress_advanced_internal (srcSize:%u)", (unsigned)srcSize); 5341 FORWARD_IF_ERROR( ZSTD_compressBegin_internal(cctx, 5342 dict, dictSize, ZSTD_dct_auto, ZSTD_dtlm_fast, NULL, 5343 params, srcSize, ZSTDb_not_buffered) , ""); 5344 return ZSTD_compressEnd_public(cctx, dst, dstCapacity, src, srcSize); 5345 } 5346 5347 size_t ZSTD_compress_usingDict(ZSTD_CCtx* cctx, 5348 void* dst, size_t dstCapacity, 5349 const void* src, size_t srcSize, 5350 const void* dict, size_t dictSize, 5351 int compressionLevel) 5352 { 5353 { 5354 ZSTD_parameters const params = ZSTD_getParams_internal(compressionLevel, srcSize, dict ? dictSize : 0, ZSTD_cpm_noAttachDict); 5355 assert(params.fParams.contentSizeFlag == 1); 5356 ZSTD_CCtxParams_init_internal(&cctx->simpleApiParams, ¶ms, (compressionLevel == 0) ? ZSTD_CLEVEL_DEFAULT: compressionLevel); 5357 } 5358 DEBUGLOG(4, "ZSTD_compress_usingDict (srcSize=%u)", (unsigned)srcSize); 5359 return ZSTD_compress_advanced_internal(cctx, dst, dstCapacity, src, srcSize, dict, dictSize, &cctx->simpleApiParams); 5360 } 5361 5362 size_t ZSTD_compressCCtx(ZSTD_CCtx* cctx, 5363 void* dst, size_t dstCapacity, 5364 const void* src, size_t srcSize, 5365 int compressionLevel) 5366 { 5367 DEBUGLOG(4, "ZSTD_compressCCtx (srcSize=%u)", (unsigned)srcSize); 5368 assert(cctx != NULL); 5369 return ZSTD_compress_usingDict(cctx, dst, dstCapacity, src, srcSize, NULL, 0, compressionLevel); 5370 } 5371 5372 size_t ZSTD_compress(void* dst, size_t dstCapacity, 5373 const void* src, size_t srcSize, 5374 int compressionLevel) 5375 { 5376 size_t result; 5377 #if ZSTD_COMPRESS_HEAPMODE 5378 ZSTD_CCtx* cctx = ZSTD_createCCtx(); 5379 RETURN_ERROR_IF(!cctx, memory_allocation, "ZSTD_createCCtx failed"); 5380 result = ZSTD_compressCCtx(cctx, dst, dstCapacity, src, srcSize, compressionLevel); 5381 ZSTD_freeCCtx(cctx); 5382 #else 5383 ZSTD_CCtx ctxBody; 5384 ZSTD_initCCtx(&ctxBody, ZSTD_defaultCMem); 5385 result = ZSTD_compressCCtx(&ctxBody, dst, dstCapacity, src, srcSize, compressionLevel); 5386 ZSTD_freeCCtxContent(&ctxBody); /* can't free ctxBody itself, as it's on stack; free only heap content */ 5387 #endif 5388 return result; 5389 } 5390 5391 5392 /* ===== Dictionary API ===== */ 5393 5394 /*! ZSTD_estimateCDictSize_advanced() : 5395 * Estimate amount of memory that will be needed to create a dictionary with following arguments */ 5396 size_t ZSTD_estimateCDictSize_advanced( 5397 size_t dictSize, ZSTD_compressionParameters cParams, 5398 ZSTD_dictLoadMethod_e dictLoadMethod) 5399 { 5400 DEBUGLOG(5, "sizeof(ZSTD_CDict) : %u", (unsigned)sizeof(ZSTD_CDict)); 5401 return ZSTD_cwksp_alloc_size(sizeof(ZSTD_CDict)) 5402 + ZSTD_cwksp_alloc_size(HUF_WORKSPACE_SIZE) 5403 /* enableDedicatedDictSearch == 1 ensures that CDict estimation will not be too small 5404 * in case we are using DDS with row-hash. */ 5405 + ZSTD_sizeof_matchState(&cParams, ZSTD_resolveRowMatchFinderMode(ZSTD_ps_auto, &cParams), 5406 /* enableDedicatedDictSearch */ 1, /* forCCtx */ 0) 5407 + (dictLoadMethod == ZSTD_dlm_byRef ? 0 5408 : ZSTD_cwksp_alloc_size(ZSTD_cwksp_align(dictSize, sizeof(void *)))); 5409 } 5410 5411 size_t ZSTD_estimateCDictSize(size_t dictSize, int compressionLevel) 5412 { 5413 ZSTD_compressionParameters const cParams = ZSTD_getCParams_internal(compressionLevel, ZSTD_CONTENTSIZE_UNKNOWN, dictSize, ZSTD_cpm_createCDict); 5414 return ZSTD_estimateCDictSize_advanced(dictSize, cParams, ZSTD_dlm_byCopy); 5415 } 5416 5417 size_t ZSTD_sizeof_CDict(const ZSTD_CDict* cdict) 5418 { 5419 if (cdict==NULL) return 0; /* support sizeof on NULL */ 5420 DEBUGLOG(5, "sizeof(*cdict) : %u", (unsigned)sizeof(*cdict)); 5421 /* cdict may be in the workspace */ 5422 return (cdict->workspace.workspace == cdict ? 0 : sizeof(*cdict)) 5423 + ZSTD_cwksp_sizeof(&cdict->workspace); 5424 } 5425 5426 static size_t ZSTD_initCDict_internal( 5427 ZSTD_CDict* cdict, 5428 const void* dictBuffer, size_t dictSize, 5429 ZSTD_dictLoadMethod_e dictLoadMethod, 5430 ZSTD_dictContentType_e dictContentType, 5431 ZSTD_CCtx_params params) 5432 { 5433 DEBUGLOG(3, "ZSTD_initCDict_internal (dictContentType:%u)", (unsigned)dictContentType); 5434 assert(!ZSTD_checkCParams(params.cParams)); 5435 cdict->matchState.cParams = params.cParams; 5436 cdict->matchState.dedicatedDictSearch = params.enableDedicatedDictSearch; 5437 if ((dictLoadMethod == ZSTD_dlm_byRef) || (!dictBuffer) || (!dictSize)) { 5438 cdict->dictContent = dictBuffer; 5439 } else { 5440 void *internalBuffer = ZSTD_cwksp_reserve_object(&cdict->workspace, ZSTD_cwksp_align(dictSize, sizeof(void*))); 5441 RETURN_ERROR_IF(!internalBuffer, memory_allocation, "NULL pointer!"); 5442 cdict->dictContent = internalBuffer; 5443 ZSTD_memcpy(internalBuffer, dictBuffer, dictSize); 5444 } 5445 cdict->dictContentSize = dictSize; 5446 cdict->dictContentType = dictContentType; 5447 5448 cdict->entropyWorkspace = (U32*)ZSTD_cwksp_reserve_object(&cdict->workspace, HUF_WORKSPACE_SIZE); 5449 5450 5451 /* Reset the state to no dictionary */ 5452 ZSTD_reset_compressedBlockState(&cdict->cBlockState); 5453 FORWARD_IF_ERROR(ZSTD_reset_matchState( 5454 &cdict->matchState, 5455 &cdict->workspace, 5456 ¶ms.cParams, 5457 params.useRowMatchFinder, 5458 ZSTDcrp_makeClean, 5459 ZSTDirp_reset, 5460 ZSTD_resetTarget_CDict), ""); 5461 /* (Maybe) load the dictionary 5462 * Skips loading the dictionary if it is < 8 bytes. 5463 */ 5464 { params.compressionLevel = ZSTD_CLEVEL_DEFAULT; 5465 params.fParams.contentSizeFlag = 1; 5466 { size_t const dictID = ZSTD_compress_insertDictionary( 5467 &cdict->cBlockState, &cdict->matchState, NULL, &cdict->workspace, 5468 ¶ms, cdict->dictContent, cdict->dictContentSize, 5469 dictContentType, ZSTD_dtlm_full, ZSTD_tfp_forCDict, cdict->entropyWorkspace); 5470 FORWARD_IF_ERROR(dictID, "ZSTD_compress_insertDictionary failed"); 5471 assert(dictID <= (size_t)(U32)-1); 5472 cdict->dictID = (U32)dictID; 5473 } 5474 } 5475 5476 return 0; 5477 } 5478 5479 static ZSTD_CDict* ZSTD_createCDict_advanced_internal(size_t dictSize, 5480 ZSTD_dictLoadMethod_e dictLoadMethod, 5481 ZSTD_compressionParameters cParams, 5482 ZSTD_paramSwitch_e useRowMatchFinder, 5483 U32 enableDedicatedDictSearch, 5484 ZSTD_customMem customMem) 5485 { 5486 if ((!customMem.customAlloc) ^ (!customMem.customFree)) return NULL; 5487 5488 { size_t const workspaceSize = 5489 ZSTD_cwksp_alloc_size(sizeof(ZSTD_CDict)) + 5490 ZSTD_cwksp_alloc_size(HUF_WORKSPACE_SIZE) + 5491 ZSTD_sizeof_matchState(&cParams, useRowMatchFinder, enableDedicatedDictSearch, /* forCCtx */ 0) + 5492 (dictLoadMethod == ZSTD_dlm_byRef ? 0 5493 : ZSTD_cwksp_alloc_size(ZSTD_cwksp_align(dictSize, sizeof(void*)))); 5494 void* const workspace = ZSTD_customMalloc(workspaceSize, customMem); 5495 ZSTD_cwksp ws; 5496 ZSTD_CDict* cdict; 5497 5498 if (!workspace) { 5499 ZSTD_customFree(workspace, customMem); 5500 return NULL; 5501 } 5502 5503 ZSTD_cwksp_init(&ws, workspace, workspaceSize, ZSTD_cwksp_dynamic_alloc); 5504 5505 cdict = (ZSTD_CDict*)ZSTD_cwksp_reserve_object(&ws, sizeof(ZSTD_CDict)); 5506 assert(cdict != NULL); 5507 ZSTD_cwksp_move(&cdict->workspace, &ws); 5508 cdict->customMem = customMem; 5509 cdict->compressionLevel = ZSTD_NO_CLEVEL; /* signals advanced API usage */ 5510 cdict->useRowMatchFinder = useRowMatchFinder; 5511 return cdict; 5512 } 5513 } 5514 5515 ZSTD_CDict* ZSTD_createCDict_advanced(const void* dictBuffer, size_t dictSize, 5516 ZSTD_dictLoadMethod_e dictLoadMethod, 5517 ZSTD_dictContentType_e dictContentType, 5518 ZSTD_compressionParameters cParams, 5519 ZSTD_customMem customMem) 5520 { 5521 ZSTD_CCtx_params cctxParams; 5522 ZSTD_memset(&cctxParams, 0, sizeof(cctxParams)); 5523 ZSTD_CCtxParams_init(&cctxParams, 0); 5524 cctxParams.cParams = cParams; 5525 cctxParams.customMem = customMem; 5526 return ZSTD_createCDict_advanced2( 5527 dictBuffer, dictSize, 5528 dictLoadMethod, dictContentType, 5529 &cctxParams, customMem); 5530 } 5531 5532 ZSTD_CDict* ZSTD_createCDict_advanced2( 5533 const void* dict, size_t dictSize, 5534 ZSTD_dictLoadMethod_e dictLoadMethod, 5535 ZSTD_dictContentType_e dictContentType, 5536 const ZSTD_CCtx_params* originalCctxParams, 5537 ZSTD_customMem customMem) 5538 { 5539 ZSTD_CCtx_params cctxParams = *originalCctxParams; 5540 ZSTD_compressionParameters cParams; 5541 ZSTD_CDict* cdict; 5542 5543 DEBUGLOG(3, "ZSTD_createCDict_advanced2, mode %u", (unsigned)dictContentType); 5544 if (!customMem.customAlloc ^ !customMem.customFree) return NULL; 5545 5546 if (cctxParams.enableDedicatedDictSearch) { 5547 cParams = ZSTD_dedicatedDictSearch_getCParams( 5548 cctxParams.compressionLevel, dictSize); 5549 ZSTD_overrideCParams(&cParams, &cctxParams.cParams); 5550 } else { 5551 cParams = ZSTD_getCParamsFromCCtxParams( 5552 &cctxParams, ZSTD_CONTENTSIZE_UNKNOWN, dictSize, ZSTD_cpm_createCDict); 5553 } 5554 5555 if (!ZSTD_dedicatedDictSearch_isSupported(&cParams)) { 5556 /* Fall back to non-DDSS params */ 5557 cctxParams.enableDedicatedDictSearch = 0; 5558 cParams = ZSTD_getCParamsFromCCtxParams( 5559 &cctxParams, ZSTD_CONTENTSIZE_UNKNOWN, dictSize, ZSTD_cpm_createCDict); 5560 } 5561 5562 DEBUGLOG(3, "ZSTD_createCDict_advanced2: DDS: %u", cctxParams.enableDedicatedDictSearch); 5563 cctxParams.cParams = cParams; 5564 cctxParams.useRowMatchFinder = ZSTD_resolveRowMatchFinderMode(cctxParams.useRowMatchFinder, &cParams); 5565 5566 cdict = ZSTD_createCDict_advanced_internal(dictSize, 5567 dictLoadMethod, cctxParams.cParams, 5568 cctxParams.useRowMatchFinder, cctxParams.enableDedicatedDictSearch, 5569 customMem); 5570 5571 if (!cdict || ZSTD_isError( ZSTD_initCDict_internal(cdict, 5572 dict, dictSize, 5573 dictLoadMethod, dictContentType, 5574 cctxParams) )) { 5575 ZSTD_freeCDict(cdict); 5576 return NULL; 5577 } 5578 5579 return cdict; 5580 } 5581 5582 ZSTD_CDict* ZSTD_createCDict(const void* dict, size_t dictSize, int compressionLevel) 5583 { 5584 ZSTD_compressionParameters cParams = ZSTD_getCParams_internal(compressionLevel, ZSTD_CONTENTSIZE_UNKNOWN, dictSize, ZSTD_cpm_createCDict); 5585 ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dict, dictSize, 5586 ZSTD_dlm_byCopy, ZSTD_dct_auto, 5587 cParams, ZSTD_defaultCMem); 5588 if (cdict) 5589 cdict->compressionLevel = (compressionLevel == 0) ? ZSTD_CLEVEL_DEFAULT : compressionLevel; 5590 return cdict; 5591 } 5592 5593 ZSTD_CDict* ZSTD_createCDict_byReference(const void* dict, size_t dictSize, int compressionLevel) 5594 { 5595 ZSTD_compressionParameters cParams = ZSTD_getCParams_internal(compressionLevel, ZSTD_CONTENTSIZE_UNKNOWN, dictSize, ZSTD_cpm_createCDict); 5596 ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dict, dictSize, 5597 ZSTD_dlm_byRef, ZSTD_dct_auto, 5598 cParams, ZSTD_defaultCMem); 5599 if (cdict) 5600 cdict->compressionLevel = (compressionLevel == 0) ? ZSTD_CLEVEL_DEFAULT : compressionLevel; 5601 return cdict; 5602 } 5603 5604 size_t ZSTD_freeCDict(ZSTD_CDict* cdict) 5605 { 5606 if (cdict==NULL) return 0; /* support free on NULL */ 5607 { ZSTD_customMem const cMem = cdict->customMem; 5608 int cdictInWorkspace = ZSTD_cwksp_owns_buffer(&cdict->workspace, cdict); 5609 ZSTD_cwksp_free(&cdict->workspace, cMem); 5610 if (!cdictInWorkspace) { 5611 ZSTD_customFree(cdict, cMem); 5612 } 5613 return 0; 5614 } 5615 } 5616 5617 /*! ZSTD_initStaticCDict_advanced() : 5618 * Generate a digested dictionary in provided memory area. 5619 * workspace: The memory area to emplace the dictionary into. 5620 * Provided pointer must 8-bytes aligned. 5621 * It must outlive dictionary usage. 5622 * workspaceSize: Use ZSTD_estimateCDictSize() 5623 * to determine how large workspace must be. 5624 * cParams : use ZSTD_getCParams() to transform a compression level 5625 * into its relevants cParams. 5626 * @return : pointer to ZSTD_CDict*, or NULL if error (size too small) 5627 * Note : there is no corresponding "free" function. 5628 * Since workspace was allocated externally, it must be freed externally. 5629 */ 5630 const ZSTD_CDict* ZSTD_initStaticCDict( 5631 void* workspace, size_t workspaceSize, 5632 const void* dict, size_t dictSize, 5633 ZSTD_dictLoadMethod_e dictLoadMethod, 5634 ZSTD_dictContentType_e dictContentType, 5635 ZSTD_compressionParameters cParams) 5636 { 5637 ZSTD_paramSwitch_e const useRowMatchFinder = ZSTD_resolveRowMatchFinderMode(ZSTD_ps_auto, &cParams); 5638 /* enableDedicatedDictSearch == 1 ensures matchstate is not too small in case this CDict will be used for DDS + row hash */ 5639 size_t const matchStateSize = ZSTD_sizeof_matchState(&cParams, useRowMatchFinder, /* enableDedicatedDictSearch */ 1, /* forCCtx */ 0); 5640 size_t const neededSize = ZSTD_cwksp_alloc_size(sizeof(ZSTD_CDict)) 5641 + (dictLoadMethod == ZSTD_dlm_byRef ? 0 5642 : ZSTD_cwksp_alloc_size(ZSTD_cwksp_align(dictSize, sizeof(void*)))) 5643 + ZSTD_cwksp_alloc_size(HUF_WORKSPACE_SIZE) 5644 + matchStateSize; 5645 ZSTD_CDict* cdict; 5646 ZSTD_CCtx_params params; 5647 5648 if ((size_t)workspace & 7) return NULL; /* 8-aligned */ 5649 5650 { 5651 ZSTD_cwksp ws; 5652 ZSTD_cwksp_init(&ws, workspace, workspaceSize, ZSTD_cwksp_static_alloc); 5653 cdict = (ZSTD_CDict*)ZSTD_cwksp_reserve_object(&ws, sizeof(ZSTD_CDict)); 5654 if (cdict == NULL) return NULL; 5655 ZSTD_cwksp_move(&cdict->workspace, &ws); 5656 } 5657 5658 DEBUGLOG(4, "(workspaceSize < neededSize) : (%u < %u) => %u", 5659 (unsigned)workspaceSize, (unsigned)neededSize, (unsigned)(workspaceSize < neededSize)); 5660 if (workspaceSize < neededSize) return NULL; 5661 5662 ZSTD_CCtxParams_init(¶ms, 0); 5663 params.cParams = cParams; 5664 params.useRowMatchFinder = useRowMatchFinder; 5665 cdict->useRowMatchFinder = useRowMatchFinder; 5666 cdict->compressionLevel = ZSTD_NO_CLEVEL; 5667 5668 if (ZSTD_isError( ZSTD_initCDict_internal(cdict, 5669 dict, dictSize, 5670 dictLoadMethod, dictContentType, 5671 params) )) 5672 return NULL; 5673 5674 return cdict; 5675 } 5676 5677 ZSTD_compressionParameters ZSTD_getCParamsFromCDict(const ZSTD_CDict* cdict) 5678 { 5679 assert(cdict != NULL); 5680 return cdict->matchState.cParams; 5681 } 5682 5683 /*! ZSTD_getDictID_fromCDict() : 5684 * Provides the dictID of the dictionary loaded into `cdict`. 5685 * If @return == 0, the dictionary is not conformant to Zstandard specification, or empty. 5686 * Non-conformant dictionaries can still be loaded, but as content-only dictionaries. */ 5687 unsigned ZSTD_getDictID_fromCDict(const ZSTD_CDict* cdict) 5688 { 5689 if (cdict==NULL) return 0; 5690 return cdict->dictID; 5691 } 5692 5693 /* ZSTD_compressBegin_usingCDict_internal() : 5694 * Implementation of various ZSTD_compressBegin_usingCDict* functions. 5695 */ 5696 static size_t ZSTD_compressBegin_usingCDict_internal( 5697 ZSTD_CCtx* const cctx, const ZSTD_CDict* const cdict, 5698 ZSTD_frameParameters const fParams, unsigned long long const pledgedSrcSize) 5699 { 5700 ZSTD_CCtx_params cctxParams; 5701 DEBUGLOG(4, "ZSTD_compressBegin_usingCDict_internal"); 5702 RETURN_ERROR_IF(cdict==NULL, dictionary_wrong, "NULL pointer!"); 5703 /* Initialize the cctxParams from the cdict */ 5704 { 5705 ZSTD_parameters params; 5706 params.fParams = fParams; 5707 params.cParams = ( pledgedSrcSize < ZSTD_USE_CDICT_PARAMS_SRCSIZE_CUTOFF 5708 || pledgedSrcSize < cdict->dictContentSize * ZSTD_USE_CDICT_PARAMS_DICTSIZE_MULTIPLIER 5709 || pledgedSrcSize == ZSTD_CONTENTSIZE_UNKNOWN 5710 || cdict->compressionLevel == 0 ) ? 5711 ZSTD_getCParamsFromCDict(cdict) 5712 : ZSTD_getCParams(cdict->compressionLevel, 5713 pledgedSrcSize, 5714 cdict->dictContentSize); 5715 ZSTD_CCtxParams_init_internal(&cctxParams, ¶ms, cdict->compressionLevel); 5716 } 5717 /* Increase window log to fit the entire dictionary and source if the 5718 * source size is known. Limit the increase to 19, which is the 5719 * window log for compression level 1 with the largest source size. 5720 */ 5721 if (pledgedSrcSize != ZSTD_CONTENTSIZE_UNKNOWN) { 5722 U32 const limitedSrcSize = (U32)MIN(pledgedSrcSize, 1U << 19); 5723 U32 const limitedSrcLog = limitedSrcSize > 1 ? ZSTD_highbit32(limitedSrcSize - 1) + 1 : 1; 5724 cctxParams.cParams.windowLog = MAX(cctxParams.cParams.windowLog, limitedSrcLog); 5725 } 5726 return ZSTD_compressBegin_internal(cctx, 5727 NULL, 0, ZSTD_dct_auto, ZSTD_dtlm_fast, 5728 cdict, 5729 &cctxParams, pledgedSrcSize, 5730 ZSTDb_not_buffered); 5731 } 5732 5733 5734 /* ZSTD_compressBegin_usingCDict_advanced() : 5735 * This function is DEPRECATED. 5736 * cdict must be != NULL */ 5737 size_t ZSTD_compressBegin_usingCDict_advanced( 5738 ZSTD_CCtx* const cctx, const ZSTD_CDict* const cdict, 5739 ZSTD_frameParameters const fParams, unsigned long long const pledgedSrcSize) 5740 { 5741 return ZSTD_compressBegin_usingCDict_internal(cctx, cdict, fParams, pledgedSrcSize); 5742 } 5743 5744 /* ZSTD_compressBegin_usingCDict() : 5745 * cdict must be != NULL */ 5746 size_t ZSTD_compressBegin_usingCDict_deprecated(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict) 5747 { 5748 ZSTD_frameParameters const fParams = { 0 /*content*/, 0 /*checksum*/, 0 /*noDictID*/ }; 5749 return ZSTD_compressBegin_usingCDict_internal(cctx, cdict, fParams, ZSTD_CONTENTSIZE_UNKNOWN); 5750 } 5751 5752 size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict) 5753 { 5754 return ZSTD_compressBegin_usingCDict_deprecated(cctx, cdict); 5755 } 5756 5757 /*! ZSTD_compress_usingCDict_internal(): 5758 * Implementation of various ZSTD_compress_usingCDict* functions. 5759 */ 5760 static size_t ZSTD_compress_usingCDict_internal(ZSTD_CCtx* cctx, 5761 void* dst, size_t dstCapacity, 5762 const void* src, size_t srcSize, 5763 const ZSTD_CDict* cdict, ZSTD_frameParameters fParams) 5764 { 5765 FORWARD_IF_ERROR(ZSTD_compressBegin_usingCDict_internal(cctx, cdict, fParams, srcSize), ""); /* will check if cdict != NULL */ 5766 return ZSTD_compressEnd_public(cctx, dst, dstCapacity, src, srcSize); 5767 } 5768 5769 /*! ZSTD_compress_usingCDict_advanced(): 5770 * This function is DEPRECATED. 5771 */ 5772 size_t ZSTD_compress_usingCDict_advanced(ZSTD_CCtx* cctx, 5773 void* dst, size_t dstCapacity, 5774 const void* src, size_t srcSize, 5775 const ZSTD_CDict* cdict, ZSTD_frameParameters fParams) 5776 { 5777 return ZSTD_compress_usingCDict_internal(cctx, dst, dstCapacity, src, srcSize, cdict, fParams); 5778 } 5779 5780 /*! ZSTD_compress_usingCDict() : 5781 * Compression using a digested Dictionary. 5782 * Faster startup than ZSTD_compress_usingDict(), recommended when same dictionary is used multiple times. 5783 * Note that compression parameters are decided at CDict creation time 5784 * while frame parameters are hardcoded */ 5785 size_t ZSTD_compress_usingCDict(ZSTD_CCtx* cctx, 5786 void* dst, size_t dstCapacity, 5787 const void* src, size_t srcSize, 5788 const ZSTD_CDict* cdict) 5789 { 5790 ZSTD_frameParameters const fParams = { 1 /*content*/, 0 /*checksum*/, 0 /*noDictID*/ }; 5791 return ZSTD_compress_usingCDict_internal(cctx, dst, dstCapacity, src, srcSize, cdict, fParams); 5792 } 5793 5794 5795 5796 /* ****************************************************************** 5797 * Streaming 5798 ********************************************************************/ 5799 5800 ZSTD_CStream* ZSTD_createCStream(void) 5801 { 5802 DEBUGLOG(3, "ZSTD_createCStream"); 5803 return ZSTD_createCStream_advanced(ZSTD_defaultCMem); 5804 } 5805 5806 ZSTD_CStream* ZSTD_initStaticCStream(void *workspace, size_t workspaceSize) 5807 { 5808 return ZSTD_initStaticCCtx(workspace, workspaceSize); 5809 } 5810 5811 ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem) 5812 { /* CStream and CCtx are now same object */ 5813 return ZSTD_createCCtx_advanced(customMem); 5814 } 5815 5816 size_t ZSTD_freeCStream(ZSTD_CStream* zcs) 5817 { 5818 return ZSTD_freeCCtx(zcs); /* same object */ 5819 } 5820 5821 5822 5823 /*====== Initialization ======*/ 5824 5825 size_t ZSTD_CStreamInSize(void) { return ZSTD_BLOCKSIZE_MAX; } 5826 5827 size_t ZSTD_CStreamOutSize(void) 5828 { 5829 return ZSTD_compressBound(ZSTD_BLOCKSIZE_MAX) + ZSTD_blockHeaderSize + 4 /* 32-bits hash */ ; 5830 } 5831 5832 static ZSTD_cParamMode_e ZSTD_getCParamMode(ZSTD_CDict const* cdict, ZSTD_CCtx_params const* params, U64 pledgedSrcSize) 5833 { 5834 if (cdict != NULL && ZSTD_shouldAttachDict(cdict, params, pledgedSrcSize)) 5835 return ZSTD_cpm_attachDict; 5836 else 5837 return ZSTD_cpm_noAttachDict; 5838 } 5839 5840 /* ZSTD_resetCStream(): 5841 * pledgedSrcSize == 0 means "unknown" */ 5842 size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pss) 5843 { 5844 /* temporary : 0 interpreted as "unknown" during transition period. 5845 * Users willing to specify "unknown" **must** use ZSTD_CONTENTSIZE_UNKNOWN. 5846 * 0 will be interpreted as "empty" in the future. 5847 */ 5848 U64 const pledgedSrcSize = (pss==0) ? ZSTD_CONTENTSIZE_UNKNOWN : pss; 5849 DEBUGLOG(4, "ZSTD_resetCStream: pledgedSrcSize = %u", (unsigned)pledgedSrcSize); 5850 FORWARD_IF_ERROR( ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only) , ""); 5851 FORWARD_IF_ERROR( ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize) , ""); 5852 return 0; 5853 } 5854 5855 /*! ZSTD_initCStream_internal() : 5856 * Note : for lib/compress only. Used by zstdmt_compress.c. 5857 * Assumption 1 : params are valid 5858 * Assumption 2 : either dict, or cdict, is defined, not both */ 5859 size_t ZSTD_initCStream_internal(ZSTD_CStream* zcs, 5860 const void* dict, size_t dictSize, const ZSTD_CDict* cdict, 5861 const ZSTD_CCtx_params* params, 5862 unsigned long long pledgedSrcSize) 5863 { 5864 DEBUGLOG(4, "ZSTD_initCStream_internal"); 5865 FORWARD_IF_ERROR( ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only) , ""); 5866 FORWARD_IF_ERROR( ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize) , ""); 5867 assert(!ZSTD_isError(ZSTD_checkCParams(params->cParams))); 5868 zcs->requestedParams = *params; 5869 assert(!((dict) && (cdict))); /* either dict or cdict, not both */ 5870 if (dict) { 5871 FORWARD_IF_ERROR( ZSTD_CCtx_loadDictionary(zcs, dict, dictSize) , ""); 5872 } else { 5873 /* Dictionary is cleared if !cdict */ 5874 FORWARD_IF_ERROR( ZSTD_CCtx_refCDict(zcs, cdict) , ""); 5875 } 5876 return 0; 5877 } 5878 5879 /* ZSTD_initCStream_usingCDict_advanced() : 5880 * same as ZSTD_initCStream_usingCDict(), with control over frame parameters */ 5881 size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, 5882 const ZSTD_CDict* cdict, 5883 ZSTD_frameParameters fParams, 5884 unsigned long long pledgedSrcSize) 5885 { 5886 DEBUGLOG(4, "ZSTD_initCStream_usingCDict_advanced"); 5887 FORWARD_IF_ERROR( ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only) , ""); 5888 FORWARD_IF_ERROR( ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize) , ""); 5889 zcs->requestedParams.fParams = fParams; 5890 FORWARD_IF_ERROR( ZSTD_CCtx_refCDict(zcs, cdict) , ""); 5891 return 0; 5892 } 5893 5894 /* note : cdict must outlive compression session */ 5895 size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict) 5896 { 5897 DEBUGLOG(4, "ZSTD_initCStream_usingCDict"); 5898 FORWARD_IF_ERROR( ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only) , ""); 5899 FORWARD_IF_ERROR( ZSTD_CCtx_refCDict(zcs, cdict) , ""); 5900 return 0; 5901 } 5902 5903 5904 /* ZSTD_initCStream_advanced() : 5905 * pledgedSrcSize must be exact. 5906 * if srcSize is not known at init time, use value ZSTD_CONTENTSIZE_UNKNOWN. 5907 * dict is loaded with default parameters ZSTD_dct_auto and ZSTD_dlm_byCopy. */ 5908 size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, 5909 const void* dict, size_t dictSize, 5910 ZSTD_parameters params, unsigned long long pss) 5911 { 5912 /* for compatibility with older programs relying on this behavior. 5913 * Users should now specify ZSTD_CONTENTSIZE_UNKNOWN. 5914 * This line will be removed in the future. 5915 */ 5916 U64 const pledgedSrcSize = (pss==0 && params.fParams.contentSizeFlag==0) ? ZSTD_CONTENTSIZE_UNKNOWN : pss; 5917 DEBUGLOG(4, "ZSTD_initCStream_advanced"); 5918 FORWARD_IF_ERROR( ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only) , ""); 5919 FORWARD_IF_ERROR( ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize) , ""); 5920 FORWARD_IF_ERROR( ZSTD_checkCParams(params.cParams) , ""); 5921 ZSTD_CCtxParams_setZstdParams(&zcs->requestedParams, ¶ms); 5922 FORWARD_IF_ERROR( ZSTD_CCtx_loadDictionary(zcs, dict, dictSize) , ""); 5923 return 0; 5924 } 5925 5926 size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel) 5927 { 5928 DEBUGLOG(4, "ZSTD_initCStream_usingDict"); 5929 FORWARD_IF_ERROR( ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only) , ""); 5930 FORWARD_IF_ERROR( ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel) , ""); 5931 FORWARD_IF_ERROR( ZSTD_CCtx_loadDictionary(zcs, dict, dictSize) , ""); 5932 return 0; 5933 } 5934 5935 size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pss) 5936 { 5937 /* temporary : 0 interpreted as "unknown" during transition period. 5938 * Users willing to specify "unknown" **must** use ZSTD_CONTENTSIZE_UNKNOWN. 5939 * 0 will be interpreted as "empty" in the future. 5940 */ 5941 U64 const pledgedSrcSize = (pss==0) ? ZSTD_CONTENTSIZE_UNKNOWN : pss; 5942 DEBUGLOG(4, "ZSTD_initCStream_srcSize"); 5943 FORWARD_IF_ERROR( ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only) , ""); 5944 FORWARD_IF_ERROR( ZSTD_CCtx_refCDict(zcs, NULL) , ""); 5945 FORWARD_IF_ERROR( ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel) , ""); 5946 FORWARD_IF_ERROR( ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize) , ""); 5947 return 0; 5948 } 5949 5950 size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel) 5951 { 5952 DEBUGLOG(4, "ZSTD_initCStream"); 5953 FORWARD_IF_ERROR( ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only) , ""); 5954 FORWARD_IF_ERROR( ZSTD_CCtx_refCDict(zcs, NULL) , ""); 5955 FORWARD_IF_ERROR( ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel) , ""); 5956 return 0; 5957 } 5958 5959 /*====== Compression ======*/ 5960 5961 static size_t ZSTD_nextInputSizeHint(const ZSTD_CCtx* cctx) 5962 { 5963 if (cctx->appliedParams.inBufferMode == ZSTD_bm_stable) { 5964 return cctx->blockSize - cctx->stableIn_notConsumed; 5965 } 5966 assert(cctx->appliedParams.inBufferMode == ZSTD_bm_buffered); 5967 { size_t hintInSize = cctx->inBuffTarget - cctx->inBuffPos; 5968 if (hintInSize==0) hintInSize = cctx->blockSize; 5969 return hintInSize; 5970 } 5971 } 5972 5973 /** ZSTD_compressStream_generic(): 5974 * internal function for all *compressStream*() variants 5975 * @return : hint size for next input to complete ongoing block */ 5976 static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs, 5977 ZSTD_outBuffer* output, 5978 ZSTD_inBuffer* input, 5979 ZSTD_EndDirective const flushMode) 5980 { 5981 const char* const istart = (assert(input != NULL), (const char*)input->src); 5982 const char* const iend = (istart != NULL) ? istart + input->size : istart; 5983 const char* ip = (istart != NULL) ? istart + input->pos : istart; 5984 char* const ostart = (assert(output != NULL), (char*)output->dst); 5985 char* const oend = (ostart != NULL) ? ostart + output->size : ostart; 5986 char* op = (ostart != NULL) ? ostart + output->pos : ostart; 5987 U32 someMoreWork = 1; 5988 5989 /* check expectations */ 5990 DEBUGLOG(5, "ZSTD_compressStream_generic, flush=%i, srcSize = %zu", (int)flushMode, input->size - input->pos); 5991 assert(zcs != NULL); 5992 if (zcs->appliedParams.inBufferMode == ZSTD_bm_stable) { 5993 assert(input->pos >= zcs->stableIn_notConsumed); 5994 input->pos -= zcs->stableIn_notConsumed; 5995 if (ip) ip -= zcs->stableIn_notConsumed; 5996 zcs->stableIn_notConsumed = 0; 5997 } 5998 if (zcs->appliedParams.inBufferMode == ZSTD_bm_buffered) { 5999 assert(zcs->inBuff != NULL); 6000 assert(zcs->inBuffSize > 0); 6001 } 6002 if (zcs->appliedParams.outBufferMode == ZSTD_bm_buffered) { 6003 assert(zcs->outBuff != NULL); 6004 assert(zcs->outBuffSize > 0); 6005 } 6006 if (input->src == NULL) assert(input->size == 0); 6007 assert(input->pos <= input->size); 6008 if (output->dst == NULL) assert(output->size == 0); 6009 assert(output->pos <= output->size); 6010 assert((U32)flushMode <= (U32)ZSTD_e_end); 6011 6012 while (someMoreWork) { 6013 switch(zcs->streamStage) 6014 { 6015 case zcss_init: 6016 RETURN_ERROR(init_missing, "call ZSTD_initCStream() first!"); 6017 6018 case zcss_load: 6019 if ( (flushMode == ZSTD_e_end) 6020 && ( (size_t)(oend-op) >= ZSTD_compressBound(iend-ip) /* Enough output space */ 6021 || zcs->appliedParams.outBufferMode == ZSTD_bm_stable) /* OR we are allowed to return dstSizeTooSmall */ 6022 && (zcs->inBuffPos == 0) ) { 6023 /* shortcut to compression pass directly into output buffer */ 6024 size_t const cSize = ZSTD_compressEnd_public(zcs, 6025 op, oend-op, ip, iend-ip); 6026 DEBUGLOG(4, "ZSTD_compressEnd : cSize=%u", (unsigned)cSize); 6027 FORWARD_IF_ERROR(cSize, "ZSTD_compressEnd failed"); 6028 ip = iend; 6029 op += cSize; 6030 zcs->frameEnded = 1; 6031 ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); 6032 someMoreWork = 0; break; 6033 } 6034 /* complete loading into inBuffer in buffered mode */ 6035 if (zcs->appliedParams.inBufferMode == ZSTD_bm_buffered) { 6036 size_t const toLoad = zcs->inBuffTarget - zcs->inBuffPos; 6037 size_t const loaded = ZSTD_limitCopy( 6038 zcs->inBuff + zcs->inBuffPos, toLoad, 6039 ip, iend-ip); 6040 zcs->inBuffPos += loaded; 6041 if (ip) ip += loaded; 6042 if ( (flushMode == ZSTD_e_continue) 6043 && (zcs->inBuffPos < zcs->inBuffTarget) ) { 6044 /* not enough input to fill full block : stop here */ 6045 someMoreWork = 0; break; 6046 } 6047 if ( (flushMode == ZSTD_e_flush) 6048 && (zcs->inBuffPos == zcs->inToCompress) ) { 6049 /* empty */ 6050 someMoreWork = 0; break; 6051 } 6052 } else { 6053 assert(zcs->appliedParams.inBufferMode == ZSTD_bm_stable); 6054 if ( (flushMode == ZSTD_e_continue) 6055 && ( (size_t)(iend - ip) < zcs->blockSize) ) { 6056 /* can't compress a full block : stop here */ 6057 zcs->stableIn_notConsumed = (size_t)(iend - ip); 6058 ip = iend; /* pretend to have consumed input */ 6059 someMoreWork = 0; break; 6060 } 6061 if ( (flushMode == ZSTD_e_flush) 6062 && (ip == iend) ) { 6063 /* empty */ 6064 someMoreWork = 0; break; 6065 } 6066 } 6067 /* compress current block (note : this stage cannot be stopped in the middle) */ 6068 DEBUGLOG(5, "stream compression stage (flushMode==%u)", flushMode); 6069 { int const inputBuffered = (zcs->appliedParams.inBufferMode == ZSTD_bm_buffered); 6070 void* cDst; 6071 size_t cSize; 6072 size_t oSize = oend-op; 6073 size_t const iSize = inputBuffered ? zcs->inBuffPos - zcs->inToCompress 6074 : MIN((size_t)(iend - ip), zcs->blockSize); 6075 if (oSize >= ZSTD_compressBound(iSize) || zcs->appliedParams.outBufferMode == ZSTD_bm_stable) 6076 cDst = op; /* compress into output buffer, to skip flush stage */ 6077 else 6078 cDst = zcs->outBuff, oSize = zcs->outBuffSize; 6079 if (inputBuffered) { 6080 unsigned const lastBlock = (flushMode == ZSTD_e_end) && (ip==iend); 6081 cSize = lastBlock ? 6082 ZSTD_compressEnd_public(zcs, cDst, oSize, 6083 zcs->inBuff + zcs->inToCompress, iSize) : 6084 ZSTD_compressContinue_public(zcs, cDst, oSize, 6085 zcs->inBuff + zcs->inToCompress, iSize); 6086 FORWARD_IF_ERROR(cSize, "%s", lastBlock ? "ZSTD_compressEnd failed" : "ZSTD_compressContinue failed"); 6087 zcs->frameEnded = lastBlock; 6088 /* prepare next block */ 6089 zcs->inBuffTarget = zcs->inBuffPos + zcs->blockSize; 6090 if (zcs->inBuffTarget > zcs->inBuffSize) 6091 zcs->inBuffPos = 0, zcs->inBuffTarget = zcs->blockSize; 6092 DEBUGLOG(5, "inBuffTarget:%u / inBuffSize:%u", 6093 (unsigned)zcs->inBuffTarget, (unsigned)zcs->inBuffSize); 6094 if (!lastBlock) 6095 assert(zcs->inBuffTarget <= zcs->inBuffSize); 6096 zcs->inToCompress = zcs->inBuffPos; 6097 } else { /* !inputBuffered, hence ZSTD_bm_stable */ 6098 unsigned const lastBlock = (flushMode == ZSTD_e_end) && (ip + iSize == iend); 6099 cSize = lastBlock ? 6100 ZSTD_compressEnd_public(zcs, cDst, oSize, ip, iSize) : 6101 ZSTD_compressContinue_public(zcs, cDst, oSize, ip, iSize); 6102 /* Consume the input prior to error checking to mirror buffered mode. */ 6103 if (ip) ip += iSize; 6104 FORWARD_IF_ERROR(cSize, "%s", lastBlock ? "ZSTD_compressEnd failed" : "ZSTD_compressContinue failed"); 6105 zcs->frameEnded = lastBlock; 6106 if (lastBlock) assert(ip == iend); 6107 } 6108 if (cDst == op) { /* no need to flush */ 6109 op += cSize; 6110 if (zcs->frameEnded) { 6111 DEBUGLOG(5, "Frame completed directly in outBuffer"); 6112 someMoreWork = 0; 6113 ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); 6114 } 6115 break; 6116 } 6117 zcs->outBuffContentSize = cSize; 6118 zcs->outBuffFlushedSize = 0; 6119 zcs->streamStage = zcss_flush; /* pass-through to flush stage */ 6120 } 6121 ZSTD_FALLTHROUGH; 6122 case zcss_flush: 6123 DEBUGLOG(5, "flush stage"); 6124 assert(zcs->appliedParams.outBufferMode == ZSTD_bm_buffered); 6125 { size_t const toFlush = zcs->outBuffContentSize - zcs->outBuffFlushedSize; 6126 size_t const flushed = ZSTD_limitCopy(op, (size_t)(oend-op), 6127 zcs->outBuff + zcs->outBuffFlushedSize, toFlush); 6128 DEBUGLOG(5, "toFlush: %u into %u ==> flushed: %u", 6129 (unsigned)toFlush, (unsigned)(oend-op), (unsigned)flushed); 6130 if (flushed) 6131 op += flushed; 6132 zcs->outBuffFlushedSize += flushed; 6133 if (toFlush!=flushed) { 6134 /* flush not fully completed, presumably because dst is too small */ 6135 assert(op==oend); 6136 someMoreWork = 0; 6137 break; 6138 } 6139 zcs->outBuffContentSize = zcs->outBuffFlushedSize = 0; 6140 if (zcs->frameEnded) { 6141 DEBUGLOG(5, "Frame completed on flush"); 6142 someMoreWork = 0; 6143 ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); 6144 break; 6145 } 6146 zcs->streamStage = zcss_load; 6147 break; 6148 } 6149 6150 default: /* impossible */ 6151 assert(0); 6152 } 6153 } 6154 6155 input->pos = ip - istart; 6156 output->pos = op - ostart; 6157 if (zcs->frameEnded) return 0; 6158 return ZSTD_nextInputSizeHint(zcs); 6159 } 6160 6161 static size_t ZSTD_nextInputSizeHint_MTorST(const ZSTD_CCtx* cctx) 6162 { 6163 #ifdef ZSTD_MULTITHREAD 6164 if (cctx->appliedParams.nbWorkers >= 1) { 6165 assert(cctx->mtctx != NULL); 6166 return ZSTDMT_nextInputSizeHint(cctx->mtctx); 6167 } 6168 #endif 6169 return ZSTD_nextInputSizeHint(cctx); 6170 6171 } 6172 6173 size_t ZSTD_compressStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output, ZSTD_inBuffer* input) 6174 { 6175 FORWARD_IF_ERROR( ZSTD_compressStream2(zcs, output, input, ZSTD_e_continue) , ""); 6176 return ZSTD_nextInputSizeHint_MTorST(zcs); 6177 } 6178 6179 /* After a compression call set the expected input/output buffer. 6180 * This is validated at the start of the next compression call. 6181 */ 6182 static void 6183 ZSTD_setBufferExpectations(ZSTD_CCtx* cctx, const ZSTD_outBuffer* output, const ZSTD_inBuffer* input) 6184 { 6185 DEBUGLOG(5, "ZSTD_setBufferExpectations (for advanced stable in/out modes)"); 6186 if (cctx->appliedParams.inBufferMode == ZSTD_bm_stable) { 6187 cctx->expectedInBuffer = *input; 6188 } 6189 if (cctx->appliedParams.outBufferMode == ZSTD_bm_stable) { 6190 cctx->expectedOutBufferSize = output->size - output->pos; 6191 } 6192 } 6193 6194 /* Validate that the input/output buffers match the expectations set by 6195 * ZSTD_setBufferExpectations. 6196 */ 6197 static size_t ZSTD_checkBufferStability(ZSTD_CCtx const* cctx, 6198 ZSTD_outBuffer const* output, 6199 ZSTD_inBuffer const* input, 6200 ZSTD_EndDirective endOp) 6201 { 6202 if (cctx->appliedParams.inBufferMode == ZSTD_bm_stable) { 6203 ZSTD_inBuffer const expect = cctx->expectedInBuffer; 6204 if (expect.src != input->src || expect.pos != input->pos) 6205 RETURN_ERROR(stabilityCondition_notRespected, "ZSTD_c_stableInBuffer enabled but input differs!"); 6206 } 6207 (void)endOp; 6208 if (cctx->appliedParams.outBufferMode == ZSTD_bm_stable) { 6209 size_t const outBufferSize = output->size - output->pos; 6210 if (cctx->expectedOutBufferSize != outBufferSize) 6211 RETURN_ERROR(stabilityCondition_notRespected, "ZSTD_c_stableOutBuffer enabled but output size differs!"); 6212 } 6213 return 0; 6214 } 6215 6216 static size_t ZSTD_CCtx_init_compressStream2(ZSTD_CCtx* cctx, 6217 ZSTD_EndDirective endOp, 6218 size_t inSize) 6219 { 6220 ZSTD_CCtx_params params = cctx->requestedParams; 6221 ZSTD_prefixDict const prefixDict = cctx->prefixDict; 6222 FORWARD_IF_ERROR( ZSTD_initLocalDict(cctx) , ""); /* Init the local dict if present. */ 6223 ZSTD_memset(&cctx->prefixDict, 0, sizeof(cctx->prefixDict)); /* single usage */ 6224 assert(prefixDict.dict==NULL || cctx->cdict==NULL); /* only one can be set */ 6225 if (cctx->cdict && !cctx->localDict.cdict) { 6226 /* Let the cdict's compression level take priority over the requested params. 6227 * But do not take the cdict's compression level if the "cdict" is actually a localDict 6228 * generated from ZSTD_initLocalDict(). 6229 */ 6230 params.compressionLevel = cctx->cdict->compressionLevel; 6231 } 6232 DEBUGLOG(4, "ZSTD_compressStream2 : transparent init stage"); 6233 if (endOp == ZSTD_e_end) cctx->pledgedSrcSizePlusOne = inSize + 1; /* auto-determine pledgedSrcSize */ 6234 6235 { size_t const dictSize = prefixDict.dict 6236 ? prefixDict.dictSize 6237 : (cctx->cdict ? cctx->cdict->dictContentSize : 0); 6238 ZSTD_cParamMode_e const mode = ZSTD_getCParamMode(cctx->cdict, ¶ms, cctx->pledgedSrcSizePlusOne - 1); 6239 params.cParams = ZSTD_getCParamsFromCCtxParams( 6240 ¶ms, cctx->pledgedSrcSizePlusOne-1, 6241 dictSize, mode); 6242 } 6243 6244 params.useBlockSplitter = ZSTD_resolveBlockSplitterMode(params.useBlockSplitter, ¶ms.cParams); 6245 params.ldmParams.enableLdm = ZSTD_resolveEnableLdm(params.ldmParams.enableLdm, ¶ms.cParams); 6246 params.useRowMatchFinder = ZSTD_resolveRowMatchFinderMode(params.useRowMatchFinder, ¶ms.cParams); 6247 params.validateSequences = ZSTD_resolveExternalSequenceValidation(params.validateSequences); 6248 params.maxBlockSize = ZSTD_resolveMaxBlockSize(params.maxBlockSize); 6249 params.searchForExternalRepcodes = ZSTD_resolveExternalRepcodeSearch(params.searchForExternalRepcodes, params.compressionLevel); 6250 6251 #ifdef ZSTD_MULTITHREAD 6252 /* If external matchfinder is enabled, make sure to fail before checking job size (for consistency) */ 6253 RETURN_ERROR_IF( 6254 ZSTD_hasExtSeqProd(¶ms) && params.nbWorkers >= 1, 6255 parameter_combination_unsupported, 6256 "External sequence producer isn't supported with nbWorkers >= 1" 6257 ); 6258 6259 if ((cctx->pledgedSrcSizePlusOne-1) <= ZSTDMT_JOBSIZE_MIN) { 6260 params.nbWorkers = 0; /* do not invoke multi-threading when src size is too small */ 6261 } 6262 if (params.nbWorkers > 0) { 6263 #if ZSTD_TRACE 6264 cctx->traceCtx = (ZSTD_trace_compress_begin != NULL) ? ZSTD_trace_compress_begin(cctx) : 0; 6265 #endif 6266 /* mt context creation */ 6267 if (cctx->mtctx == NULL) { 6268 DEBUGLOG(4, "ZSTD_compressStream2: creating new mtctx for nbWorkers=%u", 6269 params.nbWorkers); 6270 cctx->mtctx = ZSTDMT_createCCtx_advanced((U32)params.nbWorkers, cctx->customMem, cctx->pool); 6271 RETURN_ERROR_IF(cctx->mtctx == NULL, memory_allocation, "NULL pointer!"); 6272 } 6273 /* mt compression */ 6274 DEBUGLOG(4, "call ZSTDMT_initCStream_internal as nbWorkers=%u", params.nbWorkers); 6275 FORWARD_IF_ERROR( ZSTDMT_initCStream_internal( 6276 cctx->mtctx, 6277 prefixDict.dict, prefixDict.dictSize, prefixDict.dictContentType, 6278 cctx->cdict, params, cctx->pledgedSrcSizePlusOne-1) , ""); 6279 cctx->dictID = cctx->cdict ? cctx->cdict->dictID : 0; 6280 cctx->dictContentSize = cctx->cdict ? cctx->cdict->dictContentSize : prefixDict.dictSize; 6281 cctx->consumedSrcSize = 0; 6282 cctx->producedCSize = 0; 6283 cctx->streamStage = zcss_load; 6284 cctx->appliedParams = params; 6285 } else 6286 #endif /* ZSTD_MULTITHREAD */ 6287 { U64 const pledgedSrcSize = cctx->pledgedSrcSizePlusOne - 1; 6288 assert(!ZSTD_isError(ZSTD_checkCParams(params.cParams))); 6289 FORWARD_IF_ERROR( ZSTD_compressBegin_internal(cctx, 6290 prefixDict.dict, prefixDict.dictSize, prefixDict.dictContentType, ZSTD_dtlm_fast, 6291 cctx->cdict, 6292 ¶ms, pledgedSrcSize, 6293 ZSTDb_buffered) , ""); 6294 assert(cctx->appliedParams.nbWorkers == 0); 6295 cctx->inToCompress = 0; 6296 cctx->inBuffPos = 0; 6297 if (cctx->appliedParams.inBufferMode == ZSTD_bm_buffered) { 6298 /* for small input: avoid automatic flush on reaching end of block, since 6299 * it would require to add a 3-bytes null block to end frame 6300 */ 6301 cctx->inBuffTarget = cctx->blockSize + (cctx->blockSize == pledgedSrcSize); 6302 } else { 6303 cctx->inBuffTarget = 0; 6304 } 6305 cctx->outBuffContentSize = cctx->outBuffFlushedSize = 0; 6306 cctx->streamStage = zcss_load; 6307 cctx->frameEnded = 0; 6308 } 6309 return 0; 6310 } 6311 6312 /* @return provides a minimum amount of data remaining to be flushed from internal buffers 6313 */ 6314 size_t ZSTD_compressStream2( ZSTD_CCtx* cctx, 6315 ZSTD_outBuffer* output, 6316 ZSTD_inBuffer* input, 6317 ZSTD_EndDirective endOp) 6318 { 6319 DEBUGLOG(5, "ZSTD_compressStream2, endOp=%u ", (unsigned)endOp); 6320 /* check conditions */ 6321 RETURN_ERROR_IF(output->pos > output->size, dstSize_tooSmall, "invalid output buffer"); 6322 RETURN_ERROR_IF(input->pos > input->size, srcSize_wrong, "invalid input buffer"); 6323 RETURN_ERROR_IF((U32)endOp > (U32)ZSTD_e_end, parameter_outOfBound, "invalid endDirective"); 6324 assert(cctx != NULL); 6325 6326 /* transparent initialization stage */ 6327 if (cctx->streamStage == zcss_init) { 6328 size_t const inputSize = input->size - input->pos; /* no obligation to start from pos==0 */ 6329 size_t const totalInputSize = inputSize + cctx->stableIn_notConsumed; 6330 if ( (cctx->requestedParams.inBufferMode == ZSTD_bm_stable) /* input is presumed stable, across invocations */ 6331 && (endOp == ZSTD_e_continue) /* no flush requested, more input to come */ 6332 && (totalInputSize < ZSTD_BLOCKSIZE_MAX) ) { /* not even reached one block yet */ 6333 if (cctx->stableIn_notConsumed) { /* not the first time */ 6334 /* check stable source guarantees */ 6335 RETURN_ERROR_IF(input->src != cctx->expectedInBuffer.src, stabilityCondition_notRespected, "stableInBuffer condition not respected: wrong src pointer"); 6336 RETURN_ERROR_IF(input->pos != cctx->expectedInBuffer.size, stabilityCondition_notRespected, "stableInBuffer condition not respected: externally modified pos"); 6337 } 6338 /* pretend input was consumed, to give a sense forward progress */ 6339 input->pos = input->size; 6340 /* save stable inBuffer, for later control, and flush/end */ 6341 cctx->expectedInBuffer = *input; 6342 /* but actually input wasn't consumed, so keep track of position from where compression shall resume */ 6343 cctx->stableIn_notConsumed += inputSize; 6344 /* don't initialize yet, wait for the first block of flush() order, for better parameters adaptation */ 6345 return ZSTD_FRAMEHEADERSIZE_MIN(cctx->requestedParams.format); /* at least some header to produce */ 6346 } 6347 FORWARD_IF_ERROR(ZSTD_CCtx_init_compressStream2(cctx, endOp, totalInputSize), "compressStream2 initialization failed"); 6348 ZSTD_setBufferExpectations(cctx, output, input); /* Set initial buffer expectations now that we've initialized */ 6349 } 6350 /* end of transparent initialization stage */ 6351 6352 FORWARD_IF_ERROR(ZSTD_checkBufferStability(cctx, output, input, endOp), "invalid buffers"); 6353 /* compression stage */ 6354 #ifdef ZSTD_MULTITHREAD 6355 if (cctx->appliedParams.nbWorkers > 0) { 6356 size_t flushMin; 6357 if (cctx->cParamsChanged) { 6358 ZSTDMT_updateCParams_whileCompressing(cctx->mtctx, &cctx->requestedParams); 6359 cctx->cParamsChanged = 0; 6360 } 6361 if (cctx->stableIn_notConsumed) { 6362 assert(cctx->appliedParams.inBufferMode == ZSTD_bm_stable); 6363 /* some early data was skipped - make it available for consumption */ 6364 assert(input->pos >= cctx->stableIn_notConsumed); 6365 input->pos -= cctx->stableIn_notConsumed; 6366 cctx->stableIn_notConsumed = 0; 6367 } 6368 for (;;) { 6369 size_t const ipos = input->pos; 6370 size_t const opos = output->pos; 6371 flushMin = ZSTDMT_compressStream_generic(cctx->mtctx, output, input, endOp); 6372 cctx->consumedSrcSize += (U64)(input->pos - ipos); 6373 cctx->producedCSize += (U64)(output->pos - opos); 6374 if ( ZSTD_isError(flushMin) 6375 || (endOp == ZSTD_e_end && flushMin == 0) ) { /* compression completed */ 6376 if (flushMin == 0) 6377 ZSTD_CCtx_trace(cctx, 0); 6378 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_only); 6379 } 6380 FORWARD_IF_ERROR(flushMin, "ZSTDMT_compressStream_generic failed"); 6381 6382 if (endOp == ZSTD_e_continue) { 6383 /* We only require some progress with ZSTD_e_continue, not maximal progress. 6384 * We're done if we've consumed or produced any bytes, or either buffer is 6385 * full. 6386 */ 6387 if (input->pos != ipos || output->pos != opos || input->pos == input->size || output->pos == output->size) 6388 break; 6389 } else { 6390 assert(endOp == ZSTD_e_flush || endOp == ZSTD_e_end); 6391 /* We require maximal progress. We're done when the flush is complete or the 6392 * output buffer is full. 6393 */ 6394 if (flushMin == 0 || output->pos == output->size) 6395 break; 6396 } 6397 } 6398 DEBUGLOG(5, "completed ZSTD_compressStream2 delegating to ZSTDMT_compressStream_generic"); 6399 /* Either we don't require maximum forward progress, we've finished the 6400 * flush, or we are out of output space. 6401 */ 6402 assert(endOp == ZSTD_e_continue || flushMin == 0 || output->pos == output->size); 6403 ZSTD_setBufferExpectations(cctx, output, input); 6404 return flushMin; 6405 } 6406 #endif /* ZSTD_MULTITHREAD */ 6407 FORWARD_IF_ERROR( ZSTD_compressStream_generic(cctx, output, input, endOp) , ""); 6408 DEBUGLOG(5, "completed ZSTD_compressStream2"); 6409 ZSTD_setBufferExpectations(cctx, output, input); 6410 return cctx->outBuffContentSize - cctx->outBuffFlushedSize; /* remaining to flush */ 6411 } 6412 6413 size_t ZSTD_compressStream2_simpleArgs ( 6414 ZSTD_CCtx* cctx, 6415 void* dst, size_t dstCapacity, size_t* dstPos, 6416 const void* src, size_t srcSize, size_t* srcPos, 6417 ZSTD_EndDirective endOp) 6418 { 6419 ZSTD_outBuffer output; 6420 ZSTD_inBuffer input; 6421 output.dst = dst; 6422 output.size = dstCapacity; 6423 output.pos = *dstPos; 6424 input.src = src; 6425 input.size = srcSize; 6426 input.pos = *srcPos; 6427 /* ZSTD_compressStream2() will check validity of dstPos and srcPos */ 6428 { size_t const cErr = ZSTD_compressStream2(cctx, &output, &input, endOp); 6429 *dstPos = output.pos; 6430 *srcPos = input.pos; 6431 return cErr; 6432 } 6433 } 6434 6435 size_t ZSTD_compress2(ZSTD_CCtx* cctx, 6436 void* dst, size_t dstCapacity, 6437 const void* src, size_t srcSize) 6438 { 6439 ZSTD_bufferMode_e const originalInBufferMode = cctx->requestedParams.inBufferMode; 6440 ZSTD_bufferMode_e const originalOutBufferMode = cctx->requestedParams.outBufferMode; 6441 DEBUGLOG(4, "ZSTD_compress2 (srcSize=%u)", (unsigned)srcSize); 6442 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_only); 6443 /* Enable stable input/output buffers. */ 6444 cctx->requestedParams.inBufferMode = ZSTD_bm_stable; 6445 cctx->requestedParams.outBufferMode = ZSTD_bm_stable; 6446 { size_t oPos = 0; 6447 size_t iPos = 0; 6448 size_t const result = ZSTD_compressStream2_simpleArgs(cctx, 6449 dst, dstCapacity, &oPos, 6450 src, srcSize, &iPos, 6451 ZSTD_e_end); 6452 /* Reset to the original values. */ 6453 cctx->requestedParams.inBufferMode = originalInBufferMode; 6454 cctx->requestedParams.outBufferMode = originalOutBufferMode; 6455 6456 FORWARD_IF_ERROR(result, "ZSTD_compressStream2_simpleArgs failed"); 6457 if (result != 0) { /* compression not completed, due to lack of output space */ 6458 assert(oPos == dstCapacity); 6459 RETURN_ERROR(dstSize_tooSmall, ""); 6460 } 6461 assert(iPos == srcSize); /* all input is expected consumed */ 6462 return oPos; 6463 } 6464 } 6465 6466 /* ZSTD_validateSequence() : 6467 * @offCode : is presumed to follow format required by ZSTD_storeSeq() 6468 * @returns a ZSTD error code if sequence is not valid 6469 */ 6470 static size_t 6471 ZSTD_validateSequence(U32 offCode, U32 matchLength, U32 minMatch, 6472 size_t posInSrc, U32 windowLog, size_t dictSize, int useSequenceProducer) 6473 { 6474 U32 const windowSize = 1u << windowLog; 6475 /* posInSrc represents the amount of data the decoder would decode up to this point. 6476 * As long as the amount of data decoded is less than or equal to window size, offsets may be 6477 * larger than the total length of output decoded in order to reference the dict, even larger than 6478 * window size. After output surpasses windowSize, we're limited to windowSize offsets again. 6479 */ 6480 size_t const offsetBound = posInSrc > windowSize ? (size_t)windowSize : posInSrc + (size_t)dictSize; 6481 size_t const matchLenLowerBound = (minMatch == 3 || useSequenceProducer) ? 3 : 4; 6482 RETURN_ERROR_IF(offCode > OFFSET_TO_OFFBASE(offsetBound), externalSequences_invalid, "Offset too large!"); 6483 /* Validate maxNbSeq is large enough for the given matchLength and minMatch */ 6484 RETURN_ERROR_IF(matchLength < matchLenLowerBound, externalSequences_invalid, "Matchlength too small for the minMatch"); 6485 return 0; 6486 } 6487 6488 /* Returns an offset code, given a sequence's raw offset, the ongoing repcode array, and whether litLength == 0 */ 6489 static U32 ZSTD_finalizeOffBase(U32 rawOffset, const U32 rep[ZSTD_REP_NUM], U32 ll0) 6490 { 6491 U32 offBase = OFFSET_TO_OFFBASE(rawOffset); 6492 6493 if (!ll0 && rawOffset == rep[0]) { 6494 offBase = REPCODE1_TO_OFFBASE; 6495 } else if (rawOffset == rep[1]) { 6496 offBase = REPCODE_TO_OFFBASE(2 - ll0); 6497 } else if (rawOffset == rep[2]) { 6498 offBase = REPCODE_TO_OFFBASE(3 - ll0); 6499 } else if (ll0 && rawOffset == rep[0] - 1) { 6500 offBase = REPCODE3_TO_OFFBASE; 6501 } 6502 return offBase; 6503 } 6504 6505 size_t 6506 ZSTD_copySequencesToSeqStoreExplicitBlockDelim(ZSTD_CCtx* cctx, 6507 ZSTD_sequencePosition* seqPos, 6508 const ZSTD_Sequence* const inSeqs, size_t inSeqsSize, 6509 const void* src, size_t blockSize, 6510 ZSTD_paramSwitch_e externalRepSearch) 6511 { 6512 U32 idx = seqPos->idx; 6513 U32 const startIdx = idx; 6514 BYTE const* ip = (BYTE const*)(src); 6515 const BYTE* const iend = ip + blockSize; 6516 repcodes_t updatedRepcodes; 6517 U32 dictSize; 6518 6519 DEBUGLOG(5, "ZSTD_copySequencesToSeqStoreExplicitBlockDelim (blockSize = %zu)", blockSize); 6520 6521 if (cctx->cdict) { 6522 dictSize = (U32)cctx->cdict->dictContentSize; 6523 } else if (cctx->prefixDict.dict) { 6524 dictSize = (U32)cctx->prefixDict.dictSize; 6525 } else { 6526 dictSize = 0; 6527 } 6528 ZSTD_memcpy(updatedRepcodes.rep, cctx->blockState.prevCBlock->rep, sizeof(repcodes_t)); 6529 for (; idx < inSeqsSize && (inSeqs[idx].matchLength != 0 || inSeqs[idx].offset != 0); ++idx) { 6530 U32 const litLength = inSeqs[idx].litLength; 6531 U32 const matchLength = inSeqs[idx].matchLength; 6532 U32 offBase; 6533 6534 if (externalRepSearch == ZSTD_ps_disable) { 6535 offBase = OFFSET_TO_OFFBASE(inSeqs[idx].offset); 6536 } else { 6537 U32 const ll0 = (litLength == 0); 6538 offBase = ZSTD_finalizeOffBase(inSeqs[idx].offset, updatedRepcodes.rep, ll0); 6539 ZSTD_updateRep(updatedRepcodes.rep, offBase, ll0); 6540 } 6541 6542 DEBUGLOG(6, "Storing sequence: (of: %u, ml: %u, ll: %u)", offBase, matchLength, litLength); 6543 if (cctx->appliedParams.validateSequences) { 6544 seqPos->posInSrc += litLength + matchLength; 6545 FORWARD_IF_ERROR(ZSTD_validateSequence(offBase, matchLength, cctx->appliedParams.cParams.minMatch, seqPos->posInSrc, 6546 cctx->appliedParams.cParams.windowLog, dictSize, ZSTD_hasExtSeqProd(&cctx->appliedParams)), 6547 "Sequence validation failed"); 6548 } 6549 RETURN_ERROR_IF(idx - seqPos->idx >= cctx->seqStore.maxNbSeq, externalSequences_invalid, 6550 "Not enough memory allocated. Try adjusting ZSTD_c_minMatch."); 6551 ZSTD_storeSeq(&cctx->seqStore, litLength, ip, iend, offBase, matchLength); 6552 ip += matchLength + litLength; 6553 } 6554 6555 /* If we skipped repcode search while parsing, we need to update repcodes now */ 6556 assert(externalRepSearch != ZSTD_ps_auto); 6557 assert(idx >= startIdx); 6558 if (externalRepSearch == ZSTD_ps_disable && idx != startIdx) { 6559 U32* const rep = updatedRepcodes.rep; 6560 U32 lastSeqIdx = idx - 1; /* index of last non-block-delimiter sequence */ 6561 6562 if (lastSeqIdx >= startIdx + 2) { 6563 rep[2] = inSeqs[lastSeqIdx - 2].offset; 6564 rep[1] = inSeqs[lastSeqIdx - 1].offset; 6565 rep[0] = inSeqs[lastSeqIdx].offset; 6566 } else if (lastSeqIdx == startIdx + 1) { 6567 rep[2] = rep[0]; 6568 rep[1] = inSeqs[lastSeqIdx - 1].offset; 6569 rep[0] = inSeqs[lastSeqIdx].offset; 6570 } else { 6571 assert(lastSeqIdx == startIdx); 6572 rep[2] = rep[1]; 6573 rep[1] = rep[0]; 6574 rep[0] = inSeqs[lastSeqIdx].offset; 6575 } 6576 } 6577 6578 ZSTD_memcpy(cctx->blockState.nextCBlock->rep, updatedRepcodes.rep, sizeof(repcodes_t)); 6579 6580 if (inSeqs[idx].litLength) { 6581 DEBUGLOG(6, "Storing last literals of size: %u", inSeqs[idx].litLength); 6582 ZSTD_storeLastLiterals(&cctx->seqStore, ip, inSeqs[idx].litLength); 6583 ip += inSeqs[idx].litLength; 6584 seqPos->posInSrc += inSeqs[idx].litLength; 6585 } 6586 RETURN_ERROR_IF(ip != iend, externalSequences_invalid, "Blocksize doesn't agree with block delimiter!"); 6587 seqPos->idx = idx+1; 6588 return 0; 6589 } 6590 6591 size_t 6592 ZSTD_copySequencesToSeqStoreNoBlockDelim(ZSTD_CCtx* cctx, ZSTD_sequencePosition* seqPos, 6593 const ZSTD_Sequence* const inSeqs, size_t inSeqsSize, 6594 const void* src, size_t blockSize, ZSTD_paramSwitch_e externalRepSearch) 6595 { 6596 U32 idx = seqPos->idx; 6597 U32 startPosInSequence = seqPos->posInSequence; 6598 U32 endPosInSequence = seqPos->posInSequence + (U32)blockSize; 6599 size_t dictSize; 6600 BYTE const* ip = (BYTE const*)(src); 6601 BYTE const* iend = ip + blockSize; /* May be adjusted if we decide to process fewer than blockSize bytes */ 6602 repcodes_t updatedRepcodes; 6603 U32 bytesAdjustment = 0; 6604 U32 finalMatchSplit = 0; 6605 6606 /* TODO(embg) support fast parsing mode in noBlockDelim mode */ 6607 (void)externalRepSearch; 6608 6609 if (cctx->cdict) { 6610 dictSize = cctx->cdict->dictContentSize; 6611 } else if (cctx->prefixDict.dict) { 6612 dictSize = cctx->prefixDict.dictSize; 6613 } else { 6614 dictSize = 0; 6615 } 6616 DEBUGLOG(5, "ZSTD_copySequencesToSeqStoreNoBlockDelim: idx: %u PIS: %u blockSize: %zu", idx, startPosInSequence, blockSize); 6617 DEBUGLOG(5, "Start seq: idx: %u (of: %u ml: %u ll: %u)", idx, inSeqs[idx].offset, inSeqs[idx].matchLength, inSeqs[idx].litLength); 6618 ZSTD_memcpy(updatedRepcodes.rep, cctx->blockState.prevCBlock->rep, sizeof(repcodes_t)); 6619 while (endPosInSequence && idx < inSeqsSize && !finalMatchSplit) { 6620 const ZSTD_Sequence currSeq = inSeqs[idx]; 6621 U32 litLength = currSeq.litLength; 6622 U32 matchLength = currSeq.matchLength; 6623 U32 const rawOffset = currSeq.offset; 6624 U32 offBase; 6625 6626 /* Modify the sequence depending on where endPosInSequence lies */ 6627 if (endPosInSequence >= currSeq.litLength + currSeq.matchLength) { 6628 if (startPosInSequence >= litLength) { 6629 startPosInSequence -= litLength; 6630 litLength = 0; 6631 matchLength -= startPosInSequence; 6632 } else { 6633 litLength -= startPosInSequence; 6634 } 6635 /* Move to the next sequence */ 6636 endPosInSequence -= currSeq.litLength + currSeq.matchLength; 6637 startPosInSequence = 0; 6638 } else { 6639 /* This is the final (partial) sequence we're adding from inSeqs, and endPosInSequence 6640 does not reach the end of the match. So, we have to split the sequence */ 6641 DEBUGLOG(6, "Require a split: diff: %u, idx: %u PIS: %u", 6642 currSeq.litLength + currSeq.matchLength - endPosInSequence, idx, endPosInSequence); 6643 if (endPosInSequence > litLength) { 6644 U32 firstHalfMatchLength; 6645 litLength = startPosInSequence >= litLength ? 0 : litLength - startPosInSequence; 6646 firstHalfMatchLength = endPosInSequence - startPosInSequence - litLength; 6647 if (matchLength > blockSize && firstHalfMatchLength >= cctx->appliedParams.cParams.minMatch) { 6648 /* Only ever split the match if it is larger than the block size */ 6649 U32 secondHalfMatchLength = currSeq.matchLength + currSeq.litLength - endPosInSequence; 6650 if (secondHalfMatchLength < cctx->appliedParams.cParams.minMatch) { 6651 /* Move the endPosInSequence backward so that it creates match of minMatch length */ 6652 endPosInSequence -= cctx->appliedParams.cParams.minMatch - secondHalfMatchLength; 6653 bytesAdjustment = cctx->appliedParams.cParams.minMatch - secondHalfMatchLength; 6654 firstHalfMatchLength -= bytesAdjustment; 6655 } 6656 matchLength = firstHalfMatchLength; 6657 /* Flag that we split the last match - after storing the sequence, exit the loop, 6658 but keep the value of endPosInSequence */ 6659 finalMatchSplit = 1; 6660 } else { 6661 /* Move the position in sequence backwards so that we don't split match, and break to store 6662 * the last literals. We use the original currSeq.litLength as a marker for where endPosInSequence 6663 * should go. We prefer to do this whenever it is not necessary to split the match, or if doing so 6664 * would cause the first half of the match to be too small 6665 */ 6666 bytesAdjustment = endPosInSequence - currSeq.litLength; 6667 endPosInSequence = currSeq.litLength; 6668 break; 6669 } 6670 } else { 6671 /* This sequence ends inside the literals, break to store the last literals */ 6672 break; 6673 } 6674 } 6675 /* Check if this offset can be represented with a repcode */ 6676 { U32 const ll0 = (litLength == 0); 6677 offBase = ZSTD_finalizeOffBase(rawOffset, updatedRepcodes.rep, ll0); 6678 ZSTD_updateRep(updatedRepcodes.rep, offBase, ll0); 6679 } 6680 6681 if (cctx->appliedParams.validateSequences) { 6682 seqPos->posInSrc += litLength + matchLength; 6683 FORWARD_IF_ERROR(ZSTD_validateSequence(offBase, matchLength, cctx->appliedParams.cParams.minMatch, seqPos->posInSrc, 6684 cctx->appliedParams.cParams.windowLog, dictSize, ZSTD_hasExtSeqProd(&cctx->appliedParams)), 6685 "Sequence validation failed"); 6686 } 6687 DEBUGLOG(6, "Storing sequence: (of: %u, ml: %u, ll: %u)", offBase, matchLength, litLength); 6688 RETURN_ERROR_IF(idx - seqPos->idx >= cctx->seqStore.maxNbSeq, externalSequences_invalid, 6689 "Not enough memory allocated. Try adjusting ZSTD_c_minMatch."); 6690 ZSTD_storeSeq(&cctx->seqStore, litLength, ip, iend, offBase, matchLength); 6691 ip += matchLength + litLength; 6692 if (!finalMatchSplit) 6693 idx++; /* Next Sequence */ 6694 } 6695 DEBUGLOG(5, "Ending seq: idx: %u (of: %u ml: %u ll: %u)", idx, inSeqs[idx].offset, inSeqs[idx].matchLength, inSeqs[idx].litLength); 6696 assert(idx == inSeqsSize || endPosInSequence <= inSeqs[idx].litLength + inSeqs[idx].matchLength); 6697 seqPos->idx = idx; 6698 seqPos->posInSequence = endPosInSequence; 6699 ZSTD_memcpy(cctx->blockState.nextCBlock->rep, updatedRepcodes.rep, sizeof(repcodes_t)); 6700 6701 iend -= bytesAdjustment; 6702 if (ip != iend) { 6703 /* Store any last literals */ 6704 U32 lastLLSize = (U32)(iend - ip); 6705 assert(ip <= iend); 6706 DEBUGLOG(6, "Storing last literals of size: %u", lastLLSize); 6707 ZSTD_storeLastLiterals(&cctx->seqStore, ip, lastLLSize); 6708 seqPos->posInSrc += lastLLSize; 6709 } 6710 6711 return bytesAdjustment; 6712 } 6713 6714 typedef size_t (*ZSTD_sequenceCopier) (ZSTD_CCtx* cctx, ZSTD_sequencePosition* seqPos, 6715 const ZSTD_Sequence* const inSeqs, size_t inSeqsSize, 6716 const void* src, size_t blockSize, ZSTD_paramSwitch_e externalRepSearch); 6717 static ZSTD_sequenceCopier ZSTD_selectSequenceCopier(ZSTD_sequenceFormat_e mode) 6718 { 6719 ZSTD_sequenceCopier sequenceCopier = NULL; 6720 assert(ZSTD_cParam_withinBounds(ZSTD_c_blockDelimiters, mode)); 6721 if (mode == ZSTD_sf_explicitBlockDelimiters) { 6722 return ZSTD_copySequencesToSeqStoreExplicitBlockDelim; 6723 } else if (mode == ZSTD_sf_noBlockDelimiters) { 6724 return ZSTD_copySequencesToSeqStoreNoBlockDelim; 6725 } 6726 assert(sequenceCopier != NULL); 6727 return sequenceCopier; 6728 } 6729 6730 /* Discover the size of next block by searching for the delimiter. 6731 * Note that a block delimiter **must** exist in this mode, 6732 * otherwise it's an input error. 6733 * The block size retrieved will be later compared to ensure it remains within bounds */ 6734 static size_t 6735 blockSize_explicitDelimiter(const ZSTD_Sequence* inSeqs, size_t inSeqsSize, ZSTD_sequencePosition seqPos) 6736 { 6737 int end = 0; 6738 size_t blockSize = 0; 6739 size_t spos = seqPos.idx; 6740 DEBUGLOG(6, "blockSize_explicitDelimiter : seq %zu / %zu", spos, inSeqsSize); 6741 assert(spos <= inSeqsSize); 6742 while (spos < inSeqsSize) { 6743 end = (inSeqs[spos].offset == 0); 6744 blockSize += inSeqs[spos].litLength + inSeqs[spos].matchLength; 6745 if (end) { 6746 if (inSeqs[spos].matchLength != 0) 6747 RETURN_ERROR(externalSequences_invalid, "delimiter format error : both matchlength and offset must be == 0"); 6748 break; 6749 } 6750 spos++; 6751 } 6752 if (!end) 6753 RETURN_ERROR(externalSequences_invalid, "Reached end of sequences without finding a block delimiter"); 6754 return blockSize; 6755 } 6756 6757 /* More a "target" block size */ 6758 static size_t blockSize_noDelimiter(size_t blockSize, size_t remaining) 6759 { 6760 int const lastBlock = (remaining <= blockSize); 6761 return lastBlock ? remaining : blockSize; 6762 } 6763 6764 static size_t determine_blockSize(ZSTD_sequenceFormat_e mode, 6765 size_t blockSize, size_t remaining, 6766 const ZSTD_Sequence* inSeqs, size_t inSeqsSize, ZSTD_sequencePosition seqPos) 6767 { 6768 DEBUGLOG(6, "determine_blockSize : remainingSize = %zu", remaining); 6769 if (mode == ZSTD_sf_noBlockDelimiters) 6770 return blockSize_noDelimiter(blockSize, remaining); 6771 { size_t const explicitBlockSize = blockSize_explicitDelimiter(inSeqs, inSeqsSize, seqPos); 6772 FORWARD_IF_ERROR(explicitBlockSize, "Error while determining block size with explicit delimiters"); 6773 if (explicitBlockSize > blockSize) 6774 RETURN_ERROR(externalSequences_invalid, "sequences incorrectly define a too large block"); 6775 if (explicitBlockSize > remaining) 6776 RETURN_ERROR(externalSequences_invalid, "sequences define a frame longer than source"); 6777 return explicitBlockSize; 6778 } 6779 } 6780 6781 /* Compress, block-by-block, all of the sequences given. 6782 * 6783 * Returns the cumulative size of all compressed blocks (including their headers), 6784 * otherwise a ZSTD error. 6785 */ 6786 static size_t 6787 ZSTD_compressSequences_internal(ZSTD_CCtx* cctx, 6788 void* dst, size_t dstCapacity, 6789 const ZSTD_Sequence* inSeqs, size_t inSeqsSize, 6790 const void* src, size_t srcSize) 6791 { 6792 size_t cSize = 0; 6793 size_t remaining = srcSize; 6794 ZSTD_sequencePosition seqPos = {0, 0, 0}; 6795 6796 BYTE const* ip = (BYTE const*)src; 6797 BYTE* op = (BYTE*)dst; 6798 ZSTD_sequenceCopier const sequenceCopier = ZSTD_selectSequenceCopier(cctx->appliedParams.blockDelimiters); 6799 6800 DEBUGLOG(4, "ZSTD_compressSequences_internal srcSize: %zu, inSeqsSize: %zu", srcSize, inSeqsSize); 6801 /* Special case: empty frame */ 6802 if (remaining == 0) { 6803 U32 const cBlockHeader24 = 1 /* last block */ + (((U32)bt_raw)<<1); 6804 RETURN_ERROR_IF(dstCapacity<4, dstSize_tooSmall, "No room for empty frame block header"); 6805 MEM_writeLE32(op, cBlockHeader24); 6806 op += ZSTD_blockHeaderSize; 6807 dstCapacity -= ZSTD_blockHeaderSize; 6808 cSize += ZSTD_blockHeaderSize; 6809 } 6810 6811 while (remaining) { 6812 size_t compressedSeqsSize; 6813 size_t cBlockSize; 6814 size_t additionalByteAdjustment; 6815 size_t blockSize = determine_blockSize(cctx->appliedParams.blockDelimiters, 6816 cctx->blockSize, remaining, 6817 inSeqs, inSeqsSize, seqPos); 6818 U32 const lastBlock = (blockSize == remaining); 6819 FORWARD_IF_ERROR(blockSize, "Error while trying to determine block size"); 6820 assert(blockSize <= remaining); 6821 ZSTD_resetSeqStore(&cctx->seqStore); 6822 DEBUGLOG(5, "Working on new block. Blocksize: %zu (total:%zu)", blockSize, (ip - (const BYTE*)src) + blockSize); 6823 6824 additionalByteAdjustment = sequenceCopier(cctx, &seqPos, inSeqs, inSeqsSize, ip, blockSize, cctx->appliedParams.searchForExternalRepcodes); 6825 FORWARD_IF_ERROR(additionalByteAdjustment, "Bad sequence copy"); 6826 blockSize -= additionalByteAdjustment; 6827 6828 /* If blocks are too small, emit as a nocompress block */ 6829 /* TODO: See 3090. We reduced MIN_CBLOCK_SIZE from 3 to 2 so to compensate we are adding 6830 * additional 1. We need to revisit and change this logic to be more consistent */ 6831 if (blockSize < MIN_CBLOCK_SIZE+ZSTD_blockHeaderSize+1+1) { 6832 cBlockSize = ZSTD_noCompressBlock(op, dstCapacity, ip, blockSize, lastBlock); 6833 FORWARD_IF_ERROR(cBlockSize, "Nocompress block failed"); 6834 DEBUGLOG(5, "Block too small, writing out nocompress block: cSize: %zu", cBlockSize); 6835 cSize += cBlockSize; 6836 ip += blockSize; 6837 op += cBlockSize; 6838 remaining -= blockSize; 6839 dstCapacity -= cBlockSize; 6840 continue; 6841 } 6842 6843 RETURN_ERROR_IF(dstCapacity < ZSTD_blockHeaderSize, dstSize_tooSmall, "not enough dstCapacity to write a new compressed block"); 6844 compressedSeqsSize = ZSTD_entropyCompressSeqStore(&cctx->seqStore, 6845 &cctx->blockState.prevCBlock->entropy, &cctx->blockState.nextCBlock->entropy, 6846 &cctx->appliedParams, 6847 op + ZSTD_blockHeaderSize /* Leave space for block header */, dstCapacity - ZSTD_blockHeaderSize, 6848 blockSize, 6849 cctx->entropyWorkspace, ENTROPY_WORKSPACE_SIZE /* statically allocated in resetCCtx */, 6850 cctx->bmi2); 6851 FORWARD_IF_ERROR(compressedSeqsSize, "Compressing sequences of block failed"); 6852 DEBUGLOG(5, "Compressed sequences size: %zu", compressedSeqsSize); 6853 6854 if (!cctx->isFirstBlock && 6855 ZSTD_maybeRLE(&cctx->seqStore) && 6856 ZSTD_isRLE(ip, blockSize)) { 6857 /* We don't want to emit our first block as a RLE even if it qualifies because 6858 * doing so will cause the decoder (cli only) to throw a "should consume all input error." 6859 * This is only an issue for zstd <= v1.4.3 6860 */ 6861 compressedSeqsSize = 1; 6862 } 6863 6864 if (compressedSeqsSize == 0) { 6865 /* ZSTD_noCompressBlock writes the block header as well */ 6866 cBlockSize = ZSTD_noCompressBlock(op, dstCapacity, ip, blockSize, lastBlock); 6867 FORWARD_IF_ERROR(cBlockSize, "ZSTD_noCompressBlock failed"); 6868 DEBUGLOG(5, "Writing out nocompress block, size: %zu", cBlockSize); 6869 } else if (compressedSeqsSize == 1) { 6870 cBlockSize = ZSTD_rleCompressBlock(op, dstCapacity, *ip, blockSize, lastBlock); 6871 FORWARD_IF_ERROR(cBlockSize, "ZSTD_rleCompressBlock failed"); 6872 DEBUGLOG(5, "Writing out RLE block, size: %zu", cBlockSize); 6873 } else { 6874 U32 cBlockHeader; 6875 /* Error checking and repcodes update */ 6876 ZSTD_blockState_confirmRepcodesAndEntropyTables(&cctx->blockState); 6877 if (cctx->blockState.prevCBlock->entropy.fse.offcode_repeatMode == FSE_repeat_valid) 6878 cctx->blockState.prevCBlock->entropy.fse.offcode_repeatMode = FSE_repeat_check; 6879 6880 /* Write block header into beginning of block*/ 6881 cBlockHeader = lastBlock + (((U32)bt_compressed)<<1) + (U32)(compressedSeqsSize << 3); 6882 MEM_writeLE24(op, cBlockHeader); 6883 cBlockSize = ZSTD_blockHeaderSize + compressedSeqsSize; 6884 DEBUGLOG(5, "Writing out compressed block, size: %zu", cBlockSize); 6885 } 6886 6887 cSize += cBlockSize; 6888 6889 if (lastBlock) { 6890 break; 6891 } else { 6892 ip += blockSize; 6893 op += cBlockSize; 6894 remaining -= blockSize; 6895 dstCapacity -= cBlockSize; 6896 cctx->isFirstBlock = 0; 6897 } 6898 DEBUGLOG(5, "cSize running total: %zu (remaining dstCapacity=%zu)", cSize, dstCapacity); 6899 } 6900 6901 DEBUGLOG(4, "cSize final total: %zu", cSize); 6902 return cSize; 6903 } 6904 6905 size_t ZSTD_compressSequences(ZSTD_CCtx* cctx, 6906 void* dst, size_t dstCapacity, 6907 const ZSTD_Sequence* inSeqs, size_t inSeqsSize, 6908 const void* src, size_t srcSize) 6909 { 6910 BYTE* op = (BYTE*)dst; 6911 size_t cSize = 0; 6912 size_t compressedBlocksSize = 0; 6913 size_t frameHeaderSize = 0; 6914 6915 /* Transparent initialization stage, same as compressStream2() */ 6916 DEBUGLOG(4, "ZSTD_compressSequences (dstCapacity=%zu)", dstCapacity); 6917 assert(cctx != NULL); 6918 FORWARD_IF_ERROR(ZSTD_CCtx_init_compressStream2(cctx, ZSTD_e_end, srcSize), "CCtx initialization failed"); 6919 /* Begin writing output, starting with frame header */ 6920 frameHeaderSize = ZSTD_writeFrameHeader(op, dstCapacity, &cctx->appliedParams, srcSize, cctx->dictID); 6921 op += frameHeaderSize; 6922 dstCapacity -= frameHeaderSize; 6923 cSize += frameHeaderSize; 6924 if (cctx->appliedParams.fParams.checksumFlag && srcSize) { 6925 XXH64_update(&cctx->xxhState, src, srcSize); 6926 } 6927 /* cSize includes block header size and compressed sequences size */ 6928 compressedBlocksSize = ZSTD_compressSequences_internal(cctx, 6929 op, dstCapacity, 6930 inSeqs, inSeqsSize, 6931 src, srcSize); 6932 FORWARD_IF_ERROR(compressedBlocksSize, "Compressing blocks failed!"); 6933 cSize += compressedBlocksSize; 6934 dstCapacity -= compressedBlocksSize; 6935 6936 if (cctx->appliedParams.fParams.checksumFlag) { 6937 U32 const checksum = (U32) XXH64_digest(&cctx->xxhState); 6938 RETURN_ERROR_IF(dstCapacity<4, dstSize_tooSmall, "no room for checksum"); 6939 DEBUGLOG(4, "Write checksum : %08X", (unsigned)checksum); 6940 MEM_writeLE32((char*)dst + cSize, checksum); 6941 cSize += 4; 6942 } 6943 6944 DEBUGLOG(4, "Final compressed size: %zu", cSize); 6945 return cSize; 6946 } 6947 6948 /*====== Finalize ======*/ 6949 6950 static ZSTD_inBuffer inBuffer_forEndFlush(const ZSTD_CStream* zcs) 6951 { 6952 const ZSTD_inBuffer nullInput = { NULL, 0, 0 }; 6953 const int stableInput = (zcs->appliedParams.inBufferMode == ZSTD_bm_stable); 6954 return stableInput ? zcs->expectedInBuffer : nullInput; 6955 } 6956 6957 /*! ZSTD_flushStream() : 6958 * @return : amount of data remaining to flush */ 6959 size_t ZSTD_flushStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output) 6960 { 6961 ZSTD_inBuffer input = inBuffer_forEndFlush(zcs); 6962 input.size = input.pos; /* do not ingest more input during flush */ 6963 return ZSTD_compressStream2(zcs, output, &input, ZSTD_e_flush); 6964 } 6965 6966 6967 size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output) 6968 { 6969 ZSTD_inBuffer input = inBuffer_forEndFlush(zcs); 6970 size_t const remainingToFlush = ZSTD_compressStream2(zcs, output, &input, ZSTD_e_end); 6971 FORWARD_IF_ERROR(remainingToFlush , "ZSTD_compressStream2(,,ZSTD_e_end) failed"); 6972 if (zcs->appliedParams.nbWorkers > 0) return remainingToFlush; /* minimal estimation */ 6973 /* single thread mode : attempt to calculate remaining to flush more precisely */ 6974 { size_t const lastBlockSize = zcs->frameEnded ? 0 : ZSTD_BLOCKHEADERSIZE; 6975 size_t const checksumSize = (size_t)(zcs->frameEnded ? 0 : zcs->appliedParams.fParams.checksumFlag * 4); 6976 size_t const toFlush = remainingToFlush + lastBlockSize + checksumSize; 6977 DEBUGLOG(4, "ZSTD_endStream : remaining to flush : %u", (unsigned)toFlush); 6978 return toFlush; 6979 } 6980 } 6981 6982 6983 /*-===== Pre-defined compression levels =====-*/ 6984 #include "clevels.h" 6985 6986 int ZSTD_maxCLevel(void) { return ZSTD_MAX_CLEVEL; } 6987 int ZSTD_minCLevel(void) { return (int)-ZSTD_TARGETLENGTH_MAX; } 6988 int ZSTD_defaultCLevel(void) { return ZSTD_CLEVEL_DEFAULT; } 6989 6990 static ZSTD_compressionParameters ZSTD_dedicatedDictSearch_getCParams(int const compressionLevel, size_t const dictSize) 6991 { 6992 ZSTD_compressionParameters cParams = ZSTD_getCParams_internal(compressionLevel, 0, dictSize, ZSTD_cpm_createCDict); 6993 switch (cParams.strategy) { 6994 case ZSTD_fast: 6995 case ZSTD_dfast: 6996 break; 6997 case ZSTD_greedy: 6998 case ZSTD_lazy: 6999 case ZSTD_lazy2: 7000 cParams.hashLog += ZSTD_LAZY_DDSS_BUCKET_LOG; 7001 break; 7002 case ZSTD_btlazy2: 7003 case ZSTD_btopt: 7004 case ZSTD_btultra: 7005 case ZSTD_btultra2: 7006 break; 7007 } 7008 return cParams; 7009 } 7010 7011 static int ZSTD_dedicatedDictSearch_isSupported( 7012 ZSTD_compressionParameters const* cParams) 7013 { 7014 return (cParams->strategy >= ZSTD_greedy) 7015 && (cParams->strategy <= ZSTD_lazy2) 7016 && (cParams->hashLog > cParams->chainLog) 7017 && (cParams->chainLog <= 24); 7018 } 7019 7020 /** 7021 * Reverses the adjustment applied to cparams when enabling dedicated dict 7022 * search. This is used to recover the params set to be used in the working 7023 * context. (Otherwise, those tables would also grow.) 7024 */ 7025 static void ZSTD_dedicatedDictSearch_revertCParams( 7026 ZSTD_compressionParameters* cParams) { 7027 switch (cParams->strategy) { 7028 case ZSTD_fast: 7029 case ZSTD_dfast: 7030 break; 7031 case ZSTD_greedy: 7032 case ZSTD_lazy: 7033 case ZSTD_lazy2: 7034 cParams->hashLog -= ZSTD_LAZY_DDSS_BUCKET_LOG; 7035 if (cParams->hashLog < ZSTD_HASHLOG_MIN) { 7036 cParams->hashLog = ZSTD_HASHLOG_MIN; 7037 } 7038 break; 7039 case ZSTD_btlazy2: 7040 case ZSTD_btopt: 7041 case ZSTD_btultra: 7042 case ZSTD_btultra2: 7043 break; 7044 } 7045 } 7046 7047 static U64 ZSTD_getCParamRowSize(U64 srcSizeHint, size_t dictSize, ZSTD_cParamMode_e mode) 7048 { 7049 switch (mode) { 7050 case ZSTD_cpm_unknown: 7051 case ZSTD_cpm_noAttachDict: 7052 case ZSTD_cpm_createCDict: 7053 break; 7054 case ZSTD_cpm_attachDict: 7055 dictSize = 0; 7056 break; 7057 default: 7058 assert(0); 7059 break; 7060 } 7061 { int const unknown = srcSizeHint == ZSTD_CONTENTSIZE_UNKNOWN; 7062 size_t const addedSize = unknown && dictSize > 0 ? 500 : 0; 7063 return unknown && dictSize == 0 ? ZSTD_CONTENTSIZE_UNKNOWN : srcSizeHint+dictSize+addedSize; 7064 } 7065 } 7066 7067 /*! ZSTD_getCParams_internal() : 7068 * @return ZSTD_compressionParameters structure for a selected compression level, srcSize and dictSize. 7069 * Note: srcSizeHint 0 means 0, use ZSTD_CONTENTSIZE_UNKNOWN for unknown. 7070 * Use dictSize == 0 for unknown or unused. 7071 * Note: `mode` controls how we treat the `dictSize`. See docs for `ZSTD_cParamMode_e`. */ 7072 static ZSTD_compressionParameters ZSTD_getCParams_internal(int compressionLevel, unsigned long long srcSizeHint, size_t dictSize, ZSTD_cParamMode_e mode) 7073 { 7074 U64 const rSize = ZSTD_getCParamRowSize(srcSizeHint, dictSize, mode); 7075 U32 const tableID = (rSize <= 256 KB) + (rSize <= 128 KB) + (rSize <= 16 KB); 7076 int row; 7077 DEBUGLOG(5, "ZSTD_getCParams_internal (cLevel=%i)", compressionLevel); 7078 7079 /* row */ 7080 if (compressionLevel == 0) row = ZSTD_CLEVEL_DEFAULT; /* 0 == default */ 7081 else if (compressionLevel < 0) row = 0; /* entry 0 is baseline for fast mode */ 7082 else if (compressionLevel > ZSTD_MAX_CLEVEL) row = ZSTD_MAX_CLEVEL; 7083 else row = compressionLevel; 7084 7085 { ZSTD_compressionParameters cp = ZSTD_defaultCParameters[tableID][row]; 7086 DEBUGLOG(5, "ZSTD_getCParams_internal selected tableID: %u row: %u strat: %u", tableID, row, (U32)cp.strategy); 7087 /* acceleration factor */ 7088 if (compressionLevel < 0) { 7089 int const clampedCompressionLevel = MAX(ZSTD_minCLevel(), compressionLevel); 7090 cp.targetLength = (unsigned)(-clampedCompressionLevel); 7091 } 7092 /* refine parameters based on srcSize & dictSize */ 7093 return ZSTD_adjustCParams_internal(cp, srcSizeHint, dictSize, mode, ZSTD_ps_auto); 7094 } 7095 } 7096 7097 /*! ZSTD_getCParams() : 7098 * @return ZSTD_compressionParameters structure for a selected compression level, srcSize and dictSize. 7099 * Size values are optional, provide 0 if not known or unused */ 7100 ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, unsigned long long srcSizeHint, size_t dictSize) 7101 { 7102 if (srcSizeHint == 0) srcSizeHint = ZSTD_CONTENTSIZE_UNKNOWN; 7103 return ZSTD_getCParams_internal(compressionLevel, srcSizeHint, dictSize, ZSTD_cpm_unknown); 7104 } 7105 7106 /*! ZSTD_getParams() : 7107 * same idea as ZSTD_getCParams() 7108 * @return a `ZSTD_parameters` structure (instead of `ZSTD_compressionParameters`). 7109 * Fields of `ZSTD_frameParameters` are set to default values */ 7110 static ZSTD_parameters ZSTD_getParams_internal(int compressionLevel, unsigned long long srcSizeHint, size_t dictSize, ZSTD_cParamMode_e mode) { 7111 ZSTD_parameters params; 7112 ZSTD_compressionParameters const cParams = ZSTD_getCParams_internal(compressionLevel, srcSizeHint, dictSize, mode); 7113 DEBUGLOG(5, "ZSTD_getParams (cLevel=%i)", compressionLevel); 7114 ZSTD_memset(¶ms, 0, sizeof(params)); 7115 params.cParams = cParams; 7116 params.fParams.contentSizeFlag = 1; 7117 return params; 7118 } 7119 7120 /*! ZSTD_getParams() : 7121 * same idea as ZSTD_getCParams() 7122 * @return a `ZSTD_parameters` structure (instead of `ZSTD_compressionParameters`). 7123 * Fields of `ZSTD_frameParameters` are set to default values */ 7124 ZSTD_parameters ZSTD_getParams(int compressionLevel, unsigned long long srcSizeHint, size_t dictSize) { 7125 if (srcSizeHint == 0) srcSizeHint = ZSTD_CONTENTSIZE_UNKNOWN; 7126 return ZSTD_getParams_internal(compressionLevel, srcSizeHint, dictSize, ZSTD_cpm_unknown); 7127 } 7128 7129 void ZSTD_registerSequenceProducer( 7130 ZSTD_CCtx* zc, 7131 void* extSeqProdState, 7132 ZSTD_sequenceProducer_F extSeqProdFunc 7133 ) { 7134 assert(zc != NULL); 7135 ZSTD_CCtxParams_registerSequenceProducer( 7136 &zc->requestedParams, extSeqProdState, extSeqProdFunc 7137 ); 7138 } 7139 7140 void ZSTD_CCtxParams_registerSequenceProducer( 7141 ZSTD_CCtx_params* params, 7142 void* extSeqProdState, 7143 ZSTD_sequenceProducer_F extSeqProdFunc 7144 ) { 7145 assert(params != NULL); 7146 if (extSeqProdFunc != NULL) { 7147 params->extSeqProdFunc = extSeqProdFunc; 7148 params->extSeqProdState = extSeqProdState; 7149 } else { 7150 params->extSeqProdFunc = NULL; 7151 params->extSeqProdState = NULL; 7152 } 7153 } 7154
Indexes created Tue Feb 24 08:35:24 UTC 2026