1 /* 2 * Copyright (c) Meta Platforms, Inc. and affiliates. 3 * All rights reserved. 4 * 5 * This source code is licensed under both the BSD-style license (found in the 6 * LICENSE file in the root directory of this source tree) and the GPLv2 (found 7 * in the COPYING file in the root directory of this source tree). 8 * You may select, at your option, one of the above-listed licenses. 9 */ 10 11 12 /*-************************************ 13 * Compiler specific 14 **************************************/ 15 #ifdef _MSC_VER /* Visual Studio */ 16 # define _CRT_SECURE_NO_WARNINGS /* fgets */ 17 # pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ 18 # pragma warning(disable : 4204) /* disable: C4204: non-constant aggregate initializer */ 19 #endif 20 21 22 /*-************************************ 23 * Includes 24 **************************************/ 25 #include <stdlib.h> /* free */ 26 #include <stdio.h> /* fgets, sscanf */ 27 #include <string.h> /* strcmp */ 28 #include <time.h> /* time(), time_t */ 29 #undef NDEBUG /* always enable assert() */ 30 #include <assert.h> 31 #define ZSTD_STATIC_LINKING_ONLY /* ZSTD_compressContinue, ZSTD_compressBlock */ 32 #include "debug.h" /* DEBUG_STATIC_ASSERT */ 33 #include "fse.h" 34 #define ZSTD_DISABLE_DEPRECATE_WARNINGS /* No deprecation warnings, we still test some deprecated functions */ 35 #include "zstd.h" /* ZSTD_VERSION_STRING */ 36 #include "zstd_errors.h" /* ZSTD_getErrorCode */ 37 #define ZDICT_STATIC_LINKING_ONLY 38 #include "zdict.h" /* ZDICT_trainFromBuffer */ 39 #include "mem.h" 40 #include "datagen.h" /* RDG_genBuffer */ 41 #define XXH_STATIC_LINKING_ONLY /* XXH64_state_t */ 42 #include "xxhash.h" /* XXH64 */ 43 #include "timefn.h" /* SEC_TO_MICRO, UTIL_time_t, UTIL_TIME_INITIALIZER, UTIL_clockSpanMicro, UTIL_getTime */ 44 /* must be included after util.h, due to ERROR macro redefinition issue on Visual Studio */ 45 #include "zstd_internal.h" /* ZSTD_WORKSPACETOOLARGE_MAXDURATION, ZSTD_WORKSPACETOOLARGE_FACTOR, KB, MB */ 46 #include "threading.h" /* ZSTD_pthread_create, ZSTD_pthread_join */ 47 48 49 /*-************************************ 50 * Constants 51 **************************************/ 52 #define GB *(1U<<30) 53 54 static const int FUZ_compressibility_default = 50; 55 static const int nbTestsDefault = 30000; 56 57 58 /*-************************************ 59 * Display Macros 60 **************************************/ 61 #define DISPLAY(...) fprintf(stderr, __VA_ARGS__) 62 #define DISPLAYLEVEL(l, ...) if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); } 63 static U32 g_displayLevel = 2; 64 65 static const U64 g_refreshRate = SEC_TO_MICRO / 6; 66 static UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER; 67 68 #define DISPLAYUPDATE(l, ...) \ 69 if (g_displayLevel>=l) { \ 70 if ((UTIL_clockSpanMicro(g_displayClock) > g_refreshRate) || (g_displayLevel>=4)) \ 71 { g_displayClock = UTIL_getTime(); DISPLAY(__VA_ARGS__); \ 72 if (g_displayLevel>=4) fflush(stderr); } \ 73 } 74 75 76 /*-******************************************************* 77 * Compile time test 78 *********************************************************/ 79 #undef MIN 80 #undef MAX 81 /* Declaring the function, to avoid -Wmissing-prototype */ 82 void FUZ_bug976(void); 83 void FUZ_bug976(void) 84 { /* these constants shall not depend on MIN() macro */ 85 DEBUG_STATIC_ASSERT(ZSTD_HASHLOG_MAX < 31); 86 DEBUG_STATIC_ASSERT(ZSTD_CHAINLOG_MAX < 31); 87 } 88 89 90 /*-******************************************************* 91 * Internal functions 92 *********************************************************/ 93 #define MIN(a,b) ((a)<(b)?(a):(b)) 94 #define MAX(a,b) ((a)>(b)?(a):(b)) 95 96 #define FUZ_rotl32(x,r) ((x << r) | (x >> (32 - r))) 97 static U32 FUZ_rand(U32* src) 98 { 99 static const U32 prime1 = 2654435761U; 100 static const U32 prime2 = 2246822519U; 101 U32 rand32 = *src; 102 rand32 *= prime1; 103 rand32 += prime2; 104 rand32 = FUZ_rotl32(rand32, 13); 105 *src = rand32; 106 return rand32 >> 5; 107 } 108 109 static U32 FUZ_highbit32(U32 v32) 110 { 111 unsigned nbBits = 0; 112 if (v32==0) return 0; 113 while (v32) v32 >>= 1, nbBits++; 114 return nbBits; 115 } 116 117 118 /*============================================= 119 * Test macros 120 =============================================*/ 121 #define CHECK(fn) { if(!(fn)) { DISPLAYLEVEL(1, "Error : test (%s) failed \n", #fn); exit(1); } } 122 123 #define CHECK_Z(f) { \ 124 size_t const err = f; \ 125 if (ZSTD_isError(err)) { \ 126 DISPLAY("Error => %s : %s ", \ 127 #f, ZSTD_getErrorName(err)); \ 128 exit(1); \ 129 } } 130 131 #define CHECK_VAR(var, fn) var = fn; if (ZSTD_isError(var)) { DISPLAYLEVEL(1, "%s : fails : %s \n", #fn, ZSTD_getErrorName(var)); exit(1); } 132 #define CHECK_NEWV(var, fn) size_t const CHECK_VAR(var, fn) 133 #define CHECKPLUS(var, fn, more) { CHECK_NEWV(var, fn); more; } 134 135 #define CHECK_OP(op, lhs, rhs) { \ 136 if (!((lhs) op (rhs))) { \ 137 DISPLAY("Error L%u => FAILED %s %s %s ", __LINE__, #lhs, #op, #rhs); \ 138 exit(1); \ 139 } \ 140 } 141 #define CHECK_EQ(lhs, rhs) CHECK_OP(==, lhs, rhs) 142 #define CHECK_LT(lhs, rhs) CHECK_OP(<, lhs, rhs) 143 144 145 /*============================================= 146 * Memory Tests 147 =============================================*/ 148 #if defined(__APPLE__) && defined(__MACH__) 149 150 #include <malloc/malloc.h> /* malloc_size */ 151 152 typedef struct { 153 unsigned long long totalMalloc; 154 size_t currentMalloc; 155 size_t peakMalloc; 156 unsigned nbMalloc; 157 unsigned nbFree; 158 } mallocCounter_t; 159 160 static const mallocCounter_t INIT_MALLOC_COUNTER = { 0, 0, 0, 0, 0 }; 161 162 static void* FUZ_mallocDebug(void* counter, size_t size) 163 { 164 mallocCounter_t* const mcPtr = (mallocCounter_t*)counter; 165 void* const ptr = malloc(size); 166 if (ptr==NULL) return NULL; 167 DISPLAYLEVEL(4, "allocating %u KB => effectively %u KB \n", 168 (unsigned)(size >> 10), (unsigned)(malloc_size(ptr) >> 10)); /* OS-X specific */ 169 mcPtr->totalMalloc += size; 170 mcPtr->currentMalloc += size; 171 if (mcPtr->currentMalloc > mcPtr->peakMalloc) 172 mcPtr->peakMalloc = mcPtr->currentMalloc; 173 mcPtr->nbMalloc += 1; 174 return ptr; 175 } 176 177 static void FUZ_freeDebug(void* counter, void* address) 178 { 179 mallocCounter_t* const mcPtr = (mallocCounter_t*)counter; 180 DISPLAYLEVEL(4, "freeing %u KB \n", (unsigned)(malloc_size(address) >> 10)); 181 mcPtr->nbFree += 1; 182 mcPtr->currentMalloc -= malloc_size(address); /* OS-X specific */ 183 free(address); 184 } 185 186 static void FUZ_displayMallocStats(mallocCounter_t count) 187 { 188 DISPLAYLEVEL(3, "peak:%6u KB, nbMallocs:%2u, total:%6u KB \n", 189 (unsigned)(count.peakMalloc >> 10), 190 count.nbMalloc, 191 (unsigned)(count.totalMalloc >> 10)); 192 } 193 194 static int FUZ_mallocTests_internal(unsigned seed, double compressibility, unsigned part, 195 void* inBuffer, size_t inSize, void* outBuffer, size_t outSize) 196 { 197 /* test only played in verbose mode, as they are long */ 198 if (g_displayLevel<3) return 0; 199 200 /* Create compressible noise */ 201 if (!inBuffer || !outBuffer) { 202 DISPLAY("Not enough memory, aborting\n"); 203 exit(1); 204 } 205 RDG_genBuffer(inBuffer, inSize, compressibility, 0. /*auto*/, seed); 206 207 /* simple compression tests */ 208 if (part <= 1) 209 { int compressionLevel; 210 for (compressionLevel=1; compressionLevel<=6; compressionLevel++) { 211 mallocCounter_t malcount = INIT_MALLOC_COUNTER; 212 ZSTD_customMem const cMem = { FUZ_mallocDebug, FUZ_freeDebug, &malcount }; 213 ZSTD_CCtx* const cctx = ZSTD_createCCtx_advanced(cMem); 214 CHECK_Z( ZSTD_compressCCtx(cctx, outBuffer, outSize, inBuffer, inSize, compressionLevel) ); 215 ZSTD_freeCCtx(cctx); 216 DISPLAYLEVEL(3, "compressCCtx level %i : ", compressionLevel); 217 FUZ_displayMallocStats(malcount); 218 } } 219 220 /* streaming compression tests */ 221 if (part <= 2) 222 { int compressionLevel; 223 for (compressionLevel=1; compressionLevel<=6; compressionLevel++) { 224 mallocCounter_t malcount = INIT_MALLOC_COUNTER; 225 ZSTD_customMem const cMem = { FUZ_mallocDebug, FUZ_freeDebug, &malcount }; 226 ZSTD_CCtx* const cstream = ZSTD_createCStream_advanced(cMem); 227 ZSTD_outBuffer out = { outBuffer, outSize, 0 }; 228 ZSTD_inBuffer in = { inBuffer, inSize, 0 }; 229 CHECK_Z( ZSTD_initCStream(cstream, compressionLevel) ); 230 CHECK_Z( ZSTD_compressStream(cstream, &out, &in) ); 231 CHECK_Z( ZSTD_endStream(cstream, &out) ); 232 ZSTD_freeCStream(cstream); 233 DISPLAYLEVEL(3, "compressStream level %i : ", compressionLevel); 234 FUZ_displayMallocStats(malcount); 235 } } 236 237 /* advanced MT API test */ 238 if (part <= 3) 239 { int nbThreads; 240 for (nbThreads=1; nbThreads<=4; nbThreads++) { 241 int compressionLevel; 242 for (compressionLevel=1; compressionLevel<=6; compressionLevel++) { 243 mallocCounter_t malcount = INIT_MALLOC_COUNTER; 244 ZSTD_customMem const cMem = { FUZ_mallocDebug, FUZ_freeDebug, &malcount }; 245 ZSTD_CCtx* const cctx = ZSTD_createCCtx_advanced(cMem); 246 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, compressionLevel) ); 247 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, nbThreads) ); 248 CHECK_Z( ZSTD_compress2(cctx, outBuffer, outSize, inBuffer, inSize) ); 249 ZSTD_freeCCtx(cctx); 250 DISPLAYLEVEL(3, "compress_generic,-T%i,end level %i : ", 251 nbThreads, compressionLevel); 252 FUZ_displayMallocStats(malcount); 253 } } } 254 255 /* advanced MT streaming API test */ 256 if (part <= 4) 257 { int nbThreads; 258 for (nbThreads=1; nbThreads<=4; nbThreads++) { 259 int compressionLevel; 260 for (compressionLevel=1; compressionLevel<=6; compressionLevel++) { 261 mallocCounter_t malcount = INIT_MALLOC_COUNTER; 262 ZSTD_customMem const cMem = { FUZ_mallocDebug, FUZ_freeDebug, &malcount }; 263 ZSTD_CCtx* const cctx = ZSTD_createCCtx_advanced(cMem); 264 ZSTD_outBuffer out = { outBuffer, outSize, 0 }; 265 ZSTD_inBuffer in = { inBuffer, inSize, 0 }; 266 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, compressionLevel) ); 267 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, nbThreads) ); 268 CHECK_Z( ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_continue) ); 269 while ( ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end) ) {} 270 ZSTD_freeCCtx(cctx); 271 DISPLAYLEVEL(3, "compress_generic,-T%i,continue level %i : ", 272 nbThreads, compressionLevel); 273 FUZ_displayMallocStats(malcount); 274 } } } 275 276 return 0; 277 } 278 279 static int FUZ_mallocTests(unsigned seed, double compressibility, unsigned part) 280 { 281 size_t const inSize = 64 MB + 16 MB + 4 MB + 1 MB + 256 KB + 64 KB; /* 85.3 MB */ 282 size_t const outSize = ZSTD_compressBound(inSize); 283 void* const inBuffer = malloc(inSize); 284 void* const outBuffer = malloc(outSize); 285 int result; 286 287 /* Create compressible noise */ 288 if (!inBuffer || !outBuffer) { 289 DISPLAY("Not enough memory, aborting \n"); 290 exit(1); 291 } 292 293 result = FUZ_mallocTests_internal(seed, compressibility, part, 294 inBuffer, inSize, outBuffer, outSize); 295 296 free(inBuffer); 297 free(outBuffer); 298 return result; 299 } 300 301 #else 302 303 static int FUZ_mallocTests(unsigned seed, double compressibility, unsigned part) 304 { 305 (void)seed; (void)compressibility; (void)part; 306 return 0; 307 } 308 309 #endif 310 311 static void FUZ_decodeSequences(BYTE* dst, ZSTD_Sequence* seqs, size_t seqsSize, 312 BYTE* src, size_t size, ZSTD_SequenceFormat_e format) 313 { 314 size_t i; 315 size_t j; 316 for(i = 0; i < seqsSize; ++i) { 317 assert(dst + seqs[i].litLength + seqs[i].matchLength <= dst + size); 318 assert(src + seqs[i].litLength + seqs[i].matchLength <= src + size); 319 if (format == ZSTD_sf_noBlockDelimiters) { 320 assert(seqs[i].matchLength != 0 || seqs[i].offset != 0); 321 } 322 323 memcpy(dst, src, seqs[i].litLength); 324 dst += seqs[i].litLength; 325 src += seqs[i].litLength; 326 size -= seqs[i].litLength; 327 328 if (seqs[i].offset != 0) { 329 for (j = 0; j < seqs[i].matchLength; ++j) 330 dst[j] = dst[(ptrdiff_t)(j - seqs[i].offset)]; 331 dst += seqs[i].matchLength; 332 src += seqs[i].matchLength; 333 size -= seqs[i].matchLength; 334 } 335 } 336 if (format == ZSTD_sf_noBlockDelimiters) { 337 memcpy(dst, src, size); 338 } 339 } 340 341 static size_t FUZ_getLitSize(const ZSTD_Sequence* seqs, size_t nbSeqs) 342 { 343 size_t n, litSize = 0; 344 assert(seqs != NULL); 345 for (n=0; n<nbSeqs; n++) { 346 litSize += seqs[n].litLength; 347 } 348 return litSize; 349 } 350 351 static void 352 FUZ_transferLiterals(void* dst, size_t dstCapacity, 353 const void* src, size_t srcSize, 354 const ZSTD_Sequence* seqs, size_t nbSeqs) 355 { 356 size_t n; 357 const char* ip = (const char*)src; 358 char* op = (char*)dst; 359 size_t const litSize = FUZ_getLitSize(seqs, nbSeqs); 360 assert(litSize <= dstCapacity); 361 for (n=0; n<nbSeqs; n++) { 362 size_t const ll = seqs[n].litLength; 363 memcpy(op, ip, ll); 364 op += ll; 365 ip += ll + seqs[n].matchLength; 366 } 367 assert((size_t)(ip - (const char*)src) == srcSize); 368 } 369 370 #ifdef ZSTD_MULTITHREAD 371 372 typedef struct { 373 ZSTD_CCtx* cctx; 374 ZSTD_threadPool* pool; 375 void* CNBuffer; 376 size_t CNBuffSize; 377 void* compressedBuffer; 378 size_t compressedBufferSize; 379 void* decodedBuffer; 380 int err; 381 } threadPoolTests_compressionJob_payload; 382 383 static void* threadPoolTests_compressionJob(void* payload) { 384 threadPoolTests_compressionJob_payload* args = (threadPoolTests_compressionJob_payload*)payload; 385 size_t cSize; 386 if (ZSTD_isError(ZSTD_CCtx_refThreadPool(args->cctx, args->pool))) args->err = 1; 387 cSize = ZSTD_compress2(args->cctx, args->compressedBuffer, args->compressedBufferSize, args->CNBuffer, args->CNBuffSize); 388 if (ZSTD_isError(cSize)) args->err = 1; 389 if (ZSTD_isError(ZSTD_decompress(args->decodedBuffer, args->CNBuffSize, args->compressedBuffer, cSize))) args->err = 1; 390 return payload; 391 } 392 393 static int threadPoolTests(void) { 394 int testResult = 0; 395 size_t err; 396 397 size_t const CNBuffSize = 5 MB; 398 void* const CNBuffer = malloc(CNBuffSize); 399 size_t const compressedBufferSize = ZSTD_compressBound(CNBuffSize); 400 void* const compressedBuffer = malloc(compressedBufferSize); 401 void* const decodedBuffer = malloc(CNBuffSize); 402 403 size_t const kPoolNumThreads = 8; 404 405 RDG_genBuffer(CNBuffer, CNBuffSize, 0.5, 0.5, 0); 406 407 DISPLAYLEVEL(3, "thread pool test : threadPool reuse roundtrips: "); 408 { 409 ZSTD_CCtx* cctx = ZSTD_createCCtx(); 410 ZSTD_threadPool* pool = ZSTD_createThreadPool(kPoolNumThreads); 411 412 size_t nbThreads = 1; 413 for (; nbThreads <= kPoolNumThreads; ++nbThreads) { 414 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters); 415 ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, (int)nbThreads); 416 err = ZSTD_CCtx_refThreadPool(cctx, pool); 417 if (ZSTD_isError(err)) { 418 DISPLAYLEVEL(3, "refThreadPool error!\n"); 419 ZSTD_freeCCtx(cctx); 420 goto _output_error; 421 } 422 err = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize); 423 if (ZSTD_isError(err)) { 424 DISPLAYLEVEL(3, "Compression error!\n"); 425 ZSTD_freeCCtx(cctx); 426 goto _output_error; 427 } 428 err = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, err); 429 if (ZSTD_isError(err)) { 430 DISPLAYLEVEL(3, "Decompression error!\n"); 431 ZSTD_freeCCtx(cctx); 432 goto _output_error; 433 } 434 } 435 436 ZSTD_freeCCtx(cctx); 437 ZSTD_freeThreadPool(pool); 438 } 439 DISPLAYLEVEL(3, "OK \n"); 440 441 DISPLAYLEVEL(3, "thread pool test : threadPool simultaneous usage: "); 442 { 443 void* const decodedBuffer2 = malloc(CNBuffSize); 444 void* const compressedBuffer2 = malloc(compressedBufferSize); 445 ZSTD_threadPool* pool = ZSTD_createThreadPool(kPoolNumThreads); 446 ZSTD_CCtx* cctx1 = ZSTD_createCCtx(); 447 ZSTD_CCtx* cctx2 = ZSTD_createCCtx(); 448 449 ZSTD_pthread_t t1; 450 ZSTD_pthread_t t2; 451 threadPoolTests_compressionJob_payload p1 = {cctx1, pool, CNBuffer, CNBuffSize, 452 compressedBuffer, compressedBufferSize, decodedBuffer, 0 /* err */}; 453 threadPoolTests_compressionJob_payload p2 = {cctx2, pool, CNBuffer, CNBuffSize, 454 compressedBuffer2, compressedBufferSize, decodedBuffer2, 0 /* err */}; 455 456 ZSTD_CCtx_setParameter(cctx1, ZSTD_c_nbWorkers, 2); 457 ZSTD_CCtx_setParameter(cctx2, ZSTD_c_nbWorkers, 2); 458 ZSTD_CCtx_refThreadPool(cctx1, pool); 459 ZSTD_CCtx_refThreadPool(cctx2, pool); 460 461 ZSTD_pthread_create(&t1, NULL, threadPoolTests_compressionJob, &p1); 462 ZSTD_pthread_create(&t2, NULL, threadPoolTests_compressionJob, &p2); 463 ZSTD_pthread_join(t1); 464 ZSTD_pthread_join(t2); 465 466 assert(!memcmp(decodedBuffer, decodedBuffer2, CNBuffSize)); 467 free(decodedBuffer2); 468 free(compressedBuffer2); 469 470 ZSTD_freeThreadPool(pool); 471 ZSTD_freeCCtx(cctx1); 472 ZSTD_freeCCtx(cctx2); 473 474 if (p1.err || p2.err) goto _output_error; 475 } 476 DISPLAYLEVEL(3, "OK \n"); 477 478 _end: 479 free(CNBuffer); 480 free(compressedBuffer); 481 free(decodedBuffer); 482 return testResult; 483 484 _output_error: 485 testResult = 1; 486 DISPLAY("Error detected in Unit tests ! \n"); 487 goto _end; 488 } 489 #endif /* ZSTD_MULTITHREAD */ 490 491 /*============================================= 492 * Unit tests 493 =============================================*/ 494 495 static void test_compressBound(unsigned tnb) 496 { 497 DISPLAYLEVEL(3, "test%3u : compressBound : ", tnb); 498 499 /* check ZSTD_compressBound == ZSTD_COMPRESSBOUND 500 * for a large range of known valid values */ 501 DEBUG_STATIC_ASSERT(sizeof(size_t) >= 4); 502 { int s; 503 for (s=0; s<30; s++) { 504 size_t const w = (size_t)1 << s; 505 CHECK_EQ(ZSTD_compressBound(w), ZSTD_COMPRESSBOUND(w)); 506 } } 507 508 /* Ensure error if srcSize too big */ 509 { size_t const w = ZSTD_MAX_INPUT_SIZE + 1; 510 CHECK(ZSTD_isError(ZSTD_compressBound(w))); /* must fail */ 511 CHECK_EQ(ZSTD_COMPRESSBOUND(w), 0); 512 } 513 514 DISPLAYLEVEL(3, "OK \n"); 515 } 516 517 static void test_decompressBound(unsigned tnb) 518 { 519 DISPLAYLEVEL(3, "test%3u : decompressBound : ", tnb); 520 521 /* Simple compression, with size : should provide size; */ 522 { const char example[] = "abcd"; 523 char cBuffer[ZSTD_COMPRESSBOUND(sizeof(example))]; 524 size_t const cSize = ZSTD_compress(cBuffer, sizeof(cBuffer), example, sizeof(example), 0); 525 CHECK_Z(cSize); 526 CHECK_EQ(ZSTD_decompressBound(cBuffer, cSize), (unsigned long long)sizeof(example)); 527 } 528 529 /* Simple small compression without size : should provide 1 block size */ 530 { char cBuffer[ZSTD_COMPRESSBOUND(0)]; 531 ZSTD_outBuffer out = { cBuffer, sizeof(cBuffer), 0 }; 532 ZSTD_inBuffer in = { NULL, 0, 0 }; 533 ZSTD_CCtx* const cctx = ZSTD_createCCtx(); 534 assert(cctx); 535 CHECK_Z( ZSTD_initCStream(cctx, 0) ); 536 CHECK_Z( ZSTD_compressStream(cctx, &out, &in) ); 537 CHECK_EQ( ZSTD_endStream(cctx, &out), 0 ); 538 CHECK_EQ( ZSTD_decompressBound(cBuffer, out.pos), ZSTD_BLOCKSIZE_MAX ); 539 ZSTD_freeCCtx(cctx); 540 } 541 542 /* Attempt to overflow 32-bit intermediate multiplication result 543 * This requires dBound >= 4 GB, aka 2^32. 544 * This requires 2^32 / 2^17 = 2^15 blocks 545 * => create 2^15 blocks (can be empty, or just 1 byte). */ 546 { const char input[] = "a"; 547 size_t const nbBlocks = (1 << 15) + 1; 548 size_t blockNb; 549 size_t const outCapacity = 1 << 18; /* large margin */ 550 char* const outBuffer = malloc (outCapacity); 551 ZSTD_outBuffer out = { outBuffer, outCapacity, 0 }; 552 ZSTD_CCtx* const cctx = ZSTD_createCCtx(); 553 assert(cctx); 554 assert(outBuffer); 555 CHECK_Z( ZSTD_initCStream(cctx, 0) ); 556 for (blockNb=0; blockNb<nbBlocks; blockNb++) { 557 ZSTD_inBuffer in = { input, sizeof(input), 0 }; 558 CHECK_Z( ZSTD_compressStream(cctx, &out, &in) ); 559 CHECK_EQ( ZSTD_flushStream(cctx, &out), 0 ); 560 } 561 CHECK_EQ( ZSTD_endStream(cctx, &out), 0 ); 562 CHECK( ZSTD_decompressBound(outBuffer, out.pos) > 0x100000000ULL /* 4 GB */ ); 563 ZSTD_freeCCtx(cctx); 564 free(outBuffer); 565 } 566 567 DISPLAYLEVEL(3, "OK \n"); 568 } 569 570 static void test_setCParams(unsigned tnb) 571 { 572 ZSTD_CCtx* const cctx = ZSTD_createCCtx(); 573 ZSTD_compressionParameters cparams; 574 assert(cctx); 575 576 DISPLAYLEVEL(3, "test%3u : ZSTD_CCtx_setCParams : ", tnb); 577 578 /* valid cparams */ 579 cparams = ZSTD_getCParams(1, 0, 0); 580 CHECK_Z(ZSTD_CCtx_setCParams(cctx, cparams)); 581 582 /* invalid cparams (must fail) */ 583 cparams.windowLog = 99; 584 CHECK(ZSTD_isError(ZSTD_CCtx_setCParams(cctx, cparams))); 585 586 free(cctx); 587 DISPLAYLEVEL(3, "OK \n"); 588 } 589 590 static void test_blockSplitter_incompressibleExpansionProtection(unsigned testNb, unsigned seed) 591 { 592 DISPLAYLEVEL(3, "test%3i : Check block splitter doesn't oversplit incompressible data (seed %u): ", testNb, seed); 593 { ZSTD_CCtx* cctx = ZSTD_createCCtx(); 594 size_t const srcSize = 256 * 1024; /* needs to be at least 2 blocks */ 595 void* incompressible = malloc(srcSize); 596 size_t const dstCapacity = ZSTD_compressBound(srcSize); 597 void* cBuffer = malloc(dstCapacity); 598 size_t const chunkSize = 8 KB; 599 size_t const nbChunks = srcSize / chunkSize; 600 size_t chunkNb, cSizeNoSplit, cSizeWithSplit; 601 assert(cctx != NULL); 602 assert(incompressible != NULL); 603 assert(cBuffer != NULL); 604 605 /* let's fill input with random noise (incompressible) */ 606 RDG_genBuffer(incompressible, srcSize, 0.0, 0.0, seed); 607 608 /* this pattern targets the fastest _byChunk variant's sampling (level 3). 609 * manually checked that, without the @savings protection, it would over-split. 610 */ 611 for (chunkNb=0; chunkNb<nbChunks; chunkNb++) { 612 BYTE* const p = (BYTE*)incompressible + chunkNb * chunkSize; 613 size_t const samplingRate = 43; 614 int addOrRemove = chunkNb % 2; 615 size_t n; 616 for (n=0; n<chunkSize; n+=samplingRate) { 617 if (addOrRemove) { 618 p[n] &= 0x80; 619 } else { 620 p[n] |= 0x80; 621 } 622 } 623 } 624 625 /* run first without splitting */ 626 ZSTD_CCtx_setParameter(cctx, ZSTD_c_blockSplitterLevel, 1 /* no split */); 627 cSizeNoSplit = ZSTD_compress2(cctx, cBuffer, dstCapacity, incompressible, srcSize); 628 629 /* run with sample43 splitter, check it's still the same */ 630 ZSTD_CCtx_setParameter(cctx, ZSTD_c_blockSplitterLevel, 3 /* sample43, fastest _byChunk variant */); 631 cSizeWithSplit = ZSTD_compress2(cctx, cBuffer, dstCapacity, incompressible, srcSize); 632 633 if (cSizeWithSplit != cSizeNoSplit) { 634 DISPLAYLEVEL(1, "invalid compressed size: cSizeWithSplit %u != %u cSizeNoSplit \n", 635 (unsigned)cSizeWithSplit, (unsigned)cSizeNoSplit); 636 abort(); 637 } 638 DISPLAYLEVEL(4, "compressed size: cSizeWithSplit %u == %u cSizeNoSplit : ", 639 (unsigned)cSizeWithSplit, (unsigned)cSizeNoSplit); 640 641 free(incompressible); 642 free(cBuffer); 643 ZSTD_freeCCtx(cctx); 644 } 645 DISPLAYLEVEL(3, "OK \n"); 646 } 647 648 /* ============================================================= */ 649 650 static int basicUnitTests(U32 const seed, double compressibility) 651 { 652 size_t const CNBuffSize = 5 MB; 653 void* const CNBuffer = malloc(CNBuffSize); 654 size_t const compressedBufferSize = ZSTD_compressBound(CNBuffSize); 655 void* const compressedBuffer = malloc(compressedBufferSize); 656 void* const decodedBuffer = malloc(CNBuffSize); 657 int testResult = 0; 658 unsigned testNb=0; 659 size_t cSize; 660 661 /* Create compressible noise */ 662 if (!CNBuffer || !compressedBuffer || !decodedBuffer) { 663 DISPLAY("Not enough memory, aborting\n"); 664 testResult = 1; 665 goto _end; 666 } 667 RDG_genBuffer(CNBuffer, CNBuffSize, compressibility, 0., seed); 668 669 /* Basic tests */ 670 DISPLAYLEVEL(3, "test%3u : ZSTD_getErrorName : ", testNb++); 671 { const char* errorString = ZSTD_getErrorName(0); 672 DISPLAYLEVEL(3, "OK : %s \n", errorString); 673 } 674 675 DISPLAYLEVEL(3, "test%3u : ZSTD_getErrorName with wrong value : ", testNb++); 676 { const char* errorString = ZSTD_getErrorName(499); 677 DISPLAYLEVEL(3, "OK : %s \n", errorString); 678 } 679 680 DISPLAYLEVEL(3, "test%3u : min compression level : ", testNb++); 681 { int const mcl = ZSTD_minCLevel(); 682 DISPLAYLEVEL(3, "%i (OK) \n", mcl); 683 } 684 685 DISPLAYLEVEL(3, "test%3u : default compression level : ", testNb++); 686 { int const defaultCLevel = ZSTD_defaultCLevel(); 687 if (defaultCLevel != ZSTD_CLEVEL_DEFAULT) goto _output_error; 688 DISPLAYLEVEL(3, "%i (OK) \n", defaultCLevel); 689 } 690 691 DISPLAYLEVEL(3, "test%3u : ZSTD_versionNumber : ", testNb++); 692 { unsigned const vn = ZSTD_versionNumber(); 693 DISPLAYLEVEL(3, "%u (OK) \n", vn); 694 } 695 696 test_compressBound(testNb++); 697 698 test_decompressBound(testNb++); 699 700 test_setCParams(testNb++); 701 702 DISPLAYLEVEL(3, "test%3u : ZSTD_adjustCParams : ", testNb++); 703 { 704 ZSTD_compressionParameters params; 705 memset(¶ms, 0, sizeof(params)); 706 params.windowLog = 10; 707 params.hashLog = 19; 708 params.chainLog = 19; 709 params = ZSTD_adjustCParams(params, 1000, 100000); 710 if (params.hashLog != 18) goto _output_error; 711 if (params.chainLog != 17) goto _output_error; 712 } 713 DISPLAYLEVEL(3, "OK \n"); 714 715 DISPLAYLEVEL(3, "test%3u : compress %u bytes : ", testNb++, (unsigned)CNBuffSize); 716 { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); 717 if (cctx==NULL) goto _output_error; 718 CHECK_VAR(cSize, ZSTD_compressCCtx(cctx, 719 compressedBuffer, compressedBufferSize, 720 CNBuffer, CNBuffSize, 1) ); 721 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBuffSize*100); 722 723 DISPLAYLEVEL(3, "test%3i : size of cctx for level 1 : ", testNb++); 724 { size_t const cctxSize = ZSTD_sizeof_CCtx(cctx); 725 DISPLAYLEVEL(3, "%u bytes \n", (unsigned)cctxSize); 726 } 727 ZSTD_freeCCtx(cctx); 728 } 729 730 DISPLAYLEVEL(3, "test%3i : decompress skippable frame -8 size : ", testNb++); 731 { 732 char const skippable8[] = "\x50\x2a\x4d\x18\xf8\xff\xff\xff"; 733 size_t const size = ZSTD_decompress(NULL, 0, skippable8, 8); 734 if (!ZSTD_isError(size)) goto _output_error; 735 } 736 DISPLAYLEVEL(3, "OK \n"); 737 738 DISPLAYLEVEL(3, "test%3i : ZSTD_getFrameContentSize test : ", testNb++); 739 { unsigned long long const rSize = ZSTD_getFrameContentSize(compressedBuffer, cSize); 740 if (rSize != CNBuffSize) goto _output_error; 741 } 742 DISPLAYLEVEL(3, "OK \n"); 743 744 DISPLAYLEVEL(3, "test%3i : ZSTD_getDecompressedSize test : ", testNb++); 745 { unsigned long long const rSize = ZSTD_getDecompressedSize(compressedBuffer, cSize); 746 if (rSize != CNBuffSize) goto _output_error; 747 } 748 DISPLAYLEVEL(3, "OK \n"); 749 750 DISPLAYLEVEL(3, "test%3i : ZSTD_findDecompressedSize test : ", testNb++); 751 { unsigned long long const rSize = ZSTD_findDecompressedSize(compressedBuffer, cSize); 752 if (rSize != CNBuffSize) goto _output_error; 753 } 754 DISPLAYLEVEL(3, "OK \n"); 755 756 DISPLAYLEVEL(3, "test%3i : tight ZSTD_decompressBound test : ", testNb++); 757 { 758 unsigned long long bound = ZSTD_decompressBound(compressedBuffer, cSize); 759 if (bound != CNBuffSize) goto _output_error; 760 } 761 DISPLAYLEVEL(3, "OK \n"); 762 763 DISPLAYLEVEL(3, "test%3i : ZSTD_decompressBound test with invalid srcSize : ", testNb++); 764 { 765 unsigned long long bound = ZSTD_decompressBound(compressedBuffer, cSize - 1); 766 if (bound != ZSTD_CONTENTSIZE_ERROR) goto _output_error; 767 } 768 DISPLAYLEVEL(3, "OK \n"); 769 770 DISPLAYLEVEL(3, "test%3i : decompress %u bytes : ", testNb++, (unsigned)CNBuffSize); 771 { size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize); 772 if (r != CNBuffSize) goto _output_error; } 773 DISPLAYLEVEL(3, "OK \n"); 774 775 DISPLAYLEVEL(3, "test%3i : decompress %u bytes with Huffman assembly disabled : ", testNb++, (unsigned)CNBuffSize); 776 { 777 ZSTD_DCtx* dctx = ZSTD_createDCtx(); 778 size_t r; 779 CHECK_Z(ZSTD_DCtx_setParameter(dctx, ZSTD_d_disableHuffmanAssembly, 1)); 780 r = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize); 781 if (r != CNBuffSize || memcmp(decodedBuffer, CNBuffer, CNBuffSize)) goto _output_error; 782 ZSTD_freeDCtx(dctx); 783 } 784 DISPLAYLEVEL(3, "OK \n"); 785 786 DISPLAYLEVEL(3, "test%3i : check decompressed result : ", testNb++); 787 { size_t u; 788 for (u=0; u<CNBuffSize; u++) { 789 if (((BYTE*)decodedBuffer)[u] != ((BYTE*)CNBuffer)[u]) goto _output_error; 790 } } 791 DISPLAYLEVEL(3, "OK \n"); 792 793 DISPLAYLEVEL(3, "test%3u : invalid endDirective : ", testNb++); 794 { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); 795 ZSTD_inBuffer inb = { CNBuffer, CNBuffSize, 0 }; 796 ZSTD_outBuffer outb = { compressedBuffer, compressedBufferSize, 0 }; 797 if (cctx==NULL) goto _output_error; 798 CHECK( ZSTD_isError( ZSTD_compressStream2(cctx, &outb, &inb, (ZSTD_EndDirective) 3) ) ); /* must fail */ 799 CHECK( ZSTD_isError( ZSTD_compressStream2(cctx, &outb, &inb, (ZSTD_EndDirective)-1) ) ); /* must fail */ 800 ZSTD_freeCCtx(cctx); 801 } 802 DISPLAYLEVEL(3, "OK \n"); 803 804 DISPLAYLEVEL(3, "test%3i : ZSTD_checkCParams : ", testNb++); 805 { 806 ZSTD_parameters params = ZSTD_getParams(3, 0, 0); 807 assert(!ZSTD_checkCParams(params.cParams)); 808 } 809 DISPLAYLEVEL(3, "OK \n"); 810 811 DISPLAYLEVEL(3, "test%3i : ZSTD_createDCtx_advanced and ZSTD_sizeof_DCtx: ", testNb++); 812 { 813 ZSTD_DCtx* const dctx = ZSTD_createDCtx_advanced(ZSTD_defaultCMem); 814 assert(dctx != NULL); 815 assert(ZSTD_sizeof_DCtx(dctx) != 0); 816 ZSTD_freeDCtx(dctx); 817 } 818 DISPLAYLEVEL(3, "OK \n"); 819 820 DISPLAYLEVEL(3, "test%3i : misc unaccounted for zstd symbols : ", testNb++); 821 { 822 /* %p takes a void*. In ISO C, it's illegal to cast a function pointer 823 * to a data pointer. (Although in POSIX you're required to be allowed 824 * to do it...) So we have to fall back to our trusty friend memcpy. */ 825 unsigned (* const funcptr_getDictID)(const ZSTD_DDict* ddict) = 826 ZSTD_getDictID_fromDDict; 827 ZSTD_DStream* (* const funcptr_createDStream)( 828 ZSTD_customMem customMem) = ZSTD_createDStream_advanced; 829 void (* const funcptr_copyDCtx)( 830 ZSTD_DCtx* dctx, const ZSTD_DCtx* preparedDCtx) = ZSTD_copyDCtx; 831 ZSTD_nextInputType_e (* const funcptr_nextInputType)(ZSTD_DCtx* dctx) = 832 ZSTD_nextInputType; 833 const void *voidptr_getDictID; 834 const void *voidptr_createDStream; 835 const void *voidptr_copyDCtx; 836 const void *voidptr_nextInputType; 837 DEBUG_STATIC_ASSERT(sizeof(funcptr_getDictID) == sizeof(voidptr_getDictID)); 838 memcpy( 839 (void*)&voidptr_getDictID, 840 (const void*)&funcptr_getDictID, 841 sizeof(void*)); 842 memcpy( 843 (void*)&voidptr_createDStream, 844 (const void*)&funcptr_createDStream, 845 sizeof(void*)); 846 memcpy( 847 (void*)&voidptr_copyDCtx, 848 (const void*)&funcptr_copyDCtx, 849 sizeof(void*)); 850 memcpy( 851 (void*)&voidptr_nextInputType, 852 (const void*)&funcptr_nextInputType, 853 sizeof(void*)); 854 DISPLAYLEVEL(3, "%p ", voidptr_getDictID); 855 DISPLAYLEVEL(3, "%p ", voidptr_createDStream); 856 DISPLAYLEVEL(3, "%p ", voidptr_copyDCtx); 857 DISPLAYLEVEL(3, "%p ", voidptr_nextInputType); 858 } 859 DISPLAYLEVEL(3, ": OK \n"); 860 861 DISPLAYLEVEL(3, "test%3i : decompress with null dict : ", testNb++); 862 { ZSTD_DCtx* const dctx = ZSTD_createDCtx(); assert(dctx != NULL); 863 { size_t const r = ZSTD_decompress_usingDict(dctx, 864 decodedBuffer, CNBuffSize, 865 compressedBuffer, cSize, 866 NULL, 0); 867 if (r != CNBuffSize) goto _output_error; 868 } 869 ZSTD_freeDCtx(dctx); 870 } 871 DISPLAYLEVEL(3, "OK \n"); 872 873 DISPLAYLEVEL(3, "test%3i : decompress with null DDict : ", testNb++); 874 { ZSTD_DCtx* const dctx = ZSTD_createDCtx(); assert(dctx != NULL); 875 { size_t const r = ZSTD_decompress_usingDDict(dctx, 876 decodedBuffer, CNBuffSize, 877 compressedBuffer, cSize, 878 NULL); 879 if (r != CNBuffSize) goto _output_error; 880 } 881 ZSTD_freeDCtx(dctx); 882 } 883 DISPLAYLEVEL(3, "OK \n"); 884 885 DISPLAYLEVEL(3, "test%3i : decompress with 1 missing byte : ", testNb++); 886 { size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize-1); 887 if (!ZSTD_isError(r)) goto _output_error; 888 if (ZSTD_getErrorCode((size_t)r) != ZSTD_error_srcSize_wrong) goto _output_error; } 889 DISPLAYLEVEL(3, "OK \n"); 890 891 DISPLAYLEVEL(3, "test%3i : decompress with 1 too much byte : ", testNb++); 892 { size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize+1); 893 if (!ZSTD_isError(r)) goto _output_error; 894 if (ZSTD_getErrorCode(r) != ZSTD_error_srcSize_wrong) goto _output_error; } 895 DISPLAYLEVEL(3, "OK \n"); 896 897 DISPLAYLEVEL(3, "test%3i : decompress too large input : ", testNb++); 898 { size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, compressedBufferSize); 899 if (!ZSTD_isError(r)) goto _output_error; 900 if (ZSTD_getErrorCode(r) != ZSTD_error_srcSize_wrong) goto _output_error; } 901 DISPLAYLEVEL(3, "OK \n"); 902 903 DISPLAYLEVEL(3, "test%3i : decompress into NULL buffer : ", testNb++); 904 { size_t const r = ZSTD_decompress(NULL, 0, compressedBuffer, compressedBufferSize); 905 if (!ZSTD_isError(r)) goto _output_error; 906 if (ZSTD_getErrorCode(r) != ZSTD_error_dstSize_tooSmall) goto _output_error; } 907 DISPLAYLEVEL(3, "OK \n"); 908 909 DISPLAYLEVEL(3, "test%3i : decompress with corrupted checksum : ", testNb++); 910 { /* create compressed buffer with checksumming enabled */ 911 ZSTD_CCtx* const cctx = ZSTD_createCCtx(); 912 if (!cctx) { 913 DISPLAY("Not enough memory, aborting\n"); 914 testResult = 1; 915 goto _end; 916 } 917 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1) ); 918 CHECK_VAR(cSize, ZSTD_compress2(cctx, 919 compressedBuffer, compressedBufferSize, 920 CNBuffer, CNBuffSize) ); 921 ZSTD_freeCCtx(cctx); 922 } 923 { /* copy the compressed buffer and corrupt the checksum */ 924 size_t r; 925 ZSTD_DCtx* const dctx = ZSTD_createDCtx(); 926 if (!dctx) { 927 DISPLAY("Not enough memory, aborting\n"); 928 testResult = 1; 929 goto _end; 930 } 931 932 ((char*)compressedBuffer)[cSize-1] += 1; 933 r = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize); 934 if (!ZSTD_isError(r)) goto _output_error; 935 if (ZSTD_getErrorCode(r) != ZSTD_error_checksum_wrong) goto _output_error; 936 937 CHECK_Z(ZSTD_DCtx_setParameter(dctx, ZSTD_d_forceIgnoreChecksum, ZSTD_d_ignoreChecksum)); 938 r = ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize-1); 939 if (!ZSTD_isError(r)) goto _output_error; /* wrong checksum size should still throw error */ 940 r = ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize); 941 if (ZSTD_isError(r)) goto _output_error; 942 943 ZSTD_freeDCtx(dctx); 944 } 945 DISPLAYLEVEL(3, "OK \n"); 946 947 948 DISPLAYLEVEL(3, "test%3i : ZSTD_decompressBound test with content size missing : ", testNb++); 949 { /* create compressed buffer with content size missing */ 950 ZSTD_CCtx* const cctx = ZSTD_createCCtx(); 951 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_contentSizeFlag, 0) ); 952 CHECK_VAR(cSize, ZSTD_compress2(cctx, 953 compressedBuffer, compressedBufferSize, 954 CNBuffer, CNBuffSize) ); 955 ZSTD_freeCCtx(cctx); 956 } 957 { /* ensure frame content size is missing */ 958 ZSTD_FrameHeader zfh; 959 size_t const ret = ZSTD_getFrameHeader(&zfh, compressedBuffer, compressedBufferSize); 960 if (ret != 0 || zfh.frameContentSize != ZSTD_CONTENTSIZE_UNKNOWN) goto _output_error; 961 } 962 { /* ensure CNBuffSize <= decompressBound */ 963 unsigned long long const bound = ZSTD_decompressBound(compressedBuffer, compressedBufferSize); 964 if (CNBuffSize > bound) goto _output_error; 965 } 966 DISPLAYLEVEL(3, "OK \n"); 967 968 DISPLAYLEVEL(3, "test%3d: check DCtx size is reduced after many oversized calls : ", testNb++); 969 { 970 size_t const largeFrameSrcSize = 200; 971 size_t const smallFrameSrcSize = 10; 972 size_t const nbFrames = 256; 973 974 size_t i = 0, consumed = 0, produced = 0, prevDCtxSize = 0; 975 int sizeReduced = 0; 976 977 BYTE* const dst = (BYTE*)compressedBuffer; 978 ZSTD_DCtx* dctx = ZSTD_createDCtx(); 979 980 /* create a large frame and then a bunch of small frames */ 981 size_t srcSize = ZSTD_compress((void*)dst, 982 compressedBufferSize, CNBuffer, largeFrameSrcSize, 3); 983 for (i = 0; i < nbFrames; i++) 984 srcSize += ZSTD_compress((void*)(dst + srcSize), 985 compressedBufferSize - srcSize, CNBuffer, 986 smallFrameSrcSize, 3); 987 988 /* decompressStream and make sure that dctx size was reduced at least once */ 989 while (consumed < srcSize) { 990 ZSTD_inBuffer in = {(void*)(dst + consumed), MIN(1, srcSize - consumed), 0}; 991 ZSTD_outBuffer out = {(BYTE*)CNBuffer + produced, CNBuffSize - produced, 0}; 992 ZSTD_decompressStream(dctx, &out, &in); 993 consumed += in.pos; 994 produced += out.pos; 995 996 /* success! size was reduced from the previous frame */ 997 if (prevDCtxSize > ZSTD_sizeof_DCtx(dctx)) 998 sizeReduced = 1; 999 1000 prevDCtxSize = ZSTD_sizeof_DCtx(dctx); 1001 } 1002 1003 assert(sizeReduced); 1004 1005 ZSTD_freeDCtx(dctx); 1006 } 1007 DISPLAYLEVEL(3, "OK \n"); 1008 1009 { 1010 ZSTD_CCtx* const cctx = ZSTD_createCCtx(); 1011 ZSTD_CDict* const cdict = ZSTD_createCDict(CNBuffer, 100, 1); 1012 ZSTD_parameters const params = ZSTD_getParams(1, 0, 0); 1013 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_format, ZSTD_f_zstd1_magicless) ); 1014 1015 DISPLAYLEVEL(3, "test%3i : ZSTD_compressCCtx() doesn't use advanced parameters", testNb++); 1016 CHECK_Z(ZSTD_compressCCtx(cctx, compressedBuffer, compressedBufferSize, NULL, 0, 1)); 1017 if (MEM_readLE32(compressedBuffer) != ZSTD_MAGICNUMBER) goto _output_error; 1018 DISPLAYLEVEL(3, "OK \n"); 1019 1020 DISPLAYLEVEL(3, "test%3i : ZSTD_compress_usingDict() doesn't use advanced parameters: ", testNb++); 1021 CHECK_Z(ZSTD_compress_usingDict(cctx, compressedBuffer, compressedBufferSize, NULL, 0, NULL, 0, 1)); 1022 if (MEM_readLE32(compressedBuffer) != ZSTD_MAGICNUMBER) goto _output_error; 1023 DISPLAYLEVEL(3, "OK \n"); 1024 1025 DISPLAYLEVEL(3, "test%3i : ZSTD_compress_usingCDict() doesn't use advanced parameters: ", testNb++); 1026 CHECK_Z(ZSTD_compress_usingCDict(cctx, compressedBuffer, compressedBufferSize, NULL, 0, cdict)); 1027 if (MEM_readLE32(compressedBuffer) != ZSTD_MAGICNUMBER) goto _output_error; 1028 DISPLAYLEVEL(3, "OK \n"); 1029 1030 DISPLAYLEVEL(3, "test%3i : ZSTD_compress_advanced() doesn't use advanced parameters: ", testNb++); 1031 CHECK_Z(ZSTD_compress_advanced(cctx, compressedBuffer, compressedBufferSize, NULL, 0, NULL, 0, params)); 1032 if (MEM_readLE32(compressedBuffer) != ZSTD_MAGICNUMBER) goto _output_error; 1033 DISPLAYLEVEL(3, "OK \n"); 1034 1035 DISPLAYLEVEL(3, "test%3i : ZSTD_compress_usingCDict_advanced() doesn't use advanced parameters: ", testNb++); 1036 CHECK_Z(ZSTD_compress_usingCDict_advanced(cctx, compressedBuffer, compressedBufferSize, NULL, 0, cdict, params.fParams)); 1037 if (MEM_readLE32(compressedBuffer) != ZSTD_MAGICNUMBER) goto _output_error; 1038 DISPLAYLEVEL(3, "OK \n"); 1039 1040 ZSTD_freeCDict(cdict); 1041 ZSTD_freeCCtx(cctx); 1042 } 1043 1044 DISPLAYLEVEL(3, "test%3i : maxBlockSize = 2K", testNb++); 1045 { 1046 ZSTD_CCtx* cctx = ZSTD_createCCtx(); 1047 ZSTD_DCtx* dctx = ZSTD_createDCtx(); 1048 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1)); 1049 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_maxBlockSize, 2048)); 1050 CHECK_Z(ZSTD_DCtx_setParameter(dctx, ZSTD_d_maxBlockSize, 2048)); 1051 1052 cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize); 1053 CHECK_Z(cSize); 1054 CHECK_Z(ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize)); 1055 1056 CHECK_Z(ZSTD_DCtx_setParameter(dctx, ZSTD_d_maxBlockSize, 1024)); 1057 CHECK(ZSTD_isError(ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize))); 1058 1059 ZSTD_freeDCtx(dctx); 1060 ZSTD_freeCCtx(cctx); 1061 } 1062 1063 DISPLAYLEVEL(3, "test%3i : ldm fill dict out-of-bounds check", testNb++); 1064 { 1065 ZSTD_CCtx* const cctx = ZSTD_createCCtx(); 1066 1067 size_t const size = (1U << 10); 1068 size_t const dstCapacity = ZSTD_compressBound(size); 1069 void* dict = (void*)malloc(size); 1070 void* src = (void*)malloc(size); 1071 void* dst = (void*)malloc(dstCapacity); 1072 1073 RDG_genBuffer(dict, size, 0.5, 0.5, seed); 1074 RDG_genBuffer(src, size, 0.5, 0.5, seed); 1075 1076 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable)); 1077 assert(!ZSTD_isError(ZSTD_compress_usingDict(cctx, dst, dstCapacity, src, size, dict, size, 3))); 1078 1079 ZSTD_freeCCtx(cctx); 1080 free(dict); 1081 free(src); 1082 free(dst); 1083 } 1084 DISPLAYLEVEL(3, "OK \n"); 1085 1086 DISPLAYLEVEL(3, "test%3i : testing dict compression with enableLdm and forceMaxWindow : ", testNb++); 1087 { 1088 ZSTD_CCtx* const cctx = ZSTD_createCCtx(); 1089 ZSTD_DCtx* const dctx = ZSTD_createDCtx(); 1090 void* dict = (void*)malloc(CNBuffSize); 1091 int nbWorkers; 1092 1093 for (nbWorkers = 0; nbWorkers < 3; ++nbWorkers) { 1094 RDG_genBuffer(dict, CNBuffSize, 0.5, 0.5, seed); 1095 RDG_genBuffer(CNBuffer, CNBuffSize, 0.6, 0.6, seed); 1096 1097 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, nbWorkers)); 1098 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1)); 1099 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_forceMaxWindow, 1)); 1100 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable)); 1101 CHECK_Z(ZSTD_CCtx_refPrefix(cctx, dict, CNBuffSize)); 1102 cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize); 1103 CHECK_Z(cSize); 1104 CHECK_Z(ZSTD_decompress_usingDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, dict, CNBuffSize)); 1105 } 1106 1107 ZSTD_freeCCtx(cctx); 1108 ZSTD_freeDCtx(dctx); 1109 free(dict); 1110 } 1111 DISPLAYLEVEL(3, "OK \n"); 1112 1113 DISPLAYLEVEL(3, "test%3i : testing dict compression for determinism : ", testNb++); 1114 { 1115 size_t const testSize = 1024; 1116 ZSTD_CCtx* const cctx = ZSTD_createCCtx(); 1117 ZSTD_DCtx* const dctx = ZSTD_createDCtx(); 1118 char* dict = (char*)malloc(2 * testSize); 1119 int ldmEnabled, level; 1120 1121 RDG_genBuffer(dict, testSize, 0.5, 0.5, seed); 1122 RDG_genBuffer(CNBuffer, testSize, 0.6, 0.6, seed); 1123 memcpy(dict + testSize, CNBuffer, testSize); 1124 for (level = 1; level <= 5; ++level) { 1125 for (ldmEnabled = ZSTD_ps_enable; ldmEnabled <= ZSTD_ps_disable; ++ldmEnabled) { 1126 size_t cSize0; 1127 XXH64_hash_t compressedChecksum0; 1128 1129 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1)); 1130 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, level)); 1131 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ldmEnabled)); 1132 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_deterministicRefPrefix, 1)); 1133 1134 CHECK_Z(ZSTD_CCtx_refPrefix(cctx, dict, testSize)); 1135 cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, testSize); 1136 CHECK_Z(cSize); 1137 CHECK_Z(ZSTD_decompress_usingDict(dctx, decodedBuffer, testSize, compressedBuffer, cSize, dict, testSize)); 1138 1139 cSize0 = cSize; 1140 compressedChecksum0 = XXH64(compressedBuffer, cSize, 0); 1141 1142 CHECK_Z(ZSTD_CCtx_refPrefix(cctx, dict, testSize)); 1143 cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, dict + testSize, testSize); 1144 CHECK_Z(cSize); 1145 1146 if (cSize != cSize0) goto _output_error; 1147 if (XXH64(compressedBuffer, cSize, 0) != compressedChecksum0) goto _output_error; 1148 } 1149 } 1150 1151 ZSTD_freeCCtx(cctx); 1152 ZSTD_freeDCtx(dctx); 1153 free(dict); 1154 } 1155 DISPLAYLEVEL(3, "OK \n"); 1156 1157 DISPLAYLEVEL(3, "test%3i : LDM + opt parser with small uncompressible block ", testNb++); 1158 { ZSTD_CCtx* cctx = ZSTD_createCCtx(); 1159 ZSTD_DCtx* dctx = ZSTD_createDCtx(); 1160 size_t const srcSize = 300 KB; 1161 size_t const flushSize = 128 KB + 5; 1162 size_t const dstSize = ZSTD_compressBound(srcSize); 1163 char* src = (char*)CNBuffer; 1164 char* dst = (char*)compressedBuffer; 1165 1166 ZSTD_outBuffer out = { dst, dstSize, 0 }; 1167 ZSTD_inBuffer in = { src, flushSize, 0 }; 1168 1169 if (!cctx || !dctx) { 1170 DISPLAY("Not enough memory, aborting\n"); 1171 testResult = 1; 1172 goto _end; 1173 } 1174 1175 RDG_genBuffer(src, srcSize, 0.5, 0.5, seed); 1176 /* Force an LDM to exist that crosses block boundary into uncompressible block */ 1177 memcpy(src + 125 KB, src, 3 KB + 5); 1178 1179 /* Enable MT, LDM, and opt parser */ 1180 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 1)); 1181 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable)); 1182 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1)); 1183 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 19)); 1184 1185 /* Flushes a block of 128 KB and block of 5 bytes */ 1186 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush)); 1187 1188 /* Compress the rest */ 1189 in.size = 300 KB; 1190 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end)); 1191 1192 CHECK_Z(ZSTD_decompress(decodedBuffer, CNBuffSize, dst, out.pos)); 1193 1194 ZSTD_freeCCtx(cctx); 1195 ZSTD_freeDCtx(dctx); 1196 } 1197 DISPLAYLEVEL(3, "OK \n"); 1198 1199 DISPLAYLEVEL(3, "test%3i : testing ldm dictionary gets invalidated : ", testNb++); 1200 { 1201 ZSTD_CCtx* const cctx = ZSTD_createCCtx(); 1202 ZSTD_DCtx* const dctx = ZSTD_createDCtx(); 1203 void* dict = (void*)malloc(CNBuffSize); 1204 size_t const kWindowLog = 10; 1205 size_t const kWindowSize = (size_t)1 << kWindowLog; 1206 size_t const dictSize = kWindowSize * 10; 1207 size_t const srcSize1 = kWindowSize / 2; 1208 size_t const srcSize2 = kWindowSize * 10; 1209 1210 CHECK(cctx!=NULL); 1211 CHECK(dctx!=NULL); 1212 CHECK(dict!=NULL); 1213 if (CNBuffSize < dictSize) goto _output_error; 1214 1215 RDG_genBuffer(dict, dictSize, 0.5, 0.5, seed); 1216 RDG_genBuffer(CNBuffer, srcSize1 + srcSize2, 0.5, 0.5, seed); 1217 1218 /* Enable checksum to verify round trip. */ 1219 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1)); 1220 /* Disable content size to skip single-pass decompression. */ 1221 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_contentSizeFlag, 0)); 1222 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, (int)kWindowLog)); 1223 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable)); 1224 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_ldmMinMatch, 32)); 1225 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_ldmHashRateLog, 1)); 1226 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_ldmHashLog, 16)); 1227 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_ldmBucketSizeLog, 3)); 1228 1229 /* Round trip once with a dictionary. */ 1230 CHECK_Z(ZSTD_CCtx_refPrefix(cctx, dict, dictSize)); 1231 cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, srcSize1); 1232 CHECK_Z(cSize); 1233 CHECK_Z(ZSTD_decompress_usingDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, dict, dictSize)); 1234 1235 cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, srcSize2); 1236 /* Streaming decompression to catch out of bounds offsets. */ 1237 { 1238 ZSTD_inBuffer in = {compressedBuffer, cSize, 0}; 1239 ZSTD_outBuffer out = {decodedBuffer, CNBuffSize, 0}; 1240 size_t const dSize = ZSTD_decompressStream(dctx, &out, &in); 1241 CHECK_Z(dSize); 1242 if (dSize != 0) goto _output_error; 1243 } 1244 1245 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 2)); 1246 /* Round trip once with a dictionary. */ 1247 CHECK_Z(ZSTD_CCtx_refPrefix(cctx, dict, dictSize)); 1248 { ZSTD_inBuffer in = {CNBuffer, srcSize1, 0}; 1249 ZSTD_outBuffer out = {compressedBuffer, compressedBufferSize, 0}; 1250 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush)); 1251 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end)); 1252 cSize = out.pos; 1253 } 1254 CHECK_Z(ZSTD_decompress_usingDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, dict, dictSize)); 1255 1256 { ZSTD_inBuffer in = {CNBuffer, srcSize2, 0}; 1257 ZSTD_outBuffer out = {compressedBuffer, compressedBufferSize, 0}; 1258 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush)); 1259 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end)); 1260 cSize = out.pos; 1261 } 1262 /* Streaming decompression to catch out of bounds offsets. */ 1263 { ZSTD_inBuffer in = {compressedBuffer, cSize, 0}; 1264 ZSTD_outBuffer out = {decodedBuffer, CNBuffSize, 0}; 1265 size_t const dSize = ZSTD_decompressStream(dctx, &out, &in); 1266 CHECK_Z(dSize); 1267 if (dSize != 0) goto _output_error; 1268 } 1269 1270 ZSTD_freeCCtx(cctx); 1271 ZSTD_freeDCtx(dctx); 1272 free(dict); 1273 } 1274 DISPLAYLEVEL(3, "OK \n"); 1275 1276 /* Note: this test takes 0.5 seconds to run */ 1277 DISPLAYLEVEL(3, "test%3i : testing refPrefx vs refPrefx + ldm (size comparison) : ", testNb++); 1278 { 1279 /* test a big buffer so that ldm can take effect */ 1280 size_t const size = 100 MB; 1281 int const windowLog = 27; 1282 size_t const dstSize = ZSTD_compressBound(size); 1283 1284 void* dict = (void*)malloc(size); 1285 void* src = (void*)malloc(size); 1286 void* dst = (void*)malloc(dstSize); 1287 void* recon = (void*)malloc(size); 1288 1289 size_t refPrefixCompressedSize = 0; 1290 size_t refPrefixLdmCompressedSize = 0; 1291 size_t reconSize = 0; 1292 1293 ZSTD_CCtx* const cctx = ZSTD_createCCtx(); 1294 ZSTD_DCtx* const dctx = ZSTD_createDCtx(); 1295 1296 /* make dict and src the same uncompressible data */ 1297 RDG_genBuffer(src, size, 0, 0, seed); 1298 memcpy(dict, src, size); 1299 assert(!memcmp(dict, src, size)); 1300 1301 /* set level 1 and windowLog to cover src */ 1302 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 1)); 1303 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, windowLog)); 1304 1305 /* compress on level 1 using just refPrefix and no ldm */ 1306 ZSTD_CCtx_refPrefix(cctx, dict, size); 1307 refPrefixCompressedSize = ZSTD_compress2(cctx, dst, dstSize, src, size); 1308 assert(!ZSTD_isError(refPrefixCompressedSize)); 1309 1310 /* test round trip just refPrefix */ 1311 ZSTD_DCtx_refPrefix(dctx, dict, size); 1312 reconSize = ZSTD_decompressDCtx(dctx, recon, size, dst, refPrefixCompressedSize); 1313 assert(!ZSTD_isError(reconSize)); 1314 assert(reconSize == size); 1315 assert(!memcmp(recon, src, size)); 1316 1317 /* compress on level 1 using refPrefix and ldm */ 1318 ZSTD_CCtx_refPrefix(cctx, dict, size);; 1319 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable)) 1320 refPrefixLdmCompressedSize = ZSTD_compress2(cctx, dst, dstSize, src, size); 1321 assert(!ZSTD_isError(refPrefixLdmCompressedSize)); 1322 1323 /* test round trip refPrefix + ldm*/ 1324 ZSTD_DCtx_refPrefix(dctx, dict, size); 1325 reconSize = ZSTD_decompressDCtx(dctx, recon, size, dst, refPrefixLdmCompressedSize); 1326 assert(!ZSTD_isError(reconSize)); 1327 assert(reconSize == size); 1328 assert(!memcmp(recon, src, size)); 1329 1330 /* make sure that refPrefixCompressedSize is significantly greater */ 1331 assert(refPrefixCompressedSize > 10 * refPrefixLdmCompressedSize); 1332 /* make sure the ldm compressed size is less than 1% of original */ 1333 assert((double)refPrefixLdmCompressedSize / (double)size < 0.01); 1334 1335 ZSTD_freeDCtx(dctx); 1336 ZSTD_freeCCtx(cctx); 1337 free(recon); 1338 free(dict); 1339 free(src); 1340 free(dst); 1341 } 1342 DISPLAYLEVEL(3, "OK \n"); 1343 1344 DISPLAYLEVEL(3, "test%3i : in-place decompression : ", testNb++); 1345 cSize = ZSTD_compress(compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize, -ZSTD_BLOCKSIZE_MAX); 1346 CHECK_Z(cSize); 1347 CHECK_LT(CNBuffSize, cSize); 1348 { 1349 size_t const margin = ZSTD_decompressionMargin(compressedBuffer, cSize); 1350 size_t const outputSize = (CNBuffSize + margin); 1351 char* output = malloc(outputSize); 1352 char* input = output + outputSize - cSize; 1353 CHECK_LT(cSize, CNBuffSize + margin); 1354 CHECK(output != NULL); 1355 CHECK_Z(margin); 1356 CHECK(margin <= ZSTD_DECOMPRESSION_MARGIN(CNBuffSize, ZSTD_BLOCKSIZE_MAX)); 1357 memcpy(input, compressedBuffer, cSize); 1358 1359 { 1360 size_t const dSize = ZSTD_decompress(output, outputSize, input, cSize); 1361 CHECK_Z(dSize); 1362 CHECK_EQ(dSize, CNBuffSize); 1363 } 1364 CHECK(!memcmp(output, CNBuffer, CNBuffSize)); 1365 free(output); 1366 } 1367 DISPLAYLEVEL(3, "OK \n"); 1368 1369 DISPLAYLEVEL(3, "test%3i : in-place decompression with 2 frames : ", testNb++); 1370 cSize = ZSTD_compress(compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize / 3, -ZSTD_BLOCKSIZE_MAX); 1371 CHECK_Z(cSize); 1372 { 1373 size_t const cSize2 = ZSTD_compress((char*)compressedBuffer + cSize, compressedBufferSize - cSize, (char const*)CNBuffer + (CNBuffSize / 3), CNBuffSize / 3, -ZSTD_BLOCKSIZE_MAX); 1374 CHECK_Z(cSize2); 1375 cSize += cSize2; 1376 } 1377 { 1378 size_t const srcSize = (CNBuffSize / 3) * 2; 1379 size_t const margin = ZSTD_decompressionMargin(compressedBuffer, cSize); 1380 size_t const outputSize = (CNBuffSize + margin); 1381 char* output = malloc(outputSize); 1382 char* input = output + outputSize - cSize; 1383 CHECK_LT(cSize, CNBuffSize + margin); 1384 CHECK(output != NULL); 1385 CHECK_Z(margin); 1386 memcpy(input, compressedBuffer, cSize); 1387 1388 { 1389 size_t const dSize = ZSTD_decompress(output, outputSize, input, cSize); 1390 CHECK_Z(dSize); 1391 CHECK_EQ(dSize, srcSize); 1392 } 1393 CHECK(!memcmp(output, CNBuffer, srcSize)); 1394 free(output); 1395 } 1396 DISPLAYLEVEL(3, "OK \n"); 1397 1398 DISPLAYLEVEL(3, "test%3i : Check block splitter with 64K literal length : ", testNb++); 1399 { ZSTD_CCtx* cctx = ZSTD_createCCtx(); 1400 size_t const srcSize = 256 * 1024; 1401 U32 const compressibleLenU32 = 32 * 1024 / 4; 1402 U32 const blockSizeU32 = 128 * 1024 / 4; 1403 U32 const litLenU32 = 64 * 1024 / 4; 1404 U32* data = (U32*)malloc(srcSize); 1405 size_t dSize; 1406 1407 if (data == NULL || cctx == NULL) goto _output_error; 1408 1409 /* Generate data without any matches */ 1410 RDG_genBuffer(data, srcSize, 0.0, 0.01, 2654435761U); 1411 /* Generate 32K of compressible data */ 1412 RDG_genBuffer(data, compressibleLenU32 * 4, 0.5, 0.5, 0xcafebabe); 1413 1414 /* Add a match of offset=12, length=8 at idx=16, 32, 48, 64 */ 1415 data[compressibleLenU32 + 0] = 0xFFFFFFFF; 1416 data[compressibleLenU32 + 1] = 0xEEEEEEEE; 1417 data[compressibleLenU32 + 4] = 0xFFFFFFFF; 1418 data[compressibleLenU32 + 5] = 0xEEEEEEEE; 1419 1420 /* Add a match of offset=16, length=8 at idx=64K + 64. 1421 * This generates a sequence with llen=64K, and repeat code 1. 1422 * The block splitter thought this was ll0, and corrupted the 1423 * repeat offset history. 1424 */ 1425 data[compressibleLenU32 + litLenU32 + 2 + 0] = 0xDDDDDDDD; 1426 data[compressibleLenU32 + litLenU32 + 2 + 1] = 0xCCCCCCCC; 1427 data[compressibleLenU32 + litLenU32 + 2 + 4] = 0xDDDDDDDD; 1428 data[compressibleLenU32 + litLenU32 + 2 + 5] = 0xCCCCCCCC; 1429 1430 /* Add a match of offset=16, length=8 at idx=128K + 16. 1431 * This should generate a sequence with repeat code = 1. 1432 * But the block splitters mistake caused zstd to generate 1433 * repeat code = 2, corrupting the data. 1434 */ 1435 data[blockSizeU32] = 0xBBBBBBBB; 1436 data[blockSizeU32 + 1] = 0xAAAAAAAA; 1437 data[blockSizeU32 + 4] = 0xBBBBBBBB; 1438 data[blockSizeU32 + 5] = 0xAAAAAAAA; 1439 1440 /* Generate a golden file from this data in case datagen changes and 1441 * doesn't generate the exact same data. We will also test this golden file. 1442 */ 1443 if (0) { 1444 FILE* f = fopen("golden-compression/PR-3517-block-splitter-corruption-test", "wb"); 1445 fwrite(data, 1, srcSize, f); 1446 fclose(f); 1447 } 1448 1449 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 19)); 1450 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_minMatch, 7)); 1451 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_splitAfterSequences, ZSTD_ps_enable)); 1452 1453 cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, data, srcSize); 1454 CHECK_Z(cSize); 1455 dSize = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize); 1456 CHECK_Z(dSize); 1457 CHECK_EQ(dSize, srcSize); 1458 CHECK(!memcmp(decodedBuffer, data, srcSize)); 1459 1460 free(data); 1461 ZSTD_freeCCtx(cctx); 1462 } 1463 DISPLAYLEVEL(3, "OK \n"); 1464 1465 test_blockSplitter_incompressibleExpansionProtection(testNb++, seed); 1466 1467 DISPLAYLEVEL(3, "test%3d : superblock uncompressible data: too many nocompress superblocks : ", testNb++); 1468 { 1469 ZSTD_CCtx* const cctx = ZSTD_createCCtx(); 1470 const BYTE* src = (BYTE*)CNBuffer; BYTE* dst = (BYTE*)compressedBuffer; 1471 size_t srcSize = 321656; size_t dstCapacity = ZSTD_compressBound(srcSize); 1472 1473 /* This is the number of bytes to stream before ending. This value 1474 * was obtained by trial and error :/. */ 1475 1476 const size_t streamCompressThreshold = 161792; 1477 const size_t streamCompressDelta = 1024; 1478 1479 /* The first 1/5 of the buffer is compressible and the last 4/5 is 1480 * uncompressible. This is an approximation of the type of data 1481 * the fuzzer generated to catch this bug. Streams like this were making 1482 * zstd generate noCompress superblocks (which are larger than the src 1483 * they come from). Do this enough times, and we'll run out of room 1484 * and throw a dstSize_tooSmall error. */ 1485 1486 const size_t compressiblePartSize = srcSize/5; 1487 const size_t uncompressiblePartSize = srcSize-compressiblePartSize; 1488 RDG_genBuffer(CNBuffer, compressiblePartSize, 0.5, 0.5, seed); 1489 RDG_genBuffer((BYTE*)CNBuffer+compressiblePartSize, uncompressiblePartSize, 0, 0, seed); 1490 1491 /* Setting target block size so that superblock is used */ 1492 1493 assert(cctx != NULL); 1494 ZSTD_CCtx_setParameter(cctx, ZSTD_c_targetCBlockSize, 81); 1495 1496 { size_t read; 1497 for (read = 0; read < streamCompressThreshold; read += streamCompressDelta) { 1498 ZSTD_inBuffer in = {src, streamCompressDelta, 0}; 1499 ZSTD_outBuffer out = {dst, dstCapacity, 0}; 1500 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_continue)); 1501 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end)); 1502 src += streamCompressDelta; srcSize -= streamCompressDelta; 1503 dst += out.pos; dstCapacity -= out.pos; 1504 } } 1505 1506 /* This is trying to catch a dstSize_tooSmall error */ 1507 1508 { ZSTD_inBuffer in = {src, srcSize, 0}; 1509 ZSTD_outBuffer out = {dst, dstCapacity, 0}; 1510 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end)); 1511 } 1512 ZSTD_freeCCtx(cctx); 1513 } 1514 DISPLAYLEVEL(3, "OK \n"); 1515 1516 DISPLAYLEVEL(3, "test%3d: superblock with no literals : ", testNb++); 1517 /* Generate the same data 20 times over */ 1518 { size_t const avgChunkSize = CNBuffSize / 20; 1519 size_t b; 1520 for (b = 0; b < CNBuffSize; b += avgChunkSize) { 1521 size_t const chunkSize = MIN(CNBuffSize - b, avgChunkSize); 1522 RDG_genBuffer((char*)CNBuffer + b, chunkSize, compressibility, 0. /* auto */, seed); 1523 } } 1524 { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); 1525 size_t const normalCSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize); 1526 size_t const allowedExpansion = (CNBuffSize * 3 / 1000); 1527 size_t superCSize; 1528 CHECK_Z(normalCSize); 1529 ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 19); 1530 ZSTD_CCtx_setParameter(cctx, ZSTD_c_targetCBlockSize, 1000); 1531 superCSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize); 1532 CHECK_Z(superCSize); 1533 if (superCSize > normalCSize + allowedExpansion) { 1534 DISPLAYLEVEL(1, "Superblock too big: %u > %u + %u \n", (U32)superCSize, (U32)normalCSize, (U32)allowedExpansion); 1535 goto _output_error; 1536 } 1537 ZSTD_freeCCtx(cctx); 1538 } 1539 DISPLAYLEVEL(3, "OK \n"); 1540 1541 RDG_genBuffer(CNBuffer, CNBuffSize, compressibility, 0. /*auto*/, seed); 1542 DISPLAYLEVEL(3, "test%3d: superblock enough room for checksum : ", testNb++) 1543 /* This tests whether or not we leave enough room for the checksum at the end 1544 * of the dst buffer. The bug that motivated this test was found by the 1545 * stream_round_trip fuzzer but this crashes for the same reason and is 1546 * far more compact than re-creating the stream_round_trip fuzzer's code path */ 1547 { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); 1548 ZSTD_CCtx_setParameter(cctx, ZSTD_c_targetCBlockSize, 64); 1549 assert(!ZSTD_isError(ZSTD_compress2(cctx, compressedBuffer, 1339, CNBuffer, 1278))); 1550 ZSTD_freeCCtx(cctx); 1551 } 1552 DISPLAYLEVEL(3, "OK \n"); 1553 1554 DISPLAYLEVEL(3, "test%3i : compress a NULL input with each level : ", testNb++); 1555 { int level = -1; 1556 ZSTD_CCtx* const cctx = ZSTD_createCCtx(); 1557 if (!cctx) goto _output_error; 1558 for (level = -1; level <= ZSTD_maxCLevel(); ++level) { 1559 CHECK_Z( ZSTD_compress(compressedBuffer, compressedBufferSize, NULL, 0, level) ); 1560 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, level) ); 1561 CHECK_Z( ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, NULL, 0) ); 1562 } 1563 ZSTD_freeCCtx(cctx); 1564 } 1565 DISPLAYLEVEL(3, "OK \n"); 1566 1567 DISPLAYLEVEL(3, "test%3d : check CCtx size after compressing empty input : ", testNb++); 1568 { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); 1569 size_t const r = ZSTD_compressCCtx(cctx, compressedBuffer, compressedBufferSize, NULL, 0, 19); 1570 if (ZSTD_isError(r)) goto _output_error; 1571 if (ZSTD_sizeof_CCtx(cctx) > (1U << 20)) goto _output_error; 1572 ZSTD_freeCCtx(cctx); 1573 cSize = r; 1574 } 1575 DISPLAYLEVEL(3, "OK \n"); 1576 1577 DISPLAYLEVEL(3, "test%3d : decompress empty frame into NULL : ", testNb++); 1578 { size_t const r = ZSTD_decompress(NULL, 0, compressedBuffer, cSize); 1579 if (ZSTD_isError(r)) goto _output_error; 1580 if (r != 0) goto _output_error; 1581 } 1582 { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); 1583 ZSTD_outBuffer output; 1584 if (cctx==NULL) goto _output_error; 1585 output.dst = compressedBuffer; 1586 output.size = compressedBufferSize; 1587 output.pos = 0; 1588 CHECK_Z( ZSTD_initCStream(cctx, 1) ); /* content size unknown */ 1589 CHECK_Z( ZSTD_flushStream(cctx, &output) ); /* ensure no possibility to "concatenate" and determine the content size */ 1590 CHECK_Z( ZSTD_endStream(cctx, &output) ); 1591 ZSTD_freeCCtx(cctx); 1592 /* single scan decompression */ 1593 { size_t const r = ZSTD_decompress(NULL, 0, compressedBuffer, output.pos); 1594 if (ZSTD_isError(r)) goto _output_error; 1595 if (r != 0) goto _output_error; 1596 } 1597 /* streaming decompression */ 1598 { ZSTD_DCtx* const dstream = ZSTD_createDStream(); 1599 ZSTD_inBuffer dinput; 1600 ZSTD_outBuffer doutput; 1601 size_t ipos; 1602 if (dstream==NULL) goto _output_error; 1603 dinput.src = compressedBuffer; 1604 dinput.size = 0; 1605 dinput.pos = 0; 1606 doutput.dst = NULL; 1607 doutput.size = 0; 1608 doutput.pos = 0; 1609 CHECK_Z ( ZSTD_initDStream(dstream) ); 1610 for (ipos=1; ipos<=output.pos; ipos++) { 1611 dinput.size = ipos; 1612 CHECK_Z ( ZSTD_decompressStream(dstream, &doutput, &dinput) ); 1613 } 1614 if (doutput.pos != 0) goto _output_error; 1615 ZSTD_freeDStream(dstream); 1616 } 1617 } 1618 DISPLAYLEVEL(3, "OK \n"); 1619 1620 DISPLAYLEVEL(3, "test%3d : reuse CCtx with expanding block size : ", testNb++); 1621 { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); 1622 ZSTD_parameters const params = ZSTD_getParams(1, ZSTD_CONTENTSIZE_UNKNOWN, 0); 1623 assert(params.fParams.contentSizeFlag == 1); /* block size will be adapted if pledgedSrcSize is enabled */ 1624 CHECK_Z( ZSTD_compressBegin_advanced(cctx, NULL, 0, params, 1 /*pledgedSrcSize*/) ); 1625 CHECK_Z( ZSTD_compressEnd(cctx, compressedBuffer, compressedBufferSize, CNBuffer, 1) ); /* creates a block size of 1 */ 1626 1627 CHECK_Z( ZSTD_compressBegin_advanced(cctx, NULL, 0, params, ZSTD_CONTENTSIZE_UNKNOWN) ); /* reuse same parameters */ 1628 { size_t const inSize = 2* 128 KB; 1629 size_t const outSize = ZSTD_compressBound(inSize); 1630 CHECK_Z( ZSTD_compressEnd(cctx, compressedBuffer, outSize, CNBuffer, inSize) ); 1631 /* will fail if blockSize is not resized */ 1632 } 1633 ZSTD_freeCCtx(cctx); 1634 } 1635 DISPLAYLEVEL(3, "OK \n"); 1636 1637 DISPLAYLEVEL(3, "test%3d : re-using a CCtx should compress the same : ", testNb++); 1638 { size_t const sampleSize = 30; 1639 int i; 1640 for (i=0; i<20; i++) 1641 ((char*)CNBuffer)[i] = (char)i; /* ensure no match during initial section */ 1642 memcpy((char*)CNBuffer + 20, CNBuffer, 10); /* create one match, starting from beginning of sample, which is the difficult case (see #1241) */ 1643 for (i=1; i<=19; i++) { 1644 ZSTD_CCtx* const cctx = ZSTD_createCCtx(); 1645 size_t size1, size2; 1646 DISPLAYLEVEL(5, "l%i ", i); 1647 size1 = ZSTD_compressCCtx(cctx, compressedBuffer, compressedBufferSize, CNBuffer, sampleSize, i); 1648 CHECK_Z(size1); 1649 1650 size2 = ZSTD_compressCCtx(cctx, compressedBuffer, compressedBufferSize, CNBuffer, sampleSize, i); 1651 CHECK_Z(size2); 1652 CHECK_EQ(size1, size2); 1653 1654 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, i) ); 1655 size2 = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, sampleSize); 1656 CHECK_Z(size2); 1657 CHECK_EQ(size1, size2); 1658 1659 size2 = ZSTD_compress2(cctx, compressedBuffer, ZSTD_compressBound(sampleSize) - 1, CNBuffer, sampleSize); /* force streaming, as output buffer is not large enough to guarantee success */ 1660 CHECK_Z(size2); 1661 CHECK_EQ(size1, size2); 1662 1663 { ZSTD_inBuffer inb; 1664 ZSTD_outBuffer outb; 1665 inb.src = CNBuffer; 1666 inb.pos = 0; 1667 inb.size = sampleSize; 1668 outb.dst = compressedBuffer; 1669 outb.pos = 0; 1670 outb.size = ZSTD_compressBound(sampleSize) - 1; /* force streaming, as output buffer is not large enough to guarantee success */ 1671 CHECK_Z( ZSTD_compressStream2(cctx, &outb, &inb, ZSTD_e_end) ); 1672 assert(inb.pos == inb.size); 1673 CHECK_EQ(size1, outb.pos); 1674 } 1675 1676 ZSTD_freeCCtx(cctx); 1677 } 1678 } 1679 DISPLAYLEVEL(3, "OK \n"); 1680 1681 DISPLAYLEVEL(3, "test%3d : btultra2 & 1st block : ", testNb++); 1682 { size_t const sampleSize = 1024; 1683 ZSTD_CCtx* const cctx = ZSTD_createCCtx(); 1684 ZSTD_inBuffer inb; 1685 ZSTD_outBuffer outb; 1686 inb.src = CNBuffer; 1687 inb.pos = 0; 1688 inb.size = 0; 1689 outb.dst = compressedBuffer; 1690 outb.pos = 0; 1691 outb.size = compressedBufferSize; 1692 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, ZSTD_maxCLevel()) ); 1693 1694 inb.size = sampleSize; /* start with something, so that context is already used */ 1695 CHECK_Z( ZSTD_compressStream2(cctx, &outb, &inb, ZSTD_e_end) ); /* will break internal assert if stats_init is not disabled */ 1696 assert(inb.pos == inb.size); 1697 outb.pos = 0; /* cancel output */ 1698 1699 CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(cctx, sampleSize) ); 1700 inb.size = 4; /* too small size : compression will be skipped */ 1701 inb.pos = 0; 1702 CHECK_Z( ZSTD_compressStream2(cctx, &outb, &inb, ZSTD_e_flush) ); 1703 assert(inb.pos == inb.size); 1704 1705 inb.size += 5; /* too small size : compression will be skipped */ 1706 CHECK_Z( ZSTD_compressStream2(cctx, &outb, &inb, ZSTD_e_flush) ); 1707 assert(inb.pos == inb.size); 1708 1709 inb.size += 11; /* small enough to attempt compression */ 1710 CHECK_Z( ZSTD_compressStream2(cctx, &outb, &inb, ZSTD_e_flush) ); 1711 assert(inb.pos == inb.size); 1712 1713 assert(inb.pos < sampleSize); 1714 inb.size = sampleSize; /* large enough to trigger stats_init, but no longer at beginning */ 1715 CHECK_Z( ZSTD_compressStream2(cctx, &outb, &inb, ZSTD_e_end) ); /* will break internal assert if stats_init is not disabled */ 1716 assert(inb.pos == inb.size); 1717 ZSTD_freeCCtx(cctx); 1718 } 1719 DISPLAYLEVEL(3, "OK \n"); 1720 1721 DISPLAYLEVEL(3, "test%3d : ZSTD_CCtx_getParameter() : ", testNb++); 1722 { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); 1723 ZSTD_outBuffer out = {NULL, 0, 0}; 1724 ZSTD_inBuffer in = {NULL, 0, 0}; 1725 int value; 1726 1727 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_compressionLevel, &value)); 1728 CHECK_EQ(value, 3); 1729 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_hashLog, &value)); 1730 CHECK_EQ(value, 0); 1731 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_hashLog, ZSTD_HASHLOG_MIN)); 1732 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_compressionLevel, &value)); 1733 CHECK_EQ(value, 3); 1734 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_hashLog, &value)); 1735 CHECK_EQ(value, ZSTD_HASHLOG_MIN); 1736 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 7)); 1737 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_compressionLevel, &value)); 1738 CHECK_EQ(value, 7); 1739 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_hashLog, &value)); 1740 CHECK_EQ(value, ZSTD_HASHLOG_MIN); 1741 /* Start a compression job */ 1742 ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_continue); 1743 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_compressionLevel, &value)); 1744 CHECK_EQ(value, 7); 1745 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_hashLog, &value)); 1746 CHECK_EQ(value, ZSTD_HASHLOG_MIN); 1747 /* Reset the CCtx */ 1748 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_only); 1749 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_compressionLevel, &value)); 1750 CHECK_EQ(value, 7); 1751 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_hashLog, &value)); 1752 CHECK_EQ(value, ZSTD_HASHLOG_MIN); 1753 /* Reset the parameters */ 1754 ZSTD_CCtx_reset(cctx, ZSTD_reset_parameters); 1755 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_compressionLevel, &value)); 1756 CHECK_EQ(value, 3); 1757 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_hashLog, &value)); 1758 CHECK_EQ(value, 0); 1759 1760 ZSTD_freeCCtx(cctx); 1761 } 1762 DISPLAYLEVEL(3, "OK \n"); 1763 1764 DISPLAYLEVEL(3, "test%3d : ZSTD_CCtx_setCParams() : ", testNb++); 1765 { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); 1766 int value; 1767 ZSTD_compressionParameters cparams = ZSTD_getCParams(1, 0, 0); 1768 cparams.strategy = (ZSTD_strategy)-1; /* set invalid value, on purpose */ 1769 /* Set invalid cParams == error out, and no change. */ 1770 CHECK(ZSTD_isError(ZSTD_CCtx_setCParams(cctx, cparams))); 1771 1772 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_windowLog, &value)); 1773 CHECK_EQ(value, 0); 1774 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_chainLog, &value)); 1775 CHECK_EQ(value, 0); 1776 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_hashLog, &value)); 1777 CHECK_EQ(value, 0); 1778 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_searchLog, &value)); 1779 CHECK_EQ(value, 0); 1780 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_minMatch, &value)); 1781 CHECK_EQ(value, 0); 1782 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_targetLength, &value)); 1783 CHECK_EQ(value, 0); 1784 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_strategy, &value)); 1785 CHECK_EQ(value, 0); 1786 1787 cparams = ZSTD_getCParams(12, 0, 0); 1788 CHECK_Z(ZSTD_CCtx_setCParams(cctx, cparams)); 1789 1790 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_windowLog, &value)); 1791 CHECK_EQ(value, (int)cparams.windowLog); 1792 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_chainLog, &value)); 1793 CHECK_EQ(value, (int)cparams.chainLog); 1794 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_hashLog, &value)); 1795 CHECK_EQ(value, (int)cparams.hashLog); 1796 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_searchLog, &value)); 1797 CHECK_EQ(value, (int)cparams.searchLog); 1798 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_minMatch, &value)); 1799 CHECK_EQ(value, (int)cparams.minMatch); 1800 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_targetLength, &value)); 1801 CHECK_EQ(value, (int)cparams.targetLength); 1802 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_strategy, &value)); 1803 CHECK_EQ(value, (int)cparams.strategy); 1804 1805 ZSTD_freeCCtx(cctx); 1806 } 1807 1808 DISPLAYLEVEL(3, "test%3d : ZSTD_CCtx_setFParams() : ", testNb++); 1809 { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); 1810 int value; 1811 ZSTD_frameParameters fparams = {0, 1, 1}; 1812 1813 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_contentSizeFlag, &value)); 1814 CHECK_EQ(value, 1); 1815 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_checksumFlag, &value)); 1816 CHECK_EQ(value, 0); 1817 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_dictIDFlag, &value)); 1818 CHECK_EQ(value, 1); 1819 1820 CHECK_Z(ZSTD_CCtx_setFParams(cctx, fparams)); 1821 1822 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_contentSizeFlag, &value)); 1823 CHECK_EQ(value, fparams.contentSizeFlag); 1824 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_checksumFlag, &value)); 1825 CHECK_EQ(value, fparams.checksumFlag); 1826 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_dictIDFlag, &value)); 1827 CHECK_EQ(value, !fparams.noDictIDFlag); 1828 1829 ZSTD_freeCCtx(cctx); 1830 } 1831 1832 DISPLAYLEVEL(3, "test%3d : ZSTD_CCtx_setParams() : ", testNb++); 1833 { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); 1834 int value; 1835 ZSTD_parameters params = ZSTD_getParams(1, 0, 0); 1836 params.cParams.strategy = (ZSTD_strategy)-1; /* set invalid value, on purpose */ 1837 /* Set invalid params == error out, and no change. */ 1838 CHECK(ZSTD_isError(ZSTD_CCtx_setParams(cctx, params))); 1839 1840 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_windowLog, &value)); 1841 CHECK_EQ(value, 0); 1842 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_chainLog, &value)); 1843 CHECK_EQ(value, 0); 1844 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_hashLog, &value)); 1845 CHECK_EQ(value, 0); 1846 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_searchLog, &value)); 1847 CHECK_EQ(value, 0); 1848 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_minMatch, &value)); 1849 CHECK_EQ(value, 0); 1850 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_targetLength, &value)); 1851 CHECK_EQ(value, 0); 1852 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_strategy, &value)); 1853 CHECK_EQ(value, 0); 1854 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_contentSizeFlag, &value)); 1855 CHECK_EQ(value, 1); 1856 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_checksumFlag, &value)); 1857 CHECK_EQ(value, 0); 1858 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_dictIDFlag, &value)); 1859 CHECK_EQ(value, 1); 1860 1861 params = ZSTD_getParams(12, 0, 0); 1862 params.fParams.contentSizeFlag = 0; 1863 params.fParams.checksumFlag = 1; 1864 params.fParams.noDictIDFlag = 1; 1865 CHECK_Z(ZSTD_CCtx_setParams(cctx, params)); 1866 1867 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_windowLog, &value)); 1868 CHECK_EQ(value, (int)params.cParams.windowLog); 1869 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_chainLog, &value)); 1870 CHECK_EQ(value, (int)params.cParams.chainLog); 1871 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_hashLog, &value)); 1872 CHECK_EQ(value, (int)params.cParams.hashLog); 1873 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_searchLog, &value)); 1874 CHECK_EQ(value, (int)params.cParams.searchLog); 1875 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_minMatch, &value)); 1876 CHECK_EQ(value, (int)params.cParams.minMatch); 1877 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_targetLength, &value)); 1878 CHECK_EQ(value, (int)params.cParams.targetLength); 1879 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_strategy, &value)); 1880 CHECK_EQ(value, (int)params.cParams.strategy); 1881 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_contentSizeFlag, &value)); 1882 CHECK_EQ(value, params.fParams.contentSizeFlag); 1883 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_checksumFlag, &value)); 1884 CHECK_EQ(value, params.fParams.checksumFlag); 1885 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_dictIDFlag, &value)); 1886 CHECK_EQ(value, !params.fParams.noDictIDFlag); 1887 1888 ZSTD_freeCCtx(cctx); 1889 } 1890 1891 DISPLAYLEVEL(3, "test%3d : ldm conditionally enabled by default doesn't change cctx params: ", testNb++); 1892 { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); 1893 ZSTD_outBuffer out = {NULL, 0, 0}; 1894 ZSTD_inBuffer in = {NULL, 0, 0}; 1895 int value; 1896 1897 /* Even if LDM will be enabled by default in the applied params (since wlog >= 27 and strategy >= btopt), 1898 * we should not modify the actual parameter specified by the user within the CCtx 1899 */ 1900 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, 27)); 1901 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_strategy, ZSTD_btopt)); 1902 1903 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_continue)); 1904 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_enableLongDistanceMatching, &value)); 1905 CHECK_EQ(value, 0); 1906 1907 ZSTD_freeCCtx(cctx); 1908 } 1909 DISPLAYLEVEL(3, "OK \n"); 1910 1911 /* this test is really too long, and should be made faster */ 1912 DISPLAYLEVEL(3, "test%3d : overflow protection with large windowLog : ", testNb++); 1913 { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); 1914 ZSTD_parameters params = ZSTD_getParams(-999, ZSTD_CONTENTSIZE_UNKNOWN, 0); 1915 size_t const nbCompressions = ((1U << 31) / CNBuffSize) + 2; /* ensure U32 overflow protection is triggered */ 1916 size_t cnb; 1917 assert(cctx != NULL); 1918 params.fParams.contentSizeFlag = 0; 1919 params.cParams.windowLog = ZSTD_WINDOWLOG_MAX; 1920 for (cnb = 0; cnb < nbCompressions; ++cnb) { 1921 DISPLAYLEVEL(6, "run %u / %u \n", (unsigned)cnb, (unsigned)nbCompressions); 1922 CHECK_Z( ZSTD_compressBegin_advanced(cctx, NULL, 0, params, ZSTD_CONTENTSIZE_UNKNOWN) ); /* reuse same parameters */ 1923 CHECK_Z( ZSTD_compressEnd(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize) ); 1924 } 1925 ZSTD_freeCCtx(cctx); 1926 } 1927 DISPLAYLEVEL(3, "OK \n"); 1928 1929 DISPLAYLEVEL(3, "test%3d : size down context : ", testNb++); 1930 { ZSTD_CCtx* const largeCCtx = ZSTD_createCCtx(); 1931 assert(largeCCtx != NULL); 1932 CHECK_Z( ZSTD_compressBegin(largeCCtx, 19) ); /* streaming implies ZSTD_CONTENTSIZE_UNKNOWN, which maximizes memory usage */ 1933 CHECK_Z( ZSTD_compressEnd(largeCCtx, compressedBuffer, compressedBufferSize, CNBuffer, 1) ); 1934 { size_t const largeCCtxSize = ZSTD_sizeof_CCtx(largeCCtx); /* size of context must be measured after compression */ 1935 { ZSTD_CCtx* const smallCCtx = ZSTD_createCCtx(); 1936 assert(smallCCtx != NULL); 1937 CHECK_Z(ZSTD_compressCCtx(smallCCtx, compressedBuffer, compressedBufferSize, CNBuffer, 1, 1)); 1938 { size_t const smallCCtxSize = ZSTD_sizeof_CCtx(smallCCtx); 1939 DISPLAYLEVEL(5, "(large) %uKB > 32*%uKB (small) : ", 1940 (unsigned)(largeCCtxSize>>10), (unsigned)(smallCCtxSize>>10)); 1941 assert(largeCCtxSize > 32* smallCCtxSize); /* note : "too large" definition is handled within zstd_compress.c . 1942 * make this test case extreme, so that it doesn't depend on a possibly fluctuating definition */ 1943 } 1944 ZSTD_freeCCtx(smallCCtx); 1945 } 1946 { U32 const maxNbAttempts = 1100; /* nb of usages before triggering size down is handled within zstd_compress.c. 1947 * currently defined as 128x, but could be adjusted in the future. 1948 * make this test long enough so that it's not too much tied to the current definition within zstd_compress.c */ 1949 unsigned u; 1950 for (u=0; u<maxNbAttempts; u++) { 1951 CHECK_Z(ZSTD_compressCCtx(largeCCtx, compressedBuffer, compressedBufferSize, CNBuffer, 1, 1)); 1952 if (ZSTD_sizeof_CCtx(largeCCtx) < largeCCtxSize) break; /* sized down */ 1953 } 1954 DISPLAYLEVEL(5, "size down after %u attempts : ", u); 1955 if (u==maxNbAttempts) goto _output_error; /* no sizedown happened */ 1956 } 1957 } 1958 ZSTD_freeCCtx(largeCCtx); 1959 } 1960 DISPLAYLEVEL(3, "OK \n"); 1961 1962 /* Static CCtx tests */ 1963 #define STATIC_CCTX_LEVEL 4 1964 DISPLAYLEVEL(3, "test%3i : create static CCtx for level %u : ", testNb++, STATIC_CCTX_LEVEL); 1965 { size_t const staticCStreamSize = ZSTD_estimateCStreamSize(STATIC_CCTX_LEVEL); 1966 void* const staticCCtxBuffer = malloc(staticCStreamSize); 1967 size_t const staticDCtxSize = ZSTD_estimateDCtxSize(); 1968 void* const staticDCtxBuffer = malloc(staticDCtxSize); 1969 DISPLAYLEVEL(4, "CStream size = %u, ", (U32)staticCStreamSize); 1970 if (staticCCtxBuffer==NULL || staticDCtxBuffer==NULL) { 1971 free(staticCCtxBuffer); 1972 free(staticDCtxBuffer); 1973 DISPLAY("Not enough memory, aborting\n"); 1974 testResult = 1; 1975 goto _end; 1976 } 1977 { size_t const smallInSize = 32 KB; 1978 ZSTD_compressionParameters const cparams_small = ZSTD_getCParams(STATIC_CCTX_LEVEL, smallInSize, 0); 1979 size_t const smallCCtxSize = ZSTD_estimateCCtxSize_usingCParams(cparams_small); 1980 size_t const staticCCtxSize = ZSTD_estimateCCtxSize(STATIC_CCTX_LEVEL); 1981 ZSTD_CCtx* staticCCtx = ZSTD_initStaticCCtx(staticCCtxBuffer, smallCCtxSize); 1982 ZSTD_DCtx* const staticDCtx = ZSTD_initStaticDCtx(staticDCtxBuffer, staticDCtxSize); 1983 DISPLAYLEVEL(4, "Full CCtx size = %u, ", (U32)staticCCtxSize); 1984 DISPLAYLEVEL(4, "CCtx for 32 KB = %u, ", (U32)smallCCtxSize); 1985 if ((staticCCtx==NULL) || (staticDCtx==NULL)) goto _output_error; 1986 DISPLAYLEVEL(3, "OK \n"); 1987 1988 DISPLAYLEVEL(3, "test%3i : compress small input with small static CCtx : ", testNb++); 1989 CHECK_VAR(cSize, ZSTD_compressCCtx(staticCCtx, 1990 compressedBuffer, compressedBufferSize, 1991 CNBuffer, smallInSize, STATIC_CCTX_LEVEL) ); 1992 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", 1993 (unsigned)cSize, (double)cSize/smallInSize*100); 1994 1995 DISPLAYLEVEL(3, "test%3i : compress large input with small static CCtx (must fail) : ", testNb++); 1996 { size_t const r = ZSTD_compressCCtx(staticCCtx, 1997 compressedBuffer, compressedBufferSize, 1998 CNBuffer, CNBuffSize, STATIC_CCTX_LEVEL); 1999 if (ZSTD_getErrorCode((size_t)r) != ZSTD_error_memory_allocation) goto _output_error; 2000 } 2001 DISPLAYLEVEL(3, "OK \n"); 2002 2003 DISPLAYLEVEL(3, "test%3i : resize context to full CCtx size : ", testNb++); 2004 staticCCtx = ZSTD_initStaticCStream(staticCCtxBuffer, staticCCtxSize); 2005 DISPLAYLEVEL(4, "staticCCtxBuffer = %p, staticCCtx = %p , ", staticCCtxBuffer, (void*)staticCCtx); 2006 if (staticCCtx == NULL) goto _output_error; 2007 DISPLAYLEVEL(3, "OK \n"); 2008 2009 DISPLAYLEVEL(3, "test%3i : compress large input with static CCtx : ", testNb++); 2010 CHECK_VAR(cSize, ZSTD_compressCCtx(staticCCtx, 2011 compressedBuffer, compressedBufferSize, 2012 CNBuffer, CNBuffSize, STATIC_CCTX_LEVEL) ); 2013 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", 2014 (unsigned)cSize, (double)cSize/CNBuffSize*100); 2015 2016 DISPLAYLEVEL(3, "test%3i : compress small input often enough to trigger context reduce : ", testNb++); 2017 { int nbc; 2018 assert(staticCCtxSize > smallCCtxSize * ZSTD_WORKSPACETOOLARGE_FACTOR); /* ensure size down scenario */ 2019 assert(CNBuffSize > smallInSize + ZSTD_WORKSPACETOOLARGE_MAXDURATION + 3); 2020 for (nbc=0; nbc<ZSTD_WORKSPACETOOLARGE_MAXDURATION+2; nbc++) { 2021 CHECK_Z(ZSTD_compressCCtx(staticCCtx, 2022 compressedBuffer, compressedBufferSize, 2023 (char*)CNBuffer + nbc, smallInSize, 2024 STATIC_CCTX_LEVEL) ); 2025 } } 2026 DISPLAYLEVEL(3, "OK \n") 2027 2028 DISPLAYLEVEL(3, "test%3i : init CCtx for level %u : ", testNb++, STATIC_CCTX_LEVEL); 2029 CHECK_Z( ZSTD_compressBegin(staticCCtx, STATIC_CCTX_LEVEL) ); 2030 DISPLAYLEVEL(3, "OK \n"); 2031 2032 DISPLAYLEVEL(3, "test%3i : compression again with static CCtx : ", testNb++); 2033 CHECK_VAR(cSize, ZSTD_compressCCtx(staticCCtx, 2034 compressedBuffer, compressedBufferSize, 2035 CNBuffer, CNBuffSize, STATIC_CCTX_LEVEL) ); 2036 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", 2037 (unsigned)cSize, (double)cSize/CNBuffSize*100); 2038 2039 DISPLAYLEVEL(3, "test%3i : simple decompression test with static DCtx : ", testNb++); 2040 { size_t const r = ZSTD_decompressDCtx(staticDCtx, 2041 decodedBuffer, CNBuffSize, 2042 compressedBuffer, cSize); 2043 if (r != CNBuffSize) goto _output_error; } 2044 DISPLAYLEVEL(3, "OK \n"); 2045 2046 DISPLAYLEVEL(3, "test%3i : check decompressed result : ", testNb++); 2047 if (memcmp(decodedBuffer, CNBuffer, CNBuffSize)) goto _output_error; 2048 DISPLAYLEVEL(3, "OK \n"); 2049 2050 DISPLAYLEVEL(3, "test%3i : init CCtx for too large level (must fail) : ", testNb++); 2051 { size_t const r = ZSTD_compressBegin(staticCCtx, ZSTD_maxCLevel()); 2052 if (!ZSTD_isError(r)) goto _output_error; } 2053 DISPLAYLEVEL(3, "OK \n"); 2054 2055 DISPLAYLEVEL(3, "test%3i : init CCtx for small level %u (should work again) : ", testNb++, 1); 2056 CHECK_Z( ZSTD_compressBegin(staticCCtx, 1) ); 2057 DISPLAYLEVEL(3, "OK \n"); 2058 2059 DISPLAYLEVEL(3, "test%3i : use CStream on CCtx-sized static context (should fail) : ", testNb++); 2060 CHECK_Z( ZSTD_initCStream(staticCCtx, STATIC_CCTX_LEVEL) ); /* note : doesn't allocate */ 2061 { ZSTD_outBuffer output = { compressedBuffer, compressedBufferSize, 0 }; 2062 ZSTD_inBuffer input = { CNBuffer, CNBuffSize, 0 }; 2063 size_t const r = ZSTD_compressStream(staticCCtx, &output, &input); /* now allocates, should fail */ 2064 if (!ZSTD_isError(r)) goto _output_error; 2065 } 2066 DISPLAYLEVEL(3, "OK \n"); 2067 2068 DISPLAYLEVEL(3, "test%3i : resize context to CStream size, then stream compress : ", testNb++); 2069 staticCCtx = ZSTD_initStaticCStream(staticCCtxBuffer, staticCStreamSize); 2070 assert(staticCCtx != NULL); 2071 CHECK_Z( ZSTD_initCStream(staticCCtx, STATIC_CCTX_LEVEL) ); /* note : doesn't allocate */ 2072 { ZSTD_outBuffer output = { compressedBuffer, compressedBufferSize, 0 }; 2073 ZSTD_inBuffer input = { CNBuffer, CNBuffSize, 0 }; 2074 CHECK_Z( ZSTD_compressStream(staticCCtx, &output, &input) ); 2075 } 2076 DISPLAYLEVEL(3, "OK \n"); 2077 2078 DISPLAYLEVEL(3, "test%3i : CStream for small level %u : ", testNb++, 1); 2079 CHECK_Z( ZSTD_initCStream(staticCCtx, 1) ); /* note : doesn't allocate */ 2080 { ZSTD_outBuffer output = { compressedBuffer, compressedBufferSize, 0 }; 2081 ZSTD_inBuffer input = { CNBuffer, CNBuffSize, 0 }; 2082 CHECK_Z( ZSTD_compressStream(staticCCtx, &output, &input) ); 2083 } 2084 DISPLAYLEVEL(3, "OK \n"); 2085 2086 DISPLAYLEVEL(3, "test%3i : init static CStream with dictionary (should fail) : ", testNb++); 2087 { size_t const r = ZSTD_initCStream_usingDict(staticCCtx, CNBuffer, 64 KB, 1); 2088 if (!ZSTD_isError(r)) goto _output_error; } 2089 DISPLAYLEVEL(3, "OK \n"); 2090 2091 DISPLAYLEVEL(3, "test%3i : use DStream on DCtx-sized static context (should fail) : ", testNb++); 2092 CHECK_Z( ZSTD_initDStream(staticDCtx) ); 2093 { ZSTD_outBuffer output = { decodedBuffer, CNBuffSize, 0 }; 2094 ZSTD_inBuffer input = { compressedBuffer, ZSTD_FRAMEHEADERSIZE_MAX+1, 0 }; 2095 size_t const r = ZSTD_decompressStream(staticDCtx, &output, &input); 2096 if (!ZSTD_isError(r)) goto _output_error; 2097 } 2098 DISPLAYLEVEL(3, "OK \n"); 2099 2100 DISPLAYLEVEL(3, "test%3i : test estimation functions with default cctx params : ", testNb++); 2101 { 2102 // Test ZSTD_estimateCCtxSize_usingCCtxParams 2103 { 2104 ZSTD_CCtx_params* params = ZSTD_createCCtxParams(); 2105 size_t const cctxSizeDefault = ZSTD_estimateCCtxSize_usingCCtxParams(params); 2106 staticCCtx = ZSTD_initStaticCCtx(staticCCtxBuffer, cctxSizeDefault); 2107 CHECK_VAR(cSize, ZSTD_compressCCtx(staticCCtx, 2108 compressedBuffer, compressedBufferSize, 2109 CNBuffer, CNBuffSize, 3)); 2110 2111 { 2112 size_t const r = ZSTD_decompressDCtx(staticDCtx, 2113 decodedBuffer, CNBuffSize, 2114 compressedBuffer, cSize); 2115 if (r != CNBuffSize) goto _output_error; 2116 if (memcmp(decodedBuffer, CNBuffer, CNBuffSize)) goto _output_error; 2117 } 2118 ZSTD_freeCCtxParams(params); 2119 } 2120 2121 // Test ZSTD_estimateCStreamSize_usingCCtxParams 2122 { 2123 ZSTD_CCtx_params* params = ZSTD_createCCtxParams(); 2124 size_t const cctxSizeDefault = ZSTD_estimateCStreamSize_usingCCtxParams(params); 2125 staticCCtx = ZSTD_initStaticCCtx(staticCCtxBuffer, cctxSizeDefault); 2126 CHECK_VAR(cSize, ZSTD_compressCCtx(staticCCtx, 2127 compressedBuffer, compressedBufferSize, 2128 CNBuffer, CNBuffSize, 3) ); 2129 2130 { 2131 size_t const r = ZSTD_decompressDCtx(staticDCtx, 2132 decodedBuffer, CNBuffSize, 2133 compressedBuffer, cSize); 2134 if (r != CNBuffSize) goto _output_error; 2135 if (memcmp(decodedBuffer, CNBuffer, CNBuffSize)) goto _output_error; 2136 } 2137 ZSTD_freeCCtxParams(params); 2138 } 2139 } 2140 DISPLAYLEVEL(3, "OK \n"); 2141 2142 DISPLAYLEVEL(3, "test%3i : test estimation functions with maxBlockSize = 0 : ", testNb++); 2143 { 2144 // Test ZSTD_estimateCCtxSize_usingCCtxParams 2145 { 2146 ZSTD_CCtx_params* params = ZSTD_createCCtxParams(); 2147 size_t cctxSizeDefault; 2148 CHECK_Z(ZSTD_CCtxParams_setParameter(params, ZSTD_c_maxBlockSize, 0)); 2149 cctxSizeDefault = ZSTD_estimateCCtxSize_usingCCtxParams(params); 2150 staticCCtx = ZSTD_initStaticCCtx(staticCCtxBuffer, cctxSizeDefault); 2151 CHECK_VAR(cSize, ZSTD_compressCCtx(staticCCtx, 2152 compressedBuffer, compressedBufferSize, 2153 CNBuffer, CNBuffSize, 3) ); 2154 2155 { 2156 size_t const r = ZSTD_decompressDCtx(staticDCtx, 2157 decodedBuffer, CNBuffSize, 2158 compressedBuffer, cSize); 2159 if (r != CNBuffSize) goto _output_error; 2160 if (memcmp(decodedBuffer, CNBuffer, CNBuffSize)) goto _output_error; 2161 } 2162 ZSTD_freeCCtxParams(params); 2163 } 2164 2165 // Test ZSTD_estimateCStreamSize_usingCCtxParams 2166 { 2167 ZSTD_CCtx_params* params = ZSTD_createCCtxParams(); 2168 size_t cctxSizeDefault; 2169 CHECK_Z(ZSTD_CCtxParams_setParameter(params, ZSTD_c_maxBlockSize, 0)); 2170 cctxSizeDefault = ZSTD_estimateCStreamSize_usingCCtxParams(params); 2171 staticCCtx = ZSTD_initStaticCCtx(staticCCtxBuffer, cctxSizeDefault); 2172 CHECK_VAR(cSize, ZSTD_compressCCtx(staticCCtx, 2173 compressedBuffer, compressedBufferSize, 2174 CNBuffer, CNBuffSize, 3) ); 2175 2176 { 2177 size_t const r = ZSTD_decompressDCtx(staticDCtx, 2178 decodedBuffer, CNBuffSize, 2179 compressedBuffer, cSize); 2180 if (r != CNBuffSize) goto _output_error; 2181 if (memcmp(decodedBuffer, CNBuffer, CNBuffSize)) goto _output_error; 2182 } 2183 ZSTD_freeCCtxParams(params); 2184 } 2185 } 2186 DISPLAYLEVEL(3, "OK \n"); 2187 } 2188 free(staticCCtxBuffer); 2189 free(staticDCtxBuffer); 2190 } 2191 2192 DISPLAYLEVEL(3, "test%3i : Static context sizes for negative levels : ", testNb++); 2193 { size_t const cctxSizeN1 = ZSTD_estimateCCtxSize(-1); 2194 size_t const cctxSizeP1 = ZSTD_estimateCCtxSize(1); 2195 size_t const cstreamSizeN1 = ZSTD_estimateCStreamSize(-1); 2196 size_t const cstreamSizeP1 = ZSTD_estimateCStreamSize(1); 2197 2198 if (!(0 < cctxSizeN1 && cctxSizeN1 <= cctxSizeP1)) goto _output_error; 2199 if (!(0 < cstreamSizeN1 && cstreamSizeN1 <= cstreamSizeP1)) goto _output_error; 2200 } 2201 DISPLAYLEVEL(3, "OK \n"); 2202 2203 2204 /* ZSTDMT simple MT compression test */ 2205 DISPLAYLEVEL(3, "test%3i : create ZSTDMT CCtx : ", testNb++); 2206 { ZSTD_CCtx* const mtctx = ZSTD_createCCtx(); 2207 if (mtctx==NULL) { 2208 DISPLAY("mtctx : not enough memory, aborting \n"); 2209 testResult = 1; 2210 goto _end; 2211 } 2212 CHECK_Z( ZSTD_CCtx_setParameter(mtctx, ZSTD_c_nbWorkers, 2) ); 2213 CHECK_Z( ZSTD_CCtx_setParameter(mtctx, ZSTD_c_compressionLevel, 1) ); 2214 DISPLAYLEVEL(3, "OK \n"); 2215 2216 DISPLAYLEVEL(3, "test%3u : compress %u bytes with 2 threads : ", testNb++, (unsigned)CNBuffSize); 2217 CHECK_VAR(cSize, ZSTD_compress2(mtctx, 2218 compressedBuffer, compressedBufferSize, 2219 CNBuffer, CNBuffSize) ); 2220 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBuffSize*100); 2221 2222 DISPLAYLEVEL(3, "test%3i : decompressed size test : ", testNb++); 2223 { unsigned long long const rSize = ZSTD_getFrameContentSize(compressedBuffer, cSize); 2224 if (rSize != CNBuffSize) { 2225 DISPLAY("ZSTD_getFrameContentSize incorrect : %u != %u \n", (unsigned)rSize, (unsigned)CNBuffSize); 2226 goto _output_error; 2227 } } 2228 DISPLAYLEVEL(3, "OK \n"); 2229 2230 DISPLAYLEVEL(3, "test%3i : decompress %u bytes : ", testNb++, (unsigned)CNBuffSize); 2231 { size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize); 2232 if (r != CNBuffSize) goto _output_error; } 2233 DISPLAYLEVEL(3, "OK \n"); 2234 2235 DISPLAYLEVEL(3, "test%3i : check decompressed result : ", testNb++); 2236 { size_t u; 2237 for (u=0; u<CNBuffSize; u++) { 2238 if (((BYTE*)decodedBuffer)[u] != ((BYTE*)CNBuffer)[u]) goto _output_error; 2239 } } 2240 DISPLAYLEVEL(3, "OK \n"); 2241 2242 DISPLAYLEVEL(3, "test%3i : compress -T2 with checksum : ", testNb++); 2243 CHECK_Z( ZSTD_CCtx_setParameter(mtctx, ZSTD_c_checksumFlag, 1) ); 2244 CHECK_Z( ZSTD_CCtx_setParameter(mtctx, ZSTD_c_contentSizeFlag, 1) ); 2245 CHECK_Z( ZSTD_CCtx_setParameter(mtctx, ZSTD_c_overlapLog, 3) ); 2246 CHECK_VAR(cSize, ZSTD_compress2(mtctx, 2247 compressedBuffer, compressedBufferSize, 2248 CNBuffer, CNBuffSize) ); 2249 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBuffSize*100); 2250 2251 DISPLAYLEVEL(3, "test%3i : decompress %u bytes : ", testNb++, (unsigned)CNBuffSize); 2252 { size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize); 2253 if (r != CNBuffSize) goto _output_error; } 2254 DISPLAYLEVEL(3, "OK \n"); 2255 2256 ZSTD_freeCCtx(mtctx); 2257 } 2258 2259 DISPLAYLEVEL(3, "test%3u : compress empty string and decompress with small window log : ", testNb++); 2260 { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); 2261 ZSTD_DCtx* const dctx = ZSTD_createDCtx(); 2262 char out[32]; 2263 if (cctx == NULL || dctx == NULL) goto _output_error; 2264 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_contentSizeFlag, 0) ); 2265 CHECK_VAR(cSize, ZSTD_compress2(cctx, out, sizeof(out), NULL, 0) ); 2266 DISPLAYLEVEL(3, "OK (%u bytes)\n", (unsigned)cSize); 2267 2268 CHECK_Z( ZSTD_DCtx_setParameter(dctx, ZSTD_d_windowLogMax, 10) ); 2269 { char const* outPtr = out; 2270 ZSTD_inBuffer inBuffer = { outPtr, cSize, 0 }; 2271 ZSTD_outBuffer outBuffer = { NULL, 0, 0 }; 2272 size_t dSize; 2273 CHECK_VAR(dSize, ZSTD_decompressStream(dctx, &outBuffer, &inBuffer) ); 2274 if (dSize != 0) goto _output_error; 2275 } 2276 2277 ZSTD_freeDCtx(dctx); 2278 ZSTD_freeCCtx(cctx); 2279 } 2280 2281 DISPLAYLEVEL(3, "test%3i : compress with block splitting : ", testNb++) 2282 { ZSTD_CCtx* cctx = ZSTD_createCCtx(); 2283 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_splitAfterSequences, ZSTD_ps_enable) ); 2284 cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize); 2285 CHECK_Z(cSize); 2286 ZSTD_freeCCtx(cctx); 2287 } 2288 DISPLAYLEVEL(3, "OK \n"); 2289 2290 DISPLAYLEVEL(3, "test%3i : compress -T2 with/without literals compression : ", testNb++) 2291 { ZSTD_CCtx* cctx = ZSTD_createCCtx(); 2292 size_t cSize1, cSize2; 2293 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 1) ); 2294 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 2) ); 2295 cSize1 = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize); 2296 CHECK_Z(cSize1); 2297 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_literalCompressionMode, ZSTD_ps_disable) ); 2298 cSize2 = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize); 2299 CHECK_Z(cSize2); 2300 CHECK_LT(cSize1, cSize2); 2301 ZSTD_freeCCtx(cctx); 2302 } 2303 DISPLAYLEVEL(3, "OK \n"); 2304 2305 DISPLAYLEVEL(3, "test%3i : Multithreaded ZSTD_compress2() with rsyncable : ", testNb++) 2306 { ZSTD_CCtx* cctx = ZSTD_createCCtx(); 2307 /* Set rsyncable and don't give the ZSTD_compressBound(CNBuffSize) so 2308 * ZSTDMT is forced to not take the shortcut. 2309 */ 2310 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 1) ); 2311 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 1) ); 2312 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_rsyncable, 1) ); 2313 CHECK_Z( ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize - 1, CNBuffer, CNBuffSize) ); 2314 ZSTD_freeCCtx(cctx); 2315 } 2316 DISPLAYLEVEL(3, "OK \n"); 2317 2318 DISPLAYLEVEL(3, "test%3i : setting multithreaded parameters : ", testNb++) 2319 { ZSTD_CCtx_params* params = ZSTD_createCCtxParams(); 2320 int const jobSize = 512 KB; 2321 int value; 2322 /* Check that the overlap log and job size are unset. */ 2323 CHECK_Z( ZSTD_CCtxParams_getParameter(params, ZSTD_c_overlapLog, &value) ); 2324 CHECK_EQ(value, 0); 2325 CHECK_Z( ZSTD_CCtxParams_getParameter(params, ZSTD_c_jobSize, &value) ); 2326 CHECK_EQ(value, 0); 2327 /* Set and check the overlap log and job size. */ 2328 CHECK_Z( ZSTD_CCtxParams_setParameter(params, ZSTD_c_overlapLog, 5) ); 2329 CHECK_Z( ZSTD_CCtxParams_setParameter(params, ZSTD_c_jobSize, jobSize) ); 2330 CHECK_Z( ZSTD_CCtxParams_getParameter(params, ZSTD_c_overlapLog, &value) ); 2331 CHECK_EQ(value, 5); 2332 CHECK_Z( ZSTD_CCtxParams_getParameter(params, ZSTD_c_jobSize, &value) ); 2333 CHECK_EQ(value, jobSize); 2334 /* Set the number of workers and check the overlap log and job size. */ 2335 CHECK_Z( ZSTD_CCtxParams_setParameter(params, ZSTD_c_nbWorkers, 2) ); 2336 CHECK_Z( ZSTD_CCtxParams_getParameter(params, ZSTD_c_overlapLog, &value) ); 2337 CHECK_EQ(value, 5); 2338 CHECK_Z( ZSTD_CCtxParams_getParameter(params, ZSTD_c_jobSize, &value) ); 2339 CHECK_EQ(value, jobSize); 2340 ZSTD_freeCCtxParams(params); 2341 } 2342 DISPLAYLEVEL(3, "OK \n"); 2343 2344 /* Simple API multiframe test */ 2345 DISPLAYLEVEL(3, "test%3i : compress multiple frames : ", testNb++); 2346 { size_t off = 0; 2347 int i; 2348 int const segs = 4; 2349 /* only use the first half so we don't push against size limit of compressedBuffer */ 2350 size_t const segSize = (CNBuffSize / 2) / segs; 2351 2352 const U32 skipLen = 129 KB; 2353 char* const skipBuff = (char*)malloc(skipLen); 2354 assert(skipBuff != NULL); 2355 memset(skipBuff, 0, skipLen); 2356 for (i = 0; i < segs; i++) { 2357 CHECK_NEWV(r, ZSTD_compress( 2358 (BYTE*)compressedBuffer + off, CNBuffSize - off, 2359 (BYTE*)CNBuffer + segSize * (size_t)i, segSize, 2360 5) ); 2361 off += r; 2362 if (i == segs/2) { 2363 /* insert skippable frame */ 2364 size_t const skippableSize = 2365 ZSTD_writeSkippableFrame((BYTE*)compressedBuffer + off, compressedBufferSize, 2366 skipBuff, skipLen, seed % 15); 2367 CHECK_Z(skippableSize); 2368 off += skippableSize; 2369 } 2370 } 2371 cSize = off; 2372 free(skipBuff); 2373 } 2374 DISPLAYLEVEL(3, "OK \n"); 2375 2376 DISPLAYLEVEL(3, "test%3i : get decompressed size of multiple frames : ", testNb++); 2377 { unsigned long long const r = ZSTD_findDecompressedSize(compressedBuffer, cSize); 2378 if (r != CNBuffSize / 2) goto _output_error; } 2379 DISPLAYLEVEL(3, "OK \n"); 2380 2381 DISPLAYLEVEL(3, "test%3i : get tight decompressed bound of multiple frames : ", testNb++); 2382 { unsigned long long const bound = ZSTD_decompressBound(compressedBuffer, cSize); 2383 if (bound != CNBuffSize / 2) goto _output_error; } 2384 DISPLAYLEVEL(3, "OK \n"); 2385 2386 DISPLAYLEVEL(3, "test%3i : decompress multiple frames : ", testNb++); 2387 { CHECK_NEWV(r, ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize)); 2388 if (r != CNBuffSize / 2) goto _output_error; } 2389 DISPLAYLEVEL(3, "OK \n"); 2390 2391 DISPLAYLEVEL(3, "test%3i : check decompressed result : ", testNb++); 2392 if (memcmp(decodedBuffer, CNBuffer, CNBuffSize / 2) != 0) goto _output_error; 2393 DISPLAYLEVEL(3, "OK \n"); 2394 2395 /* Simple API skippable frame test */ 2396 DISPLAYLEVEL(3, "test%3i : read/write a skippable frame : ", testNb++); 2397 { U32 i; 2398 unsigned readMagic; 2399 unsigned long long receivedSize; 2400 size_t skippableSize; 2401 const U32 skipLen = 129 KB; 2402 char* const skipBuff = (char*)malloc(skipLen); 2403 assert(skipBuff != NULL); 2404 for (i = 0; i < skipLen; i++) 2405 skipBuff[i] = (char) ((seed + i) % 256); 2406 skippableSize = ZSTD_writeSkippableFrame( 2407 compressedBuffer, compressedBufferSize, 2408 skipBuff, skipLen, seed % 15); 2409 CHECK_Z(skippableSize); 2410 CHECK_EQ(1, ZSTD_isSkippableFrame(compressedBuffer, skippableSize)); 2411 receivedSize = ZSTD_readSkippableFrame(decodedBuffer, CNBuffSize, &readMagic, compressedBuffer, skippableSize); 2412 CHECK_EQ(skippableSize, receivedSize + ZSTD_SKIPPABLEHEADERSIZE); 2413 CHECK_EQ(seed % 15, readMagic); 2414 if (memcmp(decodedBuffer, skipBuff, skipLen) != 0) goto _output_error; 2415 2416 free(skipBuff); 2417 } 2418 DISPLAYLEVEL(3, "OK \n"); 2419 2420 DISPLAYLEVEL(3, "test%3i : read/write an empty skippable frame : ", testNb++); 2421 { 2422 unsigned readMagic; 2423 unsigned long long receivedSize; 2424 size_t skippableSize; 2425 skippableSize = ZSTD_writeSkippableFrame( 2426 compressedBuffer, compressedBufferSize, 2427 CNBuffer, 0, seed % 15); 2428 CHECK_EQ(ZSTD_SKIPPABLEHEADERSIZE, skippableSize); 2429 CHECK_EQ(1, ZSTD_isSkippableFrame(compressedBuffer, skippableSize)); 2430 receivedSize = ZSTD_readSkippableFrame(NULL, 0, &readMagic, compressedBuffer, skippableSize); 2431 CHECK_EQ(skippableSize, receivedSize + ZSTD_SKIPPABLEHEADERSIZE); 2432 CHECK_EQ(seed % 15, readMagic); 2433 } 2434 DISPLAYLEVEL(3, "OK \n"); 2435 2436 /* Dictionary and CCtx Duplication tests */ 2437 { ZSTD_CCtx* const ctxOrig = ZSTD_createCCtx(); 2438 ZSTD_CCtx* const ctxDuplicated = ZSTD_createCCtx(); 2439 ZSTD_DCtx* const dctx = ZSTD_createDCtx(); 2440 static const size_t dictSize = 551; 2441 assert(dctx != NULL); assert(ctxOrig != NULL); assert(ctxDuplicated != NULL); 2442 2443 DISPLAYLEVEL(3, "test%3i : copy context too soon : ", testNb++); 2444 { size_t const copyResult = ZSTD_copyCCtx(ctxDuplicated, ctxOrig, 0); 2445 if (!ZSTD_isError(copyResult)) goto _output_error; } /* error must be detected */ 2446 DISPLAYLEVEL(3, "OK \n"); 2447 2448 DISPLAYLEVEL(3, "test%3i : load dictionary into context : ", testNb++); 2449 CHECK_Z( ZSTD_compressBegin_usingDict(ctxOrig, CNBuffer, dictSize, 2) ); 2450 CHECK_Z( ZSTD_copyCCtx(ctxDuplicated, ctxOrig, 0) ); /* Begin_usingDict implies unknown srcSize, so match that */ 2451 DISPLAYLEVEL(3, "OK \n"); 2452 2453 DISPLAYLEVEL(3, "test%3i : compress with flat dictionary : ", testNb++); 2454 cSize = 0; 2455 CHECKPLUS(r, ZSTD_compressEnd(ctxOrig, 2456 compressedBuffer, compressedBufferSize, 2457 (const char*)CNBuffer + dictSize, CNBuffSize - dictSize), 2458 cSize += r); 2459 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBuffSize*100); 2460 2461 DISPLAYLEVEL(3, "test%3i : frame built with flat dictionary should be decompressible : ", testNb++); 2462 CHECKPLUS(r, ZSTD_decompress_usingDict(dctx, 2463 decodedBuffer, CNBuffSize, 2464 compressedBuffer, cSize, 2465 CNBuffer, dictSize), 2466 if (r != CNBuffSize - dictSize) goto _output_error); 2467 DISPLAYLEVEL(3, "OK \n"); 2468 2469 DISPLAYLEVEL(3, "test%3i : compress with duplicated context : ", testNb++); 2470 { size_t const cSizeOrig = cSize; 2471 cSize = 0; 2472 CHECKPLUS(r, ZSTD_compressEnd(ctxDuplicated, 2473 compressedBuffer, compressedBufferSize, 2474 (const char*)CNBuffer + dictSize, CNBuffSize - dictSize), 2475 cSize += r); 2476 if (cSize != cSizeOrig) goto _output_error; /* should be identical ==> same size */ 2477 } 2478 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBuffSize*100); 2479 2480 DISPLAYLEVEL(3, "test%3i : frame built with duplicated context should be decompressible : ", testNb++); 2481 CHECKPLUS(r, ZSTD_decompress_usingDict(dctx, 2482 decodedBuffer, CNBuffSize, 2483 compressedBuffer, cSize, 2484 CNBuffer, dictSize), 2485 if (r != CNBuffSize - dictSize) goto _output_error); 2486 DISPLAYLEVEL(3, "OK \n"); 2487 2488 DISPLAYLEVEL(3, "test%3i : decompress with DDict : ", testNb++); 2489 { ZSTD_DDict* const ddict = ZSTD_createDDict(CNBuffer, dictSize); 2490 size_t const r = ZSTD_decompress_usingDDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, ddict); 2491 if (r != CNBuffSize - dictSize) goto _output_error; 2492 DISPLAYLEVEL(3, "OK (size of DDict : %u) \n", (unsigned)ZSTD_sizeof_DDict(ddict)); 2493 ZSTD_freeDDict(ddict); 2494 } 2495 2496 DISPLAYLEVEL(3, "test%3i : decompress with static DDict : ", testNb++); 2497 { size_t const ddictBufferSize = ZSTD_estimateDDictSize(dictSize, ZSTD_dlm_byCopy); 2498 void* const ddictBuffer = malloc(ddictBufferSize); 2499 if (ddictBuffer == NULL) goto _output_error; 2500 { const ZSTD_DDict* const ddict = ZSTD_initStaticDDict(ddictBuffer, ddictBufferSize, CNBuffer, dictSize, ZSTD_dlm_byCopy, ZSTD_dct_auto); 2501 size_t const r = ZSTD_decompress_usingDDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, ddict); 2502 if (r != CNBuffSize - dictSize) goto _output_error; 2503 } 2504 free(ddictBuffer); 2505 DISPLAYLEVEL(3, "OK (size of static DDict : %u) \n", (unsigned)ddictBufferSize); 2506 } 2507 2508 DISPLAYLEVEL(3, "test%3i : check content size on duplicated context : ", testNb++); 2509 { size_t const testSize = CNBuffSize / 3; 2510 CHECK_Z( ZSTD_compressBegin(ctxOrig, ZSTD_defaultCLevel()) ); 2511 CHECK_Z( ZSTD_copyCCtx(ctxDuplicated, ctxOrig, testSize) ); 2512 2513 CHECK_VAR(cSize, ZSTD_compressEnd(ctxDuplicated, compressedBuffer, ZSTD_compressBound(testSize), 2514 (const char*)CNBuffer + dictSize, testSize) ); 2515 { ZSTD_FrameHeader zfh; 2516 if (ZSTD_getFrameHeader(&zfh, compressedBuffer, cSize)) goto _output_error; 2517 if ((zfh.frameContentSize != testSize) && (zfh.frameContentSize != 0)) goto _output_error; 2518 } } 2519 DISPLAYLEVEL(3, "OK \n"); 2520 2521 #if !defined(ZSTD_EXCLUDE_DFAST_BLOCK_COMPRESSOR) \ 2522 && !defined(ZSTD_EXCLUDE_GREEDY_BLOCK_COMPRESSOR) \ 2523 && !defined(ZSTD_EXCLUDE_DFAST_BLOCK_COMPRESSOR) \ 2524 && !defined(ZSTD_EXCLUDE_LAZY_BLOCK_COMPRESSOR) \ 2525 && !defined(ZSTD_EXCLUDE_LAZY2_BLOCK_COMPRESSOR) \ 2526 && !defined(ZSTD_EXCLUDE_BTLAZY2_BLOCK_COMPRESSOR) \ 2527 && !defined(ZSTD_EXCLUDE_BTOPT_BLOCK_COMPRESSOR) \ 2528 && !defined(ZSTD_EXCLUDE_BTULTRA_BLOCK_COMPRESSOR) 2529 /* Note : these tests should be replaced by proper regression tests, 2530 * but existing ones do not focus on small data + dictionary + all levels. 2531 */ 2532 if ((int)(compressibility * 100 + 0.1) == FUZ_compressibility_default) { /* test only valid with known input */ 2533 size_t const flatdictSize = 22 KB; 2534 size_t const contentSize = 9 KB; 2535 const void* const dict = (const char*)CNBuffer; 2536 const void* const contentStart = (const char*)dict + flatdictSize; 2537 /* These upper bounds are generally within a few bytes of the compressed size */ 2538 size_t target_nodict_cSize[22+1] = { 3840, 3770, 3870, 3830, 3770, 2539 3770, 3770, 3770, 3750, 3750, 2540 3742, 3675, 3674, 3665, 3664, 2541 3663, 3662, 3661, 3660, 3660, 2542 3660, 3660, 3660 }; 2543 size_t const target_wdict_cSize[22+1] = { 2830, 2896, 2893, 2840, 2950, 2544 2950, 2950, 2925, 2900, 2892, 2545 2910, 2910, 2910, 2780, 2775, 2546 2765, 2760, 2755, 2754, 2753, 2547 2753, 2753, 2753 }; 2548 int l = 1; 2549 int const maxLevel = ZSTD_maxCLevel(); 2550 /* clevels with strategies that support rowhash on small inputs */ 2551 int rowLevel = 4; 2552 int const rowLevelEnd = 8; 2553 2554 DISPLAYLEVEL(3, "test%3i : flat-dictionary efficiency test : \n", testNb++); 2555 assert(maxLevel == 22); 2556 RDG_genBuffer(CNBuffer, flatdictSize + contentSize, compressibility, 0., seed); 2557 DISPLAYLEVEL(4, "content hash : %016llx; dict hash : %016llx \n", 2558 (unsigned long long)XXH64(contentStart, contentSize, 0), 2559 (unsigned long long)XXH64(dict, flatdictSize, 0)); 2560 2561 for ( ; l <= maxLevel; l++) { 2562 size_t const nodict_cSize = ZSTD_compress(compressedBuffer, compressedBufferSize, 2563 contentStart, contentSize, l); 2564 if (nodict_cSize > target_nodict_cSize[l]) { 2565 DISPLAYLEVEL(1, "error : compression at level %i worse than expected (%u > %u) \n", 2566 l, (unsigned)nodict_cSize, (unsigned)target_nodict_cSize[l]); 2567 goto _output_error; 2568 } 2569 DISPLAYLEVEL(4, "level %i : max expected %u >= reached %u \n", 2570 l, (unsigned)target_nodict_cSize[l], (unsigned)nodict_cSize); 2571 } 2572 for ( l=1 ; l <= maxLevel; l++) { 2573 size_t const wdict_cSize = ZSTD_compress_usingDict(ctxOrig, 2574 compressedBuffer, compressedBufferSize, 2575 contentStart, contentSize, 2576 dict, flatdictSize, 2577 l); 2578 if (wdict_cSize > target_wdict_cSize[l]) { 2579 DISPLAYLEVEL(1, "error : compression with dictionary at level %i worse than expected (%u > %u) \n", 2580 l, (unsigned)wdict_cSize, (unsigned)target_wdict_cSize[l]); 2581 goto _output_error; 2582 } 2583 DISPLAYLEVEL(4, "level %i with dictionary : max expected %u >= reached %u \n", 2584 l, (unsigned)target_wdict_cSize[l], (unsigned)wdict_cSize); 2585 } 2586 /* Compression with ZSTD_compress2 and row match finder force enabled. 2587 * Give some slack for force-enabled row matchfinder since we're on a small input (9KB) 2588 */ 2589 for ( ; rowLevel <= rowLevelEnd; ++rowLevel) target_nodict_cSize[rowLevel] += 5; 2590 for (l=1 ; l <= maxLevel; l++) { 2591 ZSTD_CCtx* const cctx = ZSTD_createCCtx(); 2592 size_t nodict_cSize; 2593 ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, l); 2594 ZSTD_CCtx_setParameter(cctx, ZSTD_c_useRowMatchFinder, ZSTD_ps_enable); 2595 nodict_cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, 2596 contentStart, contentSize); 2597 if (nodict_cSize > target_nodict_cSize[l]) { 2598 DISPLAYLEVEL(1, "error : compression with compress2 at level %i worse than expected (%u > %u) \n", 2599 l, (unsigned)nodict_cSize, (unsigned)target_nodict_cSize[l]); 2600 ZSTD_freeCCtx(cctx); 2601 goto _output_error; 2602 } 2603 DISPLAYLEVEL(4, "level %i with compress2 : max expected %u >= reached %u \n", 2604 l, (unsigned)target_nodict_cSize[l], (unsigned)nodict_cSize); 2605 ZSTD_freeCCtx(cctx); 2606 } 2607 /* Dict compression with DMS */ 2608 for ( l=1 ; l <= maxLevel; l++) { 2609 size_t wdict_cSize; 2610 CHECK_Z( ZSTD_CCtx_loadDictionary(ctxOrig, dict, flatdictSize) ); 2611 CHECK_Z( ZSTD_CCtx_setParameter(ctxOrig, ZSTD_c_compressionLevel, l) ); 2612 CHECK_Z( ZSTD_CCtx_setParameter(ctxOrig, ZSTD_c_enableDedicatedDictSearch, 0) ); 2613 CHECK_Z( ZSTD_CCtx_setParameter(ctxOrig, ZSTD_c_forceAttachDict, ZSTD_dictForceAttach) ); 2614 CHECK_Z( ZSTD_CCtx_setParameter(ctxOrig, ZSTD_c_prefetchCDictTables, seed % 3) ); 2615 wdict_cSize = ZSTD_compress2(ctxOrig, compressedBuffer, compressedBufferSize, contentStart, contentSize); 2616 if (wdict_cSize > target_wdict_cSize[l]) { 2617 DISPLAYLEVEL(1, "error : compression with dictionary and compress2 at level %i worse than expected (%u > %u) \n", 2618 l, (unsigned)wdict_cSize, (unsigned)target_wdict_cSize[l]); 2619 goto _output_error; 2620 } 2621 DISPLAYLEVEL(4, "level %i with dictionary and compress2 : max expected %u >= reached %u \n", 2622 l, (unsigned)target_wdict_cSize[l], (unsigned)wdict_cSize); 2623 } 2624 2625 DISPLAYLEVEL(4, "compression efficiency tests OK \n"); 2626 } 2627 #endif 2628 2629 ZSTD_freeCCtx(ctxOrig); 2630 ZSTD_freeCCtx(ctxDuplicated); 2631 ZSTD_freeDCtx(dctx); 2632 } 2633 2634 /* Dictionary and dictBuilder tests */ 2635 { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); 2636 size_t const dictBufferCapacity = 16 KB; 2637 void* const dictBuffer = malloc(dictBufferCapacity); 2638 size_t const totalSampleSize = 1 MB; 2639 size_t const sampleUnitSize = 8 KB; 2640 U32 const nbSamples = (U32)(totalSampleSize / sampleUnitSize); 2641 size_t* const samplesSizes = (size_t*) malloc(nbSamples * sizeof(size_t)); 2642 size_t dictSize; 2643 U32 dictID; 2644 size_t dictHeaderSize; 2645 size_t dictBufferFixedSize = 144; 2646 unsigned char const dictBufferFixed[144] = {0x37, 0xa4, 0x30, 0xec, 0x63, 0x00, 0x00, 0x00, 0x08, 0x10, 0x00, 0x1f, 2647 0x0f, 0x00, 0x28, 0xe5, 0x03, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 2648 0x00, 0x80, 0x0f, 0x9e, 0x0f, 0x00, 0x00, 0x24, 0x40, 0x80, 0x00, 0x01, 2649 0x02, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0xde, 0x08, 2650 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 2651 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 2652 0x08, 0x08, 0x08, 0x08, 0xbc, 0xe1, 0x4b, 0x92, 0x0e, 0xb4, 0x7b, 0x18, 2653 0x86, 0x61, 0x18, 0xc6, 0x18, 0x63, 0x8c, 0x31, 0xc6, 0x18, 0x63, 0x8c, 2654 0x31, 0x66, 0x66, 0x66, 0x66, 0xb6, 0x6d, 0x01, 0x00, 0x00, 0x00, 0x04, 2655 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x20, 0x73, 0x6f, 0x64, 0x61, 2656 0x6c, 0x65, 0x73, 0x20, 0x74, 0x6f, 0x72, 0x74, 0x6f, 0x72, 0x20, 0x65, 2657 0x6c, 0x65, 0x69, 0x66, 0x65, 0x6e, 0x64, 0x2e, 0x20, 0x41, 0x6c, 0x69}; 2658 2659 if (dictBuffer==NULL || samplesSizes==NULL) { 2660 free(dictBuffer); 2661 free(samplesSizes); 2662 goto _output_error; 2663 } 2664 2665 DISPLAYLEVEL(3, "test%3i : dictBuilder on cyclic data : ", testNb++); 2666 assert(compressedBufferSize >= totalSampleSize); 2667 { U32 u; for (u=0; u<totalSampleSize; u++) ((BYTE*)decodedBuffer)[u] = (BYTE)u; } 2668 { U32 u; for (u=0; u<nbSamples; u++) samplesSizes[u] = sampleUnitSize; } 2669 { size_t const sDictSize = ZDICT_trainFromBuffer(dictBuffer, dictBufferCapacity, 2670 decodedBuffer, samplesSizes, nbSamples); 2671 if (ZDICT_isError(sDictSize)) goto _output_error; 2672 DISPLAYLEVEL(3, "OK, created dictionary of size %u \n", (unsigned)sDictSize); 2673 } 2674 2675 DISPLAYLEVEL(3, "test%3i : dictBuilder : ", testNb++); 2676 { U32 u; for (u=0; u<nbSamples; u++) samplesSizes[u] = sampleUnitSize; } 2677 dictSize = ZDICT_trainFromBuffer(dictBuffer, dictBufferCapacity, 2678 CNBuffer, samplesSizes, nbSamples); 2679 if (ZDICT_isError(dictSize)) goto _output_error; 2680 DISPLAYLEVEL(3, "OK, created dictionary of size %u \n", (unsigned)dictSize); 2681 2682 DISPLAYLEVEL(3, "test%3i : Multithreaded COVER dictBuilder : ", testNb++); 2683 { U32 u; for (u=0; u<nbSamples; u++) samplesSizes[u] = sampleUnitSize; } 2684 { ZDICT_cover_params_t coverParams; 2685 memset(&coverParams, 0, sizeof(coverParams)); 2686 coverParams.steps = 8; 2687 coverParams.nbThreads = 4; 2688 dictSize = ZDICT_optimizeTrainFromBuffer_cover( 2689 dictBuffer, dictBufferCapacity, 2690 CNBuffer, samplesSizes, nbSamples/8, /* less samples for faster tests */ 2691 &coverParams); 2692 if (ZDICT_isError(dictSize)) goto _output_error; 2693 } 2694 DISPLAYLEVEL(3, "OK, created dictionary of size %u \n", (unsigned)dictSize); 2695 2696 DISPLAYLEVEL(3, "test%3i : COVER dictBuilder with shrinkDict: ", testNb++); 2697 { U32 u; for (u=0; u<nbSamples; u++) samplesSizes[u] = sampleUnitSize; } 2698 { ZDICT_cover_params_t coverParams; 2699 memset(&coverParams, 0, sizeof(coverParams)); 2700 coverParams.steps = 8; 2701 coverParams.nbThreads = 4; 2702 coverParams.shrinkDict = 1; 2703 coverParams.shrinkDictMaxRegression = 1; 2704 dictSize = ZDICT_optimizeTrainFromBuffer_cover( 2705 dictBuffer, dictBufferCapacity, 2706 CNBuffer, samplesSizes, nbSamples/8, /* less samples for faster tests */ 2707 &coverParams); 2708 if (ZDICT_isError(dictSize)) goto _output_error; 2709 } 2710 DISPLAYLEVEL(3, "OK, created dictionary of size %u \n", (unsigned)dictSize); 2711 2712 DISPLAYLEVEL(3, "test%3i : Multithreaded FASTCOVER dictBuilder : ", testNb++); 2713 { U32 u; for (u=0; u<nbSamples; u++) samplesSizes[u] = sampleUnitSize; } 2714 { ZDICT_fastCover_params_t fastCoverParams; 2715 memset(&fastCoverParams, 0, sizeof(fastCoverParams)); 2716 fastCoverParams.steps = 8; 2717 fastCoverParams.nbThreads = 4; 2718 dictSize = ZDICT_optimizeTrainFromBuffer_fastCover( 2719 dictBuffer, dictBufferCapacity, 2720 CNBuffer, samplesSizes, nbSamples, 2721 &fastCoverParams); 2722 if (ZDICT_isError(dictSize)) goto _output_error; 2723 } 2724 DISPLAYLEVEL(3, "OK, created dictionary of size %u \n", (unsigned)dictSize); 2725 2726 DISPLAYLEVEL(3, "test%3i : FASTCOVER dictBuilder with shrinkDict: ", testNb++); 2727 { U32 u; for (u=0; u<nbSamples; u++) samplesSizes[u] = sampleUnitSize; } 2728 { ZDICT_fastCover_params_t fastCoverParams; 2729 memset(&fastCoverParams, 0, sizeof(fastCoverParams)); 2730 fastCoverParams.steps = 8; 2731 fastCoverParams.nbThreads = 4; 2732 fastCoverParams.shrinkDict = 1; 2733 fastCoverParams.shrinkDictMaxRegression = 1; 2734 dictSize = ZDICT_optimizeTrainFromBuffer_fastCover( 2735 dictBuffer, dictBufferCapacity, 2736 CNBuffer, samplesSizes, nbSamples, 2737 &fastCoverParams); 2738 if (ZDICT_isError(dictSize)) goto _output_error; 2739 } 2740 DISPLAYLEVEL(3, "OK, created dictionary of size %u \n", (unsigned)dictSize); 2741 2742 DISPLAYLEVEL(3, "test%3i : check dictID : ", testNb++); 2743 dictID = ZDICT_getDictID(dictBuffer, dictSize); 2744 if (dictID==0) goto _output_error; 2745 DISPLAYLEVEL(3, "OK : %u \n", (unsigned)dictID); 2746 2747 DISPLAYLEVEL(3, "test%3i : check dict header size no error : ", testNb++); 2748 dictHeaderSize = ZDICT_getDictHeaderSize(dictBuffer, dictSize); 2749 if (dictHeaderSize==0) goto _output_error; 2750 DISPLAYLEVEL(3, "OK : %u \n", (unsigned)dictHeaderSize); 2751 2752 DISPLAYLEVEL(3, "test%3i : check dict header size correctness : ", testNb++); 2753 { dictHeaderSize = ZDICT_getDictHeaderSize(dictBufferFixed, dictBufferFixedSize); 2754 if (dictHeaderSize != 115) goto _output_error; 2755 } 2756 DISPLAYLEVEL(3, "OK : %u \n", (unsigned)dictHeaderSize); 2757 2758 DISPLAYLEVEL(3, "test%3i : compress with dictionary : ", testNb++); 2759 cSize = ZSTD_compress_usingDict(cctx, compressedBuffer, compressedBufferSize, 2760 CNBuffer, CNBuffSize, 2761 dictBuffer, dictSize, 4); 2762 if (ZSTD_isError(cSize)) goto _output_error; 2763 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBuffSize*100); 2764 2765 DISPLAYLEVEL(3, "test%3i : retrieve dictID from dictionary : ", testNb++); 2766 { U32 const did = ZSTD_getDictID_fromDict(dictBuffer, dictSize); 2767 if (did != dictID) goto _output_error; /* non-conformant (content-only) dictionary */ 2768 } 2769 DISPLAYLEVEL(3, "OK \n"); 2770 2771 DISPLAYLEVEL(3, "test%3i : retrieve dictID from frame : ", testNb++); 2772 { U32 const did = ZSTD_getDictID_fromFrame(compressedBuffer, cSize); 2773 if (did != dictID) goto _output_error; /* non-conformant (content-only) dictionary */ 2774 } 2775 DISPLAYLEVEL(3, "OK \n"); 2776 2777 DISPLAYLEVEL(3, "test%3i : frame built with dictionary should be decompressible : ", testNb++); 2778 { ZSTD_DCtx* const dctx = ZSTD_createDCtx(); assert(dctx != NULL); 2779 CHECKPLUS(r, ZSTD_decompress_usingDict(dctx, 2780 decodedBuffer, CNBuffSize, 2781 compressedBuffer, cSize, 2782 dictBuffer, dictSize), 2783 if (r != CNBuffSize) goto _output_error); 2784 ZSTD_freeDCtx(dctx); 2785 } 2786 DISPLAYLEVEL(3, "OK \n"); 2787 2788 DISPLAYLEVEL(3, "test%3i : estimate CDict size : ", testNb++); 2789 { ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize); 2790 size_t const estimatedSize = ZSTD_estimateCDictSize_advanced(dictSize, cParams, ZSTD_dlm_byRef); 2791 DISPLAYLEVEL(3, "OK : %u \n", (unsigned)estimatedSize); 2792 } 2793 2794 DISPLAYLEVEL(3, "test%3i : compress with CDict ", testNb++); 2795 { ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize); 2796 ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictBuffer, dictSize, 2797 ZSTD_dlm_byRef, ZSTD_dct_auto, 2798 cParams, ZSTD_defaultCMem); 2799 assert(cdict != NULL); 2800 DISPLAYLEVEL(3, "(size : %u) : ", (unsigned)ZSTD_sizeof_CDict(cdict)); 2801 assert(ZSTD_getDictID_fromDict(dictBuffer, dictSize) == ZSTD_getDictID_fromCDict(cdict)); 2802 cSize = ZSTD_compress_usingCDict(cctx, compressedBuffer, compressedBufferSize, 2803 CNBuffer, CNBuffSize, cdict); 2804 ZSTD_freeCDict(cdict); 2805 if (ZSTD_isError(cSize)) goto _output_error; 2806 } 2807 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBuffSize*100); 2808 2809 DISPLAYLEVEL(3, "test%3i : retrieve dictID from frame : ", testNb++); 2810 { U32 const did = ZSTD_getDictID_fromFrame(compressedBuffer, cSize); 2811 if (did != dictID) goto _output_error; /* non-conformant (content-only) dictionary */ 2812 } 2813 DISPLAYLEVEL(3, "OK \n"); 2814 2815 DISPLAYLEVEL(3, "test%3i : frame built with dictionary should be decompressible : ", testNb++); 2816 { ZSTD_DCtx* const dctx = ZSTD_createDCtx(); assert(dctx != NULL); 2817 CHECKPLUS(r, ZSTD_decompress_usingDict(dctx, 2818 decodedBuffer, CNBuffSize, 2819 compressedBuffer, cSize, 2820 dictBuffer, dictSize), 2821 if (r != CNBuffSize) goto _output_error); 2822 ZSTD_freeDCtx(dctx); 2823 } 2824 DISPLAYLEVEL(3, "OK \n"); 2825 2826 DISPLAYLEVEL(3, "test%3i : compress with static CDict : ", testNb++); 2827 { int const maxLevel = ZSTD_maxCLevel(); 2828 int level; 2829 for (level = 1; level <= maxLevel; ++level) { 2830 ZSTD_compressionParameters const cParams = ZSTD_getCParams(level, CNBuffSize, dictSize); 2831 size_t const cdictSize = ZSTD_estimateCDictSize_advanced(dictSize, cParams, ZSTD_dlm_byCopy); 2832 void* const cdictBuffer = malloc(cdictSize); 2833 if (cdictBuffer==NULL) goto _output_error; 2834 { const ZSTD_CDict* const cdict = ZSTD_initStaticCDict( 2835 cdictBuffer, cdictSize, 2836 dictBuffer, dictSize, 2837 ZSTD_dlm_byCopy, ZSTD_dct_auto, 2838 cParams); 2839 if (cdict == NULL) { 2840 DISPLAY("ZSTD_initStaticCDict failed "); 2841 goto _output_error; 2842 } 2843 cSize = ZSTD_compress_usingCDict(cctx, 2844 compressedBuffer, compressedBufferSize, 2845 CNBuffer, MIN(10 KB, CNBuffSize), cdict); 2846 if (ZSTD_isError(cSize)) { 2847 DISPLAY("ZSTD_compress_usingCDict failed "); 2848 goto _output_error; 2849 } } 2850 free(cdictBuffer); 2851 } } 2852 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBuffSize*100); 2853 2854 DISPLAYLEVEL(3, "test%3i : ZSTD_compress_usingCDict_advanced, no contentSize, no dictID : ", testNb++); 2855 { ZSTD_frameParameters const fParams = { 0 /* frameSize */, 1 /* checksum */, 1 /* noDictID*/ }; 2856 ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize); 2857 ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dct_auto, cParams, ZSTD_defaultCMem); 2858 assert(cdict != NULL); 2859 cSize = ZSTD_compress_usingCDict_advanced(cctx, 2860 compressedBuffer, compressedBufferSize, 2861 CNBuffer, CNBuffSize, 2862 cdict, fParams); 2863 ZSTD_freeCDict(cdict); 2864 if (ZSTD_isError(cSize)) goto _output_error; 2865 } 2866 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBuffSize*100); 2867 2868 DISPLAYLEVEL(3, "test%3i : try retrieving contentSize from frame : ", testNb++); 2869 { U64 const contentSize = ZSTD_getFrameContentSize(compressedBuffer, cSize); 2870 if (contentSize != ZSTD_CONTENTSIZE_UNKNOWN) goto _output_error; 2871 } 2872 DISPLAYLEVEL(3, "OK (unknown)\n"); 2873 2874 DISPLAYLEVEL(3, "test%3i : frame built without dictID should be decompressible : ", testNb++); 2875 { ZSTD_DCtx* const dctx = ZSTD_createDCtx(); 2876 assert(dctx != NULL); 2877 CHECKPLUS(r, ZSTD_decompress_usingDict(dctx, 2878 decodedBuffer, CNBuffSize, 2879 compressedBuffer, cSize, 2880 dictBuffer, dictSize), 2881 if (r != CNBuffSize) goto _output_error); 2882 ZSTD_freeDCtx(dctx); 2883 } 2884 DISPLAYLEVEL(3, "OK \n"); 2885 2886 DISPLAYLEVEL(3, "test%3i : ZSTD_compress_advanced, no dictID : ", testNb++); 2887 { ZSTD_parameters p = ZSTD_getParams(3, CNBuffSize, dictSize); 2888 p.fParams.noDictIDFlag = 1; 2889 cSize = ZSTD_compress_advanced(cctx, compressedBuffer, compressedBufferSize, 2890 CNBuffer, CNBuffSize, 2891 dictBuffer, dictSize, p); 2892 if (ZSTD_isError(cSize)) goto _output_error; 2893 } 2894 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBuffSize*100); 2895 2896 DISPLAYLEVEL(3, "test%3i : frame built without dictID should be decompressible : ", testNb++); 2897 { ZSTD_DCtx* const dctx = ZSTD_createDCtx(); assert(dctx != NULL); 2898 CHECKPLUS(r, ZSTD_decompress_usingDict(dctx, 2899 decodedBuffer, CNBuffSize, 2900 compressedBuffer, cSize, 2901 dictBuffer, dictSize), 2902 if (r != CNBuffSize) goto _output_error); 2903 ZSTD_freeDCtx(dctx); 2904 } 2905 DISPLAYLEVEL(3, "OK \n"); 2906 2907 DISPLAYLEVEL(3, "test%3i : dictionary containing only header should return error : ", testNb++); 2908 { ZSTD_DCtx* const dctx = ZSTD_createDCtx(); 2909 assert(dctx != NULL); 2910 { const size_t ret = ZSTD_decompress_usingDict( 2911 dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, 2912 "\x37\xa4\x30\xec\x11\x22\x33\x44", 8); 2913 if (ZSTD_getErrorCode(ret) != ZSTD_error_dictionary_corrupted) 2914 goto _output_error; 2915 } 2916 ZSTD_freeDCtx(dctx); 2917 } 2918 DISPLAYLEVEL(3, "OK \n"); 2919 2920 DISPLAYLEVEL(3, "test%3d : bufferless api with cdict : ", testNb++); 2921 { ZSTD_CDict* const cdict = ZSTD_createCDict(dictBuffer, dictSize, 1); 2922 ZSTD_DCtx* const dctx = ZSTD_createDCtx(); 2923 ZSTD_frameParameters const fParams = { 0, 1, 0 }; 2924 size_t cBlockSize; 2925 cSize = 0; 2926 CHECK_Z(ZSTD_compressBegin_usingCDict_advanced(cctx, cdict, fParams, ZSTD_CONTENTSIZE_UNKNOWN)); 2927 cBlockSize = ZSTD_compressContinue(cctx, (char*)compressedBuffer + cSize, compressedBufferSize - cSize, CNBuffer, 1000); 2928 CHECK_Z(cBlockSize); 2929 cSize += cBlockSize; 2930 cBlockSize = ZSTD_compressEnd(cctx, (char*)compressedBuffer + cSize, compressedBufferSize - cSize, (char const*)CNBuffer + 2000, 1000); 2931 CHECK_Z(cBlockSize); 2932 cSize += cBlockSize; 2933 2934 CHECK_Z(ZSTD_decompress_usingDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, dictBuffer, dictSize)); 2935 2936 ZSTD_freeCDict(cdict); 2937 ZSTD_freeDCtx(dctx); 2938 } 2939 DISPLAYLEVEL(3, "OK \n"); 2940 2941 DISPLAYLEVEL(3, "test%3i : Building cdict w/ ZSTD_dct_fullDict on a good dictionary : ", testNb++); 2942 { ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize); 2943 ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dct_fullDict, cParams, ZSTD_defaultCMem); 2944 if (cdict==NULL) goto _output_error; 2945 ZSTD_freeCDict(cdict); 2946 } 2947 DISPLAYLEVEL(3, "OK \n"); 2948 2949 DISPLAYLEVEL(3, "test%3i : Building cdict w/ ZSTD_dct_fullDict on a rawContent (must fail) : ", testNb++); 2950 { ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize); 2951 ZSTD_CDict* const cdict = ZSTD_createCDict_advanced((const char*)dictBuffer+1, dictSize-1, ZSTD_dlm_byRef, ZSTD_dct_fullDict, cParams, ZSTD_defaultCMem); 2952 if (cdict!=NULL) goto _output_error; 2953 ZSTD_freeCDict(cdict); 2954 } 2955 DISPLAYLEVEL(3, "OK \n"); 2956 2957 { char* rawDictBuffer = (char*)malloc(dictSize); 2958 assert(rawDictBuffer); 2959 memcpy(rawDictBuffer, (char*)dictBuffer + 2, dictSize - 2); 2960 memset(rawDictBuffer + dictSize - 2, 0, 2); 2961 MEM_writeLE32((char*)rawDictBuffer, ZSTD_MAGIC_DICTIONARY); 2962 2963 DISPLAYLEVEL(3, "test%3i : Loading rawContent starting with dict header w/ ZSTD_dct_auto should fail : ", testNb++); 2964 { 2965 size_t ret; 2966 /* Either operation is allowed to fail, but one must fail. */ 2967 ret = ZSTD_CCtx_loadDictionary_advanced( 2968 cctx, (const char*)rawDictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dct_auto); 2969 if (!ZSTD_isError(ret)) { 2970 ret = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100)); 2971 if (!ZSTD_isError(ret)) goto _output_error; 2972 } 2973 } 2974 DISPLAYLEVEL(3, "OK \n"); 2975 2976 DISPLAYLEVEL(3, "test%3i : Loading rawContent starting with dict header w/ ZSTD_dct_rawContent should pass : ", testNb++); 2977 { 2978 size_t ret; 2979 ret = ZSTD_CCtx_loadDictionary_advanced( 2980 cctx, (const char*)rawDictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dct_rawContent); 2981 if (ZSTD_isError(ret)) goto _output_error; 2982 ret = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100)); 2983 if (ZSTD_isError(ret)) goto _output_error; 2984 } 2985 DISPLAYLEVEL(3, "OK \n"); 2986 2987 DISPLAYLEVEL(3, "test%3i : Testing non-attached CDict with ZSTD_dct_rawContent : ", testNb++); 2988 { size_t const srcSize = MIN(CNBuffSize, 100); 2989 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters); 2990 /* Force the dictionary to be reloaded in raw content mode */ 2991 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_forceAttachDict, ZSTD_dictForceLoad)); 2992 CHECK_Z(ZSTD_CCtx_loadDictionary_advanced(cctx, rawDictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dct_rawContent)); 2993 cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, srcSize); 2994 CHECK_Z(cSize); 2995 } 2996 DISPLAYLEVEL(3, "OK \n"); 2997 2998 free(rawDictBuffer); 2999 } 3000 3001 DISPLAYLEVEL(3, "test%3i : ZSTD_CCtx_refCDict() then set parameters : ", testNb++); 3002 { ZSTD_CDict* const cdict = ZSTD_createCDict(CNBuffer, dictSize, 1); 3003 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters); 3004 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 1) ); 3005 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_hashLog, 12 )); 3006 CHECK_Z( ZSTD_CCtx_refCDict(cctx, cdict) ); 3007 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 1) ); 3008 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_hashLog, 12 )); 3009 ZSTD_freeCDict(cdict); 3010 } 3011 DISPLAYLEVEL(3, "OK \n"); 3012 3013 DISPLAYLEVEL(3, "test%3i : Loading dictionary before setting parameters is the same as loading after : ", testNb++); 3014 { 3015 size_t size1, size2; 3016 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters); 3017 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 7) ); 3018 CHECK_Z( ZSTD_CCtx_loadDictionary(cctx, CNBuffer, MIN(CNBuffSize, 10 KB)) ); 3019 size1 = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100 KB)); 3020 if (ZSTD_isError(size1)) goto _output_error; 3021 3022 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters); 3023 CHECK_Z( ZSTD_CCtx_loadDictionary(cctx, CNBuffer, MIN(CNBuffSize, 10 KB)) ); 3024 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 7) ); 3025 size2 = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100 KB)); 3026 if (ZSTD_isError(size2)) goto _output_error; 3027 3028 if (size1 != size2) goto _output_error; 3029 } 3030 DISPLAYLEVEL(3, "OK \n"); 3031 3032 DISPLAYLEVEL(3, "test%3i : Loading a dictionary clears the prefix : ", testNb++); 3033 { 3034 CHECK_Z( ZSTD_CCtx_refPrefix(cctx, (const char*)dictBuffer, dictSize) ); 3035 CHECK_Z( ZSTD_CCtx_loadDictionary(cctx, (const char*)dictBuffer, dictSize) ); 3036 CHECK_Z( ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100)) ); 3037 } 3038 DISPLAYLEVEL(3, "OK \n"); 3039 3040 DISPLAYLEVEL(3, "test%3i : Loading a dictionary clears the cdict : ", testNb++); 3041 { 3042 ZSTD_CDict* const cdict = ZSTD_createCDict(dictBuffer, dictSize, 1); 3043 CHECK_Z( ZSTD_CCtx_refCDict(cctx, cdict) ); 3044 CHECK_Z( ZSTD_CCtx_loadDictionary(cctx, (const char*)dictBuffer, dictSize) ); 3045 CHECK_Z( ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100)) ); 3046 ZSTD_freeCDict(cdict); 3047 } 3048 DISPLAYLEVEL(3, "OK \n"); 3049 3050 DISPLAYLEVEL(3, "test%3i : Loading a cdict clears the prefix : ", testNb++); 3051 { 3052 ZSTD_CDict* const cdict = ZSTD_createCDict(dictBuffer, dictSize, 1); 3053 CHECK_Z( ZSTD_CCtx_refPrefix(cctx, (const char*)dictBuffer, dictSize) ); 3054 CHECK_Z( ZSTD_CCtx_refCDict(cctx, cdict) ); 3055 CHECK_Z( ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100)) ); 3056 ZSTD_freeCDict(cdict); 3057 } 3058 DISPLAYLEVEL(3, "OK \n"); 3059 3060 DISPLAYLEVEL(3, "test%3i : Loading a cdict clears the dictionary : ", testNb++); 3061 { 3062 ZSTD_CDict* const cdict = ZSTD_createCDict(dictBuffer, dictSize, 1); 3063 CHECK_Z( ZSTD_CCtx_loadDictionary(cctx, (const char*)dictBuffer, dictSize) ); 3064 CHECK_Z( ZSTD_CCtx_refCDict(cctx, cdict) ); 3065 CHECK_Z( ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100)) ); 3066 ZSTD_freeCDict(cdict); 3067 } 3068 DISPLAYLEVEL(3, "OK \n"); 3069 3070 DISPLAYLEVEL(3, "test%3i : Loading a prefix clears the dictionary : ", testNb++); 3071 { 3072 CHECK_Z( ZSTD_CCtx_loadDictionary(cctx, (const char*)dictBuffer, dictSize) ); 3073 CHECK_Z( ZSTD_CCtx_refPrefix(cctx, (const char*)dictBuffer, dictSize) ); 3074 CHECK_Z( ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100)) ); 3075 } 3076 DISPLAYLEVEL(3, "OK \n"); 3077 3078 DISPLAYLEVEL(3, "test%3i : Loading a prefix clears the cdict : ", testNb++); 3079 { 3080 ZSTD_CDict* const cdict = ZSTD_createCDict(dictBuffer, dictSize, 1); 3081 CHECK_Z( ZSTD_CCtx_refCDict(cctx, cdict) ); 3082 CHECK_Z( ZSTD_CCtx_refPrefix(cctx, (const char*)dictBuffer, dictSize) ); 3083 CHECK_Z( ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100)) ); 3084 ZSTD_freeCDict(cdict); 3085 } 3086 DISPLAYLEVEL(3, "OK \n"); 3087 3088 DISPLAYLEVEL(3, "test%3i : Loaded dictionary persists across reset session : ", testNb++); 3089 { 3090 size_t size1, size2; 3091 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters); 3092 CHECK_Z( ZSTD_CCtx_loadDictionary(cctx, CNBuffer, MIN(CNBuffSize, 10 KB)) ); 3093 size1 = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100 KB)); 3094 if (ZSTD_isError(size1)) goto _output_error; 3095 3096 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_only); 3097 size2 = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100 KB)); 3098 if (ZSTD_isError(size2)) goto _output_error; 3099 3100 if (size1 != size2) goto _output_error; 3101 } 3102 DISPLAYLEVEL(3, "OK \n"); 3103 3104 DISPLAYLEVEL(3, "test%3i : Loaded dictionary is cleared after resetting parameters : ", testNb++); 3105 { 3106 size_t size1, size2; 3107 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters); 3108 CHECK_Z( ZSTD_CCtx_loadDictionary(cctx, CNBuffer, MIN(CNBuffSize, 10 KB)) ); 3109 size1 = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100 KB)); 3110 if (ZSTD_isError(size1)) goto _output_error; 3111 3112 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters); 3113 size2 = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100 KB)); 3114 if (ZSTD_isError(size2)) goto _output_error; 3115 3116 if (size1 == size2) goto _output_error; 3117 } 3118 DISPLAYLEVEL(3, "OK \n"); 3119 3120 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters); 3121 CHECK_Z( ZSTD_CCtx_loadDictionary(cctx, dictBuffer, dictSize) ); 3122 cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100 KB)); 3123 CHECK_Z(cSize); 3124 DISPLAYLEVEL(3, "test%3i : ZSTD_decompressDCtx() with dictionary : ", testNb++); 3125 { 3126 ZSTD_DCtx* dctx = ZSTD_createDCtx(); 3127 size_t ret; 3128 /* We should fail to decompress without a dictionary. */ 3129 ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters); 3130 ret = ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize); 3131 if (!ZSTD_isError(ret)) goto _output_error; 3132 /* We should succeed to decompress with the dictionary. */ 3133 ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters); 3134 CHECK_Z( ZSTD_DCtx_loadDictionary(dctx, dictBuffer, dictSize) ); 3135 CHECK_Z( ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize) ); 3136 /* The dictionary should persist across calls. */ 3137 CHECK_Z( ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize) ); 3138 /* When we reset the context the dictionary is cleared. */ 3139 ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters); 3140 ret = ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize); 3141 if (!ZSTD_isError(ret)) goto _output_error; 3142 ZSTD_freeDCtx(dctx); 3143 } 3144 DISPLAYLEVEL(3, "OK \n"); 3145 3146 DISPLAYLEVEL(3, "test%3i : ZSTD_decompressDCtx() with ddict : ", testNb++); 3147 { 3148 ZSTD_DCtx* dctx = ZSTD_createDCtx(); 3149 ZSTD_DDict* ddict = ZSTD_createDDict(dictBuffer, dictSize); 3150 size_t ret; 3151 /* We should succeed to decompress with the ddict. */ 3152 ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters); 3153 CHECK_Z( ZSTD_DCtx_refDDict(dctx, ddict) ); 3154 CHECK_Z( ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize) ); 3155 /* The ddict should persist across calls. */ 3156 CHECK_Z( ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize) ); 3157 /* When we reset the context the ddict is cleared. */ 3158 ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters); 3159 ret = ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize); 3160 if (!ZSTD_isError(ret)) goto _output_error; 3161 ZSTD_freeDCtx(dctx); 3162 ZSTD_freeDDict(ddict); 3163 } 3164 DISPLAYLEVEL(3, "OK \n"); 3165 3166 DISPLAYLEVEL(3, "test%3i : ZSTD_decompressDCtx() with prefix : ", testNb++); 3167 { 3168 ZSTD_DCtx* dctx = ZSTD_createDCtx(); 3169 size_t ret; 3170 /* We should succeed to decompress with the prefix. */ 3171 ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters); 3172 CHECK_Z( ZSTD_DCtx_refPrefix_advanced(dctx, dictBuffer, dictSize, ZSTD_dct_auto) ); 3173 CHECK_Z( ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize) ); 3174 /* The prefix should be cleared after the first compression. */ 3175 ret = ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize); 3176 if (!ZSTD_isError(ret)) goto _output_error; 3177 ZSTD_freeDCtx(dctx); 3178 } 3179 DISPLAYLEVEL(3, "OK \n"); 3180 3181 DISPLAYLEVEL(3, "test%3i : ZSTD_fast attach dictionary with hashLog = 25 and chainLog = 25 : ", testNb++); 3182 { 3183 ZSTD_CCtx_params* cctxParams = ZSTD_createCCtxParams(); 3184 ZSTD_customMem customMem = {NULL, NULL, NULL}; 3185 ZSTD_DCtx* dctx = ZSTD_createDCtx(); 3186 ZSTD_CDict* cdict; 3187 CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_strategy, ZSTD_fast)); 3188 /* Set windowLog to 25 so hash/chain logs don't get sized down */ 3189 CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_windowLog, 25)); 3190 CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_hashLog, 25)); 3191 CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_chainLog, 25)); 3192 /* Set srcSizeHint to 2^25 so hash/chain logs don't get sized down */ 3193 CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_srcSizeHint, 1u << 25)); 3194 cdict = ZSTD_createCDict_advanced2(dictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dct_auto, cctxParams, customMem); 3195 CHECK_Z(ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters)); 3196 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_forceAttachDict, ZSTD_dictForceAttach)); 3197 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1)); 3198 CHECK_Z(ZSTD_CCtx_refCDict(cctx, cdict)); 3199 cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize); 3200 CHECK_Z(cSize); 3201 CHECK_Z(ZSTD_decompress_usingDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, dictBuffer, dictSize)); 3202 ZSTD_freeCDict(cdict); 3203 ZSTD_freeDCtx(dctx); 3204 ZSTD_freeCCtxParams(cctxParams); 3205 } 3206 DISPLAYLEVEL(3, "OK \n"); 3207 3208 DISPLAYLEVEL(3, "test%3i : ZSTD_dfast attach dictionary with hashLog = 25 and chainLog = 25 : ", testNb++); 3209 { 3210 ZSTD_CCtx_params* cctxParams = ZSTD_createCCtxParams(); 3211 ZSTD_customMem customMem = {NULL, NULL, NULL}; 3212 ZSTD_DCtx* dctx = ZSTD_createDCtx(); 3213 ZSTD_CDict* cdict; 3214 CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_strategy, ZSTD_dfast)); 3215 /* Set windowLog to 25 so hash/chain logs don't get sized down */ 3216 CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_windowLog, 25)); 3217 CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_hashLog, 25)); 3218 CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_chainLog, 25)); 3219 /* Set srcSizeHint to 2^25 so hash/chain logs don't get sized down */ 3220 CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_srcSizeHint, 1u << 25)); 3221 cdict = ZSTD_createCDict_advanced2(dictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dct_auto, cctxParams, customMem); 3222 CHECK_Z(ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters)); 3223 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_forceAttachDict, ZSTD_dictForceAttach)); 3224 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1)); 3225 CHECK_Z(ZSTD_CCtx_refCDict(cctx, cdict)); 3226 cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize); 3227 CHECK_Z(cSize); 3228 CHECK_Z(ZSTD_decompress_usingDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, dictBuffer, dictSize)); 3229 ZSTD_freeCDict(cdict); 3230 ZSTD_freeDCtx(dctx); 3231 ZSTD_freeCCtxParams(cctxParams); 3232 } 3233 DISPLAYLEVEL(3, "OK \n"); 3234 3235 DISPLAYLEVEL(3, "test%3i : ZSTD_lazy attach dictionary with hashLog = 29 and searchLog = 4 : ", testNb++); 3236 if (MEM_64bits()) { 3237 ZSTD_CCtx_params* cctxParams = ZSTD_createCCtxParams(); 3238 ZSTD_customMem customMem = {NULL, NULL, NULL}; 3239 ZSTD_DCtx* dctx = ZSTD_createDCtx(); 3240 ZSTD_CDict* cdict; 3241 CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_strategy, ZSTD_lazy)); 3242 /* Force enable row based match finder, and disable dedicated dict search. */ 3243 CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_useRowMatchFinder, ZSTD_ps_enable)); 3244 CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_enableDedicatedDictSearch, 0)); 3245 CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_searchLog, 4)); 3246 /* Set windowLog to 29 so hash/chain logs don't get sized down */ 3247 CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_windowLog, 29)); 3248 CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_hashLog, 29)); 3249 /* Set srcSizeHint to 2^29 so hash/chain logs don't get sized down */ 3250 CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_srcSizeHint, 1u << 29)); 3251 cdict = ZSTD_createCDict_advanced2(dictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dct_auto, cctxParams, customMem); 3252 CHECK_Z(ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters)); 3253 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_forceAttachDict, ZSTD_dictForceAttach)); 3254 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1)); 3255 CHECK_Z(ZSTD_CCtx_refCDict(cctx, cdict)); 3256 cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize); 3257 CHECK_Z(cSize); 3258 CHECK_Z(ZSTD_decompress_usingDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, dictBuffer, dictSize)); 3259 ZSTD_freeCDict(cdict); 3260 ZSTD_freeDCtx(dctx); 3261 ZSTD_freeCCtxParams(cctxParams); 3262 } 3263 DISPLAYLEVEL(3, "OK \n"); 3264 3265 DISPLAYLEVEL(3, "test%3i : Dictionary with non-default repcodes : ", testNb++); 3266 { U32 u; for (u=0; u<nbSamples; u++) samplesSizes[u] = sampleUnitSize; } 3267 dictSize = ZDICT_trainFromBuffer(dictBuffer, dictSize, 3268 CNBuffer, samplesSizes, nbSamples); 3269 if (ZDICT_isError(dictSize)) goto _output_error; 3270 /* Set all the repcodes to non-default */ 3271 { 3272 BYTE* dictPtr = (BYTE*)dictBuffer; 3273 BYTE* dictLimit = dictPtr + dictSize - 12; 3274 /* Find the repcodes */ 3275 while (dictPtr < dictLimit && 3276 (MEM_readLE32(dictPtr) != 1 || MEM_readLE32(dictPtr + 4) != 4 || 3277 MEM_readLE32(dictPtr + 8) != 8)) { 3278 ++dictPtr; 3279 } 3280 if (dictPtr >= dictLimit) goto _output_error; 3281 MEM_writeLE32(dictPtr + 0, 10); 3282 MEM_writeLE32(dictPtr + 4, 10); 3283 MEM_writeLE32(dictPtr + 8, 10); 3284 /* Set the last 8 bytes to 'x' */ 3285 memset((BYTE*)dictBuffer + dictSize - 8, 'x', 8); 3286 } 3287 /* The optimal parser checks all the repcodes. 3288 * Make sure at least one is a match >= targetLength so that it is 3289 * immediately chosen. This will make sure that the compressor and 3290 * decompressor agree on at least one of the repcodes. 3291 */ 3292 { size_t dSize; 3293 BYTE data[1024]; 3294 ZSTD_DCtx* const dctx = ZSTD_createDCtx(); 3295 ZSTD_compressionParameters const cParams = ZSTD_getCParams(19, CNBuffSize, dictSize); 3296 ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictBuffer, dictSize, 3297 ZSTD_dlm_byRef, ZSTD_dct_auto, 3298 cParams, ZSTD_defaultCMem); 3299 assert(dctx != NULL); assert(cdict != NULL); 3300 memset(data, 'x', sizeof(data)); 3301 cSize = ZSTD_compress_usingCDict(cctx, compressedBuffer, compressedBufferSize, 3302 data, sizeof(data), cdict); 3303 ZSTD_freeCDict(cdict); 3304 if (ZSTD_isError(cSize)) { DISPLAYLEVEL(5, "Compression error %s : ", ZSTD_getErrorName(cSize)); goto _output_error; } 3305 dSize = ZSTD_decompress_usingDict(dctx, decodedBuffer, sizeof(data), compressedBuffer, cSize, dictBuffer, dictSize); 3306 if (ZSTD_isError(dSize)) { DISPLAYLEVEL(5, "Decompression error %s : ", ZSTD_getErrorName(dSize)); goto _output_error; } 3307 if (memcmp(data, decodedBuffer, sizeof(data))) { DISPLAYLEVEL(5, "Data corruption : "); goto _output_error; } 3308 ZSTD_freeDCtx(dctx); 3309 } 3310 DISPLAYLEVEL(3, "OK \n"); 3311 3312 DISPLAYLEVEL(3, "test%3i : ZSTD_decompressDCtx() with multiple ddicts : ", testNb++); 3313 { 3314 const size_t numDicts = 128; 3315 const size_t numFrames = 4; 3316 size_t i; 3317 ZSTD_DCtx* dctx = ZSTD_createDCtx(); 3318 ZSTD_DDict** ddictTable = (ZSTD_DDict**)malloc(sizeof(ZSTD_DDict*)*numDicts); 3319 ZSTD_CDict** cdictTable = (ZSTD_CDict**)malloc(sizeof(ZSTD_CDict*)*numDicts); 3320 U32 dictIDSeed = seed; 3321 /* Create new compressed buffer that will hold frames with differing dictIDs */ 3322 char* dictBufferMulti = (char*)malloc(sizeof(char) * dictBufferFixedSize); /* Modifiable copy of fixed full dict buffer */ 3323 3324 ZSTD_memcpy(dictBufferMulti, dictBufferFixed, dictBufferFixedSize); 3325 /* Create a bunch of DDicts with random dict IDs */ 3326 for (i = 0; i < numDicts; ++i) { 3327 U32 currDictID = FUZ_rand(&dictIDSeed); 3328 MEM_writeLE32(dictBufferMulti+ZSTD_FRAMEIDSIZE, currDictID); 3329 ddictTable[i] = ZSTD_createDDict(dictBufferMulti, dictBufferFixedSize); 3330 cdictTable[i] = ZSTD_createCDict(dictBufferMulti, dictBufferFixedSize, 3); 3331 if (!ddictTable[i] || !cdictTable[i] || ZSTD_getDictID_fromCDict(cdictTable[i]) != ZSTD_getDictID_fromDDict(ddictTable[i])) { 3332 goto _output_error; 3333 } 3334 } 3335 /* Compress a few frames using random CDicts */ 3336 { 3337 size_t off = 0; 3338 /* only use the first half so we don't push against size limit of compressedBuffer */ 3339 size_t const segSize = (CNBuffSize / 2) / numFrames; 3340 for (i = 0; i < numFrames; i++) { 3341 size_t dictIdx = FUZ_rand(&dictIDSeed) % numDicts; 3342 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters); 3343 { CHECK_NEWV(r, ZSTD_compress_usingCDict(cctx, 3344 (BYTE*)compressedBuffer + off, CNBuffSize - off, 3345 (BYTE*)CNBuffer + segSize * (size_t)i, segSize, 3346 cdictTable[dictIdx])); 3347 off += r; 3348 } 3349 } 3350 cSize = off; 3351 } 3352 3353 /* We should succeed to decompression even though different dicts were used on different frames */ 3354 ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters); 3355 ZSTD_DCtx_setParameter(dctx, ZSTD_d_refMultipleDDicts, ZSTD_rmd_refMultipleDDicts); 3356 /* Reference every single ddict we made */ 3357 for (i = 0; i < numDicts; ++i) { 3358 CHECK_Z( ZSTD_DCtx_refDDict(dctx, ddictTable[i])); 3359 } 3360 CHECK_Z( ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize) ); 3361 /* Streaming decompression should also work */ 3362 { 3363 ZSTD_inBuffer in = {compressedBuffer, cSize, 0}; 3364 ZSTD_outBuffer out = {decodedBuffer, CNBuffSize, 0}; 3365 while (in.pos < in.size) { 3366 CHECK_Z(ZSTD_decompressStream(dctx, &out, &in)); 3367 } 3368 } 3369 ZSTD_freeDCtx(dctx); 3370 for (i = 0; i < numDicts; ++i) { 3371 ZSTD_freeCDict(cdictTable[i]); 3372 ZSTD_freeDDict(ddictTable[i]); 3373 } 3374 free(dictBufferMulti); 3375 free(ddictTable); 3376 free(cdictTable); 3377 } 3378 DISPLAYLEVEL(3, "OK \n"); 3379 3380 ZSTD_freeCCtx(cctx); 3381 free(dictBuffer); 3382 free(samplesSizes); 3383 } 3384 3385 /* COVER dictionary builder tests */ 3386 { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); 3387 size_t dictSize = 16 KB; 3388 size_t optDictSize = dictSize; 3389 void* dictBuffer = malloc(dictSize); 3390 size_t const totalSampleSize = 1 MB; 3391 size_t const sampleUnitSize = 8 KB; 3392 U32 const nbSamples = (U32)(totalSampleSize / sampleUnitSize); 3393 size_t* const samplesSizes = (size_t*) malloc(nbSamples * sizeof(size_t)); 3394 U32 seed32 = seed; 3395 ZDICT_cover_params_t params; 3396 U32 dictID; 3397 3398 if (dictBuffer==NULL || samplesSizes==NULL) { 3399 free(dictBuffer); 3400 free(samplesSizes); 3401 goto _output_error; 3402 } 3403 3404 DISPLAYLEVEL(3, "test%3i : ZDICT_trainFromBuffer_cover : ", testNb++); 3405 { U32 u; for (u=0; u<nbSamples; u++) samplesSizes[u] = sampleUnitSize; } 3406 memset(¶ms, 0, sizeof(params)); 3407 params.d = 1 + (FUZ_rand(&seed32) % 16); 3408 params.k = params.d + (FUZ_rand(&seed32) % 256); 3409 dictSize = ZDICT_trainFromBuffer_cover(dictBuffer, dictSize, 3410 CNBuffer, samplesSizes, nbSamples, 3411 params); 3412 if (ZDICT_isError(dictSize)) goto _output_error; 3413 DISPLAYLEVEL(3, "OK, created dictionary of size %u \n", (unsigned)dictSize); 3414 3415 DISPLAYLEVEL(3, "test%3i : check dictID : ", testNb++); 3416 dictID = ZDICT_getDictID(dictBuffer, dictSize); 3417 if (dictID==0) goto _output_error; 3418 DISPLAYLEVEL(3, "OK : %u \n", (unsigned)dictID); 3419 3420 DISPLAYLEVEL(3, "test%3i : ZDICT_optimizeTrainFromBuffer_cover : ", testNb++); 3421 memset(¶ms, 0, sizeof(params)); 3422 params.steps = 4; 3423 optDictSize = ZDICT_optimizeTrainFromBuffer_cover(dictBuffer, optDictSize, 3424 CNBuffer, samplesSizes, 3425 nbSamples / 4, ¶ms); 3426 if (ZDICT_isError(optDictSize)) goto _output_error; 3427 DISPLAYLEVEL(3, "OK, created dictionary of size %u \n", (unsigned)optDictSize); 3428 3429 DISPLAYLEVEL(3, "test%3i : check dictID : ", testNb++); 3430 dictID = ZDICT_getDictID(dictBuffer, optDictSize); 3431 if (dictID==0) goto _output_error; 3432 DISPLAYLEVEL(3, "OK : %u \n", (unsigned)dictID); 3433 3434 ZSTD_freeCCtx(cctx); 3435 free(dictBuffer); 3436 free(samplesSizes); 3437 } 3438 3439 /* Decompression defense tests */ 3440 DISPLAYLEVEL(3, "test%3i : Check input length for magic number : ", testNb++); 3441 { size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, CNBuffer, 3); /* too small input */ 3442 if (!ZSTD_isError(r)) goto _output_error; 3443 if (ZSTD_getErrorCode(r) != ZSTD_error_srcSize_wrong) goto _output_error; } 3444 DISPLAYLEVEL(3, "OK \n"); 3445 3446 DISPLAYLEVEL(3, "test%3i : Check magic Number : ", testNb++); 3447 ((char*)(CNBuffer))[0] = 1; 3448 { size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, CNBuffer, 4); 3449 if (!ZSTD_isError(r)) goto _output_error; } 3450 DISPLAYLEVEL(3, "OK \n"); 3451 3452 /* content size verification test */ 3453 DISPLAYLEVEL(3, "test%3i : Content size verification : ", testNb++); 3454 { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); 3455 size_t const srcSize = 5000; 3456 size_t const wrongSrcSize = (srcSize + 1000); 3457 ZSTD_parameters params = ZSTD_getParams(1, wrongSrcSize, 0); 3458 params.fParams.contentSizeFlag = 1; 3459 CHECK_Z( ZSTD_compressBegin_advanced(cctx, NULL, 0, params, wrongSrcSize) ); 3460 { size_t const result = ZSTD_compressEnd(cctx, decodedBuffer, CNBuffSize, CNBuffer, srcSize); 3461 if (!ZSTD_isError(result)) goto _output_error; 3462 if (ZSTD_getErrorCode(result) != ZSTD_error_srcSize_wrong) goto _output_error; 3463 DISPLAYLEVEL(3, "OK : %s \n", ZSTD_getErrorName(result)); 3464 } 3465 ZSTD_freeCCtx(cctx); 3466 } 3467 3468 /* negative compression level test : ensure simple API and advanced API produce same result */ 3469 DISPLAYLEVEL(3, "test%3i : negative compression level : ", testNb++); 3470 { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); 3471 size_t const srcSize = CNBuffSize / 5; 3472 int const compressionLevel = -1; 3473 3474 assert(cctx != NULL); 3475 { size_t const cSize_1pass = ZSTD_compress(compressedBuffer, compressedBufferSize, 3476 CNBuffer, srcSize, compressionLevel); 3477 if (ZSTD_isError(cSize_1pass)) goto _output_error; 3478 3479 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, compressionLevel) ); 3480 { size_t const compressionResult = ZSTD_compress2(cctx, 3481 compressedBuffer, compressedBufferSize, 3482 CNBuffer, srcSize); 3483 DISPLAYLEVEL(5, "simple=%u vs %u=advanced : ", (unsigned)cSize_1pass, (unsigned)compressionResult); 3484 if (ZSTD_isError(compressionResult)) goto _output_error; 3485 if (compressionResult != cSize_1pass) goto _output_error; 3486 } } 3487 ZSTD_freeCCtx(cctx); 3488 } 3489 DISPLAYLEVEL(3, "OK \n"); 3490 3491 /* parameters order test */ 3492 { size_t const inputSize = CNBuffSize / 2; 3493 U64 xxh64; 3494 3495 { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); 3496 DISPLAYLEVEL(3, "test%3i : parameters in order : ", testNb++); 3497 assert(cctx != NULL); 3498 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 2) ); 3499 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable) ); 3500 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, 18) ); 3501 { size_t const compressedSize = ZSTD_compress2(cctx, 3502 compressedBuffer, ZSTD_compressBound(inputSize), 3503 CNBuffer, inputSize); 3504 CHECK_Z(compressedSize); 3505 cSize = compressedSize; 3506 xxh64 = XXH64(compressedBuffer, compressedSize, 0); 3507 } 3508 DISPLAYLEVEL(3, "OK (compress : %u -> %u bytes)\n", (unsigned)inputSize, (unsigned)cSize); 3509 ZSTD_freeCCtx(cctx); 3510 } 3511 3512 { ZSTD_CCtx* cctx = ZSTD_createCCtx(); 3513 DISPLAYLEVEL(3, "test%3i : parameters disordered : ", testNb++); 3514 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, 18) ); 3515 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable) ); 3516 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 2) ); 3517 { size_t const result = ZSTD_compress2(cctx, 3518 compressedBuffer, ZSTD_compressBound(inputSize), 3519 CNBuffer, inputSize); 3520 CHECK_Z(result); 3521 if (result != cSize) goto _output_error; /* must result in same compressed result, hence same size */ 3522 if (XXH64(compressedBuffer, result, 0) != xxh64) goto _output_error; /* must result in exactly same content, hence same hash */ 3523 DISPLAYLEVEL(3, "OK (compress : %u -> %u bytes)\n", (unsigned)inputSize, (unsigned)result); 3524 } 3525 ZSTD_freeCCtx(cctx); 3526 } 3527 } 3528 3529 /* advanced parameters for decompression */ 3530 { ZSTD_DCtx* const dctx = ZSTD_createDCtx(); 3531 assert(dctx != NULL); 3532 3533 DISPLAYLEVEL(3, "test%3i : get dParameter bounds ", testNb++); 3534 { ZSTD_bounds const bounds = ZSTD_dParam_getBounds(ZSTD_d_windowLogMax); 3535 CHECK_Z(bounds.error); 3536 } 3537 DISPLAYLEVEL(3, "OK \n"); 3538 3539 DISPLAYLEVEL(3, "test%3i : wrong dParameter : ", testNb++); 3540 { size_t const sr = ZSTD_DCtx_setParameter(dctx, (ZSTD_dParameter)999999, 0); 3541 if (!ZSTD_isError(sr)) goto _output_error; 3542 } 3543 { ZSTD_bounds const bounds = ZSTD_dParam_getBounds((ZSTD_dParameter)999998); 3544 if (!ZSTD_isError(bounds.error)) goto _output_error; 3545 } 3546 DISPLAYLEVEL(3, "OK \n"); 3547 3548 DISPLAYLEVEL(3, "test%3i : out of bound dParameter : ", testNb++); 3549 { size_t const sr = ZSTD_DCtx_setParameter(dctx, ZSTD_d_windowLogMax, 9999); 3550 if (!ZSTD_isError(sr)) goto _output_error; 3551 } 3552 { size_t const sr = ZSTD_DCtx_setParameter(dctx, ZSTD_d_format, (ZSTD_format_e)888); 3553 if (!ZSTD_isError(sr)) goto _output_error; 3554 } 3555 DISPLAYLEVEL(3, "OK \n"); 3556 3557 ZSTD_freeDCtx(dctx); 3558 } 3559 3560 3561 /* custom formats tests */ 3562 { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); 3563 ZSTD_DCtx* const dctx = ZSTD_createDCtx(); 3564 size_t const inputSize = CNBuffSize / 2; /* won't cause pb with small dict size */ 3565 assert(dctx != NULL); assert(cctx != NULL); 3566 3567 /* basic block compression */ 3568 DISPLAYLEVEL(3, "test%3i : magic-less format test : ", testNb++); 3569 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_format, ZSTD_f_zstd1_magicless) ); 3570 { ZSTD_inBuffer in = { CNBuffer, inputSize, 0 }; 3571 ZSTD_outBuffer out = { compressedBuffer, ZSTD_compressBound(inputSize), 0 }; 3572 size_t const result = ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end); 3573 if (result != 0) goto _output_error; 3574 if (in.pos != in.size) goto _output_error; 3575 cSize = out.pos; 3576 } 3577 DISPLAYLEVEL(3, "OK (compress : %u -> %u bytes)\n", (unsigned)inputSize, (unsigned)cSize); 3578 3579 DISPLAYLEVEL(3, "test%3i : decompress normally (should fail) : ", testNb++); 3580 { size_t const decodeResult = ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize); 3581 if (ZSTD_getErrorCode(decodeResult) != ZSTD_error_prefix_unknown) goto _output_error; 3582 DISPLAYLEVEL(3, "OK : %s \n", ZSTD_getErrorName(decodeResult)); 3583 } 3584 3585 DISPLAYLEVEL(3, "test%3i : decompress of magic-less frame : ", testNb++); 3586 ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters); 3587 CHECK_Z( ZSTD_DCtx_setParameter(dctx, ZSTD_d_format, ZSTD_f_zstd1_magicless) ); 3588 { ZSTD_FrameHeader zfh; 3589 size_t const zfhrt = ZSTD_getFrameHeader_advanced(&zfh, compressedBuffer, cSize, ZSTD_f_zstd1_magicless); 3590 if (zfhrt != 0) goto _output_error; 3591 } 3592 /* one shot */ 3593 { size_t const result = ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize); 3594 if (result != inputSize) goto _output_error; 3595 DISPLAYLEVEL(3, "one-shot OK, "); 3596 } 3597 /* streaming */ 3598 { ZSTD_inBuffer in = { compressedBuffer, cSize, 0 }; 3599 ZSTD_outBuffer out = { decodedBuffer, CNBuffSize, 0 }; 3600 size_t const result = ZSTD_decompressStream(dctx, &out, &in); 3601 if (result != 0) goto _output_error; 3602 if (in.pos != in.size) goto _output_error; 3603 if (out.pos != inputSize) goto _output_error; 3604 DISPLAYLEVEL(3, "streaming OK : regenerated %u bytes \n", (unsigned)out.pos); 3605 } 3606 3607 /* basic block compression */ 3608 DISPLAYLEVEL(3, "test%3i : empty magic-less format test : ", testNb++); 3609 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_format, ZSTD_f_zstd1_magicless) ); 3610 { ZSTD_inBuffer in = { CNBuffer, 0, 0 }; 3611 ZSTD_outBuffer out = { compressedBuffer, ZSTD_compressBound(0), 0 }; 3612 size_t const result = ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end); 3613 if (result != 0) goto _output_error; 3614 if (in.pos != in.size) goto _output_error; 3615 cSize = out.pos; 3616 } 3617 DISPLAYLEVEL(3, "OK (compress : %u -> %u bytes)\n", (unsigned)0, (unsigned)cSize); 3618 3619 DISPLAYLEVEL(3, "test%3i : decompress of empty magic-less frame : ", testNb++); 3620 ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters); 3621 CHECK_Z( ZSTD_DCtx_setParameter(dctx, ZSTD_d_format, ZSTD_f_zstd1_magicless) ); 3622 /* one shot */ 3623 { size_t const result = ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize); 3624 if (result != 0) goto _output_error; 3625 DISPLAYLEVEL(3, "one-shot OK, "); 3626 } 3627 /* streaming */ 3628 { ZSTD_inBuffer in = { compressedBuffer, cSize, 0 }; 3629 ZSTD_outBuffer out = { decodedBuffer, CNBuffSize, 0 }; 3630 size_t const result = ZSTD_decompressStream(dctx, &out, &in); 3631 if (result != 0) goto _output_error; 3632 if (in.pos != in.size) goto _output_error; 3633 if (out.pos != 0) goto _output_error; 3634 DISPLAYLEVEL(3, "streaming OK : regenerated %u bytes \n", (unsigned)out.pos); 3635 } 3636 3637 ZSTD_freeCCtx(cctx); 3638 ZSTD_freeDCtx(dctx); 3639 } 3640 3641 DISPLAYLEVEL(3, "test%3i : Decompression parameter reset test : ", testNb++); 3642 { 3643 ZSTD_DCtx* const dctx = ZSTD_createDCtx(); 3644 /* Attempt to future proof this to new parameters. */ 3645 int const maxParam = 2000; 3646 int param; 3647 if (ZSTD_d_experimentalParam3 > maxParam) goto _output_error; 3648 for (param = 0; param < maxParam; ++param) { 3649 ZSTD_dParameter dParam = (ZSTD_dParameter)param; 3650 ZSTD_bounds bounds = ZSTD_dParam_getBounds(dParam); 3651 int value1; 3652 int value2; 3653 int check; 3654 if (ZSTD_isError(bounds.error)) 3655 continue; 3656 CHECK_Z(ZSTD_DCtx_getParameter(dctx, dParam, &value1)); 3657 value2 = (value1 != bounds.lowerBound) ? bounds.lowerBound : bounds.upperBound; 3658 CHECK_Z(ZSTD_DCtx_setParameter(dctx, dParam, value2)); 3659 CHECK_Z(ZSTD_DCtx_getParameter(dctx, dParam, &check)); 3660 if (check != value2) goto _output_error; 3661 CHECK_Z(ZSTD_DCtx_reset(dctx, ZSTD_reset_parameters)); 3662 CHECK_Z(ZSTD_DCtx_getParameter(dctx, dParam, &check)); 3663 if (check != value1) goto _output_error; 3664 } 3665 ZSTD_freeDCtx(dctx); 3666 } 3667 DISPLAYLEVEL(3, "OK \n"); 3668 3669 /* block API tests */ 3670 { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); 3671 ZSTD_DCtx* const dctx = ZSTD_createDCtx(); 3672 static const size_t dictSize = 65 KB; 3673 static const size_t blockSize = 100 KB; /* won't cause pb with small dict size */ 3674 size_t cSize2; 3675 assert(cctx != NULL); assert(dctx != NULL); 3676 3677 /* basic block compression */ 3678 DISPLAYLEVEL(3, "test%3i : Block compression test : ", testNb++); 3679 CHECK_Z( ZSTD_compressBegin(cctx, 5) ); 3680 CHECK_Z( ZSTD_getBlockSize(cctx) >= blockSize); 3681 CHECK_VAR(cSize, ZSTD_compressBlock(cctx, compressedBuffer, ZSTD_compressBound(blockSize), CNBuffer, blockSize) ); 3682 DISPLAYLEVEL(3, "OK \n"); 3683 3684 DISPLAYLEVEL(3, "test%3i : Block decompression test : ", testNb++); 3685 CHECK_Z( ZSTD_decompressBegin(dctx) ); 3686 { CHECK_NEWV(r, ZSTD_decompressBlock(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize) ); 3687 if (r != blockSize) goto _output_error; } 3688 DISPLAYLEVEL(3, "OK \n"); 3689 3690 /* very long stream of block compression */ 3691 DISPLAYLEVEL(3, "test%3i : Huge block streaming compression test : ", testNb++); 3692 CHECK_Z( ZSTD_compressBegin(cctx, -199) ); /* we just want to quickly overflow internal U32 index */ 3693 CHECK_Z( ZSTD_getBlockSize(cctx) >= blockSize); 3694 { U64 const toCompress = 5000000000ULL; /* > 4 GB */ 3695 U64 compressed = 0; 3696 while (compressed < toCompress) { 3697 size_t const blockCSize = ZSTD_compressBlock(cctx, compressedBuffer, ZSTD_compressBound(blockSize), CNBuffer, blockSize); 3698 assert(blockCSize != 0); 3699 if (ZSTD_isError(blockCSize)) goto _output_error; 3700 compressed += blockCSize; 3701 } } 3702 DISPLAYLEVEL(3, "OK \n"); 3703 3704 /* dictionary block compression */ 3705 DISPLAYLEVEL(3, "test%3i : Dictionary Block compression test : ", testNb++); 3706 CHECK_Z( ZSTD_compressBegin_usingDict(cctx, CNBuffer, dictSize, 5) ); 3707 CHECK_VAR(cSize, ZSTD_compressBlock(cctx, compressedBuffer, ZSTD_compressBound(blockSize), (char*)CNBuffer+dictSize, blockSize)); 3708 RDG_genBuffer((char*)CNBuffer+dictSize+blockSize, blockSize, 0.0, 0.0, seed); /* create a non-compressible second block */ 3709 { CHECK_NEWV(r, ZSTD_compressBlock(cctx, (char*)compressedBuffer+cSize, ZSTD_compressBound(blockSize), (char*)CNBuffer+dictSize+blockSize, blockSize) ); /* for cctx history consistency */ 3710 assert(r == 0); /* non-compressible block */ } 3711 memcpy((char*)compressedBuffer+cSize, (char*)CNBuffer+dictSize+blockSize, blockSize); /* send non-compressed block (without header) */ 3712 CHECK_VAR(cSize2, ZSTD_compressBlock(cctx, (char*)compressedBuffer+cSize+blockSize, ZSTD_compressBound(blockSize), 3713 (char*)CNBuffer+dictSize+2*blockSize, blockSize)); 3714 DISPLAYLEVEL(3, "OK \n"); 3715 3716 DISPLAYLEVEL(3, "test%3i : Dictionary Block decompression test : ", testNb++); 3717 CHECK_Z( ZSTD_decompressBegin_usingDict(dctx, CNBuffer, dictSize) ); 3718 { CHECK_NEWV( r, ZSTD_decompressBlock(dctx, decodedBuffer, blockSize, compressedBuffer, cSize) ); 3719 if (r != blockSize) { 3720 DISPLAYLEVEL(1, "ZSTD_decompressBlock() with _usingDict() fails : %u, instead of %u expected \n", (unsigned)r, (unsigned)blockSize); 3721 goto _output_error; 3722 } } 3723 memcpy((char*)decodedBuffer+blockSize, (char*)compressedBuffer+cSize, blockSize); 3724 ZSTD_insertBlock(dctx, (char*)decodedBuffer+blockSize, blockSize); /* insert non-compressed block into dctx history */ 3725 { CHECK_NEWV( r, ZSTD_decompressBlock(dctx, (char*)decodedBuffer+2*blockSize, blockSize, (char*)compressedBuffer+cSize+blockSize, cSize2) ); 3726 if (r != blockSize) { 3727 DISPLAYLEVEL(1, "ZSTD_decompressBlock() with _usingDict() and after insertBlock() fails : %u, instead of %u expected \n", (unsigned)r, (unsigned)blockSize); 3728 goto _output_error; 3729 } } 3730 assert(memcpy((char*)CNBuffer+dictSize, decodedBuffer, blockSize*3)); /* ensure regenerated content is identical to origin */ 3731 DISPLAYLEVEL(3, "OK \n"); 3732 3733 DISPLAYLEVEL(3, "test%3i : Block compression with CDict : ", testNb++); 3734 { ZSTD_CDict* const cdict = ZSTD_createCDict(CNBuffer, dictSize, 3); 3735 if (cdict==NULL) goto _output_error; 3736 CHECK_Z( ZSTD_compressBegin_usingCDict(cctx, cdict) ); 3737 CHECK_Z( ZSTD_compressBlock(cctx, compressedBuffer, ZSTD_compressBound(blockSize), (char*)CNBuffer+dictSize, blockSize) ); 3738 ZSTD_freeCDict(cdict); 3739 } 3740 DISPLAYLEVEL(3, "OK \n"); 3741 3742 ZSTD_freeCCtx(cctx); 3743 ZSTD_freeDCtx(dctx); 3744 } 3745 3746 /* rle detection test: must compress better blocks with a single identical byte repeated */ 3747 { size_t sampleSize = 0; 3748 size_t maxCompressedSize = 46; /* block 1, 2: compressed, block 3: RLE, zstd 1.4.4 */ 3749 DISPLAYLEVEL(3, "test%3i : RLE detection test : ", testNb++); 3750 memset((char*)CNBuffer+sampleSize, 'B', 256 KB - 2); 3751 sampleSize += 256 KB - 2; 3752 memset((char*)CNBuffer+sampleSize, 'A', 100 KB); 3753 sampleSize += 100 KB; 3754 cSize = ZSTD_compress(compressedBuffer, ZSTD_compressBound(sampleSize), CNBuffer, sampleSize, 1); 3755 if (ZSTD_isError(cSize) || cSize > maxCompressedSize) { 3756 DISPLAYLEVEL(4, "error: cSize %u > %u expected ! \n", (unsigned)cSize, (unsigned)maxCompressedSize); 3757 goto _output_error; 3758 } 3759 { CHECK_NEWV(regenSize, ZSTD_decompress(decodedBuffer, sampleSize, compressedBuffer, cSize)); 3760 if (regenSize!=sampleSize) goto _output_error; } 3761 DISPLAYLEVEL(3, "OK \n"); 3762 } 3763 3764 DISPLAYLEVEL(3, "test%3i : ZSTD_generateSequences decode from sequences test : ", testNb++); 3765 { 3766 size_t srcSize = 150 KB; 3767 BYTE* src = (BYTE*)CNBuffer; 3768 BYTE* decoded = (BYTE*)compressedBuffer; 3769 3770 ZSTD_CCtx* cctx = ZSTD_createCCtx(); 3771 ZSTD_Sequence* seqs = (ZSTD_Sequence*)malloc(srcSize * sizeof(ZSTD_Sequence)); 3772 size_t seqsSize; 3773 3774 if (seqs == NULL) goto _output_error; 3775 assert(cctx != NULL); 3776 ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 19); 3777 /* Populate src with random data */ 3778 RDG_genBuffer(CNBuffer, srcSize, compressibility, 0.5, seed); 3779 3780 /* Test with block delimiters roundtrip */ 3781 seqsSize = ZSTD_generateSequences(cctx, seqs, srcSize, src, srcSize); 3782 CHECK_Z(seqsSize); 3783 FUZ_decodeSequences(decoded, seqs, seqsSize, src, srcSize, ZSTD_sf_explicitBlockDelimiters); 3784 assert(!memcmp(CNBuffer, compressedBuffer, srcSize)); 3785 3786 /* Test no block delimiters roundtrip */ 3787 seqsSize = ZSTD_mergeBlockDelimiters(seqs, seqsSize); 3788 CHECK_Z(seqsSize); 3789 FUZ_decodeSequences(decoded, seqs, seqsSize, src, srcSize, ZSTD_sf_noBlockDelimiters); 3790 assert(!memcmp(CNBuffer, compressedBuffer, srcSize)); 3791 3792 ZSTD_freeCCtx(cctx); 3793 free(seqs); 3794 } 3795 DISPLAYLEVEL(3, "OK \n"); 3796 3797 DISPLAYLEVEL(3, "test%3i : ZSTD_generateSequences too small output buffer : ", testNb++); 3798 { 3799 const size_t seqsCapacity = 10; 3800 const size_t srcSize = 150 KB; 3801 const BYTE* src = (BYTE*)CNBuffer; 3802 3803 ZSTD_CCtx* const cctx = ZSTD_createCCtx(); 3804 ZSTD_Sequence* const seqs = (ZSTD_Sequence*)malloc(seqsCapacity * sizeof(ZSTD_Sequence)); 3805 3806 if (seqs == NULL) goto _output_error; 3807 if (cctx == NULL) goto _output_error; 3808 /* Populate src with random data */ 3809 RDG_genBuffer(CNBuffer, srcSize, compressibility, 0.5, seed); 3810 3811 /* Test with block delimiters roundtrip */ 3812 { 3813 size_t const seqsSize = ZSTD_generateSequences(cctx, seqs, seqsCapacity, src, srcSize); 3814 if (!ZSTD_isError(seqsSize)) goto _output_error; 3815 } 3816 3817 ZSTD_freeCCtx(cctx); 3818 free(seqs); 3819 } 3820 DISPLAYLEVEL(3, "OK \n"); 3821 3822 DISPLAYLEVEL(3, "test%3i : ZSTD_getSequences followed by ZSTD_compressSequences : ", testNb++); 3823 { 3824 const size_t srcSize = 500 KB; 3825 const BYTE* const src = (BYTE*)CNBuffer; 3826 BYTE* const dst = (BYTE*)compressedBuffer; 3827 const size_t dstCapacity = ZSTD_compressBound(srcSize); 3828 const size_t decompressSize = srcSize; 3829 char* const decompressBuffer = (char*)malloc(decompressSize); 3830 size_t compressedSize; 3831 3832 ZSTD_CCtx* const cctx = ZSTD_createCCtx(); 3833 ZSTD_Sequence* const seqs = (ZSTD_Sequence*)malloc(srcSize * sizeof(ZSTD_Sequence)); 3834 size_t nbSeqs; 3835 3836 if (seqs == NULL) goto _output_error; 3837 assert(cctx != NULL); 3838 3839 /* Populate src with compressible random data */ 3840 RDG_genBuffer(CNBuffer, srcSize, compressibility, 0., seed); 3841 3842 /* Roundtrip Test with block delimiters generated by ZSTD_generateSequences() */ 3843 nbSeqs = ZSTD_generateSequences(cctx, seqs, srcSize, src, srcSize); 3844 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters); 3845 ZSTD_CCtx_setParameter(cctx, ZSTD_c_blockDelimiters, ZSTD_sf_explicitBlockDelimiters); 3846 compressedSize = ZSTD_compressSequences(cctx, dst, dstCapacity, seqs, nbSeqs, src, srcSize); 3847 if (ZSTD_isError(compressedSize)) { 3848 DISPLAY("Error in sequence compression with block delims\n"); 3849 goto _output_error; 3850 } 3851 { size_t const dSize = ZSTD_decompress(decompressBuffer, decompressSize, dst, compressedSize); 3852 if (ZSTD_isError(dSize)) { 3853 DISPLAY("Error in sequence compression roundtrip with block delims\n"); 3854 goto _output_error; 3855 } } 3856 assert(!memcmp(decompressBuffer, src, srcSize)); 3857 3858 /* Roundtrip Test with no block delimiters */ 3859 { size_t const nbSeqsAfterMerge = ZSTD_mergeBlockDelimiters(seqs, nbSeqs); 3860 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters); 3861 ZSTD_CCtx_setParameter(cctx, ZSTD_c_blockDelimiters, ZSTD_sf_noBlockDelimiters); 3862 compressedSize = ZSTD_compressSequences(cctx, dst, dstCapacity, seqs, nbSeqsAfterMerge, src, srcSize); 3863 } 3864 if (ZSTD_isError(compressedSize)) { 3865 DISPLAY("Error in sequence compression with no block delims\n"); 3866 goto _output_error; 3867 } 3868 { size_t const dSize = ZSTD_decompress(decompressBuffer, decompressSize, dst, compressedSize); 3869 if (ZSTD_isError(dSize)) { 3870 DISPLAY("Error in sequence compression roundtrip with no block delims\n"); 3871 goto _output_error; 3872 } } 3873 assert(!memcmp(decompressBuffer, src, srcSize)); 3874 3875 ZSTD_freeCCtx(cctx); 3876 free(decompressBuffer); 3877 free(seqs); 3878 } 3879 DISPLAYLEVEL(3, "OK \n"); 3880 3881 DISPLAYLEVEL(3, "test%3i : ZSTD_compressSequencesAndLiterals : ", testNb++); 3882 { 3883 const size_t srcSize = 497000; 3884 const BYTE* const src = (BYTE*)CNBuffer; 3885 BYTE* const dst = (BYTE*)compressedBuffer; 3886 const size_t dstCapacity = ZSTD_compressBound(srcSize); 3887 const size_t decompressSize = srcSize; 3888 char* const decompressBuffer = (char*)malloc(decompressSize); 3889 char* const litBuffer = (char*)malloc(decompressSize); 3890 size_t compressedSize; 3891 3892 ZSTD_CCtx* const cctx = ZSTD_createCCtx(); 3893 ZSTD_Sequence* const seqs = (ZSTD_Sequence*)malloc(srcSize * sizeof(ZSTD_Sequence)); 3894 size_t nbSeqs; 3895 3896 if (litBuffer == NULL) goto _output_error; 3897 if (decompressBuffer == NULL) goto _output_error; 3898 if (seqs == NULL) goto _output_error; 3899 assert(cctx != NULL); 3900 3901 /* Populate src with compressible random data */ 3902 RDG_genBuffer(CNBuffer, srcSize, compressibility, 0., seed); 3903 3904 /* Roundtrip Test using the AndLiterals() variant */ 3905 nbSeqs = ZSTD_generateSequences(cctx, seqs, srcSize, src, srcSize); 3906 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters); 3907 ZSTD_CCtx_setParameter(cctx, ZSTD_c_blockDelimiters, ZSTD_sf_explicitBlockDelimiters); 3908 { size_t const litSize = FUZ_getLitSize(seqs, nbSeqs); 3909 FUZ_transferLiterals(litBuffer, decompressSize, CNBuffer, srcSize, seqs, nbSeqs); 3910 3911 /* not enough literals: must fail */ 3912 compressedSize = ZSTD_compressSequencesAndLiterals(cctx, dst, dstCapacity, seqs, nbSeqs, src, litSize-1, decompressSize, srcSize); 3913 if (!ZSTD_isError(compressedSize)) { 3914 DISPLAY("ZSTD_compressSequencesAndLiterals() should have failed: not enough literals provided\n"); 3915 goto _output_error; 3916 } 3917 3918 /* too many literals: must fail */ 3919 compressedSize = ZSTD_compressSequencesAndLiterals(cctx, dst, dstCapacity, seqs, nbSeqs, src, litSize+1, decompressSize, srcSize); 3920 if (!ZSTD_isError(compressedSize)) { 3921 DISPLAY("ZSTD_compressSequencesAndLiterals() should have failed: too many literals provided\n"); 3922 goto _output_error; 3923 } 3924 3925 /* srcSize too large: must fail */ 3926 compressedSize = ZSTD_compressSequencesAndLiterals(cctx, dst, dstCapacity, seqs, nbSeqs, litBuffer, litSize, decompressSize, srcSize+1); 3927 if (!ZSTD_isError(compressedSize)) { 3928 DISPLAY("ZSTD_compressSequencesAndLiterals() should have failed: srcSize is too large\n"); 3929 goto _output_error; 3930 } 3931 3932 /* srcSize too small: must fail */ 3933 compressedSize = ZSTD_compressSequencesAndLiterals(cctx, dst, dstCapacity, seqs, nbSeqs, litBuffer, litSize, decompressSize, srcSize-1); 3934 if (!ZSTD_isError(compressedSize)) { 3935 DISPLAY("ZSTD_compressSequencesAndLiterals() should have failed: srcSize is too small\n"); 3936 goto _output_error; 3937 } 3938 3939 /* correct amount of literals: should compress successfully */ 3940 compressedSize = ZSTD_compressSequencesAndLiterals(cctx, dst, dstCapacity, seqs, nbSeqs, litBuffer, litSize, decompressSize, srcSize); 3941 if (ZSTD_isError(compressedSize)) { 3942 DISPLAY("Error in ZSTD_compressSequencesAndLiterals()\n"); 3943 goto _output_error; 3944 } 3945 } 3946 { ZSTD_FrameHeader zfh; 3947 size_t const zfhStatus = ZSTD_getFrameHeader(&zfh, dst, compressedSize); 3948 if (zfhStatus != 0) { 3949 DISPLAY("Error reading frame header\n"); 3950 goto _output_error; 3951 } 3952 if (zfh.frameContentSize != srcSize) { 3953 DISPLAY("Error: ZSTD_compressSequencesAndLiterals() did not report srcSize in the frame header\n"); 3954 goto _output_error; 3955 } 3956 if (zfh.windowSize > srcSize) { 3957 DISPLAY("Error: ZSTD_compressSequencesAndLiterals() did not resized window size to smaller contentSize\n"); 3958 goto _output_error; 3959 } 3960 } 3961 { size_t const dSize = ZSTD_decompress(decompressBuffer, decompressSize, dst, compressedSize); 3962 if (ZSTD_isError(dSize)) { 3963 DISPLAY("Error during decompression of frame produced by ZSTD_compressSequencesAndLiterals()\n"); 3964 goto _output_error; 3965 } 3966 if (dSize != srcSize) { 3967 DISPLAY("Error: decompression of frame produced by ZSTD_compressSequencesAndLiterals() has different size\n"); 3968 goto _output_error; 3969 } 3970 if (memcmp(decompressBuffer, src, srcSize)) { 3971 DISPLAY("Error: decompression of frame produced by ZSTD_compressSequencesAndLiterals() produces a different content (of same size)\n"); 3972 goto _output_error; 3973 } 3974 } 3975 3976 ZSTD_freeCCtx(cctx); 3977 free(litBuffer); 3978 free(decompressBuffer); 3979 free(seqs); 3980 } 3981 DISPLAYLEVEL(3, "OK \n"); 3982 3983 /* Multiple blocks of zeros test */ 3984 #define LONGZEROSLENGTH 1000000 /* 1MB of zeros */ 3985 DISPLAYLEVEL(3, "test%3i : compress %u zeroes : ", testNb++, LONGZEROSLENGTH); 3986 memset(CNBuffer, 0, LONGZEROSLENGTH); 3987 CHECK_VAR(cSize, ZSTD_compress(compressedBuffer, ZSTD_compressBound(LONGZEROSLENGTH), CNBuffer, LONGZEROSLENGTH, 1) ); 3988 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/LONGZEROSLENGTH*100); 3989 3990 DISPLAYLEVEL(3, "test%3i : decompress %u zeroes : ", testNb++, LONGZEROSLENGTH); 3991 { CHECK_NEWV(r, ZSTD_decompress(decodedBuffer, LONGZEROSLENGTH, compressedBuffer, cSize) ); 3992 if (r != LONGZEROSLENGTH) goto _output_error; } 3993 DISPLAYLEVEL(3, "OK \n"); 3994 3995 /* All zeroes test (test bug #137) */ 3996 #define ZEROESLENGTH 100 3997 DISPLAYLEVEL(3, "test%3i : compress %u zeroes : ", testNb++, ZEROESLENGTH); 3998 memset(CNBuffer, 0, ZEROESLENGTH); 3999 CHECK_VAR(cSize, ZSTD_compress(compressedBuffer, ZSTD_compressBound(ZEROESLENGTH), CNBuffer, ZEROESLENGTH, 1) ); 4000 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/ZEROESLENGTH*100); 4001 4002 DISPLAYLEVEL(3, "test%3i : decompress %u zeroes : ", testNb++, ZEROESLENGTH); 4003 { CHECK_NEWV(r, ZSTD_decompress(decodedBuffer, ZEROESLENGTH, compressedBuffer, cSize) ); 4004 if (r != ZEROESLENGTH) goto _output_error; } 4005 DISPLAYLEVEL(3, "OK \n"); 4006 4007 /* nbSeq limit test */ 4008 #define _3BYTESTESTLENGTH 131000 4009 #define NB3BYTESSEQLOG 9 4010 #define NB3BYTESSEQ (1 << NB3BYTESSEQLOG) 4011 #define NB3BYTESSEQMASK (NB3BYTESSEQ-1) 4012 /* creates a buffer full of 3-bytes sequences */ 4013 { BYTE _3BytesSeqs[NB3BYTESSEQ][3]; 4014 U32 rSeed = 1; 4015 4016 /* create batch of 3-bytes sequences */ 4017 { int i; 4018 for (i=0; i < NB3BYTESSEQ; i++) { 4019 _3BytesSeqs[i][0] = (BYTE)(FUZ_rand(&rSeed) & 255); 4020 _3BytesSeqs[i][1] = (BYTE)(FUZ_rand(&rSeed) & 255); 4021 _3BytesSeqs[i][2] = (BYTE)(FUZ_rand(&rSeed) & 255); 4022 } } 4023 4024 /* randomly fills CNBuffer with prepared 3-bytes sequences */ 4025 { int i; 4026 for (i=0; i < _3BYTESTESTLENGTH; i += 3) { /* note : CNBuffer size > _3BYTESTESTLENGTH+3 */ 4027 U32 const id = FUZ_rand(&rSeed) & NB3BYTESSEQMASK; 4028 ((BYTE*)CNBuffer)[i+0] = _3BytesSeqs[id][0]; 4029 ((BYTE*)CNBuffer)[i+1] = _3BytesSeqs[id][1]; 4030 ((BYTE*)CNBuffer)[i+2] = _3BytesSeqs[id][2]; 4031 } } } 4032 DISPLAYLEVEL(3, "test%3i : growing nbSeq : ", testNb++); 4033 { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); 4034 size_t const maxNbSeq = _3BYTESTESTLENGTH / 3; 4035 size_t const bound = ZSTD_compressBound(_3BYTESTESTLENGTH); 4036 size_t nbSeq = 1; 4037 while (nbSeq <= maxNbSeq) { 4038 CHECK_Z(ZSTD_compressCCtx(cctx, compressedBuffer, bound, CNBuffer, nbSeq * 3, 19)); 4039 /* Check every sequence for the first 100, then skip more rapidly. */ 4040 if (nbSeq < 100) { 4041 ++nbSeq; 4042 } else { 4043 nbSeq += (nbSeq >> 2); 4044 } 4045 } 4046 ZSTD_freeCCtx(cctx); 4047 } 4048 DISPLAYLEVEL(3, "OK \n"); 4049 4050 DISPLAYLEVEL(3, "test%3i : compress lots 3-bytes sequences : ", testNb++); 4051 CHECK_VAR(cSize, ZSTD_compress(compressedBuffer, ZSTD_compressBound(_3BYTESTESTLENGTH), 4052 CNBuffer, _3BYTESTESTLENGTH, 19) ); 4053 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/_3BYTESTESTLENGTH*100); 4054 4055 DISPLAYLEVEL(3, "test%3i : decompress lots 3-bytes sequence : ", testNb++); 4056 { CHECK_NEWV(r, ZSTD_decompress(decodedBuffer, _3BYTESTESTLENGTH, compressedBuffer, cSize) ); 4057 if (r != _3BYTESTESTLENGTH) goto _output_error; } 4058 DISPLAYLEVEL(3, "OK \n"); 4059 4060 4061 DISPLAYLEVEL(3, "test%3i : growing literals buffer : ", testNb++); 4062 RDG_genBuffer(CNBuffer, CNBuffSize, 0.0, 0.1, seed); 4063 { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); 4064 size_t const bound = ZSTD_compressBound(CNBuffSize); 4065 size_t size = 1; 4066 while (size <= CNBuffSize) { 4067 CHECK_Z(ZSTD_compressCCtx(cctx, compressedBuffer, bound, CNBuffer, size, 3)); 4068 /* Check every size for the first 100, then skip more rapidly. */ 4069 if (size < 100) { 4070 ++size; 4071 } else { 4072 size += (size >> 2); 4073 } 4074 } 4075 ZSTD_freeCCtx(cctx); 4076 } 4077 DISPLAYLEVEL(3, "OK \n"); 4078 4079 DISPLAYLEVEL(3, "test%3i : incompressible data and ill suited dictionary : ", testNb++); 4080 { /* Train a dictionary on low characters */ 4081 size_t dictSize = 16 KB; 4082 void* const dictBuffer = malloc(dictSize); 4083 size_t const totalSampleSize = 1 MB; 4084 size_t const sampleUnitSize = 8 KB; 4085 U32 const nbSamples = (U32)(totalSampleSize / sampleUnitSize); 4086 size_t* const samplesSizes = (size_t*) malloc(nbSamples * sizeof(size_t)); 4087 if (!dictBuffer || !samplesSizes) goto _output_error; 4088 { U32 u; for (u=0; u<nbSamples; u++) samplesSizes[u] = sampleUnitSize; } 4089 dictSize = ZDICT_trainFromBuffer(dictBuffer, dictSize, CNBuffer, samplesSizes, nbSamples); 4090 if (ZDICT_isError(dictSize)) goto _output_error; 4091 /* Reverse the characters to make the dictionary ill suited */ 4092 { U32 u; 4093 for (u = 0; u < CNBuffSize; ++u) { 4094 ((BYTE*)CNBuffer)[u] = 255 - ((BYTE*)CNBuffer)[u]; 4095 } } 4096 { /* Compress the data */ 4097 size_t const inputSize = 500; 4098 size_t const outputSize = ZSTD_compressBound(inputSize); 4099 void* const outputBuffer = malloc(outputSize); 4100 ZSTD_CCtx* const cctx = ZSTD_createCCtx(); 4101 if (!outputBuffer || !cctx) goto _output_error; 4102 CHECK_Z(ZSTD_compress_usingDict(cctx, outputBuffer, outputSize, CNBuffer, inputSize, dictBuffer, dictSize, 1)); 4103 free(outputBuffer); 4104 ZSTD_freeCCtx(cctx); 4105 } 4106 4107 free(dictBuffer); 4108 free(samplesSizes); 4109 } 4110 DISPLAYLEVEL(3, "OK \n"); 4111 4112 4113 /* frame operations on skippable frames */ 4114 { const char skippableFrame[] = "\x52\x2a\x4d\x18\x05\x0\x0\0abcde"; 4115 size_t const skippableFrameSize = sizeof(skippableFrame) - 1 /* remove the terminating /0 */; 4116 4117 DISPLAYLEVEL(3, "test%3i : ZSTD_findFrameCompressedSize on skippable frame : ", testNb++); 4118 CHECK(ZSTD_findFrameCompressedSize(skippableFrame, skippableFrameSize) == skippableFrameSize); 4119 DISPLAYLEVEL(3, "OK \n"); 4120 4121 DISPLAYLEVEL(3, "test%3i : ZSTD_getFrameContentSize on skippable frame : ", testNb++); 4122 CHECK(ZSTD_getFrameContentSize(skippableFrame, skippableFrameSize) == 0); 4123 DISPLAYLEVEL(3, "OK \n"); 4124 4125 DISPLAYLEVEL(3, "test%3i : ZSTD_getFrameHeader on skippable frame : ", testNb++); 4126 { ZSTD_FrameHeader zfh; 4127 size_t const s = ZSTD_getFrameHeader(&zfh, skippableFrame, skippableFrameSize); 4128 CHECK_Z(s); 4129 CHECK(s == 0); /* success */ 4130 CHECK(zfh.frameType == ZSTD_skippableFrame); 4131 CHECK(zfh.headerSize == ZSTD_SKIPPABLEHEADERSIZE); 4132 CHECK(zfh.dictID == 2); /* magic variant */ 4133 assert(skippableFrameSize >= ZSTD_SKIPPABLEHEADERSIZE); 4134 CHECK(zfh.frameContentSize == skippableFrameSize - ZSTD_SKIPPABLEHEADERSIZE); 4135 } 4136 DISPLAYLEVEL(3, "OK \n"); 4137 } 4138 4139 /* error string tests */ 4140 DISPLAYLEVEL(3, "test%3i : testing ZSTD error code strings : ", testNb++); 4141 if (strcmp("No error detected", ZSTD_getErrorName((ZSTD_ErrorCode)(0-ZSTD_error_no_error))) != 0) goto _output_error; 4142 if (strcmp("No error detected", ZSTD_getErrorString(ZSTD_error_no_error)) != 0) goto _output_error; 4143 if (strcmp("Unspecified error code", ZSTD_getErrorString((ZSTD_ErrorCode)(0-ZSTD_error_GENERIC))) != 0) goto _output_error; 4144 if (strcmp("Error (generic)", ZSTD_getErrorName((size_t)0-ZSTD_error_GENERIC)) != 0) goto _output_error; 4145 if (strcmp("Error (generic)", ZSTD_getErrorString(ZSTD_error_GENERIC)) != 0) goto _output_error; 4146 if (strcmp("No error detected", ZSTD_getErrorName(ZSTD_error_GENERIC)) != 0) goto _output_error; 4147 DISPLAYLEVEL(3, "OK \n"); 4148 4149 DISPLAYLEVEL(3, "test%3i : testing ZSTD dictionary sizes : ", testNb++); 4150 RDG_genBuffer(CNBuffer, CNBuffSize, compressibility, 0., seed); 4151 { 4152 size_t const size = MIN(128 KB, CNBuffSize); 4153 ZSTD_CCtx* const cctx = ZSTD_createCCtx(); 4154 ZSTD_CDict* const lgCDict = ZSTD_createCDict(CNBuffer, size, 1); 4155 ZSTD_CDict* const smCDict = ZSTD_createCDict(CNBuffer, 1 KB, 1); 4156 ZSTD_FrameHeader lgHeader; 4157 ZSTD_FrameHeader smHeader; 4158 4159 CHECK_Z(ZSTD_compress_usingCDict(cctx, compressedBuffer, compressedBufferSize, CNBuffer, size, lgCDict)); 4160 CHECK_Z(ZSTD_getFrameHeader(&lgHeader, compressedBuffer, compressedBufferSize)); 4161 CHECK_Z(ZSTD_compress_usingCDict(cctx, compressedBuffer, compressedBufferSize, CNBuffer, size, smCDict)); 4162 CHECK_Z(ZSTD_getFrameHeader(&smHeader, compressedBuffer, compressedBufferSize)); 4163 4164 if (lgHeader.windowSize != smHeader.windowSize) goto _output_error; 4165 4166 ZSTD_freeCDict(smCDict); 4167 ZSTD_freeCDict(lgCDict); 4168 ZSTD_freeCCtx(cctx); 4169 } 4170 DISPLAYLEVEL(3, "OK \n"); 4171 4172 DISPLAYLEVEL(3, "test%3i : testing FSE_normalizeCount() PR#1255: ", testNb++); 4173 { 4174 short norm[32]; 4175 unsigned count[32]; 4176 unsigned const tableLog = 5; 4177 size_t const nbSeq = 32; 4178 unsigned const maxSymbolValue = 31; 4179 size_t i; 4180 4181 for (i = 0; i < 32; ++i) 4182 count[i] = 1; 4183 /* Calling FSE_normalizeCount() on a uniform distribution should not 4184 * cause a division by zero. 4185 */ 4186 FSE_normalizeCount(norm, tableLog, count, nbSeq, maxSymbolValue, /* useLowProbCount */ 1); 4187 } 4188 DISPLAYLEVEL(3, "OK \n"); 4189 4190 DISPLAYLEVEL(3, "test%3i : testing FSE_writeNCount() PR#2779: ", testNb++); 4191 { 4192 size_t const outBufSize = 9; 4193 short const count[11] = {1, 0, 1, 0, 1, 0, 1, 0, 1, 9, 18}; 4194 unsigned const tableLog = 5; 4195 unsigned const maxSymbolValue = 10; 4196 BYTE* outBuf = (BYTE*)malloc(outBufSize*sizeof(BYTE)); 4197 4198 /* Ensure that this write doesn't write out of bounds, and that 4199 * FSE_writeNCount_generic() is *not* called with writeIsSafe == 1. 4200 */ 4201 FSE_writeNCount(outBuf, outBufSize, count, maxSymbolValue, tableLog); 4202 free(outBuf); 4203 } 4204 DISPLAYLEVEL(3, "OK \n"); 4205 4206 DISPLAYLEVEL(3, "test%3i : testing bitwise intrinsics PR#3045: ", testNb++); 4207 { 4208 U32 seed_copy = seed; /* need non-const seed to avoid compiler warning for FUZ_rand(&seed) */ 4209 U32 rand32 = FUZ_rand(&seed_copy); 4210 U64 rand64 = ((U64)FUZ_rand(&seed_copy) << 32) | FUZ_rand(&seed_copy); 4211 U32 lowbit_only_32 = 1; 4212 U64 lowbit_only_64 = 1; 4213 U32 highbit_only_32 = (U32)1 << 31; 4214 U64 highbit_only_64 = (U64)1 << 63; 4215 U32 i; 4216 if (rand32 == 0) rand32 = 1; /* CLZ and CTZ are undefined on 0 */ 4217 if (rand64 == 0) rand64 = 1; /* CLZ and CTZ are undefined on 0 */ 4218 4219 /* Test ZSTD_countTrailingZeros32 */ 4220 CHECK_EQ(ZSTD_countTrailingZeros32(lowbit_only_32), 0u); 4221 CHECK_EQ(ZSTD_countTrailingZeros32(highbit_only_32), 31u); 4222 CHECK_EQ(ZSTD_countTrailingZeros32(rand32), ZSTD_countTrailingZeros32_fallback(rand32)); 4223 4224 /* Test ZSTD_countLeadingZeros32 */ 4225 CHECK_EQ(ZSTD_countLeadingZeros32(lowbit_only_32), 31u); 4226 CHECK_EQ(ZSTD_countLeadingZeros32(highbit_only_32), 0u); 4227 CHECK_EQ(ZSTD_countLeadingZeros32(rand32), ZSTD_countLeadingZeros32_fallback(rand32)); 4228 4229 /* Test ZSTD_countTrailingZeros64 */ 4230 CHECK_EQ(ZSTD_countTrailingZeros64(lowbit_only_64), 0u); 4231 CHECK_EQ(ZSTD_countTrailingZeros64(highbit_only_64), 63u); 4232 4233 /* Test ZSTD_countLeadingZeros64 */ 4234 CHECK_EQ(ZSTD_countLeadingZeros64(lowbit_only_64), 63u); 4235 CHECK_EQ(ZSTD_countLeadingZeros64(highbit_only_64), 0u); 4236 4237 /* Test ZSTD_highbit32 */ 4238 CHECK_EQ(ZSTD_highbit32(lowbit_only_32), 0u); 4239 CHECK_EQ(ZSTD_highbit32(highbit_only_32), 31u); 4240 4241 /* Test ZSTD_NbCommonBytes */ 4242 if (MEM_isLittleEndian()) { 4243 if (MEM_64bits()) { 4244 CHECK_EQ(ZSTD_NbCommonBytes(lowbit_only_32), 0u); 4245 CHECK_EQ(ZSTD_NbCommonBytes(highbit_only_32), 3u); 4246 } else { 4247 CHECK_EQ(ZSTD_NbCommonBytes(lowbit_only_32), 0u); 4248 CHECK_EQ(ZSTD_NbCommonBytes(highbit_only_32), 3u); 4249 } 4250 } else { 4251 if (MEM_64bits()) { 4252 CHECK_EQ(ZSTD_NbCommonBytes(lowbit_only_32), 7u); 4253 CHECK_EQ(ZSTD_NbCommonBytes(highbit_only_32), 4u); 4254 } else { 4255 CHECK_EQ(ZSTD_NbCommonBytes(lowbit_only_32), 3u); 4256 CHECK_EQ(ZSTD_NbCommonBytes(highbit_only_32), 0u); 4257 } 4258 } 4259 4260 /* Test MEM_ intrinsics */ 4261 CHECK_EQ(MEM_swap32(rand32), MEM_swap32_fallback(rand32)); 4262 CHECK_EQ(MEM_swap64(rand64), MEM_swap64_fallback(rand64)); 4263 4264 /* Test fallbacks vs intrinsics on a range of small integers */ 4265 for (i=1; i <= 1000; i++) { 4266 CHECK_EQ(MEM_swap32(i), MEM_swap32_fallback(i)); 4267 CHECK_EQ(MEM_swap64((U64)i), MEM_swap64_fallback((U64)i)); 4268 CHECK_EQ(ZSTD_countTrailingZeros32(i), ZSTD_countTrailingZeros32_fallback(i)); 4269 CHECK_EQ(ZSTD_countLeadingZeros32(i), ZSTD_countLeadingZeros32_fallback(i)); 4270 } 4271 } 4272 DISPLAYLEVEL(3, "OK \n"); 4273 4274 #ifdef ZSTD_MULTITHREAD 4275 DISPLAYLEVEL(3, "test%3i : passing wrong full dict should fail on compressStream2 refPrefix ", testNb++); 4276 { ZSTD_CCtx* cctx = ZSTD_createCCtx(); 4277 size_t const srcSize = 1 MB + 5; /* A little more than ZSTDMT_JOBSIZE_MIN */ 4278 size_t const dstSize = ZSTD_compressBound(srcSize); 4279 void* const src = CNBuffer; 4280 void* const dst = compressedBuffer; 4281 void* dict = (void*)malloc(srcSize); 4282 4283 RDG_genBuffer(src, srcSize, compressibility, 0.5, seed); 4284 RDG_genBuffer(dict, srcSize, compressibility, 0., seed); 4285 4286 /* Make sure there is no ZSTD_MAGIC_NUMBER */ 4287 memset(dict, 0, sizeof(U32)); 4288 4289 /* something more than 1 */ 4290 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 2)); 4291 /* lie and claim this is a full dict */ 4292 CHECK_Z(ZSTD_CCtx_refPrefix_advanced(cctx, dict, srcSize, ZSTD_dct_fullDict)); 4293 4294 { ZSTD_outBuffer out = {dst, dstSize, 0}; 4295 ZSTD_inBuffer in = {src, srcSize, 0}; 4296 /* should fail because its not a full dict like we said it was */ 4297 assert(ZSTD_isError(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush))); 4298 } 4299 4300 ZSTD_freeCCtx(cctx); 4301 free(dict); 4302 } 4303 DISPLAYLEVEL(3, "OK \n"); 4304 4305 DISPLAYLEVEL(3, "test%3i : small dictionary with multithreading and LDM ", testNb++); 4306 { ZSTD_CCtx* cctx = ZSTD_createCCtx(); 4307 size_t const srcSize = 1 MB + 5; /* A little more than ZSTDMT_JOBSIZE_MIN */ 4308 size_t const dictSize = 10; 4309 size_t const dstSize = ZSTD_compressBound(srcSize); 4310 void* const src = CNBuffer; 4311 void* const dst = compressedBuffer; 4312 void* dict = (void*)malloc(dictSize); 4313 4314 RDG_genBuffer(src, srcSize, compressibility, 0.5, seed); 4315 RDG_genBuffer(dict, dictSize, compressibility, 0., seed); 4316 4317 /* Make sure there is no ZSTD_MAGIC_NUMBER */ 4318 memset(dict, 0, sizeof(U32)); 4319 4320 /* Enable MT, LDM, and use refPrefix() for a small dict */ 4321 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 2)); 4322 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable)); 4323 CHECK_Z(ZSTD_CCtx_refPrefix(cctx, dict, dictSize)); 4324 4325 CHECK_Z(ZSTD_compress2(cctx, dst, dstSize, src, srcSize)); 4326 4327 ZSTD_freeCCtx(cctx); 4328 free(dict); 4329 } 4330 DISPLAYLEVEL(3, "OK \n"); 4331 4332 DISPLAYLEVEL(3, "test%3i : ZSTD_getCParams() + dictionary ", testNb++); 4333 { 4334 ZSTD_compressionParameters const medium = ZSTD_getCParams(1, 16*1024-1, 0); 4335 ZSTD_compressionParameters const large = ZSTD_getCParams(1, 128*1024-1, 0); 4336 ZSTD_compressionParameters const smallDict = ZSTD_getCParams(1, 0, 400); 4337 ZSTD_compressionParameters const mediumDict = ZSTD_getCParams(1, 0, 10000); 4338 ZSTD_compressionParameters const largeDict = ZSTD_getCParams(1, 0, 100000); 4339 4340 assert(!memcmp(&smallDict, &mediumDict, sizeof(smallDict))); 4341 assert(!memcmp(&medium, &mediumDict, sizeof(medium))); 4342 assert(!memcmp(&large, &largeDict, sizeof(large))); 4343 } 4344 DISPLAYLEVEL(3, "OK \n"); 4345 4346 DISPLAYLEVEL(3, "test%3i : ZSTD_adjustCParams() + dictionary ", testNb++); 4347 { 4348 ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, 0, 0); 4349 ZSTD_compressionParameters const smallDict = ZSTD_adjustCParams(cParams, 0, 400); 4350 ZSTD_compressionParameters const smallSrcAndDict = ZSTD_adjustCParams(cParams, 500, 400); 4351 4352 assert(smallSrcAndDict.windowLog == 10); 4353 assert(!memcmp(&cParams, &smallDict, sizeof(cParams))); 4354 } 4355 DISPLAYLEVEL(3, "OK \n"); 4356 4357 DISPLAYLEVEL(3, "test%3i : check compression mem usage monotonicity over levels for estimateCCtxSize() : ", testNb++); 4358 { 4359 int level = 1; 4360 size_t prevSize = 0; 4361 for (; level < ZSTD_maxCLevel(); ++level) { 4362 size_t const currSize = ZSTD_estimateCCtxSize(level); 4363 if (prevSize > currSize) { 4364 DISPLAYLEVEL(3, "Error! previous cctx size: %u at level: %d is larger than current cctx size: %u at level: %d", 4365 (unsigned)prevSize, level-1, (unsigned)currSize, level); 4366 goto _output_error; 4367 } 4368 prevSize = currSize; 4369 } 4370 } 4371 DISPLAYLEVEL(3, "OK \n"); 4372 4373 DISPLAYLEVEL(3, "test%3i : check estimateCCtxSize() always larger or equal to ZSTD_estimateCCtxSize_usingCParams() : ", testNb++); 4374 { 4375 size_t const kSizeIncrement = 2 KB; 4376 int level = -3; 4377 4378 for (; level <= ZSTD_maxCLevel(); ++level) { 4379 size_t dictSize = 0; 4380 for (; dictSize <= 256 KB; dictSize += 8 * kSizeIncrement) { 4381 size_t srcSize = 2 KB; 4382 for (; srcSize < 300 KB; srcSize += kSizeIncrement) { 4383 ZSTD_compressionParameters const cParams = ZSTD_getCParams(level, srcSize, dictSize); 4384 size_t const cctxSizeUsingCParams = ZSTD_estimateCCtxSize_usingCParams(cParams); 4385 size_t const cctxSizeUsingLevel = ZSTD_estimateCCtxSize(level); 4386 if (cctxSizeUsingLevel < cctxSizeUsingCParams 4387 || ZSTD_isError(cctxSizeUsingCParams) 4388 || ZSTD_isError(cctxSizeUsingLevel)) { 4389 DISPLAYLEVEL(3, "error! l: %d dict: %u srcSize: %u cctx size cpar: %u, cctx size level: %u\n", 4390 level, (unsigned)dictSize, (unsigned)srcSize, (unsigned)cctxSizeUsingCParams, (unsigned)cctxSizeUsingLevel); 4391 goto _output_error; 4392 } } } } } 4393 DISPLAYLEVEL(3, "OK \n"); 4394 4395 DISPLAYLEVEL(3, "test%3i : thread pool API tests : \n", testNb++) 4396 { 4397 int const threadPoolTestResult = threadPoolTests(); 4398 if (threadPoolTestResult) { 4399 goto _output_error; 4400 } 4401 } 4402 DISPLAYLEVEL(3, "thread pool tests OK \n"); 4403 4404 #endif /* ZSTD_MULTITHREAD */ 4405 4406 _end: 4407 free(CNBuffer); 4408 free(compressedBuffer); 4409 free(decodedBuffer); 4410 return testResult; 4411 4412 _output_error: 4413 testResult = 1; 4414 DISPLAY("Error detected in Unit tests ! \n"); 4415 goto _end; 4416 } 4417 4418 static int longUnitTests(U32 const seed, double compressibility) 4419 { 4420 size_t const CNBuffSize = 5 MB; 4421 void* const CNBuffer = malloc(CNBuffSize); 4422 size_t const compressedBufferSize = ZSTD_compressBound(CNBuffSize); 4423 void* const compressedBuffer = malloc(compressedBufferSize); 4424 void* const decodedBuffer = malloc(CNBuffSize); 4425 int testResult = 0; 4426 unsigned testNb=0; 4427 size_t cSize; 4428 4429 /* Create compressible noise */ 4430 if (!CNBuffer || !compressedBuffer || !decodedBuffer) { 4431 DISPLAY("Not enough memory, aborting\n"); 4432 testResult = 1; 4433 goto _end; 4434 } 4435 RDG_genBuffer(CNBuffer, CNBuffSize, compressibility, 0., seed); 4436 4437 /* note : this test is rather long, it would be great to find a way to speed up its execution */ 4438 DISPLAYLEVEL(3, "longtest%3i : table cleanliness through index reduction : ", testNb++); 4439 { int cLevel; 4440 size_t approxIndex = 0; 4441 size_t maxIndex = ((3U << 29) + (1U << ZSTD_WINDOWLOG_MAX)); /* ZSTD_CURRENT_MAX from zstd_compress_internal.h */ 4442 4443 /* Provision enough space in a static context so that we can do all 4444 * this without ever reallocating, which would reset the indices. */ 4445 size_t const staticCCtxSize = ZSTD_estimateCStreamSize(22); 4446 void* const staticCCtxBuffer = malloc(staticCCtxSize); 4447 ZSTD_CCtx* const cctx = ZSTD_initStaticCCtx(staticCCtxBuffer, staticCCtxSize); 4448 4449 /* bump the indices so the following compressions happen at high 4450 * indices. */ 4451 { ZSTD_outBuffer out = { compressedBuffer, compressedBufferSize, 0 }; 4452 ZSTD_inBuffer in = { CNBuffer, CNBuffSize, 0 }; 4453 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters); 4454 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, -500)); 4455 while (approxIndex <= (maxIndex / 4) * 3) { 4456 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush)); 4457 approxIndex += in.pos; 4458 CHECK_Z(in.pos == in.size); 4459 in.pos = 0; 4460 out.pos = 0; 4461 } 4462 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end)); 4463 } 4464 4465 /* spew a bunch of stuff into the table area */ 4466 for (cLevel = 1; cLevel <= 22; cLevel++) { 4467 ZSTD_outBuffer out = { compressedBuffer, compressedBufferSize / (unsigned)cLevel, 0 }; 4468 ZSTD_inBuffer in = { CNBuffer, CNBuffSize, 0 }; 4469 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters); 4470 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, cLevel)); 4471 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush)); 4472 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end)); 4473 approxIndex += in.pos; 4474 } 4475 4476 /* now crank the indices so we overflow */ 4477 { ZSTD_outBuffer out = { compressedBuffer, compressedBufferSize, 0 }; 4478 ZSTD_inBuffer in = { CNBuffer, CNBuffSize, 0 }; 4479 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters); 4480 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, -500)); 4481 while (approxIndex <= maxIndex) { 4482 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush)); 4483 approxIndex += in.pos; 4484 CHECK_Z(in.pos == in.size); 4485 in.pos = 0; 4486 out.pos = 0; 4487 } 4488 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end)); 4489 } 4490 4491 /* do a bunch of compressions again in low indices and ensure we don't 4492 * hit untracked invalid indices */ 4493 for (cLevel = 1; cLevel <= 22; cLevel++) { 4494 ZSTD_outBuffer out = { compressedBuffer, compressedBufferSize / (unsigned)cLevel, 0 }; 4495 ZSTD_inBuffer in = { CNBuffer, CNBuffSize, 0 }; 4496 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters); 4497 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, cLevel)); 4498 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush)); 4499 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end)); 4500 approxIndex += in.pos; 4501 } 4502 4503 free(staticCCtxBuffer); 4504 } 4505 DISPLAYLEVEL(3, "OK \n"); 4506 4507 DISPLAYLEVEL(3, "longtest%3i : testing ldm no regressions in size for opt parser : ", testNb++); 4508 { size_t cSizeLdm; 4509 size_t cSizeNoLdm; 4510 ZSTD_CCtx* const cctx = ZSTD_createCCtx(); 4511 4512 RDG_genBuffer(CNBuffer, CNBuffSize, 0.5, 0.5, seed); 4513 4514 /* Enable checksum to verify round trip. */ 4515 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1)); 4516 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable)); 4517 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 19)); 4518 4519 /* Round trip once with ldm. */ 4520 cSizeLdm = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize); 4521 CHECK_Z(cSizeLdm); 4522 CHECK_Z(ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSizeLdm)); 4523 4524 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters); 4525 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1)); 4526 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_disable)); 4527 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 19)); 4528 4529 /* Round trip once without ldm. */ 4530 cSizeNoLdm = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize); 4531 CHECK_Z(cSizeNoLdm); 4532 CHECK_Z(ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSizeNoLdm)); 4533 4534 if (cSizeLdm > cSizeNoLdm) { 4535 DISPLAY("Using long mode should not cause regressions for btopt+\n"); 4536 testResult = 1; 4537 goto _end; 4538 } 4539 4540 ZSTD_freeCCtx(cctx); 4541 } 4542 DISPLAYLEVEL(3, "OK \n"); 4543 4544 DISPLAYLEVEL(3, "longtest%3i : testing cdict compression with different attachment strategies : ", testNb++); 4545 { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); 4546 ZSTD_DCtx* const dctx = ZSTD_createDCtx(); 4547 size_t dictSize = CNBuffSize; 4548 void* dict = (void*)malloc(dictSize); 4549 ZSTD_CCtx_params* cctx_params = ZSTD_createCCtxParams(); 4550 ZSTD_dictAttachPref_e const attachPrefs[] = { 4551 ZSTD_dictDefaultAttach, 4552 ZSTD_dictForceAttach, 4553 ZSTD_dictForceCopy, 4554 ZSTD_dictForceLoad, 4555 ZSTD_dictDefaultAttach, 4556 ZSTD_dictForceAttach, 4557 ZSTD_dictForceCopy, 4558 ZSTD_dictForceLoad 4559 }; 4560 int const enableDedicatedDictSearch[] = {0, 0, 0, 0, 1, 1, 1, 1}; 4561 int cLevel; 4562 int i; 4563 4564 RDG_genBuffer(dict, dictSize, 0.5, 0.5, seed); 4565 RDG_genBuffer(CNBuffer, CNBuffSize, 0.6, 0.6, seed); 4566 4567 CHECK_Z(cctx_params != NULL); 4568 4569 for (dictSize = CNBuffSize; dictSize; dictSize = dictSize >> 3) { 4570 DISPLAYLEVEL(3, "\n Testing with dictSize %u ", (U32)dictSize); 4571 for (cLevel = 4; cLevel < 13; cLevel++) { 4572 for (i = 0; i < 8; ++i) { 4573 ZSTD_dictAttachPref_e const attachPref = attachPrefs[i]; 4574 int const enableDDS = enableDedicatedDictSearch[i]; 4575 ZSTD_CDict* cdict; 4576 4577 DISPLAYLEVEL(5, "\n dictSize %u cLevel %d iter %d ", (U32)dictSize, cLevel, i); 4578 4579 ZSTD_CCtxParams_init(cctx_params, cLevel); 4580 CHECK_Z(ZSTD_CCtxParams_setParameter(cctx_params, ZSTD_c_enableDedicatedDictSearch, enableDDS)); 4581 4582 cdict = ZSTD_createCDict_advanced2(dict, dictSize, ZSTD_dlm_byRef, ZSTD_dct_auto, cctx_params, ZSTD_defaultCMem); 4583 CHECK(cdict != NULL); 4584 4585 CHECK_Z(ZSTD_CCtx_refCDict(cctx, cdict)); 4586 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_forceAttachDict, (int)attachPref)); 4587 4588 cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize); 4589 CHECK_Z(cSize); 4590 CHECK_Z(ZSTD_decompress_usingDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, dict, dictSize)); 4591 4592 DISPLAYLEVEL(5, "compressed to %u bytes ", (U32)cSize); 4593 4594 CHECK_Z(ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters)); 4595 ZSTD_freeCDict(cdict); 4596 } } } 4597 4598 ZSTD_freeCCtx(cctx); 4599 ZSTD_freeDCtx(dctx); 4600 ZSTD_freeCCtxParams(cctx_params); 4601 free(dict); 4602 } 4603 DISPLAYLEVEL(3, "OK \n"); 4604 4605 _end: 4606 free(CNBuffer); 4607 free(compressedBuffer); 4608 free(decodedBuffer); 4609 return testResult; 4610 } 4611 4612 4613 static size_t findDiff(const void* buf1, const void* buf2, size_t max) 4614 { 4615 const BYTE* b1 = (const BYTE*)buf1; 4616 const BYTE* b2 = (const BYTE*)buf2; 4617 size_t u; 4618 for (u=0; u<max; u++) { 4619 if (b1[u] != b2[u]) break; 4620 } 4621 return u; 4622 } 4623 4624 4625 static ZSTD_parameters FUZ_makeParams(ZSTD_compressionParameters cParams, ZSTD_frameParameters fParams) 4626 { 4627 ZSTD_parameters params; 4628 params.cParams = cParams; 4629 params.fParams = fParams; 4630 return params; 4631 } 4632 4633 static size_t FUZ_rLogLength(U32* seed, U32 logLength) 4634 { 4635 size_t const lengthMask = ((size_t)1 << logLength) - 1; 4636 return (lengthMask+1) + (FUZ_rand(seed) & lengthMask); 4637 } 4638 4639 static size_t FUZ_randomLength(U32* seed, U32 maxLog) 4640 { 4641 U32 const logLength = FUZ_rand(seed) % maxLog; 4642 return FUZ_rLogLength(seed, logLength); 4643 } 4644 4645 #undef CHECK 4646 #define CHECK(cond, ...) { \ 4647 if (cond) { \ 4648 DISPLAY("Error => "); \ 4649 DISPLAY(__VA_ARGS__); \ 4650 DISPLAY(" (seed %u, test nb %u) \n", (unsigned)seed, testNb); \ 4651 goto _output_error; \ 4652 } } 4653 4654 #undef CHECK_Z 4655 #define CHECK_Z(f) { \ 4656 size_t const err = f; \ 4657 if (ZSTD_isError(err)) { \ 4658 DISPLAY("Error => %s : %s ", \ 4659 #f, ZSTD_getErrorName(err)); \ 4660 DISPLAY(" (seed %u, test nb %u) \n", (unsigned)seed, testNb); \ 4661 goto _output_error; \ 4662 } } 4663 4664 4665 static int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, U32 const maxDurationS, double compressibility, int bigTests) 4666 { 4667 static const U32 maxSrcLog = 23; 4668 static const U32 maxSampleLog = 22; 4669 size_t const srcBufferSize = (size_t)1<<maxSrcLog; 4670 size_t const dstBufferSize = (size_t)1<<maxSampleLog; 4671 size_t const cBufferSize = ZSTD_compressBound(dstBufferSize); 4672 BYTE* cNoiseBuffer[5]; 4673 BYTE* const cBuffer = (BYTE*) malloc (cBufferSize); 4674 BYTE* const dstBuffer = (BYTE*) malloc (dstBufferSize); 4675 BYTE* const mirrorBuffer = (BYTE*) malloc (dstBufferSize); 4676 ZSTD_CCtx* const refCtx = ZSTD_createCCtx(); 4677 ZSTD_CCtx* const ctx = ZSTD_createCCtx(); 4678 ZSTD_DCtx* const dctx = ZSTD_createDCtx(); 4679 U32 result = 0; 4680 unsigned testNb = 0; 4681 U32 coreSeed = seed; 4682 UTIL_time_t const startClock = UTIL_getTime(); 4683 U64 const maxClockSpan = maxDurationS * SEC_TO_MICRO; 4684 int const cLevelLimiter = bigTests ? 3 : 2; 4685 4686 /* allocation */ 4687 cNoiseBuffer[0] = (BYTE*)malloc (srcBufferSize); 4688 cNoiseBuffer[1] = (BYTE*)malloc (srcBufferSize); 4689 cNoiseBuffer[2] = (BYTE*)malloc (srcBufferSize); 4690 cNoiseBuffer[3] = (BYTE*)malloc (srcBufferSize); 4691 cNoiseBuffer[4] = (BYTE*)malloc (srcBufferSize); 4692 CHECK (!cNoiseBuffer[0] || !cNoiseBuffer[1] || !cNoiseBuffer[2] || !cNoiseBuffer[3] || !cNoiseBuffer[4] 4693 || !dstBuffer || !mirrorBuffer || !cBuffer || !refCtx || !ctx || !dctx, 4694 "Not enough memory, fuzzer tests cancelled"); 4695 4696 /* Create initial samples */ 4697 RDG_genBuffer(cNoiseBuffer[0], srcBufferSize, 0.00, 0., coreSeed); /* pure noise */ 4698 RDG_genBuffer(cNoiseBuffer[1], srcBufferSize, 0.05, 0., coreSeed); /* barely compressible */ 4699 RDG_genBuffer(cNoiseBuffer[2], srcBufferSize, compressibility, 0., coreSeed); 4700 RDG_genBuffer(cNoiseBuffer[3], srcBufferSize, 0.95, 0., coreSeed); /* highly compressible */ 4701 RDG_genBuffer(cNoiseBuffer[4], srcBufferSize, 1.00, 0., coreSeed); /* sparse content */ 4702 4703 /* catch up testNb */ 4704 for (testNb=1; testNb < startTest; testNb++) FUZ_rand(&coreSeed); 4705 4706 /* main test loop */ 4707 for ( ; (testNb <= nbTests) || (UTIL_clockSpanMicro(startClock) < maxClockSpan); testNb++ ) { 4708 BYTE* srcBuffer; /* jumping pointer */ 4709 U32 lseed; 4710 size_t sampleSize, maxTestSize, totalTestSize; 4711 size_t cSize, totalCSize, totalGenSize; 4712 U64 crcOrig; 4713 BYTE* sampleBuffer; 4714 const BYTE* dict; 4715 size_t dictSize; 4716 4717 /* notification */ 4718 if (nbTests >= testNb) { DISPLAYUPDATE(2, "\r%6u/%6u ", testNb, nbTests); } 4719 else { DISPLAYUPDATE(2, "\r%6u ", testNb); } 4720 4721 FUZ_rand(&coreSeed); 4722 { U32 const prime1 = 2654435761U; lseed = coreSeed ^ prime1; } 4723 4724 /* srcBuffer selection [0-4] */ 4725 { U32 buffNb = FUZ_rand(&lseed) & 0x7F; 4726 if (buffNb & 7) buffNb=2; /* most common : compressible (P) */ 4727 else { 4728 buffNb >>= 3; 4729 if (buffNb & 7) { 4730 const U32 tnb[2] = { 1, 3 }; /* barely/highly compressible */ 4731 buffNb = tnb[buffNb >> 3]; 4732 } else { 4733 const U32 tnb[2] = { 0, 4 }; /* not compressible / sparse */ 4734 buffNb = tnb[buffNb >> 3]; 4735 } } 4736 srcBuffer = cNoiseBuffer[buffNb]; 4737 } 4738 4739 /* select src segment */ 4740 sampleSize = FUZ_randomLength(&lseed, maxSampleLog); 4741 4742 /* create sample buffer (to catch read error with valgrind & sanitizers) */ 4743 sampleBuffer = (BYTE*)malloc(sampleSize); 4744 CHECK(sampleBuffer==NULL, "not enough memory for sample buffer"); 4745 { size_t const sampleStart = FUZ_rand(&lseed) % (srcBufferSize - sampleSize); 4746 memcpy(sampleBuffer, srcBuffer + sampleStart, sampleSize); } 4747 crcOrig = XXH64(sampleBuffer, sampleSize, 0); 4748 4749 /* compression tests */ 4750 { int const cLevelPositive = (int) 4751 ( FUZ_rand(&lseed) % 4752 ((U32)ZSTD_maxCLevel() - (FUZ_highbit32((U32)sampleSize) / (U32)cLevelLimiter)) ) 4753 + 1; 4754 int const cLevel = ((FUZ_rand(&lseed) & 15) == 3) ? 4755 - (int)((FUZ_rand(&lseed) & 7) + 1) : /* test negative cLevel */ 4756 cLevelPositive; 4757 DISPLAYLEVEL(5, "fuzzer t%u: Simple compression test (level %i) \n", testNb, cLevel); 4758 cSize = ZSTD_compressCCtx(ctx, cBuffer, cBufferSize, sampleBuffer, sampleSize, cLevel); 4759 CHECK(ZSTD_isError(cSize), "ZSTD_compressCCtx failed : %s", ZSTD_getErrorName(cSize)); 4760 4761 /* compression failure test : too small dest buffer */ 4762 assert(cSize > 3); 4763 { const size_t missing = (FUZ_rand(&lseed) % (cSize-2)) + 1; 4764 const size_t tooSmallSize = cSize - missing; 4765 const unsigned endMark = 0x4DC2B1A9; 4766 memcpy(dstBuffer+tooSmallSize, &endMark, sizeof(endMark)); 4767 DISPLAYLEVEL(5, "fuzzer t%u: compress into too small buffer of size %u (missing %u bytes) \n", 4768 testNb, (unsigned)tooSmallSize, (unsigned)missing); 4769 { size_t const errorCode = ZSTD_compressCCtx(ctx, dstBuffer, tooSmallSize, sampleBuffer, sampleSize, cLevel); 4770 CHECK(ZSTD_getErrorCode(errorCode) != ZSTD_error_dstSize_tooSmall, "ZSTD_compressCCtx should have failed ! (buffer too small : %u < %u)", (unsigned)tooSmallSize, (unsigned)cSize); } 4771 { unsigned endCheck; memcpy(&endCheck, dstBuffer+tooSmallSize, sizeof(endCheck)); 4772 CHECK(endCheck != endMark, "ZSTD_compressCCtx : dst buffer overflow (check.%08X != %08X.mark)", endCheck, endMark); } 4773 } } 4774 4775 /* frame header decompression test */ 4776 { ZSTD_FrameHeader zfh; 4777 CHECK_Z( ZSTD_getFrameHeader(&zfh, cBuffer, cSize) ); 4778 CHECK(zfh.frameContentSize != sampleSize, "Frame content size incorrect"); 4779 } 4780 4781 /* Decompressed size test */ 4782 { unsigned long long const rSize = ZSTD_findDecompressedSize(cBuffer, cSize); 4783 CHECK(rSize != sampleSize, "decompressed size incorrect"); 4784 } 4785 4786 /* successful decompression test */ 4787 DISPLAYLEVEL(5, "fuzzer t%u: simple decompression test \n", testNb); 4788 { size_t const margin = (FUZ_rand(&lseed) & 1) ? 0 : (FUZ_rand(&lseed) & 31) + 1; 4789 size_t const dSize = ZSTD_decompress(dstBuffer, sampleSize + margin, cBuffer, cSize); 4790 CHECK(dSize != sampleSize, "ZSTD_decompress failed (%s) (srcSize : %u ; cSize : %u)", ZSTD_getErrorName(dSize), (unsigned)sampleSize, (unsigned)cSize); 4791 { U64 const crcDest = XXH64(dstBuffer, sampleSize, 0); 4792 CHECK(crcOrig != crcDest, "decompression result corrupted (pos %u / %u)", (unsigned)findDiff(sampleBuffer, dstBuffer, sampleSize), (unsigned)sampleSize); 4793 } } 4794 4795 free(sampleBuffer); /* no longer useful after this point */ 4796 4797 /* truncated src decompression test */ 4798 DISPLAYLEVEL(5, "fuzzer t%u: decompression of truncated source \n", testNb); 4799 { size_t const missing = (FUZ_rand(&lseed) % (cSize-2)) + 1; /* no problem, as cSize > 4 (frameHeaderSizer) */ 4800 size_t const tooSmallSize = cSize - missing; 4801 void* cBufferTooSmall = malloc(tooSmallSize); /* valgrind will catch read overflows */ 4802 CHECK(cBufferTooSmall == NULL, "not enough memory !"); 4803 memcpy(cBufferTooSmall, cBuffer, tooSmallSize); 4804 { size_t const errorCode = ZSTD_decompress(dstBuffer, dstBufferSize, cBufferTooSmall, tooSmallSize); 4805 CHECK(!ZSTD_isError(errorCode), "ZSTD_decompress should have failed ! (truncated src buffer)"); } 4806 free(cBufferTooSmall); 4807 } 4808 4809 /* too small dst decompression test */ 4810 DISPLAYLEVEL(5, "fuzzer t%u: decompress into too small dst buffer \n", testNb); 4811 if (sampleSize > 3) { 4812 size_t const missing = (FUZ_rand(&lseed) % (sampleSize-2)) + 1; /* no problem, as cSize > 4 (frameHeaderSizer) */ 4813 size_t const tooSmallSize = sampleSize - missing; 4814 static const BYTE token = 0xA9; 4815 dstBuffer[tooSmallSize] = token; 4816 { size_t const errorCode = ZSTD_decompress(dstBuffer, tooSmallSize, cBuffer, cSize); 4817 CHECK(ZSTD_getErrorCode(errorCode) != ZSTD_error_dstSize_tooSmall, "ZSTD_decompress should have failed : %u > %u (dst buffer too small)", (unsigned)errorCode, (unsigned)tooSmallSize); } 4818 CHECK(dstBuffer[tooSmallSize] != token, "ZSTD_decompress : dst buffer overflow"); 4819 } 4820 4821 /* noisy src decompression test */ 4822 if (cSize > 6) { 4823 /* insert noise into src */ 4824 { U32 const maxNbBits = FUZ_highbit32((U32)(cSize-4)); 4825 size_t pos = 4; /* preserve magic number (too easy to detect) */ 4826 for (;;) { 4827 /* keep some original src */ 4828 { U32 const nbBits = FUZ_rand(&lseed) % maxNbBits; 4829 size_t const mask = (1<<nbBits) - 1; 4830 size_t const skipLength = FUZ_rand(&lseed) & mask; 4831 pos += skipLength; 4832 } 4833 if (pos >= cSize) break; 4834 /* add noise */ 4835 { U32 const nbBitsCodes = FUZ_rand(&lseed) % maxNbBits; 4836 U32 const nbBits = nbBitsCodes ? nbBitsCodes-1 : 0; 4837 size_t const mask = (1<<nbBits) - 1; 4838 size_t const rNoiseLength = (FUZ_rand(&lseed) & mask) + 1; 4839 size_t const noiseLength = MIN(rNoiseLength, cSize-pos); 4840 size_t const noiseStart = FUZ_rand(&lseed) % (srcBufferSize - noiseLength); 4841 memcpy(cBuffer + pos, srcBuffer + noiseStart, noiseLength); 4842 pos += noiseLength; 4843 } } } 4844 4845 /* decompress noisy source */ 4846 DISPLAYLEVEL(5, "fuzzer t%u: decompress noisy source \n", testNb); 4847 { U32 const endMark = 0xA9B1C3D6; 4848 memcpy(dstBuffer+sampleSize, &endMark, 4); 4849 { size_t const decompressResult = ZSTD_decompress(dstBuffer, sampleSize, cBuffer, cSize); 4850 /* result *may* be an unlikely success, but even then, it must strictly respect dst buffer boundaries */ 4851 CHECK((!ZSTD_isError(decompressResult)) && (decompressResult>sampleSize), 4852 "ZSTD_decompress on noisy src : result is too large : %u > %u (dst buffer)", (unsigned)decompressResult, (unsigned)sampleSize); 4853 } 4854 { U32 endCheck; memcpy(&endCheck, dstBuffer+sampleSize, 4); 4855 CHECK(endMark!=endCheck, "ZSTD_decompress on noisy src : dst buffer overflow"); 4856 } } } /* noisy src decompression test */ 4857 4858 /*===== Bufferless streaming compression test, scattered segments and dictionary =====*/ 4859 DISPLAYLEVEL(5, "fuzzer t%u: Bufferless streaming compression test \n", testNb); 4860 { U32 const testLog = FUZ_rand(&lseed) % maxSrcLog; 4861 U32 const dictLog = FUZ_rand(&lseed) % maxSrcLog; 4862 int const cLevel = (int)(FUZ_rand(&lseed) % 4863 ((U32)ZSTD_maxCLevel() - 4864 (MAX(testLog, dictLog) / (U32)cLevelLimiter))) + 4865 1; 4866 maxTestSize = FUZ_rLogLength(&lseed, testLog); 4867 if (maxTestSize >= dstBufferSize) maxTestSize = dstBufferSize-1; 4868 4869 dictSize = FUZ_rLogLength(&lseed, dictLog); /* needed also for decompression */ 4870 dict = srcBuffer + (FUZ_rand(&lseed) % (srcBufferSize - dictSize)); 4871 4872 DISPLAYLEVEL(6, "fuzzer t%u: Compressing up to <=%u bytes at level %i with dictionary size %u \n", 4873 testNb, (unsigned)maxTestSize, cLevel, (unsigned)dictSize); 4874 4875 if (FUZ_rand(&lseed) & 0xF) { 4876 CHECK_Z ( ZSTD_compressBegin_usingDict(refCtx, dict, dictSize, cLevel) ); 4877 } else { 4878 ZSTD_compressionParameters const cPar = ZSTD_getCParams(cLevel, ZSTD_CONTENTSIZE_UNKNOWN, dictSize); 4879 ZSTD_frameParameters const fPar = { FUZ_rand(&lseed)&1 /* contentSizeFlag */, 4880 !(FUZ_rand(&lseed)&3) /* contentChecksumFlag*/, 4881 0 /*NodictID*/ }; /* note : since dictionary is fake, dictIDflag has no impact */ 4882 ZSTD_parameters const p = FUZ_makeParams(cPar, fPar); 4883 CHECK_Z ( ZSTD_compressBegin_advanced(refCtx, dict, dictSize, p, 0) ); 4884 } 4885 CHECK_Z( ZSTD_copyCCtx(ctx, refCtx, 0) ); 4886 } 4887 4888 { U32 const nbChunks = (FUZ_rand(&lseed) & 127) + 2; 4889 U32 n; 4890 XXH64_state_t xxhState; 4891 XXH64_reset(&xxhState, 0); 4892 for (totalTestSize=0, cSize=0, n=0 ; n<nbChunks ; n++) { 4893 size_t const segmentSize = FUZ_randomLength(&lseed, maxSampleLog); 4894 size_t const segmentStart = FUZ_rand(&lseed) % (srcBufferSize - segmentSize); 4895 4896 if (cBufferSize-cSize < ZSTD_compressBound(segmentSize)) break; /* avoid invalid dstBufferTooSmall */ 4897 if (totalTestSize+segmentSize > maxTestSize) break; 4898 4899 { size_t const compressResult = ZSTD_compressContinue(ctx, cBuffer+cSize, cBufferSize-cSize, srcBuffer+segmentStart, segmentSize); 4900 CHECK (ZSTD_isError(compressResult), "multi-segments compression error : %s", ZSTD_getErrorName(compressResult)); 4901 cSize += compressResult; 4902 } 4903 XXH64_update(&xxhState, srcBuffer+segmentStart, segmentSize); 4904 memcpy(mirrorBuffer + totalTestSize, srcBuffer+segmentStart, segmentSize); 4905 totalTestSize += segmentSize; 4906 } 4907 4908 { size_t const flushResult = ZSTD_compressEnd(ctx, cBuffer+cSize, cBufferSize-cSize, NULL, 0); 4909 CHECK (ZSTD_isError(flushResult), "multi-segments epilogue error : %s", ZSTD_getErrorName(flushResult)); 4910 cSize += flushResult; 4911 } 4912 crcOrig = XXH64_digest(&xxhState); 4913 } 4914 4915 /* streaming decompression test */ 4916 DISPLAYLEVEL(5, "fuzzer t%u: Bufferless streaming decompression test \n", testNb); 4917 /* ensure memory requirement is good enough (should always be true) */ 4918 { ZSTD_FrameHeader zfh; 4919 CHECK( ZSTD_getFrameHeader(&zfh, cBuffer, ZSTD_FRAMEHEADERSIZE_MAX), 4920 "ZSTD_getFrameHeader(): error retrieving frame information"); 4921 { size_t const roundBuffSize = ZSTD_decodingBufferSize_min(zfh.windowSize, zfh.frameContentSize); 4922 CHECK_Z(roundBuffSize); 4923 CHECK((roundBuffSize > totalTestSize) && (zfh.frameContentSize!=ZSTD_CONTENTSIZE_UNKNOWN), 4924 "ZSTD_decodingBufferSize_min() requires more memory (%u) than necessary (%u)", 4925 (unsigned)roundBuffSize, (unsigned)totalTestSize ); 4926 } } 4927 if (dictSize<8) dictSize=0, dict=NULL; /* disable dictionary */ 4928 CHECK_Z( ZSTD_decompressBegin_usingDict(dctx, dict, dictSize) ); 4929 totalCSize = 0; 4930 totalGenSize = 0; 4931 while (totalCSize < cSize) { 4932 size_t const inSize = ZSTD_nextSrcSizeToDecompress(dctx); 4933 size_t const genSize = ZSTD_decompressContinue(dctx, dstBuffer+totalGenSize, dstBufferSize-totalGenSize, cBuffer+totalCSize, inSize); 4934 CHECK (ZSTD_isError(genSize), "ZSTD_decompressContinue error : %s", ZSTD_getErrorName(genSize)); 4935 totalGenSize += genSize; 4936 totalCSize += inSize; 4937 } 4938 CHECK (ZSTD_nextSrcSizeToDecompress(dctx) != 0, "frame not fully decoded"); 4939 CHECK (totalGenSize != totalTestSize, "streaming decompressed data : wrong size") 4940 CHECK (totalCSize != cSize, "compressed data should be fully read") 4941 { U64 const crcDest = XXH64(dstBuffer, totalTestSize, 0); 4942 CHECK(crcOrig != crcDest, "streaming decompressed data corrupted (pos %u / %u)", 4943 (unsigned)findDiff(mirrorBuffer, dstBuffer, totalTestSize), (unsigned)totalTestSize); 4944 } 4945 } /* for ( ; (testNb <= nbTests) */ 4946 DISPLAY("\r%u fuzzer tests completed \n", testNb-1); 4947 4948 _cleanup: 4949 ZSTD_freeCCtx(refCtx); 4950 ZSTD_freeCCtx(ctx); 4951 ZSTD_freeDCtx(dctx); 4952 free(cNoiseBuffer[0]); 4953 free(cNoiseBuffer[1]); 4954 free(cNoiseBuffer[2]); 4955 free(cNoiseBuffer[3]); 4956 free(cNoiseBuffer[4]); 4957 free(cBuffer); 4958 free(dstBuffer); 4959 free(mirrorBuffer); 4960 return (int)result; 4961 4962 _output_error: 4963 result = 1; 4964 goto _cleanup; 4965 } 4966 4967 4968 /*_******************************************************* 4969 * Command line 4970 *********************************************************/ 4971 static int FUZ_usage(const char* programName) 4972 { 4973 DISPLAY( "Usage :\n"); 4974 DISPLAY( " %s [args]\n", programName); 4975 DISPLAY( "\n"); 4976 DISPLAY( "Arguments :\n"); 4977 DISPLAY( " -i# : Number of tests (default:%i)\n", nbTestsDefault); 4978 DISPLAY( " -T# : Max duration to run for. Overrides number of tests. (e.g. -T1m or -T60s for one minute)\n"); 4979 DISPLAY( " -s# : Select seed (default:prompt user)\n"); 4980 DISPLAY( " -t# : Select starting test number (default:0)\n"); 4981 DISPLAY( " -P# : Select compressibility in %% (default:%i%%)\n", FUZ_compressibility_default); 4982 DISPLAY( " -v : verbose\n"); 4983 DISPLAY( " -p : pause at the end\n"); 4984 DISPLAY( " -h : display help and exit\n"); 4985 return 0; 4986 } 4987 4988 /*! readU32FromChar() : 4989 @return : unsigned integer value read from input in `char` format 4990 allows and interprets K, KB, KiB, M, MB and MiB suffix. 4991 Will also modify `*stringPtr`, advancing it to position where it stopped reading. 4992 Note : function result can overflow if digit string > MAX_UINT */ 4993 static unsigned readU32FromChar(const char** stringPtr) 4994 { 4995 unsigned result = 0; 4996 while ((**stringPtr >='0') && (**stringPtr <='9')) 4997 result *= 10, result += (unsigned)(**stringPtr - '0'), (*stringPtr)++ ; 4998 if ((**stringPtr=='K') || (**stringPtr=='M')) { 4999 result <<= 10; 5000 if (**stringPtr=='M') result <<= 10; 5001 (*stringPtr)++ ; 5002 if (**stringPtr=='i') (*stringPtr)++; 5003 if (**stringPtr=='B') (*stringPtr)++; 5004 } 5005 return result; 5006 } 5007 5008 /** longCommandWArg() : 5009 * check if *stringPtr is the same as longCommand. 5010 * If yes, @return 1 and advances *stringPtr to the position which immediately follows longCommand. 5011 * @return 0 and doesn't modify *stringPtr otherwise. 5012 */ 5013 static int longCommandWArg(const char** stringPtr, const char* longCommand) 5014 { 5015 size_t const comSize = strlen(longCommand); 5016 int const result = !strncmp(*stringPtr, longCommand, comSize); 5017 if (result) *stringPtr += comSize; 5018 return result; 5019 } 5020 5021 int main(int argc, const char** argv) 5022 { 5023 U32 seed = 0; 5024 int seedset = 0; 5025 int argNb; 5026 int nbTests = nbTestsDefault; 5027 int testNb = 0; 5028 int proba = FUZ_compressibility_default; 5029 double probfloat; 5030 int result = 0; 5031 U32 mainPause = 0; 5032 U32 maxDuration = 0; 5033 int bigTests = 1; 5034 int longTests = 0; 5035 U32 memTestsOnly = 0; 5036 const char* const programName = argv[0]; 5037 5038 /* Check command line */ 5039 for (argNb=1; argNb<argc; argNb++) { 5040 const char* argument = argv[argNb]; 5041 if(!argument) continue; /* Protection if argument empty */ 5042 5043 /* Handle commands. Aggregated commands are allowed */ 5044 if (argument[0]=='-') { 5045 5046 if (longCommandWArg(&argument, "--memtest=")) { memTestsOnly = readU32FromChar(&argument); continue; } 5047 5048 if (!strcmp(argument, "--memtest")) { memTestsOnly=1; continue; } 5049 if (!strcmp(argument, "--no-big-tests")) { bigTests=0; continue; } 5050 if (!strcmp(argument, "--long-tests")) { longTests=1; continue; } 5051 if (!strcmp(argument, "--no-long-tests")) { longTests=0; continue; } 5052 5053 argument++; 5054 while (*argument!=0) { 5055 switch(*argument) 5056 { 5057 case 'h': 5058 return FUZ_usage(programName); 5059 5060 case 'v': 5061 argument++; 5062 g_displayLevel++; 5063 break; 5064 5065 case 'q': 5066 argument++; 5067 g_displayLevel--; 5068 break; 5069 5070 case 'p': /* pause at the end */ 5071 argument++; 5072 mainPause = 1; 5073 break; 5074 5075 case 'i': 5076 argument++; maxDuration = 0; 5077 nbTests = (int)readU32FromChar(&argument); 5078 break; 5079 5080 case 'T': 5081 argument++; 5082 nbTests = 0; 5083 maxDuration = readU32FromChar(&argument); 5084 if (*argument=='s') argument++; /* seconds */ 5085 if (*argument=='m') maxDuration *= 60, argument++; /* minutes */ 5086 if (*argument=='n') argument++; 5087 break; 5088 5089 case 's': 5090 argument++; 5091 seedset = 1; 5092 seed = readU32FromChar(&argument); 5093 break; 5094 5095 case 't': 5096 argument++; 5097 testNb = (int)readU32FromChar(&argument); 5098 break; 5099 5100 case 'P': /* compressibility % */ 5101 argument++; 5102 proba = (int)readU32FromChar(&argument); 5103 if (proba>100) proba = 100; 5104 break; 5105 5106 default: 5107 return (FUZ_usage(programName), 1); 5108 } } } } /* for (argNb=1; argNb<argc; argNb++) */ 5109 5110 /* Get Seed */ 5111 DISPLAY("Starting zstd tester (%i-bits, %s)\n", (int)(sizeof(size_t)*8), ZSTD_VERSION_STRING); 5112 5113 if (!seedset) { 5114 time_t const t = time(NULL); 5115 U32 const h = XXH32(&t, sizeof(t), 1); 5116 seed = h % 10000; 5117 } 5118 5119 DISPLAY("Seed = %u\n", (unsigned)seed); 5120 if (proba!=FUZ_compressibility_default) DISPLAY("Compressibility : %i%%\n", proba); 5121 5122 probfloat = ((double)proba) / 100; 5123 5124 if (memTestsOnly) { 5125 g_displayLevel = MAX(3, g_displayLevel); 5126 return FUZ_mallocTests(seed, probfloat, memTestsOnly); 5127 } 5128 5129 if (nbTests < testNb) nbTests = testNb; 5130 5131 if (testNb==0) { 5132 result = basicUnitTests(0, probfloat); /* constant seed for predictability */ 5133 5134 if (!result && longTests) { 5135 result = longUnitTests(0, probfloat); 5136 } 5137 } 5138 if (!result) 5139 result = fuzzerTests(seed, (unsigned)nbTests, (unsigned)testNb, maxDuration, ((double)proba) / 100, bigTests); 5140 if (mainPause) { 5141 int unused; 5142 DISPLAY("Press Enter \n"); 5143 unused = getchar(); 5144 (void)unused; 5145 } 5146 return result; 5147 } 5148
Indexes created Wed Jun 17 00:25:26 UTC 2026