1 1.1 christos /* 2 1.1 christos * Copyright (c) Meta Platforms, Inc. and affiliates. 3 1.1 christos * All rights reserved. 4 1.1 christos * 5 1.1 christos * This source code is licensed under both the BSD-style license (found in the 6 1.1 christos * LICENSE file in the root directory of this source tree) and the GPLv2 (found 7 1.1 christos * in the COPYING file in the root directory of this source tree). 8 1.1 christos * You may select, at your option, one of the above-listed licenses. 9 1.1 christos */ 10 1.1 christos 11 1.1 christos 12 1.1 christos /*-************************************ 13 1.1 christos * Compiler specific 14 1.1 christos **************************************/ 15 1.1 christos #ifdef _MSC_VER /* Visual Studio */ 16 1.1 christos # define _CRT_SECURE_NO_WARNINGS /* fgets */ 17 1.1 christos # pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ 18 1.1 christos # pragma warning(disable : 4204) /* disable: C4204: non-constant aggregate initializer */ 19 1.1 christos #endif 20 1.1 christos 21 1.1 christos 22 1.1 christos /*-************************************ 23 1.1 christos * Includes 24 1.1 christos **************************************/ 25 1.1 christos #include <stdlib.h> /* free */ 26 1.1 christos #include <stdio.h> /* fgets, sscanf */ 27 1.1 christos #include <string.h> /* strcmp */ 28 1.1 christos #include <time.h> /* time(), time_t */ 29 1.1 christos #undef NDEBUG /* always enable assert() */ 30 1.1 christos #include <assert.h> 31 1.1 christos #define ZSTD_STATIC_LINKING_ONLY /* ZSTD_compressContinue, ZSTD_compressBlock */ 32 1.1 christos #include "debug.h" /* DEBUG_STATIC_ASSERT */ 33 1.1 christos #include "fse.h" 34 1.1 christos #define ZSTD_DISABLE_DEPRECATE_WARNINGS /* No deprecation warnings, we still test some deprecated functions */ 35 1.1 christos #include "zstd.h" /* ZSTD_VERSION_STRING */ 36 1.1 christos #include "zstd_errors.h" /* ZSTD_getErrorCode */ 37 1.1 christos #define ZDICT_STATIC_LINKING_ONLY 38 1.1 christos #include "zdict.h" /* ZDICT_trainFromBuffer */ 39 1.1 christos #include "mem.h" 40 1.1 christos #include "datagen.h" /* RDG_genBuffer */ 41 1.1 christos #define XXH_STATIC_LINKING_ONLY /* XXH64_state_t */ 42 1.1 christos #include "xxhash.h" /* XXH64 */ 43 1.1 christos #include "util.h" 44 1.1 christos #include "timefn.h" /* SEC_TO_MICRO, UTIL_time_t, UTIL_TIME_INITIALIZER, UTIL_clockSpanMicro, UTIL_getTime */ 45 1.1 christos /* must be included after util.h, due to ERROR macro redefinition issue on Visual Studio */ 46 1.1 christos #include "zstd_internal.h" /* ZSTD_WORKSPACETOOLARGE_MAXDURATION, ZSTD_WORKSPACETOOLARGE_FACTOR, KB, MB */ 47 1.1 christos #include "threading.h" /* ZSTD_pthread_create, ZSTD_pthread_join */ 48 1.1 christos 49 1.1 christos 50 1.1 christos /*-************************************ 51 1.1 christos * Constants 52 1.1 christos **************************************/ 53 1.1 christos #define GB *(1U<<30) 54 1.1 christos 55 1.1 christos static const int FUZ_compressibility_default = 50; 56 1.1 christos static const int nbTestsDefault = 30000; 57 1.1 christos 58 1.1 christos 59 1.1 christos /*-************************************ 60 1.1 christos * Display Macros 61 1.1 christos **************************************/ 62 1.1 christos #define DISPLAY(...) fprintf(stderr, __VA_ARGS__) 63 1.1 christos #define DISPLAYLEVEL(l, ...) if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); } 64 1.1 christos static U32 g_displayLevel = 2; 65 1.1 christos 66 1.1 christos static const U64 g_refreshRate = SEC_TO_MICRO / 6; 67 1.1 christos static UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER; 68 1.1 christos 69 1.1 christos #define DISPLAYUPDATE(l, ...) \ 70 1.1 christos if (g_displayLevel>=l) { \ 71 1.1 christos if ((UTIL_clockSpanMicro(g_displayClock) > g_refreshRate) || (g_displayLevel>=4)) \ 72 1.1 christos { g_displayClock = UTIL_getTime(); DISPLAY(__VA_ARGS__); \ 73 1.1 christos if (g_displayLevel>=4) fflush(stderr); } \ 74 1.1 christos } 75 1.1 christos 76 1.1 christos 77 1.1 christos /*-******************************************************* 78 1.1 christos * Compile time test 79 1.1 christos *********************************************************/ 80 1.1 christos #undef MIN 81 1.1 christos #undef MAX 82 1.1 christos /* Declaring the function, to avoid -Wmissing-prototype */ 83 1.1 christos void FUZ_bug976(void); 84 1.1 christos void FUZ_bug976(void) 85 1.1 christos { /* these constants shall not depend on MIN() macro */ 86 1.1 christos DEBUG_STATIC_ASSERT(ZSTD_HASHLOG_MAX < 31); 87 1.1 christos DEBUG_STATIC_ASSERT(ZSTD_CHAINLOG_MAX < 31); 88 1.1 christos } 89 1.1 christos 90 1.1 christos 91 1.1 christos /*-******************************************************* 92 1.1 christos * Internal functions 93 1.1 christos *********************************************************/ 94 1.1 christos #define MIN(a,b) ((a)<(b)?(a):(b)) 95 1.1 christos #define MAX(a,b) ((a)>(b)?(a):(b)) 96 1.1 christos 97 1.1 christos #define FUZ_rotl32(x,r) ((x << r) | (x >> (32 - r))) 98 1.1 christos static U32 FUZ_rand(U32* src) 99 1.1 christos { 100 1.1 christos static const U32 prime1 = 2654435761U; 101 1.1 christos static const U32 prime2 = 2246822519U; 102 1.1 christos U32 rand32 = *src; 103 1.1 christos rand32 *= prime1; 104 1.1 christos rand32 += prime2; 105 1.1 christos rand32 = FUZ_rotl32(rand32, 13); 106 1.1 christos *src = rand32; 107 1.1 christos return rand32 >> 5; 108 1.1 christos } 109 1.1 christos 110 1.1 christos static U32 FUZ_highbit32(U32 v32) 111 1.1 christos { 112 1.1 christos unsigned nbBits = 0; 113 1.1 christos if (v32==0) return 0; 114 1.1 christos while (v32) v32 >>= 1, nbBits++; 115 1.1 christos return nbBits; 116 1.1 christos } 117 1.1 christos 118 1.1 christos 119 1.1 christos /*============================================= 120 1.1 christos * Test macros 121 1.1 christos =============================================*/ 122 1.1 christos #define CHECK(fn) { if(!(fn)) { DISPLAYLEVEL(1, "Error : test (%s) failed \n", #fn); exit(1); } } 123 1.1 christos 124 1.1 christos #define CHECK_Z(f) { \ 125 1.1 christos size_t const err = f; \ 126 1.1 christos if (ZSTD_isError(err)) { \ 127 1.1 christos DISPLAY("Error => %s : %s ", \ 128 1.1 christos #f, ZSTD_getErrorName(err)); \ 129 1.1 christos exit(1); \ 130 1.1 christos } } 131 1.1 christos 132 1.1 christos #define CHECK_VAR(var, fn) var = fn; if (ZSTD_isError(var)) { DISPLAYLEVEL(1, "%s : fails : %s \n", #fn, ZSTD_getErrorName(var)); exit(1); } 133 1.1 christos #define CHECK_NEWV(var, fn) size_t const CHECK_VAR(var, fn) 134 1.1 christos #define CHECKPLUS(var, fn, more) { CHECK_NEWV(var, fn); more; } 135 1.1 christos 136 1.1 christos #define CHECK_OP(op, lhs, rhs) { \ 137 1.1 christos if (!((lhs) op (rhs))) { \ 138 1.1 christos DISPLAY("Error L%u => FAILED %s %s %s ", __LINE__, #lhs, #op, #rhs); \ 139 1.1 christos exit(1); \ 140 1.1 christos } \ 141 1.1 christos } 142 1.1 christos #define CHECK_EQ(lhs, rhs) CHECK_OP(==, lhs, rhs) 143 1.1 christos #define CHECK_LT(lhs, rhs) CHECK_OP(<, lhs, rhs) 144 1.1 christos 145 1.1 christos 146 1.1 christos /*============================================= 147 1.1 christos * Memory Tests 148 1.1 christos =============================================*/ 149 1.1 christos #if defined(__APPLE__) && defined(__MACH__) 150 1.1 christos 151 1.1 christos #include <malloc/malloc.h> /* malloc_size */ 152 1.1 christos 153 1.1 christos typedef struct { 154 1.1 christos unsigned long long totalMalloc; 155 1.1 christos size_t currentMalloc; 156 1.1 christos size_t peakMalloc; 157 1.1 christos unsigned nbMalloc; 158 1.1 christos unsigned nbFree; 159 1.1 christos } mallocCounter_t; 160 1.1 christos 161 1.1 christos static const mallocCounter_t INIT_MALLOC_COUNTER = { 0, 0, 0, 0, 0 }; 162 1.1 christos 163 1.1 christos static void* FUZ_mallocDebug(void* counter, size_t size) 164 1.1 christos { 165 1.1 christos mallocCounter_t* const mcPtr = (mallocCounter_t*)counter; 166 1.1 christos void* const ptr = malloc(size); 167 1.1 christos if (ptr==NULL) return NULL; 168 1.1 christos DISPLAYLEVEL(4, "allocating %u KB => effectively %u KB \n", 169 1.1 christos (unsigned)(size >> 10), (unsigned)(malloc_size(ptr) >> 10)); /* OS-X specific */ 170 1.1 christos mcPtr->totalMalloc += size; 171 1.1 christos mcPtr->currentMalloc += size; 172 1.1 christos if (mcPtr->currentMalloc > mcPtr->peakMalloc) 173 1.1 christos mcPtr->peakMalloc = mcPtr->currentMalloc; 174 1.1 christos mcPtr->nbMalloc += 1; 175 1.1 christos return ptr; 176 1.1 christos } 177 1.1 christos 178 1.1 christos static void FUZ_freeDebug(void* counter, void* address) 179 1.1 christos { 180 1.1 christos mallocCounter_t* const mcPtr = (mallocCounter_t*)counter; 181 1.1 christos DISPLAYLEVEL(4, "freeing %u KB \n", (unsigned)(malloc_size(address) >> 10)); 182 1.1 christos mcPtr->nbFree += 1; 183 1.1 christos mcPtr->currentMalloc -= malloc_size(address); /* OS-X specific */ 184 1.1 christos free(address); 185 1.1 christos } 186 1.1 christos 187 1.1 christos static void FUZ_displayMallocStats(mallocCounter_t count) 188 1.1 christos { 189 1.1 christos DISPLAYLEVEL(3, "peak:%6u KB, nbMallocs:%2u, total:%6u KB \n", 190 1.1 christos (unsigned)(count.peakMalloc >> 10), 191 1.1 christos count.nbMalloc, 192 1.1 christos (unsigned)(count.totalMalloc >> 10)); 193 1.1 christos } 194 1.1 christos 195 1.1 christos static int FUZ_mallocTests_internal(unsigned seed, double compressibility, unsigned part, 196 1.1 christos void* inBuffer, size_t inSize, void* outBuffer, size_t outSize) 197 1.1 christos { 198 1.1 christos /* test only played in verbose mode, as they are long */ 199 1.1 christos if (g_displayLevel<3) return 0; 200 1.1 christos 201 1.1 christos /* Create compressible noise */ 202 1.1 christos if (!inBuffer || !outBuffer) { 203 1.1 christos DISPLAY("Not enough memory, aborting\n"); 204 1.1 christos exit(1); 205 1.1 christos } 206 1.1 christos RDG_genBuffer(inBuffer, inSize, compressibility, 0. /*auto*/, seed); 207 1.1 christos 208 1.1 christos /* simple compression tests */ 209 1.1 christos if (part <= 1) 210 1.1 christos { int compressionLevel; 211 1.1 christos for (compressionLevel=1; compressionLevel<=6; compressionLevel++) { 212 1.1 christos mallocCounter_t malcount = INIT_MALLOC_COUNTER; 213 1.1 christos ZSTD_customMem const cMem = { FUZ_mallocDebug, FUZ_freeDebug, &malcount }; 214 1.1 christos ZSTD_CCtx* const cctx = ZSTD_createCCtx_advanced(cMem); 215 1.1 christos CHECK_Z( ZSTD_compressCCtx(cctx, outBuffer, outSize, inBuffer, inSize, compressionLevel) ); 216 1.1 christos ZSTD_freeCCtx(cctx); 217 1.1 christos DISPLAYLEVEL(3, "compressCCtx level %i : ", compressionLevel); 218 1.1 christos FUZ_displayMallocStats(malcount); 219 1.1 christos } } 220 1.1 christos 221 1.1 christos /* streaming compression tests */ 222 1.1 christos if (part <= 2) 223 1.1 christos { int compressionLevel; 224 1.1 christos for (compressionLevel=1; compressionLevel<=6; compressionLevel++) { 225 1.1 christos mallocCounter_t malcount = INIT_MALLOC_COUNTER; 226 1.1 christos ZSTD_customMem const cMem = { FUZ_mallocDebug, FUZ_freeDebug, &malcount }; 227 1.1 christos ZSTD_CCtx* const cstream = ZSTD_createCStream_advanced(cMem); 228 1.1 christos ZSTD_outBuffer out = { outBuffer, outSize, 0 }; 229 1.1 christos ZSTD_inBuffer in = { inBuffer, inSize, 0 }; 230 1.1 christos CHECK_Z( ZSTD_initCStream(cstream, compressionLevel) ); 231 1.1 christos CHECK_Z( ZSTD_compressStream(cstream, &out, &in) ); 232 1.1 christos CHECK_Z( ZSTD_endStream(cstream, &out) ); 233 1.1 christos ZSTD_freeCStream(cstream); 234 1.1 christos DISPLAYLEVEL(3, "compressStream level %i : ", compressionLevel); 235 1.1 christos FUZ_displayMallocStats(malcount); 236 1.1 christos } } 237 1.1 christos 238 1.1 christos /* advanced MT API test */ 239 1.1 christos if (part <= 3) 240 1.1 christos { int nbThreads; 241 1.1 christos for (nbThreads=1; nbThreads<=4; nbThreads++) { 242 1.1 christos int compressionLevel; 243 1.1 christos for (compressionLevel=1; compressionLevel<=6; compressionLevel++) { 244 1.1 christos mallocCounter_t malcount = INIT_MALLOC_COUNTER; 245 1.1 christos ZSTD_customMem const cMem = { FUZ_mallocDebug, FUZ_freeDebug, &malcount }; 246 1.1 christos ZSTD_CCtx* const cctx = ZSTD_createCCtx_advanced(cMem); 247 1.1 christos CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, compressionLevel) ); 248 1.1 christos CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, nbThreads) ); 249 1.1 christos CHECK_Z( ZSTD_compress2(cctx, outBuffer, outSize, inBuffer, inSize) ); 250 1.1 christos ZSTD_freeCCtx(cctx); 251 1.1 christos DISPLAYLEVEL(3, "compress_generic,-T%i,end level %i : ", 252 1.1 christos nbThreads, compressionLevel); 253 1.1 christos FUZ_displayMallocStats(malcount); 254 1.1 christos } } } 255 1.1 christos 256 1.1 christos /* advanced MT streaming API test */ 257 1.1 christos if (part <= 4) 258 1.1 christos { int nbThreads; 259 1.1 christos for (nbThreads=1; nbThreads<=4; nbThreads++) { 260 1.1 christos int compressionLevel; 261 1.1 christos for (compressionLevel=1; compressionLevel<=6; compressionLevel++) { 262 1.1 christos mallocCounter_t malcount = INIT_MALLOC_COUNTER; 263 1.1 christos ZSTD_customMem const cMem = { FUZ_mallocDebug, FUZ_freeDebug, &malcount }; 264 1.1 christos ZSTD_CCtx* const cctx = ZSTD_createCCtx_advanced(cMem); 265 1.1 christos ZSTD_outBuffer out = { outBuffer, outSize, 0 }; 266 1.1 christos ZSTD_inBuffer in = { inBuffer, inSize, 0 }; 267 1.1 christos CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, compressionLevel) ); 268 1.1 christos CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, nbThreads) ); 269 1.1 christos CHECK_Z( ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_continue) ); 270 1.1 christos while ( ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end) ) {} 271 1.1 christos ZSTD_freeCCtx(cctx); 272 1.1 christos DISPLAYLEVEL(3, "compress_generic,-T%i,continue level %i : ", 273 1.1 christos nbThreads, compressionLevel); 274 1.1 christos FUZ_displayMallocStats(malcount); 275 1.1 christos } } } 276 1.1 christos 277 1.1 christos return 0; 278 1.1 christos } 279 1.1 christos 280 1.1 christos static int FUZ_mallocTests(unsigned seed, double compressibility, unsigned part) 281 1.1 christos { 282 1.1 christos size_t const inSize = 64 MB + 16 MB + 4 MB + 1 MB + 256 KB + 64 KB; /* 85.3 MB */ 283 1.1 christos size_t const outSize = ZSTD_compressBound(inSize); 284 1.1 christos void* const inBuffer = malloc(inSize); 285 1.1 christos void* const outBuffer = malloc(outSize); 286 1.1 christos int result; 287 1.1 christos 288 1.1 christos /* Create compressible noise */ 289 1.1 christos if (!inBuffer || !outBuffer) { 290 1.1 christos DISPLAY("Not enough memory, aborting \n"); 291 1.1 christos exit(1); 292 1.1 christos } 293 1.1 christos 294 1.1 christos result = FUZ_mallocTests_internal(seed, compressibility, part, 295 1.1 christos inBuffer, inSize, outBuffer, outSize); 296 1.1 christos 297 1.1 christos free(inBuffer); 298 1.1 christos free(outBuffer); 299 1.1 christos return result; 300 1.1 christos } 301 1.1 christos 302 1.1 christos #else 303 1.1 christos 304 1.1 christos static int FUZ_mallocTests(unsigned seed, double compressibility, unsigned part) 305 1.1 christos { 306 1.1 christos (void)seed; (void)compressibility; (void)part; 307 1.1 christos return 0; 308 1.1 christos } 309 1.1 christos 310 1.1 christos #endif 311 1.1 christos 312 1.1 christos static void FUZ_decodeSequences(BYTE* dst, ZSTD_Sequence* seqs, size_t seqsSize, 313 1.1 christos BYTE* src, size_t size, ZSTD_sequenceFormat_e format) 314 1.1 christos { 315 1.1 christos size_t i; 316 1.1 christos size_t j; 317 1.1 christos for(i = 0; i < seqsSize; ++i) { 318 1.1 christos assert(dst + seqs[i].litLength + seqs[i].matchLength <= dst + size); 319 1.1 christos assert(src + seqs[i].litLength + seqs[i].matchLength <= src + size); 320 1.1 christos if (format == ZSTD_sf_noBlockDelimiters) { 321 1.1 christos assert(seqs[i].matchLength != 0 || seqs[i].offset != 0); 322 1.1 christos } 323 1.1 christos 324 1.1 christos memcpy(dst, src, seqs[i].litLength); 325 1.1 christos dst += seqs[i].litLength; 326 1.1 christos src += seqs[i].litLength; 327 1.1 christos size -= seqs[i].litLength; 328 1.1 christos 329 1.1 christos if (seqs[i].offset != 0) { 330 1.1 christos for (j = 0; j < seqs[i].matchLength; ++j) 331 1.1 christos dst[j] = dst[(ptrdiff_t)(j - seqs[i].offset)]; 332 1.1 christos dst += seqs[i].matchLength; 333 1.1 christos src += seqs[i].matchLength; 334 1.1 christos size -= seqs[i].matchLength; 335 1.1 christos } 336 1.1 christos } 337 1.1 christos if (format == ZSTD_sf_noBlockDelimiters) { 338 1.1 christos memcpy(dst, src, size); 339 1.1 christos } 340 1.1 christos } 341 1.1 christos 342 1.1 christos #ifdef ZSTD_MULTITHREAD 343 1.1 christos 344 1.1 christos typedef struct { 345 1.1 christos ZSTD_CCtx* cctx; 346 1.1 christos ZSTD_threadPool* pool; 347 1.1 christos void* CNBuffer; 348 1.1 christos size_t CNBuffSize; 349 1.1 christos void* compressedBuffer; 350 1.1 christos size_t compressedBufferSize; 351 1.1 christos void* decodedBuffer; 352 1.1 christos int err; 353 1.1 christos } threadPoolTests_compressionJob_payload; 354 1.1 christos 355 1.1 christos static void* threadPoolTests_compressionJob(void* payload) { 356 1.1 christos threadPoolTests_compressionJob_payload* args = (threadPoolTests_compressionJob_payload*)payload; 357 1.1 christos size_t cSize; 358 1.1 christos if (ZSTD_isError(ZSTD_CCtx_refThreadPool(args->cctx, args->pool))) args->err = 1; 359 1.1 christos cSize = ZSTD_compress2(args->cctx, args->compressedBuffer, args->compressedBufferSize, args->CNBuffer, args->CNBuffSize); 360 1.1 christos if (ZSTD_isError(cSize)) args->err = 1; 361 1.1 christos if (ZSTD_isError(ZSTD_decompress(args->decodedBuffer, args->CNBuffSize, args->compressedBuffer, cSize))) args->err = 1; 362 1.1 christos return payload; 363 1.1 christos } 364 1.1 christos 365 1.1 christos static int threadPoolTests(void) { 366 1.1 christos int testResult = 0; 367 1.1 christos size_t err; 368 1.1 christos 369 1.1 christos size_t const CNBuffSize = 5 MB; 370 1.1 christos void* const CNBuffer = malloc(CNBuffSize); 371 1.1 christos size_t const compressedBufferSize = ZSTD_compressBound(CNBuffSize); 372 1.1 christos void* const compressedBuffer = malloc(compressedBufferSize); 373 1.1 christos void* const decodedBuffer = malloc(CNBuffSize); 374 1.1 christos 375 1.1 christos size_t const kPoolNumThreads = 8; 376 1.1 christos 377 1.1 christos RDG_genBuffer(CNBuffer, CNBuffSize, 0.5, 0.5, 0); 378 1.1 christos 379 1.1 christos DISPLAYLEVEL(3, "thread pool test : threadPool reuse roundtrips: "); 380 1.1 christos { 381 1.1 christos ZSTD_CCtx* cctx = ZSTD_createCCtx(); 382 1.1 christos ZSTD_threadPool* pool = ZSTD_createThreadPool(kPoolNumThreads); 383 1.1 christos 384 1.1 christos size_t nbThreads = 1; 385 1.1 christos for (; nbThreads <= kPoolNumThreads; ++nbThreads) { 386 1.1 christos ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters); 387 1.1 christos ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, (int)nbThreads); 388 1.1 christos err = ZSTD_CCtx_refThreadPool(cctx, pool); 389 1.1 christos if (ZSTD_isError(err)) { 390 1.1 christos DISPLAYLEVEL(3, "refThreadPool error!\n"); 391 1.1 christos ZSTD_freeCCtx(cctx); 392 1.1 christos goto _output_error; 393 1.1 christos } 394 1.1 christos err = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize); 395 1.1 christos if (ZSTD_isError(err)) { 396 1.1 christos DISPLAYLEVEL(3, "Compression error!\n"); 397 1.1 christos ZSTD_freeCCtx(cctx); 398 1.1 christos goto _output_error; 399 1.1 christos } 400 1.1 christos err = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, err); 401 1.1 christos if (ZSTD_isError(err)) { 402 1.1 christos DISPLAYLEVEL(3, "Decompression error!\n"); 403 1.1 christos ZSTD_freeCCtx(cctx); 404 1.1 christos goto _output_error; 405 1.1 christos } 406 1.1 christos } 407 1.1 christos 408 1.1 christos ZSTD_freeCCtx(cctx); 409 1.1 christos ZSTD_freeThreadPool(pool); 410 1.1 christos } 411 1.1 christos DISPLAYLEVEL(3, "OK \n"); 412 1.1 christos 413 1.1 christos DISPLAYLEVEL(3, "thread pool test : threadPool simultaneous usage: "); 414 1.1 christos { 415 1.1 christos void* const decodedBuffer2 = malloc(CNBuffSize); 416 1.1 christos void* const compressedBuffer2 = malloc(compressedBufferSize); 417 1.1 christos ZSTD_threadPool* pool = ZSTD_createThreadPool(kPoolNumThreads); 418 1.1 christos ZSTD_CCtx* cctx1 = ZSTD_createCCtx(); 419 1.1 christos ZSTD_CCtx* cctx2 = ZSTD_createCCtx(); 420 1.1 christos 421 1.1 christos ZSTD_pthread_t t1; 422 1.1 christos ZSTD_pthread_t t2; 423 1.1 christos threadPoolTests_compressionJob_payload p1 = {cctx1, pool, CNBuffer, CNBuffSize, 424 1.1 christos compressedBuffer, compressedBufferSize, decodedBuffer, 0 /* err */}; 425 1.1 christos threadPoolTests_compressionJob_payload p2 = {cctx2, pool, CNBuffer, CNBuffSize, 426 1.1 christos compressedBuffer2, compressedBufferSize, decodedBuffer2, 0 /* err */}; 427 1.1 christos 428 1.1 christos ZSTD_CCtx_setParameter(cctx1, ZSTD_c_nbWorkers, 2); 429 1.1 christos ZSTD_CCtx_setParameter(cctx2, ZSTD_c_nbWorkers, 2); 430 1.1 christos ZSTD_CCtx_refThreadPool(cctx1, pool); 431 1.1 christos ZSTD_CCtx_refThreadPool(cctx2, pool); 432 1.1 christos 433 1.1 christos ZSTD_pthread_create(&t1, NULL, threadPoolTests_compressionJob, &p1); 434 1.1 christos ZSTD_pthread_create(&t2, NULL, threadPoolTests_compressionJob, &p2); 435 1.1 christos ZSTD_pthread_join(t1); 436 1.1 christos ZSTD_pthread_join(t2); 437 1.1 christos 438 1.1 christos assert(!memcmp(decodedBuffer, decodedBuffer2, CNBuffSize)); 439 1.1 christos free(decodedBuffer2); 440 1.1 christos free(compressedBuffer2); 441 1.1 christos 442 1.1 christos ZSTD_freeThreadPool(pool); 443 1.1 christos ZSTD_freeCCtx(cctx1); 444 1.1 christos ZSTD_freeCCtx(cctx2); 445 1.1 christos 446 1.1 christos if (p1.err || p2.err) goto _output_error; 447 1.1 christos } 448 1.1 christos DISPLAYLEVEL(3, "OK \n"); 449 1.1 christos 450 1.1 christos _end: 451 1.1 christos free(CNBuffer); 452 1.1 christos free(compressedBuffer); 453 1.1 christos free(decodedBuffer); 454 1.1 christos return testResult; 455 1.1 christos 456 1.1 christos _output_error: 457 1.1 christos testResult = 1; 458 1.1 christos DISPLAY("Error detected in Unit tests ! \n"); 459 1.1 christos goto _end; 460 1.1 christos } 461 1.1 christos #endif /* ZSTD_MULTITHREAD */ 462 1.1 christos 463 1.1 christos /*============================================= 464 1.1 christos * Unit tests 465 1.1 christos =============================================*/ 466 1.1 christos 467 1.1 christos static void test_compressBound(unsigned tnb) 468 1.1 christos { 469 1.1 christos DISPLAYLEVEL(3, "test%3u : compressBound : ", tnb); 470 1.1 christos 471 1.1 christos /* check ZSTD_compressBound == ZSTD_COMPRESSBOUND 472 1.1 christos * for a large range of known valid values */ 473 1.1 christos DEBUG_STATIC_ASSERT(sizeof(size_t) >= 4); 474 1.1 christos { int s; 475 1.1 christos for (s=0; s<30; s++) { 476 1.1 christos size_t const w = (size_t)1 << s; 477 1.1 christos CHECK_EQ(ZSTD_compressBound(w), ZSTD_COMPRESSBOUND(w)); 478 1.1 christos } } 479 1.1 christos 480 1.1 christos /* Ensure error if srcSize too big */ 481 1.1 christos { size_t const w = ZSTD_MAX_INPUT_SIZE + 1; 482 1.1 christos CHECK(ZSTD_isError(ZSTD_compressBound(w))); /* must fail */ 483 1.1 christos CHECK_EQ(ZSTD_COMPRESSBOUND(w), 0); 484 1.1 christos } 485 1.1 christos 486 1.1 christos DISPLAYLEVEL(3, "OK \n"); 487 1.1 christos } 488 1.1 christos 489 1.1 christos static void test_decompressBound(unsigned tnb) 490 1.1 christos { 491 1.1 christos DISPLAYLEVEL(3, "test%3u : decompressBound : ", tnb); 492 1.1 christos 493 1.1 christos /* Simple compression, with size : should provide size; */ 494 1.1 christos { const char example[] = "abcd"; 495 1.1 christos char cBuffer[ZSTD_COMPRESSBOUND(sizeof(example))]; 496 1.1 christos size_t const cSize = ZSTD_compress(cBuffer, sizeof(cBuffer), example, sizeof(example), 0); 497 1.1 christos CHECK_Z(cSize); 498 1.1 christos CHECK_EQ(ZSTD_decompressBound(cBuffer, cSize), (unsigned long long)sizeof(example)); 499 1.1 christos } 500 1.1 christos 501 1.1 christos /* Simple small compression without size : should provide 1 block size */ 502 1.1 christos { char cBuffer[ZSTD_COMPRESSBOUND(0)]; 503 1.1 christos ZSTD_outBuffer out = { cBuffer, sizeof(cBuffer), 0 }; 504 1.1 christos ZSTD_inBuffer in = { NULL, 0, 0 }; 505 1.1 christos ZSTD_CCtx* const cctx = ZSTD_createCCtx(); 506 1.1 christos assert(cctx); 507 1.1 christos CHECK_Z( ZSTD_initCStream(cctx, 0) ); 508 1.1 christos CHECK_Z( ZSTD_compressStream(cctx, &out, &in) ); 509 1.1 christos CHECK_EQ( ZSTD_endStream(cctx, &out), 0 ); 510 1.1 christos CHECK_EQ( ZSTD_decompressBound(cBuffer, out.pos), ZSTD_BLOCKSIZE_MAX ); 511 1.1 christos ZSTD_freeCCtx(cctx); 512 1.1 christos } 513 1.1 christos 514 1.1 christos /* Attempt to overflow 32-bit intermediate multiplication result 515 1.1 christos * This requires dBound >= 4 GB, aka 2^32. 516 1.1 christos * This requires 2^32 / 2^17 = 2^15 blocks 517 1.1 christos * => create 2^15 blocks (can be empty, or just 1 byte). */ 518 1.1 christos { const char input[] = "a"; 519 1.1 christos size_t const nbBlocks = (1 << 15) + 1; 520 1.1 christos size_t blockNb; 521 1.1 christos size_t const outCapacity = 1 << 18; /* large margin */ 522 1.1 christos char* const outBuffer = malloc (outCapacity); 523 1.1 christos ZSTD_outBuffer out = { outBuffer, outCapacity, 0 }; 524 1.1 christos ZSTD_CCtx* const cctx = ZSTD_createCCtx(); 525 1.1 christos assert(cctx); 526 1.1 christos assert(outBuffer); 527 1.1 christos CHECK_Z( ZSTD_initCStream(cctx, 0) ); 528 1.1 christos for (blockNb=0; blockNb<nbBlocks; blockNb++) { 529 1.1 christos ZSTD_inBuffer in = { input, sizeof(input), 0 }; 530 1.1 christos CHECK_Z( ZSTD_compressStream(cctx, &out, &in) ); 531 1.1 christos CHECK_EQ( ZSTD_flushStream(cctx, &out), 0 ); 532 1.1 christos } 533 1.1 christos CHECK_EQ( ZSTD_endStream(cctx, &out), 0 ); 534 1.1 christos CHECK( ZSTD_decompressBound(outBuffer, out.pos) > 0x100000000ULL /* 4 GB */ ); 535 1.1 christos ZSTD_freeCCtx(cctx); 536 1.1 christos free(outBuffer); 537 1.1 christos } 538 1.1 christos 539 1.1 christos DISPLAYLEVEL(3, "OK \n"); 540 1.1 christos } 541 1.1 christos 542 1.1 christos static void test_setCParams(unsigned tnb) 543 1.1 christos { 544 1.1 christos ZSTD_CCtx* const cctx = ZSTD_createCCtx(); 545 1.1 christos ZSTD_compressionParameters cparams; 546 1.1 christos assert(cctx); 547 1.1 christos 548 1.1 christos DISPLAYLEVEL(3, "test%3u : ZSTD_CCtx_setCParams : ", tnb); 549 1.1 christos 550 1.1 christos /* valid cparams */ 551 1.1 christos cparams = ZSTD_getCParams(1, 0, 0); 552 1.1 christos CHECK_Z(ZSTD_CCtx_setCParams(cctx, cparams)); 553 1.1 christos 554 1.1 christos /* invalid cparams (must fail) */ 555 1.1 christos cparams.windowLog = 99; 556 1.1 christos CHECK(ZSTD_isError(ZSTD_CCtx_setCParams(cctx, cparams))); 557 1.1 christos 558 1.1 christos free(cctx); 559 1.1 christos DISPLAYLEVEL(3, "OK \n"); 560 1.1 christos } 561 1.1 christos 562 1.1 christos static int basicUnitTests(U32 const seed, double compressibility) 563 1.1 christos { 564 1.1 christos size_t const CNBuffSize = 5 MB; 565 1.1 christos void* const CNBuffer = malloc(CNBuffSize); 566 1.1 christos size_t const compressedBufferSize = ZSTD_compressBound(CNBuffSize); 567 1.1 christos void* const compressedBuffer = malloc(compressedBufferSize); 568 1.1 christos void* const decodedBuffer = malloc(CNBuffSize); 569 1.1 christos int testResult = 0; 570 1.1 christos unsigned testNb=0; 571 1.1 christos size_t cSize; 572 1.1 christos 573 1.1 christos /* Create compressible noise */ 574 1.1 christos if (!CNBuffer || !compressedBuffer || !decodedBuffer) { 575 1.1 christos DISPLAY("Not enough memory, aborting\n"); 576 1.1 christos testResult = 1; 577 1.1 christos goto _end; 578 1.1 christos } 579 1.1 christos RDG_genBuffer(CNBuffer, CNBuffSize, compressibility, 0., seed); 580 1.1 christos 581 1.1 christos /* Basic tests */ 582 1.1 christos DISPLAYLEVEL(3, "test%3u : ZSTD_getErrorName : ", testNb++); 583 1.1 christos { const char* errorString = ZSTD_getErrorName(0); 584 1.1 christos DISPLAYLEVEL(3, "OK : %s \n", errorString); 585 1.1 christos } 586 1.1 christos 587 1.1 christos DISPLAYLEVEL(3, "test%3u : ZSTD_getErrorName with wrong value : ", testNb++); 588 1.1 christos { const char* errorString = ZSTD_getErrorName(499); 589 1.1 christos DISPLAYLEVEL(3, "OK : %s \n", errorString); 590 1.1 christos } 591 1.1 christos 592 1.1 christos DISPLAYLEVEL(3, "test%3u : min compression level : ", testNb++); 593 1.1 christos { int const mcl = ZSTD_minCLevel(); 594 1.1 christos DISPLAYLEVEL(3, "%i (OK) \n", mcl); 595 1.1 christos } 596 1.1 christos 597 1.1 christos DISPLAYLEVEL(3, "test%3u : default compression level : ", testNb++); 598 1.1 christos { int const defaultCLevel = ZSTD_defaultCLevel(); 599 1.1 christos if (defaultCLevel != ZSTD_CLEVEL_DEFAULT) goto _output_error; 600 1.1 christos DISPLAYLEVEL(3, "%i (OK) \n", defaultCLevel); 601 1.1 christos } 602 1.1 christos 603 1.1 christos DISPLAYLEVEL(3, "test%3u : ZSTD_versionNumber : ", testNb++); 604 1.1 christos { unsigned const vn = ZSTD_versionNumber(); 605 1.1 christos DISPLAYLEVEL(3, "%u (OK) \n", vn); 606 1.1 christos } 607 1.1 christos 608 1.1 christos test_compressBound(testNb++); 609 1.1 christos 610 1.1 christos test_decompressBound(testNb++); 611 1.1 christos 612 1.1 christos test_setCParams(testNb++); 613 1.1 christos 614 1.1 christos DISPLAYLEVEL(3, "test%3u : ZSTD_adjustCParams : ", testNb++); 615 1.1 christos { 616 1.1 christos ZSTD_compressionParameters params; 617 1.1 christos memset(¶ms, 0, sizeof(params)); 618 1.1 christos params.windowLog = 10; 619 1.1 christos params.hashLog = 19; 620 1.1 christos params.chainLog = 19; 621 1.1 christos params = ZSTD_adjustCParams(params, 1000, 100000); 622 1.1 christos if (params.hashLog != 18) goto _output_error; 623 1.1 christos if (params.chainLog != 17) goto _output_error; 624 1.1 christos } 625 1.1 christos DISPLAYLEVEL(3, "OK \n"); 626 1.1 christos 627 1.1 christos DISPLAYLEVEL(3, "test%3u : compress %u bytes : ", testNb++, (unsigned)CNBuffSize); 628 1.1 christos { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); 629 1.1 christos if (cctx==NULL) goto _output_error; 630 1.1 christos CHECK_VAR(cSize, ZSTD_compressCCtx(cctx, 631 1.1 christos compressedBuffer, compressedBufferSize, 632 1.1 christos CNBuffer, CNBuffSize, 1) ); 633 1.1 christos DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBuffSize*100); 634 1.1 christos 635 1.1 christos DISPLAYLEVEL(3, "test%3i : size of cctx for level 1 : ", testNb++); 636 1.1 christos { size_t const cctxSize = ZSTD_sizeof_CCtx(cctx); 637 1.1 christos DISPLAYLEVEL(3, "%u bytes \n", (unsigned)cctxSize); 638 1.1 christos } 639 1.1 christos ZSTD_freeCCtx(cctx); 640 1.1 christos } 641 1.1 christos 642 1.1 christos DISPLAYLEVEL(3, "test%3i : decompress skippable frame -8 size : ", testNb++); 643 1.1 christos { 644 1.1 christos char const skippable8[] = "\x50\x2a\x4d\x18\xf8\xff\xff\xff"; 645 1.1 christos size_t const size = ZSTD_decompress(NULL, 0, skippable8, 8); 646 1.1 christos if (!ZSTD_isError(size)) goto _output_error; 647 1.1 christos } 648 1.1 christos DISPLAYLEVEL(3, "OK \n"); 649 1.1 christos 650 1.1 christos DISPLAYLEVEL(3, "test%3i : ZSTD_getFrameContentSize test : ", testNb++); 651 1.1 christos { unsigned long long const rSize = ZSTD_getFrameContentSize(compressedBuffer, cSize); 652 1.1 christos if (rSize != CNBuffSize) goto _output_error; 653 1.1 christos } 654 1.1 christos DISPLAYLEVEL(3, "OK \n"); 655 1.1 christos 656 1.1 christos DISPLAYLEVEL(3, "test%3i : ZSTD_getDecompressedSize test : ", testNb++); 657 1.1 christos { unsigned long long const rSize = ZSTD_getDecompressedSize(compressedBuffer, cSize); 658 1.1 christos if (rSize != CNBuffSize) goto _output_error; 659 1.1 christos } 660 1.1 christos DISPLAYLEVEL(3, "OK \n"); 661 1.1 christos 662 1.1 christos DISPLAYLEVEL(3, "test%3i : ZSTD_findDecompressedSize test : ", testNb++); 663 1.1 christos { unsigned long long const rSize = ZSTD_findDecompressedSize(compressedBuffer, cSize); 664 1.1 christos if (rSize != CNBuffSize) goto _output_error; 665 1.1 christos } 666 1.1 christos DISPLAYLEVEL(3, "OK \n"); 667 1.1 christos 668 1.1 christos DISPLAYLEVEL(3, "test%3i : tight ZSTD_decompressBound test : ", testNb++); 669 1.1 christos { 670 1.1 christos unsigned long long bound = ZSTD_decompressBound(compressedBuffer, cSize); 671 1.1 christos if (bound != CNBuffSize) goto _output_error; 672 1.1 christos } 673 1.1 christos DISPLAYLEVEL(3, "OK \n"); 674 1.1 christos 675 1.1 christos DISPLAYLEVEL(3, "test%3i : ZSTD_decompressBound test with invalid srcSize : ", testNb++); 676 1.1 christos { 677 1.1 christos unsigned long long bound = ZSTD_decompressBound(compressedBuffer, cSize - 1); 678 1.1 christos if (bound != ZSTD_CONTENTSIZE_ERROR) goto _output_error; 679 1.1 christos } 680 1.1 christos DISPLAYLEVEL(3, "OK \n"); 681 1.1 christos 682 1.1 christos DISPLAYLEVEL(3, "test%3i : decompress %u bytes : ", testNb++, (unsigned)CNBuffSize); 683 1.1 christos { size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize); 684 1.1 christos if (r != CNBuffSize) goto _output_error; } 685 1.1 christos DISPLAYLEVEL(3, "OK \n"); 686 1.1 christos 687 1.1 christos DISPLAYLEVEL(3, "test%3i : decompress %u bytes with Huffman assembly disabled : ", testNb++, (unsigned)CNBuffSize); 688 1.1 christos { 689 1.1 christos ZSTD_DCtx* dctx = ZSTD_createDCtx(); 690 1.1 christos size_t r; 691 1.1 christos CHECK_Z(ZSTD_DCtx_setParameter(dctx, ZSTD_d_disableHuffmanAssembly, 1)); 692 1.1 christos r = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize); 693 1.1 christos if (r != CNBuffSize || memcmp(decodedBuffer, CNBuffer, CNBuffSize)) goto _output_error; 694 1.1 christos ZSTD_freeDCtx(dctx); 695 1.1 christos } 696 1.1 christos DISPLAYLEVEL(3, "OK \n"); 697 1.1 christos 698 1.1 christos DISPLAYLEVEL(3, "test%3i : check decompressed result : ", testNb++); 699 1.1 christos { size_t u; 700 1.1 christos for (u=0; u<CNBuffSize; u++) { 701 1.1 christos if (((BYTE*)decodedBuffer)[u] != ((BYTE*)CNBuffer)[u]) goto _output_error; 702 1.1 christos } } 703 1.1 christos DISPLAYLEVEL(3, "OK \n"); 704 1.1 christos 705 1.1 christos DISPLAYLEVEL(3, "test%3u : invalid endDirective : ", testNb++); 706 1.1 christos { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); 707 1.1 christos ZSTD_inBuffer inb = { CNBuffer, CNBuffSize, 0 }; 708 1.1 christos ZSTD_outBuffer outb = { compressedBuffer, compressedBufferSize, 0 }; 709 1.1 christos if (cctx==NULL) goto _output_error; 710 1.1 christos CHECK( ZSTD_isError( ZSTD_compressStream2(cctx, &outb, &inb, (ZSTD_EndDirective) 3) ) ); /* must fail */ 711 1.1 christos CHECK( ZSTD_isError( ZSTD_compressStream2(cctx, &outb, &inb, (ZSTD_EndDirective)-1) ) ); /* must fail */ 712 1.1 christos ZSTD_freeCCtx(cctx); 713 1.1 christos } 714 1.1 christos DISPLAYLEVEL(3, "OK \n"); 715 1.1 christos 716 1.1 christos DISPLAYLEVEL(3, "test%3i : ZSTD_checkCParams : ", testNb++); 717 1.1 christos { 718 1.1 christos ZSTD_parameters params = ZSTD_getParams(3, 0, 0); 719 1.1 christos assert(!ZSTD_checkCParams(params.cParams)); 720 1.1 christos } 721 1.1 christos DISPLAYLEVEL(3, "OK \n"); 722 1.1 christos 723 1.1 christos DISPLAYLEVEL(3, "test%3i : ZSTD_createDCtx_advanced and ZSTD_sizeof_DCtx: ", testNb++); 724 1.1 christos { 725 1.1 christos ZSTD_DCtx* const dctx = ZSTD_createDCtx_advanced(ZSTD_defaultCMem); 726 1.1 christos assert(dctx != NULL); 727 1.1 christos assert(ZSTD_sizeof_DCtx(dctx) != 0); 728 1.1 christos ZSTD_freeDCtx(dctx); 729 1.1 christos } 730 1.1 christos DISPLAYLEVEL(3, "OK \n"); 731 1.1 christos 732 1.1 christos DISPLAYLEVEL(3, "test%3i : misc unaccounted for zstd symbols : ", testNb++); 733 1.1 christos { 734 1.1 christos /* %p takes a void*. In ISO C, it's illegal to cast a function pointer 735 1.1 christos * to a data pointer. (Although in POSIX you're required to be allowed 736 1.1 christos * to do it...) So we have to fall back to our trusty friend memcpy. */ 737 1.1 christos unsigned (* const funcptr_getDictID)(const ZSTD_DDict* ddict) = 738 1.1 christos ZSTD_getDictID_fromDDict; 739 1.1 christos ZSTD_DStream* (* const funcptr_createDStream)( 740 1.1 christos ZSTD_customMem customMem) = ZSTD_createDStream_advanced; 741 1.1 christos void (* const funcptr_copyDCtx)( 742 1.1 christos ZSTD_DCtx* dctx, const ZSTD_DCtx* preparedDCtx) = ZSTD_copyDCtx; 743 1.1 christos ZSTD_nextInputType_e (* const funcptr_nextInputType)(ZSTD_DCtx* dctx) = 744 1.1 christos ZSTD_nextInputType; 745 1.1 christos const void *voidptr_getDictID; 746 1.1 christos const void *voidptr_createDStream; 747 1.1 christos const void *voidptr_copyDCtx; 748 1.1 christos const void *voidptr_nextInputType; 749 1.1 christos DEBUG_STATIC_ASSERT(sizeof(funcptr_getDictID) == sizeof(voidptr_getDictID)); 750 1.1 christos memcpy( 751 1.1 christos (void*)&voidptr_getDictID, 752 1.1 christos (const void*)&funcptr_getDictID, 753 1.1 christos sizeof(void*)); 754 1.1 christos memcpy( 755 1.1 christos (void*)&voidptr_createDStream, 756 1.1 christos (const void*)&funcptr_createDStream, 757 1.1 christos sizeof(void*)); 758 1.1 christos memcpy( 759 1.1 christos (void*)&voidptr_copyDCtx, 760 1.1 christos (const void*)&funcptr_copyDCtx, 761 1.1 christos sizeof(void*)); 762 1.1 christos memcpy( 763 1.1 christos (void*)&voidptr_nextInputType, 764 1.1 christos (const void*)&funcptr_nextInputType, 765 1.1 christos sizeof(void*)); 766 1.1 christos DISPLAYLEVEL(3, "%p ", voidptr_getDictID); 767 1.1 christos DISPLAYLEVEL(3, "%p ", voidptr_createDStream); 768 1.1 christos DISPLAYLEVEL(3, "%p ", voidptr_copyDCtx); 769 1.1 christos DISPLAYLEVEL(3, "%p ", voidptr_nextInputType); 770 1.1 christos } 771 1.1 christos DISPLAYLEVEL(3, ": OK \n"); 772 1.1 christos 773 1.1 christos DISPLAYLEVEL(3, "test%3i : decompress with null dict : ", testNb++); 774 1.1 christos { ZSTD_DCtx* const dctx = ZSTD_createDCtx(); assert(dctx != NULL); 775 1.1 christos { size_t const r = ZSTD_decompress_usingDict(dctx, 776 1.1 christos decodedBuffer, CNBuffSize, 777 1.1 christos compressedBuffer, cSize, 778 1.1 christos NULL, 0); 779 1.1 christos if (r != CNBuffSize) goto _output_error; 780 1.1 christos } 781 1.1 christos ZSTD_freeDCtx(dctx); 782 1.1 christos } 783 1.1 christos DISPLAYLEVEL(3, "OK \n"); 784 1.1 christos 785 1.1 christos DISPLAYLEVEL(3, "test%3i : decompress with null DDict : ", testNb++); 786 1.1 christos { ZSTD_DCtx* const dctx = ZSTD_createDCtx(); assert(dctx != NULL); 787 1.1 christos { size_t const r = ZSTD_decompress_usingDDict(dctx, 788 1.1 christos decodedBuffer, CNBuffSize, 789 1.1 christos compressedBuffer, cSize, 790 1.1 christos NULL); 791 1.1 christos if (r != CNBuffSize) goto _output_error; 792 1.1 christos } 793 1.1 christos ZSTD_freeDCtx(dctx); 794 1.1 christos } 795 1.1 christos DISPLAYLEVEL(3, "OK \n"); 796 1.1 christos 797 1.1 christos DISPLAYLEVEL(3, "test%3i : decompress with 1 missing byte : ", testNb++); 798 1.1 christos { size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize-1); 799 1.1 christos if (!ZSTD_isError(r)) goto _output_error; 800 1.1 christos if (ZSTD_getErrorCode((size_t)r) != ZSTD_error_srcSize_wrong) goto _output_error; } 801 1.1 christos DISPLAYLEVEL(3, "OK \n"); 802 1.1 christos 803 1.1 christos DISPLAYLEVEL(3, "test%3i : decompress with 1 too much byte : ", testNb++); 804 1.1 christos { size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize+1); 805 1.1 christos if (!ZSTD_isError(r)) goto _output_error; 806 1.1 christos if (ZSTD_getErrorCode(r) != ZSTD_error_srcSize_wrong) goto _output_error; } 807 1.1 christos DISPLAYLEVEL(3, "OK \n"); 808 1.1 christos 809 1.1 christos DISPLAYLEVEL(3, "test%3i : decompress too large input : ", testNb++); 810 1.1 christos { size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, compressedBufferSize); 811 1.1 christos if (!ZSTD_isError(r)) goto _output_error; 812 1.1 christos if (ZSTD_getErrorCode(r) != ZSTD_error_srcSize_wrong) goto _output_error; } 813 1.1 christos DISPLAYLEVEL(3, "OK \n"); 814 1.1 christos 815 1.1 christos DISPLAYLEVEL(3, "test%3i : decompress into NULL buffer : ", testNb++); 816 1.1 christos { size_t const r = ZSTD_decompress(NULL, 0, compressedBuffer, compressedBufferSize); 817 1.1 christos if (!ZSTD_isError(r)) goto _output_error; 818 1.1 christos if (ZSTD_getErrorCode(r) != ZSTD_error_dstSize_tooSmall) goto _output_error; } 819 1.1 christos DISPLAYLEVEL(3, "OK \n"); 820 1.1 christos 821 1.1 christos DISPLAYLEVEL(3, "test%3i : decompress with corrupted checksum : ", testNb++); 822 1.1 christos { /* create compressed buffer with checksumming enabled */ 823 1.1 christos ZSTD_CCtx* const cctx = ZSTD_createCCtx(); 824 1.1 christos if (!cctx) { 825 1.1 christos DISPLAY("Not enough memory, aborting\n"); 826 1.1 christos testResult = 1; 827 1.1 christos goto _end; 828 1.1 christos } 829 1.1 christos CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1) ); 830 1.1 christos CHECK_VAR(cSize, ZSTD_compress2(cctx, 831 1.1 christos compressedBuffer, compressedBufferSize, 832 1.1 christos CNBuffer, CNBuffSize) ); 833 1.1 christos ZSTD_freeCCtx(cctx); 834 1.1 christos } 835 1.1 christos { /* copy the compressed buffer and corrupt the checksum */ 836 1.1 christos size_t r; 837 1.1 christos ZSTD_DCtx* const dctx = ZSTD_createDCtx(); 838 1.1 christos if (!dctx) { 839 1.1 christos DISPLAY("Not enough memory, aborting\n"); 840 1.1 christos testResult = 1; 841 1.1 christos goto _end; 842 1.1 christos } 843 1.1 christos 844 1.1 christos ((char*)compressedBuffer)[cSize-1] += 1; 845 1.1 christos r = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize); 846 1.1 christos if (!ZSTD_isError(r)) goto _output_error; 847 1.1 christos if (ZSTD_getErrorCode(r) != ZSTD_error_checksum_wrong) goto _output_error; 848 1.1 christos 849 1.1 christos CHECK_Z(ZSTD_DCtx_setParameter(dctx, ZSTD_d_forceIgnoreChecksum, ZSTD_d_ignoreChecksum)); 850 1.1 christos r = ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize-1); 851 1.1 christos if (!ZSTD_isError(r)) goto _output_error; /* wrong checksum size should still throw error */ 852 1.1 christos r = ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize); 853 1.1 christos if (ZSTD_isError(r)) goto _output_error; 854 1.1 christos 855 1.1 christos ZSTD_freeDCtx(dctx); 856 1.1 christos } 857 1.1 christos DISPLAYLEVEL(3, "OK \n"); 858 1.1 christos 859 1.1 christos 860 1.1 christos DISPLAYLEVEL(3, "test%3i : ZSTD_decompressBound test with content size missing : ", testNb++); 861 1.1 christos { /* create compressed buffer with content size missing */ 862 1.1 christos ZSTD_CCtx* const cctx = ZSTD_createCCtx(); 863 1.1 christos CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_contentSizeFlag, 0) ); 864 1.1 christos CHECK_VAR(cSize, ZSTD_compress2(cctx, 865 1.1 christos compressedBuffer, compressedBufferSize, 866 1.1 christos CNBuffer, CNBuffSize) ); 867 1.1 christos ZSTD_freeCCtx(cctx); 868 1.1 christos } 869 1.1 christos { /* ensure frame content size is missing */ 870 1.1 christos ZSTD_frameHeader zfh; 871 1.1 christos size_t const ret = ZSTD_getFrameHeader(&zfh, compressedBuffer, compressedBufferSize); 872 1.1 christos if (ret != 0 || zfh.frameContentSize != ZSTD_CONTENTSIZE_UNKNOWN) goto _output_error; 873 1.1 christos } 874 1.1 christos { /* ensure CNBuffSize <= decompressBound */ 875 1.1 christos unsigned long long const bound = ZSTD_decompressBound(compressedBuffer, compressedBufferSize); 876 1.1 christos if (CNBuffSize > bound) goto _output_error; 877 1.1 christos } 878 1.1 christos DISPLAYLEVEL(3, "OK \n"); 879 1.1 christos 880 1.1 christos DISPLAYLEVEL(3, "test%3d: check DCtx size is reduced after many oversized calls : ", testNb++); 881 1.1 christos { 882 1.1 christos size_t const largeFrameSrcSize = 200; 883 1.1 christos size_t const smallFrameSrcSize = 10; 884 1.1 christos size_t const nbFrames = 256; 885 1.1 christos 886 1.1 christos size_t i = 0, consumed = 0, produced = 0, prevDCtxSize = 0; 887 1.1 christos int sizeReduced = 0; 888 1.1 christos 889 1.1 christos BYTE* const dst = (BYTE*)compressedBuffer; 890 1.1 christos ZSTD_DCtx* dctx = ZSTD_createDCtx(); 891 1.1 christos 892 1.1 christos /* create a large frame and then a bunch of small frames */ 893 1.1 christos size_t srcSize = ZSTD_compress((void*)dst, 894 1.1 christos compressedBufferSize, CNBuffer, largeFrameSrcSize, 3); 895 1.1 christos for (i = 0; i < nbFrames; i++) 896 1.1 christos srcSize += ZSTD_compress((void*)(dst + srcSize), 897 1.1 christos compressedBufferSize - srcSize, CNBuffer, 898 1.1 christos smallFrameSrcSize, 3); 899 1.1 christos 900 1.1 christos /* decompressStream and make sure that dctx size was reduced at least once */ 901 1.1 christos while (consumed < srcSize) { 902 1.1 christos ZSTD_inBuffer in = {(void*)(dst + consumed), MIN(1, srcSize - consumed), 0}; 903 1.1 christos ZSTD_outBuffer out = {(BYTE*)CNBuffer + produced, CNBuffSize - produced, 0}; 904 1.1 christos ZSTD_decompressStream(dctx, &out, &in); 905 1.1 christos consumed += in.pos; 906 1.1 christos produced += out.pos; 907 1.1 christos 908 1.1 christos /* success! size was reduced from the previous frame */ 909 1.1 christos if (prevDCtxSize > ZSTD_sizeof_DCtx(dctx)) 910 1.1 christos sizeReduced = 1; 911 1.1 christos 912 1.1 christos prevDCtxSize = ZSTD_sizeof_DCtx(dctx); 913 1.1 christos } 914 1.1 christos 915 1.1 christos assert(sizeReduced); 916 1.1 christos 917 1.1 christos ZSTD_freeDCtx(dctx); 918 1.1 christos } 919 1.1 christos DISPLAYLEVEL(3, "OK \n"); 920 1.1 christos 921 1.1 christos { 922 1.1 christos ZSTD_CCtx* const cctx = ZSTD_createCCtx(); 923 1.1 christos ZSTD_CDict* const cdict = ZSTD_createCDict(CNBuffer, 100, 1); 924 1.1 christos ZSTD_parameters const params = ZSTD_getParams(1, 0, 0); 925 1.1 christos CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_format, ZSTD_f_zstd1_magicless) ); 926 1.1 christos 927 1.1 christos DISPLAYLEVEL(3, "test%3i : ZSTD_compressCCtx() doesn't use advanced parameters", testNb++); 928 1.1 christos CHECK_Z(ZSTD_compressCCtx(cctx, compressedBuffer, compressedBufferSize, NULL, 0, 1)); 929 1.1 christos if (MEM_readLE32(compressedBuffer) != ZSTD_MAGICNUMBER) goto _output_error; 930 1.1 christos DISPLAYLEVEL(3, "OK \n"); 931 1.1 christos 932 1.1 christos DISPLAYLEVEL(3, "test%3i : ZSTD_compress_usingDict() doesn't use advanced parameters: ", testNb++); 933 1.1 christos CHECK_Z(ZSTD_compress_usingDict(cctx, compressedBuffer, compressedBufferSize, NULL, 0, NULL, 0, 1)); 934 1.1 christos if (MEM_readLE32(compressedBuffer) != ZSTD_MAGICNUMBER) goto _output_error; 935 1.1 christos DISPLAYLEVEL(3, "OK \n"); 936 1.1 christos 937 1.1 christos DISPLAYLEVEL(3, "test%3i : ZSTD_compress_usingCDict() doesn't use advanced parameters: ", testNb++); 938 1.1 christos CHECK_Z(ZSTD_compress_usingCDict(cctx, compressedBuffer, compressedBufferSize, NULL, 0, cdict)); 939 1.1 christos if (MEM_readLE32(compressedBuffer) != ZSTD_MAGICNUMBER) goto _output_error; 940 1.1 christos DISPLAYLEVEL(3, "OK \n"); 941 1.1 christos 942 1.1 christos DISPLAYLEVEL(3, "test%3i : ZSTD_compress_advanced() doesn't use advanced parameters: ", testNb++); 943 1.1 christos CHECK_Z(ZSTD_compress_advanced(cctx, compressedBuffer, compressedBufferSize, NULL, 0, NULL, 0, params)); 944 1.1 christos if (MEM_readLE32(compressedBuffer) != ZSTD_MAGICNUMBER) goto _output_error; 945 1.1 christos DISPLAYLEVEL(3, "OK \n"); 946 1.1 christos 947 1.1 christos DISPLAYLEVEL(3, "test%3i : ZSTD_compress_usingCDict_advanced() doesn't use advanced parameters: ", testNb++); 948 1.1 christos CHECK_Z(ZSTD_compress_usingCDict_advanced(cctx, compressedBuffer, compressedBufferSize, NULL, 0, cdict, params.fParams)); 949 1.1 christos if (MEM_readLE32(compressedBuffer) != ZSTD_MAGICNUMBER) goto _output_error; 950 1.1 christos DISPLAYLEVEL(3, "OK \n"); 951 1.1 christos 952 1.1 christos ZSTD_freeCDict(cdict); 953 1.1 christos ZSTD_freeCCtx(cctx); 954 1.1 christos } 955 1.1 christos 956 1.1 christos DISPLAYLEVEL(3, "test%3i : maxBlockSize = 2K", testNb++); 957 1.1 christos { 958 1.1 christos ZSTD_CCtx* cctx = ZSTD_createCCtx(); 959 1.1 christos ZSTD_DCtx* dctx = ZSTD_createDCtx(); 960 1.1 christos CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1)); 961 1.1 christos CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_maxBlockSize, 2048)); 962 1.1 christos CHECK_Z(ZSTD_DCtx_setParameter(dctx, ZSTD_d_maxBlockSize, 2048)); 963 1.1 christos 964 1.1 christos cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize); 965 1.1 christos CHECK_Z(cSize); 966 1.1 christos CHECK_Z(ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize)); 967 1.1 christos 968 1.1 christos CHECK_Z(ZSTD_DCtx_setParameter(dctx, ZSTD_d_maxBlockSize, 1024)); 969 1.1 christos CHECK(ZSTD_isError(ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize))); 970 1.1 christos 971 1.1 christos ZSTD_freeDCtx(dctx); 972 1.1 christos ZSTD_freeCCtx(cctx); 973 1.1 christos } 974 1.1 christos 975 1.1 christos DISPLAYLEVEL(3, "test%3i : ldm fill dict out-of-bounds check", testNb++); 976 1.1 christos { 977 1.1 christos ZSTD_CCtx* const cctx = ZSTD_createCCtx(); 978 1.1 christos 979 1.1 christos size_t const size = (1U << 10); 980 1.1 christos size_t const dstCapacity = ZSTD_compressBound(size); 981 1.1 christos void* dict = (void*)malloc(size); 982 1.1 christos void* src = (void*)malloc(size); 983 1.1 christos void* dst = (void*)malloc(dstCapacity); 984 1.1 christos 985 1.1 christos RDG_genBuffer(dict, size, 0.5, 0.5, seed); 986 1.1 christos RDG_genBuffer(src, size, 0.5, 0.5, seed); 987 1.1 christos 988 1.1 christos CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable)); 989 1.1 christos assert(!ZSTD_isError(ZSTD_compress_usingDict(cctx, dst, dstCapacity, src, size, dict, size, 3))); 990 1.1 christos 991 1.1 christos ZSTD_freeCCtx(cctx); 992 1.1 christos free(dict); 993 1.1 christos free(src); 994 1.1 christos free(dst); 995 1.1 christos } 996 1.1 christos DISPLAYLEVEL(3, "OK \n"); 997 1.1 christos 998 1.1 christos DISPLAYLEVEL(3, "test%3i : testing dict compression with enableLdm and forceMaxWindow : ", testNb++); 999 1.1 christos { 1000 1.1 christos ZSTD_CCtx* const cctx = ZSTD_createCCtx(); 1001 1.1 christos ZSTD_DCtx* const dctx = ZSTD_createDCtx(); 1002 1.1 christos void* dict = (void*)malloc(CNBuffSize); 1003 1.1 christos int nbWorkers; 1004 1.1 christos 1005 1.1 christos for (nbWorkers = 0; nbWorkers < 3; ++nbWorkers) { 1006 1.1 christos RDG_genBuffer(dict, CNBuffSize, 0.5, 0.5, seed); 1007 1.1 christos RDG_genBuffer(CNBuffer, CNBuffSize, 0.6, 0.6, seed); 1008 1.1 christos 1009 1.1 christos CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, nbWorkers)); 1010 1.1 christos CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1)); 1011 1.1 christos CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_forceMaxWindow, 1)); 1012 1.1 christos CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable)); 1013 1.1 christos CHECK_Z(ZSTD_CCtx_refPrefix(cctx, dict, CNBuffSize)); 1014 1.1 christos cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize); 1015 1.1 christos CHECK_Z(cSize); 1016 1.1 christos CHECK_Z(ZSTD_decompress_usingDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, dict, CNBuffSize)); 1017 1.1 christos } 1018 1.1 christos 1019 1.1 christos ZSTD_freeCCtx(cctx); 1020 1.1 christos ZSTD_freeDCtx(dctx); 1021 1.1 christos free(dict); 1022 1.1 christos } 1023 1.1 christos DISPLAYLEVEL(3, "OK \n"); 1024 1.1 christos 1025 1.1 christos DISPLAYLEVEL(3, "test%3i : testing dict compression for determinism : ", testNb++); 1026 1.1 christos { 1027 1.1 christos size_t const testSize = 1024; 1028 1.1 christos ZSTD_CCtx* const cctx = ZSTD_createCCtx(); 1029 1.1 christos ZSTD_DCtx* const dctx = ZSTD_createDCtx(); 1030 1.1 christos char* dict = (char*)malloc(2 * testSize); 1031 1.1 christos int ldmEnabled, level; 1032 1.1 christos 1033 1.1 christos RDG_genBuffer(dict, testSize, 0.5, 0.5, seed); 1034 1.1 christos RDG_genBuffer(CNBuffer, testSize, 0.6, 0.6, seed); 1035 1.1 christos memcpy(dict + testSize, CNBuffer, testSize); 1036 1.1 christos for (level = 1; level <= 5; ++level) { 1037 1.1 christos for (ldmEnabled = ZSTD_ps_enable; ldmEnabled <= ZSTD_ps_disable; ++ldmEnabled) { 1038 1.1 christos size_t cSize0; 1039 1.1 christos XXH64_hash_t compressedChecksum0; 1040 1.1 christos 1041 1.1 christos CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1)); 1042 1.1 christos CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, level)); 1043 1.1 christos CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ldmEnabled)); 1044 1.1 christos CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_deterministicRefPrefix, 1)); 1045 1.1 christos 1046 1.1 christos CHECK_Z(ZSTD_CCtx_refPrefix(cctx, dict, testSize)); 1047 1.1 christos cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, testSize); 1048 1.1 christos CHECK_Z(cSize); 1049 1.1 christos CHECK_Z(ZSTD_decompress_usingDict(dctx, decodedBuffer, testSize, compressedBuffer, cSize, dict, testSize)); 1050 1.1 christos 1051 1.1 christos cSize0 = cSize; 1052 1.1 christos compressedChecksum0 = XXH64(compressedBuffer, cSize, 0); 1053 1.1 christos 1054 1.1 christos CHECK_Z(ZSTD_CCtx_refPrefix(cctx, dict, testSize)); 1055 1.1 christos cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, dict + testSize, testSize); 1056 1.1 christos CHECK_Z(cSize); 1057 1.1 christos 1058 1.1 christos if (cSize != cSize0) goto _output_error; 1059 1.1 christos if (XXH64(compressedBuffer, cSize, 0) != compressedChecksum0) goto _output_error; 1060 1.1 christos } 1061 1.1 christos } 1062 1.1 christos 1063 1.1 christos ZSTD_freeCCtx(cctx); 1064 1.1 christos ZSTD_freeDCtx(dctx); 1065 1.1 christos free(dict); 1066 1.1 christos } 1067 1.1 christos DISPLAYLEVEL(3, "OK \n"); 1068 1.1 christos 1069 1.1 christos DISPLAYLEVEL(3, "test%3i : LDM + opt parser with small uncompressible block ", testNb++); 1070 1.1 christos { ZSTD_CCtx* cctx = ZSTD_createCCtx(); 1071 1.1 christos ZSTD_DCtx* dctx = ZSTD_createDCtx(); 1072 1.1 christos size_t const srcSize = 300 KB; 1073 1.1 christos size_t const flushSize = 128 KB + 5; 1074 1.1 christos size_t const dstSize = ZSTD_compressBound(srcSize); 1075 1.1 christos char* src = (char*)CNBuffer; 1076 1.1 christos char* dst = (char*)compressedBuffer; 1077 1.1 christos 1078 1.1 christos ZSTD_outBuffer out = { dst, dstSize, 0 }; 1079 1.1 christos ZSTD_inBuffer in = { src, flushSize, 0 }; 1080 1.1 christos 1081 1.1 christos if (!cctx || !dctx) { 1082 1.1 christos DISPLAY("Not enough memory, aborting\n"); 1083 1.1 christos testResult = 1; 1084 1.1 christos goto _end; 1085 1.1 christos } 1086 1.1 christos 1087 1.1 christos RDG_genBuffer(src, srcSize, 0.5, 0.5, seed); 1088 1.1 christos /* Force an LDM to exist that crosses block boundary into uncompressible block */ 1089 1.1 christos memcpy(src + 125 KB, src, 3 KB + 5); 1090 1.1 christos 1091 1.1 christos /* Enable MT, LDM, and opt parser */ 1092 1.1 christos CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 1)); 1093 1.1 christos CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable)); 1094 1.1 christos CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1)); 1095 1.1 christos CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 19)); 1096 1.1 christos 1097 1.1 christos /* Flushes a block of 128 KB and block of 5 bytes */ 1098 1.1 christos CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush)); 1099 1.1 christos 1100 1.1 christos /* Compress the rest */ 1101 1.1 christos in.size = 300 KB; 1102 1.1 christos CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end)); 1103 1.1 christos 1104 1.1 christos CHECK_Z(ZSTD_decompress(decodedBuffer, CNBuffSize, dst, out.pos)); 1105 1.1 christos 1106 1.1 christos ZSTD_freeCCtx(cctx); 1107 1.1 christos ZSTD_freeDCtx(dctx); 1108 1.1 christos } 1109 1.1 christos DISPLAYLEVEL(3, "OK \n"); 1110 1.1 christos 1111 1.1 christos DISPLAYLEVEL(3, "test%3i : testing ldm dictionary gets invalidated : ", testNb++); 1112 1.1 christos { 1113 1.1 christos ZSTD_CCtx* const cctx = ZSTD_createCCtx(); 1114 1.1 christos ZSTD_DCtx* const dctx = ZSTD_createDCtx(); 1115 1.1 christos void* dict = (void*)malloc(CNBuffSize); 1116 1.1 christos size_t const kWindowLog = 10; 1117 1.1 christos size_t const kWindowSize = (size_t)1 << kWindowLog; 1118 1.1 christos size_t const dictSize = kWindowSize * 10; 1119 1.1 christos size_t const srcSize1 = kWindowSize / 2; 1120 1.1 christos size_t const srcSize2 = kWindowSize * 10; 1121 1.1 christos 1122 1.1 christos CHECK(cctx!=NULL); 1123 1.1 christos CHECK(dctx!=NULL); 1124 1.1 christos CHECK(dict!=NULL); 1125 1.1 christos if (CNBuffSize < dictSize) goto _output_error; 1126 1.1 christos 1127 1.1 christos RDG_genBuffer(dict, dictSize, 0.5, 0.5, seed); 1128 1.1 christos RDG_genBuffer(CNBuffer, srcSize1 + srcSize2, 0.5, 0.5, seed); 1129 1.1 christos 1130 1.1 christos /* Enable checksum to verify round trip. */ 1131 1.1 christos CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1)); 1132 1.1 christos /* Disable content size to skip single-pass decompression. */ 1133 1.1 christos CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_contentSizeFlag, 0)); 1134 1.1 christos CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, (int)kWindowLog)); 1135 1.1 christos CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable)); 1136 1.1 christos CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_ldmMinMatch, 32)); 1137 1.1 christos CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_ldmHashRateLog, 1)); 1138 1.1 christos CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_ldmHashLog, 16)); 1139 1.1 christos CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_ldmBucketSizeLog, 3)); 1140 1.1 christos 1141 1.1 christos /* Round trip once with a dictionary. */ 1142 1.1 christos CHECK_Z(ZSTD_CCtx_refPrefix(cctx, dict, dictSize)); 1143 1.1 christos cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, srcSize1); 1144 1.1 christos CHECK_Z(cSize); 1145 1.1 christos CHECK_Z(ZSTD_decompress_usingDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, dict, dictSize)); 1146 1.1 christos 1147 1.1 christos cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, srcSize2); 1148 1.1 christos /* Streaming decompression to catch out of bounds offsets. */ 1149 1.1 christos { 1150 1.1 christos ZSTD_inBuffer in = {compressedBuffer, cSize, 0}; 1151 1.1 christos ZSTD_outBuffer out = {decodedBuffer, CNBuffSize, 0}; 1152 1.1 christos size_t const dSize = ZSTD_decompressStream(dctx, &out, &in); 1153 1.1 christos CHECK_Z(dSize); 1154 1.1 christos if (dSize != 0) goto _output_error; 1155 1.1 christos } 1156 1.1 christos 1157 1.1 christos CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 2)); 1158 1.1 christos /* Round trip once with a dictionary. */ 1159 1.1 christos CHECK_Z(ZSTD_CCtx_refPrefix(cctx, dict, dictSize)); 1160 1.1 christos { ZSTD_inBuffer in = {CNBuffer, srcSize1, 0}; 1161 1.1 christos ZSTD_outBuffer out = {compressedBuffer, compressedBufferSize, 0}; 1162 1.1 christos CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush)); 1163 1.1 christos CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end)); 1164 1.1 christos cSize = out.pos; 1165 1.1 christos } 1166 1.1 christos CHECK_Z(ZSTD_decompress_usingDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, dict, dictSize)); 1167 1.1 christos 1168 1.1 christos { ZSTD_inBuffer in = {CNBuffer, srcSize2, 0}; 1169 1.1 christos ZSTD_outBuffer out = {compressedBuffer, compressedBufferSize, 0}; 1170 1.1 christos CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush)); 1171 1.1 christos CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end)); 1172 1.1 christos cSize = out.pos; 1173 1.1 christos } 1174 1.1 christos /* Streaming decompression to catch out of bounds offsets. */ 1175 1.1 christos { ZSTD_inBuffer in = {compressedBuffer, cSize, 0}; 1176 1.1 christos ZSTD_outBuffer out = {decodedBuffer, CNBuffSize, 0}; 1177 1.1 christos size_t const dSize = ZSTD_decompressStream(dctx, &out, &in); 1178 1.1 christos CHECK_Z(dSize); 1179 1.1 christos if (dSize != 0) goto _output_error; 1180 1.1 christos } 1181 1.1 christos 1182 1.1 christos ZSTD_freeCCtx(cctx); 1183 1.1 christos ZSTD_freeDCtx(dctx); 1184 1.1 christos free(dict); 1185 1.1 christos } 1186 1.1 christos DISPLAYLEVEL(3, "OK \n"); 1187 1.1 christos 1188 1.1 christos /* Note: this test takes 0.5 seconds to run */ 1189 1.1 christos DISPLAYLEVEL(3, "test%3i : testing refPrefx vs refPrefx + ldm (size comparison) : ", testNb++); 1190 1.1 christos { 1191 1.1 christos /* test a big buffer so that ldm can take effect */ 1192 1.1 christos size_t const size = 100 MB; 1193 1.1 christos int const windowLog = 27; 1194 1.1 christos size_t const dstSize = ZSTD_compressBound(size); 1195 1.1 christos 1196 1.1 christos void* dict = (void*)malloc(size); 1197 1.1 christos void* src = (void*)malloc(size); 1198 1.1 christos void* dst = (void*)malloc(dstSize); 1199 1.1 christos void* recon = (void*)malloc(size); 1200 1.1 christos 1201 1.1 christos size_t refPrefixCompressedSize = 0; 1202 1.1 christos size_t refPrefixLdmCompressedSize = 0; 1203 1.1 christos size_t reconSize = 0; 1204 1.1 christos 1205 1.1 christos ZSTD_CCtx* const cctx = ZSTD_createCCtx(); 1206 1.1 christos ZSTD_DCtx* const dctx = ZSTD_createDCtx(); 1207 1.1 christos 1208 1.1 christos /* make dict and src the same uncompressible data */ 1209 1.1 christos RDG_genBuffer(src, size, 0, 0, seed); 1210 1.1 christos memcpy(dict, src, size); 1211 1.1 christos assert(!memcmp(dict, src, size)); 1212 1.1 christos 1213 1.1 christos /* set level 1 and windowLog to cover src */ 1214 1.1 christos CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 1)); 1215 1.1 christos CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, windowLog)); 1216 1.1 christos 1217 1.1 christos /* compress on level 1 using just refPrefix and no ldm */ 1218 1.1 christos ZSTD_CCtx_refPrefix(cctx, dict, size); 1219 1.1 christos refPrefixCompressedSize = ZSTD_compress2(cctx, dst, dstSize, src, size); 1220 1.1 christos assert(!ZSTD_isError(refPrefixCompressedSize)); 1221 1.1 christos 1222 1.1 christos /* test round trip just refPrefix */ 1223 1.1 christos ZSTD_DCtx_refPrefix(dctx, dict, size); 1224 1.1 christos reconSize = ZSTD_decompressDCtx(dctx, recon, size, dst, refPrefixCompressedSize); 1225 1.1 christos assert(!ZSTD_isError(reconSize)); 1226 1.1 christos assert(reconSize == size); 1227 1.1 christos assert(!memcmp(recon, src, size)); 1228 1.1 christos 1229 1.1 christos /* compress on level 1 using refPrefix and ldm */ 1230 1.1 christos ZSTD_CCtx_refPrefix(cctx, dict, size);; 1231 1.1 christos CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable)) 1232 1.1 christos refPrefixLdmCompressedSize = ZSTD_compress2(cctx, dst, dstSize, src, size); 1233 1.1 christos assert(!ZSTD_isError(refPrefixLdmCompressedSize)); 1234 1.1 christos 1235 1.1 christos /* test round trip refPrefix + ldm*/ 1236 1.1 christos ZSTD_DCtx_refPrefix(dctx, dict, size); 1237 1.1 christos reconSize = ZSTD_decompressDCtx(dctx, recon, size, dst, refPrefixLdmCompressedSize); 1238 1.1 christos assert(!ZSTD_isError(reconSize)); 1239 1.1 christos assert(reconSize == size); 1240 1.1 christos assert(!memcmp(recon, src, size)); 1241 1.1 christos 1242 1.1 christos /* make sure that refPrefixCompressedSize is significantly greater */ 1243 1.1 christos assert(refPrefixCompressedSize > 10 * refPrefixLdmCompressedSize); 1244 1.1 christos /* make sure the ldm compressed size is less than 1% of original */ 1245 1.1 christos assert((double)refPrefixLdmCompressedSize / (double)size < 0.01); 1246 1.1 christos 1247 1.1 christos ZSTD_freeDCtx(dctx); 1248 1.1 christos ZSTD_freeCCtx(cctx); 1249 1.1 christos free(recon); 1250 1.1 christos free(dict); 1251 1.1 christos free(src); 1252 1.1 christos free(dst); 1253 1.1 christos } 1254 1.1 christos DISPLAYLEVEL(3, "OK \n"); 1255 1.1 christos 1256 1.1 christos DISPLAYLEVEL(3, "test%3i : in-place decompression : ", testNb++); 1257 1.1 christos cSize = ZSTD_compress(compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize, -ZSTD_BLOCKSIZE_MAX); 1258 1.1 christos CHECK_Z(cSize); 1259 1.1 christos CHECK_LT(CNBuffSize, cSize); 1260 1.1 christos { 1261 1.1 christos size_t const margin = ZSTD_decompressionMargin(compressedBuffer, cSize); 1262 1.1 christos size_t const outputSize = (CNBuffSize + margin); 1263 1.1 christos char* output = malloc(outputSize); 1264 1.1 christos char* input = output + outputSize - cSize; 1265 1.1 christos CHECK_LT(cSize, CNBuffSize + margin); 1266 1.1 christos CHECK(output != NULL); 1267 1.1 christos CHECK_Z(margin); 1268 1.1 christos CHECK(margin <= ZSTD_DECOMPRESSION_MARGIN(CNBuffSize, ZSTD_BLOCKSIZE_MAX)); 1269 1.1 christos memcpy(input, compressedBuffer, cSize); 1270 1.1 christos 1271 1.1 christos { 1272 1.1 christos size_t const dSize = ZSTD_decompress(output, outputSize, input, cSize); 1273 1.1 christos CHECK_Z(dSize); 1274 1.1 christos CHECK_EQ(dSize, CNBuffSize); 1275 1.1 christos } 1276 1.1 christos CHECK(!memcmp(output, CNBuffer, CNBuffSize)); 1277 1.1 christos free(output); 1278 1.1 christos } 1279 1.1 christos DISPLAYLEVEL(3, "OK \n"); 1280 1.1 christos 1281 1.1 christos DISPLAYLEVEL(3, "test%3i : in-place decompression with 2 frames : ", testNb++); 1282 1.1 christos cSize = ZSTD_compress(compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize / 3, -ZSTD_BLOCKSIZE_MAX); 1283 1.1 christos CHECK_Z(cSize); 1284 1.1 christos { 1285 1.1 christos size_t const cSize2 = ZSTD_compress((char*)compressedBuffer + cSize, compressedBufferSize - cSize, (char const*)CNBuffer + (CNBuffSize / 3), CNBuffSize / 3, -ZSTD_BLOCKSIZE_MAX); 1286 1.1 christos CHECK_Z(cSize2); 1287 1.1 christos cSize += cSize2; 1288 1.1 christos } 1289 1.1 christos { 1290 1.1 christos size_t const srcSize = (CNBuffSize / 3) * 2; 1291 1.1 christos size_t const margin = ZSTD_decompressionMargin(compressedBuffer, cSize); 1292 1.1 christos size_t const outputSize = (CNBuffSize + margin); 1293 1.1 christos char* output = malloc(outputSize); 1294 1.1 christos char* input = output + outputSize - cSize; 1295 1.1 christos CHECK_LT(cSize, CNBuffSize + margin); 1296 1.1 christos CHECK(output != NULL); 1297 1.1 christos CHECK_Z(margin); 1298 1.1 christos memcpy(input, compressedBuffer, cSize); 1299 1.1 christos 1300 1.1 christos { 1301 1.1 christos size_t const dSize = ZSTD_decompress(output, outputSize, input, cSize); 1302 1.1 christos CHECK_Z(dSize); 1303 1.1 christos CHECK_EQ(dSize, srcSize); 1304 1.1 christos } 1305 1.1 christos CHECK(!memcmp(output, CNBuffer, srcSize)); 1306 1.1 christos free(output); 1307 1.1 christos } 1308 1.1 christos DISPLAYLEVEL(3, "OK \n"); 1309 1.1 christos 1310 1.1 christos DISPLAYLEVEL(3, "test%3i : Check block splitter with 64K literal length : ", testNb++); 1311 1.1 christos { ZSTD_CCtx* cctx = ZSTD_createCCtx(); 1312 1.1 christos size_t const srcSize = 256 * 1024; 1313 1.1 christos U32 const compressibleLenU32 = 32 * 1024 / 4; 1314 1.1 christos U32 const blockSizeU32 = 128 * 1024 / 4; 1315 1.1 christos U32 const litLenU32 = 64 * 1024 / 4; 1316 1.1 christos U32* data = (U32*)malloc(srcSize); 1317 1.1 christos size_t dSize; 1318 1.1 christos 1319 1.1 christos if (data == NULL || cctx == NULL) goto _output_error; 1320 1.1 christos 1321 1.1 christos /* Generate data without any matches */ 1322 1.1 christos RDG_genBuffer(data, srcSize, 0.0, 0.01, 2654435761U); 1323 1.1 christos /* Generate 32K of compressible data */ 1324 1.1 christos RDG_genBuffer(data, compressibleLenU32 * 4, 0.5, 0.5, 0xcafebabe); 1325 1.1 christos 1326 1.1 christos /* Add a match of offset=12, length=8 at idx=16, 32, 48, 64 */ 1327 1.1 christos data[compressibleLenU32 + 0] = 0xFFFFFFFF; 1328 1.1 christos data[compressibleLenU32 + 1] = 0xEEEEEEEE; 1329 1.1 christos data[compressibleLenU32 + 4] = 0xFFFFFFFF; 1330 1.1 christos data[compressibleLenU32 + 5] = 0xEEEEEEEE; 1331 1.1 christos 1332 1.1 christos /* Add a match of offset=16, length=8 at idx=64K + 64. 1333 1.1 christos * This generates a sequence with llen=64K, and repeat code 1. 1334 1.1 christos * The block splitter thought this was ll0, and corrupted the 1335 1.1 christos * repeat offset history. 1336 1.1 christos */ 1337 1.1 christos data[compressibleLenU32 + litLenU32 + 2 + 0] = 0xDDDDDDDD; 1338 1.1 christos data[compressibleLenU32 + litLenU32 + 2 + 1] = 0xCCCCCCCC; 1339 1.1 christos data[compressibleLenU32 + litLenU32 + 2 + 4] = 0xDDDDDDDD; 1340 1.1 christos data[compressibleLenU32 + litLenU32 + 2 + 5] = 0xCCCCCCCC; 1341 1.1 christos 1342 1.1 christos /* Add a match of offset=16, length=8 at idx=128K + 16. 1343 1.1 christos * This should generate a sequence with repeat code = 1. 1344 1.1 christos * But the block splitters mistake caused zstd to generate 1345 1.1 christos * repeat code = 2, corrupting the data. 1346 1.1 christos */ 1347 1.1 christos data[blockSizeU32] = 0xBBBBBBBB; 1348 1.1 christos data[blockSizeU32 + 1] = 0xAAAAAAAA; 1349 1.1 christos data[blockSizeU32 + 4] = 0xBBBBBBBB; 1350 1.1 christos data[blockSizeU32 + 5] = 0xAAAAAAAA; 1351 1.1 christos 1352 1.1 christos /* Generate a golden file from this data in case datagen changes and 1353 1.1 christos * doesn't generate the exact same data. We will also test this golden file. 1354 1.1 christos */ 1355 1.1 christos if (0) { 1356 1.1 christos FILE* f = fopen("golden-compression/PR-3517-block-splitter-corruption-test", "wb"); 1357 1.1 christos fwrite(data, 1, srcSize, f); 1358 1.1 christos fclose(f); 1359 1.1 christos } 1360 1.1 christos 1361 1.1 christos CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 19)); 1362 1.1 christos CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_minMatch, 7)); 1363 1.1 christos CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_useBlockSplitter, ZSTD_ps_enable)); 1364 1.1 christos 1365 1.1 christos cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, data, srcSize); 1366 1.1 christos CHECK_Z(cSize); 1367 1.1 christos dSize = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize); 1368 1.1 christos CHECK_Z(dSize); 1369 1.1 christos CHECK_EQ(dSize, srcSize); 1370 1.1 christos CHECK(!memcmp(decodedBuffer, data, srcSize)); 1371 1.1 christos 1372 1.1 christos free(data); 1373 1.1 christos ZSTD_freeCCtx(cctx); 1374 1.1 christos } 1375 1.1 christos DISPLAYLEVEL(3, "OK \n"); 1376 1.1 christos 1377 1.1 christos DISPLAYLEVEL(3, "test%3d : superblock uncompressible data: too many nocompress superblocks : ", testNb++); 1378 1.1 christos { 1379 1.1 christos ZSTD_CCtx* const cctx = ZSTD_createCCtx(); 1380 1.1 christos const BYTE* src = (BYTE*)CNBuffer; BYTE* dst = (BYTE*)compressedBuffer; 1381 1.1 christos size_t srcSize = 321656; size_t dstCapacity = ZSTD_compressBound(srcSize); 1382 1.1 christos 1383 1.1 christos /* This is the number of bytes to stream before ending. This value 1384 1.1 christos * was obtained by trial and error :/. */ 1385 1.1 christos 1386 1.1 christos const size_t streamCompressThreshold = 161792; 1387 1.1 christos const size_t streamCompressDelta = 1024; 1388 1.1 christos 1389 1.1 christos /* The first 1/5 of the buffer is compressible and the last 4/5 is 1390 1.1 christos * uncompressible. This is an approximation of the type of data 1391 1.1 christos * the fuzzer generated to catch this bug. Streams like this were making 1392 1.1 christos * zstd generate noCompress superblocks (which are larger than the src 1393 1.1 christos * they come from). Do this enough times, and we'll run out of room 1394 1.1 christos * and throw a dstSize_tooSmall error. */ 1395 1.1 christos 1396 1.1 christos const size_t compressiblePartSize = srcSize/5; 1397 1.1 christos const size_t uncompressiblePartSize = srcSize-compressiblePartSize; 1398 1.1 christos RDG_genBuffer(CNBuffer, compressiblePartSize, 0.5, 0.5, seed); 1399 1.1 christos RDG_genBuffer((BYTE*)CNBuffer+compressiblePartSize, uncompressiblePartSize, 0, 0, seed); 1400 1.1 christos 1401 1.1 christos /* Setting target block size so that superblock is used */ 1402 1.1 christos 1403 1.1 christos assert(cctx != NULL); 1404 1.1 christos ZSTD_CCtx_setParameter(cctx, ZSTD_c_targetCBlockSize, 81); 1405 1.1 christos 1406 1.1 christos { size_t read; 1407 1.1 christos for (read = 0; read < streamCompressThreshold; read += streamCompressDelta) { 1408 1.1 christos ZSTD_inBuffer in = {src, streamCompressDelta, 0}; 1409 1.1 christos ZSTD_outBuffer out = {dst, dstCapacity, 0}; 1410 1.1 christos CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_continue)); 1411 1.1 christos CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end)); 1412 1.1 christos src += streamCompressDelta; srcSize -= streamCompressDelta; 1413 1.1 christos dst += out.pos; dstCapacity -= out.pos; 1414 1.1 christos } } 1415 1.1 christos 1416 1.1 christos /* This is trying to catch a dstSize_tooSmall error */ 1417 1.1 christos 1418 1.1 christos { ZSTD_inBuffer in = {src, srcSize, 0}; 1419 1.1 christos ZSTD_outBuffer out = {dst, dstCapacity, 0}; 1420 1.1 christos CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end)); 1421 1.1 christos } 1422 1.1 christos ZSTD_freeCCtx(cctx); 1423 1.1 christos } 1424 1.1 christos DISPLAYLEVEL(3, "OK \n"); 1425 1.1 christos 1426 1.1 christos DISPLAYLEVEL(3, "test%3d: superblock with no literals : ", testNb++); 1427 1.1 christos /* Generate the same data 20 times over */ 1428 1.1 christos { size_t const avgChunkSize = CNBuffSize / 20; 1429 1.1 christos size_t b; 1430 1.1 christos for (b = 0; b < CNBuffSize; b += avgChunkSize) { 1431 1.1 christos size_t const chunkSize = MIN(CNBuffSize - b, avgChunkSize); 1432 1.1 christos RDG_genBuffer((char*)CNBuffer + b, chunkSize, compressibility, 0. /* auto */, seed); 1433 1.1 christos } } 1434 1.1 christos { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); 1435 1.1 christos size_t const normalCSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize); 1436 1.1 christos size_t const allowedExpansion = (CNBuffSize * 3 / 1000); 1437 1.1 christos size_t superCSize; 1438 1.1 christos CHECK_Z(normalCSize); 1439 1.1 christos ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 19); 1440 1.1 christos ZSTD_CCtx_setParameter(cctx, ZSTD_c_targetCBlockSize, 1000); 1441 1.1 christos superCSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize); 1442 1.1 christos CHECK_Z(superCSize); 1443 1.1 christos if (superCSize > normalCSize + allowedExpansion) { 1444 1.1 christos DISPLAYLEVEL(1, "Superblock too big: %u > %u + %u \n", (U32)superCSize, (U32)normalCSize, (U32)allowedExpansion); 1445 1.1 christos goto _output_error; 1446 1.1 christos } 1447 1.1 christos ZSTD_freeCCtx(cctx); 1448 1.1 christos } 1449 1.1 christos DISPLAYLEVEL(3, "OK \n"); 1450 1.1 christos 1451 1.1 christos RDG_genBuffer(CNBuffer, CNBuffSize, compressibility, 0. /*auto*/, seed); 1452 1.1 christos DISPLAYLEVEL(3, "test%3d: superblock enough room for checksum : ", testNb++) 1453 1.1 christos /* This tests whether or not we leave enough room for the checksum at the end 1454 1.1 christos * of the dst buffer. The bug that motivated this test was found by the 1455 1.1 christos * stream_round_trip fuzzer but this crashes for the same reason and is 1456 1.1 christos * far more compact than re-creating the stream_round_trip fuzzer's code path */ 1457 1.1 christos { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); 1458 1.1 christos ZSTD_CCtx_setParameter(cctx, ZSTD_c_targetCBlockSize, 64); 1459 1.1 christos assert(!ZSTD_isError(ZSTD_compress2(cctx, compressedBuffer, 1339, CNBuffer, 1278))); 1460 1.1 christos ZSTD_freeCCtx(cctx); 1461 1.1 christos } 1462 1.1 christos DISPLAYLEVEL(3, "OK \n"); 1463 1.1 christos 1464 1.1 christos DISPLAYLEVEL(3, "test%3i : compress a NULL input with each level : ", testNb++); 1465 1.1 christos { int level = -1; 1466 1.1 christos ZSTD_CCtx* const cctx = ZSTD_createCCtx(); 1467 1.1 christos if (!cctx) goto _output_error; 1468 1.1 christos for (level = -1; level <= ZSTD_maxCLevel(); ++level) { 1469 1.1 christos CHECK_Z( ZSTD_compress(compressedBuffer, compressedBufferSize, NULL, 0, level) ); 1470 1.1 christos CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, level) ); 1471 1.1 christos CHECK_Z( ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, NULL, 0) ); 1472 1.1 christos } 1473 1.1 christos ZSTD_freeCCtx(cctx); 1474 1.1 christos } 1475 1.1 christos DISPLAYLEVEL(3, "OK \n"); 1476 1.1 christos 1477 1.1 christos DISPLAYLEVEL(3, "test%3d : check CCtx size after compressing empty input : ", testNb++); 1478 1.1 christos { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); 1479 1.1 christos size_t const r = ZSTD_compressCCtx(cctx, compressedBuffer, compressedBufferSize, NULL, 0, 19); 1480 1.1 christos if (ZSTD_isError(r)) goto _output_error; 1481 1.1 christos if (ZSTD_sizeof_CCtx(cctx) > (1U << 20)) goto _output_error; 1482 1.1 christos ZSTD_freeCCtx(cctx); 1483 1.1 christos cSize = r; 1484 1.1 christos } 1485 1.1 christos DISPLAYLEVEL(3, "OK \n"); 1486 1.1 christos 1487 1.1 christos DISPLAYLEVEL(3, "test%3d : decompress empty frame into NULL : ", testNb++); 1488 1.1 christos { size_t const r = ZSTD_decompress(NULL, 0, compressedBuffer, cSize); 1489 1.1 christos if (ZSTD_isError(r)) goto _output_error; 1490 1.1 christos if (r != 0) goto _output_error; 1491 1.1 christos } 1492 1.1 christos { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); 1493 1.1 christos ZSTD_outBuffer output; 1494 1.1 christos if (cctx==NULL) goto _output_error; 1495 1.1 christos output.dst = compressedBuffer; 1496 1.1 christos output.size = compressedBufferSize; 1497 1.1 christos output.pos = 0; 1498 1.1 christos CHECK_Z( ZSTD_initCStream(cctx, 1) ); /* content size unknown */ 1499 1.1 christos CHECK_Z( ZSTD_flushStream(cctx, &output) ); /* ensure no possibility to "concatenate" and determine the content size */ 1500 1.1 christos CHECK_Z( ZSTD_endStream(cctx, &output) ); 1501 1.1 christos ZSTD_freeCCtx(cctx); 1502 1.1 christos /* single scan decompression */ 1503 1.1 christos { size_t const r = ZSTD_decompress(NULL, 0, compressedBuffer, output.pos); 1504 1.1 christos if (ZSTD_isError(r)) goto _output_error; 1505 1.1 christos if (r != 0) goto _output_error; 1506 1.1 christos } 1507 1.1 christos /* streaming decompression */ 1508 1.1 christos { ZSTD_DCtx* const dstream = ZSTD_createDStream(); 1509 1.1 christos ZSTD_inBuffer dinput; 1510 1.1 christos ZSTD_outBuffer doutput; 1511 1.1 christos size_t ipos; 1512 1.1 christos if (dstream==NULL) goto _output_error; 1513 1.1 christos dinput.src = compressedBuffer; 1514 1.1 christos dinput.size = 0; 1515 1.1 christos dinput.pos = 0; 1516 1.1 christos doutput.dst = NULL; 1517 1.1 christos doutput.size = 0; 1518 1.1 christos doutput.pos = 0; 1519 1.1 christos CHECK_Z ( ZSTD_initDStream(dstream) ); 1520 1.1 christos for (ipos=1; ipos<=output.pos; ipos++) { 1521 1.1 christos dinput.size = ipos; 1522 1.1 christos CHECK_Z ( ZSTD_decompressStream(dstream, &doutput, &dinput) ); 1523 1.1 christos } 1524 1.1 christos if (doutput.pos != 0) goto _output_error; 1525 1.1 christos ZSTD_freeDStream(dstream); 1526 1.1 christos } 1527 1.1 christos } 1528 1.1 christos DISPLAYLEVEL(3, "OK \n"); 1529 1.1 christos 1530 1.1 christos DISPLAYLEVEL(3, "test%3d : reuse CCtx with expanding block size : ", testNb++); 1531 1.1 christos { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); 1532 1.1 christos ZSTD_parameters const params = ZSTD_getParams(1, ZSTD_CONTENTSIZE_UNKNOWN, 0); 1533 1.1 christos assert(params.fParams.contentSizeFlag == 1); /* block size will be adapted if pledgedSrcSize is enabled */ 1534 1.1 christos CHECK_Z( ZSTD_compressBegin_advanced(cctx, NULL, 0, params, 1 /*pledgedSrcSize*/) ); 1535 1.1 christos CHECK_Z( ZSTD_compressEnd(cctx, compressedBuffer, compressedBufferSize, CNBuffer, 1) ); /* creates a block size of 1 */ 1536 1.1 christos 1537 1.1 christos CHECK_Z( ZSTD_compressBegin_advanced(cctx, NULL, 0, params, ZSTD_CONTENTSIZE_UNKNOWN) ); /* reuse same parameters */ 1538 1.1 christos { size_t const inSize = 2* 128 KB; 1539 1.1 christos size_t const outSize = ZSTD_compressBound(inSize); 1540 1.1 christos CHECK_Z( ZSTD_compressEnd(cctx, compressedBuffer, outSize, CNBuffer, inSize) ); 1541 1.1 christos /* will fail if blockSize is not resized */ 1542 1.1 christos } 1543 1.1 christos ZSTD_freeCCtx(cctx); 1544 1.1 christos } 1545 1.1 christos DISPLAYLEVEL(3, "OK \n"); 1546 1.1 christos 1547 1.1 christos DISPLAYLEVEL(3, "test%3d : re-using a CCtx should compress the same : ", testNb++); 1548 1.1 christos { size_t const sampleSize = 30; 1549 1.1 christos int i; 1550 1.1 christos for (i=0; i<20; i++) 1551 1.1 christos ((char*)CNBuffer)[i] = (char)i; /* ensure no match during initial section */ 1552 1.1 christos memcpy((char*)CNBuffer + 20, CNBuffer, 10); /* create one match, starting from beginning of sample, which is the difficult case (see #1241) */ 1553 1.1 christos for (i=1; i<=19; i++) { 1554 1.1 christos ZSTD_CCtx* const cctx = ZSTD_createCCtx(); 1555 1.1 christos size_t size1, size2; 1556 1.1 christos DISPLAYLEVEL(5, "l%i ", i); 1557 1.1 christos size1 = ZSTD_compressCCtx(cctx, compressedBuffer, compressedBufferSize, CNBuffer, sampleSize, i); 1558 1.1 christos CHECK_Z(size1); 1559 1.1 christos 1560 1.1 christos size2 = ZSTD_compressCCtx(cctx, compressedBuffer, compressedBufferSize, CNBuffer, sampleSize, i); 1561 1.1 christos CHECK_Z(size2); 1562 1.1 christos CHECK_EQ(size1, size2); 1563 1.1 christos 1564 1.1 christos CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, i) ); 1565 1.1 christos size2 = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, sampleSize); 1566 1.1 christos CHECK_Z(size2); 1567 1.1 christos CHECK_EQ(size1, size2); 1568 1.1 christos 1569 1.1 christos size2 = ZSTD_compress2(cctx, compressedBuffer, ZSTD_compressBound(sampleSize) - 1, CNBuffer, sampleSize); /* force streaming, as output buffer is not large enough to guarantee success */ 1570 1.1 christos CHECK_Z(size2); 1571 1.1 christos CHECK_EQ(size1, size2); 1572 1.1 christos 1573 1.1 christos { ZSTD_inBuffer inb; 1574 1.1 christos ZSTD_outBuffer outb; 1575 1.1 christos inb.src = CNBuffer; 1576 1.1 christos inb.pos = 0; 1577 1.1 christos inb.size = sampleSize; 1578 1.1 christos outb.dst = compressedBuffer; 1579 1.1 christos outb.pos = 0; 1580 1.1 christos outb.size = ZSTD_compressBound(sampleSize) - 1; /* force streaming, as output buffer is not large enough to guarantee success */ 1581 1.1 christos CHECK_Z( ZSTD_compressStream2(cctx, &outb, &inb, ZSTD_e_end) ); 1582 1.1 christos assert(inb.pos == inb.size); 1583 1.1 christos CHECK_EQ(size1, outb.pos); 1584 1.1 christos } 1585 1.1 christos 1586 1.1 christos ZSTD_freeCCtx(cctx); 1587 1.1 christos } 1588 1.1 christos } 1589 1.1 christos DISPLAYLEVEL(3, "OK \n"); 1590 1.1 christos 1591 1.1 christos DISPLAYLEVEL(3, "test%3d : btultra2 & 1st block : ", testNb++); 1592 1.1 christos { size_t const sampleSize = 1024; 1593 1.1 christos ZSTD_CCtx* const cctx = ZSTD_createCCtx(); 1594 1.1 christos ZSTD_inBuffer inb; 1595 1.1 christos ZSTD_outBuffer outb; 1596 1.1 christos inb.src = CNBuffer; 1597 1.1 christos inb.pos = 0; 1598 1.1 christos inb.size = 0; 1599 1.1 christos outb.dst = compressedBuffer; 1600 1.1 christos outb.pos = 0; 1601 1.1 christos outb.size = compressedBufferSize; 1602 1.1 christos CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, ZSTD_maxCLevel()) ); 1603 1.1 christos 1604 1.1 christos inb.size = sampleSize; /* start with something, so that context is already used */ 1605 1.1 christos CHECK_Z( ZSTD_compressStream2(cctx, &outb, &inb, ZSTD_e_end) ); /* will break internal assert if stats_init is not disabled */ 1606 1.1 christos assert(inb.pos == inb.size); 1607 1.1 christos outb.pos = 0; /* cancel output */ 1608 1.1 christos 1609 1.1 christos CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(cctx, sampleSize) ); 1610 1.1 christos inb.size = 4; /* too small size : compression will be skipped */ 1611 1.1 christos inb.pos = 0; 1612 1.1 christos CHECK_Z( ZSTD_compressStream2(cctx, &outb, &inb, ZSTD_e_flush) ); 1613 1.1 christos assert(inb.pos == inb.size); 1614 1.1 christos 1615 1.1 christos inb.size += 5; /* too small size : compression will be skipped */ 1616 1.1 christos CHECK_Z( ZSTD_compressStream2(cctx, &outb, &inb, ZSTD_e_flush) ); 1617 1.1 christos assert(inb.pos == inb.size); 1618 1.1 christos 1619 1.1 christos inb.size += 11; /* small enough to attempt compression */ 1620 1.1 christos CHECK_Z( ZSTD_compressStream2(cctx, &outb, &inb, ZSTD_e_flush) ); 1621 1.1 christos assert(inb.pos == inb.size); 1622 1.1 christos 1623 1.1 christos assert(inb.pos < sampleSize); 1624 1.1 christos inb.size = sampleSize; /* large enough to trigger stats_init, but no longer at beginning */ 1625 1.1 christos CHECK_Z( ZSTD_compressStream2(cctx, &outb, &inb, ZSTD_e_end) ); /* will break internal assert if stats_init is not disabled */ 1626 1.1 christos assert(inb.pos == inb.size); 1627 1.1 christos ZSTD_freeCCtx(cctx); 1628 1.1 christos } 1629 1.1 christos DISPLAYLEVEL(3, "OK \n"); 1630 1.1 christos 1631 1.1 christos DISPLAYLEVEL(3, "test%3d : ZSTD_CCtx_getParameter() : ", testNb++); 1632 1.1 christos { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); 1633 1.1 christos ZSTD_outBuffer out = {NULL, 0, 0}; 1634 1.1 christos ZSTD_inBuffer in = {NULL, 0, 0}; 1635 1.1 christos int value; 1636 1.1 christos 1637 1.1 christos CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_compressionLevel, &value)); 1638 1.1 christos CHECK_EQ(value, 3); 1639 1.1 christos CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_hashLog, &value)); 1640 1.1 christos CHECK_EQ(value, 0); 1641 1.1 christos CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_hashLog, ZSTD_HASHLOG_MIN)); 1642 1.1 christos CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_compressionLevel, &value)); 1643 1.1 christos CHECK_EQ(value, 3); 1644 1.1 christos CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_hashLog, &value)); 1645 1.1 christos CHECK_EQ(value, ZSTD_HASHLOG_MIN); 1646 1.1 christos CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 7)); 1647 1.1 christos CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_compressionLevel, &value)); 1648 1.1 christos CHECK_EQ(value, 7); 1649 1.1 christos CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_hashLog, &value)); 1650 1.1 christos CHECK_EQ(value, ZSTD_HASHLOG_MIN); 1651 1.1 christos /* Start a compression job */ 1652 1.1 christos ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_continue); 1653 1.1 christos CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_compressionLevel, &value)); 1654 1.1 christos CHECK_EQ(value, 7); 1655 1.1 christos CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_hashLog, &value)); 1656 1.1 christos CHECK_EQ(value, ZSTD_HASHLOG_MIN); 1657 1.1 christos /* Reset the CCtx */ 1658 1.1 christos ZSTD_CCtx_reset(cctx, ZSTD_reset_session_only); 1659 1.1 christos CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_compressionLevel, &value)); 1660 1.1 christos CHECK_EQ(value, 7); 1661 1.1 christos CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_hashLog, &value)); 1662 1.1 christos CHECK_EQ(value, ZSTD_HASHLOG_MIN); 1663 1.1 christos /* Reset the parameters */ 1664 1.1 christos ZSTD_CCtx_reset(cctx, ZSTD_reset_parameters); 1665 1.1 christos CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_compressionLevel, &value)); 1666 1.1 christos CHECK_EQ(value, 3); 1667 1.1 christos CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_hashLog, &value)); 1668 1.1 christos CHECK_EQ(value, 0); 1669 1.1 christos 1670 1.1 christos ZSTD_freeCCtx(cctx); 1671 1.1 christos } 1672 1.1 christos DISPLAYLEVEL(3, "OK \n"); 1673 1.1 christos 1674 1.1 christos DISPLAYLEVEL(3, "test%3d : ZSTD_CCtx_setCParams() : ", testNb++); 1675 1.1 christos { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); 1676 1.1 christos int value; 1677 1.1 christos ZSTD_compressionParameters cparams = ZSTD_getCParams(1, 0, 0); 1678 1.1 christos cparams.strategy = -1; 1679 1.1 christos /* Set invalid cParams == no change. */ 1680 1.1 christos CHECK(ZSTD_isError(ZSTD_CCtx_setCParams(cctx, cparams))); 1681 1.1 christos 1682 1.1 christos CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_windowLog, &value)); 1683 1.1 christos CHECK_EQ(value, 0); 1684 1.1 christos CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_chainLog, &value)); 1685 1.1 christos CHECK_EQ(value, 0); 1686 1.1 christos CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_hashLog, &value)); 1687 1.1 christos CHECK_EQ(value, 0); 1688 1.1 christos CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_searchLog, &value)); 1689 1.1 christos CHECK_EQ(value, 0); 1690 1.1 christos CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_minMatch, &value)); 1691 1.1 christos CHECK_EQ(value, 0); 1692 1.1 christos CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_targetLength, &value)); 1693 1.1 christos CHECK_EQ(value, 0); 1694 1.1 christos CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_strategy, &value)); 1695 1.1 christos CHECK_EQ(value, 0); 1696 1.1 christos 1697 1.1 christos cparams = ZSTD_getCParams(12, 0, 0); 1698 1.1 christos CHECK_Z(ZSTD_CCtx_setCParams(cctx, cparams)); 1699 1.1 christos 1700 1.1 christos CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_windowLog, &value)); 1701 1.1 christos CHECK_EQ(value, (int)cparams.windowLog); 1702 1.1 christos CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_chainLog, &value)); 1703 1.1 christos CHECK_EQ(value, (int)cparams.chainLog); 1704 1.1 christos CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_hashLog, &value)); 1705 1.1 christos CHECK_EQ(value, (int)cparams.hashLog); 1706 1.1 christos CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_searchLog, &value)); 1707 1.1 christos CHECK_EQ(value, (int)cparams.searchLog); 1708 1.1 christos CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_minMatch, &value)); 1709 1.1 christos CHECK_EQ(value, (int)cparams.minMatch); 1710 1.1 christos CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_targetLength, &value)); 1711 1.1 christos CHECK_EQ(value, (int)cparams.targetLength); 1712 1.1 christos CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_strategy, &value)); 1713 1.1 christos CHECK_EQ(value, (int)cparams.strategy); 1714 1.1 christos 1715 1.1 christos ZSTD_freeCCtx(cctx); 1716 1.1 christos } 1717 1.1 christos 1718 1.1 christos DISPLAYLEVEL(3, "test%3d : ZSTD_CCtx_setFParams() : ", testNb++); 1719 1.1 christos { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); 1720 1.1 christos int value; 1721 1.1 christos ZSTD_frameParameters fparams = {0, 1, 1}; 1722 1.1 christos 1723 1.1 christos CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_contentSizeFlag, &value)); 1724 1.1 christos CHECK_EQ(value, 1); 1725 1.1 christos CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_checksumFlag, &value)); 1726 1.1 christos CHECK_EQ(value, 0); 1727 1.1 christos CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_dictIDFlag, &value)); 1728 1.1 christos CHECK_EQ(value, 1); 1729 1.1 christos 1730 1.1 christos CHECK_Z(ZSTD_CCtx_setFParams(cctx, fparams)); 1731 1.1 christos 1732 1.1 christos CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_contentSizeFlag, &value)); 1733 1.1 christos CHECK_EQ(value, fparams.contentSizeFlag); 1734 1.1 christos CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_checksumFlag, &value)); 1735 1.1 christos CHECK_EQ(value, fparams.checksumFlag); 1736 1.1 christos CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_dictIDFlag, &value)); 1737 1.1 christos CHECK_EQ(value, !fparams.noDictIDFlag); 1738 1.1 christos 1739 1.1 christos ZSTD_freeCCtx(cctx); 1740 1.1 christos } 1741 1.1 christos 1742 1.1 christos DISPLAYLEVEL(3, "test%3d : ZSTD_CCtx_setCarams() : ", testNb++); 1743 1.1 christos { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); 1744 1.1 christos int value; 1745 1.1 christos ZSTD_parameters params = ZSTD_getParams(1, 0, 0); 1746 1.1 christos params.cParams.strategy = -1; 1747 1.1 christos /* Set invalid params == no change. */ 1748 1.1 christos CHECK(ZSTD_isError(ZSTD_CCtx_setParams(cctx, params))); 1749 1.1 christos 1750 1.1 christos CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_windowLog, &value)); 1751 1.1 christos CHECK_EQ(value, 0); 1752 1.1 christos CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_chainLog, &value)); 1753 1.1 christos CHECK_EQ(value, 0); 1754 1.1 christos CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_hashLog, &value)); 1755 1.1 christos CHECK_EQ(value, 0); 1756 1.1 christos CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_searchLog, &value)); 1757 1.1 christos CHECK_EQ(value, 0); 1758 1.1 christos CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_minMatch, &value)); 1759 1.1 christos CHECK_EQ(value, 0); 1760 1.1 christos CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_targetLength, &value)); 1761 1.1 christos CHECK_EQ(value, 0); 1762 1.1 christos CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_strategy, &value)); 1763 1.1 christos CHECK_EQ(value, 0); 1764 1.1 christos CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_contentSizeFlag, &value)); 1765 1.1 christos CHECK_EQ(value, 1); 1766 1.1 christos CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_checksumFlag, &value)); 1767 1.1 christos CHECK_EQ(value, 0); 1768 1.1 christos CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_dictIDFlag, &value)); 1769 1.1 christos CHECK_EQ(value, 1); 1770 1.1 christos 1771 1.1 christos params = ZSTD_getParams(12, 0, 0); 1772 1.1 christos params.fParams.contentSizeFlag = 0; 1773 1.1 christos params.fParams.checksumFlag = 1; 1774 1.1 christos params.fParams.noDictIDFlag = 1; 1775 1.1 christos CHECK_Z(ZSTD_CCtx_setParams(cctx, params)); 1776 1.1 christos 1777 1.1 christos CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_windowLog, &value)); 1778 1.1 christos CHECK_EQ(value, (int)params.cParams.windowLog); 1779 1.1 christos CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_chainLog, &value)); 1780 1.1 christos CHECK_EQ(value, (int)params.cParams.chainLog); 1781 1.1 christos CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_hashLog, &value)); 1782 1.1 christos CHECK_EQ(value, (int)params.cParams.hashLog); 1783 1.1 christos CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_searchLog, &value)); 1784 1.1 christos CHECK_EQ(value, (int)params.cParams.searchLog); 1785 1.1 christos CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_minMatch, &value)); 1786 1.1 christos CHECK_EQ(value, (int)params.cParams.minMatch); 1787 1.1 christos CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_targetLength, &value)); 1788 1.1 christos CHECK_EQ(value, (int)params.cParams.targetLength); 1789 1.1 christos CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_strategy, &value)); 1790 1.1 christos CHECK_EQ(value, (int)params.cParams.strategy); 1791 1.1 christos CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_contentSizeFlag, &value)); 1792 1.1 christos CHECK_EQ(value, params.fParams.contentSizeFlag); 1793 1.1 christos CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_checksumFlag, &value)); 1794 1.1 christos CHECK_EQ(value, params.fParams.checksumFlag); 1795 1.1 christos CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_dictIDFlag, &value)); 1796 1.1 christos CHECK_EQ(value, !params.fParams.noDictIDFlag); 1797 1.1 christos 1798 1.1 christos ZSTD_freeCCtx(cctx); 1799 1.1 christos } 1800 1.1 christos 1801 1.1 christos DISPLAYLEVEL(3, "test%3d : ldm conditionally enabled by default doesn't change cctx params: ", testNb++); 1802 1.1 christos { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); 1803 1.1 christos ZSTD_outBuffer out = {NULL, 0, 0}; 1804 1.1 christos ZSTD_inBuffer in = {NULL, 0, 0}; 1805 1.1 christos int value; 1806 1.1 christos 1807 1.1 christos /* Even if LDM will be enabled by default in the applied params (since wlog >= 27 and strategy >= btopt), 1808 1.1 christos * we should not modify the actual parameter specified by the user within the CCtx 1809 1.1 christos */ 1810 1.1 christos CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, 27)); 1811 1.1 christos CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_strategy, ZSTD_btopt)); 1812 1.1 christos 1813 1.1 christos CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_continue)); 1814 1.1 christos CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_enableLongDistanceMatching, &value)); 1815 1.1 christos CHECK_EQ(value, 0); 1816 1.1 christos 1817 1.1 christos ZSTD_freeCCtx(cctx); 1818 1.1 christos } 1819 1.1 christos DISPLAYLEVEL(3, "OK \n"); 1820 1.1 christos 1821 1.1 christos /* this test is really too long, and should be made faster */ 1822 1.1 christos DISPLAYLEVEL(3, "test%3d : overflow protection with large windowLog : ", testNb++); 1823 1.1 christos { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); 1824 1.1 christos ZSTD_parameters params = ZSTD_getParams(-999, ZSTD_CONTENTSIZE_UNKNOWN, 0); 1825 1.1 christos size_t const nbCompressions = ((1U << 31) / CNBuffSize) + 2; /* ensure U32 overflow protection is triggered */ 1826 1.1 christos size_t cnb; 1827 1.1 christos assert(cctx != NULL); 1828 1.1 christos params.fParams.contentSizeFlag = 0; 1829 1.1 christos params.cParams.windowLog = ZSTD_WINDOWLOG_MAX; 1830 1.1 christos for (cnb = 0; cnb < nbCompressions; ++cnb) { 1831 1.1 christos DISPLAYLEVEL(6, "run %zu / %zu \n", cnb, nbCompressions); 1832 1.1 christos CHECK_Z( ZSTD_compressBegin_advanced(cctx, NULL, 0, params, ZSTD_CONTENTSIZE_UNKNOWN) ); /* reuse same parameters */ 1833 1.1 christos CHECK_Z( ZSTD_compressEnd(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize) ); 1834 1.1 christos } 1835 1.1 christos ZSTD_freeCCtx(cctx); 1836 1.1 christos } 1837 1.1 christos DISPLAYLEVEL(3, "OK \n"); 1838 1.1 christos 1839 1.1 christos DISPLAYLEVEL(3, "test%3d : size down context : ", testNb++); 1840 1.1 christos { ZSTD_CCtx* const largeCCtx = ZSTD_createCCtx(); 1841 1.1 christos assert(largeCCtx != NULL); 1842 1.1 christos CHECK_Z( ZSTD_compressBegin(largeCCtx, 19) ); /* streaming implies ZSTD_CONTENTSIZE_UNKNOWN, which maximizes memory usage */ 1843 1.1 christos CHECK_Z( ZSTD_compressEnd(largeCCtx, compressedBuffer, compressedBufferSize, CNBuffer, 1) ); 1844 1.1 christos { size_t const largeCCtxSize = ZSTD_sizeof_CCtx(largeCCtx); /* size of context must be measured after compression */ 1845 1.1 christos { ZSTD_CCtx* const smallCCtx = ZSTD_createCCtx(); 1846 1.1 christos assert(smallCCtx != NULL); 1847 1.1 christos CHECK_Z(ZSTD_compressCCtx(smallCCtx, compressedBuffer, compressedBufferSize, CNBuffer, 1, 1)); 1848 1.1 christos { size_t const smallCCtxSize = ZSTD_sizeof_CCtx(smallCCtx); 1849 1.1 christos DISPLAYLEVEL(5, "(large) %zuKB > 32*%zuKB (small) : ", 1850 1.1 christos largeCCtxSize>>10, smallCCtxSize>>10); 1851 1.1 christos assert(largeCCtxSize > 32* smallCCtxSize); /* note : "too large" definition is handled within zstd_compress.c . 1852 1.1 christos * make this test case extreme, so that it doesn't depend on a possibly fluctuating definition */ 1853 1.1 christos } 1854 1.1 christos ZSTD_freeCCtx(smallCCtx); 1855 1.1 christos } 1856 1.1 christos { U32 const maxNbAttempts = 1100; /* nb of usages before triggering size down is handled within zstd_compress.c. 1857 1.1 christos * currently defined as 128x, but could be adjusted in the future. 1858 1.1 christos * make this test long enough so that it's not too much tied to the current definition within zstd_compress.c */ 1859 1.1 christos unsigned u; 1860 1.1 christos for (u=0; u<maxNbAttempts; u++) { 1861 1.1 christos CHECK_Z(ZSTD_compressCCtx(largeCCtx, compressedBuffer, compressedBufferSize, CNBuffer, 1, 1)); 1862 1.1 christos if (ZSTD_sizeof_CCtx(largeCCtx) < largeCCtxSize) break; /* sized down */ 1863 1.1 christos } 1864 1.1 christos DISPLAYLEVEL(5, "size down after %u attempts : ", u); 1865 1.1 christos if (u==maxNbAttempts) goto _output_error; /* no sizedown happened */ 1866 1.1 christos } 1867 1.1 christos } 1868 1.1 christos ZSTD_freeCCtx(largeCCtx); 1869 1.1 christos } 1870 1.1 christos DISPLAYLEVEL(3, "OK \n"); 1871 1.1 christos 1872 1.1 christos /* Static CCtx tests */ 1873 1.1 christos #define STATIC_CCTX_LEVEL 4 1874 1.1 christos DISPLAYLEVEL(3, "test%3i : create static CCtx for level %u : ", testNb++, STATIC_CCTX_LEVEL); 1875 1.1 christos { size_t const staticCStreamSize = ZSTD_estimateCStreamSize(STATIC_CCTX_LEVEL); 1876 1.1 christos void* const staticCCtxBuffer = malloc(staticCStreamSize); 1877 1.1 christos size_t const staticDCtxSize = ZSTD_estimateDCtxSize(); 1878 1.1 christos void* const staticDCtxBuffer = malloc(staticDCtxSize); 1879 1.1 christos DISPLAYLEVEL(4, "CStream size = %u, ", (U32)staticCStreamSize); 1880 1.1 christos if (staticCCtxBuffer==NULL || staticDCtxBuffer==NULL) { 1881 1.1 christos free(staticCCtxBuffer); 1882 1.1 christos free(staticDCtxBuffer); 1883 1.1 christos DISPLAY("Not enough memory, aborting\n"); 1884 1.1 christos testResult = 1; 1885 1.1 christos goto _end; 1886 1.1 christos } 1887 1.1 christos { size_t const smallInSize = 32 KB; 1888 1.1 christos ZSTD_compressionParameters const cparams_small = ZSTD_getCParams(STATIC_CCTX_LEVEL, smallInSize, 0); 1889 1.1 christos size_t const smallCCtxSize = ZSTD_estimateCCtxSize_usingCParams(cparams_small); 1890 1.1 christos size_t const staticCCtxSize = ZSTD_estimateCCtxSize(STATIC_CCTX_LEVEL); 1891 1.1 christos ZSTD_CCtx* staticCCtx = ZSTD_initStaticCCtx(staticCCtxBuffer, smallCCtxSize); 1892 1.1 christos ZSTD_DCtx* const staticDCtx = ZSTD_initStaticDCtx(staticDCtxBuffer, staticDCtxSize); 1893 1.1 christos DISPLAYLEVEL(4, "Full CCtx size = %u, ", (U32)staticCCtxSize); 1894 1.1 christos DISPLAYLEVEL(4, "CCtx for 32 KB = %u, ", (U32)smallCCtxSize); 1895 1.1 christos if ((staticCCtx==NULL) || (staticDCtx==NULL)) goto _output_error; 1896 1.1 christos DISPLAYLEVEL(3, "OK \n"); 1897 1.1 christos 1898 1.1 christos DISPLAYLEVEL(3, "test%3i : compress small input with small static CCtx : ", testNb++); 1899 1.1 christos CHECK_VAR(cSize, ZSTD_compressCCtx(staticCCtx, 1900 1.1 christos compressedBuffer, compressedBufferSize, 1901 1.1 christos CNBuffer, smallInSize, STATIC_CCTX_LEVEL) ); 1902 1.1 christos DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", 1903 1.1 christos (unsigned)cSize, (double)cSize/smallInSize*100); 1904 1.1 christos 1905 1.1 christos DISPLAYLEVEL(3, "test%3i : compress large input with small static CCtx (must fail) : ", testNb++); 1906 1.1 christos { size_t const r = ZSTD_compressCCtx(staticCCtx, 1907 1.1 christos compressedBuffer, compressedBufferSize, 1908 1.1 christos CNBuffer, CNBuffSize, STATIC_CCTX_LEVEL); 1909 1.1 christos if (ZSTD_getErrorCode((size_t)r) != ZSTD_error_memory_allocation) goto _output_error; 1910 1.1 christos } 1911 1.1 christos DISPLAYLEVEL(3, "OK \n"); 1912 1.1 christos 1913 1.1 christos DISPLAYLEVEL(3, "test%3i : resize context to full CCtx size : ", testNb++); 1914 1.1 christos staticCCtx = ZSTD_initStaticCStream(staticCCtxBuffer, staticCCtxSize); 1915 1.1 christos DISPLAYLEVEL(4, "staticCCtxBuffer = %p, staticCCtx = %p , ", staticCCtxBuffer, (void*)staticCCtx); 1916 1.1 christos if (staticCCtx == NULL) goto _output_error; 1917 1.1 christos DISPLAYLEVEL(3, "OK \n"); 1918 1.1 christos 1919 1.1 christos DISPLAYLEVEL(3, "test%3i : compress large input with static CCtx : ", testNb++); 1920 1.1 christos CHECK_VAR(cSize, ZSTD_compressCCtx(staticCCtx, 1921 1.1 christos compressedBuffer, compressedBufferSize, 1922 1.1 christos CNBuffer, CNBuffSize, STATIC_CCTX_LEVEL) ); 1923 1.1 christos DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", 1924 1.1 christos (unsigned)cSize, (double)cSize/CNBuffSize*100); 1925 1.1 christos 1926 1.1 christos DISPLAYLEVEL(3, "test%3i : compress small input often enough to trigger context reduce : ", testNb++); 1927 1.1 christos { int nbc; 1928 1.1 christos assert(staticCCtxSize > smallCCtxSize * ZSTD_WORKSPACETOOLARGE_FACTOR); /* ensure size down scenario */ 1929 1.1 christos assert(CNBuffSize > smallInSize + ZSTD_WORKSPACETOOLARGE_MAXDURATION + 3); 1930 1.1 christos for (nbc=0; nbc<ZSTD_WORKSPACETOOLARGE_MAXDURATION+2; nbc++) { 1931 1.1 christos CHECK_Z(ZSTD_compressCCtx(staticCCtx, 1932 1.1 christos compressedBuffer, compressedBufferSize, 1933 1.1 christos (char*)CNBuffer + nbc, smallInSize, 1934 1.1 christos STATIC_CCTX_LEVEL) ); 1935 1.1 christos } } 1936 1.1 christos DISPLAYLEVEL(3, "OK \n") 1937 1.1 christos 1938 1.1 christos DISPLAYLEVEL(3, "test%3i : init CCtx for level %u : ", testNb++, STATIC_CCTX_LEVEL); 1939 1.1 christos CHECK_Z( ZSTD_compressBegin(staticCCtx, STATIC_CCTX_LEVEL) ); 1940 1.1 christos DISPLAYLEVEL(3, "OK \n"); 1941 1.1 christos 1942 1.1 christos DISPLAYLEVEL(3, "test%3i : compression again with static CCtx : ", testNb++); 1943 1.1 christos CHECK_VAR(cSize, ZSTD_compressCCtx(staticCCtx, 1944 1.1 christos compressedBuffer, compressedBufferSize, 1945 1.1 christos CNBuffer, CNBuffSize, STATIC_CCTX_LEVEL) ); 1946 1.1 christos DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", 1947 1.1 christos (unsigned)cSize, (double)cSize/CNBuffSize*100); 1948 1.1 christos 1949 1.1 christos DISPLAYLEVEL(3, "test%3i : simple decompression test with static DCtx : ", testNb++); 1950 1.1 christos { size_t const r = ZSTD_decompressDCtx(staticDCtx, 1951 1.1 christos decodedBuffer, CNBuffSize, 1952 1.1 christos compressedBuffer, cSize); 1953 1.1 christos if (r != CNBuffSize) goto _output_error; } 1954 1.1 christos DISPLAYLEVEL(3, "OK \n"); 1955 1.1 christos 1956 1.1 christos DISPLAYLEVEL(3, "test%3i : check decompressed result : ", testNb++); 1957 1.1 christos if (memcmp(decodedBuffer, CNBuffer, CNBuffSize)) goto _output_error; 1958 1.1 christos DISPLAYLEVEL(3, "OK \n"); 1959 1.1 christos 1960 1.1 christos DISPLAYLEVEL(3, "test%3i : init CCtx for too large level (must fail) : ", testNb++); 1961 1.1 christos { size_t const r = ZSTD_compressBegin(staticCCtx, ZSTD_maxCLevel()); 1962 1.1 christos if (!ZSTD_isError(r)) goto _output_error; } 1963 1.1 christos DISPLAYLEVEL(3, "OK \n"); 1964 1.1 christos 1965 1.1 christos DISPLAYLEVEL(3, "test%3i : init CCtx for small level %u (should work again) : ", testNb++, 1); 1966 1.1 christos CHECK_Z( ZSTD_compressBegin(staticCCtx, 1) ); 1967 1.1 christos DISPLAYLEVEL(3, "OK \n"); 1968 1.1 christos 1969 1.1 christos DISPLAYLEVEL(3, "test%3i : use CStream on CCtx-sized static context (should fail) : ", testNb++); 1970 1.1 christos CHECK_Z( ZSTD_initCStream(staticCCtx, STATIC_CCTX_LEVEL) ); /* note : doesn't allocate */ 1971 1.1 christos { ZSTD_outBuffer output = { compressedBuffer, compressedBufferSize, 0 }; 1972 1.1 christos ZSTD_inBuffer input = { CNBuffer, CNBuffSize, 0 }; 1973 1.1 christos size_t const r = ZSTD_compressStream(staticCCtx, &output, &input); /* now allocates, should fail */ 1974 1.1 christos if (!ZSTD_isError(r)) goto _output_error; 1975 1.1 christos } 1976 1.1 christos DISPLAYLEVEL(3, "OK \n"); 1977 1.1 christos 1978 1.1 christos DISPLAYLEVEL(3, "test%3i : resize context to CStream size, then stream compress : ", testNb++); 1979 1.1 christos staticCCtx = ZSTD_initStaticCStream(staticCCtxBuffer, staticCStreamSize); 1980 1.1 christos assert(staticCCtx != NULL); 1981 1.1 christos CHECK_Z( ZSTD_initCStream(staticCCtx, STATIC_CCTX_LEVEL) ); /* note : doesn't allocate */ 1982 1.1 christos { ZSTD_outBuffer output = { compressedBuffer, compressedBufferSize, 0 }; 1983 1.1 christos ZSTD_inBuffer input = { CNBuffer, CNBuffSize, 0 }; 1984 1.1 christos CHECK_Z( ZSTD_compressStream(staticCCtx, &output, &input) ); 1985 1.1 christos } 1986 1.1 christos DISPLAYLEVEL(3, "OK \n"); 1987 1.1 christos 1988 1.1 christos DISPLAYLEVEL(3, "test%3i : CStream for small level %u : ", testNb++, 1); 1989 1.1 christos CHECK_Z( ZSTD_initCStream(staticCCtx, 1) ); /* note : doesn't allocate */ 1990 1.1 christos { ZSTD_outBuffer output = { compressedBuffer, compressedBufferSize, 0 }; 1991 1.1 christos ZSTD_inBuffer input = { CNBuffer, CNBuffSize, 0 }; 1992 1.1 christos CHECK_Z( ZSTD_compressStream(staticCCtx, &output, &input) ); 1993 1.1 christos } 1994 1.1 christos DISPLAYLEVEL(3, "OK \n"); 1995 1.1 christos 1996 1.1 christos DISPLAYLEVEL(3, "test%3i : init static CStream with dictionary (should fail) : ", testNb++); 1997 1.1 christos { size_t const r = ZSTD_initCStream_usingDict(staticCCtx, CNBuffer, 64 KB, 1); 1998 1.1 christos if (!ZSTD_isError(r)) goto _output_error; } 1999 1.1 christos DISPLAYLEVEL(3, "OK \n"); 2000 1.1 christos 2001 1.1 christos DISPLAYLEVEL(3, "test%3i : use DStream on DCtx-sized static context (should fail) : ", testNb++); 2002 1.1 christos CHECK_Z( ZSTD_initDStream(staticDCtx) ); 2003 1.1 christos { ZSTD_outBuffer output = { decodedBuffer, CNBuffSize, 0 }; 2004 1.1 christos ZSTD_inBuffer input = { compressedBuffer, ZSTD_FRAMEHEADERSIZE_MAX+1, 0 }; 2005 1.1 christos size_t const r = ZSTD_decompressStream(staticDCtx, &output, &input); 2006 1.1 christos if (!ZSTD_isError(r)) goto _output_error; 2007 1.1 christos } 2008 1.1 christos DISPLAYLEVEL(3, "OK \n"); 2009 1.1 christos 2010 1.1 christos DISPLAYLEVEL(3, "test%3i : test estimation functions with default cctx params : ", testNb++); 2011 1.1 christos { 2012 1.1 christos // Test ZSTD_estimateCCtxSize_usingCCtxParams 2013 1.1 christos { 2014 1.1 christos ZSTD_CCtx_params* params = ZSTD_createCCtxParams(); 2015 1.1 christos size_t const cctxSizeDefault = ZSTD_estimateCCtxSize_usingCCtxParams(params); 2016 1.1 christos staticCCtx = ZSTD_initStaticCCtx(staticCCtxBuffer, cctxSizeDefault); 2017 1.1 christos CHECK_VAR(cSize, ZSTD_compressCCtx(staticCCtx, 2018 1.1 christos compressedBuffer, compressedBufferSize, 2019 1.1 christos CNBuffer, CNBuffSize, 3)); 2020 1.1 christos 2021 1.1 christos { 2022 1.1 christos size_t const r = ZSTD_decompressDCtx(staticDCtx, 2023 1.1 christos decodedBuffer, CNBuffSize, 2024 1.1 christos compressedBuffer, cSize); 2025 1.1 christos if (r != CNBuffSize) goto _output_error; 2026 1.1 christos if (memcmp(decodedBuffer, CNBuffer, CNBuffSize)) goto _output_error; 2027 1.1 christos } 2028 1.1 christos ZSTD_freeCCtxParams(params); 2029 1.1 christos } 2030 1.1 christos 2031 1.1 christos // Test ZSTD_estimateCStreamSize_usingCCtxParams 2032 1.1 christos { 2033 1.1 christos ZSTD_CCtx_params* params = ZSTD_createCCtxParams(); 2034 1.1 christos size_t const cctxSizeDefault = ZSTD_estimateCStreamSize_usingCCtxParams(params); 2035 1.1 christos staticCCtx = ZSTD_initStaticCCtx(staticCCtxBuffer, cctxSizeDefault); 2036 1.1 christos CHECK_VAR(cSize, ZSTD_compressCCtx(staticCCtx, 2037 1.1 christos compressedBuffer, compressedBufferSize, 2038 1.1 christos CNBuffer, CNBuffSize, 3) ); 2039 1.1 christos 2040 1.1 christos { 2041 1.1 christos size_t const r = ZSTD_decompressDCtx(staticDCtx, 2042 1.1 christos decodedBuffer, CNBuffSize, 2043 1.1 christos compressedBuffer, cSize); 2044 1.1 christos if (r != CNBuffSize) goto _output_error; 2045 1.1 christos if (memcmp(decodedBuffer, CNBuffer, CNBuffSize)) goto _output_error; 2046 1.1 christos } 2047 1.1 christos ZSTD_freeCCtxParams(params); 2048 1.1 christos } 2049 1.1 christos } 2050 1.1 christos DISPLAYLEVEL(3, "OK \n"); 2051 1.1 christos 2052 1.1 christos DISPLAYLEVEL(3, "test%3i : test estimation functions with maxBlockSize = 0 : ", testNb++); 2053 1.1 christos { 2054 1.1 christos // Test ZSTD_estimateCCtxSize_usingCCtxParams 2055 1.1 christos { 2056 1.1 christos ZSTD_CCtx_params* params = ZSTD_createCCtxParams(); 2057 1.1 christos size_t cctxSizeDefault; 2058 1.1 christos CHECK_Z(ZSTD_CCtxParams_setParameter(params, ZSTD_c_maxBlockSize, 0)); 2059 1.1 christos cctxSizeDefault = ZSTD_estimateCCtxSize_usingCCtxParams(params); 2060 1.1 christos staticCCtx = ZSTD_initStaticCCtx(staticCCtxBuffer, cctxSizeDefault); 2061 1.1 christos CHECK_VAR(cSize, ZSTD_compressCCtx(staticCCtx, 2062 1.1 christos compressedBuffer, compressedBufferSize, 2063 1.1 christos CNBuffer, CNBuffSize, 3) ); 2064 1.1 christos 2065 1.1 christos { 2066 1.1 christos size_t const r = ZSTD_decompressDCtx(staticDCtx, 2067 1.1 christos decodedBuffer, CNBuffSize, 2068 1.1 christos compressedBuffer, cSize); 2069 1.1 christos if (r != CNBuffSize) goto _output_error; 2070 1.1 christos if (memcmp(decodedBuffer, CNBuffer, CNBuffSize)) goto _output_error; 2071 1.1 christos } 2072 1.1 christos ZSTD_freeCCtxParams(params); 2073 1.1 christos } 2074 1.1 christos 2075 1.1 christos // Test ZSTD_estimateCStreamSize_usingCCtxParams 2076 1.1 christos { 2077 1.1 christos ZSTD_CCtx_params* params = ZSTD_createCCtxParams(); 2078 1.1 christos size_t cctxSizeDefault; 2079 1.1 christos CHECK_Z(ZSTD_CCtxParams_setParameter(params, ZSTD_c_maxBlockSize, 0)); 2080 1.1 christos cctxSizeDefault = ZSTD_estimateCStreamSize_usingCCtxParams(params); 2081 1.1 christos staticCCtx = ZSTD_initStaticCCtx(staticCCtxBuffer, cctxSizeDefault); 2082 1.1 christos CHECK_VAR(cSize, ZSTD_compressCCtx(staticCCtx, 2083 1.1 christos compressedBuffer, compressedBufferSize, 2084 1.1 christos CNBuffer, CNBuffSize, 3) ); 2085 1.1 christos 2086 1.1 christos { 2087 1.1 christos size_t const r = ZSTD_decompressDCtx(staticDCtx, 2088 1.1 christos decodedBuffer, CNBuffSize, 2089 1.1 christos compressedBuffer, cSize); 2090 1.1 christos if (r != CNBuffSize) goto _output_error; 2091 1.1 christos if (memcmp(decodedBuffer, CNBuffer, CNBuffSize)) goto _output_error; 2092 1.1 christos } 2093 1.1 christos ZSTD_freeCCtxParams(params); 2094 1.1 christos } 2095 1.1 christos } 2096 1.1 christos DISPLAYLEVEL(3, "OK \n"); 2097 1.1 christos } 2098 1.1 christos free(staticCCtxBuffer); 2099 1.1 christos free(staticDCtxBuffer); 2100 1.1 christos } 2101 1.1 christos 2102 1.1 christos DISPLAYLEVEL(3, "test%3i : Static context sizes for negative levels : ", testNb++); 2103 1.1 christos { size_t const cctxSizeN1 = ZSTD_estimateCCtxSize(-1); 2104 1.1 christos size_t const cctxSizeP1 = ZSTD_estimateCCtxSize(1); 2105 1.1 christos size_t const cstreamSizeN1 = ZSTD_estimateCStreamSize(-1); 2106 1.1 christos size_t const cstreamSizeP1 = ZSTD_estimateCStreamSize(1); 2107 1.1 christos 2108 1.1 christos if (!(0 < cctxSizeN1 && cctxSizeN1 <= cctxSizeP1)) goto _output_error; 2109 1.1 christos if (!(0 < cstreamSizeN1 && cstreamSizeN1 <= cstreamSizeP1)) goto _output_error; 2110 1.1 christos } 2111 1.1 christos DISPLAYLEVEL(3, "OK \n"); 2112 1.1 christos 2113 1.1 christos 2114 1.1 christos /* ZSTDMT simple MT compression test */ 2115 1.1 christos DISPLAYLEVEL(3, "test%3i : create ZSTDMT CCtx : ", testNb++); 2116 1.1 christos { ZSTD_CCtx* const mtctx = ZSTD_createCCtx(); 2117 1.1 christos if (mtctx==NULL) { 2118 1.1 christos DISPLAY("mtctx : not enough memory, aborting \n"); 2119 1.1 christos testResult = 1; 2120 1.1 christos goto _end; 2121 1.1 christos } 2122 1.1 christos CHECK_Z( ZSTD_CCtx_setParameter(mtctx, ZSTD_c_nbWorkers, 2) ); 2123 1.1 christos CHECK_Z( ZSTD_CCtx_setParameter(mtctx, ZSTD_c_compressionLevel, 1) ); 2124 1.1 christos DISPLAYLEVEL(3, "OK \n"); 2125 1.1 christos 2126 1.1 christos DISPLAYLEVEL(3, "test%3u : compress %u bytes with 2 threads : ", testNb++, (unsigned)CNBuffSize); 2127 1.1 christos CHECK_VAR(cSize, ZSTD_compress2(mtctx, 2128 1.1 christos compressedBuffer, compressedBufferSize, 2129 1.1 christos CNBuffer, CNBuffSize) ); 2130 1.1 christos DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBuffSize*100); 2131 1.1 christos 2132 1.1 christos DISPLAYLEVEL(3, "test%3i : decompressed size test : ", testNb++); 2133 1.1 christos { unsigned long long const rSize = ZSTD_getFrameContentSize(compressedBuffer, cSize); 2134 1.1 christos if (rSize != CNBuffSize) { 2135 1.1 christos DISPLAY("ZSTD_getFrameContentSize incorrect : %u != %u \n", (unsigned)rSize, (unsigned)CNBuffSize); 2136 1.1 christos goto _output_error; 2137 1.1 christos } } 2138 1.1 christos DISPLAYLEVEL(3, "OK \n"); 2139 1.1 christos 2140 1.1 christos DISPLAYLEVEL(3, "test%3i : decompress %u bytes : ", testNb++, (unsigned)CNBuffSize); 2141 1.1 christos { size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize); 2142 1.1 christos if (r != CNBuffSize) goto _output_error; } 2143 1.1 christos DISPLAYLEVEL(3, "OK \n"); 2144 1.1 christos 2145 1.1 christos DISPLAYLEVEL(3, "test%3i : check decompressed result : ", testNb++); 2146 1.1 christos { size_t u; 2147 1.1 christos for (u=0; u<CNBuffSize; u++) { 2148 1.1 christos if (((BYTE*)decodedBuffer)[u] != ((BYTE*)CNBuffer)[u]) goto _output_error; 2149 1.1 christos } } 2150 1.1 christos DISPLAYLEVEL(3, "OK \n"); 2151 1.1 christos 2152 1.1 christos DISPLAYLEVEL(3, "test%3i : compress -T2 with checksum : ", testNb++); 2153 1.1 christos CHECK_Z( ZSTD_CCtx_setParameter(mtctx, ZSTD_c_checksumFlag, 1) ); 2154 1.1 christos CHECK_Z( ZSTD_CCtx_setParameter(mtctx, ZSTD_c_contentSizeFlag, 1) ); 2155 1.1 christos CHECK_Z( ZSTD_CCtx_setParameter(mtctx, ZSTD_c_overlapLog, 3) ); 2156 1.1 christos CHECK_VAR(cSize, ZSTD_compress2(mtctx, 2157 1.1 christos compressedBuffer, compressedBufferSize, 2158 1.1 christos CNBuffer, CNBuffSize) ); 2159 1.1 christos DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBuffSize*100); 2160 1.1 christos 2161 1.1 christos DISPLAYLEVEL(3, "test%3i : decompress %u bytes : ", testNb++, (unsigned)CNBuffSize); 2162 1.1 christos { size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize); 2163 1.1 christos if (r != CNBuffSize) goto _output_error; } 2164 1.1 christos DISPLAYLEVEL(3, "OK \n"); 2165 1.1 christos 2166 1.1 christos ZSTD_freeCCtx(mtctx); 2167 1.1 christos } 2168 1.1 christos 2169 1.1 christos DISPLAYLEVEL(3, "test%3u : compress empty string and decompress with small window log : ", testNb++); 2170 1.1 christos { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); 2171 1.1 christos ZSTD_DCtx* const dctx = ZSTD_createDCtx(); 2172 1.1 christos char out[32]; 2173 1.1 christos if (cctx == NULL || dctx == NULL) goto _output_error; 2174 1.1 christos CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_contentSizeFlag, 0) ); 2175 1.1 christos CHECK_VAR(cSize, ZSTD_compress2(cctx, out, sizeof(out), NULL, 0) ); 2176 1.1 christos DISPLAYLEVEL(3, "OK (%u bytes)\n", (unsigned)cSize); 2177 1.1 christos 2178 1.1 christos CHECK_Z( ZSTD_DCtx_setParameter(dctx, ZSTD_d_windowLogMax, 10) ); 2179 1.1 christos { char const* outPtr = out; 2180 1.1 christos ZSTD_inBuffer inBuffer = { outPtr, cSize, 0 }; 2181 1.1 christos ZSTD_outBuffer outBuffer = { NULL, 0, 0 }; 2182 1.1 christos size_t dSize; 2183 1.1 christos CHECK_VAR(dSize, ZSTD_decompressStream(dctx, &outBuffer, &inBuffer) ); 2184 1.1 christos if (dSize != 0) goto _output_error; 2185 1.1 christos } 2186 1.1 christos 2187 1.1 christos ZSTD_freeDCtx(dctx); 2188 1.1 christos ZSTD_freeCCtx(cctx); 2189 1.1 christos } 2190 1.1 christos 2191 1.1 christos DISPLAYLEVEL(3, "test%3i : compress with block splitting : ", testNb++) 2192 1.1 christos { ZSTD_CCtx* cctx = ZSTD_createCCtx(); 2193 1.1 christos CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_useBlockSplitter, ZSTD_ps_enable) ); 2194 1.1 christos cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize); 2195 1.1 christos CHECK_Z(cSize); 2196 1.1 christos ZSTD_freeCCtx(cctx); 2197 1.1 christos } 2198 1.1 christos DISPLAYLEVEL(3, "OK \n"); 2199 1.1 christos 2200 1.1 christos DISPLAYLEVEL(3, "test%3i : compress -T2 with/without literals compression : ", testNb++) 2201 1.1 christos { ZSTD_CCtx* cctx = ZSTD_createCCtx(); 2202 1.1 christos size_t cSize1, cSize2; 2203 1.1 christos CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 1) ); 2204 1.1 christos CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 2) ); 2205 1.1 christos cSize1 = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize); 2206 1.1 christos CHECK_Z(cSize1); 2207 1.1 christos CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_literalCompressionMode, ZSTD_ps_disable) ); 2208 1.1 christos cSize2 = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize); 2209 1.1 christos CHECK_Z(cSize2); 2210 1.1 christos CHECK_LT(cSize1, cSize2); 2211 1.1 christos ZSTD_freeCCtx(cctx); 2212 1.1 christos } 2213 1.1 christos DISPLAYLEVEL(3, "OK \n"); 2214 1.1 christos 2215 1.1 christos DISPLAYLEVEL(3, "test%3i : Multithreaded ZSTD_compress2() with rsyncable : ", testNb++) 2216 1.1 christos { ZSTD_CCtx* cctx = ZSTD_createCCtx(); 2217 1.1 christos /* Set rsyncable and don't give the ZSTD_compressBound(CNBuffSize) so 2218 1.1 christos * ZSTDMT is forced to not take the shortcut. 2219 1.1 christos */ 2220 1.1 christos CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 1) ); 2221 1.1 christos CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 1) ); 2222 1.1 christos CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_rsyncable, 1) ); 2223 1.1 christos CHECK_Z( ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize - 1, CNBuffer, CNBuffSize) ); 2224 1.1 christos ZSTD_freeCCtx(cctx); 2225 1.1 christos } 2226 1.1 christos DISPLAYLEVEL(3, "OK \n"); 2227 1.1 christos 2228 1.1 christos DISPLAYLEVEL(3, "test%3i : setting multithreaded parameters : ", testNb++) 2229 1.1 christos { ZSTD_CCtx_params* params = ZSTD_createCCtxParams(); 2230 1.1 christos int const jobSize = 512 KB; 2231 1.1 christos int value; 2232 1.1 christos /* Check that the overlap log and job size are unset. */ 2233 1.1 christos CHECK_Z( ZSTD_CCtxParams_getParameter(params, ZSTD_c_overlapLog, &value) ); 2234 1.1 christos CHECK_EQ(value, 0); 2235 1.1 christos CHECK_Z( ZSTD_CCtxParams_getParameter(params, ZSTD_c_jobSize, &value) ); 2236 1.1 christos CHECK_EQ(value, 0); 2237 1.1 christos /* Set and check the overlap log and job size. */ 2238 1.1 christos CHECK_Z( ZSTD_CCtxParams_setParameter(params, ZSTD_c_overlapLog, 5) ); 2239 1.1 christos CHECK_Z( ZSTD_CCtxParams_setParameter(params, ZSTD_c_jobSize, jobSize) ); 2240 1.1 christos CHECK_Z( ZSTD_CCtxParams_getParameter(params, ZSTD_c_overlapLog, &value) ); 2241 1.1 christos CHECK_EQ(value, 5); 2242 1.1 christos CHECK_Z( ZSTD_CCtxParams_getParameter(params, ZSTD_c_jobSize, &value) ); 2243 1.1 christos CHECK_EQ(value, jobSize); 2244 1.1 christos /* Set the number of workers and check the overlap log and job size. */ 2245 1.1 christos CHECK_Z( ZSTD_CCtxParams_setParameter(params, ZSTD_c_nbWorkers, 2) ); 2246 1.1 christos CHECK_Z( ZSTD_CCtxParams_getParameter(params, ZSTD_c_overlapLog, &value) ); 2247 1.1 christos CHECK_EQ(value, 5); 2248 1.1 christos CHECK_Z( ZSTD_CCtxParams_getParameter(params, ZSTD_c_jobSize, &value) ); 2249 1.1 christos CHECK_EQ(value, jobSize); 2250 1.1 christos ZSTD_freeCCtxParams(params); 2251 1.1 christos } 2252 1.1 christos DISPLAYLEVEL(3, "OK \n"); 2253 1.1 christos 2254 1.1 christos /* Simple API multiframe test */ 2255 1.1 christos DISPLAYLEVEL(3, "test%3i : compress multiple frames : ", testNb++); 2256 1.1 christos { size_t off = 0; 2257 1.1 christos int i; 2258 1.1 christos int const segs = 4; 2259 1.1 christos /* only use the first half so we don't push against size limit of compressedBuffer */ 2260 1.1 christos size_t const segSize = (CNBuffSize / 2) / segs; 2261 1.1 christos 2262 1.1 christos const U32 skipLen = 129 KB; 2263 1.1 christos char* const skipBuff = (char*)malloc(skipLen); 2264 1.1 christos assert(skipBuff != NULL); 2265 1.1 christos memset(skipBuff, 0, skipLen); 2266 1.1 christos for (i = 0; i < segs; i++) { 2267 1.1 christos CHECK_NEWV(r, ZSTD_compress( 2268 1.1 christos (BYTE*)compressedBuffer + off, CNBuffSize - off, 2269 1.1 christos (BYTE*)CNBuffer + segSize * (size_t)i, segSize, 2270 1.1 christos 5) ); 2271 1.1 christos off += r; 2272 1.1 christos if (i == segs/2) { 2273 1.1 christos /* insert skippable frame */ 2274 1.1 christos size_t const skippableSize = 2275 1.1 christos ZSTD_writeSkippableFrame((BYTE*)compressedBuffer + off, compressedBufferSize, 2276 1.1 christos skipBuff, skipLen, seed % 15); 2277 1.1 christos CHECK_Z(skippableSize); 2278 1.1 christos off += skippableSize; 2279 1.1 christos } 2280 1.1 christos } 2281 1.1 christos cSize = off; 2282 1.1 christos free(skipBuff); 2283 1.1 christos } 2284 1.1 christos DISPLAYLEVEL(3, "OK \n"); 2285 1.1 christos 2286 1.1 christos DISPLAYLEVEL(3, "test%3i : get decompressed size of multiple frames : ", testNb++); 2287 1.1 christos { unsigned long long const r = ZSTD_findDecompressedSize(compressedBuffer, cSize); 2288 1.1 christos if (r != CNBuffSize / 2) goto _output_error; } 2289 1.1 christos DISPLAYLEVEL(3, "OK \n"); 2290 1.1 christos 2291 1.1 christos DISPLAYLEVEL(3, "test%3i : get tight decompressed bound of multiple frames : ", testNb++); 2292 1.1 christos { unsigned long long const bound = ZSTD_decompressBound(compressedBuffer, cSize); 2293 1.1 christos if (bound != CNBuffSize / 2) goto _output_error; } 2294 1.1 christos DISPLAYLEVEL(3, "OK \n"); 2295 1.1 christos 2296 1.1 christos DISPLAYLEVEL(3, "test%3i : decompress multiple frames : ", testNb++); 2297 1.1 christos { CHECK_NEWV(r, ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize)); 2298 1.1 christos if (r != CNBuffSize / 2) goto _output_error; } 2299 1.1 christos DISPLAYLEVEL(3, "OK \n"); 2300 1.1 christos 2301 1.1 christos DISPLAYLEVEL(3, "test%3i : check decompressed result : ", testNb++); 2302 1.1 christos if (memcmp(decodedBuffer, CNBuffer, CNBuffSize / 2) != 0) goto _output_error; 2303 1.1 christos DISPLAYLEVEL(3, "OK \n"); 2304 1.1 christos 2305 1.1 christos /* Simple API skippable frame test */ 2306 1.1 christos DISPLAYLEVEL(3, "test%3i : read/write a skippable frame : ", testNb++); 2307 1.1 christos { U32 i; 2308 1.1 christos unsigned readMagic; 2309 1.1 christos unsigned long long receivedSize; 2310 1.1 christos size_t skippableSize; 2311 1.1 christos const U32 skipLen = 129 KB; 2312 1.1 christos char* const skipBuff = (char*)malloc(skipLen); 2313 1.1 christos assert(skipBuff != NULL); 2314 1.1 christos for (i = 0; i < skipLen; i++) 2315 1.1 christos skipBuff[i] = (char) ((seed + i) % 256); 2316 1.1 christos skippableSize = ZSTD_writeSkippableFrame( 2317 1.1 christos compressedBuffer, compressedBufferSize, 2318 1.1 christos skipBuff, skipLen, seed % 15); 2319 1.1 christos CHECK_Z(skippableSize); 2320 1.1 christos CHECK_EQ(1, ZSTD_isSkippableFrame(compressedBuffer, skippableSize)); 2321 1.1 christos receivedSize = ZSTD_readSkippableFrame(decodedBuffer, CNBuffSize, &readMagic, compressedBuffer, skippableSize); 2322 1.1 christos CHECK_EQ(skippableSize, receivedSize + ZSTD_SKIPPABLEHEADERSIZE); 2323 1.1 christos CHECK_EQ(seed % 15, readMagic); 2324 1.1 christos if (memcmp(decodedBuffer, skipBuff, skipLen) != 0) goto _output_error; 2325 1.1 christos 2326 1.1 christos free(skipBuff); 2327 1.1 christos } 2328 1.1 christos DISPLAYLEVEL(3, "OK \n"); 2329 1.1 christos 2330 1.1 christos DISPLAYLEVEL(3, "test%3i : read/write an empty skippable frame : ", testNb++); 2331 1.1 christos { 2332 1.1 christos unsigned readMagic; 2333 1.1 christos unsigned long long receivedSize; 2334 1.1 christos size_t skippableSize; 2335 1.1 christos skippableSize = ZSTD_writeSkippableFrame( 2336 1.1 christos compressedBuffer, compressedBufferSize, 2337 1.1 christos CNBuffer, 0, seed % 15); 2338 1.1 christos CHECK_EQ(ZSTD_SKIPPABLEHEADERSIZE, skippableSize); 2339 1.1 christos CHECK_EQ(1, ZSTD_isSkippableFrame(compressedBuffer, skippableSize)); 2340 1.1 christos receivedSize = ZSTD_readSkippableFrame(NULL, 0, &readMagic, compressedBuffer, skippableSize); 2341 1.1 christos CHECK_EQ(skippableSize, receivedSize + ZSTD_SKIPPABLEHEADERSIZE); 2342 1.1 christos CHECK_EQ(seed % 15, readMagic); 2343 1.1 christos } 2344 1.1 christos DISPLAYLEVEL(3, "OK \n"); 2345 1.1 christos 2346 1.1 christos /* Dictionary and CCtx Duplication tests */ 2347 1.1 christos { ZSTD_CCtx* const ctxOrig = ZSTD_createCCtx(); 2348 1.1 christos ZSTD_CCtx* const ctxDuplicated = ZSTD_createCCtx(); 2349 1.1 christos ZSTD_DCtx* const dctx = ZSTD_createDCtx(); 2350 1.1 christos static const size_t dictSize = 551; 2351 1.1 christos assert(dctx != NULL); assert(ctxOrig != NULL); assert(ctxDuplicated != NULL); 2352 1.1 christos 2353 1.1 christos DISPLAYLEVEL(3, "test%3i : copy context too soon : ", testNb++); 2354 1.1 christos { size_t const copyResult = ZSTD_copyCCtx(ctxDuplicated, ctxOrig, 0); 2355 1.1 christos if (!ZSTD_isError(copyResult)) goto _output_error; } /* error must be detected */ 2356 1.1 christos DISPLAYLEVEL(3, "OK \n"); 2357 1.1 christos 2358 1.1 christos DISPLAYLEVEL(3, "test%3i : load dictionary into context : ", testNb++); 2359 1.1 christos CHECK_Z( ZSTD_compressBegin_usingDict(ctxOrig, CNBuffer, dictSize, 2) ); 2360 1.1 christos CHECK_Z( ZSTD_copyCCtx(ctxDuplicated, ctxOrig, 0) ); /* Begin_usingDict implies unknown srcSize, so match that */ 2361 1.1 christos DISPLAYLEVEL(3, "OK \n"); 2362 1.1 christos 2363 1.1 christos DISPLAYLEVEL(3, "test%3i : compress with flat dictionary : ", testNb++); 2364 1.1 christos cSize = 0; 2365 1.1 christos CHECKPLUS(r, ZSTD_compressEnd(ctxOrig, 2366 1.1 christos compressedBuffer, compressedBufferSize, 2367 1.1 christos (const char*)CNBuffer + dictSize, CNBuffSize - dictSize), 2368 1.1 christos cSize += r); 2369 1.1 christos DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBuffSize*100); 2370 1.1 christos 2371 1.1 christos DISPLAYLEVEL(3, "test%3i : frame built with flat dictionary should be decompressible : ", testNb++); 2372 1.1 christos CHECKPLUS(r, ZSTD_decompress_usingDict(dctx, 2373 1.1 christos decodedBuffer, CNBuffSize, 2374 1.1 christos compressedBuffer, cSize, 2375 1.1 christos CNBuffer, dictSize), 2376 1.1 christos if (r != CNBuffSize - dictSize) goto _output_error); 2377 1.1 christos DISPLAYLEVEL(3, "OK \n"); 2378 1.1 christos 2379 1.1 christos DISPLAYLEVEL(3, "test%3i : compress with duplicated context : ", testNb++); 2380 1.1 christos { size_t const cSizeOrig = cSize; 2381 1.1 christos cSize = 0; 2382 1.1 christos CHECKPLUS(r, ZSTD_compressEnd(ctxDuplicated, 2383 1.1 christos compressedBuffer, compressedBufferSize, 2384 1.1 christos (const char*)CNBuffer + dictSize, CNBuffSize - dictSize), 2385 1.1 christos cSize += r); 2386 1.1 christos if (cSize != cSizeOrig) goto _output_error; /* should be identical ==> same size */ 2387 1.1 christos } 2388 1.1 christos DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBuffSize*100); 2389 1.1 christos 2390 1.1 christos DISPLAYLEVEL(3, "test%3i : frame built with duplicated context should be decompressible : ", testNb++); 2391 1.1 christos CHECKPLUS(r, ZSTD_decompress_usingDict(dctx, 2392 1.1 christos decodedBuffer, CNBuffSize, 2393 1.1 christos compressedBuffer, cSize, 2394 1.1 christos CNBuffer, dictSize), 2395 1.1 christos if (r != CNBuffSize - dictSize) goto _output_error); 2396 1.1 christos DISPLAYLEVEL(3, "OK \n"); 2397 1.1 christos 2398 1.1 christos DISPLAYLEVEL(3, "test%3i : decompress with DDict : ", testNb++); 2399 1.1 christos { ZSTD_DDict* const ddict = ZSTD_createDDict(CNBuffer, dictSize); 2400 1.1 christos size_t const r = ZSTD_decompress_usingDDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, ddict); 2401 1.1 christos if (r != CNBuffSize - dictSize) goto _output_error; 2402 1.1 christos DISPLAYLEVEL(3, "OK (size of DDict : %u) \n", (unsigned)ZSTD_sizeof_DDict(ddict)); 2403 1.1 christos ZSTD_freeDDict(ddict); 2404 1.1 christos } 2405 1.1 christos 2406 1.1 christos DISPLAYLEVEL(3, "test%3i : decompress with static DDict : ", testNb++); 2407 1.1 christos { size_t const ddictBufferSize = ZSTD_estimateDDictSize(dictSize, ZSTD_dlm_byCopy); 2408 1.1 christos void* const ddictBuffer = malloc(ddictBufferSize); 2409 1.1 christos if (ddictBuffer == NULL) goto _output_error; 2410 1.1 christos { const ZSTD_DDict* const ddict = ZSTD_initStaticDDict(ddictBuffer, ddictBufferSize, CNBuffer, dictSize, ZSTD_dlm_byCopy, ZSTD_dct_auto); 2411 1.1 christos size_t const r = ZSTD_decompress_usingDDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, ddict); 2412 1.1 christos if (r != CNBuffSize - dictSize) goto _output_error; 2413 1.1 christos } 2414 1.1 christos free(ddictBuffer); 2415 1.1 christos DISPLAYLEVEL(3, "OK (size of static DDict : %u) \n", (unsigned)ddictBufferSize); 2416 1.1 christos } 2417 1.1 christos 2418 1.1 christos DISPLAYLEVEL(3, "test%3i : check content size on duplicated context : ", testNb++); 2419 1.1 christos { size_t const testSize = CNBuffSize / 3; 2420 1.1 christos CHECK_Z( ZSTD_compressBegin(ctxOrig, ZSTD_defaultCLevel()) ); 2421 1.1 christos CHECK_Z( ZSTD_copyCCtx(ctxDuplicated, ctxOrig, testSize) ); 2422 1.1 christos 2423 1.1 christos CHECK_VAR(cSize, ZSTD_compressEnd(ctxDuplicated, compressedBuffer, ZSTD_compressBound(testSize), 2424 1.1 christos (const char*)CNBuffer + dictSize, testSize) ); 2425 1.1 christos { ZSTD_frameHeader zfh; 2426 1.1 christos if (ZSTD_getFrameHeader(&zfh, compressedBuffer, cSize)) goto _output_error; 2427 1.1 christos if ((zfh.frameContentSize != testSize) && (zfh.frameContentSize != 0)) goto _output_error; 2428 1.1 christos } } 2429 1.1 christos DISPLAYLEVEL(3, "OK \n"); 2430 1.1 christos 2431 1.1 christos #if !defined(ZSTD_EXCLUDE_DFAST_BLOCK_COMPRESSOR) \ 2432 1.1 christos && !defined(ZSTD_EXCLUDE_GREEDY_BLOCK_COMPRESSOR) \ 2433 1.1 christos && !defined(ZSTD_EXCLUDE_DFAST_BLOCK_COMPRESSOR) \ 2434 1.1 christos && !defined(ZSTD_EXCLUDE_LAZY_BLOCK_COMPRESSOR) \ 2435 1.1 christos && !defined(ZSTD_EXCLUDE_LAZY2_BLOCK_COMPRESSOR) \ 2436 1.1 christos && !defined(ZSTD_EXCLUDE_BTLAZY2_BLOCK_COMPRESSOR) \ 2437 1.1 christos && !defined(ZSTD_EXCLUDE_BTOPT_BLOCK_COMPRESSOR) \ 2438 1.1 christos && !defined(ZSTD_EXCLUDE_BTULTRA_BLOCK_COMPRESSOR) 2439 1.1 christos /* Note : these tests should be replaced by proper regression tests, 2440 1.1 christos * but existing ones do not focus on small data + dictionary + all levels. 2441 1.1 christos */ 2442 1.1 christos if ((int)(compressibility * 100 + 0.1) == FUZ_compressibility_default) { /* test only valid with known input */ 2443 1.1 christos size_t const flatdictSize = 22 KB; 2444 1.1 christos size_t const contentSize = 9 KB; 2445 1.1 christos const void* const dict = (const char*)CNBuffer; 2446 1.1 christos const void* const contentStart = (const char*)dict + flatdictSize; 2447 1.1 christos /* These upper bounds are generally within a few bytes of the compressed size */ 2448 1.1 christos size_t target_nodict_cSize[22+1] = { 3840, 3770, 3870, 3830, 3770, 2449 1.1 christos 3770, 3770, 3770, 3750, 3750, 2450 1.1 christos 3742, 3675, 3674, 3665, 3664, 2451 1.1 christos 3663, 3662, 3661, 3660, 3660, 2452 1.1 christos 3660, 3660, 3660 }; 2453 1.1 christos size_t const target_wdict_cSize[22+1] = { 2830, 2896, 2893, 2820, 2940, 2454 1.1 christos 2950, 2950, 2925, 2900, 2892, 2455 1.1 christos 2910, 2910, 2910, 2780, 2775, 2456 1.1 christos 2765, 2760, 2755, 2754, 2753, 2457 1.1 christos 2753, 2753, 2753 }; 2458 1.1 christos int l = 1; 2459 1.1 christos int const maxLevel = ZSTD_maxCLevel(); 2460 1.1 christos /* clevels with strategies that support rowhash on small inputs */ 2461 1.1 christos int rowLevel = 4; 2462 1.1 christos int const rowLevelEnd = 8; 2463 1.1 christos 2464 1.1 christos DISPLAYLEVEL(3, "test%3i : flat-dictionary efficiency test : \n", testNb++); 2465 1.1 christos assert(maxLevel == 22); 2466 1.1 christos RDG_genBuffer(CNBuffer, flatdictSize + contentSize, compressibility, 0., seed); 2467 1.1 christos DISPLAYLEVEL(4, "content hash : %016llx; dict hash : %016llx \n", 2468 1.1 christos (unsigned long long)XXH64(contentStart, contentSize, 0), 2469 1.1 christos (unsigned long long)XXH64(dict, flatdictSize, 0)); 2470 1.1 christos 2471 1.1 christos for ( ; l <= maxLevel; l++) { 2472 1.1 christos size_t const nodict_cSize = ZSTD_compress(compressedBuffer, compressedBufferSize, 2473 1.1 christos contentStart, contentSize, l); 2474 1.1 christos if (nodict_cSize > target_nodict_cSize[l]) { 2475 1.1 christos DISPLAYLEVEL(1, "error : compression at level %i worse than expected (%u > %u) \n", 2476 1.1 christos l, (unsigned)nodict_cSize, (unsigned)target_nodict_cSize[l]); 2477 1.1 christos goto _output_error; 2478 1.1 christos } 2479 1.1 christos DISPLAYLEVEL(4, "level %i : max expected %u >= reached %u \n", 2480 1.1 christos l, (unsigned)target_nodict_cSize[l], (unsigned)nodict_cSize); 2481 1.1 christos } 2482 1.1 christos for ( l=1 ; l <= maxLevel; l++) { 2483 1.1 christos size_t const wdict_cSize = ZSTD_compress_usingDict(ctxOrig, 2484 1.1 christos compressedBuffer, compressedBufferSize, 2485 1.1 christos contentStart, contentSize, 2486 1.1 christos dict, flatdictSize, 2487 1.1 christos l); 2488 1.1 christos if (wdict_cSize > target_wdict_cSize[l]) { 2489 1.1 christos DISPLAYLEVEL(1, "error : compression with dictionary at level %i worse than expected (%u > %u) \n", 2490 1.1 christos l, (unsigned)wdict_cSize, (unsigned)target_wdict_cSize[l]); 2491 1.1 christos goto _output_error; 2492 1.1 christos } 2493 1.1 christos DISPLAYLEVEL(4, "level %i with dictionary : max expected %u >= reached %u \n", 2494 1.1 christos l, (unsigned)target_wdict_cSize[l], (unsigned)wdict_cSize); 2495 1.1 christos } 2496 1.1 christos /* Compression with ZSTD_compress2 and row match finder force enabled. 2497 1.1 christos * Give some slack for force-enabled row matchfinder since we're on a small input (9KB) 2498 1.1 christos */ 2499 1.1 christos for ( ; rowLevel <= rowLevelEnd; ++rowLevel) target_nodict_cSize[rowLevel] += 5; 2500 1.1 christos for (l=1 ; l <= maxLevel; l++) { 2501 1.1 christos ZSTD_CCtx* const cctx = ZSTD_createCCtx(); 2502 1.1 christos size_t nodict_cSize; 2503 1.1 christos ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, l); 2504 1.1 christos ZSTD_CCtx_setParameter(cctx, ZSTD_c_useRowMatchFinder, ZSTD_ps_enable); 2505 1.1 christos nodict_cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, 2506 1.1 christos contentStart, contentSize); 2507 1.1 christos if (nodict_cSize > target_nodict_cSize[l]) { 2508 1.1 christos DISPLAYLEVEL(1, "error : compression with compress2 at level %i worse than expected (%u > %u) \n", 2509 1.1 christos l, (unsigned)nodict_cSize, (unsigned)target_nodict_cSize[l]); 2510 1.1 christos ZSTD_freeCCtx(cctx); 2511 1.1 christos goto _output_error; 2512 1.1 christos } 2513 1.1 christos DISPLAYLEVEL(4, "level %i with compress2 : max expected %u >= reached %u \n", 2514 1.1 christos l, (unsigned)target_nodict_cSize[l], (unsigned)nodict_cSize); 2515 1.1 christos ZSTD_freeCCtx(cctx); 2516 1.1 christos } 2517 1.1 christos /* Dict compression with DMS */ 2518 1.1 christos for ( l=1 ; l <= maxLevel; l++) { 2519 1.1 christos size_t wdict_cSize; 2520 1.1 christos CHECK_Z( ZSTD_CCtx_loadDictionary(ctxOrig, dict, flatdictSize) ); 2521 1.1 christos CHECK_Z( ZSTD_CCtx_setParameter(ctxOrig, ZSTD_c_compressionLevel, l) ); 2522 1.1 christos CHECK_Z( ZSTD_CCtx_setParameter(ctxOrig, ZSTD_c_enableDedicatedDictSearch, 0) ); 2523 1.1 christos CHECK_Z( ZSTD_CCtx_setParameter(ctxOrig, ZSTD_c_forceAttachDict, ZSTD_dictForceAttach) ); 2524 1.1 christos CHECK_Z( ZSTD_CCtx_setParameter(ctxOrig, ZSTD_c_prefetchCDictTables, seed % 3) ); 2525 1.1 christos wdict_cSize = ZSTD_compress2(ctxOrig, compressedBuffer, compressedBufferSize, contentStart, contentSize); 2526 1.1 christos if (wdict_cSize > target_wdict_cSize[l]) { 2527 1.1 christos DISPLAYLEVEL(1, "error : compression with dictionary and compress2 at level %i worse than expected (%u > %u) \n", 2528 1.1 christos l, (unsigned)wdict_cSize, (unsigned)target_wdict_cSize[l]); 2529 1.1 christos goto _output_error; 2530 1.1 christos } 2531 1.1 christos DISPLAYLEVEL(4, "level %i with dictionary and compress2 : max expected %u >= reached %u \n", 2532 1.1 christos l, (unsigned)target_wdict_cSize[l], (unsigned)wdict_cSize); 2533 1.1 christos } 2534 1.1 christos 2535 1.1 christos DISPLAYLEVEL(4, "compression efficiency tests OK \n"); 2536 1.1 christos } 2537 1.1 christos #endif 2538 1.1 christos 2539 1.1 christos ZSTD_freeCCtx(ctxOrig); 2540 1.1 christos ZSTD_freeCCtx(ctxDuplicated); 2541 1.1 christos ZSTD_freeDCtx(dctx); 2542 1.1 christos } 2543 1.1 christos 2544 1.1 christos /* Dictionary and dictBuilder tests */ 2545 1.1 christos { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); 2546 1.1 christos size_t const dictBufferCapacity = 16 KB; 2547 1.1 christos void* const dictBuffer = malloc(dictBufferCapacity); 2548 1.1 christos size_t const totalSampleSize = 1 MB; 2549 1.1 christos size_t const sampleUnitSize = 8 KB; 2550 1.1 christos U32 const nbSamples = (U32)(totalSampleSize / sampleUnitSize); 2551 1.1 christos size_t* const samplesSizes = (size_t*) malloc(nbSamples * sizeof(size_t)); 2552 1.1 christos size_t dictSize; 2553 1.1 christos U32 dictID; 2554 1.1 christos size_t dictHeaderSize; 2555 1.1 christos size_t dictBufferFixedSize = 144; 2556 1.1 christos unsigned char const dictBufferFixed[144] = {0x37, 0xa4, 0x30, 0xec, 0x63, 0x00, 0x00, 0x00, 0x08, 0x10, 0x00, 0x1f, 2557 1.1 christos 0x0f, 0x00, 0x28, 0xe5, 0x03, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 2558 1.1 christos 0x00, 0x80, 0x0f, 0x9e, 0x0f, 0x00, 0x00, 0x24, 0x40, 0x80, 0x00, 0x01, 2559 1.1 christos 0x02, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0xde, 0x08, 2560 1.1 christos 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 2561 1.1 christos 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 2562 1.1 christos 0x08, 0x08, 0x08, 0x08, 0xbc, 0xe1, 0x4b, 0x92, 0x0e, 0xb4, 0x7b, 0x18, 2563 1.1 christos 0x86, 0x61, 0x18, 0xc6, 0x18, 0x63, 0x8c, 0x31, 0xc6, 0x18, 0x63, 0x8c, 2564 1.1 christos 0x31, 0x66, 0x66, 0x66, 0x66, 0xb6, 0x6d, 0x01, 0x00, 0x00, 0x00, 0x04, 2565 1.1 christos 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x20, 0x73, 0x6f, 0x64, 0x61, 2566 1.1 christos 0x6c, 0x65, 0x73, 0x20, 0x74, 0x6f, 0x72, 0x74, 0x6f, 0x72, 0x20, 0x65, 2567 1.1 christos 0x6c, 0x65, 0x69, 0x66, 0x65, 0x6e, 0x64, 0x2e, 0x20, 0x41, 0x6c, 0x69}; 2568 1.1 christos 2569 1.1 christos if (dictBuffer==NULL || samplesSizes==NULL) { 2570 1.1 christos free(dictBuffer); 2571 1.1 christos free(samplesSizes); 2572 1.1 christos goto _output_error; 2573 1.1 christos } 2574 1.1 christos 2575 1.1 christos DISPLAYLEVEL(3, "test%3i : dictBuilder on cyclic data : ", testNb++); 2576 1.1 christos assert(compressedBufferSize >= totalSampleSize); 2577 1.1 christos { U32 u; for (u=0; u<totalSampleSize; u++) ((BYTE*)decodedBuffer)[u] = (BYTE)u; } 2578 1.1 christos { U32 u; for (u=0; u<nbSamples; u++) samplesSizes[u] = sampleUnitSize; } 2579 1.1 christos { size_t const sDictSize = ZDICT_trainFromBuffer(dictBuffer, dictBufferCapacity, 2580 1.1 christos decodedBuffer, samplesSizes, nbSamples); 2581 1.1 christos if (ZDICT_isError(sDictSize)) goto _output_error; 2582 1.1 christos DISPLAYLEVEL(3, "OK, created dictionary of size %u \n", (unsigned)sDictSize); 2583 1.1 christos } 2584 1.1 christos 2585 1.1 christos DISPLAYLEVEL(3, "test%3i : dictBuilder : ", testNb++); 2586 1.1 christos { U32 u; for (u=0; u<nbSamples; u++) samplesSizes[u] = sampleUnitSize; } 2587 1.1 christos dictSize = ZDICT_trainFromBuffer(dictBuffer, dictBufferCapacity, 2588 1.1 christos CNBuffer, samplesSizes, nbSamples); 2589 1.1 christos if (ZDICT_isError(dictSize)) goto _output_error; 2590 1.1 christos DISPLAYLEVEL(3, "OK, created dictionary of size %u \n", (unsigned)dictSize); 2591 1.1 christos 2592 1.1 christos DISPLAYLEVEL(3, "test%3i : Multithreaded COVER dictBuilder : ", testNb++); 2593 1.1 christos { U32 u; for (u=0; u<nbSamples; u++) samplesSizes[u] = sampleUnitSize; } 2594 1.1 christos { ZDICT_cover_params_t coverParams; 2595 1.1 christos memset(&coverParams, 0, sizeof(coverParams)); 2596 1.1 christos coverParams.steps = 8; 2597 1.1 christos coverParams.nbThreads = 4; 2598 1.1 christos dictSize = ZDICT_optimizeTrainFromBuffer_cover( 2599 1.1 christos dictBuffer, dictBufferCapacity, 2600 1.1 christos CNBuffer, samplesSizes, nbSamples/8, /* less samples for faster tests */ 2601 1.1 christos &coverParams); 2602 1.1 christos if (ZDICT_isError(dictSize)) goto _output_error; 2603 1.1 christos } 2604 1.1 christos DISPLAYLEVEL(3, "OK, created dictionary of size %u \n", (unsigned)dictSize); 2605 1.1 christos 2606 1.1 christos DISPLAYLEVEL(3, "test%3i : COVER dictBuilder with shrinkDict: ", testNb++); 2607 1.1 christos { U32 u; for (u=0; u<nbSamples; u++) samplesSizes[u] = sampleUnitSize; } 2608 1.1 christos { ZDICT_cover_params_t coverParams; 2609 1.1 christos memset(&coverParams, 0, sizeof(coverParams)); 2610 1.1 christos coverParams.steps = 8; 2611 1.1 christos coverParams.nbThreads = 4; 2612 1.1 christos coverParams.shrinkDict = 1; 2613 1.1 christos coverParams.shrinkDictMaxRegression = 1; 2614 1.1 christos dictSize = ZDICT_optimizeTrainFromBuffer_cover( 2615 1.1 christos dictBuffer, dictBufferCapacity, 2616 1.1 christos CNBuffer, samplesSizes, nbSamples/8, /* less samples for faster tests */ 2617 1.1 christos &coverParams); 2618 1.1 christos if (ZDICT_isError(dictSize)) goto _output_error; 2619 1.1 christos } 2620 1.1 christos DISPLAYLEVEL(3, "OK, created dictionary of size %u \n", (unsigned)dictSize); 2621 1.1 christos 2622 1.1 christos DISPLAYLEVEL(3, "test%3i : Multithreaded FASTCOVER dictBuilder : ", testNb++); 2623 1.1 christos { U32 u; for (u=0; u<nbSamples; u++) samplesSizes[u] = sampleUnitSize; } 2624 1.1 christos { ZDICT_fastCover_params_t fastCoverParams; 2625 1.1 christos memset(&fastCoverParams, 0, sizeof(fastCoverParams)); 2626 1.1 christos fastCoverParams.steps = 8; 2627 1.1 christos fastCoverParams.nbThreads = 4; 2628 1.1 christos dictSize = ZDICT_optimizeTrainFromBuffer_fastCover( 2629 1.1 christos dictBuffer, dictBufferCapacity, 2630 1.1 christos CNBuffer, samplesSizes, nbSamples, 2631 1.1 christos &fastCoverParams); 2632 1.1 christos if (ZDICT_isError(dictSize)) goto _output_error; 2633 1.1 christos } 2634 1.1 christos DISPLAYLEVEL(3, "OK, created dictionary of size %u \n", (unsigned)dictSize); 2635 1.1 christos 2636 1.1 christos DISPLAYLEVEL(3, "test%3i : FASTCOVER dictBuilder with shrinkDict: ", testNb++); 2637 1.1 christos { U32 u; for (u=0; u<nbSamples; u++) samplesSizes[u] = sampleUnitSize; } 2638 1.1 christos { ZDICT_fastCover_params_t fastCoverParams; 2639 1.1 christos memset(&fastCoverParams, 0, sizeof(fastCoverParams)); 2640 1.1 christos fastCoverParams.steps = 8; 2641 1.1 christos fastCoverParams.nbThreads = 4; 2642 1.1 christos fastCoverParams.shrinkDict = 1; 2643 1.1 christos fastCoverParams.shrinkDictMaxRegression = 1; 2644 1.1 christos dictSize = ZDICT_optimizeTrainFromBuffer_fastCover( 2645 1.1 christos dictBuffer, dictBufferCapacity, 2646 1.1 christos CNBuffer, samplesSizes, nbSamples, 2647 1.1 christos &fastCoverParams); 2648 1.1 christos if (ZDICT_isError(dictSize)) goto _output_error; 2649 1.1 christos } 2650 1.1 christos DISPLAYLEVEL(3, "OK, created dictionary of size %u \n", (unsigned)dictSize); 2651 1.1 christos 2652 1.1 christos DISPLAYLEVEL(3, "test%3i : check dictID : ", testNb++); 2653 1.1 christos dictID = ZDICT_getDictID(dictBuffer, dictSize); 2654 1.1 christos if (dictID==0) goto _output_error; 2655 1.1 christos DISPLAYLEVEL(3, "OK : %u \n", (unsigned)dictID); 2656 1.1 christos 2657 1.1 christos DISPLAYLEVEL(3, "test%3i : check dict header size no error : ", testNb++); 2658 1.1 christos dictHeaderSize = ZDICT_getDictHeaderSize(dictBuffer, dictSize); 2659 1.1 christos if (dictHeaderSize==0) goto _output_error; 2660 1.1 christos DISPLAYLEVEL(3, "OK : %u \n", (unsigned)dictHeaderSize); 2661 1.1 christos 2662 1.1 christos DISPLAYLEVEL(3, "test%3i : check dict header size correctness : ", testNb++); 2663 1.1 christos { dictHeaderSize = ZDICT_getDictHeaderSize(dictBufferFixed, dictBufferFixedSize); 2664 1.1 christos if (dictHeaderSize != 115) goto _output_error; 2665 1.1 christos } 2666 1.1 christos DISPLAYLEVEL(3, "OK : %u \n", (unsigned)dictHeaderSize); 2667 1.1 christos 2668 1.1 christos DISPLAYLEVEL(3, "test%3i : compress with dictionary : ", testNb++); 2669 1.1 christos cSize = ZSTD_compress_usingDict(cctx, compressedBuffer, compressedBufferSize, 2670 1.1 christos CNBuffer, CNBuffSize, 2671 1.1 christos dictBuffer, dictSize, 4); 2672 1.1 christos if (ZSTD_isError(cSize)) goto _output_error; 2673 1.1 christos DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBuffSize*100); 2674 1.1 christos 2675 1.1 christos DISPLAYLEVEL(3, "test%3i : retrieve dictID from dictionary : ", testNb++); 2676 1.1 christos { U32 const did = ZSTD_getDictID_fromDict(dictBuffer, dictSize); 2677 1.1 christos if (did != dictID) goto _output_error; /* non-conformant (content-only) dictionary */ 2678 1.1 christos } 2679 1.1 christos DISPLAYLEVEL(3, "OK \n"); 2680 1.1 christos 2681 1.1 christos DISPLAYLEVEL(3, "test%3i : retrieve dictID from frame : ", testNb++); 2682 1.1 christos { U32 const did = ZSTD_getDictID_fromFrame(compressedBuffer, cSize); 2683 1.1 christos if (did != dictID) goto _output_error; /* non-conformant (content-only) dictionary */ 2684 1.1 christos } 2685 1.1 christos DISPLAYLEVEL(3, "OK \n"); 2686 1.1 christos 2687 1.1 christos DISPLAYLEVEL(3, "test%3i : frame built with dictionary should be decompressible : ", testNb++); 2688 1.1 christos { ZSTD_DCtx* const dctx = ZSTD_createDCtx(); assert(dctx != NULL); 2689 1.1 christos CHECKPLUS(r, ZSTD_decompress_usingDict(dctx, 2690 1.1 christos decodedBuffer, CNBuffSize, 2691 1.1 christos compressedBuffer, cSize, 2692 1.1 christos dictBuffer, dictSize), 2693 1.1 christos if (r != CNBuffSize) goto _output_error); 2694 1.1 christos ZSTD_freeDCtx(dctx); 2695 1.1 christos } 2696 1.1 christos DISPLAYLEVEL(3, "OK \n"); 2697 1.1 christos 2698 1.1 christos DISPLAYLEVEL(3, "test%3i : estimate CDict size : ", testNb++); 2699 1.1 christos { ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize); 2700 1.1 christos size_t const estimatedSize = ZSTD_estimateCDictSize_advanced(dictSize, cParams, ZSTD_dlm_byRef); 2701 1.1 christos DISPLAYLEVEL(3, "OK : %u \n", (unsigned)estimatedSize); 2702 1.1 christos } 2703 1.1 christos 2704 1.1 christos DISPLAYLEVEL(3, "test%3i : compress with CDict ", testNb++); 2705 1.1 christos { ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize); 2706 1.1 christos ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictBuffer, dictSize, 2707 1.1 christos ZSTD_dlm_byRef, ZSTD_dct_auto, 2708 1.1 christos cParams, ZSTD_defaultCMem); 2709 1.1 christos assert(cdict != NULL); 2710 1.1 christos DISPLAYLEVEL(3, "(size : %u) : ", (unsigned)ZSTD_sizeof_CDict(cdict)); 2711 1.1 christos assert(ZSTD_getDictID_fromDict(dictBuffer, dictSize) == ZSTD_getDictID_fromCDict(cdict)); 2712 1.1 christos cSize = ZSTD_compress_usingCDict(cctx, compressedBuffer, compressedBufferSize, 2713 1.1 christos CNBuffer, CNBuffSize, cdict); 2714 1.1 christos ZSTD_freeCDict(cdict); 2715 1.1 christos if (ZSTD_isError(cSize)) goto _output_error; 2716 1.1 christos } 2717 1.1 christos DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBuffSize*100); 2718 1.1 christos 2719 1.1 christos DISPLAYLEVEL(3, "test%3i : retrieve dictID from frame : ", testNb++); 2720 1.1 christos { U32 const did = ZSTD_getDictID_fromFrame(compressedBuffer, cSize); 2721 1.1 christos if (did != dictID) goto _output_error; /* non-conformant (content-only) dictionary */ 2722 1.1 christos } 2723 1.1 christos DISPLAYLEVEL(3, "OK \n"); 2724 1.1 christos 2725 1.1 christos DISPLAYLEVEL(3, "test%3i : frame built with dictionary should be decompressible : ", testNb++); 2726 1.1 christos { ZSTD_DCtx* const dctx = ZSTD_createDCtx(); assert(dctx != NULL); 2727 1.1 christos CHECKPLUS(r, ZSTD_decompress_usingDict(dctx, 2728 1.1 christos decodedBuffer, CNBuffSize, 2729 1.1 christos compressedBuffer, cSize, 2730 1.1 christos dictBuffer, dictSize), 2731 1.1 christos if (r != CNBuffSize) goto _output_error); 2732 1.1 christos ZSTD_freeDCtx(dctx); 2733 1.1 christos } 2734 1.1 christos DISPLAYLEVEL(3, "OK \n"); 2735 1.1 christos 2736 1.1 christos DISPLAYLEVEL(3, "test%3i : compress with static CDict : ", testNb++); 2737 1.1 christos { int const maxLevel = ZSTD_maxCLevel(); 2738 1.1 christos int level; 2739 1.1 christos for (level = 1; level <= maxLevel; ++level) { 2740 1.1 christos ZSTD_compressionParameters const cParams = ZSTD_getCParams(level, CNBuffSize, dictSize); 2741 1.1 christos size_t const cdictSize = ZSTD_estimateCDictSize_advanced(dictSize, cParams, ZSTD_dlm_byCopy); 2742 1.1 christos void* const cdictBuffer = malloc(cdictSize); 2743 1.1 christos if (cdictBuffer==NULL) goto _output_error; 2744 1.1 christos { const ZSTD_CDict* const cdict = ZSTD_initStaticCDict( 2745 1.1 christos cdictBuffer, cdictSize, 2746 1.1 christos dictBuffer, dictSize, 2747 1.1 christos ZSTD_dlm_byCopy, ZSTD_dct_auto, 2748 1.1 christos cParams); 2749 1.1 christos if (cdict == NULL) { 2750 1.1 christos DISPLAY("ZSTD_initStaticCDict failed "); 2751 1.1 christos goto _output_error; 2752 1.1 christos } 2753 1.1 christos cSize = ZSTD_compress_usingCDict(cctx, 2754 1.1 christos compressedBuffer, compressedBufferSize, 2755 1.1 christos CNBuffer, MIN(10 KB, CNBuffSize), cdict); 2756 1.1 christos if (ZSTD_isError(cSize)) { 2757 1.1 christos DISPLAY("ZSTD_compress_usingCDict failed "); 2758 1.1 christos goto _output_error; 2759 1.1 christos } } 2760 1.1 christos free(cdictBuffer); 2761 1.1 christos } } 2762 1.1 christos DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBuffSize*100); 2763 1.1 christos 2764 1.1 christos DISPLAYLEVEL(3, "test%3i : ZSTD_compress_usingCDict_advanced, no contentSize, no dictID : ", testNb++); 2765 1.1 christos { ZSTD_frameParameters const fParams = { 0 /* frameSize */, 1 /* checksum */, 1 /* noDictID*/ }; 2766 1.1 christos ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize); 2767 1.1 christos ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dct_auto, cParams, ZSTD_defaultCMem); 2768 1.1 christos assert(cdict != NULL); 2769 1.1 christos cSize = ZSTD_compress_usingCDict_advanced(cctx, 2770 1.1 christos compressedBuffer, compressedBufferSize, 2771 1.1 christos CNBuffer, CNBuffSize, 2772 1.1 christos cdict, fParams); 2773 1.1 christos ZSTD_freeCDict(cdict); 2774 1.1 christos if (ZSTD_isError(cSize)) goto _output_error; 2775 1.1 christos } 2776 1.1 christos DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBuffSize*100); 2777 1.1 christos 2778 1.1 christos DISPLAYLEVEL(3, "test%3i : try retrieving contentSize from frame : ", testNb++); 2779 1.1 christos { U64 const contentSize = ZSTD_getFrameContentSize(compressedBuffer, cSize); 2780 1.1 christos if (contentSize != ZSTD_CONTENTSIZE_UNKNOWN) goto _output_error; 2781 1.1 christos } 2782 1.1 christos DISPLAYLEVEL(3, "OK (unknown)\n"); 2783 1.1 christos 2784 1.1 christos DISPLAYLEVEL(3, "test%3i : frame built without dictID should be decompressible : ", testNb++); 2785 1.1 christos { ZSTD_DCtx* const dctx = ZSTD_createDCtx(); 2786 1.1 christos assert(dctx != NULL); 2787 1.1 christos CHECKPLUS(r, ZSTD_decompress_usingDict(dctx, 2788 1.1 christos decodedBuffer, CNBuffSize, 2789 1.1 christos compressedBuffer, cSize, 2790 1.1 christos dictBuffer, dictSize), 2791 1.1 christos if (r != CNBuffSize) goto _output_error); 2792 1.1 christos ZSTD_freeDCtx(dctx); 2793 1.1 christos } 2794 1.1 christos DISPLAYLEVEL(3, "OK \n"); 2795 1.1 christos 2796 1.1 christos DISPLAYLEVEL(3, "test%3i : ZSTD_compress_advanced, no dictID : ", testNb++); 2797 1.1 christos { ZSTD_parameters p = ZSTD_getParams(3, CNBuffSize, dictSize); 2798 1.1 christos p.fParams.noDictIDFlag = 1; 2799 1.1 christos cSize = ZSTD_compress_advanced(cctx, compressedBuffer, compressedBufferSize, 2800 1.1 christos CNBuffer, CNBuffSize, 2801 1.1 christos dictBuffer, dictSize, p); 2802 1.1 christos if (ZSTD_isError(cSize)) goto _output_error; 2803 1.1 christos } 2804 1.1 christos DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBuffSize*100); 2805 1.1 christos 2806 1.1 christos DISPLAYLEVEL(3, "test%3i : frame built without dictID should be decompressible : ", testNb++); 2807 1.1 christos { ZSTD_DCtx* const dctx = ZSTD_createDCtx(); assert(dctx != NULL); 2808 1.1 christos CHECKPLUS(r, ZSTD_decompress_usingDict(dctx, 2809 1.1 christos decodedBuffer, CNBuffSize, 2810 1.1 christos compressedBuffer, cSize, 2811 1.1 christos dictBuffer, dictSize), 2812 1.1 christos if (r != CNBuffSize) goto _output_error); 2813 1.1 christos ZSTD_freeDCtx(dctx); 2814 1.1 christos } 2815 1.1 christos DISPLAYLEVEL(3, "OK \n"); 2816 1.1 christos 2817 1.1 christos DISPLAYLEVEL(3, "test%3i : dictionary containing only header should return error : ", testNb++); 2818 1.1 christos { ZSTD_DCtx* const dctx = ZSTD_createDCtx(); 2819 1.1 christos assert(dctx != NULL); 2820 1.1 christos { const size_t ret = ZSTD_decompress_usingDict( 2821 1.1 christos dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, 2822 1.1 christos "\x37\xa4\x30\xec\x11\x22\x33\x44", 8); 2823 1.1 christos if (ZSTD_getErrorCode(ret) != ZSTD_error_dictionary_corrupted) 2824 1.1 christos goto _output_error; 2825 1.1 christos } 2826 1.1 christos ZSTD_freeDCtx(dctx); 2827 1.1 christos } 2828 1.1 christos DISPLAYLEVEL(3, "OK \n"); 2829 1.1 christos 2830 1.1 christos DISPLAYLEVEL(3, "test%3d : bufferless api with cdict : ", testNb++); 2831 1.1 christos { ZSTD_CDict* const cdict = ZSTD_createCDict(dictBuffer, dictSize, 1); 2832 1.1 christos ZSTD_DCtx* const dctx = ZSTD_createDCtx(); 2833 1.1 christos ZSTD_frameParameters const fParams = { 0, 1, 0 }; 2834 1.1 christos size_t cBlockSize; 2835 1.1 christos cSize = 0; 2836 1.1 christos CHECK_Z(ZSTD_compressBegin_usingCDict_advanced(cctx, cdict, fParams, ZSTD_CONTENTSIZE_UNKNOWN)); 2837 1.1 christos cBlockSize = ZSTD_compressContinue(cctx, (char*)compressedBuffer + cSize, compressedBufferSize - cSize, CNBuffer, 1000); 2838 1.1 christos CHECK_Z(cBlockSize); 2839 1.1 christos cSize += cBlockSize; 2840 1.1 christos cBlockSize = ZSTD_compressEnd(cctx, (char*)compressedBuffer + cSize, compressedBufferSize - cSize, (char const*)CNBuffer + 2000, 1000); 2841 1.1 christos CHECK_Z(cBlockSize); 2842 1.1 christos cSize += cBlockSize; 2843 1.1 christos 2844 1.1 christos CHECK_Z(ZSTD_decompress_usingDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, dictBuffer, dictSize)); 2845 1.1 christos 2846 1.1 christos ZSTD_freeCDict(cdict); 2847 1.1 christos ZSTD_freeDCtx(dctx); 2848 1.1 christos } 2849 1.1 christos DISPLAYLEVEL(3, "OK \n"); 2850 1.1 christos 2851 1.1 christos DISPLAYLEVEL(3, "test%3i : Building cdict w/ ZSTD_dct_fullDict on a good dictionary : ", testNb++); 2852 1.1 christos { ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize); 2853 1.1 christos ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dct_fullDict, cParams, ZSTD_defaultCMem); 2854 1.1 christos if (cdict==NULL) goto _output_error; 2855 1.1 christos ZSTD_freeCDict(cdict); 2856 1.1 christos } 2857 1.1 christos DISPLAYLEVEL(3, "OK \n"); 2858 1.1 christos 2859 1.1 christos DISPLAYLEVEL(3, "test%3i : Building cdict w/ ZSTD_dct_fullDict on a rawContent (must fail) : ", testNb++); 2860 1.1 christos { ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize); 2861 1.1 christos ZSTD_CDict* const cdict = ZSTD_createCDict_advanced((const char*)dictBuffer+1, dictSize-1, ZSTD_dlm_byRef, ZSTD_dct_fullDict, cParams, ZSTD_defaultCMem); 2862 1.1 christos if (cdict!=NULL) goto _output_error; 2863 1.1 christos ZSTD_freeCDict(cdict); 2864 1.1 christos } 2865 1.1 christos DISPLAYLEVEL(3, "OK \n"); 2866 1.1 christos 2867 1.1 christos { char* rawDictBuffer = (char*)malloc(dictSize); 2868 1.1 christos assert(rawDictBuffer); 2869 1.1 christos memcpy(rawDictBuffer, (char*)dictBuffer + 2, dictSize - 2); 2870 1.1 christos memset(rawDictBuffer + dictSize - 2, 0, 2); 2871 1.1 christos MEM_writeLE32((char*)rawDictBuffer, ZSTD_MAGIC_DICTIONARY); 2872 1.1 christos 2873 1.1 christos DISPLAYLEVEL(3, "test%3i : Loading rawContent starting with dict header w/ ZSTD_dct_auto should fail : ", testNb++); 2874 1.1 christos { 2875 1.1 christos size_t ret; 2876 1.1 christos /* Either operation is allowed to fail, but one must fail. */ 2877 1.1 christos ret = ZSTD_CCtx_loadDictionary_advanced( 2878 1.1 christos cctx, (const char*)rawDictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dct_auto); 2879 1.1 christos if (!ZSTD_isError(ret)) { 2880 1.1 christos ret = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100)); 2881 1.1 christos if (!ZSTD_isError(ret)) goto _output_error; 2882 1.1 christos } 2883 1.1 christos } 2884 1.1 christos DISPLAYLEVEL(3, "OK \n"); 2885 1.1 christos 2886 1.1 christos DISPLAYLEVEL(3, "test%3i : Loading rawContent starting with dict header w/ ZSTD_dct_rawContent should pass : ", testNb++); 2887 1.1 christos { 2888 1.1 christos size_t ret; 2889 1.1 christos ret = ZSTD_CCtx_loadDictionary_advanced( 2890 1.1 christos cctx, (const char*)rawDictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dct_rawContent); 2891 1.1 christos if (ZSTD_isError(ret)) goto _output_error; 2892 1.1 christos ret = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100)); 2893 1.1 christos if (ZSTD_isError(ret)) goto _output_error; 2894 1.1 christos } 2895 1.1 christos DISPLAYLEVEL(3, "OK \n"); 2896 1.1 christos 2897 1.1 christos DISPLAYLEVEL(3, "test%3i : Testing non-attached CDict with ZSTD_dct_rawContent : ", testNb++); 2898 1.1 christos { size_t const srcSize = MIN(CNBuffSize, 100); 2899 1.1 christos ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters); 2900 1.1 christos /* Force the dictionary to be reloaded in raw content mode */ 2901 1.1 christos CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_forceAttachDict, ZSTD_dictForceLoad)); 2902 1.1 christos CHECK_Z(ZSTD_CCtx_loadDictionary_advanced(cctx, rawDictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dct_rawContent)); 2903 1.1 christos cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, srcSize); 2904 1.1 christos CHECK_Z(cSize); 2905 1.1 christos } 2906 1.1 christos DISPLAYLEVEL(3, "OK \n"); 2907 1.1 christos 2908 1.1 christos free(rawDictBuffer); 2909 1.1 christos } 2910 1.1 christos 2911 1.1 christos DISPLAYLEVEL(3, "test%3i : ZSTD_CCtx_refCDict() then set parameters : ", testNb++); 2912 1.1 christos { ZSTD_CDict* const cdict = ZSTD_createCDict(CNBuffer, dictSize, 1); 2913 1.1 christos ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters); 2914 1.1 christos CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 1) ); 2915 1.1 christos CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_hashLog, 12 )); 2916 1.1 christos CHECK_Z( ZSTD_CCtx_refCDict(cctx, cdict) ); 2917 1.1 christos CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 1) ); 2918 1.1 christos CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_hashLog, 12 )); 2919 1.1 christos ZSTD_freeCDict(cdict); 2920 1.1 christos } 2921 1.1 christos DISPLAYLEVEL(3, "OK \n"); 2922 1.1 christos 2923 1.1 christos DISPLAYLEVEL(3, "test%3i : Loading dictionary before setting parameters is the same as loading after : ", testNb++); 2924 1.1 christos { 2925 1.1 christos size_t size1, size2; 2926 1.1 christos ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters); 2927 1.1 christos CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 7) ); 2928 1.1 christos CHECK_Z( ZSTD_CCtx_loadDictionary(cctx, CNBuffer, MIN(CNBuffSize, 10 KB)) ); 2929 1.1 christos size1 = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100 KB)); 2930 1.1 christos if (ZSTD_isError(size1)) goto _output_error; 2931 1.1 christos 2932 1.1 christos ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters); 2933 1.1 christos CHECK_Z( ZSTD_CCtx_loadDictionary(cctx, CNBuffer, MIN(CNBuffSize, 10 KB)) ); 2934 1.1 christos CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 7) ); 2935 1.1 christos size2 = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100 KB)); 2936 1.1 christos if (ZSTD_isError(size2)) goto _output_error; 2937 1.1 christos 2938 1.1 christos if (size1 != size2) goto _output_error; 2939 1.1 christos } 2940 1.1 christos DISPLAYLEVEL(3, "OK \n"); 2941 1.1 christos 2942 1.1 christos DISPLAYLEVEL(3, "test%3i : Loading a dictionary clears the prefix : ", testNb++); 2943 1.1 christos { 2944 1.1 christos CHECK_Z( ZSTD_CCtx_refPrefix(cctx, (const char*)dictBuffer, dictSize) ); 2945 1.1 christos CHECK_Z( ZSTD_CCtx_loadDictionary(cctx, (const char*)dictBuffer, dictSize) ); 2946 1.1 christos CHECK_Z( ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100)) ); 2947 1.1 christos } 2948 1.1 christos DISPLAYLEVEL(3, "OK \n"); 2949 1.1 christos 2950 1.1 christos DISPLAYLEVEL(3, "test%3i : Loading a dictionary clears the cdict : ", testNb++); 2951 1.1 christos { 2952 1.1 christos ZSTD_CDict* const cdict = ZSTD_createCDict(dictBuffer, dictSize, 1); 2953 1.1 christos CHECK_Z( ZSTD_CCtx_refCDict(cctx, cdict) ); 2954 1.1 christos CHECK_Z( ZSTD_CCtx_loadDictionary(cctx, (const char*)dictBuffer, dictSize) ); 2955 1.1 christos CHECK_Z( ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100)) ); 2956 1.1 christos ZSTD_freeCDict(cdict); 2957 1.1 christos } 2958 1.1 christos DISPLAYLEVEL(3, "OK \n"); 2959 1.1 christos 2960 1.1 christos DISPLAYLEVEL(3, "test%3i : Loading a cdict clears the prefix : ", testNb++); 2961 1.1 christos { 2962 1.1 christos ZSTD_CDict* const cdict = ZSTD_createCDict(dictBuffer, dictSize, 1); 2963 1.1 christos CHECK_Z( ZSTD_CCtx_refPrefix(cctx, (const char*)dictBuffer, dictSize) ); 2964 1.1 christos CHECK_Z( ZSTD_CCtx_refCDict(cctx, cdict) ); 2965 1.1 christos CHECK_Z( ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100)) ); 2966 1.1 christos ZSTD_freeCDict(cdict); 2967 1.1 christos } 2968 1.1 christos DISPLAYLEVEL(3, "OK \n"); 2969 1.1 christos 2970 1.1 christos DISPLAYLEVEL(3, "test%3i : Loading a cdict clears the dictionary : ", testNb++); 2971 1.1 christos { 2972 1.1 christos ZSTD_CDict* const cdict = ZSTD_createCDict(dictBuffer, dictSize, 1); 2973 1.1 christos CHECK_Z( ZSTD_CCtx_loadDictionary(cctx, (const char*)dictBuffer, dictSize) ); 2974 1.1 christos CHECK_Z( ZSTD_CCtx_refCDict(cctx, cdict) ); 2975 1.1 christos CHECK_Z( ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100)) ); 2976 1.1 christos ZSTD_freeCDict(cdict); 2977 1.1 christos } 2978 1.1 christos DISPLAYLEVEL(3, "OK \n"); 2979 1.1 christos 2980 1.1 christos DISPLAYLEVEL(3, "test%3i : Loading a prefix clears the dictionary : ", testNb++); 2981 1.1 christos { 2982 1.1 christos CHECK_Z( ZSTD_CCtx_loadDictionary(cctx, (const char*)dictBuffer, dictSize) ); 2983 1.1 christos CHECK_Z( ZSTD_CCtx_refPrefix(cctx, (const char*)dictBuffer, dictSize) ); 2984 1.1 christos CHECK_Z( ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100)) ); 2985 1.1 christos } 2986 1.1 christos DISPLAYLEVEL(3, "OK \n"); 2987 1.1 christos 2988 1.1 christos DISPLAYLEVEL(3, "test%3i : Loading a prefix clears the cdict : ", testNb++); 2989 1.1 christos { 2990 1.1 christos ZSTD_CDict* const cdict = ZSTD_createCDict(dictBuffer, dictSize, 1); 2991 1.1 christos CHECK_Z( ZSTD_CCtx_refCDict(cctx, cdict) ); 2992 1.1 christos CHECK_Z( ZSTD_CCtx_refPrefix(cctx, (const char*)dictBuffer, dictSize) ); 2993 1.1 christos CHECK_Z( ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100)) ); 2994 1.1 christos ZSTD_freeCDict(cdict); 2995 1.1 christos } 2996 1.1 christos DISPLAYLEVEL(3, "OK \n"); 2997 1.1 christos 2998 1.1 christos DISPLAYLEVEL(3, "test%3i : Loaded dictionary persists across reset session : ", testNb++); 2999 1.1 christos { 3000 1.1 christos size_t size1, size2; 3001 1.1 christos ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters); 3002 1.1 christos CHECK_Z( ZSTD_CCtx_loadDictionary(cctx, CNBuffer, MIN(CNBuffSize, 10 KB)) ); 3003 1.1 christos size1 = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100 KB)); 3004 1.1 christos if (ZSTD_isError(size1)) goto _output_error; 3005 1.1 christos 3006 1.1 christos ZSTD_CCtx_reset(cctx, ZSTD_reset_session_only); 3007 1.1 christos size2 = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100 KB)); 3008 1.1 christos if (ZSTD_isError(size2)) goto _output_error; 3009 1.1 christos 3010 1.1 christos if (size1 != size2) goto _output_error; 3011 1.1 christos } 3012 1.1 christos DISPLAYLEVEL(3, "OK \n"); 3013 1.1 christos 3014 1.1 christos DISPLAYLEVEL(3, "test%3i : Loaded dictionary is cleared after resetting parameters : ", testNb++); 3015 1.1 christos { 3016 1.1 christos size_t size1, size2; 3017 1.1 christos ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters); 3018 1.1 christos CHECK_Z( ZSTD_CCtx_loadDictionary(cctx, CNBuffer, MIN(CNBuffSize, 10 KB)) ); 3019 1.1 christos size1 = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100 KB)); 3020 1.1 christos if (ZSTD_isError(size1)) goto _output_error; 3021 1.1 christos 3022 1.1 christos ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters); 3023 1.1 christos size2 = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100 KB)); 3024 1.1 christos if (ZSTD_isError(size2)) goto _output_error; 3025 1.1 christos 3026 1.1 christos if (size1 == size2) goto _output_error; 3027 1.1 christos } 3028 1.1 christos DISPLAYLEVEL(3, "OK \n"); 3029 1.1 christos 3030 1.1 christos ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters); 3031 1.1 christos CHECK_Z( ZSTD_CCtx_loadDictionary(cctx, dictBuffer, dictSize) ); 3032 1.1 christos cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100 KB)); 3033 1.1 christos CHECK_Z(cSize); 3034 1.1 christos DISPLAYLEVEL(3, "test%3i : ZSTD_decompressDCtx() with dictionary : ", testNb++); 3035 1.1 christos { 3036 1.1 christos ZSTD_DCtx* dctx = ZSTD_createDCtx(); 3037 1.1 christos size_t ret; 3038 1.1 christos /* We should fail to decompress without a dictionary. */ 3039 1.1 christos ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters); 3040 1.1 christos ret = ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize); 3041 1.1 christos if (!ZSTD_isError(ret)) goto _output_error; 3042 1.1 christos /* We should succeed to decompress with the dictionary. */ 3043 1.1 christos ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters); 3044 1.1 christos CHECK_Z( ZSTD_DCtx_loadDictionary(dctx, dictBuffer, dictSize) ); 3045 1.1 christos CHECK_Z( ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize) ); 3046 1.1 christos /* The dictionary should persist across calls. */ 3047 1.1 christos CHECK_Z( ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize) ); 3048 1.1 christos /* When we reset the context the dictionary is cleared. */ 3049 1.1 christos ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters); 3050 1.1 christos ret = ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize); 3051 1.1 christos if (!ZSTD_isError(ret)) goto _output_error; 3052 1.1 christos ZSTD_freeDCtx(dctx); 3053 1.1 christos } 3054 1.1 christos DISPLAYLEVEL(3, "OK \n"); 3055 1.1 christos 3056 1.1 christos DISPLAYLEVEL(3, "test%3i : ZSTD_decompressDCtx() with ddict : ", testNb++); 3057 1.1 christos { 3058 1.1 christos ZSTD_DCtx* dctx = ZSTD_createDCtx(); 3059 1.1 christos ZSTD_DDict* ddict = ZSTD_createDDict(dictBuffer, dictSize); 3060 1.1 christos size_t ret; 3061 1.1 christos /* We should succeed to decompress with the ddict. */ 3062 1.1 christos ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters); 3063 1.1 christos CHECK_Z( ZSTD_DCtx_refDDict(dctx, ddict) ); 3064 1.1 christos CHECK_Z( ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize) ); 3065 1.1 christos /* The ddict should persist across calls. */ 3066 1.1 christos CHECK_Z( ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize) ); 3067 1.1 christos /* When we reset the context the ddict is cleared. */ 3068 1.1 christos ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters); 3069 1.1 christos ret = ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize); 3070 1.1 christos if (!ZSTD_isError(ret)) goto _output_error; 3071 1.1 christos ZSTD_freeDCtx(dctx); 3072 1.1 christos ZSTD_freeDDict(ddict); 3073 1.1 christos } 3074 1.1 christos DISPLAYLEVEL(3, "OK \n"); 3075 1.1 christos 3076 1.1 christos DISPLAYLEVEL(3, "test%3i : ZSTD_decompressDCtx() with prefix : ", testNb++); 3077 1.1 christos { 3078 1.1 christos ZSTD_DCtx* dctx = ZSTD_createDCtx(); 3079 1.1 christos size_t ret; 3080 1.1 christos /* We should succeed to decompress with the prefix. */ 3081 1.1 christos ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters); 3082 1.1 christos CHECK_Z( ZSTD_DCtx_refPrefix_advanced(dctx, dictBuffer, dictSize, ZSTD_dct_auto) ); 3083 1.1 christos CHECK_Z( ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize) ); 3084 1.1 christos /* The prefix should be cleared after the first compression. */ 3085 1.1 christos ret = ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize); 3086 1.1 christos if (!ZSTD_isError(ret)) goto _output_error; 3087 1.1 christos ZSTD_freeDCtx(dctx); 3088 1.1 christos } 3089 1.1 christos DISPLAYLEVEL(3, "OK \n"); 3090 1.1 christos 3091 1.1 christos DISPLAYLEVEL(3, "test%3i : ZSTD_fast attach dictionary with hashLog = 25 and chainLog = 25 : ", testNb++); 3092 1.1 christos { 3093 1.1 christos ZSTD_CCtx_params* cctxParams = ZSTD_createCCtxParams(); 3094 1.1 christos ZSTD_customMem customMem = {NULL, NULL, NULL}; 3095 1.1 christos ZSTD_DCtx* dctx = ZSTD_createDCtx(); 3096 1.1 christos ZSTD_CDict* cdict; 3097 1.1 christos CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_strategy, ZSTD_fast)); 3098 1.1 christos /* Set windowLog to 25 so hash/chain logs don't get sized down */ 3099 1.1 christos CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_windowLog, 25)); 3100 1.1 christos CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_hashLog, 25)); 3101 1.1 christos CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_chainLog, 25)); 3102 1.1 christos /* Set srcSizeHint to 2^25 so hash/chain logs don't get sized down */ 3103 1.1 christos CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_srcSizeHint, 1u << 25)); 3104 1.1 christos cdict = ZSTD_createCDict_advanced2(dictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dct_auto, cctxParams, customMem); 3105 1.1 christos CHECK_Z(ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters)); 3106 1.1 christos CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_forceAttachDict, ZSTD_dictForceAttach)); 3107 1.1 christos CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1)); 3108 1.1 christos CHECK_Z(ZSTD_CCtx_refCDict(cctx, cdict)); 3109 1.1 christos cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize); 3110 1.1 christos CHECK_Z(cSize); 3111 1.1 christos CHECK_Z(ZSTD_decompress_usingDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, dictBuffer, dictSize)); 3112 1.1 christos ZSTD_freeCDict(cdict); 3113 1.1 christos ZSTD_freeDCtx(dctx); 3114 1.1 christos ZSTD_freeCCtxParams(cctxParams); 3115 1.1 christos } 3116 1.1 christos DISPLAYLEVEL(3, "OK \n"); 3117 1.1 christos 3118 1.1 christos DISPLAYLEVEL(3, "test%3i : ZSTD_dfast attach dictionary with hashLog = 25 and chainLog = 25 : ", testNb++); 3119 1.1 christos { 3120 1.1 christos ZSTD_CCtx_params* cctxParams = ZSTD_createCCtxParams(); 3121 1.1 christos ZSTD_customMem customMem = {NULL, NULL, NULL}; 3122 1.1 christos ZSTD_DCtx* dctx = ZSTD_createDCtx(); 3123 1.1 christos ZSTD_CDict* cdict; 3124 1.1 christos CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_strategy, ZSTD_dfast)); 3125 1.1 christos /* Set windowLog to 25 so hash/chain logs don't get sized down */ 3126 1.1 christos CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_windowLog, 25)); 3127 1.1 christos CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_hashLog, 25)); 3128 1.1 christos CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_chainLog, 25)); 3129 1.1 christos /* Set srcSizeHint to 2^25 so hash/chain logs don't get sized down */ 3130 1.1 christos CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_srcSizeHint, 1u << 25)); 3131 1.1 christos cdict = ZSTD_createCDict_advanced2(dictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dct_auto, cctxParams, customMem); 3132 1.1 christos CHECK_Z(ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters)); 3133 1.1 christos CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_forceAttachDict, ZSTD_dictForceAttach)); 3134 1.1 christos CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1)); 3135 1.1 christos CHECK_Z(ZSTD_CCtx_refCDict(cctx, cdict)); 3136 1.1 christos cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize); 3137 1.1 christos CHECK_Z(cSize); 3138 1.1 christos CHECK_Z(ZSTD_decompress_usingDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, dictBuffer, dictSize)); 3139 1.1 christos ZSTD_freeCDict(cdict); 3140 1.1 christos ZSTD_freeDCtx(dctx); 3141 1.1 christos ZSTD_freeCCtxParams(cctxParams); 3142 1.1 christos } 3143 1.1 christos DISPLAYLEVEL(3, "OK \n"); 3144 1.1 christos 3145 1.1 christos DISPLAYLEVEL(3, "test%3i : ZSTD_lazy attach dictionary with hashLog = 29 and searchLog = 4 : ", testNb++); 3146 1.1 christos if (MEM_64bits()) { 3147 1.1 christos ZSTD_CCtx_params* cctxParams = ZSTD_createCCtxParams(); 3148 1.1 christos ZSTD_customMem customMem = {NULL, NULL, NULL}; 3149 1.1 christos ZSTD_DCtx* dctx = ZSTD_createDCtx(); 3150 1.1 christos ZSTD_CDict* cdict; 3151 1.1 christos CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_strategy, ZSTD_lazy)); 3152 1.1 christos /* Force enable row based match finder, and disable dedicated dict search. */ 3153 1.1 christos CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_useRowMatchFinder, ZSTD_ps_enable)); 3154 1.1 christos CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_enableDedicatedDictSearch, 0)); 3155 1.1 christos CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_searchLog, 4)); 3156 1.1 christos /* Set windowLog to 29 so hash/chain logs don't get sized down */ 3157 1.1 christos CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_windowLog, 29)); 3158 1.1 christos CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_hashLog, 29)); 3159 1.1 christos /* Set srcSizeHint to 2^29 so hash/chain logs don't get sized down */ 3160 1.1 christos CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_srcSizeHint, 1u << 29)); 3161 1.1 christos cdict = ZSTD_createCDict_advanced2(dictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dct_auto, cctxParams, customMem); 3162 1.1 christos CHECK_Z(ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters)); 3163 1.1 christos CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_forceAttachDict, ZSTD_dictForceAttach)); 3164 1.1 christos CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1)); 3165 1.1 christos CHECK_Z(ZSTD_CCtx_refCDict(cctx, cdict)); 3166 1.1 christos cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize); 3167 1.1 christos CHECK_Z(cSize); 3168 1.1 christos CHECK_Z(ZSTD_decompress_usingDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, dictBuffer, dictSize)); 3169 1.1 christos ZSTD_freeCDict(cdict); 3170 1.1 christos ZSTD_freeDCtx(dctx); 3171 1.1 christos ZSTD_freeCCtxParams(cctxParams); 3172 1.1 christos } 3173 1.1 christos DISPLAYLEVEL(3, "OK \n"); 3174 1.1 christos 3175 1.1 christos DISPLAYLEVEL(3, "test%3i : Dictionary with non-default repcodes : ", testNb++); 3176 1.1 christos { U32 u; for (u=0; u<nbSamples; u++) samplesSizes[u] = sampleUnitSize; } 3177 1.1 christos dictSize = ZDICT_trainFromBuffer(dictBuffer, dictSize, 3178 1.1 christos CNBuffer, samplesSizes, nbSamples); 3179 1.1 christos if (ZDICT_isError(dictSize)) goto _output_error; 3180 1.1 christos /* Set all the repcodes to non-default */ 3181 1.1 christos { 3182 1.1 christos BYTE* dictPtr = (BYTE*)dictBuffer; 3183 1.1 christos BYTE* dictLimit = dictPtr + dictSize - 12; 3184 1.1 christos /* Find the repcodes */ 3185 1.1 christos while (dictPtr < dictLimit && 3186 1.1 christos (MEM_readLE32(dictPtr) != 1 || MEM_readLE32(dictPtr + 4) != 4 || 3187 1.1 christos MEM_readLE32(dictPtr + 8) != 8)) { 3188 1.1 christos ++dictPtr; 3189 1.1 christos } 3190 1.1 christos if (dictPtr >= dictLimit) goto _output_error; 3191 1.1 christos MEM_writeLE32(dictPtr + 0, 10); 3192 1.1 christos MEM_writeLE32(dictPtr + 4, 10); 3193 1.1 christos MEM_writeLE32(dictPtr + 8, 10); 3194 1.1 christos /* Set the last 8 bytes to 'x' */ 3195 1.1 christos memset((BYTE*)dictBuffer + dictSize - 8, 'x', 8); 3196 1.1 christos } 3197 1.1 christos /* The optimal parser checks all the repcodes. 3198 1.1 christos * Make sure at least one is a match >= targetLength so that it is 3199 1.1 christos * immediately chosen. This will make sure that the compressor and 3200 1.1 christos * decompressor agree on at least one of the repcodes. 3201 1.1 christos */ 3202 1.1 christos { size_t dSize; 3203 1.1 christos BYTE data[1024]; 3204 1.1 christos ZSTD_DCtx* const dctx = ZSTD_createDCtx(); 3205 1.1 christos ZSTD_compressionParameters const cParams = ZSTD_getCParams(19, CNBuffSize, dictSize); 3206 1.1 christos ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictBuffer, dictSize, 3207 1.1 christos ZSTD_dlm_byRef, ZSTD_dct_auto, 3208 1.1 christos cParams, ZSTD_defaultCMem); 3209 1.1 christos assert(dctx != NULL); assert(cdict != NULL); 3210 1.1 christos memset(data, 'x', sizeof(data)); 3211 1.1 christos cSize = ZSTD_compress_usingCDict(cctx, compressedBuffer, compressedBufferSize, 3212 1.1 christos data, sizeof(data), cdict); 3213 1.1 christos ZSTD_freeCDict(cdict); 3214 1.1 christos if (ZSTD_isError(cSize)) { DISPLAYLEVEL(5, "Compression error %s : ", ZSTD_getErrorName(cSize)); goto _output_error; } 3215 1.1 christos dSize = ZSTD_decompress_usingDict(dctx, decodedBuffer, sizeof(data), compressedBuffer, cSize, dictBuffer, dictSize); 3216 1.1 christos if (ZSTD_isError(dSize)) { DISPLAYLEVEL(5, "Decompression error %s : ", ZSTD_getErrorName(dSize)); goto _output_error; } 3217 1.1 christos if (memcmp(data, decodedBuffer, sizeof(data))) { DISPLAYLEVEL(5, "Data corruption : "); goto _output_error; } 3218 1.1 christos ZSTD_freeDCtx(dctx); 3219 1.1 christos } 3220 1.1 christos DISPLAYLEVEL(3, "OK \n"); 3221 1.1 christos 3222 1.1 christos DISPLAYLEVEL(3, "test%3i : ZSTD_decompressDCtx() with multiple ddicts : ", testNb++); 3223 1.1 christos { 3224 1.1 christos const size_t numDicts = 128; 3225 1.1 christos const size_t numFrames = 4; 3226 1.1 christos size_t i; 3227 1.1 christos ZSTD_DCtx* dctx = ZSTD_createDCtx(); 3228 1.1 christos ZSTD_DDict** ddictTable = (ZSTD_DDict**)malloc(sizeof(ZSTD_DDict*)*numDicts); 3229 1.1 christos ZSTD_CDict** cdictTable = (ZSTD_CDict**)malloc(sizeof(ZSTD_CDict*)*numDicts); 3230 1.1 christos U32 dictIDSeed = seed; 3231 1.1 christos /* Create new compressed buffer that will hold frames with differing dictIDs */ 3232 1.1 christos char* dictBufferMulti = (char*)malloc(sizeof(char) * dictBufferFixedSize); /* Modifiable copy of fixed full dict buffer */ 3233 1.1 christos 3234 1.1 christos ZSTD_memcpy(dictBufferMulti, dictBufferFixed, dictBufferFixedSize); 3235 1.1 christos /* Create a bunch of DDicts with random dict IDs */ 3236 1.1 christos for (i = 0; i < numDicts; ++i) { 3237 1.1 christos U32 currDictID = FUZ_rand(&dictIDSeed); 3238 1.1 christos MEM_writeLE32(dictBufferMulti+ZSTD_FRAMEIDSIZE, currDictID); 3239 1.1 christos ddictTable[i] = ZSTD_createDDict(dictBufferMulti, dictBufferFixedSize); 3240 1.1 christos cdictTable[i] = ZSTD_createCDict(dictBufferMulti, dictBufferFixedSize, 3); 3241 1.1 christos if (!ddictTable[i] || !cdictTable[i] || ZSTD_getDictID_fromCDict(cdictTable[i]) != ZSTD_getDictID_fromDDict(ddictTable[i])) { 3242 1.1 christos goto _output_error; 3243 1.1 christos } 3244 1.1 christos } 3245 1.1 christos /* Compress a few frames using random CDicts */ 3246 1.1 christos { 3247 1.1 christos size_t off = 0; 3248 1.1 christos /* only use the first half so we don't push against size limit of compressedBuffer */ 3249 1.1 christos size_t const segSize = (CNBuffSize / 2) / numFrames; 3250 1.1 christos for (i = 0; i < numFrames; i++) { 3251 1.1 christos size_t dictIdx = FUZ_rand(&dictIDSeed) % numDicts; 3252 1.1 christos ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters); 3253 1.1 christos { CHECK_NEWV(r, ZSTD_compress_usingCDict(cctx, 3254 1.1 christos (BYTE*)compressedBuffer + off, CNBuffSize - off, 3255 1.1 christos (BYTE*)CNBuffer + segSize * (size_t)i, segSize, 3256 1.1 christos cdictTable[dictIdx])); 3257 1.1 christos off += r; 3258 1.1 christos } 3259 1.1 christos } 3260 1.1 christos cSize = off; 3261 1.1 christos } 3262 1.1 christos 3263 1.1 christos /* We should succeed to decompression even though different dicts were used on different frames */ 3264 1.1 christos ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters); 3265 1.1 christos ZSTD_DCtx_setParameter(dctx, ZSTD_d_refMultipleDDicts, ZSTD_rmd_refMultipleDDicts); 3266 1.1 christos /* Reference every single ddict we made */ 3267 1.1 christos for (i = 0; i < numDicts; ++i) { 3268 1.1 christos CHECK_Z( ZSTD_DCtx_refDDict(dctx, ddictTable[i])); 3269 1.1 christos } 3270 1.1 christos CHECK_Z( ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize) ); 3271 1.1 christos /* Streaming decompression should also work */ 3272 1.1 christos { 3273 1.1 christos ZSTD_inBuffer in = {compressedBuffer, cSize, 0}; 3274 1.1 christos ZSTD_outBuffer out = {decodedBuffer, CNBuffSize, 0}; 3275 1.1 christos while (in.pos < in.size) { 3276 1.1 christos CHECK_Z(ZSTD_decompressStream(dctx, &out, &in)); 3277 1.1 christos } 3278 1.1 christos } 3279 1.1 christos ZSTD_freeDCtx(dctx); 3280 1.1 christos for (i = 0; i < numDicts; ++i) { 3281 1.1 christos ZSTD_freeCDict(cdictTable[i]); 3282 1.1 christos ZSTD_freeDDict(ddictTable[i]); 3283 1.1 christos } 3284 1.1 christos free(dictBufferMulti); 3285 1.1 christos free(ddictTable); 3286 1.1 christos free(cdictTable); 3287 1.1 christos } 3288 1.1 christos DISPLAYLEVEL(3, "OK \n"); 3289 1.1 christos 3290 1.1 christos ZSTD_freeCCtx(cctx); 3291 1.1 christos free(dictBuffer); 3292 1.1 christos free(samplesSizes); 3293 1.1 christos } 3294 1.1 christos 3295 1.1 christos /* COVER dictionary builder tests */ 3296 1.1 christos { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); 3297 1.1 christos size_t dictSize = 16 KB; 3298 1.1 christos size_t optDictSize = dictSize; 3299 1.1 christos void* dictBuffer = malloc(dictSize); 3300 1.1 christos size_t const totalSampleSize = 1 MB; 3301 1.1 christos size_t const sampleUnitSize = 8 KB; 3302 1.1 christos U32 const nbSamples = (U32)(totalSampleSize / sampleUnitSize); 3303 1.1 christos size_t* const samplesSizes = (size_t*) malloc(nbSamples * sizeof(size_t)); 3304 1.1 christos U32 seed32 = seed; 3305 1.1 christos ZDICT_cover_params_t params; 3306 1.1 christos U32 dictID; 3307 1.1 christos 3308 1.1 christos if (dictBuffer==NULL || samplesSizes==NULL) { 3309 1.1 christos free(dictBuffer); 3310 1.1 christos free(samplesSizes); 3311 1.1 christos goto _output_error; 3312 1.1 christos } 3313 1.1 christos 3314 1.1 christos DISPLAYLEVEL(3, "test%3i : ZDICT_trainFromBuffer_cover : ", testNb++); 3315 1.1 christos { U32 u; for (u=0; u<nbSamples; u++) samplesSizes[u] = sampleUnitSize; } 3316 1.1 christos memset(¶ms, 0, sizeof(params)); 3317 1.1 christos params.d = 1 + (FUZ_rand(&seed32) % 16); 3318 1.1 christos params.k = params.d + (FUZ_rand(&seed32) % 256); 3319 1.1 christos dictSize = ZDICT_trainFromBuffer_cover(dictBuffer, dictSize, 3320 1.1 christos CNBuffer, samplesSizes, nbSamples, 3321 1.1 christos params); 3322 1.1 christos if (ZDICT_isError(dictSize)) goto _output_error; 3323 1.1 christos DISPLAYLEVEL(3, "OK, created dictionary of size %u \n", (unsigned)dictSize); 3324 1.1 christos 3325 1.1 christos DISPLAYLEVEL(3, "test%3i : check dictID : ", testNb++); 3326 1.1 christos dictID = ZDICT_getDictID(dictBuffer, dictSize); 3327 1.1 christos if (dictID==0) goto _output_error; 3328 1.1 christos DISPLAYLEVEL(3, "OK : %u \n", (unsigned)dictID); 3329 1.1 christos 3330 1.1 christos DISPLAYLEVEL(3, "test%3i : ZDICT_optimizeTrainFromBuffer_cover : ", testNb++); 3331 1.1 christos memset(¶ms, 0, sizeof(params)); 3332 1.1 christos params.steps = 4; 3333 1.1 christos optDictSize = ZDICT_optimizeTrainFromBuffer_cover(dictBuffer, optDictSize, 3334 1.1 christos CNBuffer, samplesSizes, 3335 1.1 christos nbSamples / 4, ¶ms); 3336 1.1 christos if (ZDICT_isError(optDictSize)) goto _output_error; 3337 1.1 christos DISPLAYLEVEL(3, "OK, created dictionary of size %u \n", (unsigned)optDictSize); 3338 1.1 christos 3339 1.1 christos DISPLAYLEVEL(3, "test%3i : check dictID : ", testNb++); 3340 1.1 christos dictID = ZDICT_getDictID(dictBuffer, optDictSize); 3341 1.1 christos if (dictID==0) goto _output_error; 3342 1.1 christos DISPLAYLEVEL(3, "OK : %u \n", (unsigned)dictID); 3343 1.1 christos 3344 1.1 christos ZSTD_freeCCtx(cctx); 3345 1.1 christos free(dictBuffer); 3346 1.1 christos free(samplesSizes); 3347 1.1 christos } 3348 1.1 christos 3349 1.1 christos /* Decompression defense tests */ 3350 1.1 christos DISPLAYLEVEL(3, "test%3i : Check input length for magic number : ", testNb++); 3351 1.1 christos { size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, CNBuffer, 3); /* too small input */ 3352 1.1 christos if (!ZSTD_isError(r)) goto _output_error; 3353 1.1 christos if (ZSTD_getErrorCode(r) != ZSTD_error_srcSize_wrong) goto _output_error; } 3354 1.1 christos DISPLAYLEVEL(3, "OK \n"); 3355 1.1 christos 3356 1.1 christos DISPLAYLEVEL(3, "test%3i : Check magic Number : ", testNb++); 3357 1.1 christos ((char*)(CNBuffer))[0] = 1; 3358 1.1 christos { size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, CNBuffer, 4); 3359 1.1 christos if (!ZSTD_isError(r)) goto _output_error; } 3360 1.1 christos DISPLAYLEVEL(3, "OK \n"); 3361 1.1 christos 3362 1.1 christos /* content size verification test */ 3363 1.1 christos DISPLAYLEVEL(3, "test%3i : Content size verification : ", testNb++); 3364 1.1 christos { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); 3365 1.1 christos size_t const srcSize = 5000; 3366 1.1 christos size_t const wrongSrcSize = (srcSize + 1000); 3367 1.1 christos ZSTD_parameters params = ZSTD_getParams(1, wrongSrcSize, 0); 3368 1.1 christos params.fParams.contentSizeFlag = 1; 3369 1.1 christos CHECK_Z( ZSTD_compressBegin_advanced(cctx, NULL, 0, params, wrongSrcSize) ); 3370 1.1 christos { size_t const result = ZSTD_compressEnd(cctx, decodedBuffer, CNBuffSize, CNBuffer, srcSize); 3371 1.1 christos if (!ZSTD_isError(result)) goto _output_error; 3372 1.1 christos if (ZSTD_getErrorCode(result) != ZSTD_error_srcSize_wrong) goto _output_error; 3373 1.1 christos DISPLAYLEVEL(3, "OK : %s \n", ZSTD_getErrorName(result)); 3374 1.1 christos } 3375 1.1 christos ZSTD_freeCCtx(cctx); 3376 1.1 christos } 3377 1.1 christos 3378 1.1 christos /* negative compression level test : ensure simple API and advanced API produce same result */ 3379 1.1 christos DISPLAYLEVEL(3, "test%3i : negative compression level : ", testNb++); 3380 1.1 christos { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); 3381 1.1 christos size_t const srcSize = CNBuffSize / 5; 3382 1.1 christos int const compressionLevel = -1; 3383 1.1 christos 3384 1.1 christos assert(cctx != NULL); 3385 1.1 christos { size_t const cSize_1pass = ZSTD_compress(compressedBuffer, compressedBufferSize, 3386 1.1 christos CNBuffer, srcSize, compressionLevel); 3387 1.1 christos if (ZSTD_isError(cSize_1pass)) goto _output_error; 3388 1.1 christos 3389 1.1 christos CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, compressionLevel) ); 3390 1.1 christos { size_t const compressionResult = ZSTD_compress2(cctx, 3391 1.1 christos compressedBuffer, compressedBufferSize, 3392 1.1 christos CNBuffer, srcSize); 3393 1.1 christos DISPLAYLEVEL(5, "simple=%zu vs %zu=advanced : ", cSize_1pass, compressionResult); 3394 1.1 christos if (ZSTD_isError(compressionResult)) goto _output_error; 3395 1.1 christos if (compressionResult != cSize_1pass) goto _output_error; 3396 1.1 christos } } 3397 1.1 christos ZSTD_freeCCtx(cctx); 3398 1.1 christos } 3399 1.1 christos DISPLAYLEVEL(3, "OK \n"); 3400 1.1 christos 3401 1.1 christos /* parameters order test */ 3402 1.1 christos { size_t const inputSize = CNBuffSize / 2; 3403 1.1 christos U64 xxh64; 3404 1.1 christos 3405 1.1 christos { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); 3406 1.1 christos DISPLAYLEVEL(3, "test%3i : parameters in order : ", testNb++); 3407 1.1 christos assert(cctx != NULL); 3408 1.1 christos CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 2) ); 3409 1.1 christos CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable) ); 3410 1.1 christos CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, 18) ); 3411 1.1 christos { size_t const compressedSize = ZSTD_compress2(cctx, 3412 1.1 christos compressedBuffer, ZSTD_compressBound(inputSize), 3413 1.1 christos CNBuffer, inputSize); 3414 1.1 christos CHECK_Z(compressedSize); 3415 1.1 christos cSize = compressedSize; 3416 1.1 christos xxh64 = XXH64(compressedBuffer, compressedSize, 0); 3417 1.1 christos } 3418 1.1 christos DISPLAYLEVEL(3, "OK (compress : %u -> %u bytes)\n", (unsigned)inputSize, (unsigned)cSize); 3419 1.1 christos ZSTD_freeCCtx(cctx); 3420 1.1 christos } 3421 1.1 christos 3422 1.1 christos { ZSTD_CCtx* cctx = ZSTD_createCCtx(); 3423 1.1 christos DISPLAYLEVEL(3, "test%3i : parameters disordered : ", testNb++); 3424 1.1 christos CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, 18) ); 3425 1.1 christos CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable) ); 3426 1.1 christos CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 2) ); 3427 1.1 christos { size_t const result = ZSTD_compress2(cctx, 3428 1.1 christos compressedBuffer, ZSTD_compressBound(inputSize), 3429 1.1 christos CNBuffer, inputSize); 3430 1.1 christos CHECK_Z(result); 3431 1.1 christos if (result != cSize) goto _output_error; /* must result in same compressed result, hence same size */ 3432 1.1 christos if (XXH64(compressedBuffer, result, 0) != xxh64) goto _output_error; /* must result in exactly same content, hence same hash */ 3433 1.1 christos DISPLAYLEVEL(3, "OK (compress : %u -> %u bytes)\n", (unsigned)inputSize, (unsigned)result); 3434 1.1 christos } 3435 1.1 christos ZSTD_freeCCtx(cctx); 3436 1.1 christos } 3437 1.1 christos } 3438 1.1 christos 3439 1.1 christos /* advanced parameters for decompression */ 3440 1.1 christos { ZSTD_DCtx* const dctx = ZSTD_createDCtx(); 3441 1.1 christos assert(dctx != NULL); 3442 1.1 christos 3443 1.1 christos DISPLAYLEVEL(3, "test%3i : get dParameter bounds ", testNb++); 3444 1.1 christos { ZSTD_bounds const bounds = ZSTD_dParam_getBounds(ZSTD_d_windowLogMax); 3445 1.1 christos CHECK_Z(bounds.error); 3446 1.1 christos } 3447 1.1 christos DISPLAYLEVEL(3, "OK \n"); 3448 1.1 christos 3449 1.1 christos DISPLAYLEVEL(3, "test%3i : wrong dParameter : ", testNb++); 3450 1.1 christos { size_t const sr = ZSTD_DCtx_setParameter(dctx, (ZSTD_dParameter)999999, 0); 3451 1.1 christos if (!ZSTD_isError(sr)) goto _output_error; 3452 1.1 christos } 3453 1.1 christos { ZSTD_bounds const bounds = ZSTD_dParam_getBounds((ZSTD_dParameter)999998); 3454 1.1 christos if (!ZSTD_isError(bounds.error)) goto _output_error; 3455 1.1 christos } 3456 1.1 christos DISPLAYLEVEL(3, "OK \n"); 3457 1.1 christos 3458 1.1 christos DISPLAYLEVEL(3, "test%3i : out of bound dParameter : ", testNb++); 3459 1.1 christos { size_t const sr = ZSTD_DCtx_setParameter(dctx, ZSTD_d_windowLogMax, 9999); 3460 1.1 christos if (!ZSTD_isError(sr)) goto _output_error; 3461 1.1 christos } 3462 1.1 christos { size_t const sr = ZSTD_DCtx_setParameter(dctx, ZSTD_d_format, (ZSTD_format_e)888); 3463 1.1 christos if (!ZSTD_isError(sr)) goto _output_error; 3464 1.1 christos } 3465 1.1 christos DISPLAYLEVEL(3, "OK \n"); 3466 1.1 christos 3467 1.1 christos ZSTD_freeDCtx(dctx); 3468 1.1 christos } 3469 1.1 christos 3470 1.1 christos 3471 1.1 christos /* custom formats tests */ 3472 1.1 christos { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); 3473 1.1 christos ZSTD_DCtx* const dctx = ZSTD_createDCtx(); 3474 1.1 christos size_t const inputSize = CNBuffSize / 2; /* won't cause pb with small dict size */ 3475 1.1 christos assert(dctx != NULL); assert(cctx != NULL); 3476 1.1 christos 3477 1.1 christos /* basic block compression */ 3478 1.1 christos DISPLAYLEVEL(3, "test%3i : magic-less format test : ", testNb++); 3479 1.1 christos CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_format, ZSTD_f_zstd1_magicless) ); 3480 1.1 christos { ZSTD_inBuffer in = { CNBuffer, inputSize, 0 }; 3481 1.1 christos ZSTD_outBuffer out = { compressedBuffer, ZSTD_compressBound(inputSize), 0 }; 3482 1.1 christos size_t const result = ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end); 3483 1.1 christos if (result != 0) goto _output_error; 3484 1.1 christos if (in.pos != in.size) goto _output_error; 3485 1.1 christos cSize = out.pos; 3486 1.1 christos } 3487 1.1 christos DISPLAYLEVEL(3, "OK (compress : %u -> %u bytes)\n", (unsigned)inputSize, (unsigned)cSize); 3488 1.1 christos 3489 1.1 christos DISPLAYLEVEL(3, "test%3i : decompress normally (should fail) : ", testNb++); 3490 1.1 christos { size_t const decodeResult = ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize); 3491 1.1 christos if (ZSTD_getErrorCode(decodeResult) != ZSTD_error_prefix_unknown) goto _output_error; 3492 1.1 christos DISPLAYLEVEL(3, "OK : %s \n", ZSTD_getErrorName(decodeResult)); 3493 1.1 christos } 3494 1.1 christos 3495 1.1 christos DISPLAYLEVEL(3, "test%3i : decompress of magic-less frame : ", testNb++); 3496 1.1 christos ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters); 3497 1.1 christos CHECK_Z( ZSTD_DCtx_setParameter(dctx, ZSTD_d_format, ZSTD_f_zstd1_magicless) ); 3498 1.1 christos { ZSTD_frameHeader zfh; 3499 1.1 christos size_t const zfhrt = ZSTD_getFrameHeader_advanced(&zfh, compressedBuffer, cSize, ZSTD_f_zstd1_magicless); 3500 1.1 christos if (zfhrt != 0) goto _output_error; 3501 1.1 christos } 3502 1.1 christos /* one shot */ 3503 1.1 christos { size_t const result = ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize); 3504 1.1 christos if (result != inputSize) goto _output_error; 3505 1.1 christos DISPLAYLEVEL(3, "one-shot OK, "); 3506 1.1 christos } 3507 1.1 christos /* streaming */ 3508 1.1 christos { ZSTD_inBuffer in = { compressedBuffer, cSize, 0 }; 3509 1.1 christos ZSTD_outBuffer out = { decodedBuffer, CNBuffSize, 0 }; 3510 1.1 christos size_t const result = ZSTD_decompressStream(dctx, &out, &in); 3511 1.1 christos if (result != 0) goto _output_error; 3512 1.1 christos if (in.pos != in.size) goto _output_error; 3513 1.1 christos if (out.pos != inputSize) goto _output_error; 3514 1.1 christos DISPLAYLEVEL(3, "streaming OK : regenerated %u bytes \n", (unsigned)out.pos); 3515 1.1 christos } 3516 1.1 christos 3517 1.1 christos /* basic block compression */ 3518 1.1 christos DISPLAYLEVEL(3, "test%3i : empty magic-less format test : ", testNb++); 3519 1.1 christos CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_format, ZSTD_f_zstd1_magicless) ); 3520 1.1 christos { ZSTD_inBuffer in = { CNBuffer, 0, 0 }; 3521 1.1 christos ZSTD_outBuffer out = { compressedBuffer, ZSTD_compressBound(0), 0 }; 3522 1.1 christos size_t const result = ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end); 3523 1.1 christos if (result != 0) goto _output_error; 3524 1.1 christos if (in.pos != in.size) goto _output_error; 3525 1.1 christos cSize = out.pos; 3526 1.1 christos } 3527 1.1 christos DISPLAYLEVEL(3, "OK (compress : %u -> %u bytes)\n", (unsigned)0, (unsigned)cSize); 3528 1.1 christos 3529 1.1 christos DISPLAYLEVEL(3, "test%3i : decompress of empty magic-less frame : ", testNb++); 3530 1.1 christos ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters); 3531 1.1 christos CHECK_Z( ZSTD_DCtx_setParameter(dctx, ZSTD_d_format, ZSTD_f_zstd1_magicless) ); 3532 1.1 christos /* one shot */ 3533 1.1 christos { size_t const result = ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize); 3534 1.1 christos if (result != 0) goto _output_error; 3535 1.1 christos DISPLAYLEVEL(3, "one-shot OK, "); 3536 1.1 christos } 3537 1.1 christos /* streaming */ 3538 1.1 christos { ZSTD_inBuffer in = { compressedBuffer, cSize, 0 }; 3539 1.1 christos ZSTD_outBuffer out = { decodedBuffer, CNBuffSize, 0 }; 3540 1.1 christos size_t const result = ZSTD_decompressStream(dctx, &out, &in); 3541 1.1 christos if (result != 0) goto _output_error; 3542 1.1 christos if (in.pos != in.size) goto _output_error; 3543 1.1 christos if (out.pos != 0) goto _output_error; 3544 1.1 christos DISPLAYLEVEL(3, "streaming OK : regenerated %u bytes \n", (unsigned)out.pos); 3545 1.1 christos } 3546 1.1 christos 3547 1.1 christos ZSTD_freeCCtx(cctx); 3548 1.1 christos ZSTD_freeDCtx(dctx); 3549 1.1 christos } 3550 1.1 christos 3551 1.1 christos DISPLAYLEVEL(3, "test%3i : Decompression parameter reset test : ", testNb++); 3552 1.1 christos { 3553 1.1 christos ZSTD_DCtx* const dctx = ZSTD_createDCtx(); 3554 1.1 christos /* Attempt to future proof this to new parameters. */ 3555 1.1 christos int const maxParam = 2000; 3556 1.1 christos int param; 3557 1.1 christos if (ZSTD_d_experimentalParam3 > maxParam) goto _output_error; 3558 1.1 christos for (param = 0; param < maxParam; ++param) { 3559 1.1 christos ZSTD_dParameter dParam = (ZSTD_dParameter)param; 3560 1.1 christos ZSTD_bounds bounds = ZSTD_dParam_getBounds(dParam); 3561 1.1 christos int value1; 3562 1.1 christos int value2; 3563 1.1 christos int check; 3564 1.1 christos if (ZSTD_isError(bounds.error)) 3565 1.1 christos continue; 3566 1.1 christos CHECK_Z(ZSTD_DCtx_getParameter(dctx, dParam, &value1)); 3567 1.1 christos value2 = (value1 != bounds.lowerBound) ? bounds.lowerBound : bounds.upperBound; 3568 1.1 christos CHECK_Z(ZSTD_DCtx_setParameter(dctx, dParam, value2)); 3569 1.1 christos CHECK_Z(ZSTD_DCtx_getParameter(dctx, dParam, &check)); 3570 1.1 christos if (check != value2) goto _output_error; 3571 1.1 christos CHECK_Z(ZSTD_DCtx_reset(dctx, ZSTD_reset_parameters)); 3572 1.1 christos CHECK_Z(ZSTD_DCtx_getParameter(dctx, dParam, &check)); 3573 1.1 christos if (check != value1) goto _output_error; 3574 1.1 christos } 3575 1.1 christos ZSTD_freeDCtx(dctx); 3576 1.1 christos } 3577 1.1 christos DISPLAYLEVEL(3, "OK \n"); 3578 1.1 christos 3579 1.1 christos /* block API tests */ 3580 1.1 christos { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); 3581 1.1 christos ZSTD_DCtx* const dctx = ZSTD_createDCtx(); 3582 1.1 christos static const size_t dictSize = 65 KB; 3583 1.1 christos static const size_t blockSize = 100 KB; /* won't cause pb with small dict size */ 3584 1.1 christos size_t cSize2; 3585 1.1 christos assert(cctx != NULL); assert(dctx != NULL); 3586 1.1 christos 3587 1.1 christos /* basic block compression */ 3588 1.1 christos DISPLAYLEVEL(3, "test%3i : Block compression test : ", testNb++); 3589 1.1 christos CHECK_Z( ZSTD_compressBegin(cctx, 5) ); 3590 1.1 christos CHECK_Z( ZSTD_getBlockSize(cctx) >= blockSize); 3591 1.1 christos CHECK_VAR(cSize, ZSTD_compressBlock(cctx, compressedBuffer, ZSTD_compressBound(blockSize), CNBuffer, blockSize) ); 3592 1.1 christos DISPLAYLEVEL(3, "OK \n"); 3593 1.1 christos 3594 1.1 christos DISPLAYLEVEL(3, "test%3i : Block decompression test : ", testNb++); 3595 1.1 christos CHECK_Z( ZSTD_decompressBegin(dctx) ); 3596 1.1 christos { CHECK_NEWV(r, ZSTD_decompressBlock(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize) ); 3597 1.1 christos if (r != blockSize) goto _output_error; } 3598 1.1 christos DISPLAYLEVEL(3, "OK \n"); 3599 1.1 christos 3600 1.1 christos /* very long stream of block compression */ 3601 1.1 christos DISPLAYLEVEL(3, "test%3i : Huge block streaming compression test : ", testNb++); 3602 1.1 christos CHECK_Z( ZSTD_compressBegin(cctx, -199) ); /* we just want to quickly overflow internal U32 index */ 3603 1.1 christos CHECK_Z( ZSTD_getBlockSize(cctx) >= blockSize); 3604 1.1 christos { U64 const toCompress = 5000000000ULL; /* > 4 GB */ 3605 1.1 christos U64 compressed = 0; 3606 1.1 christos while (compressed < toCompress) { 3607 1.1 christos size_t const blockCSize = ZSTD_compressBlock(cctx, compressedBuffer, ZSTD_compressBound(blockSize), CNBuffer, blockSize); 3608 1.1 christos assert(blockCSize != 0); 3609 1.1 christos if (ZSTD_isError(blockCSize)) goto _output_error; 3610 1.1 christos compressed += blockCSize; 3611 1.1 christos } } 3612 1.1 christos DISPLAYLEVEL(3, "OK \n"); 3613 1.1 christos 3614 1.1 christos /* dictionary block compression */ 3615 1.1 christos DISPLAYLEVEL(3, "test%3i : Dictionary Block compression test : ", testNb++); 3616 1.1 christos CHECK_Z( ZSTD_compressBegin_usingDict(cctx, CNBuffer, dictSize, 5) ); 3617 1.1 christos CHECK_VAR(cSize, ZSTD_compressBlock(cctx, compressedBuffer, ZSTD_compressBound(blockSize), (char*)CNBuffer+dictSize, blockSize)); 3618 1.1 christos RDG_genBuffer((char*)CNBuffer+dictSize+blockSize, blockSize, 0.0, 0.0, seed); /* create a non-compressible second block */ 3619 1.1 christos { CHECK_NEWV(r, ZSTD_compressBlock(cctx, (char*)compressedBuffer+cSize, ZSTD_compressBound(blockSize), (char*)CNBuffer+dictSize+blockSize, blockSize) ); /* for cctx history consistency */ 3620 1.1 christos assert(r == 0); /* non-compressible block */ } 3621 1.1 christos memcpy((char*)compressedBuffer+cSize, (char*)CNBuffer+dictSize+blockSize, blockSize); /* send non-compressed block (without header) */ 3622 1.1 christos CHECK_VAR(cSize2, ZSTD_compressBlock(cctx, (char*)compressedBuffer+cSize+blockSize, ZSTD_compressBound(blockSize), 3623 1.1 christos (char*)CNBuffer+dictSize+2*blockSize, blockSize)); 3624 1.1 christos DISPLAYLEVEL(3, "OK \n"); 3625 1.1 christos 3626 1.1 christos DISPLAYLEVEL(3, "test%3i : Dictionary Block decompression test : ", testNb++); 3627 1.1 christos CHECK_Z( ZSTD_decompressBegin_usingDict(dctx, CNBuffer, dictSize) ); 3628 1.1 christos { CHECK_NEWV( r, ZSTD_decompressBlock(dctx, decodedBuffer, blockSize, compressedBuffer, cSize) ); 3629 1.1 christos if (r != blockSize) { 3630 1.1 christos DISPLAYLEVEL(1, "ZSTD_decompressBlock() with _usingDict() fails : %u, instead of %u expected \n", (unsigned)r, (unsigned)blockSize); 3631 1.1 christos goto _output_error; 3632 1.1 christos } } 3633 1.1 christos memcpy((char*)decodedBuffer+blockSize, (char*)compressedBuffer+cSize, blockSize); 3634 1.1 christos ZSTD_insertBlock(dctx, (char*)decodedBuffer+blockSize, blockSize); /* insert non-compressed block into dctx history */ 3635 1.1 christos { CHECK_NEWV( r, ZSTD_decompressBlock(dctx, (char*)decodedBuffer+2*blockSize, blockSize, (char*)compressedBuffer+cSize+blockSize, cSize2) ); 3636 1.1 christos if (r != blockSize) { 3637 1.1 christos DISPLAYLEVEL(1, "ZSTD_decompressBlock() with _usingDict() and after insertBlock() fails : %u, instead of %u expected \n", (unsigned)r, (unsigned)blockSize); 3638 1.1 christos goto _output_error; 3639 1.1 christos } } 3640 1.1 christos assert(memcpy((char*)CNBuffer+dictSize, decodedBuffer, blockSize*3)); /* ensure regenerated content is identical to origin */ 3641 1.1 christos DISPLAYLEVEL(3, "OK \n"); 3642 1.1 christos 3643 1.1 christos DISPLAYLEVEL(3, "test%3i : Block compression with CDict : ", testNb++); 3644 1.1 christos { ZSTD_CDict* const cdict = ZSTD_createCDict(CNBuffer, dictSize, 3); 3645 1.1 christos if (cdict==NULL) goto _output_error; 3646 1.1 christos CHECK_Z( ZSTD_compressBegin_usingCDict(cctx, cdict) ); 3647 1.1 christos CHECK_Z( ZSTD_compressBlock(cctx, compressedBuffer, ZSTD_compressBound(blockSize), (char*)CNBuffer+dictSize, blockSize) ); 3648 1.1 christos ZSTD_freeCDict(cdict); 3649 1.1 christos } 3650 1.1 christos DISPLAYLEVEL(3, "OK \n"); 3651 1.1 christos 3652 1.1 christos ZSTD_freeCCtx(cctx); 3653 1.1 christos ZSTD_freeDCtx(dctx); 3654 1.1 christos } 3655 1.1 christos 3656 1.1 christos /* long rle test */ 3657 1.1 christos { size_t sampleSize = 0; 3658 1.1 christos size_t expectedCompressedSize = 39; /* block 1, 2: compressed, block 3: RLE, zstd 1.4.4 */ 3659 1.1 christos DISPLAYLEVEL(3, "test%3i : Long RLE test : ", testNb++); 3660 1.1 christos memset((char*)CNBuffer+sampleSize, 'B', 256 KB - 1); 3661 1.1 christos sampleSize += 256 KB - 1; 3662 1.1 christos memset((char*)CNBuffer+sampleSize, 'A', 96 KB); 3663 1.1 christos sampleSize += 96 KB; 3664 1.1 christos cSize = ZSTD_compress(compressedBuffer, ZSTD_compressBound(sampleSize), CNBuffer, sampleSize, 1); 3665 1.1 christos if (ZSTD_isError(cSize) || cSize > expectedCompressedSize) goto _output_error; 3666 1.1 christos { CHECK_NEWV(regenSize, ZSTD_decompress(decodedBuffer, sampleSize, compressedBuffer, cSize)); 3667 1.1 christos if (regenSize!=sampleSize) goto _output_error; } 3668 1.1 christos DISPLAYLEVEL(3, "OK \n"); 3669 1.1 christos } 3670 1.1 christos 3671 1.1 christos DISPLAYLEVEL(3, "test%3i : ZSTD_generateSequences decode from sequences test : ", testNb++); 3672 1.1 christos { 3673 1.1 christos size_t srcSize = 150 KB; 3674 1.1 christos BYTE* src = (BYTE*)CNBuffer; 3675 1.1 christos BYTE* decoded = (BYTE*)compressedBuffer; 3676 1.1 christos 3677 1.1 christos ZSTD_CCtx* cctx = ZSTD_createCCtx(); 3678 1.1 christos ZSTD_Sequence* seqs = (ZSTD_Sequence*)malloc(srcSize * sizeof(ZSTD_Sequence)); 3679 1.1 christos size_t seqsSize; 3680 1.1 christos 3681 1.1 christos if (seqs == NULL) goto _output_error; 3682 1.1 christos assert(cctx != NULL); 3683 1.1 christos ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 19); 3684 1.1 christos /* Populate src with random data */ 3685 1.1 christos RDG_genBuffer(CNBuffer, srcSize, compressibility, 0.5, seed); 3686 1.1 christos 3687 1.1 christos /* Test with block delimiters roundtrip */ 3688 1.1 christos seqsSize = ZSTD_generateSequences(cctx, seqs, srcSize, src, srcSize); 3689 1.1 christos CHECK_Z(seqsSize); 3690 1.1 christos FUZ_decodeSequences(decoded, seqs, seqsSize, src, srcSize, ZSTD_sf_explicitBlockDelimiters); 3691 1.1 christos assert(!memcmp(CNBuffer, compressedBuffer, srcSize)); 3692 1.1 christos 3693 1.1 christos /* Test no block delimiters roundtrip */ 3694 1.1 christos seqsSize = ZSTD_mergeBlockDelimiters(seqs, seqsSize); 3695 1.1 christos CHECK_Z(seqsSize); 3696 1.1 christos FUZ_decodeSequences(decoded, seqs, seqsSize, src, srcSize, ZSTD_sf_noBlockDelimiters); 3697 1.1 christos assert(!memcmp(CNBuffer, compressedBuffer, srcSize)); 3698 1.1 christos 3699 1.1 christos ZSTD_freeCCtx(cctx); 3700 1.1 christos free(seqs); 3701 1.1 christos } 3702 1.1 christos DISPLAYLEVEL(3, "OK \n"); 3703 1.1 christos 3704 1.1 christos DISPLAYLEVEL(3, "test%3i : ZSTD_generateSequences too small output buffer : ", testNb++); 3705 1.1 christos { 3706 1.1 christos const size_t seqsCapacity = 10; 3707 1.1 christos const size_t srcSize = 150 KB; 3708 1.1 christos const BYTE* src = (BYTE*)CNBuffer; 3709 1.1 christos 3710 1.1 christos ZSTD_CCtx* const cctx = ZSTD_createCCtx(); 3711 1.1 christos ZSTD_Sequence* const seqs = (ZSTD_Sequence*)malloc(seqsCapacity * sizeof(ZSTD_Sequence)); 3712 1.1 christos 3713 1.1 christos if (seqs == NULL) goto _output_error; 3714 1.1 christos if (cctx == NULL) goto _output_error; 3715 1.1 christos /* Populate src with random data */ 3716 1.1 christos RDG_genBuffer(CNBuffer, srcSize, compressibility, 0.5, seed); 3717 1.1 christos 3718 1.1 christos /* Test with block delimiters roundtrip */ 3719 1.1 christos { 3720 1.1 christos size_t const seqsSize = ZSTD_generateSequences(cctx, seqs, seqsCapacity, src, srcSize); 3721 1.1 christos if (!ZSTD_isError(seqsSize)) goto _output_error; 3722 1.1 christos } 3723 1.1 christos 3724 1.1 christos ZSTD_freeCCtx(cctx); 3725 1.1 christos free(seqs); 3726 1.1 christos } 3727 1.1 christos DISPLAYLEVEL(3, "OK \n"); 3728 1.1 christos 3729 1.1 christos DISPLAYLEVEL(3, "test%3i : ZSTD_getSequences followed by ZSTD_compressSequences : ", testNb++); 3730 1.1 christos { 3731 1.1 christos const size_t srcSize = 500 KB; 3732 1.1 christos const BYTE* const src = (BYTE*)CNBuffer; 3733 1.1 christos BYTE* const dst = (BYTE*)compressedBuffer; 3734 1.1 christos const size_t dstCapacity = ZSTD_compressBound(srcSize); 3735 1.1 christos const size_t decompressSize = srcSize; 3736 1.1 christos char* const decompressBuffer = (char*)malloc(decompressSize); 3737 1.1 christos size_t compressedSize; 3738 1.1 christos 3739 1.1 christos ZSTD_CCtx* const cctx = ZSTD_createCCtx(); 3740 1.1 christos ZSTD_Sequence* const seqs = (ZSTD_Sequence*)malloc(srcSize * sizeof(ZSTD_Sequence)); 3741 1.1 christos size_t nbSeqs; 3742 1.1 christos 3743 1.1 christos if (seqs == NULL) goto _output_error; 3744 1.1 christos assert(cctx != NULL); 3745 1.1 christos 3746 1.1 christos /* Populate src with random data */ 3747 1.1 christos RDG_genBuffer(CNBuffer, srcSize, compressibility, 0., seed); 3748 1.1 christos 3749 1.1 christos /* Roundtrip Test with block delimiters generated by ZSTD_generateSequences() */ 3750 1.1 christos nbSeqs = ZSTD_generateSequences(cctx, seqs, srcSize, src, srcSize); 3751 1.1 christos ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters); 3752 1.1 christos ZSTD_CCtx_setParameter(cctx, ZSTD_c_blockDelimiters, ZSTD_sf_explicitBlockDelimiters); 3753 1.1 christos compressedSize = ZSTD_compressSequences(cctx, dst, dstCapacity, seqs, nbSeqs, src, srcSize); 3754 1.1 christos if (ZSTD_isError(compressedSize)) { 3755 1.1 christos DISPLAY("Error in sequence compression with block delims\n"); 3756 1.1 christos goto _output_error; 3757 1.1 christos } 3758 1.1 christos { size_t const dSize = ZSTD_decompress(decompressBuffer, decompressSize, dst, compressedSize); 3759 1.1 christos if (ZSTD_isError(dSize)) { 3760 1.1 christos DISPLAY("Error in sequence compression roundtrip with block delims\n"); 3761 1.1 christos goto _output_error; 3762 1.1 christos } } 3763 1.1 christos assert(!memcmp(decompressBuffer, src, srcSize)); 3764 1.1 christos 3765 1.1 christos /* Roundtrip Test with no block delimiters */ 3766 1.1 christos { size_t const nbSeqsAfterMerge = ZSTD_mergeBlockDelimiters(seqs, nbSeqs); 3767 1.1 christos ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters); 3768 1.1 christos ZSTD_CCtx_setParameter(cctx, ZSTD_c_blockDelimiters, ZSTD_sf_noBlockDelimiters); 3769 1.1 christos compressedSize = ZSTD_compressSequences(cctx, dst, dstCapacity, seqs, nbSeqsAfterMerge, src, srcSize); 3770 1.1 christos } 3771 1.1 christos if (ZSTD_isError(compressedSize)) { 3772 1.1 christos DISPLAY("Error in sequence compression with no block delims\n"); 3773 1.1 christos goto _output_error; 3774 1.1 christos } 3775 1.1 christos { size_t const dSize = ZSTD_decompress(decompressBuffer, decompressSize, dst, compressedSize); 3776 1.1 christos if (ZSTD_isError(dSize)) { 3777 1.1 christos DISPLAY("Error in sequence compression roundtrip with no block delims\n"); 3778 1.1 christos goto _output_error; 3779 1.1 christos } } 3780 1.1 christos assert(!memcmp(decompressBuffer, src, srcSize)); 3781 1.1 christos 3782 1.1 christos ZSTD_freeCCtx(cctx); 3783 1.1 christos free(decompressBuffer); 3784 1.1 christos free(seqs); 3785 1.1 christos } 3786 1.1 christos DISPLAYLEVEL(3, "OK \n"); 3787 1.1 christos 3788 1.1 christos /* Multiple blocks of zeros test */ 3789 1.1 christos #define LONGZEROSLENGTH 1000000 /* 1MB of zeros */ 3790 1.1 christos DISPLAYLEVEL(3, "test%3i : compress %u zeroes : ", testNb++, LONGZEROSLENGTH); 3791 1.1 christos memset(CNBuffer, 0, LONGZEROSLENGTH); 3792 1.1 christos CHECK_VAR(cSize, ZSTD_compress(compressedBuffer, ZSTD_compressBound(LONGZEROSLENGTH), CNBuffer, LONGZEROSLENGTH, 1) ); 3793 1.1 christos DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/LONGZEROSLENGTH*100); 3794 1.1 christos 3795 1.1 christos DISPLAYLEVEL(3, "test%3i : decompress %u zeroes : ", testNb++, LONGZEROSLENGTH); 3796 1.1 christos { CHECK_NEWV(r, ZSTD_decompress(decodedBuffer, LONGZEROSLENGTH, compressedBuffer, cSize) ); 3797 1.1 christos if (r != LONGZEROSLENGTH) goto _output_error; } 3798 1.1 christos DISPLAYLEVEL(3, "OK \n"); 3799 1.1 christos 3800 1.1 christos /* All zeroes test (test bug #137) */ 3801 1.1 christos #define ZEROESLENGTH 100 3802 1.1 christos DISPLAYLEVEL(3, "test%3i : compress %u zeroes : ", testNb++, ZEROESLENGTH); 3803 1.1 christos memset(CNBuffer, 0, ZEROESLENGTH); 3804 1.1 christos CHECK_VAR(cSize, ZSTD_compress(compressedBuffer, ZSTD_compressBound(ZEROESLENGTH), CNBuffer, ZEROESLENGTH, 1) ); 3805 1.1 christos DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/ZEROESLENGTH*100); 3806 1.1 christos 3807 1.1 christos DISPLAYLEVEL(3, "test%3i : decompress %u zeroes : ", testNb++, ZEROESLENGTH); 3808 1.1 christos { CHECK_NEWV(r, ZSTD_decompress(decodedBuffer, ZEROESLENGTH, compressedBuffer, cSize) ); 3809 1.1 christos if (r != ZEROESLENGTH) goto _output_error; } 3810 1.1 christos DISPLAYLEVEL(3, "OK \n"); 3811 1.1 christos 3812 1.1 christos /* nbSeq limit test */ 3813 1.1 christos #define _3BYTESTESTLENGTH 131000 3814 1.1 christos #define NB3BYTESSEQLOG 9 3815 1.1 christos #define NB3BYTESSEQ (1 << NB3BYTESSEQLOG) 3816 1.1 christos #define NB3BYTESSEQMASK (NB3BYTESSEQ-1) 3817 1.1 christos /* creates a buffer full of 3-bytes sequences */ 3818 1.1 christos { BYTE _3BytesSeqs[NB3BYTESSEQ][3]; 3819 1.1 christos U32 rSeed = 1; 3820 1.1 christos 3821 1.1 christos /* create batch of 3-bytes sequences */ 3822 1.1 christos { int i; 3823 1.1 christos for (i=0; i < NB3BYTESSEQ; i++) { 3824 1.1 christos _3BytesSeqs[i][0] = (BYTE)(FUZ_rand(&rSeed) & 255); 3825 1.1 christos _3BytesSeqs[i][1] = (BYTE)(FUZ_rand(&rSeed) & 255); 3826 1.1 christos _3BytesSeqs[i][2] = (BYTE)(FUZ_rand(&rSeed) & 255); 3827 1.1 christos } } 3828 1.1 christos 3829 1.1 christos /* randomly fills CNBuffer with prepared 3-bytes sequences */ 3830 1.1 christos { int i; 3831 1.1 christos for (i=0; i < _3BYTESTESTLENGTH; i += 3) { /* note : CNBuffer size > _3BYTESTESTLENGTH+3 */ 3832 1.1 christos U32 const id = FUZ_rand(&rSeed) & NB3BYTESSEQMASK; 3833 1.1 christos ((BYTE*)CNBuffer)[i+0] = _3BytesSeqs[id][0]; 3834 1.1 christos ((BYTE*)CNBuffer)[i+1] = _3BytesSeqs[id][1]; 3835 1.1 christos ((BYTE*)CNBuffer)[i+2] = _3BytesSeqs[id][2]; 3836 1.1 christos } } } 3837 1.1 christos DISPLAYLEVEL(3, "test%3i : growing nbSeq : ", testNb++); 3838 1.1 christos { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); 3839 1.1 christos size_t const maxNbSeq = _3BYTESTESTLENGTH / 3; 3840 1.1 christos size_t const bound = ZSTD_compressBound(_3BYTESTESTLENGTH); 3841 1.1 christos size_t nbSeq = 1; 3842 1.1 christos while (nbSeq <= maxNbSeq) { 3843 1.1 christos CHECK_Z(ZSTD_compressCCtx(cctx, compressedBuffer, bound, CNBuffer, nbSeq * 3, 19)); 3844 1.1 christos /* Check every sequence for the first 100, then skip more rapidly. */ 3845 1.1 christos if (nbSeq < 100) { 3846 1.1 christos ++nbSeq; 3847 1.1 christos } else { 3848 1.1 christos nbSeq += (nbSeq >> 2); 3849 1.1 christos } 3850 1.1 christos } 3851 1.1 christos ZSTD_freeCCtx(cctx); 3852 1.1 christos } 3853 1.1 christos DISPLAYLEVEL(3, "OK \n"); 3854 1.1 christos 3855 1.1 christos DISPLAYLEVEL(3, "test%3i : compress lots 3-bytes sequences : ", testNb++); 3856 1.1 christos CHECK_VAR(cSize, ZSTD_compress(compressedBuffer, ZSTD_compressBound(_3BYTESTESTLENGTH), 3857 1.1 christos CNBuffer, _3BYTESTESTLENGTH, 19) ); 3858 1.1 christos DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/_3BYTESTESTLENGTH*100); 3859 1.1 christos 3860 1.1 christos DISPLAYLEVEL(3, "test%3i : decompress lots 3-bytes sequence : ", testNb++); 3861 1.1 christos { CHECK_NEWV(r, ZSTD_decompress(decodedBuffer, _3BYTESTESTLENGTH, compressedBuffer, cSize) ); 3862 1.1 christos if (r != _3BYTESTESTLENGTH) goto _output_error; } 3863 1.1 christos DISPLAYLEVEL(3, "OK \n"); 3864 1.1 christos 3865 1.1 christos 3866 1.1 christos DISPLAYLEVEL(3, "test%3i : growing literals buffer : ", testNb++); 3867 1.1 christos RDG_genBuffer(CNBuffer, CNBuffSize, 0.0, 0.1, seed); 3868 1.1 christos { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); 3869 1.1 christos size_t const bound = ZSTD_compressBound(CNBuffSize); 3870 1.1 christos size_t size = 1; 3871 1.1 christos while (size <= CNBuffSize) { 3872 1.1 christos CHECK_Z(ZSTD_compressCCtx(cctx, compressedBuffer, bound, CNBuffer, size, 3)); 3873 1.1 christos /* Check every size for the first 100, then skip more rapidly. */ 3874 1.1 christos if (size < 100) { 3875 1.1 christos ++size; 3876 1.1 christos } else { 3877 1.1 christos size += (size >> 2); 3878 1.1 christos } 3879 1.1 christos } 3880 1.1 christos ZSTD_freeCCtx(cctx); 3881 1.1 christos } 3882 1.1 christos DISPLAYLEVEL(3, "OK \n"); 3883 1.1 christos 3884 1.1 christos DISPLAYLEVEL(3, "test%3i : incompressible data and ill suited dictionary : ", testNb++); 3885 1.1 christos { /* Train a dictionary on low characters */ 3886 1.1 christos size_t dictSize = 16 KB; 3887 1.1 christos void* const dictBuffer = malloc(dictSize); 3888 1.1 christos size_t const totalSampleSize = 1 MB; 3889 1.1 christos size_t const sampleUnitSize = 8 KB; 3890 1.1 christos U32 const nbSamples = (U32)(totalSampleSize / sampleUnitSize); 3891 1.1 christos size_t* const samplesSizes = (size_t*) malloc(nbSamples * sizeof(size_t)); 3892 1.1 christos if (!dictBuffer || !samplesSizes) goto _output_error; 3893 1.1 christos { U32 u; for (u=0; u<nbSamples; u++) samplesSizes[u] = sampleUnitSize; } 3894 1.1 christos dictSize = ZDICT_trainFromBuffer(dictBuffer, dictSize, CNBuffer, samplesSizes, nbSamples); 3895 1.1 christos if (ZDICT_isError(dictSize)) goto _output_error; 3896 1.1 christos /* Reverse the characters to make the dictionary ill suited */ 3897 1.1 christos { U32 u; 3898 1.1 christos for (u = 0; u < CNBuffSize; ++u) { 3899 1.1 christos ((BYTE*)CNBuffer)[u] = 255 - ((BYTE*)CNBuffer)[u]; 3900 1.1 christos } } 3901 1.1 christos { /* Compress the data */ 3902 1.1 christos size_t const inputSize = 500; 3903 1.1 christos size_t const outputSize = ZSTD_compressBound(inputSize); 3904 1.1 christos void* const outputBuffer = malloc(outputSize); 3905 1.1 christos ZSTD_CCtx* const cctx = ZSTD_createCCtx(); 3906 1.1 christos if (!outputBuffer || !cctx) goto _output_error; 3907 1.1 christos CHECK_Z(ZSTD_compress_usingDict(cctx, outputBuffer, outputSize, CNBuffer, inputSize, dictBuffer, dictSize, 1)); 3908 1.1 christos free(outputBuffer); 3909 1.1 christos ZSTD_freeCCtx(cctx); 3910 1.1 christos } 3911 1.1 christos 3912 1.1 christos free(dictBuffer); 3913 1.1 christos free(samplesSizes); 3914 1.1 christos } 3915 1.1 christos DISPLAYLEVEL(3, "OK \n"); 3916 1.1 christos 3917 1.1 christos 3918 1.1 christos /* findFrameCompressedSize on skippable frames */ 3919 1.1 christos DISPLAYLEVEL(3, "test%3i : frame compressed size of skippable frame : ", testNb++); 3920 1.1 christos { const char* frame = "\x50\x2a\x4d\x18\x05\x0\x0\0abcde"; 3921 1.1 christos size_t const frameSrcSize = 13; 3922 1.1 christos if (ZSTD_findFrameCompressedSize(frame, frameSrcSize) != frameSrcSize) goto _output_error; } 3923 1.1 christos DISPLAYLEVEL(3, "OK \n"); 3924 1.1 christos 3925 1.1 christos /* error string tests */ 3926 1.1 christos DISPLAYLEVEL(3, "test%3i : testing ZSTD error code strings : ", testNb++); 3927 1.1 christos if (strcmp("No error detected", ZSTD_getErrorName((ZSTD_ErrorCode)(0-ZSTD_error_no_error))) != 0) goto _output_error; 3928 1.1 christos if (strcmp("No error detected", ZSTD_getErrorString(ZSTD_error_no_error)) != 0) goto _output_error; 3929 1.1 christos if (strcmp("Unspecified error code", ZSTD_getErrorString((ZSTD_ErrorCode)(0-ZSTD_error_GENERIC))) != 0) goto _output_error; 3930 1.1 christos if (strcmp("Error (generic)", ZSTD_getErrorName((size_t)0-ZSTD_error_GENERIC)) != 0) goto _output_error; 3931 1.1 christos if (strcmp("Error (generic)", ZSTD_getErrorString(ZSTD_error_GENERIC)) != 0) goto _output_error; 3932 1.1 christos if (strcmp("No error detected", ZSTD_getErrorName(ZSTD_error_GENERIC)) != 0) goto _output_error; 3933 1.1 christos DISPLAYLEVEL(3, "OK \n"); 3934 1.1 christos 3935 1.1 christos DISPLAYLEVEL(3, "test%3i : testing ZSTD dictionary sizes : ", testNb++); 3936 1.1 christos RDG_genBuffer(CNBuffer, CNBuffSize, compressibility, 0., seed); 3937 1.1 christos { 3938 1.1 christos size_t const size = MIN(128 KB, CNBuffSize); 3939 1.1 christos ZSTD_CCtx* const cctx = ZSTD_createCCtx(); 3940 1.1 christos ZSTD_CDict* const lgCDict = ZSTD_createCDict(CNBuffer, size, 1); 3941 1.1 christos ZSTD_CDict* const smCDict = ZSTD_createCDict(CNBuffer, 1 KB, 1); 3942 1.1 christos ZSTD_frameHeader lgHeader; 3943 1.1 christos ZSTD_frameHeader smHeader; 3944 1.1 christos 3945 1.1 christos CHECK_Z(ZSTD_compress_usingCDict(cctx, compressedBuffer, compressedBufferSize, CNBuffer, size, lgCDict)); 3946 1.1 christos CHECK_Z(ZSTD_getFrameHeader(&lgHeader, compressedBuffer, compressedBufferSize)); 3947 1.1 christos CHECK_Z(ZSTD_compress_usingCDict(cctx, compressedBuffer, compressedBufferSize, CNBuffer, size, smCDict)); 3948 1.1 christos CHECK_Z(ZSTD_getFrameHeader(&smHeader, compressedBuffer, compressedBufferSize)); 3949 1.1 christos 3950 1.1 christos if (lgHeader.windowSize != smHeader.windowSize) goto _output_error; 3951 1.1 christos 3952 1.1 christos ZSTD_freeCDict(smCDict); 3953 1.1 christos ZSTD_freeCDict(lgCDict); 3954 1.1 christos ZSTD_freeCCtx(cctx); 3955 1.1 christos } 3956 1.1 christos DISPLAYLEVEL(3, "OK \n"); 3957 1.1 christos 3958 1.1 christos DISPLAYLEVEL(3, "test%3i : testing FSE_normalizeCount() PR#1255: ", testNb++); 3959 1.1 christos { 3960 1.1 christos short norm[32]; 3961 1.1 christos unsigned count[32]; 3962 1.1 christos unsigned const tableLog = 5; 3963 1.1 christos size_t const nbSeq = 32; 3964 1.1 christos unsigned const maxSymbolValue = 31; 3965 1.1 christos size_t i; 3966 1.1 christos 3967 1.1 christos for (i = 0; i < 32; ++i) 3968 1.1 christos count[i] = 1; 3969 1.1 christos /* Calling FSE_normalizeCount() on a uniform distribution should not 3970 1.1 christos * cause a division by zero. 3971 1.1 christos */ 3972 1.1 christos FSE_normalizeCount(norm, tableLog, count, nbSeq, maxSymbolValue, /* useLowProbCount */ 1); 3973 1.1 christos } 3974 1.1 christos DISPLAYLEVEL(3, "OK \n"); 3975 1.1 christos 3976 1.1 christos DISPLAYLEVEL(3, "test%3i : testing FSE_writeNCount() PR#2779: ", testNb++); 3977 1.1 christos { 3978 1.1 christos size_t const outBufSize = 9; 3979 1.1 christos short const count[11] = {1, 0, 1, 0, 1, 0, 1, 0, 1, 9, 18}; 3980 1.1 christos unsigned const tableLog = 5; 3981 1.1 christos unsigned const maxSymbolValue = 10; 3982 1.1 christos BYTE* outBuf = (BYTE*)malloc(outBufSize*sizeof(BYTE)); 3983 1.1 christos 3984 1.1 christos /* Ensure that this write doesn't write out of bounds, and that 3985 1.1 christos * FSE_writeNCount_generic() is *not* called with writeIsSafe == 1. 3986 1.1 christos */ 3987 1.1 christos FSE_writeNCount(outBuf, outBufSize, count, maxSymbolValue, tableLog); 3988 1.1 christos free(outBuf); 3989 1.1 christos } 3990 1.1 christos DISPLAYLEVEL(3, "OK \n"); 3991 1.1 christos 3992 1.1 christos DISPLAYLEVEL(3, "test%3i : testing bitwise intrinsics PR#3045: ", testNb++); 3993 1.1 christos { 3994 1.1 christos U32 seed_copy = seed; /* need non-const seed to avoid compiler warning for FUZ_rand(&seed) */ 3995 1.1 christos U32 rand32 = FUZ_rand(&seed_copy); 3996 1.1 christos U64 rand64 = ((U64)FUZ_rand(&seed_copy) << 32) | FUZ_rand(&seed_copy); 3997 1.1 christos U32 lowbit_only_32 = 1; 3998 1.1 christos U64 lowbit_only_64 = 1; 3999 1.1 christos U32 highbit_only_32 = (U32)1 << 31; 4000 1.1 christos U64 highbit_only_64 = (U64)1 << 63; 4001 1.1 christos U32 i; 4002 1.1 christos if (rand32 == 0) rand32 = 1; /* CLZ and CTZ are undefined on 0 */ 4003 1.1 christos if (rand64 == 0) rand64 = 1; /* CLZ and CTZ are undefined on 0 */ 4004 1.1 christos 4005 1.1 christos /* Test ZSTD_countTrailingZeros32 */ 4006 1.1 christos CHECK_EQ(ZSTD_countTrailingZeros32(lowbit_only_32), 0u); 4007 1.1 christos CHECK_EQ(ZSTD_countTrailingZeros32(highbit_only_32), 31u); 4008 1.1 christos CHECK_EQ(ZSTD_countTrailingZeros32(rand32), ZSTD_countTrailingZeros32_fallback(rand32)); 4009 1.1 christos 4010 1.1 christos /* Test ZSTD_countLeadingZeros32 */ 4011 1.1 christos CHECK_EQ(ZSTD_countLeadingZeros32(lowbit_only_32), 31u); 4012 1.1 christos CHECK_EQ(ZSTD_countLeadingZeros32(highbit_only_32), 0u); 4013 1.1 christos CHECK_EQ(ZSTD_countLeadingZeros32(rand32), ZSTD_countLeadingZeros32_fallback(rand32)); 4014 1.1 christos 4015 1.1 christos /* Test ZSTD_countTrailingZeros64 */ 4016 1.1 christos CHECK_EQ(ZSTD_countTrailingZeros64(lowbit_only_64), 0u); 4017 1.1 christos CHECK_EQ(ZSTD_countTrailingZeros64(highbit_only_64), 63u); 4018 1.1 christos 4019 1.1 christos /* Test ZSTD_countLeadingZeros64 */ 4020 1.1 christos CHECK_EQ(ZSTD_countLeadingZeros64(lowbit_only_64), 63u); 4021 1.1 christos CHECK_EQ(ZSTD_countLeadingZeros64(highbit_only_64), 0u); 4022 1.1 christos 4023 1.1 christos /* Test ZSTD_highbit32 */ 4024 1.1 christos CHECK_EQ(ZSTD_highbit32(lowbit_only_32), 0u); 4025 1.1 christos CHECK_EQ(ZSTD_highbit32(highbit_only_32), 31u); 4026 1.1 christos 4027 1.1 christos /* Test ZSTD_NbCommonBytes */ 4028 1.1 christos if (MEM_isLittleEndian()) { 4029 1.1 christos if (MEM_64bits()) { 4030 1.1 christos CHECK_EQ(ZSTD_NbCommonBytes(lowbit_only_32), 0u); 4031 1.1 christos CHECK_EQ(ZSTD_NbCommonBytes(highbit_only_32), 3u); 4032 1.1 christos } else { 4033 1.1 christos CHECK_EQ(ZSTD_NbCommonBytes(lowbit_only_32), 0u); 4034 1.1 christos CHECK_EQ(ZSTD_NbCommonBytes(highbit_only_32), 3u); 4035 1.1 christos } 4036 1.1 christos } else { 4037 1.1 christos if (MEM_64bits()) { 4038 1.1 christos CHECK_EQ(ZSTD_NbCommonBytes(lowbit_only_32), 7u); 4039 1.1 christos CHECK_EQ(ZSTD_NbCommonBytes(highbit_only_32), 4u); 4040 1.1 christos } else { 4041 1.1 christos CHECK_EQ(ZSTD_NbCommonBytes(lowbit_only_32), 3u); 4042 1.1 christos CHECK_EQ(ZSTD_NbCommonBytes(highbit_only_32), 0u); 4043 1.1 christos } 4044 1.1 christos } 4045 1.1 christos 4046 1.1 christos /* Test MEM_ intrinsics */ 4047 1.1 christos CHECK_EQ(MEM_swap32(rand32), MEM_swap32_fallback(rand32)); 4048 1.1 christos CHECK_EQ(MEM_swap64(rand64), MEM_swap64_fallback(rand64)); 4049 1.1 christos 4050 1.1 christos /* Test fallbacks vs intrinsics on a range of small integers */ 4051 1.1 christos for (i=1; i <= 1000; i++) { 4052 1.1 christos CHECK_EQ(MEM_swap32(i), MEM_swap32_fallback(i)); 4053 1.1 christos CHECK_EQ(MEM_swap64((U64)i), MEM_swap64_fallback((U64)i)); 4054 1.1 christos CHECK_EQ(ZSTD_countTrailingZeros32(i), ZSTD_countTrailingZeros32_fallback(i)); 4055 1.1 christos CHECK_EQ(ZSTD_countLeadingZeros32(i), ZSTD_countLeadingZeros32_fallback(i)); 4056 1.1 christos } 4057 1.1 christos } 4058 1.1 christos DISPLAYLEVEL(3, "OK \n"); 4059 1.1 christos 4060 1.1 christos #ifdef ZSTD_MULTITHREAD 4061 1.1 christos DISPLAYLEVEL(3, "test%3i : passing wrong full dict should fail on compressStream2 refPrefix ", testNb++); 4062 1.1 christos { ZSTD_CCtx* cctx = ZSTD_createCCtx(); 4063 1.1 christos size_t const srcSize = 1 MB + 5; /* A little more than ZSTDMT_JOBSIZE_MIN */ 4064 1.1 christos size_t const dstSize = ZSTD_compressBound(srcSize); 4065 1.1 christos void* const src = CNBuffer; 4066 1.1 christos void* const dst = compressedBuffer; 4067 1.1 christos void* dict = (void*)malloc(srcSize); 4068 1.1 christos 4069 1.1 christos RDG_genBuffer(src, srcSize, compressibility, 0.5, seed); 4070 1.1 christos RDG_genBuffer(dict, srcSize, compressibility, 0., seed); 4071 1.1 christos 4072 1.1 christos /* Make sure there is no ZSTD_MAGIC_NUMBER */ 4073 1.1 christos memset(dict, 0, sizeof(U32)); 4074 1.1 christos 4075 1.1 christos /* something more than 1 */ 4076 1.1 christos CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 2)); 4077 1.1 christos /* lie and claim this is a full dict */ 4078 1.1 christos CHECK_Z(ZSTD_CCtx_refPrefix_advanced(cctx, dict, srcSize, ZSTD_dct_fullDict)); 4079 1.1 christos 4080 1.1 christos { ZSTD_outBuffer out = {dst, dstSize, 0}; 4081 1.1 christos ZSTD_inBuffer in = {src, srcSize, 0}; 4082 1.1 christos /* should fail because its not a full dict like we said it was */ 4083 1.1 christos assert(ZSTD_isError(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush))); 4084 1.1 christos } 4085 1.1 christos 4086 1.1 christos ZSTD_freeCCtx(cctx); 4087 1.1 christos free(dict); 4088 1.1 christos } 4089 1.1 christos DISPLAYLEVEL(3, "OK \n"); 4090 1.1 christos 4091 1.1 christos DISPLAYLEVEL(3, "test%3i : small dictionary with multithreading and LDM ", testNb++); 4092 1.1 christos { ZSTD_CCtx* cctx = ZSTD_createCCtx(); 4093 1.1 christos size_t const srcSize = 1 MB + 5; /* A little more than ZSTDMT_JOBSIZE_MIN */ 4094 1.1 christos size_t const dictSize = 10; 4095 1.1 christos size_t const dstSize = ZSTD_compressBound(srcSize); 4096 1.1 christos void* const src = CNBuffer; 4097 1.1 christos void* const dst = compressedBuffer; 4098 1.1 christos void* dict = (void*)malloc(dictSize); 4099 1.1 christos 4100 1.1 christos RDG_genBuffer(src, srcSize, compressibility, 0.5, seed); 4101 1.1 christos RDG_genBuffer(dict, dictSize, compressibility, 0., seed); 4102 1.1 christos 4103 1.1 christos /* Make sure there is no ZSTD_MAGIC_NUMBER */ 4104 1.1 christos memset(dict, 0, sizeof(U32)); 4105 1.1 christos 4106 1.1 christos /* Enable MT, LDM, and use refPrefix() for a small dict */ 4107 1.1 christos CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 2)); 4108 1.1 christos CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable)); 4109 1.1 christos CHECK_Z(ZSTD_CCtx_refPrefix(cctx, dict, dictSize)); 4110 1.1 christos 4111 1.1 christos CHECK_Z(ZSTD_compress2(cctx, dst, dstSize, src, srcSize)); 4112 1.1 christos 4113 1.1 christos ZSTD_freeCCtx(cctx); 4114 1.1 christos free(dict); 4115 1.1 christos } 4116 1.1 christos DISPLAYLEVEL(3, "OK \n"); 4117 1.1 christos 4118 1.1 christos DISPLAYLEVEL(3, "test%3i : ZSTD_getCParams() + dictionary ", testNb++); 4119 1.1 christos { 4120 1.1 christos ZSTD_compressionParameters const medium = ZSTD_getCParams(1, 16*1024-1, 0); 4121 1.1 christos ZSTD_compressionParameters const large = ZSTD_getCParams(1, 128*1024-1, 0); 4122 1.1 christos ZSTD_compressionParameters const smallDict = ZSTD_getCParams(1, 0, 400); 4123 1.1 christos ZSTD_compressionParameters const mediumDict = ZSTD_getCParams(1, 0, 10000); 4124 1.1 christos ZSTD_compressionParameters const largeDict = ZSTD_getCParams(1, 0, 100000); 4125 1.1 christos 4126 1.1 christos assert(!memcmp(&smallDict, &mediumDict, sizeof(smallDict))); 4127 1.1 christos assert(!memcmp(&medium, &mediumDict, sizeof(medium))); 4128 1.1 christos assert(!memcmp(&large, &largeDict, sizeof(large))); 4129 1.1 christos } 4130 1.1 christos DISPLAYLEVEL(3, "OK \n"); 4131 1.1 christos 4132 1.1 christos DISPLAYLEVEL(3, "test%3i : ZSTD_adjustCParams() + dictionary ", testNb++); 4133 1.1 christos { 4134 1.1 christos ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, 0, 0); 4135 1.1 christos ZSTD_compressionParameters const smallDict = ZSTD_adjustCParams(cParams, 0, 400); 4136 1.1 christos ZSTD_compressionParameters const smallSrcAndDict = ZSTD_adjustCParams(cParams, 500, 400); 4137 1.1 christos 4138 1.1 christos assert(smallSrcAndDict.windowLog == 10); 4139 1.1 christos assert(!memcmp(&cParams, &smallDict, sizeof(cParams))); 4140 1.1 christos } 4141 1.1 christos DISPLAYLEVEL(3, "OK \n"); 4142 1.1 christos 4143 1.1 christos DISPLAYLEVEL(3, "test%3i : check compression mem usage monotonicity over levels for estimateCCtxSize() : ", testNb++); 4144 1.1 christos { 4145 1.1 christos int level = 1; 4146 1.1 christos size_t prevSize = 0; 4147 1.1 christos for (; level < ZSTD_maxCLevel(); ++level) { 4148 1.1 christos size_t const currSize = ZSTD_estimateCCtxSize(level); 4149 1.1 christos if (prevSize > currSize) { 4150 1.1 christos DISPLAYLEVEL(3, "Error! previous cctx size: %zu at level: %d is larger than current cctx size: %zu at level: %d", 4151 1.1 christos prevSize, level-1, currSize, level); 4152 1.1 christos goto _output_error; 4153 1.1 christos } 4154 1.1 christos prevSize = currSize; 4155 1.1 christos } 4156 1.1 christos } 4157 1.1 christos DISPLAYLEVEL(3, "OK \n"); 4158 1.1 christos 4159 1.1 christos DISPLAYLEVEL(3, "test%3i : check estimateCCtxSize() always larger or equal to ZSTD_estimateCCtxSize_usingCParams() : ", testNb++); 4160 1.1 christos { 4161 1.1 christos size_t const kSizeIncrement = 2 KB; 4162 1.1 christos int level = -3; 4163 1.1 christos 4164 1.1 christos for (; level <= ZSTD_maxCLevel(); ++level) { 4165 1.1 christos size_t dictSize = 0; 4166 1.1 christos for (; dictSize <= 256 KB; dictSize += 8 * kSizeIncrement) { 4167 1.1 christos size_t srcSize = 2 KB; 4168 1.1 christos for (; srcSize < 300 KB; srcSize += kSizeIncrement) { 4169 1.1 christos ZSTD_compressionParameters const cParams = ZSTD_getCParams(level, srcSize, dictSize); 4170 1.1 christos size_t const cctxSizeUsingCParams = ZSTD_estimateCCtxSize_usingCParams(cParams); 4171 1.1 christos size_t const cctxSizeUsingLevel = ZSTD_estimateCCtxSize(level); 4172 1.1 christos if (cctxSizeUsingLevel < cctxSizeUsingCParams 4173 1.1 christos || ZSTD_isError(cctxSizeUsingCParams) 4174 1.1 christos || ZSTD_isError(cctxSizeUsingLevel)) { 4175 1.1 christos DISPLAYLEVEL(3, "error! l: %d dict: %zu srcSize: %zu cctx size cpar: %zu, cctx size level: %zu\n", 4176 1.1 christos level, dictSize, srcSize, cctxSizeUsingCParams, cctxSizeUsingLevel); 4177 1.1 christos goto _output_error; 4178 1.1 christos } } } } } 4179 1.1 christos DISPLAYLEVEL(3, "OK \n"); 4180 1.1 christos 4181 1.1 christos DISPLAYLEVEL(3, "test%3i : thread pool API tests : \n", testNb++) 4182 1.1 christos { 4183 1.1 christos int const threadPoolTestResult = threadPoolTests(); 4184 1.1 christos if (threadPoolTestResult) { 4185 1.1 christos goto _output_error; 4186 1.1 christos } 4187 1.1 christos } 4188 1.1 christos DISPLAYLEVEL(3, "thread pool tests OK \n"); 4189 1.1 christos 4190 1.1 christos #endif /* ZSTD_MULTITHREAD */ 4191 1.1 christos 4192 1.1 christos _end: 4193 1.1 christos free(CNBuffer); 4194 1.1 christos free(compressedBuffer); 4195 1.1 christos free(decodedBuffer); 4196 1.1 christos return testResult; 4197 1.1 christos 4198 1.1 christos _output_error: 4199 1.1 christos testResult = 1; 4200 1.1 christos DISPLAY("Error detected in Unit tests ! \n"); 4201 1.1 christos goto _end; 4202 1.1 christos } 4203 1.1 christos 4204 1.1 christos static int longUnitTests(U32 const seed, double compressibility) 4205 1.1 christos { 4206 1.1 christos size_t const CNBuffSize = 5 MB; 4207 1.1 christos void* const CNBuffer = malloc(CNBuffSize); 4208 1.1 christos size_t const compressedBufferSize = ZSTD_compressBound(CNBuffSize); 4209 1.1 christos void* const compressedBuffer = malloc(compressedBufferSize); 4210 1.1 christos void* const decodedBuffer = malloc(CNBuffSize); 4211 1.1 christos int testResult = 0; 4212 1.1 christos unsigned testNb=0; 4213 1.1 christos size_t cSize; 4214 1.1 christos 4215 1.1 christos /* Create compressible noise */ 4216 1.1 christos if (!CNBuffer || !compressedBuffer || !decodedBuffer) { 4217 1.1 christos DISPLAY("Not enough memory, aborting\n"); 4218 1.1 christos testResult = 1; 4219 1.1 christos goto _end; 4220 1.1 christos } 4221 1.1 christos RDG_genBuffer(CNBuffer, CNBuffSize, compressibility, 0., seed); 4222 1.1 christos 4223 1.1 christos /* note : this test is rather long, it would be great to find a way to speed up its execution */ 4224 1.1 christos DISPLAYLEVEL(3, "longtest%3i : table cleanliness through index reduction : ", testNb++); 4225 1.1 christos { int cLevel; 4226 1.1 christos size_t approxIndex = 0; 4227 1.1 christos size_t maxIndex = ((3U << 29) + (1U << ZSTD_WINDOWLOG_MAX)); /* ZSTD_CURRENT_MAX from zstd_compress_internal.h */ 4228 1.1 christos 4229 1.1 christos /* Provision enough space in a static context so that we can do all 4230 1.1 christos * this without ever reallocating, which would reset the indices. */ 4231 1.1 christos size_t const staticCCtxSize = ZSTD_estimateCStreamSize(22); 4232 1.1 christos void* const staticCCtxBuffer = malloc(staticCCtxSize); 4233 1.1 christos ZSTD_CCtx* const cctx = ZSTD_initStaticCCtx(staticCCtxBuffer, staticCCtxSize); 4234 1.1 christos 4235 1.1 christos /* bump the indices so the following compressions happen at high 4236 1.1 christos * indices. */ 4237 1.1 christos { ZSTD_outBuffer out = { compressedBuffer, compressedBufferSize, 0 }; 4238 1.1 christos ZSTD_inBuffer in = { CNBuffer, CNBuffSize, 0 }; 4239 1.1 christos ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters); 4240 1.1 christos CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, -500)); 4241 1.1 christos while (approxIndex <= (maxIndex / 4) * 3) { 4242 1.1 christos CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush)); 4243 1.1 christos approxIndex += in.pos; 4244 1.1 christos CHECK_Z(in.pos == in.size); 4245 1.1 christos in.pos = 0; 4246 1.1 christos out.pos = 0; 4247 1.1 christos } 4248 1.1 christos CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end)); 4249 1.1 christos } 4250 1.1 christos 4251 1.1 christos /* spew a bunch of stuff into the table area */ 4252 1.1 christos for (cLevel = 1; cLevel <= 22; cLevel++) { 4253 1.1 christos ZSTD_outBuffer out = { compressedBuffer, compressedBufferSize / (unsigned)cLevel, 0 }; 4254 1.1 christos ZSTD_inBuffer in = { CNBuffer, CNBuffSize, 0 }; 4255 1.1 christos ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters); 4256 1.1 christos CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, cLevel)); 4257 1.1 christos CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush)); 4258 1.1 christos CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end)); 4259 1.1 christos approxIndex += in.pos; 4260 1.1 christos } 4261 1.1 christos 4262 1.1 christos /* now crank the indices so we overflow */ 4263 1.1 christos { ZSTD_outBuffer out = { compressedBuffer, compressedBufferSize, 0 }; 4264 1.1 christos ZSTD_inBuffer in = { CNBuffer, CNBuffSize, 0 }; 4265 1.1 christos ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters); 4266 1.1 christos CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, -500)); 4267 1.1 christos while (approxIndex <= maxIndex) { 4268 1.1 christos CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush)); 4269 1.1 christos approxIndex += in.pos; 4270 1.1 christos CHECK_Z(in.pos == in.size); 4271 1.1 christos in.pos = 0; 4272 1.1 christos out.pos = 0; 4273 1.1 christos } 4274 1.1 christos CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end)); 4275 1.1 christos } 4276 1.1 christos 4277 1.1 christos /* do a bunch of compressions again in low indices and ensure we don't 4278 1.1 christos * hit untracked invalid indices */ 4279 1.1 christos for (cLevel = 1; cLevel <= 22; cLevel++) { 4280 1.1 christos ZSTD_outBuffer out = { compressedBuffer, compressedBufferSize / (unsigned)cLevel, 0 }; 4281 1.1 christos ZSTD_inBuffer in = { CNBuffer, CNBuffSize, 0 }; 4282 1.1 christos ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters); 4283 1.1 christos CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, cLevel)); 4284 1.1 christos CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush)); 4285 1.1 christos CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end)); 4286 1.1 christos approxIndex += in.pos; 4287 1.1 christos } 4288 1.1 christos 4289 1.1 christos free(staticCCtxBuffer); 4290 1.1 christos } 4291 1.1 christos DISPLAYLEVEL(3, "OK \n"); 4292 1.1 christos 4293 1.1 christos DISPLAYLEVEL(3, "longtest%3i : testing ldm no regressions in size for opt parser : ", testNb++); 4294 1.1 christos { size_t cSizeLdm; 4295 1.1 christos size_t cSizeNoLdm; 4296 1.1 christos ZSTD_CCtx* const cctx = ZSTD_createCCtx(); 4297 1.1 christos 4298 1.1 christos RDG_genBuffer(CNBuffer, CNBuffSize, 0.5, 0.5, seed); 4299 1.1 christos 4300 1.1 christos /* Enable checksum to verify round trip. */ 4301 1.1 christos CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1)); 4302 1.1 christos CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable)); 4303 1.1 christos CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 19)); 4304 1.1 christos 4305 1.1 christos /* Round trip once with ldm. */ 4306 1.1 christos cSizeLdm = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize); 4307 1.1 christos CHECK_Z(cSizeLdm); 4308 1.1 christos CHECK_Z(ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSizeLdm)); 4309 1.1 christos 4310 1.1 christos ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters); 4311 1.1 christos CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1)); 4312 1.1 christos CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_disable)); 4313 1.1 christos CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 19)); 4314 1.1 christos 4315 1.1 christos /* Round trip once without ldm. */ 4316 1.1 christos cSizeNoLdm = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize); 4317 1.1 christos CHECK_Z(cSizeNoLdm); 4318 1.1 christos CHECK_Z(ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSizeNoLdm)); 4319 1.1 christos 4320 1.1 christos if (cSizeLdm > cSizeNoLdm) { 4321 1.1 christos DISPLAY("Using long mode should not cause regressions for btopt+\n"); 4322 1.1 christos testResult = 1; 4323 1.1 christos goto _end; 4324 1.1 christos } 4325 1.1 christos 4326 1.1 christos ZSTD_freeCCtx(cctx); 4327 1.1 christos } 4328 1.1 christos DISPLAYLEVEL(3, "OK \n"); 4329 1.1 christos 4330 1.1 christos DISPLAYLEVEL(3, "longtest%3i : testing cdict compression with different attachment strategies : ", testNb++); 4331 1.1 christos { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); 4332 1.1 christos ZSTD_DCtx* const dctx = ZSTD_createDCtx(); 4333 1.1 christos size_t dictSize = CNBuffSize; 4334 1.1 christos void* dict = (void*)malloc(dictSize); 4335 1.1 christos ZSTD_CCtx_params* cctx_params = ZSTD_createCCtxParams(); 4336 1.1 christos ZSTD_dictAttachPref_e const attachPrefs[] = { 4337 1.1 christos ZSTD_dictDefaultAttach, 4338 1.1 christos ZSTD_dictForceAttach, 4339 1.1 christos ZSTD_dictForceCopy, 4340 1.1 christos ZSTD_dictForceLoad, 4341 1.1 christos ZSTD_dictDefaultAttach, 4342 1.1 christos ZSTD_dictForceAttach, 4343 1.1 christos ZSTD_dictForceCopy, 4344 1.1 christos ZSTD_dictForceLoad 4345 1.1 christos }; 4346 1.1 christos int const enableDedicatedDictSearch[] = {0, 0, 0, 0, 1, 1, 1, 1}; 4347 1.1 christos int cLevel; 4348 1.1 christos int i; 4349 1.1 christos 4350 1.1 christos RDG_genBuffer(dict, dictSize, 0.5, 0.5, seed); 4351 1.1 christos RDG_genBuffer(CNBuffer, CNBuffSize, 0.6, 0.6, seed); 4352 1.1 christos 4353 1.1 christos CHECK_Z(cctx_params != NULL); 4354 1.1 christos 4355 1.1 christos for (dictSize = CNBuffSize; dictSize; dictSize = dictSize >> 3) { 4356 1.1 christos DISPLAYLEVEL(3, "\n Testing with dictSize %u ", (U32)dictSize); 4357 1.1 christos for (cLevel = 4; cLevel < 13; cLevel++) { 4358 1.1 christos for (i = 0; i < 8; ++i) { 4359 1.1 christos ZSTD_dictAttachPref_e const attachPref = attachPrefs[i]; 4360 1.1 christos int const enableDDS = enableDedicatedDictSearch[i]; 4361 1.1 christos ZSTD_CDict* cdict; 4362 1.1 christos 4363 1.1 christos DISPLAYLEVEL(5, "\n dictSize %u cLevel %d iter %d ", (U32)dictSize, cLevel, i); 4364 1.1 christos 4365 1.1 christos ZSTD_CCtxParams_init(cctx_params, cLevel); 4366 1.1 christos CHECK_Z(ZSTD_CCtxParams_setParameter(cctx_params, ZSTD_c_enableDedicatedDictSearch, enableDDS)); 4367 1.1 christos 4368 1.1 christos cdict = ZSTD_createCDict_advanced2(dict, dictSize, ZSTD_dlm_byRef, ZSTD_dct_auto, cctx_params, ZSTD_defaultCMem); 4369 1.1 christos CHECK(cdict != NULL); 4370 1.1 christos 4371 1.1 christos CHECK_Z(ZSTD_CCtx_refCDict(cctx, cdict)); 4372 1.1 christos CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_forceAttachDict, (int)attachPref)); 4373 1.1 christos 4374 1.1 christos cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize); 4375 1.1 christos CHECK_Z(cSize); 4376 1.1 christos CHECK_Z(ZSTD_decompress_usingDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, dict, dictSize)); 4377 1.1 christos 4378 1.1 christos DISPLAYLEVEL(5, "compressed to %u bytes ", (U32)cSize); 4379 1.1 christos 4380 1.1 christos CHECK_Z(ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters)); 4381 1.1 christos ZSTD_freeCDict(cdict); 4382 1.1 christos } } } 4383 1.1 christos 4384 1.1 christos ZSTD_freeCCtx(cctx); 4385 1.1 christos ZSTD_freeDCtx(dctx); 4386 1.1 christos ZSTD_freeCCtxParams(cctx_params); 4387 1.1 christos free(dict); 4388 1.1 christos } 4389 1.1 christos DISPLAYLEVEL(3, "OK \n"); 4390 1.1 christos 4391 1.1 christos _end: 4392 1.1 christos free(CNBuffer); 4393 1.1 christos free(compressedBuffer); 4394 1.1 christos free(decodedBuffer); 4395 1.1 christos return testResult; 4396 1.1 christos } 4397 1.1 christos 4398 1.1 christos 4399 1.1 christos static size_t findDiff(const void* buf1, const void* buf2, size_t max) 4400 1.1 christos { 4401 1.1 christos const BYTE* b1 = (const BYTE*)buf1; 4402 1.1 christos const BYTE* b2 = (const BYTE*)buf2; 4403 1.1 christos size_t u; 4404 1.1 christos for (u=0; u<max; u++) { 4405 1.1 christos if (b1[u] != b2[u]) break; 4406 1.1 christos } 4407 1.1 christos return u; 4408 1.1 christos } 4409 1.1 christos 4410 1.1 christos 4411 1.1 christos static ZSTD_parameters FUZ_makeParams(ZSTD_compressionParameters cParams, ZSTD_frameParameters fParams) 4412 1.1 christos { 4413 1.1 christos ZSTD_parameters params; 4414 1.1 christos params.cParams = cParams; 4415 1.1 christos params.fParams = fParams; 4416 1.1 christos return params; 4417 1.1 christos } 4418 1.1 christos 4419 1.1 christos static size_t FUZ_rLogLength(U32* seed, U32 logLength) 4420 1.1 christos { 4421 1.1 christos size_t const lengthMask = ((size_t)1 << logLength) - 1; 4422 1.1 christos return (lengthMask+1) + (FUZ_rand(seed) & lengthMask); 4423 1.1 christos } 4424 1.1 christos 4425 1.1 christos static size_t FUZ_randomLength(U32* seed, U32 maxLog) 4426 1.1 christos { 4427 1.1 christos U32 const logLength = FUZ_rand(seed) % maxLog; 4428 1.1 christos return FUZ_rLogLength(seed, logLength); 4429 1.1 christos } 4430 1.1 christos 4431 1.1 christos #undef CHECK 4432 1.1 christos #define CHECK(cond, ...) { \ 4433 1.1 christos if (cond) { \ 4434 1.1 christos DISPLAY("Error => "); \ 4435 1.1 christos DISPLAY(__VA_ARGS__); \ 4436 1.1 christos DISPLAY(" (seed %u, test nb %u) \n", (unsigned)seed, testNb); \ 4437 1.1 christos goto _output_error; \ 4438 1.1 christos } } 4439 1.1 christos 4440 1.1 christos #undef CHECK_Z 4441 1.1 christos #define CHECK_Z(f) { \ 4442 1.1 christos size_t const err = f; \ 4443 1.1 christos if (ZSTD_isError(err)) { \ 4444 1.1 christos DISPLAY("Error => %s : %s ", \ 4445 1.1 christos #f, ZSTD_getErrorName(err)); \ 4446 1.1 christos DISPLAY(" (seed %u, test nb %u) \n", (unsigned)seed, testNb); \ 4447 1.1 christos goto _output_error; \ 4448 1.1 christos } } 4449 1.1 christos 4450 1.1 christos 4451 1.1 christos static int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, U32 const maxDurationS, double compressibility, int bigTests) 4452 1.1 christos { 4453 1.1 christos static const U32 maxSrcLog = 23; 4454 1.1 christos static const U32 maxSampleLog = 22; 4455 1.1 christos size_t const srcBufferSize = (size_t)1<<maxSrcLog; 4456 1.1 christos size_t const dstBufferSize = (size_t)1<<maxSampleLog; 4457 1.1 christos size_t const cBufferSize = ZSTD_compressBound(dstBufferSize); 4458 1.1 christos BYTE* cNoiseBuffer[5]; 4459 1.1 christos BYTE* const cBuffer = (BYTE*) malloc (cBufferSize); 4460 1.1 christos BYTE* const dstBuffer = (BYTE*) malloc (dstBufferSize); 4461 1.1 christos BYTE* const mirrorBuffer = (BYTE*) malloc (dstBufferSize); 4462 1.1 christos ZSTD_CCtx* const refCtx = ZSTD_createCCtx(); 4463 1.1 christos ZSTD_CCtx* const ctx = ZSTD_createCCtx(); 4464 1.1 christos ZSTD_DCtx* const dctx = ZSTD_createDCtx(); 4465 1.1 christos U32 result = 0; 4466 1.1 christos unsigned testNb = 0; 4467 1.1 christos U32 coreSeed = seed; 4468 1.1 christos UTIL_time_t const startClock = UTIL_getTime(); 4469 1.1 christos U64 const maxClockSpan = maxDurationS * SEC_TO_MICRO; 4470 1.1 christos int const cLevelLimiter = bigTests ? 3 : 2; 4471 1.1 christos 4472 1.1 christos /* allocation */ 4473 1.1 christos cNoiseBuffer[0] = (BYTE*)malloc (srcBufferSize); 4474 1.1 christos cNoiseBuffer[1] = (BYTE*)malloc (srcBufferSize); 4475 1.1 christos cNoiseBuffer[2] = (BYTE*)malloc (srcBufferSize); 4476 1.1 christos cNoiseBuffer[3] = (BYTE*)malloc (srcBufferSize); 4477 1.1 christos cNoiseBuffer[4] = (BYTE*)malloc (srcBufferSize); 4478 1.1 christos CHECK (!cNoiseBuffer[0] || !cNoiseBuffer[1] || !cNoiseBuffer[2] || !cNoiseBuffer[3] || !cNoiseBuffer[4] 4479 1.1 christos || !dstBuffer || !mirrorBuffer || !cBuffer || !refCtx || !ctx || !dctx, 4480 1.1 christos "Not enough memory, fuzzer tests cancelled"); 4481 1.1 christos 4482 1.1 christos /* Create initial samples */ 4483 1.1 christos RDG_genBuffer(cNoiseBuffer[0], srcBufferSize, 0.00, 0., coreSeed); /* pure noise */ 4484 1.1 christos RDG_genBuffer(cNoiseBuffer[1], srcBufferSize, 0.05, 0., coreSeed); /* barely compressible */ 4485 1.1 christos RDG_genBuffer(cNoiseBuffer[2], srcBufferSize, compressibility, 0., coreSeed); 4486 1.1 christos RDG_genBuffer(cNoiseBuffer[3], srcBufferSize, 0.95, 0., coreSeed); /* highly compressible */ 4487 1.1 christos RDG_genBuffer(cNoiseBuffer[4], srcBufferSize, 1.00, 0., coreSeed); /* sparse content */ 4488 1.1 christos 4489 1.1 christos /* catch up testNb */ 4490 1.1 christos for (testNb=1; testNb < startTest; testNb++) FUZ_rand(&coreSeed); 4491 1.1 christos 4492 1.1 christos /* main test loop */ 4493 1.1 christos for ( ; (testNb <= nbTests) || (UTIL_clockSpanMicro(startClock) < maxClockSpan); testNb++ ) { 4494 1.1 christos BYTE* srcBuffer; /* jumping pointer */ 4495 1.1 christos U32 lseed; 4496 1.1 christos size_t sampleSize, maxTestSize, totalTestSize; 4497 1.1 christos size_t cSize, totalCSize, totalGenSize; 4498 1.1 christos U64 crcOrig; 4499 1.1 christos BYTE* sampleBuffer; 4500 1.1 christos const BYTE* dict; 4501 1.1 christos size_t dictSize; 4502 1.1 christos 4503 1.1 christos /* notification */ 4504 1.1 christos if (nbTests >= testNb) { DISPLAYUPDATE(2, "\r%6u/%6u ", testNb, nbTests); } 4505 1.1 christos else { DISPLAYUPDATE(2, "\r%6u ", testNb); } 4506 1.1 christos 4507 1.1 christos FUZ_rand(&coreSeed); 4508 1.1 christos { U32 const prime1 = 2654435761U; lseed = coreSeed ^ prime1; } 4509 1.1 christos 4510 1.1 christos /* srcBuffer selection [0-4] */ 4511 1.1 christos { U32 buffNb = FUZ_rand(&lseed) & 0x7F; 4512 1.1 christos if (buffNb & 7) buffNb=2; /* most common : compressible (P) */ 4513 1.1 christos else { 4514 1.1 christos buffNb >>= 3; 4515 1.1 christos if (buffNb & 7) { 4516 1.1 christos const U32 tnb[2] = { 1, 3 }; /* barely/highly compressible */ 4517 1.1 christos buffNb = tnb[buffNb >> 3]; 4518 1.1 christos } else { 4519 1.1 christos const U32 tnb[2] = { 0, 4 }; /* not compressible / sparse */ 4520 1.1 christos buffNb = tnb[buffNb >> 3]; 4521 1.1 christos } } 4522 1.1 christos srcBuffer = cNoiseBuffer[buffNb]; 4523 1.1 christos } 4524 1.1 christos 4525 1.1 christos /* select src segment */ 4526 1.1 christos sampleSize = FUZ_randomLength(&lseed, maxSampleLog); 4527 1.1 christos 4528 1.1 christos /* create sample buffer (to catch read error with valgrind & sanitizers) */ 4529 1.1 christos sampleBuffer = (BYTE*)malloc(sampleSize); 4530 1.1 christos CHECK(sampleBuffer==NULL, "not enough memory for sample buffer"); 4531 1.1 christos { size_t const sampleStart = FUZ_rand(&lseed) % (srcBufferSize - sampleSize); 4532 1.1 christos memcpy(sampleBuffer, srcBuffer + sampleStart, sampleSize); } 4533 1.1 christos crcOrig = XXH64(sampleBuffer, sampleSize, 0); 4534 1.1 christos 4535 1.1 christos /* compression tests */ 4536 1.1 christos { int const cLevelPositive = (int) 4537 1.1 christos ( FUZ_rand(&lseed) % 4538 1.1 christos ((U32)ZSTD_maxCLevel() - (FUZ_highbit32((U32)sampleSize) / (U32)cLevelLimiter)) ) 4539 1.1 christos + 1; 4540 1.1 christos int const cLevel = ((FUZ_rand(&lseed) & 15) == 3) ? 4541 1.1 christos - (int)((FUZ_rand(&lseed) & 7) + 1) : /* test negative cLevel */ 4542 1.1 christos cLevelPositive; 4543 1.1 christos DISPLAYLEVEL(5, "fuzzer t%u: Simple compression test (level %i) \n", testNb, cLevel); 4544 1.1 christos cSize = ZSTD_compressCCtx(ctx, cBuffer, cBufferSize, sampleBuffer, sampleSize, cLevel); 4545 1.1 christos CHECK(ZSTD_isError(cSize), "ZSTD_compressCCtx failed : %s", ZSTD_getErrorName(cSize)); 4546 1.1 christos 4547 1.1 christos /* compression failure test : too small dest buffer */ 4548 1.1 christos assert(cSize > 3); 4549 1.1 christos { const size_t missing = (FUZ_rand(&lseed) % (cSize-2)) + 1; 4550 1.1 christos const size_t tooSmallSize = cSize - missing; 4551 1.1 christos const unsigned endMark = 0x4DC2B1A9; 4552 1.1 christos memcpy(dstBuffer+tooSmallSize, &endMark, sizeof(endMark)); 4553 1.1 christos DISPLAYLEVEL(5, "fuzzer t%u: compress into too small buffer of size %u (missing %u bytes) \n", 4554 1.1 christos testNb, (unsigned)tooSmallSize, (unsigned)missing); 4555 1.1 christos { size_t const errorCode = ZSTD_compressCCtx(ctx, dstBuffer, tooSmallSize, sampleBuffer, sampleSize, cLevel); 4556 1.1 christos CHECK(ZSTD_getErrorCode(errorCode) != ZSTD_error_dstSize_tooSmall, "ZSTD_compressCCtx should have failed ! (buffer too small : %u < %u)", (unsigned)tooSmallSize, (unsigned)cSize); } 4557 1.1 christos { unsigned endCheck; memcpy(&endCheck, dstBuffer+tooSmallSize, sizeof(endCheck)); 4558 1.1 christos CHECK(endCheck != endMark, "ZSTD_compressCCtx : dst buffer overflow (check.%08X != %08X.mark)", endCheck, endMark); } 4559 1.1 christos } } 4560 1.1 christos 4561 1.1 christos /* frame header decompression test */ 4562 1.1 christos { ZSTD_frameHeader zfh; 4563 1.1 christos CHECK_Z( ZSTD_getFrameHeader(&zfh, cBuffer, cSize) ); 4564 1.1 christos CHECK(zfh.frameContentSize != sampleSize, "Frame content size incorrect"); 4565 1.1 christos } 4566 1.1 christos 4567 1.1 christos /* Decompressed size test */ 4568 1.1 christos { unsigned long long const rSize = ZSTD_findDecompressedSize(cBuffer, cSize); 4569 1.1 christos CHECK(rSize != sampleSize, "decompressed size incorrect"); 4570 1.1 christos } 4571 1.1 christos 4572 1.1 christos /* successful decompression test */ 4573 1.1 christos DISPLAYLEVEL(5, "fuzzer t%u: simple decompression test \n", testNb); 4574 1.1 christos { size_t const margin = (FUZ_rand(&lseed) & 1) ? 0 : (FUZ_rand(&lseed) & 31) + 1; 4575 1.1 christos size_t const dSize = ZSTD_decompress(dstBuffer, sampleSize + margin, cBuffer, cSize); 4576 1.1 christos CHECK(dSize != sampleSize, "ZSTD_decompress failed (%s) (srcSize : %u ; cSize : %u)", ZSTD_getErrorName(dSize), (unsigned)sampleSize, (unsigned)cSize); 4577 1.1 christos { U64 const crcDest = XXH64(dstBuffer, sampleSize, 0); 4578 1.1 christos CHECK(crcOrig != crcDest, "decompression result corrupted (pos %u / %u)", (unsigned)findDiff(sampleBuffer, dstBuffer, sampleSize), (unsigned)sampleSize); 4579 1.1 christos } } 4580 1.1 christos 4581 1.1 christos free(sampleBuffer); /* no longer useful after this point */ 4582 1.1 christos 4583 1.1 christos /* truncated src decompression test */ 4584 1.1 christos DISPLAYLEVEL(5, "fuzzer t%u: decompression of truncated source \n", testNb); 4585 1.1 christos { size_t const missing = (FUZ_rand(&lseed) % (cSize-2)) + 1; /* no problem, as cSize > 4 (frameHeaderSizer) */ 4586 1.1 christos size_t const tooSmallSize = cSize - missing; 4587 1.1 christos void* cBufferTooSmall = malloc(tooSmallSize); /* valgrind will catch read overflows */ 4588 1.1 christos CHECK(cBufferTooSmall == NULL, "not enough memory !"); 4589 1.1 christos memcpy(cBufferTooSmall, cBuffer, tooSmallSize); 4590 1.1 christos { size_t const errorCode = ZSTD_decompress(dstBuffer, dstBufferSize, cBufferTooSmall, tooSmallSize); 4591 1.1 christos CHECK(!ZSTD_isError(errorCode), "ZSTD_decompress should have failed ! (truncated src buffer)"); } 4592 1.1 christos free(cBufferTooSmall); 4593 1.1 christos } 4594 1.1 christos 4595 1.1 christos /* too small dst decompression test */ 4596 1.1 christos DISPLAYLEVEL(5, "fuzzer t%u: decompress into too small dst buffer \n", testNb); 4597 1.1 christos if (sampleSize > 3) { 4598 1.1 christos size_t const missing = (FUZ_rand(&lseed) % (sampleSize-2)) + 1; /* no problem, as cSize > 4 (frameHeaderSizer) */ 4599 1.1 christos size_t const tooSmallSize = sampleSize - missing; 4600 1.1 christos static const BYTE token = 0xA9; 4601 1.1 christos dstBuffer[tooSmallSize] = token; 4602 1.1 christos { size_t const errorCode = ZSTD_decompress(dstBuffer, tooSmallSize, cBuffer, cSize); 4603 1.1 christos CHECK(ZSTD_getErrorCode(errorCode) != ZSTD_error_dstSize_tooSmall, "ZSTD_decompress should have failed : %u > %u (dst buffer too small)", (unsigned)errorCode, (unsigned)tooSmallSize); } 4604 1.1 christos CHECK(dstBuffer[tooSmallSize] != token, "ZSTD_decompress : dst buffer overflow"); 4605 1.1 christos } 4606 1.1 christos 4607 1.1 christos /* noisy src decompression test */ 4608 1.1 christos if (cSize > 6) { 4609 1.1 christos /* insert noise into src */ 4610 1.1 christos { U32 const maxNbBits = FUZ_highbit32((U32)(cSize-4)); 4611 1.1 christos size_t pos = 4; /* preserve magic number (too easy to detect) */ 4612 1.1 christos for (;;) { 4613 1.1 christos /* keep some original src */ 4614 1.1 christos { U32 const nbBits = FUZ_rand(&lseed) % maxNbBits; 4615 1.1 christos size_t const mask = (1<<nbBits) - 1; 4616 1.1 christos size_t const skipLength = FUZ_rand(&lseed) & mask; 4617 1.1 christos pos += skipLength; 4618 1.1 christos } 4619 1.1 christos if (pos >= cSize) break; 4620 1.1 christos /* add noise */ 4621 1.1 christos { U32 const nbBitsCodes = FUZ_rand(&lseed) % maxNbBits; 4622 1.1 christos U32 const nbBits = nbBitsCodes ? nbBitsCodes-1 : 0; 4623 1.1 christos size_t const mask = (1<<nbBits) - 1; 4624 1.1 christos size_t const rNoiseLength = (FUZ_rand(&lseed) & mask) + 1; 4625 1.1 christos size_t const noiseLength = MIN(rNoiseLength, cSize-pos); 4626 1.1 christos size_t const noiseStart = FUZ_rand(&lseed) % (srcBufferSize - noiseLength); 4627 1.1 christos memcpy(cBuffer + pos, srcBuffer + noiseStart, noiseLength); 4628 1.1 christos pos += noiseLength; 4629 1.1 christos } } } 4630 1.1 christos 4631 1.1 christos /* decompress noisy source */ 4632 1.1 christos DISPLAYLEVEL(5, "fuzzer t%u: decompress noisy source \n", testNb); 4633 1.1 christos { U32 const endMark = 0xA9B1C3D6; 4634 1.1 christos memcpy(dstBuffer+sampleSize, &endMark, 4); 4635 1.1 christos { size_t const decompressResult = ZSTD_decompress(dstBuffer, sampleSize, cBuffer, cSize); 4636 1.1 christos /* result *may* be an unlikely success, but even then, it must strictly respect dst buffer boundaries */ 4637 1.1 christos CHECK((!ZSTD_isError(decompressResult)) && (decompressResult>sampleSize), 4638 1.1 christos "ZSTD_decompress on noisy src : result is too large : %u > %u (dst buffer)", (unsigned)decompressResult, (unsigned)sampleSize); 4639 1.1 christos } 4640 1.1 christos { U32 endCheck; memcpy(&endCheck, dstBuffer+sampleSize, 4); 4641 1.1 christos CHECK(endMark!=endCheck, "ZSTD_decompress on noisy src : dst buffer overflow"); 4642 1.1 christos } } } /* noisy src decompression test */ 4643 1.1 christos 4644 1.1 christos /*===== Bufferless streaming compression test, scattered segments and dictionary =====*/ 4645 1.1 christos DISPLAYLEVEL(5, "fuzzer t%u: Bufferless streaming compression test \n", testNb); 4646 1.1 christos { U32 const testLog = FUZ_rand(&lseed) % maxSrcLog; 4647 1.1 christos U32 const dictLog = FUZ_rand(&lseed) % maxSrcLog; 4648 1.1 christos int const cLevel = (int)(FUZ_rand(&lseed) % 4649 1.1 christos ((U32)ZSTD_maxCLevel() - 4650 1.1 christos (MAX(testLog, dictLog) / (U32)cLevelLimiter))) + 4651 1.1 christos 1; 4652 1.1 christos maxTestSize = FUZ_rLogLength(&lseed, testLog); 4653 1.1 christos if (maxTestSize >= dstBufferSize) maxTestSize = dstBufferSize-1; 4654 1.1 christos 4655 1.1 christos dictSize = FUZ_rLogLength(&lseed, dictLog); /* needed also for decompression */ 4656 1.1 christos dict = srcBuffer + (FUZ_rand(&lseed) % (srcBufferSize - dictSize)); 4657 1.1 christos 4658 1.1 christos DISPLAYLEVEL(6, "fuzzer t%u: Compressing up to <=%u bytes at level %i with dictionary size %u \n", 4659 1.1 christos testNb, (unsigned)maxTestSize, cLevel, (unsigned)dictSize); 4660 1.1 christos 4661 1.1 christos if (FUZ_rand(&lseed) & 0xF) { 4662 1.1 christos CHECK_Z ( ZSTD_compressBegin_usingDict(refCtx, dict, dictSize, cLevel) ); 4663 1.1 christos } else { 4664 1.1 christos ZSTD_compressionParameters const cPar = ZSTD_getCParams(cLevel, ZSTD_CONTENTSIZE_UNKNOWN, dictSize); 4665 1.1 christos ZSTD_frameParameters const fPar = { FUZ_rand(&lseed)&1 /* contentSizeFlag */, 4666 1.1 christos !(FUZ_rand(&lseed)&3) /* contentChecksumFlag*/, 4667 1.1 christos 0 /*NodictID*/ }; /* note : since dictionary is fake, dictIDflag has no impact */ 4668 1.1 christos ZSTD_parameters const p = FUZ_makeParams(cPar, fPar); 4669 1.1 christos CHECK_Z ( ZSTD_compressBegin_advanced(refCtx, dict, dictSize, p, 0) ); 4670 1.1 christos } 4671 1.1 christos CHECK_Z( ZSTD_copyCCtx(ctx, refCtx, 0) ); 4672 1.1 christos } 4673 1.1 christos 4674 1.1 christos { U32 const nbChunks = (FUZ_rand(&lseed) & 127) + 2; 4675 1.1 christos U32 n; 4676 1.1 christos XXH64_state_t xxhState; 4677 1.1 christos XXH64_reset(&xxhState, 0); 4678 1.1 christos for (totalTestSize=0, cSize=0, n=0 ; n<nbChunks ; n++) { 4679 1.1 christos size_t const segmentSize = FUZ_randomLength(&lseed, maxSampleLog); 4680 1.1 christos size_t const segmentStart = FUZ_rand(&lseed) % (srcBufferSize - segmentSize); 4681 1.1 christos 4682 1.1 christos if (cBufferSize-cSize < ZSTD_compressBound(segmentSize)) break; /* avoid invalid dstBufferTooSmall */ 4683 1.1 christos if (totalTestSize+segmentSize > maxTestSize) break; 4684 1.1 christos 4685 1.1 christos { size_t const compressResult = ZSTD_compressContinue(ctx, cBuffer+cSize, cBufferSize-cSize, srcBuffer+segmentStart, segmentSize); 4686 1.1 christos CHECK (ZSTD_isError(compressResult), "multi-segments compression error : %s", ZSTD_getErrorName(compressResult)); 4687 1.1 christos cSize += compressResult; 4688 1.1 christos } 4689 1.1 christos XXH64_update(&xxhState, srcBuffer+segmentStart, segmentSize); 4690 1.1 christos memcpy(mirrorBuffer + totalTestSize, srcBuffer+segmentStart, segmentSize); 4691 1.1 christos totalTestSize += segmentSize; 4692 1.1 christos } 4693 1.1 christos 4694 1.1 christos { size_t const flushResult = ZSTD_compressEnd(ctx, cBuffer+cSize, cBufferSize-cSize, NULL, 0); 4695 1.1 christos CHECK (ZSTD_isError(flushResult), "multi-segments epilogue error : %s", ZSTD_getErrorName(flushResult)); 4696 1.1 christos cSize += flushResult; 4697 1.1 christos } 4698 1.1 christos crcOrig = XXH64_digest(&xxhState); 4699 1.1 christos } 4700 1.1 christos 4701 1.1 christos /* streaming decompression test */ 4702 1.1 christos DISPLAYLEVEL(5, "fuzzer t%u: Bufferless streaming decompression test \n", testNb); 4703 1.1 christos /* ensure memory requirement is good enough (should always be true) */ 4704 1.1 christos { ZSTD_frameHeader zfh; 4705 1.1 christos CHECK( ZSTD_getFrameHeader(&zfh, cBuffer, ZSTD_FRAMEHEADERSIZE_MAX), 4706 1.1 christos "ZSTD_getFrameHeader(): error retrieving frame information"); 4707 1.1 christos { size_t const roundBuffSize = ZSTD_decodingBufferSize_min(zfh.windowSize, zfh.frameContentSize); 4708 1.1 christos CHECK_Z(roundBuffSize); 4709 1.1 christos CHECK((roundBuffSize > totalTestSize) && (zfh.frameContentSize!=ZSTD_CONTENTSIZE_UNKNOWN), 4710 1.1 christos "ZSTD_decodingBufferSize_min() requires more memory (%u) than necessary (%u)", 4711 1.1 christos (unsigned)roundBuffSize, (unsigned)totalTestSize ); 4712 1.1 christos } } 4713 1.1 christos if (dictSize<8) dictSize=0, dict=NULL; /* disable dictionary */ 4714 1.1 christos CHECK_Z( ZSTD_decompressBegin_usingDict(dctx, dict, dictSize) ); 4715 1.1 christos totalCSize = 0; 4716 1.1 christos totalGenSize = 0; 4717 1.1 christos while (totalCSize < cSize) { 4718 1.1 christos size_t const inSize = ZSTD_nextSrcSizeToDecompress(dctx); 4719 1.1 christos size_t const genSize = ZSTD_decompressContinue(dctx, dstBuffer+totalGenSize, dstBufferSize-totalGenSize, cBuffer+totalCSize, inSize); 4720 1.1 christos CHECK (ZSTD_isError(genSize), "ZSTD_decompressContinue error : %s", ZSTD_getErrorName(genSize)); 4721 1.1 christos totalGenSize += genSize; 4722 1.1 christos totalCSize += inSize; 4723 1.1 christos } 4724 1.1 christos CHECK (ZSTD_nextSrcSizeToDecompress(dctx) != 0, "frame not fully decoded"); 4725 1.1 christos CHECK (totalGenSize != totalTestSize, "streaming decompressed data : wrong size") 4726 1.1 christos CHECK (totalCSize != cSize, "compressed data should be fully read") 4727 1.1 christos { U64 const crcDest = XXH64(dstBuffer, totalTestSize, 0); 4728 1.1 christos CHECK(crcOrig != crcDest, "streaming decompressed data corrupted (pos %u / %u)", 4729 1.1 christos (unsigned)findDiff(mirrorBuffer, dstBuffer, totalTestSize), (unsigned)totalTestSize); 4730 1.1 christos } 4731 1.1 christos } /* for ( ; (testNb <= nbTests) */ 4732 1.1 christos DISPLAY("\r%u fuzzer tests completed \n", testNb-1); 4733 1.1 christos 4734 1.1 christos _cleanup: 4735 1.1 christos ZSTD_freeCCtx(refCtx); 4736 1.1 christos ZSTD_freeCCtx(ctx); 4737 1.1 christos ZSTD_freeDCtx(dctx); 4738 1.1 christos free(cNoiseBuffer[0]); 4739 1.1 christos free(cNoiseBuffer[1]); 4740 1.1 christos free(cNoiseBuffer[2]); 4741 1.1 christos free(cNoiseBuffer[3]); 4742 1.1 christos free(cNoiseBuffer[4]); 4743 1.1 christos free(cBuffer); 4744 1.1 christos free(dstBuffer); 4745 1.1 christos free(mirrorBuffer); 4746 1.1 christos return (int)result; 4747 1.1 christos 4748 1.1 christos _output_error: 4749 1.1 christos result = 1; 4750 1.1 christos goto _cleanup; 4751 1.1 christos } 4752 1.1 christos 4753 1.1 christos 4754 1.1 christos /*_******************************************************* 4755 1.1 christos * Command line 4756 1.1 christos *********************************************************/ 4757 1.1 christos static int FUZ_usage(const char* programName) 4758 1.1 christos { 4759 1.1 christos DISPLAY( "Usage :\n"); 4760 1.1 christos DISPLAY( " %s [args]\n", programName); 4761 1.1 christos DISPLAY( "\n"); 4762 1.1 christos DISPLAY( "Arguments :\n"); 4763 1.1 christos DISPLAY( " -i# : Number of tests (default:%i)\n", nbTestsDefault); 4764 1.1 christos DISPLAY( " -T# : Max duration to run for. Overrides number of tests. (e.g. -T1m or -T60s for one minute)\n"); 4765 1.1 christos DISPLAY( " -s# : Select seed (default:prompt user)\n"); 4766 1.1 christos DISPLAY( " -t# : Select starting test number (default:0)\n"); 4767 1.1 christos DISPLAY( " -P# : Select compressibility in %% (default:%i%%)\n", FUZ_compressibility_default); 4768 1.1 christos DISPLAY( " -v : verbose\n"); 4769 1.1 christos DISPLAY( " -p : pause at the end\n"); 4770 1.1 christos DISPLAY( " -h : display help and exit\n"); 4771 1.1 christos return 0; 4772 1.1 christos } 4773 1.1 christos 4774 1.1 christos /*! readU32FromChar() : 4775 1.1 christos @return : unsigned integer value read from input in `char` format 4776 1.1 christos allows and interprets K, KB, KiB, M, MB and MiB suffix. 4777 1.1 christos Will also modify `*stringPtr`, advancing it to position where it stopped reading. 4778 1.1 christos Note : function result can overflow if digit string > MAX_UINT */ 4779 1.1 christos static unsigned readU32FromChar(const char** stringPtr) 4780 1.1 christos { 4781 1.1 christos unsigned result = 0; 4782 1.1 christos while ((**stringPtr >='0') && (**stringPtr <='9')) 4783 1.1 christos result *= 10, result += (unsigned)(**stringPtr - '0'), (*stringPtr)++ ; 4784 1.1 christos if ((**stringPtr=='K') || (**stringPtr=='M')) { 4785 1.1 christos result <<= 10; 4786 1.1 christos if (**stringPtr=='M') result <<= 10; 4787 1.1 christos (*stringPtr)++ ; 4788 1.1 christos if (**stringPtr=='i') (*stringPtr)++; 4789 1.1 christos if (**stringPtr=='B') (*stringPtr)++; 4790 1.1 christos } 4791 1.1 christos return result; 4792 1.1 christos } 4793 1.1 christos 4794 1.1 christos /** longCommandWArg() : 4795 1.1 christos * check if *stringPtr is the same as longCommand. 4796 1.1 christos * If yes, @return 1 and advances *stringPtr to the position which immediately follows longCommand. 4797 1.1 christos * @return 0 and doesn't modify *stringPtr otherwise. 4798 1.1 christos */ 4799 1.1 christos static int longCommandWArg(const char** stringPtr, const char* longCommand) 4800 1.1 christos { 4801 1.1 christos size_t const comSize = strlen(longCommand); 4802 1.1 christos int const result = !strncmp(*stringPtr, longCommand, comSize); 4803 1.1 christos if (result) *stringPtr += comSize; 4804 1.1 christos return result; 4805 1.1 christos } 4806 1.1 christos 4807 1.1 christos int main(int argc, const char** argv) 4808 1.1 christos { 4809 1.1 christos U32 seed = 0; 4810 1.1 christos int seedset = 0; 4811 1.1 christos int argNb; 4812 1.1 christos int nbTests = nbTestsDefault; 4813 1.1 christos int testNb = 0; 4814 1.1 christos int proba = FUZ_compressibility_default; 4815 1.1 christos double probfloat; 4816 1.1 christos int result = 0; 4817 1.1 christos U32 mainPause = 0; 4818 1.1 christos U32 maxDuration = 0; 4819 1.1 christos int bigTests = 1; 4820 1.1 christos int longTests = 0; 4821 1.1 christos U32 memTestsOnly = 0; 4822 1.1 christos const char* const programName = argv[0]; 4823 1.1 christos 4824 1.1 christos /* Check command line */ 4825 1.1 christos for (argNb=1; argNb<argc; argNb++) { 4826 1.1 christos const char* argument = argv[argNb]; 4827 1.1 christos if(!argument) continue; /* Protection if argument empty */ 4828 1.1 christos 4829 1.1 christos /* Handle commands. Aggregated commands are allowed */ 4830 1.1 christos if (argument[0]=='-') { 4831 1.1 christos 4832 1.1 christos if (longCommandWArg(&argument, "--memtest=")) { memTestsOnly = readU32FromChar(&argument); continue; } 4833 1.1 christos 4834 1.1 christos if (!strcmp(argument, "--memtest")) { memTestsOnly=1; continue; } 4835 1.1 christos if (!strcmp(argument, "--no-big-tests")) { bigTests=0; continue; } 4836 1.1 christos if (!strcmp(argument, "--long-tests")) { longTests=1; continue; } 4837 1.1 christos if (!strcmp(argument, "--no-long-tests")) { longTests=0; continue; } 4838 1.1 christos 4839 1.1 christos argument++; 4840 1.1 christos while (*argument!=0) { 4841 1.1 christos switch(*argument) 4842 1.1 christos { 4843 1.1 christos case 'h': 4844 1.1 christos return FUZ_usage(programName); 4845 1.1 christos 4846 1.1 christos case 'v': 4847 1.1 christos argument++; 4848 1.1 christos g_displayLevel++; 4849 1.1 christos break; 4850 1.1 christos 4851 1.1 christos case 'q': 4852 1.1 christos argument++; 4853 1.1 christos g_displayLevel--; 4854 1.1 christos break; 4855 1.1 christos 4856 1.1 christos case 'p': /* pause at the end */ 4857 1.1 christos argument++; 4858 1.1 christos mainPause = 1; 4859 1.1 christos break; 4860 1.1 christos 4861 1.1 christos case 'i': 4862 1.1 christos argument++; maxDuration = 0; 4863 1.1 christos nbTests = (int)readU32FromChar(&argument); 4864 1.1 christos break; 4865 1.1 christos 4866 1.1 christos case 'T': 4867 1.1 christos argument++; 4868 1.1 christos nbTests = 0; 4869 1.1 christos maxDuration = readU32FromChar(&argument); 4870 1.1 christos if (*argument=='s') argument++; /* seconds */ 4871 1.1 christos if (*argument=='m') maxDuration *= 60, argument++; /* minutes */ 4872 1.1 christos if (*argument=='n') argument++; 4873 1.1 christos break; 4874 1.1 christos 4875 1.1 christos case 's': 4876 1.1 christos argument++; 4877 1.1 christos seedset = 1; 4878 1.1 christos seed = readU32FromChar(&argument); 4879 1.1 christos break; 4880 1.1 christos 4881 1.1 christos case 't': 4882 1.1 christos argument++; 4883 1.1 christos testNb = (int)readU32FromChar(&argument); 4884 1.1 christos break; 4885 1.1 christos 4886 1.1 christos case 'P': /* compressibility % */ 4887 1.1 christos argument++; 4888 1.1 christos proba = (int)readU32FromChar(&argument); 4889 1.1 christos if (proba>100) proba = 100; 4890 1.1 christos break; 4891 1.1 christos 4892 1.1 christos default: 4893 1.1 christos return (FUZ_usage(programName), 1); 4894 1.1 christos } } } } /* for (argNb=1; argNb<argc; argNb++) */ 4895 1.1 christos 4896 1.1 christos /* Get Seed */ 4897 1.1 christos DISPLAY("Starting zstd tester (%i-bits, %s)\n", (int)(sizeof(size_t)*8), ZSTD_VERSION_STRING); 4898 1.1 christos 4899 1.1 christos if (!seedset) { 4900 1.1 christos time_t const t = time(NULL); 4901 1.1 christos U32 const h = XXH32(&t, sizeof(t), 1); 4902 1.1 christos seed = h % 10000; 4903 1.1 christos } 4904 1.1 christos 4905 1.1 christos DISPLAY("Seed = %u\n", (unsigned)seed); 4906 1.1 christos if (proba!=FUZ_compressibility_default) DISPLAY("Compressibility : %i%%\n", proba); 4907 1.1 christos 4908 1.1 christos probfloat = ((double)proba) / 100; 4909 1.1 christos 4910 1.1 christos if (memTestsOnly) { 4911 1.1 christos g_displayLevel = MAX(3, g_displayLevel); 4912 1.1 christos return FUZ_mallocTests(seed, probfloat, memTestsOnly); 4913 1.1 christos } 4914 1.1 christos 4915 1.1 christos if (nbTests < testNb) nbTests = testNb; 4916 1.1 christos 4917 1.1 christos if (testNb==0) { 4918 1.1 christos result = basicUnitTests(0, probfloat); /* constant seed for predictability */ 4919 1.1 christos 4920 1.1 christos if (!result && longTests) { 4921 1.1 christos result = longUnitTests(0, probfloat); 4922 1.1 christos } 4923 1.1 christos } 4924 1.1 christos if (!result) 4925 1.1 christos result = fuzzerTests(seed, (unsigned)nbTests, (unsigned)testNb, maxDuration, ((double)proba) / 100, bigTests); 4926 1.1 christos if (mainPause) { 4927 1.1 christos int unused; 4928 1.1 christos DISPLAY("Press Enter \n"); 4929 1.1 christos unused = getchar(); 4930 1.1 christos (void)unused; 4931 1.1 christos } 4932 1.1 christos return result; 4933 1.1 christos } 4934