1 /*- 2 * Copyright (c) 2017 Sean Purcell 3 * Copyright (c) 2023-2024 Klara, Inc. 4 * All rights reserved. 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR(S) ``AS IS'' AND ANY EXPRESS OR 16 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 17 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 18 * IN NO EVENT SHALL THE AUTHOR(S) BE LIABLE FOR ANY DIRECT, INDIRECT, 19 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 20 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 21 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 22 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 23 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 24 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25 */ 26 27 #include "archive_platform.h" 28 29 #ifdef HAVE_ERRNO_H 30 #include <errno.h> 31 #endif 32 #ifdef HAVE_LIMITS_H 33 #include <limits.h> 34 #endif 35 #ifdef HAVE_STDINT_H 36 #include <stdint.h> 37 #endif 38 #ifdef HAVE_STDLIB_H 39 #include <stdlib.h> 40 #endif 41 #ifdef HAVE_STRING_H 42 #include <string.h> 43 #endif 44 #ifdef HAVE_UNISTD_H 45 #include <unistd.h> 46 #endif 47 #ifdef HAVE_ZSTD_H 48 #include <zstd.h> 49 #endif 50 51 #include "archive.h" 52 #include "archive_private.h" 53 #include "archive_string.h" 54 #include "archive_write_private.h" 55 56 /* Don't compile this if we don't have zstd.h */ 57 58 struct private_data { 59 int compression_level; 60 int threads; 61 int long_distance; 62 #if HAVE_ZSTD_H && HAVE_ZSTD_compressStream 63 enum { 64 running, 65 finishing, 66 resetting, 67 } state; 68 int frame_per_file; 69 size_t min_frame_in; 70 size_t max_frame_in; 71 size_t min_frame_out; 72 size_t max_frame_out; 73 size_t cur_frame; 74 size_t cur_frame_in; 75 size_t cur_frame_out; 76 size_t total_in; 77 ZSTD_CStream *cstream; 78 ZSTD_outBuffer out; 79 #else 80 struct archive_write_program_data *pdata; 81 #endif 82 }; 83 84 /* If we don't have the library use default range values (zstdcli.c v1.4.0) */ 85 #define CLEVEL_MIN -99 86 #define CLEVEL_STD_MIN 0 /* prior to 1.3.4 and more recent without using --fast */ 87 #define CLEVEL_DEFAULT 3 88 #define CLEVEL_STD_MAX 19 /* without using --ultra */ 89 #define CLEVEL_MAX 22 90 91 #define LONG_STD 27 92 93 #define MINVER_NEGCLEVEL 10304 94 #define MINVER_MINCLEVEL 10306 95 #define MINVER_LONG 10302 96 97 static int archive_compressor_zstd_options(struct archive_write_filter *, 98 const char *, const char *); 99 static int archive_compressor_zstd_open(struct archive_write_filter *); 100 static int archive_compressor_zstd_write(struct archive_write_filter *, 101 const void *, size_t); 102 static int archive_compressor_zstd_flush(struct archive_write_filter *); 103 static int archive_compressor_zstd_close(struct archive_write_filter *); 104 static int archive_compressor_zstd_free(struct archive_write_filter *); 105 #if HAVE_ZSTD_H && HAVE_ZSTD_compressStream 106 static int drive_compressor(struct archive_write_filter *, 107 struct private_data *, int, const void *, size_t); 108 #endif 109 110 111 /* 112 * Add a zstd compression filter to this write handle. 113 */ 114 int 115 archive_write_add_filter_zstd(struct archive *_a) 116 { 117 struct archive_write *a = (struct archive_write *)_a; 118 struct archive_write_filter *f = __archive_write_allocate_filter(_a); 119 struct private_data *data; 120 archive_check_magic(&a->archive, ARCHIVE_WRITE_MAGIC, 121 ARCHIVE_STATE_NEW, "archive_write_add_filter_zstd"); 122 123 data = calloc(1, sizeof(*data)); 124 if (data == NULL) { 125 archive_set_error(&a->archive, ENOMEM, "Out of memory"); 126 return (ARCHIVE_FATAL); 127 } 128 f->data = data; 129 f->open = &archive_compressor_zstd_open; 130 f->options = &archive_compressor_zstd_options; 131 f->flush = &archive_compressor_zstd_flush; 132 f->close = &archive_compressor_zstd_close; 133 f->free = &archive_compressor_zstd_free; 134 f->code = ARCHIVE_FILTER_ZSTD; 135 f->name = "zstd"; 136 data->compression_level = CLEVEL_DEFAULT; 137 data->threads = 0; 138 data->long_distance = 0; 139 #if HAVE_ZSTD_H && HAVE_ZSTD_compressStream 140 data->frame_per_file = 0; 141 data->min_frame_in = 0; 142 data->max_frame_in = SIZE_MAX; 143 data->min_frame_out = 0; 144 data->max_frame_out = SIZE_MAX; 145 data->cur_frame_in = 0; 146 data->cur_frame_out = 0; 147 data->cstream = ZSTD_createCStream(); 148 if (data->cstream == NULL) { 149 free(data); 150 archive_set_error(&a->archive, ENOMEM, 151 "Failed to allocate zstd compressor object"); 152 return (ARCHIVE_FATAL); 153 } 154 155 return (ARCHIVE_OK); 156 #else 157 data->pdata = __archive_write_program_allocate("zstd"); 158 if (data->pdata == NULL) { 159 free(data); 160 archive_set_error(&a->archive, ENOMEM, "Out of memory"); 161 return (ARCHIVE_FATAL); 162 } 163 archive_set_error(&a->archive, ARCHIVE_ERRNO_MISC, 164 "Using external zstd program"); 165 return (ARCHIVE_WARN); 166 #endif 167 } 168 169 static int 170 archive_compressor_zstd_free(struct archive_write_filter *f) 171 { 172 struct private_data *data = (struct private_data *)f->data; 173 #if HAVE_ZSTD_H && HAVE_ZSTD_compressStream 174 ZSTD_freeCStream(data->cstream); 175 free(data->out.dst); 176 #else 177 __archive_write_program_free(data->pdata); 178 #endif 179 free(data); 180 f->data = NULL; 181 return (ARCHIVE_OK); 182 } 183 184 static int 185 string_to_number(const char *string, intmax_t *numberp) 186 { 187 char *end; 188 189 if (string == NULL || *string == '\0') 190 return (ARCHIVE_WARN); 191 *numberp = strtoimax(string, &end, 10); 192 if (end == string || *end != '\0' || errno == EOVERFLOW) { 193 *numberp = 0; 194 return (ARCHIVE_WARN); 195 } 196 return (ARCHIVE_OK); 197 } 198 199 #if HAVE_ZSTD_H && HAVE_ZSTD_compressStream 200 static int 201 string_to_size(const char *string, size_t *numberp) 202 { 203 uintmax_t number; 204 char *end; 205 unsigned int shift = 0; 206 207 if (string == NULL || *string == '\0' || *string == '-') 208 return (ARCHIVE_WARN); 209 number = strtoumax(string, &end, 10); 210 if (end > string) { 211 if (*end == 'K' || *end == 'k') { 212 shift = 10; 213 end++; 214 } else if (*end == 'M' || *end == 'm') { 215 shift = 20; 216 end++; 217 } else if (*end == 'G' || *end == 'g') { 218 shift = 30; 219 end++; 220 } 221 if (*end == 'B' || *end == 'b') { 222 end++; 223 } 224 } 225 if (end == string || *end != '\0' || errno == EOVERFLOW) { 226 return (ARCHIVE_WARN); 227 } 228 if (number > (uintmax_t)SIZE_MAX >> shift) { 229 return (ARCHIVE_WARN); 230 } 231 *numberp = (size_t)(number << shift); 232 return (ARCHIVE_OK); 233 } 234 #endif 235 236 /* 237 * Set write options. 238 */ 239 static int 240 archive_compressor_zstd_options(struct archive_write_filter *f, const char *key, 241 const char *value) 242 { 243 struct private_data *data = (struct private_data *)f->data; 244 245 if (strcmp(key, "compression-level") == 0) { 246 intmax_t level; 247 if (string_to_number(value, &level) != ARCHIVE_OK) { 248 archive_set_error(f->archive, ARCHIVE_ERRNO_MISC, 249 "compression-level invalid"); 250 return (ARCHIVE_FAILED); 251 } 252 /* If we don't have the library, hard-code the max level */ 253 int minimum = CLEVEL_MIN; 254 int maximum = CLEVEL_MAX; 255 #if HAVE_ZSTD_H && HAVE_ZSTD_compressStream 256 maximum = ZSTD_maxCLevel(); 257 #if ZSTD_VERSION_NUMBER >= MINVER_MINCLEVEL 258 if (ZSTD_versionNumber() >= MINVER_MINCLEVEL) { 259 minimum = ZSTD_minCLevel(); 260 } 261 else 262 #endif 263 if (ZSTD_versionNumber() < MINVER_NEGCLEVEL) { 264 minimum = CLEVEL_STD_MIN; 265 } 266 #endif 267 if (level < minimum || level > maximum) { 268 archive_set_error(f->archive, ARCHIVE_ERRNO_MISC, 269 "compression-level out of range"); 270 return (ARCHIVE_FAILED); 271 } 272 data->compression_level = (int)level; 273 return (ARCHIVE_OK); 274 } else if (strcmp(key, "threads") == 0) { 275 intmax_t threads; 276 if (string_to_number(value, &threads) != ARCHIVE_OK) { 277 archive_set_error(f->archive, ARCHIVE_ERRNO_MISC, 278 "threads invalid"); 279 return (ARCHIVE_FAILED); 280 } 281 282 #if defined(HAVE_SYSCONF) && defined(_SC_NPROCESSORS_ONLN) 283 if (threads == 0) { 284 threads = sysconf(_SC_NPROCESSORS_ONLN); 285 } 286 #elif !defined(__CYGWIN__) && defined(_WIN32_WINNT) && \ 287 _WIN32_WINNT >= 0x0601 /* _WIN32_WINNT_WIN7 */ 288 if (threads == 0) { 289 DWORD winCores = GetActiveProcessorCount( 290 ALL_PROCESSOR_GROUPS); 291 threads = (intmax_t)winCores; 292 } 293 #endif 294 if (threads < 0 || threads > INT_MAX) { 295 archive_set_error(f->archive, ARCHIVE_ERRNO_MISC, 296 "threads out of rnage"); 297 return (ARCHIVE_FAILED); 298 } 299 data->threads = (int)threads; 300 return (ARCHIVE_OK); 301 #if HAVE_ZSTD_H && HAVE_ZSTD_compressStream 302 } else if (strcmp(key, "frame-per-file") == 0) { 303 data->frame_per_file = 1; 304 return (ARCHIVE_OK); 305 } else if (strcmp(key, "min-frame-in") == 0) { 306 if (string_to_size(value, &data->min_frame_in) != ARCHIVE_OK) { 307 archive_set_error(f->archive, ARCHIVE_ERRNO_MISC, 308 "min-frame-in invalid"); 309 return (ARCHIVE_FAILED); 310 } 311 return (ARCHIVE_OK); 312 } else if (strcmp(key, "min-frame-out") == 0 || 313 strcmp(key, "min-frame-size") == 0) { 314 if (string_to_size(value, &data->min_frame_out) != ARCHIVE_OK) { 315 archive_set_error(f->archive, ARCHIVE_ERRNO_MISC, 316 "min-frame-out invalid"); 317 return (ARCHIVE_FAILED); 318 } 319 return (ARCHIVE_OK); 320 } else if (strcmp(key, "max-frame-in") == 0 || 321 strcmp(key, "max-frame-size") == 0) { 322 if (string_to_size(value, &data->max_frame_in) != ARCHIVE_OK || 323 data->max_frame_in < 1024) { 324 archive_set_error(f->archive, ARCHIVE_ERRNO_MISC, 325 "max-frame-size invalid"); 326 return (ARCHIVE_FAILED); 327 } 328 return (ARCHIVE_OK); 329 } else if (strcmp(key, "max-frame-out") == 0) { 330 if (string_to_size(value, &data->max_frame_out) != ARCHIVE_OK || 331 data->max_frame_out < 1024) { 332 archive_set_error(f->archive, ARCHIVE_ERRNO_MISC, 333 "max-frame-out invalid"); 334 return (ARCHIVE_FAILED); 335 } 336 return (ARCHIVE_OK); 337 #endif 338 } 339 else if (strcmp(key, "long") == 0) { 340 intmax_t long_distance; 341 if (string_to_number(value, &long_distance) != ARCHIVE_OK) { 342 archive_set_error(f->archive, ARCHIVE_ERRNO_MISC, 343 "long invalid"); 344 return (ARCHIVE_FAILED); 345 } 346 #if HAVE_ZSTD_H && HAVE_ZSTD_compressStream && ZSTD_VERSION_NUMBER >= MINVER_LONG 347 ZSTD_bounds bounds = ZSTD_cParam_getBounds(ZSTD_c_windowLog); 348 if (ZSTD_isError(bounds.error)) { 349 int max_distance = ((int)(sizeof(size_t) == 4 ? 30 : 31)); 350 if (((int)long_distance) < 10 || (int)long_distance > max_distance) { 351 archive_set_error(f->archive, ARCHIVE_ERRNO_MISC, 352 "long out of range"); 353 return (ARCHIVE_FAILED); 354 } 355 } else { 356 if ((int)long_distance < bounds.lowerBound || (int)long_distance > bounds.upperBound) { 357 archive_set_error(f->archive, ARCHIVE_ERRNO_MISC, 358 "long out of range"); 359 return (ARCHIVE_FAILED); 360 } 361 } 362 #else 363 int max_distance = ((int)(sizeof(size_t) == 4 ? 30 : 31)); 364 if (((int)long_distance) < 10 || (int)long_distance > max_distance) 365 return (ARCHIVE_FAILED); 366 #endif 367 data->long_distance = (int)long_distance; 368 return (ARCHIVE_OK); 369 } 370 371 /* Note: The "warn" return is just to inform the options 372 * supervisor that we didn't handle it. It will generate 373 * a suitable error if no one used this option. */ 374 return (ARCHIVE_WARN); 375 } 376 377 #if HAVE_ZSTD_H && HAVE_ZSTD_compressStream 378 /* 379 * Setup callback. 380 */ 381 static int 382 archive_compressor_zstd_open(struct archive_write_filter *f) 383 { 384 struct private_data *data = (struct private_data *)f->data; 385 386 if (data->out.dst == NULL) { 387 size_t bs = ZSTD_CStreamOutSize(), bpb; 388 if (f->archive->magic == ARCHIVE_WRITE_MAGIC) { 389 /* Buffer size should be a multiple number of 390 * the of bytes per block for performance. */ 391 bpb = archive_write_get_bytes_per_block(f->archive); 392 if (bpb > bs) 393 bs = bpb; 394 else if (bpb != 0) 395 bs -= bs % bpb; 396 } 397 data->out.size = bs; 398 data->out.pos = 0; 399 data->out.dst = malloc(data->out.size); 400 if (data->out.dst == NULL) { 401 archive_set_error(f->archive, ENOMEM, 402 "Can't allocate data for compression buffer"); 403 return (ARCHIVE_FATAL); 404 } 405 } 406 407 f->write = archive_compressor_zstd_write; 408 409 if (ZSTD_isError(ZSTD_initCStream(data->cstream, 410 data->compression_level))) { 411 archive_set_error(f->archive, ARCHIVE_ERRNO_MISC, 412 "Internal error initializing zstd compressor object"); 413 return (ARCHIVE_FATAL); 414 } 415 416 ZSTD_CCtx_setParameter(data->cstream, ZSTD_c_nbWorkers, data->threads); 417 418 ZSTD_CCtx_setParameter(data->cstream, ZSTD_c_checksumFlag, 1); 419 420 #if ZSTD_VERSION_NUMBER >= MINVER_LONG 421 ZSTD_CCtx_setParameter(data->cstream, ZSTD_c_windowLog, data->long_distance); 422 #endif 423 424 return (ARCHIVE_OK); 425 } 426 427 /* 428 * Write data to the compressed stream. 429 */ 430 static int 431 archive_compressor_zstd_write(struct archive_write_filter *f, const void *buff, 432 size_t length) 433 { 434 struct private_data *data = (struct private_data *)f->data; 435 436 return (drive_compressor(f, data, 0, buff, length)); 437 } 438 439 /* 440 * Flush the compressed stream. 441 */ 442 static int 443 archive_compressor_zstd_flush(struct archive_write_filter *f) 444 { 445 struct private_data *data = (struct private_data *)f->data; 446 447 if (data->frame_per_file && data->state == running) { 448 if (data->cur_frame_in > data->min_frame_in && 449 data->cur_frame_out > data->min_frame_out) { 450 data->state = finishing; 451 } 452 } 453 return (drive_compressor(f, data, 1, NULL, 0)); 454 } 455 456 /* 457 * Finish the compression... 458 */ 459 static int 460 archive_compressor_zstd_close(struct archive_write_filter *f) 461 { 462 struct private_data *data = (struct private_data *)f->data; 463 464 if (data->state == running) 465 data->state = finishing; 466 return (drive_compressor(f, data, 1, NULL, 0)); 467 } 468 469 /* 470 * Utility function to push input data through compressor, 471 * writing full output blocks as necessary. 472 */ 473 static int 474 drive_compressor(struct archive_write_filter *f, 475 struct private_data *data, int flush, const void *src, size_t length) 476 { 477 ZSTD_inBuffer in = { .src = src, .size = length, .pos = 0 }; 478 size_t ipos, opos, zstdret = 0; 479 int ret; 480 481 for (;;) { 482 ipos = in.pos; 483 opos = data->out.pos; 484 switch (data->state) { 485 case running: 486 if (in.pos == in.size) 487 return (ARCHIVE_OK); 488 zstdret = ZSTD_compressStream(data->cstream, 489 &data->out, &in); 490 if (ZSTD_isError(zstdret)) 491 goto zstd_fatal; 492 break; 493 case finishing: 494 zstdret = ZSTD_endStream(data->cstream, &data->out); 495 if (ZSTD_isError(zstdret)) 496 goto zstd_fatal; 497 if (zstdret == 0) 498 data->state = resetting; 499 break; 500 case resetting: 501 ZSTD_CCtx_reset(data->cstream, ZSTD_reset_session_only); 502 data->cur_frame++; 503 data->cur_frame_in = 0; 504 data->cur_frame_out = 0; 505 data->state = running; 506 break; 507 } 508 data->total_in += in.pos - ipos; 509 data->cur_frame_in += in.pos - ipos; 510 data->cur_frame_out += data->out.pos - opos; 511 if (data->state == running) { 512 if (data->cur_frame_in >= data->max_frame_in || 513 data->cur_frame_out >= data->max_frame_out) { 514 data->state = finishing; 515 } 516 } 517 if (data->out.pos == data->out.size || 518 (flush && data->out.pos > 0)) { 519 ret = __archive_write_filter(f->next_filter, 520 data->out.dst, data->out.pos); 521 if (ret != ARCHIVE_OK) 522 goto fatal; 523 data->out.pos = 0; 524 } 525 } 526 zstd_fatal: 527 archive_set_error(f->archive, ARCHIVE_ERRNO_MISC, 528 "Zstd compression failed: %s", 529 ZSTD_getErrorName(zstdret)); 530 fatal: 531 return (ARCHIVE_FATAL); 532 } 533 534 #else /* HAVE_ZSTD_H && HAVE_ZSTD_compressStream */ 535 536 static int 537 archive_compressor_zstd_open(struct archive_write_filter *f) 538 { 539 struct private_data *data = (struct private_data *)f->data; 540 struct archive_string as; 541 int r; 542 543 archive_string_init(&as); 544 /* --no-check matches library default */ 545 archive_strcpy(&as, "zstd --no-check"); 546 547 if (data->compression_level < CLEVEL_STD_MIN) { 548 archive_string_sprintf(&as, " --fast=%d", -data->compression_level); 549 } else { 550 archive_string_sprintf(&as, " -%d", data->compression_level); 551 } 552 553 if (data->compression_level > CLEVEL_STD_MAX) { 554 archive_strcat(&as, " --ultra"); 555 } 556 557 if (data->threads != 0) { 558 archive_string_sprintf(&as, " --threads=%d", data->threads); 559 } 560 561 if (data->long_distance != 0) { 562 archive_string_sprintf(&as, " --long=%d", data->long_distance); 563 } 564 565 f->write = archive_compressor_zstd_write; 566 r = __archive_write_program_open(f, data->pdata, as.s); 567 archive_string_free(&as); 568 return (r); 569 } 570 571 static int 572 archive_compressor_zstd_write(struct archive_write_filter *f, const void *buff, 573 size_t length) 574 { 575 struct private_data *data = (struct private_data *)f->data; 576 577 return __archive_write_program_write(f, data->pdata, buff, length); 578 } 579 580 static int 581 archive_compressor_zstd_flush(struct archive_write_filter *f) 582 { 583 (void)f; /* UNUSED */ 584 585 return (ARCHIVE_OK); 586 } 587 588 static int 589 archive_compressor_zstd_close(struct archive_write_filter *f) 590 { 591 struct private_data *data = (struct private_data *)f->data; 592 593 return __archive_write_program_close(f, data->pdata); 594 } 595 596 #endif /* HAVE_ZSTD_H && HAVE_ZSTD_compressStream */ 597
Indexes created Wed Sep 16 00:24:39 UTC 2026