1 /* $NetBSD: compress.c,v 1.24 2026/06/10 20:54:16 christos Exp $ */ 2 3 /* 4 * Copyright (c) Ian F. Darwin 1986-1995. 5 * Software written by Ian F. Darwin and others; 6 * maintained 1995-present by Christos Zoulas and others. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice immediately at the beginning of the file, without modification, 13 * this list of conditions, and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 19 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 21 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR 22 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 24 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 28 * SUCH DAMAGE. 29 */ 30 /* 31 * compress routines: 32 * zmagic() - returns 0 if not recognized, uncompresses and prints 33 * information if recognized 34 * uncompress(method, old, n, newch) - uncompress old into new, 35 * using method, return sizeof new 36 */ 37 #include "file.h" 38 39 #ifndef lint 40 #if 0 41 FILE_RCSID("@(#)$File: compress.c,v 1.162 2026/05/17 17:10:25 christos Exp $") 42 #else 43 __RCSID("$NetBSD: compress.c,v 1.24 2026/06/10 20:54:16 christos Exp $"); 44 #endif 45 #endif 46 47 #include "magic.h" 48 #include <stdlib.h> 49 #ifdef HAVE_UNISTD_H 50 #include <unistd.h> 51 #endif 52 #ifdef HAVE_SPAWN_H 53 #include <spawn.h> 54 #endif 55 #include <stdio.h> 56 #include <string.h> 57 #include <errno.h> 58 #include <ctype.h> 59 #include <stdarg.h> 60 #include <signal.h> 61 #ifndef HAVE_SIG_T 62 typedef void (*sig_t)(int); 63 #endif /* HAVE_SIG_T */ 64 #ifdef HAVE_SYS_IOCTL_H 65 #include <sys/ioctl.h> 66 #endif 67 #ifdef HAVE_SYS_WAIT_H 68 #include <sys/wait.h> 69 #endif 70 #if defined(HAVE_SYS_TIME_H) 71 #include <sys/time.h> 72 #endif 73 74 #if defined(HAVE_ZLIB_H) && defined(ZLIBSUPPORT) 75 #define BUILTIN_DECOMPRESS 76 #include <zlib.h> 77 #endif 78 79 #if defined(HAVE_BZLIB_H) && defined(BZLIBSUPPORT) 80 #define BUILTIN_BZLIB 81 #include <bzlib.h> 82 #endif 83 84 #if defined(HAVE_LZMA_H) && defined(XZLIBSUPPORT) 85 #define BUILTIN_XZLIB 86 #include <lzma.h> 87 #endif 88 89 #if defined(HAVE_ZSTD_H) && defined(ZSTDLIBSUPPORT) 90 #define BUILTIN_ZSTDLIB 91 #include <zstd.h> 92 #include <zstd_errors.h> 93 #endif 94 95 #if defined(HAVE_LZLIB_H) && defined(LZLIBSUPPORT) 96 #define BUILTIN_LZLIB 97 #include <lzlib.h> 98 #endif 99 100 #ifdef notyet 101 #if defined(HAVE_LRZIP_H) && defined(LRZIPLIBSUPPORT) 102 #define BUILTIN_LRZIP 103 #include <Lrzip.h> 104 #endif 105 #endif 106 107 #ifdef DEBUG 108 int tty = -1; 109 #define DPRINTF(...) do { \ 110 if (tty == -1) \ 111 tty = open("/dev/tty", O_RDWR); \ 112 if (tty == -1) \ 113 abort(); \ 114 dprintf(tty, __VA_ARGS__); \ 115 } while (/*CONSTCOND*/0) 116 #else 117 #define DPRINTF(...) 118 #endif 119 120 #ifdef ZLIBSUPPORT 121 /* 122 * The following python code is not really used because ZLIBSUPPORT is only 123 * defined if we have a built-in zlib, and the built-in zlib handles that. 124 * That is not true for android where we have zlib.h and not -lz. 125 */ 126 static const char zlibcode[] = 127 "import sys, zlib; sys.stdout.write(zlib.decompress(sys.stdin.read()))"; 128 129 static const char *zlib_args[] = { "python", "-c", zlibcode, NULL }; 130 131 static int 132 zlibcmp(const unsigned char *buf) 133 { 134 unsigned short x; 135 136 if ((buf[0] & 0xf) != 8 || (buf[0] & 0x80) != 0) 137 return 0; 138 if (file_bigendian()) /* endianness test */ 139 x = buf[0] | (buf[1] << 8); 140 else 141 x = buf[1] | (buf[0] << 8); 142 if (x % 31) 143 return 0; 144 return 1; 145 } 146 #endif 147 148 static int 149 lzmacmp(const unsigned char *buf) 150 { 151 if (buf[0] != 0x5d || buf[1] || buf[2]) 152 return 0; 153 if (buf[12] && buf[12] != 0xff) 154 return 0; 155 return 1; 156 } 157 158 #define gzip_flags "-cd" 159 #define lzip_flags gzip_flags 160 161 static const char *gzip_args[] = { 162 "gzip", gzip_flags, NULL 163 }; 164 static const char *uncompress_args[] = { 165 "uncompress", "-c", NULL 166 }; 167 static const char *bzip2_args[] = { 168 "bzip2", "-cd", NULL 169 }; 170 static const char *lzip_args[] = { 171 "lzip", lzip_flags, NULL 172 }; 173 static const char *xz_args[] = { 174 "xz", "-cd", NULL 175 }; 176 static const char *lrzip_args[] = { 177 "lrzip", "-qdf", "-", NULL 178 }; 179 static const char *lz4_args[] = { 180 "lz4", "-cd", NULL 181 }; 182 static const char *zstd_args[] = { 183 "zstd", "-cd", NULL 184 }; 185 186 #define do_zlib NULL 187 #define do_bzlib NULL 188 189 file_private const struct { 190 union { 191 const char *magic; 192 int (*func)(const unsigned char *); 193 } u; 194 int maglen; 195 const char **argv; 196 void *unused; 197 } compr[] = { 198 #define METH_FROZEN 2 199 #define METH_BZIP 7 200 #define METH_XZ 9 201 #define METH_LZIP 8 202 #define METH_LRZIP 10 203 #define METH_ZSTD 12 204 #define METH_LZMA 13 205 #define METH_ZLIB 14 206 { { .magic = "\037\235" }, 2, gzip_args, NULL }, /* 0, compressed */ 207 /* Uncompress can get stuck; so use gzip first if we have it 208 * Idea from Damien Clark, thanks! */ 209 { { .magic = "\037\235" }, 2, uncompress_args, NULL },/* 1, compressed */ 210 { { .magic = "\037\213" }, 2, gzip_args, do_zlib },/* 2, gzipped */ 211 { { .magic = "\037\236" }, 2, gzip_args, NULL }, /* 3, frozen */ 212 { { .magic = "\037\240" }, 2, gzip_args, NULL }, /* 4, SCO LZH */ 213 /* the standard pack utilities do not accept standard input */ 214 { { .magic = "\037\036" }, 2, gzip_args, NULL }, /* 5, packed */ 215 { { .magic = "PK\3\4" }, 4, gzip_args, NULL }, /* 6, pkziped */ 216 /* ...only first file examined */ 217 { { .magic = "BZh" }, 3, bzip2_args, do_bzlib },/* 7, bzip2-ed */ 218 { { .magic = "LZIP" }, 4, lzip_args, NULL }, /* 8, lzip-ed */ 219 { { .magic = "\3757zXZ\0" },6, xz_args, NULL }, /* 9, XZ Util */ 220 { { .magic = "LRZI" }, 4, lrzip_args, NULL }, /* 10, LRZIP */ 221 { { .magic = "\004\"M\030" },4, lz4_args, NULL }, /* 11, LZ4 */ 222 { { .magic = "\x28\xB5\x2F\xFD" }, 4, zstd_args, NULL },/* 12, zstd */ 223 { { .func = lzmacmp }, -13, xz_args, NULL }, /* 13, lzma */ 224 #ifdef ZLIBSUPPORT 225 { { .func = zlibcmp }, -2, zlib_args, NULL }, /* 14, zlib */ 226 #endif 227 }; 228 229 #define OKDATA 0 230 #define NODATA 1 231 #define ERRDATA 2 232 233 file_private ssize_t swrite(int, const void *, size_t); 234 #if HAVE_FORK 235 file_private size_t ncompr = __arraycount(compr); 236 file_private int uncompressbuf(int, size_t, size_t, int, const unsigned char *, 237 unsigned char **, size_t *); 238 #ifdef BUILTIN_DECOMPRESS 239 file_private int uncompresszlib(const unsigned char *, unsigned char **, size_t, 240 size_t *, int); 241 file_private int uncompressgzipped(const unsigned char *, unsigned char **, size_t, 242 size_t *, int); 243 #endif 244 #ifdef BUILTIN_BZLIB 245 file_private int uncompressbzlib(const unsigned char *, unsigned char **, size_t, 246 size_t *, int); 247 #endif 248 #ifdef BUILTIN_XZLIB 249 file_private int uncompressxzlib(const unsigned char *, unsigned char **, size_t, 250 size_t *, int); 251 #endif 252 #ifdef BUILTIN_ZSTDLIB 253 file_private int uncompresszstd(const unsigned char *, unsigned char **, size_t, 254 size_t *, int); 255 #endif 256 #ifdef BUILTIN_LZLIB 257 file_private int uncompresslzlib(const unsigned char *, unsigned char **, size_t, 258 size_t *, int); 259 #endif 260 #ifdef BUILTIN_LRZIP 261 file_private int uncompresslrzip(const unsigned char *, unsigned char **, size_t, 262 size_t *, int); 263 #endif 264 265 266 static int makeerror(unsigned char **, size_t *, const char *, ...) 267 __attribute__((__format__(__printf__, 3, 4))); 268 file_private const char *methodname(size_t); 269 270 file_private int 271 format_decompression_error(struct magic_set *ms, size_t i, unsigned char *buf) 272 { 273 unsigned char *p; 274 int mime = ms->flags & MAGIC_MIME; 275 276 if (!mime) 277 return file_printf(ms, "ERROR:[%s: %s]", methodname(i), buf); 278 279 for (p = buf; *p; p++) 280 if (!isalnum(*p)) 281 *p = '-'; 282 283 return file_printf(ms, "application/x-decompression-error-%s-%s", 284 methodname(i), buf); 285 } 286 287 file_protected int 288 file_zmagic(struct magic_set *ms, const struct buffer *b, const char *name) 289 { 290 unsigned char *newbuf = NULL; 291 size_t i, nsz; 292 char *rbuf; 293 file_pushbuf_t *pb; 294 int urv, prv, rv = 0; 295 int mime = ms->flags & MAGIC_MIME; 296 int fd = b->fd; 297 const unsigned char *buf = CAST(const unsigned char *, b->fbuf); 298 size_t nbytes = b->flen; 299 int sa_saved = 0; 300 struct sigaction sig_act; 301 302 if ((ms->flags & MAGIC_COMPRESS) == 0) 303 return 0; 304 305 for (i = 0; i < ncompr; i++) { 306 int zm; 307 if (nbytes < CAST(size_t, abs(compr[i].maglen))) 308 continue; 309 if (compr[i].maglen < 0) { 310 zm = (*compr[i].u.func)(buf); 311 } else { 312 zm = memcmp(buf, compr[i].u.magic, 313 CAST(size_t, compr[i].maglen)) == 0; 314 } 315 316 if (!zm) 317 continue; 318 319 /* Prevent SIGPIPE death if child dies unexpectedly */ 320 if (!sa_saved) { 321 //We can use sig_act for both new and old, but 322 struct sigaction new_act; 323 memset(&new_act, 0, sizeof(new_act)); 324 new_act.sa_handler = SIG_IGN; 325 sa_saved = sigaction(SIGPIPE, &new_act, &sig_act) != -1; 326 } 327 328 nsz = nbytes; 329 free(newbuf); 330 urv = uncompressbuf(fd, ms->bytes_max, i, 331 (ms->flags & MAGIC_NO_COMPRESS_FORK), buf, &newbuf, &nsz); 332 DPRINTF("uncompressbuf = %d, %s, %" SIZE_T_FORMAT "u\n", urv, 333 (char *)newbuf, nsz); 334 switch (urv) { 335 case OKDATA: 336 case ERRDATA: 337 ms->flags &= ~MAGIC_COMPRESS; 338 if (urv == ERRDATA) 339 prv = format_decompression_error(ms, i, newbuf); 340 else 341 prv = file_buffer(ms, -1, NULL, name, newbuf, 342 nsz); 343 if (prv == -1) 344 goto error; 345 rv = 1; 346 if ((ms->flags & MAGIC_COMPRESS_TRANSP) != 0) 347 goto out; 348 if (mime != MAGIC_MIME && mime != 0) 349 goto out; 350 if ((file_printf(ms, 351 mime ? " compressed-encoding=" : " (")) == -1) 352 goto error; 353 if ((pb = file_push_buffer(ms)) == NULL) 354 goto error; 355 /* 356 * XXX: If file_buffer fails here, we overwrite 357 * the compressed text. FIXME. 358 */ 359 if (file_buffer(ms, -1, NULL, NULL, buf, nbytes) == -1) 360 { 361 if (file_pop_buffer(ms, pb) != NULL) 362 abort(); 363 goto error; 364 } 365 if ((rbuf = file_pop_buffer(ms, pb)) != NULL) { 366 if (file_printf(ms, "%s", rbuf) == -1) { 367 free(rbuf); 368 goto error; 369 } 370 free(rbuf); 371 } 372 if (!mime && file_printf(ms, ")") == -1) 373 goto error; 374 /*FALLTHROUGH*/ 375 case NODATA: 376 break; 377 default: 378 abort(); 379 /*NOTREACHED*/ 380 error: 381 rv = -1; 382 break; 383 } 384 } 385 out: 386 DPRINTF("rv = %d\n", rv); 387 388 if (sa_saved && sig_act.sa_handler != SIG_IGN) 389 (void)sigaction(SIGPIPE, &sig_act, NULL); 390 391 free(newbuf); 392 ms->flags |= MAGIC_COMPRESS; 393 DPRINTF("Zmagic returns %d\n", rv); 394 return rv; 395 } 396 #endif 397 /* 398 * `safe' write for sockets and pipes. 399 */ 400 file_private ssize_t 401 swrite(int fd, const void *buf, size_t n) 402 { 403 ssize_t rv; 404 size_t rn = n; 405 406 do 407 switch (rv = write(fd, buf, n)) { 408 case -1: 409 if (errno == EINTR) 410 continue; 411 return -1; 412 default: 413 n -= rv; 414 buf = CAST(const char *, buf) + rv; 415 break; 416 } 417 while (n > 0); 418 return rn; 419 } 420 421 422 /* 423 * `safe' read for sockets and pipes. 424 */ 425 file_protected ssize_t 426 sread(int fd, void *buf, size_t n, int canbepipe __attribute__((__unused__))) 427 { 428 ssize_t rv; 429 #if defined(FIONREAD) && !defined(__MINGW32__) 430 int t = 0; 431 #endif 432 size_t rn = n; 433 434 if (fd == STDIN_FILENO) 435 goto nocheck; 436 437 #if defined(FIONREAD) && !defined(__MINGW32__) 438 if (canbepipe && (ioctl(fd, FIONREAD, &t) == -1 || t == 0)) { 439 #ifdef FD_ZERO 440 ssize_t cnt; 441 for (cnt = 0;; cnt++) { 442 fd_set check; 443 struct timeval tout = {0, 100 * 1000}; 444 int selrv; 445 446 FD_ZERO(&check); 447 FD_SET(fd, &check); 448 449 /* 450 * Avoid soft deadlock: do not read if there 451 * is nothing to read from sockets and pipes. 452 */ 453 selrv = select(fd + 1, &check, NULL, NULL, &tout); 454 if (selrv == -1) { 455 if (errno == EINTR || errno == EAGAIN) 456 continue; 457 } else if (selrv == 0 && cnt >= 5) { 458 return 0; 459 } else 460 break; 461 } 462 #endif 463 (void)ioctl(fd, FIONREAD, &t); 464 } 465 466 if (t > 0 && CAST(size_t, t) < n) { 467 n = t; 468 rn = n; 469 } 470 #endif 471 472 nocheck: 473 do 474 switch ((rv = read(fd, buf, n))) { 475 case -1: 476 if (errno == EINTR) 477 continue; 478 return -1; 479 case 0: 480 return rn - n; 481 default: 482 n -= rv; 483 buf = CAST(char *, CCAST(void *, buf)) + rv; 484 break; 485 } 486 while (n > 0); 487 return rn; 488 } 489 490 file_protected int 491 file_pipe2file(struct magic_set *ms, int fd, const void *startbuf, 492 size_t nbytes) 493 { 494 char buf[4096]; 495 ssize_t r; 496 int tfd; 497 498 #ifdef WIN32 499 const char *t; 500 buf[0] = '\0'; 501 if ((t = getenv("TEMP")) != NULL) 502 (void)strlcpy(buf, t, sizeof(buf)); 503 else if ((t = getenv("TMP")) != NULL) 504 (void)strlcpy(buf, t, sizeof(buf)); 505 else if ((t = getenv("TMPDIR")) != NULL) 506 (void)strlcpy(buf, t, sizeof(buf)); 507 if (buf[0] != '\0') 508 (void)strlcat(buf, "/", sizeof(buf)); 509 (void)strlcat(buf, "file.XXXXXX", sizeof(buf)); 510 #else 511 (void)strlcpy(buf, "/tmp/file.XXXXXX", sizeof(buf)); 512 #endif 513 #ifndef HAVE_MKSTEMP 514 { 515 char *ptr = mktemp(buf); 516 tfd = open(ptr, O_RDWR|O_TRUNC|O_EXCL|O_CREAT, 0600); 517 r = errno; 518 (void)unlink(ptr); 519 errno = r; 520 } 521 #else 522 { 523 int te; 524 mode_t ou = umask(0); 525 tfd = mkstemp(buf); 526 (void)umask(ou); 527 te = errno; 528 (void)unlink(buf); 529 errno = te; 530 } 531 #endif 532 if (tfd == -1) { 533 file_error(ms, errno, 534 "cannot create temporary file for pipe copy"); 535 return -1; 536 } 537 538 if (swrite(tfd, startbuf, nbytes) != CAST(ssize_t, nbytes)) 539 r = 1; 540 else { 541 while ((r = sread(fd, buf, sizeof(buf), 1)) > 0) 542 if (swrite(tfd, buf, CAST(size_t, r)) != r) 543 break; 544 } 545 546 switch (r) { 547 case -1: 548 file_error(ms, errno, "error copying from pipe to temp file"); 549 return -1; 550 case 0: 551 break; 552 default: 553 file_error(ms, errno, "error while writing to temp file"); 554 return -1; 555 } 556 557 /* 558 * We duplicate the file descriptor, because fclose on a 559 * tmpfile will delete the file, but any open descriptors 560 * can still access the phantom inode. 561 */ 562 if ((fd = dup2(tfd, fd)) == -1) { 563 file_error(ms, errno, "could not dup descriptor for temp file"); 564 return -1; 565 } 566 (void)close(tfd); 567 if (lseek(fd, CAST(off_t, 0), SEEK_SET) == CAST(off_t, -1)) { 568 file_badseek(ms); 569 return -1; 570 } 571 return fd; 572 } 573 #if HAVE_FORK 574 #ifdef BUILTIN_DECOMPRESS 575 576 #define FHCRC (1 << 1) 577 #define FEXTRA (1 << 2) 578 #define FNAME (1 << 3) 579 #define FCOMMENT (1 << 4) 580 581 582 file_private int 583 uncompressgzipped(const unsigned char *old, unsigned char **newch, 584 size_t bytes_max, size_t *n, int extra __attribute__((__unused__))) 585 { 586 unsigned char flg; 587 size_t data_start = 10; 588 589 if (*n < 4) { 590 goto err; 591 } 592 593 flg = old[3]; 594 595 if (flg & FEXTRA) { 596 if (data_start + 1 >= *n) 597 goto err; 598 data_start += 2 + old[data_start] + old[data_start + 1] * 256; 599 } 600 if (flg & FNAME) { 601 while(data_start < *n && old[data_start]) 602 data_start++; 603 data_start++; 604 } 605 if (flg & FCOMMENT) { 606 while(data_start < *n && old[data_start]) 607 data_start++; 608 data_start++; 609 } 610 if (flg & FHCRC) 611 data_start += 2; 612 613 if (data_start >= *n) 614 goto err; 615 616 *n -= data_start; 617 old += data_start; 618 return uncompresszlib(old, newch, bytes_max, n, 0); 619 err: 620 return makeerror(newch, n, "File too short"); 621 } 622 623 file_private int 624 uncompresszlib(const unsigned char *old, unsigned char **newch, 625 size_t bytes_max, size_t *n, int zlib) 626 { 627 int rc; 628 z_stream z; 629 630 DPRINTF("builtin zlib decompression\n"); 631 z.next_in = CCAST(Bytef *, old); 632 z.avail_in = CAST(uint32_t, *n); 633 z.next_out = *newch; 634 z.avail_out = CAST(unsigned int, bytes_max); 635 z.zalloc = Z_NULL; 636 z.zfree = Z_NULL; 637 z.opaque = Z_NULL; 638 639 /* LINTED bug in header macro */ 640 rc = zlib ? inflateInit(&z) : inflateInit2(&z, -15); 641 if (rc != Z_OK) 642 goto err; 643 644 rc = inflate(&z, Z_SYNC_FLUSH); 645 if (rc != Z_OK && rc != Z_STREAM_END) { 646 inflateEnd(&z); 647 goto err; 648 } 649 650 *n = CAST(size_t, z.total_out); 651 rc = inflateEnd(&z); 652 if (rc != Z_OK) 653 goto err; 654 655 /* let's keep the nul-terminate tradition */ 656 (*newch)[*n] = '\0'; 657 658 return OKDATA; 659 err: 660 return makeerror(newch, n, "%s", z.msg ? z.msg : zError(rc)); 661 } 662 #endif 663 664 #ifdef BUILTIN_BZLIB 665 file_private int 666 uncompressbzlib(const unsigned char *old, unsigned char **newch, 667 size_t bytes_max, size_t *n, int extra __attribute__((__unused__))) 668 { 669 int rc; 670 bz_stream bz; 671 672 DPRINTF("builtin bzlib decompression\n"); 673 memset(&bz, 0, sizeof(bz)); 674 rc = BZ2_bzDecompressInit(&bz, 0, 0); 675 if (rc != BZ_OK) 676 goto err; 677 678 bz.next_in = CCAST(char *, RCAST(const char *, old)); 679 bz.avail_in = CAST(uint32_t, *n); 680 bz.next_out = RCAST(char *, *newch); 681 bz.avail_out = CAST(unsigned int, bytes_max); 682 683 rc = BZ2_bzDecompress(&bz); 684 if (rc != BZ_OK && rc != BZ_STREAM_END) { 685 BZ2_bzDecompressEnd(&bz); 686 goto err; 687 } 688 689 /* Assume byte_max is within 32bit */ 690 /* assert(bz.total_out_hi32 == 0); */ 691 *n = CAST(size_t, bz.total_out_lo32); 692 rc = BZ2_bzDecompressEnd(&bz); 693 if (rc != BZ_OK) 694 goto err; 695 696 /* let's keep the nul-terminate tradition */ 697 (*newch)[*n] = '\0'; 698 699 return OKDATA; 700 err: 701 return makeerror(newch, n, "bunzip error %d", rc); 702 } 703 #endif 704 705 #ifdef BUILTIN_XZLIB 706 file_private int 707 uncompressxzlib(const unsigned char *old, unsigned char **newch, 708 size_t bytes_max, size_t *n, int extra __attribute__((__unused__))) 709 { 710 int rc; 711 lzma_stream xz; 712 713 DPRINTF("builtin xzlib decompression\n"); 714 memset(&xz, 0, sizeof(xz)); 715 rc = lzma_auto_decoder(&xz, UINT64_MAX, 0); 716 if (rc != LZMA_OK) 717 goto err; 718 719 xz.next_in = CCAST(const uint8_t *, old); 720 xz.avail_in = CAST(uint32_t, *n); 721 xz.next_out = RCAST(uint8_t *, *newch); 722 xz.avail_out = CAST(unsigned int, bytes_max); 723 724 rc = lzma_code(&xz, LZMA_RUN); 725 if (rc != LZMA_OK && rc != LZMA_STREAM_END) { 726 lzma_end(&xz); 727 goto err; 728 } 729 730 *n = CAST(size_t, xz.total_out); 731 732 lzma_end(&xz); 733 734 /* let's keep the nul-terminate tradition */ 735 (*newch)[*n] = '\0'; 736 737 return OKDATA; 738 err: 739 return makeerror(newch, n, "unxz error %d", rc); 740 } 741 #endif 742 743 #ifdef BUILTIN_ZSTDLIB 744 file_private int 745 uncompresszstd(const unsigned char *old, unsigned char **newch, 746 size_t bytes_max, size_t *n, int extra __attribute__((__unused__))) 747 { 748 size_t rc; 749 ZSTD_DStream *zstd; 750 ZSTD_inBuffer in; 751 ZSTD_outBuffer out; 752 753 DPRINTF("builtin zstd decompression\n"); 754 if ((zstd = ZSTD_createDStream()) == NULL) { 755 return makeerror(newch, n, "No ZSTD decompression stream, %s", 756 strerror(errno)); 757 } 758 759 rc = ZSTD_DCtx_reset(zstd, ZSTD_reset_session_only); 760 if (ZSTD_isError(rc)) 761 goto err; 762 763 in.src = CCAST(const void *, old); 764 in.size = *n; 765 in.pos = 0; 766 out.dst = RCAST(void *, *newch); 767 out.size = bytes_max; 768 out.pos = 0; 769 770 rc = ZSTD_decompressStream(zstd, &out, &in); 771 if (ZSTD_isError(rc)) 772 goto err; 773 774 *n = out.pos; 775 776 ZSTD_freeDStream(zstd); 777 778 /* let's keep the nul-terminate tradition */ 779 (*newch)[*n] = '\0'; 780 781 return OKDATA; 782 err: 783 ZSTD_freeDStream(zstd); 784 return makeerror(newch, n, "zstd error %d", ZSTD_getErrorCode(rc)); 785 } 786 #endif 787 788 #ifdef BUILTIN_LZLIB 789 file_private int 790 uncompresslzlib(const unsigned char *old, unsigned char **newch, 791 size_t bytes_max, size_t *n, int extra __attribute__((__unused__))) 792 { 793 enum LZ_Errno err; 794 size_t old_remaining = *n; 795 size_t new_remaining = bytes_max; 796 size_t total_read = 0; 797 unsigned char *bufp; 798 struct LZ_Decoder *dec; 799 800 bufp = *newch; 801 802 DPRINTF("builtin lzlib decompression\n"); 803 dec = LZ_decompress_open(); 804 if (!dec) { 805 return makeerror(newch, n, "unable to allocate LZ_Decoder"); 806 } 807 if (LZ_decompress_errno(dec) != LZ_ok) 808 goto err; 809 810 for (;;) { 811 // LZ_decompress_read() stops at member boundaries, so we may 812 // have more than one successful read after writing all data 813 // we have. 814 if (old_remaining > 0) { 815 int wr = LZ_decompress_write(dec, old, old_remaining); 816 if (wr < 0) 817 goto err; 818 old_remaining -= wr; 819 old += wr; 820 } 821 822 int rd = LZ_decompress_read(dec, bufp, new_remaining); 823 if (rd > 0) { 824 new_remaining -= rd; 825 bufp += rd; 826 total_read += rd; 827 } 828 829 if (rd < 0 || LZ_decompress_errno(dec) != LZ_ok) 830 goto err; 831 if (new_remaining == 0) 832 break; 833 if (old_remaining == 0 && rd == 0) 834 break; 835 } 836 837 LZ_decompress_close(dec); 838 *n = total_read; 839 840 /* let's keep the nul-terminate tradition */ 841 *bufp = '\0'; 842 843 return OKDATA; 844 err: 845 err = LZ_decompress_errno(dec); 846 LZ_decompress_close(dec); 847 return makeerror(newch, n, "lzlib error: %s", LZ_strerror(err)); 848 } 849 #endif 850 851 #ifdef BUILTIN_LRZIP 852 file_private int 853 uncompresslrzip(const unsigned char *old, unsigned char **newch, 854 size_t bytes_max, size_t *n, int extra __attribute__((__unused__))) 855 { 856 Lrzip *lr; 857 FILE *in, *out; 858 int res = OKDATA; 859 860 DPRINTF("builtin rlzip decompression\n"); 861 lr = lrzip_new(LRZIP_MODE_DECOMPRESS); 862 if (lr == NULL) { 863 res = makeerror(newch, n, "unable to create an lrzip decoder"); 864 goto out0; 865 } 866 lrzip_config_env(lr); 867 in = fmemopen(RCAST(void *, old), *n, "r"); 868 if (in == NULL) { 869 res = makeerror(newch, n, "unable to construct input file"); 870 goto out1; 871 } 872 if (!lrzip_file_add(lr, in)) { 873 res = makeerror(newch, n, "unable to add input file"); 874 goto out2; 875 } 876 free(*newch); 877 *newch = calloc(*n = 2 * bytes_max, 1); 878 if (*newch == NULL) { 879 res = makeerror(newch, n, "unable to allocate output buffer"); 880 goto out2; 881 } 882 out = fmemopen(*newch, *n, "w"); 883 if (out == NULL) { 884 free(*newch); 885 res = makeerror(newch, n, "unable to allocate output file"); 886 goto out2; 887 } 888 lrzip_outfile_set(lr, out); 889 if (lrzip_run(lr)) { 890 free(*newch); 891 res = makeerror(newch, n, "unable to decompress file"); 892 goto out3; 893 } 894 *n = (size_t)ftell(out); 895 out3: 896 fclose(out); 897 out2: 898 fclose(in); 899 out1: 900 lrzip_free(lr); 901 out0: 902 return res; 903 } 904 #endif 905 906 static int 907 makeerror(unsigned char **buf, size_t *len, const char *fmt, ...) 908 { 909 char *msg; 910 va_list ap; 911 int rv; 912 913 DPRINTF("Makeerror %s\n", fmt); 914 free(*buf); 915 va_start(ap, fmt); 916 rv = vasprintf(&msg, fmt, ap); 917 va_end(ap); 918 if (rv < 0) { 919 DPRINTF("Makeerror failed"); 920 *buf = NULL; 921 *len = 0; 922 return NODATA; 923 } 924 *buf = RCAST(unsigned char *, msg); 925 *len = strlen(msg); 926 return ERRDATA; 927 } 928 929 static void 930 closefd(int *fd, size_t i) 931 { 932 if (fd[i] == -1) 933 return; 934 (void) close(fd[i]); 935 fd[i] = -1; 936 } 937 938 static void 939 closep(int *fd) 940 { 941 size_t i; 942 for (i = 0; i < 2; i++) 943 closefd(fd, i); 944 } 945 946 static void 947 movedesc(void *v, int i, int fd) 948 { 949 if (fd == i) 950 return; /* "no dup was necessary" */ 951 #ifdef HAVE_POSIX_SPAWNP 952 posix_spawn_file_actions_t *fa = RCAST(posix_spawn_file_actions_t *, v); 953 posix_spawn_file_actions_adddup2(fa, fd, i); 954 posix_spawn_file_actions_addclose(fa, fd); 955 #else 956 if (dup2(fd, i) == -1) { 957 DPRINTF("dup(%d, %d) failed (%s)\n", fd, i, strerror(errno)); 958 exit(EXIT_FAILURE); 959 } 960 close(v ? fd : fd); 961 #endif 962 } 963 964 static void 965 closedesc(void *v, int fd) 966 { 967 #ifdef HAVE_POSIX_SPAWNP 968 posix_spawn_file_actions_t *fa = RCAST(posix_spawn_file_actions_t *, v); 969 posix_spawn_file_actions_addclose(fa, fd); 970 #else 971 close(v ? fd : fd); 972 #endif 973 } 974 975 static void 976 handledesc(void *v, int fd, int fdp[3][2]) 977 { 978 if (fd != -1) { 979 (void) lseek(fd, CAST(off_t, 0), SEEK_SET); 980 movedesc(v, STDIN_FILENO, fd); 981 } else { 982 movedesc(v, STDIN_FILENO, fdp[STDIN_FILENO][0]); 983 if (fdp[STDIN_FILENO][1] > 2) 984 closedesc(v, fdp[STDIN_FILENO][1]); 985 } 986 987 file_clear_closexec(STDIN_FILENO); 988 989 ///FIXME: if one of the fdp[i][j] is 0 or 1, this can bomb spectacularly 990 movedesc(v, STDOUT_FILENO, fdp[STDOUT_FILENO][1]); 991 if (fdp[STDOUT_FILENO][0] > 2) 992 closedesc(v, fdp[STDOUT_FILENO][0]); 993 994 file_clear_closexec(STDOUT_FILENO); 995 996 movedesc(v, STDERR_FILENO, fdp[STDERR_FILENO][1]); 997 if (fdp[STDERR_FILENO][0] > 2) 998 closedesc(v, fdp[STDERR_FILENO][0]); 999 1000 file_clear_closexec(STDERR_FILENO); 1001 } 1002 1003 static pid_t 1004 writechild(int fd, const void *old, size_t n) 1005 { 1006 pid_t pid; 1007 1008 /* 1009 * fork again, to avoid blocking because both 1010 * pipes filled 1011 */ 1012 pid = fork(); 1013 if (pid == -1) { 1014 DPRINTF("Fork failed (%s)\n", strerror(errno)); 1015 return -1; 1016 } 1017 if (pid == 0) { 1018 /* child */ 1019 if (swrite(fd, old, n) != CAST(ssize_t, n)) { 1020 DPRINTF("Write failed (%s)\n", strerror(errno)); 1021 exit(EXIT_FAILURE); 1022 } 1023 exit(EXIT_SUCCESS); 1024 } 1025 /* parent */ 1026 return pid; 1027 } 1028 1029 static ssize_t 1030 filter_error(unsigned char *ubuf, ssize_t n) 1031 { 1032 char *p; 1033 char *buf; 1034 1035 ubuf[n] = '\0'; 1036 buf = RCAST(char *, ubuf); 1037 while (isspace(CAST(unsigned char, *buf))) 1038 buf++; 1039 DPRINTF("Filter error[[[%s]]]\n", buf); 1040 if ((p = strchr(CAST(char *, buf), '\n')) != NULL) 1041 *p = '\0'; 1042 if ((p = strchr(CAST(char *, buf), ';')) != NULL) 1043 *p = '\0'; 1044 if ((p = strrchr(CAST(char *, buf), ':')) != NULL) { 1045 ++p; 1046 while (isspace(CAST(unsigned char, *p))) 1047 p++; 1048 n = strlen(p); 1049 memmove(ubuf, p, CAST(size_t, n + 1)); 1050 } 1051 DPRINTF("Filter error after[[[%s]]]\n", (char *)ubuf); 1052 if (islower(*ubuf)) 1053 *ubuf = toupper(*ubuf); 1054 return n; 1055 } 1056 1057 file_private const char * 1058 methodname(size_t method) 1059 { 1060 switch (method) { 1061 #ifdef BUILTIN_DECOMPRESS 1062 case METH_FROZEN: 1063 case METH_ZLIB: 1064 return "zlib"; 1065 #endif 1066 #ifdef BUILTIN_BZLIB 1067 case METH_BZIP: 1068 return "bzlib"; 1069 #endif 1070 #ifdef BUILTIN_XZLIB 1071 case METH_XZ: 1072 case METH_LZMA: 1073 return "xzlib"; 1074 #endif 1075 #ifdef BUILTIN_ZSTDLIB 1076 case METH_ZSTD: 1077 return "zstd"; 1078 #endif 1079 #ifdef BUILTIN_LZLIB 1080 case METH_LZIP: 1081 return "lzlib"; 1082 #endif 1083 #ifdef BUILTIN_LRZIP 1084 case METH_LRZIP: 1085 return "lrzip"; 1086 #endif 1087 default: 1088 return compr[method].argv[0]; 1089 } 1090 } 1091 1092 file_private int (* 1093 getdecompressor(size_t method))(const unsigned char *, unsigned char **, size_t, 1094 size_t *, int) 1095 { 1096 switch (method) { 1097 #ifdef BUILTIN_DECOMPRESS 1098 case METH_FROZEN: 1099 return uncompressgzipped; 1100 case METH_ZLIB: 1101 return uncompresszlib; 1102 #endif 1103 #ifdef BUILTIN_BZLIB 1104 case METH_BZIP: 1105 return uncompressbzlib; 1106 #endif 1107 #ifdef BUILTIN_XZLIB 1108 case METH_XZ: 1109 case METH_LZMA: 1110 return uncompressxzlib; 1111 #endif 1112 #ifdef BUILTIN_ZSTDLIB 1113 case METH_ZSTD: 1114 return uncompresszstd; 1115 #endif 1116 #ifdef BUILTIN_LZLIB 1117 case METH_LZIP: 1118 return uncompresslzlib; 1119 #endif 1120 #ifdef BUILTIN_LRZIP 1121 case METH_LRZIP: 1122 return uncompresslrzip; 1123 #endif 1124 default: 1125 return NULL; 1126 } 1127 } 1128 1129 file_private int 1130 uncompressbuf(int fd, size_t bytes_max, size_t method, int nofork, 1131 const unsigned char *old, unsigned char **newch, size_t* n) 1132 { 1133 int fdp[3][2]; 1134 int status, rv, w; 1135 pid_t pid; 1136 pid_t writepid = -1; 1137 size_t i; 1138 ssize_t r, re; 1139 char *const *args; 1140 #ifdef HAVE_POSIX_SPAWNP 1141 posix_spawn_file_actions_t fa; 1142 #endif 1143 int (*decompress)(const unsigned char *, unsigned char **, 1144 size_t, size_t *, int) = getdecompressor(method); 1145 1146 *newch = CAST(unsigned char *, malloc(bytes_max + 1)); 1147 if (*newch == NULL) 1148 return makeerror(newch, n, "No buffer, %s", strerror(errno)); 1149 1150 if (decompress) { 1151 if (nofork) { 1152 return makeerror(newch, n, 1153 "Fork is required to uncompress, but disabled"); 1154 } 1155 return (*decompress)(old, newch, bytes_max, n, 1); 1156 } 1157 1158 (void)fflush(stdout); 1159 (void)fflush(stderr); 1160 1161 for (i = 0; i < __arraycount(fdp); i++) 1162 fdp[i][0] = fdp[i][1] = -1; 1163 1164 /* 1165 * There are multithreaded users who run magic_file() 1166 * from dozens of threads. If two parallel magic_file() calls 1167 * analyze two large compressed files, both will spawn 1168 * an uncompressing child here, which writes out uncompressed data. 1169 * We read some portion, then close the pipe, then waitpid() the child. 1170 * If uncompressed data is larger, child should get EPIPE and exit. 1171 * However, with *parallel* calls OTHER child may unintentionally 1172 * inherit pipe fds, thus keeping pipe open and making writes in 1173 * our child block instead of failing with EPIPE! 1174 * (For the bug to occur, two threads must mutually inherit their pipes, 1175 * and both must have large outputs. Thus it happens not that often). 1176 * To avoid this, be sure to create pipes with O_CLOEXEC. 1177 */ 1178 if ((fd == -1 && file_pipe_closexec(fdp[STDIN_FILENO]) == -1) || 1179 file_pipe_closexec(fdp[STDOUT_FILENO]) == -1 || 1180 file_pipe_closexec(fdp[STDERR_FILENO]) == -1) { 1181 closep(fdp[STDIN_FILENO]); 1182 closep(fdp[STDOUT_FILENO]); 1183 return makeerror(newch, n, "Cannot create pipe, %s", 1184 strerror(errno)); 1185 } 1186 1187 args = RCAST(char *const *, RCAST(intptr_t, compr[method].argv)); 1188 #ifdef HAVE_POSIX_SPAWNP 1189 posix_spawn_file_actions_init(&fa); 1190 1191 handledesc(&fa, fd, fdp); 1192 1193 DPRINTF("Executing %s\n", compr[method].argv[0]); 1194 status = posix_spawnp(&pid, compr[method].argv[0], &fa, NULL, 1195 args, NULL); 1196 1197 posix_spawn_file_actions_destroy(&fa); 1198 1199 if (status != 0) { 1200 return makeerror(newch, n, "Cannot posix_spawn `%s', %s", 1201 compr[method].argv[0], strerror(status)); 1202 } 1203 #else 1204 /* For processes with large mapped virtual sizes, vfork 1205 * may be _much_ faster (10-100 times) than fork. 1206 */ 1207 pid = vfork(); 1208 if (pid == -1) { 1209 return makeerror(newch, n, "Cannot vfork, %s", 1210 strerror(errno)); 1211 } 1212 if (pid == 0) { 1213 /* child */ 1214 /* Note: we are after vfork, do not modify memory 1215 * in a way which confuses parent. In particular, 1216 * do not modify fdp[i][j]. 1217 */ 1218 handledesc(NULL, fd, fdp); 1219 DPRINTF("Executing %s\n", compr[method].argv[0]); 1220 1221 (void)execvp(compr[method].argv[0], args); 1222 dprintf(STDERR_FILENO, "exec `%s' failed, %s", 1223 compr[method].argv[0], strerror(errno)); 1224 _exit(EXIT_FAILURE); /* _exit(), not exit(), because of vfork */ 1225 } 1226 #endif 1227 /* parent */ 1228 /* Close write sides of child stdout/err pipes */ 1229 for (i = 1; i < __arraycount(fdp); i++) 1230 closefd(fdp[i], 1); 1231 /* Write the buffer data to child stdin, if we don't have fd */ 1232 if (fd == -1) { 1233 closefd(fdp[STDIN_FILENO], 0); 1234 writepid = writechild(fdp[STDIN_FILENO][1], old, *n); 1235 if (writepid == (pid_t)-1) { 1236 rv = makeerror(newch, n, "Write to child failed, %s", 1237 strerror(errno)); 1238 DPRINTF("Write to child failed\n"); 1239 goto err; 1240 } 1241 closefd(fdp[STDIN_FILENO], 1); 1242 } 1243 1244 rv = OKDATA; 1245 r = sread(fdp[STDOUT_FILENO][0], *newch, bytes_max, 0); 1246 DPRINTF("read got %zd\n", r); 1247 if (r < 0) { 1248 rv = ERRDATA; 1249 DPRINTF("Read stdout failed %d (%s)\n", fdp[STDOUT_FILENO][0], 1250 strerror(errno)); 1251 goto err; 1252 } 1253 if (CAST(size_t, r) == bytes_max) { 1254 /* 1255 * close fd so that the child exits with sigpipe and ignore 1256 * errors, otherwise we risk the child blocking and never 1257 * exiting. 1258 */ 1259 DPRINTF("Closing stdout for bytes_max\n"); 1260 closefd(fdp[STDOUT_FILENO], 0); 1261 goto ok; 1262 } 1263 if ((re = sread(fdp[STDERR_FILENO][0], *newch, bytes_max, 0)) > 0) { 1264 DPRINTF("Got stuff from stderr %s\n", *newch); 1265 rv = ERRDATA; 1266 r = filter_error(*newch, r); 1267 goto ok; 1268 } 1269 if (re == 0) 1270 goto ok; 1271 rv = makeerror(newch, n, "Read stderr failed, %s", 1272 strerror(errno)); 1273 goto err; 1274 ok: 1275 *n = r; 1276 /* NUL terminate, as every buffer is handled here. */ 1277 (*newch)[*n] = '\0'; 1278 err: 1279 closefd(fdp[STDIN_FILENO], 1); 1280 closefd(fdp[STDOUT_FILENO], 0); 1281 closefd(fdp[STDERR_FILENO], 0); 1282 1283 w = waitpid(pid, &status, 0); 1284 wait_err: 1285 if (w == -1) { 1286 rv = makeerror(newch, n, "Wait failed, %s", strerror(errno)); 1287 DPRINTF("Child wait return %#x\n", status); 1288 } else if (!WIFEXITED(status)) { 1289 DPRINTF("Child not exited (%#x)\n", status); 1290 } else if (WEXITSTATUS(status) != 0) { 1291 DPRINTF("Child exited (%#x)\n", WEXITSTATUS(status)); 1292 } 1293 if (writepid > 0) { 1294 /* _After_ we know decompressor has exited, our input writer 1295 * definitely will exit now (at worst, writing fails in it, 1296 * since output fd is closed now on the reading size). 1297 */ 1298 w = waitpid(writepid, &status, 0); 1299 writepid = -1; 1300 goto wait_err; 1301 } 1302 1303 closefd(fdp[STDIN_FILENO], 0); //why? it is already closed here! 1304 DPRINTF("Returning %p n=%" SIZE_T_FORMAT "u rv=%d\n", *newch, *n, rv); 1305 1306 return rv; 1307 } 1308 #endif 1309
Indexes created Sun Sep 13 00:25:22 UTC 2026