1 /* $XTermId: ptydata.c,v 1.163 2024/12/01 23:48:07 tom Exp $ */ 2 3 /* 4 * Copyright 1999-2023,2024 by Thomas E. Dickey 5 * 6 * All Rights Reserved 7 * 8 * Permission is hereby granted, free of charge, to any person obtaining a 9 * copy of this software and associated documentation files (the 10 * "Software"), to deal in the Software without restriction, including 11 * without limitation the rights to use, copy, modify, merge, publish, 12 * distribute, sublicense, and/or sell copies of the Software, and to 13 * permit persons to whom the Software is furnished to do so, subject to 14 * the following conditions: 15 * 16 * The above copyright notice and this permission notice shall be included 17 * in all copies or substantial portions of the Software. 18 * 19 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 20 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 21 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. 22 * IN NO EVENT SHALL THE ABOVE LISTED COPYRIGHT HOLDER(S) BE LIABLE FOR ANY 23 * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, 24 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE 25 * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 26 * 27 * Except as contained in this notice, the name(s) of the above copyright 28 * holders shall not be used in advertising or otherwise to promote the 29 * sale, use or other dealings in this Software without prior written 30 * authorization. 31 */ 32 33 #include <data.h> 34 35 #if OPT_WIDE_CHARS 36 #include <menu.h> 37 #include <wcwidth.h> 38 #endif 39 40 #ifdef TEST_DRIVER 41 #undef TRACE 42 #define TRACE(p) if (1) printf p 43 #undef TRACE2 44 #define TRACE2(p) if (0) printf p 45 #define visibleChars(buf, len) "buffer" 46 #endif 47 48 /* 49 * Check for both EAGAIN and EWOULDBLOCK, because some supposedly POSIX 50 * systems are broken and return EWOULDBLOCK when they should return EAGAIN. 51 * Note that this macro may evaluate its argument more than once. 52 */ 53 #if defined(EAGAIN) && defined(EWOULDBLOCK) 54 #define E_TEST(err) ((err) == EAGAIN || (err) == EWOULDBLOCK) 55 #else 56 #ifdef EAGAIN 57 #define E_TEST(err) ((err) == EAGAIN) 58 #else 59 #define E_TEST(err) ((err) == EWOULDBLOCK) 60 #endif 61 #endif 62 63 #if OPT_WIDE_CHARS 64 /* 65 * Convert the 8-bit codes in data->buffer[] into Unicode in data->utf_data. 66 * The number of bytes converted will be nonzero iff there is data. 67 */ 68 Bool 69 decodeUtf8(TScreen *screen, PtyData *data) 70 { 71 size_t i; 72 size_t length = (size_t) (data->last - data->next); 73 int utf_count = 0; 74 unsigned utf_char = 0; 75 76 data->utf_size = 0; 77 for (i = 0; i < length; i++) { 78 unsigned c = data->next[i]; 79 80 /* Combine UTF-8 into Unicode */ 81 if (c < 0x80) { 82 /* We received an ASCII character */ 83 if (utf_count > 0) { 84 data->utf_data = UCS_REPL; /* prev. sequence incomplete */ 85 data->utf_size = i; 86 } else { 87 data->utf_data = (IChar) c; 88 data->utf_size = 1; 89 } 90 break; 91 } else if (screen->vt100_graphics 92 && (c < 0x100) 93 && (utf_count == 0) 94 && screen->gsets[(int) screen->curgr] != nrc_ASCII) { 95 data->utf_data = (IChar) c; 96 data->utf_size = 1; 97 break; 98 } else if (c < 0xc0) { 99 /* We received a continuation byte */ 100 if (utf_count < 1) { 101 if (screen->c1_printable) { 102 data->utf_data = (IChar) c; 103 } else if ((i + 1) < length 104 && data->next[i + 1] > 0x20 105 && data->next[i + 1] < 0x80) { 106 /* 107 * Allow for C1 control string if the next byte is 108 * available for inspection. 109 */ 110 data->utf_data = (IChar) c; 111 } else { 112 /* 113 * We received a continuation byte before receiving a 114 * sequence state, or a failed attempt to use a C1 control 115 * string. 116 */ 117 data->utf_data = (IChar) UCS_REPL; 118 } 119 data->utf_size = (i + 1); 120 break; 121 } else if (screen->utf8_weblike 122 && (utf_count == 3 123 && utf_char == 0x04 124 && c >= 0x90)) { 125 /* The encoding would form a code point beyond U+10FFFF. */ 126 data->utf_size = i; 127 data->utf_data = UCS_REPL; 128 break; 129 } else if (screen->utf8_weblike 130 && (utf_count == 2 131 && utf_char == 0x0d 132 && c >= 0xa0)) { 133 /* The encoding would form a surrogate code point. */ 134 data->utf_size = i; 135 data->utf_data = UCS_REPL; 136 break; 137 } else { 138 /* Check for overlong UTF-8 sequences for which a shorter 139 * encoding would exist and replace them with UCS_REPL. 140 * An overlong UTF-8 sequence can have any of the following 141 * forms: 142 * 1100000x 10xxxxxx 143 * 11100000 100xxxxx 10xxxxxx 144 * 11110000 1000xxxx 10xxxxxx 10xxxxxx 145 * 11111000 10000xxx 10xxxxxx 10xxxxxx 10xxxxxx 146 * 11111100 100000xx 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx 147 */ 148 if (!utf_char && !((c & 0x7f) >> (7 - utf_count))) { 149 if (screen->utf8_weblike) { 150 /* overlong sequence continued */ 151 data->utf_data = UCS_REPL; 152 data->utf_size = i; 153 break; 154 } else { 155 utf_char = UCS_REPL; 156 } 157 } 158 utf_char <<= 6; 159 utf_char |= (c & 0x3f); 160 if ((utf_char >= 0xd800 && 161 utf_char <= 0xdfff) || 162 (utf_char == 0xfffe) || 163 (utf_char == HIDDEN_CHAR)) { 164 utf_char = UCS_REPL; 165 } 166 utf_count--; 167 if (utf_count == 0) { 168 #if !OPT_WIDER_ICHAR 169 /* characters outside UCS-2 become UCS_REPL */ 170 if (utf_char > NARROW_ICHAR) { 171 TRACE(("using replacement for %#x\n", utf_char)); 172 utf_char = UCS_REPL; 173 } 174 #endif 175 data->utf_data = (IChar) utf_char; 176 data->utf_size = (i + 1); 177 break; 178 } 179 } 180 } else { 181 /* We received a sequence start byte */ 182 if (utf_count > 0) { 183 /* previous sequence is incomplete */ 184 data->utf_data = UCS_REPL; 185 data->utf_size = i; 186 break; 187 } 188 if (screen->utf8_weblike) { 189 if (c < 0xe0) { 190 if (!(c & 0x1e)) { 191 /* overlong sequence start */ 192 data->utf_data = UCS_REPL; 193 data->utf_size = (i + 1); 194 break; 195 } 196 utf_count = 1; 197 utf_char = (c & 0x1f); 198 } else if (c < 0xf0) { 199 utf_count = 2; 200 utf_char = (c & 0x0f); 201 } else if (c < 0xf5) { 202 utf_count = 3; 203 utf_char = (c & 0x07); 204 } else { 205 data->utf_data = UCS_REPL; 206 data->utf_size = (i + 1); 207 break; 208 } 209 } else { 210 if (c < 0xe0) { 211 utf_count = 1; 212 utf_char = (c & 0x1f); 213 if (!(c & 0x1e)) { 214 /* overlong sequence */ 215 utf_char = UCS_REPL; 216 } 217 } else if (c < 0xf0) { 218 utf_count = 2; 219 utf_char = (c & 0x0f); 220 } else if (c < 0xf8) { 221 utf_count = 3; 222 utf_char = (c & 0x07); 223 } else if (c < 0xfc) { 224 utf_count = 4; 225 utf_char = (c & 0x03); 226 } else if (c < 0xfe) { 227 utf_count = 5; 228 utf_char = (c & 0x01); 229 } else { 230 data->utf_data = UCS_REPL; 231 data->utf_size = (i + 1); 232 break; 233 } 234 } 235 } 236 } 237 #if OPT_TRACE > 1 238 TRACE(("UTF-8 char %04X [%lu..%lu]\n", 239 data->utf_data, 240 (unsigned long) (data->next - data->buffer), 241 (unsigned long) (data->next - data->buffer + data->utf_size - 1))); 242 #endif 243 244 return (data->utf_size != 0); 245 } 246 #endif 247 248 int 249 readPtyData(XtermWidget xw, PtySelect * select_mask, PtyData *data) 250 { 251 TScreen *screen = TScreenOf(xw); 252 int size = 0; 253 254 if (FD_ISSET(screen->respond, select_mask)) { 255 int save_err; 256 trimPtyData(xw, data); 257 258 size = (int) read(screen->respond, (char *) data->last, (size_t) FRG_SIZE); 259 save_err = errno; 260 #if (defined(i386) && defined(SVR4) && defined(sun)) || defined(__CYGWIN__) 261 /* 262 * Yes, I know this is a majorly f*ugly hack, however it seems to 263 * be necessary for Solaris x86. DWH 11/15/94 264 * Dunno why though.. 265 * (and now CYGWIN, alanh (at) xfree86.org 08/15/01 266 */ 267 if (size <= 0) { 268 if (save_err == EIO || save_err == 0) 269 NormalExit(); 270 else if (!E_TEST(save_err)) 271 Panic("input: read returned unexpected error (%d)\n", save_err); 272 size = 0; 273 } 274 #else /* !f*ugly */ 275 if (size < 0) { 276 if (save_err == EIO) 277 NormalExit(); 278 else if (!E_TEST(save_err)) 279 Panic("input: read returned unexpected error (%d)\n", save_err); 280 size = 0; 281 } else if (size == 0) { 282 #if defined(__FreeBSD__) 283 NormalExit(); 284 #else 285 Panic("input: read returned zero\n", 0); 286 #endif 287 } 288 #endif /* f*ugly */ 289 } 290 291 if (size) { 292 #if OPT_TRACE 293 int i; 294 295 TRACE(("read %d bytes from pty\n", size)); 296 for (i = 0; i < size; i++) { 297 if (!(i % 16)) 298 TRACE(("%s", i ? "\n " : "READ")); 299 TRACE((" %02X", data->last[i])); 300 } 301 TRACE(("\n")); 302 #endif 303 data->last += size; 304 #ifdef ALLOWLOGGING 305 TScreenOf(term)->logstart = VTbuffer->next; 306 #endif 307 } 308 309 return (size); 310 } 311 312 /* 313 * Return the next value from the input buffer. Note that morePtyData() is 314 * always called before this function, so we can do the UTF-8 input conversion 315 * in that function and simply return the result here. 316 */ 317 #if OPT_WIDE_CHARS 318 IChar 319 nextPtyData(TScreen *screen, PtyData *data) 320 { 321 IChar result; 322 if (screen->utf8_inparse) { 323 skipPtyData(data, result); 324 } else { 325 result = *((data)->next++); 326 if (!screen->output_eight_bits) { 327 result = (IChar) (result & 0x7f); 328 } 329 } 330 TRACE2(("nextPtyData returns %#x\n", result)); 331 return result; 332 } 333 #endif 334 335 #if OPT_WIDE_CHARS 336 /* 337 * Called when UTF-8 mode has been turned on/off. 338 */ 339 void 340 switchPtyData(TScreen *screen, int flag) 341 { 342 if (screen->utf8_mode != flag) { 343 screen->utf8_mode = flag; 344 screen->utf8_inparse = (Boolean) (flag != 0); 345 mk_wcwidth_init(screen->utf8_mode); 346 347 TRACE(("turning UTF-8 mode %s\n", BtoS(flag))); 348 if (flag) { 349 saveCharsets(screen, screen->ansi_save_gsets); 350 resetCharsets(screen); 351 } else { 352 restoreCharsets(screen, screen->ansi_save_gsets); 353 } 354 update_font_utf8_mode(); 355 } 356 } 357 #endif 358 359 /* 360 * Allocate a buffer. 361 */ 362 void 363 initPtyData(PtyData **result) 364 { 365 PtyData *data; 366 367 TRACE2(("initPtyData given minBufSize %d, maxBufSize %d\n", 368 FRG_SIZE, BUF_SIZE)); 369 370 if (FRG_SIZE < 64) 371 FRG_SIZE = 64; 372 if (BUF_SIZE < FRG_SIZE) 373 BUF_SIZE = FRG_SIZE; 374 if (BUF_SIZE % FRG_SIZE) 375 BUF_SIZE = BUF_SIZE + FRG_SIZE - (BUF_SIZE % FRG_SIZE); 376 377 TRACE2(("initPtyData using minBufSize %d, maxBufSize %d\n", 378 FRG_SIZE, BUF_SIZE)); 379 380 data = TypeXtMallocX(PtyData, (BUF_SIZE + FRG_SIZE)); 381 382 memset(data, 0, sizeof(*data)); 383 data->next = data->buffer; 384 data->last = data->buffer; 385 *result = data; 386 } 387 388 /* 389 * Initialize a buffer for the caller, using its data in 'next'. 390 */ 391 #if OPT_WIDE_CHARS 392 PtyData * 393 fakePtyData(PtyData *result, Char *next, Char *last) 394 { 395 PtyData *data = result; 396 397 memset(data, 0, sizeof(*data)); 398 data->next = next; 399 data->last = last; 400 401 return data; 402 } 403 #endif 404 405 /* 406 * Remove used data by shifting the buffer down, to make room for more data, 407 * e.g., a continuation-read. 408 */ 409 void 410 trimPtyData(XtermWidget xw, PtyData *data) 411 { 412 (void) xw; 413 FlushLog(xw); 414 415 if (data->next != data->buffer) { 416 size_t i; 417 size_t n = (size_t) (data->last - data->next); 418 419 TRACE(("shifting buffer down by %lu\n", (unsigned long) n)); 420 for (i = 0; i < n; ++i) { 421 data->buffer[i] = data->next[i]; 422 } 423 data->next = data->buffer; 424 data->last = data->next + n; 425 } 426 427 } 428 429 /* 430 * Insert new data into the input buffer so the next calls to morePtyData() 431 * and nextPtyData() will return that. 432 */ 433 void 434 fillPtyData(XtermWidget xw, PtyData *data, const char *value, size_t length) 435 { 436 size_t size; 437 size_t n; 438 439 /* remove the used portion of the buffer */ 440 trimPtyData(xw, data); 441 442 VTbuffer->last += length; 443 size = (size_t) (VTbuffer->last - VTbuffer->next); 444 445 /* shift the unused portion up to make room */ 446 for (n = size; n >= length; --n) 447 VTbuffer->next[n] = VTbuffer->next[n - length]; 448 449 /* insert the new bytes to interpret */ 450 for (n = 0; n < length; n++) 451 VTbuffer->next[n] = CharOf(value[n]); 452 } 453 454 #if OPT_WIDE_CHARS 455 /* 456 * Convert an ISO-8859-1 code 'c' to UTF-8, storing the result in the target 457 * 'lp', and returning a pointer past the converted character. 458 */ 459 Char * 460 convertToUTF8(Char *lp, unsigned c) 461 { 462 #define CH(n) (Char)((c) >> ((n) * 8)) 463 if (c < 0x80) { 464 /* 0******* */ 465 *lp++ = (Char) CH(0); 466 } else if (c < 0x800) { 467 /* 110***** 10****** */ 468 *lp++ = (Char) (0xc0 | (CH(0) >> 6) | ((CH(1) & 0x07) << 2)); 469 *lp++ = (Char) (0x80 | (CH(0) & 0x3f)); 470 } else if (c < 0x00010000) { 471 /* 1110**** 10****** 10****** */ 472 *lp++ = (Char) (0xe0 | ((int) (CH(1) & 0xf0) >> 4)); 473 *lp++ = (Char) (0x80 | (CH(0) >> 6) | ((CH(1) & 0x0f) << 2)); 474 *lp++ = (Char) (0x80 | (CH(0) & 0x3f)); 475 } else if (c < 0x00200000) { 476 *lp++ = (Char) (0xf0 | ((int) (CH(2) & 0x1f) >> 2)); 477 *lp++ = (Char) (0x80 | 478 ((int) (CH(1) & 0xf0) >> 4) | 479 ((int) (CH(2) & 0x03) << 4)); 480 *lp++ = (Char) (0x80 | (CH(0) >> 6) | ((CH(1) & 0x0f) << 2)); 481 *lp++ = (Char) (0x80 | (CH(0) & 0x3f)); 482 } else if (c < 0x04000000) { 483 *lp++ = (Char) (0xf8 | (CH(3) & 0x03)); 484 *lp++ = (Char) (0x80 | (CH(2) >> 2)); 485 *lp++ = (Char) (0x80 | 486 ((int) (CH(1) & 0xf0) >> 4) | 487 ((int) (CH(2) & 0x03) << 4)); 488 *lp++ = (Char) (0x80 | (CH(0) >> 6) | ((CH(1) & 0x0f) << 2)); 489 *lp++ = (Char) (0x80 | (CH(0) & 0x3f)); 490 } else { 491 *lp++ = (Char) (0xfc | ((int) (CH(3) & 0x40) >> 6)); 492 *lp++ = (Char) (0x80 | (CH(3) & 0x3f)); 493 *lp++ = (Char) (0x80 | (CH(2) >> 2)); 494 *lp++ = (Char) (0x80 | (CH(1) >> 4) | ((CH(2) & 0x03) << 4)); 495 *lp++ = (Char) (0x80 | (CH(0) >> 6) | ((CH(1) & 0x0f) << 2)); 496 *lp++ = (Char) (0x80 | (CH(0) & 0x3f)); 497 } 498 return lp; 499 #undef CH 500 } 501 502 /* 503 * Convert a UTF-8 multibyte character to an Unicode value, returning a pointer 504 * past the converted UTF-8 input. The first 256 values align with ISO-8859-1, 505 * making it possible to use this to convert to Latin-1. 506 * 507 * If the conversion fails, return null. 508 */ 509 Char * 510 convertFromUTF8(Char *lp, unsigned *cp) 511 { 512 int want; 513 514 /* 515 * Find the number of bytes we will need from the source. 516 */ 517 if ((*lp & 0x80) == 0) { 518 want = 1; 519 } else if ((*lp & 0xe0) == 0xc0) { 520 want = 2; 521 } else if ((*lp & 0xf0) == 0xe0) { 522 want = 3; 523 } else if ((*lp & 0xf8) == 0xf0) { 524 want = 4; 525 } else if ((*lp & 0xfc) == 0xf8) { 526 want = 5; 527 } else if ((*lp & 0xfe) == 0xfc) { 528 want = 6; 529 } else { 530 want = 0; 531 } 532 533 if (want) { 534 int have = 1; 535 536 while (lp[have] != '\0') { 537 if ((lp[have] & 0xc0) != 0x80) 538 break; 539 ++have; 540 } 541 if (want == have) { 542 unsigned mask = 0; 543 int j; 544 int shift = 0; 545 546 *cp = 0; 547 switch (want) { 548 case 1: 549 mask = (*lp); 550 break; 551 case 2: 552 mask = (*lp & 0x1f); 553 break; 554 case 3: 555 mask = (*lp & 0x0f); 556 break; 557 case 4: 558 mask = (*lp & 0x07); 559 break; 560 case 5: 561 mask = (*lp & 0x03); 562 break; 563 case 6: 564 mask = (*lp & 0x01); 565 break; 566 default: 567 mask = 0; 568 break; 569 } 570 571 for (j = 1; j < want; j++) { 572 *cp |= (unsigned) ((lp[want - j] & 0x3f) << shift); 573 shift += 6; 574 } 575 *cp |= mask << shift; 576 lp += want; 577 } else { 578 *cp = BAD_ASCII; 579 lp = NULL; 580 } 581 } else { 582 *cp = BAD_ASCII; 583 lp = NULL; 584 } 585 return lp; 586 } 587 588 /* 589 * Returns true if the entire string is valid UTF-8. 590 */ 591 Boolean 592 isValidUTF8(Char *lp) 593 { 594 Boolean result = True; 595 while (*lp) { 596 unsigned ch; 597 Char *next = convertFromUTF8(lp, &ch); 598 if (next == NULL || ch == 0) { 599 result = False; 600 break; 601 } 602 lp = next; 603 } 604 return result; 605 } 606 607 /* 608 * Write data back to the PTY 609 */ 610 void 611 writePtyData(int f, IChar *d, size_t len) 612 { 613 size_t n = (len << 1); 614 615 if (VTbuffer->write_len <= len) { 616 VTbuffer->write_len = n; 617 VTbuffer->write_buf = realloc(VTbuffer->write_buf, VTbuffer->write_len); 618 } 619 620 for (n = 0; n < len; n++) 621 VTbuffer->write_buf[n] = (Char) d[n]; 622 623 TRACE(("writePtyData %lu:%s\n", (unsigned long) n, 624 visibleChars(VTbuffer->write_buf, n))); 625 v_write(f, VTbuffer->write_buf, n); 626 } 627 #endif /* OPT_WIDE_CHARS */ 628 629 #ifdef NO_LEAKS 630 void 631 noleaks_ptydata(void) 632 { 633 if (VTbuffer != NULL) { 634 #if OPT_WIDE_CHARS 635 free(VTbuffer->write_buf); 636 #endif 637 FreeAndNull(VTbuffer); 638 } 639 } 640 #endif 641 642 #ifdef TEST_DRIVER 643 644 #include "data.c" 645 646 void 647 NormalExit(void) 648 { 649 fprintf(stderr, "NormalExit!\n"); 650 exit(EXIT_SUCCESS); 651 } 652 653 void 654 Panic(const char *s, int a) 655 { 656 (void) s; 657 (void) a; 658 fprintf(stderr, "Panic!\n"); 659 exit(EXIT_FAILURE); 660 } 661 662 #if OPT_WIDE_CHARS 663 void 664 saveCharsets(TScreen *screen, DECNRCM_codes * target) 665 { 666 (void) screen; 667 (void) target; 668 } 669 670 void 671 restoreCharsets(TScreen *screen, const DECNRCM_codes * target) 672 { 673 (void) screen; 674 (void) target; 675 } 676 677 void 678 resetCharsets(TScreen *screen) 679 { 680 (void) screen; 681 } 682 683 #ifdef ALLOWLOGGING 684 void 685 FlushLog(XtermWidget xw) 686 { 687 (void) xw; 688 } 689 #endif 690 691 void 692 v_write(int f, const Char *data, size_t len) 693 { 694 (void) f; 695 (void) data; 696 (void) len; 697 } 698 699 void 700 mk_wcwidth_init(int mode) 701 { 702 (void) mode; 703 } 704 705 void 706 update_font_utf8_mode(void) 707 { 708 } 709 710 static int message_level = 0; 711 static int opt_all = 0; 712 static int opt_illegal = 0; 713 static int opt_convert = 0; 714 static int opt_reverse = 0; 715 static long total_test = 0; 716 static long total_errs = 0; 717 718 static void 719 usage(void) 720 { 721 static const char *msg[] = 722 { 723 "Usage: test_ptydata [options] [c1[-c1b] [c2-[c2b] [...]]]", 724 "", 725 "Options:", 726 " -a exercise all legal encode/decode to/from UTF-8", 727 " -c call convertFromUTF8 rather than decodeUTF8", 728 " -i ignore illegal UTF-8 when testing -r option", 729 " -q quieter", 730 " -r reverse/decode from UTF-8 byte-string to/from Unicode", 731 " -v more verbose" 732 }; 733 size_t n; 734 for (n = 0; n < sizeof(msg) / sizeof(msg[0]); ++n) { 735 fprintf(stderr, "%s\n", msg[n]); 736 } 737 exit(EXIT_FAILURE); 738 } 739 740 /* 741 * http://www.unicode.org/versions/corrigendum1.html, table 3.1B 742 */ 743 #define OkRange(n,lo,hi) \ 744 if (value[n] < lo || value[n] > hi) { \ 745 result = False; \ 746 break; \ 747 } 748 static Bool 749 is_legal_utf8(const Char *value) 750 { 751 Bool result = True; 752 Char ch; 753 while ((ch = *value) != '\0') { 754 if (ch <= 0x7f) { 755 ++value; 756 } else if (ch >= 0xc2 && ch <= 0xdf) { 757 OkRange(1, 0x80, 0xbf); 758 value += 2; 759 } else if (ch == 0xe0) { 760 OkRange(1, 0xa0, 0xbf); 761 OkRange(2, 0x80, 0xbf); 762 value += 3; 763 } else if (ch >= 0xe1 && ch <= 0xef) { 764 OkRange(1, 0x80, 0xbf); 765 OkRange(2, 0x80, 0xbf); 766 value += 3; 767 } else if (ch == 0xf0) { 768 OkRange(1, 0x90, 0xbf); 769 OkRange(2, 0x80, 0xbf); 770 OkRange(3, 0x80, 0xbf); 771 value += 4; 772 } else if (ch >= 0xf1 && ch <= 0xf3) { 773 OkRange(1, 0x80, 0xbf); 774 OkRange(2, 0x80, 0xbf); 775 OkRange(3, 0x80, 0xbf); 776 value += 4; 777 } else if (ch == 0xf4) { 778 OkRange(1, 0x80, 0x8f); 779 OkRange(2, 0x80, 0xbf); 780 OkRange(3, 0x80, 0xbf); 781 value += 4; 782 } else { 783 result = False; 784 break; 785 } 786 } 787 return result; 788 } 789 790 static void 791 test_utf8_convert(void) 792 { 793 unsigned c_in, c_out; 794 Char buffer[10]; 795 Char *result; 796 unsigned limit = 0x110000; 797 unsigned success = 0; 798 unsigned bucket[256]; 799 800 memset(bucket, 0, sizeof(bucket)); 801 for (c_in = 0; c_in < limit; ++c_in) { 802 memset(buffer, 0, sizeof(buffer)); 803 if ((result = convertToUTF8(buffer, c_in)) == NULL) { 804 TRACE(("conversion of U+%04X to UTF-8 failed\n", c_in)); 805 } else { 806 if ((result = convertFromUTF8(buffer, &c_out)) == NULL) { 807 TRACE(("conversion of U+%04X from UTF-8 failed\n", c_in)); 808 } else if (c_in != c_out) { 809 TRACE(("conversion of U+%04X to/from UTF-8 gave U+%04X\n", 810 c_in, c_out)); 811 } else { 812 while (result-- != buffer) { 813 bucket[*result]++; 814 } 815 ++success; 816 } 817 } 818 } 819 TRACE(("%u/%u successful\n", success, limit)); 820 for (c_in = 0; c_in < 256; ++c_in) { 821 if ((c_in % 8) == 0) { 822 TRACE((" %02X:", c_in)); 823 } 824 TRACE((" %8X", bucket[c_in])); 825 if (((c_in + 1) % 8) == 0) { 826 TRACE(("\n")); 827 } 828 } 829 } 830 831 static int 832 decode_one(const char *source, char **target) 833 { 834 int result = -1; 835 long check; 836 int radix = 0; 837 if ((source[0] == 'u' || source[0] == 'U') && source[1] == '+') { 838 source += 2; 839 radix = 16; 840 } else if (source[0] == '0' && source[1] == 'b') { 841 source += 2; 842 radix = 2; 843 } 844 check = strtol(source, target, radix); 845 if (*target != NULL && *target != source) 846 result = (int) check; 847 return result; 848 } 849 850 static int 851 decode_range(const char *source, int *lo, int *hi) 852 { 853 int result = 0; 854 char *after1; 855 char *after2; 856 if ((*lo = decode_one(source, &after1)) >= 0) { 857 after1 += strspn(after1, ":-.\t "); 858 if ((*hi = decode_one(after1, &after2)) < 0) { 859 *hi = *lo; 860 } 861 result = 1; 862 } 863 return result; 864 } 865 866 #define MAX_BYTES 6 867 868 static void 869 do_range(const char *source) 870 { 871 int lo, hi; 872 873 TScreen screen; 874 memset(&screen, 0, sizeof(screen)); 875 876 if (decode_range(source, &lo, &hi)) { 877 while (lo <= hi) { 878 unsigned c_in = (unsigned) lo++; 879 PtyData *data; 880 Char *next; 881 Char buffer[MAX_BYTES + 1]; 882 883 if (opt_reverse) { 884 Bool skip = False; 885 Bool first = True; 886 int j, k; 887 for (j = 0; j < MAX_BYTES; ++j) { 888 unsigned long bits = ((unsigned long) c_in >> (8 * j)); 889 if ((buffer[j] = (Char) bits) == 0) { 890 skip = (bits != 0); 891 break; 892 } 893 } 894 if (skip) 895 continue; 896 initPtyData(&data); 897 for (k = 0; k <= j; ++k) { 898 data->buffer[k] = buffer[j - k - 1]; 899 } 900 if (opt_illegal && !is_legal_utf8(data->buffer)) { 901 free(data); 902 continue; 903 } 904 if (message_level > 1) { 905 printf("TEST "); 906 for (k = 0; k < j; ++k) { 907 printf("%02X", data->buffer[k]); 908 } 909 } 910 data->next = data->buffer; 911 data->last = data->buffer + j; 912 while (decodeUtf8(&screen, data)) { 913 total_test++; 914 if (is_UCS_SPECIAL(data->utf_data)) 915 total_errs++; 916 data->next += data->utf_size; 917 if (message_level > 1) { 918 printf("%s%04X", first ? " ->" : ", ", data->utf_data); 919 } 920 first = False; 921 } 922 if (!first) 923 total_test--; 924 if (message_level > 1) { 925 printf("\n"); 926 fflush(stdout); 927 } 928 free(data); 929 } else if (opt_convert) { 930 unsigned c_out; 931 Char *result; 932 933 memset(buffer, 0, sizeof(buffer)); 934 if ((result = next = convertToUTF8(buffer, c_in)) == NULL) { 935 fprintf(stderr, 936 "conversion of U+%04X to UTF-8 failed\n", c_in); 937 } else if ((result = convertFromUTF8(buffer, &c_out)) == NULL) { 938 fprintf(stderr, 939 "conversion of U+%04X from UTF-8 failed\n", c_in); 940 total_errs++; 941 } else if (c_in != c_out) { 942 fprintf(stderr, 943 "conversion of U+%04X to/from UTF-8 gave U+%04X\n", 944 c_in, c_out); 945 } else if (message_level > 1) { 946 *next = '\0'; 947 printf("TEST %04X (%lu:%s) ->%04X\n", c_in, 948 (unsigned long) (next - buffer), 949 buffer, 950 c_out); 951 fflush(stdout); 952 } 953 } else { 954 initPtyData(&data); 955 next = convertToUTF8(data->buffer, c_in); 956 *next = 0; 957 data->next = data->buffer; 958 data->last = next; 959 decodeUtf8(&screen, data); 960 if (message_level > 1) { 961 printf("TEST %04X (%lu:%s) ->%04X\n", c_in, 962 (unsigned long) (next - data->buffer), 963 data->buffer, 964 data->utf_data); 965 fflush(stdout); 966 } 967 if (c_in != data->utf_data) { 968 fprintf(stderr, "Mismatch: %04X vs %04X\n", c_in, data->utf_data); 969 total_errs++; 970 } 971 free(data); 972 } 973 total_test++; 974 } 975 } 976 } 977 978 int 979 main(int argc, char **argv) 980 { 981 int ch; 982 983 setlocale(LC_ALL, ""); 984 while ((ch = getopt(argc, argv, "aciqrv")) != -1) { 985 switch (ch) { 986 case 'a': 987 opt_all = 1; 988 break; 989 case 'c': 990 opt_convert = 1; 991 break; 992 case 'i': 993 opt_illegal = 1; 994 break; 995 case 'q': 996 message_level--; 997 break; 998 case 'r': 999 opt_reverse = 1; 1000 break; 1001 case 'v': 1002 message_level++; 1003 break; 1004 default: 1005 usage(); 1006 } 1007 } 1008 if (opt_all) { 1009 test_utf8_convert(); 1010 } else { 1011 if (optind >= argc) 1012 usage(); 1013 while (optind < argc) { 1014 do_range(argv[optind++]); 1015 } 1016 if (total_test) { 1017 printf("%ld/%ld mismatches (%.0f%%)\n", 1018 total_errs, 1019 total_test, 1020 (100.0 * (double) total_errs) / (double) total_test); 1021 } 1022 } 1023 return EXIT_SUCCESS; 1024 } 1025 #else 1026 int 1027 main(int argc, char **argv) 1028 { 1029 (void) argc; 1030 (void) argv; 1031 printf("Nothing to be done here...\n"); 1032 return EXIT_SUCCESS; 1033 } 1034 #endif /* OPT_WIDE_CHARS */ 1035 #endif 1036
Indexes created Mon Aug 17 00:26:26 UTC 2026