lexi.c revision 1.86
1 /* $NetBSD: lexi.c,v 1.86 2021/10/08 23:43:33 rillig Exp $ */ 2 3 /*- 4 * SPDX-License-Identifier: BSD-4-Clause 5 * 6 * Copyright (c) 1985 Sun Microsystems, Inc. 7 * Copyright (c) 1980, 1993 8 * The Regents of the University of California. All rights reserved. 9 * All rights reserved. 10 * 11 * Redistribution and use in source and binary forms, with or without 12 * modification, are permitted provided that the following conditions 13 * are met: 14 * 1. Redistributions of source code must retain the above copyright 15 * notice, this list of conditions and the following disclaimer. 16 * 2. Redistributions in binary form must reproduce the above copyright 17 * notice, this list of conditions and the following disclaimer in the 18 * documentation and/or other materials provided with the distribution. 19 * 3. All advertising materials mentioning features or use of this software 20 * must display the following acknowledgement: 21 * This product includes software developed by the University of 22 * California, Berkeley and its contributors. 23 * 4. Neither the name of the University nor the names of its contributors 24 * may be used to endorse or promote products derived from this software 25 * without specific prior written permission. 26 * 27 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 28 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 29 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 30 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 31 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 32 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 33 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 34 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 35 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 36 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 37 * SUCH DAMAGE. 38 */ 39 40 #if 0 41 static char sccsid[] = "@(#)lexi.c 8.1 (Berkeley) 6/6/93"; 42 #endif 43 44 #include <sys/cdefs.h> 45 #if defined(__NetBSD__) 46 __RCSID("$NetBSD: lexi.c,v 1.86 2021/10/08 23:43:33 rillig Exp $"); 47 #elif defined(__FreeBSD__) 48 __FBSDID("$FreeBSD: head/usr.bin/indent/lexi.c 337862 2018-08-15 18:19:45Z pstef $"); 49 #endif 50 51 #include <assert.h> 52 #include <stdio.h> 53 #include <ctype.h> 54 #include <stdlib.h> 55 #include <string.h> 56 #include <sys/param.h> 57 58 #include "indent.h" 59 60 /* must be sorted alphabetically, is used in binary search */ 61 static const struct keyword { 62 const char *name; 63 enum keyword_kind kind; 64 } keywords[] = { 65 {"_Bool", kw_type}, 66 {"_Complex", kw_type}, 67 {"_Imaginary", kw_type}, 68 {"auto", kw_storage_class}, 69 {"bool", kw_type}, 70 {"break", kw_jump}, 71 {"case", kw_case_or_default}, 72 {"char", kw_type}, 73 {"complex", kw_type}, 74 {"const", kw_type}, 75 {"continue", kw_jump}, 76 {"default", kw_case_or_default}, 77 {"do", kw_do_or_else}, 78 {"double", kw_type}, 79 {"else", kw_do_or_else}, 80 {"enum", kw_struct_or_union_or_enum}, 81 {"extern", kw_storage_class}, 82 {"float", kw_type}, 83 {"for", kw_for_or_if_or_while}, 84 {"goto", kw_jump}, 85 {"if", kw_for_or_if_or_while}, 86 {"imaginary", kw_type}, 87 {"inline", kw_inline_or_restrict}, 88 {"int", kw_type}, 89 {"long", kw_type}, 90 {"offsetof", kw_offsetof}, 91 {"register", kw_storage_class}, 92 {"restrict", kw_inline_or_restrict}, 93 {"return", kw_jump}, 94 {"short", kw_type}, 95 {"signed", kw_type}, 96 {"sizeof", kw_sizeof}, 97 {"static", kw_storage_class}, 98 {"struct", kw_struct_or_union_or_enum}, 99 {"switch", kw_switch}, 100 {"typedef", kw_typedef}, 101 {"union", kw_struct_or_union_or_enum}, 102 {"unsigned", kw_type}, 103 {"void", kw_type}, 104 {"volatile", kw_type}, 105 {"while", kw_for_or_if_or_while} 106 }; 107 108 static struct { 109 const char **items; 110 unsigned int len; 111 unsigned int cap; 112 } typenames; 113 114 /* 115 * The transition table below was rewritten by hand from lx's output, given 116 * the following definitions. lx is Katherine Flavel's lexer generator. 117 * 118 * O = /[0-7]/; D = /[0-9]/; NZ = /[1-9]/; 119 * H = /[a-f0-9]/i; B = /[0-1]/; HP = /0x/i; 120 * BP = /0b/i; E = /e[+\-]?/i D+; P = /p[+\-]?/i D+; 121 * FS = /[fl]/i; IS = /u/i /(l|L|ll|LL)/? | /(l|L|ll|LL)/ /u/i?; 122 * 123 * D+ E FS? -> $float; 124 * D* "." D+ E? FS? -> $float; 125 * D+ "." E? FS? -> $float; HP H+ IS? -> $int; 126 * HP H+ P FS? -> $float; NZ D* IS? -> $int; 127 * HP H* "." H+ P FS? -> $float; "0" O* IS? -> $int; 128 * HP H+ "." P FS -> $float; BP B+ IS? -> $int; 129 */ 130 /* INDENT OFF */ 131 static const unsigned char lex_number_state[][26] = { 132 /* examples: 133 00 134 s 0xx 135 t 00xaa 136 a 11 101100xxa.. 137 r 11ee0001101lbuuxx.a.pp 138 t.01.e+008bLuxll0Ll.aa.p+0 139 states: ABCDEFGHIJKLMNOPQRSTUVWXYZ */ 140 [0] = "uuiifuufiuuiiuiiiiiuiuuuuu", /* (other) */ 141 [1] = "CEIDEHHHIJQ U Q VUVVZZZ", /* 0 */ 142 [2] = "DEIDEHHHIJQ U Q VUVVZZZ", /* 1 */ 143 [3] = "DEIDEHHHIJ U VUVVZZZ", /* 2 3 4 5 6 7 */ 144 [4] = "DEJDEHHHJJ U VUVVZZZ", /* 8 9 */ 145 [5] = " U VUVV ", /* A a C c D d */ 146 [6] = " K U VUVV ", /* B b */ 147 [7] = " FFF FF U VUVV ", /* E e */ 148 [8] = " f f U VUVV f", /* F f */ 149 [9] = " LLf fL PR Li L f", /* L */ 150 [10] = " OOf fO S P O i O f", /* l */ 151 [11] = " FFX ", /* P p */ 152 [12] = " MM M i iiM M ", /* U u */ 153 [13] = " N ", /* X x */ 154 [14] = " G Y ", /* + - */ 155 [15] = "B EE EE T W ", /* . */ 156 /* ABCDEFGHIJKLMNOPQRSTUVWXYZ */ 157 }; 158 /* INDENT ON */ 159 160 static const uint8_t lex_number_row[] = { 161 ['0'] = 1, 162 ['1'] = 2, 163 ['2'] = 3, ['3'] = 3, ['4'] = 3, ['5'] = 3, ['6'] = 3, ['7'] = 3, 164 ['8'] = 4, ['9'] = 4, 165 ['A'] = 5, ['a'] = 5, ['C'] = 5, ['c'] = 5, ['D'] = 5, ['d'] = 5, 166 ['B'] = 6, ['b'] = 6, 167 ['E'] = 7, ['e'] = 7, 168 ['F'] = 8, ['f'] = 8, 169 ['L'] = 9, 170 ['l'] = 10, 171 ['P'] = 11, ['p'] = 11, 172 ['U'] = 12, ['u'] = 12, 173 ['X'] = 13, ['x'] = 13, 174 ['+'] = 14, ['-'] = 14, 175 ['.'] = 15, 176 }; 177 178 static char 179 inbuf_peek(void) 180 { 181 return *inp.s; 182 } 183 184 void 185 inbuf_skip(void) 186 { 187 inp.s++; 188 if (inp.s >= inp.e) 189 inbuf_read_line(); 190 } 191 192 char 193 inbuf_next(void) 194 { 195 char ch = inbuf_peek(); 196 inbuf_skip(); 197 return ch; 198 } 199 200 static void 201 check_size_token(size_t desired_size) 202 { 203 if (token.e + desired_size >= token.l) 204 buf_expand(&token, desired_size); 205 } 206 207 static int 208 cmp_keyword_by_name(const void *key, const void *elem) 209 { 210 return strcmp(key, ((const struct keyword *)elem)->name); 211 } 212 213 #ifdef debug 214 const char * 215 token_type_name(token_type ttype) 216 { 217 static const char *const name[] = { 218 "end_of_file", "newline", "lparen_or_lbracket", "rparen_or_rbracket", 219 "unary_op", "binary_op", "postfix_op", "question", 220 "case_label", "colon", 221 "semicolon", "lbrace", "rbrace", "ident", "comma", 222 "comment", "switch_expr", "preprocessing", "form_feed", "decl", 223 "keyword_for_if_while", "keyword_do_else", 224 "if_expr", "while_expr", "for_exprs", 225 "stmt", "stmt_list", "keyword_else", "keyword_do", "do_stmt", 226 "if_expr_stmt", "if_expr_stmt_else", "period", "string_prefix", 227 "storage_class", "funcname", "type_def", "keyword_struct_union_enum" 228 }; 229 230 assert(0 <= ttype && ttype < nitems(name)); 231 232 return name[ttype]; 233 } 234 235 static void 236 debug_print_buf(const char *name, const struct buffer *buf) 237 { 238 if (buf->s < buf->e) { 239 debug_printf(" %s ", name); 240 debug_vis_range("\"", buf->s, buf->e, "\""); 241 } 242 } 243 244 static token_type 245 lexi_end(token_type ttype) 246 { 247 debug_printf("in line %d, lexi returns '%s'", 248 line_no, token_type_name(ttype)); 249 debug_print_buf("token", &token); 250 debug_print_buf("label", &lab); 251 debug_print_buf("code", &code); 252 debug_print_buf("comment", &com); 253 debug_printf("\n"); 254 255 return ttype; 256 } 257 #else 258 #define lexi_end(tk) (tk) 259 #endif 260 261 static void 262 lex_number(void) 263 { 264 for (uint8_t s = 'A'; s != 'f' && s != 'i' && s != 'u';) { 265 uint8_t ch = (uint8_t)*inp.s; 266 if (ch >= nitems(lex_number_row) || lex_number_row[ch] == 0) 267 break; 268 269 uint8_t row = lex_number_row[ch]; 270 if (lex_number_state[row][s - 'A'] == ' ') { 271 /*- 272 * lex_number_state[0][s - 'A'] now indicates the type: 273 * f = floating, i = integer, u = unknown 274 */ 275 break; 276 } 277 278 s = lex_number_state[row][s - 'A']; 279 check_size_token(1); 280 *token.e++ = inbuf_next(); 281 } 282 } 283 284 static void 285 lex_word(void) 286 { 287 while (isalnum((unsigned char)*inp.s) || 288 *inp.s == '\\' || 289 *inp.s == '_' || *inp.s == '$') { 290 291 if (*inp.s == '\\') { 292 if (inp.s[1] == '\n') { 293 inp.s += 2; 294 if (inp.s >= inp.e) 295 inbuf_read_line(); 296 } else 297 break; 298 } 299 300 check_size_token(1); 301 *token.e++ = inbuf_next(); 302 } 303 } 304 305 static void 306 lex_char_or_string(void) 307 { 308 for (char delim = *token.s;;) { 309 if (*inp.s == '\n') { 310 diag(1, "Unterminated literal"); 311 return; 312 } 313 314 check_size_token(2); 315 *token.e++ = inbuf_next(); 316 if (token.e[-1] == delim) 317 return; 318 319 if (token.e[-1] == '\\') { 320 if (*inp.s == '\n') 321 ++line_no; 322 *token.e++ = inbuf_next(); 323 } 324 } 325 } 326 327 /* Guess whether the current token is a declared type. */ 328 static bool 329 probably_typename(const struct parser_state *state) 330 { 331 if (state->p_l_follow != 0) 332 return false; 333 if (state->block_init || state->in_stmt) 334 return false; 335 if (inp.s[0] == '*' && inp.s[1] != '=') 336 goto maybe; 337 if (isalpha((unsigned char)*inp.s)) 338 goto maybe; 339 return false; 340 maybe: 341 return state->last_token == semicolon || 342 state->last_token == lbrace || 343 state->last_token == rbrace; 344 } 345 346 static int 347 bsearch_typenames(const char *key) 348 { 349 const char **arr = typenames.items; 350 int lo = 0; 351 int hi = (int)typenames.len - 1; 352 353 while (lo <= hi) { 354 int mid = (int)((unsigned)(lo + hi) >> 1); 355 int cmp = strcmp(arr[mid], key); 356 if (cmp < 0) 357 lo = mid + 1; 358 else if (cmp > 0) 359 hi = mid - 1; 360 else 361 return mid; 362 } 363 return -(lo + 1); 364 } 365 366 static bool 367 is_typename(void) 368 { 369 if (opt.auto_typedefs && 370 token.e - token.s >= 2 && memcmp(token.e - 2, "_t", 2) == 0) 371 return true; 372 373 return bsearch_typenames(token.s) >= 0; 374 } 375 376 /* Reads the next token, placing it in the global variable "token". */ 377 token_type 378 lexi(struct parser_state *state) 379 { 380 bool unary_delim; /* whether the current token forces a 381 * following operator to be unary */ 382 token_type ttype; 383 384 token.e = token.s; /* point to start of place to save token */ 385 unary_delim = false; 386 state->col_1 = state->last_nl; /* tell world that this token started 387 * in column 1 iff the last thing 388 * scanned was a newline */ 389 state->last_nl = false; 390 391 while (is_hspace(*inp.s)) { 392 state->col_1 = false; 393 inbuf_skip(); 394 } 395 396 /* Scan an alphanumeric token */ 397 if (isalnum((unsigned char)*inp.s) || 398 *inp.s == '_' || *inp.s == '$' || 399 (inp.s[0] == '.' && isdigit((unsigned char)inp.s[1]))) { 400 struct keyword *kw; 401 402 if (isdigit((unsigned char)*inp.s) || 403 (inp.s[0] == '.' && isdigit((unsigned char)inp.s[1]))) { 404 lex_number(); 405 } else { 406 lex_word(); 407 } 408 *token.e = '\0'; 409 410 if (token.s[0] == 'L' && token.s[1] == '\0' && 411 (*inp.s == '"' || *inp.s == '\'')) 412 return lexi_end(string_prefix); 413 414 while (is_hspace(inbuf_peek())) 415 inbuf_skip(); 416 state->keyword = kw_0; 417 418 if (state->last_token == keyword_struct_union_enum && 419 state->p_l_follow == 0) { 420 state->last_u_d = true; 421 return lexi_end(decl); 422 } 423 /* 424 * Operator after identifier is binary unless last token was 'struct' 425 */ 426 state->last_u_d = (state->last_token == keyword_struct_union_enum); 427 428 kw = bsearch(token.s, keywords, nitems(keywords), 429 sizeof(keywords[0]), cmp_keyword_by_name); 430 if (kw == NULL) { 431 if (is_typename()) { 432 state->keyword = kw_type; 433 state->last_u_d = true; 434 goto found_typename; 435 } 436 437 } else { /* we have a keyword */ 438 state->keyword = kw->kind; 439 state->last_u_d = true; 440 441 switch (kw->kind) { 442 case kw_switch: 443 return lexi_end(switch_expr); 444 445 case kw_case_or_default: 446 return lexi_end(case_label); 447 448 case kw_struct_or_union_or_enum: 449 case kw_type: 450 found_typename: 451 if (state->p_l_follow != 0) { 452 /* inside parens: cast, param list, offsetof or sizeof */ 453 state->cast_mask |= (1 << state->p_l_follow) & ~state->not_cast_mask; 454 } 455 if (state->last_token == period || state->last_token == unary_op) { 456 state->keyword = kw_0; 457 break; 458 } 459 if (kw != NULL && kw->kind == kw_struct_or_union_or_enum) 460 return lexi_end(keyword_struct_union_enum); 461 if (state->p_l_follow != 0) 462 break; 463 return lexi_end(decl); 464 465 case kw_for_or_if_or_while: 466 return lexi_end(keyword_for_if_while); 467 468 case kw_do_or_else: 469 return lexi_end(keyword_do_else); 470 471 case kw_storage_class: 472 return lexi_end(storage_class); 473 474 case kw_typedef: 475 return lexi_end(type_def); 476 477 default: /* all others are treated like any other 478 * identifier */ 479 return lexi_end(ident); 480 } /* end of switch */ 481 } /* end of if (found_it) */ 482 483 if (*inp.s == '(' && state->tos <= 1 && state->ind_level == 0 && 484 !state->in_parameter_declaration && !state->block_init) { 485 486 for (const char *p = inp.s; p < inp.e;) 487 if (*p++ == ')' && (*p == ';' || *p == ',')) 488 goto not_proc; 489 490 strncpy(state->procname, token.s, sizeof state->procname - 1); 491 if (state->in_decl) 492 state->in_parameter_declaration = true; 493 return lexi_end(funcname); 494 not_proc:; 495 496 } else if (probably_typename(state)) { 497 state->keyword = kw_type; 498 state->last_u_d = true; 499 return lexi_end(decl); 500 } 501 502 if (state->last_token == decl) /* if this is a declared variable, 503 * then following sign is unary */ 504 state->last_u_d = true; /* will make "int a -1" work */ 505 506 return lexi_end(ident); /* the ident is not in the list */ 507 } /* end of processing for alphanum character */ 508 509 /* Scan a non-alphanumeric token */ 510 511 check_size_token(3); /* things like "<<=" */ 512 *token.e++ = inbuf_next(); /* if it is only a one-character token, it is 513 * moved here */ 514 *token.e = '\0'; 515 516 switch (*token.s) { 517 case '\n': 518 unary_delim = state->last_u_d; 519 state->last_nl = true; /* remember that we just had a newline */ 520 /* if data has been exhausted, the newline is a dummy. */ 521 ttype = had_eof ? end_of_file : newline; 522 break; 523 524 case '\'': 525 case '"': 526 lex_char_or_string(); 527 ttype = ident; 528 break; 529 530 case '(': 531 case '[': 532 unary_delim = true; 533 ttype = lparen_or_lbracket; 534 break; 535 536 case ')': 537 case ']': 538 ttype = rparen_or_rbracket; 539 break; 540 541 case '#': 542 unary_delim = state->last_u_d; 543 ttype = preprocessing; 544 break; 545 546 case '?': 547 unary_delim = true; 548 ttype = question; 549 break; 550 551 case ':': 552 ttype = colon; 553 unary_delim = true; 554 break; 555 556 case ';': 557 unary_delim = true; 558 ttype = semicolon; 559 break; 560 561 case '{': 562 unary_delim = true; 563 ttype = lbrace; 564 break; 565 566 case '}': 567 unary_delim = true; 568 ttype = rbrace; 569 break; 570 571 case '\f': 572 unary_delim = state->last_u_d; 573 state->last_nl = true; /* remember this, so we can set 'state->col_1' 574 * right */ 575 ttype = form_feed; 576 break; 577 578 case ',': 579 unary_delim = true; 580 ttype = comma; 581 break; 582 583 case '.': 584 unary_delim = false; 585 ttype = period; 586 break; 587 588 case '-': 589 case '+': /* check for -, +, --, ++ */ 590 ttype = state->last_u_d ? unary_op : binary_op; 591 unary_delim = true; 592 593 if (*inp.s == token.s[0]) { 594 /* check for doubled character */ 595 *token.e++ = *inp.s++; 596 /* buffer overflow will be checked at end of loop */ 597 if (state->last_token == ident || 598 state->last_token == rparen_or_rbracket) { 599 ttype = state->last_u_d ? unary_op : postfix_op; 600 /* check for following ++ or -- */ 601 unary_delim = false; 602 } 603 604 } else if (*inp.s == '=') { 605 /* check for operator += */ 606 *token.e++ = *inp.s++; 607 608 } else if (*inp.s == '>') { 609 /* check for operator -> */ 610 *token.e++ = *inp.s++; 611 unary_delim = false; 612 ttype = unary_op; 613 state->want_blank = false; 614 } 615 break; /* buffer overflow will be checked at end of 616 * switch */ 617 618 case '=': 619 if (state->init_or_struct) 620 state->block_init = true; 621 if (*inp.s == '=') { /* == */ 622 *token.e++ = *inp.s++; 623 *token.e = '\0'; 624 } 625 ttype = binary_op; 626 unary_delim = true; 627 break; 628 629 case '>': 630 case '<': 631 case '!': /* ops like <, <<, <=, !=, etc */ 632 if (*inp.s == '>' || *inp.s == '<' || *inp.s == '=') 633 *token.e++ = inbuf_next(); 634 if (*inp.s == '=') 635 *token.e++ = *inp.s++; 636 ttype = state->last_u_d ? unary_op : binary_op; 637 unary_delim = true; 638 break; 639 640 case '*': 641 unary_delim = true; 642 if (!state->last_u_d) { 643 if (*inp.s == '=') 644 *token.e++ = *inp.s++; 645 ttype = binary_op; 646 break; 647 } 648 649 while (*inp.s == '*' || isspace((unsigned char)*inp.s)) { 650 if (*inp.s == '*') { 651 check_size_token(1); 652 *token.e++ = *inp.s; 653 } 654 inbuf_skip(); 655 } 656 657 if (ps.in_decl) { 658 char *tp = inp.s; 659 660 while (isalpha((unsigned char)*tp) || 661 isspace((unsigned char)*tp)) { 662 if (++tp >= inp.e) 663 inbuf_read_line(); 664 } 665 if (*tp == '(') 666 ps.procname[0] = ' '; 667 } 668 669 ttype = unary_op; 670 break; 671 672 default: 673 if (token.s[0] == '/' && (*inp.s == '*' || *inp.s == '/')) { 674 /* it is start of comment */ 675 *token.e++ = inbuf_next(); 676 677 ttype = comment; 678 unary_delim = state->last_u_d; 679 break; 680 } 681 682 while (token.e[-1] == *inp.s || *inp.s == '=') { 683 /* 684 * handle ||, &&, etc, and also things as in int *****i 685 */ 686 check_size_token(1); 687 *token.e++ = inbuf_next(); 688 } 689 690 ttype = state->last_u_d ? unary_op : binary_op; 691 unary_delim = true; 692 } 693 694 if (inp.s >= inp.e) /* check for input buffer empty */ 695 inbuf_read_line(); 696 697 state->last_u_d = unary_delim; 698 699 check_size_token(1); 700 *token.e = '\0'; 701 702 return lexi_end(ttype); 703 } 704 705 void 706 add_typename(const char *name) 707 { 708 if (typenames.len >= typenames.cap) { 709 typenames.cap = 16 + 2 * typenames.cap; 710 typenames.items = xrealloc(typenames.items, 711 sizeof(typenames.items[0]) * typenames.cap); 712 } 713 714 int pos = bsearch_typenames(name); 715 if (pos >= 0) 716 return; /* already in the list */ 717 718 pos = -(pos + 1); 719 memmove(typenames.items + pos + 1, typenames.items + pos, 720 sizeof(typenames.items[0]) * (typenames.len++ - (unsigned)pos)); 721 typenames.items[pos] = xstrdup(name); 722 } 723
Indexes created Wed Sep 24 05:09:52 GMT 2025