lexi.c revision 1.82
1 /* $NetBSD: lexi.c,v 1.82 2021/10/08 21:16:23 rillig Exp $ */ 2 3 /*- 4 * SPDX-License-Identifier: BSD-4-Clause 5 * 6 * Copyright (c) 1985 Sun Microsystems, Inc. 7 * Copyright (c) 1980, 1993 8 * The Regents of the University of California. All rights reserved. 9 * All rights reserved. 10 * 11 * Redistribution and use in source and binary forms, with or without 12 * modification, are permitted provided that the following conditions 13 * are met: 14 * 1. Redistributions of source code must retain the above copyright 15 * notice, this list of conditions and the following disclaimer. 16 * 2. Redistributions in binary form must reproduce the above copyright 17 * notice, this list of conditions and the following disclaimer in the 18 * documentation and/or other materials provided with the distribution. 19 * 3. All advertising materials mentioning features or use of this software 20 * must display the following acknowledgement: 21 * This product includes software developed by the University of 22 * California, Berkeley and its contributors. 23 * 4. Neither the name of the University nor the names of its contributors 24 * may be used to endorse or promote products derived from this software 25 * without specific prior written permission. 26 * 27 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 28 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 29 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 30 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 31 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 32 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 33 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 34 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 35 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 36 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 37 * SUCH DAMAGE. 38 */ 39 40 #if 0 41 static char sccsid[] = "@(#)lexi.c 8.1 (Berkeley) 6/6/93"; 42 #endif 43 44 #include <sys/cdefs.h> 45 #if defined(__NetBSD__) 46 __RCSID("$NetBSD: lexi.c,v 1.82 2021/10/08 21:16:23 rillig Exp $"); 47 #elif defined(__FreeBSD__) 48 __FBSDID("$FreeBSD: head/usr.bin/indent/lexi.c 337862 2018-08-15 18:19:45Z pstef $"); 49 #endif 50 51 #include <assert.h> 52 #include <stdio.h> 53 #include <ctype.h> 54 #include <stdlib.h> 55 #include <string.h> 56 #include <sys/param.h> 57 58 #include "indent.h" 59 60 /* must be sorted alphabetically, is used in binary search */ 61 static const struct keyword { 62 const char *name; 63 enum keyword_kind kind; 64 } keywords[] = { 65 {"_Bool", kw_type}, 66 {"_Complex", kw_type}, 67 {"_Imaginary", kw_type}, 68 {"auto", kw_storage_class}, 69 {"bool", kw_type}, 70 {"break", kw_jump}, 71 {"case", kw_case_or_default}, 72 {"char", kw_type}, 73 {"complex", kw_type}, 74 {"const", kw_type}, 75 {"continue", kw_jump}, 76 {"default", kw_case_or_default}, 77 {"do", kw_do_or_else}, 78 {"double", kw_type}, 79 {"else", kw_do_or_else}, 80 {"enum", kw_struct_or_union_or_enum}, 81 {"extern", kw_storage_class}, 82 {"float", kw_type}, 83 {"for", kw_for_or_if_or_while}, 84 {"global", kw_type}, 85 {"goto", kw_jump}, 86 {"if", kw_for_or_if_or_while}, 87 {"imaginary", kw_type}, 88 {"inline", kw_inline_or_restrict}, 89 {"int", kw_type}, 90 {"long", kw_type}, 91 {"offsetof", kw_offsetof}, 92 {"register", kw_storage_class}, 93 {"restrict", kw_inline_or_restrict}, 94 {"return", kw_jump}, 95 {"short", kw_type}, 96 {"signed", kw_type}, 97 {"sizeof", kw_sizeof}, 98 {"static", kw_storage_class}, 99 {"struct", kw_struct_or_union_or_enum}, 100 {"switch", kw_switch}, 101 {"typedef", kw_typedef}, 102 {"union", kw_struct_or_union_or_enum}, 103 {"unsigned", kw_type}, 104 {"void", kw_type}, 105 {"volatile", kw_type}, 106 {"while", kw_for_or_if_or_while} 107 }; 108 109 struct { 110 const char **items; 111 unsigned int len; 112 unsigned int cap; 113 } typenames; 114 115 /* 116 * The transition table below was rewritten by hand from lx's output, given 117 * the following definitions. lx is Katherine Flavel's lexer generator. 118 * 119 * O = /[0-7]/; D = /[0-9]/; NZ = /[1-9]/; 120 * H = /[a-f0-9]/i; B = /[0-1]/; HP = /0x/i; 121 * BP = /0b/i; E = /e[+\-]?/i D+; P = /p[+\-]?/i D+; 122 * FS = /[fl]/i; IS = /u/i /(l|L|ll|LL)/? | /(l|L|ll|LL)/ /u/i?; 123 * 124 * D+ E FS? -> $float; 125 * D* "." D+ E? FS? -> $float; 126 * D+ "." E? FS? -> $float; HP H+ IS? -> $int; 127 * HP H+ P FS? -> $float; NZ D* IS? -> $int; 128 * HP H* "." H+ P FS? -> $float; "0" O* IS? -> $int; 129 * HP H+ "." P FS -> $float; BP B+ IS? -> $int; 130 */ 131 /* INDENT OFF */ 132 static const unsigned char lex_number_state[][26] = { 133 /* examples: 134 00 135 s 0xx 136 t 00xaa 137 a 11 101100xxa.. 138 r 11ee0001101lbuuxx.a.pp 139 t.01.e+008bLuxll0Ll.aa.p+0 140 states: ABCDEFGHIJKLMNOPQRSTUVWXYZ */ 141 [0] = "uuiifuufiuuiiuiiiiiuiuuuuu", 142 [1] = "CEIDEHHHIJQ U Q VUVVZZZ", 143 [2] = "DEIDEHHHIJQ U Q VUVVZZZ", 144 [3] = "DEIDEHHHIJ U VUVVZZZ", 145 [4] = "DEJDEHHHJJ U VUVVZZZ", 146 [5] = " U VUVV ", 147 [6] = " K U VUVV ", 148 [7] = " FFF FF U VUVV ", 149 [8] = " f f U VUVV f", 150 [9] = " LLf fL PR Li L f", 151 [10] = " OOf fO S P O i O f", 152 [11] = " FFX ", 153 [12] = " MM M i iiM M ", 154 [13] = " N ", 155 [14] = " G Y ", 156 [15] = "B EE EE T W ", 157 /* ABCDEFGHIJKLMNOPQRSTUVWXYZ */ 158 }; 159 /* INDENT ON */ 160 161 static const uint8_t lex_number_row[] = { 162 ['0'] = 1, 163 ['1'] = 2, 164 ['2'] = 3, ['3'] = 3, ['4'] = 3, ['5'] = 3, ['6'] = 3, ['7'] = 3, 165 ['8'] = 4, ['9'] = 4, 166 ['A'] = 5, ['a'] = 5, ['C'] = 5, ['c'] = 5, ['D'] = 5, ['d'] = 5, 167 ['B'] = 6, ['b'] = 6, 168 ['E'] = 7, ['e'] = 7, 169 ['F'] = 8, ['f'] = 8, 170 ['L'] = 9, 171 ['l'] = 10, 172 ['P'] = 11, ['p'] = 11, 173 ['U'] = 12, ['u'] = 12, 174 ['X'] = 13, ['x'] = 13, 175 ['+'] = 14, ['-'] = 14, 176 ['.'] = 15, 177 }; 178 179 static char 180 inbuf_peek(void) 181 { 182 return *inp.s; 183 } 184 185 void 186 inbuf_skip(void) 187 { 188 inp.s++; 189 if (inp.s >= inp.e) 190 inbuf_read_line(); 191 } 192 193 char 194 inbuf_next(void) 195 { 196 char ch = inbuf_peek(); 197 inbuf_skip(); 198 return ch; 199 } 200 201 static void 202 check_size_token(size_t desired_size) 203 { 204 if (token.e + desired_size >= token.l) 205 buf_expand(&token, desired_size); 206 } 207 208 static int 209 cmp_keyword_by_name(const void *key, const void *elem) 210 { 211 return strcmp(key, ((const struct keyword *)elem)->name); 212 } 213 214 static int 215 cmp_type_by_name(const void *key, const void *elem) 216 { 217 return strcmp(key, *((const char *const *)elem)); 218 } 219 220 #ifdef debug 221 const char * 222 token_type_name(token_type ttype) 223 { 224 static const char *const name[] = { 225 "end_of_file", "newline", "lparen_or_lbracket", "rparen_or_rbracket", 226 "unary_op", "binary_op", "postfix_op", "question", 227 "case_label", "colon", 228 "semicolon", "lbrace", "rbrace", "ident", "comma", 229 "comment", "switch_expr", "preprocessing", "form_feed", "decl", 230 "keyword_for_if_while", "keyword_do_else", 231 "if_expr", "while_expr", "for_exprs", 232 "stmt", "stmt_list", "keyword_else", "keyword_do", "do_stmt", 233 "if_expr_stmt", "if_expr_stmt_else", "period", "string_prefix", 234 "storage_class", "funcname", "type_def", "keyword_struct_union_enum" 235 }; 236 237 assert(0 <= ttype && ttype < nitems(name)); 238 239 return name[ttype]; 240 } 241 242 static void 243 debug_print_buf(const char *name, const struct buffer *buf) 244 { 245 if (buf->s < buf->e) { 246 debug_printf(" %s ", name); 247 debug_vis_range("\"", buf->s, buf->e, "\""); 248 } 249 } 250 251 static token_type 252 lexi_end(token_type ttype) 253 { 254 debug_printf("in line %d, lexi returns '%s'", 255 line_no, token_type_name(ttype)); 256 debug_print_buf("token", &token); 257 debug_print_buf("label", &lab); 258 debug_print_buf("code", &code); 259 debug_print_buf("comment", &com); 260 debug_printf("\n"); 261 262 return ttype; 263 } 264 #else 265 #define lexi_end(tk) (tk) 266 #endif 267 268 static void 269 lex_number(void) 270 { 271 for (uint8_t s = 'A'; s != 'f' && s != 'i' && s != 'u';) { 272 uint8_t ch = (uint8_t)*inp.s; 273 if (ch >= nitems(lex_number_row) || lex_number_row[ch] == 0) 274 break; 275 276 uint8_t row = lex_number_row[ch]; 277 if (lex_number_state[row][s - 'A'] == ' ') { 278 /*- 279 * lex_number_state[0][s - 'A'] now indicates the type: 280 * f = floating, i = integer, u = unknown 281 */ 282 break; 283 } 284 285 s = lex_number_state[row][s - 'A']; 286 check_size_token(1); 287 *token.e++ = inbuf_next(); 288 } 289 } 290 291 static void 292 lex_word(void) 293 { 294 while (isalnum((unsigned char)*inp.s) || 295 *inp.s == '\\' || 296 *inp.s == '_' || *inp.s == '$') { 297 298 if (*inp.s == '\\') { 299 if (inp.s[1] == '\n') { 300 inp.s += 2; 301 if (inp.s >= inp.e) 302 inbuf_read_line(); 303 } else 304 break; 305 } 306 307 check_size_token(1); 308 *token.e++ = inbuf_next(); 309 } 310 } 311 312 static void 313 lex_char_or_string(void) 314 { 315 for (char delim = *token.s;;) { 316 if (*inp.s == '\n') { 317 diag(1, "Unterminated literal"); 318 return; 319 } 320 321 check_size_token(2); 322 *token.e++ = inbuf_next(); 323 if (token.e[-1] == delim) 324 return; 325 326 if (token.e[-1] == '\\') { 327 if (*inp.s == '\n') 328 ++line_no; 329 *token.e++ = inbuf_next(); 330 } 331 } 332 } 333 334 /* 335 * This hack attempts to guess whether the current token is in fact a 336 * declaration keyword -- one that has been defined by typedef. 337 */ 338 static bool 339 probably_typedef(const struct parser_state *state) 340 { 341 if (state->p_l_follow != 0) 342 return false; 343 if (state->block_init || state->in_stmt) 344 return false; 345 if (inp.s[0] == '*' && inp.s[1] != '=') 346 goto maybe; 347 if (isalpha((unsigned char)*inp.s)) 348 goto maybe; 349 return false; 350 maybe: 351 return state->last_token == semicolon || 352 state->last_token == lbrace || 353 state->last_token == rbrace; 354 } 355 356 static bool 357 is_typename(void) 358 { 359 if (opt.auto_typedefs) { 360 const char *u; 361 if ((u = strrchr(token.s, '_')) != NULL && strcmp(u, "_t") == 0) 362 return true; 363 } 364 365 if (typenames.len == 0) 366 return false; 367 return bsearch(token.s, typenames.items, (size_t)typenames.len, 368 sizeof(typenames.items[0]), cmp_type_by_name) != NULL; 369 } 370 371 /* Reads the next token, placing it in the global variable "token". */ 372 token_type 373 lexi(struct parser_state *state) 374 { 375 bool unary_delim; /* whether the current token forces a 376 * following operator to be unary */ 377 token_type ttype; 378 379 token.e = token.s; /* point to start of place to save token */ 380 unary_delim = false; 381 state->col_1 = state->last_nl; /* tell world that this token started 382 * in column 1 iff the last thing 383 * scanned was a newline */ 384 state->last_nl = false; 385 386 while (is_hspace(*inp.s)) { 387 state->col_1 = false; 388 inbuf_skip(); 389 } 390 391 /* Scan an alphanumeric token */ 392 if (isalnum((unsigned char)*inp.s) || 393 *inp.s == '_' || *inp.s == '$' || 394 (inp.s[0] == '.' && isdigit((unsigned char)inp.s[1]))) { 395 struct keyword *kw; 396 397 if (isdigit((unsigned char)*inp.s) || 398 (inp.s[0] == '.' && isdigit((unsigned char)inp.s[1]))) { 399 lex_number(); 400 } else { 401 lex_word(); 402 } 403 *token.e = '\0'; 404 405 if (token.s[0] == 'L' && token.s[1] == '\0' && 406 (*inp.s == '"' || *inp.s == '\'')) 407 return lexi_end(string_prefix); 408 409 while (is_hspace(inbuf_peek())) 410 inbuf_skip(); 411 state->keyword = kw_0; 412 413 if (state->last_token == keyword_struct_union_enum && 414 state->p_l_follow == 0) { 415 state->last_u_d = true; 416 return lexi_end(decl); 417 } 418 /* 419 * Operator after identifier is binary unless last token was 'struct' 420 */ 421 state->last_u_d = (state->last_token == keyword_struct_union_enum); 422 423 kw = bsearch(token.s, keywords, nitems(keywords), 424 sizeof(keywords[0]), cmp_keyword_by_name); 425 if (kw == NULL) { 426 if (is_typename()) { 427 state->keyword = kw_type; 428 state->last_u_d = true; 429 goto found_typename; 430 } 431 432 } else { /* we have a keyword */ 433 state->keyword = kw->kind; 434 state->last_u_d = true; 435 436 switch (kw->kind) { 437 case kw_switch: 438 return lexi_end(switch_expr); 439 440 case kw_case_or_default: 441 return lexi_end(case_label); 442 443 case kw_struct_or_union_or_enum: 444 case kw_type: 445 found_typename: 446 if (state->p_l_follow != 0) { 447 /* inside parens: cast, param list, offsetof or sizeof */ 448 state->cast_mask |= (1 << state->p_l_follow) & ~state->not_cast_mask; 449 } 450 if (state->last_token == period || state->last_token == unary_op) { 451 state->keyword = kw_0; 452 break; 453 } 454 if (kw != NULL && kw->kind == kw_struct_or_union_or_enum) 455 return lexi_end(keyword_struct_union_enum); 456 if (state->p_l_follow != 0) 457 break; 458 return lexi_end(decl); 459 460 case kw_for_or_if_or_while: 461 return lexi_end(keyword_for_if_while); 462 463 case kw_do_or_else: 464 return lexi_end(keyword_do_else); 465 466 case kw_storage_class: 467 return lexi_end(storage_class); 468 469 case kw_typedef: 470 return lexi_end(type_def); 471 472 default: /* all others are treated like any other 473 * identifier */ 474 return lexi_end(ident); 475 } /* end of switch */ 476 } /* end of if (found_it) */ 477 478 if (*inp.s == '(' && state->tos <= 1 && state->ind_level == 0 && 479 !state->in_parameter_declaration && !state->block_init) { 480 481 for (const char *p = inp.s; p < inp.e;) 482 if (*p++ == ')' && (*p == ';' || *p == ',')) 483 goto not_proc; 484 485 strncpy(state->procname, token.s, sizeof state->procname - 1); 486 if (state->in_decl) 487 state->in_parameter_declaration = true; 488 return lexi_end(funcname); 489 not_proc:; 490 491 } else if (probably_typedef(state)) { 492 state->keyword = kw_type; 493 state->last_u_d = true; 494 return lexi_end(decl); 495 } 496 497 if (state->last_token == decl) /* if this is a declared variable, 498 * then following sign is unary */ 499 state->last_u_d = true; /* will make "int a -1" work */ 500 501 return lexi_end(ident); /* the ident is not in the list */ 502 } /* end of processing for alphanum character */ 503 504 /* Scan a non-alphanumeric token */ 505 506 check_size_token(3); /* things like "<<=" */ 507 *token.e++ = inbuf_next(); /* if it is only a one-character token, it is 508 * moved here */ 509 *token.e = '\0'; 510 511 switch (*token.s) { 512 case '\n': 513 unary_delim = state->last_u_d; 514 state->last_nl = true; /* remember that we just had a newline */ 515 /* if data has been exhausted, the newline is a dummy. */ 516 ttype = had_eof ? end_of_file : newline; 517 break; 518 519 case '\'': 520 case '"': 521 lex_char_or_string(); 522 ttype = ident; 523 break; 524 525 case '(': 526 case '[': 527 unary_delim = true; 528 ttype = lparen_or_lbracket; 529 break; 530 531 case ')': 532 case ']': 533 ttype = rparen_or_rbracket; 534 break; 535 536 case '#': 537 unary_delim = state->last_u_d; 538 ttype = preprocessing; 539 break; 540 541 case '?': 542 unary_delim = true; 543 ttype = question; 544 break; 545 546 case ':': 547 ttype = colon; 548 unary_delim = true; 549 break; 550 551 case ';': 552 unary_delim = true; 553 ttype = semicolon; 554 break; 555 556 case '{': 557 unary_delim = true; 558 ttype = lbrace; 559 break; 560 561 case '}': 562 unary_delim = true; 563 ttype = rbrace; 564 break; 565 566 case '\f': 567 unary_delim = state->last_u_d; 568 state->last_nl = true; /* remember this, so we can set 'state->col_1' 569 * right */ 570 ttype = form_feed; 571 break; 572 573 case ',': 574 unary_delim = true; 575 ttype = comma; 576 break; 577 578 case '.': 579 unary_delim = false; 580 ttype = period; 581 break; 582 583 case '-': 584 case '+': /* check for -, +, --, ++ */ 585 ttype = state->last_u_d ? unary_op : binary_op; 586 unary_delim = true; 587 588 if (*inp.s == token.s[0]) { 589 /* check for doubled character */ 590 *token.e++ = *inp.s++; 591 /* buffer overflow will be checked at end of loop */ 592 if (state->last_token == ident || 593 state->last_token == rparen_or_rbracket) { 594 ttype = state->last_u_d ? unary_op : postfix_op; 595 /* check for following ++ or -- */ 596 unary_delim = false; 597 } 598 599 } else if (*inp.s == '=') { 600 /* check for operator += */ 601 *token.e++ = *inp.s++; 602 603 } else if (*inp.s == '>') { 604 /* check for operator -> */ 605 *token.e++ = *inp.s++; 606 unary_delim = false; 607 ttype = unary_op; 608 state->want_blank = false; 609 } 610 break; /* buffer overflow will be checked at end of 611 * switch */ 612 613 case '=': 614 if (state->in_or_st) 615 state->block_init = true; 616 if (*inp.s == '=') { /* == */ 617 *token.e++ = *inp.s++; 618 *token.e = '\0'; 619 } 620 ttype = binary_op; 621 unary_delim = true; 622 break; 623 624 case '>': 625 case '<': 626 case '!': /* ops like <, <<, <=, !=, etc */ 627 if (*inp.s == '>' || *inp.s == '<' || *inp.s == '=') 628 *token.e++ = inbuf_next(); 629 if (*inp.s == '=') 630 *token.e++ = *inp.s++; 631 ttype = state->last_u_d ? unary_op : binary_op; 632 unary_delim = true; 633 break; 634 635 case '*': 636 unary_delim = true; 637 if (!state->last_u_d) { 638 if (*inp.s == '=') 639 *token.e++ = *inp.s++; 640 ttype = binary_op; 641 break; 642 } 643 644 while (*inp.s == '*' || isspace((unsigned char)*inp.s)) { 645 if (*inp.s == '*') { 646 check_size_token(1); 647 *token.e++ = *inp.s; 648 } 649 inbuf_skip(); 650 } 651 652 if (ps.in_decl) { 653 char *tp = inp.s; 654 655 while (isalpha((unsigned char)*tp) || 656 isspace((unsigned char)*tp)) { 657 if (++tp >= inp.e) 658 inbuf_read_line(); 659 } 660 if (*tp == '(') 661 ps.procname[0] = ' '; 662 } 663 664 ttype = unary_op; 665 break; 666 667 default: 668 if (token.s[0] == '/' && (*inp.s == '*' || *inp.s == '/')) { 669 /* it is start of comment */ 670 *token.e++ = inbuf_next(); 671 672 ttype = comment; 673 unary_delim = state->last_u_d; 674 break; 675 } 676 677 while (token.e[-1] == *inp.s || *inp.s == '=') { 678 /* 679 * handle ||, &&, etc, and also things as in int *****i 680 */ 681 check_size_token(1); 682 *token.e++ = inbuf_next(); 683 } 684 685 ttype = state->last_u_d ? unary_op : binary_op; 686 unary_delim = true; 687 } 688 689 if (inp.s >= inp.e) /* check for input buffer empty */ 690 inbuf_read_line(); 691 692 state->last_u_d = unary_delim; 693 694 check_size_token(1); 695 *token.e = '\0'; 696 697 return lexi_end(ttype); 698 } 699 700 static int 701 insert_pos(const char *key, const char **arr, unsigned int len) 702 { 703 int lo = 0; 704 int hi = (int)len - 1; 705 706 while (lo <= hi) { 707 int mid = (int)((unsigned)(lo + hi) >> 1); 708 int cmp = strcmp(arr[mid], key); 709 if (cmp < 0) 710 lo = mid + 1; 711 else if (cmp > 0) 712 hi = mid - 1; 713 else 714 return mid; 715 } 716 return -(lo + 1); 717 } 718 719 void 720 add_typename(const char *name) 721 { 722 if (typenames.len >= typenames.cap) { 723 typenames.cap = 16 + 2 * typenames.cap; 724 typenames.items = xrealloc(typenames.items, 725 sizeof(typenames.items[0]) * typenames.cap); 726 } 727 728 int pos = insert_pos(name, typenames.items, typenames.len); 729 if (pos >= 0) 730 return; /* already in the list */ 731 732 pos = -(pos + 1); 733 memmove(typenames.items + pos + 1, typenames.items + pos, 734 sizeof(typenames.items[0]) * (typenames.len++ - (unsigned)pos)); 735 typenames.items[pos] = xstrdup(name); 736 } 737
Indexes created Fri Sep 26 20:09:58 GMT 2025