lexi.c revision 1.204
1 /* $NetBSD: lexi.c,v 1.204 2023/05/23 06:35:01 rillig Exp $ */ 2 3 /*- 4 * SPDX-License-Identifier: BSD-4-Clause 5 * 6 * Copyright (c) 1985 Sun Microsystems, Inc. 7 * Copyright (c) 1980, 1993 8 * The Regents of the University of California. All rights reserved. 9 * All rights reserved. 10 * 11 * Redistribution and use in source and binary forms, with or without 12 * modification, are permitted provided that the following conditions 13 * are met: 14 * 1. Redistributions of source code must retain the above copyright 15 * notice, this list of conditions and the following disclaimer. 16 * 2. Redistributions in binary form must reproduce the above copyright 17 * notice, this list of conditions and the following disclaimer in the 18 * documentation and/or other materials provided with the distribution. 19 * 3. All advertising materials mentioning features or use of this software 20 * must display the following acknowledgement: 21 * This product includes software developed by the University of 22 * California, Berkeley and its contributors. 23 * 4. Neither the name of the University nor the names of its contributors 24 * may be used to endorse or promote products derived from this software 25 * without specific prior written permission. 26 * 27 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 28 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 29 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 30 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 31 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 32 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 33 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 34 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 35 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 36 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 37 * SUCH DAMAGE. 38 */ 39 40 #include <sys/cdefs.h> 41 __RCSID("$NetBSD: lexi.c,v 1.204 2023/05/23 06:35:01 rillig Exp $"); 42 43 #include <stdlib.h> 44 #include <string.h> 45 46 #include "indent.h" 47 48 /* In lexi_alnum, this constant marks a type, independent of parentheses. */ 49 #define lsym_type lsym_type_outside_parentheses 50 #define lsym_type_modifier lsym_storage_class 51 52 /* must be sorted alphabetically, is used in binary search */ 53 static const struct keyword { 54 const char name[12]; 55 lexer_symbol lsym; 56 } keywords[] = { 57 {"_Bool", lsym_type}, 58 {"_Complex", lsym_type}, 59 {"_Imaginary", lsym_type}, 60 {"auto", lsym_storage_class}, 61 {"bool", lsym_type}, 62 {"break", lsym_word}, 63 {"case", lsym_case_label}, 64 {"char", lsym_type}, 65 {"complex", lsym_type}, 66 {"const", lsym_type_modifier}, 67 {"continue", lsym_word}, 68 {"default", lsym_case_label}, 69 {"do", lsym_do}, 70 {"double", lsym_type}, 71 {"else", lsym_else}, 72 {"enum", lsym_tag}, 73 {"extern", lsym_storage_class}, 74 {"float", lsym_type}, 75 {"for", lsym_for}, 76 {"goto", lsym_word}, 77 {"if", lsym_if}, 78 {"imaginary", lsym_type}, 79 {"inline", lsym_word}, 80 {"int", lsym_type}, 81 {"long", lsym_type}, 82 {"offsetof", lsym_offsetof}, 83 {"register", lsym_storage_class}, 84 {"restrict", lsym_word}, 85 {"return", lsym_return}, 86 {"short", lsym_type}, 87 {"signed", lsym_type}, 88 {"sizeof", lsym_sizeof}, 89 {"static", lsym_storage_class}, 90 {"struct", lsym_tag}, 91 {"switch", lsym_switch}, 92 {"typedef", lsym_typedef}, 93 {"union", lsym_tag}, 94 {"unsigned", lsym_type}, 95 {"void", lsym_type}, 96 {"volatile", lsym_type_modifier}, 97 {"while", lsym_while} 98 }; 99 100 static struct { 101 const char **items; 102 unsigned int len; 103 unsigned int cap; 104 } typenames; 105 106 /*- 107 * The transition table below was rewritten by hand from lx's output, given 108 * the following definitions. lx is Katherine Flavel's lexer generator. 109 * 110 * O = /[0-7]/; D = /[0-9]/; NZ = /[1-9]/; 111 * H = /[a-f0-9]/i; B = /[0-1]/; HP = /0x/i; 112 * BP = /0b/i; E = /e[+\-]?/i D+; P = /p[+\-]?/i D+; 113 * FS = /[fl]/i; IS = /u/i /(l|L|ll|LL)/? | /(l|L|ll|LL)/ /u/i?; 114 * 115 * D+ E FS? -> $float; 116 * D* "." D+ E? FS? -> $float; 117 * D+ "." E? FS? -> $float; HP H+ IS? -> $int; 118 * HP H+ P FS? -> $float; NZ D* IS? -> $int; 119 * HP H* "." H+ P FS? -> $float; "0" O* IS? -> $int; 120 * HP H+ "." P FS -> $float; BP B+ IS? -> $int; 121 */ 122 /* INDENT OFF */ 123 static const unsigned char lex_number_state[][26] = { 124 /* examples: 125 00 126 s 0xx 127 t 00xaa 128 a 11 101100xxa.. 129 r 11ee0001101lbuuxx.a.pp 130 t.01.e+008bLuxll0Ll.aa.p+0 131 states: ABCDEFGHIJKLMNOPQRSTUVWXYZ */ 132 [0] = "uuiifuufiuuiiuiiiiiuiuuuuu", /* (other) */ 133 [1] = "CEIDEHHHIJQ U Q VUVVZZZ", /* 0 */ 134 [2] = "DEIDEHHHIJQ U Q VUVVZZZ", /* 1 */ 135 [3] = "DEIDEHHHIJ U VUVVZZZ", /* 2 3 4 5 6 7 */ 136 [4] = "DEJDEHHHJJ U VUVVZZZ", /* 8 9 */ 137 [5] = " U VUVV ", /* A a C c D d */ 138 [6] = " K U VUVV ", /* B b */ 139 [7] = " FFF FF U VUVV ", /* E e */ 140 [8] = " f f U VUVV f", /* F f */ 141 [9] = " LLf fL PR Li L f", /* L */ 142 [10] = " OOf fO S P O i O f", /* l */ 143 [11] = " FFX ", /* P p */ 144 [12] = " MM M i iiM M ", /* U u */ 145 [13] = " N ", /* X x */ 146 [14] = " G Y ", /* + - */ 147 [15] = "B EE EE T W ", /* . */ 148 /* ABCDEFGHIJKLMNOPQRSTUVWXYZ */ 149 }; 150 /* INDENT ON */ 151 152 static const unsigned char lex_number_row[] = { 153 ['0'] = 1, 154 ['1'] = 2, 155 ['2'] = 3, ['3'] = 3, ['4'] = 3, ['5'] = 3, ['6'] = 3, ['7'] = 3, 156 ['8'] = 4, ['9'] = 4, 157 ['A'] = 5, ['a'] = 5, ['C'] = 5, ['c'] = 5, ['D'] = 5, ['d'] = 5, 158 ['B'] = 6, ['b'] = 6, 159 ['E'] = 7, ['e'] = 7, 160 ['F'] = 8, ['f'] = 8, 161 ['L'] = 9, 162 ['l'] = 10, 163 ['P'] = 11, ['p'] = 11, 164 ['U'] = 12, ['u'] = 12, 165 ['X'] = 13, ['x'] = 13, 166 ['+'] = 14, ['-'] = 14, 167 ['.'] = 15, 168 }; 169 170 static void 171 token_add_char(char ch) 172 { 173 buf_add_char(&token, ch); 174 } 175 176 static void 177 lex_number(void) 178 { 179 for (unsigned char s = 'A'; s != 'f' && s != 'i' && s != 'u';) { 180 unsigned char ch = (unsigned char)inp.st[0]; 181 if (ch == '\\' && inp.st[1] == '\n') { 182 inp.st++; 183 inp_skip(); 184 line_no++; 185 continue; 186 } 187 if (ch >= array_length(lex_number_row) 188 || lex_number_row[ch] == 0) 189 break; 190 191 unsigned char row = lex_number_row[ch]; 192 if (lex_number_state[row][s - 'A'] == ' ') { 193 /*- 194 * lex_number_state[0][s - 'A'] now indicates the type: 195 * f = floating, i = integer, u = unknown 196 */ 197 return; 198 } 199 200 s = lex_number_state[row][s - 'A']; 201 token_add_char(inp_next()); 202 } 203 } 204 205 static bool 206 is_identifier_start(char ch) 207 { 208 return ch_isalpha(ch) || ch == '_' || ch == '$'; 209 } 210 211 static bool 212 is_identifier_part(char ch) 213 { 214 return ch_isalnum(ch) || ch == '_' || ch == '$'; 215 } 216 217 static void 218 lex_word(void) 219 { 220 for (;;) { 221 if (is_identifier_part(inp.st[0])) 222 token_add_char(*inp.st++); 223 else if (inp.st[0] == '\\' && inp.st[1] == '\n') { 224 inp.st++; 225 inp_skip(); 226 line_no++; 227 } else 228 return; 229 } 230 } 231 232 static void 233 lex_char_or_string(void) 234 { 235 for (char delim = token.mem[token.len - 1];;) { 236 if (inp.st[0] == '\n') { 237 diag(1, "Unterminated literal"); 238 return; 239 } 240 241 token_add_char(*inp.st++); 242 if (token.mem[token.len - 1] == delim) 243 return; 244 245 if (token.mem[token.len - 1] == '\\') { 246 if (inp.st[0] == '\n') 247 ++line_no; 248 token_add_char(inp_next()); 249 } 250 } 251 } 252 253 /* Guess whether the current token is a declared type. */ 254 static bool 255 probably_typename(void) 256 { 257 if (ps.prev_token == lsym_storage_class) 258 return true; 259 if (ps.block_init) 260 return false; 261 if (ps.in_stmt_or_decl) /* XXX: this condition looks incorrect */ 262 return false; 263 if (inp.st[0] == '*' && inp.st[1] != '=') 264 goto maybe; 265 /* XXX: is_identifier_start */ 266 if (ch_isalpha(inp.st[0])) 267 goto maybe; 268 return false; 269 maybe: 270 return ps.prev_token == lsym_semicolon || 271 ps.prev_token == lsym_lbrace || 272 ps.prev_token == lsym_rbrace; 273 } 274 275 static int 276 bsearch_typenames(const char *key) 277 { 278 const char **arr = typenames.items; 279 int lo = 0; 280 int hi = (int)typenames.len - 1; 281 282 while (lo <= hi) { 283 int mid = (int)((unsigned)(lo + hi) >> 1); 284 int cmp = strcmp(arr[mid], key); 285 if (cmp < 0) 286 lo = mid + 1; 287 else if (cmp > 0) 288 hi = mid - 1; 289 else 290 return mid; 291 } 292 return -(lo + 1); 293 } 294 295 static bool 296 is_typename(void) 297 { 298 if (opt.auto_typedefs && 299 token.len >= 2 && memcmp(token.mem + token.len - 2, "_t", 2) == 0) 300 return true; 301 302 return bsearch_typenames(token.st) >= 0; 303 } 304 305 static int 306 cmp_keyword_by_name(const void *key, const void *elem) 307 { 308 return strcmp(key, ((const struct keyword *)elem)->name); 309 } 310 311 /* 312 * Looking at something like 'function_name(...)' in a line, guess whether 313 * this starts a function definition or a declaration. 314 */ 315 static bool 316 probably_looking_at_definition(void) 317 { 318 int paren_level = 0; 319 for (const char *p = inp.st; *p != '\n'; p++) { 320 if (*p == '(') 321 paren_level++; 322 if (*p == ')' && --paren_level == 0) { 323 p++; 324 325 while (*p != '\n' 326 && (ch_isspace(*p) || is_identifier_part(*p))) 327 p++; /* '__dead' or '__unused' */ 328 329 if (*p == '\n') /* func(...) */ 330 break; 331 if (*p == ';') /* func(...); */ 332 return false; 333 if (*p == ',') /* double abs(), pi; */ 334 return false; 335 if (*p == '(') /* func(...) __attribute__((...)) */ 336 paren_level++; /* func(...) __printflike(...) 337 */ 338 else 339 break; /* func(...) { ... */ 340 } 341 } 342 343 /* To further reduce the cases where indent wrongly treats an 344 * incomplete function declaration as a function definition, thus 345 * adding a newline before the function name, it may be worth looking 346 * for parameter names, as these are often omitted in function 347 * declarations and only included in function definitions. Or just 348 * increase the lookahead to more than just the current line of input, 349 * until the next '{'. */ 350 return true; 351 } 352 353 /* Read an alphanumeric token into 'token', or return lsym_eof. */ 354 static lexer_symbol 355 lexi_alnum(void) 356 { 357 if (ch_isdigit(inp.st[0]) || 358 (inp.st[0] == '.' && ch_isdigit(inp.st[1]))) { 359 lex_number(); 360 } else if (is_identifier_start(inp.st[0])) { 361 lex_word(); 362 363 if (token.len == 1 && token.st[0] == 'L' && 364 (inp.st[0] == '"' || inp.st[0] == '\'')) { 365 token_add_char(*inp.st++); 366 lex_char_or_string(); 367 ps.next_unary = false; 368 return lsym_word; 369 } 370 } else 371 return lsym_eof; /* just as a placeholder */ 372 373 while (ch_isblank(inp.st[0])) 374 inp.st++; 375 376 ps.next_unary = ps.prev_token == lsym_tag 377 || ps.prev_token == lsym_typedef; 378 379 if (ps.prev_token == lsym_tag && ps.nparen == 0) 380 return lsym_type_outside_parentheses; 381 382 token_add_char('\0'); 383 token.len--; 384 const struct keyword *kw = bsearch(token.st, keywords, 385 array_length(keywords), sizeof(keywords[0]), cmp_keyword_by_name); 386 lexer_symbol lsym = lsym_word; 387 if (kw != NULL) { 388 if (kw->lsym == lsym_type) 389 lsym = lsym_type_in_parentheses; 390 ps.next_unary = true; 391 if (kw->lsym == lsym_tag || kw->lsym == lsym_type) 392 goto found_typename; 393 return kw->lsym; 394 } 395 396 if (is_typename()) { 397 lsym = lsym_type_in_parentheses; 398 ps.next_unary = true; 399 if (ps.in_enum == in_enum_enum) 400 ps.in_enum = in_enum_type; 401 found_typename: 402 if (ps.nparen > 0) { 403 /* inside parentheses: cast, param list, offsetof or 404 * sizeof */ 405 if (ps.paren[ps.nparen - 1].cast == cast_unknown) 406 ps.paren[ps.nparen - 1].cast = cast_maybe; 407 } 408 if (ps.prev_token != lsym_period 409 && ps.prev_token != lsym_unary_op) { 410 if (kw != NULL && kw->lsym == lsym_tag) { 411 if (token.st[0] == 'e' /* enum */) 412 ps.in_enum = in_enum_enum; 413 return lsym_tag; 414 } 415 if (ps.nparen == 0) 416 return lsym_type_outside_parentheses; 417 } 418 } 419 420 if (inp.st[0] == '(' && ps.tos <= 1 && ps.ind_level == 0 && 421 !ps.in_func_def_params && !ps.block_init) { 422 423 if (ps.nparen == 0 && probably_looking_at_definition()) { 424 ps.is_function_definition = true; 425 if (ps.in_decl) 426 ps.in_func_def_params = true; 427 return lsym_funcname; 428 } 429 430 } else if (ps.nparen == 0 && probably_typename()) { 431 ps.next_unary = true; 432 return lsym_type_outside_parentheses; 433 } 434 435 return lsym; 436 } 437 438 static bool 439 is_asterisk_unary(void) 440 { 441 if (ps.decl_ptr == dp_word) 442 return true; 443 if (ps.next_unary || ps.in_func_def_params) 444 return true; 445 if (ps.prev_token == lsym_word || 446 ps.prev_token == lsym_rparen_or_rbracket) 447 return false; 448 return ps.in_decl && ps.nparen > 0; 449 } 450 451 static bool 452 probably_in_function_definition(void) 453 { 454 for (const char *tp = inp.st; *tp != '\n';) { 455 if (ch_isspace(*tp)) 456 tp++; 457 else if (is_identifier_start(*tp)) { 458 tp++; 459 while (is_identifier_part(*tp)) 460 tp++; 461 } else 462 return *tp == '('; 463 } 464 return false; 465 } 466 467 static void 468 lex_asterisk_unary(void) 469 { 470 while (inp.st[0] == '*' || ch_isspace(inp.st[0])) { 471 if (inp.st[0] == '*') 472 token_add_char('*'); 473 inp_skip(); 474 } 475 476 if (ps.in_decl && probably_in_function_definition()) 477 ps.is_function_definition = true; 478 } 479 480 static void 481 skip_blank(const char **pp) 482 { 483 while (ch_isblank(**pp)) 484 (*pp)++; 485 } 486 487 static bool 488 skip_string(const char **pp, const char *s) 489 { 490 size_t len = strlen(s); 491 if (strncmp(*pp, s, len) == 0) { 492 *pp += len; 493 return true; 494 } 495 return false; 496 } 497 498 static void 499 lex_indent_comment(void) 500 { 501 const char *p = inp.mem; 502 503 skip_blank(&p); 504 if (!skip_string(&p, "/*")) 505 return; 506 skip_blank(&p); 507 if (!skip_string(&p, "INDENT")) 508 return; 509 510 enum indent_enabled enabled; 511 skip_blank(&p); 512 if (*p == '*' || skip_string(&p, "ON")) 513 enabled = indent_last_off_line; 514 else if (skip_string(&p, "OFF")) 515 enabled = indent_off; 516 else 517 return; 518 519 skip_blank(&p); 520 if (!skip_string(&p, "*/\n")) 521 return; 522 523 if (lab.len > 0 || code.len > 0 || com.len > 0) 524 output_line(); 525 526 indent_enabled = enabled; 527 } 528 529 /* Reads the next token, placing it in the global variable "token". */ 530 lexer_symbol 531 lexi(void) 532 { 533 token.len = 0; 534 ps.curr_col_1 = ps.next_col_1; 535 ps.next_col_1 = false; 536 537 for (;;) { 538 if (ch_isblank(inp.st[0])) { 539 ps.curr_col_1 = false; 540 inp.st++; 541 } else if (inp.st[0] == '\\' && inp.st[1] == '\n') { 542 inp.st++; 543 inp_skip(); 544 line_no++; 545 } else 546 break; 547 } 548 549 lexer_symbol alnum_lsym = lexi_alnum(); 550 if (alnum_lsym != lsym_eof) { 551 debug_parser_state(alnum_lsym); 552 return alnum_lsym; 553 } 554 555 /* Scan a non-alphanumeric token */ 556 557 token_add_char(inp_next()); 558 559 lexer_symbol lsym; 560 bool next_unary; 561 562 switch (token.mem[token.len - 1]) { 563 564 /* INDENT OFF */ 565 case '(': 566 case '[': lsym = lsym_lparen_or_lbracket; next_unary = true; break; 567 case ')': 568 case ']': lsym = lsym_rparen_or_rbracket; next_unary = false; break; 569 case '?': lsym = lsym_question; next_unary = true; break; 570 case ':': lsym = lsym_colon; next_unary = true; break; 571 case ';': lsym = lsym_semicolon; next_unary = true; break; 572 case '{': lsym = lsym_lbrace; next_unary = true; break; 573 case '}': lsym = lsym_rbrace; next_unary = true; break; 574 case ',': lsym = lsym_comma; next_unary = true; break; 575 case '.': lsym = lsym_period; next_unary = false; break; 576 /* INDENT ON */ 577 578 case '\n': 579 /* if data has been exhausted, the '\n' is a dummy. */ 580 lsym = had_eof ? lsym_eof : lsym_newline; 581 next_unary = ps.next_unary; 582 ps.next_col_1 = true; 583 break; 584 585 case '#': 586 lsym = lsym_preprocessing; 587 next_unary = ps.next_unary; 588 break; 589 590 case '\'': 591 case '"': 592 lex_char_or_string(); 593 lsym = lsym_word; 594 next_unary = false; 595 break; 596 597 case '-': 598 case '+': 599 lsym = ps.next_unary ? lsym_unary_op : lsym_binary_op; 600 next_unary = true; 601 602 /* '++' or '--' */ 603 if (inp.st[0] == token.mem[token.len - 1]) { 604 token_add_char(*inp.st++); 605 if (ps.prev_token == lsym_word || 606 ps.prev_token == lsym_rparen_or_rbracket) { 607 lsym = ps.next_unary 608 ? lsym_unary_op : lsym_postfix_op; 609 next_unary = false; 610 } 611 612 } else if (inp.st[0] == '=') { /* '+=' or '-=' */ 613 token_add_char(*inp.st++); 614 615 } else if (inp.st[0] == '>') { /* '->' */ 616 token_add_char(*inp.st++); 617 lsym = lsym_unary_op; 618 next_unary = false; 619 ps.want_blank = false; 620 } 621 break; 622 623 case '=': 624 if (ps.init_or_struct) 625 ps.block_init = true; 626 if (inp.st[0] == '=') 627 token_add_char(*inp.st++); 628 lsym = lsym_binary_op; 629 next_unary = true; 630 break; 631 632 case '>': 633 case '<': 634 case '!': /* ops like <, <<, <=, !=, etc */ 635 if (inp.st[0] == '>' || inp.st[0] == '<' || inp.st[0] == '=') 636 token_add_char(*inp.st++); 637 if (inp.st[0] == '=') 638 token_add_char(*inp.st++); 639 lsym = ps.next_unary ? lsym_unary_op : lsym_binary_op; 640 next_unary = true; 641 break; 642 643 case '*': 644 if (is_asterisk_unary()) { 645 lex_asterisk_unary(); 646 lsym = lsym_unary_op; 647 next_unary = true; 648 } else { 649 if (inp.st[0] == '=') 650 token_add_char(*inp.st++); 651 lsym = lsym_binary_op; 652 next_unary = true; 653 } 654 break; 655 656 default: 657 if (token.mem[token.len - 1] == '/' 658 && (inp.st[0] == '*' || inp.st[0] == '/')) { 659 enum indent_enabled prev = indent_enabled; 660 lex_indent_comment(); 661 if (prev == indent_on && indent_enabled == indent_off) 662 out.indent_off_text.len = 0; 663 token_add_char(*inp.st++); 664 lsym = lsym_comment; 665 next_unary = ps.next_unary; 666 break; 667 } 668 669 /* things like '||', '&&', '<<=', 'int *****i' */ 670 while (inp.st[0] == token.mem[token.len - 1] 671 || inp.st[0] == '=') 672 token_add_char(*inp.st++); 673 674 lsym = ps.next_unary ? lsym_unary_op : lsym_binary_op; 675 next_unary = true; 676 } 677 678 if (ps.in_enum == in_enum_enum || ps.in_enum == in_enum_type) 679 ps.in_enum = lsym == lsym_lbrace ? in_enum_brace : in_enum_no; 680 if (lsym == lsym_rbrace) 681 ps.in_enum = in_enum_no; 682 683 ps.next_unary = next_unary; 684 685 debug_parser_state(lsym); 686 return lsym; 687 } 688 689 void 690 register_typename(const char *name) 691 { 692 if (typenames.len >= typenames.cap) { 693 typenames.cap = 16 + 2 * typenames.cap; 694 typenames.items = nonnull(realloc(typenames.items, 695 sizeof(typenames.items[0]) * typenames.cap)); 696 } 697 698 int pos = bsearch_typenames(name); 699 if (pos >= 0) 700 return; /* already in the list */ 701 702 pos = -(pos + 1); 703 memmove(typenames.items + pos + 1, typenames.items + pos, 704 sizeof(typenames.items[0]) * (typenames.len++ - (unsigned)pos)); 705 typenames.items[pos] = nonnull(strdup(name)); 706 } 707
Indexes created Fri Sep 26 08:10:20 GMT 2025