lexi.c revision 1.226
1 /* $NetBSD: lexi.c,v 1.226 2023/06/14 08:25:15 rillig Exp $ */ 2 3 /*- 4 * SPDX-License-Identifier: BSD-4-Clause 5 * 6 * Copyright (c) 1985 Sun Microsystems, Inc. 7 * Copyright (c) 1980, 1993 8 * The Regents of the University of California. All rights reserved. 9 * All rights reserved. 10 * 11 * Redistribution and use in source and binary forms, with or without 12 * modification, are permitted provided that the following conditions 13 * are met: 14 * 1. Redistributions of source code must retain the above copyright 15 * notice, this list of conditions and the following disclaimer. 16 * 2. Redistributions in binary form must reproduce the above copyright 17 * notice, this list of conditions and the following disclaimer in the 18 * documentation and/or other materials provided with the distribution. 19 * 3. All advertising materials mentioning features or use of this software 20 * must display the following acknowledgement: 21 * This product includes software developed by the University of 22 * California, Berkeley and its contributors. 23 * 4. Neither the name of the University nor the names of its contributors 24 * may be used to endorse or promote products derived from this software 25 * without specific prior written permission. 26 * 27 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 28 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 29 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 30 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 31 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 32 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 33 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 34 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 35 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 36 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 37 * SUCH DAMAGE. 38 */ 39 40 #include <sys/cdefs.h> 41 __RCSID("$NetBSD: lexi.c,v 1.226 2023/06/14 08:25:15 rillig Exp $"); 42 43 #include <stdlib.h> 44 #include <string.h> 45 46 #include "indent.h" 47 48 /* In lexi_alnum, this constant marks a type, independent of parentheses. */ 49 #define lsym_type lsym_type_outside_parentheses 50 51 /* must be sorted alphabetically, is used in binary search */ 52 static const struct keyword { 53 const char name[12]; 54 lexer_symbol lsym; 55 } keywords[] = { 56 {"_Bool", lsym_type}, 57 {"_Complex", lsym_type}, 58 {"_Imaginary", lsym_type}, 59 {"auto", lsym_modifier}, 60 {"bool", lsym_type}, 61 {"break", lsym_word}, 62 {"case", lsym_case}, 63 {"char", lsym_type}, 64 {"complex", lsym_type}, 65 {"const", lsym_modifier}, 66 {"continue", lsym_word}, 67 {"default", lsym_default}, 68 {"do", lsym_do}, 69 {"double", lsym_type}, 70 {"else", lsym_else}, 71 {"enum", lsym_tag}, 72 {"extern", lsym_modifier}, 73 {"float", lsym_type}, 74 {"for", lsym_for}, 75 {"goto", lsym_word}, 76 {"if", lsym_if}, 77 {"imaginary", lsym_type}, 78 {"inline", lsym_modifier}, 79 {"int", lsym_type}, 80 {"long", lsym_type}, 81 {"offsetof", lsym_offsetof}, 82 {"register", lsym_modifier}, 83 {"restrict", lsym_word}, 84 {"return", lsym_return}, 85 {"short", lsym_type}, 86 {"signed", lsym_type}, 87 {"sizeof", lsym_sizeof}, 88 {"static", lsym_modifier}, 89 {"struct", lsym_tag}, 90 {"switch", lsym_switch}, 91 {"typedef", lsym_typedef}, 92 {"union", lsym_tag}, 93 {"unsigned", lsym_type}, 94 {"void", lsym_type}, 95 {"volatile", lsym_modifier}, 96 {"while", lsym_while} 97 }; 98 99 static struct { 100 const char **items; 101 unsigned int len; 102 unsigned int cap; 103 } typenames; 104 105 /*- 106 * The transition table below was rewritten by hand from lx's output, given 107 * the following definitions. lx is Katherine Flavel's lexer generator. 108 * 109 * O = /[0-7]/; D = /[0-9]/; NZ = /[1-9]/; 110 * H = /[a-f0-9]/i; B = /[0-1]/; HP = /0x/i; 111 * BP = /0b/i; E = /e[+\-]?/i D+; P = /p[+\-]?/i D+; 112 * FS = /[fl]/i; IS = /u/i /(l|L|ll|LL)/? | /(l|L|ll|LL)/ /u/i?; 113 * 114 * D+ E FS? -> $float; 115 * D* "." D+ E? FS? -> $float; 116 * D+ "." E? FS? -> $float; HP H+ IS? -> $int; 117 * HP H+ P FS? -> $float; NZ D* IS? -> $int; 118 * HP H* "." H+ P FS? -> $float; "0" O* IS? -> $int; 119 * HP H+ "." P FS -> $float; BP B+ IS? -> $int; 120 */ 121 /* INDENT OFF */ 122 static const unsigned char lex_number_state[][26] = { 123 /* examples: 124 00 125 s 0xx 126 t 00xaa 127 a 11 101100xxa.. 128 r 11ee0001101lbuuxx.a.pp 129 t.01.e+008bLuxll0Ll.aa.p+0 130 states: ABCDEFGHIJKLMNOPQRSTUVWXYZ */ 131 [0] = "uuiifuufiuuiiuiiiiiuiuuuuu", /* (other) */ 132 [1] = "CEIDEHHHIJQ U Q VUVVZZZ", /* 0 */ 133 [2] = "DEIDEHHHIJQ U Q VUVVZZZ", /* 1 */ 134 [3] = "DEIDEHHHIJ U VUVVZZZ", /* 2 3 4 5 6 7 */ 135 [4] = "DEJDEHHHJJ U VUVVZZZ", /* 8 9 */ 136 [5] = " U VUVV ", /* A a C c D d */ 137 [6] = " K U VUVV ", /* B b */ 138 [7] = " FFF FF U VUVV ", /* E e */ 139 [8] = " f f U VUVV f", /* F f */ 140 [9] = " LLf fL PR Li L f", /* L */ 141 [10] = " OOf fO S P O i O f", /* l */ 142 [11] = " FFX ", /* P p */ 143 [12] = " MM M i iiM M ", /* U u */ 144 [13] = " N ", /* X x */ 145 [14] = " G Y ", /* + - */ 146 [15] = "B EE EE T W ", /* . */ 147 /* ABCDEFGHIJKLMNOPQRSTUVWXYZ */ 148 }; 149 /* INDENT ON */ 150 151 static const unsigned char lex_number_row[] = { 152 ['0'] = 1, 153 ['1'] = 2, 154 ['2'] = 3, ['3'] = 3, ['4'] = 3, ['5'] = 3, ['6'] = 3, ['7'] = 3, 155 ['8'] = 4, ['9'] = 4, 156 ['A'] = 5, ['a'] = 5, ['C'] = 5, ['c'] = 5, ['D'] = 5, ['d'] = 5, 157 ['B'] = 6, ['b'] = 6, 158 ['E'] = 7, ['e'] = 7, 159 ['F'] = 8, ['f'] = 8, 160 ['L'] = 9, 161 ['l'] = 10, 162 ['P'] = 11, ['p'] = 11, 163 ['U'] = 12, ['u'] = 12, 164 ['X'] = 13, ['x'] = 13, 165 ['+'] = 14, ['-'] = 14, 166 ['.'] = 15, 167 }; 168 169 170 static bool 171 is_identifier_start(char ch) 172 { 173 return ch_isalpha(ch) || ch == '_' || ch == '$'; 174 } 175 176 static bool 177 is_identifier_part(char ch) 178 { 179 return ch_isalnum(ch) || ch == '_' || ch == '$'; 180 } 181 182 static void 183 token_add_char(char ch) 184 { 185 buf_add_char(&token, ch); 186 } 187 188 static void 189 lex_number(void) 190 { 191 for (unsigned char s = 'A'; s != 'f' && s != 'i' && s != 'u';) { 192 unsigned char ch = (unsigned char)inp_p[0]; 193 if (ch == '\\' && inp_p[1] == '\n') { 194 inp_p++; 195 inp_skip(); 196 line_no++; 197 continue; 198 } 199 if (ch >= array_length(lex_number_row) 200 || lex_number_row[ch] == 0) 201 break; 202 203 unsigned char row = lex_number_row[ch]; 204 if (lex_number_state[row][s - 'A'] == ' ') { 205 /*- 206 * lex_number_state[0][s - 'A'] now indicates the type: 207 * f = floating, i = integer, u = unknown 208 */ 209 return; 210 } 211 212 s = lex_number_state[row][s - 'A']; 213 token_add_char(inp_next()); 214 } 215 } 216 217 static void 218 lex_word(void) 219 { 220 for (;;) { 221 if (is_identifier_part(inp_p[0])) 222 token_add_char(*inp_p++); 223 else if (inp_p[0] == '\\' && inp_p[1] == '\n') { 224 inp_p++; 225 inp_skip(); 226 line_no++; 227 } else 228 return; 229 } 230 } 231 232 static void 233 lex_char_or_string(void) 234 { 235 for (char delim = token.s[token.len - 1];;) { 236 if (inp_p[0] == '\n') { 237 diag(1, "Unterminated literal"); 238 return; 239 } 240 241 token_add_char(*inp_p++); 242 if (token.s[token.len - 1] == delim) 243 return; 244 245 if (token.s[token.len - 1] == '\\') { 246 if (inp_p[0] == '\n') 247 ++line_no; 248 token_add_char(inp_next()); 249 } 250 } 251 } 252 253 /* Guess whether the current token is a declared type. */ 254 static bool 255 probably_typename(void) 256 { 257 if (ps.prev_lsym == lsym_modifier) 258 return true; 259 if (ps.in_init) 260 return false; 261 if (ps.in_stmt_or_decl) /* XXX: this condition looks incorrect */ 262 return false; 263 if (ps.prev_lsym == lsym_semicolon 264 || ps.prev_lsym == lsym_lbrace 265 || ps.prev_lsym == lsym_rbrace) { 266 if (inp_p[0] == '*' && inp_p[1] != '=') 267 return true; 268 /* XXX: is_identifier_start */ 269 if (ch_isalpha(inp_p[0])) 270 return true; 271 } 272 return false; 273 } 274 275 static int 276 bsearch_typenames(const char *key) 277 { 278 const char **arr = typenames.items; 279 unsigned lo = 0; 280 unsigned hi = typenames.len; 281 282 while (lo < hi) { 283 unsigned mid = (lo + hi) / 2; 284 int cmp = strcmp(arr[mid], key); 285 if (cmp < 0) 286 lo = mid + 1; 287 else if (cmp > 0) 288 hi = mid; 289 else 290 return (int)mid; 291 } 292 return -1 - (int)lo; 293 } 294 295 static bool 296 is_typename(void) 297 { 298 if (opt.auto_typedefs && 299 token.len >= 2 && memcmp(token.s + token.len - 2, "_t", 2) == 0) 300 return true; 301 302 return bsearch_typenames(token.s) >= 0; 303 } 304 305 void 306 register_typename(const char *name) 307 { 308 if (typenames.len >= typenames.cap) { 309 typenames.cap = 16 + 2 * typenames.cap; 310 typenames.items = nonnull(realloc(typenames.items, 311 sizeof(typenames.items[0]) * typenames.cap)); 312 } 313 314 int pos = bsearch_typenames(name); 315 if (pos >= 0) 316 return; /* already in the list */ 317 318 pos = -1 - pos; 319 memmove(typenames.items + pos + 1, typenames.items + pos, 320 sizeof(typenames.items[0]) * (typenames.len++ - (unsigned)pos)); 321 typenames.items[pos] = nonnull(strdup(name)); 322 } 323 324 static int 325 cmp_keyword_by_name(const void *key, const void *elem) 326 { 327 return strcmp(key, ((const struct keyword *)elem)->name); 328 } 329 330 /* 331 * Looking at something like 'function_name(...)' in a line, guess whether 332 * this starts a function definition or a declaration. 333 */ 334 static bool 335 probably_function_definition(void) 336 { 337 int paren_level = 0; 338 for (const char *p = inp_p; *p != '\n'; p++) { 339 if (*p == '(') 340 paren_level++; 341 if (*p == ')' && --paren_level == 0) { 342 p++; 343 344 while (*p != '\n' 345 && (ch_isspace(*p) || is_identifier_part(*p))) 346 p++; /* '__dead' or '__unused' */ 347 348 if (*p == '\n') /* func(...) */ 349 break; 350 if (*p == ';') /* func(...); */ 351 return false; 352 if (*p == ',') /* double abs(), pi; */ 353 return false; 354 if (*p == '(') /* func(...) __attribute__((...)) */ 355 paren_level++; /* func(...) __printflike(...) 356 */ 357 else 358 break; /* func(...) { ... */ 359 } 360 361 if (paren_level == 1 && p[0] == '*' && p[1] == ',') 362 return false; 363 } 364 365 /* To further reduce the cases where indent wrongly treats an 366 * incomplete function declaration as a function definition, thus 367 * adding a newline before the function name, it may be worth looking 368 * for parameter names, as these are often omitted in function 369 * declarations and only included in function definitions. Or just 370 * increase the lookahead to more than just the current line of input, 371 * until the next '{'. */ 372 return true; 373 } 374 375 static lexer_symbol 376 lexi_alnum(void) 377 { 378 if (ch_isdigit(inp_p[0]) || 379 (inp_p[0] == '.' && ch_isdigit(inp_p[1]))) { 380 lex_number(); 381 } else if (is_identifier_start(inp_p[0])) { 382 lex_word(); 383 384 if (token.len == 1 && token.s[0] == 'L' && 385 (inp_p[0] == '"' || inp_p[0] == '\'')) { 386 token_add_char(*inp_p++); 387 lex_char_or_string(); 388 ps.next_unary = false; 389 return lsym_word; 390 } 391 } else 392 return lsym_eof; /* just as a placeholder */ 393 394 while (ch_isblank(inp_p[0])) 395 inp_p++; 396 397 ps.next_unary = ps.prev_lsym == lsym_tag 398 || ps.prev_lsym == lsym_typedef; 399 400 if (ps.prev_lsym == lsym_tag && ps.nparen == 0) 401 return lsym_type_outside_parentheses; 402 403 token_add_char('\0'); 404 token.len--; 405 const struct keyword *kw = bsearch(token.s, keywords, 406 array_length(keywords), sizeof(keywords[0]), cmp_keyword_by_name); 407 lexer_symbol lsym = lsym_word; 408 if (kw != NULL) { 409 if (kw->lsym == lsym_type) 410 lsym = lsym_type_in_parentheses; 411 ps.next_unary = true; 412 if (kw->lsym == lsym_tag || kw->lsym == lsym_type) 413 goto found_typename; 414 return kw->lsym; 415 } 416 417 if (is_typename()) { 418 lsym = lsym_type_in_parentheses; 419 ps.next_unary = true; 420 found_typename: 421 if (ps.nparen > 0) { 422 /* inside parentheses: cast, param list, offsetof or 423 * sizeof */ 424 if (ps.paren[ps.nparen - 1].cast == cast_unknown) 425 ps.paren[ps.nparen - 1].cast = cast_maybe; 426 } 427 if (ps.prev_lsym != lsym_period 428 && ps.prev_lsym != lsym_unary_op) { 429 if (kw != NULL && kw->lsym == lsym_tag) 430 return lsym_tag; 431 if (ps.nparen == 0) 432 return lsym_type_outside_parentheses; 433 } 434 } 435 436 if (inp_p[0] == '(' && ps.psyms.top <= 1 && ps.ind_level == 0 && 437 !ps.in_func_def_params && !ps.in_init) { 438 439 if (ps.nparen == 0 && probably_function_definition()) { 440 ps.line_has_func_def = true; 441 if (ps.in_decl) 442 ps.in_func_def_params = true; 443 return lsym_funcname; 444 } 445 446 } else if (ps.nparen == 0 && probably_typename()) { 447 ps.next_unary = true; 448 return lsym_type_outside_parentheses; 449 } 450 451 return lsym; 452 } 453 454 static bool 455 is_asterisk_pointer(void) 456 { 457 if (inp_p[strspn(inp_p, "* \t")] == ')') 458 return true; 459 if (ps.next_unary || ps.in_func_def_params) 460 return true; 461 if (ps.prev_lsym == lsym_word || 462 ps.prev_lsym == lsym_rparen || 463 ps.prev_lsym == lsym_rbracket) 464 return false; 465 return ps.in_decl && ps.nparen > 0; 466 } 467 468 static bool 469 probably_in_function_definition(void) 470 { 471 for (const char *tp = inp_p; *tp != '\n';) { 472 if (ch_isspace(*tp)) 473 tp++; 474 else if (is_identifier_start(*tp)) { 475 tp++; 476 while (is_identifier_part(*tp)) 477 tp++; 478 } else 479 return *tp == '('; 480 } 481 return false; 482 } 483 484 static void 485 lex_asterisk_pointer(void) 486 { 487 while (inp_p[0] == '*' || ch_isspace(inp_p[0])) { 488 if (inp_p[0] == '*') 489 token_add_char('*'); 490 inp_skip(); 491 } 492 493 if (ps.in_decl && probably_in_function_definition()) 494 ps.line_has_func_def = true; 495 } 496 497 static bool 498 skip(const char **pp, const char *s) 499 { 500 size_t len = strlen(s); 501 while (ch_isblank(**pp)) 502 (*pp)++; 503 if (strncmp(*pp, s, len) == 0) { 504 *pp += len; 505 return true; 506 } 507 return false; 508 } 509 510 static void 511 lex_indent_comment(void) 512 { 513 const char *p = inp.s; 514 if (skip(&p, "/*") && skip(&p, "INDENT")) { 515 enum indent_enabled enabled; 516 if (skip(&p, "ON") || *p == '*') 517 enabled = indent_last_off_line; 518 else if (skip(&p, "OFF")) 519 enabled = indent_off; 520 else 521 return; 522 if (skip(&p, "*/\n")) { 523 if (lab.len > 0 || code.len > 0 || com.len > 0) 524 output_line(); 525 indent_enabled = enabled; 526 } 527 } 528 } 529 530 /* Reads the next token, placing it in the global variable "token". */ 531 lexer_symbol 532 lexi(void) 533 { 534 buf_clear(&token); 535 ps.next_col_1 = false; 536 537 for (;;) { 538 if (ch_isblank(inp_p[0])) 539 inp_p++; 540 else if (inp_p[0] == '\\' && inp_p[1] == '\n') { 541 inp_p++; 542 inp_skip(); 543 line_no++; 544 } else 545 break; 546 } 547 548 lexer_symbol alnum_lsym = lexi_alnum(); 549 if (alnum_lsym != lsym_eof) 550 return alnum_lsym; 551 552 /* Scan a non-alphanumeric token */ 553 554 token_add_char(inp_next()); 555 556 lexer_symbol lsym; 557 bool next_unary; 558 559 switch (token.s[token.len - 1]) { 560 561 case '#': 562 lsym = lsym_preprocessing; 563 next_unary = ps.next_unary; 564 break; 565 566 case '\n': 567 /* if data has been exhausted, the '\n' is a dummy. */ 568 lsym = had_eof ? lsym_eof : lsym_newline; 569 next_unary = ps.next_unary; 570 ps.next_col_1 = true; 571 break; 572 573 /* INDENT OFF */ 574 case '(': lsym = lsym_lparen; next_unary = true; break; 575 case ')': lsym = lsym_rparen; next_unary = false; break; 576 case '[': lsym = lsym_lbracket; next_unary = true; break; 577 case ']': lsym = lsym_rbracket; next_unary = false; break; 578 case '{': lsym = lsym_lbrace; next_unary = true; break; 579 case '}': lsym = lsym_rbrace; next_unary = true; break; 580 case '.': lsym = lsym_period; next_unary = false; break; 581 case '?': lsym = lsym_question; next_unary = true; break; 582 case ',': lsym = lsym_comma; next_unary = true; break; 583 case ';': lsym = lsym_semicolon; next_unary = true; break; 584 /* INDENT ON */ 585 586 case '-': 587 case '+': 588 lsym = ps.next_unary ? lsym_unary_op : lsym_binary_op; 589 next_unary = true; 590 591 /* '++' or '--' */ 592 if (inp_p[0] == token.s[token.len - 1]) { 593 token_add_char(*inp_p++); 594 if (ps.prev_lsym == lsym_word || 595 ps.prev_lsym == lsym_rparen || 596 ps.prev_lsym == lsym_rbracket) { 597 lsym = ps.next_unary 598 ? lsym_unary_op : lsym_postfix_op; 599 next_unary = false; 600 } 601 602 } else if (inp_p[0] == '=') { /* '+=' or '-=' */ 603 token_add_char(*inp_p++); 604 605 } else if (inp_p[0] == '>') { /* '->' */ 606 token_add_char(*inp_p++); 607 lsym = lsym_unary_op; 608 next_unary = false; 609 ps.want_blank = false; 610 } 611 break; 612 613 case ':': 614 lsym = ps.quest_level > 0 615 ? (ps.quest_level--, lsym_question_colon) 616 : ps.in_var_decl ? lsym_other_colon : lsym_label_colon; 617 next_unary = true; 618 break; 619 620 case '*': 621 if (inp_p[0] == '=') { 622 token_add_char(*inp_p++); 623 lsym = lsym_binary_op; 624 } else if (is_asterisk_pointer()) { 625 lex_asterisk_pointer(); 626 lsym = lsym_unary_op; 627 } else 628 lsym = lsym_binary_op; 629 next_unary = true; 630 break; 631 632 case '=': 633 if (ps.in_var_decl) 634 ps.in_init = true; 635 if (inp_p[0] == '=') 636 token_add_char(*inp_p++); 637 lsym = lsym_binary_op; 638 next_unary = true; 639 break; 640 641 case '>': 642 case '<': 643 case '!': /* ops like <, <<, <=, !=, etc. */ 644 if (inp_p[0] == '>' || inp_p[0] == '<' || inp_p[0] == '=') 645 token_add_char(*inp_p++); 646 if (inp_p[0] == '=') 647 token_add_char(*inp_p++); 648 lsym = ps.next_unary ? lsym_unary_op : lsym_binary_op; 649 next_unary = true; 650 break; 651 652 case '\'': 653 case '"': 654 lex_char_or_string(); 655 lsym = lsym_word; 656 next_unary = false; 657 break; 658 659 default: 660 if (token.s[token.len - 1] == '/' 661 && (inp_p[0] == '*' || inp_p[0] == '/')) { 662 enum indent_enabled prev = indent_enabled; 663 lex_indent_comment(); 664 if (prev == indent_on && indent_enabled == indent_off) 665 buf_clear(&out.indent_off_text); 666 token_add_char(*inp_p++); 667 lsym = lsym_comment; 668 next_unary = ps.next_unary; 669 break; 670 } 671 672 /* punctuation like '%', '&&', '/', '^', '||', '~' */ 673 lsym = ps.next_unary ? lsym_unary_op : lsym_binary_op; 674 if (inp_p[0] == token.s[token.len - 1]) 675 token_add_char(*inp_p++), lsym = lsym_binary_op; 676 if (inp_p[0] == '=') 677 token_add_char(*inp_p++), lsym = lsym_binary_op; 678 679 next_unary = true; 680 } 681 682 ps.next_unary = next_unary; 683 684 return lsym; 685 } 686
Indexes created Wed Sep 24 05:09:52 GMT 2025