lexi.c revision 1.187
1 /* $NetBSD: lexi.c,v 1.187 2023/05/15 08:02:01 rillig Exp $ */ 2 3 /*- 4 * SPDX-License-Identifier: BSD-4-Clause 5 * 6 * Copyright (c) 1985 Sun Microsystems, Inc. 7 * Copyright (c) 1980, 1993 8 * The Regents of the University of California. All rights reserved. 9 * All rights reserved. 10 * 11 * Redistribution and use in source and binary forms, with or without 12 * modification, are permitted provided that the following conditions 13 * are met: 14 * 1. Redistributions of source code must retain the above copyright 15 * notice, this list of conditions and the following disclaimer. 16 * 2. Redistributions in binary form must reproduce the above copyright 17 * notice, this list of conditions and the following disclaimer in the 18 * documentation and/or other materials provided with the distribution. 19 * 3. All advertising materials mentioning features or use of this software 20 * must display the following acknowledgement: 21 * This product includes software developed by the University of 22 * California, Berkeley and its contributors. 23 * 4. Neither the name of the University nor the names of its contributors 24 * may be used to endorse or promote products derived from this software 25 * without specific prior written permission. 26 * 27 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 28 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 29 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 30 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 31 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 32 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 33 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 34 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 35 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 36 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 37 * SUCH DAMAGE. 38 */ 39 40 #include <sys/cdefs.h> 41 __RCSID("$NetBSD: lexi.c,v 1.187 2023/05/15 08:02:01 rillig Exp $"); 42 43 #include <stdlib.h> 44 #include <string.h> 45 46 #include "indent.h" 47 48 /* In lexi_alnum, this constant marks a type, independent of parentheses. */ 49 #define lsym_type lsym_type_outside_parentheses 50 51 /* must be sorted alphabetically, is used in binary search */ 52 static const struct keyword { 53 const char name[12]; 54 lexer_symbol lsym; 55 } keywords[] = { 56 {"_Bool", lsym_type}, 57 {"_Complex", lsym_type}, 58 {"_Imaginary", lsym_type}, 59 {"auto", lsym_storage_class}, 60 {"bool", lsym_type}, 61 {"break", lsym_word}, 62 {"case", lsym_case_label}, 63 {"char", lsym_type}, 64 {"complex", lsym_type}, 65 {"const", lsym_type}, 66 {"continue", lsym_word}, 67 {"default", lsym_case_label}, 68 {"do", lsym_do}, 69 {"double", lsym_type}, 70 {"else", lsym_else}, 71 {"enum", lsym_tag}, 72 {"extern", lsym_storage_class}, 73 {"float", lsym_type}, 74 {"for", lsym_for}, 75 {"goto", lsym_word}, 76 {"if", lsym_if}, 77 {"imaginary", lsym_type}, 78 {"inline", lsym_word}, 79 {"int", lsym_type}, 80 {"long", lsym_type}, 81 {"offsetof", lsym_offsetof}, 82 {"register", lsym_storage_class}, 83 {"restrict", lsym_word}, 84 {"return", lsym_return}, 85 {"short", lsym_type}, 86 {"signed", lsym_type}, 87 {"sizeof", lsym_sizeof}, 88 {"static", lsym_storage_class}, 89 {"struct", lsym_tag}, 90 {"switch", lsym_switch}, 91 {"typedef", lsym_typedef}, 92 {"union", lsym_tag}, 93 {"unsigned", lsym_type}, 94 {"void", lsym_type}, 95 {"volatile", lsym_type}, 96 {"while", lsym_while} 97 }; 98 99 static struct { 100 const char **items; 101 unsigned int len; 102 unsigned int cap; 103 } typenames; 104 105 /* 106 * The transition table below was rewritten by hand from lx's output, given 107 * the following definitions. lx is Katherine Flavel's lexer generator. 108 * 109 * O = /[0-7]/; D = /[0-9]/; NZ = /[1-9]/; 110 * H = /[a-f0-9]/i; B = /[0-1]/; HP = /0x/i; 111 * BP = /0b/i; E = /e[+\-]?/i D+; P = /p[+\-]?/i D+; 112 * FS = /[fl]/i; IS = /u/i /(l|L|ll|LL)/? | /(l|L|ll|LL)/ /u/i?; 113 * 114 * D+ E FS? -> $float; 115 * D* "." D+ E? FS? -> $float; 116 * D+ "." E? FS? -> $float; HP H+ IS? -> $int; 117 * HP H+ P FS? -> $float; NZ D* IS? -> $int; 118 * HP H* "." H+ P FS? -> $float; "0" O* IS? -> $int; 119 * HP H+ "." P FS -> $float; BP B+ IS? -> $int; 120 */ 121 /* INDENT OFF */ 122 static const unsigned char lex_number_state[][26] = { 123 /* examples: 124 00 125 s 0xx 126 t 00xaa 127 a 11 101100xxa.. 128 r 11ee0001101lbuuxx.a.pp 129 t.01.e+008bLuxll0Ll.aa.p+0 130 states: ABCDEFGHIJKLMNOPQRSTUVWXYZ */ 131 [0] = "uuiifuufiuuiiuiiiiiuiuuuuu", /* (other) */ 132 [1] = "CEIDEHHHIJQ U Q VUVVZZZ", /* 0 */ 133 [2] = "DEIDEHHHIJQ U Q VUVVZZZ", /* 1 */ 134 [3] = "DEIDEHHHIJ U VUVVZZZ", /* 2 3 4 5 6 7 */ 135 [4] = "DEJDEHHHJJ U VUVVZZZ", /* 8 9 */ 136 [5] = " U VUVV ", /* A a C c D d */ 137 [6] = " K U VUVV ", /* B b */ 138 [7] = " FFF FF U VUVV ", /* E e */ 139 [8] = " f f U VUVV f", /* F f */ 140 [9] = " LLf fL PR Li L f", /* L */ 141 [10] = " OOf fO S P O i O f", /* l */ 142 [11] = " FFX ", /* P p */ 143 [12] = " MM M i iiM M ", /* U u */ 144 [13] = " N ", /* X x */ 145 [14] = " G Y ", /* + - */ 146 [15] = "B EE EE T W ", /* . */ 147 /* ABCDEFGHIJKLMNOPQRSTUVWXYZ */ 148 }; 149 /* INDENT ON */ 150 151 static const unsigned char lex_number_row[] = { 152 ['0'] = 1, 153 ['1'] = 2, 154 ['2'] = 3, ['3'] = 3, ['4'] = 3, ['5'] = 3, ['6'] = 3, ['7'] = 3, 155 ['8'] = 4, ['9'] = 4, 156 ['A'] = 5, ['a'] = 5, ['C'] = 5, ['c'] = 5, ['D'] = 5, ['d'] = 5, 157 ['B'] = 6, ['b'] = 6, 158 ['E'] = 7, ['e'] = 7, 159 ['F'] = 8, ['f'] = 8, 160 ['L'] = 9, 161 ['l'] = 10, 162 ['P'] = 11, ['p'] = 11, 163 ['U'] = 12, ['u'] = 12, 164 ['X'] = 13, ['x'] = 13, 165 ['+'] = 14, ['-'] = 14, 166 ['.'] = 15, 167 }; 168 169 static void 170 token_add_char(char ch) 171 { 172 buf_add_char(&token, ch); 173 } 174 175 176 static void 177 lex_number(void) 178 { 179 for (unsigned char s = 'A'; s != 'f' && s != 'i' && s != 'u';) { 180 unsigned char ch = (unsigned char)inp_peek(); 181 if (ch == '\\' && inp_lookahead(1) == '\n') { 182 inp_skip(); 183 inp_skip(); 184 line_no++; 185 continue; 186 } 187 if (ch >= array_length(lex_number_row) || lex_number_row[ch] == 0) 188 break; 189 190 unsigned char row = lex_number_row[ch]; 191 if (lex_number_state[row][s - 'A'] == ' ') { 192 /*- 193 * lex_number_state[0][s - 'A'] now indicates the type: 194 * f = floating, i = integer, u = unknown 195 */ 196 return; 197 } 198 199 s = lex_number_state[row][s - 'A']; 200 token_add_char(inp_next()); 201 } 202 } 203 204 static bool 205 is_identifier_start(char ch) 206 { 207 return ch_isalpha(ch) || ch == '_' || ch == '$'; 208 } 209 210 static bool 211 is_identifier_part(char ch) 212 { 213 return ch_isalnum(ch) || ch == '_' || ch == '$'; 214 } 215 216 static void 217 lex_word(void) 218 { 219 for (;;) { 220 if (is_identifier_part(inp_peek())) 221 token_add_char(inp_next()); 222 else if (inp_peek() == '\\' && inp_lookahead(1) == '\n') { 223 inp_skip(); 224 inp_skip(); 225 line_no++; 226 } else 227 return; 228 } 229 } 230 231 static void 232 lex_char_or_string(void) 233 { 234 for (char delim = token.mem[token.len - 1];;) { 235 if (inp_peek() == '\n') { 236 diag(1, "Unterminated literal"); 237 return; 238 } 239 240 token_add_char(inp_next()); 241 if (token.mem[token.len - 1] == delim) 242 return; 243 244 if (token.mem[token.len - 1] == '\\') { 245 if (inp_peek() == '\n') 246 ++line_no; 247 token_add_char(inp_next()); 248 } 249 } 250 } 251 252 /* Guess whether the current token is a declared type. */ 253 static bool 254 probably_typename(void) 255 { 256 if (ps.prev_token == lsym_storage_class) 257 return true; 258 if (ps.block_init) 259 return false; 260 if (ps.in_stmt_or_decl) /* XXX: this condition looks incorrect */ 261 return false; 262 if (inp_peek() == '*' && inp_lookahead(1) != '=') 263 goto maybe; 264 /* XXX: is_identifier_start */ 265 if (ch_isalpha(inp_peek())) 266 goto maybe; 267 return false; 268 maybe: 269 return ps.prev_token == lsym_semicolon || 270 ps.prev_token == lsym_lbrace || 271 ps.prev_token == lsym_rbrace; 272 } 273 274 static int 275 bsearch_typenames(const char *key) 276 { 277 const char **arr = typenames.items; 278 int lo = 0; 279 int hi = (int)typenames.len - 1; 280 281 while (lo <= hi) { 282 int mid = (int)((unsigned)(lo + hi) >> 1); 283 int cmp = strcmp(arr[mid], key); 284 if (cmp < 0) 285 lo = mid + 1; 286 else if (cmp > 0) 287 hi = mid - 1; 288 else 289 return mid; 290 } 291 return -(lo + 1); 292 } 293 294 static bool 295 is_typename(void) 296 { 297 if (opt.auto_typedefs && 298 token.len >= 2 && memcmp(token.mem + token.len - 2, "_t", 2) == 0) 299 return true; 300 301 return bsearch_typenames(token.st) >= 0; 302 } 303 304 static int 305 cmp_keyword_by_name(const void *key, const void *elem) 306 { 307 return strcmp(key, ((const struct keyword *)elem)->name); 308 } 309 310 /* 311 * Looking at something like 'function_name(...)' in a line, guess whether 312 * this starts a function definition or a declaration. 313 */ 314 static bool 315 probably_looking_at_definition(void) 316 { 317 int paren_level = 0; 318 for (const char *p = inp_p(); *p != '\n'; p++) { 319 if (*p == '(') 320 paren_level++; 321 if (*p == ')' && --paren_level == 0) { 322 p++; 323 324 while (*p != '\n' && (ch_isspace(*p) || is_identifier_part(*p))) 325 p++; /* '__dead' or '__unused' */ 326 327 if (*p == '\n') /* func(...) */ 328 break; 329 if (*p == ';') /* func(...); */ 330 return false; 331 if (*p == ',') /* double abs(), pi; */ 332 return false; 333 if (*p == '(') /* func(...) __attribute__((...)) */ 334 paren_level++; /* func(...) __printflike(...) */ 335 else 336 break; /* func(...) { ... */ 337 } 338 } 339 340 /* 341 * To further reduce the cases where indent wrongly treats an incomplete 342 * function declaration as a function definition, thus adding a newline 343 * before the function name, it may be worth looking for parameter names, 344 * as these are often omitted in function declarations and only included 345 * in function definitions. Or just increase the lookahead to more than 346 * just the current line of input, until the next '{'. 347 */ 348 return true; 349 } 350 351 /* Read an alphanumeric token into 'token', or return lsym_eof. */ 352 static lexer_symbol 353 lexi_alnum(void) 354 { 355 if (ch_isdigit(inp_peek()) || 356 (inp_peek() == '.' && ch_isdigit(inp_lookahead(1)))) { 357 lex_number(); 358 } else if (is_identifier_start(inp_peek())) { 359 lex_word(); 360 361 if (token.len == 1 && token.st[0] == 'L' && 362 (inp_peek() == '"' || inp_peek() == '\'')) { 363 token_add_char(inp_next()); 364 lex_char_or_string(); 365 ps.next_unary = false; 366 return lsym_word; 367 } 368 } else 369 return lsym_eof; /* just as a placeholder */ 370 371 while (ch_isblank(inp_peek())) 372 inp_skip(); 373 374 ps.next_unary = ps.prev_token == lsym_tag; /* for 'struct s *' */ 375 376 if (ps.prev_token == lsym_tag && ps.nparen == 0) 377 return lsym_type_outside_parentheses; 378 379 token_add_char('\0'); 380 token.len--; 381 const struct keyword *kw = bsearch(token.st, keywords, 382 array_length(keywords), sizeof(keywords[0]), cmp_keyword_by_name); 383 bool is_type = false; 384 if (kw == NULL) { 385 if (is_typename()) { 386 is_type = true; 387 ps.next_unary = true; 388 if (ps.in_enum == in_enum_enum) 389 ps.in_enum = in_enum_type; 390 goto found_typename; 391 } 392 393 } else { /* we have a keyword */ 394 is_type = kw->lsym == lsym_type; 395 ps.next_unary = true; 396 if (kw->lsym != lsym_tag && kw->lsym != lsym_type) 397 return kw->lsym; 398 399 found_typename: 400 if (ps.nparen > 0) { 401 /* inside parentheses: cast, param list, offsetof or sizeof */ 402 if (!ps.paren[ps.nparen - 1].no_cast) 403 ps.paren[ps.nparen - 1].maybe_cast = true; 404 } 405 if (ps.prev_token != lsym_period && ps.prev_token != lsym_unary_op) { 406 if (kw != NULL && kw->lsym == lsym_tag) { 407 if (token.st[0] == 'e' /* enum */) 408 ps.in_enum = in_enum_enum; 409 return lsym_tag; 410 } 411 if (ps.nparen == 0) 412 return lsym_type_outside_parentheses; 413 } 414 } 415 416 if (inp_peek() == '(' && ps.tos <= 1 && ps.ind_level == 0 && 417 !ps.in_func_def_params && !ps.block_init) { 418 419 if (ps.nparen == 0 && probably_looking_at_definition()) { 420 ps.is_function_definition = true; 421 if (ps.in_decl) 422 ps.in_func_def_params = true; 423 return lsym_funcname; 424 } 425 426 } else if (ps.nparen == 0 && probably_typename()) { 427 ps.next_unary = true; 428 return lsym_type_outside_parentheses; 429 } 430 431 return is_type ? lsym_type_in_parentheses : lsym_word; 432 } 433 434 static bool 435 is_asterisk_unary(void) 436 { 437 if (ps.next_unary || ps.in_func_def_params) 438 return true; 439 if (ps.prev_token == lsym_word || 440 ps.prev_token == lsym_rparen_or_rbracket) 441 return false; 442 return ps.in_decl && ps.nparen > 0; 443 } 444 445 static void 446 lex_asterisk_unary(void) 447 { 448 while (inp_peek() == '*' || ch_isspace(inp_peek())) { 449 if (inp_peek() == '*') 450 token_add_char('*'); 451 inp_skip(); 452 } 453 454 if (ps.in_decl) { 455 for (const char *tp = inp_p(); *tp != '\n';) { 456 if (ch_isspace(*tp)) 457 tp++; 458 else if (is_identifier_start(*tp)) { 459 tp++; 460 while (is_identifier_part(*tp)) 461 tp++; 462 } else { 463 if (*tp == '(') 464 ps.is_function_definition = true; 465 break; 466 } 467 } 468 } 469 } 470 471 /* Reads the next token, placing it in the global variable "token". */ 472 lexer_symbol 473 lexi(void) 474 { 475 token.len = 0; 476 ps.curr_col_1 = ps.next_col_1; 477 ps.next_col_1 = false; 478 479 while (ch_isblank(inp_peek())) { 480 ps.curr_col_1 = false; 481 inp_skip(); 482 } 483 484 lexer_symbol alnum_lsym = lexi_alnum(); 485 if (alnum_lsym != lsym_eof) { 486 debug_parser_state(alnum_lsym); 487 return alnum_lsym; 488 } 489 490 /* Scan a non-alphanumeric token */ 491 492 token_add_char(inp_next()); 493 494 lexer_symbol lsym; 495 bool next_unary; 496 497 switch (token.mem[token.len - 1]) { 498 499 /* INDENT OFF */ 500 case '(': 501 case '[': lsym = lsym_lparen_or_lbracket; next_unary = true; break; 502 case ')': 503 case ']': lsym = lsym_rparen_or_rbracket; next_unary = false; break; 504 case '?': lsym = lsym_question; next_unary = true; break; 505 case ':': lsym = lsym_colon; next_unary = true; break; 506 case ';': lsym = lsym_semicolon; next_unary = true; break; 507 case '{': lsym = lsym_lbrace; next_unary = true; break; 508 case '}': lsym = lsym_rbrace; next_unary = true; break; 509 case ',': lsym = lsym_comma; next_unary = true; break; 510 case '.': lsym = lsym_period; next_unary = false; break; 511 /* INDENT ON */ 512 513 case '\n': 514 /* if data has been exhausted, the '\n' is a dummy. */ 515 lsym = had_eof ? lsym_eof : lsym_newline; 516 next_unary = ps.next_unary; 517 ps.next_col_1 = true; 518 break; 519 520 case '\f': 521 lsym = lsym_form_feed; 522 next_unary = ps.next_unary; 523 ps.next_col_1 = true; 524 break; 525 526 case '#': 527 lsym = lsym_preprocessing; 528 next_unary = ps.next_unary; 529 break; 530 531 case '\'': 532 case '"': 533 lex_char_or_string(); 534 lsym = lsym_word; 535 next_unary = false; 536 break; 537 538 case '-': 539 case '+': 540 lsym = ps.next_unary ? lsym_unary_op : lsym_binary_op; 541 next_unary = true; 542 543 if (inp_peek() == token.mem[token.len - 1]) { /* '++' or '--' */ 544 token_add_char(inp_next()); 545 if (ps.prev_token == lsym_word || 546 ps.prev_token == lsym_rparen_or_rbracket) { 547 lsym = ps.next_unary ? lsym_unary_op : lsym_postfix_op; 548 next_unary = false; 549 } 550 551 } else if (inp_peek() == '=') { /* '+=' or '-=' */ 552 token_add_char(inp_next()); 553 554 } else if (inp_peek() == '>') { /* '->' */ 555 token_add_char(inp_next()); 556 lsym = lsym_unary_op; 557 next_unary = false; 558 ps.want_blank = false; 559 } 560 break; 561 562 case '=': 563 if (ps.init_or_struct) 564 ps.block_init = true; 565 if (inp_peek() == '=') 566 token_add_char(inp_next()); 567 lsym = lsym_binary_op; 568 next_unary = true; 569 break; 570 571 case '>': 572 case '<': 573 case '!': /* ops like <, <<, <=, !=, etc */ 574 if (inp_peek() == '>' || inp_peek() == '<' || inp_peek() == '=') 575 token_add_char(inp_next()); 576 if (inp_peek() == '=') 577 token_add_char(inp_next()); 578 lsym = ps.next_unary ? lsym_unary_op : lsym_binary_op; 579 next_unary = true; 580 break; 581 582 case '*': 583 if (is_asterisk_unary()) { 584 lex_asterisk_unary(); 585 lsym = lsym_unary_op; 586 next_unary = true; 587 } else { 588 if (inp_peek() == '=') 589 token_add_char(inp_next()); 590 lsym = lsym_binary_op; 591 next_unary = true; 592 } 593 break; 594 595 default: 596 if (token.mem[token.len - 1] == '/' 597 && (inp_peek() == '*' || inp_peek() == '/')) { 598 token_add_char(inp_next()); 599 lsym = lsym_comment; 600 next_unary = ps.next_unary; 601 break; 602 } 603 604 /* handle '||', '&&', etc., and also things as in 'int *****i' */ 605 while (token.mem[token.len - 1] == inp_peek() || inp_peek() == '=') 606 token_add_char(inp_next()); 607 608 lsym = ps.next_unary ? lsym_unary_op : lsym_binary_op; 609 next_unary = true; 610 } 611 612 if (ps.in_enum == in_enum_enum || ps.in_enum == in_enum_type) 613 ps.in_enum = lsym == lsym_lbrace ? in_enum_brace : in_enum_no; 614 if (lsym == lsym_rbrace) 615 ps.in_enum = in_enum_no; 616 617 ps.next_unary = next_unary; 618 619 debug_parser_state(lsym); 620 return lsym; 621 } 622 623 void 624 register_typename(const char *name) 625 { 626 if (typenames.len >= typenames.cap) { 627 typenames.cap = 16 + 2 * typenames.cap; 628 typenames.items = nonnull(realloc(typenames.items, 629 sizeof(typenames.items[0]) * typenames.cap)); 630 } 631 632 int pos = bsearch_typenames(name); 633 if (pos >= 0) 634 return; /* already in the list */ 635 636 pos = -(pos + 1); 637 memmove(typenames.items + pos + 1, typenames.items + pos, 638 sizeof(typenames.items[0]) * (typenames.len++ - (unsigned)pos)); 639 typenames.items[pos] = nonnull(strdup(name)); 640 } 641
Indexes created Sat Sep 27 14:09:57 GMT 2025