lexi.c revision 1.61
1 /* $NetBSD: lexi.c,v 1.61 2021/09/26 21:23:31 rillig Exp $ */ 2 3 /*- 4 * SPDX-License-Identifier: BSD-4-Clause 5 * 6 * Copyright (c) 1985 Sun Microsystems, Inc. 7 * Copyright (c) 1980, 1993 8 * The Regents of the University of California. All rights reserved. 9 * All rights reserved. 10 * 11 * Redistribution and use in source and binary forms, with or without 12 * modification, are permitted provided that the following conditions 13 * are met: 14 * 1. Redistributions of source code must retain the above copyright 15 * notice, this list of conditions and the following disclaimer. 16 * 2. Redistributions in binary form must reproduce the above copyright 17 * notice, this list of conditions and the following disclaimer in the 18 * documentation and/or other materials provided with the distribution. 19 * 3. All advertising materials mentioning features or use of this software 20 * must display the following acknowledgement: 21 * This product includes software developed by the University of 22 * California, Berkeley and its contributors. 23 * 4. Neither the name of the University nor the names of its contributors 24 * may be used to endorse or promote products derived from this software 25 * without specific prior written permission. 26 * 27 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 28 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 29 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 30 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 31 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 32 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 33 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 34 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 35 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 36 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 37 * SUCH DAMAGE. 38 */ 39 40 #if 0 41 static char sccsid[] = "@(#)lexi.c 8.1 (Berkeley) 6/6/93"; 42 #endif 43 44 #include <sys/cdefs.h> 45 #if defined(__NetBSD__) 46 __RCSID("$NetBSD: lexi.c,v 1.61 2021/09/26 21:23:31 rillig Exp $"); 47 #elif defined(__FreeBSD__) 48 __FBSDID("$FreeBSD: head/usr.bin/indent/lexi.c 337862 2018-08-15 18:19:45Z pstef $"); 49 #endif 50 51 #include <assert.h> 52 #include <err.h> 53 #include <stdio.h> 54 #include <ctype.h> 55 #include <stdint.h> 56 #include <stdlib.h> 57 #include <string.h> 58 #include <sys/param.h> 59 60 #include "indent.h" 61 62 /* must be sorted alphabetically, is used in binary search */ 63 static const struct special { 64 const char *rwd; 65 enum rwcode rwcode; 66 } specials[] = { 67 {"_Bool", rw_type}, 68 {"_Complex", rw_type}, 69 {"_Imaginary", rw_type}, 70 {"auto", rw_storage_class}, 71 {"bool", rw_type}, 72 {"break", rw_jump}, 73 {"case", rw_case_or_default}, 74 {"char", rw_type}, 75 {"complex", rw_type}, 76 {"const", rw_type}, 77 {"continue", rw_jump}, 78 {"default", rw_case_or_default}, 79 {"do", rw_do_or_else}, 80 {"double", rw_type}, 81 {"else", rw_do_or_else}, 82 {"enum", rw_struct_or_union_or_enum}, 83 {"extern", rw_storage_class}, 84 {"float", rw_type}, 85 {"for", rw_for_or_if_or_while}, 86 {"global", rw_type}, 87 {"goto", rw_jump}, 88 {"if", rw_for_or_if_or_while}, 89 {"imaginary", rw_type}, 90 {"inline", rw_inline_or_restrict}, 91 {"int", rw_type}, 92 {"long", rw_type}, 93 {"offsetof", rw_offsetof}, 94 {"register", rw_storage_class}, 95 {"restrict", rw_inline_or_restrict}, 96 {"return", rw_jump}, 97 {"short", rw_type}, 98 {"signed", rw_type}, 99 {"sizeof", rw_sizeof}, 100 {"static", rw_storage_class}, 101 {"struct", rw_struct_or_union_or_enum}, 102 {"switch", rw_switch}, 103 {"typedef", rw_typedef}, 104 {"union", rw_struct_or_union_or_enum}, 105 {"unsigned", rw_type}, 106 {"void", rw_type}, 107 {"volatile", rw_type}, 108 {"while", rw_for_or_if_or_while} 109 }; 110 111 static const char **typenames; 112 static int typename_count; 113 static int typename_top = -1; 114 115 /* 116 * The transition table below was rewritten by hand from lx's output, given 117 * the following definitions. lx is Katherine Flavel's lexer generator. 118 * 119 * O = /[0-7]/; D = /[0-9]/; NZ = /[1-9]/; 120 * H = /[a-f0-9]/i; B = /[0-1]/; HP = /0x/i; 121 * BP = /0b/i; E = /e[+\-]?/i D+; P = /p[+\-]?/i D+; 122 * FS = /[fl]/i; IS = /u/i /(l|L|ll|LL)/? | /(l|L|ll|LL)/ /u/i?; 123 * 124 * D+ E FS? -> $float; 125 * D* "." D+ E? FS? -> $float; 126 * D+ "." E? FS? -> $float; HP H+ IS? -> $int; 127 * HP H+ P FS? -> $float; NZ D* IS? -> $int; 128 * HP H* "." H+ P FS? -> $float; "0" O* IS? -> $int; 129 * HP H+ "." P FS -> $float; BP B+ IS? -> $int; 130 */ 131 static const char num_lex_state[][26] = { 132 /* examples: 133 00 134 s 0xx 135 t 00xaa 136 a 11 101100xxa.. 137 r 11ee0001101lbuuxx.a.pp 138 t.01.e+008bLuxll0Ll.aa.p+0 139 states: ABCDEFGHIJKLMNOPQRSTUVWXYZ */ 140 [0] = "uuiifuufiuuiiuiiiiiuiuuuuu", 141 [1] = "CEIDEHHHIJQ U Q VUVVZZZ", 142 [2] = "DEIDEHHHIJQ U Q VUVVZZZ", 143 [3] = "DEIDEHHHIJ U VUVVZZZ", 144 [4] = "DEJDEHHHJJ U VUVVZZZ", 145 [5] = " U VUVV ", 146 [6] = " K U VUVV ", 147 [7] = " FFF FF U VUVV ", 148 [8] = " f f U VUVV f", 149 [9] = " LLf fL PR Li L f", 150 [10] = " OOf fO S P O i O f", 151 [11] = " FFX ", 152 [12] = " MM M i iiM M ", 153 [13] = " N ", 154 [14] = " G Y ", 155 [15] = "B EE EE T W ", 156 /* ABCDEFGHIJKLMNOPQRSTUVWXYZ */ 157 }; 158 159 static const uint8_t num_lex_row[] = { 160 ['0'] = 1, 161 ['1'] = 2, 162 ['2'] = 3, ['3'] = 3, ['4'] = 3, ['5'] = 3, ['6'] = 3, ['7'] = 3, 163 ['8'] = 4, ['9'] = 4, 164 ['A'] = 5, ['a'] = 5, ['C'] = 5, ['c'] = 5, ['D'] = 5, ['d'] = 5, 165 ['B'] = 6, ['b'] = 6, 166 ['E'] = 7, ['e'] = 7, 167 ['F'] = 8, ['f'] = 8, 168 ['L'] = 9, 169 ['l'] = 10, 170 ['P'] = 11, ['p'] = 11, 171 ['U'] = 12, ['u'] = 12, 172 ['X'] = 13, ['x'] = 13, 173 ['+'] = 14, ['-'] = 14, 174 ['.'] = 15, 175 }; 176 177 static char 178 inbuf_peek(void) 179 { 180 return *buf_ptr; 181 } 182 183 static void 184 inbuf_skip(void) 185 { 186 buf_ptr++; 187 if (buf_ptr >= buf_end) 188 fill_buffer(); 189 } 190 191 static char 192 inbuf_next(void) 193 { 194 char ch = inbuf_peek(); 195 inbuf_skip(); 196 return ch; 197 } 198 199 static void 200 check_size_token(size_t desired_size) 201 { 202 if (token.e + desired_size >= token.l) 203 buf_expand(&token, desired_size); 204 } 205 206 static int 207 compare_special_array(const void *key, const void *elem) 208 { 209 return strcmp(key, ((const struct special *)elem)->rwd); 210 } 211 212 static int 213 compare_string_array(const void *key, const void *elem) 214 { 215 return strcmp(key, *((const char *const *)elem)); 216 } 217 218 #ifdef debug 219 const char * 220 token_type_name(token_type ttype) 221 { 222 static const char *const name[] = { 223 "end_of_file", "newline", "lparen", "rparen", "unary_op", 224 "binary_op", "postfix_op", "question", "case_label", "colon", 225 "semicolon", "lbrace", "rbrace", "ident", "comma", 226 "comment", "switch_expr", "preprocessing", "form_feed", "decl", 227 "keyword_for_if_while", "keyword_do_else", 228 "if_expr", "while_expr", "for_exprs", 229 "stmt", "stmt_list", "keyword_else", "keyword_do", "do_stmt", 230 "if_expr_stmt", "if_expr_stmt_else", "period", "string_prefix", 231 "storage_class", "funcname", "type_def", "keyword_struct_union_enum" 232 }; 233 234 assert(0 <= ttype && ttype < sizeof name / sizeof name[0]); 235 236 return name[ttype]; 237 } 238 239 static void 240 print_buf(const char *name, const char *s, const char *e) 241 { 242 if (s < e) { 243 debug_printf(" %s ", name); 244 debug_vis_range("\"", s, e, "\""); 245 } 246 } 247 248 static token_type 249 lexi_end(token_type ttype) 250 { 251 debug_printf("in line %d, lexi returns '%s'", 252 line_no, token_type_name(ttype)); 253 print_buf("token", token.s, token.e); 254 print_buf("label", lab.s, lab.e); 255 print_buf("code", code.s, code.e); 256 print_buf("comment", com.s, com.e); 257 debug_printf("\n"); 258 259 return ttype; 260 } 261 #else 262 # define lexi_end(tk) (tk) 263 #endif 264 265 static void 266 lex_number(void) 267 { 268 for (uint8_t s = 'A'; s != 'f' && s != 'i' && s != 'u'; ) { 269 uint8_t ch = (uint8_t)*buf_ptr; 270 if (ch >= nitems(num_lex_row) || num_lex_row[ch] == 0) 271 break; 272 uint8_t row = num_lex_row[ch]; 273 if (num_lex_state[row][s - 'A'] == ' ') { 274 /* 275 * num_lex_state[0][s - 'A'] now indicates the type: 276 * f = floating, ch = integer, u = unknown 277 */ 278 break; 279 } 280 s = num_lex_state[row][s - 'A']; 281 check_size_token(1); 282 *token.e++ = inbuf_next(); 283 } 284 } 285 286 static void 287 lex_word(void) 288 { 289 while (isalnum((unsigned char)*buf_ptr) || 290 *buf_ptr == '\\' || 291 *buf_ptr == '_' || *buf_ptr == '$') { 292 /* fill_buffer() terminates buffer with newline */ 293 if (*buf_ptr == '\\') { 294 if (buf_ptr[1] == '\n') { 295 buf_ptr += 2; 296 if (buf_ptr >= buf_end) 297 fill_buffer(); 298 } else 299 break; 300 } 301 check_size_token(1); 302 *token.e++ = inbuf_next(); 303 } 304 } 305 306 static void 307 lex_char_or_string(void) 308 { 309 for (char delim = *token.s;;) { 310 if (*buf_ptr == '\n') { 311 diag(1, "Unterminated literal"); 312 return; 313 } 314 check_size_token(2); 315 *token.e++ = inbuf_next(); 316 if (token.e[-1] == delim) 317 return; 318 if (token.e[-1] == '\\') { 319 if (*buf_ptr == '\n') 320 ++line_no; 321 *token.e++ = inbuf_next(); 322 } 323 } 324 } 325 326 /* 327 * This hack attempts to guess whether the current token is in fact a 328 * declaration keyword -- one that has been defined by typedef. 329 */ 330 static bool 331 probably_typedef(const struct parser_state *state) 332 { 333 return state->p_l_follow == 0 && !state->block_init && !state->in_stmt && 334 ((*buf_ptr == '*' && buf_ptr[1] != '=') || 335 isalpha((unsigned char)*buf_ptr)) && 336 (state->last_token == semicolon || state->last_token == lbrace || 337 state->last_token == rbrace); 338 } 339 340 /* Reads the next token, placing it in the global variable "token". */ 341 token_type 342 lexi(struct parser_state *state) 343 { 344 bool unary_delim; /* whether the current token forces a 345 * following operator to be unary */ 346 token_type ttype; 347 348 token.e = token.s; /* point to start of place to save token */ 349 unary_delim = false; 350 state->col_1 = state->last_nl; /* tell world that this token started 351 * in column 1 iff the last thing 352 * scanned was a newline */ 353 state->last_nl = false; 354 355 while (*buf_ptr == ' ' || *buf_ptr == '\t') { /* get rid of blanks */ 356 state->col_1 = false; /* leading blanks imply token is not in column 357 * 1 */ 358 inbuf_skip(); 359 } 360 361 /* Scan an alphanumeric token */ 362 if (isalnum((unsigned char)*buf_ptr) || 363 *buf_ptr == '_' || *buf_ptr == '$' || 364 (buf_ptr[0] == '.' && isdigit((unsigned char)buf_ptr[1]))) { 365 struct special *p; 366 367 if (isdigit((unsigned char)*buf_ptr) || 368 (buf_ptr[0] == '.' && isdigit((unsigned char)buf_ptr[1]))) { 369 lex_number(); 370 } else { 371 lex_word(); 372 } 373 *token.e = '\0'; 374 375 if (token.s[0] == 'L' && token.s[1] == '\0' && 376 (*buf_ptr == '"' || *buf_ptr == '\'')) 377 return lexi_end(string_prefix); 378 379 while (*buf_ptr == ' ' || *buf_ptr == '\t') /* get rid of blanks */ 380 inbuf_next(); 381 state->keyword = rw_0; 382 383 if (state->last_token == keyword_struct_union_enum && 384 state->p_l_follow == 0) { 385 state->last_u_d = true; 386 return lexi_end(decl); 387 } 388 /* 389 * Operator after identifier is binary unless last token was 'struct' 390 */ 391 state->last_u_d = (state->last_token == keyword_struct_union_enum); 392 393 p = bsearch(token.s, specials, sizeof specials / sizeof specials[0], 394 sizeof specials[0], compare_special_array); 395 if (p == NULL) { /* not a special keyword... */ 396 char *u; 397 398 /* ... so maybe a type_t or a typedef */ 399 if ((opt.auto_typedefs && ((u = strrchr(token.s, '_')) != NULL) && 400 strcmp(u, "_t") == 0) || (typename_top >= 0 && 401 bsearch(token.s, typenames, (size_t)typename_top + 1, 402 sizeof typenames[0], compare_string_array) != NULL)) { 403 state->keyword = rw_type; 404 state->last_u_d = true; 405 goto found_typename; 406 } 407 } else { /* we have a keyword */ 408 state->keyword = p->rwcode; 409 state->last_u_d = true; 410 switch (p->rwcode) { 411 case rw_switch: 412 return lexi_end(switch_expr); 413 case rw_case_or_default: 414 return lexi_end(case_label); 415 case rw_struct_or_union_or_enum: 416 case rw_type: 417 found_typename: 418 if (state->p_l_follow != 0) { 419 /* inside parens: cast, param list, offsetof or sizeof */ 420 state->cast_mask |= (1 << state->p_l_follow) & ~state->not_cast_mask; 421 } 422 if (state->last_token == period || state->last_token == unary_op) { 423 state->keyword = rw_0; 424 break; 425 } 426 if (p != NULL && p->rwcode == rw_struct_or_union_or_enum) 427 return lexi_end(keyword_struct_union_enum); 428 if (state->p_l_follow != 0) 429 break; 430 return lexi_end(decl); 431 432 case rw_for_or_if_or_while: 433 return lexi_end(keyword_for_if_while); 434 435 case rw_do_or_else: 436 return lexi_end(keyword_do_else); 437 438 case rw_storage_class: 439 return lexi_end(storage_class); 440 441 case rw_typedef: 442 return lexi_end(type_def); 443 444 default: /* all others are treated like any other 445 * identifier */ 446 return lexi_end(ident); 447 } /* end of switch */ 448 } /* end of if (found_it) */ 449 if (*buf_ptr == '(' && state->tos <= 1 && state->ind_level == 0 && 450 !state->in_parameter_declaration && !state->block_init) { 451 char *tp = buf_ptr; 452 while (tp < buf_end) 453 if (*tp++ == ')' && (*tp == ';' || *tp == ',')) 454 goto not_proc; 455 strncpy(state->procname, token.s, sizeof state->procname - 1); 456 if (state->in_decl) 457 state->in_parameter_declaration = true; 458 return lexi_end(funcname); 459 not_proc:; 460 } else if (probably_typedef(state)) { 461 state->keyword = rw_type; 462 state->last_u_d = true; 463 return lexi_end(decl); 464 } 465 if (state->last_token == decl) /* if this is a declared variable, 466 * then following sign is unary */ 467 state->last_u_d = true; /* will make "int a -1" work */ 468 return lexi_end(ident); /* the ident is not in the list */ 469 } /* end of procesing for alpanum character */ 470 471 /* Scan a non-alphanumeric token */ 472 473 check_size_token(3); /* things like "<<=" */ 474 *token.e++ = inbuf_next(); /* if it is only a one-character token, it is 475 * moved here */ 476 *token.e = '\0'; 477 478 switch (*token.s) { 479 case '\n': 480 unary_delim = state->last_u_d; 481 state->last_nl = true; /* remember that we just had a newline */ 482 /* if data has been exhausted, the newline is a dummy. */ 483 ttype = had_eof ? end_of_file : newline; 484 break; 485 486 case '\'': 487 case '"': 488 lex_char_or_string(); 489 ttype = ident; 490 break; 491 492 case '(': 493 case '[': 494 unary_delim = true; 495 ttype = lparen; 496 break; 497 498 case ')': 499 case ']': 500 ttype = rparen; 501 break; 502 503 case '#': 504 unary_delim = state->last_u_d; 505 ttype = preprocessing; 506 break; 507 508 case '?': 509 unary_delim = true; 510 ttype = question; 511 break; 512 513 case ':': 514 ttype = colon; 515 unary_delim = true; 516 break; 517 518 case ';': 519 unary_delim = true; 520 ttype = semicolon; 521 break; 522 523 case '{': 524 unary_delim = true; 525 ttype = lbrace; 526 break; 527 528 case '}': 529 unary_delim = true; 530 ttype = rbrace; 531 break; 532 533 case 014: /* a form feed */ 534 unary_delim = state->last_u_d; 535 state->last_nl = true; /* remember this so we can set 'state->col_1' 536 * right */ 537 ttype = form_feed; 538 break; 539 540 case ',': 541 unary_delim = true; 542 ttype = comma; 543 break; 544 545 case '.': 546 unary_delim = false; 547 ttype = period; 548 break; 549 550 case '-': 551 case '+': /* check for -, +, --, ++ */ 552 ttype = state->last_u_d ? unary_op : binary_op; 553 unary_delim = true; 554 555 if (*buf_ptr == token.s[0]) { 556 /* check for doubled character */ 557 *token.e++ = *buf_ptr++; 558 /* buffer overflow will be checked at end of loop */ 559 if (state->last_token == ident || state->last_token == rparen) { 560 ttype = state->last_u_d ? unary_op : postfix_op; 561 /* check for following ++ or -- */ 562 unary_delim = false; 563 } 564 } else if (*buf_ptr == '=') 565 /* check for operator += */ 566 *token.e++ = *buf_ptr++; 567 else if (*buf_ptr == '>') { 568 /* check for operator -> */ 569 *token.e++ = *buf_ptr++; 570 unary_delim = false; 571 ttype = unary_op; 572 state->want_blank = false; 573 } 574 break; /* buffer overflow will be checked at end of 575 * switch */ 576 577 case '=': 578 if (state->in_or_st) 579 state->block_init = true; 580 if (*buf_ptr == '=') { /* == */ 581 *token.e++ = '='; /* Flip =+ to += */ 582 buf_ptr++; 583 *token.e = 0; 584 } 585 ttype = binary_op; 586 unary_delim = true; 587 break; 588 /* can drop thru!!! */ 589 590 case '>': 591 case '<': 592 case '!': /* ops like <, <<, <=, !=, etc */ 593 if (*buf_ptr == '>' || *buf_ptr == '<' || *buf_ptr == '=') 594 *token.e++ = inbuf_next(); 595 if (*buf_ptr == '=') 596 *token.e++ = *buf_ptr++; 597 ttype = state->last_u_d ? unary_op : binary_op; 598 unary_delim = true; 599 break; 600 601 case '*': 602 unary_delim = true; 603 if (!state->last_u_d) { 604 if (*buf_ptr == '=') 605 *token.e++ = *buf_ptr++; 606 ttype = binary_op; 607 break; 608 } 609 while (*buf_ptr == '*' || isspace((unsigned char)*buf_ptr)) { 610 if (*buf_ptr == '*') { 611 check_size_token(1); 612 *token.e++ = *buf_ptr; 613 } 614 inbuf_skip(); 615 } 616 if (ps.in_decl) { 617 char *tp = buf_ptr; 618 619 while (isalpha((unsigned char)*tp) || 620 isspace((unsigned char)*tp)) { 621 if (++tp >= buf_end) 622 fill_buffer(); 623 } 624 if (*tp == '(') 625 ps.procname[0] = ' '; 626 } 627 ttype = unary_op; 628 break; 629 630 default: 631 if (token.s[0] == '/' && (*buf_ptr == '*' || *buf_ptr == '/')) { 632 /* it is start of comment */ 633 *token.e++ = inbuf_next(); 634 635 ttype = comment; 636 unary_delim = state->last_u_d; 637 break; 638 } 639 while (token.e[-1] == *buf_ptr || *buf_ptr == '=') { 640 /* 641 * handle ||, &&, etc, and also things as in int *****i 642 */ 643 check_size_token(1); 644 *token.e++ = inbuf_next(); 645 } 646 ttype = state->last_u_d ? unary_op : binary_op; 647 unary_delim = true; 648 } 649 650 if (buf_ptr >= buf_end) /* check for input buffer empty */ 651 fill_buffer(); 652 state->last_u_d = unary_delim; 653 check_size_token(1); 654 *token.e = '\0'; 655 return lexi_end(ttype); 656 } 657 658 void 659 alloc_typenames(void) 660 { 661 662 typenames = xmalloc(sizeof(typenames[0]) * (typename_count = 16)); 663 } 664 665 void 666 add_typename(const char *key) 667 { 668 int comparison; 669 670 if (typename_top + 1 >= typename_count) { 671 typenames = xrealloc((void *)typenames, 672 sizeof(typenames[0]) * (typename_count *= 2)); 673 } 674 if (typename_top == -1) 675 typenames[++typename_top] = xstrdup(key); 676 else if ((comparison = strcmp(key, typenames[typename_top])) >= 0) { 677 /* take advantage of sorted input */ 678 if (comparison == 0) /* remove duplicates */ 679 return; 680 typenames[++typename_top] = xstrdup(key); 681 } else { 682 int p; 683 684 for (p = 0; (comparison = strcmp(key, typenames[p])) > 0; p++) 685 /* find place for the new key */; 686 if (comparison == 0) /* remove duplicates */ 687 return; 688 memmove(&typenames[p + 1], &typenames[p], 689 sizeof(typenames[0]) * (++typename_top - p)); 690 typenames[p] = xstrdup(key); 691 } 692 } 693
Indexes created Thu Oct 23 22:10:10 GMT 2025