lexi.c revision 1.55
1 /* $NetBSD: lexi.c,v 1.55 2021/09/25 17:36:51 rillig Exp $ */ 2 3 /*- 4 * SPDX-License-Identifier: BSD-4-Clause 5 * 6 * Copyright (c) 1985 Sun Microsystems, Inc. 7 * Copyright (c) 1980, 1993 8 * The Regents of the University of California. All rights reserved. 9 * All rights reserved. 10 * 11 * Redistribution and use in source and binary forms, with or without 12 * modification, are permitted provided that the following conditions 13 * are met: 14 * 1. Redistributions of source code must retain the above copyright 15 * notice, this list of conditions and the following disclaimer. 16 * 2. Redistributions in binary form must reproduce the above copyright 17 * notice, this list of conditions and the following disclaimer in the 18 * documentation and/or other materials provided with the distribution. 19 * 3. All advertising materials mentioning features or use of this software 20 * must display the following acknowledgement: 21 * This product includes software developed by the University of 22 * California, Berkeley and its contributors. 23 * 4. Neither the name of the University nor the names of its contributors 24 * may be used to endorse or promote products derived from this software 25 * without specific prior written permission. 26 * 27 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 28 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 29 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 30 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 31 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 32 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 33 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 34 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 35 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 36 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 37 * SUCH DAMAGE. 38 */ 39 40 #if 0 41 static char sccsid[] = "@(#)lexi.c 8.1 (Berkeley) 6/6/93"; 42 #endif 43 44 #include <sys/cdefs.h> 45 #if defined(__NetBSD__) 46 __RCSID("$NetBSD: lexi.c,v 1.55 2021/09/25 17:36:51 rillig Exp $"); 47 #elif defined(__FreeBSD__) 48 __FBSDID("$FreeBSD: head/usr.bin/indent/lexi.c 337862 2018-08-15 18:19:45Z pstef $"); 49 #endif 50 51 #include <assert.h> 52 #include <err.h> 53 #include <stdio.h> 54 #include <ctype.h> 55 #include <stdlib.h> 56 #include <string.h> 57 #include <sys/param.h> 58 59 #include "indent.h" 60 61 struct templ { 62 const char *rwd; 63 enum rwcode rwcode; 64 }; 65 66 /* 67 * This table has to be sorted alphabetically, because it'll be used in binary 68 * search. 69 */ 70 const struct templ specials[] = 71 { 72 {"_Bool", rw_type}, 73 {"_Complex", rw_type}, 74 {"_Imaginary", rw_type}, 75 {"auto", rw_storage_class}, 76 {"bool", rw_type}, 77 {"break", rw_jump}, 78 {"case", rw_case_or_default}, 79 {"char", rw_type}, 80 {"complex", rw_type}, 81 {"const", rw_type}, 82 {"continue", rw_jump}, 83 {"default", rw_case_or_default}, 84 {"do", rw_do_or_else}, 85 {"double", rw_type}, 86 {"else", rw_do_or_else}, 87 {"enum", rw_struct_or_union_or_enum}, 88 {"extern", rw_storage_class}, 89 {"float", rw_type}, 90 {"for", rw_for_or_if_or_while}, 91 {"global", rw_type}, 92 {"goto", rw_jump}, 93 {"if", rw_for_or_if_or_while}, 94 {"imaginary", rw_type}, 95 {"inline", rw_inline_or_restrict}, 96 {"int", rw_type}, 97 {"long", rw_type}, 98 {"offsetof", rw_offsetof}, 99 {"register", rw_storage_class}, 100 {"restrict", rw_inline_or_restrict}, 101 {"return", rw_jump}, 102 {"short", rw_type}, 103 {"signed", rw_type}, 104 {"sizeof", rw_sizeof}, 105 {"static", rw_storage_class}, 106 {"struct", rw_struct_or_union_or_enum}, 107 {"switch", rw_switch}, 108 {"typedef", rw_typedef}, 109 {"union", rw_struct_or_union_or_enum}, 110 {"unsigned", rw_type}, 111 {"void", rw_type}, 112 {"volatile", rw_type}, 113 {"while", rw_for_or_if_or_while} 114 }; 115 116 const char **typenames; 117 int typename_count; 118 int typename_top = -1; 119 120 /* 121 * The transition table below was rewritten by hand from lx's output, given 122 * the following definitions. lx is Katherine Flavel's lexer generator. 123 * 124 * O = /[0-7]/; D = /[0-9]/; NZ = /[1-9]/; 125 * H = /[a-f0-9]/i; B = /[0-1]/; HP = /0x/i; 126 * BP = /0b/i; E = /e[+\-]?/i D+; P = /p[+\-]?/i D+; 127 * FS = /[fl]/i; IS = /u/i /(l|L|ll|LL)/? | /(l|L|ll|LL)/ /u/i?; 128 * 129 * D+ E FS? -> $float; 130 * D* "." D+ E? FS? -> $float; 131 * D+ "." E? FS? -> $float; HP H+ IS? -> $int; 132 * HP H+ P FS? -> $float; NZ D* IS? -> $int; 133 * HP H* "." H+ P FS? -> $float; "0" O* IS? -> $int; 134 * HP H+ "." P FS -> $float; BP B+ IS? -> $int; 135 */ 136 static char const *table[] = { 137 /* examples: 138 00 139 s 0xx 140 t 00xaa 141 a 11 101100xxa.. 142 r 11ee0001101lbuuxx.a.pp 143 t.01.e+008bLuxll0Ll.aa.p+0 144 states: ABCDEFGHIJKLMNOPQRSTUVWXYZ */ 145 ['0'] = "CEIDEHHHIJQ U Q VUVVZZZ", 146 ['1'] = "DEIDEHHHIJQ U Q VUVVZZZ", 147 ['7'] = "DEIDEHHHIJ U VUVVZZZ", 148 ['9'] = "DEJDEHHHJJ U VUVVZZZ", 149 ['a'] = " U VUVV ", 150 ['b'] = " K U VUVV ", 151 ['e'] = " FFF FF U VUVV ", 152 ['f'] = " f f U VUVV f", 153 ['u'] = " MM M i iiM M ", 154 ['x'] = " N ", 155 ['p'] = " FFX ", 156 ['L'] = " LLf fL PR Li L f", 157 ['l'] = " OOf fO S P O i O f", 158 ['+'] = " G Y ", 159 ['.'] = "B EE EE T W ", 160 /* ABCDEFGHIJKLMNOPQRSTUVWXYZ */ 161 [0] = "uuiifuufiuuiiuiiiiiuiuuuuu", 162 }; 163 164 /* Initialize constant transition table */ 165 void 166 init_constant_tt(void) 167 { 168 table['-'] = table['+']; 169 table['8'] = table['9']; 170 table['2'] = table['3'] = table['4'] = table['5'] = table['6'] = table['7']; 171 table['A'] = table['C'] = table['D'] = table['c'] = table['d'] = table['a']; 172 table['B'] = table['b']; 173 table['E'] = table['e']; 174 table['U'] = table['u']; 175 table['X'] = table['x']; 176 table['P'] = table['p']; 177 table['F'] = table['f']; 178 } 179 180 static char 181 inbuf_peek(void) 182 { 183 return *buf_ptr; 184 } 185 186 static void 187 inbuf_skip(void) 188 { 189 buf_ptr++; 190 if (buf_ptr >= buf_end) 191 fill_buffer(); 192 } 193 194 static char 195 inbuf_next(void) 196 { 197 char ch = inbuf_peek(); 198 inbuf_skip(); 199 return ch; 200 } 201 202 static void 203 check_size_token(size_t desired_size) 204 { 205 if (token.e + (desired_size) < token.l) 206 return; 207 208 size_t nsize = token.l - token.s + 400 + desired_size; 209 size_t token_len = token.e - token.s; 210 token.buf = xrealloc(token.buf, nsize); 211 token.e = token.buf + token_len + 1; 212 token.l = token.buf + nsize - 5; 213 token.s = token.buf + 1; 214 } 215 216 static int 217 compare_templ_array(const void *key, const void *elem) 218 { 219 return strcmp(key, ((const struct templ *)elem)->rwd); 220 } 221 222 static int 223 compare_string_array(const void *key, const void *elem) 224 { 225 return strcmp(key, *((const char *const *)elem)); 226 } 227 228 #ifdef debug 229 const char * 230 token_type_name(token_type ttype) 231 { 232 static const char *const name[] = { 233 "end_of_file", "newline", "lparen", "rparen", "unary_op", 234 "binary_op", "postfix_op", "question", "case_label", "colon", 235 "semicolon", "lbrace", "rbrace", "ident", "comma", 236 "comment", "switch_expr", "preprocessing", "form_feed", "decl", 237 "keyword_for_if_while", "keyword_do_else", 238 "if_expr", "while_expr", "for_exprs", 239 "stmt", "stmt_list", "keyword_else", "keyword_do", "do_stmt", 240 "if_expr_stmt", "if_expr_stmt_else", "period", "string_prefix", 241 "storage_class", "funcname", "type_def", "keyword_struct_union_enum" 242 }; 243 244 assert(0 <= ttype && ttype < sizeof name / sizeof name[0]); 245 246 return name[ttype]; 247 } 248 249 static void 250 print_buf(const char *name, const char *s, const char *e) 251 { 252 if (s < e) { 253 debug_printf(" %s ", name); 254 debug_vis_range("\"", s, e, "\""); 255 } 256 } 257 258 static token_type 259 lexi_end(token_type ttype) 260 { 261 debug_printf("in line %d, lexi returns '%s'", 262 line_no, token_type_name(ttype)); 263 print_buf("token", token.s, token.e); 264 print_buf("label", lab.s, lab.e); 265 print_buf("code", code.s, code.e); 266 print_buf("comment", com.s, com.e); 267 debug_printf("\n"); 268 269 return ttype; 270 } 271 #else 272 # define lexi_end(tk) (tk) 273 #endif 274 275 static void 276 lex_number(void) 277 { 278 char s; 279 unsigned char i; 280 281 for (s = 'A'; s != 'f' && s != 'i' && s != 'u'; ) { 282 i = (unsigned char)*buf_ptr; 283 if (i >= nitems(table) || table[i] == NULL || 284 table[i][s - 'A'] == ' ') { 285 s = table[0][s - 'A']; 286 break; 287 } 288 s = table[i][s - 'A']; 289 check_size_token(1); 290 *token.e++ = inbuf_next(); 291 } 292 /* s now indicates the type: f(loating), i(integer), u(nknown) */ 293 } 294 295 static void 296 lex_word(void) 297 { 298 while (isalnum((unsigned char)*buf_ptr) || 299 *buf_ptr == '\\' || 300 *buf_ptr == '_' || *buf_ptr == '$') { 301 /* fill_buffer() terminates buffer with newline */ 302 if (*buf_ptr == '\\') { 303 if (buf_ptr[1] == '\n') { 304 buf_ptr += 2; 305 if (buf_ptr >= buf_end) 306 fill_buffer(); 307 } else 308 break; 309 } 310 check_size_token(1); 311 *token.e++ = inbuf_next(); 312 } 313 } 314 315 static void 316 lex_char_or_string(void) 317 { 318 for (char delim = *token.s;;) { 319 if (*buf_ptr == '\n') { 320 diag(1, "Unterminated literal"); 321 return; 322 } 323 check_size_token(2); 324 *token.e++ = inbuf_next(); 325 if (token.e[-1] == delim) 326 return; 327 if (token.e[-1] == '\\') { 328 if (*buf_ptr == '\n') 329 ++line_no; 330 *token.e++ = inbuf_next(); 331 } 332 } 333 } 334 335 /* Reads the next token, placing it in the global variable "token". */ 336 token_type 337 lexi(struct parser_state *state) 338 { 339 bool unary_delim; /* whether the current token 340 * forces a following operator to be unary */ 341 token_type ttype; 342 343 token.e = token.s; /* point to start of place to save token */ 344 unary_delim = false; 345 state->col_1 = state->last_nl; /* tell world that this token started 346 * in column 1 iff the last thing 347 * scanned was a newline */ 348 state->last_nl = false; 349 350 while (*buf_ptr == ' ' || *buf_ptr == '\t') { /* get rid of blanks */ 351 state->col_1 = false; /* leading blanks imply token is not in column 352 * 1 */ 353 inbuf_skip(); 354 } 355 356 /* Scan an alphanumeric token */ 357 if (isalnum((unsigned char)*buf_ptr) || 358 *buf_ptr == '_' || *buf_ptr == '$' || 359 (buf_ptr[0] == '.' && isdigit((unsigned char)buf_ptr[1]))) { 360 /* 361 * we have a letter or number 362 */ 363 struct templ *p; 364 365 if (isdigit((unsigned char)*buf_ptr) || 366 (buf_ptr[0] == '.' && isdigit((unsigned char)buf_ptr[1]))) { 367 lex_number(); 368 } else { 369 lex_word(); 370 } 371 *token.e = '\0'; 372 373 if (token.s[0] == 'L' && token.s[1] == '\0' && 374 (*buf_ptr == '"' || *buf_ptr == '\'')) 375 return lexi_end(string_prefix); 376 377 while (*buf_ptr == ' ' || *buf_ptr == '\t') /* get rid of blanks */ 378 inbuf_next(); 379 state->keyword = rw_0; 380 if (state->last_token == keyword_struct_union_enum && 381 state->p_l_follow == 0) { 382 /* if last token was 'struct' and we're not in parentheses, then 383 * this token should be treated as a declaration */ 384 state->last_u_d = true; 385 return lexi_end(decl); 386 } 387 /* 388 * Operator after identifier is binary unless last token was 'struct' 389 */ 390 state->last_u_d = (state->last_token == keyword_struct_union_enum); 391 392 p = bsearch(token.s, specials, sizeof specials / sizeof specials[0], 393 sizeof specials[0], compare_templ_array); 394 if (p == NULL) { /* not a special keyword... */ 395 char *u; 396 397 /* ... so maybe a type_t or a typedef */ 398 if ((opt.auto_typedefs && ((u = strrchr(token.s, '_')) != NULL) && 399 strcmp(u, "_t") == 0) || (typename_top >= 0 && 400 bsearch(token.s, typenames, (size_t)typename_top + 1, 401 sizeof typenames[0], compare_string_array) != NULL)) { 402 state->keyword = rw_type; 403 state->last_u_d = true; 404 goto found_typename; 405 } 406 } else { /* we have a keyword */ 407 state->keyword = p->rwcode; 408 state->last_u_d = true; 409 switch (p->rwcode) { 410 case rw_switch: 411 return lexi_end(switch_expr); 412 case rw_case_or_default: 413 return lexi_end(case_label); 414 case rw_struct_or_union_or_enum: 415 case rw_type: 416 found_typename: 417 if (state->p_l_follow != 0) { 418 /* inside parens: cast, param list, offsetof or sizeof */ 419 state->cast_mask |= (1 << state->p_l_follow) & ~state->not_cast_mask; 420 } 421 if (state->last_token == period || state->last_token == unary_op) { 422 state->keyword = rw_0; 423 break; 424 } 425 if (p != NULL && p->rwcode == rw_struct_or_union_or_enum) 426 return lexi_end(keyword_struct_union_enum); 427 if (state->p_l_follow != 0) 428 break; 429 return lexi_end(decl); 430 431 case rw_for_or_if_or_while: 432 return lexi_end(keyword_for_if_while); 433 434 case rw_do_or_else: 435 return lexi_end(keyword_do_else); 436 437 case rw_storage_class: 438 return lexi_end(storage_class); 439 440 case rw_typedef: 441 return lexi_end(type_def); 442 443 default: /* all others are treated like any other 444 * identifier */ 445 return lexi_end(ident); 446 } /* end of switch */ 447 } /* end of if (found_it) */ 448 if (*buf_ptr == '(' && state->tos <= 1 && state->ind_level == 0 && 449 !state->in_parameter_declaration && !state->block_init) { 450 char *tp = buf_ptr; 451 while (tp < buf_end) 452 if (*tp++ == ')' && (*tp == ';' || *tp == ',')) 453 goto not_proc; 454 strncpy(state->procname, token.s, sizeof state->procname - 1); 455 if (state->in_decl) 456 state->in_parameter_declaration = true; 457 return lexi_end(funcname); 458 not_proc:; 459 } 460 /* 461 * The following hack attempts to guess whether or not the current 462 * token is in fact a declaration keyword -- one that has been 463 * typedefd 464 */ 465 else if (state->p_l_follow == 0 && !state->block_init && 466 !state->in_stmt && 467 ((*buf_ptr == '*' && buf_ptr[1] != '=') || 468 isalpha((unsigned char)*buf_ptr)) && 469 (state->last_token == semicolon || state->last_token == lbrace || 470 state->last_token == rbrace)) { 471 state->keyword = rw_type; 472 state->last_u_d = true; 473 return lexi_end(decl); 474 } 475 if (state->last_token == decl) /* if this is a declared variable, 476 * then following sign is unary */ 477 state->last_u_d = true; /* will make "int a -1" work */ 478 return lexi_end(ident); /* the ident is not in the list */ 479 } /* end of procesing for alpanum character */ 480 481 /* Scan a non-alphanumeric token */ 482 483 check_size_token(3); /* things like "<<=" */ 484 *token.e++ = inbuf_next(); /* if it is only a one-character token, it is 485 * moved here */ 486 *token.e = '\0'; 487 488 switch (*token.s) { 489 case '\n': 490 unary_delim = state->last_u_d; 491 state->last_nl = true; /* remember that we just had a newline */ 492 /* if data has been exhausted, the newline is a dummy. */ 493 ttype = had_eof ? end_of_file : newline; 494 break; 495 496 case '\'': 497 case '"': 498 lex_char_or_string(); 499 ttype = ident; 500 break; 501 502 case '(': 503 case '[': 504 unary_delim = true; 505 ttype = lparen; 506 break; 507 508 case ')': 509 case ']': 510 ttype = rparen; 511 break; 512 513 case '#': 514 unary_delim = state->last_u_d; 515 ttype = preprocessing; 516 break; 517 518 case '?': 519 unary_delim = true; 520 ttype = question; 521 break; 522 523 case ':': 524 ttype = colon; 525 unary_delim = true; 526 break; 527 528 case ';': 529 unary_delim = true; 530 ttype = semicolon; 531 break; 532 533 case '{': 534 unary_delim = true; 535 ttype = lbrace; 536 break; 537 538 case '}': 539 unary_delim = true; 540 ttype = rbrace; 541 break; 542 543 case 014: /* a form feed */ 544 unary_delim = state->last_u_d; 545 state->last_nl = true; /* remember this so we can set 'state->col_1' 546 * right */ 547 ttype = form_feed; 548 break; 549 550 case ',': 551 unary_delim = true; 552 ttype = comma; 553 break; 554 555 case '.': 556 unary_delim = false; 557 ttype = period; 558 break; 559 560 case '-': 561 case '+': /* check for -, +, --, ++ */ 562 ttype = state->last_u_d ? unary_op : binary_op; 563 unary_delim = true; 564 565 if (*buf_ptr == token.s[0]) { 566 /* check for doubled character */ 567 *token.e++ = *buf_ptr++; 568 /* buffer overflow will be checked at end of loop */ 569 if (state->last_token == ident || state->last_token == rparen) { 570 ttype = state->last_u_d ? unary_op : postfix_op; 571 /* check for following ++ or -- */ 572 unary_delim = false; 573 } 574 } else if (*buf_ptr == '=') 575 /* check for operator += */ 576 *token.e++ = *buf_ptr++; 577 else if (*buf_ptr == '>') { 578 /* check for operator -> */ 579 *token.e++ = *buf_ptr++; 580 unary_delim = false; 581 ttype = unary_op; 582 state->want_blank = false; 583 } 584 break; /* buffer overflow will be checked at end of 585 * switch */ 586 587 case '=': 588 if (state->in_or_st) 589 state->block_init = true; 590 if (*buf_ptr == '=') { /* == */ 591 *token.e++ = '='; /* Flip =+ to += */ 592 buf_ptr++; 593 *token.e = 0; 594 } 595 ttype = binary_op; 596 unary_delim = true; 597 break; 598 /* can drop thru!!! */ 599 600 case '>': 601 case '<': 602 case '!': /* ops like <, <<, <=, !=, etc */ 603 if (*buf_ptr == '>' || *buf_ptr == '<' || *buf_ptr == '=') 604 *token.e++ = inbuf_next(); 605 if (*buf_ptr == '=') 606 *token.e++ = *buf_ptr++; 607 ttype = state->last_u_d ? unary_op : binary_op; 608 unary_delim = true; 609 break; 610 611 case '*': 612 unary_delim = true; 613 if (!state->last_u_d) { 614 if (*buf_ptr == '=') 615 *token.e++ = *buf_ptr++; 616 ttype = binary_op; 617 break; 618 } 619 while (*buf_ptr == '*' || isspace((unsigned char)*buf_ptr)) { 620 if (*buf_ptr == '*') { 621 check_size_token(1); 622 *token.e++ = *buf_ptr; 623 } 624 inbuf_skip(); 625 } 626 if (ps.in_decl) { 627 char *tp = buf_ptr; 628 629 while (isalpha((unsigned char)*tp) || 630 isspace((unsigned char)*tp)) { 631 if (++tp >= buf_end) 632 fill_buffer(); 633 } 634 if (*tp == '(') 635 ps.procname[0] = ' '; 636 } 637 ttype = unary_op; 638 break; 639 640 default: 641 if (token.s[0] == '/' && (*buf_ptr == '*' || *buf_ptr == '/')) { 642 /* it is start of comment */ 643 *token.e++ = inbuf_next(); 644 645 ttype = comment; 646 unary_delim = state->last_u_d; 647 break; 648 } 649 while (token.e[-1] == *buf_ptr || *buf_ptr == '=') { 650 /* 651 * handle ||, &&, etc, and also things as in int *****i 652 */ 653 check_size_token(1); 654 *token.e++ = inbuf_next(); 655 } 656 ttype = state->last_u_d ? unary_op : binary_op; 657 unary_delim = true; 658 } 659 660 if (buf_ptr >= buf_end) /* check for input buffer empty */ 661 fill_buffer(); 662 state->last_u_d = unary_delim; 663 check_size_token(1); 664 *token.e = '\0'; 665 return lexi_end(ttype); 666 } 667 668 void 669 alloc_typenames(void) 670 { 671 672 typenames = xmalloc(sizeof(typenames[0]) * (typename_count = 16)); 673 } 674 675 void 676 add_typename(const char *key) 677 { 678 int comparison; 679 680 if (typename_top + 1 >= typename_count) { 681 typenames = xrealloc((void *)typenames, 682 sizeof(typenames[0]) * (typename_count *= 2)); 683 } 684 if (typename_top == -1) 685 typenames[++typename_top] = xstrdup(key); 686 else if ((comparison = strcmp(key, typenames[typename_top])) >= 0) { 687 /* take advantage of sorted input */ 688 if (comparison == 0) /* remove duplicates */ 689 return; 690 typenames[++typename_top] = xstrdup(key); 691 } else { 692 int p; 693 694 for (p = 0; (comparison = strcmp(key, typenames[p])) > 0; p++) 695 /* find place for the new key */; 696 if (comparison == 0) /* remove duplicates */ 697 return; 698 memmove(&typenames[p + 1], &typenames[p], 699 sizeof(typenames[0]) * (++typename_top - p)); 700 typenames[p] = xstrdup(key); 701 } 702 } 703
Indexes created Fri Sep 26 20:09:58 GMT 2025