lexi.c revision 1.63
1 /* $NetBSD: lexi.c,v 1.63 2021/09/27 17:33:07 rillig Exp $ */ 2 3 /*- 4 * SPDX-License-Identifier: BSD-4-Clause 5 * 6 * Copyright (c) 1985 Sun Microsystems, Inc. 7 * Copyright (c) 1980, 1993 8 * The Regents of the University of California. All rights reserved. 9 * All rights reserved. 10 * 11 * Redistribution and use in source and binary forms, with or without 12 * modification, are permitted provided that the following conditions 13 * are met: 14 * 1. Redistributions of source code must retain the above copyright 15 * notice, this list of conditions and the following disclaimer. 16 * 2. Redistributions in binary form must reproduce the above copyright 17 * notice, this list of conditions and the following disclaimer in the 18 * documentation and/or other materials provided with the distribution. 19 * 3. All advertising materials mentioning features or use of this software 20 * must display the following acknowledgement: 21 * This product includes software developed by the University of 22 * California, Berkeley and its contributors. 23 * 4. Neither the name of the University nor the names of its contributors 24 * may be used to endorse or promote products derived from this software 25 * without specific prior written permission. 26 * 27 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 28 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 29 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 30 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 31 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 32 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 33 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 34 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 35 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 36 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 37 * SUCH DAMAGE. 38 */ 39 40 #if 0 41 static char sccsid[] = "@(#)lexi.c 8.1 (Berkeley) 6/6/93"; 42 #endif 43 44 #include <sys/cdefs.h> 45 #if defined(__NetBSD__) 46 __RCSID("$NetBSD: lexi.c,v 1.63 2021/09/27 17:33:07 rillig Exp $"); 47 #elif defined(__FreeBSD__) 48 __FBSDID("$FreeBSD: head/usr.bin/indent/lexi.c 337862 2018-08-15 18:19:45Z pstef $"); 49 #endif 50 51 #include <assert.h> 52 #include <stdio.h> 53 #include <ctype.h> 54 #include <stdlib.h> 55 #include <string.h> 56 #include <sys/param.h> 57 58 #include "indent.h" 59 60 /* must be sorted alphabetically, is used in binary search */ 61 static const struct keyword { 62 const char *name; 63 enum keyword_kind kind; 64 } keywords[] = { 65 {"_Bool", kw_type}, 66 {"_Complex", kw_type}, 67 {"_Imaginary", kw_type}, 68 {"auto", kw_storage_class}, 69 {"bool", kw_type}, 70 {"break", kw_jump}, 71 {"case", kw_case_or_default}, 72 {"char", kw_type}, 73 {"complex", kw_type}, 74 {"const", kw_type}, 75 {"continue", kw_jump}, 76 {"default", kw_case_or_default}, 77 {"do", kw_do_or_else}, 78 {"double", kw_type}, 79 {"else", kw_do_or_else}, 80 {"enum", kw_struct_or_union_or_enum}, 81 {"extern", kw_storage_class}, 82 {"float", kw_type}, 83 {"for", kw_for_or_if_or_while}, 84 {"global", kw_type}, 85 {"goto", kw_jump}, 86 {"if", kw_for_or_if_or_while}, 87 {"imaginary", kw_type}, 88 {"inline", kw_inline_or_restrict}, 89 {"int", kw_type}, 90 {"long", kw_type}, 91 {"offsetof", kw_offsetof}, 92 {"register", kw_storage_class}, 93 {"restrict", kw_inline_or_restrict}, 94 {"return", kw_jump}, 95 {"short", kw_type}, 96 {"signed", kw_type}, 97 {"sizeof", kw_sizeof}, 98 {"static", kw_storage_class}, 99 {"struct", kw_struct_or_union_or_enum}, 100 {"switch", kw_switch}, 101 {"typedef", kw_typedef}, 102 {"union", kw_struct_or_union_or_enum}, 103 {"unsigned", kw_type}, 104 {"void", kw_type}, 105 {"volatile", kw_type}, 106 {"while", kw_for_or_if_or_while} 107 }; 108 109 static const char **typenames; 110 static int typename_count; 111 static int typename_top = -1; 112 113 /* 114 * The transition table below was rewritten by hand from lx's output, given 115 * the following definitions. lx is Katherine Flavel's lexer generator. 116 * 117 * O = /[0-7]/; D = /[0-9]/; NZ = /[1-9]/; 118 * H = /[a-f0-9]/i; B = /[0-1]/; HP = /0x/i; 119 * BP = /0b/i; E = /e[+\-]?/i D+; P = /p[+\-]?/i D+; 120 * FS = /[fl]/i; IS = /u/i /(l|L|ll|LL)/? | /(l|L|ll|LL)/ /u/i?; 121 * 122 * D+ E FS? -> $float; 123 * D* "." D+ E? FS? -> $float; 124 * D+ "." E? FS? -> $float; HP H+ IS? -> $int; 125 * HP H+ P FS? -> $float; NZ D* IS? -> $int; 126 * HP H* "." H+ P FS? -> $float; "0" O* IS? -> $int; 127 * HP H+ "." P FS -> $float; BP B+ IS? -> $int; 128 */ 129 static const char num_lex_state[][26] = { 130 /* examples: 131 00 132 s 0xx 133 t 00xaa 134 a 11 101100xxa.. 135 r 11ee0001101lbuuxx.a.pp 136 t.01.e+008bLuxll0Ll.aa.p+0 137 states: ABCDEFGHIJKLMNOPQRSTUVWXYZ */ 138 [0] = "uuiifuufiuuiiuiiiiiuiuuuuu", 139 [1] = "CEIDEHHHIJQ U Q VUVVZZZ", 140 [2] = "DEIDEHHHIJQ U Q VUVVZZZ", 141 [3] = "DEIDEHHHIJ U VUVVZZZ", 142 [4] = "DEJDEHHHJJ U VUVVZZZ", 143 [5] = " U VUVV ", 144 [6] = " K U VUVV ", 145 [7] = " FFF FF U VUVV ", 146 [8] = " f f U VUVV f", 147 [9] = " LLf fL PR Li L f", 148 [10] = " OOf fO S P O i O f", 149 [11] = " FFX ", 150 [12] = " MM M i iiM M ", 151 [13] = " N ", 152 [14] = " G Y ", 153 [15] = "B EE EE T W ", 154 /* ABCDEFGHIJKLMNOPQRSTUVWXYZ */ 155 }; 156 157 static const uint8_t num_lex_row[] = { 158 ['0'] = 1, 159 ['1'] = 2, 160 ['2'] = 3, ['3'] = 3, ['4'] = 3, ['5'] = 3, ['6'] = 3, ['7'] = 3, 161 ['8'] = 4, ['9'] = 4, 162 ['A'] = 5, ['a'] = 5, ['C'] = 5, ['c'] = 5, ['D'] = 5, ['d'] = 5, 163 ['B'] = 6, ['b'] = 6, 164 ['E'] = 7, ['e'] = 7, 165 ['F'] = 8, ['f'] = 8, 166 ['L'] = 9, 167 ['l'] = 10, 168 ['P'] = 11, ['p'] = 11, 169 ['U'] = 12, ['u'] = 12, 170 ['X'] = 13, ['x'] = 13, 171 ['+'] = 14, ['-'] = 14, 172 ['.'] = 15, 173 }; 174 175 static char 176 inbuf_peek(void) 177 { 178 return *buf_ptr; 179 } 180 181 static void 182 inbuf_skip(void) 183 { 184 buf_ptr++; 185 if (buf_ptr >= buf_end) 186 fill_buffer(); 187 } 188 189 static char 190 inbuf_next(void) 191 { 192 char ch = inbuf_peek(); 193 inbuf_skip(); 194 return ch; 195 } 196 197 static void 198 check_size_token(size_t desired_size) 199 { 200 if (token.e + desired_size >= token.l) 201 buf_expand(&token, desired_size); 202 } 203 204 static int 205 cmp_keyword_by_name(const void *key, const void *elem) 206 { 207 return strcmp(key, ((const struct keyword *)elem)->name); 208 } 209 210 static int 211 cmp_type_by_name(const void *key, const void *elem) 212 { 213 return strcmp(key, *((const char *const *)elem)); 214 } 215 216 #ifdef debug 217 const char * 218 token_type_name(token_type ttype) 219 { 220 static const char *const name[] = { 221 "end_of_file", "newline", "lparen", "rparen", "unary_op", 222 "binary_op", "postfix_op", "question", "case_label", "colon", 223 "semicolon", "lbrace", "rbrace", "ident", "comma", 224 "comment", "switch_expr", "preprocessing", "form_feed", "decl", 225 "keyword_for_if_while", "keyword_do_else", 226 "if_expr", "while_expr", "for_exprs", 227 "stmt", "stmt_list", "keyword_else", "keyword_do", "do_stmt", 228 "if_expr_stmt", "if_expr_stmt_else", "period", "string_prefix", 229 "storage_class", "funcname", "type_def", "keyword_struct_union_enum" 230 }; 231 232 assert(0 <= ttype && ttype < nitems(name)); 233 234 return name[ttype]; 235 } 236 237 static void 238 print_buf(const char *name, const char *s, const char *e) 239 { 240 if (s < e) { 241 debug_printf(" %s ", name); 242 debug_vis_range("\"", s, e, "\""); 243 } 244 } 245 246 static token_type 247 lexi_end(token_type ttype) 248 { 249 debug_printf("in line %d, lexi returns '%s'", 250 line_no, token_type_name(ttype)); 251 print_buf("token", token.s, token.e); 252 print_buf("label", lab.s, lab.e); 253 print_buf("code", code.s, code.e); 254 print_buf("comment", com.s, com.e); 255 debug_printf("\n"); 256 257 return ttype; 258 } 259 #else 260 # define lexi_end(tk) (tk) 261 #endif 262 263 static void 264 lex_number(void) 265 { 266 for (uint8_t s = 'A'; s != 'f' && s != 'i' && s != 'u'; ) { 267 uint8_t ch = (uint8_t)*buf_ptr; 268 if (ch >= nitems(num_lex_row) || num_lex_row[ch] == 0) 269 break; 270 uint8_t row = num_lex_row[ch]; 271 if (num_lex_state[row][s - 'A'] == ' ') { 272 /* 273 * num_lex_state[0][s - 'A'] now indicates the type: 274 * f = floating, ch = integer, u = unknown 275 */ 276 break; 277 } 278 s = num_lex_state[row][s - 'A']; 279 check_size_token(1); 280 *token.e++ = inbuf_next(); 281 } 282 } 283 284 static void 285 lex_word(void) 286 { 287 while (isalnum((unsigned char)*buf_ptr) || 288 *buf_ptr == '\\' || 289 *buf_ptr == '_' || *buf_ptr == '$') { 290 /* fill_buffer() terminates buffer with newline */ 291 if (*buf_ptr == '\\') { 292 if (buf_ptr[1] == '\n') { 293 buf_ptr += 2; 294 if (buf_ptr >= buf_end) 295 fill_buffer(); 296 } else 297 break; 298 } 299 check_size_token(1); 300 *token.e++ = inbuf_next(); 301 } 302 } 303 304 static void 305 lex_char_or_string(void) 306 { 307 for (char delim = *token.s;;) { 308 if (*buf_ptr == '\n') { 309 diag(1, "Unterminated literal"); 310 return; 311 } 312 check_size_token(2); 313 *token.e++ = inbuf_next(); 314 if (token.e[-1] == delim) 315 return; 316 if (token.e[-1] == '\\') { 317 if (*buf_ptr == '\n') 318 ++line_no; 319 *token.e++ = inbuf_next(); 320 } 321 } 322 } 323 324 /* 325 * This hack attempts to guess whether the current token is in fact a 326 * declaration keyword -- one that has been defined by typedef. 327 */ 328 static bool 329 probably_typedef(const struct parser_state *state) 330 { 331 return state->p_l_follow == 0 && !state->block_init && !state->in_stmt && 332 ((*buf_ptr == '*' && buf_ptr[1] != '=') || 333 isalpha((unsigned char)*buf_ptr)) && 334 (state->last_token == semicolon || state->last_token == lbrace || 335 state->last_token == rbrace); 336 } 337 338 static bool 339 is_typename(void) 340 { 341 if (opt.auto_typedefs) { 342 const char *u; 343 if ((u = strrchr(token.s, '_')) != NULL && strcmp(u, "_t") == 0) 344 return true; 345 } 346 347 if (typename_top < 0) 348 return false; 349 return bsearch(token.s, typenames, (size_t)typename_top + 1, 350 sizeof(typenames[0]), cmp_type_by_name) != NULL; 351 } 352 353 /* Reads the next token, placing it in the global variable "token". */ 354 token_type 355 lexi(struct parser_state *state) 356 { 357 bool unary_delim; /* whether the current token forces a 358 * following operator to be unary */ 359 token_type ttype; 360 361 token.e = token.s; /* point to start of place to save token */ 362 unary_delim = false; 363 state->col_1 = state->last_nl; /* tell world that this token started 364 * in column 1 iff the last thing 365 * scanned was a newline */ 366 state->last_nl = false; 367 368 while (*buf_ptr == ' ' || *buf_ptr == '\t') { /* get rid of blanks */ 369 state->col_1 = false; /* leading blanks imply token is not in column 370 * 1 */ 371 inbuf_skip(); 372 } 373 374 /* Scan an alphanumeric token */ 375 if (isalnum((unsigned char)*buf_ptr) || 376 *buf_ptr == '_' || *buf_ptr == '$' || 377 (buf_ptr[0] == '.' && isdigit((unsigned char)buf_ptr[1]))) { 378 struct keyword *kw; 379 380 if (isdigit((unsigned char)*buf_ptr) || 381 (buf_ptr[0] == '.' && isdigit((unsigned char)buf_ptr[1]))) { 382 lex_number(); 383 } else { 384 lex_word(); 385 } 386 *token.e = '\0'; 387 388 if (token.s[0] == 'L' && token.s[1] == '\0' && 389 (*buf_ptr == '"' || *buf_ptr == '\'')) 390 return lexi_end(string_prefix); 391 392 while (*buf_ptr == ' ' || *buf_ptr == '\t') /* get rid of blanks */ 393 inbuf_next(); 394 state->keyword = kw_0; 395 396 if (state->last_token == keyword_struct_union_enum && 397 state->p_l_follow == 0) { 398 state->last_u_d = true; 399 return lexi_end(decl); 400 } 401 /* 402 * Operator after identifier is binary unless last token was 'struct' 403 */ 404 state->last_u_d = (state->last_token == keyword_struct_union_enum); 405 406 kw = bsearch(token.s, keywords, nitems(keywords), 407 sizeof(keywords[0]), cmp_keyword_by_name); 408 if (kw == NULL) { 409 if (is_typename()) { 410 state->keyword = kw_type; 411 state->last_u_d = true; 412 goto found_typename; 413 } 414 } else { /* we have a keyword */ 415 state->keyword = kw->kind; 416 state->last_u_d = true; 417 switch (kw->kind) { 418 case kw_switch: 419 return lexi_end(switch_expr); 420 case kw_case_or_default: 421 return lexi_end(case_label); 422 case kw_struct_or_union_or_enum: 423 case kw_type: 424 found_typename: 425 if (state->p_l_follow != 0) { 426 /* inside parens: cast, param list, offsetof or sizeof */ 427 state->cast_mask |= (1 << state->p_l_follow) & ~state->not_cast_mask; 428 } 429 if (state->last_token == period || state->last_token == unary_op) { 430 state->keyword = kw_0; 431 break; 432 } 433 if (kw != NULL && kw->kind == kw_struct_or_union_or_enum) 434 return lexi_end(keyword_struct_union_enum); 435 if (state->p_l_follow != 0) 436 break; 437 return lexi_end(decl); 438 439 case kw_for_or_if_or_while: 440 return lexi_end(keyword_for_if_while); 441 442 case kw_do_or_else: 443 return lexi_end(keyword_do_else); 444 445 case kw_storage_class: 446 return lexi_end(storage_class); 447 448 case kw_typedef: 449 return lexi_end(type_def); 450 451 default: /* all others are treated like any other 452 * identifier */ 453 return lexi_end(ident); 454 } /* end of switch */ 455 } /* end of if (found_it) */ 456 if (*buf_ptr == '(' && state->tos <= 1 && state->ind_level == 0 && 457 !state->in_parameter_declaration && !state->block_init) { 458 char *tp = buf_ptr; 459 while (tp < buf_end) 460 if (*tp++ == ')' && (*tp == ';' || *tp == ',')) 461 goto not_proc; 462 strncpy(state->procname, token.s, sizeof state->procname - 1); 463 if (state->in_decl) 464 state->in_parameter_declaration = true; 465 return lexi_end(funcname); 466 not_proc:; 467 } else if (probably_typedef(state)) { 468 state->keyword = kw_type; 469 state->last_u_d = true; 470 return lexi_end(decl); 471 } 472 if (state->last_token == decl) /* if this is a declared variable, 473 * then following sign is unary */ 474 state->last_u_d = true; /* will make "int a -1" work */ 475 return lexi_end(ident); /* the ident is not in the list */ 476 } /* end of procesing for alpanum character */ 477 478 /* Scan a non-alphanumeric token */ 479 480 check_size_token(3); /* things like "<<=" */ 481 *token.e++ = inbuf_next(); /* if it is only a one-character token, it is 482 * moved here */ 483 *token.e = '\0'; 484 485 switch (*token.s) { 486 case '\n': 487 unary_delim = state->last_u_d; 488 state->last_nl = true; /* remember that we just had a newline */ 489 /* if data has been exhausted, the newline is a dummy. */ 490 ttype = had_eof ? end_of_file : newline; 491 break; 492 493 case '\'': 494 case '"': 495 lex_char_or_string(); 496 ttype = ident; 497 break; 498 499 case '(': 500 case '[': 501 unary_delim = true; 502 ttype = lparen; 503 break; 504 505 case ')': 506 case ']': 507 ttype = rparen; 508 break; 509 510 case '#': 511 unary_delim = state->last_u_d; 512 ttype = preprocessing; 513 break; 514 515 case '?': 516 unary_delim = true; 517 ttype = question; 518 break; 519 520 case ':': 521 ttype = colon; 522 unary_delim = true; 523 break; 524 525 case ';': 526 unary_delim = true; 527 ttype = semicolon; 528 break; 529 530 case '{': 531 unary_delim = true; 532 ttype = lbrace; 533 break; 534 535 case '}': 536 unary_delim = true; 537 ttype = rbrace; 538 break; 539 540 case 014: /* a form feed */ 541 unary_delim = state->last_u_d; 542 state->last_nl = true; /* remember this so we can set 'state->col_1' 543 * right */ 544 ttype = form_feed; 545 break; 546 547 case ',': 548 unary_delim = true; 549 ttype = comma; 550 break; 551 552 case '.': 553 unary_delim = false; 554 ttype = period; 555 break; 556 557 case '-': 558 case '+': /* check for -, +, --, ++ */ 559 ttype = state->last_u_d ? unary_op : binary_op; 560 unary_delim = true; 561 562 if (*buf_ptr == token.s[0]) { 563 /* check for doubled character */ 564 *token.e++ = *buf_ptr++; 565 /* buffer overflow will be checked at end of loop */ 566 if (state->last_token == ident || state->last_token == rparen) { 567 ttype = state->last_u_d ? unary_op : postfix_op; 568 /* check for following ++ or -- */ 569 unary_delim = false; 570 } 571 } else if (*buf_ptr == '=') 572 /* check for operator += */ 573 *token.e++ = *buf_ptr++; 574 else if (*buf_ptr == '>') { 575 /* check for operator -> */ 576 *token.e++ = *buf_ptr++; 577 unary_delim = false; 578 ttype = unary_op; 579 state->want_blank = false; 580 } 581 break; /* buffer overflow will be checked at end of 582 * switch */ 583 584 case '=': 585 if (state->in_or_st) 586 state->block_init = true; 587 if (*buf_ptr == '=') { /* == */ 588 *token.e++ = '='; /* Flip =+ to += */ 589 buf_ptr++; 590 *token.e = 0; 591 } 592 ttype = binary_op; 593 unary_delim = true; 594 break; 595 /* can drop thru!!! */ 596 597 case '>': 598 case '<': 599 case '!': /* ops like <, <<, <=, !=, etc */ 600 if (*buf_ptr == '>' || *buf_ptr == '<' || *buf_ptr == '=') 601 *token.e++ = inbuf_next(); 602 if (*buf_ptr == '=') 603 *token.e++ = *buf_ptr++; 604 ttype = state->last_u_d ? unary_op : binary_op; 605 unary_delim = true; 606 break; 607 608 case '*': 609 unary_delim = true; 610 if (!state->last_u_d) { 611 if (*buf_ptr == '=') 612 *token.e++ = *buf_ptr++; 613 ttype = binary_op; 614 break; 615 } 616 while (*buf_ptr == '*' || isspace((unsigned char)*buf_ptr)) { 617 if (*buf_ptr == '*') { 618 check_size_token(1); 619 *token.e++ = *buf_ptr; 620 } 621 inbuf_skip(); 622 } 623 if (ps.in_decl) { 624 char *tp = buf_ptr; 625 626 while (isalpha((unsigned char)*tp) || 627 isspace((unsigned char)*tp)) { 628 if (++tp >= buf_end) 629 fill_buffer(); 630 } 631 if (*tp == '(') 632 ps.procname[0] = ' '; 633 } 634 ttype = unary_op; 635 break; 636 637 default: 638 if (token.s[0] == '/' && (*buf_ptr == '*' || *buf_ptr == '/')) { 639 /* it is start of comment */ 640 *token.e++ = inbuf_next(); 641 642 ttype = comment; 643 unary_delim = state->last_u_d; 644 break; 645 } 646 while (token.e[-1] == *buf_ptr || *buf_ptr == '=') { 647 /* 648 * handle ||, &&, etc, and also things as in int *****i 649 */ 650 check_size_token(1); 651 *token.e++ = inbuf_next(); 652 } 653 ttype = state->last_u_d ? unary_op : binary_op; 654 unary_delim = true; 655 } 656 657 if (buf_ptr >= buf_end) /* check for input buffer empty */ 658 fill_buffer(); 659 state->last_u_d = unary_delim; 660 check_size_token(1); 661 *token.e = '\0'; 662 return lexi_end(ttype); 663 } 664 665 void 666 alloc_typenames(void) 667 { 668 669 typenames = xmalloc(sizeof(typenames[0]) * (typename_count = 16)); 670 } 671 672 void 673 add_typename(const char *key) 674 { 675 int comparison; 676 677 if (typename_top + 1 >= typename_count) { 678 typenames = xrealloc((void *)typenames, 679 sizeof(typenames[0]) * (typename_count *= 2)); 680 } 681 if (typename_top == -1) 682 typenames[++typename_top] = xstrdup(key); 683 else if ((comparison = strcmp(key, typenames[typename_top])) >= 0) { 684 /* take advantage of sorted input */ 685 if (comparison == 0) /* remove duplicates */ 686 return; 687 typenames[++typename_top] = xstrdup(key); 688 } else { 689 int p; 690 691 for (p = 0; (comparison = strcmp(key, typenames[p])) > 0; p++) 692 /* find place for the new key */; 693 if (comparison == 0) /* remove duplicates */ 694 return; 695 memmove(&typenames[p + 1], &typenames[p], 696 sizeof(typenames[0]) * (++typename_top - p)); 697 typenames[p] = xstrdup(key); 698 } 699 } 700
Indexes created Fri Sep 26 20:09:58 GMT 2025