lexi.c revision 1.16
1 /* $NetBSD: lexi.c,v 1.16 2019/04/04 15:22:13 kamil Exp $ */ 2 3 /*- 4 * SPDX-License-Identifier: BSD-4-Clause 5 * 6 * Copyright (c) 1985 Sun Microsystems, Inc. 7 * Copyright (c) 1980, 1993 8 * The Regents of the University of California. All rights reserved. 9 * All rights reserved. 10 * 11 * Redistribution and use in source and binary forms, with or without 12 * modification, are permitted provided that the following conditions 13 * are met: 14 * 1. Redistributions of source code must retain the above copyright 15 * notice, this list of conditions and the following disclaimer. 16 * 2. Redistributions in binary form must reproduce the above copyright 17 * notice, this list of conditions and the following disclaimer in the 18 * documentation and/or other materials provided with the distribution. 19 * 3. All advertising materials mentioning features or use of this software 20 * must display the following acknowledgement: 21 * This product includes software developed by the University of 22 * California, Berkeley and its contributors. 23 * 4. Neither the name of the University nor the names of its contributors 24 * may be used to endorse or promote products derived from this software 25 * without specific prior written permission. 26 * 27 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 28 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 29 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 30 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 31 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 32 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 33 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 34 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 35 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 36 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 37 * SUCH DAMAGE. 38 */ 39 40 #if 0 41 #ifndef lint 42 static char sccsid[] = "@(#)lexi.c 8.1 (Berkeley) 6/6/93"; 43 #endif /* not lint */ 44 #endif 45 46 #include <sys/cdefs.h> 47 #ifndef lint 48 #if defined(__NetBSD__) 49 __RCSID("$NetBSD: lexi.c,v 1.16 2019/04/04 15:22:13 kamil Exp $"); 50 #elif defined(__FreeBSD__) 51 __FBSDID("$FreeBSD: head/usr.bin/indent/lexi.c 337862 2018-08-15 18:19:45Z pstef $"); 52 #endif 53 #endif 54 55 /* 56 * Here we have the token scanner for indent. It scans off one token and puts 57 * it in the global variable "token". It returns a code, indicating the type 58 * of token scanned. 59 */ 60 61 #include <err.h> 62 #include <stdio.h> 63 #include <ctype.h> 64 #include <stdlib.h> 65 #include <string.h> 66 #include <sys/param.h> 67 68 #include "indent_globs.h" 69 #include "indent_codes.h" 70 #include "indent.h" 71 72 struct templ { 73 const char *rwd; 74 int rwcode; 75 }; 76 77 /* 78 * This table has to be sorted alphabetically, because it'll be used in binary 79 * search. For the same reason, string must be the first thing in struct templ. 80 */ 81 struct templ specials[] = 82 { 83 {"_Bool", 4}, 84 {"_Complex", 4}, 85 {"_Imaginary", 4}, 86 {"auto", 10}, 87 {"bool", 4}, 88 {"break", 9}, 89 {"case", 8}, 90 {"char", 4}, 91 {"complex", 4}, 92 {"const", 4}, 93 {"continue", 12}, 94 {"default", 8}, 95 {"do", 6}, 96 {"double", 4}, 97 {"else", 6}, 98 {"enum", 3}, 99 {"extern", 10}, 100 {"float", 4}, 101 {"for", 5}, 102 {"global", 4}, 103 {"goto", 9}, 104 {"if", 5}, 105 {"imaginary", 4}, 106 {"inline", 12}, 107 {"int", 4}, 108 {"long", 4}, 109 {"offsetof", 1}, 110 {"register", 10}, 111 {"restrict", 12}, 112 {"return", 9}, 113 {"short", 4}, 114 {"signed", 4}, 115 {"sizeof", 2}, 116 {"static", 10}, 117 {"struct", 3}, 118 {"switch", 7}, 119 {"typedef", 11}, 120 {"union", 3}, 121 {"unsigned", 4}, 122 {"void", 4}, 123 {"volatile", 4}, 124 {"while", 5} 125 }; 126 127 const char **typenames; 128 int typename_count; 129 int typename_top = -1; 130 131 /* 132 * The transition table below was rewritten by hand from lx's output, given 133 * the following definitions. lx is Katherine Flavel's lexer generator. 134 * 135 * O = /[0-7]/; D = /[0-9]/; NZ = /[1-9]/; 136 * H = /[a-f0-9]/i; B = /[0-1]/; HP = /0x/i; 137 * BP = /0b/i; E = /e[+\-]?/i D+; P = /p[+\-]?/i D+; 138 * FS = /[fl]/i; IS = /u/i /(l|L|ll|LL)/? | /(l|L|ll|LL)/ /u/i?; 139 * 140 * D+ E FS? -> $float; 141 * D* "." D+ E? FS? -> $float; 142 * D+ "." E? FS? -> $float; HP H+ IS? -> $int; 143 * HP H+ P FS? -> $float; NZ D* IS? -> $int; 144 * HP H* "." H+ P FS? -> $float; "0" O* IS? -> $int; 145 * HP H+ "." P FS -> $float; BP B+ IS? -> $int; 146 */ 147 static char const *table[] = { 148 /* examples: 149 00 150 s 0xx 151 t 00xaa 152 a 11 101100xxa.. 153 r 11ee0001101lbuuxx.a.pp 154 t.01.e+008bLuxll0Ll.aa.p+0 155 states: ABCDEFGHIJKLMNOPQRSTUVWXYZ */ 156 ['0'] = "CEIDEHHHIJQ U Q VUVVZZZ", 157 ['1'] = "DEIDEHHHIJQ U Q VUVVZZZ", 158 ['7'] = "DEIDEHHHIJ U VUVVZZZ", 159 ['9'] = "DEJDEHHHJJ U VUVVZZZ", 160 ['a'] = " U VUVV ", 161 ['b'] = " K U VUVV ", 162 ['e'] = " FFF FF U VUVV ", 163 ['f'] = " f f U VUVV f", 164 ['u'] = " MM M i iiM M ", 165 ['x'] = " N ", 166 ['p'] = " FFX ", 167 ['L'] = " LLf fL PR Li L f", 168 ['l'] = " OOf fO S P O i O f", 169 ['+'] = " G Y ", 170 ['.'] = "B EE EE T W ", 171 /* ABCDEFGHIJKLMNOPQRSTUVWXYZ */ 172 [0] = "uuiifuufiuuiiuiiiiiuiuuuuu", 173 }; 174 175 static int 176 strcmp_type(const void *e1, const void *e2) 177 { 178 return (strcmp(e1, *(const char * const *)e2)); 179 } 180 181 int 182 lexi(struct parser_state *state) 183 { 184 int unary_delim; /* this is set to 1 if the current token 185 * forces a following operator to be unary */ 186 int code; /* internal code to be returned */ 187 char qchar; /* the delimiter character for a string */ 188 189 e_token = s_token; /* point to start of place to save token */ 190 unary_delim = false; 191 state->col_1 = state->last_nl; /* tell world that this token started 192 * in column 1 iff the last thing 193 * scanned was a newline */ 194 state->last_nl = false; 195 196 while (*buf_ptr == ' ' || *buf_ptr == '\t') { /* get rid of blanks */ 197 state->col_1 = false; /* leading blanks imply token is not in column 198 * 1 */ 199 if (++buf_ptr >= buf_end) 200 fill_buffer(); 201 } 202 203 /* Scan an alphanumeric token */ 204 if (isalnum((unsigned char)*buf_ptr) || 205 *buf_ptr == '_' || *buf_ptr == '$' || 206 (buf_ptr[0] == '.' && isdigit((unsigned char)buf_ptr[1]))) { 207 /* 208 * we have a character or number 209 */ 210 struct templ *p; 211 212 if (isdigit((unsigned char)*buf_ptr) || 213 (buf_ptr[0] == '.' && isdigit((unsigned char)buf_ptr[1]))) { 214 char s; 215 unsigned char i; 216 217 for (s = 'A'; s != 'f' && s != 'i' && s != 'u'; ) { 218 i = (unsigned char)*buf_ptr; 219 if (i >= nitems(table) || table[i] == NULL || 220 table[i][s - 'A'] == ' ') { 221 s = table[0][s - 'A']; 222 break; 223 } 224 s = table[i][s - 'A']; 225 CHECK_SIZE_TOKEN(1); 226 *e_token++ = *buf_ptr++; 227 if (buf_ptr >= buf_end) 228 fill_buffer(); 229 } 230 /* s now indicates the type: f(loating), i(integer), u(nknown) */ 231 } 232 else 233 while (isalnum((unsigned char)*buf_ptr) || 234 *buf_ptr == BACKSLASH || 235 *buf_ptr == '_' || *buf_ptr == '$') { 236 /* fill_buffer() terminates buffer with newline */ 237 if (*buf_ptr == BACKSLASH) { 238 if (*(buf_ptr + 1) == '\n') { 239 buf_ptr += 2; 240 if (buf_ptr >= buf_end) 241 fill_buffer(); 242 } else 243 break; 244 } 245 CHECK_SIZE_TOKEN(1); 246 /* copy it over */ 247 *e_token++ = *buf_ptr++; 248 if (buf_ptr >= buf_end) 249 fill_buffer(); 250 } 251 *e_token = '\0'; 252 253 if (s_token[0] == 'L' && s_token[1] == '\0' && 254 (*buf_ptr == '"' || *buf_ptr == '\'')) 255 return (strpfx); 256 257 while (*buf_ptr == ' ' || *buf_ptr == '\t') { /* get rid of blanks */ 258 if (++buf_ptr >= buf_end) 259 fill_buffer(); 260 } 261 state->keyword = 0; 262 if (state->last_token == structure && !state->p_l_follow) { 263 /* if last token was 'struct' and we're not 264 * in parentheses, then this token 265 * should be treated as a declaration */ 266 state->last_u_d = true; 267 return (decl); 268 } 269 /* 270 * Operator after identifier is binary unless last token was 'struct' 271 */ 272 state->last_u_d = (state->last_token == structure); 273 274 p = bsearch(s_token, 275 specials, 276 sizeof(specials) / sizeof(specials[0]), 277 sizeof(specials[0]), 278 strcmp_type); 279 if (p == NULL) { /* not a special keyword... */ 280 char *u; 281 282 /* ... so maybe a type_t or a typedef */ 283 if ((opt.auto_typedefs && ((u = strrchr(s_token, '_')) != NULL) && 284 strcmp(u, "_t") == 0) || (typename_top >= 0 && 285 bsearch(s_token, typenames, typename_top + 1, 286 sizeof(typenames[0]), strcmp_type))) { 287 state->keyword = 4; /* a type name */ 288 state->last_u_d = true; 289 goto found_typename; 290 } 291 } else { /* we have a keyword */ 292 state->keyword = p->rwcode; 293 state->last_u_d = true; 294 switch (p->rwcode) { 295 case 7: /* it is a switch */ 296 return (swstmt); 297 case 8: /* a case or default */ 298 return (casestmt); 299 300 case 3: /* a "struct" */ 301 /* FALLTHROUGH */ 302 case 4: /* one of the declaration keywords */ 303 found_typename: 304 if (state->p_l_follow) { 305 /* inside parens: cast, param list, offsetof or sizeof */ 306 state->cast_mask |= (1 << state->p_l_follow) & ~state->not_cast_mask; 307 } 308 if (state->last_token == period || state->last_token == unary_op) { 309 state->keyword = 0; 310 break; 311 } 312 if (p != NULL && p->rwcode == 3) 313 return (structure); 314 if (state->p_l_follow) 315 break; 316 return (decl); 317 318 case 5: /* if, while, for */ 319 return (sp_paren); 320 321 case 6: /* do, else */ 322 return (sp_nparen); 323 324 case 10: /* storage class specifier */ 325 return (storage); 326 327 case 11: /* typedef */ 328 return (type_def); 329 330 default: /* all others are treated like any other 331 * identifier */ 332 return (ident); 333 } /* end of switch */ 334 } /* end of if (found_it) */ 335 if (*buf_ptr == '(' && state->tos <= 1 && state->ind_level == 0 && 336 state->in_parameter_declaration == 0 && state->block_init == 0) { 337 char *tp = buf_ptr; 338 while (tp < buf_end) 339 if (*tp++ == ')' && (*tp == ';' || *tp == ',')) 340 goto not_proc; 341 strncpy(state->procname, token, sizeof state->procname - 1); 342 if (state->in_decl) 343 state->in_parameter_declaration = 1; 344 return (funcname); 345 not_proc:; 346 } 347 /* 348 * The following hack attempts to guess whether or not the current 349 * token is in fact a declaration keyword -- one that has been 350 * typedefd 351 */ 352 else if (!state->p_l_follow && !state->block_init && 353 !state->in_stmt && 354 ((*buf_ptr == '*' && buf_ptr[1] != '=') || 355 isalpha((unsigned char)*buf_ptr)) && 356 (state->last_token == semicolon || state->last_token == lbrace || 357 state->last_token == rbrace)) { 358 state->keyword = 4; /* a type name */ 359 state->last_u_d = true; 360 return decl; 361 } 362 if (state->last_token == decl) /* if this is a declared variable, 363 * then following sign is unary */ 364 state->last_u_d = true; /* will make "int a -1" work */ 365 return (ident); /* the ident is not in the list */ 366 } /* end of procesing for alpanum character */ 367 368 /* Scan a non-alphanumeric token */ 369 370 CHECK_SIZE_TOKEN(3); /* things like "<<=" */ 371 *e_token++ = *buf_ptr; /* if it is only a one-character token, it is 372 * moved here */ 373 *e_token = '\0'; 374 if (++buf_ptr >= buf_end) 375 fill_buffer(); 376 377 switch (*token) { 378 case '\n': 379 unary_delim = state->last_u_d; 380 state->last_nl = true; /* remember that we just had a newline */ 381 code = (had_eof ? 0 : newline); 382 383 /* 384 * if data has been exhausted, the newline is a dummy, and we should 385 * return code to stop 386 */ 387 break; 388 389 case '\'': /* start of quoted character */ 390 case '"': /* start of string */ 391 qchar = *token; 392 do { /* copy the string */ 393 while (1) { /* move one character or [/<char>]<char> */ 394 if (*buf_ptr == '\n') { 395 diag2(1, "Unterminated literal"); 396 goto stop_lit; 397 } 398 CHECK_SIZE_TOKEN(2); 399 *e_token = *buf_ptr++; 400 if (buf_ptr >= buf_end) 401 fill_buffer(); 402 if (*e_token == BACKSLASH) { /* if escape, copy extra char */ 403 if (*buf_ptr == '\n') /* check for escaped newline */ 404 ++line_no; 405 *++e_token = *buf_ptr++; 406 ++e_token; /* we must increment this again because we 407 * copied two chars */ 408 if (buf_ptr >= buf_end) 409 fill_buffer(); 410 } 411 else 412 break; /* we copied one character */ 413 } /* end of while (1) */ 414 } while (*e_token++ != qchar); 415 stop_lit: 416 code = ident; 417 break; 418 419 case ('('): 420 case ('['): 421 unary_delim = true; 422 code = lparen; 423 break; 424 425 case (')'): 426 case (']'): 427 code = rparen; 428 break; 429 430 case '#': 431 unary_delim = state->last_u_d; 432 code = preesc; 433 break; 434 435 case '?': 436 unary_delim = true; 437 code = question; 438 break; 439 440 case (':'): 441 code = colon; 442 unary_delim = true; 443 break; 444 445 case (';'): 446 unary_delim = true; 447 code = semicolon; 448 break; 449 450 case ('{'): 451 unary_delim = true; 452 453 /* 454 * if (state->in_or_st) state->block_init = 1; 455 */ 456 /* ? code = state->block_init ? lparen : lbrace; */ 457 code = lbrace; 458 break; 459 460 case ('}'): 461 unary_delim = true; 462 /* ? code = state->block_init ? rparen : rbrace; */ 463 code = rbrace; 464 break; 465 466 case 014: /* a form feed */ 467 unary_delim = state->last_u_d; 468 state->last_nl = true; /* remember this so we can set 'state->col_1' 469 * right */ 470 code = form_feed; 471 break; 472 473 case (','): 474 unary_delim = true; 475 code = comma; 476 break; 477 478 case '.': 479 unary_delim = false; 480 code = period; 481 break; 482 483 case '-': 484 case '+': /* check for -, +, --, ++ */ 485 code = (state->last_u_d ? unary_op : binary_op); 486 unary_delim = true; 487 488 if (*buf_ptr == token[0]) { 489 /* check for doubled character */ 490 *e_token++ = *buf_ptr++; 491 /* buffer overflow will be checked at end of loop */ 492 if (state->last_token == ident || state->last_token == rparen) { 493 code = (state->last_u_d ? unary_op : postop); 494 /* check for following ++ or -- */ 495 unary_delim = false; 496 } 497 } 498 else if (*buf_ptr == '=') 499 /* check for operator += */ 500 *e_token++ = *buf_ptr++; 501 else if (*buf_ptr == '>') { 502 /* check for operator -> */ 503 *e_token++ = *buf_ptr++; 504 unary_delim = false; 505 code = unary_op; 506 state->want_blank = false; 507 } 508 break; /* buffer overflow will be checked at end of 509 * switch */ 510 511 case '=': 512 if (state->in_or_st) 513 state->block_init = 1; 514 if (*buf_ptr == '=') {/* == */ 515 *e_token++ = '='; /* Flip =+ to += */ 516 buf_ptr++; 517 *e_token = 0; 518 } 519 code = binary_op; 520 unary_delim = true; 521 break; 522 /* can drop thru!!! */ 523 524 case '>': 525 case '<': 526 case '!': /* ops like <, <<, <=, !=, etc */ 527 if (*buf_ptr == '>' || *buf_ptr == '<' || *buf_ptr == '=') { 528 *e_token++ = *buf_ptr; 529 if (++buf_ptr >= buf_end) 530 fill_buffer(); 531 } 532 if (*buf_ptr == '=') 533 *e_token++ = *buf_ptr++; 534 code = (state->last_u_d ? unary_op : binary_op); 535 unary_delim = true; 536 break; 537 538 case '*': 539 unary_delim = true; 540 if (!state->last_u_d) { 541 if (*buf_ptr == '=') 542 *e_token++ = *buf_ptr++; 543 code = binary_op; 544 break; 545 } 546 while (*buf_ptr == '*' || isspace((unsigned char)*buf_ptr)) { 547 if (*buf_ptr == '*') { 548 CHECK_SIZE_TOKEN(1); 549 *e_token++ = *buf_ptr; 550 } 551 if (++buf_ptr >= buf_end) 552 fill_buffer(); 553 } 554 if (ps.in_decl) { 555 char *tp = buf_ptr; 556 557 while (isalpha((unsigned char)*tp) || 558 isspace((unsigned char)*tp)) { 559 if (++tp >= buf_end) 560 fill_buffer(); 561 } 562 if (*tp == '(') 563 ps.procname[0] = ' '; 564 } 565 code = unary_op; 566 break; 567 568 default: 569 if (token[0] == '/' && *buf_ptr == '*') { 570 /* it is start of comment */ 571 *e_token++ = '*'; 572 573 if (++buf_ptr >= buf_end) 574 fill_buffer(); 575 576 code = comment; 577 unary_delim = state->last_u_d; 578 break; 579 } 580 while (*(e_token - 1) == *buf_ptr || *buf_ptr == '=') { 581 /* 582 * handle ||, &&, etc, and also things as in int *****i 583 */ 584 CHECK_SIZE_TOKEN(1); 585 *e_token++ = *buf_ptr; 586 if (++buf_ptr >= buf_end) 587 fill_buffer(); 588 } 589 code = (state->last_u_d ? unary_op : binary_op); 590 unary_delim = true; 591 592 593 } /* end of switch */ 594 if (buf_ptr >= buf_end) /* check for input buffer empty */ 595 fill_buffer(); 596 state->last_u_d = unary_delim; 597 CHECK_SIZE_TOKEN(1); 598 *e_token = '\0'; /* null terminate the token */ 599 return (code); 600 } 601 602 /* Initialize constant transition table */ 603 void 604 init_constant_tt(void) 605 { 606 table['-'] = table['+']; 607 table['8'] = table['9']; 608 table['2'] = table['3'] = table['4'] = table['5'] = table['6'] = table['7']; 609 table['A'] = table['C'] = table['D'] = table['c'] = table['d'] = table['a']; 610 table['B'] = table['b']; 611 table['E'] = table['e']; 612 table['U'] = table['u']; 613 table['X'] = table['x']; 614 table['P'] = table['p']; 615 table['F'] = table['f']; 616 } 617 618 void 619 alloc_typenames(void) 620 { 621 622 typenames = (const char **)malloc(sizeof(typenames[0]) * 623 (typename_count = 16)); 624 if (typenames == NULL) 625 err(1, NULL); 626 } 627 628 void 629 add_typename(const char *key) 630 { 631 int comparison; 632 const char *copy; 633 634 if (typename_top + 1 >= typename_count) { 635 typenames = realloc((void *)typenames, 636 sizeof(typenames[0]) * (typename_count *= 2)); 637 if (typenames == NULL) 638 err(1, NULL); 639 } 640 if (typename_top == -1) 641 typenames[++typename_top] = copy = strdup(key); 642 else if ((comparison = strcmp(key, typenames[typename_top])) >= 0) { 643 /* take advantage of sorted input */ 644 if (comparison == 0) /* remove duplicates */ 645 return; 646 typenames[++typename_top] = copy = strdup(key); 647 } 648 else { 649 int p; 650 651 for (p = 0; (comparison = strcmp(key, typenames[p])) > 0; p++) 652 /* find place for the new key */; 653 if (comparison == 0) /* remove duplicates */ 654 return; 655 memmove(&typenames[p + 1], &typenames[p], 656 sizeof(typenames[0]) * (++typename_top - p)); 657 typenames[p] = copy = strdup(key); 658 } 659 660 if (copy == NULL) 661 err(1, NULL); 662 } 663
Indexes created Wed Sep 24 15:09:51 GMT 2025