lexi.c revision 1.49 1 /* $NetBSD: lexi.c,v 1.49 2021/09/25 07:59:52 rillig Exp $ */
2
3 /*-
4 * SPDX-License-Identifier: BSD-4-Clause
5 *
6 * Copyright (c) 1985 Sun Microsystems, Inc.
7 * Copyright (c) 1980, 1993
8 * The Regents of the University of California. All rights reserved.
9 * All rights reserved.
10 *
11 * Redistribution and use in source and binary forms, with or without
12 * modification, are permitted provided that the following conditions
13 * are met:
14 * 1. Redistributions of source code must retain the above copyright
15 * notice, this list of conditions and the following disclaimer.
16 * 2. Redistributions in binary form must reproduce the above copyright
17 * notice, this list of conditions and the following disclaimer in the
18 * documentation and/or other materials provided with the distribution.
19 * 3. All advertising materials mentioning features or use of this software
20 * must display the following acknowledgement:
21 * This product includes software developed by the University of
22 * California, Berkeley and its contributors.
23 * 4. Neither the name of the University nor the names of its contributors
24 * may be used to endorse or promote products derived from this software
25 * without specific prior written permission.
26 *
27 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
28 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
29 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
30 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
31 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
32 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
33 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
34 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
35 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
36 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
37 * SUCH DAMAGE.
38 */
39
40 #if 0
41 #ifndef lint
42 static char sccsid[] = "@(#)lexi.c 8.1 (Berkeley) 6/6/93";
43 #endif /* not lint */
44 #endif
45
46 #include <sys/cdefs.h>
47 #ifndef lint
48 #if defined(__NetBSD__)
49 __RCSID("$NetBSD: lexi.c,v 1.49 2021/09/25 07:59:52 rillig Exp $");
50 #elif defined(__FreeBSD__)
51 __FBSDID("$FreeBSD: head/usr.bin/indent/lexi.c 337862 2018-08-15 18:19:45Z pstef $");
52 #endif
53 #endif
54
55 #include <assert.h>
56 #include <err.h>
57 #include <stdio.h>
58 #include <ctype.h>
59 #include <stdlib.h>
60 #include <string.h>
61 #include <sys/param.h>
62
63 #include "indent.h"
64
65 struct templ {
66 const char *rwd;
67 enum rwcode rwcode;
68 };
69
70 /*
71 * This table has to be sorted alphabetically, because it'll be used in binary
72 * search.
73 */
74 const struct templ specials[] =
75 {
76 {"_Bool", rw_type},
77 {"_Complex", rw_type},
78 {"_Imaginary", rw_type},
79 {"auto", rw_storage_class},
80 {"bool", rw_type},
81 {"break", rw_jump},
82 {"case", rw_case_or_default},
83 {"char", rw_type},
84 {"complex", rw_type},
85 {"const", rw_type},
86 {"continue", rw_jump},
87 {"default", rw_case_or_default},
88 {"do", rw_do_or_else},
89 {"double", rw_type},
90 {"else", rw_do_or_else},
91 {"enum", rw_struct_or_union_or_enum},
92 {"extern", rw_storage_class},
93 {"float", rw_type},
94 {"for", rw_for_or_if_or_while},
95 {"global", rw_type},
96 {"goto", rw_jump},
97 {"if", rw_for_or_if_or_while},
98 {"imaginary", rw_type},
99 {"inline", rw_inline_or_restrict},
100 {"int", rw_type},
101 {"long", rw_type},
102 {"offsetof", rw_offsetof},
103 {"register", rw_storage_class},
104 {"restrict", rw_inline_or_restrict},
105 {"return", rw_jump},
106 {"short", rw_type},
107 {"signed", rw_type},
108 {"sizeof", rw_sizeof},
109 {"static", rw_storage_class},
110 {"struct", rw_struct_or_union_or_enum},
111 {"switch", rw_switch},
112 {"typedef", rw_typedef},
113 {"union", rw_struct_or_union_or_enum},
114 {"unsigned", rw_type},
115 {"void", rw_type},
116 {"volatile", rw_type},
117 {"while", rw_for_or_if_or_while}
118 };
119
120 const char **typenames;
121 int typename_count;
122 int typename_top = -1;
123
124 /*
125 * The transition table below was rewritten by hand from lx's output, given
126 * the following definitions. lx is Katherine Flavel's lexer generator.
127 *
128 * O = /[0-7]/; D = /[0-9]/; NZ = /[1-9]/;
129 * H = /[a-f0-9]/i; B = /[0-1]/; HP = /0x/i;
130 * BP = /0b/i; E = /e[+\-]?/i D+; P = /p[+\-]?/i D+;
131 * FS = /[fl]/i; IS = /u/i /(l|L|ll|LL)/? | /(l|L|ll|LL)/ /u/i?;
132 *
133 * D+ E FS? -> $float;
134 * D* "." D+ E? FS? -> $float;
135 * D+ "." E? FS? -> $float; HP H+ IS? -> $int;
136 * HP H+ P FS? -> $float; NZ D* IS? -> $int;
137 * HP H* "." H+ P FS? -> $float; "0" O* IS? -> $int;
138 * HP H+ "." P FS -> $float; BP B+ IS? -> $int;
139 */
140 static char const *table[] = {
141 /* examples:
142 00
143 s 0xx
144 t 00xaa
145 a 11 101100xxa..
146 r 11ee0001101lbuuxx.a.pp
147 t.01.e+008bLuxll0Ll.aa.p+0
148 states: ABCDEFGHIJKLMNOPQRSTUVWXYZ */
149 ['0'] = "CEIDEHHHIJQ U Q VUVVZZZ",
150 ['1'] = "DEIDEHHHIJQ U Q VUVVZZZ",
151 ['7'] = "DEIDEHHHIJ U VUVVZZZ",
152 ['9'] = "DEJDEHHHJJ U VUVVZZZ",
153 ['a'] = " U VUVV ",
154 ['b'] = " K U VUVV ",
155 ['e'] = " FFF FF U VUVV ",
156 ['f'] = " f f U VUVV f",
157 ['u'] = " MM M i iiM M ",
158 ['x'] = " N ",
159 ['p'] = " FFX ",
160 ['L'] = " LLf fL PR Li L f",
161 ['l'] = " OOf fO S P O i O f",
162 ['+'] = " G Y ",
163 ['.'] = "B EE EE T W ",
164 /* ABCDEFGHIJKLMNOPQRSTUVWXYZ */
165 [0] = "uuiifuufiuuiiuiiiiiuiuuuuu",
166 };
167
168 /* Initialize constant transition table */
169 void
170 init_constant_tt(void)
171 {
172 table['-'] = table['+'];
173 table['8'] = table['9'];
174 table['2'] = table['3'] = table['4'] = table['5'] = table['6'] = table['7'];
175 table['A'] = table['C'] = table['D'] = table['c'] = table['d'] = table['a'];
176 table['B'] = table['b'];
177 table['E'] = table['e'];
178 table['U'] = table['u'];
179 table['X'] = table['x'];
180 table['P'] = table['p'];
181 table['F'] = table['f'];
182 }
183
184 static char
185 inbuf_peek(void)
186 {
187 return *buf_ptr;
188 }
189
190 static void
191 inbuf_skip(void)
192 {
193 buf_ptr++;
194 if (buf_ptr >= buf_end)
195 fill_buffer();
196 }
197
198 static char
199 inbuf_next(void)
200 {
201 char ch = inbuf_peek();
202 inbuf_skip();
203 return ch;
204 }
205
206 static void
207 check_size_token(size_t desired_size)
208 {
209 if (e_token + (desired_size) < l_token)
210 return;
211
212 size_t nsize = l_token - s_token + 400 + desired_size;
213 size_t token_len = e_token - s_token;
214 tokenbuf = realloc(tokenbuf, nsize);
215 if (tokenbuf == NULL)
216 err(1, NULL);
217 e_token = tokenbuf + token_len + 1;
218 l_token = tokenbuf + nsize - 5;
219 s_token = tokenbuf + 1;
220 }
221
222 static int
223 compare_templ_array(const void *key, const void *elem)
224 {
225 return strcmp(key, ((const struct templ *)elem)->rwd);
226 }
227
228 static int
229 compare_string_array(const void *key, const void *elem)
230 {
231 return strcmp(key, *((const char *const *)elem));
232 }
233
234 #ifdef debug
235 const char *
236 token_type_name(token_type ttype)
237 {
238 static const char *const name[] = {
239 "end_of_file", "newline", "lparen", "rparen", "unary_op",
240 "binary_op", "postfix_op", "question", "case_label", "colon",
241 "semicolon", "lbrace", "rbrace", "ident", "comma",
242 "comment", "switch_expr", "preprocessing", "form_feed", "decl",
243 "keyword_for_if_while", "keyword_do_else",
244 "if_expr", "while_expr", "for_exprs",
245 "stmt", "stmt_list", "keyword_else", "keyword_do", "do_stmt",
246 "if_expr_stmt", "if_expr_stmt_else", "period", "string_prefix",
247 "storage_class", "funcname", "type_def", "keyword_struct_union_enum"
248 };
249
250 assert(0 <= ttype && ttype < sizeof name / sizeof name[0]);
251
252 return name[ttype];
253 }
254
255 static void
256 print_buf(const char *name, const char *s, const char *e)
257 {
258 if (s < e) {
259 debug_printf(" %s ", name);
260 debug_vis_range("\"", s, e, "\"");
261 }
262 }
263
264 static token_type
265 lexi_end(token_type ttype)
266 {
267 debug_printf("in line %d, lexi returns '%s'",
268 line_no, token_type_name(ttype));
269 print_buf("token", s_token, e_token);
270 print_buf("label", lab.s, lab.e);
271 print_buf("code", code.s, code.e);
272 print_buf("comment", com.s, com.e);
273 debug_printf("\n");
274
275 return ttype;
276 }
277 #else
278 # define lexi_end(tk) (tk)
279 #endif
280
281 static void
282 lex_number(void)
283 {
284 char s;
285 unsigned char i;
286
287 for (s = 'A'; s != 'f' && s != 'i' && s != 'u'; ) {
288 i = (unsigned char)*buf_ptr;
289 if (i >= nitems(table) || table[i] == NULL ||
290 table[i][s - 'A'] == ' ') {
291 s = table[0][s - 'A'];
292 break;
293 }
294 s = table[i][s - 'A'];
295 check_size_token(1);
296 *e_token++ = inbuf_next();
297 }
298 /* s now indicates the type: f(loating), i(integer), u(nknown) */
299 }
300
301 static void
302 lex_word(void)
303 {
304 while (isalnum((unsigned char)*buf_ptr) ||
305 *buf_ptr == '\\' ||
306 *buf_ptr == '_' || *buf_ptr == '$') {
307 /* fill_buffer() terminates buffer with newline */
308 if (*buf_ptr == '\\') {
309 if (buf_ptr[1] == '\n') {
310 buf_ptr += 2;
311 if (buf_ptr >= buf_end)
312 fill_buffer();
313 } else
314 break;
315 }
316 check_size_token(1);
317 *e_token++ = inbuf_next();
318 }
319 }
320
321 static void
322 lex_char_or_string(void)
323 {
324 char delim;
325
326 delim = *s_token;
327 do { /* copy the string */
328 for (;;) { /* move one character or [/<char>]<char> */
329 if (*buf_ptr == '\n') {
330 diag(1, "Unterminated literal");
331 return;
332 }
333 check_size_token(2);
334 *e_token = inbuf_next();
335 if (*e_token == '\\') { /* if escape, copy extra char */
336 if (*buf_ptr == '\n') /* check for escaped newline */
337 ++line_no;
338 *++e_token = inbuf_next();
339 ++e_token; /* we must increment this again because we
340 * copied two chars */
341 } else
342 break; /* we copied one character */
343 } /* end of while (1) */
344 } while (*e_token++ != delim);
345 }
346
347 /* Reads the next token, placing it in the global variable "token". */
348 token_type
349 lexi(struct parser_state *state)
350 {
351 int unary_delim; /* this is set to 1 if the current token
352 * forces a following operator to be unary */
353 token_type ttype;
354
355 e_token = s_token; /* point to start of place to save token */
356 unary_delim = false;
357 state->col_1 = state->last_nl; /* tell world that this token started
358 * in column 1 iff the last thing
359 * scanned was a newline */
360 state->last_nl = false;
361
362 while (*buf_ptr == ' ' || *buf_ptr == '\t') { /* get rid of blanks */
363 state->col_1 = false; /* leading blanks imply token is not in column
364 * 1 */
365 inbuf_skip();
366 }
367
368 /* Scan an alphanumeric token */
369 if (isalnum((unsigned char)*buf_ptr) ||
370 *buf_ptr == '_' || *buf_ptr == '$' ||
371 (buf_ptr[0] == '.' && isdigit((unsigned char)buf_ptr[1]))) {
372 /*
373 * we have a letter or number
374 */
375 struct templ *p;
376
377 if (isdigit((unsigned char)*buf_ptr) ||
378 (buf_ptr[0] == '.' && isdigit((unsigned char)buf_ptr[1]))) {
379 lex_number();
380 } else {
381 lex_word();
382 }
383 *e_token = '\0';
384
385 if (s_token[0] == 'L' && s_token[1] == '\0' &&
386 (*buf_ptr == '"' || *buf_ptr == '\''))
387 return lexi_end(string_prefix);
388
389 while (*buf_ptr == ' ' || *buf_ptr == '\t') /* get rid of blanks */
390 inbuf_next();
391 state->keyword = rw_0;
392 if (state->last_token == keyword_struct_union_enum &&
393 !state->p_l_follow) {
394 /* if last token was 'struct' and we're not in parentheses, then
395 * this token should be treated as a declaration */
396 state->last_u_d = true;
397 return lexi_end(decl);
398 }
399 /*
400 * Operator after identifier is binary unless last token was 'struct'
401 */
402 state->last_u_d = (state->last_token == keyword_struct_union_enum);
403
404 p = bsearch(s_token, specials, sizeof specials / sizeof specials[0],
405 sizeof specials[0], compare_templ_array);
406 if (p == NULL) { /* not a special keyword... */
407 char *u;
408
409 /* ... so maybe a type_t or a typedef */
410 if ((opt.auto_typedefs && ((u = strrchr(s_token, '_')) != NULL) &&
411 strcmp(u, "_t") == 0) || (typename_top >= 0 &&
412 bsearch(s_token, typenames, (size_t)typename_top + 1,
413 sizeof typenames[0], compare_string_array))) {
414 state->keyword = rw_type;
415 state->last_u_d = true;
416 goto found_typename;
417 }
418 } else { /* we have a keyword */
419 state->keyword = p->rwcode;
420 state->last_u_d = true;
421 switch (p->rwcode) {
422 case rw_switch:
423 return lexi_end(switch_expr);
424 case rw_case_or_default:
425 return lexi_end(case_label);
426 case rw_struct_or_union_or_enum:
427 case rw_type:
428 found_typename:
429 if (state->p_l_follow) {
430 /* inside parens: cast, param list, offsetof or sizeof */
431 state->cast_mask |= (1 << state->p_l_follow) & ~state->not_cast_mask;
432 }
433 if (state->last_token == period || state->last_token == unary_op) {
434 state->keyword = rw_0;
435 break;
436 }
437 if (p != NULL && p->rwcode == rw_struct_or_union_or_enum)
438 return lexi_end(keyword_struct_union_enum);
439 if (state->p_l_follow)
440 break;
441 return lexi_end(decl);
442
443 case rw_for_or_if_or_while:
444 return lexi_end(keyword_for_if_while);
445
446 case rw_do_or_else:
447 return lexi_end(keyword_do_else);
448
449 case rw_storage_class:
450 return lexi_end(storage_class);
451
452 case rw_typedef:
453 return lexi_end(type_def);
454
455 default: /* all others are treated like any other
456 * identifier */
457 return lexi_end(ident);
458 } /* end of switch */
459 } /* end of if (found_it) */
460 if (*buf_ptr == '(' && state->tos <= 1 && state->ind_level == 0 &&
461 state->in_parameter_declaration == 0 && state->block_init == 0) {
462 char *tp = buf_ptr;
463 while (tp < buf_end)
464 if (*tp++ == ')' && (*tp == ';' || *tp == ','))
465 goto not_proc;
466 strncpy(state->procname, s_token, sizeof state->procname - 1);
467 if (state->in_decl)
468 state->in_parameter_declaration = 1;
469 return lexi_end(funcname);
470 not_proc:;
471 }
472 /*
473 * The following hack attempts to guess whether or not the current
474 * token is in fact a declaration keyword -- one that has been
475 * typedefd
476 */
477 else if (!state->p_l_follow && !state->block_init &&
478 !state->in_stmt &&
479 ((*buf_ptr == '*' && buf_ptr[1] != '=') ||
480 isalpha((unsigned char)*buf_ptr)) &&
481 (state->last_token == semicolon || state->last_token == lbrace ||
482 state->last_token == rbrace)) {
483 state->keyword = rw_type;
484 state->last_u_d = true;
485 return lexi_end(decl);
486 }
487 if (state->last_token == decl) /* if this is a declared variable,
488 * then following sign is unary */
489 state->last_u_d = true; /* will make "int a -1" work */
490 return lexi_end(ident); /* the ident is not in the list */
491 } /* end of procesing for alpanum character */
492
493 /* Scan a non-alphanumeric token */
494
495 check_size_token(3); /* things like "<<=" */
496 *e_token++ = inbuf_next(); /* if it is only a one-character token, it is
497 * moved here */
498 *e_token = '\0';
499
500 switch (*s_token) {
501 case '\n':
502 unary_delim = state->last_u_d;
503 state->last_nl = true; /* remember that we just had a newline */
504 /* if data has been exhausted, the newline is a dummy. */
505 ttype = had_eof ? end_of_file : newline;
506 break;
507
508 case '\'':
509 case '"':
510 lex_char_or_string();
511 ttype = ident;
512 break;
513
514 case '(':
515 case '[':
516 unary_delim = true;
517 ttype = lparen;
518 break;
519
520 case ')':
521 case ']':
522 ttype = rparen;
523 break;
524
525 case '#':
526 unary_delim = state->last_u_d;
527 ttype = preprocessing;
528 break;
529
530 case '?':
531 unary_delim = true;
532 ttype = question;
533 break;
534
535 case ':':
536 ttype = colon;
537 unary_delim = true;
538 break;
539
540 case ';':
541 unary_delim = true;
542 ttype = semicolon;
543 break;
544
545 case '{':
546 unary_delim = true;
547 ttype = lbrace;
548 break;
549
550 case '}':
551 unary_delim = true;
552 ttype = rbrace;
553 break;
554
555 case 014: /* a form feed */
556 unary_delim = state->last_u_d;
557 state->last_nl = true; /* remember this so we can set 'state->col_1'
558 * right */
559 ttype = form_feed;
560 break;
561
562 case ',':
563 unary_delim = true;
564 ttype = comma;
565 break;
566
567 case '.':
568 unary_delim = false;
569 ttype = period;
570 break;
571
572 case '-':
573 case '+': /* check for -, +, --, ++ */
574 ttype = state->last_u_d ? unary_op : binary_op;
575 unary_delim = true;
576
577 if (*buf_ptr == s_token[0]) {
578 /* check for doubled character */
579 *e_token++ = *buf_ptr++;
580 /* buffer overflow will be checked at end of loop */
581 if (state->last_token == ident || state->last_token == rparen) {
582 ttype = state->last_u_d ? unary_op : postfix_op;
583 /* check for following ++ or -- */
584 unary_delim = false;
585 }
586 } else if (*buf_ptr == '=')
587 /* check for operator += */
588 *e_token++ = *buf_ptr++;
589 else if (*buf_ptr == '>') {
590 /* check for operator -> */
591 *e_token++ = *buf_ptr++;
592 unary_delim = false;
593 ttype = unary_op;
594 state->want_blank = false;
595 }
596 break; /* buffer overflow will be checked at end of
597 * switch */
598
599 case '=':
600 if (state->in_or_st)
601 state->block_init = 1;
602 if (*buf_ptr == '=') { /* == */
603 *e_token++ = '='; /* Flip =+ to += */
604 buf_ptr++;
605 *e_token = 0;
606 }
607 ttype = binary_op;
608 unary_delim = true;
609 break;
610 /* can drop thru!!! */
611
612 case '>':
613 case '<':
614 case '!': /* ops like <, <<, <=, !=, etc */
615 if (*buf_ptr == '>' || *buf_ptr == '<' || *buf_ptr == '=')
616 *e_token++ = inbuf_next();
617 if (*buf_ptr == '=')
618 *e_token++ = *buf_ptr++;
619 ttype = state->last_u_d ? unary_op : binary_op;
620 unary_delim = true;
621 break;
622
623 case '*':
624 unary_delim = true;
625 if (!state->last_u_d) {
626 if (*buf_ptr == '=')
627 *e_token++ = *buf_ptr++;
628 ttype = binary_op;
629 break;
630 }
631 while (*buf_ptr == '*' || isspace((unsigned char)*buf_ptr)) {
632 if (*buf_ptr == '*') {
633 check_size_token(1);
634 *e_token++ = *buf_ptr;
635 }
636 inbuf_skip();
637 }
638 if (ps.in_decl) {
639 char *tp = buf_ptr;
640
641 while (isalpha((unsigned char)*tp) ||
642 isspace((unsigned char)*tp)) {
643 if (++tp >= buf_end)
644 fill_buffer();
645 }
646 if (*tp == '(')
647 ps.procname[0] = ' ';
648 }
649 ttype = unary_op;
650 break;
651
652 default:
653 if (s_token[0] == '/' && (*buf_ptr == '*' || *buf_ptr == '/')) {
654 /* it is start of comment */
655 *e_token++ = inbuf_next();
656
657 ttype = comment;
658 unary_delim = state->last_u_d;
659 break;
660 }
661 while (e_token[-1] == *buf_ptr || *buf_ptr == '=') {
662 /*
663 * handle ||, &&, etc, and also things as in int *****i
664 */
665 check_size_token(1);
666 *e_token++ = inbuf_next();
667 }
668 ttype = state->last_u_d ? unary_op : binary_op;
669 unary_delim = true;
670 }
671
672 if (buf_ptr >= buf_end) /* check for input buffer empty */
673 fill_buffer();
674 state->last_u_d = unary_delim;
675 check_size_token(1);
676 *e_token = '\0';
677 return lexi_end(ttype);
678 }
679
680 void
681 alloc_typenames(void)
682 {
683
684 typenames = malloc(sizeof(typenames[0]) * (typename_count = 16));
685 if (typenames == NULL)
686 err(1, NULL);
687 }
688
689 void
690 add_typename(const char *key)
691 {
692 int comparison;
693 const char *copy;
694
695 if (typename_top + 1 >= typename_count) {
696 typenames = realloc((void *)typenames,
697 sizeof(typenames[0]) * (typename_count *= 2));
698 if (typenames == NULL)
699 err(1, NULL);
700 }
701 if (typename_top == -1)
702 typenames[++typename_top] = copy = strdup(key);
703 else if ((comparison = strcmp(key, typenames[typename_top])) >= 0) {
704 /* take advantage of sorted input */
705 if (comparison == 0) /* remove duplicates */
706 return;
707 typenames[++typename_top] = copy = strdup(key);
708 } else {
709 int p;
710
711 for (p = 0; (comparison = strcmp(key, typenames[p])) > 0; p++)
712 /* find place for the new key */;
713 if (comparison == 0) /* remove duplicates */
714 return;
715 memmove(&typenames[p + 1], &typenames[p],
716 sizeof(typenames[0]) * (++typename_top - p));
717 typenames[p] = copy = strdup(key);
718 }
719
720 if (copy == NULL)
721 err(1, NULL);
722 }
723