Home | History | Annotate | Line # | Download | only in indent
lexi.c revision 1.30
      1 /*	$NetBSD: lexi.c,v 1.30 2021/03/09 19:14:39 rillig Exp $	*/
      2 
      3 /*-
      4  * SPDX-License-Identifier: BSD-4-Clause
      5  *
      6  * Copyright (c) 1985 Sun Microsystems, Inc.
      7  * Copyright (c) 1980, 1993
      8  *	The Regents of the University of California.  All rights reserved.
      9  * All rights reserved.
     10  *
     11  * Redistribution and use in source and binary forms, with or without
     12  * modification, are permitted provided that the following conditions
     13  * are met:
     14  * 1. Redistributions of source code must retain the above copyright
     15  *    notice, this list of conditions and the following disclaimer.
     16  * 2. Redistributions in binary form must reproduce the above copyright
     17  *    notice, this list of conditions and the following disclaimer in the
     18  *    documentation and/or other materials provided with the distribution.
     19  * 3. All advertising materials mentioning features or use of this software
     20  *    must display the following acknowledgement:
     21  *	This product includes software developed by the University of
     22  *	California, Berkeley and its contributors.
     23  * 4. Neither the name of the University nor the names of its contributors
     24  *    may be used to endorse or promote products derived from this software
     25  *    without specific prior written permission.
     26  *
     27  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
     28  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
     29  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
     30  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
     31  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
     32  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
     33  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
     34  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
     35  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
     36  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
     37  * SUCH DAMAGE.
     38  */
     39 
     40 #if 0
     41 #ifndef lint
     42 static char sccsid[] = "@(#)lexi.c	8.1 (Berkeley) 6/6/93";
     43 #endif /* not lint */
     44 #endif
     45 
     46 #include <sys/cdefs.h>
     47 #ifndef lint
     48 #if defined(__NetBSD__)
     49 __RCSID("$NetBSD: lexi.c,v 1.30 2021/03/09 19:14:39 rillig Exp $");
     50 #elif defined(__FreeBSD__)
     51 __FBSDID("$FreeBSD: head/usr.bin/indent/lexi.c 337862 2018-08-15 18:19:45Z pstef $");
     52 #endif
     53 #endif
     54 
     55 /*
     56  * Here we have the token scanner for indent.  It scans off one token and puts
     57  * it in the global variable "token".  It returns a code, indicating the type
     58  * of token scanned.
     59  */
     60 
     61 #include <assert.h>
     62 #include <err.h>
     63 #include <stdio.h>
     64 #include <ctype.h>
     65 #include <stdlib.h>
     66 #include <string.h>
     67 #include <sys/param.h>
     68 
     69 #include "indent.h"
     70 
     71 struct templ {
     72     const char *rwd;
     73     enum rwcode rwcode;
     74 };
     75 
     76 /*
     77  * This table has to be sorted alphabetically, because it'll be used in binary
     78  * search.
     79  */
     80 const struct templ specials[] =
     81 {
     82     {"_Bool", rw_type},
     83     {"_Complex", rw_type},
     84     {"_Imaginary", rw_type},
     85     {"auto", rw_storage_class},
     86     {"bool", rw_type},
     87     {"break", rw_jump},
     88     {"case", rw_case_or_default},
     89     {"char", rw_type},
     90     {"complex", rw_type},
     91     {"const", rw_type},
     92     {"continue", rw_jump},
     93     {"default", rw_case_or_default},
     94     {"do", rw_do_or_else},
     95     {"double", rw_type},
     96     {"else", rw_do_or_else},
     97     {"enum", rw_struct_or_union_or_enum},
     98     {"extern", rw_storage_class},
     99     {"float", rw_type},
    100     {"for", rw_for_or_if_or_while},
    101     {"global", rw_type},
    102     {"goto", rw_jump},
    103     {"if", rw_for_or_if_or_while},
    104     {"imaginary", rw_type},
    105     {"inline", rw_inline_or_restrict},
    106     {"int", rw_type},
    107     {"long", rw_type},
    108     {"offsetof", rw_offsetof},
    109     {"register", rw_storage_class},
    110     {"restrict", rw_inline_or_restrict},
    111     {"return", rw_jump},
    112     {"short", rw_type},
    113     {"signed", rw_type},
    114     {"sizeof", rw_sizeof},
    115     {"static", rw_storage_class},
    116     {"struct", rw_struct_or_union_or_enum},
    117     {"switch", rw_switch},
    118     {"typedef", rw_typedef},
    119     {"union", rw_struct_or_union_or_enum},
    120     {"unsigned", rw_type},
    121     {"void", rw_type},
    122     {"volatile", rw_type},
    123     {"while", rw_for_or_if_or_while}
    124 };
    125 
    126 const char **typenames;
    127 int         typename_count;
    128 int         typename_top = -1;
    129 
    130 /*
    131  * The transition table below was rewritten by hand from lx's output, given
    132  * the following definitions. lx is Katherine Flavel's lexer generator.
    133  *
    134  * O  = /[0-7]/;        D  = /[0-9]/;          NZ = /[1-9]/;
    135  * H  = /[a-f0-9]/i;    B  = /[0-1]/;          HP = /0x/i;
    136  * BP = /0b/i;          E  = /e[+\-]?/i D+;    P  = /p[+\-]?/i D+;
    137  * FS = /[fl]/i;        IS = /u/i /(l|L|ll|LL)/? | /(l|L|ll|LL)/ /u/i?;
    138  *
    139  * D+           E  FS? -> $float;
    140  * D*    "." D+ E? FS? -> $float;
    141  * D+    "."    E? FS? -> $float;    HP H+           IS? -> $int;
    142  * HP H+        P  FS? -> $float;    NZ D*           IS? -> $int;
    143  * HP H* "." H+ P  FS? -> $float;    "0" O*          IS? -> $int;
    144  * HP H+ "."    P  FS  -> $float;    BP B+           IS? -> $int;
    145  */
    146 static char const *table[] = {
    147     /*                examples:
    148                                      00
    149              s                      0xx
    150              t                    00xaa
    151              a     11       101100xxa..
    152              r   11ee0001101lbuuxx.a.pp
    153              t.01.e+008bLuxll0Ll.aa.p+0
    154     states:  ABCDEFGHIJKLMNOPQRSTUVWXYZ */
    155     ['0'] = "CEIDEHHHIJQ  U  Q  VUVVZZZ",
    156     ['1'] = "DEIDEHHHIJQ  U  Q  VUVVZZZ",
    157     ['7'] = "DEIDEHHHIJ   U     VUVVZZZ",
    158     ['9'] = "DEJDEHHHJJ   U     VUVVZZZ",
    159     ['a'] = "             U     VUVV   ",
    160     ['b'] = "  K          U     VUVV   ",
    161     ['e'] = "  FFF   FF   U     VUVV   ",
    162     ['f'] = "    f  f     U     VUVV  f",
    163     ['u'] = "  MM    M  i  iiM   M     ",
    164     ['x'] = "  N                       ",
    165     ['p'] = "                    FFX   ",
    166     ['L'] = "  LLf  fL  PR   Li  L    f",
    167     ['l'] = "  OOf  fO   S P O i O    f",
    168     ['+'] = "     G                 Y  ",
    169     ['.'] = "B EE    EE   T      W     ",
    170     /*       ABCDEFGHIJKLMNOPQRSTUVWXYZ */
    171     [0]   = "uuiifuufiuuiiuiiiiiuiuuuuu",
    172 };
    173 
    174 static void
    175 check_size_token(size_t desired_size)
    176 {
    177     if (e_token + (desired_size) >= l_token) {
    178 	int nsize = l_token - s_token + 400 + desired_size;
    179 	int token_len = e_token - s_token;
    180 	tokenbuf = (char *)realloc(tokenbuf, nsize);
    181 	if (tokenbuf == NULL)
    182 	    err(1, NULL);
    183 	e_token = tokenbuf + token_len + 1;
    184 	l_token = tokenbuf + nsize - 5;
    185 	s_token = tokenbuf + 1;
    186     }
    187 }
    188 
    189 static int
    190 compare_templ_array(const void *key, const void *elem)
    191 {
    192     return strcmp(key, ((const struct templ *)elem)->rwd);
    193 }
    194 
    195 static int
    196 compare_string_array(const void *key, const void *elem)
    197 {
    198     return strcmp(key, *((const char *const *)elem));
    199 }
    200 
    201 #ifdef debug
    202 const char *
    203 token_type_name(token_type tk)
    204 {
    205     static const char *const name[] = {
    206 	"end_of_file", "newline", "lparen", "rparen", "unary_op",
    207 	"binary_op", "postop", "question", "case_label", "colon",
    208 	"semicolon", "lbrace", "rbrace", "ident", "comma",
    209 	"comment", "switch_expr", "preesc", "form_feed", "decl",
    210 	"keyword_for_if_while", "keyword_do_else",
    211 	"if_expr", "while_expr", "for_exprs",
    212 	"stmt", "stmt_list", "keyword_else", "keyword_do", "do_stmt",
    213 	"if_expr_stmt", "if_expr_stmt_else", "period", "strpfx", "storage",
    214 	"funcname", "type_def", "structure"
    215     };
    216 
    217     assert(0 <= tk && tk < sizeof name / sizeof name[0]);
    218 
    219     return name[tk];
    220 }
    221 
    222 static void
    223 print_buf(const char *name, const char *s, const char *e)
    224 {
    225     if (s == e)
    226 	return;
    227 
    228     printf(" %s \"", name);
    229     for (const char *p = s; p < e; p++) {
    230 	if (isprint((unsigned char)*p) && *p != '\\' && *p != '"')
    231 	    printf("%c", *p);
    232 	else if (*p == '\n')
    233 	    printf("\\n");
    234 	else if (*p == '\t')
    235 	    printf("\\t");
    236 	else
    237 	    printf("\\x%02x", *p);
    238     }
    239     printf("\"");
    240 }
    241 
    242 static token_type
    243 lexi_end(token_type code)
    244 {
    245     printf("in line %d, lexi returns '%s'", line_no, token_type_name(code));
    246     print_buf("token", s_token, e_token);
    247     print_buf("label", s_lab, e_lab);
    248     print_buf("code", s_code, e_code);
    249     print_buf("comment", s_com, e_com);
    250     printf("\n");
    251 
    252     return code;
    253 }
    254 #else
    255 #  define lexi_end(tk) (tk)
    256 #endif
    257 
    258 token_type
    259 lexi(struct parser_state *state)
    260 {
    261     int         unary_delim;	/* this is set to 1 if the current token
    262 				 * forces a following operator to be unary */
    263     token_type  code;		/* internal code to be returned */
    264     char        qchar;		/* the delimiter character for a string */
    265 
    266     e_token = s_token;		/* point to start of place to save token */
    267     unary_delim = false;
    268     state->col_1 = state->last_nl;	/* tell world that this token started
    269 					 * in column 1 iff the last thing
    270 					 * scanned was a newline */
    271     state->last_nl = false;
    272 
    273     while (*buf_ptr == ' ' || *buf_ptr == '\t') {	/* get rid of blanks */
    274 	state->col_1 = false;	/* leading blanks imply token is not in column
    275 				 * 1 */
    276 	if (++buf_ptr >= buf_end)
    277 	    fill_buffer();
    278     }
    279 
    280     /* Scan an alphanumeric token */
    281     if (isalnum((unsigned char)*buf_ptr) ||
    282 	*buf_ptr == '_' || *buf_ptr == '$' ||
    283 	(buf_ptr[0] == '.' && isdigit((unsigned char)buf_ptr[1]))) {
    284 	/*
    285 	 * we have a character or number
    286 	 */
    287 	struct templ *p;
    288 
    289 	if (isdigit((unsigned char)*buf_ptr) ||
    290 	    (buf_ptr[0] == '.' && isdigit((unsigned char)buf_ptr[1]))) {
    291 	    char s;
    292 	    unsigned char i;
    293 
    294 	    for (s = 'A'; s != 'f' && s != 'i' && s != 'u'; ) {
    295 		i = (unsigned char)*buf_ptr;
    296 		if (i >= nitems(table) || table[i] == NULL ||
    297 		    table[i][s - 'A'] == ' ') {
    298 		    s = table[0][s - 'A'];
    299 		    break;
    300 		}
    301 		s = table[i][s - 'A'];
    302 		check_size_token(1);
    303 		*e_token++ = *buf_ptr++;
    304 		if (buf_ptr >= buf_end)
    305 		    fill_buffer();
    306 	    }
    307 	    /* s now indicates the type: f(loating), i(integer), u(nknown) */
    308 	}
    309 	else
    310 	    while (isalnum((unsigned char)*buf_ptr) ||
    311 	        *buf_ptr == '\\' ||
    312 		*buf_ptr == '_' || *buf_ptr == '$') {
    313 		/* fill_buffer() terminates buffer with newline */
    314 		if (*buf_ptr == '\\') {
    315 		    if (*(buf_ptr + 1) == '\n') {
    316 			buf_ptr += 2;
    317 			if (buf_ptr >= buf_end)
    318 			    fill_buffer();
    319 			} else
    320 			    break;
    321 		}
    322 		check_size_token(1);
    323 		/* copy it over */
    324 		*e_token++ = *buf_ptr++;
    325 		if (buf_ptr >= buf_end)
    326 		    fill_buffer();
    327 	    }
    328 	*e_token = '\0';
    329 
    330 	if (s_token[0] == 'L' && s_token[1] == '\0' &&
    331 	      (*buf_ptr == '"' || *buf_ptr == '\''))
    332 	    return lexi_end(strpfx);
    333 
    334 	while (*buf_ptr == ' ' || *buf_ptr == '\t') {	/* get rid of blanks */
    335 	    if (++buf_ptr >= buf_end)
    336 		fill_buffer();
    337 	}
    338 	state->keyword = rw_0;
    339 	if (state->last_token == structure && !state->p_l_follow) {
    340 				/* if last token was 'struct' and we're not
    341 				 * in parentheses, then this token
    342 				 * should be treated as a declaration */
    343 	    state->last_u_d = true;
    344 	    return lexi_end(decl);
    345 	}
    346 	/*
    347 	 * Operator after identifier is binary unless last token was 'struct'
    348 	 */
    349 	state->last_u_d = (state->last_token == structure);
    350 
    351 	p = bsearch(s_token, specials, sizeof specials / sizeof specials[0],
    352 	    sizeof specials[0], compare_templ_array);
    353 	if (p == NULL) {	/* not a special keyword... */
    354 	    char *u;
    355 
    356 	    /* ... so maybe a type_t or a typedef */
    357 	    if ((opt.auto_typedefs && ((u = strrchr(s_token, '_')) != NULL) &&
    358 	        strcmp(u, "_t") == 0) || (typename_top >= 0 &&
    359 		  bsearch(s_token, typenames, typename_top + 1,
    360 		    sizeof typenames[0], compare_string_array))) {
    361 		state->keyword = rw_type;
    362 		state->last_u_d = true;
    363 	        goto found_typename;
    364 	    }
    365 	} else {			/* we have a keyword */
    366 	    state->keyword = p->rwcode;
    367 	    state->last_u_d = true;
    368 	    switch (p->rwcode) {
    369 	    case rw_switch:
    370 		return lexi_end(switch_expr);
    371 	    case rw_case_or_default:
    372 		return lexi_end(case_label);
    373 	    case rw_struct_or_union_or_enum:
    374 	    case rw_type:
    375 	    found_typename:
    376 		if (state->p_l_follow) {
    377 		    /* inside parens: cast, param list, offsetof or sizeof */
    378 		    state->cast_mask |= (1 << state->p_l_follow) & ~state->not_cast_mask;
    379 		}
    380 		if (state->last_token == period || state->last_token == unary_op) {
    381 		    state->keyword = rw_0;
    382 		    break;
    383 		}
    384 		if (p != NULL && p->rwcode == rw_struct_or_union_or_enum)
    385 		    return lexi_end(structure);
    386 		if (state->p_l_follow)
    387 		    break;
    388 		return lexi_end(decl);
    389 
    390 	    case rw_for_or_if_or_while:
    391 		return lexi_end(keyword_for_if_while);
    392 
    393 	    case rw_do_or_else:
    394 		return lexi_end(keyword_do_else);
    395 
    396 	    case rw_storage_class:
    397 		return lexi_end(storage);
    398 
    399 	    case rw_typedef:
    400 		return lexi_end(type_def);
    401 
    402 	    default:		/* all others are treated like any other
    403 				 * identifier */
    404 		return lexi_end(ident);
    405 	    }			/* end of switch */
    406 	}			/* end of if (found_it) */
    407 	if (*buf_ptr == '(' && state->tos <= 1 && state->ind_level == 0 &&
    408 	    state->in_parameter_declaration == 0 && state->block_init == 0) {
    409 	    char *tp = buf_ptr;
    410 	    while (tp < buf_end)
    411 		if (*tp++ == ')' && (*tp == ';' || *tp == ','))
    412 		    goto not_proc;
    413 	    strncpy(state->procname, token, sizeof state->procname - 1);
    414 	    if (state->in_decl)
    415 		state->in_parameter_declaration = 1;
    416 	    return lexi_end(funcname);
    417     not_proc:;
    418 	}
    419 	/*
    420 	 * The following hack attempts to guess whether or not the current
    421 	 * token is in fact a declaration keyword -- one that has been
    422 	 * typedefd
    423 	 */
    424 	else if (!state->p_l_follow && !state->block_init &&
    425 	    !state->in_stmt &&
    426 	    ((*buf_ptr == '*' && buf_ptr[1] != '=') ||
    427 		isalpha((unsigned char)*buf_ptr)) &&
    428 	    (state->last_token == semicolon || state->last_token == lbrace ||
    429 		state->last_token == rbrace)) {
    430 	    state->keyword = rw_type;
    431 	    state->last_u_d = true;
    432 	    return lexi_end(decl);
    433 	}
    434 	if (state->last_token == decl)	/* if this is a declared variable,
    435 					 * then following sign is unary */
    436 	    state->last_u_d = true;	/* will make "int a -1" work */
    437 	return lexi_end(ident);		/* the ident is not in the list */
    438     }				/* end of procesing for alpanum character */
    439 
    440     /* Scan a non-alphanumeric token */
    441 
    442     check_size_token(3);	/* things like "<<=" */
    443     *e_token++ = *buf_ptr;	/* if it is only a one-character token, it is
    444 				 * moved here */
    445     *e_token = '\0';
    446     if (++buf_ptr >= buf_end)
    447 	fill_buffer();
    448 
    449     switch (*token) {
    450     case '\n':
    451 	unary_delim = state->last_u_d;
    452 	state->last_nl = true;	/* remember that we just had a newline */
    453 	code = (had_eof ? end_of_file : newline);
    454 
    455 	/*
    456 	 * if data has been exhausted, the newline is a dummy, and we should
    457 	 * return code to stop
    458 	 */
    459 	break;
    460 
    461     case '\'':			/* start of quoted character */
    462     case '"':			/* start of string */
    463 	qchar = *token;
    464 	do {			/* copy the string */
    465 	    while (1) {		/* move one character or [/<char>]<char> */
    466 		if (*buf_ptr == '\n') {
    467 		    diag(1, "Unterminated literal");
    468 		    goto stop_lit;
    469 		}
    470 		check_size_token(2);
    471 		*e_token = *buf_ptr++;
    472 		if (buf_ptr >= buf_end)
    473 		    fill_buffer();
    474 		if (*e_token == '\\') {		/* if escape, copy extra char */
    475 		    if (*buf_ptr == '\n')	/* check for escaped newline */
    476 			++line_no;
    477 		    *++e_token = *buf_ptr++;
    478 		    ++e_token;	/* we must increment this again because we
    479 				 * copied two chars */
    480 		    if (buf_ptr >= buf_end)
    481 			fill_buffer();
    482 		}
    483 		else
    484 		    break;	/* we copied one character */
    485 	    }			/* end of while (1) */
    486 	} while (*e_token++ != qchar);
    487 stop_lit:
    488 	code = ident;
    489 	break;
    490 
    491     case ('('):
    492     case ('['):
    493 	unary_delim = true;
    494 	code = lparen;
    495 	break;
    496 
    497     case (')'):
    498     case (']'):
    499 	code = rparen;
    500 	break;
    501 
    502     case '#':
    503 	unary_delim = state->last_u_d;
    504 	code = preesc;
    505 	break;
    506 
    507     case '?':
    508 	unary_delim = true;
    509 	code = question;
    510 	break;
    511 
    512     case (':'):
    513 	code = colon;
    514 	unary_delim = true;
    515 	break;
    516 
    517     case (';'):
    518 	unary_delim = true;
    519 	code = semicolon;
    520 	break;
    521 
    522     case ('{'):
    523 	unary_delim = true;
    524 
    525 	/*
    526 	 * if (state->in_or_st) state->block_init = 1;
    527 	 */
    528 	/* ?	code = state->block_init ? lparen : lbrace; */
    529 	code = lbrace;
    530 	break;
    531 
    532     case ('}'):
    533 	unary_delim = true;
    534 	/* ?	code = state->block_init ? rparen : rbrace; */
    535 	code = rbrace;
    536 	break;
    537 
    538     case 014:			/* a form feed */
    539 	unary_delim = state->last_u_d;
    540 	state->last_nl = true;	/* remember this so we can set 'state->col_1'
    541 				 * right */
    542 	code = form_feed;
    543 	break;
    544 
    545     case (','):
    546 	unary_delim = true;
    547 	code = comma;
    548 	break;
    549 
    550     case '.':
    551 	unary_delim = false;
    552 	code = period;
    553 	break;
    554 
    555     case '-':
    556     case '+':			/* check for -, +, --, ++ */
    557 	code = (state->last_u_d ? unary_op : binary_op);
    558 	unary_delim = true;
    559 
    560 	if (*buf_ptr == token[0]) {
    561 	    /* check for doubled character */
    562 	    *e_token++ = *buf_ptr++;
    563 	    /* buffer overflow will be checked at end of loop */
    564 	    if (state->last_token == ident || state->last_token == rparen) {
    565 		code = (state->last_u_d ? unary_op : postop);
    566 		/* check for following ++ or -- */
    567 		unary_delim = false;
    568 	    }
    569 	}
    570 	else if (*buf_ptr == '=')
    571 	    /* check for operator += */
    572 	    *e_token++ = *buf_ptr++;
    573 	else if (*buf_ptr == '>') {
    574 	    /* check for operator -> */
    575 	    *e_token++ = *buf_ptr++;
    576 	    unary_delim = false;
    577 	    code = unary_op;
    578 	    state->want_blank = false;
    579 	}
    580 	break;			/* buffer overflow will be checked at end of
    581 				 * switch */
    582 
    583     case '=':
    584 	if (state->in_or_st)
    585 	    state->block_init = 1;
    586 	if (*buf_ptr == '=') {	/* == */
    587 	    *e_token++ = '=';	/* Flip =+ to += */
    588 	    buf_ptr++;
    589 	    *e_token = 0;
    590 	}
    591 	code = binary_op;
    592 	unary_delim = true;
    593 	break;
    594 	/* can drop thru!!! */
    595 
    596     case '>':
    597     case '<':
    598     case '!':			/* ops like <, <<, <=, !=, etc */
    599 	if (*buf_ptr == '>' || *buf_ptr == '<' || *buf_ptr == '=') {
    600 	    *e_token++ = *buf_ptr;
    601 	    if (++buf_ptr >= buf_end)
    602 		fill_buffer();
    603 	}
    604 	if (*buf_ptr == '=')
    605 	    *e_token++ = *buf_ptr++;
    606 	code = (state->last_u_d ? unary_op : binary_op);
    607 	unary_delim = true;
    608 	break;
    609 
    610     case '*':
    611 	unary_delim = true;
    612 	if (!state->last_u_d) {
    613 	    if (*buf_ptr == '=')
    614 		*e_token++ = *buf_ptr++;
    615 	    code = binary_op;
    616 	    break;
    617 	}
    618 	while (*buf_ptr == '*' || isspace((unsigned char)*buf_ptr)) {
    619 	    if (*buf_ptr == '*') {
    620 		check_size_token(1);
    621 		*e_token++ = *buf_ptr;
    622 	    }
    623 	    if (++buf_ptr >= buf_end)
    624 		fill_buffer();
    625 	}
    626 	if (ps.in_decl) {
    627 	    char *tp = buf_ptr;
    628 
    629 	    while (isalpha((unsigned char)*tp) ||
    630 		   isspace((unsigned char)*tp)) {
    631 		if (++tp >= buf_end)
    632 		    fill_buffer();
    633 	    }
    634 	    if (*tp == '(')
    635 		ps.procname[0] = ' ';
    636 	}
    637 	code = unary_op;
    638 	break;
    639 
    640     default:
    641 	if (token[0] == '/' && (*buf_ptr == '*' || *buf_ptr == '/')) {
    642 	    /* it is start of comment */
    643 	    *e_token++ = *buf_ptr;
    644 
    645 	    if (++buf_ptr >= buf_end)
    646 		fill_buffer();
    647 
    648 	    code = comment;
    649 	    unary_delim = state->last_u_d;
    650 	    break;
    651 	}
    652 	while (*(e_token - 1) == *buf_ptr || *buf_ptr == '=') {
    653 	    /*
    654 	     * handle ||, &&, etc, and also things as in int *****i
    655 	     */
    656 	    check_size_token(1);
    657 	    *e_token++ = *buf_ptr;
    658 	    if (++buf_ptr >= buf_end)
    659 		fill_buffer();
    660 	}
    661 	code = (state->last_u_d ? unary_op : binary_op);
    662 	unary_delim = true;
    663 
    664 
    665     }				/* end of switch */
    666     if (buf_ptr >= buf_end)	/* check for input buffer empty */
    667 	fill_buffer();
    668     state->last_u_d = unary_delim;
    669     check_size_token(1);
    670     *e_token = '\0';		/* null terminate the token */
    671     return lexi_end(code);
    672 }
    673 
    674 /* Initialize constant transition table */
    675 void
    676 init_constant_tt(void)
    677 {
    678     table['-'] = table['+'];
    679     table['8'] = table['9'];
    680     table['2'] = table['3'] = table['4'] = table['5'] = table['6'] = table['7'];
    681     table['A'] = table['C'] = table['D'] = table['c'] = table['d'] = table['a'];
    682     table['B'] = table['b'];
    683     table['E'] = table['e'];
    684     table['U'] = table['u'];
    685     table['X'] = table['x'];
    686     table['P'] = table['p'];
    687     table['F'] = table['f'];
    688 }
    689 
    690 void
    691 alloc_typenames(void)
    692 {
    693 
    694     typenames = (const char **)malloc(sizeof(typenames[0]) *
    695         (typename_count = 16));
    696     if (typenames == NULL)
    697 	err(1, NULL);
    698 }
    699 
    700 void
    701 add_typename(const char *key)
    702 {
    703     int comparison;
    704     const char *copy;
    705 
    706     if (typename_top + 1 >= typename_count) {
    707 	typenames = realloc((void *)typenames,
    708 	    sizeof(typenames[0]) * (typename_count *= 2));
    709 	if (typenames == NULL)
    710 	    err(1, NULL);
    711     }
    712     if (typename_top == -1)
    713 	typenames[++typename_top] = copy = strdup(key);
    714     else if ((comparison = strcmp(key, typenames[typename_top])) >= 0) {
    715 	/* take advantage of sorted input */
    716 	if (comparison == 0)	/* remove duplicates */
    717 	    return;
    718 	typenames[++typename_top] = copy = strdup(key);
    719     }
    720     else {
    721 	int p;
    722 
    723 	for (p = 0; (comparison = strcmp(key, typenames[p])) > 0; p++)
    724 	    /* find place for the new key */;
    725 	if (comparison == 0)	/* remove duplicates */
    726 	    return;
    727 	memmove(&typenames[p + 1], &typenames[p],
    728 	    sizeof(typenames[0]) * (++typename_top - p));
    729 	typenames[p] = copy = strdup(key);
    730     }
    731 
    732     if (copy == NULL)
    733 	err(1, NULL);
    734 }
    735