Home | History | Annotate | Line # | Download | only in sh
parser.c revision 1.110
      1 /*	$NetBSD: parser.c,v 1.110 2016/03/27 14:36:29 christos Exp $	*/
      2 
      3 /*-
      4  * Copyright (c) 1991, 1993
      5  *	The Regents of the University of California.  All rights reserved.
      6  *
      7  * This code is derived from software contributed to Berkeley by
      8  * Kenneth Almquist.
      9  *
     10  * Redistribution and use in source and binary forms, with or without
     11  * modification, are permitted provided that the following conditions
     12  * are met:
     13  * 1. Redistributions of source code must retain the above copyright
     14  *    notice, this list of conditions and the following disclaimer.
     15  * 2. Redistributions in binary form must reproduce the above copyright
     16  *    notice, this list of conditions and the following disclaimer in the
     17  *    documentation and/or other materials provided with the distribution.
     18  * 3. Neither the name of the University nor the names of its contributors
     19  *    may be used to endorse or promote products derived from this software
     20  *    without specific prior written permission.
     21  *
     22  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
     23  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
     24  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
     25  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
     26  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
     27  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
     28  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
     29  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
     30  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
     31  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
     32  * SUCH DAMAGE.
     33  */
     34 
     35 #include <sys/cdefs.h>
     36 #ifndef lint
     37 #if 0
     38 static char sccsid[] = "@(#)parser.c	8.7 (Berkeley) 5/16/95";
     39 #else
     40 __RCSID("$NetBSD: parser.c,v 1.110 2016/03/27 14:36:29 christos Exp $");
     41 #endif
     42 #endif /* not lint */
     43 
     44 #include <stdio.h>
     45 #include <stdlib.h>
     46 #include <limits.h>
     47 
     48 #include "shell.h"
     49 #include "parser.h"
     50 #include "nodes.h"
     51 #include "expand.h"	/* defines rmescapes() */
     52 #include "eval.h"	/* defines commandname */
     53 #include "redir.h"	/* defines copyfd() */
     54 #include "syntax.h"
     55 #include "options.h"
     56 #include "input.h"
     57 #include "output.h"
     58 #include "var.h"
     59 #include "error.h"
     60 #include "memalloc.h"
     61 #include "mystring.h"
     62 #include "alias.h"
     63 #include "show.h"
     64 #ifndef SMALL
     65 #include "myhistedit.h"
     66 #endif
     67 
     68 /*
     69  * Shell command parser.
     70  */
     71 
     72 #define EOFMARKLEN 79
     73 
     74 /* values returned by readtoken */
     75 #include "token.h"
     76 
     77 #define OPENBRACE '{'
     78 #define CLOSEBRACE '}'
     79 
     80 
     81 struct heredoc {
     82 	struct heredoc *next;	/* next here document in list */
     83 	union node *here;		/* redirection node */
     84 	char *eofmark;		/* string indicating end of input */
     85 	int striptabs;		/* if set, strip leading tabs */
     86 };
     87 
     88 
     89 
     90 static int noalias = 0;		/* when set, don't handle aliases */
     91 struct heredoc *heredoclist;	/* list of here documents to read */
     92 int parsebackquote;		/* nonzero if we are inside backquotes */
     93 int doprompt;			/* if set, prompt the user */
     94 int needprompt;			/* true if interactive and at start of line */
     95 int lasttoken;			/* last token read */
     96 MKINIT int tokpushback;		/* last token pushed back */
     97 char *wordtext;			/* text of last word returned by readtoken */
     98 MKINIT int checkkwd;		/* 1 == check for kwds, 2 == also eat newlines */
     99 struct nodelist *backquotelist;
    100 union node *redirnode;
    101 struct heredoc *heredoc;
    102 int quoteflag;			/* set if (part of) last token was quoted */
    103 int startlinno;			/* line # where last token started */
    104 int funclinno;			/* line # where the current function started */
    105 
    106 
    107 STATIC union node *list(int, int);
    108 STATIC union node *andor(void);
    109 STATIC union node *pipeline(void);
    110 STATIC union node *command(void);
    111 STATIC union node *simplecmd(union node **, union node *);
    112 STATIC union node *makename(void);
    113 STATIC void parsefname(void);
    114 STATIC void parseheredoc(void);
    115 STATIC int peektoken(void);
    116 STATIC int readtoken(void);
    117 STATIC int xxreadtoken(void);
    118 STATIC int readtoken1(int, char const *, char *, int);
    119 STATIC int noexpand(char *);
    120 STATIC void synexpect(int, const char *) __dead;
    121 STATIC void synerror(const char *) __dead;
    122 STATIC void setprompt(int);
    123 
    124 
    125 static const char EOFhere[] = "EOF reading here (<<) document";
    126 
    127 
    128 /*
    129  * Read and parse a command.  Returns NEOF on end of file.  (NULL is a
    130  * valid parse tree indicating a blank line.)
    131  */
    132 
    133 union node *
    134 parsecmd(int interact)
    135 {
    136 	int t;
    137 
    138 	tokpushback = 0;
    139 	doprompt = interact;
    140 	if (doprompt)
    141 		setprompt(1);
    142 	else
    143 		setprompt(0);
    144 	needprompt = 0;
    145 	t = readtoken();
    146 	if (t == TEOF)
    147 		return NEOF;
    148 	if (t == TNL)
    149 		return NULL;
    150 	tokpushback++;
    151 	return list(1, 0);
    152 }
    153 
    154 
    155 STATIC union node *
    156 list(int nlflag, int erflag)
    157 {
    158 	union node *n1, *n2, *n3;
    159 	int tok;
    160 	TRACE(("list(%d,%d): entered\n", nlflag, erflag));
    161 
    162 	checkkwd = 2;
    163 	if (nlflag == 0 && tokendlist[peektoken()])
    164 		return NULL;
    165 	n1 = NULL;
    166 	for (;;) {
    167 		n2 = andor();
    168 		tok = readtoken();
    169 		if (tok == TBACKGND) {
    170 			if (n2->type == NCMD || n2->type == NPIPE) {
    171 				n2->ncmd.backgnd = 1;
    172 			} else if (n2->type == NREDIR) {
    173 				n2->type = NBACKGND;
    174 			} else {
    175 				n3 = stalloc(sizeof(struct nredir));
    176 				n3->type = NBACKGND;
    177 				n3->nredir.n = n2;
    178 				n3->nredir.redirect = NULL;
    179 				n2 = n3;
    180 			}
    181 		}
    182 		if (n1 == NULL) {
    183 			n1 = n2;
    184 		}
    185 		else {
    186 			n3 = stalloc(sizeof(struct nbinary));
    187 			n3->type = NSEMI;
    188 			n3->nbinary.ch1 = n1;
    189 			n3->nbinary.ch2 = n2;
    190 			n1 = n3;
    191 		}
    192 		switch (tok) {
    193 		case TBACKGND:
    194 		case TSEMI:
    195 			tok = readtoken();
    196 			/* FALLTHROUGH */
    197 		case TNL:
    198 			if (tok == TNL) {
    199 				parseheredoc();
    200 				if (nlflag)
    201 					return n1;
    202 			} else {
    203 				tokpushback++;
    204 			}
    205 			checkkwd = 2;
    206 			if (tokendlist[peektoken()])
    207 				return n1;
    208 			break;
    209 		case TEOF:
    210 			if (heredoclist)
    211 				parseheredoc();
    212 			else
    213 				pungetc();	/* push back EOF on input */
    214 			return n1;
    215 		default:
    216 			if (nlflag || erflag)
    217 				synexpect(-1, 0);
    218 			tokpushback++;
    219 			return n1;
    220 		}
    221 	}
    222 }
    223 
    224 STATIC union node *
    225 andor(void)
    226 {
    227 	union node *n1, *n2, *n3;
    228 	int t;
    229 
    230 	TRACE(("andor: entered\n"));
    231 	n1 = pipeline();
    232 	for (;;) {
    233 		if ((t = readtoken()) == TAND) {
    234 			t = NAND;
    235 		} else if (t == TOR) {
    236 			t = NOR;
    237 		} else {
    238 			tokpushback++;
    239 			return n1;
    240 		}
    241 		n2 = pipeline();
    242 		n3 = stalloc(sizeof(struct nbinary));
    243 		n3->type = t;
    244 		n3->nbinary.ch1 = n1;
    245 		n3->nbinary.ch2 = n2;
    246 		n1 = n3;
    247 	}
    248 }
    249 
    250 STATIC union node *
    251 pipeline(void)
    252 {
    253 	union node *n1, *n2, *pipenode;
    254 	struct nodelist *lp, *prev;
    255 	int negate;
    256 
    257 	TRACE(("pipeline: entered\n"));
    258 
    259 	negate = 0;
    260 	checkkwd = 2;
    261 	while (readtoken() == TNOT) {
    262 		TRACE(("pipeline: TNOT recognized\n"));
    263 		negate = !negate;
    264 	}
    265 	tokpushback++;
    266 	n1 = command();
    267 	if (readtoken() == TPIPE) {
    268 		pipenode = stalloc(sizeof(struct npipe));
    269 		pipenode->type = NPIPE;
    270 		pipenode->npipe.backgnd = 0;
    271 		lp = stalloc(sizeof(struct nodelist));
    272 		pipenode->npipe.cmdlist = lp;
    273 		lp->n = n1;
    274 		do {
    275 			prev = lp;
    276 			lp = stalloc(sizeof(struct nodelist));
    277 			lp->n = command();
    278 			prev->next = lp;
    279 		} while (readtoken() == TPIPE);
    280 		lp->next = NULL;
    281 		n1 = pipenode;
    282 	}
    283 	tokpushback++;
    284 	if (negate) {
    285 		TRACE(("negate pipeline\n"));
    286 		n2 = stalloc(sizeof(struct nnot));
    287 		n2->type = NNOT;
    288 		n2->nnot.com = n1;
    289 		return n2;
    290 	} else
    291 		return n1;
    292 }
    293 
    294 
    295 
    296 STATIC union node *
    297 command(void)
    298 {
    299 	union node *n1, *n2;
    300 	union node *ap, **app;
    301 	union node *cp, **cpp;
    302 	union node *redir, **rpp;
    303 	int t, negate = 0;
    304 
    305 	TRACE(("command: entered\n"));
    306 
    307 	checkkwd = 2;
    308 	redir = NULL;
    309 	n1 = NULL;
    310 	rpp = &redir;
    311 
    312 	/* Check for redirection which may precede command */
    313 	while (readtoken() == TREDIR) {
    314 		*rpp = n2 = redirnode;
    315 		rpp = &n2->nfile.next;
    316 		parsefname();
    317 	}
    318 	tokpushback++;
    319 
    320 	while (readtoken() == TNOT) {
    321 		TRACE(("command: TNOT recognized\n"));
    322 		negate = !negate;
    323 	}
    324 	tokpushback++;
    325 
    326 	switch (readtoken()) {
    327 	case TIF:
    328 		n1 = stalloc(sizeof(struct nif));
    329 		n1->type = NIF;
    330 		n1->nif.test = list(0, 0);
    331 		if (readtoken() != TTHEN)
    332 			synexpect(TTHEN, 0);
    333 		n1->nif.ifpart = list(0, 0);
    334 		n2 = n1;
    335 		while (readtoken() == TELIF) {
    336 			n2->nif.elsepart = stalloc(sizeof(struct nif));
    337 			n2 = n2->nif.elsepart;
    338 			n2->type = NIF;
    339 			n2->nif.test = list(0, 0);
    340 			if (readtoken() != TTHEN)
    341 				synexpect(TTHEN, 0);
    342 			n2->nif.ifpart = list(0, 0);
    343 		}
    344 		if (lasttoken == TELSE)
    345 			n2->nif.elsepart = list(0, 0);
    346 		else {
    347 			n2->nif.elsepart = NULL;
    348 			tokpushback++;
    349 		}
    350 		if (readtoken() != TFI)
    351 			synexpect(TFI, 0);
    352 		checkkwd = 1;
    353 		break;
    354 	case TWHILE:
    355 	case TUNTIL: {
    356 		int got;
    357 		n1 = stalloc(sizeof(struct nbinary));
    358 		n1->type = (lasttoken == TWHILE)? NWHILE : NUNTIL;
    359 		n1->nbinary.ch1 = list(0, 0);
    360 		if ((got=readtoken()) != TDO) {
    361 TRACE(("expecting DO got %s %s\n", tokname[got], got == TWORD ? wordtext : ""));
    362 			synexpect(TDO, 0);
    363 		}
    364 		n1->nbinary.ch2 = list(0, 0);
    365 		if (readtoken() != TDONE)
    366 			synexpect(TDONE, 0);
    367 		checkkwd = 1;
    368 		break;
    369 	}
    370 	case TFOR:
    371 		if (readtoken() != TWORD || quoteflag || ! goodname(wordtext))
    372 			synerror("Bad for loop variable");
    373 		n1 = stalloc(sizeof(struct nfor));
    374 		n1->type = NFOR;
    375 		n1->nfor.var = wordtext;
    376 		if (readtoken() == TWORD && ! quoteflag && equal(wordtext, "in")) {
    377 			app = &ap;
    378 			while (readtoken() == TWORD) {
    379 				n2 = stalloc(sizeof(struct narg));
    380 				n2->type = NARG;
    381 				n2->narg.text = wordtext;
    382 				n2->narg.backquote = backquotelist;
    383 				*app = n2;
    384 				app = &n2->narg.next;
    385 			}
    386 			*app = NULL;
    387 			n1->nfor.args = ap;
    388 			if (lasttoken != TNL && lasttoken != TSEMI)
    389 				synexpect(-1, 0);
    390 		} else {
    391 			static char argvars[5] = {
    392 			    CTLVAR, VSNORMAL|VSQUOTE, '@', '=', '\0'
    393 			};
    394 			n2 = stalloc(sizeof(struct narg));
    395 			n2->type = NARG;
    396 			n2->narg.text = argvars;
    397 			n2->narg.backquote = NULL;
    398 			n2->narg.next = NULL;
    399 			n1->nfor.args = n2;
    400 			/*
    401 			 * Newline or semicolon here is optional (but note
    402 			 * that the original Bourne shell only allowed NL).
    403 			 */
    404 			if (lasttoken != TNL && lasttoken != TSEMI)
    405 				tokpushback++;
    406 		}
    407 		checkkwd = 2;
    408 		if ((t = readtoken()) == TDO)
    409 			t = TDONE;
    410 		else if (t == TBEGIN)
    411 			t = TEND;
    412 		else
    413 			synexpect(-1, 0);
    414 		n1->nfor.body = list(0, 0);
    415 		if (readtoken() != t)
    416 			synexpect(t, 0);
    417 		checkkwd = 1;
    418 		break;
    419 	case TCASE:
    420 		n1 = stalloc(sizeof(struct ncase));
    421 		n1->type = NCASE;
    422 		if (readtoken() != TWORD)
    423 			synexpect(TWORD, 0);
    424 		n1->ncase.expr = n2 = stalloc(sizeof(struct narg));
    425 		n2->type = NARG;
    426 		n2->narg.text = wordtext;
    427 		n2->narg.backquote = backquotelist;
    428 		n2->narg.next = NULL;
    429 		while (readtoken() == TNL);
    430 		if (lasttoken != TWORD || ! equal(wordtext, "in"))
    431 			synexpect(-1, "in");
    432 		cpp = &n1->ncase.cases;
    433 		noalias = 1;
    434 		checkkwd = 2, readtoken();
    435 		/*
    436 		 * Both ksh and bash accept 'case x in esac'
    437 		 * so configure scripts started taking advantage of this.
    438 		 * The page: http://pubs.opengroup.org/onlinepubs/\
    439 		 * 009695399/utilities/xcu_chap02.html contradicts itself,
    440 		 * as to if this is legal; the "Case Conditional Format"
    441 		 * paragraph shows one case is required, but the "Grammar"
    442 		 * section shows a grammar that explicitly allows the no
    443 		 * case option.
    444 		 */
    445 		while (lasttoken != TESAC) {
    446 			*cpp = cp = stalloc(sizeof(struct nclist));
    447 			if (lasttoken == TLP)
    448 				readtoken();
    449 			cp->type = NCLIST;
    450 			app = &cp->nclist.pattern;
    451 			for (;;) {
    452 				*app = ap = stalloc(sizeof(struct narg));
    453 				ap->type = NARG;
    454 				ap->narg.text = wordtext;
    455 				ap->narg.backquote = backquotelist;
    456 				if (checkkwd = 2, readtoken() != TPIPE)
    457 					break;
    458 				app = &ap->narg.next;
    459 				readtoken();
    460 			}
    461 			ap->narg.next = NULL;
    462 			noalias = 0;
    463 			if (lasttoken != TRP) {
    464 				synexpect(TRP, 0);
    465 			}
    466 			cp->nclist.body = list(0, 0);
    467 
    468 			checkkwd = 2;
    469 			if ((t = readtoken()) != TESAC) {
    470 				if (t != TENDCASE) {
    471 					noalias = 0;
    472 					synexpect(TENDCASE, 0);
    473 				} else {
    474 					noalias = 1;
    475 					checkkwd = 2;
    476 					readtoken();
    477 				}
    478 			}
    479 			cpp = &cp->nclist.next;
    480 		}
    481 		noalias = 0;
    482 		*cpp = NULL;
    483 		checkkwd = 1;
    484 		break;
    485 	case TLP:
    486 		n1 = stalloc(sizeof(struct nredir));
    487 		n1->type = NSUBSHELL;
    488 		n1->nredir.n = list(0, 0);
    489 		n1->nredir.redirect = NULL;
    490 		if (readtoken() != TRP)
    491 			synexpect(TRP, 0);
    492 		checkkwd = 1;
    493 		break;
    494 	case TBEGIN:
    495 		n1 = list(0, 0);
    496 		if (readtoken() != TEND)
    497 			synexpect(TEND, 0);
    498 		checkkwd = 1;
    499 		break;
    500 	/* Handle an empty command like other simple commands.  */
    501 	case TSEMI:
    502 		/*
    503 		 * An empty command before a ; doesn't make much sense, and
    504 		 * should certainly be disallowed in the case of `if ;'.
    505 		 */
    506 		if (!redir)
    507 			synexpect(-1, 0);
    508 	case TAND:
    509 	case TOR:
    510 	case TNL:
    511 	case TEOF:
    512 	case TWORD:
    513 	case TRP:
    514 		tokpushback++;
    515 		n1 = simplecmd(rpp, redir);
    516 		goto checkneg;
    517 	case TENDCASE:
    518 		if (redir) {
    519 			tokpushback++;
    520 			goto checkneg;
    521 		}
    522 		/* FALLTHROUGH */
    523 	default:
    524 		synexpect(-1, 0);
    525 		/* NOTREACHED */
    526 	}
    527 
    528 	/* Now check for redirection which may follow command */
    529 	while (readtoken() == TREDIR) {
    530 		*rpp = n2 = redirnode;
    531 		rpp = &n2->nfile.next;
    532 		parsefname();
    533 	}
    534 	tokpushback++;
    535 	*rpp = NULL;
    536 	if (redir) {
    537 		if (n1->type != NSUBSHELL) {
    538 			n2 = stalloc(sizeof(struct nredir));
    539 			n2->type = NREDIR;
    540 			n2->nredir.n = n1;
    541 			n1 = n2;
    542 		}
    543 		n1->nredir.redirect = redir;
    544 	}
    545 
    546 checkneg:
    547 	if (negate) {
    548 		TRACE(("negate command\n"));
    549 		n2 = stalloc(sizeof(struct nnot));
    550 		n2->type = NNOT;
    551 		n2->nnot.com = n1;
    552 		return n2;
    553 	}
    554 	else
    555 		return n1;
    556 }
    557 
    558 
    559 STATIC union node *
    560 simplecmd(union node **rpp, union node *redir)
    561 {
    562 	union node *args, **app;
    563 	union node *n = NULL, *n2;
    564 	int negate = 0;
    565 
    566 	/* If we don't have any redirections already, then we must reset */
    567 	/* rpp to be the address of the local redir variable.  */
    568 	if (redir == 0)
    569 		rpp = &redir;
    570 
    571 	args = NULL;
    572 	app = &args;
    573 
    574 	while (readtoken() == TNOT) {
    575 		TRACE(("simplcmd: TNOT recognized\n"));
    576 		negate = !negate;
    577 	}
    578 	tokpushback++;
    579 
    580 	for (;;) {
    581 		if (readtoken() == TWORD) {
    582 			n = stalloc(sizeof(struct narg));
    583 			n->type = NARG;
    584 			n->narg.text = wordtext;
    585 			n->narg.backquote = backquotelist;
    586 			*app = n;
    587 			app = &n->narg.next;
    588 		} else if (lasttoken == TREDIR) {
    589 			*rpp = n = redirnode;
    590 			rpp = &n->nfile.next;
    591 			parsefname();	/* read name of redirection file */
    592 		} else if (lasttoken == TLP && app == &args->narg.next
    593 					    && redir == 0) {
    594 			/* We have a function */
    595 			if (readtoken() != TRP)
    596 				synexpect(TRP, 0);
    597 			funclinno = plinno;
    598 			rmescapes(n->narg.text);
    599 			if (!goodname(n->narg.text))
    600 				synerror("Bad function name");
    601 			n->type = NDEFUN;
    602 			n->narg.next = command();
    603 			funclinno = 0;
    604 			goto checkneg;
    605 		} else {
    606 			tokpushback++;
    607 			break;
    608 		}
    609 	}
    610 	*app = NULL;
    611 	*rpp = NULL;
    612 	n = stalloc(sizeof(struct ncmd));
    613 	n->type = NCMD;
    614 	n->ncmd.backgnd = 0;
    615 	n->ncmd.args = args;
    616 	n->ncmd.redirect = redir;
    617 
    618 checkneg:
    619 	if (negate) {
    620 		TRACE(("negate simplecmd\n"));
    621 		n2 = stalloc(sizeof(struct nnot));
    622 		n2->type = NNOT;
    623 		n2->nnot.com = n;
    624 		return n2;
    625 	}
    626 	else
    627 		return n;
    628 }
    629 
    630 STATIC union node *
    631 makename(void)
    632 {
    633 	union node *n;
    634 
    635 	n = stalloc(sizeof(struct narg));
    636 	n->type = NARG;
    637 	n->narg.next = NULL;
    638 	n->narg.text = wordtext;
    639 	n->narg.backquote = backquotelist;
    640 	return n;
    641 }
    642 
    643 void
    644 fixredir(union node *n, const char *text, int err)
    645 {
    646 	TRACE(("Fix redir %s %d\n", text, err));
    647 	if (!err)
    648 		n->ndup.vname = NULL;
    649 
    650 	if (is_number(text))
    651 		n->ndup.dupfd = number(text);
    652 	else if (text[0] == '-' && text[1] == '\0')
    653 		n->ndup.dupfd = -1;
    654 	else {
    655 
    656 		if (err)
    657 			synerror("Bad fd number");
    658 		else
    659 			n->ndup.vname = makename();
    660 	}
    661 }
    662 
    663 
    664 STATIC void
    665 parsefname(void)
    666 {
    667 	union node *n = redirnode;
    668 
    669 	if (readtoken() != TWORD)
    670 		synexpect(-1, 0);
    671 	if (n->type == NHERE) {
    672 		struct heredoc *here = heredoc;
    673 		struct heredoc *p;
    674 		int i;
    675 
    676 		if (quoteflag == 0)
    677 			n->type = NXHERE;
    678 		TRACE(("Here document %d\n", n->type));
    679 		if (here->striptabs) {
    680 			while (*wordtext == '\t')
    681 				wordtext++;
    682 		}
    683 		if (! noexpand(wordtext) || (i = strlen(wordtext)) == 0 || i > EOFMARKLEN)
    684 			synerror("Illegal eof marker for << redirection");
    685 		rmescapes(wordtext);
    686 		here->eofmark = wordtext;
    687 		here->next = NULL;
    688 		if (heredoclist == NULL)
    689 			heredoclist = here;
    690 		else {
    691 			for (p = heredoclist ; p->next ; p = p->next)
    692 				continue;
    693 			p->next = here;
    694 		}
    695 	} else if (n->type == NTOFD || n->type == NFROMFD) {
    696 		fixredir(n, wordtext, 0);
    697 	} else {
    698 		n->nfile.fname = makename();
    699 	}
    700 }
    701 
    702 
    703 /*
    704  * Input any here documents.
    705  */
    706 
    707 STATIC void
    708 parseheredoc(void)
    709 {
    710 	struct heredoc *here;
    711 	union node *n;
    712 
    713 	while (heredoclist) {
    714 		int c;
    715 
    716 		here = heredoclist;
    717 		heredoclist = here->next;
    718 		if (needprompt) {
    719 			setprompt(2);
    720 			needprompt = 0;
    721 		}
    722 		if ((c = pgetc()) == PEOF) {
    723 			synerror(EOFhere);
    724 			/* NOTREACHED */
    725 		}
    726 		readtoken1(c, here->here->type == NHERE? SQSYNTAX : DQSYNTAX,
    727 		    here->eofmark, here->striptabs);
    728 		n = stalloc(sizeof(struct narg));
    729 		n->narg.type = NARG;
    730 		n->narg.next = NULL;
    731 		n->narg.text = wordtext;
    732 		n->narg.backquote = backquotelist;
    733 		here->here->nhere.doc = n;
    734 	}
    735 }
    736 
    737 STATIC int
    738 peektoken(void)
    739 {
    740 	int t;
    741 
    742 	t = readtoken();
    743 	tokpushback++;
    744 	return (t);
    745 }
    746 
    747 STATIC int
    748 readtoken(void)
    749 {
    750 	int t;
    751 	int savecheckkwd = checkkwd;
    752 #ifdef DEBUG
    753 	int alreadyseen = tokpushback;
    754 #endif
    755 	struct alias *ap;
    756 
    757 	top:
    758 	t = xxreadtoken();
    759 
    760 	if (checkkwd) {
    761 		/*
    762 		 * eat newlines
    763 		 */
    764 		if (checkkwd == 2) {
    765 			checkkwd = 0;
    766 			while (t == TNL) {
    767 				parseheredoc();
    768 				t = xxreadtoken();
    769 			}
    770 		} else
    771 			checkkwd = 0;
    772 		/*
    773 		 * check for keywords and aliases
    774 		 */
    775 		if (t == TWORD && !quoteflag) {
    776 			const char *const *pp;
    777 
    778 			for (pp = parsekwd; *pp; pp++) {
    779 				if (**pp == *wordtext && equal(*pp, wordtext)) {
    780 					lasttoken = t = pp -
    781 					    parsekwd + KWDOFFSET;
    782 					TRACE(("keyword %s recognized\n", tokname[t]));
    783 					goto out;
    784 				}
    785 			}
    786 			if (!noalias &&
    787 			    (ap = lookupalias(wordtext, 1)) != NULL) {
    788 				pushstring(ap->val, strlen(ap->val), ap);
    789 				checkkwd = savecheckkwd;
    790 				goto top;
    791 			}
    792 		}
    793 out:
    794 		checkkwd = (t == TNOT) ? savecheckkwd : 0;
    795 	}
    796 	TRACE(("%stoken %s %s\n", alreadyseen ? "reread " : "", tokname[t], t == TWORD ? wordtext : ""));
    797 	return (t);
    798 }
    799 
    800 
    801 /*
    802  * Read the next input token.
    803  * If the token is a word, we set backquotelist to the list of cmds in
    804  *	backquotes.  We set quoteflag to true if any part of the word was
    805  *	quoted.
    806  * If the token is TREDIR, then we set redirnode to a structure containing
    807  *	the redirection.
    808  * In all cases, the variable startlinno is set to the number of the line
    809  *	on which the token starts.
    810  *
    811  * [Change comment:  here documents and internal procedures]
    812  * [Readtoken shouldn't have any arguments.  Perhaps we should make the
    813  *  word parsing code into a separate routine.  In this case, readtoken
    814  *  doesn't need to have any internal procedures, but parseword does.
    815  *  We could also make parseoperator in essence the main routine, and
    816  *  have parseword (readtoken1?) handle both words and redirection.]
    817  */
    818 
    819 #define RETURN(token)	return lasttoken = token
    820 
    821 STATIC int
    822 xxreadtoken(void)
    823 {
    824 	int c;
    825 
    826 	if (tokpushback) {
    827 		tokpushback = 0;
    828 		return lasttoken;
    829 	}
    830 	if (needprompt) {
    831 		setprompt(2);
    832 		needprompt = 0;
    833 	}
    834 	startlinno = plinno;
    835 	for (;;) {	/* until token or start of word found */
    836 		c = pgetc_macro();
    837 		switch (c) {
    838 		case ' ': case '\t':
    839 			continue;
    840 		case '#':
    841 			while ((c = pgetc()) != '\n' && c != PEOF)
    842 				continue;
    843 			pungetc();
    844 			continue;
    845 
    846 		case '\n':
    847 			plinno++;
    848 			needprompt = doprompt;
    849 			RETURN(TNL);
    850 		case PEOF:
    851 			RETURN(TEOF);
    852 
    853 		case '&':
    854 			if (pgetc() == '&')
    855 				RETURN(TAND);
    856 			pungetc();
    857 			RETURN(TBACKGND);
    858 		case '|':
    859 			if (pgetc() == '|')
    860 				RETURN(TOR);
    861 			pungetc();
    862 			RETURN(TPIPE);
    863 		case ';':
    864 			if (pgetc() == ';')
    865 				RETURN(TENDCASE);
    866 			pungetc();
    867 			RETURN(TSEMI);
    868 		case '(':
    869 			RETURN(TLP);
    870 		case ')':
    871 			RETURN(TRP);
    872 
    873 		case '\\':
    874 			switch (pgetc()) {
    875 			case '\n':
    876 				startlinno = ++plinno;
    877 				if (doprompt)
    878 					setprompt(2);
    879 				else
    880 					setprompt(0);
    881 				continue;
    882 			case PEOF:
    883 				RETURN(TEOF);
    884 			default:
    885 				pungetc();
    886 				break;
    887 			}
    888 			/* FALLTHROUGH */
    889 		default:
    890 			return readtoken1(c, BASESYNTAX, NULL, 0);
    891 		}
    892 	}
    893 #undef RETURN
    894 }
    895 
    896 
    897 
    898 /*
    899  * If eofmark is NULL, read a word or a redirection symbol.  If eofmark
    900  * is not NULL, read a here document.  In the latter case, eofmark is the
    901  * word which marks the end of the document and striptabs is true if
    902  * leading tabs should be stripped from the document.  The argument firstc
    903  * is the first character of the input token or document.
    904  *
    905  * Because C does not have internal subroutines, I have simulated them
    906  * using goto's to implement the subroutine linkage.  The following macros
    907  * will run code that appears at the end of readtoken1.
    908  */
    909 
    910 /*
    911  * We used to remember only the current syntax, variable nesting level,
    912  * double quote state for each var nesting level, and arith nesting
    913  * level (unrelated to var nesting) and one prev syntax when in arith
    914  * syntax.  This worked for simple cases, but can't handle arith inside
    915  * var expansion inside arith inside var with some quoted and some not.
    916  *
    917  * Inspired by FreeBSD's implementation (though it was the obvious way)
    918  * though implemented differently, we now have a stack that keeps track
    919  * of what we are doing now, and what we were doing previously.
    920  * Every time something changes, which will eventually end and should
    921  * revert to the previous state, we push this stack, and then pop it
    922  * again later (that is every ${} with an operator (to parse the word
    923  * or pattern that follows) ${x} and $x are too simple to need it)
    924  * $(( )) $( ) and "...".   Always.   Really, always!
    925  *
    926  * The stack is implemented as one static (on the C stack) base block
    927  * containing LEVELS_PER_BLOCK (8) stack entries, which should be
    928  * enough for the vast majority of cases.  For torture tests, we
    929  * malloc more blocks as needed.  All accesses through the inline
    930  * functions below.
    931  */
    932 
    933 /*
    934  * varnest & arinest will typically be 0 or 1
    935  * (varnest can increment in usages like ${x=${y}} but probably
    936  *  does not really need to)
    937  * parenlevel allows balancing parens inside a $(( )), it is reset
    938  * at each new nesting level ( $(( ( x + 3 ${unset-)} )) does not work.
    939  * quoted is special - we need to know 2 things ... are we inside "..."
    940  * (even if inherited from some previous nesting level) and was there
    941  * an opening '"' at this level (so the next will be closing).
    942  * "..." can span nesting levels, but cannot be opened in one and
    943  * closed in a different one.
    944  * To handle this, "quoted" has two fields, the bottom 4 (really 2)
    945  * bits are 0, 1, or 2, for un, single, and double quoted (single quoted
    946  * is really so special that this setting is not very important)
    947  * and 0x10 that indicates that an opening quote has been seen.
    948  * The bottom 4 bits are inherited, the 0x10 bit is not.
    949  */
    950 struct tokenstate {
    951 	const char *ts_syntax;
    952 	unsigned short ts_parenlevel;	/* counters */
    953 	unsigned short ts_varnest;	/* 64000 levels should be enough! */
    954 	unsigned short ts_arinest;
    955 	unsigned short ts_quoted;	/* 1 -> single, 2 -> double */
    956 };
    957 
    958 #define	NQ	0x00	/* Unquoted */
    959 #define	SQ	0x01	/* Single Quotes */
    960 #define	DQ	0x02	/* Double Quotes (or equivalent) */
    961 #define	QF	0x0F		/* Mask to extract previous values */
    962 #define	QS	0x10	/* Quoting started at this level in stack */
    963 
    964 #define	LEVELS_PER_BLOCK	8
    965 #define	VSS			struct statestack
    966 
    967 struct statestack {
    968 	VSS *prev;		/* previous block in list */
    969 	int cur;		/* which of our tokenstates is current */
    970 	struct tokenstate tokenstate[LEVELS_PER_BLOCK];
    971 };
    972 
    973 static inline struct tokenstate *
    974 currentstate(VSS *stack)
    975 {
    976 	return &stack->tokenstate[stack->cur];
    977 }
    978 
    979 static inline struct tokenstate *
    980 prevstate(VSS *stack)
    981 {
    982 	if (stack->cur != 0)
    983 		return &stack->tokenstate[stack->cur - 1];
    984 	if (stack->prev == NULL)	/* cannot drop below base */
    985 		return &stack->tokenstate[0];
    986 	return &stack->prev->tokenstate[LEVELS_PER_BLOCK - 1];
    987 }
    988 
    989 static inline VSS *
    990 bump_state_level(VSS *stack)
    991 {
    992 	struct tokenstate *os, *ts;
    993 
    994 	os = currentstate(stack);
    995 
    996 	if (++stack->cur >= LEVELS_PER_BLOCK) {
    997 		VSS *ss;
    998 
    999 		ss = (VSS *)ckmalloc(sizeof (struct statestack));
   1000 		ss->cur = 0;
   1001 		ss->prev = stack;
   1002 		stack = ss;
   1003 	}
   1004 
   1005 	ts = currentstate(stack);
   1006 
   1007 	ts->ts_parenlevel = 0;	/* parens inside never match outside */
   1008 
   1009 	ts->ts_quoted  = os->ts_quoted & QF;	/* these are default settings */
   1010 	ts->ts_varnest = os->ts_varnest;
   1011 	ts->ts_arinest = os->ts_arinest;	/* when appropriate	   */
   1012 	ts->ts_syntax  = os->ts_syntax;		/*    they will be altered */
   1013 
   1014 	return stack;
   1015 }
   1016 
   1017 static inline VSS *
   1018 drop_state_level(VSS *stack)
   1019 {
   1020 	if (stack->cur == 0) {
   1021 		VSS *ss;
   1022 
   1023 		ss = stack;
   1024 		stack = ss->prev;
   1025 		if (stack == NULL)
   1026 			return ss;
   1027 		ckfree(ss);
   1028 	}
   1029 	--stack->cur;
   1030 	return stack;
   1031 }
   1032 
   1033 static inline void
   1034 cleanup_state_stack(VSS *stack)
   1035 {
   1036 	while (stack->prev != NULL) {
   1037 		stack->cur = 0;
   1038 		stack = drop_state_level(stack);
   1039 	}
   1040 }
   1041 
   1042 #define	CHECKEND()	{goto checkend; checkend_return:;}
   1043 #define	PARSEREDIR()	{goto parseredir; parseredir_return:;}
   1044 #define	PARSESUB()	{goto parsesub; parsesub_return:;}
   1045 #define	PARSEARITH()	{goto parsearith; parsearith_return:;}
   1046 
   1047 /*
   1048  * The following macros all assume the existance of a local var "stack"
   1049  * which contains a pointer to the current struct stackstate
   1050  */
   1051 
   1052 /*
   1053  * These are macros rather than inline funcs to avoid code churn as much
   1054  * as possible - they replace macros of the same name used previously.
   1055  */
   1056 #define	ISDBLQUOTE()	(currentstate(stack)->ts_quoted & QS)
   1057 #define	SETDBLQUOTE()	(currentstate(stack)->ts_quoted = QS | DQ)
   1058 #define	CLRDBLQUOTE()	(currentstate(stack)->ts_quoted =		\
   1059 			    stack->cur != 0 || stack->prev ?		\
   1060 				prevstate(stack)->ts_quoted & QF : 0)
   1061 
   1062 /*
   1063  * This set are just to avoid excess typing and line lengths...
   1064  * The ones that "look like" var names must be implemented to be lvalues
   1065  */
   1066 #define	syntax		(currentstate(stack)->ts_syntax)
   1067 #define	parenlevel	(currentstate(stack)->ts_parenlevel)
   1068 #define	varnest		(currentstate(stack)->ts_varnest)
   1069 #define	arinest		(currentstate(stack)->ts_arinest)
   1070 #define	quoted		(currentstate(stack)->ts_quoted)
   1071 #define	TS_PUSH()	(stack = bump_state_level(stack))
   1072 #define	TS_POP()	(stack = drop_state_level(stack))
   1073 
   1074 /*
   1075  * Called to parse command substitutions.  oldstyle is true if the command
   1076  * is enclosed inside `` (otherwise it was enclosed in "$( )")
   1077  *
   1078  * Internally nlpp is a pointer to the head of the linked
   1079  * list of commands (passed by reference), and savelen is the number of
   1080  * characters on the top of the stack which must be preserved.
   1081  */
   1082 static char *
   1083 parsebackq(VSS *const stack, char * const in,
   1084     struct nodelist **const pbqlist, const int oldstyle)
   1085 {
   1086 	struct nodelist **nlpp;
   1087 	const int savepbq = parsebackquote;
   1088 	union node *n;
   1089 	char *out;
   1090 	char *str = NULL;
   1091 	char *volatile sstr = str;
   1092 	struct jmploc jmploc;
   1093 	struct jmploc *const savehandler = handler;
   1094 	const int savelen = in - stackblock();
   1095 	int saveprompt;
   1096 
   1097 	if (setjmp(jmploc.loc)) {
   1098 		if (sstr)
   1099 			ckfree(__UNVOLATILE(sstr));
   1100 		cleanup_state_stack(stack);
   1101 		parsebackquote = 0;
   1102 		handler = savehandler;
   1103 		longjmp(handler->loc, 1);
   1104 	}
   1105 	INTOFF;
   1106 	sstr = str = NULL;
   1107 	if (savelen > 0) {
   1108 		sstr = str = ckmalloc(savelen);
   1109 		memcpy(str, stackblock(), savelen);
   1110 	}
   1111 	handler = &jmploc;
   1112 	INTON;
   1113         if (oldstyle) {
   1114                 /* We must read until the closing backquote, giving special
   1115                    treatment to some slashes, and then push the string and
   1116                    reread it as input, interpreting it normally.  */
   1117                 int pc;
   1118                 int psavelen;
   1119                 char *pstr;
   1120 
   1121 		/*
   1122 		 * Because the entire `...` is read here, we don't
   1123 		 * need to bother the state stack.  That will be used
   1124 		 * (as appropriate) when the processed string is re-read.
   1125 		 */
   1126                 STARTSTACKSTR(out);
   1127 		for (;;) {
   1128 			if (needprompt) {
   1129 				setprompt(2);
   1130 				needprompt = 0;
   1131 			}
   1132 			switch (pc = pgetc()) {
   1133 			case '`':
   1134 				goto done;
   1135 
   1136 			case '\\':
   1137                                 if ((pc = pgetc()) == '\n') {
   1138 					plinno++;
   1139 					if (doprompt)
   1140 						setprompt(2);
   1141 					else
   1142 						setprompt(0);
   1143 					/*
   1144 					 * If eating a newline, avoid putting
   1145 					 * the newline into the new character
   1146 					 * stream (via the STPUTC after the
   1147 					 * switch).
   1148 					 */
   1149 					continue;
   1150 				}
   1151                                 if (pc != '\\' && pc != '`' && pc != '$'
   1152                                     && (!ISDBLQUOTE() || pc != '"'))
   1153                                         STPUTC('\\', out);
   1154 				break;
   1155 
   1156 			case '\n':
   1157 				plinno++;
   1158 				needprompt = doprompt;
   1159 				break;
   1160 
   1161 			case PEOF:
   1162 			        startlinno = plinno;
   1163 				synerror("EOF in backquote substitution");
   1164  				break;
   1165 
   1166 			default:
   1167 				break;
   1168 			}
   1169 			STPUTC(pc, out);
   1170                 }
   1171 done:
   1172                 STPUTC('\0', out);
   1173                 psavelen = out - stackblock();
   1174                 if (psavelen > 0) {
   1175 			pstr = grabstackstr(out);
   1176 			setinputstring(pstr, 1);
   1177                 }
   1178         }
   1179 	nlpp = pbqlist;
   1180 	while (*nlpp)
   1181 		nlpp = &(*nlpp)->next;
   1182 	*nlpp = stalloc(sizeof(struct nodelist));
   1183 	(*nlpp)->next = NULL;
   1184 	parsebackquote = oldstyle;
   1185 
   1186 	if (oldstyle) {
   1187 		saveprompt = doprompt;
   1188 		doprompt = 0;
   1189 	} else
   1190 		saveprompt = 0;
   1191 
   1192 	n = list(0, oldstyle);
   1193 
   1194 	if (oldstyle)
   1195 		doprompt = saveprompt;
   1196 	else {
   1197 		if (readtoken() != TRP) {
   1198 			cleanup_state_stack(stack);
   1199 			synexpect(TRP, 0);
   1200 		}
   1201 	}
   1202 
   1203 	(*nlpp)->n = n;
   1204         if (oldstyle) {
   1205 		/*
   1206 		 * Start reading from old file again, ignoring any pushed back
   1207 		 * tokens left from the backquote parsing
   1208 		 */
   1209                 popfile();
   1210 		tokpushback = 0;
   1211 	}
   1212 
   1213 	while (stackblocksize() <= savelen)
   1214 		growstackblock();
   1215 	STARTSTACKSTR(out);
   1216 	if (str) {
   1217 		memcpy(out, str, savelen);
   1218 		STADJUST(savelen, out);
   1219 		INTOFF;
   1220 		ckfree(str);
   1221 		sstr = str = NULL;
   1222 		INTON;
   1223 	}
   1224 	parsebackquote = savepbq;
   1225 	handler = savehandler;
   1226 	if (arinest || ISDBLQUOTE())
   1227 		USTPUTC(CTLBACKQ | CTLQUOTE, out);
   1228 	else
   1229 		USTPUTC(CTLBACKQ, out);
   1230 
   1231 	return out;
   1232 }
   1233 
   1234 STATIC int
   1235 readtoken1(int firstc, char const *syn, char *eofmark, int striptabs)
   1236 {
   1237 	int c = firstc;
   1238 	char * out;
   1239 	int len;
   1240 	char line[EOFMARKLEN + 1];
   1241 	struct nodelist *bqlist;
   1242 	int quotef;
   1243 	VSS static_stack;
   1244 	VSS *stack = &static_stack;
   1245 
   1246 	stack->prev = NULL;
   1247 	stack->cur = 0;
   1248 
   1249 	syntax = syn;
   1250 
   1251 	startlinno = plinno;
   1252 	varnest = 0;
   1253 	quoted = 0;
   1254 	if (syntax == DQSYNTAX)
   1255 		SETDBLQUOTE();
   1256 	quotef = 0;
   1257 	bqlist = NULL;
   1258 	arinest = 0;
   1259 	parenlevel = 0;
   1260 
   1261 	STARTSTACKSTR(out);
   1262 	loop: {	/* for each line, until end of word */
   1263 		CHECKEND();	/* set c to PEOF if at end of here document */
   1264 		for (;;) {	/* until end of line or end of word */
   1265 			CHECKSTRSPACE(4, out);	/* permit 4 calls to USTPUTC */
   1266 			switch(syntax[c]) {
   1267 			case CNL:	/* '\n' */
   1268 				if (syntax == BASESYNTAX)
   1269 					goto endword;	/* exit outer loop */
   1270 				USTPUTC(c, out);
   1271 				plinno++;
   1272 				if (doprompt)
   1273 					setprompt(2);
   1274 				else
   1275 					setprompt(0);
   1276 				c = pgetc();
   1277 				goto loop;		/* continue outer loop */
   1278 			case CWORD:
   1279 				USTPUTC(c, out);
   1280 				break;
   1281 			case CCTL:
   1282 				if (eofmark == NULL || ISDBLQUOTE())
   1283 					USTPUTC(CTLESC, out);
   1284 				USTPUTC(c, out);
   1285 				break;
   1286 			case CBACK:	/* backslash */
   1287 				c = pgetc();
   1288 				if (c == PEOF) {
   1289 					USTPUTC('\\', out);
   1290 					pungetc();
   1291 					break;
   1292 				}
   1293 				if (c == '\n') {
   1294 					plinno++;
   1295 					if (doprompt)
   1296 						setprompt(2);
   1297 					else
   1298 						setprompt(0);
   1299 					break;
   1300 				}
   1301 				quotef = 1;
   1302 				if (ISDBLQUOTE() && c != '\\' &&
   1303 				    c != '`' && c != '$' &&
   1304 				    (c != '"' || eofmark != NULL))
   1305 					USTPUTC('\\', out);
   1306 				if (SQSYNTAX[c] == CCTL)
   1307 					USTPUTC(CTLESC, out);
   1308 				else if (eofmark == NULL) {
   1309 					USTPUTC(CTLQUOTEMARK, out);
   1310 					USTPUTC(c, out);
   1311 					if (varnest != 0)
   1312 						USTPUTC(CTLQUOTEEND, out);
   1313 					break;
   1314 				}
   1315 				USTPUTC(c, out);
   1316 				break;
   1317 			case CSQUOTE:
   1318 				if (syntax != SQSYNTAX) {
   1319 					if (eofmark == NULL)
   1320 						USTPUTC(CTLQUOTEMARK, out);
   1321 					quotef = 1;
   1322 					TS_PUSH();
   1323 					syntax = SQSYNTAX;
   1324 					quoted = SQ;
   1325 					break;
   1326 				}
   1327 				if (eofmark != NULL && arinest == 0 &&
   1328 				    varnest == 0) {
   1329 					/* Ignore inside quoted here document */
   1330 					USTPUTC(c, out);
   1331 					break;
   1332 				}
   1333 				/* End of single quotes... */
   1334 				TS_POP();
   1335 				if (syntax == BASESYNTAX && varnest != 0)
   1336 					USTPUTC(CTLQUOTEEND, out);
   1337 				break;
   1338 			case CDQUOTE:
   1339 				if (eofmark != NULL && arinest == 0 &&
   1340 				    varnest == 0) {
   1341 					/* Ignore inside here document */
   1342 					USTPUTC(c, out);
   1343 					break;
   1344 				}
   1345 				quotef = 1;
   1346 				if (arinest) {
   1347 					if (ISDBLQUOTE()) {
   1348 						TS_POP();
   1349 					} else {
   1350 						TS_PUSH();
   1351 						syntax = DQSYNTAX;
   1352 						SETDBLQUOTE();
   1353 						USTPUTC(CTLQUOTEMARK, out);
   1354 					}
   1355 					break;
   1356 				}
   1357 				if (eofmark != NULL)
   1358 					break;
   1359 				if (ISDBLQUOTE()) {
   1360 					TS_POP();
   1361 					if (varnest != 0)
   1362 						USTPUTC(CTLQUOTEEND, out);
   1363 				} else {
   1364 					TS_PUSH();
   1365 					syntax = DQSYNTAX;
   1366 					SETDBLQUOTE();
   1367 					USTPUTC(CTLQUOTEMARK, out);
   1368 				}
   1369 				break;
   1370 			case CVAR:	/* '$' */
   1371 				PARSESUB();		/* parse substitution */
   1372 				break;
   1373 			case CENDVAR:	/* CLOSEBRACE */
   1374 				if (varnest > 0 && !ISDBLQUOTE()) {
   1375 					TS_POP();
   1376 					USTPUTC(CTLENDVAR, out);
   1377 				} else {
   1378 					USTPUTC(c, out);
   1379 				}
   1380 				break;
   1381 			case CLP:	/* '(' in arithmetic */
   1382 				parenlevel++;
   1383 				USTPUTC(c, out);
   1384 				break;
   1385 			case CRP:	/* ')' in arithmetic */
   1386 				if (parenlevel > 0) {
   1387 					USTPUTC(c, out);
   1388 					--parenlevel;
   1389 				} else {
   1390 					if (pgetc() == ')') {
   1391 						if (--arinest == 0) {
   1392 							TS_POP();
   1393 							USTPUTC(CTLENDARI, out);
   1394 						} else
   1395 							USTPUTC(')', out);
   1396 					} else {
   1397 						/*
   1398 						 * unbalanced parens
   1399 						 *  (don't 2nd guess - no error)
   1400 						 */
   1401 						pungetc();
   1402 						USTPUTC(')', out);
   1403 					}
   1404 				}
   1405 				break;
   1406 			case CBQUOTE:	/* '`' */
   1407 				out = parsebackq(stack, out, &bqlist, 1);
   1408 				break;
   1409 			case CEOF:
   1410 				goto endword;		/* exit outer loop */
   1411 			default:
   1412 				if (varnest == 0 && !ISDBLQUOTE())
   1413 					goto endword;	/* exit outer loop */
   1414 				USTPUTC(c, out);
   1415 			}
   1416 			c = pgetc_macro();
   1417 		}
   1418 	}
   1419 endword:
   1420 	if (syntax == ARISYNTAX) {
   1421 		cleanup_state_stack(stack);
   1422 		synerror("Missing '))'");
   1423 	}
   1424 	if (syntax != BASESYNTAX && /* ! parsebackquote && */ eofmark == NULL) {
   1425 		cleanup_state_stack(stack);
   1426 		synerror("Unterminated quoted string");
   1427 	}
   1428 	if (varnest != 0) {
   1429 		cleanup_state_stack(stack);
   1430 		startlinno = plinno;
   1431 		/* { */
   1432 		synerror("Missing '}'");
   1433 	}
   1434 	USTPUTC('\0', out);
   1435 	len = out - stackblock();
   1436 	out = stackblock();
   1437 	if (eofmark == NULL) {
   1438 		if ((c == '>' || c == '<')
   1439 		 && quotef == 0
   1440 		 && (*out == '\0' || is_number(out))) {
   1441 			PARSEREDIR();
   1442 			cleanup_state_stack(stack);
   1443 			return lasttoken = TREDIR;
   1444 		} else {
   1445 			pungetc();
   1446 		}
   1447 	}
   1448 	quoteflag = quotef;
   1449 	backquotelist = bqlist;
   1450 	grabstackblock(len);
   1451 	wordtext = out;
   1452 	cleanup_state_stack(stack);
   1453 	return lasttoken = TWORD;
   1454 /* end of readtoken routine */
   1455 
   1456 
   1457 
   1458 /*
   1459  * Check to see whether we are at the end of the here document.  When this
   1460  * is called, c is set to the first character of the next input line.  If
   1461  * we are at the end of the here document, this routine sets the c to PEOF.
   1462  */
   1463 
   1464 checkend: {
   1465 	if (eofmark) {
   1466 		if (c == PEOF)
   1467 			synerror(EOFhere);
   1468 		if (striptabs) {
   1469 			while (c == '\t')
   1470 				c = pgetc();
   1471 		}
   1472 		if (c == *eofmark) {
   1473 			if (pfgets(line, sizeof line) != NULL) {
   1474 				char *p, *q;
   1475 
   1476 				p = line;
   1477 				for (q = eofmark + 1 ; *q && *p == *q ; p++, q++)
   1478 					continue;
   1479 				if ((*p == '\0' || *p == '\n') && *q == '\0') {
   1480 					c = PEOF;
   1481 					plinno++;
   1482 					needprompt = doprompt;
   1483 				} else {
   1484 					pushstring(line, strlen(line), NULL);
   1485 				}
   1486 			} else
   1487 				synerror(EOFhere);
   1488 		}
   1489 	}
   1490 	goto checkend_return;
   1491 }
   1492 
   1493 
   1494 /*
   1495  * Parse a redirection operator.  The variable "out" points to a string
   1496  * specifying the fd to be redirected.  The variable "c" contains the
   1497  * first character of the redirection operator.
   1498  */
   1499 
   1500 parseredir: {
   1501 	char fd[64];
   1502 	union node *np;
   1503 	strlcpy(fd, out, sizeof(fd));
   1504 
   1505 	np = stalloc(sizeof(struct nfile));
   1506 	if (c == '>') {
   1507 		np->nfile.fd = 1;
   1508 		c = pgetc();
   1509 		if (c == '>')
   1510 			np->type = NAPPEND;
   1511 		else if (c == '|')
   1512 			np->type = NCLOBBER;
   1513 		else if (c == '&')
   1514 			np->type = NTOFD;
   1515 		else {
   1516 			np->type = NTO;
   1517 			pungetc();
   1518 		}
   1519 	} else {	/* c == '<' */
   1520 		np->nfile.fd = 0;
   1521 		switch (c = pgetc()) {
   1522 		case '<':
   1523 			if (sizeof (struct nfile) != sizeof (struct nhere)) {
   1524 				np = stalloc(sizeof(struct nhere));
   1525 				np->nfile.fd = 0;
   1526 			}
   1527 			np->type = NHERE;
   1528 			heredoc = stalloc(sizeof(struct heredoc));
   1529 			heredoc->here = np;
   1530 			if ((c = pgetc()) == '-') {
   1531 				heredoc->striptabs = 1;
   1532 			} else {
   1533 				heredoc->striptabs = 0;
   1534 				pungetc();
   1535 			}
   1536 			break;
   1537 
   1538 		case '&':
   1539 			np->type = NFROMFD;
   1540 			break;
   1541 
   1542 		case '>':
   1543 			np->type = NFROMTO;
   1544 			break;
   1545 
   1546 		default:
   1547 			np->type = NFROM;
   1548 			pungetc();
   1549 			break;
   1550 		}
   1551 	}
   1552 	if (*fd != '\0')
   1553 		np->nfile.fd = number(fd);
   1554 	redirnode = np;
   1555 	goto parseredir_return;
   1556 }
   1557 
   1558 
   1559 /*
   1560  * Parse a substitution.  At this point, we have read the dollar sign
   1561  * and nothing else.
   1562  */
   1563 
   1564 parsesub: {
   1565 	char buf[10];
   1566 	int subtype;
   1567 	int typeloc;
   1568 	int flags;
   1569 	char *p;
   1570 	static const char types[] = "}-+?=";
   1571 	int i;
   1572 	int linno;
   1573 
   1574 	c = pgetc();
   1575 	if (c != '(' && c != OPENBRACE && !is_name(c) && !is_special(c)) {
   1576 		USTPUTC('$', out);
   1577 		pungetc();
   1578 	} else if (c == '(') {	/* $(command) or $((arith)) */
   1579 		if (pgetc() == '(') {
   1580 			PARSEARITH();
   1581 		} else {
   1582 			pungetc();
   1583 			out = parsebackq(stack, out, &bqlist, 0);
   1584 		}
   1585 	} else {
   1586 		USTPUTC(CTLVAR, out);
   1587 		typeloc = out - stackblock();
   1588 		USTPUTC(VSNORMAL, out);
   1589 		subtype = VSNORMAL;
   1590 		flags = 0;
   1591 		if (c == OPENBRACE) {
   1592 			c = pgetc();
   1593 			if (c == '#') {
   1594 				if ((c = pgetc()) == CLOSEBRACE)
   1595 					c = '#';
   1596 				else
   1597 					subtype = VSLENGTH;
   1598 			}
   1599 			else
   1600 				subtype = 0;
   1601 		}
   1602 		if (is_name(c)) {
   1603 			p = out;
   1604 			do {
   1605 				STPUTC(c, out);
   1606 				c = pgetc();
   1607 			} while (is_in_name(c));
   1608 			if (out - p == 6 && strncmp(p, "LINENO", 6) == 0) {
   1609 				/* Replace the variable name with the
   1610 				 * current line number. */
   1611 				linno = plinno;
   1612 				if (funclinno != 0)
   1613 					linno -= funclinno - 1;
   1614 				snprintf(buf, sizeof(buf), "%d", linno);
   1615 				STADJUST(-6, out);
   1616 				for (i = 0; buf[i] != '\0'; i++)
   1617 					STPUTC(buf[i], out);
   1618 				flags |= VSLINENO;
   1619 			}
   1620 		} else if (is_digit(c)) {
   1621 			do {
   1622 				USTPUTC(c, out);
   1623 				c = pgetc();
   1624 			} while (is_digit(c));
   1625 		}
   1626 		else if (is_special(c)) {
   1627 			USTPUTC(c, out);
   1628 			c = pgetc();
   1629 		}
   1630 		else {
   1631 badsub:
   1632 			cleanup_state_stack(stack);
   1633 			synerror("Bad substitution");
   1634 		}
   1635 
   1636 		STPUTC('=', out);
   1637 		if (subtype == 0) {
   1638 			switch (c) {
   1639 			case ':':
   1640 				flags |= VSNUL;
   1641 				c = pgetc();
   1642 				/*FALLTHROUGH*/
   1643 			default:
   1644 				p = strchr(types, c);
   1645 				if (p == NULL)
   1646 					goto badsub;
   1647 				subtype = p - types + VSNORMAL;
   1648 				break;
   1649 			case '%':
   1650 			case '#':
   1651 				{
   1652 					int cc = c;
   1653 					subtype = c == '#' ? VSTRIMLEFT :
   1654 							     VSTRIMRIGHT;
   1655 					c = pgetc();
   1656 					if (c == cc)
   1657 						subtype++;
   1658 					else
   1659 						pungetc();
   1660 					break;
   1661 				}
   1662 			}
   1663 		} else {
   1664 			pungetc();
   1665 		}
   1666 		if (ISDBLQUOTE() || arinest)
   1667 			flags |= VSQUOTE;
   1668 		if (subtype >= VSTRIMLEFT && subtype <= VSTRIMRIGHTMAX)
   1669 			flags |= VSPATQ;
   1670 		*(stackblock() + typeloc) = subtype | flags;
   1671 		if (subtype != VSNORMAL) {
   1672 			TS_PUSH();
   1673 			varnest++;
   1674 			arinest = 0;
   1675 			if (subtype > VSASSIGN) {	/* # ## % %% */
   1676 				syntax = BASESYNTAX;
   1677 				CLRDBLQUOTE();
   1678 			}
   1679 		}
   1680 	}
   1681 	goto parsesub_return;
   1682 }
   1683 
   1684 
   1685 /*
   1686  * Parse an arithmetic expansion (indicate start of one and set state)
   1687  */
   1688 parsearith: {
   1689 
   1690 	if (syntax == ARISYNTAX) {
   1691 		/*
   1692 		 * we collapse embedded arithmetic expansion to
   1693 		 * parentheses, which should be equivalent
   1694 		 */
   1695 		USTPUTC('(', out);
   1696 		USTPUTC('(', out);
   1697 		/*
   1698 		 * Need 2 of them because there will (should be)
   1699 		 * two closing ))'s to follow later.
   1700 		 */
   1701 		parenlevel += 2;
   1702 	} else {
   1703 		TS_PUSH();
   1704 		syntax = ARISYNTAX;
   1705 		++arinest;
   1706 		varnest = 0;
   1707 
   1708 		USTPUTC(CTLARI, out);
   1709 		if (ISDBLQUOTE())
   1710 			USTPUTC('"',out);
   1711 		else
   1712 			USTPUTC(' ',out);
   1713 	}
   1714 	goto parsearith_return;
   1715 }
   1716 
   1717 } /* end of readtoken */
   1718 
   1719 
   1720 
   1721 #ifdef mkinit
   1722 RESET {
   1723 	tokpushback = 0;
   1724 	checkkwd = 0;
   1725 }
   1726 #endif
   1727 
   1728 /*
   1729  * Returns true if the text contains nothing to expand (no dollar signs
   1730  * or backquotes).
   1731  */
   1732 
   1733 STATIC int
   1734 noexpand(char *text)
   1735 {
   1736 	char *p;
   1737 	char c;
   1738 
   1739 	p = text;
   1740 	while ((c = *p++) != '\0') {
   1741 		if (c == CTLQUOTEMARK)
   1742 			continue;
   1743 		if (c == CTLESC)
   1744 			p++;
   1745 		else if (BASESYNTAX[(int)c] == CCTL)
   1746 			return 0;
   1747 	}
   1748 	return 1;
   1749 }
   1750 
   1751 
   1752 /*
   1753  * Return true if the argument is a legal variable name (a letter or
   1754  * underscore followed by zero or more letters, underscores, and digits).
   1755  */
   1756 
   1757 int
   1758 goodname(char *name)
   1759 {
   1760 	char *p;
   1761 
   1762 	p = name;
   1763 	if (! is_name(*p))
   1764 		return 0;
   1765 	while (*++p) {
   1766 		if (! is_in_name(*p))
   1767 			return 0;
   1768 	}
   1769 	return 1;
   1770 }
   1771 
   1772 
   1773 /*
   1774  * Called when an unexpected token is read during the parse.  The argument
   1775  * is the token that is expected, or -1 if more than one type of token can
   1776  * occur at this point.
   1777  */
   1778 
   1779 STATIC void
   1780 synexpect(int token, const char *text)
   1781 {
   1782 	char msg[64];
   1783 	char *p;
   1784 
   1785 	if (lasttoken == TWORD) {
   1786 		size_t len = strlen(wordtext);
   1787 
   1788 		if (len <= 13)
   1789 			fmtstr(msg, 34, "Word \"%.13s\" unexpected", wordtext);
   1790 		else
   1791 			fmtstr(msg, 34,
   1792 			    "Word \"%.10s...\" unexpected", wordtext);
   1793 	} else
   1794 		fmtstr(msg, 34, "%s unexpected", tokname[lasttoken]);
   1795 
   1796 	p = strchr(msg, '\0');
   1797 	if (text)
   1798 		fmtstr(p, 30, " (expecting \"%.10s\")", text);
   1799 	else if (token >= 0)
   1800 		fmtstr(p, 30, " (expecting %s)",  tokname[token]);
   1801 
   1802 	synerror(msg);
   1803 	/* NOTREACHED */
   1804 }
   1805 
   1806 
   1807 STATIC void
   1808 synerror(const char *msg)
   1809 {
   1810 	error("%d: Syntax error: %s\n", startlinno, msg);
   1811 	/* NOTREACHED */
   1812 }
   1813 
   1814 STATIC void
   1815 setprompt(int which)
   1816 {
   1817 	whichprompt = which;
   1818 
   1819 #ifndef SMALL
   1820 	if (!el)
   1821 #endif
   1822 		out2str(getprompt(NULL));
   1823 }
   1824 
   1825 /*
   1826  * called by editline -- any expansions to the prompt
   1827  *    should be added here.
   1828  */
   1829 const char *
   1830 getprompt(void *unused)
   1831 {
   1832 	switch (whichprompt) {
   1833 	case 0:
   1834 		return "";
   1835 	case 1:
   1836 		return ps1val();
   1837 	case 2:
   1838 		return ps2val();
   1839 	default:
   1840 		return "<internal prompt error>";
   1841 	}
   1842 }
   1843