Home | History | Annotate | Line # | Download | only in sh
parser.c revision 1.97
      1 /*	$NetBSD: parser.c,v 1.97 2016/02/22 19:42:46 christos Exp $	*/
      2 
      3 /*-
      4  * Copyright (c) 1991, 1993
      5  *	The Regents of the University of California.  All rights reserved.
      6  *
      7  * This code is derived from software contributed to Berkeley by
      8  * Kenneth Almquist.
      9  *
     10  * Redistribution and use in source and binary forms, with or without
     11  * modification, are permitted provided that the following conditions
     12  * are met:
     13  * 1. Redistributions of source code must retain the above copyright
     14  *    notice, this list of conditions and the following disclaimer.
     15  * 2. Redistributions in binary form must reproduce the above copyright
     16  *    notice, this list of conditions and the following disclaimer in the
     17  *    documentation and/or other materials provided with the distribution.
     18  * 3. Neither the name of the University nor the names of its contributors
     19  *    may be used to endorse or promote products derived from this software
     20  *    without specific prior written permission.
     21  *
     22  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
     23  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
     24  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
     25  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
     26  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
     27  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
     28  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
     29  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
     30  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
     31  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
     32  * SUCH DAMAGE.
     33  */
     34 
     35 #include <sys/cdefs.h>
     36 #ifndef lint
     37 #if 0
     38 static char sccsid[] = "@(#)parser.c	8.7 (Berkeley) 5/16/95";
     39 #else
     40 __RCSID("$NetBSD: parser.c,v 1.97 2016/02/22 19:42:46 christos Exp $");
     41 #endif
     42 #endif /* not lint */
     43 
     44 #include <stdio.h>
     45 #include <stdlib.h>
     46 #include <limits.h>
     47 
     48 #include "shell.h"
     49 #include "parser.h"
     50 #include "nodes.h"
     51 #include "expand.h"	/* defines rmescapes() */
     52 #include "eval.h"	/* defines commandname */
     53 #include "redir.h"	/* defines copyfd() */
     54 #include "syntax.h"
     55 #include "options.h"
     56 #include "input.h"
     57 #include "output.h"
     58 #include "var.h"
     59 #include "error.h"
     60 #include "memalloc.h"
     61 #include "mystring.h"
     62 #include "alias.h"
     63 #include "show.h"
     64 #ifndef SMALL
     65 #include "myhistedit.h"
     66 #endif
     67 
     68 /*
     69  * Shell command parser.
     70  */
     71 
     72 #define EOFMARKLEN 79
     73 
     74 /* values returned by readtoken */
     75 #include "token.h"
     76 
     77 #define OPENBRACE '{'
     78 #define CLOSEBRACE '}'
     79 
     80 
     81 struct heredoc {
     82 	struct heredoc *next;	/* next here document in list */
     83 	union node *here;		/* redirection node */
     84 	char *eofmark;		/* string indicating end of input */
     85 	int striptabs;		/* if set, strip leading tabs */
     86 };
     87 
     88 
     89 
     90 static int noalias = 0;		/* when set, don't handle aliases */
     91 struct heredoc *heredoclist;	/* list of here documents to read */
     92 int parsebackquote;		/* nonzero if we are inside backquotes */
     93 int doprompt;			/* if set, prompt the user */
     94 int needprompt;			/* true if interactive and at start of line */
     95 int lasttoken;			/* last token read */
     96 MKINIT int tokpushback;		/* last token pushed back */
     97 char *wordtext;			/* text of last word returned by readtoken */
     98 MKINIT int checkkwd;		/* 1 == check for kwds, 2 == also eat newlines */
     99 struct nodelist *backquotelist;
    100 union node *redirnode;
    101 struct heredoc *heredoc;
    102 int quoteflag;			/* set if (part of) last token was quoted */
    103 int startlinno;			/* line # where last token started */
    104 int funclinno;			/* line # where the current function started */
    105 
    106 
    107 STATIC union node *list(int, int);
    108 STATIC union node *andor(void);
    109 STATIC union node *pipeline(void);
    110 STATIC union node *command(void);
    111 STATIC union node *simplecmd(union node **, union node *);
    112 STATIC union node *makename(void);
    113 STATIC void parsefname(void);
    114 STATIC void parseheredoc(void);
    115 STATIC int peektoken(void);
    116 STATIC int readtoken(void);
    117 STATIC int xxreadtoken(void);
    118 STATIC int readtoken1(int, char const *, char *, int);
    119 STATIC int noexpand(char *);
    120 STATIC void synexpect(int) __dead;
    121 STATIC void synerror(const char *) __dead;
    122 STATIC void setprompt(int);
    123 
    124 
    125 /*
    126  * Read and parse a command.  Returns NEOF on end of file.  (NULL is a
    127  * valid parse tree indicating a blank line.)
    128  */
    129 
    130 union node *
    131 parsecmd(int interact)
    132 {
    133 	int t;
    134 
    135 	tokpushback = 0;
    136 	doprompt = interact;
    137 	if (doprompt)
    138 		setprompt(1);
    139 	else
    140 		setprompt(0);
    141 	needprompt = 0;
    142 	t = readtoken();
    143 	if (t == TEOF)
    144 		return NEOF;
    145 	if (t == TNL)
    146 		return NULL;
    147 	tokpushback++;
    148 	return list(1, 0);
    149 }
    150 
    151 
    152 STATIC union node *
    153 list(int nlflag, int erflag)
    154 {
    155 	union node *n1, *n2, *n3;
    156 	int tok;
    157 	TRACE(("list: entered\n"));
    158 
    159 	checkkwd = 2;
    160 	if (nlflag == 0 && tokendlist[peektoken()])
    161 		return NULL;
    162 	n1 = NULL;
    163 	for (;;) {
    164 		n2 = andor();
    165 		tok = readtoken();
    166 		if (tok == TBACKGND) {
    167 			if (n2->type == NCMD || n2->type == NPIPE) {
    168 				n2->ncmd.backgnd = 1;
    169 			} else if (n2->type == NREDIR) {
    170 				n2->type = NBACKGND;
    171 			} else {
    172 				n3 = (union node *)stalloc(sizeof (struct nredir));
    173 				n3->type = NBACKGND;
    174 				n3->nredir.n = n2;
    175 				n3->nredir.redirect = NULL;
    176 				n2 = n3;
    177 			}
    178 		}
    179 		if (n1 == NULL) {
    180 			n1 = n2;
    181 		}
    182 		else {
    183 			n3 = (union node *)stalloc(sizeof (struct nbinary));
    184 			n3->type = NSEMI;
    185 			n3->nbinary.ch1 = n1;
    186 			n3->nbinary.ch2 = n2;
    187 			n1 = n3;
    188 		}
    189 		switch (tok) {
    190 		case TBACKGND:
    191 		case TSEMI:
    192 			tok = readtoken();
    193 			/* fall through */
    194 		case TNL:
    195 			if (tok == TNL) {
    196 				parseheredoc();
    197 				if (nlflag)
    198 					return n1;
    199 			} else {
    200 				tokpushback++;
    201 			}
    202 			checkkwd = 2;
    203 			if (tokendlist[peektoken()])
    204 				return n1;
    205 			break;
    206 		case TEOF:
    207 			if (heredoclist)
    208 				parseheredoc();
    209 			else
    210 				pungetc();		/* push back EOF on input */
    211 			return n1;
    212 		default:
    213 			if (nlflag || erflag)
    214 				synexpect(-1);
    215 			tokpushback++;
    216 			return n1;
    217 		}
    218 	}
    219 }
    220 
    221 
    222 
    223 STATIC union node *
    224 andor(void)
    225 {
    226 	union node *n1, *n2, *n3;
    227 	int t;
    228 
    229 	TRACE(("andor: entered\n"));
    230 	n1 = pipeline();
    231 	for (;;) {
    232 		if ((t = readtoken()) == TAND) {
    233 			t = NAND;
    234 		} else if (t == TOR) {
    235 			t = NOR;
    236 		} else {
    237 			tokpushback++;
    238 			return n1;
    239 		}
    240 		n2 = pipeline();
    241 		n3 = (union node *)stalloc(sizeof (struct nbinary));
    242 		n3->type = t;
    243 		n3->nbinary.ch1 = n1;
    244 		n3->nbinary.ch2 = n2;
    245 		n1 = n3;
    246 	}
    247 }
    248 
    249 
    250 
    251 STATIC union node *
    252 pipeline(void)
    253 {
    254 	union node *n1, *n2, *pipenode;
    255 	struct nodelist *lp, *prev;
    256 	int negate;
    257 
    258 	TRACE(("pipeline: entered\n"));
    259 
    260 	negate = 0;
    261 	checkkwd = 2;
    262 	while (readtoken() == TNOT) {
    263 		TRACE(("pipeline: TNOT recognized\n"));
    264 		negate = !negate;
    265 	}
    266 	tokpushback++;
    267 	n1 = command();
    268 	if (readtoken() == TPIPE) {
    269 		pipenode = (union node *)stalloc(sizeof (struct npipe));
    270 		pipenode->type = NPIPE;
    271 		pipenode->npipe.backgnd = 0;
    272 		lp = (struct nodelist *)stalloc(sizeof (struct nodelist));
    273 		pipenode->npipe.cmdlist = lp;
    274 		lp->n = n1;
    275 		do {
    276 			prev = lp;
    277 			lp = (struct nodelist *)stalloc(sizeof (struct nodelist));
    278 			lp->n = command();
    279 			prev->next = lp;
    280 		} while (readtoken() == TPIPE);
    281 		lp->next = NULL;
    282 		n1 = pipenode;
    283 	}
    284 	tokpushback++;
    285 	if (negate) {
    286 		TRACE(("negate pipeline\n"));
    287 		n2 = (union node *)stalloc(sizeof (struct nnot));
    288 		n2->type = NNOT;
    289 		n2->nnot.com = n1;
    290 		return n2;
    291 	} else
    292 		return n1;
    293 }
    294 
    295 
    296 
    297 STATIC union node *
    298 command(void)
    299 {
    300 	union node *n1, *n2;
    301 	union node *ap, **app;
    302 	union node *cp, **cpp;
    303 	union node *redir, **rpp;
    304 	int t, negate = 0;
    305 
    306 	TRACE(("command: entered\n"));
    307 
    308 	checkkwd = 2;
    309 	redir = NULL;
    310 	n1 = NULL;
    311 	rpp = &redir;
    312 
    313 	/* Check for redirection which may precede command */
    314 	while (readtoken() == TREDIR) {
    315 		*rpp = n2 = redirnode;
    316 		rpp = &n2->nfile.next;
    317 		parsefname();
    318 	}
    319 	tokpushback++;
    320 
    321 	while (readtoken() == TNOT) {
    322 		TRACE(("command: TNOT recognized\n"));
    323 		negate = !negate;
    324 	}
    325 	tokpushback++;
    326 
    327 	switch (readtoken()) {
    328 	case TIF:
    329 		n1 = (union node *)stalloc(sizeof (struct nif));
    330 		n1->type = NIF;
    331 		n1->nif.test = list(0, 0);
    332 		if (readtoken() != TTHEN)
    333 			synexpect(TTHEN);
    334 		n1->nif.ifpart = list(0, 0);
    335 		n2 = n1;
    336 		while (readtoken() == TELIF) {
    337 			n2->nif.elsepart = (union node *)stalloc(sizeof (struct nif));
    338 			n2 = n2->nif.elsepart;
    339 			n2->type = NIF;
    340 			n2->nif.test = list(0, 0);
    341 			if (readtoken() != TTHEN)
    342 				synexpect(TTHEN);
    343 			n2->nif.ifpart = list(0, 0);
    344 		}
    345 		if (lasttoken == TELSE)
    346 			n2->nif.elsepart = list(0, 0);
    347 		else {
    348 			n2->nif.elsepart = NULL;
    349 			tokpushback++;
    350 		}
    351 		if (readtoken() != TFI)
    352 			synexpect(TFI);
    353 		checkkwd = 1;
    354 		break;
    355 	case TWHILE:
    356 	case TUNTIL: {
    357 		int got;
    358 		n1 = (union node *)stalloc(sizeof (struct nbinary));
    359 		n1->type = (lasttoken == TWHILE)? NWHILE : NUNTIL;
    360 		n1->nbinary.ch1 = list(0, 0);
    361 		if ((got=readtoken()) != TDO) {
    362 TRACE(("expecting DO got %s %s\n", tokname[got], got == TWORD ? wordtext : ""));
    363 			synexpect(TDO);
    364 		}
    365 		n1->nbinary.ch2 = list(0, 0);
    366 		if (readtoken() != TDONE)
    367 			synexpect(TDONE);
    368 		checkkwd = 1;
    369 		break;
    370 	}
    371 	case TFOR:
    372 		if (readtoken() != TWORD || quoteflag || ! goodname(wordtext))
    373 			synerror("Bad for loop variable");
    374 		n1 = (union node *)stalloc(sizeof (struct nfor));
    375 		n1->type = NFOR;
    376 		n1->nfor.var = wordtext;
    377 		if (readtoken() == TWORD && ! quoteflag && equal(wordtext, "in")) {
    378 			app = &ap;
    379 			while (readtoken() == TWORD) {
    380 				n2 = (union node *)stalloc(sizeof (struct narg));
    381 				n2->type = NARG;
    382 				n2->narg.text = wordtext;
    383 				n2->narg.backquote = backquotelist;
    384 				*app = n2;
    385 				app = &n2->narg.next;
    386 			}
    387 			*app = NULL;
    388 			n1->nfor.args = ap;
    389 			if (lasttoken != TNL && lasttoken != TSEMI)
    390 				synexpect(-1);
    391 		} else {
    392 			static char argvars[5] = {
    393 			    CTLVAR, VSNORMAL|VSQUOTE, '@', '=', '\0'
    394 			};
    395 			n2 = (union node *)stalloc(sizeof (struct narg));
    396 			n2->type = NARG;
    397 			n2->narg.text = argvars;
    398 			n2->narg.backquote = NULL;
    399 			n2->narg.next = NULL;
    400 			n1->nfor.args = n2;
    401 			/*
    402 			 * Newline or semicolon here is optional (but note
    403 			 * that the original Bourne shell only allowed NL).
    404 			 */
    405 			if (lasttoken != TNL && lasttoken != TSEMI)
    406 				tokpushback++;
    407 		}
    408 		checkkwd = 2;
    409 		if ((t = readtoken()) == TDO)
    410 			t = TDONE;
    411 		else if (t == TBEGIN)
    412 			t = TEND;
    413 		else
    414 			synexpect(-1);
    415 		n1->nfor.body = list(0, 0);
    416 		if (readtoken() != t)
    417 			synexpect(t);
    418 		checkkwd = 1;
    419 		break;
    420 	case TCASE:
    421 		n1 = (union node *)stalloc(sizeof (struct ncase));
    422 		n1->type = NCASE;
    423 		if (readtoken() != TWORD)
    424 			synexpect(TWORD);
    425 		n1->ncase.expr = n2 = (union node *)stalloc(sizeof (struct narg));
    426 		n2->type = NARG;
    427 		n2->narg.text = wordtext;
    428 		n2->narg.backquote = backquotelist;
    429 		n2->narg.next = NULL;
    430 		while (readtoken() == TNL);
    431 		if (lasttoken != TWORD || ! equal(wordtext, "in"))
    432 			synerror("expecting \"in\"");
    433 		cpp = &n1->ncase.cases;
    434 		noalias = 1;
    435 		checkkwd = 2, readtoken();
    436 		/*
    437 		 * Both ksh and bash accept 'case x in esac'
    438 		 * so configure scripts started taking advantage of this.
    439 		 * The page: http://pubs.opengroup.org/onlinepubs/\
    440 		 * 009695399/utilities/xcu_chap02.html contradicts itself,
    441 		 * as to if this is legal; the "Case Conditional Format"
    442 		 * paragraph shows one case is required, but the "Grammar"
    443 		 * section shows a grammar that explicitly allows the no
    444 		 * case option.
    445 		 */
    446 		while (lasttoken != TESAC) {
    447 			*cpp = cp = (union node *)stalloc(sizeof (struct nclist));
    448 			if (lasttoken == TLP)
    449 				readtoken();
    450 			cp->type = NCLIST;
    451 			app = &cp->nclist.pattern;
    452 			for (;;) {
    453 				*app = ap = (union node *)stalloc(sizeof (struct narg));
    454 				ap->type = NARG;
    455 				ap->narg.text = wordtext;
    456 				ap->narg.backquote = backquotelist;
    457 				if (checkkwd = 2, readtoken() != TPIPE)
    458 					break;
    459 				app = &ap->narg.next;
    460 				readtoken();
    461 			}
    462 			ap->narg.next = NULL;
    463 			noalias = 0;
    464 			if (lasttoken != TRP) {
    465 				synexpect(TRP);
    466 			}
    467 			cp->nclist.body = list(0, 0);
    468 
    469 			checkkwd = 2;
    470 			if ((t = readtoken()) != TESAC) {
    471 				if (t != TENDCASE) {
    472 					noalias = 0;
    473 					synexpect(TENDCASE);
    474 				} else {
    475 					noalias = 1;
    476 					checkkwd = 2;
    477 					readtoken();
    478 				}
    479 			}
    480 			cpp = &cp->nclist.next;
    481 		}
    482 		noalias = 0;
    483 		*cpp = NULL;
    484 		checkkwd = 1;
    485 		break;
    486 	case TLP:
    487 		n1 = (union node *)stalloc(sizeof (struct nredir));
    488 		n1->type = NSUBSHELL;
    489 		n1->nredir.n = list(0, 0);
    490 		n1->nredir.redirect = NULL;
    491 		if (readtoken() != TRP)
    492 			synexpect(TRP);
    493 		checkkwd = 1;
    494 		break;
    495 	case TBEGIN:
    496 		n1 = list(0, 0);
    497 		if (readtoken() != TEND)
    498 			synexpect(TEND);
    499 		checkkwd = 1;
    500 		break;
    501 	/* Handle an empty command like other simple commands.  */
    502 	case TSEMI:
    503 		/*
    504 		 * An empty command before a ; doesn't make much sense, and
    505 		 * should certainly be disallowed in the case of `if ;'.
    506 		 */
    507 		if (!redir)
    508 			synexpect(-1);
    509 	case TAND:
    510 	case TOR:
    511 	case TNL:
    512 	case TEOF:
    513 	case TWORD:
    514 	case TRP:
    515 		tokpushback++;
    516 		n1 = simplecmd(rpp, redir);
    517 		goto checkneg;
    518 	case TENDCASE:
    519 		if (redir) {
    520 			tokpushback++;
    521 			goto checkneg;
    522 		}
    523 		/* FALLTHROUGH */
    524 	default:
    525 		synexpect(-1);
    526 		/* NOTREACHED */
    527 	}
    528 
    529 	/* Now check for redirection which may follow command */
    530 	while (readtoken() == TREDIR) {
    531 		*rpp = n2 = redirnode;
    532 		rpp = &n2->nfile.next;
    533 		parsefname();
    534 	}
    535 	tokpushback++;
    536 	*rpp = NULL;
    537 	if (redir) {
    538 		if (n1->type != NSUBSHELL) {
    539 			n2 = (union node *)stalloc(sizeof (struct nredir));
    540 			n2->type = NREDIR;
    541 			n2->nredir.n = n1;
    542 			n1 = n2;
    543 		}
    544 		n1->nredir.redirect = redir;
    545 	}
    546 
    547 checkneg:
    548 	if (negate) {
    549 		TRACE(("negate command\n"));
    550 		n2 = (union node *)stalloc(sizeof (struct nnot));
    551 		n2->type = NNOT;
    552 		n2->nnot.com = n1;
    553 		return n2;
    554 	}
    555 	else
    556 		return n1;
    557 }
    558 
    559 
    560 STATIC union node *
    561 simplecmd(union node **rpp, union node *redir)
    562 {
    563 	union node *args, **app;
    564 	union node **orig_rpp = rpp;
    565 	union node *n = NULL, *n2;
    566 	int negate = 0;
    567 
    568 	/* If we don't have any redirections already, then we must reset */
    569 	/* rpp to be the address of the local redir variable.  */
    570 	if (redir == 0)
    571 		rpp = &redir;
    572 
    573 	args = NULL;
    574 	app = &args;
    575 	/*
    576 	 * We save the incoming value, because we need this for shell
    577 	 * functions.  There can not be a redirect or an argument between
    578 	 * the function name and the open parenthesis.
    579 	 */
    580 	orig_rpp = rpp;
    581 
    582 	while (readtoken() == TNOT) {
    583 		TRACE(("simplcmd: TNOT recognized\n"));
    584 		negate = !negate;
    585 	}
    586 	tokpushback++;
    587 
    588 	for (;;) {
    589 		if (readtoken() == TWORD) {
    590 			n = (union node *)stalloc(sizeof (struct narg));
    591 			n->type = NARG;
    592 			n->narg.text = wordtext;
    593 			n->narg.backquote = backquotelist;
    594 			*app = n;
    595 			app = &n->narg.next;
    596 		} else if (lasttoken == TREDIR) {
    597 			*rpp = n = redirnode;
    598 			rpp = &n->nfile.next;
    599 			parsefname();	/* read name of redirection file */
    600 		} else if (lasttoken == TLP && app == &args->narg.next
    601 					    && rpp == orig_rpp) {
    602 			/* We have a function */
    603 			if (readtoken() != TRP)
    604 				synexpect(TRP);
    605 			funclinno = plinno;
    606 			rmescapes(n->narg.text);
    607 			if (!goodname(n->narg.text))
    608 				synerror("Bad function name");
    609 			n->type = NDEFUN;
    610 			n->narg.next = command();
    611 			funclinno = 0;
    612 			goto checkneg;
    613 		} else {
    614 			tokpushback++;
    615 			break;
    616 		}
    617 	}
    618 	*app = NULL;
    619 	*rpp = NULL;
    620 	n = (union node *)stalloc(sizeof (struct ncmd));
    621 	n->type = NCMD;
    622 	n->ncmd.backgnd = 0;
    623 	n->ncmd.args = args;
    624 	n->ncmd.redirect = redir;
    625 
    626 checkneg:
    627 	if (negate) {
    628 		TRACE(("negate simplecmd\n"));
    629 		n2 = (union node *)stalloc(sizeof (struct nnot));
    630 		n2->type = NNOT;
    631 		n2->nnot.com = n;
    632 		return n2;
    633 	}
    634 	else
    635 		return n;
    636 }
    637 
    638 STATIC union node *
    639 makename(void)
    640 {
    641 	union node *n;
    642 
    643 	n = (union node *)stalloc(sizeof (struct narg));
    644 	n->type = NARG;
    645 	n->narg.next = NULL;
    646 	n->narg.text = wordtext;
    647 	n->narg.backquote = backquotelist;
    648 	return n;
    649 }
    650 
    651 void
    652 fixredir(union node *n, const char *text, int err)
    653 {
    654 	TRACE(("Fix redir %s %d\n", text, err));
    655 	if (!err)
    656 		n->ndup.vname = NULL;
    657 
    658 	if (is_number(text))
    659 		n->ndup.dupfd = number(text);
    660 	else if (text[0] == '-' && text[1] == '\0')
    661 		n->ndup.dupfd = -1;
    662 	else {
    663 
    664 		if (err)
    665 			synerror("Bad fd number");
    666 		else
    667 			n->ndup.vname = makename();
    668 	}
    669 }
    670 
    671 
    672 STATIC void
    673 parsefname(void)
    674 {
    675 	union node *n = redirnode;
    676 
    677 	if (readtoken() != TWORD)
    678 		synexpect(-1);
    679 	if (n->type == NHERE) {
    680 		struct heredoc *here = heredoc;
    681 		struct heredoc *p;
    682 		int i;
    683 
    684 		if (quoteflag == 0)
    685 			n->type = NXHERE;
    686 		TRACE(("Here document %d\n", n->type));
    687 		if (here->striptabs) {
    688 			while (*wordtext == '\t')
    689 				wordtext++;
    690 		}
    691 		if (! noexpand(wordtext) || (i = strlen(wordtext)) == 0 || i > EOFMARKLEN)
    692 			synerror("Illegal eof marker for << redirection");
    693 		rmescapes(wordtext);
    694 		here->eofmark = wordtext;
    695 		here->next = NULL;
    696 		if (heredoclist == NULL)
    697 			heredoclist = here;
    698 		else {
    699 			for (p = heredoclist ; p->next ; p = p->next)
    700 				continue;
    701 			p->next = here;
    702 		}
    703 	} else if (n->type == NTOFD || n->type == NFROMFD) {
    704 		fixredir(n, wordtext, 0);
    705 	} else {
    706 		n->nfile.fname = makename();
    707 	}
    708 }
    709 
    710 
    711 /*
    712  * Input any here documents.
    713  */
    714 
    715 STATIC void
    716 parseheredoc(void)
    717 {
    718 	struct heredoc *here;
    719 	union node *n;
    720 
    721 	while (heredoclist) {
    722 		here = heredoclist;
    723 		heredoclist = here->next;
    724 		if (needprompt) {
    725 			setprompt(2);
    726 			needprompt = 0;
    727 		}
    728 		readtoken1(pgetc(), here->here->type == NHERE? SQSYNTAX : DQSYNTAX,
    729 				here->eofmark, here->striptabs);
    730 		n = (union node *)stalloc(sizeof (struct narg));
    731 		n->narg.type = NARG;
    732 		n->narg.next = NULL;
    733 		n->narg.text = wordtext;
    734 		n->narg.backquote = backquotelist;
    735 		here->here->nhere.doc = n;
    736 	}
    737 }
    738 
    739 STATIC int
    740 peektoken(void)
    741 {
    742 	int t;
    743 
    744 	t = readtoken();
    745 	tokpushback++;
    746 	return (t);
    747 }
    748 
    749 STATIC int
    750 readtoken(void)
    751 {
    752 	int t;
    753 	int savecheckkwd = checkkwd;
    754 #ifdef DEBUG
    755 	int alreadyseen = tokpushback;
    756 #endif
    757 	struct alias *ap;
    758 
    759 	top:
    760 	t = xxreadtoken();
    761 
    762 	if (checkkwd) {
    763 		/*
    764 		 * eat newlines
    765 		 */
    766 		if (checkkwd == 2) {
    767 			checkkwd = 0;
    768 			while (t == TNL) {
    769 				parseheredoc();
    770 				t = xxreadtoken();
    771 			}
    772 		} else
    773 			checkkwd = 0;
    774 		/*
    775 		 * check for keywords and aliases
    776 		 */
    777 		if (t == TWORD && !quoteflag) {
    778 			const char *const *pp;
    779 
    780 			for (pp = parsekwd; *pp; pp++) {
    781 				if (**pp == *wordtext && equal(*pp, wordtext)) {
    782 					lasttoken = t = pp -
    783 					    parsekwd + KWDOFFSET;
    784 					TRACE(("keyword %s recognized\n", tokname[t]));
    785 					goto out;
    786 				}
    787 			}
    788 			if (!noalias &&
    789 			    (ap = lookupalias(wordtext, 1)) != NULL) {
    790 				pushstring(ap->val, strlen(ap->val), ap);
    791 				checkkwd = savecheckkwd;
    792 				goto top;
    793 			}
    794 		}
    795 out:
    796 		checkkwd = (t == TNOT) ? savecheckkwd : 0;
    797 	}
    798 	TRACE(("%stoken %s %s\n", alreadyseen ? "reread " : "", tokname[t], t == TWORD ? wordtext : ""));
    799 	return (t);
    800 }
    801 
    802 
    803 /*
    804  * Read the next input token.
    805  * If the token is a word, we set backquotelist to the list of cmds in
    806  *	backquotes.  We set quoteflag to true if any part of the word was
    807  *	quoted.
    808  * If the token is TREDIR, then we set redirnode to a structure containing
    809  *	the redirection.
    810  * In all cases, the variable startlinno is set to the number of the line
    811  *	on which the token starts.
    812  *
    813  * [Change comment:  here documents and internal procedures]
    814  * [Readtoken shouldn't have any arguments.  Perhaps we should make the
    815  *  word parsing code into a separate routine.  In this case, readtoken
    816  *  doesn't need to have any internal procedures, but parseword does.
    817  *  We could also make parseoperator in essence the main routine, and
    818  *  have parseword (readtoken1?) handle both words and redirection.]
    819  */
    820 
    821 #define RETURN(token)	return lasttoken = token
    822 
    823 STATIC int
    824 xxreadtoken(void)
    825 {
    826 	int c;
    827 
    828 	if (tokpushback) {
    829 		tokpushback = 0;
    830 		return lasttoken;
    831 	}
    832 	if (needprompt) {
    833 		setprompt(2);
    834 		needprompt = 0;
    835 	}
    836 	startlinno = plinno;
    837 	for (;;) {	/* until token or start of word found */
    838 		c = pgetc_macro();
    839 		switch (c) {
    840 		case ' ': case '\t':
    841 			continue;
    842 		case '#':
    843 			while ((c = pgetc()) != '\n' && c != PEOF)
    844 				continue;
    845 			pungetc();
    846 			continue;
    847 		case '\\':
    848 			switch (pgetc()) {
    849 			case '\n':
    850 				startlinno = ++plinno;
    851 				if (doprompt)
    852 					setprompt(2);
    853 				else
    854 					setprompt(0);
    855 				continue;
    856 			case PEOF:
    857 				RETURN(TEOF);
    858 			default:
    859 				pungetc();
    860 				break;
    861 			}
    862 			goto breakloop;
    863 		case '\n':
    864 			plinno++;
    865 			needprompt = doprompt;
    866 			RETURN(TNL);
    867 		case PEOF:
    868 			RETURN(TEOF);
    869 		case '&':
    870 			if (pgetc() == '&')
    871 				RETURN(TAND);
    872 			pungetc();
    873 			RETURN(TBACKGND);
    874 		case '|':
    875 			if (pgetc() == '|')
    876 				RETURN(TOR);
    877 			pungetc();
    878 			RETURN(TPIPE);
    879 		case ';':
    880 			if (pgetc() == ';')
    881 				RETURN(TENDCASE);
    882 			pungetc();
    883 			RETURN(TSEMI);
    884 		case '(':
    885 			RETURN(TLP);
    886 		case ')':
    887 			RETURN(TRP);
    888 		default:
    889 			goto breakloop;
    890 		}
    891 	}
    892 breakloop:
    893 	return readtoken1(c, BASESYNTAX, NULL, 0);
    894 #undef RETURN
    895 }
    896 
    897 
    898 
    899 /*
    900  * If eofmark is NULL, read a word or a redirection symbol.  If eofmark
    901  * is not NULL, read a here document.  In the latter case, eofmark is the
    902  * word which marks the end of the document and striptabs is true if
    903  * leading tabs should be stripped from the document.  The argument firstc
    904  * is the first character of the input token or document.
    905  *
    906  * Because C does not have internal subroutines, I have simulated them
    907  * using goto's to implement the subroutine linkage.  The following macros
    908  * will run code that appears at the end of readtoken1.
    909  */
    910 
    911 /*
    912  * We used to remember only the current syntax, variable nesting level,
    913  * double quote state for each var nexting level, and arith nesting
    914  * level (unrelated to var nesting) and one prev syntax when in arith
    915  * syntax.  This worked for simple cases, but can't handle arith inside
    916  * var expansion inside arith inside var with some quoted and some not.
    917  *
    918  * Inspired by FreeBSD's implementation (though it was the obvious way)
    919  * though implemented differently, we now have a stack that keeps track
    920  * of what we are doing now, and what we were doing previously.
    921  * Every time something changes, which will eventually end and should
    922  * revert to the previous state, we push this stack, and then pop it
    923  * again later (that is every ${} with an operator (to parse the word
    924  * or pattern that follows) ${x} and $x are too * simple to need it)
    925  * $(( )) $( ) and "...".   Always.   Really, always!
    926  *
    927  * The stack is implemented as one static (on the C stack) base block
    928  * containing LEVELS_PER_BLOCK (8) stack entries, which should be
    929  * enough for the vast majority of cases.  For torture tests, we
    930  * malloc more blocks as needed.  All accesses through the inline
    931  * functions below.
    932  */
    933 
    934 /*
    935  * varnest & arinest will typically be 0 or 1
    936  * (varnest can increment in usages like ${x=${y}} but probably
    937  *  does not really need to)
    938  * parenlevel allows balancing parens inside a $(( )), it is reset
    939  * at each new nesting level ( $(( ( x + 3 ${unset-)} )) does not work.
    940  * quoted is special - we need to know 2 things ... are we inside "..."
    941  * (even if inherited from some previous nesting level) and was there
    942  * an opening '"' at this level (so the next will be closing).
    943  * "..." can span nexting levels, but cannot be opened in one and
    944  * closed in a different one.
    945  * To handle this, "quoted" has two fields, the bottom 4 (really 2)
    946  * bits are 0, 1, or 2, for un, single, and double quoted (single quoted
    947  * is really so special that this setting is not very important)
    948  * and 0x10 that indicates that an opening quote has been seen.
    949  * The bottom 4 bits are inherited, the 0x10 bit is not.
    950  */
    951 struct tokenstate {
    952 	const char *ts_syntax;
    953 	unsigned short ts_parenlevel;	/* counters */
    954 	unsigned short ts_varnest;	/* 64000 levels should be enough! */
    955 	unsigned short ts_arinest;
    956 	unsigned short ts_quoted;	/* 1 -> single, 2 -> double */
    957 };
    958 
    959 #define	NQ	0x00
    960 #define	SQ	0x01
    961 #define	DQ	0x02
    962 #define	QF	0x0F
    963 #define	QS	0x10
    964 
    965 #define	LEVELS_PER_BLOCK	8
    966 #define	VSS			volatile struct statestack
    967 
    968 struct statestack {
    969 	VSS *prev;		/* previous block in list */
    970 	int cur;		/* which of our tokenstates is current */
    971 	struct tokenstate tokenstate[LEVELS_PER_BLOCK];
    972 };
    973 
    974 static inline volatile struct tokenstate *
    975 currentstate(VSS *stack)
    976 {
    977 	return &stack->tokenstate[stack->cur];
    978 }
    979 
    980 static inline volatile struct tokenstate *
    981 prevstate(VSS *stack)
    982 {
    983 	if (stack->cur != 0)
    984 		return &stack->tokenstate[stack->cur - 1];
    985 	if (stack->prev == NULL)	/* cannot drop below base */
    986 		return &stack->tokenstate[0];
    987 	return &stack->prev->tokenstate[LEVELS_PER_BLOCK - 1];
    988 }
    989 
    990 static inline VSS *
    991 bump_state_level(VSS *stack)
    992 {
    993 	volatile struct tokenstate *os, *ts;
    994 
    995 	os = currentstate(stack);
    996 
    997 	if (++stack->cur >= LEVELS_PER_BLOCK) {
    998 		VSS *ss;
    999 
   1000 		ss = (VSS *)ckmalloc(sizeof (struct statestack));
   1001 		ss->cur = 0;
   1002 		ss->prev = stack;
   1003 		stack = ss;
   1004 	}
   1005 
   1006 	ts = currentstate(stack);
   1007 
   1008 	ts->ts_parenlevel = 0;	/* parens inside never match outside */
   1009 
   1010 	ts->ts_quoted  = os->ts_quoted & QF;	/* these are default settings */
   1011 	ts->ts_varnest = os->ts_varnest;
   1012 	ts->ts_arinest = os->ts_arinest;	/* when appropriate	   */
   1013 	ts->ts_syntax  = os->ts_syntax;		/*    they will be altered */
   1014 
   1015 	return stack;
   1016 }
   1017 
   1018 static inline VSS *
   1019 drop_state_level(VSS *stack)
   1020 {
   1021 	if (stack->cur == 0) {
   1022 		VSS *ss;
   1023 
   1024 		ss = stack;
   1025 		stack = ss->prev;
   1026 		if (stack == NULL)
   1027 			return ss;
   1028 		ckfree(__UNVOLATILE(ss));
   1029 	}
   1030 	--stack->cur;
   1031 	return stack;
   1032 }
   1033 
   1034 static inline void
   1035 cleanup_state_stack(VSS *stack)
   1036 {
   1037 	while (stack->prev != NULL) {
   1038 		stack->cur = 0;
   1039 		stack = drop_state_level(stack);
   1040 	}
   1041 }
   1042 
   1043 #define	CHECKEND()	{goto checkend; checkend_return:;}
   1044 #define	PARSEREDIR()	{goto parseredir; parseredir_return:;}
   1045 #define	PARSESUB()	{goto parsesub; parsesub_return:;}
   1046 #define	PARSEBACKQOLD()	{oldstyle = 1; goto parsebackq; parsebackq_oldreturn:;}
   1047 #define	PARSEBACKQNEW()	{oldstyle = 0; goto parsebackq; parsebackq_newreturn:;}
   1048 #define	PARSEARITH()	{goto parsearith; parsearith_return:;}
   1049 
   1050 /*
   1051  * The following macros all assume the existance of a local var "stack"
   1052  * which contains a pointer to the current struct stackstate
   1053  */
   1054 
   1055 /*
   1056  * These are macros rather than inline funcs to avoid code churn as much
   1057  * as possible - they replace macros of the same name used previously.
   1058  */
   1059 #define	ISDBLQUOTE()	(currentstate(stack)->ts_quoted & QS)
   1060 #define	SETDBLQUOTE()	(currentstate(stack)->ts_quoted = QS | DQ)
   1061 #define	CLRDBLQUOTE()	(currentstate(stack)->ts_quoted =		\
   1062 			    stack->cur != 0 || stack->prev ?		\
   1063 				prevstate(stack)->ts_quoted & QF : 0)
   1064 
   1065 /*
   1066  * This set are just to avoid excess typing and line lengths...
   1067  * The ones that "look like" var names must be implemented to be lvalues
   1068  */
   1069 #define	syntax		(currentstate(stack)->ts_syntax)
   1070 #define	parenlevel	(currentstate(stack)->ts_parenlevel)
   1071 #define	varnest		(currentstate(stack)->ts_varnest)
   1072 #define	arinest		(currentstate(stack)->ts_arinest)
   1073 #define	quoted		(currentstate(stack)->ts_quoted)
   1074 #define	TS_PUSH()	(stack = bump_state_level(stack))
   1075 #define	TS_POP()	(stack = drop_state_level(stack))
   1076 
   1077 STATIC int
   1078 readtoken1(int firstc, char const *syn, char *eofmark, int striptabs)
   1079 {
   1080 	int c = firstc;
   1081 	char * volatile out;
   1082 	int len;
   1083 	char line[EOFMARKLEN + 1];
   1084 	struct nodelist *bqlist;
   1085 	volatile int quotef;
   1086 	volatile int oldstyle;
   1087 	VSS static_stack;
   1088 	VSS *stack = &static_stack;
   1089 
   1090 	stack->prev = NULL;
   1091 	stack->cur = 0;
   1092 
   1093 	syntax = syn;
   1094 
   1095 	startlinno = plinno;
   1096 	varnest = 0;
   1097 	if (syntax == DQSYNTAX) {
   1098 		SETDBLQUOTE();
   1099 	}
   1100 	quotef = 0;
   1101 	bqlist = NULL;
   1102 	arinest = 0;
   1103 	parenlevel = 0;
   1104 	quoted = 0;
   1105 
   1106 	STARTSTACKSTR(out);
   1107 	loop: {	/* for each line, until end of word */
   1108 #if ATTY
   1109 		if (c == '\034' && doprompt
   1110 		 && attyset() && ! equal(termval(), "emacs")) {
   1111 			attyline();
   1112 			if (syntax == BASESYNTAX)
   1113 				return readtoken();
   1114 			c = pgetc();
   1115 			goto loop;
   1116 		}
   1117 #endif
   1118 		CHECKEND();	/* set c to PEOF if at end of here document */
   1119 		for (;;) {	/* until end of line or end of word */
   1120 			CHECKSTRSPACE(4, out);	/* permit 4 calls to USTPUTC */
   1121 			switch(syntax[c]) {
   1122 			case CNL:	/* '\n' */
   1123 				if (syntax == BASESYNTAX)
   1124 					goto endword;	/* exit outer loop */
   1125 				USTPUTC(c, out);
   1126 				plinno++;
   1127 				if (doprompt)
   1128 					setprompt(2);
   1129 				else
   1130 					setprompt(0);
   1131 				c = pgetc();
   1132 				goto loop;		/* continue outer loop */
   1133 			case CWORD:
   1134 				USTPUTC(c, out);
   1135 				break;
   1136 			case CCTL:
   1137 				if (eofmark == NULL || ISDBLQUOTE())
   1138 					USTPUTC(CTLESC, out);
   1139 				USTPUTC(c, out);
   1140 				break;
   1141 			case CBACK:	/* backslash */
   1142 				c = pgetc();
   1143 				if (c == PEOF) {
   1144 					USTPUTC('\\', out);
   1145 					pungetc();
   1146 					break;
   1147 				}
   1148 				if (c == '\n') {
   1149 					plinno++;
   1150 					if (doprompt)
   1151 						setprompt(2);
   1152 					else
   1153 						setprompt(0);
   1154 					break;
   1155 				}
   1156 				quotef = 1;
   1157 				if (ISDBLQUOTE() && c != '\\' &&
   1158 				    c != '`' && c != '$' &&
   1159 				    (c != '"' || eofmark != NULL))
   1160 					USTPUTC('\\', out);
   1161 				if (SQSYNTAX[c] == CCTL)
   1162 					USTPUTC(CTLESC, out);
   1163 				else if (eofmark == NULL) {
   1164 					USTPUTC(CTLQUOTEMARK, out);
   1165 					USTPUTC(c, out);
   1166 					if (varnest != 0)
   1167 						USTPUTC(CTLQUOTEEND, out);
   1168 					break;
   1169 				}
   1170 				USTPUTC(c, out);
   1171 				break;
   1172 			case CSQUOTE:
   1173 				if (syntax != SQSYNTAX) {
   1174 					if (eofmark == NULL)
   1175 						USTPUTC(CTLQUOTEMARK, out);
   1176 					quotef = 1;
   1177 					TS_PUSH();
   1178 					syntax = SQSYNTAX;
   1179 					quoted = SQ;
   1180 					break;
   1181 				}
   1182 				if (eofmark != NULL && arinest == 0 &&
   1183 				    varnest == 0) {
   1184 					/* Ignore inside quoted here document */
   1185 					USTPUTC(c, out);
   1186 					break;
   1187 				}
   1188 				/* End of single quotes... */
   1189 				TS_POP();
   1190 				if (syntax == BASESYNTAX && varnest != 0)
   1191 					USTPUTC(CTLQUOTEEND, out);
   1192 				break;
   1193 			case CDQUOTE:
   1194 				if (eofmark != NULL && arinest == 0 &&
   1195 				    varnest == 0) {
   1196 					/* Ignore inside here document */
   1197 					USTPUTC(c, out);
   1198 					break;
   1199 				}
   1200 				quotef = 1;
   1201 				if (arinest) {
   1202 					if (ISDBLQUOTE()) {
   1203 						TS_POP();
   1204 					} else {
   1205 						TS_PUSH();
   1206 						syntax = DQSYNTAX;
   1207 						SETDBLQUOTE();
   1208 						USTPUTC(CTLQUOTEMARK, out);
   1209 					}
   1210 					break;
   1211 				}
   1212 				if (eofmark != NULL)
   1213 					break;
   1214 				if (ISDBLQUOTE()) {
   1215 					TS_POP();
   1216 					if (varnest != 0)
   1217 						USTPUTC(CTLQUOTEEND, out);
   1218 				} else {
   1219 					TS_PUSH();
   1220 					syntax = DQSYNTAX;
   1221 					SETDBLQUOTE();
   1222 					USTPUTC(CTLQUOTEMARK, out);
   1223 				}
   1224 				break;
   1225 			case CVAR:	/* '$' */
   1226 				PARSESUB();		/* parse substitution */
   1227 				break;
   1228 			case CENDVAR:	/* CLOSEBRACE */
   1229 				if (varnest > 0 && !ISDBLQUOTE()) {
   1230 					TS_POP();
   1231 					USTPUTC(CTLENDVAR, out);
   1232 				} else {
   1233 					USTPUTC(c, out);
   1234 				}
   1235 				break;
   1236 			case CLP:	/* '(' in arithmetic */
   1237 				parenlevel++;
   1238 				USTPUTC(c, out);
   1239 				break;
   1240 			case CRP:	/* ')' in arithmetic */
   1241 				if (parenlevel > 0) {
   1242 					USTPUTC(c, out);
   1243 					--parenlevel;
   1244 				} else {
   1245 					if (pgetc() == ')') {
   1246 						if (--arinest == 0) {
   1247 							TS_POP();
   1248 							USTPUTC(CTLENDARI, out);
   1249 						} else
   1250 							USTPUTC(')', out);
   1251 					} else {
   1252 						/*
   1253 						 * unbalanced parens
   1254 						 *  (don't 2nd guess - no error)
   1255 						 */
   1256 						pungetc();
   1257 						USTPUTC(')', out);
   1258 					}
   1259 				}
   1260 				break;
   1261 			case CBQUOTE:	/* '`' */
   1262 				PARSEBACKQOLD();
   1263 				break;
   1264 			case CEOF:
   1265 				goto endword;		/* exit outer loop */
   1266 			default:
   1267 				if (varnest == 0 && !ISDBLQUOTE())
   1268 					goto endword;	/* exit outer loop */
   1269 				USTPUTC(c, out);
   1270 			}
   1271 			c = pgetc_macro();
   1272 		}
   1273 	}
   1274 endword:
   1275 	if (syntax == ARISYNTAX) {
   1276 		cleanup_state_stack(stack);
   1277 		synerror("Missing '))'");
   1278 	}
   1279 	if (syntax != BASESYNTAX && /* ! parsebackquote && */ eofmark == NULL) {
   1280 		cleanup_state_stack(stack);
   1281 		synerror("Unterminated quoted string");
   1282 	}
   1283 	if (varnest != 0) {
   1284 		cleanup_state_stack(stack);
   1285 		startlinno = plinno;
   1286 		/* { */
   1287 		synerror("Missing '}'");
   1288 	}
   1289 	USTPUTC('\0', out);
   1290 	len = out - stackblock();
   1291 	out = stackblock();
   1292 	if (eofmark == NULL) {
   1293 		if ((c == '>' || c == '<')
   1294 		 && quotef == 0
   1295 		 && (*out == '\0' || is_number(out))) {
   1296 			PARSEREDIR();
   1297 			cleanup_state_stack(stack);
   1298 			return lasttoken = TREDIR;
   1299 		} else {
   1300 			pungetc();
   1301 		}
   1302 	}
   1303 	quoteflag = quotef;
   1304 	backquotelist = bqlist;
   1305 	grabstackblock(len);
   1306 	wordtext = out;
   1307 	cleanup_state_stack(stack);
   1308 	return lasttoken = TWORD;
   1309 /* end of readtoken routine */
   1310 
   1311 
   1312 
   1313 /*
   1314  * Check to see whether we are at the end of the here document.  When this
   1315  * is called, c is set to the first character of the next input line.  If
   1316  * we are at the end of the here document, this routine sets the c to PEOF.
   1317  */
   1318 
   1319 checkend: {
   1320 	if (eofmark) {
   1321 		if (striptabs) {
   1322 			while (c == '\t')
   1323 				c = pgetc();
   1324 		}
   1325 		if (c == *eofmark) {
   1326 			if (pfgets(line, sizeof line) != NULL) {
   1327 				char *p, *q;
   1328 
   1329 				p = line;
   1330 				for (q = eofmark + 1 ; *q && *p == *q ; p++, q++)
   1331 					continue;
   1332 				if ((*p == '\0' || *p == '\n') && *q == '\0') {
   1333 					c = PEOF;
   1334 					plinno++;
   1335 					needprompt = doprompt;
   1336 				} else {
   1337 					pushstring(line, strlen(line), NULL);
   1338 				}
   1339 			}
   1340 		}
   1341 	}
   1342 	goto checkend_return;
   1343 }
   1344 
   1345 
   1346 /*
   1347  * Parse a redirection operator.  The variable "out" points to a string
   1348  * specifying the fd to be redirected.  The variable "c" contains the
   1349  * first character of the redirection operator.
   1350  */
   1351 
   1352 parseredir: {
   1353 	char fd[64];
   1354 	union node *np;
   1355 	strlcpy(fd, out, sizeof(fd));
   1356 
   1357 	np = (union node *)stalloc(sizeof (struct nfile));
   1358 	if (c == '>') {
   1359 		np->nfile.fd = 1;
   1360 		c = pgetc();
   1361 		if (c == '>')
   1362 			np->type = NAPPEND;
   1363 		else if (c == '|')
   1364 			np->type = NCLOBBER;
   1365 		else if (c == '&')
   1366 			np->type = NTOFD;
   1367 		else {
   1368 			np->type = NTO;
   1369 			pungetc();
   1370 		}
   1371 	} else {	/* c == '<' */
   1372 		np->nfile.fd = 0;
   1373 		switch (c = pgetc()) {
   1374 		case '<':
   1375 			if (sizeof (struct nfile) != sizeof (struct nhere)) {
   1376 				np = (union node *)stalloc(sizeof (struct nhere));
   1377 				np->nfile.fd = 0;
   1378 			}
   1379 			np->type = NHERE;
   1380 			heredoc = (struct heredoc *)stalloc(sizeof (struct heredoc));
   1381 			heredoc->here = np;
   1382 			if ((c = pgetc()) == '-') {
   1383 				heredoc->striptabs = 1;
   1384 			} else {
   1385 				heredoc->striptabs = 0;
   1386 				pungetc();
   1387 			}
   1388 			break;
   1389 
   1390 		case '&':
   1391 			np->type = NFROMFD;
   1392 			break;
   1393 
   1394 		case '>':
   1395 			np->type = NFROMTO;
   1396 			break;
   1397 
   1398 		default:
   1399 			np->type = NFROM;
   1400 			pungetc();
   1401 			break;
   1402 		}
   1403 	}
   1404 	if (*fd != '\0')
   1405 		np->nfile.fd = number(fd);
   1406 	redirnode = np;
   1407 	goto parseredir_return;
   1408 }
   1409 
   1410 
   1411 /*
   1412  * Parse a substitution.  At this point, we have read the dollar sign
   1413  * and nothing else.
   1414  */
   1415 
   1416 parsesub: {
   1417 	char buf[10];
   1418 	int subtype;
   1419 	int typeloc;
   1420 	int flags;
   1421 	char *p;
   1422 	static const char types[] = "}-+?=";
   1423 	int i;
   1424 	int linno;
   1425 
   1426 	c = pgetc();
   1427 	if (c != '(' && c != OPENBRACE && !is_name(c) && !is_special(c)) {
   1428 		USTPUTC('$', out);
   1429 		pungetc();
   1430 	} else if (c == '(') {	/* $(command) or $((arith)) */
   1431 		if (pgetc() == '(') {
   1432 			PARSEARITH();
   1433 		} else {
   1434 			pungetc();
   1435 			PARSEBACKQNEW();
   1436 		}
   1437 	} else {
   1438 		USTPUTC(CTLVAR, out);
   1439 		typeloc = out - stackblock();
   1440 		USTPUTC(VSNORMAL, out);
   1441 		subtype = VSNORMAL;
   1442 		flags = 0;
   1443 		if (c == OPENBRACE) {
   1444 			c = pgetc();
   1445 			if (c == '#') {
   1446 				if ((c = pgetc()) == CLOSEBRACE)
   1447 					c = '#';
   1448 				else
   1449 					subtype = VSLENGTH;
   1450 			}
   1451 			else
   1452 				subtype = 0;
   1453 		}
   1454 		if (is_name(c)) {
   1455 			p = out;
   1456 			do {
   1457 				STPUTC(c, out);
   1458 				c = pgetc();
   1459 			} while (is_in_name(c));
   1460 			if (out - p == 6 && strncmp(p, "LINENO", 6) == 0) {
   1461 				/* Replace the variable name with the
   1462 				 * current line number. */
   1463 				linno = plinno;
   1464 				if (funclinno != 0)
   1465 					linno -= funclinno - 1;
   1466 				snprintf(buf, sizeof(buf), "%d", linno);
   1467 				STADJUST(-6, out);
   1468 				for (i = 0; buf[i] != '\0'; i++)
   1469 					STPUTC(buf[i], out);
   1470 				flags |= VSLINENO;
   1471 			}
   1472 		} else if (is_digit(c)) {
   1473 			do {
   1474 				USTPUTC(c, out);
   1475 				c = pgetc();
   1476 			} while (is_digit(c));
   1477 		}
   1478 		else if (is_special(c)) {
   1479 			USTPUTC(c, out);
   1480 			c = pgetc();
   1481 		}
   1482 		else {
   1483 badsub:
   1484 			cleanup_state_stack(stack);
   1485 			synerror("Bad substitution");
   1486 		}
   1487 
   1488 		STPUTC('=', out);
   1489 		if (subtype == 0) {
   1490 			switch (c) {
   1491 			case ':':
   1492 				flags |= VSNUL;
   1493 				c = pgetc();
   1494 				/*FALLTHROUGH*/
   1495 			default:
   1496 				p = strchr(types, c);
   1497 				if (p == NULL)
   1498 					goto badsub;
   1499 				subtype = p - types + VSNORMAL;
   1500 				break;
   1501 			case '%':
   1502 			case '#':
   1503 				{
   1504 					int cc = c;
   1505 					subtype = c == '#' ? VSTRIMLEFT :
   1506 							     VSTRIMRIGHT;
   1507 					c = pgetc();
   1508 					if (c == cc)
   1509 						subtype++;
   1510 					else
   1511 						pungetc();
   1512 					break;
   1513 				}
   1514 			}
   1515 		} else {
   1516 			pungetc();
   1517 		}
   1518 		if (ISDBLQUOTE() || arinest)
   1519 			flags |= VSQUOTE;
   1520 		*(stackblock() + typeloc) = subtype | flags;
   1521 		if (subtype != VSNORMAL) {
   1522 			TS_PUSH();
   1523 			varnest++;
   1524 			arinest = 0;
   1525 			if (subtype > VSASSIGN) {	/* # ## % %% */
   1526 				syntax = BASESYNTAX;
   1527 				CLRDBLQUOTE();
   1528 			}
   1529 		}
   1530 	}
   1531 	goto parsesub_return;
   1532 }
   1533 
   1534 
   1535 /*
   1536  * Called to parse command substitutions.  Newstyle is set if the command
   1537  * is enclosed inside $(...); nlpp is a pointer to the head of the linked
   1538  * list of commands (passed by reference), and savelen is the number of
   1539  * characters on the top of the stack which must be preserved.
   1540  */
   1541 
   1542 parsebackq: {
   1543 	struct nodelist **nlpp;
   1544 	int savepbq;
   1545 	union node *n;
   1546 	char *volatile str = NULL;
   1547 	struct jmploc jmploc;
   1548 	struct jmploc *volatile savehandler = NULL;
   1549 	int savelen;
   1550 	int saveprompt;
   1551 
   1552 	savepbq = parsebackquote;
   1553 	if (setjmp(jmploc.loc)) {
   1554 		if (str)
   1555 			ckfree(str);
   1556 		cleanup_state_stack(stack);
   1557 		parsebackquote = 0;
   1558 		handler = savehandler;
   1559 		longjmp(handler->loc, 1);
   1560 	}
   1561 	INTOFF;
   1562 	str = NULL;
   1563 	savelen = out - stackblock();
   1564 	if (savelen > 0) {
   1565 		str = ckmalloc(savelen);
   1566 		memcpy(str, stackblock(), savelen);
   1567 	}
   1568 	savehandler = handler;
   1569 	handler = &jmploc;
   1570 	INTON;
   1571         if (oldstyle) {
   1572                 /* We must read until the closing backquote, giving special
   1573                    treatment to some slashes, and then push the string and
   1574                    reread it as input, interpreting it normally.  */
   1575                 char *pout;
   1576                 int pc;
   1577                 int psavelen;
   1578                 char *pstr;
   1579 
   1580 		/*
   1581 		 * Because the entire `...` is read here, we don't
   1582 		 * need to bother the state stack.  That will be used
   1583 		 * (as appropriate) when the processed string is re-read.
   1584 		 */
   1585                 STARTSTACKSTR(pout);
   1586 		for (;;) {
   1587 			if (needprompt) {
   1588 				setprompt(2);
   1589 				needprompt = 0;
   1590 			}
   1591 			switch (pc = pgetc()) {
   1592 			case '`':
   1593 				goto done;
   1594 
   1595 			case '\\':
   1596                                 if ((pc = pgetc()) == '\n') {
   1597 					plinno++;
   1598 					if (doprompt)
   1599 						setprompt(2);
   1600 					else
   1601 						setprompt(0);
   1602 					/*
   1603 					 * If eating a newline, avoid putting
   1604 					 * the newline into the new character
   1605 					 * stream (via the STPUTC after the
   1606 					 * switch).
   1607 					 */
   1608 					continue;
   1609 				}
   1610                                 if (pc != '\\' && pc != '`' && pc != '$'
   1611                                     && (!ISDBLQUOTE() || pc != '"'))
   1612                                         STPUTC('\\', pout);
   1613 				break;
   1614 
   1615 			case '\n':
   1616 				plinno++;
   1617 				needprompt = doprompt;
   1618 				break;
   1619 
   1620 			case PEOF:
   1621 			        startlinno = plinno;
   1622 				synerror("EOF in backquote substitution");
   1623  				break;
   1624 
   1625 			default:
   1626 				break;
   1627 			}
   1628 			STPUTC(pc, pout);
   1629                 }
   1630 done:
   1631                 STPUTC('\0', pout);
   1632                 psavelen = pout - stackblock();
   1633                 if (psavelen > 0) {
   1634 			pstr = grabstackstr(pout);
   1635 			setinputstring(pstr, 1);
   1636                 }
   1637         }
   1638 	nlpp = &bqlist;
   1639 	while (*nlpp)
   1640 		nlpp = &(*nlpp)->next;
   1641 	*nlpp = (struct nodelist *)stalloc(sizeof (struct nodelist));
   1642 	(*nlpp)->next = NULL;
   1643 	parsebackquote = oldstyle;
   1644 
   1645 	if (oldstyle) {
   1646 		saveprompt = doprompt;
   1647 		doprompt = 0;
   1648 	} else
   1649 		saveprompt = 0;
   1650 
   1651 	n = list(0, oldstyle);
   1652 
   1653 	if (oldstyle)
   1654 		doprompt = saveprompt;
   1655 	else {
   1656 		if (readtoken() != TRP) {
   1657 			cleanup_state_stack(stack);
   1658 			synexpect(TRP);
   1659 		}
   1660 	}
   1661 
   1662 	(*nlpp)->n = n;
   1663         if (oldstyle) {
   1664 		/*
   1665 		 * Start reading from old file again, ignoring any pushed back
   1666 		 * tokens left from the backquote parsing
   1667 		 */
   1668                 popfile();
   1669 		tokpushback = 0;
   1670 	}
   1671 	while (stackblocksize() <= savelen)
   1672 		growstackblock();
   1673 	STARTSTACKSTR(out);
   1674 	if (str) {
   1675 		memcpy(out, str, savelen);
   1676 		STADJUST(savelen, out);
   1677 		INTOFF;
   1678 		ckfree(str);
   1679 		str = NULL;
   1680 		INTON;
   1681 	}
   1682 	parsebackquote = savepbq;
   1683 	handler = savehandler;
   1684 	if (arinest || ISDBLQUOTE())
   1685 		USTPUTC(CTLBACKQ | CTLQUOTE, out);
   1686 	else
   1687 		USTPUTC(CTLBACKQ, out);
   1688 	if (oldstyle)
   1689 		goto parsebackq_oldreturn;
   1690 	else
   1691 		goto parsebackq_newreturn;
   1692 }
   1693 
   1694 /*
   1695  * Parse an arithmetic expansion (indicate start of one and set state)
   1696  */
   1697 parsearith: {
   1698 
   1699 	if (syntax == ARISYNTAX) {
   1700 		/*
   1701 		 * we collapse embedded arithmetic expansion to
   1702 		 * parentheses, which should be equivalent
   1703 		 */
   1704 		USTPUTC('(', out);
   1705 		USTPUTC('(', out);
   1706 		/*
   1707 		 * Need 2 of them because there will (should be)
   1708 		 * two closing ))'s to follow later.
   1709 		 */
   1710 		parenlevel += 2;
   1711 	} else {
   1712 		TS_PUSH();
   1713 		syntax = ARISYNTAX;
   1714 		++arinest;
   1715 		varnest = 0;
   1716 
   1717 		USTPUTC(CTLARI, out);
   1718 		if (ISDBLQUOTE())
   1719 			USTPUTC('"',out);
   1720 		else
   1721 			USTPUTC(' ',out);
   1722 	}
   1723 	goto parsearith_return;
   1724 }
   1725 
   1726 } /* end of readtoken */
   1727 
   1728 
   1729 
   1730 #ifdef mkinit
   1731 RESET {
   1732 	tokpushback = 0;
   1733 	checkkwd = 0;
   1734 }
   1735 #endif
   1736 
   1737 /*
   1738  * Returns true if the text contains nothing to expand (no dollar signs
   1739  * or backquotes).
   1740  */
   1741 
   1742 STATIC int
   1743 noexpand(char *text)
   1744 {
   1745 	char *p;
   1746 	char c;
   1747 
   1748 	p = text;
   1749 	while ((c = *p++) != '\0') {
   1750 		if (c == CTLQUOTEMARK)
   1751 			continue;
   1752 		if (c == CTLESC)
   1753 			p++;
   1754 		else if (BASESYNTAX[(int)c] == CCTL)
   1755 			return 0;
   1756 	}
   1757 	return 1;
   1758 }
   1759 
   1760 
   1761 /*
   1762  * Return true if the argument is a legal variable name (a letter or
   1763  * underscore followed by zero or more letters, underscores, and digits).
   1764  */
   1765 
   1766 int
   1767 goodname(char *name)
   1768 	{
   1769 	char *p;
   1770 
   1771 	p = name;
   1772 	if (! is_name(*p))
   1773 		return 0;
   1774 	while (*++p) {
   1775 		if (! is_in_name(*p))
   1776 			return 0;
   1777 	}
   1778 	return 1;
   1779 }
   1780 
   1781 
   1782 /*
   1783  * Called when an unexpected token is read during the parse.  The argument
   1784  * is the token that is expected, or -1 if more than one type of token can
   1785  * occur at this point.
   1786  */
   1787 
   1788 STATIC void
   1789 synexpect(int token)
   1790 {
   1791 	char msg[64];
   1792 
   1793 	if (token >= 0) {
   1794 		fmtstr(msg, 64, "%s unexpected (expecting %s)",
   1795 			tokname[lasttoken], tokname[token]);
   1796 	} else {
   1797 		fmtstr(msg, 64, "%s unexpected", tokname[lasttoken]);
   1798 	}
   1799 	synerror(msg);
   1800 	/* NOTREACHED */
   1801 }
   1802 
   1803 
   1804 STATIC void
   1805 synerror(const char *msg)
   1806 {
   1807 	if (commandname)
   1808 		outfmt(&errout, "%s: %d: ", commandname, startlinno);
   1809 	else
   1810 		outfmt(&errout, "%s: ", getprogname());
   1811 	outfmt(&errout, "Syntax error: %s\n", msg);
   1812 	error(NULL);
   1813 	/* NOTREACHED */
   1814 }
   1815 
   1816 STATIC void
   1817 setprompt(int which)
   1818 {
   1819 	whichprompt = which;
   1820 
   1821 #ifndef SMALL
   1822 	if (!el)
   1823 #endif
   1824 		out2str(getprompt(NULL));
   1825 }
   1826 
   1827 /*
   1828  * called by editline -- any expansions to the prompt
   1829  *    should be added here.
   1830  */
   1831 const char *
   1832 getprompt(void *unused)
   1833 	{
   1834 	switch (whichprompt) {
   1835 	case 0:
   1836 		return "";
   1837 	case 1:
   1838 		return ps1val();
   1839 	case 2:
   1840 		return ps2val();
   1841 	default:
   1842 		return "<internal prompt error>";
   1843 	}
   1844 }
   1845