Lines Matching refs:token
37 #define TOKEN struct tok
72 static TOKEN ltoken;
104 #define STATE(y, x) do { if (debug) printf("%lx State %s: token `%s'\n", outpos, x, y); } while (0)
492 { "set-token-table", 0x00b3, 0, NULL, NULL },
494 { "new-token
495 { "named-token", 0x00b6, 0, NULL, NULL },
514 { "external-token", 0x00ca, 0, NULL, NULL },
526 { "get-token", 0x00da, 0, NULL, NULL },
527 { "set-token", 0x00db, 0, NULL, NULL },
1082 TOKEN *token;
1087 while ((token = yylex()) != NULL) {
1088 switch (token->type) {
1090 STATE(token->text, "TOK_NUMBER");
1096 push(strtol(token->text, &end, 16));
1099 value = strtol(token->text, &end, numbase);
1126 STATE(token->text, "TOK_C_LIT");
1131 spit(token->text[1]);
1134 STATE(token->text, "TOK_STRING_LIT:");
1137 char *p = token->text;
1157 STATE(token->text, "TOK_PSTRING:");
1160 char *p = token->text;
1179 STATE(token->text, "TOK_PSTRING:");
1183 char *p = token->text;
1217 STATE(token->text, "TOK_TOKENIZE");
1222 STATE(token->text, "TOK_COMMENT:");
1229 STATE(token->text, "TOK_COLON:");
1231 token = yylex();
1232 if (token == NULL)
1239 fcode->name = estrdup(token->text);
1244 token->text);
1249 printf("Adding %s to dictionary\n", token->text);
1251 emit("new-token");
1254 emit("external-token");
1256 /* Here we have a choice of new-token or named-token */
1257 emit("named-token");
1258 sspit(token->text);
1266 STATE(token->text, "TOK_SEMICOLON:");
1279 STATE(token->text, "TOK_AGAIN");
1286 STATE(token->text, "TOK_ALIAS");
1290 token = yylex();
1291 if (token == NULL) {
1295 if (token->type != TOK_OTHER) {
1296 warnx("ENDCOMMENT aliasing weird token type %d",
1297 token->type);
1300 alias->name = estrdup(token->text);
1301 token = yylex();
1302 if (token == NULL) {
1308 alias->equiv = estrdup(token->text);
1316 STATE(token->text, "TOK_GETTOKEN");
1319 token = yylex();
1320 if (token == NULL) {
1324 if ((fcode = flookup(dictionary, token->text)) == NULL)
1326 token->text);
1330 STATE(token->text, "TOK_ASCII");
1331 token = yylex();
1332 if (token == NULL)
1338 spit(token->text[0]);
1341 STATE(token->text, "TOK_BEGIN");
1346 STATE(token->text, "TOK_BUFFER");
1348 token = yylex();
1349 if (token == NULL) {
1357 fcode->name = estrdup(token->text);
1361 emit("new-token");
1364 emit("external-token");
1366 /* Here we have a choice of new-token or named-token */
1367 emit("named-token");
1368 sspit(token->text);
1374 STATE(token->text, "TOK_CASE");
1379 STATE(token->text, "TOK_CONSTANT");
1381 token = yylex();
1382 if (token == NULL) {
1390 fcode->name = estrdup(token->text);
1394 emit("new-token");
1397 emit("external-token");
1399 /* Here we have a choice of new-token or named-token */
1400 emit("named-token");
1401 sspit(token->text);
1407 STATE(token->text, "TOK_CONTROL");
1408 token = yylex();
1409 if (token == NULL)
1415 spit(token->text[0]&0x1f);
1418 STATE(token->text, "TOK_CREATE");
1420 token = yylex();
1421 if (token == NULL) {
1429 fcode->name = estrdup(token->text);
1433 emit("new-token");
1436 emit("external-token");
1438 /* Here we have a choice of new-token or named-token */
1439 emit("named-token");
1440 sspit(token->text);
1446 STATE(token->text, "TOK_DECIMAL");
1447 if (token->text[1] != '#') {
1462 token = yylex();
1463 if (token == NULL) {
1467 if (token->type == TOK_OTHER) {
1468 if (strcmp("-1", token->text) == 0) {
1469 emit(token->text);
1473 value = strtol(token->text, &end, 10);
1476 "Illegal number conversion: %s", token->text);
1499 STATE(token->text, "TOK_DEFER");
1501 token = yylex();
1502 if (token == NULL) {
1510 fcode->name = estrdup(token->text);
1514 emit("new-token");
1517 emit("external-token");
1519 /* Here we have a choice of new-token or named-token */
1520 emit("named-token");
1521 sspit(token->text);
1527 STATE(token->text, "TOK_DO");
1546 if (token->text[0] == '?')
1555 STATE(token->text, "TOK_END0");
1561 STATE(token->text, "TOK_ELSE");
1578 STATE(token->text, "TOK_ENDCASE:");
1610 STATE(token->text, "TOK_ENDOF");
1634 STATE(token->text, "TOK_EXTERNAL");
1639 STATE(token->text, "TOK_FCODE_VERSION2");
1653 STATE(token->text, "TOK_FCODE_END");
1656 STATE(token->text, "TOK_FIELD");
1658 token = yylex();
1659 if (token == NULL) {
1667 fcode->name = estrdup(token->text);
1671 emit("new-token");
1674 emit("external-token");
1676 /* Here we have a choice of new-token or named-token */
1677 emit("named-token");
1678 sspit(token->text);
1685 STATE(token->text, "TOK_HEX");
1686 if (token->text[1] != '#') {
1701 token = yylex();
1702 if (token == NULL) {
1706 value = strtol(token->text, &end, 16);
1732 STATE(token->text, "TOK_HEADERLESS");
1736 STATE(token->text, "TOK_HEADERS");
1740 STATE(token->text, "TOK_IF");
1749 STATE(token->text, "TOK_LEAVE");
1753 STATE(token->text, "TOK_LOOP");
1755 if (token->text[0] == '+')
1772 STATE(token
1773 if (token->text[1] != '#') {
1784 token = yylex();
1785 if (token == NULL) {
1789 value = strtol(token->text, &end, 8);
1816 STATE(token->text, "TOK_OF");
1835 STATE(token->text, "TOK_OFFSET16");
1840 STATE(token->text, "TOK_REPEAT");
1858 STATE(token->text, "TOK_FCODE_VERSION2");
1862 emit(token->text);
1866 STATE(token->text, "TOK_THEN");
1875 STATE(token->text, "TOK_TO");
1880 STATE(token->text, "TOK_UNTIL");
1887 STATE(token->text, "TOK_VALUE");
1889 token = yylex();
1890 if (token == NULL) {
1898 fcode->name = estrdup(token->text);
1902 emit("new-token");
1905 emit("external-token");
1907 /* Here we have a choice of new-token or named-token */
1908 emit("named-token");
1909 sspit(token->text);
1915 STATE(token->text, "TOK_VARIABLE");
1917 token = yylex();
1918 if (token == NULL) {
1926 fcode->name = estrdup(token->text);
1930 emit("new-token");
1933 emit("external-token");
1935 /* Here we have a choice of new-token or named-token */
1936 emit("named-token");
1937 sspit(token->text);
1944 STATE(token->text, "TOK_FCODE_VERSION1");
1952 STATE(token->text, "TOK_WHILE");
1960 STATE(token->text, "TOK_BEGTOK");
1964 STATE(token->text, "TOK_EMIT_BYTE");
1968 STATE(token->text, "TOK_ENDTOK");
1975 STATE(token->text, "TOK_FLOAD");
1977 token = yylex();
1978 if ((inf = fopen(token->text, "r")) == NULL) {
1979 warn("Cannot open `%s'", token->text);
1982 infile = estrdup(token->text);
2028 STATE(token->text, "TOK_OTHER");
2029 if (apply_macros(yinput, token->text))
2031 if (emit(token->text)) {
2038 sspit(token->text);
2044 "%s: undefined token `%s'\n",
2045 __func__, token->text);
Indexes created Fri Oct 17 17:09:57 GMT 2025