subr_prf.c revision 1.169 1 /* $NetBSD: subr_prf.c,v 1.169 2018/04/14 01:45:37 kre Exp $ */
2
3 /*-
4 * Copyright (c) 1986, 1988, 1991, 1993
5 * The Regents of the University of California. All rights reserved.
6 * (c) UNIX System Laboratories, Inc.
7 * All or some portions of this file are derived from material licensed
8 * to the University of California by American Telephone and Telegraph
9 * Co. or Unix System Laboratories, Inc. and are reproduced herein with
10 * the permission of UNIX System Laboratories, Inc.
11 *
12 * Redistribution and use in source and binary forms, with or without
13 * modification, are permitted provided that the following conditions
14 * are met:
15 * 1. Redistributions of source code must retain the above copyright
16 * notice, this list of conditions and the following disclaimer.
17 * 2. Redistributions in binary form must reproduce the above copyright
18 * notice, this list of conditions and the following disclaimer in the
19 * documentation and/or other materials provided with the distribution.
20 * 3. Neither the name of the University nor the names of its contributors
21 * may be used to endorse or promote products derived from this software
22 * without specific prior written permission.
23 *
24 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
25 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
26 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
27 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
28 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
29 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
30 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
31 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
32 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
33 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
34 * SUCH DAMAGE.
35 *
36 * @(#)subr_prf.c 8.4 (Berkeley) 5/4/95
37 */
38
39 #include <sys/cdefs.h>
40 __KERNEL_RCSID(0, "$NetBSD: subr_prf.c,v 1.169 2018/04/14 01:45:37 kre Exp $");
41
42 #ifdef _KERNEL_OPT
43 #include "opt_ddb.h"
44 #include "opt_ipkdb.h"
45 #include "opt_kgdb.h"
46 #include "opt_dump.h"
47 #include "opt_rnd_printf.h"
48 #endif
49
50 #include <sys/param.h>
51 #include <sys/stdint.h>
52 #include <sys/systm.h>
53 #include <sys/buf.h>
54 #include <sys/device.h>
55 #include <sys/reboot.h>
56 #include <sys/msgbuf.h>
57 #include <sys/proc.h>
58 #include <sys/ioctl.h>
59 #include <sys/vnode.h>
60 #include <sys/file.h>
61 #include <sys/tty.h>
62 #include <sys/tprintf.h>
63 #include <sys/spldebug.h>
64 #include <sys/syslog.h>
65 #include <sys/kprintf.h>
66 #include <sys/atomic.h>
67 #include <sys/kernel.h>
68 #include <sys/cpu.h>
69 #include <sys/sha2.h>
70 #include <sys/rndsource.h>
71
72 #include <dev/cons.h>
73
74 #include <net/if.h>
75
76 #ifdef IPKDB
77 #include <ipkdb/ipkdb.h>
78 #endif
79
80 static kmutex_t kprintf_mtx;
81 static bool kprintf_inited = false;
82
83 #ifdef KGDB
84 #include <sys/kgdb.h>
85 #endif
86
87 #ifdef DDB
88 #include <ddb/ddbvar.h> /* db_panic */
89 #include <ddb/db_output.h> /* db_printf, db_putchar prototypes */
90 #endif
91
92
93 /*
94 * defines
95 */
96 #define KLOG_PRI 0x80000000
97
98
99 /*
100 * local prototypes
101 */
102
103 static void putchar(int, int, struct tty *);
104
105
106 /*
107 * globals
108 */
109
110 extern struct tty *constty; /* pointer to console "window" tty */
111 extern int log_open; /* subr_log: is /dev/klog open? */
112 extern krndsource_t rnd_printf_source;
113 const char *panicstr; /* arg to first call to panic (used as a flag
114 to indicate that panic has already been called). */
115 struct cpu_info *paniccpu; /* cpu that first paniced */
116 long panicstart, panicend; /* position in the msgbuf of the start and
117 end of the formatted panicstr. */
118 int doing_shutdown; /* set to indicate shutdown in progress */
119
120 #ifdef RND_PRINTF
121 static bool kprintf_inited_callout = false;
122 static SHA512_CTX kprnd_sha;
123 static uint8_t kprnd_accum[SHA512_DIGEST_LENGTH];
124 static int kprnd_added;
125
126 static struct callout kprnd_callout;
127 #endif
128
129 #ifndef DUMP_ON_PANIC
130 #define DUMP_ON_PANIC 1
131 #endif
132 int dumponpanic = DUMP_ON_PANIC;
133
134 /*
135 * v_putc: routine to putc on virtual console
136 *
137 * the v_putc pointer can be used to redirect the console cnputc elsewhere
138 * [e.g. to a "virtual console"].
139 */
140
141 void (*v_putc)(int) = cnputc; /* start with cnputc (normal cons) */
142 void (*v_flush)(void) = cnflush; /* start with cnflush (normal cons) */
143
144 const char hexdigits[] = "0123456789abcdef";
145 const char HEXDIGITS[] = "0123456789ABCDEF";
146
147
148 /*
149 * functions
150 */
151
152 #ifdef RND_PRINTF
153 static void kprintf_rnd_get(size_t bytes, void *priv)
154 {
155 if (kprnd_added) {
156 KASSERT(kprintf_inited);
157 if (mutex_tryenter(&kprintf_mtx)) {
158 SHA512_Final(kprnd_accum, &kprnd_sha);
159 rnd_add_data(&rnd_printf_source,
160 kprnd_accum, sizeof(kprnd_accum), 0);
161 kprnd_added = 0;
162 /* This, we must do, since we called _Final. */
163 SHA512_Init(&kprnd_sha);
164 /* This is optional but seems useful. */
165 SHA512_Update(&kprnd_sha, kprnd_accum,
166 sizeof(kprnd_accum));
167 mutex_exit(&kprintf_mtx);
168 }
169 }
170 }
171
172 static void kprintf_rnd_callout(void *arg)
173 {
174 kprintf_rnd_get(0, NULL);
175 callout_schedule(&kprnd_callout, hz);
176 }
177
178 #endif
179
180 /*
181 * Locking is inited fairly early in MI bootstrap. Before that
182 * prints are done unlocked. But that doesn't really matter,
183 * since nothing can preempt us before interrupts are enabled.
184 */
185 void
186 kprintf_init(void)
187 {
188
189 KASSERT(!kprintf_inited && cold); /* not foolproof, but ... */
190 #ifdef RND_PRINTF
191 SHA512_Init(&kprnd_sha);
192 #endif
193 mutex_init(&kprintf_mtx, MUTEX_DEFAULT, IPL_HIGH);
194 kprintf_inited = true;
195 }
196
197 #ifdef RND_PRINTF
198 void
199 kprintf_init_callout(void)
200 {
201 KASSERT(!kprintf_inited_callout);
202 callout_init(&kprnd_callout, CALLOUT_MPSAFE);
203 callout_setfunc(&kprnd_callout, kprintf_rnd_callout, NULL);
204 callout_schedule(&kprnd_callout, hz);
205 kprintf_inited_callout = true;
206 }
207 #endif
208
209 void
210 kprintf_lock(void)
211 {
212
213 if (__predict_true(kprintf_inited))
214 mutex_enter(&kprintf_mtx);
215 }
216
217 void
218 kprintf_unlock(void)
219 {
220
221 if (__predict_true(kprintf_inited)) {
222 /* assert kprintf wasn't somehow inited while we were in */
223 KASSERT(mutex_owned(&kprintf_mtx));
224 mutex_exit(&kprintf_mtx);
225 }
226 }
227
228 /*
229 * twiddle: spin a little propellor on the console.
230 */
231
232 void
233 twiddle(void)
234 {
235 static const char twiddle_chars[] = "|/-\\";
236 static int pos;
237
238 kprintf_lock();
239
240 putchar(twiddle_chars[pos++ & 3], TOCONS, NULL);
241 putchar('\b', TOCONS, NULL);
242
243 kprintf_unlock();
244 }
245
246 /*
247 * panic: handle an unresolvable fatal error
248 *
249 * prints "panic: <message>" and reboots. if called twice (i.e. recursive
250 * call) we avoid trying to dump and just reboot (to avoid recursive panics).
251 */
252
253 void
254 panic(const char *fmt, ...)
255 {
256 va_list ap;
257
258 va_start(ap, fmt);
259 vpanic(fmt, ap);
260 va_end(ap);
261 }
262
263 void
264 vpanic(const char *fmt, va_list ap)
265 {
266 CPU_INFO_ITERATOR cii;
267 struct cpu_info *ci, *oci;
268 int bootopt;
269 static char scratchstr[384]; /* stores panic message */
270
271 spldebug_stop();
272
273 if (lwp0.l_cpu && curlwp) {
274 /*
275 * Disable preemption. If already panicing on another CPU, sit
276 * here and spin until the system is rebooted. Allow the CPU that
277 * first paniced to panic again.
278 */
279 kpreempt_disable();
280 ci = curcpu();
281 oci = atomic_cas_ptr((void *)&paniccpu, NULL, ci);
282 if (oci != NULL && oci != ci) {
283 /* Give interrupts a chance to try and prevent deadlock. */
284 for (;;) {
285 #ifndef _RUMPKERNEL /* XXXpooka: temporary build fix, see kern/40505 */
286 DELAY(10);
287 #endif /* _RUMPKERNEL */
288 }
289 }
290
291 /*
292 * Convert the current thread to a bound thread and prevent all
293 * CPUs from scheduling unbound jobs. Do so without taking any
294 * locks.
295 */
296 curlwp->l_pflag |= LP_BOUND;
297 for (CPU_INFO_FOREACH(cii, ci)) {
298 ci->ci_schedstate.spc_flags |= SPCF_OFFLINE;
299 }
300 }
301
302 bootopt = RB_AUTOBOOT | RB_NOSYNC;
303 if (!doing_shutdown) {
304 if (dumponpanic)
305 bootopt |= RB_DUMP;
306 } else
307 printf("Skipping crash dump on recursive panic\n");
308
309 doing_shutdown = 1;
310
311 if (logenabled(msgbufp))
312 panicstart = msgbufp->msg_bufx;
313
314 printf("panic: ");
315 if (panicstr == NULL) {
316 /* first time in panic - store fmt first for precaution */
317 panicstr = fmt;
318
319 vsnprintf(scratchstr, sizeof(scratchstr), fmt, ap);
320 printf("%s", scratchstr);
321 panicstr = scratchstr;
322 } else {
323 vprintf(fmt, ap);
324 }
325 printf("\n");
326
327 if (logenabled(msgbufp))
328 panicend = msgbufp->msg_bufx;
329
330 #ifdef IPKDB
331 ipkdb_panic();
332 #endif
333 #ifdef KGDB
334 kgdb_panic();
335 #endif
336 #ifdef KADB
337 if (boothowto & RB_KDB)
338 kdbpanic();
339 #endif
340 #ifdef DDB
341 db_panic();
342 #endif
343 cpu_reboot(bootopt, NULL);
344 }
345
346 /*
347 * kernel logging functions: log, logpri, addlog
348 */
349
350 /*
351 * log: write to the log buffer
352 *
353 * => will not sleep [so safe to call from interrupt]
354 * => will log to console if /dev/klog isn't open
355 */
356
357 void
358 log(int level, const char *fmt, ...)
359 {
360 va_list ap;
361
362 kprintf_lock();
363
364 klogpri(level); /* log the level first */
365 va_start(ap, fmt);
366 kprintf(fmt, TOLOG, NULL, NULL, ap);
367 va_end(ap);
368 if (!log_open) {
369 va_start(ap, fmt);
370 kprintf(fmt, TOCONS, NULL, NULL, ap);
371 va_end(ap);
372 }
373
374 kprintf_unlock();
375
376 logwakeup(); /* wake up anyone waiting for log msgs */
377 }
378
379 /*
380 * vlog: write to the log buffer [already have va_list]
381 */
382
383 void
384 vlog(int level, const char *fmt, va_list ap)
385 {
386 va_list cap;
387
388 va_copy(cap, ap);
389 kprintf_lock();
390
391 klogpri(level); /* log the level first */
392 kprintf(fmt, TOLOG, NULL, NULL, ap);
393 if (!log_open)
394 kprintf(fmt, TOCONS, NULL, NULL, cap);
395
396 kprintf_unlock();
397 va_end(cap);
398
399 logwakeup(); /* wake up anyone waiting for log msgs */
400 }
401
402 /*
403 * logpri: log the priority level to the klog
404 */
405
406 void
407 logpri(int level)
408 {
409
410 kprintf_lock();
411 klogpri(level);
412 kprintf_unlock();
413 }
414
415 /*
416 * Note: we must be in the mutex here!
417 */
418 void
419 klogpri(int level)
420 {
421 KASSERT((level & KLOG_PRI) == 0);
422
423 putchar(level | KLOG_PRI, TOLOG, NULL);
424 }
425
426 /*
427 * addlog: add info to previous log message
428 */
429
430 void
431 addlog(const char *fmt, ...)
432 {
433 va_list ap;
434
435 kprintf_lock();
436
437 va_start(ap, fmt);
438 kprintf(fmt, TOLOG, NULL, NULL, ap);
439 va_end(ap);
440 if (!log_open) {
441 va_start(ap, fmt);
442 kprintf(fmt, TOCONS, NULL, NULL, ap);
443 va_end(ap);
444 }
445
446 kprintf_unlock();
447
448 logwakeup();
449 }
450
451 static void
452 putone(int c, int flags, struct tty *tp)
453 {
454 if (panicstr)
455 constty = NULL;
456
457 if ((flags & TOCONS) && tp == NULL && constty) {
458 tp = constty;
459 flags |= TOTTY;
460 }
461 if ((flags & TOTTY) && tp &&
462 tputchar(c, flags, tp) < 0 &&
463 (flags & TOCONS) && tp == constty)
464 constty = NULL;
465 if ((flags & TOLOG) &&
466 c != '\0' && c != '\r' && c != 0177)
467 logputchar(c);
468 if ((flags & TOCONS) && constty == NULL && c != '\0')
469 (*v_putc)(c);
470 }
471
472 static void
473 putlogpri(int level)
474 {
475 char *p;
476 char snbuf[KPRINTF_BUFSIZE];
477
478 putone('<', TOLOG, NULL);
479 snprintf(snbuf, sizeof(snbuf), "%d", level);
480 for (p = snbuf ; *p ; p++)
481 putone(*p, TOLOG, NULL);
482 putone('>', TOLOG, NULL);
483 }
484
485 #ifndef KLOG_NOTIMESTAMP
486 static int needtstamp = 1;
487 int log_ts_prec = 9;
488
489 static void
490 addtstamp(int flags, struct tty *tp)
491 {
492 char buf[64];
493 struct timespec ts;
494 int n, prec;
495 long fsec;
496
497 prec = log_ts_prec;
498 if (prec < 0) {
499 prec = 0;
500 log_ts_prec = prec;
501 } else if (prec > 9) {
502 prec = 9;
503 log_ts_prec = prec;
504 }
505
506 getnanouptime(&ts);
507
508 for (n = prec, fsec = ts.tv_nsec; n < 8; n++)
509 fsec /= 10;
510 if (n < 9)
511 fsec = (fsec / 10) + ((fsec % 10) >= 5);
512
513 n = snprintf(buf, sizeof(buf), "[% 4jd.%.*ld] ",
514 (intmax_t)ts.tv_sec, prec, fsec);
515
516 for (int i = 0; i < n; i++)
517 putone(buf[i], flags, tp);
518 }
519 #endif
520
521 /*
522 * putchar: print a single character on console or user terminal.
523 *
524 * => if console, then the last MSGBUFS chars are saved in msgbuf
525 * for inspection later (e.g. dmesg/syslog)
526 * => we must already be in the mutex!
527 */
528 static void
529 putchar(int c, int flags, struct tty *tp)
530 {
531 if (c & KLOG_PRI) {
532 putlogpri(c & ~KLOG_PRI);
533 return;
534 }
535
536 #ifndef KLOG_NOTIMESTAMP
537 if (c != '\0' && c != '\n' && needtstamp) {
538 addtstamp(flags, tp);
539 needtstamp = 0;
540 }
541
542 if (c == '\n')
543 needtstamp = 1;
544 #endif
545 putone(c, flags, tp);
546
547 #ifdef DDB
548 if (flags & TODDB) {
549 db_putchar(c);
550 return;
551 }
552 #endif
553
554 #ifdef RND_PRINTF
555 if (__predict_true(kprintf_inited)) {
556 static uint8_t rbuf[SHA512_BLOCK_LENGTH];
557 static int cursor;
558
559 rbuf[cursor] = c;
560 if (cursor == sizeof(rbuf) - 1) {
561 SHA512_Update(&kprnd_sha, rbuf, sizeof(rbuf));
562 kprnd_added++;
563 cursor = 0;
564 } else {
565 cursor++;
566 }
567 }
568 #endif
569 }
570
571 /*
572 * tablefull: warn that a system table is full
573 */
574
575 void
576 tablefull(const char *tab, const char *hint)
577 {
578 if (hint)
579 log(LOG_ERR, "%s: table is full - %s\n", tab, hint);
580 else
581 log(LOG_ERR, "%s: table is full\n", tab);
582 }
583
584
585 /*
586 * uprintf: print to the controlling tty of the current process
587 *
588 * => we may block if the tty queue is full
589 * => no message is printed if the queue doesn't clear in a reasonable
590 * time
591 */
592
593 void
594 uprintf(const char *fmt, ...)
595 {
596 struct proc *p = curproc;
597 va_list ap;
598
599 /* mutex_enter(proc_lock); XXXSMP */
600
601 if (p->p_lflag & PL_CONTROLT && p->p_session->s_ttyvp) {
602 /* No mutex needed; going to process TTY. */
603 va_start(ap, fmt);
604 kprintf(fmt, TOTTY, p->p_session->s_ttyp, NULL, ap);
605 va_end(ap);
606 }
607
608 /* mutex_exit(proc_lock); XXXSMP */
609 }
610
611 void
612 uprintf_locked(const char *fmt, ...)
613 {
614 struct proc *p = curproc;
615 va_list ap;
616
617 if (p->p_lflag & PL_CONTROLT && p->p_session->s_ttyvp) {
618 /* No mutex needed; going to process TTY. */
619 va_start(ap, fmt);
620 kprintf(fmt, TOTTY, p->p_session->s_ttyp, NULL, ap);
621 va_end(ap);
622 }
623 }
624
625 /*
626 * tprintf functions: used to send messages to a specific process
627 *
628 * usage:
629 * get a tpr_t handle on a process "p" by using "tprintf_open(p)"
630 * use the handle when calling "tprintf"
631 * when done, do a "tprintf_close" to drop the handle
632 */
633
634 /*
635 * tprintf_open: get a tprintf handle on a process "p"
636 *
637 * => returns NULL if process can't be printed to
638 */
639
640 tpr_t
641 tprintf_open(struct proc *p)
642 {
643 tpr_t cookie;
644
645 cookie = NULL;
646
647 mutex_enter(proc_lock);
648 if (p->p_lflag & PL_CONTROLT && p->p_session->s_ttyvp) {
649 proc_sesshold(p->p_session);
650 cookie = (tpr_t)p->p_session;
651 }
652 mutex_exit(proc_lock);
653
654 return cookie;
655 }
656
657 /*
658 * tprintf_close: dispose of a tprintf handle obtained with tprintf_open
659 */
660
661 void
662 tprintf_close(tpr_t sess)
663 {
664
665 if (sess) {
666 mutex_enter(proc_lock);
667 /* Releases proc_lock. */
668 proc_sessrele((struct session *)sess);
669 }
670 }
671
672 /*
673 * tprintf: given tprintf handle to a process [obtained with tprintf_open],
674 * send a message to the controlling tty for that process.
675 *
676 * => also sends message to /dev/klog
677 */
678 void
679 tprintf(tpr_t tpr, const char *fmt, ...)
680 {
681 struct session *sess = (struct session *)tpr;
682 struct tty *tp = NULL;
683 int flags = TOLOG;
684 va_list ap;
685
686 /* mutex_enter(proc_lock); XXXSMP */
687 if (sess && sess->s_ttyvp && ttycheckoutq(sess->s_ttyp, 0)) {
688 flags |= TOTTY;
689 tp = sess->s_ttyp;
690 }
691
692 kprintf_lock();
693
694 klogpri(LOG_INFO);
695 va_start(ap, fmt);
696 kprintf(fmt, flags, tp, NULL, ap);
697 va_end(ap);
698
699 kprintf_unlock();
700 /* mutex_exit(proc_lock); XXXSMP */
701
702 logwakeup();
703 }
704
705
706 /*
707 * ttyprintf: send a message to a specific tty
708 *
709 * => should be used only by tty driver or anything that knows the
710 * underlying tty will not be revoked(2)'d away. [otherwise,
711 * use tprintf]
712 */
713 void
714 ttyprintf(struct tty *tp, const char *fmt, ...)
715 {
716 va_list ap;
717
718 /* No mutex needed; going to process TTY. */
719 va_start(ap, fmt);
720 kprintf(fmt, TOTTY, tp, NULL, ap);
721 va_end(ap);
722 }
723
724 #ifdef DDB
725
726 /*
727 * db_printf: printf for DDB (via db_putchar)
728 */
729
730 void
731 db_printf(const char *fmt, ...)
732 {
733 va_list ap;
734
735 /* No mutex needed; DDB pauses all processors. */
736 va_start(ap, fmt);
737 kprintf(fmt, TODDB, NULL, NULL, ap);
738 va_end(ap);
739
740 if (db_tee_msgbuf) {
741 va_start(ap, fmt);
742 kprintf(fmt, TOLOG, NULL, NULL, ap);
743 va_end(ap);
744 }
745 }
746
747 void
748 db_vprintf(const char *fmt, va_list ap)
749 {
750 va_list cap;
751
752 va_copy(cap, ap);
753 /* No mutex needed; DDB pauses all processors. */
754 kprintf(fmt, TODDB, NULL, NULL, ap);
755 if (db_tee_msgbuf)
756 kprintf(fmt, TOLOG, NULL, NULL, cap);
757 va_end(cap);
758 }
759
760 #endif /* DDB */
761
762 static void
763 kprintf_internal(const char *fmt, int oflags, void *vp, char *sbuf, ...)
764 {
765 va_list ap;
766
767 va_start(ap, sbuf);
768 (void)kprintf(fmt, oflags, vp, sbuf, ap);
769 va_end(ap);
770 }
771
772 /*
773 * Device autoconfiguration printf routines. These change their
774 * behavior based on the AB_* flags in boothowto. If AB_SILENT
775 * is set, messages never go to the console (but they still always
776 * go to the log). AB_VERBOSE overrides AB_SILENT.
777 */
778
779 /*
780 * aprint_normal: Send to console unless AB_QUIET. Always goes
781 * to the log.
782 */
783 static void
784 aprint_normal_internal(const char *prefix, const char *fmt, va_list ap)
785 {
786 int flags = TOLOG;
787
788 if ((boothowto & (AB_SILENT|AB_QUIET)) == 0 ||
789 (boothowto & AB_VERBOSE) != 0)
790 flags |= TOCONS;
791
792 kprintf_lock();
793
794 if (prefix)
795 kprintf_internal("%s: ", flags, NULL, NULL, prefix);
796 kprintf(fmt, flags, NULL, NULL, ap);
797
798 kprintf_unlock();
799
800 if (!panicstr)
801 logwakeup();
802 }
803
804 void
805 aprint_normal(const char *fmt, ...)
806 {
807 va_list ap;
808
809 va_start(ap, fmt);
810 aprint_normal_internal(NULL, fmt, ap);
811 va_end(ap);
812 }
813
814 void
815 aprint_normal_dev(device_t dv, const char *fmt, ...)
816 {
817 va_list ap;
818
819 va_start(ap, fmt);
820 aprint_normal_internal(device_xname(dv), fmt, ap);
821 va_end(ap);
822 }
823
824 void
825 aprint_normal_ifnet(struct ifnet *ifp, const char *fmt, ...)
826 {
827 va_list ap;
828
829 va_start(ap, fmt);
830 aprint_normal_internal(ifp->if_xname, fmt, ap);
831 va_end(ap);
832 }
833
834 /*
835 * aprint_error: Send to console unless AB_QUIET. Always goes
836 * to the log. Also counts the number of times called so other
837 * parts of the kernel can report the number of errors during a
838 * given phase of system startup.
839 */
840 static int aprint_error_count;
841
842 int
843 aprint_get_error_count(void)
844 {
845 int count;
846
847 kprintf_lock();
848
849 count = aprint_error_count;
850 aprint_error_count = 0;
851
852 kprintf_unlock();
853
854 return (count);
855 }
856
857 static void
858 aprint_error_internal(const char *prefix, const char *fmt, va_list ap)
859 {
860 int flags = TOLOG;
861
862 if ((boothowto & (AB_SILENT|AB_QUIET)) == 0 ||
863 (boothowto & AB_VERBOSE) != 0)
864 flags |= TOCONS;
865
866 kprintf_lock();
867
868 aprint_error_count++;
869
870 if (prefix)
871 kprintf_internal("%s: ", flags, NULL, NULL, prefix);
872 kprintf(fmt, flags, NULL, NULL, ap);
873
874 kprintf_unlock();
875
876 if (!panicstr)
877 logwakeup();
878 }
879
880 void
881 aprint_error(const char *fmt, ...)
882 {
883 va_list ap;
884
885 va_start(ap, fmt);
886 aprint_error_internal(NULL, fmt, ap);
887 va_end(ap);
888 }
889
890 void
891 aprint_error_dev(device_t dv, const char *fmt, ...)
892 {
893 va_list ap;
894
895 va_start(ap, fmt);
896 aprint_error_internal(device_xname(dv), fmt, ap);
897 va_end(ap);
898 }
899
900 void
901 aprint_error_ifnet(struct ifnet *ifp, const char *fmt, ...)
902 {
903 va_list ap;
904
905 va_start(ap, fmt);
906 aprint_error_internal(ifp->if_xname, fmt, ap);
907 va_end(ap);
908 }
909
910 /*
911 * aprint_naive: Send to console only if AB_QUIET. Never goes
912 * to the log.
913 */
914 static void
915 aprint_naive_internal(const char *prefix, const char *fmt, va_list ap)
916 {
917 if ((boothowto & (AB_QUIET|AB_SILENT|AB_VERBOSE)) != AB_QUIET)
918 return;
919
920 kprintf_lock();
921
922 if (prefix)
923 kprintf_internal("%s: ", TOCONS, NULL, NULL, prefix);
924 kprintf(fmt, TOCONS, NULL, NULL, ap);
925
926 kprintf_unlock();
927 }
928
929 void
930 aprint_naive(const char *fmt, ...)
931 {
932 va_list ap;
933
934 va_start(ap, fmt);
935 aprint_naive_internal(NULL, fmt, ap);
936 va_end(ap);
937 }
938
939 void
940 aprint_naive_dev(device_t dv, const char *fmt, ...)
941 {
942 va_list ap;
943
944 va_start(ap, fmt);
945 aprint_naive_internal(device_xname(dv), fmt, ap);
946 va_end(ap);
947 }
948
949 void
950 aprint_naive_ifnet(struct ifnet *ifp, const char *fmt, ...)
951 {
952 va_list ap;
953
954 va_start(ap, fmt);
955 aprint_naive_internal(ifp->if_xname, fmt, ap);
956 va_end(ap);
957 }
958
959 /*
960 * aprint_verbose: Send to console only if AB_VERBOSE. Always
961 * goes to the log.
962 */
963 static void
964 aprint_verbose_internal(const char *prefix, const char *fmt, va_list ap)
965 {
966 int flags = TOLOG;
967
968 if (boothowto & AB_VERBOSE)
969 flags |= TOCONS;
970
971 kprintf_lock();
972
973 if (prefix)
974 kprintf_internal("%s: ", flags, NULL, NULL, prefix);
975 kprintf(fmt, flags, NULL, NULL, ap);
976
977 kprintf_unlock();
978
979 if (!panicstr)
980 logwakeup();
981 }
982
983 void
984 aprint_verbose(const char *fmt, ...)
985 {
986 va_list ap;
987
988 va_start(ap, fmt);
989 aprint_verbose_internal(NULL, fmt, ap);
990 va_end(ap);
991 }
992
993 void
994 aprint_verbose_dev(device_t dv, const char *fmt, ...)
995 {
996 va_list ap;
997
998 va_start(ap, fmt);
999 aprint_verbose_internal(device_xname(dv), fmt, ap);
1000 va_end(ap);
1001 }
1002
1003 void
1004 aprint_verbose_ifnet(struct ifnet *ifp, const char *fmt, ...)
1005 {
1006 va_list ap;
1007
1008 va_start(ap, fmt);
1009 aprint_verbose_internal(ifp->if_xname, fmt, ap);
1010 va_end(ap);
1011 }
1012
1013 /*
1014 * aprint_debug: Send to console and log only if AB_DEBUG.
1015 */
1016 static void
1017 aprint_debug_internal(const char *prefix, const char *fmt, va_list ap)
1018 {
1019 if ((boothowto & AB_DEBUG) == 0)
1020 return;
1021
1022 kprintf_lock();
1023
1024 if (prefix)
1025 kprintf_internal("%s: ", TOCONS | TOLOG, NULL, NULL, prefix);
1026 kprintf(fmt, TOCONS | TOLOG, NULL, NULL, ap);
1027
1028 kprintf_unlock();
1029 }
1030
1031 void
1032 aprint_debug(const char *fmt, ...)
1033 {
1034 va_list ap;
1035
1036 va_start(ap, fmt);
1037 aprint_debug_internal(NULL, fmt, ap);
1038 va_end(ap);
1039 }
1040
1041 void
1042 aprint_debug_dev(device_t dv, const char *fmt, ...)
1043 {
1044 va_list ap;
1045
1046 va_start(ap, fmt);
1047 aprint_debug_internal(device_xname(dv), fmt, ap);
1048 va_end(ap);
1049 }
1050
1051 void
1052 aprint_debug_ifnet(struct ifnet *ifp, const char *fmt, ...)
1053 {
1054 va_list ap;
1055
1056 va_start(ap, fmt);
1057 aprint_debug_internal(ifp->if_xname, fmt, ap);
1058 va_end(ap);
1059 }
1060
1061 void
1062 printf_tolog(const char *fmt, ...)
1063 {
1064 va_list ap;
1065
1066 kprintf_lock();
1067
1068 va_start(ap, fmt);
1069 kprintf(fmt, TOLOG, NULL, NULL, ap);
1070 va_end(ap);
1071
1072 kprintf_unlock();
1073 }
1074
1075 /*
1076 * printf_nolog: Like printf(), but does not send message to the log.
1077 */
1078
1079 void
1080 printf_nolog(const char *fmt, ...)
1081 {
1082 va_list ap;
1083
1084 kprintf_lock();
1085
1086 va_start(ap, fmt);
1087 kprintf(fmt, TOCONS, NULL, NULL, ap);
1088 va_end(ap);
1089
1090 kprintf_unlock();
1091 }
1092
1093 /*
1094 * normal kernel printf functions: printf, vprintf, snprintf, vsnprintf
1095 */
1096
1097 /*
1098 * printf: print a message to the console and the log
1099 */
1100 void
1101 printf(const char *fmt, ...)
1102 {
1103 va_list ap;
1104
1105 kprintf_lock();
1106
1107 va_start(ap, fmt);
1108 kprintf(fmt, TOCONS | TOLOG, NULL, NULL, ap);
1109 va_end(ap);
1110
1111 kprintf_unlock();
1112
1113 if (!panicstr)
1114 logwakeup();
1115 }
1116
1117 /*
1118 * vprintf: print a message to the console and the log [already have
1119 * va_list]
1120 */
1121
1122 void
1123 vprintf(const char *fmt, va_list ap)
1124 {
1125 kprintf_lock();
1126
1127 kprintf(fmt, TOCONS | TOLOG, NULL, NULL, ap);
1128
1129 kprintf_unlock();
1130
1131 if (!panicstr)
1132 logwakeup();
1133 }
1134
1135 /*
1136 * snprintf: print a message to a buffer
1137 */
1138 int
1139 snprintf(char *bf, size_t size, const char *fmt, ...)
1140 {
1141 int retval;
1142 va_list ap;
1143
1144 va_start(ap, fmt);
1145 retval = vsnprintf(bf, size, fmt, ap);
1146 va_end(ap);
1147
1148 return retval;
1149 }
1150
1151 /*
1152 * vsnprintf: print a message to a buffer [already have va_list]
1153 */
1154 int
1155 vsnprintf(char *bf, size_t size, const char *fmt, va_list ap)
1156 {
1157 int retval;
1158 char *p;
1159
1160 p = bf + size;
1161 retval = kprintf(fmt, TOBUFONLY, &p, bf, ap);
1162 if (bf && size > 0) {
1163 /* nul terminate */
1164 if (size <= (size_t)retval)
1165 bf[size - 1] = '\0';
1166 else
1167 bf[retval] = '\0';
1168 }
1169 return retval;
1170 }
1171
1172 /*
1173 * kprintf: scaled down version of printf(3).
1174 *
1175 * this version based on vfprintf() from libc which was derived from
1176 * software contributed to Berkeley by Chris Torek.
1177 *
1178 * NOTE: The kprintf mutex must be held if we're going TOBUF or TOCONS!
1179 */
1180
1181 /*
1182 * macros for converting digits to letters and vice versa
1183 */
1184 #define to_digit(c) ((c) - '0')
1185 #define is_digit(c) ((unsigned)to_digit(c) <= 9)
1186 #define to_char(n) ((n) + '0')
1187
1188 /*
1189 * flags used during conversion.
1190 */
1191 #define ALT 0x001 /* alternate form */
1192 #define HEXPREFIX 0x002 /* add 0x or 0X prefix */
1193 #define LADJUST 0x004 /* left adjustment */
1194 #define LONGDBL 0x008 /* long double; unimplemented */
1195 #define LONGINT 0x010 /* long integer */
1196 #define QUADINT 0x020 /* quad integer */
1197 #define SHORTINT 0x040 /* short integer */
1198 #define MAXINT 0x080 /* intmax_t */
1199 #define PTRINT 0x100 /* intptr_t */
1200 #define SIZEINT 0x200 /* size_t */
1201 #define ZEROPAD 0x400 /* zero (as opposed to blank) pad */
1202 #define FPT 0x800 /* Floating point number */
1203
1204 /*
1205 * To extend shorts properly, we need both signed and unsigned
1206 * argument extraction methods.
1207 */
1208 #define SARG() \
1209 (flags&MAXINT ? va_arg(ap, intmax_t) : \
1210 flags&PTRINT ? va_arg(ap, intptr_t) : \
1211 flags&SIZEINT ? va_arg(ap, ssize_t) : /* XXX */ \
1212 flags&QUADINT ? va_arg(ap, quad_t) : \
1213 flags&LONGINT ? va_arg(ap, long) : \
1214 flags&SHORTINT ? (long)(short)va_arg(ap, int) : \
1215 (long)va_arg(ap, int))
1216 #define UARG() \
1217 (flags&MAXINT ? va_arg(ap, uintmax_t) : \
1218 flags&PTRINT ? va_arg(ap, uintptr_t) : \
1219 flags&SIZEINT ? va_arg(ap, size_t) : \
1220 flags&QUADINT ? va_arg(ap, u_quad_t) : \
1221 flags&LONGINT ? va_arg(ap, u_long) : \
1222 flags&SHORTINT ? (u_long)(u_short)va_arg(ap, int) : \
1223 (u_long)va_arg(ap, u_int))
1224
1225 #define KPRINTF_PUTCHAR(C) { \
1226 if (oflags == TOBUFONLY) { \
1227 if (sbuf && ((vp == NULL) || (sbuf < tailp))) \
1228 *sbuf++ = (C); \
1229 } else { \
1230 putchar((C), oflags, vp); \
1231 } \
1232 }
1233
1234 void
1235 device_printf(device_t dev, const char *fmt, ...)
1236 {
1237 va_list ap;
1238
1239 va_start(ap, fmt);
1240 printf("%s: ", device_xname(dev));
1241 vprintf(fmt, ap);
1242 va_end(ap);
1243 return;
1244 }
1245
1246 /*
1247 * Guts of kernel printf. Note, we already expect to be in a mutex!
1248 */
1249 int
1250 kprintf(const char *fmt0, int oflags, void *vp, char *sbuf, va_list ap)
1251 {
1252 const char *fmt; /* format string */
1253 int ch; /* character from fmt */
1254 int n; /* handy integer (short term usage) */
1255 char *cp; /* handy char pointer (short term usage) */
1256 int flags; /* flags as above */
1257 int ret; /* return value accumulator */
1258 int width; /* width from format (%8d), or 0 */
1259 int prec; /* precision from format (%.3d), or -1 */
1260 char sign; /* sign prefix (' ', '+', '-', or \0) */
1261
1262 u_quad_t _uquad; /* integer arguments %[diouxX] */
1263 enum { OCT, DEC, HEX } base;/* base for [diouxX] conversion */
1264 int dprec; /* a copy of prec if [diouxX], 0 otherwise */
1265 int realsz; /* field size expanded by dprec */
1266 int size; /* size of converted field or string */
1267 const char *xdigs; /* digits for [xX] conversion */
1268 char bf[KPRINTF_BUFSIZE]; /* space for %c, %[diouxX] */
1269 char *tailp; /* tail pointer for snprintf */
1270
1271 if (oflags == TOBUFONLY && (vp != NULL))
1272 tailp = *(char **)vp;
1273 else
1274 tailp = NULL;
1275
1276 cp = NULL; /* XXX: shutup gcc */
1277 size = 0; /* XXX: shutup gcc */
1278
1279 fmt = fmt0;
1280 ret = 0;
1281
1282 xdigs = NULL; /* XXX: shut up gcc warning */
1283
1284 /*
1285 * Scan the format for conversions (`%' character).
1286 */
1287 for (;;) {
1288 for (; *fmt != '%' && *fmt; fmt++) {
1289 ret++;
1290 KPRINTF_PUTCHAR(*fmt);
1291 }
1292 if (*fmt == 0)
1293 goto done;
1294
1295 fmt++; /* skip over '%' */
1296
1297 flags = 0;
1298 dprec = 0;
1299 width = 0;
1300 prec = -1;
1301 sign = '\0';
1302
1303 rflag: ch = *fmt++;
1304 reswitch: switch (ch) {
1305 case ' ':
1306 /*
1307 * ``If the space and + flags both appear, the space
1308 * flag will be ignored.''
1309 * -- ANSI X3J11
1310 */
1311 if (!sign)
1312 sign = ' ';
1313 goto rflag;
1314 case '#':
1315 flags |= ALT;
1316 goto rflag;
1317 case '*':
1318 /*
1319 * ``A negative field width argument is taken as a
1320 * - flag followed by a positive field width.''
1321 * -- ANSI X3J11
1322 * They don't exclude field widths read from args.
1323 */
1324 if ((width = va_arg(ap, int)) >= 0)
1325 goto rflag;
1326 width = -width;
1327 /* FALLTHROUGH */
1328 case '-':
1329 flags |= LADJUST;
1330 goto rflag;
1331 case '+':
1332 sign = '+';
1333 goto rflag;
1334 case '.':
1335 if ((ch = *fmt++) == '*') {
1336 n = va_arg(ap, int);
1337 prec = n < 0 ? -1 : n;
1338 goto rflag;
1339 }
1340 n = 0;
1341 while (is_digit(ch)) {
1342 n = 10 * n + to_digit(ch);
1343 ch = *fmt++;
1344 }
1345 prec = n < 0 ? -1 : n;
1346 goto reswitch;
1347 case '0':
1348 /*
1349 * ``Note that 0 is taken as a flag, not as the
1350 * beginning of a field width.''
1351 * -- ANSI X3J11
1352 */
1353 flags |= ZEROPAD;
1354 goto rflag;
1355 case '1': case '2': case '3': case '4':
1356 case '5': case '6': case '7': case '8': case '9':
1357 n = 0;
1358 do {
1359 n = 10 * n + to_digit(ch);
1360 ch = *fmt++;
1361 } while (is_digit(ch));
1362 width = n;
1363 goto reswitch;
1364 case 'h':
1365 flags |= SHORTINT;
1366 goto rflag;
1367 case 'j':
1368 flags |= MAXINT;
1369 goto rflag;
1370 case 'l':
1371 if (*fmt == 'l') {
1372 fmt++;
1373 flags |= QUADINT;
1374 } else {
1375 flags |= LONGINT;
1376 }
1377 goto rflag;
1378 case 'q':
1379 flags |= QUADINT;
1380 goto rflag;
1381 case 't':
1382 flags |= PTRINT;
1383 goto rflag;
1384 case 'z':
1385 flags |= SIZEINT;
1386 goto rflag;
1387 case 'c':
1388 *(cp = bf) = va_arg(ap, int);
1389 size = 1;
1390 sign = '\0';
1391 break;
1392 case 'D':
1393 flags |= LONGINT;
1394 /*FALLTHROUGH*/
1395 case 'd':
1396 case 'i':
1397 _uquad = SARG();
1398 if ((quad_t)_uquad < 0) {
1399 _uquad = -_uquad;
1400 sign = '-';
1401 }
1402 base = DEC;
1403 goto number;
1404 case 'n':
1405 if (flags & MAXINT)
1406 *va_arg(ap, intmax_t *) = ret;
1407 else if (flags & PTRINT)
1408 *va_arg(ap, intptr_t *) = ret;
1409 else if (flags & SIZEINT)
1410 *va_arg(ap, ssize_t *) = ret;
1411 else if (flags & QUADINT)
1412 *va_arg(ap, quad_t *) = ret;
1413 else if (flags & LONGINT)
1414 *va_arg(ap, long *) = ret;
1415 else if (flags & SHORTINT)
1416 *va_arg(ap, short *) = ret;
1417 else
1418 *va_arg(ap, int *) = ret;
1419 continue; /* no output */
1420 case 'O':
1421 flags |= LONGINT;
1422 /*FALLTHROUGH*/
1423 case 'o':
1424 _uquad = UARG();
1425 base = OCT;
1426 goto nosign;
1427 case 'p':
1428 /*
1429 * ``The argument shall be a pointer to void. The
1430 * value of the pointer is converted to a sequence
1431 * of printable characters, in an implementation-
1432 * defined manner.''
1433 * -- ANSI X3J11
1434 */
1435 /* NOSTRICT */
1436 _uquad = (u_long)va_arg(ap, void *);
1437 base = HEX;
1438 xdigs = hexdigits;
1439 flags |= HEXPREFIX;
1440 ch = 'x';
1441 goto nosign;
1442 case 's':
1443 if ((cp = va_arg(ap, char *)) == NULL)
1444 /*XXXUNCONST*/
1445 cp = __UNCONST("(null)");
1446 if (prec >= 0) {
1447 /*
1448 * can't use strlen; can only look for the
1449 * NUL in the first `prec' characters, and
1450 * strlen() will go further.
1451 */
1452 char *p = memchr(cp, 0, prec);
1453
1454 if (p != NULL) {
1455 size = p - cp;
1456 if (size > prec)
1457 size = prec;
1458 } else
1459 size = prec;
1460 } else
1461 size = strlen(cp);
1462 sign = '\0';
1463 break;
1464 case 'U':
1465 flags |= LONGINT;
1466 /*FALLTHROUGH*/
1467 case 'u':
1468 _uquad = UARG();
1469 base = DEC;
1470 goto nosign;
1471 case 'X':
1472 xdigs = HEXDIGITS;
1473 goto hex;
1474 case 'x':
1475 xdigs = hexdigits;
1476 hex: _uquad = UARG();
1477 base = HEX;
1478 /* leading 0x/X only if non-zero */
1479 if (flags & ALT && _uquad != 0)
1480 flags |= HEXPREFIX;
1481
1482 /* unsigned conversions */
1483 nosign: sign = '\0';
1484 /*
1485 * ``... diouXx conversions ... if a precision is
1486 * specified, the 0 flag will be ignored.''
1487 * -- ANSI X3J11
1488 */
1489 number: if ((dprec = prec) >= 0)
1490 flags &= ~ZEROPAD;
1491
1492 /*
1493 * ``The result of converting a zero value with an
1494 * explicit precision of zero is no characters.''
1495 * -- ANSI X3J11
1496 */
1497 cp = bf + KPRINTF_BUFSIZE;
1498 if (_uquad != 0 || prec != 0) {
1499 /*
1500 * Unsigned mod is hard, and unsigned mod
1501 * by a constant is easier than that by
1502 * a variable; hence this switch.
1503 */
1504 switch (base) {
1505 case OCT:
1506 do {
1507 *--cp = to_char(_uquad & 7);
1508 _uquad >>= 3;
1509 } while (_uquad);
1510 /* handle octal leading 0 */
1511 if (flags & ALT && *cp != '0')
1512 *--cp = '0';
1513 break;
1514
1515 case DEC:
1516 /* many numbers are 1 digit */
1517 while (_uquad >= 10) {
1518 *--cp = to_char(_uquad % 10);
1519 _uquad /= 10;
1520 }
1521 *--cp = to_char(_uquad);
1522 break;
1523
1524 case HEX:
1525 do {
1526 *--cp = xdigs[_uquad & 15];
1527 _uquad >>= 4;
1528 } while (_uquad);
1529 break;
1530
1531 default:
1532 /*XXXUNCONST*/
1533 cp = __UNCONST("bug in kprintf: bad base");
1534 size = strlen(cp);
1535 goto skipsize;
1536 }
1537 }
1538 size = bf + KPRINTF_BUFSIZE - cp;
1539 skipsize:
1540 break;
1541 default: /* "%?" prints ?, unless ? is NUL */
1542 if (ch == '\0')
1543 goto done;
1544 /* pretend it was %c with argument ch */
1545 cp = bf;
1546 *cp = ch;
1547 size = 1;
1548 sign = '\0';
1549 break;
1550 }
1551
1552 /*
1553 * All reasonable formats wind up here. At this point, `cp'
1554 * points to a string which (if not flags&LADJUST) should be
1555 * padded out to `width' places. If flags&ZEROPAD, it should
1556 * first be prefixed by any sign or other prefix; otherwise,
1557 * it should be blank padded before the prefix is emitted.
1558 * After any left-hand padding and prefixing, emit zeroes
1559 * required by a decimal [diouxX] precision, then print the
1560 * string proper, then emit zeroes required by any leftover
1561 * floating precision; finally, if LADJUST, pad with blanks.
1562 *
1563 * Compute actual size, so we know how much to pad.
1564 * size excludes decimal prec; realsz includes it.
1565 */
1566 realsz = dprec > size ? dprec : size;
1567 if (sign)
1568 realsz++;
1569 else if (flags & HEXPREFIX)
1570 realsz+= 2;
1571
1572 /* adjust ret */
1573 ret += width > realsz ? width : realsz;
1574
1575 /* right-adjusting blank padding */
1576 if ((flags & (LADJUST|ZEROPAD)) == 0) {
1577 n = width - realsz;
1578 while (n-- > 0)
1579 KPRINTF_PUTCHAR(' ');
1580 }
1581
1582 /* prefix */
1583 if (sign) {
1584 KPRINTF_PUTCHAR(sign);
1585 } else if (flags & HEXPREFIX) {
1586 KPRINTF_PUTCHAR('0');
1587 KPRINTF_PUTCHAR(ch);
1588 }
1589
1590 /* right-adjusting zero padding */
1591 if ((flags & (LADJUST|ZEROPAD)) == ZEROPAD) {
1592 n = width - realsz;
1593 while (n-- > 0)
1594 KPRINTF_PUTCHAR('0');
1595 }
1596
1597 /* leading zeroes from decimal precision */
1598 n = dprec - size;
1599 while (n-- > 0)
1600 KPRINTF_PUTCHAR('0');
1601
1602 /* the string or number proper */
1603 for (; size--; cp++)
1604 KPRINTF_PUTCHAR(*cp);
1605 /* left-adjusting padding (always blank) */
1606 if (flags & LADJUST) {
1607 n = width - realsz;
1608 while (n-- > 0)
1609 KPRINTF_PUTCHAR(' ');
1610 }
1611 }
1612
1613 done:
1614 if ((oflags == TOBUFONLY) && (vp != NULL))
1615 *(char **)vp = sbuf;
1616 (*v_flush)();
1617
1618 #ifdef RND_PRINTF
1619 if (!cold) {
1620 struct timespec ts;
1621 (void)nanotime(&ts);
1622 SHA512_Update(&kprnd_sha, (char *)&ts, sizeof(ts));
1623 }
1624 #endif
1625 return ret;
1626 }
1627