kern_entropy.c revision 1.11 1 /* $NetBSD: kern_entropy.c,v 1.11 2020/05/06 18:31:05 riastradh Exp $ */
2
3 /*-
4 * Copyright (c) 2019 The NetBSD Foundation, Inc.
5 * All rights reserved.
6 *
7 * This code is derived from software contributed to The NetBSD Foundation
8 * by Taylor R. Campbell.
9 *
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions
12 * are met:
13 * 1. Redistributions of source code must retain the above copyright
14 * notice, this list of conditions and the following disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright
16 * notice, this list of conditions and the following disclaimer in the
17 * documentation and/or other materials provided with the distribution.
18 *
19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29 * POSSIBILITY OF SUCH DAMAGE.
30 */
31
32 /*
33 * Entropy subsystem
34 *
35 * * Each CPU maintains a per-CPU entropy pool so that gathering
36 * entropy requires no interprocessor synchronization, except
37 * early at boot when we may be scrambling to gather entropy as
38 * soon as possible.
39 *
40 * - entropy_enter gathers entropy and never drops it on the
41 * floor, at the cost of sometimes having to do cryptography.
42 *
43 * - entropy_enter_intr gathers entropy or drops it on the
44 * floor, with low latency. Work to stir the pool or kick the
45 * housekeeping thread is scheduled in soft interrupts.
46 *
47 * * entropy_enter immediately enters into the global pool if it
48 * can transition to full entropy in one swell foop. Otherwise,
49 * it defers to a housekeeping thread that consolidates entropy,
50 * but only when the CPUs collectively have full entropy, in
51 * order to mitigate iterative-guessing attacks.
52 *
53 * * The entropy housekeeping thread continues to consolidate
54 * entropy even after we think we have full entropy, in case we
55 * are wrong, but is limited to one discretionary consolidation
56 * per minute, and only when new entropy is actually coming in,
57 * to limit performance impact.
58 *
59 * * The entropy epoch is the number that changes when we
60 * transition from partial entropy to full entropy, so that
61 * users can easily determine when to reseed. This also
62 * facilitates an operator explicitly causing everything to
63 * reseed by sysctl -w kern.entropy.consolidate=1, e.g. if they
64 * just flipped a coin 256 times and wrote `echo tthhhhhthh... >
65 * /dev/random'.
66 *
67 * * No entropy estimation based on the sample values, which is a
68 * contradiction in terms and a potential source of side
69 * channels. It is the responsibility of the driver author to
70 * study how predictable the physical source of input can ever
71 * be, and to furnish a lower bound on the amount of entropy it
72 * has.
73 *
74 * * Entropy depletion is available for testing (or if you're into
75 * that sort of thing), with sysctl -w kern.entropy.depletion=1;
76 * the logic to support it is small, to minimize chance of bugs.
77 */
78
79 #include <sys/cdefs.h>
80 __KERNEL_RCSID(0, "$NetBSD: kern_entropy.c,v 1.11 2020/05/06 18:31:05 riastradh Exp $");
81
82 #include <sys/param.h>
83 #include <sys/types.h>
84 #include <sys/atomic.h>
85 #include <sys/compat_stub.h>
86 #include <sys/condvar.h>
87 #include <sys/cpu.h>
88 #include <sys/entropy.h>
89 #include <sys/errno.h>
90 #include <sys/evcnt.h>
91 #include <sys/event.h>
92 #include <sys/file.h>
93 #include <sys/intr.h>
94 #include <sys/kauth.h>
95 #include <sys/kernel.h>
96 #include <sys/kmem.h>
97 #include <sys/kthread.h>
98 #include <sys/module_hook.h>
99 #include <sys/mutex.h>
100 #include <sys/percpu.h>
101 #include <sys/poll.h>
102 #include <sys/queue.h>
103 #include <sys/rnd.h> /* legacy kernel API */
104 #include <sys/rndio.h> /* userland ioctl interface */
105 #include <sys/rndsource.h> /* kernel rndsource driver API */
106 #include <sys/select.h>
107 #include <sys/selinfo.h>
108 #include <sys/sha1.h> /* for boot seed checksum */
109 #include <sys/stdint.h>
110 #include <sys/sysctl.h>
111 #include <sys/systm.h>
112 #include <sys/time.h>
113 #include <sys/xcall.h>
114
115 #include <lib/libkern/entpool.h>
116
117 #include <machine/limits.h>
118
119 #ifdef __HAVE_CPU_COUNTER
120 #include <machine/cpu_counter.h>
121 #endif
122
123 /*
124 * struct entropy_cpu
125 *
126 * Per-CPU entropy state. The pool is allocated separately
127 * because percpu(9) sometimes moves per-CPU objects around
128 * without zeroing them, which would lead to unwanted copies of
129 * sensitive secrets. The evcnt is allocated separately becuase
130 * evcnt(9) assumes it stays put in memory.
131 */
132 struct entropy_cpu {
133 struct evcnt *ec_softint_evcnt;
134 struct entpool *ec_pool;
135 unsigned ec_pending;
136 bool ec_locked;
137 };
138
139 /*
140 * struct rndsource_cpu
141 *
142 * Per-CPU rndsource state.
143 */
144 struct rndsource_cpu {
145 unsigned rc_nbits; /* bits of entropy added */
146 };
147
148 /*
149 * entropy_global (a.k.a. E for short in this file)
150 *
151 * Global entropy state. Writes protected by the global lock.
152 * Some fields, marked (A), can be read outside the lock, and are
153 * maintained with atomic_load/store_relaxed.
154 */
155 struct {
156 kmutex_t lock; /* covers all global state */
157 struct entpool pool; /* global pool for extraction */
158 unsigned needed; /* (A) needed globally */
159 unsigned pending; /* (A) pending in per-CPU pools */
160 unsigned timestamp; /* (A) time of last consolidation */
161 unsigned epoch; /* (A) changes when needed -> 0 */
162 kcondvar_t cv; /* notifies state changes */
163 struct selinfo selq; /* notifies needed -> 0 */
164 struct lwp *sourcelock; /* lock on list of sources */
165 LIST_HEAD(,krndsource) sources; /* list of entropy sources */
166 enum entropy_stage {
167 ENTROPY_COLD = 0, /* single-threaded */
168 ENTROPY_WARM, /* multi-threaded at boot before CPUs */
169 ENTROPY_HOT, /* multi-threaded multi-CPU */
170 } stage;
171 bool consolidate; /* kick thread to consolidate */
172 bool seed_rndsource; /* true if seed source is attached */
173 bool seeded; /* true if seed file already loaded */
174 } entropy_global __cacheline_aligned = {
175 /* Fields that must be initialized when the kernel is loaded. */
176 .needed = ENTROPY_CAPACITY*NBBY,
177 .epoch = (unsigned)-1, /* -1 means not yet full entropy */
178 .sources = LIST_HEAD_INITIALIZER(entropy_global.sources),
179 .stage = ENTROPY_COLD,
180 };
181
182 #define E (&entropy_global) /* declutter */
183
184 /* Read-mostly globals */
185 static struct percpu *entropy_percpu __read_mostly; /* struct entropy_cpu */
186 static void *entropy_sih __read_mostly; /* softint handler */
187 static struct lwp *entropy_lwp __read_mostly; /* housekeeping thread */
188
189 int rnd_initial_entropy __read_mostly; /* XXX legacy */
190
191 static struct krndsource seed_rndsource __read_mostly;
192
193 /*
194 * Event counters
195 *
196 * Must be careful with adding these because they can serve as
197 * side channels.
198 */
199 static struct evcnt entropy_discretionary_evcnt =
200 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "entropy", "discretionary");
201 EVCNT_ATTACH_STATIC(entropy_discretionary_evcnt);
202 static struct evcnt entropy_immediate_evcnt =
203 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "entropy", "immediate");
204 EVCNT_ATTACH_STATIC(entropy_immediate_evcnt);
205 static struct evcnt entropy_partial_evcnt =
206 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "entropy", "partial");
207 EVCNT_ATTACH_STATIC(entropy_partial_evcnt);
208 static struct evcnt entropy_consolidate_evcnt =
209 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "entropy", "consolidate");
210 EVCNT_ATTACH_STATIC(entropy_consolidate_evcnt);
211 static struct evcnt entropy_extract_intr_evcnt =
212 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "entropy", "extract intr");
213 EVCNT_ATTACH_STATIC(entropy_extract_intr_evcnt);
214 static struct evcnt entropy_extract_fail_evcnt =
215 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "entropy", "extract fail");
216 EVCNT_ATTACH_STATIC(entropy_extract_fail_evcnt);
217 static struct evcnt entropy_request_evcnt =
218 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "entropy", "request");
219 EVCNT_ATTACH_STATIC(entropy_request_evcnt);
220 static struct evcnt entropy_deplete_evcnt =
221 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "entropy", "deplete");
222 EVCNT_ATTACH_STATIC(entropy_deplete_evcnt);
223 static struct evcnt entropy_notify_evcnt =
224 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "entropy", "notify");
225 EVCNT_ATTACH_STATIC(entropy_notify_evcnt);
226
227 /* Sysctl knobs */
228 bool entropy_collection = 1;
229 bool entropy_depletion = 0; /* Silly! */
230
231 static const struct sysctlnode *entropy_sysctlroot;
232 static struct sysctllog *entropy_sysctllog;
233
234 /* Forward declarations */
235 static void entropy_init_cpu(void *, void *, struct cpu_info *);
236 static void entropy_fini_cpu(void *, void *, struct cpu_info *);
237 static void entropy_account_cpu(struct entropy_cpu *);
238 static void entropy_enter(const void *, size_t, unsigned);
239 static bool entropy_enter_intr(const void *, size_t, unsigned);
240 static void entropy_softintr(void *);
241 static void entropy_thread(void *);
242 static uint32_t entropy_pending(void);
243 static void entropy_pending_cpu(void *, void *, struct cpu_info *);
244 static void entropy_consolidate(void);
245 static void entropy_gather_xc(void *, void *);
246 static void entropy_notify(void);
247 static int sysctl_entropy_consolidate(SYSCTLFN_ARGS);
248 static int sysctl_entropy_gather(SYSCTLFN_ARGS);
249 static void filt_entropy_read_detach(struct knote *);
250 static int filt_entropy_read_event(struct knote *, long);
251 static void entropy_request(size_t);
252 static void rnd_add_data_1(struct krndsource *, const void *, uint32_t,
253 uint32_t);
254 static unsigned rndsource_entropybits(struct krndsource *);
255 static void rndsource_entropybits_cpu(void *, void *, struct cpu_info *);
256 static void rndsource_to_user(struct krndsource *, rndsource_t *);
257 static void rndsource_to_user_est(struct krndsource *, rndsource_est_t *);
258
259 /*
260 * curcpu_available()
261 *
262 * True if we can inspect the current CPU. Early on this may not
263 * work. XXX On most if not all ports, this should work earlier.
264 */
265 static inline bool
266 curcpu_available(void)
267 {
268
269 return __predict_true(!cold);
270 }
271
272 /*
273 * entropy_timer()
274 *
275 * Cycle counter, time counter, or anything that changes a wee bit
276 * unpredictably.
277 */
278 static inline uint32_t
279 entropy_timer(void)
280 {
281 struct bintime bt;
282 uint32_t v;
283
284 /* Very early on, cpu_counter32() may not be available. */
285 if (!curcpu_available())
286 return 0;
287
288 /* If we have a CPU cycle counter, use the low 32 bits. */
289 #ifdef __HAVE_CPU_COUNTER
290 if (__predict_true(cpu_hascounter()))
291 return cpu_counter32();
292 #endif /* __HAVE_CPU_COUNTER */
293
294 /* If we're cold, tough. Can't binuptime while cold. */
295 if (__predict_false(cold))
296 return 0;
297
298 /* Fold the 128 bits of binuptime into 32 bits. */
299 binuptime(&bt);
300 v = bt.frac;
301 v ^= bt.frac >> 32;
302 v ^= bt.sec;
303 v ^= bt.sec >> 32;
304 return v;
305 }
306
307 static void
308 attach_seed_rndsource(void)
309 {
310
311 /*
312 * First called no later than entropy_init, while we are still
313 * single-threaded, so no need for RUN_ONCE.
314 */
315 if (E->stage >= ENTROPY_WARM || E->seed_rndsource)
316 return;
317 rnd_attach_source(&seed_rndsource, "seed", RND_TYPE_UNKNOWN,
318 RND_FLAG_COLLECT_VALUE);
319 E->seed_rndsource = true;
320 }
321
322 /*
323 * entropy_init()
324 *
325 * Initialize the entropy subsystem. Panic on failure.
326 *
327 * Requires percpu(9) and sysctl(9) to be initialized.
328 */
329 static void
330 entropy_init(void)
331 {
332 uint32_t extra[2];
333 struct krndsource *rs;
334 unsigned i = 0;
335
336 KASSERT(E->stage == ENTROPY_COLD);
337
338 /* Grab some cycle counts early at boot. */
339 extra[i++] = entropy_timer();
340
341 /* Run the entropy pool cryptography self-test. */
342 if (entpool_selftest() == -1)
343 panic("entropy pool crypto self-test failed");
344
345 /* Create the sysctl directory. */
346 sysctl_createv(&entropy_sysctllog, 0, NULL, &entropy_sysctlroot,
347 CTLFLAG_PERMANENT, CTLTYPE_NODE, "entropy",
348 SYSCTL_DESCR("Entropy (random number sources) options"),
349 NULL, 0, NULL, 0,
350 CTL_KERN, CTL_CREATE, CTL_EOL);
351
352 /* Create the sysctl knobs. */
353 /* XXX These shouldn't be writable at securelevel>0. */
354 sysctl_createv(&entropy_sysctllog, 0, &entropy_sysctlroot, NULL,
355 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, CTLTYPE_BOOL, "collection",
356 SYSCTL_DESCR("Automatically collect entropy from hardware"),
357 NULL, 0, &entropy_collection, 0, CTL_CREATE, CTL_EOL);
358 sysctl_createv(&entropy_sysctllog, 0, &entropy_sysctlroot, NULL,
359 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, CTLTYPE_BOOL, "depletion",
360 SYSCTL_DESCR("`Deplete' entropy pool when observed"),
361 NULL, 0, &entropy_depletion, 0, CTL_CREATE, CTL_EOL);
362 sysctl_createv(&entropy_sysctllog, 0, &entropy_sysctlroot, NULL,
363 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, CTLTYPE_INT, "consolidate",
364 SYSCTL_DESCR("Trigger entropy consolidation now"),
365 sysctl_entropy_consolidate, 0, NULL, 0, CTL_CREATE, CTL_EOL);
366 sysctl_createv(&entropy_sysctllog, 0, &entropy_sysctlroot, NULL,
367 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, CTLTYPE_INT, "gather",
368 SYSCTL_DESCR("Trigger entropy gathering from sources now"),
369 sysctl_entropy_gather, 0, NULL, 0, CTL_CREATE, CTL_EOL);
370 /* XXX These should maybe not be readable at securelevel>0. */
371 sysctl_createv(&entropy_sysctllog, 0, &entropy_sysctlroot, NULL,
372 CTLFLAG_PERMANENT|CTLFLAG_READONLY|CTLFLAG_PRIVATE, CTLTYPE_INT,
373 "needed", SYSCTL_DESCR("Systemwide entropy deficit"),
374 NULL, 0, &E->needed, 0, CTL_CREATE, CTL_EOL);
375 sysctl_createv(&entropy_sysctllog, 0, &entropy_sysctlroot, NULL,
376 CTLFLAG_PERMANENT|CTLFLAG_READONLY|CTLFLAG_PRIVATE, CTLTYPE_INT,
377 "pending", SYSCTL_DESCR("Entropy pending on CPUs"),
378 NULL, 0, &E->pending, 0, CTL_CREATE, CTL_EOL);
379 sysctl_createv(&entropy_sysctllog, 0, &entropy_sysctlroot, NULL,
380 CTLFLAG_PERMANENT|CTLFLAG_READONLY|CTLFLAG_PRIVATE, CTLTYPE_INT,
381 "epoch", SYSCTL_DESCR("Entropy epoch"),
382 NULL, 0, &E->epoch, 0, CTL_CREATE, CTL_EOL);
383
384 /* Initialize the global state for multithreaded operation. */
385 mutex_init(&E->lock, MUTEX_DEFAULT, IPL_VM);
386 cv_init(&E->cv, "entropy");
387 selinit(&E->selq);
388
389 /* Make sure the seed source is attached. */
390 attach_seed_rndsource();
391
392 /* Note if the bootloader didn't provide a seed. */
393 if (!E->seeded)
394 printf("entropy: no seed from bootloader\n");
395
396 /* Allocate the per-CPU records for all early entropy sources. */
397 LIST_FOREACH(rs, &E->sources, list)
398 rs->state = percpu_alloc(sizeof(struct rndsource_cpu));
399
400 /* Enter the boot cycle count to get started. */
401 extra[i++] = entropy_timer();
402 KASSERT(i == __arraycount(extra));
403 entropy_enter(extra, sizeof extra, 0);
404 explicit_memset(extra, 0, sizeof extra);
405
406 /* We are now ready for multi-threaded operation. */
407 E->stage = ENTROPY_WARM;
408 }
409
410 /*
411 * entropy_init_late()
412 *
413 * Late initialization. Panic on failure.
414 *
415 * Requires CPUs to have been detected and LWPs to have started.
416 */
417 static void
418 entropy_init_late(void)
419 {
420 int error;
421
422 KASSERT(E->stage == ENTROPY_WARM);
423
424 /* Allocate and initialize the per-CPU state. */
425 entropy_percpu = percpu_create(sizeof(struct entropy_cpu),
426 entropy_init_cpu, entropy_fini_cpu, NULL);
427
428 /*
429 * Establish the softint at the highest softint priority level.
430 * Must happen after CPU detection.
431 */
432 entropy_sih = softint_establish(SOFTINT_SERIAL|SOFTINT_MPSAFE,
433 &entropy_softintr, NULL);
434 if (entropy_sih == NULL)
435 panic("unable to establish entropy softint");
436
437 /*
438 * Create the entropy housekeeping thread. Must happen after
439 * lwpinit.
440 */
441 error = kthread_create(PRI_NONE, KTHREAD_MPSAFE|KTHREAD_TS, NULL,
442 entropy_thread, NULL, &entropy_lwp, "entbutler");
443 if (error)
444 panic("unable to create entropy housekeeping thread: %d",
445 error);
446
447 /*
448 * Wait until the per-CPU initialization has hit all CPUs
449 * before proceeding to mark the entropy system hot.
450 */
451 xc_barrier(XC_HIGHPRI);
452 E->stage = ENTROPY_HOT;
453 }
454
455 /*
456 * entropy_init_cpu(ptr, cookie, ci)
457 *
458 * percpu(9) constructor for per-CPU entropy pool.
459 */
460 static void
461 entropy_init_cpu(void *ptr, void *cookie, struct cpu_info *ci)
462 {
463 struct entropy_cpu *ec = ptr;
464
465 ec->ec_softint_evcnt = kmem_alloc(sizeof(*ec->ec_softint_evcnt),
466 KM_SLEEP);
467 ec->ec_pool = kmem_zalloc(sizeof(*ec->ec_pool), KM_SLEEP);
468 ec->ec_pending = 0;
469 ec->ec_locked = false;
470
471 evcnt_attach_dynamic(ec->ec_softint_evcnt, EVCNT_TYPE_MISC, NULL,
472 ci->ci_cpuname, "entropy softint");
473 }
474
475 /*
476 * entropy_fini_cpu(ptr, cookie, ci)
477 *
478 * percpu(9) destructor for per-CPU entropy pool.
479 */
480 static void
481 entropy_fini_cpu(void *ptr, void *cookie, struct cpu_info *ci)
482 {
483 struct entropy_cpu *ec = ptr;
484
485 /*
486 * Zero any lingering data. Disclosure of the per-CPU pool
487 * shouldn't retroactively affect the security of any keys
488 * generated, because entpool(9) erases whatever we have just
489 * drawn out of any pool, but better safe than sorry.
490 */
491 explicit_memset(ec->ec_pool, 0, sizeof(*ec->ec_pool));
492
493 evcnt_detach(ec->ec_softint_evcnt);
494
495 kmem_free(ec->ec_pool, sizeof(*ec->ec_pool));
496 kmem_free(ec->ec_softint_evcnt, sizeof(*ec->ec_softint_evcnt));
497 }
498
499 /*
500 * entropy_seed(seed)
501 *
502 * Seed the entropy pool with seed. Meant to be called as early
503 * as possible by the bootloader; may be called before or after
504 * entropy_init. Must be called before system reaches userland.
505 * Must be called in thread or soft interrupt context, not in hard
506 * interrupt context. Must be called at most once.
507 *
508 * Overwrites the seed in place. Caller may then free the memory.
509 */
510 static void
511 entropy_seed(rndsave_t *seed)
512 {
513 SHA1_CTX ctx;
514 uint8_t digest[SHA1_DIGEST_LENGTH];
515 bool seeded;
516
517 /*
518 * Verify the checksum. If the checksum fails, take the data
519 * but ignore the entropy estimate -- the file may have been
520 * incompletely written with garbage, which is harmless to add
521 * but may not be as unpredictable as alleged.
522 */
523 SHA1Init(&ctx);
524 SHA1Update(&ctx, (const void *)&seed->entropy, sizeof(seed->entropy));
525 SHA1Update(&ctx, seed->data, sizeof(seed->data));
526 SHA1Final(digest, &ctx);
527 CTASSERT(sizeof(seed->digest) == sizeof(digest));
528 if (!consttime_memequal(digest, seed->digest, sizeof(digest))) {
529 printf("entropy: invalid seed checksum\n");
530 seed->entropy = 0;
531 }
532 explicit_memset(&ctx, 0, sizeof ctx);
533 explicit_memset(digest, 0, sizeof digest);
534
535 /*
536 * If the entropy is insensibly large, try byte-swapping.
537 * Otherwise assume the file is corrupted and act as though it
538 * has zero entropy.
539 */
540 if (howmany(seed->entropy, NBBY) > sizeof(seed->data)) {
541 seed->entropy = bswap32(seed->entropy);
542 if (howmany(seed->entropy, NBBY) > sizeof(seed->data))
543 seed->entropy = 0;
544 }
545
546 /* Make sure the seed source is attached. */
547 attach_seed_rndsource();
548
549 /* Test and set E->seeded. */
550 if (E->stage >= ENTROPY_WARM)
551 mutex_enter(&E->lock);
552 seeded = E->seeded;
553 E->seeded = (seed->entropy > 0);
554 if (E->stage >= ENTROPY_WARM)
555 mutex_exit(&E->lock);
556
557 /*
558 * If we've been seeded, may be re-entering the same seed
559 * (e.g., bootloader vs module init, or something). No harm in
560 * entering it twice, but it contributes no additional entropy.
561 */
562 if (seeded) {
563 printf("entropy: double-seeded by bootloader\n");
564 seed->entropy = 0;
565 } else {
566 printf("entropy: entering seed from bootloader"
567 " with %u bits of entropy\n", (unsigned)seed->entropy);
568 }
569
570 /* Enter it into the pool and promptly zero it. */
571 rnd_add_data(&seed_rndsource, seed->data, sizeof(seed->data),
572 seed->entropy);
573 explicit_memset(seed, 0, sizeof(*seed));
574 }
575
576 /*
577 * entropy_bootrequest()
578 *
579 * Request entropy from all sources at boot, once config is
580 * complete and interrupts are running.
581 */
582 void
583 entropy_bootrequest(void)
584 {
585
586 KASSERT(E->stage >= ENTROPY_WARM);
587
588 /*
589 * Request enough to satisfy the maximum entropy shortage.
590 * This is harmless overkill if the bootloader provided a seed.
591 */
592 mutex_enter(&E->lock);
593 entropy_request(ENTROPY_CAPACITY);
594 mutex_exit(&E->lock);
595 }
596
597 /*
598 * entropy_epoch()
599 *
600 * Returns the current entropy epoch. If this changes, you should
601 * reseed. If -1, means the system has not yet reached full
602 * entropy; never reverts back to -1 after full entropy has been
603 * reached. Never zero, so you can always use zero as an
604 * uninitialized sentinel value meaning `reseed ASAP'.
605 *
606 * Usage model:
607 *
608 * struct foo {
609 * struct crypto_prng prng;
610 * unsigned epoch;
611 * } *foo;
612 *
613 * unsigned epoch = entropy_epoch();
614 * if (__predict_false(epoch != foo->epoch)) {
615 * uint8_t seed[32];
616 * if (entropy_extract(seed, sizeof seed, 0) != 0)
617 * warn("no entropy");
618 * crypto_prng_reseed(&foo->prng, seed, sizeof seed);
619 * foo->epoch = epoch;
620 * }
621 */
622 unsigned
623 entropy_epoch(void)
624 {
625
626 /*
627 * Unsigned int, so no need for seqlock for an atomic read, but
628 * make sure we read it afresh each time.
629 */
630 return atomic_load_relaxed(&E->epoch);
631 }
632
633 /*
634 * entropy_account_cpu(ec)
635 *
636 * Consider whether to consolidate entropy into the global pool
637 * after we just added some into the current CPU's pending pool.
638 *
639 * - If this CPU can provide enough entropy now, do so.
640 *
641 * - If this and whatever else is available on other CPUs can
642 * provide enough entropy, kick the consolidation thread.
643 *
644 * - Otherwise, do as little as possible, except maybe consolidate
645 * entropy at most once a minute.
646 *
647 * Caller must be bound to a CPU and therefore have exclusive
648 * access to ec. Will acquire and release the global lock.
649 */
650 static void
651 entropy_account_cpu(struct entropy_cpu *ec)
652 {
653 unsigned diff;
654
655 KASSERT(E->stage == ENTROPY_HOT);
656
657 /*
658 * If there's no entropy needed, and entropy has been
659 * consolidated in the last minute, do nothing.
660 */
661 if (__predict_true(atomic_load_relaxed(&E->needed) == 0) &&
662 __predict_true(!atomic_load_relaxed(&entropy_depletion)) &&
663 __predict_true((time_uptime - E->timestamp) <= 60))
664 return;
665
666 /* If there's nothing pending, stop here. */
667 if (ec->ec_pending == 0)
668 return;
669
670 /* Consider consolidation, under the lock. */
671 mutex_enter(&E->lock);
672 if (E->needed != 0 && E->needed <= ec->ec_pending) {
673 /*
674 * If we have not yet attained full entropy but we can
675 * now, do so. This way we disseminate entropy
676 * promptly when it becomes available early at boot;
677 * otherwise we leave it to the entropy consolidation
678 * thread, which is rate-limited to mitigate side
679 * channels and abuse.
680 */
681 uint8_t buf[ENTPOOL_CAPACITY];
682
683 /* Transfer from the local pool to the global pool. */
684 entpool_extract(ec->ec_pool, buf, sizeof buf);
685 entpool_enter(&E->pool, buf, sizeof buf);
686 atomic_store_relaxed(&ec->ec_pending, 0);
687 atomic_store_relaxed(&E->needed, 0);
688
689 /* Notify waiters that we now have full entropy. */
690 entropy_notify();
691 entropy_immediate_evcnt.ev_count++;
692 } else if (ec->ec_pending) {
693 /* Record how much we can add to the global pool. */
694 diff = MIN(ec->ec_pending, ENTROPY_CAPACITY*NBBY - E->pending);
695 E->pending += diff;
696 atomic_store_relaxed(&ec->ec_pending, ec->ec_pending - diff);
697
698 /*
699 * This should have made a difference unless we were
700 * already saturated.
701 */
702 KASSERT(diff || E->pending == ENTROPY_CAPACITY*NBBY);
703 KASSERT(E->pending);
704
705 if (E->needed <= E->pending) {
706 /*
707 * Enough entropy between all the per-CPU
708 * pools. Wake up the housekeeping thread.
709 *
710 * If we don't need any entropy, this doesn't
711 * mean much, but it is the only time we ever
712 * gather additional entropy in case the
713 * accounting has been overly optimistic. This
714 * happens at most once a minute, so there's
715 * negligible performance cost.
716 */
717 E->consolidate = true;
718 cv_broadcast(&E->cv);
719 if (E->needed == 0)
720 entropy_discretionary_evcnt.ev_count++;
721 } else {
722 /* Can't get full entropy. Keep gathering. */
723 entropy_partial_evcnt.ev_count++;
724 }
725 }
726 mutex_exit(&E->lock);
727 }
728
729 /*
730 * entropy_enter_early(buf, len, nbits)
731 *
732 * Do entropy bookkeeping globally, before we have established
733 * per-CPU pools. Enter directly into the global pool in the hope
734 * that we enter enough before the first entropy_extract to thwart
735 * iterative-guessing attacks; entropy_extract will warn if not.
736 */
737 static void
738 entropy_enter_early(const void *buf, size_t len, unsigned nbits)
739 {
740 bool notify = false;
741
742 if (E->stage >= ENTROPY_WARM)
743 mutex_enter(&E->lock);
744
745 /* Enter it into the pool. */
746 entpool_enter(&E->pool, buf, len);
747
748 /*
749 * Decide whether to notify reseed -- we will do so if either:
750 * (a) we transition from partial entropy to full entropy, or
751 * (b) we get a batch of full entropy all at once.
752 */
753 notify |= (E->needed && E->needed <= nbits);
754 notify |= (nbits >= ENTROPY_CAPACITY*NBBY);
755
756 /* Subtract from the needed count and notify if appropriate. */
757 E->needed -= MIN(E->needed, nbits);
758 if (notify) {
759 entropy_notify();
760 entropy_immediate_evcnt.ev_count++;
761 }
762
763 if (E->stage >= ENTROPY_WARM)
764 mutex_exit(&E->lock);
765 }
766
767 /*
768 * entropy_enter(buf, len, nbits)
769 *
770 * Enter len bytes of data from buf into the system's entropy
771 * pool, stirring as necessary when the internal buffer fills up.
772 * nbits is a lower bound on the number of bits of entropy in the
773 * process that led to this sample.
774 */
775 static void
776 entropy_enter(const void *buf, size_t len, unsigned nbits)
777 {
778 struct entropy_cpu *ec;
779 uint32_t pending;
780 int s;
781
782 KASSERTMSG(!curcpu_available() || !cpu_intr_p(),
783 "use entropy_enter_intr from interrupt context");
784 KASSERTMSG(howmany(nbits, NBBY) <= len,
785 "impossible entropy rate: %u bits in %zu-byte string", nbits, len);
786
787 /* If it's too early after boot, just use entropy_enter_early. */
788 if (__predict_false(E->stage < ENTROPY_HOT)) {
789 entropy_enter_early(buf, len, nbits);
790 return;
791 }
792
793 /*
794 * Acquire the per-CPU state, blocking soft interrupts and
795 * causing hard interrupts to drop samples on the floor.
796 */
797 ec = percpu_getref(entropy_percpu);
798 s = splsoftserial();
799 KASSERT(!ec->ec_locked);
800 ec->ec_locked = true;
801 __insn_barrier();
802
803 /* Enter into the per-CPU pool. */
804 entpool_enter(ec->ec_pool, buf, len);
805
806 /* Count up what we can add. */
807 pending = ec->ec_pending;
808 pending += MIN(ENTROPY_CAPACITY*NBBY - pending, nbits);
809 atomic_store_relaxed(&ec->ec_pending, pending);
810
811 /* Consolidate globally if appropriate based on what we added. */
812 entropy_account_cpu(ec);
813
814 /* Release the per-CPU state. */
815 KASSERT(ec->ec_locked);
816 __insn_barrier();
817 ec->ec_locked = false;
818 splx(s);
819 percpu_putref(entropy_percpu);
820 }
821
822 /*
823 * entropy_enter_intr(buf, len, nbits)
824 *
825 * Enter up to len bytes of data from buf into the system's
826 * entropy pool without stirring. nbits is a lower bound on the
827 * number of bits of entropy in the process that led to this
828 * sample. If the sample could be entered completely, assume
829 * nbits of entropy pending; otherwise assume none, since we don't
830 * know whether some parts of the sample are constant, for
831 * instance. Schedule a softint to stir the entropy pool if
832 * needed. Return true if used fully, false if truncated at all.
833 *
834 * Using this in thread context will work, but you might as well
835 * use entropy_enter in that case.
836 */
837 static bool
838 entropy_enter_intr(const void *buf, size_t len, unsigned nbits)
839 {
840 struct entropy_cpu *ec;
841 bool fullyused = false;
842 uint32_t pending;
843
844 KASSERTMSG(howmany(nbits, NBBY) <= len,
845 "impossible entropy rate: %u bits in %zu-byte string", nbits, len);
846
847 /* If it's too early after boot, just use entropy_enter_early. */
848 if (__predict_false(E->stage < ENTROPY_HOT)) {
849 entropy_enter_early(buf, len, nbits);
850 return true;
851 }
852
853 /*
854 * Acquire the per-CPU state. If someone is in the middle of
855 * using it, drop the sample. Otherwise, take the lock so that
856 * higher-priority interrupts will drop their samples.
857 */
858 ec = percpu_getref(entropy_percpu);
859 if (ec->ec_locked)
860 goto out0;
861 ec->ec_locked = true;
862 __insn_barrier();
863
864 /*
865 * Enter as much as we can into the per-CPU pool. If it was
866 * truncated, schedule a softint to stir the pool and stop.
867 */
868 if (!entpool_enter_nostir(ec->ec_pool, buf, len)) {
869 softint_schedule(entropy_sih);
870 goto out1;
871 }
872 fullyused = true;
873
874 /* Count up what we can contribute. */
875 pending = ec->ec_pending;
876 pending += MIN(ENTROPY_CAPACITY*NBBY - pending, nbits);
877 atomic_store_relaxed(&ec->ec_pending, pending);
878
879 /* Schedule a softint if we added anything and it matters. */
880 if (__predict_false((atomic_load_relaxed(&E->needed) != 0) ||
881 atomic_load_relaxed(&entropy_depletion)) &&
882 nbits != 0)
883 softint_schedule(entropy_sih);
884
885 out1: /* Release the per-CPU state. */
886 KASSERT(ec->ec_locked);
887 __insn_barrier();
888 ec->ec_locked = false;
889 out0: percpu_putref(entropy_percpu);
890
891 return fullyused;
892 }
893
894 /*
895 * entropy_softintr(cookie)
896 *
897 * Soft interrupt handler for entering entropy. Takes care of
898 * stirring the local CPU's entropy pool if it filled up during
899 * hard interrupts, and promptly crediting entropy from the local
900 * CPU's entropy pool to the global entropy pool if needed.
901 */
902 static void
903 entropy_softintr(void *cookie)
904 {
905 struct entropy_cpu *ec;
906
907 /*
908 * Acquire the per-CPU state. Other users can lock this only
909 * while soft interrupts are blocked. Cause hard interrupts to
910 * drop samples on the floor.
911 */
912 ec = percpu_getref(entropy_percpu);
913 KASSERT(!ec->ec_locked);
914 ec->ec_locked = true;
915 __insn_barrier();
916
917 /* Count statistics. */
918 ec->ec_softint_evcnt->ev_count++;
919
920 /* Stir the pool if necessary. */
921 entpool_stir(ec->ec_pool);
922
923 /* Consolidate globally if appropriate based on what we added. */
924 entropy_account_cpu(ec);
925
926 /* Release the per-CPU state. */
927 KASSERT(ec->ec_locked);
928 __insn_barrier();
929 ec->ec_locked = false;
930 percpu_putref(entropy_percpu);
931 }
932
933 /*
934 * entropy_thread(cookie)
935 *
936 * Handle any asynchronous entropy housekeeping.
937 */
938 static void
939 entropy_thread(void *cookie)
940 {
941 bool consolidate;
942
943 for (;;) {
944 /*
945 * Wait until there's full entropy somewhere among the
946 * CPUs, as confirmed at most once per minute, or
947 * someone wants to consolidate.
948 */
949 if (entropy_pending() >= ENTROPY_CAPACITY*NBBY) {
950 consolidate = true;
951 } else {
952 mutex_enter(&E->lock);
953 if (!E->consolidate)
954 cv_timedwait(&E->cv, &E->lock, 60*hz);
955 consolidate = E->consolidate;
956 E->consolidate = false;
957 mutex_exit(&E->lock);
958 }
959
960 if (consolidate) {
961 /* Do it. */
962 entropy_consolidate();
963
964 /* Mitigate abuse. */
965 kpause("entropy", false, hz, NULL);
966 }
967 }
968 }
969
970 /*
971 * entropy_pending()
972 *
973 * Count up the amount of entropy pending on other CPUs.
974 */
975 static uint32_t
976 entropy_pending(void)
977 {
978 uint32_t pending = 0;
979
980 percpu_foreach(entropy_percpu, &entropy_pending_cpu, &pending);
981 return pending;
982 }
983
984 static void
985 entropy_pending_cpu(void *ptr, void *cookie, struct cpu_info *ci)
986 {
987 struct entropy_cpu *ec = ptr;
988 uint32_t *pendingp = cookie;
989 uint32_t cpu_pending;
990
991 cpu_pending = atomic_load_relaxed(&ec->ec_pending);
992 *pendingp += MIN(ENTROPY_CAPACITY*NBBY - *pendingp, cpu_pending);
993 }
994
995 /*
996 * entropy_consolidate()
997 *
998 * Issue a cross-call to gather entropy on all CPUs and advance
999 * the entropy epoch.
1000 */
1001 static void
1002 entropy_consolidate(void)
1003 {
1004 static const struct timeval interval = {.tv_sec = 60, .tv_usec = 0};
1005 static struct timeval lasttime; /* serialized by E->lock */
1006 unsigned diff;
1007 uint64_t ticket;
1008
1009 /* Gather entropy on all CPUs. */
1010 ticket = xc_broadcast(0, &entropy_gather_xc, NULL, NULL);
1011 xc_wait(ticket);
1012
1013 /* Acquire the lock to notify waiters. */
1014 mutex_enter(&E->lock);
1015
1016 /* Count another consolidation. */
1017 entropy_consolidate_evcnt.ev_count++;
1018
1019 /* Note when we last consolidated, i.e. now. */
1020 E->timestamp = time_uptime;
1021
1022 /* Count the entropy that was gathered. */
1023 diff = MIN(E->needed, E->pending);
1024 atomic_store_relaxed(&E->needed, E->needed - diff);
1025 E->pending -= diff;
1026 if (__predict_false(E->needed > 0)) {
1027 if (ratecheck(&lasttime, &interval))
1028 printf("entropy: WARNING:"
1029 " consolidating less than full entropy\n");
1030 }
1031
1032 /* Advance the epoch and notify waiters. */
1033 entropy_notify();
1034
1035 /* Release the lock. */
1036 mutex_exit(&E->lock);
1037 }
1038
1039 /*
1040 * entropy_gather_xc(arg1, arg2)
1041 *
1042 * Extract output from the local CPU's input pool and enter it
1043 * into the global pool.
1044 */
1045 static void
1046 entropy_gather_xc(void *arg1 __unused, void *arg2 __unused)
1047 {
1048 struct entropy_cpu *ec;
1049 uint8_t buf[ENTPOOL_CAPACITY];
1050 uint32_t extra[7];
1051 unsigned i = 0;
1052 int s;
1053
1054 /* Grab CPU number and cycle counter to mix extra into the pool. */
1055 extra[i++] = cpu_number();
1056 extra[i++] = entropy_timer();
1057
1058 /*
1059 * Acquire the per-CPU state, blocking soft interrupts and
1060 * discarding entropy in hard interrupts, so that we can
1061 * extract from the per-CPU pool.
1062 */
1063 ec = percpu_getref(entropy_percpu);
1064 s = splsoftserial();
1065 KASSERT(!ec->ec_locked);
1066 ec->ec_locked = true;
1067 __insn_barrier();
1068 extra[i++] = entropy_timer();
1069
1070 /* Extract the data. */
1071 entpool_extract(ec->ec_pool, buf, sizeof buf);
1072 extra[i++] = entropy_timer();
1073
1074 /* Release the per-CPU state. */
1075 KASSERT(ec->ec_locked);
1076 __insn_barrier();
1077 ec->ec_locked = false;
1078 splx(s);
1079 percpu_putref(entropy_percpu);
1080 extra[i++] = entropy_timer();
1081
1082 /*
1083 * Copy over statistics, and enter the per-CPU extract and the
1084 * extra timing into the global pool, under the global lock.
1085 */
1086 mutex_enter(&E->lock);
1087 extra[i++] = entropy_timer();
1088 entpool_enter(&E->pool, buf, sizeof buf);
1089 explicit_memset(buf, 0, sizeof buf);
1090 extra[i++] = entropy_timer();
1091 KASSERT(i == __arraycount(extra));
1092 entpool_enter(&E->pool, extra, sizeof extra);
1093 explicit_memset(extra, 0, sizeof extra);
1094 mutex_exit(&E->lock);
1095 }
1096
1097 /*
1098 * entropy_notify()
1099 *
1100 * Caller just contributed entropy to the global pool. Advance
1101 * the entropy epoch and notify waiters.
1102 *
1103 * Caller must hold the global entropy lock. Except for the
1104 * `sysctl -w kern.entropy.consolidate=1` trigger, the caller must
1105 * have just have transitioned from partial entropy to full
1106 * entropy -- E->needed should be zero now.
1107 */
1108 static void
1109 entropy_notify(void)
1110 {
1111 unsigned epoch;
1112
1113 KASSERT(E->stage == ENTROPY_COLD || mutex_owned(&E->lock));
1114
1115 /*
1116 * If this is the first time, print a message to the console
1117 * that we're ready so operators can compare it to the timing
1118 * of other events.
1119 */
1120 if (E->epoch == (unsigned)-1)
1121 printf("entropy: ready\n");
1122
1123 /* Set the epoch; roll over from UINTMAX-1 to 1. */
1124 rnd_initial_entropy = 1; /* XXX legacy */
1125 epoch = E->epoch + 1;
1126 if (epoch == 0 || epoch == (unsigned)-1)
1127 epoch = 1;
1128 atomic_store_relaxed(&E->epoch, epoch);
1129
1130 /* Notify waiters. */
1131 if (E->stage >= ENTROPY_WARM) {
1132 cv_broadcast(&E->cv);
1133 selnotify(&E->selq, POLLIN|POLLRDNORM, NOTE_SUBMIT);
1134 }
1135
1136 /* Count another notification. */
1137 entropy_notify_evcnt.ev_count++;
1138 }
1139
1140 /*
1141 * sysctl -w kern.entropy.consolidate=1
1142 *
1143 * Trigger entropy consolidation and wait for it to complete.
1144 * Writable only by superuser. This is the only way for the
1145 * system to consolidate entropy if the operator knows something
1146 * the kernel doesn't about how unpredictable the pending entropy
1147 * pools are.
1148 */
1149 static int
1150 sysctl_entropy_consolidate(SYSCTLFN_ARGS)
1151 {
1152 struct sysctlnode node = *rnode;
1153 uint64_t ticket;
1154 int arg;
1155 int error;
1156
1157 KASSERT(E->stage == ENTROPY_HOT);
1158
1159 node.sysctl_data = &arg;
1160 error = sysctl_lookup(SYSCTLFN_CALL(&node));
1161 if (error || newp == NULL)
1162 return error;
1163 if (arg) {
1164 mutex_enter(&E->lock);
1165 ticket = entropy_consolidate_evcnt.ev_count;
1166 E->consolidate = true;
1167 cv_broadcast(&E->cv);
1168 while (ticket == entropy_consolidate_evcnt.ev_count) {
1169 error = cv_wait_sig(&E->cv, &E->lock);
1170 if (error)
1171 break;
1172 }
1173 mutex_exit(&E->lock);
1174 }
1175
1176 return error;
1177 }
1178
1179 /*
1180 * sysctl -w kern.entropy.gather=1
1181 *
1182 * Trigger gathering entropy from all on-demand sources, and wait
1183 * for synchronous sources (but not asynchronous sources) to
1184 * complete. Writable only by superuser.
1185 */
1186 static int
1187 sysctl_entropy_gather(SYSCTLFN_ARGS)
1188 {
1189 struct sysctlnode node = *rnode;
1190 int arg;
1191 int error;
1192
1193 KASSERT(E->stage == ENTROPY_HOT);
1194
1195 node.sysctl_data = &arg;
1196 error = sysctl_lookup(SYSCTLFN_CALL(&node));
1197 if (error || newp == NULL)
1198 return error;
1199 if (arg) {
1200 mutex_enter(&E->lock);
1201 entropy_request(ENTROPY_CAPACITY);
1202 mutex_exit(&E->lock);
1203 }
1204
1205 return 0;
1206 }
1207
1208 /*
1209 * entropy_extract(buf, len, flags)
1210 *
1211 * Extract len bytes from the global entropy pool into buf.
1212 *
1213 * Flags may have:
1214 *
1215 * ENTROPY_WAIT Wait for entropy if not available yet.
1216 * ENTROPY_SIG Allow interruption by a signal during wait.
1217 *
1218 * Return zero on success, or error on failure:
1219 *
1220 * EWOULDBLOCK No entropy and ENTROPY_WAIT not set.
1221 * EINTR/ERESTART No entropy, ENTROPY_SIG set, and interrupted.
1222 *
1223 * If ENTROPY_WAIT is set, allowed only in thread context. If
1224 * ENTROPY_WAIT is not set, allowed up to IPL_VM. (XXX That's
1225 * awfully high... Do we really need it in hard interrupts? This
1226 * arises from use of cprng_strong(9).)
1227 */
1228 int
1229 entropy_extract(void *buf, size_t len, int flags)
1230 {
1231 static const struct timeval interval = {.tv_sec = 60, .tv_usec = 0};
1232 static struct timeval lasttime; /* serialized by E->lock */
1233 int error;
1234
1235 if (ISSET(flags, ENTROPY_WAIT)) {
1236 ASSERT_SLEEPABLE();
1237 KASSERTMSG(E->stage >= ENTROPY_WARM,
1238 "can't wait for entropy until warm");
1239 }
1240
1241 /* Acquire the global lock to get at the global pool. */
1242 if (E->stage >= ENTROPY_WARM)
1243 mutex_enter(&E->lock);
1244
1245 /* Count up request for entropy in interrupt context. */
1246 if (curcpu_available() && cpu_intr_p())
1247 entropy_extract_intr_evcnt.ev_count++;
1248
1249 /* Wait until there is enough entropy in the system. */
1250 error = 0;
1251 while (E->needed) {
1252 /* Ask for more, synchronously if possible. */
1253 entropy_request(len);
1254
1255 /* If we got enough, we're done. */
1256 if (E->needed == 0) {
1257 KASSERT(error == 0);
1258 break;
1259 }
1260
1261 /* If not waiting, stop here. */
1262 if (!ISSET(flags, ENTROPY_WAIT)) {
1263 error = EWOULDBLOCK;
1264 break;
1265 }
1266
1267 /* Wait for some entropy to come in and try again. */
1268 KASSERT(E->stage >= ENTROPY_WARM);
1269 if (ISSET(flags, ENTROPY_SIG)) {
1270 error = cv_wait_sig(&E->cv, &E->lock);
1271 if (error)
1272 break;
1273 } else {
1274 cv_wait(&E->cv, &E->lock);
1275 }
1276 }
1277
1278 /* Count failure -- but fill the buffer nevertheless. */
1279 if (error)
1280 entropy_extract_fail_evcnt.ev_count++;
1281
1282 /*
1283 * Report a warning if we have never yet reached full entropy.
1284 * This is the only case where we consider entropy to be
1285 * `depleted' without kern.entropy.depletion enabled -- when we
1286 * only have partial entropy, an adversary may be able to
1287 * narrow the state of the pool down to a small number of
1288 * possibilities; the output then enables them to confirm a
1289 * guess, reducing its entropy from the adversary's perspective
1290 * to zero.
1291 */
1292 if (__predict_false(E->epoch == (unsigned)-1)) {
1293 if (ratecheck(&lasttime, &interval))
1294 printf("entropy: WARNING:"
1295 " extracting entropy too early\n");
1296 atomic_store_relaxed(&E->needed, ENTROPY_CAPACITY*NBBY);
1297 }
1298
1299 /* Extract data from the pool, and `deplete' if we're doing that. */
1300 entpool_extract(&E->pool, buf, len);
1301 if (__predict_false(atomic_load_relaxed(&entropy_depletion)) &&
1302 error == 0) {
1303 unsigned cost = MIN(len, ENTROPY_CAPACITY)*NBBY;
1304
1305 atomic_store_relaxed(&E->needed,
1306 E->needed + MIN(ENTROPY_CAPACITY*NBBY - E->needed, cost));
1307 entropy_deplete_evcnt.ev_count++;
1308 }
1309
1310 /* Release the global lock and return the error. */
1311 if (E->stage >= ENTROPY_WARM)
1312 mutex_exit(&E->lock);
1313 return error;
1314 }
1315
1316 /*
1317 * entropy_poll(events)
1318 *
1319 * Return the subset of events ready, and if it is not all of
1320 * events, record curlwp as waiting for entropy.
1321 */
1322 int
1323 entropy_poll(int events)
1324 {
1325 int revents = 0;
1326
1327 KASSERT(E->stage >= ENTROPY_WARM);
1328
1329 /* Always ready for writing. */
1330 revents |= events & (POLLOUT|POLLWRNORM);
1331
1332 /* Narrow it down to reads. */
1333 events &= POLLIN|POLLRDNORM;
1334 if (events == 0)
1335 return revents;
1336
1337 /*
1338 * If we have reached full entropy and we're not depleting
1339 * entropy, we are forever ready.
1340 */
1341 if (__predict_true(atomic_load_relaxed(&E->needed) == 0) &&
1342 __predict_true(!atomic_load_relaxed(&entropy_depletion)))
1343 return revents | events;
1344
1345 /*
1346 * Otherwise, check whether we need entropy under the lock. If
1347 * we don't, we're ready; if we do, add ourselves to the queue.
1348 */
1349 mutex_enter(&E->lock);
1350 if (E->needed == 0)
1351 revents |= events;
1352 else
1353 selrecord(curlwp, &E->selq);
1354 mutex_exit(&E->lock);
1355
1356 return revents;
1357 }
1358
1359 /*
1360 * filt_entropy_read_detach(kn)
1361 *
1362 * struct filterops::f_detach callback for entropy read events:
1363 * remove kn from the list of waiters.
1364 */
1365 static void
1366 filt_entropy_read_detach(struct knote *kn)
1367 {
1368
1369 KASSERT(E->stage >= ENTROPY_WARM);
1370
1371 mutex_enter(&E->lock);
1372 SLIST_REMOVE(&E->selq.sel_klist, kn, knote, kn_selnext);
1373 mutex_exit(&E->lock);
1374 }
1375
1376 /*
1377 * filt_entropy_read_event(kn, hint)
1378 *
1379 * struct filterops::f_event callback for entropy read events:
1380 * poll for entropy. Caller must hold the global entropy lock if
1381 * hint is NOTE_SUBMIT, and must not if hint is not NOTE_SUBMIT.
1382 */
1383 static int
1384 filt_entropy_read_event(struct knote *kn, long hint)
1385 {
1386 int ret;
1387
1388 KASSERT(E->stage >= ENTROPY_WARM);
1389
1390 /* Acquire the lock, if caller is outside entropy subsystem. */
1391 if (hint == NOTE_SUBMIT)
1392 KASSERT(mutex_owned(&E->lock));
1393 else
1394 mutex_enter(&E->lock);
1395
1396 /*
1397 * If we still need entropy, can't read anything; if not, can
1398 * read arbitrarily much.
1399 */
1400 if (E->needed != 0) {
1401 ret = 0;
1402 } else {
1403 if (atomic_load_relaxed(&entropy_depletion))
1404 kn->kn_data = ENTROPY_CAPACITY*NBBY;
1405 else
1406 kn->kn_data = MIN(INT64_MAX, SSIZE_MAX);
1407 ret = 1;
1408 }
1409
1410 /* Release the lock, if caller is outside entropy subsystem. */
1411 if (hint == NOTE_SUBMIT)
1412 KASSERT(mutex_owned(&E->lock));
1413 else
1414 mutex_exit(&E->lock);
1415
1416 return ret;
1417 }
1418
1419 static const struct filterops entropy_read_filtops = {
1420 .f_isfd = 1, /* XXX Makes sense only for /dev/u?random. */
1421 .f_attach = NULL,
1422 .f_detach = filt_entropy_read_detach,
1423 .f_event = filt_entropy_read_event,
1424 };
1425
1426 /*
1427 * entropy_kqfilter(kn)
1428 *
1429 * Register kn to receive entropy event notifications. May be
1430 * EVFILT_READ or EVFILT_WRITE; anything else yields EINVAL.
1431 */
1432 int
1433 entropy_kqfilter(struct knote *kn)
1434 {
1435
1436 KASSERT(E->stage >= ENTROPY_WARM);
1437
1438 switch (kn->kn_filter) {
1439 case EVFILT_READ:
1440 /* Enter into the global select queue. */
1441 mutex_enter(&E->lock);
1442 kn->kn_fop = &entropy_read_filtops;
1443 SLIST_INSERT_HEAD(&E->selq.sel_klist, kn, kn_selnext);
1444 mutex_exit(&E->lock);
1445 return 0;
1446 case EVFILT_WRITE:
1447 /* Can always dump entropy into the system. */
1448 kn->kn_fop = &seltrue_filtops;
1449 return 0;
1450 default:
1451 return EINVAL;
1452 }
1453 }
1454
1455 /*
1456 * rndsource_setcb(rs, get, getarg)
1457 *
1458 * Set the request callback for the entropy source rs, if it can
1459 * provide entropy on demand. Must precede rnd_attach_source.
1460 */
1461 void
1462 rndsource_setcb(struct krndsource *rs, void (*get)(size_t, void *),
1463 void *getarg)
1464 {
1465
1466 rs->get = get;
1467 rs->getarg = getarg;
1468 }
1469
1470 /*
1471 * rnd_attach_source(rs, name, type, flags)
1472 *
1473 * Attach the entropy source rs. Must be done after
1474 * rndsource_setcb, if any, and before any calls to rnd_add_data.
1475 */
1476 void
1477 rnd_attach_source(struct krndsource *rs, const char *name, uint32_t type,
1478 uint32_t flags)
1479 {
1480 uint32_t extra[4];
1481 unsigned i = 0;
1482
1483 /* Grab cycle counter to mix extra into the pool. */
1484 extra[i++] = entropy_timer();
1485
1486 /*
1487 * Apply some standard flags:
1488 *
1489 * - We do not bother with network devices by default, for
1490 * hysterical raisins (perhaps: because it is often the case
1491 * that an adversary can influence network packet timings).
1492 */
1493 switch (type) {
1494 case RND_TYPE_NET:
1495 flags |= RND_FLAG_NO_COLLECT;
1496 break;
1497 }
1498
1499 /* Sanity-check the callback if RND_FLAG_HASCB is set. */
1500 KASSERT(!ISSET(flags, RND_FLAG_HASCB) || rs->get != NULL);
1501
1502 /* Initialize the random source. */
1503 memset(rs->name, 0, sizeof(rs->name)); /* paranoia */
1504 strlcpy(rs->name, name, sizeof(rs->name));
1505 rs->total = 0;
1506 rs->type = type;
1507 rs->flags = flags;
1508 if (E->stage >= ENTROPY_WARM)
1509 rs->state = percpu_alloc(sizeof(struct rndsource_cpu));
1510 extra[i++] = entropy_timer();
1511
1512 /* Wire it into the global list of random sources. */
1513 if (E->stage >= ENTROPY_WARM)
1514 mutex_enter(&E->lock);
1515 LIST_INSERT_HEAD(&E->sources, rs, list);
1516 if (E->stage >= ENTROPY_WARM)
1517 mutex_exit(&E->lock);
1518 extra[i++] = entropy_timer();
1519
1520 /* Request that it provide entropy ASAP, if we can. */
1521 if (ISSET(flags, RND_FLAG_HASCB))
1522 (*rs->get)(ENTROPY_CAPACITY, rs->getarg);
1523 extra[i++] = entropy_timer();
1524
1525 /* Mix the extra into the pool. */
1526 KASSERT(i == __arraycount(extra));
1527 entropy_enter(extra, sizeof extra, 0);
1528 explicit_memset(extra, 0, sizeof extra);
1529 }
1530
1531 /*
1532 * rnd_detach_source(rs)
1533 *
1534 * Detach the entropy source rs. May sleep waiting for users to
1535 * drain. Further use is not allowed.
1536 */
1537 void
1538 rnd_detach_source(struct krndsource *rs)
1539 {
1540
1541 /*
1542 * If we're cold (shouldn't happen, but hey), just remove it
1543 * from the list -- there's nothing allocated.
1544 */
1545 if (E->stage == ENTROPY_COLD) {
1546 LIST_REMOVE(rs, list);
1547 return;
1548 }
1549
1550 /* We may have to wait for entropy_request. */
1551 ASSERT_SLEEPABLE();
1552
1553 /* Wait until the source list is not in use, and remove it. */
1554 mutex_enter(&E->lock);
1555 while (E->sourcelock)
1556 cv_wait(&E->cv, &E->lock);
1557 LIST_REMOVE(rs, list);
1558 mutex_exit(&E->lock);
1559
1560 /* Free the per-CPU data. */
1561 percpu_free(rs->state, sizeof(struct rndsource_cpu));
1562 }
1563
1564 /*
1565 * rnd_lock_sources()
1566 *
1567 * Prevent changes to the list of rndsources while we iterate it.
1568 * Interruptible. Caller must hold the global entropy lock. If
1569 * successful, no rndsource will go away until rnd_unlock_sources
1570 * even while the caller releases the global entropy lock.
1571 */
1572 static int
1573 rnd_lock_sources(void)
1574 {
1575 int error;
1576
1577 KASSERT(mutex_owned(&E->lock));
1578
1579 while (E->sourcelock) {
1580 error = cv_wait_sig(&E->cv, &E->lock);
1581 if (error)
1582 return error;
1583 }
1584
1585 E->sourcelock = curlwp;
1586 return 0;
1587 }
1588
1589 /*
1590 * rnd_trylock_sources()
1591 *
1592 * Try to lock the list of sources, but if it's already locked,
1593 * fail. Caller must hold the global entropy lock. If
1594 * successful, no rndsource will go away until rnd_unlock_sources
1595 * even while the caller releases the global entropy lock.
1596 */
1597 static bool
1598 rnd_trylock_sources(void)
1599 {
1600
1601 KASSERT(E->stage == ENTROPY_COLD || mutex_owned(&E->lock));
1602
1603 if (E->sourcelock)
1604 return false;
1605 E->sourcelock = (curcpu_available() ? curlwp : (void *)1);
1606 return true;
1607 }
1608
1609 /*
1610 * rnd_unlock_sources()
1611 *
1612 * Unlock the list of sources after rnd_lock_sources or
1613 * rnd_trylock_sources. Caller must hold the global entropy lock.
1614 */
1615 static void
1616 rnd_unlock_sources(void)
1617 {
1618
1619 KASSERT(E->stage == ENTROPY_COLD || mutex_owned(&E->lock));
1620
1621 KASSERTMSG(E->sourcelock == (curcpu_available() ? curlwp : (void *)1),
1622 "lwp %p releasing lock held by %p",
1623 (curcpu_available() ? curlwp : (void *)1), E->sourcelock);
1624 E->sourcelock = NULL;
1625 if (E->stage >= ENTROPY_WARM)
1626 cv_broadcast(&E->cv);
1627 }
1628
1629 /*
1630 * rnd_sources_locked()
1631 *
1632 * True if we hold the list of rndsources locked, for diagnostic
1633 * assertions.
1634 */
1635 static bool __diagused
1636 rnd_sources_locked(void)
1637 {
1638
1639 return E->sourcelock == (curcpu_available() ? curlwp : (void *)1);
1640 }
1641
1642 /*
1643 * entropy_request(nbytes)
1644 *
1645 * Request nbytes bytes of entropy from all sources in the system.
1646 * OK if we overdo it. Caller must hold the global entropy lock;
1647 * will release and re-acquire it.
1648 */
1649 static void
1650 entropy_request(size_t nbytes)
1651 {
1652 struct krndsource *rs;
1653
1654 KASSERT(E->stage == ENTROPY_COLD || mutex_owned(&E->lock));
1655
1656 /*
1657 * If there is a request in progress, let it proceed.
1658 * Otherwise, note that a request is in progress to avoid
1659 * reentry and to block rnd_detach_source until we're done.
1660 */
1661 if (!rnd_trylock_sources())
1662 return;
1663 entropy_request_evcnt.ev_count++;
1664
1665 /* Clamp to the maximum reasonable request. */
1666 nbytes = MIN(nbytes, ENTROPY_CAPACITY);
1667
1668 /* Walk the list of sources. */
1669 LIST_FOREACH(rs, &E->sources, list) {
1670 /* Skip sources without callbacks. */
1671 if (!ISSET(rs->flags, RND_FLAG_HASCB))
1672 continue;
1673
1674 /* Drop the lock while we call the callback. */
1675 if (E->stage >= ENTROPY_WARM)
1676 mutex_exit(&E->lock);
1677 (*rs->get)(nbytes, rs->getarg);
1678 if (E->stage >= ENTROPY_WARM)
1679 mutex_enter(&E->lock);
1680 }
1681
1682 /* Notify rnd_detach_source that the request is done. */
1683 rnd_unlock_sources();
1684 }
1685
1686 /*
1687 * rnd_add_uint32(rs, value)
1688 *
1689 * Enter 32 bits of data from an entropy source into the pool.
1690 *
1691 * If rs is NULL, may not be called from interrupt context.
1692 *
1693 * If rs is non-NULL, may be called from any context. May drop
1694 * data if called from interrupt context.
1695 */
1696 void
1697 rnd_add_uint32(struct krndsource *rs, uint32_t value)
1698 {
1699
1700 rnd_add_data(rs, &value, sizeof value, 0);
1701 }
1702
1703 void
1704 _rnd_add_uint32(struct krndsource *rs, uint32_t value)
1705 {
1706
1707 rnd_add_data(rs, &value, sizeof value, 0);
1708 }
1709
1710 void
1711 _rnd_add_uint64(struct krndsource *rs, uint64_t value)
1712 {
1713
1714 rnd_add_data(rs, &value, sizeof value, 0);
1715 }
1716
1717 /*
1718 * rnd_add_data(rs, buf, len, entropybits)
1719 *
1720 * Enter data from an entropy source into the pool, with a
1721 * driver's estimate of how much entropy the physical source of
1722 * the data has. If RND_FLAG_NO_ESTIMATE, we ignore the driver's
1723 * estimate and treat it as zero.
1724 *
1725 * If rs is NULL, may not be called from interrupt context.
1726 *
1727 * If rs is non-NULL, may be called from any context. May drop
1728 * data if called from interrupt context.
1729 */
1730 void
1731 rnd_add_data(struct krndsource *rs, const void *buf, uint32_t len,
1732 uint32_t entropybits)
1733 {
1734 uint32_t extra;
1735 uint32_t flags;
1736
1737 KASSERTMSG(howmany(entropybits, NBBY) <= len,
1738 "%s: impossible entropy rate:"
1739 " %"PRIu32" bits in %"PRIu32"-byte string",
1740 rs ? rs->name : "(anonymous)", entropybits, len);
1741
1742 /* If there's no rndsource, just enter the data and time now. */
1743 if (rs == NULL) {
1744 entropy_enter(buf, len, entropybits);
1745 extra = entropy_timer();
1746 entropy_enter(&extra, sizeof extra, 0);
1747 explicit_memset(&extra, 0, sizeof extra);
1748 return;
1749 }
1750
1751 /* Load a snapshot of the flags. Ioctl may change them under us. */
1752 flags = atomic_load_relaxed(&rs->flags);
1753
1754 /*
1755 * Skip if:
1756 * - we're not collecting entropy, or
1757 * - the operator doesn't want to collect entropy from this, or
1758 * - neither data nor timings are being collected from this.
1759 */
1760 if (!atomic_load_relaxed(&entropy_collection) ||
1761 ISSET(flags, RND_FLAG_NO_COLLECT) ||
1762 !ISSET(flags, RND_FLAG_COLLECT_VALUE|RND_FLAG_COLLECT_TIME))
1763 return;
1764
1765 /* If asked, ignore the estimate. */
1766 if (ISSET(flags, RND_FLAG_NO_ESTIMATE))
1767 entropybits = 0;
1768
1769 /* If we are collecting data, enter them. */
1770 if (ISSET(flags, RND_FLAG_COLLECT_VALUE))
1771 rnd_add_data_1(rs, buf, len, entropybits);
1772
1773 /* If we are collecting timings, enter one. */
1774 if (ISSET(flags, RND_FLAG_COLLECT_TIME)) {
1775 extra = entropy_timer();
1776 rnd_add_data_1(rs, &extra, sizeof extra, 0);
1777 }
1778 }
1779
1780 /*
1781 * rnd_add_data_1(rs, buf, len, entropybits)
1782 *
1783 * Internal subroutine to call either entropy_enter_intr, if we're
1784 * in interrupt context, or entropy_enter if not, and to count the
1785 * entropy in an rndsource.
1786 */
1787 static void
1788 rnd_add_data_1(struct krndsource *rs, const void *buf, uint32_t len,
1789 uint32_t entropybits)
1790 {
1791 bool fullyused;
1792
1793 /*
1794 * If we're in interrupt context, use entropy_enter_intr and
1795 * take note of whether it consumed the full sample; if not,
1796 * use entropy_enter, which always consumes the full sample.
1797 */
1798 if (curcpu_available() && cpu_intr_p()) {
1799 fullyused = entropy_enter_intr(buf, len, entropybits);
1800 } else {
1801 entropy_enter(buf, len, entropybits);
1802 fullyused = true;
1803 }
1804
1805 /*
1806 * If we used the full sample, note how many bits were
1807 * contributed from this source.
1808 */
1809 if (fullyused) {
1810 if (E->stage < ENTROPY_HOT) {
1811 if (E->stage >= ENTROPY_WARM)
1812 mutex_enter(&E->lock);
1813 rs->total += MIN(UINT_MAX - rs->total, entropybits);
1814 if (E->stage >= ENTROPY_WARM)
1815 mutex_exit(&E->lock);
1816 } else {
1817 struct rndsource_cpu *rc = percpu_getref(rs->state);
1818 unsigned nbits = rc->rc_nbits;
1819
1820 nbits += MIN(UINT_MAX - nbits, entropybits);
1821 atomic_store_relaxed(&rc->rc_nbits, nbits);
1822 percpu_putref(rs->state);
1823 }
1824 }
1825 }
1826
1827 /*
1828 * rnd_add_data_sync(rs, buf, len, entropybits)
1829 *
1830 * Same as rnd_add_data. Originally used in rndsource callbacks,
1831 * to break an unnecessary cycle; no longer really needed.
1832 */
1833 void
1834 rnd_add_data_sync(struct krndsource *rs, const void *buf, uint32_t len,
1835 uint32_t entropybits)
1836 {
1837
1838 rnd_add_data(rs, buf, len, entropybits);
1839 }
1840
1841 /*
1842 * rndsource_entropybits(rs)
1843 *
1844 * Return approximately the number of bits of entropy that have
1845 * been contributed via rs so far. Approximate if other CPUs may
1846 * be calling rnd_add_data concurrently.
1847 */
1848 static unsigned
1849 rndsource_entropybits(struct krndsource *rs)
1850 {
1851 unsigned nbits = rs->total;
1852
1853 KASSERT(E->stage >= ENTROPY_WARM);
1854 KASSERT(rnd_sources_locked());
1855 percpu_foreach(rs->state, rndsource_entropybits_cpu, &nbits);
1856 return nbits;
1857 }
1858
1859 static void
1860 rndsource_entropybits_cpu(void *ptr, void *cookie, struct cpu_info *ci)
1861 {
1862 struct rndsource_cpu *rc = ptr;
1863 unsigned *nbitsp = cookie;
1864 unsigned cpu_nbits;
1865
1866 cpu_nbits = atomic_load_relaxed(&rc->rc_nbits);
1867 *nbitsp += MIN(UINT_MAX - *nbitsp, cpu_nbits);
1868 }
1869
1870 /*
1871 * rndsource_to_user(rs, urs)
1872 *
1873 * Copy a description of rs out to urs for userland.
1874 */
1875 static void
1876 rndsource_to_user(struct krndsource *rs, rndsource_t *urs)
1877 {
1878
1879 KASSERT(E->stage >= ENTROPY_WARM);
1880 KASSERT(rnd_sources_locked());
1881
1882 /* Avoid kernel memory disclosure. */
1883 memset(urs, 0, sizeof(*urs));
1884
1885 CTASSERT(sizeof(urs->name) == sizeof(rs->name));
1886 strlcpy(urs->name, rs->name, sizeof(urs->name));
1887 urs->total = rndsource_entropybits(rs);
1888 urs->type = rs->type;
1889 urs->flags = atomic_load_relaxed(&rs->flags);
1890 }
1891
1892 /*
1893 * rndsource_to_user_est(rs, urse)
1894 *
1895 * Copy a description of rs and estimation statistics out to urse
1896 * for userland.
1897 */
1898 static void
1899 rndsource_to_user_est(struct krndsource *rs, rndsource_est_t *urse)
1900 {
1901
1902 KASSERT(E->stage >= ENTROPY_WARM);
1903 KASSERT(rnd_sources_locked());
1904
1905 /* Avoid kernel memory disclosure. */
1906 memset(urse, 0, sizeof(*urse));
1907
1908 /* Copy out the rndsource description. */
1909 rndsource_to_user(rs, &urse->rt);
1910
1911 /* Zero out the statistics because we don't do estimation. */
1912 urse->dt_samples = 0;
1913 urse->dt_total = 0;
1914 urse->dv_samples = 0;
1915 urse->dv_total = 0;
1916 }
1917
1918 /*
1919 * entropy_ioctl(cmd, data)
1920 *
1921 * Handle various /dev/random ioctl queries.
1922 */
1923 int
1924 entropy_ioctl(unsigned long cmd, void *data)
1925 {
1926 struct krndsource *rs;
1927 bool privileged;
1928 int error;
1929
1930 KASSERT(E->stage >= ENTROPY_WARM);
1931
1932 /* Verify user's authorization to perform the ioctl. */
1933 switch (cmd) {
1934 case RNDGETENTCNT:
1935 case RNDGETPOOLSTAT:
1936 case RNDGETSRCNUM:
1937 case RNDGETSRCNAME:
1938 case RNDGETESTNUM:
1939 case RNDGETESTNAME:
1940 error = kauth_authorize_device(curlwp->l_cred,
1941 KAUTH_DEVICE_RND_GETPRIV, NULL, NULL, NULL, NULL);
1942 break;
1943 case RNDCTL:
1944 error = kauth_authorize_device(curlwp->l_cred,
1945 KAUTH_DEVICE_RND_SETPRIV, NULL, NULL, NULL, NULL);
1946 break;
1947 case RNDADDDATA:
1948 error = kauth_authorize_device(curlwp->l_cred,
1949 KAUTH_DEVICE_RND_ADDDATA, NULL, NULL, NULL, NULL);
1950 /* Ascertain whether the user's inputs should be counted. */
1951 if (kauth_authorize_device(curlwp->l_cred,
1952 KAUTH_DEVICE_RND_ADDDATA_ESTIMATE,
1953 NULL, NULL, NULL, NULL) == 0)
1954 privileged = true;
1955 break;
1956 default: {
1957 /*
1958 * XXX Hack to avoid changing module ABI so this can be
1959 * pulled up. Later, we can just remove the argument.
1960 */
1961 static const struct fileops fops = {
1962 .fo_ioctl = rnd_system_ioctl,
1963 };
1964 struct file f = {
1965 .f_ops = &fops,
1966 };
1967 MODULE_HOOK_CALL(rnd_ioctl_50_hook, (&f, cmd, data),
1968 enosys(), error);
1969 #if defined(_LP64)
1970 if (error == ENOSYS)
1971 MODULE_HOOK_CALL(rnd_ioctl32_50_hook, (&f, cmd, data),
1972 enosys(), error);
1973 #endif
1974 if (error == ENOSYS)
1975 error = ENOTTY;
1976 break;
1977 }
1978 }
1979
1980 /* If anything went wrong with authorization, stop here. */
1981 if (error)
1982 return error;
1983
1984 /* Dispatch on the command. */
1985 switch (cmd) {
1986 case RNDGETENTCNT: { /* Get current entropy count in bits. */
1987 uint32_t *countp = data;
1988
1989 mutex_enter(&E->lock);
1990 *countp = ENTROPY_CAPACITY*NBBY - E->needed;
1991 mutex_exit(&E->lock);
1992
1993 break;
1994 }
1995 case RNDGETPOOLSTAT: { /* Get entropy pool statistics. */
1996 rndpoolstat_t *pstat = data;
1997
1998 mutex_enter(&E->lock);
1999
2000 /* parameters */
2001 pstat->poolsize = ENTPOOL_SIZE/sizeof(uint32_t); /* words */
2002 pstat->threshold = ENTROPY_CAPACITY*1; /* bytes */
2003 pstat->maxentropy = ENTROPY_CAPACITY*NBBY; /* bits */
2004
2005 /* state */
2006 pstat->added = 0; /* XXX total entropy_enter count */
2007 pstat->curentropy = ENTROPY_CAPACITY*NBBY - E->needed;
2008 pstat->removed = 0; /* XXX total entropy_extract count */
2009 pstat->discarded = 0; /* XXX bits of entropy beyond capacity */
2010 pstat->generated = 0; /* XXX bits of data...fabricated? */
2011
2012 mutex_exit(&E->lock);
2013 break;
2014 }
2015 case RNDGETSRCNUM: { /* Get entropy sources by number. */
2016 rndstat_t *stat = data;
2017 uint32_t start = 0, i = 0;
2018
2019 /* Skip if none requested; fail if too many requested. */
2020 if (stat->count == 0)
2021 break;
2022 if (stat->count > RND_MAXSTATCOUNT)
2023 return EINVAL;
2024
2025 /*
2026 * Under the lock, find the first one, copy out as many
2027 * as requested, and report how many we copied out.
2028 */
2029 mutex_enter(&E->lock);
2030 error = rnd_lock_sources();
2031 if (error) {
2032 mutex_exit(&E->lock);
2033 return error;
2034 }
2035 LIST_FOREACH(rs, &E->sources, list) {
2036 if (start++ == stat->start)
2037 break;
2038 }
2039 while (i < stat->count && rs != NULL) {
2040 mutex_exit(&E->lock);
2041 rndsource_to_user(rs, &stat->source[i++]);
2042 mutex_enter(&E->lock);
2043 rs = LIST_NEXT(rs, list);
2044 }
2045 KASSERT(i <= stat->count);
2046 stat->count = i;
2047 rnd_unlock_sources();
2048 mutex_exit(&E->lock);
2049 break;
2050 }
2051 case RNDGETESTNUM: { /* Get sources and estimates by number. */
2052 rndstat_est_t *estat = data;
2053 uint32_t start = 0, i = 0;
2054
2055 /* Skip if none requested; fail if too many requested. */
2056 if (estat->count == 0)
2057 break;
2058 if (estat->count > RND_MAXSTATCOUNT)
2059 return EINVAL;
2060
2061 /*
2062 * Under the lock, find the first one, copy out as many
2063 * as requested, and report how many we copied out.
2064 */
2065 mutex_enter(&E->lock);
2066 error = rnd_lock_sources();
2067 if (error) {
2068 mutex_exit(&E->lock);
2069 return error;
2070 }
2071 LIST_FOREACH(rs, &E->sources, list) {
2072 if (start++ == estat->start)
2073 break;
2074 }
2075 while (i < estat->count && rs != NULL) {
2076 mutex_exit(&E->lock);
2077 rndsource_to_user_est(rs, &estat->source[i++]);
2078 mutex_enter(&E->lock);
2079 rs = LIST_NEXT(rs, list);
2080 }
2081 KASSERT(i <= estat->count);
2082 estat->count = i;
2083 rnd_unlock_sources();
2084 mutex_exit(&E->lock);
2085 break;
2086 }
2087 case RNDGETSRCNAME: { /* Get entropy sources by name. */
2088 rndstat_name_t *nstat = data;
2089 const size_t n = sizeof(rs->name);
2090
2091 CTASSERT(sizeof(rs->name) == sizeof(nstat->name));
2092
2093 /*
2094 * Under the lock, search by name. If found, copy it
2095 * out; if not found, fail with ENOENT.
2096 */
2097 mutex_enter(&E->lock);
2098 error = rnd_lock_sources();
2099 if (error) {
2100 mutex_exit(&E->lock);
2101 return error;
2102 }
2103 LIST_FOREACH(rs, &E->sources, list) {
2104 if (strncmp(rs->name, nstat->name, n) == 0)
2105 break;
2106 }
2107 if (rs != NULL) {
2108 mutex_exit(&E->lock);
2109 rndsource_to_user(rs, &nstat->source);
2110 mutex_enter(&E->lock);
2111 } else {
2112 error = ENOENT;
2113 }
2114 rnd_unlock_sources();
2115 mutex_exit(&E->lock);
2116 break;
2117 }
2118 case RNDGETESTNAME: { /* Get sources and estimates by name. */
2119 rndstat_est_name_t *enstat = data;
2120 const size_t n = sizeof(rs->name);
2121
2122 CTASSERT(sizeof(rs->name) == sizeof(enstat->name));
2123
2124 /*
2125 * Under the lock, search by name. If found, copy it
2126 * out; if not found, fail with ENOENT.
2127 */
2128 mutex_enter(&E->lock);
2129 error = rnd_lock_sources();
2130 if (error) {
2131 mutex_exit(&E->lock);
2132 return error;
2133 }
2134 LIST_FOREACH(rs, &E->sources, list) {
2135 if (strncmp(rs->name, enstat->name, n) == 0)
2136 break;
2137 }
2138 if (rs != NULL) {
2139 mutex_exit(&E->lock);
2140 rndsource_to_user_est(rs, &enstat->source);
2141 mutex_enter(&E->lock);
2142 } else {
2143 error = ENOENT;
2144 }
2145 rnd_unlock_sources();
2146 mutex_exit(&E->lock);
2147 break;
2148 }
2149 case RNDCTL: { /* Modify entropy source flags. */
2150 rndctl_t *rndctl = data;
2151 const size_t n = sizeof(rs->name);
2152 uint32_t flags;
2153
2154 CTASSERT(sizeof(rs->name) == sizeof(rndctl->name));
2155
2156 /* Whitelist the flags that user can change. */
2157 rndctl->mask &= RND_FLAG_NO_ESTIMATE|RND_FLAG_NO_COLLECT;
2158
2159 /*
2160 * For each matching rndsource, either by type if
2161 * specified or by name if not, set the masked flags.
2162 */
2163 mutex_enter(&E->lock);
2164 LIST_FOREACH(rs, &E->sources, list) {
2165 if (rndctl->type != 0xff) {
2166 if (rs->type != rndctl->type)
2167 continue;
2168 } else {
2169 if (strncmp(rs->name, rndctl->name, n) != 0)
2170 continue;
2171 }
2172 flags = rs->flags & ~rndctl->mask;
2173 flags |= rndctl->flags & rndctl->mask;
2174 atomic_store_relaxed(&rs->flags, flags);
2175 }
2176 mutex_exit(&E->lock);
2177 break;
2178 }
2179 case RNDADDDATA: { /* Enter seed into entropy pool. */
2180 rnddata_t *rdata = data;
2181 unsigned entropybits = 0;
2182
2183 if (!atomic_load_relaxed(&entropy_collection))
2184 break; /* thanks but no thanks */
2185 if (rdata->len > MIN(sizeof(rdata->data), UINT32_MAX/NBBY))
2186 return EINVAL;
2187
2188 /*
2189 * This ioctl serves as the userland alternative a
2190 * bootloader-provided seed -- typically furnished by
2191 * /etc/rc.d/random_seed. We accept the user's entropy
2192 * claim only if
2193 *
2194 * (a) the user is privileged, and
2195 * (b) we have not entered a bootloader seed.
2196 *
2197 * under the assumption that the user may use this to
2198 * load a seed from disk that we have already loaded
2199 * from the bootloader, so we don't double-count it.
2200 */
2201 if (privileged && rdata->entropy && rdata->len) {
2202 mutex_enter(&E->lock);
2203 if (!E->seeded) {
2204 entropybits = MIN(rdata->entropy,
2205 MIN(rdata->len, ENTROPY_CAPACITY)*NBBY);
2206 E->seeded = true;
2207 }
2208 mutex_exit(&E->lock);
2209 }
2210
2211 /* Enter the data. */
2212 rnd_add_data(&seed_rndsource, rdata->data, rdata->len,
2213 entropybits);
2214 break;
2215 }
2216 default:
2217 error = ENOTTY;
2218 }
2219
2220 /* Return any error that may have come up. */
2221 return error;
2222 }
2223
2224 /* Legacy entry points */
2225
2226 void
2227 rnd_seed(void *seed, size_t len)
2228 {
2229
2230 if (len != sizeof(rndsave_t)) {
2231 printf("entropy: invalid seed length: %zu,"
2232 " expected sizeof(rndsave_t) = %zu\n",
2233 len, sizeof(rndsave_t));
2234 return;
2235 }
2236 entropy_seed(seed);
2237 }
2238
2239 void
2240 rnd_init(void)
2241 {
2242
2243 entropy_init();
2244 }
2245
2246 void
2247 rnd_init_softint(void)
2248 {
2249
2250 entropy_init_late();
2251 }
2252
2253 int
2254 rnd_system_ioctl(struct file *fp, unsigned long cmd, void *data)
2255 {
2256
2257 return entropy_ioctl(cmd, data);
2258 }
2259