kern_entropy.c revision 1.7 1 /* $NetBSD: kern_entropy.c,v 1.7 2020/04/30 20:06:40 riastradh Exp $ */
2
3 /*-
4 * Copyright (c) 2019 The NetBSD Foundation, Inc.
5 * All rights reserved.
6 *
7 * This code is derived from software contributed to The NetBSD Foundation
8 * by Taylor R. Campbell.
9 *
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions
12 * are met:
13 * 1. Redistributions of source code must retain the above copyright
14 * notice, this list of conditions and the following disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright
16 * notice, this list of conditions and the following disclaimer in the
17 * documentation and/or other materials provided with the distribution.
18 *
19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29 * POSSIBILITY OF SUCH DAMAGE.
30 */
31
32 /*
33 * Entropy subsystem
34 *
35 * * Each CPU maintains a per-CPU entropy pool so that gathering
36 * entropy requires no interprocessor synchronization, except
37 * early at boot when we may be scrambling to gather entropy as
38 * soon as possible.
39 *
40 * - entropy_enter gathers entropy and never drops it on the
41 * floor, at the cost of sometimes having to do cryptography.
42 *
43 * - entropy_enter_intr gathers entropy or drops it on the
44 * floor, with low latency. Work to stir the pool or kick the
45 * housekeeping thread is scheduled in soft interrupts.
46 *
47 * * entropy_enter immediately enters into the global pool if it
48 * can transition to full entropy in one swell foop. Otherwise,
49 * it defers to a housekeeping thread that consolidates entropy,
50 * but only when the CPUs collectively have full entropy, in
51 * order to mitigate iterative-guessing attacks.
52 *
53 * * The entropy housekeeping thread continues to consolidate
54 * entropy even after we think we have full entropy, in case we
55 * are wrong, but is limited to one discretionary consolidation
56 * per minute, and only when new entropy is actually coming in,
57 * to limit performance impact.
58 *
59 * * The entropy epoch is the number that changes when we
60 * transition from partial entropy to full entropy, so that
61 * users can easily determine when to reseed. This also
62 * facilitates an operator explicitly causing everything to
63 * reseed by sysctl -w kern.entropy.consolidate=1, e.g. if they
64 * just flipped a coin 256 times and wrote `echo tthhhhhthh... >
65 * /dev/random'.
66 *
67 * * No entropy estimation based on the sample values, which is a
68 * contradiction in terms and a potential source of side
69 * channels. It is the responsibility of the driver author to
70 * study how predictable the physical source of input can ever
71 * be, and to furnish a lower bound on the amount of entropy it
72 * has.
73 *
74 * * Entropy depletion is available for testing (or if you're into
75 * that sort of thing), with sysctl -w kern.entropy.depletion=1;
76 * the logic to support it is small, to minimize chance of bugs.
77 */
78
79 #include <sys/cdefs.h>
80 __KERNEL_RCSID(0, "$NetBSD: kern_entropy.c,v 1.7 2020/04/30 20:06:40 riastradh Exp $");
81
82 #include <sys/param.h>
83 #include <sys/types.h>
84 #include <sys/atomic.h>
85 #include <sys/compat_stub.h>
86 #include <sys/condvar.h>
87 #include <sys/cpu.h>
88 #include <sys/entropy.h>
89 #include <sys/errno.h>
90 #include <sys/evcnt.h>
91 #include <sys/event.h>
92 #include <sys/file.h>
93 #include <sys/intr.h>
94 #include <sys/kauth.h>
95 #include <sys/kernel.h>
96 #include <sys/kmem.h>
97 #include <sys/kthread.h>
98 #include <sys/module_hook.h>
99 #include <sys/mutex.h>
100 #include <sys/percpu.h>
101 #include <sys/poll.h>
102 #include <sys/queue.h>
103 #include <sys/rnd.h> /* legacy kernel API */
104 #include <sys/rndio.h> /* userland ioctl interface */
105 #include <sys/rndsource.h> /* kernel rndsource driver API */
106 #include <sys/select.h>
107 #include <sys/selinfo.h>
108 #include <sys/sha1.h> /* for boot seed checksum */
109 #include <sys/stdint.h>
110 #include <sys/sysctl.h>
111 #include <sys/systm.h>
112 #include <sys/time.h>
113 #include <sys/xcall.h>
114
115 #include <lib/libkern/entpool.h>
116
117 #include <machine/limits.h>
118
119 #ifdef __HAVE_CPU_COUNTER
120 #include <machine/cpu_counter.h>
121 #endif
122
123 /*
124 * struct entropy_cpu
125 *
126 * Per-CPU entropy state. The pool is allocated separately
127 * because percpu(9) sometimes moves per-CPU objects around
128 * without zeroing them, which would lead to unwanted copies of
129 * sensitive secrets. The evcnt is allocated separately becuase
130 * evcnt(9) assumes it stays put in memory.
131 */
132 struct entropy_cpu {
133 struct evcnt *ec_softint_evcnt;
134 struct entpool *ec_pool;
135 unsigned ec_pending;
136 bool ec_locked;
137 };
138
139 /*
140 * struct rndsource_cpu
141 *
142 * Per-CPU rndsource state.
143 */
144 struct rndsource_cpu {
145 unsigned rc_nbits; /* bits of entropy added */
146 };
147
148 /*
149 * entropy_global (a.k.a. E for short in this file)
150 *
151 * Global entropy state. Writes protected by the global lock.
152 * Some fields, marked (A), can be read outside the lock, and are
153 * maintained with atomic_load/store_relaxed.
154 */
155 struct {
156 kmutex_t lock; /* covers all global state */
157 struct entpool pool; /* global pool for extraction */
158 unsigned needed; /* (A) needed globally */
159 unsigned pending; /* (A) pending in per-CPU pools */
160 unsigned timestamp; /* (A) time of last consolidation */
161 unsigned epoch; /* (A) changes when needed -> 0 */
162 kcondvar_t cv; /* notifies state changes */
163 struct selinfo selq; /* notifies needed -> 0 */
164 struct lwp *sourcelock; /* lock on list of sources */
165 LIST_HEAD(,krndsource) sources; /* list of entropy sources */
166 enum entropy_stage {
167 ENTROPY_COLD = 0, /* single-threaded */
168 ENTROPY_WARM, /* multi-threaded at boot before CPUs */
169 ENTROPY_HOT, /* multi-threaded multi-CPU */
170 } stage;
171 bool consolidate; /* kick thread to consolidate */
172 bool seed_rndsource; /* true if seed source is attached */
173 bool seeded; /* true if seed file already loaded */
174 } entropy_global __cacheline_aligned = {
175 /* Fields that must be initialized when the kernel is loaded. */
176 .needed = ENTROPY_CAPACITY*NBBY,
177 .epoch = (unsigned)-1, /* -1 means not yet full entropy */
178 .sources = LIST_HEAD_INITIALIZER(entropy_global.sources),
179 .stage = ENTROPY_COLD,
180 };
181
182 #define E (&entropy_global) /* declutter */
183
184 /* Read-mostly globals */
185 static struct percpu *entropy_percpu __read_mostly; /* struct entropy_cpu */
186 static void *entropy_sih __read_mostly; /* softint handler */
187 static struct lwp *entropy_lwp __read_mostly; /* housekeeping thread */
188
189 int rnd_initial_entropy __read_mostly; /* XXX legacy */
190
191 static struct krndsource seed_rndsource __read_mostly;
192
193 /*
194 * Event counters
195 *
196 * Must be careful with adding these because they can serve as
197 * side channels.
198 */
199 static struct evcnt entropy_discretionary_evcnt =
200 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "entropy", "discretionary");
201 EVCNT_ATTACH_STATIC(entropy_discretionary_evcnt);
202 static struct evcnt entropy_immediate_evcnt =
203 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "entropy", "immediate");
204 EVCNT_ATTACH_STATIC(entropy_immediate_evcnt);
205 static struct evcnt entropy_partial_evcnt =
206 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "entropy", "partial");
207 EVCNT_ATTACH_STATIC(entropy_partial_evcnt);
208 static struct evcnt entropy_consolidate_evcnt =
209 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "entropy", "consolidate");
210 EVCNT_ATTACH_STATIC(entropy_consolidate_evcnt);
211 static struct evcnt entropy_extract_intr_evcnt =
212 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "entropy", "extract intr");
213 EVCNT_ATTACH_STATIC(entropy_extract_intr_evcnt);
214 static struct evcnt entropy_extract_fail_evcnt =
215 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "entropy", "extract fail");
216 EVCNT_ATTACH_STATIC(entropy_extract_fail_evcnt);
217 static struct evcnt entropy_request_evcnt =
218 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "entropy", "request");
219 EVCNT_ATTACH_STATIC(entropy_request_evcnt);
220 static struct evcnt entropy_deplete_evcnt =
221 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "entropy", "deplete");
222 EVCNT_ATTACH_STATIC(entropy_deplete_evcnt);
223 static struct evcnt entropy_notify_evcnt =
224 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "entropy", "notify");
225 EVCNT_ATTACH_STATIC(entropy_notify_evcnt);
226
227 /* Sysctl knobs */
228 bool entropy_collection = 1;
229 bool entropy_depletion = 0; /* Silly! */
230
231 static const struct sysctlnode *entropy_sysctlroot;
232 static struct sysctllog *entropy_sysctllog;
233
234 /* Forward declarations */
235 static void entropy_init_cpu(void *, void *, struct cpu_info *);
236 static void entropy_fini_cpu(void *, void *, struct cpu_info *);
237 static void entropy_account_cpu(struct entropy_cpu *);
238 static void entropy_enter(const void *, size_t, unsigned);
239 static bool entropy_enter_intr(const void *, size_t, unsigned);
240 static void entropy_softintr(void *);
241 static void entropy_thread(void *);
242 static uint32_t entropy_pending(void);
243 static void entropy_pending_cpu(void *, void *, struct cpu_info *);
244 static void entropy_consolidate(void);
245 static void entropy_gather_xc(void *, void *);
246 static void entropy_notify(void);
247 static int sysctl_entropy_consolidate(SYSCTLFN_ARGS);
248 static void filt_entropy_read_detach(struct knote *);
249 static int filt_entropy_read_event(struct knote *, long);
250 static void entropy_request(size_t);
251 static void rnd_add_data_1(struct krndsource *, const void *, uint32_t,
252 uint32_t);
253 static unsigned rndsource_entropybits(struct krndsource *);
254 static void rndsource_entropybits_cpu(void *, void *, struct cpu_info *);
255 static void rndsource_to_user(struct krndsource *, rndsource_t *);
256 static void rndsource_to_user_est(struct krndsource *, rndsource_est_t *);
257
258 /*
259 * curcpu_available()
260 *
261 * True if we can inspect the current CPU. Early on this may not
262 * work. XXX On most if not all ports, this should work earlier.
263 */
264 static inline bool
265 curcpu_available(void)
266 {
267
268 return __predict_true(!cold);
269 }
270
271 /*
272 * entropy_timer()
273 *
274 * Cycle counter, time counter, or anything that changes a wee bit
275 * unpredictably.
276 */
277 static inline uint32_t
278 entropy_timer(void)
279 {
280 struct bintime bt;
281 uint32_t v;
282
283 /* Very early on, cpu_counter32() may not be available. */
284 if (!curcpu_available())
285 return 0;
286
287 /* If we have a CPU cycle counter, use the low 32 bits. */
288 #ifdef __HAVE_CPU_COUNTER
289 if (__predict_true(cpu_hascounter()))
290 return cpu_counter32();
291 #endif /* __HAVE_CPU_COUNTER */
292
293 /* If we're cold, tough. Can't binuptime while cold. */
294 if (__predict_false(cold))
295 return 0;
296
297 /* Fold the 128 bits of binuptime into 32 bits. */
298 binuptime(&bt);
299 v = bt.frac;
300 v ^= bt.frac >> 32;
301 v ^= bt.sec;
302 v ^= bt.sec >> 32;
303 return v;
304 }
305
306 static void
307 attach_seed_rndsource(void)
308 {
309
310 /*
311 * First called no later than entropy_init, while we are still
312 * single-threaded, so no need for RUN_ONCE.
313 */
314 if (E->stage >= ENTROPY_WARM || E->seed_rndsource)
315 return;
316 rnd_attach_source(&seed_rndsource, "seed", RND_TYPE_UNKNOWN,
317 RND_FLAG_COLLECT_VALUE);
318 E->seed_rndsource = true;
319 }
320
321 /*
322 * entropy_init()
323 *
324 * Initialize the entropy subsystem. Panic on failure.
325 *
326 * Requires percpu(9) and sysctl(9) to be initialized.
327 */
328 static void
329 entropy_init(void)
330 {
331 uint32_t extra[2];
332 struct krndsource *rs;
333 unsigned i = 0;
334
335 KASSERT(E->stage == ENTROPY_COLD);
336
337 /* Grab some cycle counts early at boot. */
338 extra[i++] = entropy_timer();
339
340 /* Run the entropy pool cryptography self-test. */
341 if (entpool_selftest() == -1)
342 panic("entropy pool crypto self-test failed");
343
344 /* Create the sysctl directory. */
345 sysctl_createv(&entropy_sysctllog, 0, NULL, &entropy_sysctlroot,
346 CTLFLAG_PERMANENT, CTLTYPE_NODE, "entropy",
347 SYSCTL_DESCR("Entropy (random number sources) options"),
348 NULL, 0, NULL, 0,
349 CTL_KERN, CTL_CREATE, CTL_EOL);
350
351 /* Create the sysctl knobs. */
352 /* XXX These shouldn't be writable at securelevel>0. */
353 sysctl_createv(&entropy_sysctllog, 0, &entropy_sysctlroot, NULL,
354 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, CTLTYPE_BOOL, "collection",
355 SYSCTL_DESCR("Automatically collect entropy from hardware"),
356 NULL, 0, &entropy_collection, 0, CTL_CREATE, CTL_EOL);
357 sysctl_createv(&entropy_sysctllog, 0, &entropy_sysctlroot, NULL,
358 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, CTLTYPE_BOOL, "depletion",
359 SYSCTL_DESCR("`Deplete' entropy pool when observed"),
360 NULL, 0, &entropy_depletion, 0, CTL_CREATE, CTL_EOL);
361 sysctl_createv(&entropy_sysctllog, 0, &entropy_sysctlroot, NULL,
362 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, CTLTYPE_INT, "consolidate",
363 SYSCTL_DESCR("Trigger entropy consolidation now"),
364 sysctl_entropy_consolidate, 0, NULL, 0, CTL_CREATE, CTL_EOL);
365 /* XXX These should maybe not be readable at securelevel>0. */
366 sysctl_createv(&entropy_sysctllog, 0, &entropy_sysctlroot, NULL,
367 CTLFLAG_PERMANENT|CTLFLAG_READONLY|CTLFLAG_PRIVATE, CTLTYPE_INT,
368 "needed", SYSCTL_DESCR("Systemwide entropy deficit"),
369 NULL, 0, &E->needed, 0, CTL_CREATE, CTL_EOL);
370 sysctl_createv(&entropy_sysctllog, 0, &entropy_sysctlroot, NULL,
371 CTLFLAG_PERMANENT|CTLFLAG_READONLY|CTLFLAG_PRIVATE, CTLTYPE_INT,
372 "pending", SYSCTL_DESCR("Entropy pending on CPUs"),
373 NULL, 0, &E->pending, 0, CTL_CREATE, CTL_EOL);
374 sysctl_createv(&entropy_sysctllog, 0, &entropy_sysctlroot, NULL,
375 CTLFLAG_PERMANENT|CTLFLAG_READONLY|CTLFLAG_PRIVATE, CTLTYPE_INT,
376 "epoch", SYSCTL_DESCR("Entropy epoch"),
377 NULL, 0, &E->epoch, 0, CTL_CREATE, CTL_EOL);
378
379 /* Initialize the global state for multithreaded operation. */
380 mutex_init(&E->lock, MUTEX_DEFAULT, IPL_VM);
381 cv_init(&E->cv, "entropy");
382 selinit(&E->selq);
383
384 /* Make sure the seed source is attached. */
385 attach_seed_rndsource();
386
387 /* Note if the bootloader didn't provide a seed. */
388 if (!E->seeded)
389 printf("entropy: no seed from bootloader\n");
390
391 /* Allocate the per-CPU records for all early entropy sources. */
392 LIST_FOREACH(rs, &E->sources, list)
393 rs->state = percpu_alloc(sizeof(struct rndsource_cpu));
394
395 /* Enter the boot cycle count to get started. */
396 extra[i++] = entropy_timer();
397 KASSERT(i == __arraycount(extra));
398 entropy_enter(extra, sizeof extra, 0);
399 explicit_memset(extra, 0, sizeof extra);
400
401 /* We are now ready for multi-threaded operation. */
402 E->stage = ENTROPY_WARM;
403 }
404
405 /*
406 * entropy_init_late()
407 *
408 * Late initialization. Panic on failure.
409 *
410 * Requires CPUs to have been detected and LWPs to have started.
411 */
412 static void
413 entropy_init_late(void)
414 {
415 int error;
416
417 KASSERT(E->stage == ENTROPY_WARM);
418
419 /* Allocate and initialize the per-CPU state. */
420 entropy_percpu = percpu_create(sizeof(struct entropy_cpu),
421 entropy_init_cpu, entropy_fini_cpu, NULL);
422
423 /*
424 * Establish the softint at the highest softint priority level.
425 * Must happen after CPU detection.
426 */
427 entropy_sih = softint_establish(SOFTINT_SERIAL|SOFTINT_MPSAFE,
428 &entropy_softintr, NULL);
429 if (entropy_sih == NULL)
430 panic("unable to establish entropy softint");
431
432 /*
433 * Create the entropy housekeeping thread. Must happen after
434 * lwpinit.
435 */
436 error = kthread_create(PRI_NONE, KTHREAD_MPSAFE|KTHREAD_TS, NULL,
437 entropy_thread, NULL, &entropy_lwp, "entbutler");
438 if (error)
439 panic("unable to create entropy housekeeping thread: %d",
440 error);
441
442 /*
443 * Wait until the per-CPU initialization has hit all CPUs
444 * before proceeding to mark the entropy system hot.
445 */
446 xc_barrier(XC_HIGHPRI);
447 E->stage = ENTROPY_HOT;
448 }
449
450 /*
451 * entropy_init_cpu(ptr, cookie, ci)
452 *
453 * percpu(9) constructor for per-CPU entropy pool.
454 */
455 static void
456 entropy_init_cpu(void *ptr, void *cookie, struct cpu_info *ci)
457 {
458 struct entropy_cpu *ec = ptr;
459
460 ec->ec_softint_evcnt = kmem_alloc(sizeof(*ec->ec_softint_evcnt),
461 KM_SLEEP);
462 ec->ec_pool = kmem_zalloc(sizeof(*ec->ec_pool), KM_SLEEP);
463 ec->ec_pending = 0;
464 ec->ec_locked = false;
465
466 evcnt_attach_dynamic(ec->ec_softint_evcnt, EVCNT_TYPE_MISC, NULL,
467 ci->ci_cpuname, "entropy softint");
468 }
469
470 /*
471 * entropy_fini_cpu(ptr, cookie, ci)
472 *
473 * percpu(9) destructor for per-CPU entropy pool.
474 */
475 static void
476 entropy_fini_cpu(void *ptr, void *cookie, struct cpu_info *ci)
477 {
478 struct entropy_cpu *ec = ptr;
479
480 /*
481 * Zero any lingering data. Disclosure of the per-CPU pool
482 * shouldn't retroactively affect the security of any keys
483 * generated, because entpool(9) erases whatever we have just
484 * drawn out of any pool, but better safe than sorry.
485 */
486 explicit_memset(ec->ec_pool, 0, sizeof(*ec->ec_pool));
487
488 evcnt_detach(ec->ec_softint_evcnt);
489
490 kmem_free(ec->ec_pool, sizeof(*ec->ec_pool));
491 kmem_free(ec->ec_softint_evcnt, sizeof(*ec->ec_softint_evcnt));
492 }
493
494 /*
495 * entropy_seed(seed)
496 *
497 * Seed the entropy pool with seed. Meant to be called as early
498 * as possible by the bootloader; may be called before or after
499 * entropy_init. Must be called before system reaches userland.
500 * Must be called in thread or soft interrupt context, not in hard
501 * interrupt context. Must be called at most once.
502 *
503 * Overwrites the seed in place. Caller may then free the memory.
504 */
505 static void
506 entropy_seed(rndsave_t *seed)
507 {
508 SHA1_CTX ctx;
509 uint8_t digest[SHA1_DIGEST_LENGTH];
510 bool seeded;
511
512 /*
513 * Verify the checksum. If the checksum fails, take the data
514 * but ignore the entropy estimate -- the file may have been
515 * incompletely written with garbage, which is harmless to add
516 * but may not be as unpredictable as alleged.
517 */
518 SHA1Init(&ctx);
519 SHA1Update(&ctx, (const void *)&seed->entropy, sizeof(seed->entropy));
520 SHA1Update(&ctx, seed->data, sizeof(seed->data));
521 SHA1Final(digest, &ctx);
522 CTASSERT(sizeof(seed->digest) == sizeof(digest));
523 if (!consttime_memequal(digest, seed->digest, sizeof(digest))) {
524 printf("entropy: invalid seed checksum\n");
525 seed->entropy = 0;
526 }
527 explicit_memset(&ctx, 0, sizeof ctx);
528 explicit_memset(digest, 0, sizeof digest);
529
530 /*
531 * If the entropy is insensibly large, try byte-swapping.
532 * Otherwise assume the file is corrupted and act as though it
533 * has zero entropy.
534 */
535 if (howmany(seed->entropy, NBBY) > sizeof(seed->data)) {
536 seed->entropy = bswap32(seed->entropy);
537 if (howmany(seed->entropy, NBBY) > sizeof(seed->data))
538 seed->entropy = 0;
539 }
540
541 /* Make sure the seed source is attached. */
542 attach_seed_rndsource();
543
544 /* Test and set E->seeded. */
545 if (E->stage >= ENTROPY_WARM)
546 mutex_enter(&E->lock);
547 seeded = E->seeded;
548 E->seeded = true;
549 if (E->stage >= ENTROPY_WARM)
550 mutex_exit(&E->lock);
551
552 /*
553 * If we've been seeded, may be re-entering the same seed
554 * (e.g., bootloader vs module init, or something). No harm in
555 * entering it twice, but it contributes no additional entropy.
556 */
557 if (seeded) {
558 printf("entropy: double-seeded by bootloader\n");
559 seed->entropy = 0;
560 } else {
561 printf("entropy: entering seed from bootloader\n");
562 }
563
564 /* Enter it into the pool and promptly zero it. */
565 rnd_add_data(&seed_rndsource, seed->data, sizeof(seed->data),
566 seed->entropy);
567 explicit_memset(seed, 0, sizeof(*seed));
568 }
569
570 /*
571 * entropy_bootrequest()
572 *
573 * Request entropy from all sources at boot, once config is
574 * complete and interrupts are running.
575 */
576 void
577 entropy_bootrequest(void)
578 {
579
580 KASSERT(E->stage >= ENTROPY_WARM);
581
582 /*
583 * Request enough to satisfy the maximum entropy shortage.
584 * This is harmless overkill if the bootloader provided a seed.
585 */
586 mutex_enter(&E->lock);
587 entropy_request(ENTROPY_CAPACITY);
588 mutex_exit(&E->lock);
589 }
590
591 /*
592 * entropy_epoch()
593 *
594 * Returns the current entropy epoch. If this changes, you should
595 * reseed. If -1, means the system has not yet reached full
596 * entropy; never reverts back to -1 after full entropy has been
597 * reached. Never zero, so you can always use zero as an
598 * uninitialized sentinel value meaning `reseed ASAP'.
599 *
600 * Usage model:
601 *
602 * struct foo {
603 * struct crypto_prng prng;
604 * unsigned epoch;
605 * } *foo;
606 *
607 * unsigned epoch = entropy_epoch();
608 * if (__predict_false(epoch != foo->epoch)) {
609 * uint8_t seed[32];
610 * if (entropy_extract(seed, sizeof seed, 0) != 0)
611 * warn("no entropy");
612 * crypto_prng_reseed(&foo->prng, seed, sizeof seed);
613 * foo->epoch = epoch;
614 * }
615 */
616 unsigned
617 entropy_epoch(void)
618 {
619
620 /*
621 * Unsigned int, so no need for seqlock for an atomic read, but
622 * make sure we read it afresh each time.
623 */
624 return atomic_load_relaxed(&E->epoch);
625 }
626
627 /*
628 * entropy_account_cpu(ec)
629 *
630 * Consider whether to consolidate entropy into the global pool
631 * after we just added some into the current CPU's pending pool.
632 *
633 * - If this CPU can provide enough entropy now, do so.
634 *
635 * - If this and whatever else is available on other CPUs can
636 * provide enough entropy, kick the consolidation thread.
637 *
638 * - Otherwise, do as little as possible, except maybe consolidate
639 * entropy at most once a minute.
640 *
641 * Caller must be bound to a CPU and therefore have exclusive
642 * access to ec. Will acquire and release the global lock.
643 */
644 static void
645 entropy_account_cpu(struct entropy_cpu *ec)
646 {
647 unsigned diff;
648
649 KASSERT(E->stage == ENTROPY_HOT);
650
651 /*
652 * If there's no entropy needed, and entropy has been
653 * consolidated in the last minute, do nothing.
654 */
655 if (__predict_true(atomic_load_relaxed(&E->needed) == 0) &&
656 __predict_true(!atomic_load_relaxed(&entropy_depletion)) &&
657 __predict_true((time_uptime - E->timestamp) <= 60))
658 return;
659
660 /* If there's nothing pending, stop here. */
661 if (ec->ec_pending == 0)
662 return;
663
664 /* Consider consolidation, under the lock. */
665 mutex_enter(&E->lock);
666 if (E->needed != 0 && E->needed <= ec->ec_pending) {
667 /*
668 * If we have not yet attained full entropy but we can
669 * now, do so. This way we disseminate entropy
670 * promptly when it becomes available early at boot;
671 * otherwise we leave it to the entropy consolidation
672 * thread, which is rate-limited to mitigate side
673 * channels and abuse.
674 */
675 uint8_t buf[ENTPOOL_CAPACITY];
676
677 /* Transfer from the local pool to the global pool. */
678 entpool_extract(ec->ec_pool, buf, sizeof buf);
679 entpool_enter(&E->pool, buf, sizeof buf);
680 atomic_store_relaxed(&ec->ec_pending, 0);
681 atomic_store_relaxed(&E->needed, 0);
682
683 /* Notify waiters that we now have full entropy. */
684 entropy_notify();
685 entropy_immediate_evcnt.ev_count++;
686 } else if (ec->ec_pending) {
687 /* Record how much we can add to the global pool. */
688 diff = MIN(ec->ec_pending, ENTROPY_CAPACITY*NBBY - E->pending);
689 E->pending += diff;
690 atomic_store_relaxed(&ec->ec_pending, ec->ec_pending - diff);
691
692 /*
693 * This should have made a difference unless we were
694 * already saturated.
695 */
696 KASSERT(diff || E->pending == ENTROPY_CAPACITY*NBBY);
697 KASSERT(E->pending);
698
699 if (E->needed <= E->pending) {
700 /*
701 * Enough entropy between all the per-CPU
702 * pools. Wake up the housekeeping thread.
703 *
704 * If we don't need any entropy, this doesn't
705 * mean much, but it is the only time we ever
706 * gather additional entropy in case the
707 * accounting has been overly optimistic. This
708 * happens at most once a minute, so there's
709 * negligible performance cost.
710 */
711 E->consolidate = true;
712 cv_broadcast(&E->cv);
713 if (E->needed == 0)
714 entropy_discretionary_evcnt.ev_count++;
715 } else {
716 /* Can't get full entropy. Keep gathering. */
717 entropy_partial_evcnt.ev_count++;
718 }
719 }
720 mutex_exit(&E->lock);
721 }
722
723 /*
724 * entropy_enter_early(buf, len, nbits)
725 *
726 * Do entropy bookkeeping globally, before we have established
727 * per-CPU pools. Enter directly into the global pool in the hope
728 * that we enter enough before the first entropy_extract to thwart
729 * iterative-guessing attacks; entropy_extract will warn if not.
730 */
731 static void
732 entropy_enter_early(const void *buf, size_t len, unsigned nbits)
733 {
734 bool notify = false;
735
736 if (E->stage >= ENTROPY_WARM)
737 mutex_enter(&E->lock);
738
739 /* Enter it into the pool. */
740 entpool_enter(&E->pool, buf, len);
741
742 /*
743 * Decide whether to notify reseed -- we will do so if either:
744 * (a) we transition from partial entropy to full entropy, or
745 * (b) we get a batch of full entropy all at once.
746 */
747 notify |= (E->needed && E->needed <= nbits);
748 notify |= (nbits >= ENTROPY_CAPACITY*NBBY);
749
750 /* Subtract from the needed count and notify if appropriate. */
751 E->needed -= MIN(E->needed, nbits);
752 if (notify) {
753 entropy_notify();
754 entropy_immediate_evcnt.ev_count++;
755 }
756
757 if (E->stage >= ENTROPY_WARM)
758 mutex_exit(&E->lock);
759 }
760
761 /*
762 * entropy_enter(buf, len, nbits)
763 *
764 * Enter len bytes of data from buf into the system's entropy
765 * pool, stirring as necessary when the internal buffer fills up.
766 * nbits is a lower bound on the number of bits of entropy in the
767 * process that led to this sample.
768 */
769 static void
770 entropy_enter(const void *buf, size_t len, unsigned nbits)
771 {
772 struct entropy_cpu *ec;
773 uint32_t pending;
774 int s;
775
776 KASSERTMSG(!curcpu_available() || !cpu_intr_p(),
777 "use entropy_enter_intr from interrupt context");
778 KASSERTMSG(howmany(nbits, NBBY) <= len,
779 "impossible entropy rate: %u bits in %zu-byte string", nbits, len);
780
781 /* If it's too early after boot, just use entropy_enter_early. */
782 if (__predict_false(E->stage < ENTROPY_HOT)) {
783 entropy_enter_early(buf, len, nbits);
784 return;
785 }
786
787 /*
788 * Acquire the per-CPU state, blocking soft interrupts and
789 * causing hard interrupts to drop samples on the floor.
790 */
791 ec = percpu_getref(entropy_percpu);
792 s = splsoftserial();
793 KASSERT(!ec->ec_locked);
794 ec->ec_locked = true;
795 __insn_barrier();
796
797 /* Enter into the per-CPU pool. */
798 entpool_enter(ec->ec_pool, buf, len);
799
800 /* Count up what we can add. */
801 pending = ec->ec_pending;
802 pending += MIN(ENTROPY_CAPACITY*NBBY - pending, nbits);
803 atomic_store_relaxed(&ec->ec_pending, pending);
804
805 /* Consolidate globally if appropriate based on what we added. */
806 entropy_account_cpu(ec);
807
808 /* Release the per-CPU state. */
809 KASSERT(ec->ec_locked);
810 __insn_barrier();
811 ec->ec_locked = false;
812 splx(s);
813 percpu_putref(entropy_percpu);
814 }
815
816 /*
817 * entropy_enter_intr(buf, len, nbits)
818 *
819 * Enter up to len bytes of data from buf into the system's
820 * entropy pool without stirring. nbits is a lower bound on the
821 * number of bits of entropy in the process that led to this
822 * sample. If the sample could be entered completely, assume
823 * nbits of entropy pending; otherwise assume none, since we don't
824 * know whether some parts of the sample are constant, for
825 * instance. Schedule a softint to stir the entropy pool if
826 * needed. Return true if used fully, false if truncated at all.
827 *
828 * Using this in thread context will work, but you might as well
829 * use entropy_enter in that case.
830 */
831 static bool
832 entropy_enter_intr(const void *buf, size_t len, unsigned nbits)
833 {
834 struct entropy_cpu *ec;
835 bool fullyused = false;
836 uint32_t pending;
837
838 KASSERTMSG(howmany(nbits, NBBY) <= len,
839 "impossible entropy rate: %u bits in %zu-byte string", nbits, len);
840
841 /* If it's too early after boot, just use entropy_enter_early. */
842 if (__predict_false(E->stage < ENTROPY_HOT)) {
843 entropy_enter_early(buf, len, nbits);
844 return true;
845 }
846
847 /*
848 * Acquire the per-CPU state. If someone is in the middle of
849 * using it, drop the sample. Otherwise, take the lock so that
850 * higher-priority interrupts will drop their samples.
851 */
852 ec = percpu_getref(entropy_percpu);
853 if (ec->ec_locked)
854 goto out0;
855 ec->ec_locked = true;
856 __insn_barrier();
857
858 /*
859 * Enter as much as we can into the per-CPU pool. If it was
860 * truncated, schedule a softint to stir the pool and stop.
861 */
862 if (!entpool_enter_nostir(ec->ec_pool, buf, len)) {
863 softint_schedule(entropy_sih);
864 goto out1;
865 }
866 fullyused = true;
867
868 /* Count up what we can contribute. */
869 pending = ec->ec_pending;
870 pending += MIN(ENTROPY_CAPACITY*NBBY - pending, nbits);
871 atomic_store_relaxed(&ec->ec_pending, pending);
872
873 /* Schedule a softint if we added anything and it matters. */
874 if (__predict_false((atomic_load_relaxed(&E->needed) != 0) ||
875 atomic_load_relaxed(&entropy_depletion)) &&
876 nbits != 0)
877 softint_schedule(entropy_sih);
878
879 out1: /* Release the per-CPU state. */
880 KASSERT(ec->ec_locked);
881 __insn_barrier();
882 ec->ec_locked = false;
883 out0: percpu_putref(entropy_percpu);
884
885 return fullyused;
886 }
887
888 /*
889 * entropy_softintr(cookie)
890 *
891 * Soft interrupt handler for entering entropy. Takes care of
892 * stirring the local CPU's entropy pool if it filled up during
893 * hard interrupts, and promptly crediting entropy from the local
894 * CPU's entropy pool to the global entropy pool if needed.
895 */
896 static void
897 entropy_softintr(void *cookie)
898 {
899 struct entropy_cpu *ec;
900
901 /*
902 * Acquire the per-CPU state. Other users can lock this only
903 * while soft interrupts are blocked. Cause hard interrupts to
904 * drop samples on the floor.
905 */
906 ec = percpu_getref(entropy_percpu);
907 KASSERT(!ec->ec_locked);
908 ec->ec_locked = true;
909 __insn_barrier();
910
911 /* Count statistics. */
912 ec->ec_softint_evcnt->ev_count++;
913
914 /* Stir the pool if necessary. */
915 entpool_stir(ec->ec_pool);
916
917 /* Consolidate globally if appropriate based on what we added. */
918 entropy_account_cpu(ec);
919
920 /* Release the per-CPU state. */
921 KASSERT(ec->ec_locked);
922 __insn_barrier();
923 ec->ec_locked = false;
924 percpu_putref(entropy_percpu);
925 }
926
927 /*
928 * entropy_thread(cookie)
929 *
930 * Handle any asynchronous entropy housekeeping.
931 */
932 static void
933 entropy_thread(void *cookie)
934 {
935 bool consolidate;
936
937 for (;;) {
938 /*
939 * Wait until there's full entropy somewhere among the
940 * CPUs, as confirmed at most once per minute, or
941 * someone wants to consolidate.
942 */
943 if (entropy_pending() >= ENTROPY_CAPACITY*NBBY) {
944 consolidate = true;
945 } else {
946 mutex_enter(&E->lock);
947 if (!E->consolidate)
948 cv_timedwait(&E->cv, &E->lock, 60*hz);
949 consolidate = E->consolidate;
950 E->consolidate = false;
951 mutex_exit(&E->lock);
952 }
953
954 if (consolidate) {
955 /* Do it. */
956 entropy_consolidate();
957
958 /* Mitigate abuse. */
959 kpause("entropy", false, hz, NULL);
960 }
961 }
962 }
963
964 /*
965 * entropy_pending()
966 *
967 * Count up the amount of entropy pending on other CPUs.
968 */
969 static uint32_t
970 entropy_pending(void)
971 {
972 uint32_t pending = 0;
973
974 percpu_foreach(entropy_percpu, &entropy_pending_cpu, &pending);
975 return pending;
976 }
977
978 static void
979 entropy_pending_cpu(void *ptr, void *cookie, struct cpu_info *ci)
980 {
981 struct entropy_cpu *ec = ptr;
982 uint32_t *pendingp = cookie;
983 uint32_t cpu_pending;
984
985 cpu_pending = atomic_load_relaxed(&ec->ec_pending);
986 *pendingp += MIN(ENTROPY_CAPACITY*NBBY - *pendingp, cpu_pending);
987 }
988
989 /*
990 * entropy_consolidate()
991 *
992 * Issue a cross-call to gather entropy on all CPUs and advance
993 * the entropy epoch.
994 */
995 static void
996 entropy_consolidate(void)
997 {
998 static const struct timeval interval = {.tv_sec = 60, .tv_usec = 0};
999 static struct timeval lasttime; /* serialized by E->lock */
1000 unsigned diff;
1001 uint64_t ticket;
1002
1003 /* Gather entropy on all CPUs. */
1004 ticket = xc_broadcast(0, &entropy_gather_xc, NULL, NULL);
1005 xc_wait(ticket);
1006
1007 /* Acquire the lock to notify waiters. */
1008 mutex_enter(&E->lock);
1009
1010 /* Count another consolidation. */
1011 entropy_consolidate_evcnt.ev_count++;
1012
1013 /* Note when we last consolidated, i.e. now. */
1014 E->timestamp = time_uptime;
1015
1016 /* Count the entropy that was gathered. */
1017 diff = MIN(E->needed, E->pending);
1018 atomic_store_relaxed(&E->needed, E->needed - diff);
1019 E->pending -= diff;
1020 if (__predict_false(E->needed > 0)) {
1021 if (ratecheck(&lasttime, &interval))
1022 printf("entropy: WARNING:"
1023 " consolidating less than full entropy\n");
1024 }
1025
1026 /* Advance the epoch and notify waiters. */
1027 entropy_notify();
1028
1029 /* Release the lock. */
1030 mutex_exit(&E->lock);
1031 }
1032
1033 /*
1034 * entropy_gather_xc(arg1, arg2)
1035 *
1036 * Extract output from the local CPU's input pool and enter it
1037 * into the global pool.
1038 */
1039 static void
1040 entropy_gather_xc(void *arg1 __unused, void *arg2 __unused)
1041 {
1042 struct entropy_cpu *ec;
1043 uint8_t buf[ENTPOOL_CAPACITY];
1044 uint32_t extra[7];
1045 unsigned i = 0;
1046 int s;
1047
1048 /* Grab CPU number and cycle counter to mix extra into the pool. */
1049 extra[i++] = cpu_number();
1050 extra[i++] = entropy_timer();
1051
1052 /*
1053 * Acquire the per-CPU state, blocking soft interrupts and
1054 * discarding entropy in hard interrupts, so that we can
1055 * extract from the per-CPU pool.
1056 */
1057 ec = percpu_getref(entropy_percpu);
1058 s = splsoftserial();
1059 KASSERT(!ec->ec_locked);
1060 ec->ec_locked = true;
1061 __insn_barrier();
1062 extra[i++] = entropy_timer();
1063
1064 /* Extract the data. */
1065 entpool_extract(ec->ec_pool, buf, sizeof buf);
1066 extra[i++] = entropy_timer();
1067
1068 /* Release the per-CPU state. */
1069 KASSERT(ec->ec_locked);
1070 __insn_barrier();
1071 ec->ec_locked = false;
1072 splx(s);
1073 percpu_putref(entropy_percpu);
1074 extra[i++] = entropy_timer();
1075
1076 /*
1077 * Copy over statistics, and enter the per-CPU extract and the
1078 * extra timing into the global pool, under the global lock.
1079 */
1080 mutex_enter(&E->lock);
1081 extra[i++] = entropy_timer();
1082 entpool_enter(&E->pool, buf, sizeof buf);
1083 explicit_memset(buf, 0, sizeof buf);
1084 extra[i++] = entropy_timer();
1085 KASSERT(i == __arraycount(extra));
1086 entpool_enter(&E->pool, extra, sizeof extra);
1087 explicit_memset(extra, 0, sizeof extra);
1088 mutex_exit(&E->lock);
1089 }
1090
1091 /*
1092 * entropy_notify()
1093 *
1094 * Caller just contributed entropy to the global pool. Advance
1095 * the entropy epoch and notify waiters.
1096 *
1097 * Caller must hold the global entropy lock. Except for the
1098 * `sysctl -w kern.entropy.consolidate=1` trigger, the caller must
1099 * have just have transitioned from partial entropy to full
1100 * entropy -- E->needed should be zero now.
1101 */
1102 static void
1103 entropy_notify(void)
1104 {
1105 unsigned epoch;
1106
1107 KASSERT(E->stage == ENTROPY_COLD || mutex_owned(&E->lock));
1108
1109 /*
1110 * If this is the first time, print a message to the console
1111 * that we're ready so operators can compare it to the timing
1112 * of other events.
1113 */
1114 if (E->epoch == (unsigned)-1)
1115 printf("entropy: ready\n");
1116
1117 /* Set the epoch; roll over from UINTMAX-1 to 1. */
1118 rnd_initial_entropy = 1; /* XXX legacy */
1119 epoch = E->epoch + 1;
1120 if (epoch == 0 || epoch == (unsigned)-1)
1121 epoch = 1;
1122 atomic_store_relaxed(&E->epoch, epoch);
1123
1124 /* Notify waiters. */
1125 if (E->stage >= ENTROPY_WARM) {
1126 cv_broadcast(&E->cv);
1127 selnotify(&E->selq, POLLIN|POLLRDNORM, NOTE_SUBMIT);
1128 }
1129
1130 /* Count another notification. */
1131 entropy_notify_evcnt.ev_count++;
1132 }
1133
1134 /*
1135 * sysctl -w kern.entropy.consolidate=1
1136 *
1137 * Trigger entropy consolidation and wait for it to complete.
1138 * Writable only by superuser. This is the only way for the
1139 * system to consolidate entropy if the operator knows something
1140 * the kernel doesn't about how unpredictable the pending entropy
1141 * pools are.
1142 */
1143 static int
1144 sysctl_entropy_consolidate(SYSCTLFN_ARGS)
1145 {
1146 struct sysctlnode node = *rnode;
1147 uint64_t ticket;
1148 int arg;
1149 int error;
1150
1151 KASSERT(E->stage == ENTROPY_HOT);
1152
1153 node.sysctl_data = &arg;
1154 error = sysctl_lookup(SYSCTLFN_CALL(&node));
1155 if (error || newp == NULL)
1156 return error;
1157 if (arg) {
1158 mutex_enter(&E->lock);
1159 ticket = entropy_consolidate_evcnt.ev_count;
1160 E->consolidate = true;
1161 cv_broadcast(&E->cv);
1162 while (ticket == entropy_consolidate_evcnt.ev_count) {
1163 error = cv_wait_sig(&E->cv, &E->lock);
1164 if (error)
1165 break;
1166 }
1167 mutex_exit(&E->lock);
1168 }
1169
1170 return error;
1171 }
1172
1173 /*
1174 * entropy_extract(buf, len, flags)
1175 *
1176 * Extract len bytes from the global entropy pool into buf.
1177 *
1178 * Flags may have:
1179 *
1180 * ENTROPY_WAIT Wait for entropy if not available yet.
1181 * ENTROPY_SIG Allow interruption by a signal during wait.
1182 *
1183 * Return zero on success, or error on failure:
1184 *
1185 * EWOULDBLOCK No entropy and ENTROPY_WAIT not set.
1186 * EINTR/ERESTART No entropy, ENTROPY_SIG set, and interrupted.
1187 *
1188 * If ENTROPY_WAIT is set, allowed only in thread context. If
1189 * ENTROPY_WAIT is not set, allowed up to IPL_VM. (XXX That's
1190 * awfully high... Do we really need it in hard interrupts? This
1191 * arises from use of cprng_strong(9).)
1192 */
1193 int
1194 entropy_extract(void *buf, size_t len, int flags)
1195 {
1196 static const struct timeval interval = {.tv_sec = 60, .tv_usec = 0};
1197 static struct timeval lasttime; /* serialized by E->lock */
1198 int error;
1199
1200 if (ISSET(flags, ENTROPY_WAIT)) {
1201 ASSERT_SLEEPABLE();
1202 KASSERTMSG(E->stage >= ENTROPY_WARM,
1203 "can't wait for entropy until warm");
1204 }
1205
1206 /* Acquire the global lock to get at the global pool. */
1207 if (E->stage >= ENTROPY_WARM)
1208 mutex_enter(&E->lock);
1209
1210 /* Count up request for entropy in interrupt context. */
1211 if (curcpu_available() && cpu_intr_p())
1212 entropy_extract_intr_evcnt.ev_count++;
1213
1214 /* Wait until there is enough entropy in the system. */
1215 error = 0;
1216 while (E->needed) {
1217 /* Ask for more, synchronously if possible. */
1218 entropy_request(len);
1219
1220 /* If we got enough, we're done. */
1221 if (E->needed == 0) {
1222 KASSERT(error == 0);
1223 break;
1224 }
1225
1226 /* If not waiting, stop here. */
1227 if (!ISSET(flags, ENTROPY_WAIT)) {
1228 error = EWOULDBLOCK;
1229 break;
1230 }
1231
1232 /* Wait for some entropy to come in and try again. */
1233 KASSERT(E->stage >= ENTROPY_WARM);
1234 if (ISSET(flags, ENTROPY_SIG)) {
1235 error = cv_wait_sig(&E->cv, &E->lock);
1236 if (error)
1237 break;
1238 } else {
1239 cv_wait(&E->cv, &E->lock);
1240 }
1241 }
1242
1243 /* Count failure -- but fill the buffer nevertheless. */
1244 if (error)
1245 entropy_extract_fail_evcnt.ev_count++;
1246
1247 /*
1248 * Report a warning if we have never yet reached full entropy.
1249 * This is the only case where we consider entropy to be
1250 * `depleted' without kern.entropy.depletion enabled -- when we
1251 * only have partial entropy, an adversary may be able to
1252 * narrow the state of the pool down to a small number of
1253 * possibilities; the output then enables them to confirm a
1254 * guess, reducing its entropy from the adversary's perspective
1255 * to zero.
1256 */
1257 if (__predict_false(E->epoch == (unsigned)-1)) {
1258 if (ratecheck(&lasttime, &interval))
1259 printf("entropy: WARNING:"
1260 " extracting entropy too early\n");
1261 atomic_store_relaxed(&E->needed, ENTROPY_CAPACITY*NBBY);
1262 }
1263
1264 /* Extract data from the pool, and `deplete' if we're doing that. */
1265 entpool_extract(&E->pool, buf, len);
1266 if (__predict_false(atomic_load_relaxed(&entropy_depletion)) &&
1267 error == 0) {
1268 unsigned cost = MIN(len, ENTROPY_CAPACITY)*NBBY;
1269
1270 atomic_store_relaxed(&E->needed,
1271 E->needed + MIN(ENTROPY_CAPACITY*NBBY - E->needed, cost));
1272 entropy_deplete_evcnt.ev_count++;
1273 }
1274
1275 /* Release the global lock and return the error. */
1276 if (E->stage >= ENTROPY_WARM)
1277 mutex_exit(&E->lock);
1278 return error;
1279 }
1280
1281 /*
1282 * entropy_poll(events)
1283 *
1284 * Return the subset of events ready, and if it is not all of
1285 * events, record curlwp as waiting for entropy.
1286 */
1287 int
1288 entropy_poll(int events)
1289 {
1290 int revents = 0;
1291
1292 KASSERT(E->stage >= ENTROPY_WARM);
1293
1294 /* Always ready for writing. */
1295 revents |= events & (POLLOUT|POLLWRNORM);
1296
1297 /* Narrow it down to reads. */
1298 events &= POLLIN|POLLRDNORM;
1299 if (events == 0)
1300 return revents;
1301
1302 /*
1303 * If we have reached full entropy and we're not depleting
1304 * entropy, we are forever ready.
1305 */
1306 if (__predict_true(atomic_load_relaxed(&E->needed) == 0) &&
1307 __predict_true(!atomic_load_relaxed(&entropy_depletion)))
1308 return revents | events;
1309
1310 /*
1311 * Otherwise, check whether we need entropy under the lock. If
1312 * we don't, we're ready; if we do, add ourselves to the queue.
1313 */
1314 mutex_enter(&E->lock);
1315 if (E->needed == 0)
1316 revents |= events;
1317 else
1318 selrecord(curlwp, &E->selq);
1319 mutex_exit(&E->lock);
1320
1321 return revents;
1322 }
1323
1324 /*
1325 * filt_entropy_read_detach(kn)
1326 *
1327 * struct filterops::f_detach callback for entropy read events:
1328 * remove kn from the list of waiters.
1329 */
1330 static void
1331 filt_entropy_read_detach(struct knote *kn)
1332 {
1333
1334 KASSERT(E->stage >= ENTROPY_WARM);
1335
1336 mutex_enter(&E->lock);
1337 SLIST_REMOVE(&E->selq.sel_klist, kn, knote, kn_selnext);
1338 mutex_exit(&E->lock);
1339 }
1340
1341 /*
1342 * filt_entropy_read_event(kn, hint)
1343 *
1344 * struct filterops::f_event callback for entropy read events:
1345 * poll for entropy. Caller must hold the global entropy lock if
1346 * hint is NOTE_SUBMIT, and must not if hint is not NOTE_SUBMIT.
1347 */
1348 static int
1349 filt_entropy_read_event(struct knote *kn, long hint)
1350 {
1351 int ret;
1352
1353 KASSERT(E->stage >= ENTROPY_WARM);
1354
1355 /* Acquire the lock, if caller is outside entropy subsystem. */
1356 if (hint == NOTE_SUBMIT)
1357 KASSERT(mutex_owned(&E->lock));
1358 else
1359 mutex_enter(&E->lock);
1360
1361 /*
1362 * If we still need entropy, can't read anything; if not, can
1363 * read arbitrarily much.
1364 */
1365 if (E->needed != 0) {
1366 ret = 0;
1367 } else {
1368 if (atomic_load_relaxed(&entropy_depletion))
1369 kn->kn_data = ENTROPY_CAPACITY*NBBY;
1370 else
1371 kn->kn_data = MIN(INT64_MAX, SSIZE_MAX);
1372 ret = 1;
1373 }
1374
1375 /* Release the lock, if caller is outside entropy subsystem. */
1376 if (hint == NOTE_SUBMIT)
1377 KASSERT(mutex_owned(&E->lock));
1378 else
1379 mutex_exit(&E->lock);
1380
1381 return ret;
1382 }
1383
1384 static const struct filterops entropy_read_filtops = {
1385 .f_isfd = 1, /* XXX Makes sense only for /dev/u?random. */
1386 .f_attach = NULL,
1387 .f_detach = filt_entropy_read_detach,
1388 .f_event = filt_entropy_read_event,
1389 };
1390
1391 /*
1392 * entropy_kqfilter(kn)
1393 *
1394 * Register kn to receive entropy event notifications. May be
1395 * EVFILT_READ or EVFILT_WRITE; anything else yields EINVAL.
1396 */
1397 int
1398 entropy_kqfilter(struct knote *kn)
1399 {
1400
1401 KASSERT(E->stage >= ENTROPY_WARM);
1402
1403 switch (kn->kn_filter) {
1404 case EVFILT_READ:
1405 /* Enter into the global select queue. */
1406 mutex_enter(&E->lock);
1407 kn->kn_fop = &entropy_read_filtops;
1408 SLIST_INSERT_HEAD(&E->selq.sel_klist, kn, kn_selnext);
1409 mutex_exit(&E->lock);
1410 return 0;
1411 case EVFILT_WRITE:
1412 /* Can always dump entropy into the system. */
1413 kn->kn_fop = &seltrue_filtops;
1414 return 0;
1415 default:
1416 return EINVAL;
1417 }
1418 }
1419
1420 /*
1421 * rndsource_setcb(rs, get, getarg)
1422 *
1423 * Set the request callback for the entropy source rs, if it can
1424 * provide entropy on demand. Must precede rnd_attach_source.
1425 */
1426 void
1427 rndsource_setcb(struct krndsource *rs, void (*get)(size_t, void *),
1428 void *getarg)
1429 {
1430
1431 rs->get = get;
1432 rs->getarg = getarg;
1433 }
1434
1435 /*
1436 * rnd_attach_source(rs, name, type, flags)
1437 *
1438 * Attach the entropy source rs. Must be done after
1439 * rndsource_setcb, if any, and before any calls to rnd_add_data.
1440 */
1441 void
1442 rnd_attach_source(struct krndsource *rs, const char *name, uint32_t type,
1443 uint32_t flags)
1444 {
1445 uint32_t extra[4];
1446 unsigned i = 0;
1447
1448 /* Grab cycle counter to mix extra into the pool. */
1449 extra[i++] = entropy_timer();
1450
1451 /*
1452 * Apply some standard flags:
1453 *
1454 * - We do not bother with network devices by default, for
1455 * hysterical raisins (perhaps: because it is often the case
1456 * that an adversary can influence network packet timings).
1457 */
1458 switch (type) {
1459 case RND_TYPE_NET:
1460 flags |= RND_FLAG_NO_COLLECT;
1461 break;
1462 }
1463
1464 /* Sanity-check the callback if RND_FLAG_HASCB is set. */
1465 KASSERT(!ISSET(flags, RND_FLAG_HASCB) || rs->get != NULL);
1466
1467 /* Initialize the random source. */
1468 memset(rs->name, 0, sizeof(rs->name)); /* paranoia */
1469 strlcpy(rs->name, name, sizeof(rs->name));
1470 rs->type = type;
1471 rs->flags = flags;
1472 if (E->stage >= ENTROPY_WARM)
1473 rs->state = percpu_alloc(sizeof(struct rndsource_cpu));
1474 extra[i++] = entropy_timer();
1475
1476 /* Wire it into the global list of random sources. */
1477 if (E->stage >= ENTROPY_WARM)
1478 mutex_enter(&E->lock);
1479 LIST_INSERT_HEAD(&E->sources, rs, list);
1480 if (E->stage >= ENTROPY_WARM)
1481 mutex_exit(&E->lock);
1482 extra[i++] = entropy_timer();
1483
1484 /* Request that it provide entropy ASAP, if we can. */
1485 if (ISSET(flags, RND_FLAG_HASCB))
1486 (*rs->get)(ENTROPY_CAPACITY, rs->getarg);
1487 extra[i++] = entropy_timer();
1488
1489 /* Mix the extra into the pool. */
1490 KASSERT(i == __arraycount(extra));
1491 entropy_enter(extra, sizeof extra, 0);
1492 explicit_memset(extra, 0, sizeof extra);
1493 }
1494
1495 /*
1496 * rnd_detach_source(rs)
1497 *
1498 * Detach the entropy source rs. May sleep waiting for users to
1499 * drain. Further use is not allowed.
1500 */
1501 void
1502 rnd_detach_source(struct krndsource *rs)
1503 {
1504
1505 /*
1506 * If we're cold (shouldn't happen, but hey), just remove it
1507 * from the list -- there's nothing allocated.
1508 */
1509 if (E->stage == ENTROPY_COLD) {
1510 LIST_REMOVE(rs, list);
1511 return;
1512 }
1513
1514 /* We may have to wait for entropy_request. */
1515 ASSERT_SLEEPABLE();
1516
1517 /* Wait until the source list is not in use, and remove it. */
1518 mutex_enter(&E->lock);
1519 while (E->sourcelock)
1520 cv_wait(&E->cv, &E->lock);
1521 LIST_REMOVE(rs, list);
1522 mutex_exit(&E->lock);
1523
1524 /* Free the per-CPU data. */
1525 percpu_free(rs->state, sizeof(struct rndsource_cpu));
1526 }
1527
1528 /*
1529 * rnd_lock_sources()
1530 *
1531 * Prevent changes to the list of rndsources while we iterate it.
1532 * Interruptible. Caller must hold the global entropy lock. If
1533 * successful, no rndsource will go away until rnd_unlock_sources
1534 * even while the caller releases the global entropy lock.
1535 */
1536 static int
1537 rnd_lock_sources(void)
1538 {
1539 int error;
1540
1541 KASSERT(mutex_owned(&E->lock));
1542
1543 while (E->sourcelock) {
1544 error = cv_wait_sig(&E->cv, &E->lock);
1545 if (error)
1546 return error;
1547 }
1548
1549 E->sourcelock = curlwp;
1550 return 0;
1551 }
1552
1553 /*
1554 * rnd_trylock_sources()
1555 *
1556 * Try to lock the list of sources, but if it's already locked,
1557 * fail. Caller must hold the global entropy lock. If
1558 * successful, no rndsource will go away until rnd_unlock_sources
1559 * even while the caller releases the global entropy lock.
1560 */
1561 static bool
1562 rnd_trylock_sources(void)
1563 {
1564
1565 KASSERT(E->stage == ENTROPY_COLD || mutex_owned(&E->lock));
1566
1567 if (E->sourcelock)
1568 return false;
1569 E->sourcelock = (curcpu_available() ? (void *)1 : curlwp);
1570 return true;
1571 }
1572
1573 /*
1574 * rnd_unlock_sources()
1575 *
1576 * Unlock the list of sources after rnd_lock_sources or
1577 * rnd_trylock_sources. Caller must hold the global entropy lock.
1578 */
1579 static void
1580 rnd_unlock_sources(void)
1581 {
1582
1583 KASSERT(E->stage == ENTROPY_COLD || mutex_owned(&E->lock));
1584
1585 KASSERTMSG(E->sourcelock == (curcpu_available() ? (void *)1 : curlwp),
1586 "lwp %p releasing lock held by %p",
1587 (curcpu_available() ? (void *)1 : curlwp), E->sourcelock);
1588 E->sourcelock = NULL;
1589 if (E->stage >= ENTROPY_WARM)
1590 cv_broadcast(&E->cv);
1591 }
1592
1593 /*
1594 * rnd_sources_locked()
1595 *
1596 * True if we hold the list of rndsources locked, for diagnostic
1597 * assertions.
1598 */
1599 static bool __diagused
1600 rnd_sources_locked(void)
1601 {
1602
1603 return E->sourcelock == (curcpu_available() ? (void *)1 : curlwp);
1604 }
1605
1606 /*
1607 * entropy_request(nbytes)
1608 *
1609 * Request nbytes bytes of entropy from all sources in the system.
1610 * OK if we overdo it. Caller must hold the global entropy lock;
1611 * will release and re-acquire it.
1612 */
1613 static void
1614 entropy_request(size_t nbytes)
1615 {
1616 struct krndsource *rs;
1617
1618 KASSERT(E->stage == ENTROPY_COLD || mutex_owned(&E->lock));
1619
1620 /*
1621 * If there is a request in progress, let it proceed.
1622 * Otherwise, note that a request is in progress to avoid
1623 * reentry and to block rnd_detach_source until we're done.
1624 */
1625 if (!rnd_trylock_sources())
1626 return;
1627 entropy_request_evcnt.ev_count++;
1628
1629 /* Clamp to the maximum reasonable request. */
1630 nbytes = MIN(nbytes, ENTROPY_CAPACITY);
1631
1632 /* Walk the list of sources. */
1633 LIST_FOREACH(rs, &E->sources, list) {
1634 /* Skip sources without callbacks. */
1635 if (!ISSET(rs->flags, RND_FLAG_HASCB))
1636 continue;
1637
1638 /* Drop the lock while we call the callback. */
1639 if (E->stage >= ENTROPY_WARM)
1640 mutex_exit(&E->lock);
1641 (*rs->get)(nbytes, rs->getarg);
1642 if (E->stage >= ENTROPY_WARM)
1643 mutex_enter(&E->lock);
1644 }
1645
1646 /* Notify rnd_detach_source that the request is done. */
1647 rnd_unlock_sources();
1648 }
1649
1650 /*
1651 * rnd_add_uint32(rs, value)
1652 *
1653 * Enter 32 bits of data from an entropy source into the pool.
1654 *
1655 * If rs is NULL, may not be called from interrupt context.
1656 *
1657 * If rs is non-NULL, may be called from any context. May drop
1658 * data if called from interrupt context.
1659 */
1660 void
1661 rnd_add_uint32(struct krndsource *rs, uint32_t value)
1662 {
1663
1664 rnd_add_data(rs, &value, sizeof value, 0);
1665 }
1666
1667 void
1668 _rnd_add_uint32(struct krndsource *rs, uint32_t value)
1669 {
1670
1671 rnd_add_data(rs, &value, sizeof value, 0);
1672 }
1673
1674 void
1675 _rnd_add_uint64(struct krndsource *rs, uint64_t value)
1676 {
1677
1678 rnd_add_data(rs, &value, sizeof value, 0);
1679 }
1680
1681 /*
1682 * rnd_add_data(rs, buf, len, entropybits)
1683 *
1684 * Enter data from an entropy source into the pool, with a
1685 * driver's estimate of how much entropy the physical source of
1686 * the data has. If RND_FLAG_NO_ESTIMATE, we ignore the driver's
1687 * estimate and treat it as zero.
1688 *
1689 * If rs is NULL, may not be called from interrupt context.
1690 *
1691 * If rs is non-NULL, may be called from any context. May drop
1692 * data if called from interrupt context.
1693 */
1694 void
1695 rnd_add_data(struct krndsource *rs, const void *buf, uint32_t len,
1696 uint32_t entropybits)
1697 {
1698 uint32_t extra;
1699 uint32_t flags;
1700
1701 KASSERTMSG(howmany(entropybits, NBBY) <= len,
1702 "%s: impossible entropy rate:"
1703 " %"PRIu32" bits in %"PRIu32"-byte string",
1704 rs ? rs->name : "(anonymous)", entropybits, len);
1705
1706 /* If there's no rndsource, just enter the data and time now. */
1707 if (rs == NULL) {
1708 entropy_enter(buf, len, entropybits);
1709 extra = entropy_timer();
1710 entropy_enter(&extra, sizeof extra, 0);
1711 explicit_memset(&extra, 0, sizeof extra);
1712 return;
1713 }
1714
1715 /* Load a snapshot of the flags. Ioctl may change them under us. */
1716 flags = atomic_load_relaxed(&rs->flags);
1717
1718 /*
1719 * Skip if:
1720 * - we're not collecting entropy, or
1721 * - the operator doesn't want to collect entropy from this, or
1722 * - neither data nor timings are being collected from this.
1723 */
1724 if (!atomic_load_relaxed(&entropy_collection) ||
1725 ISSET(flags, RND_FLAG_NO_COLLECT) ||
1726 !ISSET(flags, RND_FLAG_COLLECT_VALUE|RND_FLAG_COLLECT_TIME))
1727 return;
1728
1729 /* If asked, ignore the estimate. */
1730 if (ISSET(flags, RND_FLAG_NO_ESTIMATE))
1731 entropybits = 0;
1732
1733 /* If we are collecting data, enter them. */
1734 if (ISSET(flags, RND_FLAG_COLLECT_VALUE))
1735 rnd_add_data_1(rs, buf, len, entropybits);
1736
1737 /* If we are collecting timings, enter one. */
1738 if (ISSET(flags, RND_FLAG_COLLECT_TIME)) {
1739 extra = entropy_timer();
1740 rnd_add_data_1(rs, &extra, sizeof extra, 0);
1741 }
1742 }
1743
1744 /*
1745 * rnd_add_data_1(rs, buf, len, entropybits)
1746 *
1747 * Internal subroutine to call either entropy_enter_intr, if we're
1748 * in interrupt context, or entropy_enter if not, and to count the
1749 * entropy in an rndsource.
1750 */
1751 static void
1752 rnd_add_data_1(struct krndsource *rs, const void *buf, uint32_t len,
1753 uint32_t entropybits)
1754 {
1755 bool fullyused;
1756
1757 /*
1758 * If we're in interrupt context, use entropy_enter_intr and
1759 * take note of whether it consumed the full sample; if not,
1760 * use entropy_enter, which always consumes the full sample.
1761 */
1762 if (curcpu_available() && cpu_intr_p()) {
1763 fullyused = entropy_enter_intr(buf, len, entropybits);
1764 } else {
1765 entropy_enter(buf, len, entropybits);
1766 fullyused = true;
1767 }
1768
1769 /*
1770 * If we used the full sample, note how many bits were
1771 * contributed from this source.
1772 */
1773 if (fullyused) {
1774 if (E->stage < ENTROPY_HOT) {
1775 if (E->stage >= ENTROPY_WARM)
1776 mutex_enter(&E->lock);
1777 rs->total += MIN(UINT_MAX - rs->total, entropybits);
1778 if (E->stage >= ENTROPY_WARM)
1779 mutex_exit(&E->lock);
1780 } else {
1781 struct rndsource_cpu *rc = percpu_getref(rs->state);
1782 unsigned nbits = rc->rc_nbits;
1783
1784 nbits += MIN(UINT_MAX - nbits, entropybits);
1785 atomic_store_relaxed(&rc->rc_nbits, nbits);
1786 percpu_putref(rs->state);
1787 }
1788 }
1789 }
1790
1791 /*
1792 * rnd_add_data_sync(rs, buf, len, entropybits)
1793 *
1794 * Same as rnd_add_data. Originally used in rndsource callbacks,
1795 * to break an unnecessary cycle; no longer really needed.
1796 */
1797 void
1798 rnd_add_data_sync(struct krndsource *rs, const void *buf, uint32_t len,
1799 uint32_t entropybits)
1800 {
1801
1802 rnd_add_data(rs, buf, len, entropybits);
1803 }
1804
1805 /*
1806 * rndsource_entropybits(rs)
1807 *
1808 * Return approximately the number of bits of entropy that have
1809 * been contributed via rs so far. Approximate if other CPUs may
1810 * be calling rnd_add_data concurrently.
1811 */
1812 static unsigned
1813 rndsource_entropybits(struct krndsource *rs)
1814 {
1815 unsigned nbits = rs->total;
1816
1817 KASSERT(E->stage >= ENTROPY_WARM);
1818 KASSERT(rnd_sources_locked());
1819 percpu_foreach(rs->state, rndsource_entropybits_cpu, &nbits);
1820 return nbits;
1821 }
1822
1823 static void
1824 rndsource_entropybits_cpu(void *ptr, void *cookie, struct cpu_info *ci)
1825 {
1826 struct rndsource_cpu *rc = ptr;
1827 unsigned *nbitsp = cookie;
1828 unsigned cpu_nbits;
1829
1830 cpu_nbits = atomic_load_relaxed(&rc->rc_nbits);
1831 *nbitsp += MIN(UINT_MAX - *nbitsp, cpu_nbits);
1832 }
1833
1834 /*
1835 * rndsource_to_user(rs, urs)
1836 *
1837 * Copy a description of rs out to urs for userland.
1838 */
1839 static void
1840 rndsource_to_user(struct krndsource *rs, rndsource_t *urs)
1841 {
1842
1843 KASSERT(E->stage >= ENTROPY_WARM);
1844 KASSERT(rnd_sources_locked());
1845
1846 /* Avoid kernel memory disclosure. */
1847 memset(urs, 0, sizeof(*urs));
1848
1849 CTASSERT(sizeof(urs->name) == sizeof(rs->name));
1850 strlcpy(urs->name, rs->name, sizeof(urs->name));
1851 urs->total = rndsource_entropybits(rs);
1852 urs->type = rs->type;
1853 urs->flags = atomic_load_relaxed(&rs->flags);
1854 }
1855
1856 /*
1857 * rndsource_to_user_est(rs, urse)
1858 *
1859 * Copy a description of rs and estimation statistics out to urse
1860 * for userland.
1861 */
1862 static void
1863 rndsource_to_user_est(struct krndsource *rs, rndsource_est_t *urse)
1864 {
1865
1866 KASSERT(E->stage >= ENTROPY_WARM);
1867 KASSERT(rnd_sources_locked());
1868
1869 /* Avoid kernel memory disclosure. */
1870 memset(urse, 0, sizeof(*urse));
1871
1872 /* Copy out the rndsource description. */
1873 rndsource_to_user(rs, &urse->rt);
1874
1875 /* Zero out the statistics because we don't do estimation. */
1876 urse->dt_samples = 0;
1877 urse->dt_total = 0;
1878 urse->dv_samples = 0;
1879 urse->dv_total = 0;
1880 }
1881
1882 /*
1883 * entropy_ioctl(cmd, data)
1884 *
1885 * Handle various /dev/random ioctl queries.
1886 */
1887 int
1888 entropy_ioctl(unsigned long cmd, void *data)
1889 {
1890 struct krndsource *rs;
1891 bool privileged;
1892 int error;
1893
1894 KASSERT(E->stage >= ENTROPY_WARM);
1895
1896 /* Verify user's authorization to perform the ioctl. */
1897 switch (cmd) {
1898 case RNDGETENTCNT:
1899 case RNDGETPOOLSTAT:
1900 case RNDGETSRCNUM:
1901 case RNDGETSRCNAME:
1902 case RNDGETESTNUM:
1903 case RNDGETESTNAME:
1904 error = kauth_authorize_device(curlwp->l_cred,
1905 KAUTH_DEVICE_RND_GETPRIV, NULL, NULL, NULL, NULL);
1906 break;
1907 case RNDCTL:
1908 error = kauth_authorize_device(curlwp->l_cred,
1909 KAUTH_DEVICE_RND_SETPRIV, NULL, NULL, NULL, NULL);
1910 break;
1911 case RNDADDDATA:
1912 error = kauth_authorize_device(curlwp->l_cred,
1913 KAUTH_DEVICE_RND_ADDDATA, NULL, NULL, NULL, NULL);
1914 /* Ascertain whether the user's inputs should be counted. */
1915 if (kauth_authorize_device(curlwp->l_cred,
1916 KAUTH_DEVICE_RND_ADDDATA_ESTIMATE,
1917 NULL, NULL, NULL, NULL) == 0)
1918 privileged = true;
1919 break;
1920 default: {
1921 /*
1922 * XXX Hack to avoid changing module ABI so this can be
1923 * pulled up. Later, we can just remove the argument.
1924 */
1925 static const struct fileops fops = {
1926 .fo_ioctl = rnd_system_ioctl,
1927 };
1928 struct file f = {
1929 .f_ops = &fops,
1930 };
1931 MODULE_HOOK_CALL(rnd_ioctl_50_hook, (&f, cmd, data),
1932 enosys(), error);
1933 #if defined(_LP64)
1934 if (error == ENOSYS)
1935 MODULE_HOOK_CALL(rnd_ioctl32_50_hook, (&f, cmd, data),
1936 enosys(), error);
1937 #endif
1938 if (error == ENOSYS)
1939 error = ENOTTY;
1940 break;
1941 }
1942 }
1943
1944 /* If anything went wrong with authorization, stop here. */
1945 if (error)
1946 return error;
1947
1948 /* Dispatch on the command. */
1949 switch (cmd) {
1950 case RNDGETENTCNT: { /* Get current entropy count in bits. */
1951 uint32_t *countp = data;
1952
1953 mutex_enter(&E->lock);
1954 *countp = ENTROPY_CAPACITY*NBBY - E->needed;
1955 mutex_exit(&E->lock);
1956
1957 break;
1958 }
1959 case RNDGETPOOLSTAT: { /* Get entropy pool statistics. */
1960 rndpoolstat_t *pstat = data;
1961
1962 mutex_enter(&E->lock);
1963
1964 /* parameters */
1965 pstat->poolsize = ENTPOOL_SIZE/sizeof(uint32_t); /* words */
1966 pstat->threshold = ENTROPY_CAPACITY*1; /* bytes */
1967 pstat->maxentropy = ENTROPY_CAPACITY*NBBY; /* bits */
1968
1969 /* state */
1970 pstat->added = 0; /* XXX total entropy_enter count */
1971 pstat->curentropy = ENTROPY_CAPACITY*NBBY - E->needed;
1972 pstat->removed = 0; /* XXX total entropy_extract count */
1973 pstat->discarded = 0; /* XXX bits of entropy beyond capacity */
1974 pstat->generated = 0; /* XXX bits of data...fabricated? */
1975
1976 mutex_exit(&E->lock);
1977 break;
1978 }
1979 case RNDGETSRCNUM: { /* Get entropy sources by number. */
1980 rndstat_t *stat = data;
1981 uint32_t start = 0, i = 0;
1982
1983 /* Skip if none requested; fail if too many requested. */
1984 if (stat->count == 0)
1985 break;
1986 if (stat->count > RND_MAXSTATCOUNT)
1987 return EINVAL;
1988
1989 /*
1990 * Under the lock, find the first one, copy out as many
1991 * as requested, and report how many we copied out.
1992 */
1993 mutex_enter(&E->lock);
1994 error = rnd_lock_sources();
1995 if (error) {
1996 mutex_exit(&E->lock);
1997 return error;
1998 }
1999 LIST_FOREACH(rs, &E->sources, list) {
2000 if (start++ == stat->start)
2001 break;
2002 }
2003 while (i < stat->count && rs != NULL) {
2004 mutex_exit(&E->lock);
2005 rndsource_to_user(rs, &stat->source[i++]);
2006 mutex_enter(&E->lock);
2007 rs = LIST_NEXT(rs, list);
2008 }
2009 KASSERT(i <= stat->count);
2010 stat->count = i;
2011 rnd_unlock_sources();
2012 mutex_exit(&E->lock);
2013 break;
2014 }
2015 case RNDGETESTNUM: { /* Get sources and estimates by number. */
2016 rndstat_est_t *estat = data;
2017 uint32_t start = 0, i = 0;
2018
2019 /* Skip if none requested; fail if too many requested. */
2020 if (estat->count == 0)
2021 break;
2022 if (estat->count > RND_MAXSTATCOUNT)
2023 return EINVAL;
2024
2025 /*
2026 * Under the lock, find the first one, copy out as many
2027 * as requested, and report how many we copied out.
2028 */
2029 mutex_enter(&E->lock);
2030 error = rnd_lock_sources();
2031 if (error) {
2032 mutex_exit(&E->lock);
2033 return error;
2034 }
2035 LIST_FOREACH(rs, &E->sources, list) {
2036 if (start++ == estat->start)
2037 break;
2038 }
2039 while (i < estat->count && rs != NULL) {
2040 mutex_exit(&E->lock);
2041 rndsource_to_user_est(rs, &estat->source[i++]);
2042 mutex_enter(&E->lock);
2043 rs = LIST_NEXT(rs, list);
2044 }
2045 KASSERT(i <= estat->count);
2046 estat->count = i;
2047 rnd_unlock_sources();
2048 mutex_exit(&E->lock);
2049 break;
2050 }
2051 case RNDGETSRCNAME: { /* Get entropy sources by name. */
2052 rndstat_name_t *nstat = data;
2053 const size_t n = sizeof(rs->name);
2054
2055 CTASSERT(sizeof(rs->name) == sizeof(nstat->name));
2056
2057 /*
2058 * Under the lock, search by name. If found, copy it
2059 * out; if not found, fail with ENOENT.
2060 */
2061 mutex_enter(&E->lock);
2062 error = rnd_lock_sources();
2063 if (error) {
2064 mutex_exit(&E->lock);
2065 return error;
2066 }
2067 LIST_FOREACH(rs, &E->sources, list) {
2068 if (strncmp(rs->name, nstat->name, n) == 0)
2069 break;
2070 }
2071 if (rs != NULL) {
2072 mutex_exit(&E->lock);
2073 rndsource_to_user(rs, &nstat->source);
2074 mutex_enter(&E->lock);
2075 } else {
2076 error = ENOENT;
2077 }
2078 rnd_unlock_sources();
2079 mutex_exit(&E->lock);
2080 break;
2081 }
2082 case RNDGETESTNAME: { /* Get sources and estimates by name. */
2083 rndstat_est_name_t *enstat = data;
2084 const size_t n = sizeof(rs->name);
2085
2086 CTASSERT(sizeof(rs->name) == sizeof(enstat->name));
2087
2088 /*
2089 * Under the lock, search by name. If found, copy it
2090 * out; if not found, fail with ENOENT.
2091 */
2092 mutex_enter(&E->lock);
2093 error = rnd_lock_sources();
2094 if (error) {
2095 mutex_exit(&E->lock);
2096 return error;
2097 }
2098 LIST_FOREACH(rs, &E->sources, list) {
2099 if (strncmp(rs->name, enstat->name, n) == 0)
2100 break;
2101 }
2102 if (rs != NULL) {
2103 mutex_exit(&E->lock);
2104 rndsource_to_user_est(rs, &enstat->source);
2105 mutex_enter(&E->lock);
2106 } else {
2107 error = ENOENT;
2108 }
2109 rnd_unlock_sources();
2110 mutex_exit(&E->lock);
2111 break;
2112 }
2113 case RNDCTL: { /* Modify entropy source flags. */
2114 rndctl_t *rndctl = data;
2115 const size_t n = sizeof(rs->name);
2116 uint32_t flags;
2117
2118 CTASSERT(sizeof(rs->name) == sizeof(rndctl->name));
2119
2120 /* Whitelist the flags that user can change. */
2121 rndctl->mask &= RND_FLAG_NO_ESTIMATE|RND_FLAG_NO_COLLECT;
2122
2123 /*
2124 * For each matching rndsource, either by type if
2125 * specified or by name if not, set the masked flags.
2126 */
2127 mutex_enter(&E->lock);
2128 LIST_FOREACH(rs, &E->sources, list) {
2129 if (rndctl->type != 0xff) {
2130 if (rs->type != rndctl->type)
2131 continue;
2132 } else {
2133 if (strncmp(rs->name, rndctl->name, n) != 0)
2134 continue;
2135 }
2136 flags = rs->flags & ~rndctl->mask;
2137 flags |= rndctl->flags & rndctl->mask;
2138 atomic_store_relaxed(&rs->flags, flags);
2139 }
2140 mutex_exit(&E->lock);
2141 break;
2142 }
2143 case RNDADDDATA: { /* Enter seed into entropy pool. */
2144 rnddata_t *rdata = data;
2145 unsigned entropybits = 0;
2146
2147 if (!atomic_load_relaxed(&entropy_collection))
2148 break; /* thanks but no thanks */
2149 if (rdata->len > MIN(sizeof(rdata->data), UINT32_MAX/NBBY))
2150 return EINVAL;
2151
2152 /*
2153 * This ioctl serves as the userland alternative a
2154 * bootloader-provided seed -- typically furnished by
2155 * /etc/rc.d/random_seed. We accept the user's entropy
2156 * claim only if
2157 *
2158 * (a) the user is privileged, and
2159 * (b) we have not entered a bootloader seed.
2160 *
2161 * under the assumption that the user may use this to
2162 * load a seed from disk that we have already loaded
2163 * from the bootloader, so we don't double-count it.
2164 */
2165 if (privileged) {
2166 mutex_enter(&E->lock);
2167 if (!E->seeded) {
2168 entropybits = MIN(rdata->entropy,
2169 MIN(rdata->len, ENTROPY_CAPACITY)*NBBY);
2170 E->seeded = true;
2171 }
2172 mutex_exit(&E->lock);
2173 }
2174
2175 /* Enter the data. */
2176 rnd_add_data(&seed_rndsource, rdata->data, rdata->len,
2177 entropybits);
2178 break;
2179 }
2180 default:
2181 error = ENOTTY;
2182 }
2183
2184 /* Return any error that may have come up. */
2185 return error;
2186 }
2187
2188 /* Legacy entry points */
2189
2190 void
2191 rnd_seed(void *seed, size_t len)
2192 {
2193
2194 if (len != sizeof(rndsave_t)) {
2195 printf("entropy: invalid seed length: %zu,"
2196 " expected sizeof(rndsave_t) = %zu\n",
2197 len, sizeof(rndsave_t));
2198 return;
2199 }
2200 entropy_seed(seed);
2201 }
2202
2203 void
2204 rnd_init(void)
2205 {
2206
2207 entropy_init();
2208 }
2209
2210 void
2211 rnd_init_softint(void)
2212 {
2213
2214 entropy_init_late();
2215 }
2216
2217 int
2218 rnd_system_ioctl(struct file *fp, unsigned long cmd, void *data)
2219 {
2220
2221 return entropy_ioctl(cmd, data);
2222 }
2223