kern_entropy.c revision 1.24 1 /* $NetBSD: kern_entropy.c,v 1.24 2020/09/29 07:51:01 gson Exp $ */
2
3 /*-
4 * Copyright (c) 2019 The NetBSD Foundation, Inc.
5 * All rights reserved.
6 *
7 * This code is derived from software contributed to The NetBSD Foundation
8 * by Taylor R. Campbell.
9 *
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions
12 * are met:
13 * 1. Redistributions of source code must retain the above copyright
14 * notice, this list of conditions and the following disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright
16 * notice, this list of conditions and the following disclaimer in the
17 * documentation and/or other materials provided with the distribution.
18 *
19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29 * POSSIBILITY OF SUCH DAMAGE.
30 */
31
32 /*
33 * Entropy subsystem
34 *
35 * * Each CPU maintains a per-CPU entropy pool so that gathering
36 * entropy requires no interprocessor synchronization, except
37 * early at boot when we may be scrambling to gather entropy as
38 * soon as possible.
39 *
40 * - entropy_enter gathers entropy and never drops it on the
41 * floor, at the cost of sometimes having to do cryptography.
42 *
43 * - entropy_enter_intr gathers entropy or drops it on the
44 * floor, with low latency. Work to stir the pool or kick the
45 * housekeeping thread is scheduled in soft interrupts.
46 *
47 * * entropy_enter immediately enters into the global pool if it
48 * can transition to full entropy in one swell foop. Otherwise,
49 * it defers to a housekeeping thread that consolidates entropy,
50 * but only when the CPUs collectively have full entropy, in
51 * order to mitigate iterative-guessing attacks.
52 *
53 * * The entropy housekeeping thread continues to consolidate
54 * entropy even after we think we have full entropy, in case we
55 * are wrong, but is limited to one discretionary consolidation
56 * per minute, and only when new entropy is actually coming in,
57 * to limit performance impact.
58 *
59 * * The entropy epoch is the number that changes when we
60 * transition from partial entropy to full entropy, so that
61 * users can easily determine when to reseed. This also
62 * facilitates an operator explicitly causing everything to
63 * reseed by sysctl -w kern.entropy.consolidate=1.
64 *
65 * * No entropy estimation based on the sample values, which is a
66 * contradiction in terms and a potential source of side
67 * channels. It is the responsibility of the driver author to
68 * study how predictable the physical source of input can ever
69 * be, and to furnish a lower bound on the amount of entropy it
70 * has.
71 *
72 * * Entropy depletion is available for testing (or if you're into
73 * that sort of thing), with sysctl -w kern.entropy.depletion=1;
74 * the logic to support it is small, to minimize chance of bugs.
75 */
76
77 #include <sys/cdefs.h>
78 __KERNEL_RCSID(0, "$NetBSD: kern_entropy.c,v 1.24 2020/09/29 07:51:01 gson Exp $");
79
80 #include <sys/param.h>
81 #include <sys/types.h>
82 #include <sys/atomic.h>
83 #include <sys/compat_stub.h>
84 #include <sys/condvar.h>
85 #include <sys/cpu.h>
86 #include <sys/entropy.h>
87 #include <sys/errno.h>
88 #include <sys/evcnt.h>
89 #include <sys/event.h>
90 #include <sys/file.h>
91 #include <sys/intr.h>
92 #include <sys/kauth.h>
93 #include <sys/kernel.h>
94 #include <sys/kmem.h>
95 #include <sys/kthread.h>
96 #include <sys/module_hook.h>
97 #include <sys/mutex.h>
98 #include <sys/percpu.h>
99 #include <sys/poll.h>
100 #include <sys/queue.h>
101 #include <sys/rnd.h> /* legacy kernel API */
102 #include <sys/rndio.h> /* userland ioctl interface */
103 #include <sys/rndsource.h> /* kernel rndsource driver API */
104 #include <sys/select.h>
105 #include <sys/selinfo.h>
106 #include <sys/sha1.h> /* for boot seed checksum */
107 #include <sys/stdint.h>
108 #include <sys/sysctl.h>
109 #include <sys/systm.h>
110 #include <sys/time.h>
111 #include <sys/xcall.h>
112
113 #include <lib/libkern/entpool.h>
114
115 #include <machine/limits.h>
116
117 #ifdef __HAVE_CPU_COUNTER
118 #include <machine/cpu_counter.h>
119 #endif
120
121 /*
122 * struct entropy_cpu
123 *
124 * Per-CPU entropy state. The pool is allocated separately
125 * because percpu(9) sometimes moves per-CPU objects around
126 * without zeroing them, which would lead to unwanted copies of
127 * sensitive secrets. The evcnt is allocated separately becuase
128 * evcnt(9) assumes it stays put in memory.
129 */
130 struct entropy_cpu {
131 struct evcnt *ec_softint_evcnt;
132 struct entpool *ec_pool;
133 unsigned ec_pending;
134 bool ec_locked;
135 };
136
137 /*
138 * struct rndsource_cpu
139 *
140 * Per-CPU rndsource state.
141 */
142 struct rndsource_cpu {
143 unsigned rc_nbits; /* bits of entropy added */
144 };
145
146 /*
147 * entropy_global (a.k.a. E for short in this file)
148 *
149 * Global entropy state. Writes protected by the global lock.
150 * Some fields, marked (A), can be read outside the lock, and are
151 * maintained with atomic_load/store_relaxed.
152 */
153 struct {
154 kmutex_t lock; /* covers all global state */
155 struct entpool pool; /* global pool for extraction */
156 unsigned needed; /* (A) needed globally */
157 unsigned pending; /* (A) pending in per-CPU pools */
158 unsigned timestamp; /* (A) time of last consolidation */
159 unsigned epoch; /* (A) changes when needed -> 0 */
160 kcondvar_t cv; /* notifies state changes */
161 struct selinfo selq; /* notifies needed -> 0 */
162 struct lwp *sourcelock; /* lock on list of sources */
163 LIST_HEAD(,krndsource) sources; /* list of entropy sources */
164 enum entropy_stage {
165 ENTROPY_COLD = 0, /* single-threaded */
166 ENTROPY_WARM, /* multi-threaded at boot before CPUs */
167 ENTROPY_HOT, /* multi-threaded multi-CPU */
168 } stage;
169 bool consolidate; /* kick thread to consolidate */
170 bool seed_rndsource; /* true if seed source is attached */
171 bool seeded; /* true if seed file already loaded */
172 } entropy_global __cacheline_aligned = {
173 /* Fields that must be initialized when the kernel is loaded. */
174 .needed = ENTROPY_CAPACITY*NBBY,
175 .epoch = (unsigned)-1, /* -1 means entropy never consolidated */
176 .sources = LIST_HEAD_INITIALIZER(entropy_global.sources),
177 .stage = ENTROPY_COLD,
178 };
179
180 #define E (&entropy_global) /* declutter */
181
182 /* Read-mostly globals */
183 static struct percpu *entropy_percpu __read_mostly; /* struct entropy_cpu */
184 static void *entropy_sih __read_mostly; /* softint handler */
185 static struct lwp *entropy_lwp __read_mostly; /* housekeeping thread */
186
187 int rnd_initial_entropy __read_mostly; /* XXX legacy */
188
189 static struct krndsource seed_rndsource __read_mostly;
190
191 /*
192 * Event counters
193 *
194 * Must be careful with adding these because they can serve as
195 * side channels.
196 */
197 static struct evcnt entropy_discretionary_evcnt =
198 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "entropy", "discretionary");
199 EVCNT_ATTACH_STATIC(entropy_discretionary_evcnt);
200 static struct evcnt entropy_immediate_evcnt =
201 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "entropy", "immediate");
202 EVCNT_ATTACH_STATIC(entropy_immediate_evcnt);
203 static struct evcnt entropy_partial_evcnt =
204 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "entropy", "partial");
205 EVCNT_ATTACH_STATIC(entropy_partial_evcnt);
206 static struct evcnt entropy_consolidate_evcnt =
207 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "entropy", "consolidate");
208 EVCNT_ATTACH_STATIC(entropy_consolidate_evcnt);
209 static struct evcnt entropy_extract_intr_evcnt =
210 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "entropy", "extract intr");
211 EVCNT_ATTACH_STATIC(entropy_extract_intr_evcnt);
212 static struct evcnt entropy_extract_fail_evcnt =
213 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "entropy", "extract fail");
214 EVCNT_ATTACH_STATIC(entropy_extract_fail_evcnt);
215 static struct evcnt entropy_request_evcnt =
216 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "entropy", "request");
217 EVCNT_ATTACH_STATIC(entropy_request_evcnt);
218 static struct evcnt entropy_deplete_evcnt =
219 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "entropy", "deplete");
220 EVCNT_ATTACH_STATIC(entropy_deplete_evcnt);
221 static struct evcnt entropy_notify_evcnt =
222 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "entropy", "notify");
223 EVCNT_ATTACH_STATIC(entropy_notify_evcnt);
224
225 /* Sysctl knobs */
226 static bool entropy_collection = 1;
227 static bool entropy_depletion = 0; /* Silly! */
228
229 static const struct sysctlnode *entropy_sysctlroot;
230 static struct sysctllog *entropy_sysctllog;
231
232 /* Forward declarations */
233 static void entropy_init_cpu(void *, void *, struct cpu_info *);
234 static void entropy_fini_cpu(void *, void *, struct cpu_info *);
235 static void entropy_account_cpu(struct entropy_cpu *);
236 static void entropy_enter(const void *, size_t, unsigned);
237 static bool entropy_enter_intr(const void *, size_t, unsigned);
238 static void entropy_softintr(void *);
239 static void entropy_thread(void *);
240 static uint32_t entropy_pending(void);
241 static void entropy_pending_cpu(void *, void *, struct cpu_info *);
242 static void entropy_do_consolidate(void);
243 static void entropy_consolidate_xc(void *, void *);
244 static void entropy_notify(void);
245 static int sysctl_entropy_consolidate(SYSCTLFN_ARGS);
246 static int sysctl_entropy_gather(SYSCTLFN_ARGS);
247 static void filt_entropy_read_detach(struct knote *);
248 static int filt_entropy_read_event(struct knote *, long);
249 static void entropy_request(size_t);
250 static void rnd_add_data_1(struct krndsource *, const void *, uint32_t,
251 uint32_t);
252 static unsigned rndsource_entropybits(struct krndsource *);
253 static void rndsource_entropybits_cpu(void *, void *, struct cpu_info *);
254 static void rndsource_to_user(struct krndsource *, rndsource_t *);
255 static void rndsource_to_user_est(struct krndsource *, rndsource_est_t *);
256
257 /*
258 * entropy_timer()
259 *
260 * Cycle counter, time counter, or anything that changes a wee bit
261 * unpredictably.
262 */
263 static inline uint32_t
264 entropy_timer(void)
265 {
266 struct bintime bt;
267 uint32_t v;
268
269 /* If we have a CPU cycle counter, use the low 32 bits. */
270 #ifdef __HAVE_CPU_COUNTER
271 if (__predict_true(cpu_hascounter()))
272 return cpu_counter32();
273 #endif /* __HAVE_CPU_COUNTER */
274
275 /* If we're cold, tough. Can't binuptime while cold. */
276 if (__predict_false(cold))
277 return 0;
278
279 /* Fold the 128 bits of binuptime into 32 bits. */
280 binuptime(&bt);
281 v = bt.frac;
282 v ^= bt.frac >> 32;
283 v ^= bt.sec;
284 v ^= bt.sec >> 32;
285 return v;
286 }
287
288 static void
289 attach_seed_rndsource(void)
290 {
291
292 /*
293 * First called no later than entropy_init, while we are still
294 * single-threaded, so no need for RUN_ONCE.
295 */
296 if (E->stage >= ENTROPY_WARM || E->seed_rndsource)
297 return;
298 rnd_attach_source(&seed_rndsource, "seed", RND_TYPE_UNKNOWN,
299 RND_FLAG_COLLECT_VALUE);
300 E->seed_rndsource = true;
301 }
302
303 /*
304 * entropy_init()
305 *
306 * Initialize the entropy subsystem. Panic on failure.
307 *
308 * Requires percpu(9) and sysctl(9) to be initialized.
309 */
310 static void
311 entropy_init(void)
312 {
313 uint32_t extra[2];
314 struct krndsource *rs;
315 unsigned i = 0;
316
317 KASSERT(E->stage == ENTROPY_COLD);
318
319 /* Grab some cycle counts early at boot. */
320 extra[i++] = entropy_timer();
321
322 /* Run the entropy pool cryptography self-test. */
323 if (entpool_selftest() == -1)
324 panic("entropy pool crypto self-test failed");
325
326 /* Create the sysctl directory. */
327 sysctl_createv(&entropy_sysctllog, 0, NULL, &entropy_sysctlroot,
328 CTLFLAG_PERMANENT, CTLTYPE_NODE, "entropy",
329 SYSCTL_DESCR("Entropy (random number sources) options"),
330 NULL, 0, NULL, 0,
331 CTL_KERN, CTL_CREATE, CTL_EOL);
332
333 /* Create the sysctl knobs. */
334 /* XXX These shouldn't be writable at securelevel>0. */
335 sysctl_createv(&entropy_sysctllog, 0, &entropy_sysctlroot, NULL,
336 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, CTLTYPE_BOOL, "collection",
337 SYSCTL_DESCR("Automatically collect entropy from hardware"),
338 NULL, 0, &entropy_collection, 0, CTL_CREATE, CTL_EOL);
339 sysctl_createv(&entropy_sysctllog, 0, &entropy_sysctlroot, NULL,
340 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, CTLTYPE_BOOL, "depletion",
341 SYSCTL_DESCR("`Deplete' entropy pool when observed"),
342 NULL, 0, &entropy_depletion, 0, CTL_CREATE, CTL_EOL);
343 sysctl_createv(&entropy_sysctllog, 0, &entropy_sysctlroot, NULL,
344 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, CTLTYPE_INT, "consolidate",
345 SYSCTL_DESCR("Trigger entropy consolidation now"),
346 sysctl_entropy_consolidate, 0, NULL, 0, CTL_CREATE, CTL_EOL);
347 sysctl_createv(&entropy_sysctllog, 0, &entropy_sysctlroot, NULL,
348 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, CTLTYPE_INT, "gather",
349 SYSCTL_DESCR("Trigger entropy gathering from sources now"),
350 sysctl_entropy_gather, 0, NULL, 0, CTL_CREATE, CTL_EOL);
351 /* XXX These should maybe not be readable at securelevel>0. */
352 sysctl_createv(&entropy_sysctllog, 0, &entropy_sysctlroot, NULL,
353 CTLFLAG_PERMANENT|CTLFLAG_READONLY|CTLFLAG_PRIVATE, CTLTYPE_INT,
354 "needed", SYSCTL_DESCR("Systemwide entropy deficit"),
355 NULL, 0, &E->needed, 0, CTL_CREATE, CTL_EOL);
356 sysctl_createv(&entropy_sysctllog, 0, &entropy_sysctlroot, NULL,
357 CTLFLAG_PERMANENT|CTLFLAG_READONLY|CTLFLAG_PRIVATE, CTLTYPE_INT,
358 "pending", SYSCTL_DESCR("Entropy pending on CPUs"),
359 NULL, 0, &E->pending, 0, CTL_CREATE, CTL_EOL);
360 sysctl_createv(&entropy_sysctllog, 0, &entropy_sysctlroot, NULL,
361 CTLFLAG_PERMANENT|CTLFLAG_READONLY|CTLFLAG_PRIVATE, CTLTYPE_INT,
362 "epoch", SYSCTL_DESCR("Entropy epoch"),
363 NULL, 0, &E->epoch, 0, CTL_CREATE, CTL_EOL);
364
365 /* Initialize the global state for multithreaded operation. */
366 mutex_init(&E->lock, MUTEX_DEFAULT, IPL_VM);
367 cv_init(&E->cv, "entropy");
368 selinit(&E->selq);
369
370 /* Make sure the seed source is attached. */
371 attach_seed_rndsource();
372
373 /* Note if the bootloader didn't provide a seed. */
374 if (!E->seeded)
375 printf("entropy: no seed from bootloader\n");
376
377 /* Allocate the per-CPU records for all early entropy sources. */
378 LIST_FOREACH(rs, &E->sources, list)
379 rs->state = percpu_alloc(sizeof(struct rndsource_cpu));
380
381 /* Enter the boot cycle count to get started. */
382 extra[i++] = entropy_timer();
383 KASSERT(i == __arraycount(extra));
384 entropy_enter(extra, sizeof extra, 0);
385 explicit_memset(extra, 0, sizeof extra);
386
387 /* We are now ready for multi-threaded operation. */
388 E->stage = ENTROPY_WARM;
389 }
390
391 /*
392 * entropy_init_late()
393 *
394 * Late initialization. Panic on failure.
395 *
396 * Requires CPUs to have been detected and LWPs to have started.
397 */
398 static void
399 entropy_init_late(void)
400 {
401 int error;
402
403 KASSERT(E->stage == ENTROPY_WARM);
404
405 /* Allocate and initialize the per-CPU state. */
406 entropy_percpu = percpu_create(sizeof(struct entropy_cpu),
407 entropy_init_cpu, entropy_fini_cpu, NULL);
408
409 /*
410 * Establish the softint at the highest softint priority level.
411 * Must happen after CPU detection.
412 */
413 entropy_sih = softint_establish(SOFTINT_SERIAL|SOFTINT_MPSAFE,
414 &entropy_softintr, NULL);
415 if (entropy_sih == NULL)
416 panic("unable to establish entropy softint");
417
418 /*
419 * Create the entropy housekeeping thread. Must happen after
420 * lwpinit.
421 */
422 error = kthread_create(PRI_NONE, KTHREAD_MPSAFE|KTHREAD_TS, NULL,
423 entropy_thread, NULL, &entropy_lwp, "entbutler");
424 if (error)
425 panic("unable to create entropy housekeeping thread: %d",
426 error);
427
428 /*
429 * Wait until the per-CPU initialization has hit all CPUs
430 * before proceeding to mark the entropy system hot.
431 */
432 xc_barrier(XC_HIGHPRI);
433 E->stage = ENTROPY_HOT;
434 }
435
436 /*
437 * entropy_init_cpu(ptr, cookie, ci)
438 *
439 * percpu(9) constructor for per-CPU entropy pool.
440 */
441 static void
442 entropy_init_cpu(void *ptr, void *cookie, struct cpu_info *ci)
443 {
444 struct entropy_cpu *ec = ptr;
445
446 ec->ec_softint_evcnt = kmem_alloc(sizeof(*ec->ec_softint_evcnt),
447 KM_SLEEP);
448 ec->ec_pool = kmem_zalloc(sizeof(*ec->ec_pool), KM_SLEEP);
449 ec->ec_pending = 0;
450 ec->ec_locked = false;
451
452 evcnt_attach_dynamic(ec->ec_softint_evcnt, EVCNT_TYPE_MISC, NULL,
453 ci->ci_cpuname, "entropy softint");
454 }
455
456 /*
457 * entropy_fini_cpu(ptr, cookie, ci)
458 *
459 * percpu(9) destructor for per-CPU entropy pool.
460 */
461 static void
462 entropy_fini_cpu(void *ptr, void *cookie, struct cpu_info *ci)
463 {
464 struct entropy_cpu *ec = ptr;
465
466 /*
467 * Zero any lingering data. Disclosure of the per-CPU pool
468 * shouldn't retroactively affect the security of any keys
469 * generated, because entpool(9) erases whatever we have just
470 * drawn out of any pool, but better safe than sorry.
471 */
472 explicit_memset(ec->ec_pool, 0, sizeof(*ec->ec_pool));
473
474 evcnt_detach(ec->ec_softint_evcnt);
475
476 kmem_free(ec->ec_pool, sizeof(*ec->ec_pool));
477 kmem_free(ec->ec_softint_evcnt, sizeof(*ec->ec_softint_evcnt));
478 }
479
480 /*
481 * entropy_seed(seed)
482 *
483 * Seed the entropy pool with seed. Meant to be called as early
484 * as possible by the bootloader; may be called before or after
485 * entropy_init. Must be called before system reaches userland.
486 * Must be called in thread or soft interrupt context, not in hard
487 * interrupt context. Must be called at most once.
488 *
489 * Overwrites the seed in place. Caller may then free the memory.
490 */
491 static void
492 entropy_seed(rndsave_t *seed)
493 {
494 SHA1_CTX ctx;
495 uint8_t digest[SHA1_DIGEST_LENGTH];
496 bool seeded;
497
498 /*
499 * Verify the checksum. If the checksum fails, take the data
500 * but ignore the entropy estimate -- the file may have been
501 * incompletely written with garbage, which is harmless to add
502 * but may not be as unpredictable as alleged.
503 */
504 SHA1Init(&ctx);
505 SHA1Update(&ctx, (const void *)&seed->entropy, sizeof(seed->entropy));
506 SHA1Update(&ctx, seed->data, sizeof(seed->data));
507 SHA1Final(digest, &ctx);
508 CTASSERT(sizeof(seed->digest) == sizeof(digest));
509 if (!consttime_memequal(digest, seed->digest, sizeof(digest))) {
510 printf("entropy: invalid seed checksum\n");
511 seed->entropy = 0;
512 }
513 explicit_memset(&ctx, 0, sizeof ctx);
514 explicit_memset(digest, 0, sizeof digest);
515
516 /*
517 * If the entropy is insensibly large, try byte-swapping.
518 * Otherwise assume the file is corrupted and act as though it
519 * has zero entropy.
520 */
521 if (howmany(seed->entropy, NBBY) > sizeof(seed->data)) {
522 seed->entropy = bswap32(seed->entropy);
523 if (howmany(seed->entropy, NBBY) > sizeof(seed->data))
524 seed->entropy = 0;
525 }
526
527 /* Make sure the seed source is attached. */
528 attach_seed_rndsource();
529
530 /* Test and set E->seeded. */
531 if (E->stage >= ENTROPY_WARM)
532 mutex_enter(&E->lock);
533 seeded = E->seeded;
534 E->seeded = (seed->entropy > 0);
535 if (E->stage >= ENTROPY_WARM)
536 mutex_exit(&E->lock);
537
538 /*
539 * If we've been seeded, may be re-entering the same seed
540 * (e.g., bootloader vs module init, or something). No harm in
541 * entering it twice, but it contributes no additional entropy.
542 */
543 if (seeded) {
544 printf("entropy: double-seeded by bootloader\n");
545 seed->entropy = 0;
546 } else {
547 printf("entropy: entering seed from bootloader"
548 " with %u bits of entropy\n", (unsigned)seed->entropy);
549 }
550
551 /* Enter it into the pool and promptly zero it. */
552 rnd_add_data(&seed_rndsource, seed->data, sizeof(seed->data),
553 seed->entropy);
554 explicit_memset(seed, 0, sizeof(*seed));
555 }
556
557 /*
558 * entropy_bootrequest()
559 *
560 * Request entropy from all sources at boot, once config is
561 * complete and interrupts are running.
562 */
563 void
564 entropy_bootrequest(void)
565 {
566
567 KASSERT(E->stage >= ENTROPY_WARM);
568
569 /*
570 * Request enough to satisfy the maximum entropy shortage.
571 * This is harmless overkill if the bootloader provided a seed.
572 */
573 mutex_enter(&E->lock);
574 entropy_request(ENTROPY_CAPACITY);
575 mutex_exit(&E->lock);
576 }
577
578 /*
579 * entropy_epoch()
580 *
581 * Returns the current entropy epoch. If this changes, you should
582 * reseed. If -1, means system entropy has not yet reached full
583 * entropy or been explicitly consolidated; never reverts back to
584 * -1. Never zero, so you can always use zero as an uninitialized
585 * sentinel value meaning `reseed ASAP'.
586 *
587 * Usage model:
588 *
589 * struct foo {
590 * struct crypto_prng prng;
591 * unsigned epoch;
592 * } *foo;
593 *
594 * unsigned epoch = entropy_epoch();
595 * if (__predict_false(epoch != foo->epoch)) {
596 * uint8_t seed[32];
597 * if (entropy_extract(seed, sizeof seed, 0) != 0)
598 * warn("no entropy");
599 * crypto_prng_reseed(&foo->prng, seed, sizeof seed);
600 * foo->epoch = epoch;
601 * }
602 */
603 unsigned
604 entropy_epoch(void)
605 {
606
607 /*
608 * Unsigned int, so no need for seqlock for an atomic read, but
609 * make sure we read it afresh each time.
610 */
611 return atomic_load_relaxed(&E->epoch);
612 }
613
614 /*
615 * entropy_ready()
616 *
617 * True if the entropy pool has full entropy.
618 */
619 bool
620 entropy_ready(void)
621 {
622
623 return atomic_load_relaxed(&E->needed) == 0;
624 }
625
626 /*
627 * entropy_account_cpu(ec)
628 *
629 * Consider whether to consolidate entropy into the global pool
630 * after we just added some into the current CPU's pending pool.
631 *
632 * - If this CPU can provide enough entropy now, do so.
633 *
634 * - If this and whatever else is available on other CPUs can
635 * provide enough entropy, kick the consolidation thread.
636 *
637 * - Otherwise, do as little as possible, except maybe consolidate
638 * entropy at most once a minute.
639 *
640 * Caller must be bound to a CPU and therefore have exclusive
641 * access to ec. Will acquire and release the global lock.
642 */
643 static void
644 entropy_account_cpu(struct entropy_cpu *ec)
645 {
646 unsigned diff;
647
648 KASSERT(E->stage == ENTROPY_HOT);
649
650 /*
651 * If there's no entropy needed, and entropy has been
652 * consolidated in the last minute, do nothing.
653 */
654 if (__predict_true(atomic_load_relaxed(&E->needed) == 0) &&
655 __predict_true(!atomic_load_relaxed(&entropy_depletion)) &&
656 __predict_true((time_uptime - E->timestamp) <= 60))
657 return;
658
659 /* If there's nothing pending, stop here. */
660 if (ec->ec_pending == 0)
661 return;
662
663 /* Consider consolidation, under the lock. */
664 mutex_enter(&E->lock);
665 if (E->needed != 0 && E->needed <= ec->ec_pending) {
666 /*
667 * If we have not yet attained full entropy but we can
668 * now, do so. This way we disseminate entropy
669 * promptly when it becomes available early at boot;
670 * otherwise we leave it to the entropy consolidation
671 * thread, which is rate-limited to mitigate side
672 * channels and abuse.
673 */
674 uint8_t buf[ENTPOOL_CAPACITY];
675
676 /* Transfer from the local pool to the global pool. */
677 entpool_extract(ec->ec_pool, buf, sizeof buf);
678 entpool_enter(&E->pool, buf, sizeof buf);
679 atomic_store_relaxed(&ec->ec_pending, 0);
680 atomic_store_relaxed(&E->needed, 0);
681
682 /* Notify waiters that we now have full entropy. */
683 entropy_notify();
684 entropy_immediate_evcnt.ev_count++;
685 } else {
686 /* Record how much we can add to the global pool. */
687 diff = MIN(ec->ec_pending, ENTROPY_CAPACITY*NBBY - E->pending);
688 E->pending += diff;
689 atomic_store_relaxed(&ec->ec_pending, ec->ec_pending - diff);
690
691 /*
692 * This should have made a difference unless we were
693 * already saturated.
694 */
695 KASSERT(diff || E->pending == ENTROPY_CAPACITY*NBBY);
696 KASSERT(E->pending);
697
698 if (E->needed <= E->pending) {
699 /*
700 * Enough entropy between all the per-CPU
701 * pools. Wake up the housekeeping thread.
702 *
703 * If we don't need any entropy, this doesn't
704 * mean much, but it is the only time we ever
705 * gather additional entropy in case the
706 * accounting has been overly optimistic. This
707 * happens at most once a minute, so there's
708 * negligible performance cost.
709 */
710 E->consolidate = true;
711 cv_broadcast(&E->cv);
712 if (E->needed == 0)
713 entropy_discretionary_evcnt.ev_count++;
714 } else {
715 /* Can't get full entropy. Keep gathering. */
716 entropy_partial_evcnt.ev_count++;
717 }
718 }
719 mutex_exit(&E->lock);
720 }
721
722 /*
723 * entropy_enter_early(buf, len, nbits)
724 *
725 * Do entropy bookkeeping globally, before we have established
726 * per-CPU pools. Enter directly into the global pool in the hope
727 * that we enter enough before the first entropy_extract to thwart
728 * iterative-guessing attacks; entropy_extract will warn if not.
729 */
730 static void
731 entropy_enter_early(const void *buf, size_t len, unsigned nbits)
732 {
733 bool notify = false;
734
735 if (E->stage >= ENTROPY_WARM)
736 mutex_enter(&E->lock);
737
738 /* Enter it into the pool. */
739 entpool_enter(&E->pool, buf, len);
740
741 /*
742 * Decide whether to notify reseed -- we will do so if either:
743 * (a) we transition from partial entropy to full entropy, or
744 * (b) we get a batch of full entropy all at once.
745 */
746 notify |= (E->needed && E->needed <= nbits);
747 notify |= (nbits >= ENTROPY_CAPACITY*NBBY);
748
749 /* Subtract from the needed count and notify if appropriate. */
750 E->needed -= MIN(E->needed, nbits);
751 if (notify) {
752 entropy_notify();
753 entropy_immediate_evcnt.ev_count++;
754 }
755
756 if (E->stage >= ENTROPY_WARM)
757 mutex_exit(&E->lock);
758 }
759
760 /*
761 * entropy_enter(buf, len, nbits)
762 *
763 * Enter len bytes of data from buf into the system's entropy
764 * pool, stirring as necessary when the internal buffer fills up.
765 * nbits is a lower bound on the number of bits of entropy in the
766 * process that led to this sample.
767 */
768 static void
769 entropy_enter(const void *buf, size_t len, unsigned nbits)
770 {
771 struct entropy_cpu *ec;
772 uint32_t pending;
773 int s;
774
775 KASSERTMSG(!cpu_intr_p(),
776 "use entropy_enter_intr from interrupt context");
777 KASSERTMSG(howmany(nbits, NBBY) <= len,
778 "impossible entropy rate: %u bits in %zu-byte string", nbits, len);
779
780 /* If it's too early after boot, just use entropy_enter_early. */
781 if (__predict_false(E->stage < ENTROPY_HOT)) {
782 entropy_enter_early(buf, len, nbits);
783 return;
784 }
785
786 /*
787 * Acquire the per-CPU state, blocking soft interrupts and
788 * causing hard interrupts to drop samples on the floor.
789 */
790 ec = percpu_getref(entropy_percpu);
791 s = splsoftserial();
792 KASSERT(!ec->ec_locked);
793 ec->ec_locked = true;
794 __insn_barrier();
795
796 /* Enter into the per-CPU pool. */
797 entpool_enter(ec->ec_pool, buf, len);
798
799 /* Count up what we can add. */
800 pending = ec->ec_pending;
801 pending += MIN(ENTROPY_CAPACITY*NBBY - pending, nbits);
802 atomic_store_relaxed(&ec->ec_pending, pending);
803
804 /* Consolidate globally if appropriate based on what we added. */
805 entropy_account_cpu(ec);
806
807 /* Release the per-CPU state. */
808 KASSERT(ec->ec_locked);
809 __insn_barrier();
810 ec->ec_locked = false;
811 splx(s);
812 percpu_putref(entropy_percpu);
813 }
814
815 /*
816 * entropy_enter_intr(buf, len, nbits)
817 *
818 * Enter up to len bytes of data from buf into the system's
819 * entropy pool without stirring. nbits is a lower bound on the
820 * number of bits of entropy in the process that led to this
821 * sample. If the sample could be entered completely, assume
822 * nbits of entropy pending; otherwise assume none, since we don't
823 * know whether some parts of the sample are constant, for
824 * instance. Schedule a softint to stir the entropy pool if
825 * needed. Return true if used fully, false if truncated at all.
826 *
827 * Using this in thread context will work, but you might as well
828 * use entropy_enter in that case.
829 */
830 static bool
831 entropy_enter_intr(const void *buf, size_t len, unsigned nbits)
832 {
833 struct entropy_cpu *ec;
834 bool fullyused = false;
835 uint32_t pending;
836
837 KASSERTMSG(howmany(nbits, NBBY) <= len,
838 "impossible entropy rate: %u bits in %zu-byte string", nbits, len);
839
840 /* If it's too early after boot, just use entropy_enter_early. */
841 if (__predict_false(E->stage < ENTROPY_HOT)) {
842 entropy_enter_early(buf, len, nbits);
843 return true;
844 }
845
846 /*
847 * Acquire the per-CPU state. If someone is in the middle of
848 * using it, drop the sample. Otherwise, take the lock so that
849 * higher-priority interrupts will drop their samples.
850 */
851 ec = percpu_getref(entropy_percpu);
852 if (ec->ec_locked)
853 goto out0;
854 ec->ec_locked = true;
855 __insn_barrier();
856
857 /*
858 * Enter as much as we can into the per-CPU pool. If it was
859 * truncated, schedule a softint to stir the pool and stop.
860 */
861 if (!entpool_enter_nostir(ec->ec_pool, buf, len)) {
862 softint_schedule(entropy_sih);
863 goto out1;
864 }
865 fullyused = true;
866
867 /* Count up what we can contribute. */
868 pending = ec->ec_pending;
869 pending += MIN(ENTROPY_CAPACITY*NBBY - pending, nbits);
870 atomic_store_relaxed(&ec->ec_pending, pending);
871
872 /* Schedule a softint if we added anything and it matters. */
873 if (__predict_false((atomic_load_relaxed(&E->needed) != 0) ||
874 atomic_load_relaxed(&entropy_depletion)) &&
875 nbits != 0)
876 softint_schedule(entropy_sih);
877
878 out1: /* Release the per-CPU state. */
879 KASSERT(ec->ec_locked);
880 __insn_barrier();
881 ec->ec_locked = false;
882 out0: percpu_putref(entropy_percpu);
883
884 return fullyused;
885 }
886
887 /*
888 * entropy_softintr(cookie)
889 *
890 * Soft interrupt handler for entering entropy. Takes care of
891 * stirring the local CPU's entropy pool if it filled up during
892 * hard interrupts, and promptly crediting entropy from the local
893 * CPU's entropy pool to the global entropy pool if needed.
894 */
895 static void
896 entropy_softintr(void *cookie)
897 {
898 struct entropy_cpu *ec;
899
900 /*
901 * Acquire the per-CPU state. Other users can lock this only
902 * while soft interrupts are blocked. Cause hard interrupts to
903 * drop samples on the floor.
904 */
905 ec = percpu_getref(entropy_percpu);
906 KASSERT(!ec->ec_locked);
907 ec->ec_locked = true;
908 __insn_barrier();
909
910 /* Count statistics. */
911 ec->ec_softint_evcnt->ev_count++;
912
913 /* Stir the pool if necessary. */
914 entpool_stir(ec->ec_pool);
915
916 /* Consolidate globally if appropriate based on what we added. */
917 entropy_account_cpu(ec);
918
919 /* Release the per-CPU state. */
920 KASSERT(ec->ec_locked);
921 __insn_barrier();
922 ec->ec_locked = false;
923 percpu_putref(entropy_percpu);
924 }
925
926 /*
927 * entropy_thread(cookie)
928 *
929 * Handle any asynchronous entropy housekeeping.
930 */
931 static void
932 entropy_thread(void *cookie)
933 {
934 bool consolidate;
935
936 for (;;) {
937 /*
938 * Wait until there's full entropy somewhere among the
939 * CPUs, as confirmed at most once per minute, or
940 * someone wants to consolidate.
941 */
942 if (entropy_pending() >= ENTROPY_CAPACITY*NBBY) {
943 consolidate = true;
944 } else {
945 mutex_enter(&E->lock);
946 if (!E->consolidate)
947 cv_timedwait(&E->cv, &E->lock, 60*hz);
948 consolidate = E->consolidate;
949 E->consolidate = false;
950 mutex_exit(&E->lock);
951 }
952
953 if (consolidate) {
954 /* Do it. */
955 entropy_do_consolidate();
956
957 /* Mitigate abuse. */
958 kpause("entropy", false, hz, NULL);
959 }
960 }
961 }
962
963 /*
964 * entropy_pending()
965 *
966 * Count up the amount of entropy pending on other CPUs.
967 */
968 static uint32_t
969 entropy_pending(void)
970 {
971 uint32_t pending = 0;
972
973 percpu_foreach(entropy_percpu, &entropy_pending_cpu, &pending);
974 return pending;
975 }
976
977 static void
978 entropy_pending_cpu(void *ptr, void *cookie, struct cpu_info *ci)
979 {
980 struct entropy_cpu *ec = ptr;
981 uint32_t *pendingp = cookie;
982 uint32_t cpu_pending;
983
984 cpu_pending = atomic_load_relaxed(&ec->ec_pending);
985 *pendingp += MIN(ENTROPY_CAPACITY*NBBY - *pendingp, cpu_pending);
986 }
987
988 /*
989 * entropy_do_consolidate()
990 *
991 * Issue a cross-call to gather entropy on all CPUs and advance
992 * the entropy epoch.
993 */
994 static void
995 entropy_do_consolidate(void)
996 {
997 static const struct timeval interval = {.tv_sec = 60, .tv_usec = 0};
998 static struct timeval lasttime; /* serialized by E->lock */
999 struct entpool pool;
1000 uint8_t buf[ENTPOOL_CAPACITY];
1001 unsigned diff;
1002 uint64_t ticket;
1003
1004 /* Gather entropy on all CPUs into a temporary pool. */
1005 memset(&pool, 0, sizeof pool);
1006 ticket = xc_broadcast(0, &entropy_consolidate_xc, &pool, NULL);
1007 xc_wait(ticket);
1008
1009 /* Acquire the lock to notify waiters. */
1010 mutex_enter(&E->lock);
1011
1012 /* Count another consolidation. */
1013 entropy_consolidate_evcnt.ev_count++;
1014
1015 /* Note when we last consolidated, i.e. now. */
1016 E->timestamp = time_uptime;
1017
1018 /* Mix what we gathered into the global pool. */
1019 entpool_extract(&pool, buf, sizeof buf);
1020 entpool_enter(&E->pool, buf, sizeof buf);
1021 explicit_memset(&pool, 0, sizeof pool);
1022
1023 /* Count the entropy that was gathered. */
1024 diff = MIN(E->needed, E->pending);
1025 atomic_store_relaxed(&E->needed, E->needed - diff);
1026 E->pending -= diff;
1027 if (__predict_false(E->needed > 0)) {
1028 if (ratecheck(&lasttime, &interval))
1029 printf("entropy: WARNING:"
1030 " consolidating less than full entropy\n");
1031 }
1032
1033 /* Advance the epoch and notify waiters. */
1034 entropy_notify();
1035
1036 /* Release the lock. */
1037 mutex_exit(&E->lock);
1038 }
1039
1040 /*
1041 * entropy_consolidate_xc(vpool, arg2)
1042 *
1043 * Extract output from the local CPU's input pool and enter it
1044 * into a temporary pool passed as vpool.
1045 */
1046 static void
1047 entropy_consolidate_xc(void *vpool, void *arg2 __unused)
1048 {
1049 struct entpool *pool = vpool;
1050 struct entropy_cpu *ec;
1051 uint8_t buf[ENTPOOL_CAPACITY];
1052 uint32_t extra[7];
1053 unsigned i = 0;
1054 int s;
1055
1056 /* Grab CPU number and cycle counter to mix extra into the pool. */
1057 extra[i++] = cpu_number();
1058 extra[i++] = entropy_timer();
1059
1060 /*
1061 * Acquire the per-CPU state, blocking soft interrupts and
1062 * discarding entropy in hard interrupts, so that we can
1063 * extract from the per-CPU pool.
1064 */
1065 ec = percpu_getref(entropy_percpu);
1066 s = splsoftserial();
1067 KASSERT(!ec->ec_locked);
1068 ec->ec_locked = true;
1069 __insn_barrier();
1070 extra[i++] = entropy_timer();
1071
1072 /* Extract the data and count it no longer pending. */
1073 entpool_extract(ec->ec_pool, buf, sizeof buf);
1074 atomic_store_relaxed(&ec->ec_pending, 0);
1075 extra[i++] = entropy_timer();
1076
1077 /* Release the per-CPU state. */
1078 KASSERT(ec->ec_locked);
1079 __insn_barrier();
1080 ec->ec_locked = false;
1081 splx(s);
1082 percpu_putref(entropy_percpu);
1083 extra[i++] = entropy_timer();
1084
1085 /*
1086 * Copy over statistics, and enter the per-CPU extract and the
1087 * extra timing into the temporary pool, under the global lock.
1088 */
1089 mutex_enter(&E->lock);
1090 extra[i++] = entropy_timer();
1091 entpool_enter(pool, buf, sizeof buf);
1092 explicit_memset(buf, 0, sizeof buf);
1093 extra[i++] = entropy_timer();
1094 KASSERT(i == __arraycount(extra));
1095 entpool_enter(pool, extra, sizeof extra);
1096 explicit_memset(extra, 0, sizeof extra);
1097 mutex_exit(&E->lock);
1098 }
1099
1100 /*
1101 * entropy_notify()
1102 *
1103 * Caller just contributed entropy to the global pool. Advance
1104 * the entropy epoch and notify waiters.
1105 *
1106 * Caller must hold the global entropy lock. Except for the
1107 * `sysctl -w kern.entropy.consolidate=1` trigger, the caller must
1108 * have just have transitioned from partial entropy to full
1109 * entropy -- E->needed should be zero now.
1110 */
1111 static void
1112 entropy_notify(void)
1113 {
1114 static const struct timeval interval = {.tv_sec = 60, .tv_usec = 0};
1115 static struct timeval lasttime; /* serialized by E->lock */
1116 unsigned epoch;
1117
1118 KASSERT(E->stage == ENTROPY_COLD || mutex_owned(&E->lock));
1119
1120 /*
1121 * If this is the first time, print a message to the console
1122 * that we're ready so operators can compare it to the timing
1123 * of other events.
1124 */
1125 if (__predict_false(!rnd_initial_entropy) && E->needed == 0) {
1126 printf("entropy: ready\n");
1127 rnd_initial_entropy = 1;
1128 }
1129
1130 /* Set the epoch; roll over from UINTMAX-1 to 1. */
1131 if (__predict_true(!atomic_load_relaxed(&entropy_depletion)) ||
1132 ratecheck(&lasttime, &interval)) {
1133 epoch = E->epoch + 1;
1134 if (epoch == 0 || epoch == (unsigned)-1)
1135 epoch = 1;
1136 atomic_store_relaxed(&E->epoch, epoch);
1137 }
1138
1139 /* Notify waiters. */
1140 if (E->stage >= ENTROPY_WARM) {
1141 cv_broadcast(&E->cv);
1142 selnotify(&E->selq, POLLIN|POLLRDNORM, NOTE_SUBMIT);
1143 }
1144
1145 /* Count another notification. */
1146 entropy_notify_evcnt.ev_count++;
1147 }
1148
1149 /*
1150 * entropy_consolidate()
1151 *
1152 * Trigger entropy consolidation and wait for it to complete.
1153 *
1154 * This should be used sparingly, not periodically -- requiring
1155 * conscious intervention by the operator or a clear policy
1156 * decision. Otherwise, the kernel will automatically consolidate
1157 * when enough entropy has been gathered into per-CPU pools to
1158 * transition to full entropy.
1159 */
1160 void
1161 entropy_consolidate(void)
1162 {
1163 uint64_t ticket;
1164 int error;
1165
1166 KASSERT(E->stage == ENTROPY_HOT);
1167
1168 mutex_enter(&E->lock);
1169 ticket = entropy_consolidate_evcnt.ev_count;
1170 E->consolidate = true;
1171 cv_broadcast(&E->cv);
1172 while (ticket == entropy_consolidate_evcnt.ev_count) {
1173 error = cv_wait_sig(&E->cv, &E->lock);
1174 if (error)
1175 break;
1176 }
1177 mutex_exit(&E->lock);
1178 }
1179
1180 /*
1181 * sysctl -w kern.entropy.consolidate=1
1182 *
1183 * Trigger entropy consolidation and wait for it to complete.
1184 * Writable only by superuser. This, writing to /dev/random, and
1185 * ioctl(RNDADDDATA) are the only ways for the system to
1186 * consolidate entropy if the operator knows something the kernel
1187 * doesn't about how unpredictable the pending entropy pools are.
1188 */
1189 static int
1190 sysctl_entropy_consolidate(SYSCTLFN_ARGS)
1191 {
1192 struct sysctlnode node = *rnode;
1193 int arg;
1194 int error;
1195
1196 KASSERT(E->stage == ENTROPY_HOT);
1197
1198 node.sysctl_data = &arg;
1199 error = sysctl_lookup(SYSCTLFN_CALL(&node));
1200 if (error || newp == NULL)
1201 return error;
1202 if (arg)
1203 entropy_consolidate();
1204
1205 return error;
1206 }
1207
1208 /*
1209 * sysctl -w kern.entropy.gather=1
1210 *
1211 * Trigger gathering entropy from all on-demand sources, and wait
1212 * for synchronous sources (but not asynchronous sources) to
1213 * complete. Writable only by superuser.
1214 */
1215 static int
1216 sysctl_entropy_gather(SYSCTLFN_ARGS)
1217 {
1218 struct sysctlnode node = *rnode;
1219 int arg;
1220 int error;
1221
1222 KASSERT(E->stage == ENTROPY_HOT);
1223
1224 node.sysctl_data = &arg;
1225 error = sysctl_lookup(SYSCTLFN_CALL(&node));
1226 if (error || newp == NULL)
1227 return error;
1228 if (arg) {
1229 mutex_enter(&E->lock);
1230 entropy_request(ENTROPY_CAPACITY);
1231 mutex_exit(&E->lock);
1232 }
1233
1234 return 0;
1235 }
1236
1237 /*
1238 * entropy_extract(buf, len, flags)
1239 *
1240 * Extract len bytes from the global entropy pool into buf.
1241 *
1242 * Flags may have:
1243 *
1244 * ENTROPY_WAIT Wait for entropy if not available yet.
1245 * ENTROPY_SIG Allow interruption by a signal during wait.
1246 * ENTROPY_HARDFAIL Either fill the buffer with full entropy,
1247 * or fail without filling it at all.
1248 *
1249 * Return zero on success, or error on failure:
1250 *
1251 * EWOULDBLOCK No entropy and ENTROPY_WAIT not set.
1252 * EINTR/ERESTART No entropy, ENTROPY_SIG set, and interrupted.
1253 *
1254 * If ENTROPY_WAIT is set, allowed only in thread context. If
1255 * ENTROPY_WAIT is not set, allowed up to IPL_VM. (XXX That's
1256 * awfully high... Do we really need it in hard interrupts? This
1257 * arises from use of cprng_strong(9).)
1258 */
1259 int
1260 entropy_extract(void *buf, size_t len, int flags)
1261 {
1262 static const struct timeval interval = {.tv_sec = 60, .tv_usec = 0};
1263 static struct timeval lasttime; /* serialized by E->lock */
1264 int error;
1265
1266 if (ISSET(flags, ENTROPY_WAIT)) {
1267 ASSERT_SLEEPABLE();
1268 KASSERTMSG(E->stage >= ENTROPY_WARM,
1269 "can't wait for entropy until warm");
1270 }
1271
1272 /* Acquire the global lock to get at the global pool. */
1273 if (E->stage >= ENTROPY_WARM)
1274 mutex_enter(&E->lock);
1275
1276 /* Count up request for entropy in interrupt context. */
1277 if (cpu_intr_p())
1278 entropy_extract_intr_evcnt.ev_count++;
1279
1280 /* Wait until there is enough entropy in the system. */
1281 error = 0;
1282 while (E->needed) {
1283 /* Ask for more, synchronously if possible. */
1284 entropy_request(len);
1285
1286 /* If we got enough, we're done. */
1287 if (E->needed == 0) {
1288 KASSERT(error == 0);
1289 break;
1290 }
1291
1292 /* If not waiting, stop here. */
1293 if (!ISSET(flags, ENTROPY_WAIT)) {
1294 error = EWOULDBLOCK;
1295 break;
1296 }
1297
1298 /* Wait for some entropy to come in and try again. */
1299 KASSERT(E->stage >= ENTROPY_WARM);
1300 printf("entropy: pid %d (%s) blocking due to lack of entropy\n",
1301 curproc->p_pid, curproc->p_comm);
1302
1303 if (ISSET(flags, ENTROPY_SIG)) {
1304 error = cv_wait_sig(&E->cv, &E->lock);
1305 if (error)
1306 break;
1307 } else {
1308 cv_wait(&E->cv, &E->lock);
1309 }
1310 }
1311
1312 /*
1313 * Count failure -- but fill the buffer nevertheless, unless
1314 * the caller specified ENTROPY_HARDFAIL.
1315 */
1316 if (error) {
1317 if (ISSET(flags, ENTROPY_HARDFAIL))
1318 goto out;
1319 entropy_extract_fail_evcnt.ev_count++;
1320 }
1321
1322 /*
1323 * Report a warning if we have never yet reached full entropy.
1324 * This is the only case where we consider entropy to be
1325 * `depleted' without kern.entropy.depletion enabled -- when we
1326 * only have partial entropy, an adversary may be able to
1327 * narrow the state of the pool down to a small number of
1328 * possibilities; the output then enables them to confirm a
1329 * guess, reducing its entropy from the adversary's perspective
1330 * to zero.
1331 */
1332 if (__predict_false(E->epoch == (unsigned)-1)) {
1333 if (ratecheck(&lasttime, &interval))
1334 printf("entropy: WARNING:"
1335 " extracting entropy too early\n");
1336 atomic_store_relaxed(&E->needed, ENTROPY_CAPACITY*NBBY);
1337 }
1338
1339 /* Extract data from the pool, and `deplete' if we're doing that. */
1340 entpool_extract(&E->pool, buf, len);
1341 if (__predict_false(atomic_load_relaxed(&entropy_depletion)) &&
1342 error == 0) {
1343 unsigned cost = MIN(len, ENTROPY_CAPACITY)*NBBY;
1344
1345 atomic_store_relaxed(&E->needed,
1346 E->needed + MIN(ENTROPY_CAPACITY*NBBY - E->needed, cost));
1347 entropy_deplete_evcnt.ev_count++;
1348 }
1349
1350 out: /* Release the global lock and return the error. */
1351 if (E->stage >= ENTROPY_WARM)
1352 mutex_exit(&E->lock);
1353 return error;
1354 }
1355
1356 /*
1357 * entropy_poll(events)
1358 *
1359 * Return the subset of events ready, and if it is not all of
1360 * events, record curlwp as waiting for entropy.
1361 */
1362 int
1363 entropy_poll(int events)
1364 {
1365 int revents = 0;
1366
1367 KASSERT(E->stage >= ENTROPY_WARM);
1368
1369 /* Always ready for writing. */
1370 revents |= events & (POLLOUT|POLLWRNORM);
1371
1372 /* Narrow it down to reads. */
1373 events &= POLLIN|POLLRDNORM;
1374 if (events == 0)
1375 return revents;
1376
1377 /*
1378 * If we have reached full entropy and we're not depleting
1379 * entropy, we are forever ready.
1380 */
1381 if (__predict_true(atomic_load_relaxed(&E->needed) == 0) &&
1382 __predict_true(!atomic_load_relaxed(&entropy_depletion)))
1383 return revents | events;
1384
1385 /*
1386 * Otherwise, check whether we need entropy under the lock. If
1387 * we don't, we're ready; if we do, add ourselves to the queue.
1388 */
1389 mutex_enter(&E->lock);
1390 if (E->needed == 0)
1391 revents |= events;
1392 else
1393 selrecord(curlwp, &E->selq);
1394 mutex_exit(&E->lock);
1395
1396 return revents;
1397 }
1398
1399 /*
1400 * filt_entropy_read_detach(kn)
1401 *
1402 * struct filterops::f_detach callback for entropy read events:
1403 * remove kn from the list of waiters.
1404 */
1405 static void
1406 filt_entropy_read_detach(struct knote *kn)
1407 {
1408
1409 KASSERT(E->stage >= ENTROPY_WARM);
1410
1411 mutex_enter(&E->lock);
1412 SLIST_REMOVE(&E->selq.sel_klist, kn, knote, kn_selnext);
1413 mutex_exit(&E->lock);
1414 }
1415
1416 /*
1417 * filt_entropy_read_event(kn, hint)
1418 *
1419 * struct filterops::f_event callback for entropy read events:
1420 * poll for entropy. Caller must hold the global entropy lock if
1421 * hint is NOTE_SUBMIT, and must not if hint is not NOTE_SUBMIT.
1422 */
1423 static int
1424 filt_entropy_read_event(struct knote *kn, long hint)
1425 {
1426 int ret;
1427
1428 KASSERT(E->stage >= ENTROPY_WARM);
1429
1430 /* Acquire the lock, if caller is outside entropy subsystem. */
1431 if (hint == NOTE_SUBMIT)
1432 KASSERT(mutex_owned(&E->lock));
1433 else
1434 mutex_enter(&E->lock);
1435
1436 /*
1437 * If we still need entropy, can't read anything; if not, can
1438 * read arbitrarily much.
1439 */
1440 if (E->needed != 0) {
1441 ret = 0;
1442 } else {
1443 if (atomic_load_relaxed(&entropy_depletion))
1444 kn->kn_data = ENTROPY_CAPACITY*NBBY;
1445 else
1446 kn->kn_data = MIN(INT64_MAX, SSIZE_MAX);
1447 ret = 1;
1448 }
1449
1450 /* Release the lock, if caller is outside entropy subsystem. */
1451 if (hint == NOTE_SUBMIT)
1452 KASSERT(mutex_owned(&E->lock));
1453 else
1454 mutex_exit(&E->lock);
1455
1456 return ret;
1457 }
1458
1459 static const struct filterops entropy_read_filtops = {
1460 .f_isfd = 1, /* XXX Makes sense only for /dev/u?random. */
1461 .f_attach = NULL,
1462 .f_detach = filt_entropy_read_detach,
1463 .f_event = filt_entropy_read_event,
1464 };
1465
1466 /*
1467 * entropy_kqfilter(kn)
1468 *
1469 * Register kn to receive entropy event notifications. May be
1470 * EVFILT_READ or EVFILT_WRITE; anything else yields EINVAL.
1471 */
1472 int
1473 entropy_kqfilter(struct knote *kn)
1474 {
1475
1476 KASSERT(E->stage >= ENTROPY_WARM);
1477
1478 switch (kn->kn_filter) {
1479 case EVFILT_READ:
1480 /* Enter into the global select queue. */
1481 mutex_enter(&E->lock);
1482 kn->kn_fop = &entropy_read_filtops;
1483 SLIST_INSERT_HEAD(&E->selq.sel_klist, kn, kn_selnext);
1484 mutex_exit(&E->lock);
1485 return 0;
1486 case EVFILT_WRITE:
1487 /* Can always dump entropy into the system. */
1488 kn->kn_fop = &seltrue_filtops;
1489 return 0;
1490 default:
1491 return EINVAL;
1492 }
1493 }
1494
1495 /*
1496 * rndsource_setcb(rs, get, getarg)
1497 *
1498 * Set the request callback for the entropy source rs, if it can
1499 * provide entropy on demand. Must precede rnd_attach_source.
1500 */
1501 void
1502 rndsource_setcb(struct krndsource *rs, void (*get)(size_t, void *),
1503 void *getarg)
1504 {
1505
1506 rs->get = get;
1507 rs->getarg = getarg;
1508 }
1509
1510 /*
1511 * rnd_attach_source(rs, name, type, flags)
1512 *
1513 * Attach the entropy source rs. Must be done after
1514 * rndsource_setcb, if any, and before any calls to rnd_add_data.
1515 */
1516 void
1517 rnd_attach_source(struct krndsource *rs, const char *name, uint32_t type,
1518 uint32_t flags)
1519 {
1520 uint32_t extra[4];
1521 unsigned i = 0;
1522
1523 /* Grab cycle counter to mix extra into the pool. */
1524 extra[i++] = entropy_timer();
1525
1526 /*
1527 * Apply some standard flags:
1528 *
1529 * - We do not bother with network devices by default, for
1530 * hysterical raisins (perhaps: because it is often the case
1531 * that an adversary can influence network packet timings).
1532 */
1533 switch (type) {
1534 case RND_TYPE_NET:
1535 flags |= RND_FLAG_NO_COLLECT;
1536 break;
1537 }
1538
1539 /* Sanity-check the callback if RND_FLAG_HASCB is set. */
1540 KASSERT(!ISSET(flags, RND_FLAG_HASCB) || rs->get != NULL);
1541
1542 /* Initialize the random source. */
1543 memset(rs->name, 0, sizeof(rs->name)); /* paranoia */
1544 strlcpy(rs->name, name, sizeof(rs->name));
1545 rs->total = 0;
1546 rs->type = type;
1547 rs->flags = flags;
1548 if (E->stage >= ENTROPY_WARM)
1549 rs->state = percpu_alloc(sizeof(struct rndsource_cpu));
1550 extra[i++] = entropy_timer();
1551
1552 /* Wire it into the global list of random sources. */
1553 if (E->stage >= ENTROPY_WARM)
1554 mutex_enter(&E->lock);
1555 LIST_INSERT_HEAD(&E->sources, rs, list);
1556 if (E->stage >= ENTROPY_WARM)
1557 mutex_exit(&E->lock);
1558 extra[i++] = entropy_timer();
1559
1560 /* Request that it provide entropy ASAP, if we can. */
1561 if (ISSET(flags, RND_FLAG_HASCB))
1562 (*rs->get)(ENTROPY_CAPACITY, rs->getarg);
1563 extra[i++] = entropy_timer();
1564
1565 /* Mix the extra into the pool. */
1566 KASSERT(i == __arraycount(extra));
1567 entropy_enter(extra, sizeof extra, 0);
1568 explicit_memset(extra, 0, sizeof extra);
1569 }
1570
1571 /*
1572 * rnd_detach_source(rs)
1573 *
1574 * Detach the entropy source rs. May sleep waiting for users to
1575 * drain. Further use is not allowed.
1576 */
1577 void
1578 rnd_detach_source(struct krndsource *rs)
1579 {
1580
1581 /*
1582 * If we're cold (shouldn't happen, but hey), just remove it
1583 * from the list -- there's nothing allocated.
1584 */
1585 if (E->stage == ENTROPY_COLD) {
1586 LIST_REMOVE(rs, list);
1587 return;
1588 }
1589
1590 /* We may have to wait for entropy_request. */
1591 ASSERT_SLEEPABLE();
1592
1593 /* Wait until the source list is not in use, and remove it. */
1594 mutex_enter(&E->lock);
1595 while (E->sourcelock)
1596 cv_wait(&E->cv, &E->lock);
1597 LIST_REMOVE(rs, list);
1598 mutex_exit(&E->lock);
1599
1600 /* Free the per-CPU data. */
1601 percpu_free(rs->state, sizeof(struct rndsource_cpu));
1602 }
1603
1604 /*
1605 * rnd_lock_sources()
1606 *
1607 * Prevent changes to the list of rndsources while we iterate it.
1608 * Interruptible. Caller must hold the global entropy lock. If
1609 * successful, no rndsource will go away until rnd_unlock_sources
1610 * even while the caller releases the global entropy lock.
1611 */
1612 static int
1613 rnd_lock_sources(void)
1614 {
1615 int error;
1616
1617 KASSERT(mutex_owned(&E->lock));
1618
1619 while (E->sourcelock) {
1620 error = cv_wait_sig(&E->cv, &E->lock);
1621 if (error)
1622 return error;
1623 }
1624
1625 E->sourcelock = curlwp;
1626 return 0;
1627 }
1628
1629 /*
1630 * rnd_trylock_sources()
1631 *
1632 * Try to lock the list of sources, but if it's already locked,
1633 * fail. Caller must hold the global entropy lock. If
1634 * successful, no rndsource will go away until rnd_unlock_sources
1635 * even while the caller releases the global entropy lock.
1636 */
1637 static bool
1638 rnd_trylock_sources(void)
1639 {
1640
1641 KASSERT(E->stage == ENTROPY_COLD || mutex_owned(&E->lock));
1642
1643 if (E->sourcelock)
1644 return false;
1645 E->sourcelock = curlwp;
1646 return true;
1647 }
1648
1649 /*
1650 * rnd_unlock_sources()
1651 *
1652 * Unlock the list of sources after rnd_lock_sources or
1653 * rnd_trylock_sources. Caller must hold the global entropy lock.
1654 */
1655 static void
1656 rnd_unlock_sources(void)
1657 {
1658
1659 KASSERT(E->stage == ENTROPY_COLD || mutex_owned(&E->lock));
1660
1661 KASSERTMSG(E->sourcelock == curlwp, "lwp %p releasing lock held by %p",
1662 curlwp, E->sourcelock);
1663 E->sourcelock = NULL;
1664 if (E->stage >= ENTROPY_WARM)
1665 cv_broadcast(&E->cv);
1666 }
1667
1668 /*
1669 * rnd_sources_locked()
1670 *
1671 * True if we hold the list of rndsources locked, for diagnostic
1672 * assertions.
1673 */
1674 static bool __diagused
1675 rnd_sources_locked(void)
1676 {
1677
1678 return E->sourcelock == curlwp;
1679 }
1680
1681 /*
1682 * entropy_request(nbytes)
1683 *
1684 * Request nbytes bytes of entropy from all sources in the system.
1685 * OK if we overdo it. Caller must hold the global entropy lock;
1686 * will release and re-acquire it.
1687 */
1688 static void
1689 entropy_request(size_t nbytes)
1690 {
1691 struct krndsource *rs;
1692
1693 KASSERT(E->stage == ENTROPY_COLD || mutex_owned(&E->lock));
1694
1695 /*
1696 * If there is a request in progress, let it proceed.
1697 * Otherwise, note that a request is in progress to avoid
1698 * reentry and to block rnd_detach_source until we're done.
1699 */
1700 if (!rnd_trylock_sources())
1701 return;
1702 entropy_request_evcnt.ev_count++;
1703
1704 /* Clamp to the maximum reasonable request. */
1705 nbytes = MIN(nbytes, ENTROPY_CAPACITY);
1706
1707 /* Walk the list of sources. */
1708 LIST_FOREACH(rs, &E->sources, list) {
1709 /* Skip sources without callbacks. */
1710 if (!ISSET(rs->flags, RND_FLAG_HASCB))
1711 continue;
1712
1713 /*
1714 * Skip sources that are disabled altogether -- we
1715 * would just ignore their samples anyway.
1716 */
1717 if (ISSET(rs->flags, RND_FLAG_NO_COLLECT))
1718 continue;
1719
1720 /* Drop the lock while we call the callback. */
1721 if (E->stage >= ENTROPY_WARM)
1722 mutex_exit(&E->lock);
1723 (*rs->get)(nbytes, rs->getarg);
1724 if (E->stage >= ENTROPY_WARM)
1725 mutex_enter(&E->lock);
1726 }
1727
1728 /* Notify rnd_detach_source that the request is done. */
1729 rnd_unlock_sources();
1730 }
1731
1732 /*
1733 * rnd_add_uint32(rs, value)
1734 *
1735 * Enter 32 bits of data from an entropy source into the pool.
1736 *
1737 * If rs is NULL, may not be called from interrupt context.
1738 *
1739 * If rs is non-NULL, may be called from any context. May drop
1740 * data if called from interrupt context.
1741 */
1742 void
1743 rnd_add_uint32(struct krndsource *rs, uint32_t value)
1744 {
1745
1746 rnd_add_data(rs, &value, sizeof value, 0);
1747 }
1748
1749 void
1750 _rnd_add_uint32(struct krndsource *rs, uint32_t value)
1751 {
1752
1753 rnd_add_data(rs, &value, sizeof value, 0);
1754 }
1755
1756 void
1757 _rnd_add_uint64(struct krndsource *rs, uint64_t value)
1758 {
1759
1760 rnd_add_data(rs, &value, sizeof value, 0);
1761 }
1762
1763 /*
1764 * rnd_add_data(rs, buf, len, entropybits)
1765 *
1766 * Enter data from an entropy source into the pool, with a
1767 * driver's estimate of how much entropy the physical source of
1768 * the data has. If RND_FLAG_NO_ESTIMATE, we ignore the driver's
1769 * estimate and treat it as zero.
1770 *
1771 * If rs is NULL, may not be called from interrupt context.
1772 *
1773 * If rs is non-NULL, may be called from any context. May drop
1774 * data if called from interrupt context.
1775 */
1776 void
1777 rnd_add_data(struct krndsource *rs, const void *buf, uint32_t len,
1778 uint32_t entropybits)
1779 {
1780 uint32_t extra;
1781 uint32_t flags;
1782
1783 KASSERTMSG(howmany(entropybits, NBBY) <= len,
1784 "%s: impossible entropy rate:"
1785 " %"PRIu32" bits in %"PRIu32"-byte string",
1786 rs ? rs->name : "(anonymous)", entropybits, len);
1787
1788 /* If there's no rndsource, just enter the data and time now. */
1789 if (rs == NULL) {
1790 entropy_enter(buf, len, entropybits);
1791 extra = entropy_timer();
1792 entropy_enter(&extra, sizeof extra, 0);
1793 explicit_memset(&extra, 0, sizeof extra);
1794 return;
1795 }
1796
1797 /* Load a snapshot of the flags. Ioctl may change them under us. */
1798 flags = atomic_load_relaxed(&rs->flags);
1799
1800 /*
1801 * Skip if:
1802 * - we're not collecting entropy, or
1803 * - the operator doesn't want to collect entropy from this, or
1804 * - neither data nor timings are being collected from this.
1805 */
1806 if (!atomic_load_relaxed(&entropy_collection) ||
1807 ISSET(flags, RND_FLAG_NO_COLLECT) ||
1808 !ISSET(flags, RND_FLAG_COLLECT_VALUE|RND_FLAG_COLLECT_TIME))
1809 return;
1810
1811 /* If asked, ignore the estimate. */
1812 if (ISSET(flags, RND_FLAG_NO_ESTIMATE))
1813 entropybits = 0;
1814
1815 /* If we are collecting data, enter them. */
1816 if (ISSET(flags, RND_FLAG_COLLECT_VALUE))
1817 rnd_add_data_1(rs, buf, len, entropybits);
1818
1819 /* If we are collecting timings, enter one. */
1820 if (ISSET(flags, RND_FLAG_COLLECT_TIME)) {
1821 extra = entropy_timer();
1822 rnd_add_data_1(rs, &extra, sizeof extra, 0);
1823 }
1824 }
1825
1826 /*
1827 * rnd_add_data_1(rs, buf, len, entropybits)
1828 *
1829 * Internal subroutine to call either entropy_enter_intr, if we're
1830 * in interrupt context, or entropy_enter if not, and to count the
1831 * entropy in an rndsource.
1832 */
1833 static void
1834 rnd_add_data_1(struct krndsource *rs, const void *buf, uint32_t len,
1835 uint32_t entropybits)
1836 {
1837 bool fullyused;
1838
1839 /*
1840 * If we're in interrupt context, use entropy_enter_intr and
1841 * take note of whether it consumed the full sample; if not,
1842 * use entropy_enter, which always consumes the full sample.
1843 */
1844 if (curlwp && cpu_intr_p()) {
1845 fullyused = entropy_enter_intr(buf, len, entropybits);
1846 } else {
1847 entropy_enter(buf, len, entropybits);
1848 fullyused = true;
1849 }
1850
1851 /*
1852 * If we used the full sample, note how many bits were
1853 * contributed from this source.
1854 */
1855 if (fullyused) {
1856 if (E->stage < ENTROPY_HOT) {
1857 if (E->stage >= ENTROPY_WARM)
1858 mutex_enter(&E->lock);
1859 rs->total += MIN(UINT_MAX - rs->total, entropybits);
1860 if (E->stage >= ENTROPY_WARM)
1861 mutex_exit(&E->lock);
1862 } else {
1863 struct rndsource_cpu *rc = percpu_getref(rs->state);
1864 unsigned nbits = rc->rc_nbits;
1865
1866 nbits += MIN(UINT_MAX - nbits, entropybits);
1867 atomic_store_relaxed(&rc->rc_nbits, nbits);
1868 percpu_putref(rs->state);
1869 }
1870 }
1871 }
1872
1873 /*
1874 * rnd_add_data_sync(rs, buf, len, entropybits)
1875 *
1876 * Same as rnd_add_data. Originally used in rndsource callbacks,
1877 * to break an unnecessary cycle; no longer really needed.
1878 */
1879 void
1880 rnd_add_data_sync(struct krndsource *rs, const void *buf, uint32_t len,
1881 uint32_t entropybits)
1882 {
1883
1884 rnd_add_data(rs, buf, len, entropybits);
1885 }
1886
1887 /*
1888 * rndsource_entropybits(rs)
1889 *
1890 * Return approximately the number of bits of entropy that have
1891 * been contributed via rs so far. Approximate if other CPUs may
1892 * be calling rnd_add_data concurrently.
1893 */
1894 static unsigned
1895 rndsource_entropybits(struct krndsource *rs)
1896 {
1897 unsigned nbits = rs->total;
1898
1899 KASSERT(E->stage >= ENTROPY_WARM);
1900 KASSERT(rnd_sources_locked());
1901 percpu_foreach(rs->state, rndsource_entropybits_cpu, &nbits);
1902 return nbits;
1903 }
1904
1905 static void
1906 rndsource_entropybits_cpu(void *ptr, void *cookie, struct cpu_info *ci)
1907 {
1908 struct rndsource_cpu *rc = ptr;
1909 unsigned *nbitsp = cookie;
1910 unsigned cpu_nbits;
1911
1912 cpu_nbits = atomic_load_relaxed(&rc->rc_nbits);
1913 *nbitsp += MIN(UINT_MAX - *nbitsp, cpu_nbits);
1914 }
1915
1916 /*
1917 * rndsource_to_user(rs, urs)
1918 *
1919 * Copy a description of rs out to urs for userland.
1920 */
1921 static void
1922 rndsource_to_user(struct krndsource *rs, rndsource_t *urs)
1923 {
1924
1925 KASSERT(E->stage >= ENTROPY_WARM);
1926 KASSERT(rnd_sources_locked());
1927
1928 /* Avoid kernel memory disclosure. */
1929 memset(urs, 0, sizeof(*urs));
1930
1931 CTASSERT(sizeof(urs->name) == sizeof(rs->name));
1932 strlcpy(urs->name, rs->name, sizeof(urs->name));
1933 urs->total = rndsource_entropybits(rs);
1934 urs->type = rs->type;
1935 urs->flags = atomic_load_relaxed(&rs->flags);
1936 }
1937
1938 /*
1939 * rndsource_to_user_est(rs, urse)
1940 *
1941 * Copy a description of rs and estimation statistics out to urse
1942 * for userland.
1943 */
1944 static void
1945 rndsource_to_user_est(struct krndsource *rs, rndsource_est_t *urse)
1946 {
1947
1948 KASSERT(E->stage >= ENTROPY_WARM);
1949 KASSERT(rnd_sources_locked());
1950
1951 /* Avoid kernel memory disclosure. */
1952 memset(urse, 0, sizeof(*urse));
1953
1954 /* Copy out the rndsource description. */
1955 rndsource_to_user(rs, &urse->rt);
1956
1957 /* Zero out the statistics because we don't do estimation. */
1958 urse->dt_samples = 0;
1959 urse->dt_total = 0;
1960 urse->dv_samples = 0;
1961 urse->dv_total = 0;
1962 }
1963
1964 /*
1965 * entropy_reset_xc(arg1, arg2)
1966 *
1967 * Reset the current CPU's pending entropy to zero.
1968 */
1969 static void
1970 entropy_reset_xc(void *arg1 __unused, void *arg2 __unused)
1971 {
1972 uint32_t extra = entropy_timer();
1973 struct entropy_cpu *ec;
1974 int s;
1975
1976 /*
1977 * Acquire the per-CPU state, blocking soft interrupts and
1978 * causing hard interrupts to drop samples on the floor.
1979 */
1980 ec = percpu_getref(entropy_percpu);
1981 s = splsoftserial();
1982 KASSERT(!ec->ec_locked);
1983 ec->ec_locked = true;
1984 __insn_barrier();
1985
1986 /* Zero the pending count and enter a cycle count for fun. */
1987 ec->ec_pending = 0;
1988 entpool_enter(ec->ec_pool, &extra, sizeof extra);
1989
1990 /* Release the per-CPU state. */
1991 KASSERT(ec->ec_locked);
1992 __insn_barrier();
1993 ec->ec_locked = false;
1994 splx(s);
1995 percpu_putref(entropy_percpu);
1996 }
1997
1998 /*
1999 * entropy_ioctl(cmd, data)
2000 *
2001 * Handle various /dev/random ioctl queries.
2002 */
2003 int
2004 entropy_ioctl(unsigned long cmd, void *data)
2005 {
2006 struct krndsource *rs;
2007 bool privileged;
2008 int error;
2009
2010 KASSERT(E->stage >= ENTROPY_WARM);
2011
2012 /* Verify user's authorization to perform the ioctl. */
2013 switch (cmd) {
2014 case RNDGETENTCNT:
2015 case RNDGETPOOLSTAT:
2016 case RNDGETSRCNUM:
2017 case RNDGETSRCNAME:
2018 case RNDGETESTNUM:
2019 case RNDGETESTNAME:
2020 error = kauth_authorize_device(curlwp->l_cred,
2021 KAUTH_DEVICE_RND_GETPRIV, NULL, NULL, NULL, NULL);
2022 break;
2023 case RNDCTL:
2024 error = kauth_authorize_device(curlwp->l_cred,
2025 KAUTH_DEVICE_RND_SETPRIV, NULL, NULL, NULL, NULL);
2026 break;
2027 case RNDADDDATA:
2028 error = kauth_authorize_device(curlwp->l_cred,
2029 KAUTH_DEVICE_RND_ADDDATA, NULL, NULL, NULL, NULL);
2030 /* Ascertain whether the user's inputs should be counted. */
2031 if (kauth_authorize_device(curlwp->l_cred,
2032 KAUTH_DEVICE_RND_ADDDATA_ESTIMATE,
2033 NULL, NULL, NULL, NULL) == 0)
2034 privileged = true;
2035 break;
2036 default: {
2037 /*
2038 * XXX Hack to avoid changing module ABI so this can be
2039 * pulled up. Later, we can just remove the argument.
2040 */
2041 static const struct fileops fops = {
2042 .fo_ioctl = rnd_system_ioctl,
2043 };
2044 struct file f = {
2045 .f_ops = &fops,
2046 };
2047 MODULE_HOOK_CALL(rnd_ioctl_50_hook, (&f, cmd, data),
2048 enosys(), error);
2049 #if defined(_LP64)
2050 if (error == ENOSYS)
2051 MODULE_HOOK_CALL(rnd_ioctl32_50_hook, (&f, cmd, data),
2052 enosys(), error);
2053 #endif
2054 if (error == ENOSYS)
2055 error = ENOTTY;
2056 break;
2057 }
2058 }
2059
2060 /* If anything went wrong with authorization, stop here. */
2061 if (error)
2062 return error;
2063
2064 /* Dispatch on the command. */
2065 switch (cmd) {
2066 case RNDGETENTCNT: { /* Get current entropy count in bits. */
2067 uint32_t *countp = data;
2068
2069 mutex_enter(&E->lock);
2070 *countp = ENTROPY_CAPACITY*NBBY - E->needed;
2071 mutex_exit(&E->lock);
2072
2073 break;
2074 }
2075 case RNDGETPOOLSTAT: { /* Get entropy pool statistics. */
2076 rndpoolstat_t *pstat = data;
2077
2078 mutex_enter(&E->lock);
2079
2080 /* parameters */
2081 pstat->poolsize = ENTPOOL_SIZE/sizeof(uint32_t); /* words */
2082 pstat->threshold = ENTROPY_CAPACITY*1; /* bytes */
2083 pstat->maxentropy = ENTROPY_CAPACITY*NBBY; /* bits */
2084
2085 /* state */
2086 pstat->added = 0; /* XXX total entropy_enter count */
2087 pstat->curentropy = ENTROPY_CAPACITY*NBBY - E->needed;
2088 pstat->removed = 0; /* XXX total entropy_extract count */
2089 pstat->discarded = 0; /* XXX bits of entropy beyond capacity */
2090 pstat->generated = 0; /* XXX bits of data...fabricated? */
2091
2092 mutex_exit(&E->lock);
2093 break;
2094 }
2095 case RNDGETSRCNUM: { /* Get entropy sources by number. */
2096 rndstat_t *stat = data;
2097 uint32_t start = 0, i = 0;
2098
2099 /* Skip if none requested; fail if too many requested. */
2100 if (stat->count == 0)
2101 break;
2102 if (stat->count > RND_MAXSTATCOUNT)
2103 return EINVAL;
2104
2105 /*
2106 * Under the lock, find the first one, copy out as many
2107 * as requested, and report how many we copied out.
2108 */
2109 mutex_enter(&E->lock);
2110 error = rnd_lock_sources();
2111 if (error) {
2112 mutex_exit(&E->lock);
2113 return error;
2114 }
2115 LIST_FOREACH(rs, &E->sources, list) {
2116 if (start++ == stat->start)
2117 break;
2118 }
2119 while (i < stat->count && rs != NULL) {
2120 mutex_exit(&E->lock);
2121 rndsource_to_user(rs, &stat->source[i++]);
2122 mutex_enter(&E->lock);
2123 rs = LIST_NEXT(rs, list);
2124 }
2125 KASSERT(i <= stat->count);
2126 stat->count = i;
2127 rnd_unlock_sources();
2128 mutex_exit(&E->lock);
2129 break;
2130 }
2131 case RNDGETESTNUM: { /* Get sources and estimates by number. */
2132 rndstat_est_t *estat = data;
2133 uint32_t start = 0, i = 0;
2134
2135 /* Skip if none requested; fail if too many requested. */
2136 if (estat->count == 0)
2137 break;
2138 if (estat->count > RND_MAXSTATCOUNT)
2139 return EINVAL;
2140
2141 /*
2142 * Under the lock, find the first one, copy out as many
2143 * as requested, and report how many we copied out.
2144 */
2145 mutex_enter(&E->lock);
2146 error = rnd_lock_sources();
2147 if (error) {
2148 mutex_exit(&E->lock);
2149 return error;
2150 }
2151 LIST_FOREACH(rs, &E->sources, list) {
2152 if (start++ == estat->start)
2153 break;
2154 }
2155 while (i < estat->count && rs != NULL) {
2156 mutex_exit(&E->lock);
2157 rndsource_to_user_est(rs, &estat->source[i++]);
2158 mutex_enter(&E->lock);
2159 rs = LIST_NEXT(rs, list);
2160 }
2161 KASSERT(i <= estat->count);
2162 estat->count = i;
2163 rnd_unlock_sources();
2164 mutex_exit(&E->lock);
2165 break;
2166 }
2167 case RNDGETSRCNAME: { /* Get entropy sources by name. */
2168 rndstat_name_t *nstat = data;
2169 const size_t n = sizeof(rs->name);
2170
2171 CTASSERT(sizeof(rs->name) == sizeof(nstat->name));
2172
2173 /*
2174 * Under the lock, search by name. If found, copy it
2175 * out; if not found, fail with ENOENT.
2176 */
2177 mutex_enter(&E->lock);
2178 error = rnd_lock_sources();
2179 if (error) {
2180 mutex_exit(&E->lock);
2181 return error;
2182 }
2183 LIST_FOREACH(rs, &E->sources, list) {
2184 if (strncmp(rs->name, nstat->name, n) == 0)
2185 break;
2186 }
2187 if (rs != NULL) {
2188 mutex_exit(&E->lock);
2189 rndsource_to_user(rs, &nstat->source);
2190 mutex_enter(&E->lock);
2191 } else {
2192 error = ENOENT;
2193 }
2194 rnd_unlock_sources();
2195 mutex_exit(&E->lock);
2196 break;
2197 }
2198 case RNDGETESTNAME: { /* Get sources and estimates by name. */
2199 rndstat_est_name_t *enstat = data;
2200 const size_t n = sizeof(rs->name);
2201
2202 CTASSERT(sizeof(rs->name) == sizeof(enstat->name));
2203
2204 /*
2205 * Under the lock, search by name. If found, copy it
2206 * out; if not found, fail with ENOENT.
2207 */
2208 mutex_enter(&E->lock);
2209 error = rnd_lock_sources();
2210 if (error) {
2211 mutex_exit(&E->lock);
2212 return error;
2213 }
2214 LIST_FOREACH(rs, &E->sources, list) {
2215 if (strncmp(rs->name, enstat->name, n) == 0)
2216 break;
2217 }
2218 if (rs != NULL) {
2219 mutex_exit(&E->lock);
2220 rndsource_to_user_est(rs, &enstat->source);
2221 mutex_enter(&E->lock);
2222 } else {
2223 error = ENOENT;
2224 }
2225 rnd_unlock_sources();
2226 mutex_exit(&E->lock);
2227 break;
2228 }
2229 case RNDCTL: { /* Modify entropy source flags. */
2230 rndctl_t *rndctl = data;
2231 const size_t n = sizeof(rs->name);
2232 uint32_t resetflags = RND_FLAG_NO_ESTIMATE|RND_FLAG_NO_COLLECT;
2233 uint32_t flags;
2234 bool reset = false, request = false;
2235
2236 CTASSERT(sizeof(rs->name) == sizeof(rndctl->name));
2237
2238 /* Whitelist the flags that user can change. */
2239 rndctl->mask &= RND_FLAG_NO_ESTIMATE|RND_FLAG_NO_COLLECT;
2240
2241 /*
2242 * For each matching rndsource, either by type if
2243 * specified or by name if not, set the masked flags.
2244 */
2245 mutex_enter(&E->lock);
2246 LIST_FOREACH(rs, &E->sources, list) {
2247 if (rndctl->type != 0xff) {
2248 if (rs->type != rndctl->type)
2249 continue;
2250 } else {
2251 if (strncmp(rs->name, rndctl->name, n) != 0)
2252 continue;
2253 }
2254 flags = rs->flags & ~rndctl->mask;
2255 flags |= rndctl->flags & rndctl->mask;
2256 if ((rs->flags & resetflags) == 0 &&
2257 (flags & resetflags) != 0)
2258 reset = true;
2259 if ((rs->flags ^ flags) & resetflags)
2260 request = true;
2261 atomic_store_relaxed(&rs->flags, flags);
2262 }
2263 mutex_exit(&E->lock);
2264
2265 /*
2266 * If we disabled estimation or collection, nix all the
2267 * pending entropy and set needed to the maximum.
2268 */
2269 if (reset) {
2270 xc_broadcast(0, &entropy_reset_xc, NULL, NULL);
2271 mutex_enter(&E->lock);
2272 E->pending = 0;
2273 atomic_store_relaxed(&E->needed,
2274 ENTROPY_CAPACITY*NBBY);
2275 mutex_exit(&E->lock);
2276 }
2277
2278 /*
2279 * If we changed any of the estimation or collection
2280 * flags, request new samples from everyone -- either
2281 * to make up for what we just lost, or to get new
2282 * samples from what we just added.
2283 */
2284 if (request) {
2285 mutex_enter(&E->lock);
2286 entropy_request(ENTROPY_CAPACITY);
2287 mutex_exit(&E->lock);
2288 }
2289 break;
2290 }
2291 case RNDADDDATA: { /* Enter seed into entropy pool. */
2292 rnddata_t *rdata = data;
2293 unsigned entropybits = 0;
2294
2295 if (!atomic_load_relaxed(&entropy_collection))
2296 break; /* thanks but no thanks */
2297 if (rdata->len > MIN(sizeof(rdata->data), UINT32_MAX/NBBY))
2298 return EINVAL;
2299
2300 /*
2301 * This ioctl serves as the userland alternative a
2302 * bootloader-provided seed -- typically furnished by
2303 * /etc/rc.d/random_seed. We accept the user's entropy
2304 * claim only if
2305 *
2306 * (a) the user is privileged, and
2307 * (b) we have not entered a bootloader seed.
2308 *
2309 * under the assumption that the user may use this to
2310 * load a seed from disk that we have already loaded
2311 * from the bootloader, so we don't double-count it.
2312 */
2313 if (privileged && rdata->entropy && rdata->len) {
2314 mutex_enter(&E->lock);
2315 if (!E->seeded) {
2316 entropybits = MIN(rdata->entropy,
2317 MIN(rdata->len, ENTROPY_CAPACITY)*NBBY);
2318 E->seeded = true;
2319 }
2320 mutex_exit(&E->lock);
2321 }
2322
2323 /* Enter the data and consolidate entropy. */
2324 rnd_add_data(&seed_rndsource, rdata->data, rdata->len,
2325 entropybits);
2326 entropy_consolidate();
2327 break;
2328 }
2329 default:
2330 error = ENOTTY;
2331 }
2332
2333 /* Return any error that may have come up. */
2334 return error;
2335 }
2336
2337 /* Legacy entry points */
2338
2339 void
2340 rnd_seed(void *seed, size_t len)
2341 {
2342
2343 if (len != sizeof(rndsave_t)) {
2344 printf("entropy: invalid seed length: %zu,"
2345 " expected sizeof(rndsave_t) = %zu\n",
2346 len, sizeof(rndsave_t));
2347 return;
2348 }
2349 entropy_seed(seed);
2350 }
2351
2352 void
2353 rnd_init(void)
2354 {
2355
2356 entropy_init();
2357 }
2358
2359 void
2360 rnd_init_softint(void)
2361 {
2362
2363 entropy_init_late();
2364 }
2365
2366 int
2367 rnd_system_ioctl(struct file *fp, unsigned long cmd, void *data)
2368 {
2369
2370 return entropy_ioctl(cmd, data);
2371 }
2372