Home | History | Annotate | Line # | Download | only in kern
kern_entropy.c revision 1.11
      1 /*	$NetBSD: kern_entropy.c,v 1.11 2020/05/06 18:31:05 riastradh Exp $	*/
      2 
      3 /*-
      4  * Copyright (c) 2019 The NetBSD Foundation, Inc.
      5  * All rights reserved.
      6  *
      7  * This code is derived from software contributed to The NetBSD Foundation
      8  * by Taylor R. Campbell.
      9  *
     10  * Redistribution and use in source and binary forms, with or without
     11  * modification, are permitted provided that the following conditions
     12  * are met:
     13  * 1. Redistributions of source code must retain the above copyright
     14  *    notice, this list of conditions and the following disclaimer.
     15  * 2. Redistributions in binary form must reproduce the above copyright
     16  *    notice, this list of conditions and the following disclaimer in the
     17  *    documentation and/or other materials provided with the distribution.
     18  *
     19  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
     20  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
     21  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
     22  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
     23  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
     24  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
     25  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
     26  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
     27  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
     28  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
     29  * POSSIBILITY OF SUCH DAMAGE.
     30  */
     31 
     32 /*
     33  * Entropy subsystem
     34  *
     35  *	* Each CPU maintains a per-CPU entropy pool so that gathering
     36  *	  entropy requires no interprocessor synchronization, except
     37  *	  early at boot when we may be scrambling to gather entropy as
     38  *	  soon as possible.
     39  *
     40  *	  - entropy_enter gathers entropy and never drops it on the
     41  *	    floor, at the cost of sometimes having to do cryptography.
     42  *
     43  *	  - entropy_enter_intr gathers entropy or drops it on the
     44  *	    floor, with low latency.  Work to stir the pool or kick the
     45  *	    housekeeping thread is scheduled in soft interrupts.
     46  *
     47  *	* entropy_enter immediately enters into the global pool if it
     48  *	  can transition to full entropy in one swell foop.  Otherwise,
     49  *	  it defers to a housekeeping thread that consolidates entropy,
     50  *	  but only when the CPUs collectively have full entropy, in
     51  *	  order to mitigate iterative-guessing attacks.
     52  *
     53  *	* The entropy housekeeping thread continues to consolidate
     54  *	  entropy even after we think we have full entropy, in case we
     55  *	  are wrong, but is limited to one discretionary consolidation
     56  *	  per minute, and only when new entropy is actually coming in,
     57  *	  to limit performance impact.
     58  *
     59  *	* The entropy epoch is the number that changes when we
     60  *	  transition from partial entropy to full entropy, so that
     61  *	  users can easily determine when to reseed.  This also
     62  *	  facilitates an operator explicitly causing everything to
     63  *	  reseed by sysctl -w kern.entropy.consolidate=1, e.g. if they
     64  *	  just flipped a coin 256 times and wrote `echo tthhhhhthh... >
     65  *	  /dev/random'.
     66  *
     67  *	* No entropy estimation based on the sample values, which is a
     68  *	  contradiction in terms and a potential source of side
     69  *	  channels.  It is the responsibility of the driver author to
     70  *	  study how predictable the physical source of input can ever
     71  *	  be, and to furnish a lower bound on the amount of entropy it
     72  *	  has.
     73  *
     74  *	* Entropy depletion is available for testing (or if you're into
     75  *	  that sort of thing), with sysctl -w kern.entropy.depletion=1;
     76  *	  the logic to support it is small, to minimize chance of bugs.
     77  */
     78 
     79 #include <sys/cdefs.h>
     80 __KERNEL_RCSID(0, "$NetBSD: kern_entropy.c,v 1.11 2020/05/06 18:31:05 riastradh Exp $");
     81 
     82 #include <sys/param.h>
     83 #include <sys/types.h>
     84 #include <sys/atomic.h>
     85 #include <sys/compat_stub.h>
     86 #include <sys/condvar.h>
     87 #include <sys/cpu.h>
     88 #include <sys/entropy.h>
     89 #include <sys/errno.h>
     90 #include <sys/evcnt.h>
     91 #include <sys/event.h>
     92 #include <sys/file.h>
     93 #include <sys/intr.h>
     94 #include <sys/kauth.h>
     95 #include <sys/kernel.h>
     96 #include <sys/kmem.h>
     97 #include <sys/kthread.h>
     98 #include <sys/module_hook.h>
     99 #include <sys/mutex.h>
    100 #include <sys/percpu.h>
    101 #include <sys/poll.h>
    102 #include <sys/queue.h>
    103 #include <sys/rnd.h>		/* legacy kernel API */
    104 #include <sys/rndio.h>		/* userland ioctl interface */
    105 #include <sys/rndsource.h>	/* kernel rndsource driver API */
    106 #include <sys/select.h>
    107 #include <sys/selinfo.h>
    108 #include <sys/sha1.h>		/* for boot seed checksum */
    109 #include <sys/stdint.h>
    110 #include <sys/sysctl.h>
    111 #include <sys/systm.h>
    112 #include <sys/time.h>
    113 #include <sys/xcall.h>
    114 
    115 #include <lib/libkern/entpool.h>
    116 
    117 #include <machine/limits.h>
    118 
    119 #ifdef __HAVE_CPU_COUNTER
    120 #include <machine/cpu_counter.h>
    121 #endif
    122 
    123 /*
    124  * struct entropy_cpu
    125  *
    126  *	Per-CPU entropy state.  The pool is allocated separately
    127  *	because percpu(9) sometimes moves per-CPU objects around
    128  *	without zeroing them, which would lead to unwanted copies of
    129  *	sensitive secrets.  The evcnt is allocated separately becuase
    130  *	evcnt(9) assumes it stays put in memory.
    131  */
    132 struct entropy_cpu {
    133 	struct evcnt		*ec_softint_evcnt;
    134 	struct entpool		*ec_pool;
    135 	unsigned		ec_pending;
    136 	bool			ec_locked;
    137 };
    138 
    139 /*
    140  * struct rndsource_cpu
    141  *
    142  *	Per-CPU rndsource state.
    143  */
    144 struct rndsource_cpu {
    145 	unsigned		rc_nbits; /* bits of entropy added */
    146 };
    147 
    148 /*
    149  * entropy_global (a.k.a. E for short in this file)
    150  *
    151  *	Global entropy state.  Writes protected by the global lock.
    152  *	Some fields, marked (A), can be read outside the lock, and are
    153  *	maintained with atomic_load/store_relaxed.
    154  */
    155 struct {
    156 	kmutex_t	lock;		/* covers all global state */
    157 	struct entpool	pool;		/* global pool for extraction */
    158 	unsigned	needed;		/* (A) needed globally */
    159 	unsigned	pending;	/* (A) pending in per-CPU pools */
    160 	unsigned	timestamp;	/* (A) time of last consolidation */
    161 	unsigned	epoch;		/* (A) changes when needed -> 0 */
    162 	kcondvar_t	cv;		/* notifies state changes */
    163 	struct selinfo	selq;		/* notifies needed -> 0 */
    164 	struct lwp	*sourcelock;	/* lock on list of sources */
    165 	LIST_HEAD(,krndsource) sources;	/* list of entropy sources */
    166 	enum entropy_stage {
    167 		ENTROPY_COLD = 0, /* single-threaded */
    168 		ENTROPY_WARM,	  /* multi-threaded at boot before CPUs */
    169 		ENTROPY_HOT,	  /* multi-threaded multi-CPU */
    170 	}		stage;
    171 	bool		consolidate;	/* kick thread to consolidate */
    172 	bool		seed_rndsource;	/* true if seed source is attached */
    173 	bool		seeded;		/* true if seed file already loaded */
    174 } entropy_global __cacheline_aligned = {
    175 	/* Fields that must be initialized when the kernel is loaded.  */
    176 	.needed = ENTROPY_CAPACITY*NBBY,
    177 	.epoch = (unsigned)-1,	/* -1 means not yet full entropy */
    178 	.sources = LIST_HEAD_INITIALIZER(entropy_global.sources),
    179 	.stage = ENTROPY_COLD,
    180 };
    181 
    182 #define	E	(&entropy_global)	/* declutter */
    183 
    184 /* Read-mostly globals */
    185 static struct percpu	*entropy_percpu __read_mostly; /* struct entropy_cpu */
    186 static void		*entropy_sih __read_mostly; /* softint handler */
    187 static struct lwp	*entropy_lwp __read_mostly; /* housekeeping thread */
    188 
    189 int rnd_initial_entropy __read_mostly; /* XXX legacy */
    190 
    191 static struct krndsource seed_rndsource __read_mostly;
    192 
    193 /*
    194  * Event counters
    195  *
    196  *	Must be careful with adding these because they can serve as
    197  *	side channels.
    198  */
    199 static struct evcnt entropy_discretionary_evcnt =
    200     EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "entropy", "discretionary");
    201 EVCNT_ATTACH_STATIC(entropy_discretionary_evcnt);
    202 static struct evcnt entropy_immediate_evcnt =
    203     EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "entropy", "immediate");
    204 EVCNT_ATTACH_STATIC(entropy_immediate_evcnt);
    205 static struct evcnt entropy_partial_evcnt =
    206     EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "entropy", "partial");
    207 EVCNT_ATTACH_STATIC(entropy_partial_evcnt);
    208 static struct evcnt entropy_consolidate_evcnt =
    209     EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "entropy", "consolidate");
    210 EVCNT_ATTACH_STATIC(entropy_consolidate_evcnt);
    211 static struct evcnt entropy_extract_intr_evcnt =
    212     EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "entropy", "extract intr");
    213 EVCNT_ATTACH_STATIC(entropy_extract_intr_evcnt);
    214 static struct evcnt entropy_extract_fail_evcnt =
    215     EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "entropy", "extract fail");
    216 EVCNT_ATTACH_STATIC(entropy_extract_fail_evcnt);
    217 static struct evcnt entropy_request_evcnt =
    218     EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "entropy", "request");
    219 EVCNT_ATTACH_STATIC(entropy_request_evcnt);
    220 static struct evcnt entropy_deplete_evcnt =
    221     EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "entropy", "deplete");
    222 EVCNT_ATTACH_STATIC(entropy_deplete_evcnt);
    223 static struct evcnt entropy_notify_evcnt =
    224     EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "entropy", "notify");
    225 EVCNT_ATTACH_STATIC(entropy_notify_evcnt);
    226 
    227 /* Sysctl knobs */
    228 bool	entropy_collection = 1;
    229 bool	entropy_depletion = 0; /* Silly!  */
    230 
    231 static const struct sysctlnode	*entropy_sysctlroot;
    232 static struct sysctllog		*entropy_sysctllog;
    233 
    234 /* Forward declarations */
    235 static void	entropy_init_cpu(void *, void *, struct cpu_info *);
    236 static void	entropy_fini_cpu(void *, void *, struct cpu_info *);
    237 static void	entropy_account_cpu(struct entropy_cpu *);
    238 static void	entropy_enter(const void *, size_t, unsigned);
    239 static bool	entropy_enter_intr(const void *, size_t, unsigned);
    240 static void	entropy_softintr(void *);
    241 static void	entropy_thread(void *);
    242 static uint32_t	entropy_pending(void);
    243 static void	entropy_pending_cpu(void *, void *, struct cpu_info *);
    244 static void	entropy_consolidate(void);
    245 static void	entropy_gather_xc(void *, void *);
    246 static void	entropy_notify(void);
    247 static int	sysctl_entropy_consolidate(SYSCTLFN_ARGS);
    248 static int	sysctl_entropy_gather(SYSCTLFN_ARGS);
    249 static void	filt_entropy_read_detach(struct knote *);
    250 static int	filt_entropy_read_event(struct knote *, long);
    251 static void	entropy_request(size_t);
    252 static void	rnd_add_data_1(struct krndsource *, const void *, uint32_t,
    253 		    uint32_t);
    254 static unsigned	rndsource_entropybits(struct krndsource *);
    255 static void	rndsource_entropybits_cpu(void *, void *, struct cpu_info *);
    256 static void	rndsource_to_user(struct krndsource *, rndsource_t *);
    257 static void	rndsource_to_user_est(struct krndsource *, rndsource_est_t *);
    258 
    259 /*
    260  * curcpu_available()
    261  *
    262  *	True if we can inspect the current CPU.  Early on this may not
    263  *	work.  XXX On most if not all ports, this should work earlier.
    264  */
    265 static inline bool
    266 curcpu_available(void)
    267 {
    268 
    269 	return __predict_true(!cold);
    270 }
    271 
    272 /*
    273  * entropy_timer()
    274  *
    275  *	Cycle counter, time counter, or anything that changes a wee bit
    276  *	unpredictably.
    277  */
    278 static inline uint32_t
    279 entropy_timer(void)
    280 {
    281 	struct bintime bt;
    282 	uint32_t v;
    283 
    284 	/* Very early on, cpu_counter32() may not be available.  */
    285 	if (!curcpu_available())
    286 		return 0;
    287 
    288 	/* If we have a CPU cycle counter, use the low 32 bits.  */
    289 #ifdef __HAVE_CPU_COUNTER
    290 	if (__predict_true(cpu_hascounter()))
    291 		return cpu_counter32();
    292 #endif	/* __HAVE_CPU_COUNTER */
    293 
    294 	/* If we're cold, tough.  Can't binuptime while cold.  */
    295 	if (__predict_false(cold))
    296 		return 0;
    297 
    298 	/* Fold the 128 bits of binuptime into 32 bits.  */
    299 	binuptime(&bt);
    300 	v = bt.frac;
    301 	v ^= bt.frac >> 32;
    302 	v ^= bt.sec;
    303 	v ^= bt.sec >> 32;
    304 	return v;
    305 }
    306 
    307 static void
    308 attach_seed_rndsource(void)
    309 {
    310 
    311 	/*
    312 	 * First called no later than entropy_init, while we are still
    313 	 * single-threaded, so no need for RUN_ONCE.
    314 	 */
    315 	if (E->stage >= ENTROPY_WARM || E->seed_rndsource)
    316 		return;
    317 	rnd_attach_source(&seed_rndsource, "seed", RND_TYPE_UNKNOWN,
    318 	    RND_FLAG_COLLECT_VALUE);
    319 	E->seed_rndsource = true;
    320 }
    321 
    322 /*
    323  * entropy_init()
    324  *
    325  *	Initialize the entropy subsystem.  Panic on failure.
    326  *
    327  *	Requires percpu(9) and sysctl(9) to be initialized.
    328  */
    329 static void
    330 entropy_init(void)
    331 {
    332 	uint32_t extra[2];
    333 	struct krndsource *rs;
    334 	unsigned i = 0;
    335 
    336 	KASSERT(E->stage == ENTROPY_COLD);
    337 
    338 	/* Grab some cycle counts early at boot.  */
    339 	extra[i++] = entropy_timer();
    340 
    341 	/* Run the entropy pool cryptography self-test.  */
    342 	if (entpool_selftest() == -1)
    343 		panic("entropy pool crypto self-test failed");
    344 
    345 	/* Create the sysctl directory.  */
    346 	sysctl_createv(&entropy_sysctllog, 0, NULL, &entropy_sysctlroot,
    347 	    CTLFLAG_PERMANENT, CTLTYPE_NODE, "entropy",
    348 	    SYSCTL_DESCR("Entropy (random number sources) options"),
    349 	    NULL, 0, NULL, 0,
    350 	    CTL_KERN, CTL_CREATE, CTL_EOL);
    351 
    352 	/* Create the sysctl knobs.  */
    353 	/* XXX These shouldn't be writable at securelevel>0.  */
    354 	sysctl_createv(&entropy_sysctllog, 0, &entropy_sysctlroot, NULL,
    355 	    CTLFLAG_PERMANENT|CTLFLAG_READWRITE, CTLTYPE_BOOL, "collection",
    356 	    SYSCTL_DESCR("Automatically collect entropy from hardware"),
    357 	    NULL, 0, &entropy_collection, 0, CTL_CREATE, CTL_EOL);
    358 	sysctl_createv(&entropy_sysctllog, 0, &entropy_sysctlroot, NULL,
    359 	    CTLFLAG_PERMANENT|CTLFLAG_READWRITE, CTLTYPE_BOOL, "depletion",
    360 	    SYSCTL_DESCR("`Deplete' entropy pool when observed"),
    361 	    NULL, 0, &entropy_depletion, 0, CTL_CREATE, CTL_EOL);
    362 	sysctl_createv(&entropy_sysctllog, 0, &entropy_sysctlroot, NULL,
    363 	    CTLFLAG_PERMANENT|CTLFLAG_READWRITE, CTLTYPE_INT, "consolidate",
    364 	    SYSCTL_DESCR("Trigger entropy consolidation now"),
    365 	    sysctl_entropy_consolidate, 0, NULL, 0, CTL_CREATE, CTL_EOL);
    366 	sysctl_createv(&entropy_sysctllog, 0, &entropy_sysctlroot, NULL,
    367 	    CTLFLAG_PERMANENT|CTLFLAG_READWRITE, CTLTYPE_INT, "gather",
    368 	    SYSCTL_DESCR("Trigger entropy gathering from sources now"),
    369 	    sysctl_entropy_gather, 0, NULL, 0, CTL_CREATE, CTL_EOL);
    370 	/* XXX These should maybe not be readable at securelevel>0.  */
    371 	sysctl_createv(&entropy_sysctllog, 0, &entropy_sysctlroot, NULL,
    372 	    CTLFLAG_PERMANENT|CTLFLAG_READONLY|CTLFLAG_PRIVATE, CTLTYPE_INT,
    373 	    "needed", SYSCTL_DESCR("Systemwide entropy deficit"),
    374 	    NULL, 0, &E->needed, 0, CTL_CREATE, CTL_EOL);
    375 	sysctl_createv(&entropy_sysctllog, 0, &entropy_sysctlroot, NULL,
    376 	    CTLFLAG_PERMANENT|CTLFLAG_READONLY|CTLFLAG_PRIVATE, CTLTYPE_INT,
    377 	    "pending", SYSCTL_DESCR("Entropy pending on CPUs"),
    378 	    NULL, 0, &E->pending, 0, CTL_CREATE, CTL_EOL);
    379 	sysctl_createv(&entropy_sysctllog, 0, &entropy_sysctlroot, NULL,
    380 	    CTLFLAG_PERMANENT|CTLFLAG_READONLY|CTLFLAG_PRIVATE, CTLTYPE_INT,
    381 	    "epoch", SYSCTL_DESCR("Entropy epoch"),
    382 	    NULL, 0, &E->epoch, 0, CTL_CREATE, CTL_EOL);
    383 
    384 	/* Initialize the global state for multithreaded operation.  */
    385 	mutex_init(&E->lock, MUTEX_DEFAULT, IPL_VM);
    386 	cv_init(&E->cv, "entropy");
    387 	selinit(&E->selq);
    388 
    389 	/* Make sure the seed source is attached.  */
    390 	attach_seed_rndsource();
    391 
    392 	/* Note if the bootloader didn't provide a seed.  */
    393 	if (!E->seeded)
    394 		printf("entropy: no seed from bootloader\n");
    395 
    396 	/* Allocate the per-CPU records for all early entropy sources.  */
    397 	LIST_FOREACH(rs, &E->sources, list)
    398 		rs->state = percpu_alloc(sizeof(struct rndsource_cpu));
    399 
    400 	/* Enter the boot cycle count to get started.  */
    401 	extra[i++] = entropy_timer();
    402 	KASSERT(i == __arraycount(extra));
    403 	entropy_enter(extra, sizeof extra, 0);
    404 	explicit_memset(extra, 0, sizeof extra);
    405 
    406 	/* We are now ready for multi-threaded operation.  */
    407 	E->stage = ENTROPY_WARM;
    408 }
    409 
    410 /*
    411  * entropy_init_late()
    412  *
    413  *	Late initialization.  Panic on failure.
    414  *
    415  *	Requires CPUs to have been detected and LWPs to have started.
    416  */
    417 static void
    418 entropy_init_late(void)
    419 {
    420 	int error;
    421 
    422 	KASSERT(E->stage == ENTROPY_WARM);
    423 
    424 	/* Allocate and initialize the per-CPU state.  */
    425 	entropy_percpu = percpu_create(sizeof(struct entropy_cpu),
    426 	    entropy_init_cpu, entropy_fini_cpu, NULL);
    427 
    428 	/*
    429 	 * Establish the softint at the highest softint priority level.
    430 	 * Must happen after CPU detection.
    431 	 */
    432 	entropy_sih = softint_establish(SOFTINT_SERIAL|SOFTINT_MPSAFE,
    433 	    &entropy_softintr, NULL);
    434 	if (entropy_sih == NULL)
    435 		panic("unable to establish entropy softint");
    436 
    437 	/*
    438 	 * Create the entropy housekeeping thread.  Must happen after
    439 	 * lwpinit.
    440 	 */
    441 	error = kthread_create(PRI_NONE, KTHREAD_MPSAFE|KTHREAD_TS, NULL,
    442 	    entropy_thread, NULL, &entropy_lwp, "entbutler");
    443 	if (error)
    444 		panic("unable to create entropy housekeeping thread: %d",
    445 		    error);
    446 
    447 	/*
    448 	 * Wait until the per-CPU initialization has hit all CPUs
    449 	 * before proceeding to mark the entropy system hot.
    450 	 */
    451 	xc_barrier(XC_HIGHPRI);
    452 	E->stage = ENTROPY_HOT;
    453 }
    454 
    455 /*
    456  * entropy_init_cpu(ptr, cookie, ci)
    457  *
    458  *	percpu(9) constructor for per-CPU entropy pool.
    459  */
    460 static void
    461 entropy_init_cpu(void *ptr, void *cookie, struct cpu_info *ci)
    462 {
    463 	struct entropy_cpu *ec = ptr;
    464 
    465 	ec->ec_softint_evcnt = kmem_alloc(sizeof(*ec->ec_softint_evcnt),
    466 	    KM_SLEEP);
    467 	ec->ec_pool = kmem_zalloc(sizeof(*ec->ec_pool), KM_SLEEP);
    468 	ec->ec_pending = 0;
    469 	ec->ec_locked = false;
    470 
    471 	evcnt_attach_dynamic(ec->ec_softint_evcnt, EVCNT_TYPE_MISC, NULL,
    472 	    ci->ci_cpuname, "entropy softint");
    473 }
    474 
    475 /*
    476  * entropy_fini_cpu(ptr, cookie, ci)
    477  *
    478  *	percpu(9) destructor for per-CPU entropy pool.
    479  */
    480 static void
    481 entropy_fini_cpu(void *ptr, void *cookie, struct cpu_info *ci)
    482 {
    483 	struct entropy_cpu *ec = ptr;
    484 
    485 	/*
    486 	 * Zero any lingering data.  Disclosure of the per-CPU pool
    487 	 * shouldn't retroactively affect the security of any keys
    488 	 * generated, because entpool(9) erases whatever we have just
    489 	 * drawn out of any pool, but better safe than sorry.
    490 	 */
    491 	explicit_memset(ec->ec_pool, 0, sizeof(*ec->ec_pool));
    492 
    493 	evcnt_detach(ec->ec_softint_evcnt);
    494 
    495 	kmem_free(ec->ec_pool, sizeof(*ec->ec_pool));
    496 	kmem_free(ec->ec_softint_evcnt, sizeof(*ec->ec_softint_evcnt));
    497 }
    498 
    499 /*
    500  * entropy_seed(seed)
    501  *
    502  *	Seed the entropy pool with seed.  Meant to be called as early
    503  *	as possible by the bootloader; may be called before or after
    504  *	entropy_init.  Must be called before system reaches userland.
    505  *	Must be called in thread or soft interrupt context, not in hard
    506  *	interrupt context.  Must be called at most once.
    507  *
    508  *	Overwrites the seed in place.  Caller may then free the memory.
    509  */
    510 static void
    511 entropy_seed(rndsave_t *seed)
    512 {
    513 	SHA1_CTX ctx;
    514 	uint8_t digest[SHA1_DIGEST_LENGTH];
    515 	bool seeded;
    516 
    517 	/*
    518 	 * Verify the checksum.  If the checksum fails, take the data
    519 	 * but ignore the entropy estimate -- the file may have been
    520 	 * incompletely written with garbage, which is harmless to add
    521 	 * but may not be as unpredictable as alleged.
    522 	 */
    523 	SHA1Init(&ctx);
    524 	SHA1Update(&ctx, (const void *)&seed->entropy, sizeof(seed->entropy));
    525 	SHA1Update(&ctx, seed->data, sizeof(seed->data));
    526 	SHA1Final(digest, &ctx);
    527 	CTASSERT(sizeof(seed->digest) == sizeof(digest));
    528 	if (!consttime_memequal(digest, seed->digest, sizeof(digest))) {
    529 		printf("entropy: invalid seed checksum\n");
    530 		seed->entropy = 0;
    531 	}
    532 	explicit_memset(&ctx, 0, sizeof ctx);
    533 	explicit_memset(digest, 0, sizeof digest);
    534 
    535 	/*
    536 	 * If the entropy is insensibly large, try byte-swapping.
    537 	 * Otherwise assume the file is corrupted and act as though it
    538 	 * has zero entropy.
    539 	 */
    540 	if (howmany(seed->entropy, NBBY) > sizeof(seed->data)) {
    541 		seed->entropy = bswap32(seed->entropy);
    542 		if (howmany(seed->entropy, NBBY) > sizeof(seed->data))
    543 			seed->entropy = 0;
    544 	}
    545 
    546 	/* Make sure the seed source is attached.  */
    547 	attach_seed_rndsource();
    548 
    549 	/* Test and set E->seeded.  */
    550 	if (E->stage >= ENTROPY_WARM)
    551 		mutex_enter(&E->lock);
    552 	seeded = E->seeded;
    553 	E->seeded = (seed->entropy > 0);
    554 	if (E->stage >= ENTROPY_WARM)
    555 		mutex_exit(&E->lock);
    556 
    557 	/*
    558 	 * If we've been seeded, may be re-entering the same seed
    559 	 * (e.g., bootloader vs module init, or something).  No harm in
    560 	 * entering it twice, but it contributes no additional entropy.
    561 	 */
    562 	if (seeded) {
    563 		printf("entropy: double-seeded by bootloader\n");
    564 		seed->entropy = 0;
    565 	} else {
    566 		printf("entropy: entering seed from bootloader"
    567 		    " with %u bits of entropy\n", (unsigned)seed->entropy);
    568 	}
    569 
    570 	/* Enter it into the pool and promptly zero it.  */
    571 	rnd_add_data(&seed_rndsource, seed->data, sizeof(seed->data),
    572 	    seed->entropy);
    573 	explicit_memset(seed, 0, sizeof(*seed));
    574 }
    575 
    576 /*
    577  * entropy_bootrequest()
    578  *
    579  *	Request entropy from all sources at boot, once config is
    580  *	complete and interrupts are running.
    581  */
    582 void
    583 entropy_bootrequest(void)
    584 {
    585 
    586 	KASSERT(E->stage >= ENTROPY_WARM);
    587 
    588 	/*
    589 	 * Request enough to satisfy the maximum entropy shortage.
    590 	 * This is harmless overkill if the bootloader provided a seed.
    591 	 */
    592 	mutex_enter(&E->lock);
    593 	entropy_request(ENTROPY_CAPACITY);
    594 	mutex_exit(&E->lock);
    595 }
    596 
    597 /*
    598  * entropy_epoch()
    599  *
    600  *	Returns the current entropy epoch.  If this changes, you should
    601  *	reseed.  If -1, means the system has not yet reached full
    602  *	entropy; never reverts back to -1 after full entropy has been
    603  *	reached.  Never zero, so you can always use zero as an
    604  *	uninitialized sentinel value meaning `reseed ASAP'.
    605  *
    606  *	Usage model:
    607  *
    608  *		struct foo {
    609  *			struct crypto_prng prng;
    610  *			unsigned epoch;
    611  *		} *foo;
    612  *
    613  *		unsigned epoch = entropy_epoch();
    614  *		if (__predict_false(epoch != foo->epoch)) {
    615  *			uint8_t seed[32];
    616  *			if (entropy_extract(seed, sizeof seed, 0) != 0)
    617  *				warn("no entropy");
    618  *			crypto_prng_reseed(&foo->prng, seed, sizeof seed);
    619  *			foo->epoch = epoch;
    620  *		}
    621  */
    622 unsigned
    623 entropy_epoch(void)
    624 {
    625 
    626 	/*
    627 	 * Unsigned int, so no need for seqlock for an atomic read, but
    628 	 * make sure we read it afresh each time.
    629 	 */
    630 	return atomic_load_relaxed(&E->epoch);
    631 }
    632 
    633 /*
    634  * entropy_account_cpu(ec)
    635  *
    636  *	Consider whether to consolidate entropy into the global pool
    637  *	after we just added some into the current CPU's pending pool.
    638  *
    639  *	- If this CPU can provide enough entropy now, do so.
    640  *
    641  *	- If this and whatever else is available on other CPUs can
    642  *	  provide enough entropy, kick the consolidation thread.
    643  *
    644  *	- Otherwise, do as little as possible, except maybe consolidate
    645  *	  entropy at most once a minute.
    646  *
    647  *	Caller must be bound to a CPU and therefore have exclusive
    648  *	access to ec.  Will acquire and release the global lock.
    649  */
    650 static void
    651 entropy_account_cpu(struct entropy_cpu *ec)
    652 {
    653 	unsigned diff;
    654 
    655 	KASSERT(E->stage == ENTROPY_HOT);
    656 
    657 	/*
    658 	 * If there's no entropy needed, and entropy has been
    659 	 * consolidated in the last minute, do nothing.
    660 	 */
    661 	if (__predict_true(atomic_load_relaxed(&E->needed) == 0) &&
    662 	    __predict_true(!atomic_load_relaxed(&entropy_depletion)) &&
    663 	    __predict_true((time_uptime - E->timestamp) <= 60))
    664 		return;
    665 
    666 	/* If there's nothing pending, stop here.  */
    667 	if (ec->ec_pending == 0)
    668 		return;
    669 
    670 	/* Consider consolidation, under the lock.  */
    671 	mutex_enter(&E->lock);
    672 	if (E->needed != 0 && E->needed <= ec->ec_pending) {
    673 		/*
    674 		 * If we have not yet attained full entropy but we can
    675 		 * now, do so.  This way we disseminate entropy
    676 		 * promptly when it becomes available early at boot;
    677 		 * otherwise we leave it to the entropy consolidation
    678 		 * thread, which is rate-limited to mitigate side
    679 		 * channels and abuse.
    680 		 */
    681 		uint8_t buf[ENTPOOL_CAPACITY];
    682 
    683 		/* Transfer from the local pool to the global pool.  */
    684 		entpool_extract(ec->ec_pool, buf, sizeof buf);
    685 		entpool_enter(&E->pool, buf, sizeof buf);
    686 		atomic_store_relaxed(&ec->ec_pending, 0);
    687 		atomic_store_relaxed(&E->needed, 0);
    688 
    689 		/* Notify waiters that we now have full entropy.  */
    690 		entropy_notify();
    691 		entropy_immediate_evcnt.ev_count++;
    692 	} else if (ec->ec_pending) {
    693 		/* Record how much we can add to the global pool.  */
    694 		diff = MIN(ec->ec_pending, ENTROPY_CAPACITY*NBBY - E->pending);
    695 		E->pending += diff;
    696 		atomic_store_relaxed(&ec->ec_pending, ec->ec_pending - diff);
    697 
    698 		/*
    699 		 * This should have made a difference unless we were
    700 		 * already saturated.
    701 		 */
    702 		KASSERT(diff || E->pending == ENTROPY_CAPACITY*NBBY);
    703 		KASSERT(E->pending);
    704 
    705 		if (E->needed <= E->pending) {
    706 			/*
    707 			 * Enough entropy between all the per-CPU
    708 			 * pools.  Wake up the housekeeping thread.
    709 			 *
    710 			 * If we don't need any entropy, this doesn't
    711 			 * mean much, but it is the only time we ever
    712 			 * gather additional entropy in case the
    713 			 * accounting has been overly optimistic.  This
    714 			 * happens at most once a minute, so there's
    715 			 * negligible performance cost.
    716 			 */
    717 			E->consolidate = true;
    718 			cv_broadcast(&E->cv);
    719 			if (E->needed == 0)
    720 				entropy_discretionary_evcnt.ev_count++;
    721 		} else {
    722 			/* Can't get full entropy.  Keep gathering.  */
    723 			entropy_partial_evcnt.ev_count++;
    724 		}
    725 	}
    726 	mutex_exit(&E->lock);
    727 }
    728 
    729 /*
    730  * entropy_enter_early(buf, len, nbits)
    731  *
    732  *	Do entropy bookkeeping globally, before we have established
    733  *	per-CPU pools.  Enter directly into the global pool in the hope
    734  *	that we enter enough before the first entropy_extract to thwart
    735  *	iterative-guessing attacks; entropy_extract will warn if not.
    736  */
    737 static void
    738 entropy_enter_early(const void *buf, size_t len, unsigned nbits)
    739 {
    740 	bool notify = false;
    741 
    742 	if (E->stage >= ENTROPY_WARM)
    743 		mutex_enter(&E->lock);
    744 
    745 	/* Enter it into the pool.  */
    746 	entpool_enter(&E->pool, buf, len);
    747 
    748 	/*
    749 	 * Decide whether to notify reseed -- we will do so if either:
    750 	 * (a) we transition from partial entropy to full entropy, or
    751 	 * (b) we get a batch of full entropy all at once.
    752 	 */
    753 	notify |= (E->needed && E->needed <= nbits);
    754 	notify |= (nbits >= ENTROPY_CAPACITY*NBBY);
    755 
    756 	/* Subtract from the needed count and notify if appropriate.  */
    757 	E->needed -= MIN(E->needed, nbits);
    758 	if (notify) {
    759 		entropy_notify();
    760 		entropy_immediate_evcnt.ev_count++;
    761 	}
    762 
    763 	if (E->stage >= ENTROPY_WARM)
    764 		mutex_exit(&E->lock);
    765 }
    766 
    767 /*
    768  * entropy_enter(buf, len, nbits)
    769  *
    770  *	Enter len bytes of data from buf into the system's entropy
    771  *	pool, stirring as necessary when the internal buffer fills up.
    772  *	nbits is a lower bound on the number of bits of entropy in the
    773  *	process that led to this sample.
    774  */
    775 static void
    776 entropy_enter(const void *buf, size_t len, unsigned nbits)
    777 {
    778 	struct entropy_cpu *ec;
    779 	uint32_t pending;
    780 	int s;
    781 
    782 	KASSERTMSG(!curcpu_available() || !cpu_intr_p(),
    783 	    "use entropy_enter_intr from interrupt context");
    784 	KASSERTMSG(howmany(nbits, NBBY) <= len,
    785 	    "impossible entropy rate: %u bits in %zu-byte string", nbits, len);
    786 
    787 	/* If it's too early after boot, just use entropy_enter_early.  */
    788 	if (__predict_false(E->stage < ENTROPY_HOT)) {
    789 		entropy_enter_early(buf, len, nbits);
    790 		return;
    791 	}
    792 
    793 	/*
    794 	 * Acquire the per-CPU state, blocking soft interrupts and
    795 	 * causing hard interrupts to drop samples on the floor.
    796 	 */
    797 	ec = percpu_getref(entropy_percpu);
    798 	s = splsoftserial();
    799 	KASSERT(!ec->ec_locked);
    800 	ec->ec_locked = true;
    801 	__insn_barrier();
    802 
    803 	/* Enter into the per-CPU pool.  */
    804 	entpool_enter(ec->ec_pool, buf, len);
    805 
    806 	/* Count up what we can add.  */
    807 	pending = ec->ec_pending;
    808 	pending += MIN(ENTROPY_CAPACITY*NBBY - pending, nbits);
    809 	atomic_store_relaxed(&ec->ec_pending, pending);
    810 
    811 	/* Consolidate globally if appropriate based on what we added.  */
    812 	entropy_account_cpu(ec);
    813 
    814 	/* Release the per-CPU state.  */
    815 	KASSERT(ec->ec_locked);
    816 	__insn_barrier();
    817 	ec->ec_locked = false;
    818 	splx(s);
    819 	percpu_putref(entropy_percpu);
    820 }
    821 
    822 /*
    823  * entropy_enter_intr(buf, len, nbits)
    824  *
    825  *	Enter up to len bytes of data from buf into the system's
    826  *	entropy pool without stirring.  nbits is a lower bound on the
    827  *	number of bits of entropy in the process that led to this
    828  *	sample.  If the sample could be entered completely, assume
    829  *	nbits of entropy pending; otherwise assume none, since we don't
    830  *	know whether some parts of the sample are constant, for
    831  *	instance.  Schedule a softint to stir the entropy pool if
    832  *	needed.  Return true if used fully, false if truncated at all.
    833  *
    834  *	Using this in thread context will work, but you might as well
    835  *	use entropy_enter in that case.
    836  */
    837 static bool
    838 entropy_enter_intr(const void *buf, size_t len, unsigned nbits)
    839 {
    840 	struct entropy_cpu *ec;
    841 	bool fullyused = false;
    842 	uint32_t pending;
    843 
    844 	KASSERTMSG(howmany(nbits, NBBY) <= len,
    845 	    "impossible entropy rate: %u bits in %zu-byte string", nbits, len);
    846 
    847 	/* If it's too early after boot, just use entropy_enter_early.  */
    848 	if (__predict_false(E->stage < ENTROPY_HOT)) {
    849 		entropy_enter_early(buf, len, nbits);
    850 		return true;
    851 	}
    852 
    853 	/*
    854 	 * Acquire the per-CPU state.  If someone is in the middle of
    855 	 * using it, drop the sample.  Otherwise, take the lock so that
    856 	 * higher-priority interrupts will drop their samples.
    857 	 */
    858 	ec = percpu_getref(entropy_percpu);
    859 	if (ec->ec_locked)
    860 		goto out0;
    861 	ec->ec_locked = true;
    862 	__insn_barrier();
    863 
    864 	/*
    865 	 * Enter as much as we can into the per-CPU pool.  If it was
    866 	 * truncated, schedule a softint to stir the pool and stop.
    867 	 */
    868 	if (!entpool_enter_nostir(ec->ec_pool, buf, len)) {
    869 		softint_schedule(entropy_sih);
    870 		goto out1;
    871 	}
    872 	fullyused = true;
    873 
    874 	/* Count up what we can contribute.  */
    875 	pending = ec->ec_pending;
    876 	pending += MIN(ENTROPY_CAPACITY*NBBY - pending, nbits);
    877 	atomic_store_relaxed(&ec->ec_pending, pending);
    878 
    879 	/* Schedule a softint if we added anything and it matters.  */
    880 	if (__predict_false((atomic_load_relaxed(&E->needed) != 0) ||
    881 		atomic_load_relaxed(&entropy_depletion)) &&
    882 	    nbits != 0)
    883 		softint_schedule(entropy_sih);
    884 
    885 out1:	/* Release the per-CPU state.  */
    886 	KASSERT(ec->ec_locked);
    887 	__insn_barrier();
    888 	ec->ec_locked = false;
    889 out0:	percpu_putref(entropy_percpu);
    890 
    891 	return fullyused;
    892 }
    893 
    894 /*
    895  * entropy_softintr(cookie)
    896  *
    897  *	Soft interrupt handler for entering entropy.  Takes care of
    898  *	stirring the local CPU's entropy pool if it filled up during
    899  *	hard interrupts, and promptly crediting entropy from the local
    900  *	CPU's entropy pool to the global entropy pool if needed.
    901  */
    902 static void
    903 entropy_softintr(void *cookie)
    904 {
    905 	struct entropy_cpu *ec;
    906 
    907 	/*
    908 	 * Acquire the per-CPU state.  Other users can lock this only
    909 	 * while soft interrupts are blocked.  Cause hard interrupts to
    910 	 * drop samples on the floor.
    911 	 */
    912 	ec = percpu_getref(entropy_percpu);
    913 	KASSERT(!ec->ec_locked);
    914 	ec->ec_locked = true;
    915 	__insn_barrier();
    916 
    917 	/* Count statistics.  */
    918 	ec->ec_softint_evcnt->ev_count++;
    919 
    920 	/* Stir the pool if necessary.  */
    921 	entpool_stir(ec->ec_pool);
    922 
    923 	/* Consolidate globally if appropriate based on what we added.  */
    924 	entropy_account_cpu(ec);
    925 
    926 	/* Release the per-CPU state.  */
    927 	KASSERT(ec->ec_locked);
    928 	__insn_barrier();
    929 	ec->ec_locked = false;
    930 	percpu_putref(entropy_percpu);
    931 }
    932 
    933 /*
    934  * entropy_thread(cookie)
    935  *
    936  *	Handle any asynchronous entropy housekeeping.
    937  */
    938 static void
    939 entropy_thread(void *cookie)
    940 {
    941 	bool consolidate;
    942 
    943 	for (;;) {
    944 		/*
    945 		 * Wait until there's full entropy somewhere among the
    946 		 * CPUs, as confirmed at most once per minute, or
    947 		 * someone wants to consolidate.
    948 		 */
    949 		if (entropy_pending() >= ENTROPY_CAPACITY*NBBY) {
    950 			consolidate = true;
    951 		} else {
    952 			mutex_enter(&E->lock);
    953 			if (!E->consolidate)
    954 				cv_timedwait(&E->cv, &E->lock, 60*hz);
    955 			consolidate = E->consolidate;
    956 			E->consolidate = false;
    957 			mutex_exit(&E->lock);
    958 		}
    959 
    960 		if (consolidate) {
    961 			/* Do it.  */
    962 			entropy_consolidate();
    963 
    964 			/* Mitigate abuse.  */
    965 			kpause("entropy", false, hz, NULL);
    966 		}
    967 	}
    968 }
    969 
    970 /*
    971  * entropy_pending()
    972  *
    973  *	Count up the amount of entropy pending on other CPUs.
    974  */
    975 static uint32_t
    976 entropy_pending(void)
    977 {
    978 	uint32_t pending = 0;
    979 
    980 	percpu_foreach(entropy_percpu, &entropy_pending_cpu, &pending);
    981 	return pending;
    982 }
    983 
    984 static void
    985 entropy_pending_cpu(void *ptr, void *cookie, struct cpu_info *ci)
    986 {
    987 	struct entropy_cpu *ec = ptr;
    988 	uint32_t *pendingp = cookie;
    989 	uint32_t cpu_pending;
    990 
    991 	cpu_pending = atomic_load_relaxed(&ec->ec_pending);
    992 	*pendingp += MIN(ENTROPY_CAPACITY*NBBY - *pendingp, cpu_pending);
    993 }
    994 
    995 /*
    996  * entropy_consolidate()
    997  *
    998  *	Issue a cross-call to gather entropy on all CPUs and advance
    999  *	the entropy epoch.
   1000  */
   1001 static void
   1002 entropy_consolidate(void)
   1003 {
   1004 	static const struct timeval interval = {.tv_sec = 60, .tv_usec = 0};
   1005 	static struct timeval lasttime; /* serialized by E->lock */
   1006 	unsigned diff;
   1007 	uint64_t ticket;
   1008 
   1009 	/* Gather entropy on all CPUs.  */
   1010 	ticket = xc_broadcast(0, &entropy_gather_xc, NULL, NULL);
   1011 	xc_wait(ticket);
   1012 
   1013 	/* Acquire the lock to notify waiters.  */
   1014 	mutex_enter(&E->lock);
   1015 
   1016 	/* Count another consolidation.  */
   1017 	entropy_consolidate_evcnt.ev_count++;
   1018 
   1019 	/* Note when we last consolidated, i.e. now.  */
   1020 	E->timestamp = time_uptime;
   1021 
   1022 	/* Count the entropy that was gathered.  */
   1023 	diff = MIN(E->needed, E->pending);
   1024 	atomic_store_relaxed(&E->needed, E->needed - diff);
   1025 	E->pending -= diff;
   1026 	if (__predict_false(E->needed > 0)) {
   1027 		if (ratecheck(&lasttime, &interval))
   1028 			printf("entropy: WARNING:"
   1029 			    " consolidating less than full entropy\n");
   1030 	}
   1031 
   1032 	/* Advance the epoch and notify waiters.  */
   1033 	entropy_notify();
   1034 
   1035 	/* Release the lock.  */
   1036 	mutex_exit(&E->lock);
   1037 }
   1038 
   1039 /*
   1040  * entropy_gather_xc(arg1, arg2)
   1041  *
   1042  *	Extract output from the local CPU's input pool and enter it
   1043  *	into the global pool.
   1044  */
   1045 static void
   1046 entropy_gather_xc(void *arg1 __unused, void *arg2 __unused)
   1047 {
   1048 	struct entropy_cpu *ec;
   1049 	uint8_t buf[ENTPOOL_CAPACITY];
   1050 	uint32_t extra[7];
   1051 	unsigned i = 0;
   1052 	int s;
   1053 
   1054 	/* Grab CPU number and cycle counter to mix extra into the pool.  */
   1055 	extra[i++] = cpu_number();
   1056 	extra[i++] = entropy_timer();
   1057 
   1058 	/*
   1059 	 * Acquire the per-CPU state, blocking soft interrupts and
   1060 	 * discarding entropy in hard interrupts, so that we can
   1061 	 * extract from the per-CPU pool.
   1062 	 */
   1063 	ec = percpu_getref(entropy_percpu);
   1064 	s = splsoftserial();
   1065 	KASSERT(!ec->ec_locked);
   1066 	ec->ec_locked = true;
   1067 	__insn_barrier();
   1068 	extra[i++] = entropy_timer();
   1069 
   1070 	/* Extract the data.  */
   1071 	entpool_extract(ec->ec_pool, buf, sizeof buf);
   1072 	extra[i++] = entropy_timer();
   1073 
   1074 	/* Release the per-CPU state.  */
   1075 	KASSERT(ec->ec_locked);
   1076 	__insn_barrier();
   1077 	ec->ec_locked = false;
   1078 	splx(s);
   1079 	percpu_putref(entropy_percpu);
   1080 	extra[i++] = entropy_timer();
   1081 
   1082 	/*
   1083 	 * Copy over statistics, and enter the per-CPU extract and the
   1084 	 * extra timing into the global pool, under the global lock.
   1085 	 */
   1086 	mutex_enter(&E->lock);
   1087 	extra[i++] = entropy_timer();
   1088 	entpool_enter(&E->pool, buf, sizeof buf);
   1089 	explicit_memset(buf, 0, sizeof buf);
   1090 	extra[i++] = entropy_timer();
   1091 	KASSERT(i == __arraycount(extra));
   1092 	entpool_enter(&E->pool, extra, sizeof extra);
   1093 	explicit_memset(extra, 0, sizeof extra);
   1094 	mutex_exit(&E->lock);
   1095 }
   1096 
   1097 /*
   1098  * entropy_notify()
   1099  *
   1100  *	Caller just contributed entropy to the global pool.  Advance
   1101  *	the entropy epoch and notify waiters.
   1102  *
   1103  *	Caller must hold the global entropy lock.  Except for the
   1104  *	`sysctl -w kern.entropy.consolidate=1` trigger, the caller must
   1105  *	have just have transitioned from partial entropy to full
   1106  *	entropy -- E->needed should be zero now.
   1107  */
   1108 static void
   1109 entropy_notify(void)
   1110 {
   1111 	unsigned epoch;
   1112 
   1113 	KASSERT(E->stage == ENTROPY_COLD || mutex_owned(&E->lock));
   1114 
   1115 	/*
   1116 	 * If this is the first time, print a message to the console
   1117 	 * that we're ready so operators can compare it to the timing
   1118 	 * of other events.
   1119 	 */
   1120 	if (E->epoch == (unsigned)-1)
   1121 		printf("entropy: ready\n");
   1122 
   1123 	/* Set the epoch; roll over from UINTMAX-1 to 1.  */
   1124 	rnd_initial_entropy = 1; /* XXX legacy */
   1125 	epoch = E->epoch + 1;
   1126 	if (epoch == 0 || epoch == (unsigned)-1)
   1127 		epoch = 1;
   1128 	atomic_store_relaxed(&E->epoch, epoch);
   1129 
   1130 	/* Notify waiters.  */
   1131 	if (E->stage >= ENTROPY_WARM) {
   1132 		cv_broadcast(&E->cv);
   1133 		selnotify(&E->selq, POLLIN|POLLRDNORM, NOTE_SUBMIT);
   1134 	}
   1135 
   1136 	/* Count another notification.  */
   1137 	entropy_notify_evcnt.ev_count++;
   1138 }
   1139 
   1140 /*
   1141  * sysctl -w kern.entropy.consolidate=1
   1142  *
   1143  *	Trigger entropy consolidation and wait for it to complete.
   1144  *	Writable only by superuser.  This is the only way for the
   1145  *	system to consolidate entropy if the operator knows something
   1146  *	the kernel doesn't about how unpredictable the pending entropy
   1147  *	pools are.
   1148  */
   1149 static int
   1150 sysctl_entropy_consolidate(SYSCTLFN_ARGS)
   1151 {
   1152 	struct sysctlnode node = *rnode;
   1153 	uint64_t ticket;
   1154 	int arg;
   1155 	int error;
   1156 
   1157 	KASSERT(E->stage == ENTROPY_HOT);
   1158 
   1159 	node.sysctl_data = &arg;
   1160 	error = sysctl_lookup(SYSCTLFN_CALL(&node));
   1161 	if (error || newp == NULL)
   1162 		return error;
   1163 	if (arg) {
   1164 		mutex_enter(&E->lock);
   1165 		ticket = entropy_consolidate_evcnt.ev_count;
   1166 		E->consolidate = true;
   1167 		cv_broadcast(&E->cv);
   1168 		while (ticket == entropy_consolidate_evcnt.ev_count) {
   1169 			error = cv_wait_sig(&E->cv, &E->lock);
   1170 			if (error)
   1171 				break;
   1172 		}
   1173 		mutex_exit(&E->lock);
   1174 	}
   1175 
   1176 	return error;
   1177 }
   1178 
   1179 /*
   1180  * sysctl -w kern.entropy.gather=1
   1181  *
   1182  *	Trigger gathering entropy from all on-demand sources, and wait
   1183  *	for synchronous sources (but not asynchronous sources) to
   1184  *	complete.  Writable only by superuser.
   1185  */
   1186 static int
   1187 sysctl_entropy_gather(SYSCTLFN_ARGS)
   1188 {
   1189 	struct sysctlnode node = *rnode;
   1190 	int arg;
   1191 	int error;
   1192 
   1193 	KASSERT(E->stage == ENTROPY_HOT);
   1194 
   1195 	node.sysctl_data = &arg;
   1196 	error = sysctl_lookup(SYSCTLFN_CALL(&node));
   1197 	if (error || newp == NULL)
   1198 		return error;
   1199 	if (arg) {
   1200 		mutex_enter(&E->lock);
   1201 		entropy_request(ENTROPY_CAPACITY);
   1202 		mutex_exit(&E->lock);
   1203 	}
   1204 
   1205 	return 0;
   1206 }
   1207 
   1208 /*
   1209  * entropy_extract(buf, len, flags)
   1210  *
   1211  *	Extract len bytes from the global entropy pool into buf.
   1212  *
   1213  *	Flags may have:
   1214  *
   1215  *		ENTROPY_WAIT	Wait for entropy if not available yet.
   1216  *		ENTROPY_SIG	Allow interruption by a signal during wait.
   1217  *
   1218  *	Return zero on success, or error on failure:
   1219  *
   1220  *		EWOULDBLOCK	No entropy and ENTROPY_WAIT not set.
   1221  *		EINTR/ERESTART	No entropy, ENTROPY_SIG set, and interrupted.
   1222  *
   1223  *	If ENTROPY_WAIT is set, allowed only in thread context.  If
   1224  *	ENTROPY_WAIT is not set, allowed up to IPL_VM.  (XXX That's
   1225  *	awfully high...  Do we really need it in hard interrupts?  This
   1226  *	arises from use of cprng_strong(9).)
   1227  */
   1228 int
   1229 entropy_extract(void *buf, size_t len, int flags)
   1230 {
   1231 	static const struct timeval interval = {.tv_sec = 60, .tv_usec = 0};
   1232 	static struct timeval lasttime; /* serialized by E->lock */
   1233 	int error;
   1234 
   1235 	if (ISSET(flags, ENTROPY_WAIT)) {
   1236 		ASSERT_SLEEPABLE();
   1237 		KASSERTMSG(E->stage >= ENTROPY_WARM,
   1238 		    "can't wait for entropy until warm");
   1239 	}
   1240 
   1241 	/* Acquire the global lock to get at the global pool.  */
   1242 	if (E->stage >= ENTROPY_WARM)
   1243 		mutex_enter(&E->lock);
   1244 
   1245 	/* Count up request for entropy in interrupt context.  */
   1246 	if (curcpu_available() && cpu_intr_p())
   1247 		entropy_extract_intr_evcnt.ev_count++;
   1248 
   1249 	/* Wait until there is enough entropy in the system.  */
   1250 	error = 0;
   1251 	while (E->needed) {
   1252 		/* Ask for more, synchronously if possible.  */
   1253 		entropy_request(len);
   1254 
   1255 		/* If we got enough, we're done.  */
   1256 		if (E->needed == 0) {
   1257 			KASSERT(error == 0);
   1258 			break;
   1259 		}
   1260 
   1261 		/* If not waiting, stop here.  */
   1262 		if (!ISSET(flags, ENTROPY_WAIT)) {
   1263 			error = EWOULDBLOCK;
   1264 			break;
   1265 		}
   1266 
   1267 		/* Wait for some entropy to come in and try again.  */
   1268 		KASSERT(E->stage >= ENTROPY_WARM);
   1269 		if (ISSET(flags, ENTROPY_SIG)) {
   1270 			error = cv_wait_sig(&E->cv, &E->lock);
   1271 			if (error)
   1272 				break;
   1273 		} else {
   1274 			cv_wait(&E->cv, &E->lock);
   1275 		}
   1276 	}
   1277 
   1278 	/* Count failure -- but fill the buffer nevertheless.  */
   1279 	if (error)
   1280 		entropy_extract_fail_evcnt.ev_count++;
   1281 
   1282 	/*
   1283 	 * Report a warning if we have never yet reached full entropy.
   1284 	 * This is the only case where we consider entropy to be
   1285 	 * `depleted' without kern.entropy.depletion enabled -- when we
   1286 	 * only have partial entropy, an adversary may be able to
   1287 	 * narrow the state of the pool down to a small number of
   1288 	 * possibilities; the output then enables them to confirm a
   1289 	 * guess, reducing its entropy from the adversary's perspective
   1290 	 * to zero.
   1291 	 */
   1292 	if (__predict_false(E->epoch == (unsigned)-1)) {
   1293 		if (ratecheck(&lasttime, &interval))
   1294 			printf("entropy: WARNING:"
   1295 			    " extracting entropy too early\n");
   1296 		atomic_store_relaxed(&E->needed, ENTROPY_CAPACITY*NBBY);
   1297 	}
   1298 
   1299 	/* Extract data from the pool, and `deplete' if we're doing that.  */
   1300 	entpool_extract(&E->pool, buf, len);
   1301 	if (__predict_false(atomic_load_relaxed(&entropy_depletion)) &&
   1302 	    error == 0) {
   1303 		unsigned cost = MIN(len, ENTROPY_CAPACITY)*NBBY;
   1304 
   1305 		atomic_store_relaxed(&E->needed,
   1306 		    E->needed + MIN(ENTROPY_CAPACITY*NBBY - E->needed, cost));
   1307 		entropy_deplete_evcnt.ev_count++;
   1308 	}
   1309 
   1310 	/* Release the global lock and return the error.  */
   1311 	if (E->stage >= ENTROPY_WARM)
   1312 		mutex_exit(&E->lock);
   1313 	return error;
   1314 }
   1315 
   1316 /*
   1317  * entropy_poll(events)
   1318  *
   1319  *	Return the subset of events ready, and if it is not all of
   1320  *	events, record curlwp as waiting for entropy.
   1321  */
   1322 int
   1323 entropy_poll(int events)
   1324 {
   1325 	int revents = 0;
   1326 
   1327 	KASSERT(E->stage >= ENTROPY_WARM);
   1328 
   1329 	/* Always ready for writing.  */
   1330 	revents |= events & (POLLOUT|POLLWRNORM);
   1331 
   1332 	/* Narrow it down to reads.  */
   1333 	events &= POLLIN|POLLRDNORM;
   1334 	if (events == 0)
   1335 		return revents;
   1336 
   1337 	/*
   1338 	 * If we have reached full entropy and we're not depleting
   1339 	 * entropy, we are forever ready.
   1340 	 */
   1341 	if (__predict_true(atomic_load_relaxed(&E->needed) == 0) &&
   1342 	    __predict_true(!atomic_load_relaxed(&entropy_depletion)))
   1343 		return revents | events;
   1344 
   1345 	/*
   1346 	 * Otherwise, check whether we need entropy under the lock.  If
   1347 	 * we don't, we're ready; if we do, add ourselves to the queue.
   1348 	 */
   1349 	mutex_enter(&E->lock);
   1350 	if (E->needed == 0)
   1351 		revents |= events;
   1352 	else
   1353 		selrecord(curlwp, &E->selq);
   1354 	mutex_exit(&E->lock);
   1355 
   1356 	return revents;
   1357 }
   1358 
   1359 /*
   1360  * filt_entropy_read_detach(kn)
   1361  *
   1362  *	struct filterops::f_detach callback for entropy read events:
   1363  *	remove kn from the list of waiters.
   1364  */
   1365 static void
   1366 filt_entropy_read_detach(struct knote *kn)
   1367 {
   1368 
   1369 	KASSERT(E->stage >= ENTROPY_WARM);
   1370 
   1371 	mutex_enter(&E->lock);
   1372 	SLIST_REMOVE(&E->selq.sel_klist, kn, knote, kn_selnext);
   1373 	mutex_exit(&E->lock);
   1374 }
   1375 
   1376 /*
   1377  * filt_entropy_read_event(kn, hint)
   1378  *
   1379  *	struct filterops::f_event callback for entropy read events:
   1380  *	poll for entropy.  Caller must hold the global entropy lock if
   1381  *	hint is NOTE_SUBMIT, and must not if hint is not NOTE_SUBMIT.
   1382  */
   1383 static int
   1384 filt_entropy_read_event(struct knote *kn, long hint)
   1385 {
   1386 	int ret;
   1387 
   1388 	KASSERT(E->stage >= ENTROPY_WARM);
   1389 
   1390 	/* Acquire the lock, if caller is outside entropy subsystem.  */
   1391 	if (hint == NOTE_SUBMIT)
   1392 		KASSERT(mutex_owned(&E->lock));
   1393 	else
   1394 		mutex_enter(&E->lock);
   1395 
   1396 	/*
   1397 	 * If we still need entropy, can't read anything; if not, can
   1398 	 * read arbitrarily much.
   1399 	 */
   1400 	if (E->needed != 0) {
   1401 		ret = 0;
   1402 	} else {
   1403 		if (atomic_load_relaxed(&entropy_depletion))
   1404 			kn->kn_data = ENTROPY_CAPACITY*NBBY;
   1405 		else
   1406 			kn->kn_data = MIN(INT64_MAX, SSIZE_MAX);
   1407 		ret = 1;
   1408 	}
   1409 
   1410 	/* Release the lock, if caller is outside entropy subsystem.  */
   1411 	if (hint == NOTE_SUBMIT)
   1412 		KASSERT(mutex_owned(&E->lock));
   1413 	else
   1414 		mutex_exit(&E->lock);
   1415 
   1416 	return ret;
   1417 }
   1418 
   1419 static const struct filterops entropy_read_filtops = {
   1420 	.f_isfd = 1,		/* XXX Makes sense only for /dev/u?random.  */
   1421 	.f_attach = NULL,
   1422 	.f_detach = filt_entropy_read_detach,
   1423 	.f_event = filt_entropy_read_event,
   1424 };
   1425 
   1426 /*
   1427  * entropy_kqfilter(kn)
   1428  *
   1429  *	Register kn to receive entropy event notifications.  May be
   1430  *	EVFILT_READ or EVFILT_WRITE; anything else yields EINVAL.
   1431  */
   1432 int
   1433 entropy_kqfilter(struct knote *kn)
   1434 {
   1435 
   1436 	KASSERT(E->stage >= ENTROPY_WARM);
   1437 
   1438 	switch (kn->kn_filter) {
   1439 	case EVFILT_READ:
   1440 		/* Enter into the global select queue.  */
   1441 		mutex_enter(&E->lock);
   1442 		kn->kn_fop = &entropy_read_filtops;
   1443 		SLIST_INSERT_HEAD(&E->selq.sel_klist, kn, kn_selnext);
   1444 		mutex_exit(&E->lock);
   1445 		return 0;
   1446 	case EVFILT_WRITE:
   1447 		/* Can always dump entropy into the system.  */
   1448 		kn->kn_fop = &seltrue_filtops;
   1449 		return 0;
   1450 	default:
   1451 		return EINVAL;
   1452 	}
   1453 }
   1454 
   1455 /*
   1456  * rndsource_setcb(rs, get, getarg)
   1457  *
   1458  *	Set the request callback for the entropy source rs, if it can
   1459  *	provide entropy on demand.  Must precede rnd_attach_source.
   1460  */
   1461 void
   1462 rndsource_setcb(struct krndsource *rs, void (*get)(size_t, void *),
   1463     void *getarg)
   1464 {
   1465 
   1466 	rs->get = get;
   1467 	rs->getarg = getarg;
   1468 }
   1469 
   1470 /*
   1471  * rnd_attach_source(rs, name, type, flags)
   1472  *
   1473  *	Attach the entropy source rs.  Must be done after
   1474  *	rndsource_setcb, if any, and before any calls to rnd_add_data.
   1475  */
   1476 void
   1477 rnd_attach_source(struct krndsource *rs, const char *name, uint32_t type,
   1478     uint32_t flags)
   1479 {
   1480 	uint32_t extra[4];
   1481 	unsigned i = 0;
   1482 
   1483 	/* Grab cycle counter to mix extra into the pool.  */
   1484 	extra[i++] = entropy_timer();
   1485 
   1486 	/*
   1487 	 * Apply some standard flags:
   1488 	 *
   1489 	 * - We do not bother with network devices by default, for
   1490 	 *   hysterical raisins (perhaps: because it is often the case
   1491 	 *   that an adversary can influence network packet timings).
   1492 	 */
   1493 	switch (type) {
   1494 	case RND_TYPE_NET:
   1495 		flags |= RND_FLAG_NO_COLLECT;
   1496 		break;
   1497 	}
   1498 
   1499 	/* Sanity-check the callback if RND_FLAG_HASCB is set.  */
   1500 	KASSERT(!ISSET(flags, RND_FLAG_HASCB) || rs->get != NULL);
   1501 
   1502 	/* Initialize the random source.  */
   1503 	memset(rs->name, 0, sizeof(rs->name)); /* paranoia */
   1504 	strlcpy(rs->name, name, sizeof(rs->name));
   1505 	rs->total = 0;
   1506 	rs->type = type;
   1507 	rs->flags = flags;
   1508 	if (E->stage >= ENTROPY_WARM)
   1509 		rs->state = percpu_alloc(sizeof(struct rndsource_cpu));
   1510 	extra[i++] = entropy_timer();
   1511 
   1512 	/* Wire it into the global list of random sources.  */
   1513 	if (E->stage >= ENTROPY_WARM)
   1514 		mutex_enter(&E->lock);
   1515 	LIST_INSERT_HEAD(&E->sources, rs, list);
   1516 	if (E->stage >= ENTROPY_WARM)
   1517 		mutex_exit(&E->lock);
   1518 	extra[i++] = entropy_timer();
   1519 
   1520 	/* Request that it provide entropy ASAP, if we can.  */
   1521 	if (ISSET(flags, RND_FLAG_HASCB))
   1522 		(*rs->get)(ENTROPY_CAPACITY, rs->getarg);
   1523 	extra[i++] = entropy_timer();
   1524 
   1525 	/* Mix the extra into the pool.  */
   1526 	KASSERT(i == __arraycount(extra));
   1527 	entropy_enter(extra, sizeof extra, 0);
   1528 	explicit_memset(extra, 0, sizeof extra);
   1529 }
   1530 
   1531 /*
   1532  * rnd_detach_source(rs)
   1533  *
   1534  *	Detach the entropy source rs.  May sleep waiting for users to
   1535  *	drain.  Further use is not allowed.
   1536  */
   1537 void
   1538 rnd_detach_source(struct krndsource *rs)
   1539 {
   1540 
   1541 	/*
   1542 	 * If we're cold (shouldn't happen, but hey), just remove it
   1543 	 * from the list -- there's nothing allocated.
   1544 	 */
   1545 	if (E->stage == ENTROPY_COLD) {
   1546 		LIST_REMOVE(rs, list);
   1547 		return;
   1548 	}
   1549 
   1550 	/* We may have to wait for entropy_request.  */
   1551 	ASSERT_SLEEPABLE();
   1552 
   1553 	/* Wait until the source list is not in use, and remove it.  */
   1554 	mutex_enter(&E->lock);
   1555 	while (E->sourcelock)
   1556 		cv_wait(&E->cv, &E->lock);
   1557 	LIST_REMOVE(rs, list);
   1558 	mutex_exit(&E->lock);
   1559 
   1560 	/* Free the per-CPU data.  */
   1561 	percpu_free(rs->state, sizeof(struct rndsource_cpu));
   1562 }
   1563 
   1564 /*
   1565  * rnd_lock_sources()
   1566  *
   1567  *	Prevent changes to the list of rndsources while we iterate it.
   1568  *	Interruptible.  Caller must hold the global entropy lock.  If
   1569  *	successful, no rndsource will go away until rnd_unlock_sources
   1570  *	even while the caller releases the global entropy lock.
   1571  */
   1572 static int
   1573 rnd_lock_sources(void)
   1574 {
   1575 	int error;
   1576 
   1577 	KASSERT(mutex_owned(&E->lock));
   1578 
   1579 	while (E->sourcelock) {
   1580 		error = cv_wait_sig(&E->cv, &E->lock);
   1581 		if (error)
   1582 			return error;
   1583 	}
   1584 
   1585 	E->sourcelock = curlwp;
   1586 	return 0;
   1587 }
   1588 
   1589 /*
   1590  * rnd_trylock_sources()
   1591  *
   1592  *	Try to lock the list of sources, but if it's already locked,
   1593  *	fail.  Caller must hold the global entropy lock.  If
   1594  *	successful, no rndsource will go away until rnd_unlock_sources
   1595  *	even while the caller releases the global entropy lock.
   1596  */
   1597 static bool
   1598 rnd_trylock_sources(void)
   1599 {
   1600 
   1601 	KASSERT(E->stage == ENTROPY_COLD || mutex_owned(&E->lock));
   1602 
   1603 	if (E->sourcelock)
   1604 		return false;
   1605 	E->sourcelock = (curcpu_available() ? curlwp : (void *)1);
   1606 	return true;
   1607 }
   1608 
   1609 /*
   1610  * rnd_unlock_sources()
   1611  *
   1612  *	Unlock the list of sources after rnd_lock_sources or
   1613  *	rnd_trylock_sources.  Caller must hold the global entropy lock.
   1614  */
   1615 static void
   1616 rnd_unlock_sources(void)
   1617 {
   1618 
   1619 	KASSERT(E->stage == ENTROPY_COLD || mutex_owned(&E->lock));
   1620 
   1621 	KASSERTMSG(E->sourcelock == (curcpu_available() ? curlwp : (void *)1),
   1622 	    "lwp %p releasing lock held by %p",
   1623 	    (curcpu_available() ? curlwp : (void *)1), E->sourcelock);
   1624 	E->sourcelock = NULL;
   1625 	if (E->stage >= ENTROPY_WARM)
   1626 		cv_broadcast(&E->cv);
   1627 }
   1628 
   1629 /*
   1630  * rnd_sources_locked()
   1631  *
   1632  *	True if we hold the list of rndsources locked, for diagnostic
   1633  *	assertions.
   1634  */
   1635 static bool __diagused
   1636 rnd_sources_locked(void)
   1637 {
   1638 
   1639 	return E->sourcelock == (curcpu_available() ? curlwp : (void *)1);
   1640 }
   1641 
   1642 /*
   1643  * entropy_request(nbytes)
   1644  *
   1645  *	Request nbytes bytes of entropy from all sources in the system.
   1646  *	OK if we overdo it.  Caller must hold the global entropy lock;
   1647  *	will release and re-acquire it.
   1648  */
   1649 static void
   1650 entropy_request(size_t nbytes)
   1651 {
   1652 	struct krndsource *rs;
   1653 
   1654 	KASSERT(E->stage == ENTROPY_COLD || mutex_owned(&E->lock));
   1655 
   1656 	/*
   1657 	 * If there is a request in progress, let it proceed.
   1658 	 * Otherwise, note that a request is in progress to avoid
   1659 	 * reentry and to block rnd_detach_source until we're done.
   1660 	 */
   1661 	if (!rnd_trylock_sources())
   1662 		return;
   1663 	entropy_request_evcnt.ev_count++;
   1664 
   1665 	/* Clamp to the maximum reasonable request.  */
   1666 	nbytes = MIN(nbytes, ENTROPY_CAPACITY);
   1667 
   1668 	/* Walk the list of sources.  */
   1669 	LIST_FOREACH(rs, &E->sources, list) {
   1670 		/* Skip sources without callbacks.  */
   1671 		if (!ISSET(rs->flags, RND_FLAG_HASCB))
   1672 			continue;
   1673 
   1674 		/* Drop the lock while we call the callback.  */
   1675 		if (E->stage >= ENTROPY_WARM)
   1676 			mutex_exit(&E->lock);
   1677 		(*rs->get)(nbytes, rs->getarg);
   1678 		if (E->stage >= ENTROPY_WARM)
   1679 			mutex_enter(&E->lock);
   1680 	}
   1681 
   1682 	/* Notify rnd_detach_source that the request is done.  */
   1683 	rnd_unlock_sources();
   1684 }
   1685 
   1686 /*
   1687  * rnd_add_uint32(rs, value)
   1688  *
   1689  *	Enter 32 bits of data from an entropy source into the pool.
   1690  *
   1691  *	If rs is NULL, may not be called from interrupt context.
   1692  *
   1693  *	If rs is non-NULL, may be called from any context.  May drop
   1694  *	data if called from interrupt context.
   1695  */
   1696 void
   1697 rnd_add_uint32(struct krndsource *rs, uint32_t value)
   1698 {
   1699 
   1700 	rnd_add_data(rs, &value, sizeof value, 0);
   1701 }
   1702 
   1703 void
   1704 _rnd_add_uint32(struct krndsource *rs, uint32_t value)
   1705 {
   1706 
   1707 	rnd_add_data(rs, &value, sizeof value, 0);
   1708 }
   1709 
   1710 void
   1711 _rnd_add_uint64(struct krndsource *rs, uint64_t value)
   1712 {
   1713 
   1714 	rnd_add_data(rs, &value, sizeof value, 0);
   1715 }
   1716 
   1717 /*
   1718  * rnd_add_data(rs, buf, len, entropybits)
   1719  *
   1720  *	Enter data from an entropy source into the pool, with a
   1721  *	driver's estimate of how much entropy the physical source of
   1722  *	the data has.  If RND_FLAG_NO_ESTIMATE, we ignore the driver's
   1723  *	estimate and treat it as zero.
   1724  *
   1725  *	If rs is NULL, may not be called from interrupt context.
   1726  *
   1727  *	If rs is non-NULL, may be called from any context.  May drop
   1728  *	data if called from interrupt context.
   1729  */
   1730 void
   1731 rnd_add_data(struct krndsource *rs, const void *buf, uint32_t len,
   1732     uint32_t entropybits)
   1733 {
   1734 	uint32_t extra;
   1735 	uint32_t flags;
   1736 
   1737 	KASSERTMSG(howmany(entropybits, NBBY) <= len,
   1738 	    "%s: impossible entropy rate:"
   1739 	    " %"PRIu32" bits in %"PRIu32"-byte string",
   1740 	    rs ? rs->name : "(anonymous)", entropybits, len);
   1741 
   1742 	/* If there's no rndsource, just enter the data and time now.  */
   1743 	if (rs == NULL) {
   1744 		entropy_enter(buf, len, entropybits);
   1745 		extra = entropy_timer();
   1746 		entropy_enter(&extra, sizeof extra, 0);
   1747 		explicit_memset(&extra, 0, sizeof extra);
   1748 		return;
   1749 	}
   1750 
   1751 	/* Load a snapshot of the flags.  Ioctl may change them under us.  */
   1752 	flags = atomic_load_relaxed(&rs->flags);
   1753 
   1754 	/*
   1755 	 * Skip if:
   1756 	 * - we're not collecting entropy, or
   1757 	 * - the operator doesn't want to collect entropy from this, or
   1758 	 * - neither data nor timings are being collected from this.
   1759 	 */
   1760 	if (!atomic_load_relaxed(&entropy_collection) ||
   1761 	    ISSET(flags, RND_FLAG_NO_COLLECT) ||
   1762 	    !ISSET(flags, RND_FLAG_COLLECT_VALUE|RND_FLAG_COLLECT_TIME))
   1763 		return;
   1764 
   1765 	/* If asked, ignore the estimate.  */
   1766 	if (ISSET(flags, RND_FLAG_NO_ESTIMATE))
   1767 		entropybits = 0;
   1768 
   1769 	/* If we are collecting data, enter them.  */
   1770 	if (ISSET(flags, RND_FLAG_COLLECT_VALUE))
   1771 		rnd_add_data_1(rs, buf, len, entropybits);
   1772 
   1773 	/* If we are collecting timings, enter one.  */
   1774 	if (ISSET(flags, RND_FLAG_COLLECT_TIME)) {
   1775 		extra = entropy_timer();
   1776 		rnd_add_data_1(rs, &extra, sizeof extra, 0);
   1777 	}
   1778 }
   1779 
   1780 /*
   1781  * rnd_add_data_1(rs, buf, len, entropybits)
   1782  *
   1783  *	Internal subroutine to call either entropy_enter_intr, if we're
   1784  *	in interrupt context, or entropy_enter if not, and to count the
   1785  *	entropy in an rndsource.
   1786  */
   1787 static void
   1788 rnd_add_data_1(struct krndsource *rs, const void *buf, uint32_t len,
   1789     uint32_t entropybits)
   1790 {
   1791 	bool fullyused;
   1792 
   1793 	/*
   1794 	 * If we're in interrupt context, use entropy_enter_intr and
   1795 	 * take note of whether it consumed the full sample; if not,
   1796 	 * use entropy_enter, which always consumes the full sample.
   1797 	 */
   1798 	if (curcpu_available() && cpu_intr_p()) {
   1799 		fullyused = entropy_enter_intr(buf, len, entropybits);
   1800 	} else {
   1801 		entropy_enter(buf, len, entropybits);
   1802 		fullyused = true;
   1803 	}
   1804 
   1805 	/*
   1806 	 * If we used the full sample, note how many bits were
   1807 	 * contributed from this source.
   1808 	 */
   1809 	if (fullyused) {
   1810 		if (E->stage < ENTROPY_HOT) {
   1811 			if (E->stage >= ENTROPY_WARM)
   1812 				mutex_enter(&E->lock);
   1813 			rs->total += MIN(UINT_MAX - rs->total, entropybits);
   1814 			if (E->stage >= ENTROPY_WARM)
   1815 				mutex_exit(&E->lock);
   1816 		} else {
   1817 			struct rndsource_cpu *rc = percpu_getref(rs->state);
   1818 			unsigned nbits = rc->rc_nbits;
   1819 
   1820 			nbits += MIN(UINT_MAX - nbits, entropybits);
   1821 			atomic_store_relaxed(&rc->rc_nbits, nbits);
   1822 			percpu_putref(rs->state);
   1823 		}
   1824 	}
   1825 }
   1826 
   1827 /*
   1828  * rnd_add_data_sync(rs, buf, len, entropybits)
   1829  *
   1830  *	Same as rnd_add_data.  Originally used in rndsource callbacks,
   1831  *	to break an unnecessary cycle; no longer really needed.
   1832  */
   1833 void
   1834 rnd_add_data_sync(struct krndsource *rs, const void *buf, uint32_t len,
   1835     uint32_t entropybits)
   1836 {
   1837 
   1838 	rnd_add_data(rs, buf, len, entropybits);
   1839 }
   1840 
   1841 /*
   1842  * rndsource_entropybits(rs)
   1843  *
   1844  *	Return approximately the number of bits of entropy that have
   1845  *	been contributed via rs so far.  Approximate if other CPUs may
   1846  *	be calling rnd_add_data concurrently.
   1847  */
   1848 static unsigned
   1849 rndsource_entropybits(struct krndsource *rs)
   1850 {
   1851 	unsigned nbits = rs->total;
   1852 
   1853 	KASSERT(E->stage >= ENTROPY_WARM);
   1854 	KASSERT(rnd_sources_locked());
   1855 	percpu_foreach(rs->state, rndsource_entropybits_cpu, &nbits);
   1856 	return nbits;
   1857 }
   1858 
   1859 static void
   1860 rndsource_entropybits_cpu(void *ptr, void *cookie, struct cpu_info *ci)
   1861 {
   1862 	struct rndsource_cpu *rc = ptr;
   1863 	unsigned *nbitsp = cookie;
   1864 	unsigned cpu_nbits;
   1865 
   1866 	cpu_nbits = atomic_load_relaxed(&rc->rc_nbits);
   1867 	*nbitsp += MIN(UINT_MAX - *nbitsp, cpu_nbits);
   1868 }
   1869 
   1870 /*
   1871  * rndsource_to_user(rs, urs)
   1872  *
   1873  *	Copy a description of rs out to urs for userland.
   1874  */
   1875 static void
   1876 rndsource_to_user(struct krndsource *rs, rndsource_t *urs)
   1877 {
   1878 
   1879 	KASSERT(E->stage >= ENTROPY_WARM);
   1880 	KASSERT(rnd_sources_locked());
   1881 
   1882 	/* Avoid kernel memory disclosure.  */
   1883 	memset(urs, 0, sizeof(*urs));
   1884 
   1885 	CTASSERT(sizeof(urs->name) == sizeof(rs->name));
   1886 	strlcpy(urs->name, rs->name, sizeof(urs->name));
   1887 	urs->total = rndsource_entropybits(rs);
   1888 	urs->type = rs->type;
   1889 	urs->flags = atomic_load_relaxed(&rs->flags);
   1890 }
   1891 
   1892 /*
   1893  * rndsource_to_user_est(rs, urse)
   1894  *
   1895  *	Copy a description of rs and estimation statistics out to urse
   1896  *	for userland.
   1897  */
   1898 static void
   1899 rndsource_to_user_est(struct krndsource *rs, rndsource_est_t *urse)
   1900 {
   1901 
   1902 	KASSERT(E->stage >= ENTROPY_WARM);
   1903 	KASSERT(rnd_sources_locked());
   1904 
   1905 	/* Avoid kernel memory disclosure.  */
   1906 	memset(urse, 0, sizeof(*urse));
   1907 
   1908 	/* Copy out the rndsource description.  */
   1909 	rndsource_to_user(rs, &urse->rt);
   1910 
   1911 	/* Zero out the statistics because we don't do estimation.  */
   1912 	urse->dt_samples = 0;
   1913 	urse->dt_total = 0;
   1914 	urse->dv_samples = 0;
   1915 	urse->dv_total = 0;
   1916 }
   1917 
   1918 /*
   1919  * entropy_ioctl(cmd, data)
   1920  *
   1921  *	Handle various /dev/random ioctl queries.
   1922  */
   1923 int
   1924 entropy_ioctl(unsigned long cmd, void *data)
   1925 {
   1926 	struct krndsource *rs;
   1927 	bool privileged;
   1928 	int error;
   1929 
   1930 	KASSERT(E->stage >= ENTROPY_WARM);
   1931 
   1932 	/* Verify user's authorization to perform the ioctl.  */
   1933 	switch (cmd) {
   1934 	case RNDGETENTCNT:
   1935 	case RNDGETPOOLSTAT:
   1936 	case RNDGETSRCNUM:
   1937 	case RNDGETSRCNAME:
   1938 	case RNDGETESTNUM:
   1939 	case RNDGETESTNAME:
   1940 		error = kauth_authorize_device(curlwp->l_cred,
   1941 		    KAUTH_DEVICE_RND_GETPRIV, NULL, NULL, NULL, NULL);
   1942 		break;
   1943 	case RNDCTL:
   1944 		error = kauth_authorize_device(curlwp->l_cred,
   1945 		    KAUTH_DEVICE_RND_SETPRIV, NULL, NULL, NULL, NULL);
   1946 		break;
   1947 	case RNDADDDATA:
   1948 		error = kauth_authorize_device(curlwp->l_cred,
   1949 		    KAUTH_DEVICE_RND_ADDDATA, NULL, NULL, NULL, NULL);
   1950 		/* Ascertain whether the user's inputs should be counted.  */
   1951 		if (kauth_authorize_device(curlwp->l_cred,
   1952 			KAUTH_DEVICE_RND_ADDDATA_ESTIMATE,
   1953 			NULL, NULL, NULL, NULL) == 0)
   1954 			privileged = true;
   1955 		break;
   1956 	default: {
   1957 		/*
   1958 		 * XXX Hack to avoid changing module ABI so this can be
   1959 		 * pulled up.  Later, we can just remove the argument.
   1960 		 */
   1961 		static const struct fileops fops = {
   1962 			.fo_ioctl = rnd_system_ioctl,
   1963 		};
   1964 		struct file f = {
   1965 			.f_ops = &fops,
   1966 		};
   1967 		MODULE_HOOK_CALL(rnd_ioctl_50_hook, (&f, cmd, data),
   1968 		    enosys(), error);
   1969 #if defined(_LP64)
   1970 		if (error == ENOSYS)
   1971 			MODULE_HOOK_CALL(rnd_ioctl32_50_hook, (&f, cmd, data),
   1972 			    enosys(), error);
   1973 #endif
   1974 		if (error == ENOSYS)
   1975 			error = ENOTTY;
   1976 		break;
   1977 	}
   1978 	}
   1979 
   1980 	/* If anything went wrong with authorization, stop here.  */
   1981 	if (error)
   1982 		return error;
   1983 
   1984 	/* Dispatch on the command.  */
   1985 	switch (cmd) {
   1986 	case RNDGETENTCNT: {	/* Get current entropy count in bits.  */
   1987 		uint32_t *countp = data;
   1988 
   1989 		mutex_enter(&E->lock);
   1990 		*countp = ENTROPY_CAPACITY*NBBY - E->needed;
   1991 		mutex_exit(&E->lock);
   1992 
   1993 		break;
   1994 	}
   1995 	case RNDGETPOOLSTAT: {	/* Get entropy pool statistics.  */
   1996 		rndpoolstat_t *pstat = data;
   1997 
   1998 		mutex_enter(&E->lock);
   1999 
   2000 		/* parameters */
   2001 		pstat->poolsize = ENTPOOL_SIZE/sizeof(uint32_t); /* words */
   2002 		pstat->threshold = ENTROPY_CAPACITY*1; /* bytes */
   2003 		pstat->maxentropy = ENTROPY_CAPACITY*NBBY; /* bits */
   2004 
   2005 		/* state */
   2006 		pstat->added = 0; /* XXX total entropy_enter count */
   2007 		pstat->curentropy = ENTROPY_CAPACITY*NBBY - E->needed;
   2008 		pstat->removed = 0; /* XXX total entropy_extract count */
   2009 		pstat->discarded = 0; /* XXX bits of entropy beyond capacity */
   2010 		pstat->generated = 0; /* XXX bits of data...fabricated? */
   2011 
   2012 		mutex_exit(&E->lock);
   2013 		break;
   2014 	}
   2015 	case RNDGETSRCNUM: {	/* Get entropy sources by number.  */
   2016 		rndstat_t *stat = data;
   2017 		uint32_t start = 0, i = 0;
   2018 
   2019 		/* Skip if none requested; fail if too many requested.  */
   2020 		if (stat->count == 0)
   2021 			break;
   2022 		if (stat->count > RND_MAXSTATCOUNT)
   2023 			return EINVAL;
   2024 
   2025 		/*
   2026 		 * Under the lock, find the first one, copy out as many
   2027 		 * as requested, and report how many we copied out.
   2028 		 */
   2029 		mutex_enter(&E->lock);
   2030 		error = rnd_lock_sources();
   2031 		if (error) {
   2032 			mutex_exit(&E->lock);
   2033 			return error;
   2034 		}
   2035 		LIST_FOREACH(rs, &E->sources, list) {
   2036 			if (start++ == stat->start)
   2037 				break;
   2038 		}
   2039 		while (i < stat->count && rs != NULL) {
   2040 			mutex_exit(&E->lock);
   2041 			rndsource_to_user(rs, &stat->source[i++]);
   2042 			mutex_enter(&E->lock);
   2043 			rs = LIST_NEXT(rs, list);
   2044 		}
   2045 		KASSERT(i <= stat->count);
   2046 		stat->count = i;
   2047 		rnd_unlock_sources();
   2048 		mutex_exit(&E->lock);
   2049 		break;
   2050 	}
   2051 	case RNDGETESTNUM: {	/* Get sources and estimates by number.  */
   2052 		rndstat_est_t *estat = data;
   2053 		uint32_t start = 0, i = 0;
   2054 
   2055 		/* Skip if none requested; fail if too many requested.  */
   2056 		if (estat->count == 0)
   2057 			break;
   2058 		if (estat->count > RND_MAXSTATCOUNT)
   2059 			return EINVAL;
   2060 
   2061 		/*
   2062 		 * Under the lock, find the first one, copy out as many
   2063 		 * as requested, and report how many we copied out.
   2064 		 */
   2065 		mutex_enter(&E->lock);
   2066 		error = rnd_lock_sources();
   2067 		if (error) {
   2068 			mutex_exit(&E->lock);
   2069 			return error;
   2070 		}
   2071 		LIST_FOREACH(rs, &E->sources, list) {
   2072 			if (start++ == estat->start)
   2073 				break;
   2074 		}
   2075 		while (i < estat->count && rs != NULL) {
   2076 			mutex_exit(&E->lock);
   2077 			rndsource_to_user_est(rs, &estat->source[i++]);
   2078 			mutex_enter(&E->lock);
   2079 			rs = LIST_NEXT(rs, list);
   2080 		}
   2081 		KASSERT(i <= estat->count);
   2082 		estat->count = i;
   2083 		rnd_unlock_sources();
   2084 		mutex_exit(&E->lock);
   2085 		break;
   2086 	}
   2087 	case RNDGETSRCNAME: {	/* Get entropy sources by name.  */
   2088 		rndstat_name_t *nstat = data;
   2089 		const size_t n = sizeof(rs->name);
   2090 
   2091 		CTASSERT(sizeof(rs->name) == sizeof(nstat->name));
   2092 
   2093 		/*
   2094 		 * Under the lock, search by name.  If found, copy it
   2095 		 * out; if not found, fail with ENOENT.
   2096 		 */
   2097 		mutex_enter(&E->lock);
   2098 		error = rnd_lock_sources();
   2099 		if (error) {
   2100 			mutex_exit(&E->lock);
   2101 			return error;
   2102 		}
   2103 		LIST_FOREACH(rs, &E->sources, list) {
   2104 			if (strncmp(rs->name, nstat->name, n) == 0)
   2105 				break;
   2106 		}
   2107 		if (rs != NULL) {
   2108 			mutex_exit(&E->lock);
   2109 			rndsource_to_user(rs, &nstat->source);
   2110 			mutex_enter(&E->lock);
   2111 		} else {
   2112 			error = ENOENT;
   2113 		}
   2114 		rnd_unlock_sources();
   2115 		mutex_exit(&E->lock);
   2116 		break;
   2117 	}
   2118 	case RNDGETESTNAME: {	/* Get sources and estimates by name.  */
   2119 		rndstat_est_name_t *enstat = data;
   2120 		const size_t n = sizeof(rs->name);
   2121 
   2122 		CTASSERT(sizeof(rs->name) == sizeof(enstat->name));
   2123 
   2124 		/*
   2125 		 * Under the lock, search by name.  If found, copy it
   2126 		 * out; if not found, fail with ENOENT.
   2127 		 */
   2128 		mutex_enter(&E->lock);
   2129 		error = rnd_lock_sources();
   2130 		if (error) {
   2131 			mutex_exit(&E->lock);
   2132 			return error;
   2133 		}
   2134 		LIST_FOREACH(rs, &E->sources, list) {
   2135 			if (strncmp(rs->name, enstat->name, n) == 0)
   2136 				break;
   2137 		}
   2138 		if (rs != NULL) {
   2139 			mutex_exit(&E->lock);
   2140 			rndsource_to_user_est(rs, &enstat->source);
   2141 			mutex_enter(&E->lock);
   2142 		} else {
   2143 			error = ENOENT;
   2144 		}
   2145 		rnd_unlock_sources();
   2146 		mutex_exit(&E->lock);
   2147 		break;
   2148 	}
   2149 	case RNDCTL: {		/* Modify entropy source flags.  */
   2150 		rndctl_t *rndctl = data;
   2151 		const size_t n = sizeof(rs->name);
   2152 		uint32_t flags;
   2153 
   2154 		CTASSERT(sizeof(rs->name) == sizeof(rndctl->name));
   2155 
   2156 		/* Whitelist the flags that user can change.  */
   2157 		rndctl->mask &= RND_FLAG_NO_ESTIMATE|RND_FLAG_NO_COLLECT;
   2158 
   2159 		/*
   2160 		 * For each matching rndsource, either by type if
   2161 		 * specified or by name if not, set the masked flags.
   2162 		 */
   2163 		mutex_enter(&E->lock);
   2164 		LIST_FOREACH(rs, &E->sources, list) {
   2165 			if (rndctl->type != 0xff) {
   2166 				if (rs->type != rndctl->type)
   2167 					continue;
   2168 			} else {
   2169 				if (strncmp(rs->name, rndctl->name, n) != 0)
   2170 					continue;
   2171 			}
   2172 			flags = rs->flags & ~rndctl->mask;
   2173 			flags |= rndctl->flags & rndctl->mask;
   2174 			atomic_store_relaxed(&rs->flags, flags);
   2175 		}
   2176 		mutex_exit(&E->lock);
   2177 		break;
   2178 	}
   2179 	case RNDADDDATA: {	/* Enter seed into entropy pool.  */
   2180 		rnddata_t *rdata = data;
   2181 		unsigned entropybits = 0;
   2182 
   2183 		if (!atomic_load_relaxed(&entropy_collection))
   2184 			break;	/* thanks but no thanks */
   2185 		if (rdata->len > MIN(sizeof(rdata->data), UINT32_MAX/NBBY))
   2186 			return EINVAL;
   2187 
   2188 		/*
   2189 		 * This ioctl serves as the userland alternative a
   2190 		 * bootloader-provided seed -- typically furnished by
   2191 		 * /etc/rc.d/random_seed.  We accept the user's entropy
   2192 		 * claim only if
   2193 		 *
   2194 		 * (a) the user is privileged, and
   2195 		 * (b) we have not entered a bootloader seed.
   2196 		 *
   2197 		 * under the assumption that the user may use this to
   2198 		 * load a seed from disk that we have already loaded
   2199 		 * from the bootloader, so we don't double-count it.
   2200 		 */
   2201 		if (privileged && rdata->entropy && rdata->len) {
   2202 			mutex_enter(&E->lock);
   2203 			if (!E->seeded) {
   2204 				entropybits = MIN(rdata->entropy,
   2205 				    MIN(rdata->len, ENTROPY_CAPACITY)*NBBY);
   2206 				E->seeded = true;
   2207 			}
   2208 			mutex_exit(&E->lock);
   2209 		}
   2210 
   2211 		/* Enter the data.  */
   2212 		rnd_add_data(&seed_rndsource, rdata->data, rdata->len,
   2213 		    entropybits);
   2214 		break;
   2215 	}
   2216 	default:
   2217 		error = ENOTTY;
   2218 	}
   2219 
   2220 	/* Return any error that may have come up.  */
   2221 	return error;
   2222 }
   2223 
   2224 /* Legacy entry points */
   2225 
   2226 void
   2227 rnd_seed(void *seed, size_t len)
   2228 {
   2229 
   2230 	if (len != sizeof(rndsave_t)) {
   2231 		printf("entropy: invalid seed length: %zu,"
   2232 		    " expected sizeof(rndsave_t) = %zu\n",
   2233 		    len, sizeof(rndsave_t));
   2234 		return;
   2235 	}
   2236 	entropy_seed(seed);
   2237 }
   2238 
   2239 void
   2240 rnd_init(void)
   2241 {
   2242 
   2243 	entropy_init();
   2244 }
   2245 
   2246 void
   2247 rnd_init_softint(void)
   2248 {
   2249 
   2250 	entropy_init_late();
   2251 }
   2252 
   2253 int
   2254 rnd_system_ioctl(struct file *fp, unsigned long cmd, void *data)
   2255 {
   2256 
   2257 	return entropy_ioctl(cmd, data);
   2258 }
   2259