Home | History | Annotate | Line # | Download | only in kern
kern_entropy.c revision 1.39
      1  1.39  riastrad /*	$NetBSD: kern_entropy.c,v 1.39 2022/03/18 23:35:19 riastradh Exp $	*/
      2   1.1  riastrad 
      3   1.1  riastrad /*-
      4   1.1  riastrad  * Copyright (c) 2019 The NetBSD Foundation, Inc.
      5   1.1  riastrad  * All rights reserved.
      6   1.1  riastrad  *
      7   1.1  riastrad  * This code is derived from software contributed to The NetBSD Foundation
      8   1.1  riastrad  * by Taylor R. Campbell.
      9   1.1  riastrad  *
     10   1.1  riastrad  * Redistribution and use in source and binary forms, with or without
     11   1.1  riastrad  * modification, are permitted provided that the following conditions
     12   1.1  riastrad  * are met:
     13   1.1  riastrad  * 1. Redistributions of source code must retain the above copyright
     14   1.1  riastrad  *    notice, this list of conditions and the following disclaimer.
     15   1.1  riastrad  * 2. Redistributions in binary form must reproduce the above copyright
     16   1.1  riastrad  *    notice, this list of conditions and the following disclaimer in the
     17   1.1  riastrad  *    documentation and/or other materials provided with the distribution.
     18   1.1  riastrad  *
     19   1.1  riastrad  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
     20   1.1  riastrad  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
     21   1.1  riastrad  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
     22   1.1  riastrad  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
     23   1.1  riastrad  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
     24   1.1  riastrad  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
     25   1.1  riastrad  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
     26   1.1  riastrad  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
     27   1.1  riastrad  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
     28   1.1  riastrad  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
     29   1.1  riastrad  * POSSIBILITY OF SUCH DAMAGE.
     30   1.1  riastrad  */
     31   1.1  riastrad 
     32   1.1  riastrad /*
     33   1.1  riastrad  * Entropy subsystem
     34   1.1  riastrad  *
     35   1.1  riastrad  *	* Each CPU maintains a per-CPU entropy pool so that gathering
     36   1.1  riastrad  *	  entropy requires no interprocessor synchronization, except
     37   1.1  riastrad  *	  early at boot when we may be scrambling to gather entropy as
     38   1.1  riastrad  *	  soon as possible.
     39   1.1  riastrad  *
     40   1.1  riastrad  *	  - entropy_enter gathers entropy and never drops it on the
     41   1.1  riastrad  *	    floor, at the cost of sometimes having to do cryptography.
     42   1.1  riastrad  *
     43   1.1  riastrad  *	  - entropy_enter_intr gathers entropy or drops it on the
     44   1.1  riastrad  *	    floor, with low latency.  Work to stir the pool or kick the
     45   1.1  riastrad  *	    housekeeping thread is scheduled in soft interrupts.
     46   1.1  riastrad  *
     47   1.1  riastrad  *	* entropy_enter immediately enters into the global pool if it
     48   1.1  riastrad  *	  can transition to full entropy in one swell foop.  Otherwise,
     49   1.1  riastrad  *	  it defers to a housekeeping thread that consolidates entropy,
     50   1.1  riastrad  *	  but only when the CPUs collectively have full entropy, in
     51   1.1  riastrad  *	  order to mitigate iterative-guessing attacks.
     52   1.1  riastrad  *
     53   1.1  riastrad  *	* The entropy housekeeping thread continues to consolidate
     54   1.1  riastrad  *	  entropy even after we think we have full entropy, in case we
     55   1.1  riastrad  *	  are wrong, but is limited to one discretionary consolidation
     56   1.1  riastrad  *	  per minute, and only when new entropy is actually coming in,
     57   1.1  riastrad  *	  to limit performance impact.
     58   1.1  riastrad  *
     59   1.1  riastrad  *	* The entropy epoch is the number that changes when we
     60   1.1  riastrad  *	  transition from partial entropy to full entropy, so that
     61   1.1  riastrad  *	  users can easily determine when to reseed.  This also
     62   1.1  riastrad  *	  facilitates an operator explicitly causing everything to
     63  1.13  riastrad  *	  reseed by sysctl -w kern.entropy.consolidate=1.
     64   1.1  riastrad  *
     65   1.1  riastrad  *	* No entropy estimation based on the sample values, which is a
     66   1.1  riastrad  *	  contradiction in terms and a potential source of side
     67   1.1  riastrad  *	  channels.  It is the responsibility of the driver author to
     68   1.1  riastrad  *	  study how predictable the physical source of input can ever
     69   1.1  riastrad  *	  be, and to furnish a lower bound on the amount of entropy it
     70   1.1  riastrad  *	  has.
     71   1.1  riastrad  *
     72   1.1  riastrad  *	* Entropy depletion is available for testing (or if you're into
     73   1.1  riastrad  *	  that sort of thing), with sysctl -w kern.entropy.depletion=1;
     74   1.1  riastrad  *	  the logic to support it is small, to minimize chance of bugs.
     75   1.1  riastrad  */
     76   1.1  riastrad 
     77   1.1  riastrad #include <sys/cdefs.h>
     78  1.39  riastrad __KERNEL_RCSID(0, "$NetBSD: kern_entropy.c,v 1.39 2022/03/18 23:35:19 riastradh Exp $");
     79   1.1  riastrad 
     80   1.1  riastrad #include <sys/param.h>
     81   1.1  riastrad #include <sys/types.h>
     82   1.1  riastrad #include <sys/atomic.h>
     83   1.1  riastrad #include <sys/compat_stub.h>
     84   1.1  riastrad #include <sys/condvar.h>
     85   1.1  riastrad #include <sys/cpu.h>
     86   1.1  riastrad #include <sys/entropy.h>
     87   1.1  riastrad #include <sys/errno.h>
     88   1.1  riastrad #include <sys/evcnt.h>
     89   1.1  riastrad #include <sys/event.h>
     90   1.1  riastrad #include <sys/file.h>
     91   1.1  riastrad #include <sys/intr.h>
     92   1.1  riastrad #include <sys/kauth.h>
     93   1.1  riastrad #include <sys/kernel.h>
     94   1.1  riastrad #include <sys/kmem.h>
     95   1.1  riastrad #include <sys/kthread.h>
     96   1.1  riastrad #include <sys/module_hook.h>
     97   1.1  riastrad #include <sys/mutex.h>
     98   1.1  riastrad #include <sys/percpu.h>
     99   1.1  riastrad #include <sys/poll.h>
    100   1.1  riastrad #include <sys/queue.h>
    101  1.30  jmcneill #include <sys/reboot.h>
    102   1.1  riastrad #include <sys/rnd.h>		/* legacy kernel API */
    103   1.1  riastrad #include <sys/rndio.h>		/* userland ioctl interface */
    104   1.1  riastrad #include <sys/rndsource.h>	/* kernel rndsource driver API */
    105   1.1  riastrad #include <sys/select.h>
    106   1.1  riastrad #include <sys/selinfo.h>
    107   1.1  riastrad #include <sys/sha1.h>		/* for boot seed checksum */
    108   1.1  riastrad #include <sys/stdint.h>
    109   1.1  riastrad #include <sys/sysctl.h>
    110  1.26  riastrad #include <sys/syslog.h>
    111   1.1  riastrad #include <sys/systm.h>
    112   1.1  riastrad #include <sys/time.h>
    113   1.1  riastrad #include <sys/xcall.h>
    114   1.1  riastrad 
    115   1.1  riastrad #include <lib/libkern/entpool.h>
    116   1.1  riastrad 
    117   1.1  riastrad #include <machine/limits.h>
    118   1.1  riastrad 
    119   1.1  riastrad #ifdef __HAVE_CPU_COUNTER
    120   1.1  riastrad #include <machine/cpu_counter.h>
    121   1.1  riastrad #endif
    122   1.1  riastrad 
    123   1.1  riastrad /*
    124   1.1  riastrad  * struct entropy_cpu
    125   1.1  riastrad  *
    126   1.1  riastrad  *	Per-CPU entropy state.  The pool is allocated separately
    127   1.1  riastrad  *	because percpu(9) sometimes moves per-CPU objects around
    128   1.1  riastrad  *	without zeroing them, which would lead to unwanted copies of
    129  1.34    andvar  *	sensitive secrets.  The evcnt is allocated separately because
    130   1.1  riastrad  *	evcnt(9) assumes it stays put in memory.
    131   1.1  riastrad  */
    132   1.1  riastrad struct entropy_cpu {
    133   1.1  riastrad 	struct evcnt		*ec_softint_evcnt;
    134   1.1  riastrad 	struct entpool		*ec_pool;
    135   1.1  riastrad 	unsigned		ec_pending;
    136   1.1  riastrad 	bool			ec_locked;
    137   1.1  riastrad };
    138   1.1  riastrad 
    139   1.1  riastrad /*
    140   1.1  riastrad  * struct rndsource_cpu
    141   1.1  riastrad  *
    142   1.1  riastrad  *	Per-CPU rndsource state.
    143   1.1  riastrad  */
    144   1.1  riastrad struct rndsource_cpu {
    145  1.28  riastrad 	unsigned		rc_entropybits;
    146  1.28  riastrad 	unsigned		rc_timesamples;
    147  1.28  riastrad 	unsigned		rc_datasamples;
    148   1.1  riastrad };
    149   1.1  riastrad 
    150   1.1  riastrad /*
    151   1.1  riastrad  * entropy_global (a.k.a. E for short in this file)
    152   1.1  riastrad  *
    153   1.1  riastrad  *	Global entropy state.  Writes protected by the global lock.
    154   1.1  riastrad  *	Some fields, marked (A), can be read outside the lock, and are
    155   1.1  riastrad  *	maintained with atomic_load/store_relaxed.
    156   1.1  riastrad  */
    157   1.1  riastrad struct {
    158   1.1  riastrad 	kmutex_t	lock;		/* covers all global state */
    159   1.1  riastrad 	struct entpool	pool;		/* global pool for extraction */
    160   1.1  riastrad 	unsigned	needed;		/* (A) needed globally */
    161   1.1  riastrad 	unsigned	pending;	/* (A) pending in per-CPU pools */
    162   1.1  riastrad 	unsigned	timestamp;	/* (A) time of last consolidation */
    163   1.1  riastrad 	unsigned	epoch;		/* (A) changes when needed -> 0 */
    164   1.1  riastrad 	kcondvar_t	cv;		/* notifies state changes */
    165   1.1  riastrad 	struct selinfo	selq;		/* notifies needed -> 0 */
    166   1.4  riastrad 	struct lwp	*sourcelock;	/* lock on list of sources */
    167  1.27  riastrad 	kcondvar_t	sourcelock_cv;	/* notifies sourcelock release */
    168   1.1  riastrad 	LIST_HEAD(,krndsource) sources;	/* list of entropy sources */
    169   1.1  riastrad 	enum entropy_stage {
    170   1.1  riastrad 		ENTROPY_COLD = 0, /* single-threaded */
    171   1.1  riastrad 		ENTROPY_WARM,	  /* multi-threaded at boot before CPUs */
    172   1.1  riastrad 		ENTROPY_HOT,	  /* multi-threaded multi-CPU */
    173   1.1  riastrad 	}		stage;
    174   1.1  riastrad 	bool		consolidate;	/* kick thread to consolidate */
    175   1.1  riastrad 	bool		seed_rndsource;	/* true if seed source is attached */
    176   1.1  riastrad 	bool		seeded;		/* true if seed file already loaded */
    177   1.1  riastrad } entropy_global __cacheline_aligned = {
    178   1.1  riastrad 	/* Fields that must be initialized when the kernel is loaded.  */
    179   1.1  riastrad 	.needed = ENTROPY_CAPACITY*NBBY,
    180  1.14  riastrad 	.epoch = (unsigned)-1,	/* -1 means entropy never consolidated */
    181   1.1  riastrad 	.sources = LIST_HEAD_INITIALIZER(entropy_global.sources),
    182   1.1  riastrad 	.stage = ENTROPY_COLD,
    183   1.1  riastrad };
    184   1.1  riastrad 
    185   1.1  riastrad #define	E	(&entropy_global)	/* declutter */
    186   1.1  riastrad 
    187   1.1  riastrad /* Read-mostly globals */
    188   1.1  riastrad static struct percpu	*entropy_percpu __read_mostly; /* struct entropy_cpu */
    189   1.1  riastrad static void		*entropy_sih __read_mostly; /* softint handler */
    190   1.1  riastrad static struct lwp	*entropy_lwp __read_mostly; /* housekeeping thread */
    191   1.1  riastrad 
    192   1.1  riastrad int rnd_initial_entropy __read_mostly; /* XXX legacy */
    193   1.1  riastrad 
    194   1.1  riastrad static struct krndsource seed_rndsource __read_mostly;
    195   1.1  riastrad 
    196   1.1  riastrad /*
    197   1.1  riastrad  * Event counters
    198   1.1  riastrad  *
    199   1.1  riastrad  *	Must be careful with adding these because they can serve as
    200   1.1  riastrad  *	side channels.
    201   1.1  riastrad  */
    202   1.1  riastrad static struct evcnt entropy_discretionary_evcnt =
    203   1.1  riastrad     EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "entropy", "discretionary");
    204   1.1  riastrad EVCNT_ATTACH_STATIC(entropy_discretionary_evcnt);
    205   1.1  riastrad static struct evcnt entropy_immediate_evcnt =
    206   1.1  riastrad     EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "entropy", "immediate");
    207   1.1  riastrad EVCNT_ATTACH_STATIC(entropy_immediate_evcnt);
    208   1.1  riastrad static struct evcnt entropy_partial_evcnt =
    209   1.1  riastrad     EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "entropy", "partial");
    210   1.1  riastrad EVCNT_ATTACH_STATIC(entropy_partial_evcnt);
    211   1.1  riastrad static struct evcnt entropy_consolidate_evcnt =
    212   1.1  riastrad     EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "entropy", "consolidate");
    213   1.1  riastrad EVCNT_ATTACH_STATIC(entropy_consolidate_evcnt);
    214   1.1  riastrad static struct evcnt entropy_extract_fail_evcnt =
    215   1.1  riastrad     EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "entropy", "extract fail");
    216   1.1  riastrad EVCNT_ATTACH_STATIC(entropy_extract_fail_evcnt);
    217   1.1  riastrad static struct evcnt entropy_request_evcnt =
    218   1.1  riastrad     EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "entropy", "request");
    219   1.1  riastrad EVCNT_ATTACH_STATIC(entropy_request_evcnt);
    220   1.1  riastrad static struct evcnt entropy_deplete_evcnt =
    221   1.1  riastrad     EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "entropy", "deplete");
    222   1.1  riastrad EVCNT_ATTACH_STATIC(entropy_deplete_evcnt);
    223   1.1  riastrad static struct evcnt entropy_notify_evcnt =
    224   1.1  riastrad     EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "entropy", "notify");
    225   1.1  riastrad EVCNT_ATTACH_STATIC(entropy_notify_evcnt);
    226   1.1  riastrad 
    227   1.1  riastrad /* Sysctl knobs */
    228  1.17  riastrad static bool	entropy_collection = 1;
    229  1.17  riastrad static bool	entropy_depletion = 0; /* Silly!  */
    230   1.1  riastrad 
    231   1.1  riastrad static const struct sysctlnode	*entropy_sysctlroot;
    232   1.1  riastrad static struct sysctllog		*entropy_sysctllog;
    233   1.1  riastrad 
    234   1.1  riastrad /* Forward declarations */
    235   1.1  riastrad static void	entropy_init_cpu(void *, void *, struct cpu_info *);
    236   1.1  riastrad static void	entropy_fini_cpu(void *, void *, struct cpu_info *);
    237   1.1  riastrad static void	entropy_account_cpu(struct entropy_cpu *);
    238   1.1  riastrad static void	entropy_enter(const void *, size_t, unsigned);
    239   1.1  riastrad static bool	entropy_enter_intr(const void *, size_t, unsigned);
    240   1.1  riastrad static void	entropy_softintr(void *);
    241   1.1  riastrad static void	entropy_thread(void *);
    242   1.1  riastrad static uint32_t	entropy_pending(void);
    243   1.1  riastrad static void	entropy_pending_cpu(void *, void *, struct cpu_info *);
    244  1.13  riastrad static void	entropy_do_consolidate(void);
    245  1.13  riastrad static void	entropy_consolidate_xc(void *, void *);
    246   1.1  riastrad static void	entropy_notify(void);
    247   1.1  riastrad static int	sysctl_entropy_consolidate(SYSCTLFN_ARGS);
    248  1.10  riastrad static int	sysctl_entropy_gather(SYSCTLFN_ARGS);
    249   1.1  riastrad static void	filt_entropy_read_detach(struct knote *);
    250   1.1  riastrad static int	filt_entropy_read_event(struct knote *, long);
    251   1.1  riastrad static void	entropy_request(size_t);
    252   1.1  riastrad static void	rnd_add_data_1(struct krndsource *, const void *, uint32_t,
    253  1.28  riastrad 		    uint32_t, uint32_t);
    254   1.1  riastrad static unsigned	rndsource_entropybits(struct krndsource *);
    255   1.1  riastrad static void	rndsource_entropybits_cpu(void *, void *, struct cpu_info *);
    256   1.1  riastrad static void	rndsource_to_user(struct krndsource *, rndsource_t *);
    257   1.1  riastrad static void	rndsource_to_user_est(struct krndsource *, rndsource_est_t *);
    258  1.28  riastrad static void	rndsource_to_user_est_cpu(void *, void *, struct cpu_info *);
    259   1.1  riastrad 
    260   1.1  riastrad /*
    261   1.1  riastrad  * entropy_timer()
    262   1.1  riastrad  *
    263   1.1  riastrad  *	Cycle counter, time counter, or anything that changes a wee bit
    264   1.1  riastrad  *	unpredictably.
    265   1.1  riastrad  */
    266   1.1  riastrad static inline uint32_t
    267   1.1  riastrad entropy_timer(void)
    268   1.1  riastrad {
    269   1.1  riastrad 	struct bintime bt;
    270   1.1  riastrad 	uint32_t v;
    271   1.1  riastrad 
    272   1.1  riastrad 	/* If we have a CPU cycle counter, use the low 32 bits.  */
    273   1.1  riastrad #ifdef __HAVE_CPU_COUNTER
    274   1.1  riastrad 	if (__predict_true(cpu_hascounter()))
    275   1.1  riastrad 		return cpu_counter32();
    276   1.1  riastrad #endif	/* __HAVE_CPU_COUNTER */
    277   1.1  riastrad 
    278   1.1  riastrad 	/* If we're cold, tough.  Can't binuptime while cold.  */
    279   1.1  riastrad 	if (__predict_false(cold))
    280   1.1  riastrad 		return 0;
    281   1.1  riastrad 
    282   1.1  riastrad 	/* Fold the 128 bits of binuptime into 32 bits.  */
    283   1.1  riastrad 	binuptime(&bt);
    284   1.1  riastrad 	v = bt.frac;
    285   1.1  riastrad 	v ^= bt.frac >> 32;
    286   1.1  riastrad 	v ^= bt.sec;
    287   1.1  riastrad 	v ^= bt.sec >> 32;
    288   1.1  riastrad 	return v;
    289   1.1  riastrad }
    290   1.1  riastrad 
    291   1.1  riastrad static void
    292   1.1  riastrad attach_seed_rndsource(void)
    293   1.1  riastrad {
    294   1.1  riastrad 
    295   1.1  riastrad 	/*
    296   1.1  riastrad 	 * First called no later than entropy_init, while we are still
    297   1.1  riastrad 	 * single-threaded, so no need for RUN_ONCE.
    298   1.1  riastrad 	 */
    299   1.1  riastrad 	if (E->stage >= ENTROPY_WARM || E->seed_rndsource)
    300   1.1  riastrad 		return;
    301   1.1  riastrad 	rnd_attach_source(&seed_rndsource, "seed", RND_TYPE_UNKNOWN,
    302   1.1  riastrad 	    RND_FLAG_COLLECT_VALUE);
    303   1.1  riastrad 	E->seed_rndsource = true;
    304   1.1  riastrad }
    305   1.1  riastrad 
    306   1.1  riastrad /*
    307   1.1  riastrad  * entropy_init()
    308   1.1  riastrad  *
    309   1.1  riastrad  *	Initialize the entropy subsystem.  Panic on failure.
    310   1.1  riastrad  *
    311   1.1  riastrad  *	Requires percpu(9) and sysctl(9) to be initialized.
    312   1.1  riastrad  */
    313   1.1  riastrad static void
    314   1.1  riastrad entropy_init(void)
    315   1.1  riastrad {
    316   1.1  riastrad 	uint32_t extra[2];
    317   1.1  riastrad 	struct krndsource *rs;
    318   1.1  riastrad 	unsigned i = 0;
    319   1.1  riastrad 
    320   1.1  riastrad 	KASSERT(E->stage == ENTROPY_COLD);
    321   1.1  riastrad 
    322   1.1  riastrad 	/* Grab some cycle counts early at boot.  */
    323   1.1  riastrad 	extra[i++] = entropy_timer();
    324   1.1  riastrad 
    325   1.1  riastrad 	/* Run the entropy pool cryptography self-test.  */
    326   1.1  riastrad 	if (entpool_selftest() == -1)
    327   1.1  riastrad 		panic("entropy pool crypto self-test failed");
    328   1.1  riastrad 
    329   1.1  riastrad 	/* Create the sysctl directory.  */
    330   1.1  riastrad 	sysctl_createv(&entropy_sysctllog, 0, NULL, &entropy_sysctlroot,
    331   1.1  riastrad 	    CTLFLAG_PERMANENT, CTLTYPE_NODE, "entropy",
    332   1.1  riastrad 	    SYSCTL_DESCR("Entropy (random number sources) options"),
    333   1.1  riastrad 	    NULL, 0, NULL, 0,
    334   1.1  riastrad 	    CTL_KERN, CTL_CREATE, CTL_EOL);
    335   1.1  riastrad 
    336   1.1  riastrad 	/* Create the sysctl knobs.  */
    337   1.1  riastrad 	/* XXX These shouldn't be writable at securelevel>0.  */
    338   1.1  riastrad 	sysctl_createv(&entropy_sysctllog, 0, &entropy_sysctlroot, NULL,
    339   1.1  riastrad 	    CTLFLAG_PERMANENT|CTLFLAG_READWRITE, CTLTYPE_BOOL, "collection",
    340   1.1  riastrad 	    SYSCTL_DESCR("Automatically collect entropy from hardware"),
    341   1.1  riastrad 	    NULL, 0, &entropy_collection, 0, CTL_CREATE, CTL_EOL);
    342   1.1  riastrad 	sysctl_createv(&entropy_sysctllog, 0, &entropy_sysctlroot, NULL,
    343   1.1  riastrad 	    CTLFLAG_PERMANENT|CTLFLAG_READWRITE, CTLTYPE_BOOL, "depletion",
    344   1.1  riastrad 	    SYSCTL_DESCR("`Deplete' entropy pool when observed"),
    345   1.1  riastrad 	    NULL, 0, &entropy_depletion, 0, CTL_CREATE, CTL_EOL);
    346   1.1  riastrad 	sysctl_createv(&entropy_sysctllog, 0, &entropy_sysctlroot, NULL,
    347   1.1  riastrad 	    CTLFLAG_PERMANENT|CTLFLAG_READWRITE, CTLTYPE_INT, "consolidate",
    348   1.1  riastrad 	    SYSCTL_DESCR("Trigger entropy consolidation now"),
    349   1.1  riastrad 	    sysctl_entropy_consolidate, 0, NULL, 0, CTL_CREATE, CTL_EOL);
    350  1.10  riastrad 	sysctl_createv(&entropy_sysctllog, 0, &entropy_sysctlroot, NULL,
    351  1.10  riastrad 	    CTLFLAG_PERMANENT|CTLFLAG_READWRITE, CTLTYPE_INT, "gather",
    352  1.10  riastrad 	    SYSCTL_DESCR("Trigger entropy gathering from sources now"),
    353  1.10  riastrad 	    sysctl_entropy_gather, 0, NULL, 0, CTL_CREATE, CTL_EOL);
    354   1.1  riastrad 	/* XXX These should maybe not be readable at securelevel>0.  */
    355   1.1  riastrad 	sysctl_createv(&entropy_sysctllog, 0, &entropy_sysctlroot, NULL,
    356   1.1  riastrad 	    CTLFLAG_PERMANENT|CTLFLAG_READONLY|CTLFLAG_PRIVATE, CTLTYPE_INT,
    357   1.1  riastrad 	    "needed", SYSCTL_DESCR("Systemwide entropy deficit"),
    358   1.1  riastrad 	    NULL, 0, &E->needed, 0, CTL_CREATE, CTL_EOL);
    359   1.1  riastrad 	sysctl_createv(&entropy_sysctllog, 0, &entropy_sysctlroot, NULL,
    360   1.1  riastrad 	    CTLFLAG_PERMANENT|CTLFLAG_READONLY|CTLFLAG_PRIVATE, CTLTYPE_INT,
    361   1.1  riastrad 	    "pending", SYSCTL_DESCR("Entropy pending on CPUs"),
    362   1.1  riastrad 	    NULL, 0, &E->pending, 0, CTL_CREATE, CTL_EOL);
    363   1.1  riastrad 	sysctl_createv(&entropy_sysctllog, 0, &entropy_sysctlroot, NULL,
    364   1.1  riastrad 	    CTLFLAG_PERMANENT|CTLFLAG_READONLY|CTLFLAG_PRIVATE, CTLTYPE_INT,
    365   1.1  riastrad 	    "epoch", SYSCTL_DESCR("Entropy epoch"),
    366   1.1  riastrad 	    NULL, 0, &E->epoch, 0, CTL_CREATE, CTL_EOL);
    367   1.1  riastrad 
    368   1.1  riastrad 	/* Initialize the global state for multithreaded operation.  */
    369  1.39  riastrad 	mutex_init(&E->lock, MUTEX_DEFAULT, IPL_SOFTSERIAL);
    370   1.1  riastrad 	cv_init(&E->cv, "entropy");
    371   1.1  riastrad 	selinit(&E->selq);
    372  1.27  riastrad 	cv_init(&E->sourcelock_cv, "entsrclock");
    373   1.1  riastrad 
    374   1.1  riastrad 	/* Make sure the seed source is attached.  */
    375   1.1  riastrad 	attach_seed_rndsource();
    376   1.1  riastrad 
    377   1.1  riastrad 	/* Note if the bootloader didn't provide a seed.  */
    378   1.1  riastrad 	if (!E->seeded)
    379  1.29  riastrad 		aprint_debug("entropy: no seed from bootloader\n");
    380   1.1  riastrad 
    381   1.1  riastrad 	/* Allocate the per-CPU records for all early entropy sources.  */
    382   1.1  riastrad 	LIST_FOREACH(rs, &E->sources, list)
    383   1.1  riastrad 		rs->state = percpu_alloc(sizeof(struct rndsource_cpu));
    384   1.1  riastrad 
    385  1.36  riastrad 	/* Allocate and initialize the per-CPU state.  */
    386  1.36  riastrad 	entropy_percpu = percpu_create(sizeof(struct entropy_cpu),
    387  1.36  riastrad 	    entropy_init_cpu, entropy_fini_cpu, NULL);
    388  1.36  riastrad 
    389   1.1  riastrad 	/* Enter the boot cycle count to get started.  */
    390   1.1  riastrad 	extra[i++] = entropy_timer();
    391   1.1  riastrad 	KASSERT(i == __arraycount(extra));
    392   1.1  riastrad 	entropy_enter(extra, sizeof extra, 0);
    393   1.1  riastrad 	explicit_memset(extra, 0, sizeof extra);
    394   1.1  riastrad 
    395   1.1  riastrad 	/* We are now ready for multi-threaded operation.  */
    396   1.1  riastrad 	E->stage = ENTROPY_WARM;
    397   1.1  riastrad }
    398   1.1  riastrad 
    399  1.37  riastrad static void
    400  1.37  riastrad entropy_init_late_cpu(void *a, void *b)
    401  1.37  riastrad {
    402  1.37  riastrad 
    403  1.37  riastrad 	entropy_softintr(NULL);
    404  1.37  riastrad }
    405  1.37  riastrad 
    406   1.1  riastrad /*
    407   1.1  riastrad  * entropy_init_late()
    408   1.1  riastrad  *
    409   1.1  riastrad  *	Late initialization.  Panic on failure.
    410   1.1  riastrad  *
    411   1.1  riastrad  *	Requires CPUs to have been detected and LWPs to have started.
    412   1.1  riastrad  */
    413   1.1  riastrad static void
    414   1.1  riastrad entropy_init_late(void)
    415   1.1  riastrad {
    416  1.37  riastrad 	void *sih;
    417   1.1  riastrad 	int error;
    418   1.1  riastrad 
    419   1.1  riastrad 	KASSERT(E->stage == ENTROPY_WARM);
    420   1.1  riastrad 
    421   1.1  riastrad 	/*
    422   1.1  riastrad 	 * Establish the softint at the highest softint priority level.
    423   1.1  riastrad 	 * Must happen after CPU detection.
    424   1.1  riastrad 	 */
    425  1.37  riastrad 	sih = softint_establish(SOFTINT_SERIAL|SOFTINT_MPSAFE,
    426   1.1  riastrad 	    &entropy_softintr, NULL);
    427  1.37  riastrad 	if (sih == NULL)
    428   1.1  riastrad 		panic("unable to establish entropy softint");
    429   1.1  riastrad 
    430   1.1  riastrad 	/*
    431   1.1  riastrad 	 * Create the entropy housekeeping thread.  Must happen after
    432   1.1  riastrad 	 * lwpinit.
    433   1.1  riastrad 	 */
    434   1.1  riastrad 	error = kthread_create(PRI_NONE, KTHREAD_MPSAFE|KTHREAD_TS, NULL,
    435   1.1  riastrad 	    entropy_thread, NULL, &entropy_lwp, "entbutler");
    436   1.1  riastrad 	if (error)
    437   1.1  riastrad 		panic("unable to create entropy housekeeping thread: %d",
    438   1.1  riastrad 		    error);
    439   1.1  riastrad 
    440   1.1  riastrad 	/*
    441   1.1  riastrad 	 * Wait until the per-CPU initialization has hit all CPUs
    442  1.37  riastrad 	 * before proceeding to mark the entropy system hot and
    443  1.37  riastrad 	 * enabling use of the softint.
    444   1.1  riastrad 	 */
    445   1.1  riastrad 	xc_barrier(XC_HIGHPRI);
    446   1.1  riastrad 	E->stage = ENTROPY_HOT;
    447  1.37  riastrad 	atomic_store_relaxed(&entropy_sih, sih);
    448  1.37  riastrad 
    449  1.37  riastrad 	/*
    450  1.37  riastrad 	 * At this point, entering new samples from interrupt handlers
    451  1.37  riastrad 	 * will trigger the softint to process them.  But there may be
    452  1.37  riastrad 	 * some samples that were entered from interrupt handlers
    453  1.37  riastrad 	 * before the softint was available.  Make sure we process
    454  1.37  riastrad 	 * those samples on all CPUs by running the softint logic on
    455  1.37  riastrad 	 * all CPUs.
    456  1.37  riastrad 	 */
    457  1.37  riastrad 	xc_wait(xc_broadcast(XC_HIGHPRI, entropy_init_late_cpu, NULL, NULL));
    458   1.1  riastrad }
    459   1.1  riastrad 
    460   1.1  riastrad /*
    461   1.1  riastrad  * entropy_init_cpu(ptr, cookie, ci)
    462   1.1  riastrad  *
    463   1.1  riastrad  *	percpu(9) constructor for per-CPU entropy pool.
    464   1.1  riastrad  */
    465   1.1  riastrad static void
    466   1.1  riastrad entropy_init_cpu(void *ptr, void *cookie, struct cpu_info *ci)
    467   1.1  riastrad {
    468   1.1  riastrad 	struct entropy_cpu *ec = ptr;
    469   1.1  riastrad 
    470   1.1  riastrad 	ec->ec_softint_evcnt = kmem_alloc(sizeof(*ec->ec_softint_evcnt),
    471   1.1  riastrad 	    KM_SLEEP);
    472   1.1  riastrad 	ec->ec_pool = kmem_zalloc(sizeof(*ec->ec_pool), KM_SLEEP);
    473   1.1  riastrad 	ec->ec_pending = 0;
    474   1.1  riastrad 	ec->ec_locked = false;
    475   1.1  riastrad 
    476  1.36  riastrad 	/* XXX ci_cpuname may not be initialized early enough.  */
    477   1.1  riastrad 	evcnt_attach_dynamic(ec->ec_softint_evcnt, EVCNT_TYPE_MISC, NULL,
    478  1.36  riastrad 	    ci->ci_cpuname[0] == '\0' ? "cpu0" : ci->ci_cpuname,
    479  1.36  riastrad 	    "entropy softint");
    480   1.1  riastrad }
    481   1.1  riastrad 
    482   1.1  riastrad /*
    483   1.1  riastrad  * entropy_fini_cpu(ptr, cookie, ci)
    484   1.1  riastrad  *
    485   1.1  riastrad  *	percpu(9) destructor for per-CPU entropy pool.
    486   1.1  riastrad  */
    487   1.1  riastrad static void
    488   1.1  riastrad entropy_fini_cpu(void *ptr, void *cookie, struct cpu_info *ci)
    489   1.1  riastrad {
    490   1.1  riastrad 	struct entropy_cpu *ec = ptr;
    491   1.1  riastrad 
    492   1.1  riastrad 	/*
    493   1.1  riastrad 	 * Zero any lingering data.  Disclosure of the per-CPU pool
    494   1.1  riastrad 	 * shouldn't retroactively affect the security of any keys
    495   1.1  riastrad 	 * generated, because entpool(9) erases whatever we have just
    496   1.1  riastrad 	 * drawn out of any pool, but better safe than sorry.
    497   1.1  riastrad 	 */
    498   1.1  riastrad 	explicit_memset(ec->ec_pool, 0, sizeof(*ec->ec_pool));
    499   1.1  riastrad 
    500   1.1  riastrad 	evcnt_detach(ec->ec_softint_evcnt);
    501   1.1  riastrad 
    502   1.1  riastrad 	kmem_free(ec->ec_pool, sizeof(*ec->ec_pool));
    503   1.1  riastrad 	kmem_free(ec->ec_softint_evcnt, sizeof(*ec->ec_softint_evcnt));
    504   1.1  riastrad }
    505   1.1  riastrad 
    506   1.1  riastrad /*
    507   1.1  riastrad  * entropy_seed(seed)
    508   1.1  riastrad  *
    509   1.1  riastrad  *	Seed the entropy pool with seed.  Meant to be called as early
    510   1.1  riastrad  *	as possible by the bootloader; may be called before or after
    511   1.1  riastrad  *	entropy_init.  Must be called before system reaches userland.
    512   1.1  riastrad  *	Must be called in thread or soft interrupt context, not in hard
    513   1.1  riastrad  *	interrupt context.  Must be called at most once.
    514   1.1  riastrad  *
    515   1.1  riastrad  *	Overwrites the seed in place.  Caller may then free the memory.
    516   1.1  riastrad  */
    517   1.1  riastrad static void
    518   1.1  riastrad entropy_seed(rndsave_t *seed)
    519   1.1  riastrad {
    520   1.1  riastrad 	SHA1_CTX ctx;
    521   1.1  riastrad 	uint8_t digest[SHA1_DIGEST_LENGTH];
    522   1.1  riastrad 	bool seeded;
    523   1.1  riastrad 
    524   1.1  riastrad 	/*
    525   1.1  riastrad 	 * Verify the checksum.  If the checksum fails, take the data
    526   1.1  riastrad 	 * but ignore the entropy estimate -- the file may have been
    527   1.1  riastrad 	 * incompletely written with garbage, which is harmless to add
    528   1.1  riastrad 	 * but may not be as unpredictable as alleged.
    529   1.1  riastrad 	 */
    530   1.1  riastrad 	SHA1Init(&ctx);
    531   1.1  riastrad 	SHA1Update(&ctx, (const void *)&seed->entropy, sizeof(seed->entropy));
    532   1.1  riastrad 	SHA1Update(&ctx, seed->data, sizeof(seed->data));
    533   1.1  riastrad 	SHA1Final(digest, &ctx);
    534   1.1  riastrad 	CTASSERT(sizeof(seed->digest) == sizeof(digest));
    535   1.1  riastrad 	if (!consttime_memequal(digest, seed->digest, sizeof(digest))) {
    536   1.1  riastrad 		printf("entropy: invalid seed checksum\n");
    537   1.1  riastrad 		seed->entropy = 0;
    538   1.1  riastrad 	}
    539   1.2  riastrad 	explicit_memset(&ctx, 0, sizeof ctx);
    540   1.1  riastrad 	explicit_memset(digest, 0, sizeof digest);
    541   1.1  riastrad 
    542   1.2  riastrad 	/*
    543   1.2  riastrad 	 * If the entropy is insensibly large, try byte-swapping.
    544   1.2  riastrad 	 * Otherwise assume the file is corrupted and act as though it
    545   1.2  riastrad 	 * has zero entropy.
    546   1.2  riastrad 	 */
    547   1.2  riastrad 	if (howmany(seed->entropy, NBBY) > sizeof(seed->data)) {
    548   1.2  riastrad 		seed->entropy = bswap32(seed->entropy);
    549   1.2  riastrad 		if (howmany(seed->entropy, NBBY) > sizeof(seed->data))
    550   1.2  riastrad 			seed->entropy = 0;
    551   1.2  riastrad 	}
    552   1.2  riastrad 
    553   1.1  riastrad 	/* Make sure the seed source is attached.  */
    554   1.1  riastrad 	attach_seed_rndsource();
    555   1.1  riastrad 
    556   1.1  riastrad 	/* Test and set E->seeded.  */
    557   1.1  riastrad 	if (E->stage >= ENTROPY_WARM)
    558   1.1  riastrad 		mutex_enter(&E->lock);
    559   1.1  riastrad 	seeded = E->seeded;
    560  1.11  riastrad 	E->seeded = (seed->entropy > 0);
    561   1.1  riastrad 	if (E->stage >= ENTROPY_WARM)
    562   1.1  riastrad 		mutex_exit(&E->lock);
    563   1.1  riastrad 
    564   1.1  riastrad 	/*
    565   1.1  riastrad 	 * If we've been seeded, may be re-entering the same seed
    566   1.1  riastrad 	 * (e.g., bootloader vs module init, or something).  No harm in
    567   1.1  riastrad 	 * entering it twice, but it contributes no additional entropy.
    568   1.1  riastrad 	 */
    569   1.1  riastrad 	if (seeded) {
    570   1.1  riastrad 		printf("entropy: double-seeded by bootloader\n");
    571   1.1  riastrad 		seed->entropy = 0;
    572   1.1  riastrad 	} else {
    573  1.11  riastrad 		printf("entropy: entering seed from bootloader"
    574  1.11  riastrad 		    " with %u bits of entropy\n", (unsigned)seed->entropy);
    575   1.1  riastrad 	}
    576   1.1  riastrad 
    577   1.1  riastrad 	/* Enter it into the pool and promptly zero it.  */
    578   1.1  riastrad 	rnd_add_data(&seed_rndsource, seed->data, sizeof(seed->data),
    579   1.1  riastrad 	    seed->entropy);
    580   1.1  riastrad 	explicit_memset(seed, 0, sizeof(*seed));
    581   1.1  riastrad }
    582   1.1  riastrad 
    583   1.1  riastrad /*
    584   1.1  riastrad  * entropy_bootrequest()
    585   1.1  riastrad  *
    586   1.1  riastrad  *	Request entropy from all sources at boot, once config is
    587   1.1  riastrad  *	complete and interrupts are running.
    588   1.1  riastrad  */
    589   1.1  riastrad void
    590   1.1  riastrad entropy_bootrequest(void)
    591   1.1  riastrad {
    592   1.1  riastrad 
    593   1.1  riastrad 	KASSERT(E->stage >= ENTROPY_WARM);
    594   1.1  riastrad 
    595   1.1  riastrad 	/*
    596   1.1  riastrad 	 * Request enough to satisfy the maximum entropy shortage.
    597   1.1  riastrad 	 * This is harmless overkill if the bootloader provided a seed.
    598   1.1  riastrad 	 */
    599   1.1  riastrad 	mutex_enter(&E->lock);
    600   1.1  riastrad 	entropy_request(ENTROPY_CAPACITY);
    601   1.1  riastrad 	mutex_exit(&E->lock);
    602   1.1  riastrad }
    603   1.1  riastrad 
    604   1.1  riastrad /*
    605   1.1  riastrad  * entropy_epoch()
    606   1.1  riastrad  *
    607   1.1  riastrad  *	Returns the current entropy epoch.  If this changes, you should
    608  1.14  riastrad  *	reseed.  If -1, means system entropy has not yet reached full
    609  1.14  riastrad  *	entropy or been explicitly consolidated; never reverts back to
    610  1.14  riastrad  *	-1.  Never zero, so you can always use zero as an uninitialized
    611  1.14  riastrad  *	sentinel value meaning `reseed ASAP'.
    612   1.1  riastrad  *
    613   1.1  riastrad  *	Usage model:
    614   1.1  riastrad  *
    615   1.1  riastrad  *		struct foo {
    616   1.1  riastrad  *			struct crypto_prng prng;
    617   1.1  riastrad  *			unsigned epoch;
    618   1.1  riastrad  *		} *foo;
    619   1.1  riastrad  *
    620   1.1  riastrad  *		unsigned epoch = entropy_epoch();
    621   1.1  riastrad  *		if (__predict_false(epoch != foo->epoch)) {
    622   1.1  riastrad  *			uint8_t seed[32];
    623   1.1  riastrad  *			if (entropy_extract(seed, sizeof seed, 0) != 0)
    624   1.1  riastrad  *				warn("no entropy");
    625   1.1  riastrad  *			crypto_prng_reseed(&foo->prng, seed, sizeof seed);
    626   1.1  riastrad  *			foo->epoch = epoch;
    627   1.1  riastrad  *		}
    628   1.1  riastrad  */
    629   1.1  riastrad unsigned
    630   1.1  riastrad entropy_epoch(void)
    631   1.1  riastrad {
    632   1.1  riastrad 
    633   1.1  riastrad 	/*
    634   1.1  riastrad 	 * Unsigned int, so no need for seqlock for an atomic read, but
    635   1.1  riastrad 	 * make sure we read it afresh each time.
    636   1.1  riastrad 	 */
    637   1.1  riastrad 	return atomic_load_relaxed(&E->epoch);
    638   1.1  riastrad }
    639   1.1  riastrad 
    640   1.1  riastrad /*
    641  1.23  riastrad  * entropy_ready()
    642  1.23  riastrad  *
    643  1.23  riastrad  *	True if the entropy pool has full entropy.
    644  1.23  riastrad  */
    645  1.23  riastrad bool
    646  1.23  riastrad entropy_ready(void)
    647  1.23  riastrad {
    648  1.23  riastrad 
    649  1.23  riastrad 	return atomic_load_relaxed(&E->needed) == 0;
    650  1.23  riastrad }
    651  1.23  riastrad 
    652  1.23  riastrad /*
    653   1.1  riastrad  * entropy_account_cpu(ec)
    654   1.1  riastrad  *
    655   1.1  riastrad  *	Consider whether to consolidate entropy into the global pool
    656   1.1  riastrad  *	after we just added some into the current CPU's pending pool.
    657   1.1  riastrad  *
    658   1.1  riastrad  *	- If this CPU can provide enough entropy now, do so.
    659   1.1  riastrad  *
    660   1.1  riastrad  *	- If this and whatever else is available on other CPUs can
    661   1.1  riastrad  *	  provide enough entropy, kick the consolidation thread.
    662   1.1  riastrad  *
    663   1.1  riastrad  *	- Otherwise, do as little as possible, except maybe consolidate
    664   1.1  riastrad  *	  entropy at most once a minute.
    665   1.1  riastrad  *
    666   1.1  riastrad  *	Caller must be bound to a CPU and therefore have exclusive
    667   1.1  riastrad  *	access to ec.  Will acquire and release the global lock.
    668   1.1  riastrad  */
    669   1.1  riastrad static void
    670   1.1  riastrad entropy_account_cpu(struct entropy_cpu *ec)
    671   1.1  riastrad {
    672   1.1  riastrad 	unsigned diff;
    673   1.1  riastrad 
    674  1.37  riastrad 	KASSERT(E->stage >= ENTROPY_WARM);
    675   1.1  riastrad 
    676   1.1  riastrad 	/*
    677   1.1  riastrad 	 * If there's no entropy needed, and entropy has been
    678   1.1  riastrad 	 * consolidated in the last minute, do nothing.
    679   1.1  riastrad 	 */
    680   1.1  riastrad 	if (__predict_true(atomic_load_relaxed(&E->needed) == 0) &&
    681   1.1  riastrad 	    __predict_true(!atomic_load_relaxed(&entropy_depletion)) &&
    682   1.1  riastrad 	    __predict_true((time_uptime - E->timestamp) <= 60))
    683   1.1  riastrad 		return;
    684   1.1  riastrad 
    685   1.1  riastrad 	/* If there's nothing pending, stop here.  */
    686   1.1  riastrad 	if (ec->ec_pending == 0)
    687   1.1  riastrad 		return;
    688   1.1  riastrad 
    689   1.1  riastrad 	/* Consider consolidation, under the lock.  */
    690   1.1  riastrad 	mutex_enter(&E->lock);
    691   1.1  riastrad 	if (E->needed != 0 && E->needed <= ec->ec_pending) {
    692   1.1  riastrad 		/*
    693   1.1  riastrad 		 * If we have not yet attained full entropy but we can
    694   1.1  riastrad 		 * now, do so.  This way we disseminate entropy
    695   1.1  riastrad 		 * promptly when it becomes available early at boot;
    696   1.1  riastrad 		 * otherwise we leave it to the entropy consolidation
    697   1.1  riastrad 		 * thread, which is rate-limited to mitigate side
    698   1.1  riastrad 		 * channels and abuse.
    699   1.1  riastrad 		 */
    700   1.1  riastrad 		uint8_t buf[ENTPOOL_CAPACITY];
    701   1.1  riastrad 
    702   1.1  riastrad 		/* Transfer from the local pool to the global pool.  */
    703   1.1  riastrad 		entpool_extract(ec->ec_pool, buf, sizeof buf);
    704   1.1  riastrad 		entpool_enter(&E->pool, buf, sizeof buf);
    705   1.1  riastrad 		atomic_store_relaxed(&ec->ec_pending, 0);
    706   1.1  riastrad 		atomic_store_relaxed(&E->needed, 0);
    707   1.1  riastrad 
    708   1.1  riastrad 		/* Notify waiters that we now have full entropy.  */
    709   1.1  riastrad 		entropy_notify();
    710   1.1  riastrad 		entropy_immediate_evcnt.ev_count++;
    711  1.18  riastrad 	} else {
    712   1.1  riastrad 		/* Record how much we can add to the global pool.  */
    713   1.1  riastrad 		diff = MIN(ec->ec_pending, ENTROPY_CAPACITY*NBBY - E->pending);
    714   1.1  riastrad 		E->pending += diff;
    715   1.1  riastrad 		atomic_store_relaxed(&ec->ec_pending, ec->ec_pending - diff);
    716   1.1  riastrad 
    717   1.1  riastrad 		/*
    718   1.1  riastrad 		 * This should have made a difference unless we were
    719   1.1  riastrad 		 * already saturated.
    720   1.1  riastrad 		 */
    721   1.1  riastrad 		KASSERT(diff || E->pending == ENTROPY_CAPACITY*NBBY);
    722   1.1  riastrad 		KASSERT(E->pending);
    723   1.1  riastrad 
    724   1.1  riastrad 		if (E->needed <= E->pending) {
    725   1.1  riastrad 			/*
    726   1.1  riastrad 			 * Enough entropy between all the per-CPU
    727   1.1  riastrad 			 * pools.  Wake up the housekeeping thread.
    728   1.1  riastrad 			 *
    729   1.1  riastrad 			 * If we don't need any entropy, this doesn't
    730   1.1  riastrad 			 * mean much, but it is the only time we ever
    731   1.1  riastrad 			 * gather additional entropy in case the
    732   1.1  riastrad 			 * accounting has been overly optimistic.  This
    733   1.1  riastrad 			 * happens at most once a minute, so there's
    734   1.1  riastrad 			 * negligible performance cost.
    735   1.1  riastrad 			 */
    736   1.1  riastrad 			E->consolidate = true;
    737   1.1  riastrad 			cv_broadcast(&E->cv);
    738   1.1  riastrad 			if (E->needed == 0)
    739   1.1  riastrad 				entropy_discretionary_evcnt.ev_count++;
    740   1.1  riastrad 		} else {
    741   1.1  riastrad 			/* Can't get full entropy.  Keep gathering.  */
    742   1.1  riastrad 			entropy_partial_evcnt.ev_count++;
    743   1.1  riastrad 		}
    744   1.1  riastrad 	}
    745   1.1  riastrad 	mutex_exit(&E->lock);
    746   1.1  riastrad }
    747   1.1  riastrad 
    748   1.1  riastrad /*
    749   1.1  riastrad  * entropy_enter_early(buf, len, nbits)
    750   1.1  riastrad  *
    751   1.1  riastrad  *	Do entropy bookkeeping globally, before we have established
    752   1.1  riastrad  *	per-CPU pools.  Enter directly into the global pool in the hope
    753   1.1  riastrad  *	that we enter enough before the first entropy_extract to thwart
    754   1.1  riastrad  *	iterative-guessing attacks; entropy_extract will warn if not.
    755   1.1  riastrad  */
    756   1.1  riastrad static void
    757   1.1  riastrad entropy_enter_early(const void *buf, size_t len, unsigned nbits)
    758   1.1  riastrad {
    759   1.1  riastrad 	bool notify = false;
    760   1.1  riastrad 
    761  1.37  riastrad 	KASSERT(E->stage == ENTROPY_COLD);
    762   1.1  riastrad 
    763   1.1  riastrad 	/* Enter it into the pool.  */
    764   1.1  riastrad 	entpool_enter(&E->pool, buf, len);
    765   1.1  riastrad 
    766   1.1  riastrad 	/*
    767   1.1  riastrad 	 * Decide whether to notify reseed -- we will do so if either:
    768   1.1  riastrad 	 * (a) we transition from partial entropy to full entropy, or
    769   1.1  riastrad 	 * (b) we get a batch of full entropy all at once.
    770   1.1  riastrad 	 */
    771   1.1  riastrad 	notify |= (E->needed && E->needed <= nbits);
    772   1.1  riastrad 	notify |= (nbits >= ENTROPY_CAPACITY*NBBY);
    773   1.1  riastrad 
    774   1.1  riastrad 	/* Subtract from the needed count and notify if appropriate.  */
    775   1.1  riastrad 	E->needed -= MIN(E->needed, nbits);
    776   1.1  riastrad 	if (notify) {
    777   1.1  riastrad 		entropy_notify();
    778   1.1  riastrad 		entropy_immediate_evcnt.ev_count++;
    779   1.1  riastrad 	}
    780   1.1  riastrad }
    781   1.1  riastrad 
    782   1.1  riastrad /*
    783   1.1  riastrad  * entropy_enter(buf, len, nbits)
    784   1.1  riastrad  *
    785   1.1  riastrad  *	Enter len bytes of data from buf into the system's entropy
    786   1.1  riastrad  *	pool, stirring as necessary when the internal buffer fills up.
    787   1.1  riastrad  *	nbits is a lower bound on the number of bits of entropy in the
    788   1.1  riastrad  *	process that led to this sample.
    789   1.1  riastrad  */
    790   1.1  riastrad static void
    791   1.1  riastrad entropy_enter(const void *buf, size_t len, unsigned nbits)
    792   1.1  riastrad {
    793   1.1  riastrad 	struct entropy_cpu *ec;
    794   1.1  riastrad 	uint32_t pending;
    795   1.1  riastrad 	int s;
    796   1.1  riastrad 
    797  1.16  riastrad 	KASSERTMSG(!cpu_intr_p(),
    798   1.1  riastrad 	    "use entropy_enter_intr from interrupt context");
    799   1.1  riastrad 	KASSERTMSG(howmany(nbits, NBBY) <= len,
    800   1.1  riastrad 	    "impossible entropy rate: %u bits in %zu-byte string", nbits, len);
    801   1.1  riastrad 
    802   1.1  riastrad 	/* If it's too early after boot, just use entropy_enter_early.  */
    803  1.37  riastrad 	if (__predict_false(E->stage == ENTROPY_COLD)) {
    804   1.1  riastrad 		entropy_enter_early(buf, len, nbits);
    805   1.1  riastrad 		return;
    806   1.1  riastrad 	}
    807   1.1  riastrad 
    808   1.1  riastrad 	/*
    809   1.1  riastrad 	 * Acquire the per-CPU state, blocking soft interrupts and
    810   1.1  riastrad 	 * causing hard interrupts to drop samples on the floor.
    811   1.1  riastrad 	 */
    812   1.1  riastrad 	ec = percpu_getref(entropy_percpu);
    813   1.1  riastrad 	s = splsoftserial();
    814   1.1  riastrad 	KASSERT(!ec->ec_locked);
    815   1.1  riastrad 	ec->ec_locked = true;
    816   1.1  riastrad 	__insn_barrier();
    817   1.1  riastrad 
    818   1.1  riastrad 	/* Enter into the per-CPU pool.  */
    819   1.1  riastrad 	entpool_enter(ec->ec_pool, buf, len);
    820   1.1  riastrad 
    821   1.1  riastrad 	/* Count up what we can add.  */
    822   1.1  riastrad 	pending = ec->ec_pending;
    823   1.1  riastrad 	pending += MIN(ENTROPY_CAPACITY*NBBY - pending, nbits);
    824   1.1  riastrad 	atomic_store_relaxed(&ec->ec_pending, pending);
    825   1.1  riastrad 
    826   1.1  riastrad 	/* Consolidate globally if appropriate based on what we added.  */
    827   1.1  riastrad 	entropy_account_cpu(ec);
    828   1.1  riastrad 
    829   1.1  riastrad 	/* Release the per-CPU state.  */
    830   1.1  riastrad 	KASSERT(ec->ec_locked);
    831   1.1  riastrad 	__insn_barrier();
    832   1.1  riastrad 	ec->ec_locked = false;
    833   1.1  riastrad 	splx(s);
    834   1.1  riastrad 	percpu_putref(entropy_percpu);
    835   1.1  riastrad }
    836   1.1  riastrad 
    837   1.1  riastrad /*
    838   1.1  riastrad  * entropy_enter_intr(buf, len, nbits)
    839   1.1  riastrad  *
    840   1.1  riastrad  *	Enter up to len bytes of data from buf into the system's
    841   1.1  riastrad  *	entropy pool without stirring.  nbits is a lower bound on the
    842   1.1  riastrad  *	number of bits of entropy in the process that led to this
    843   1.1  riastrad  *	sample.  If the sample could be entered completely, assume
    844   1.1  riastrad  *	nbits of entropy pending; otherwise assume none, since we don't
    845   1.1  riastrad  *	know whether some parts of the sample are constant, for
    846   1.1  riastrad  *	instance.  Schedule a softint to stir the entropy pool if
    847   1.1  riastrad  *	needed.  Return true if used fully, false if truncated at all.
    848   1.1  riastrad  *
    849   1.1  riastrad  *	Using this in thread context will work, but you might as well
    850   1.1  riastrad  *	use entropy_enter in that case.
    851   1.1  riastrad  */
    852   1.1  riastrad static bool
    853   1.1  riastrad entropy_enter_intr(const void *buf, size_t len, unsigned nbits)
    854   1.1  riastrad {
    855   1.1  riastrad 	struct entropy_cpu *ec;
    856   1.1  riastrad 	bool fullyused = false;
    857   1.1  riastrad 	uint32_t pending;
    858  1.37  riastrad 	void *sih;
    859   1.1  riastrad 
    860   1.1  riastrad 	KASSERTMSG(howmany(nbits, NBBY) <= len,
    861   1.1  riastrad 	    "impossible entropy rate: %u bits in %zu-byte string", nbits, len);
    862   1.1  riastrad 
    863   1.1  riastrad 	/* If it's too early after boot, just use entropy_enter_early.  */
    864  1.37  riastrad 	if (__predict_false(E->stage == ENTROPY_COLD)) {
    865   1.1  riastrad 		entropy_enter_early(buf, len, nbits);
    866   1.1  riastrad 		return true;
    867   1.1  riastrad 	}
    868   1.1  riastrad 
    869   1.1  riastrad 	/*
    870   1.1  riastrad 	 * Acquire the per-CPU state.  If someone is in the middle of
    871   1.1  riastrad 	 * using it, drop the sample.  Otherwise, take the lock so that
    872   1.1  riastrad 	 * higher-priority interrupts will drop their samples.
    873   1.1  riastrad 	 */
    874   1.1  riastrad 	ec = percpu_getref(entropy_percpu);
    875   1.1  riastrad 	if (ec->ec_locked)
    876   1.1  riastrad 		goto out0;
    877   1.1  riastrad 	ec->ec_locked = true;
    878   1.1  riastrad 	__insn_barrier();
    879   1.1  riastrad 
    880   1.1  riastrad 	/*
    881   1.1  riastrad 	 * Enter as much as we can into the per-CPU pool.  If it was
    882   1.1  riastrad 	 * truncated, schedule a softint to stir the pool and stop.
    883   1.1  riastrad 	 */
    884   1.1  riastrad 	if (!entpool_enter_nostir(ec->ec_pool, buf, len)) {
    885  1.37  riastrad 		sih = atomic_load_relaxed(&entropy_sih);
    886  1.37  riastrad 		if (__predict_true(sih != NULL))
    887  1.37  riastrad 			softint_schedule(sih);
    888   1.1  riastrad 		goto out1;
    889   1.1  riastrad 	}
    890   1.1  riastrad 	fullyused = true;
    891   1.1  riastrad 
    892   1.1  riastrad 	/* Count up what we can contribute.  */
    893   1.1  riastrad 	pending = ec->ec_pending;
    894   1.1  riastrad 	pending += MIN(ENTROPY_CAPACITY*NBBY - pending, nbits);
    895   1.1  riastrad 	atomic_store_relaxed(&ec->ec_pending, pending);
    896   1.1  riastrad 
    897   1.1  riastrad 	/* Schedule a softint if we added anything and it matters.  */
    898   1.1  riastrad 	if (__predict_false((atomic_load_relaxed(&E->needed) != 0) ||
    899   1.1  riastrad 		atomic_load_relaxed(&entropy_depletion)) &&
    900  1.37  riastrad 	    nbits != 0) {
    901  1.37  riastrad 		sih = atomic_load_relaxed(&entropy_sih);
    902  1.37  riastrad 		if (__predict_true(sih != NULL))
    903  1.37  riastrad 			softint_schedule(sih);
    904  1.37  riastrad 	}
    905   1.1  riastrad 
    906   1.1  riastrad out1:	/* Release the per-CPU state.  */
    907   1.1  riastrad 	KASSERT(ec->ec_locked);
    908   1.1  riastrad 	__insn_barrier();
    909   1.1  riastrad 	ec->ec_locked = false;
    910   1.1  riastrad out0:	percpu_putref(entropy_percpu);
    911   1.1  riastrad 
    912   1.1  riastrad 	return fullyused;
    913   1.1  riastrad }
    914   1.1  riastrad 
    915   1.1  riastrad /*
    916   1.1  riastrad  * entropy_softintr(cookie)
    917   1.1  riastrad  *
    918   1.1  riastrad  *	Soft interrupt handler for entering entropy.  Takes care of
    919   1.1  riastrad  *	stirring the local CPU's entropy pool if it filled up during
    920   1.1  riastrad  *	hard interrupts, and promptly crediting entropy from the local
    921   1.1  riastrad  *	CPU's entropy pool to the global entropy pool if needed.
    922   1.1  riastrad  */
    923   1.1  riastrad static void
    924   1.1  riastrad entropy_softintr(void *cookie)
    925   1.1  riastrad {
    926   1.1  riastrad 	struct entropy_cpu *ec;
    927   1.1  riastrad 
    928   1.1  riastrad 	/*
    929   1.1  riastrad 	 * Acquire the per-CPU state.  Other users can lock this only
    930   1.1  riastrad 	 * while soft interrupts are blocked.  Cause hard interrupts to
    931   1.1  riastrad 	 * drop samples on the floor.
    932   1.1  riastrad 	 */
    933   1.1  riastrad 	ec = percpu_getref(entropy_percpu);
    934   1.1  riastrad 	KASSERT(!ec->ec_locked);
    935   1.1  riastrad 	ec->ec_locked = true;
    936   1.1  riastrad 	__insn_barrier();
    937   1.1  riastrad 
    938   1.1  riastrad 	/* Count statistics.  */
    939   1.1  riastrad 	ec->ec_softint_evcnt->ev_count++;
    940   1.1  riastrad 
    941   1.1  riastrad 	/* Stir the pool if necessary.  */
    942   1.1  riastrad 	entpool_stir(ec->ec_pool);
    943   1.1  riastrad 
    944   1.1  riastrad 	/* Consolidate globally if appropriate based on what we added.  */
    945   1.1  riastrad 	entropy_account_cpu(ec);
    946   1.1  riastrad 
    947   1.1  riastrad 	/* Release the per-CPU state.  */
    948   1.1  riastrad 	KASSERT(ec->ec_locked);
    949   1.1  riastrad 	__insn_barrier();
    950   1.1  riastrad 	ec->ec_locked = false;
    951   1.1  riastrad 	percpu_putref(entropy_percpu);
    952   1.1  riastrad }
    953   1.1  riastrad 
    954   1.1  riastrad /*
    955   1.1  riastrad  * entropy_thread(cookie)
    956   1.1  riastrad  *
    957   1.1  riastrad  *	Handle any asynchronous entropy housekeeping.
    958   1.1  riastrad  */
    959   1.1  riastrad static void
    960   1.1  riastrad entropy_thread(void *cookie)
    961   1.1  riastrad {
    962   1.3  riastrad 	bool consolidate;
    963   1.1  riastrad 
    964   1.1  riastrad 	for (;;) {
    965   1.1  riastrad 		/*
    966   1.3  riastrad 		 * Wait until there's full entropy somewhere among the
    967   1.3  riastrad 		 * CPUs, as confirmed at most once per minute, or
    968   1.3  riastrad 		 * someone wants to consolidate.
    969   1.1  riastrad 		 */
    970   1.3  riastrad 		if (entropy_pending() >= ENTROPY_CAPACITY*NBBY) {
    971   1.3  riastrad 			consolidate = true;
    972   1.3  riastrad 		} else {
    973   1.3  riastrad 			mutex_enter(&E->lock);
    974   1.3  riastrad 			if (!E->consolidate)
    975   1.3  riastrad 				cv_timedwait(&E->cv, &E->lock, 60*hz);
    976   1.3  riastrad 			consolidate = E->consolidate;
    977   1.3  riastrad 			E->consolidate = false;
    978   1.3  riastrad 			mutex_exit(&E->lock);
    979   1.1  riastrad 		}
    980   1.1  riastrad 
    981   1.3  riastrad 		if (consolidate) {
    982   1.3  riastrad 			/* Do it.  */
    983  1.13  riastrad 			entropy_do_consolidate();
    984   1.1  riastrad 
    985   1.3  riastrad 			/* Mitigate abuse.  */
    986   1.3  riastrad 			kpause("entropy", false, hz, NULL);
    987   1.3  riastrad 		}
    988   1.1  riastrad 	}
    989   1.1  riastrad }
    990   1.1  riastrad 
    991   1.1  riastrad /*
    992   1.1  riastrad  * entropy_pending()
    993   1.1  riastrad  *
    994   1.1  riastrad  *	Count up the amount of entropy pending on other CPUs.
    995   1.1  riastrad  */
    996   1.1  riastrad static uint32_t
    997   1.1  riastrad entropy_pending(void)
    998   1.1  riastrad {
    999   1.1  riastrad 	uint32_t pending = 0;
   1000   1.1  riastrad 
   1001   1.1  riastrad 	percpu_foreach(entropy_percpu, &entropy_pending_cpu, &pending);
   1002   1.1  riastrad 	return pending;
   1003   1.1  riastrad }
   1004   1.1  riastrad 
   1005   1.1  riastrad static void
   1006   1.1  riastrad entropy_pending_cpu(void *ptr, void *cookie, struct cpu_info *ci)
   1007   1.1  riastrad {
   1008   1.1  riastrad 	struct entropy_cpu *ec = ptr;
   1009   1.1  riastrad 	uint32_t *pendingp = cookie;
   1010   1.1  riastrad 	uint32_t cpu_pending;
   1011   1.1  riastrad 
   1012   1.1  riastrad 	cpu_pending = atomic_load_relaxed(&ec->ec_pending);
   1013   1.1  riastrad 	*pendingp += MIN(ENTROPY_CAPACITY*NBBY - *pendingp, cpu_pending);
   1014   1.1  riastrad }
   1015   1.1  riastrad 
   1016   1.1  riastrad /*
   1017  1.13  riastrad  * entropy_do_consolidate()
   1018   1.1  riastrad  *
   1019   1.1  riastrad  *	Issue a cross-call to gather entropy on all CPUs and advance
   1020   1.1  riastrad  *	the entropy epoch.
   1021   1.1  riastrad  */
   1022   1.1  riastrad static void
   1023  1.13  riastrad entropy_do_consolidate(void)
   1024   1.1  riastrad {
   1025   1.1  riastrad 	static const struct timeval interval = {.tv_sec = 60, .tv_usec = 0};
   1026   1.1  riastrad 	static struct timeval lasttime; /* serialized by E->lock */
   1027  1.19  riastrad 	struct entpool pool;
   1028  1.19  riastrad 	uint8_t buf[ENTPOOL_CAPACITY];
   1029   1.1  riastrad 	unsigned diff;
   1030   1.1  riastrad 	uint64_t ticket;
   1031   1.1  riastrad 
   1032  1.19  riastrad 	/* Gather entropy on all CPUs into a temporary pool.  */
   1033  1.19  riastrad 	memset(&pool, 0, sizeof pool);
   1034  1.19  riastrad 	ticket = xc_broadcast(0, &entropy_consolidate_xc, &pool, NULL);
   1035   1.1  riastrad 	xc_wait(ticket);
   1036   1.1  riastrad 
   1037   1.1  riastrad 	/* Acquire the lock to notify waiters.  */
   1038   1.1  riastrad 	mutex_enter(&E->lock);
   1039   1.1  riastrad 
   1040   1.1  riastrad 	/* Count another consolidation.  */
   1041   1.1  riastrad 	entropy_consolidate_evcnt.ev_count++;
   1042   1.1  riastrad 
   1043   1.1  riastrad 	/* Note when we last consolidated, i.e. now.  */
   1044   1.1  riastrad 	E->timestamp = time_uptime;
   1045   1.1  riastrad 
   1046  1.19  riastrad 	/* Mix what we gathered into the global pool.  */
   1047  1.19  riastrad 	entpool_extract(&pool, buf, sizeof buf);
   1048  1.19  riastrad 	entpool_enter(&E->pool, buf, sizeof buf);
   1049  1.19  riastrad 	explicit_memset(&pool, 0, sizeof pool);
   1050  1.19  riastrad 
   1051   1.1  riastrad 	/* Count the entropy that was gathered.  */
   1052   1.1  riastrad 	diff = MIN(E->needed, E->pending);
   1053   1.1  riastrad 	atomic_store_relaxed(&E->needed, E->needed - diff);
   1054   1.1  riastrad 	E->pending -= diff;
   1055   1.1  riastrad 	if (__predict_false(E->needed > 0)) {
   1056  1.30  jmcneill 		if (ratecheck(&lasttime, &interval) &&
   1057  1.30  jmcneill 		    (boothowto & AB_DEBUG) != 0) {
   1058  1.30  jmcneill 			printf("entropy: WARNING:"
   1059   1.1  riastrad 			    " consolidating less than full entropy\n");
   1060  1.30  jmcneill 		}
   1061   1.1  riastrad 	}
   1062   1.1  riastrad 
   1063   1.1  riastrad 	/* Advance the epoch and notify waiters.  */
   1064   1.1  riastrad 	entropy_notify();
   1065   1.1  riastrad 
   1066   1.1  riastrad 	/* Release the lock.  */
   1067   1.1  riastrad 	mutex_exit(&E->lock);
   1068   1.1  riastrad }
   1069   1.1  riastrad 
   1070   1.1  riastrad /*
   1071  1.20  riastrad  * entropy_consolidate_xc(vpool, arg2)
   1072   1.1  riastrad  *
   1073   1.1  riastrad  *	Extract output from the local CPU's input pool and enter it
   1074  1.20  riastrad  *	into a temporary pool passed as vpool.
   1075   1.1  riastrad  */
   1076   1.1  riastrad static void
   1077  1.19  riastrad entropy_consolidate_xc(void *vpool, void *arg2 __unused)
   1078   1.1  riastrad {
   1079  1.19  riastrad 	struct entpool *pool = vpool;
   1080   1.1  riastrad 	struct entropy_cpu *ec;
   1081   1.1  riastrad 	uint8_t buf[ENTPOOL_CAPACITY];
   1082   1.1  riastrad 	uint32_t extra[7];
   1083   1.1  riastrad 	unsigned i = 0;
   1084   1.1  riastrad 	int s;
   1085   1.1  riastrad 
   1086   1.1  riastrad 	/* Grab CPU number and cycle counter to mix extra into the pool.  */
   1087   1.1  riastrad 	extra[i++] = cpu_number();
   1088   1.1  riastrad 	extra[i++] = entropy_timer();
   1089   1.1  riastrad 
   1090   1.1  riastrad 	/*
   1091   1.1  riastrad 	 * Acquire the per-CPU state, blocking soft interrupts and
   1092   1.1  riastrad 	 * discarding entropy in hard interrupts, so that we can
   1093   1.1  riastrad 	 * extract from the per-CPU pool.
   1094   1.1  riastrad 	 */
   1095   1.1  riastrad 	ec = percpu_getref(entropy_percpu);
   1096   1.1  riastrad 	s = splsoftserial();
   1097   1.1  riastrad 	KASSERT(!ec->ec_locked);
   1098   1.1  riastrad 	ec->ec_locked = true;
   1099   1.1  riastrad 	__insn_barrier();
   1100   1.1  riastrad 	extra[i++] = entropy_timer();
   1101   1.1  riastrad 
   1102  1.12  riastrad 	/* Extract the data and count it no longer pending.  */
   1103   1.1  riastrad 	entpool_extract(ec->ec_pool, buf, sizeof buf);
   1104  1.12  riastrad 	atomic_store_relaxed(&ec->ec_pending, 0);
   1105   1.1  riastrad 	extra[i++] = entropy_timer();
   1106   1.1  riastrad 
   1107   1.1  riastrad 	/* Release the per-CPU state.  */
   1108   1.1  riastrad 	KASSERT(ec->ec_locked);
   1109   1.1  riastrad 	__insn_barrier();
   1110   1.1  riastrad 	ec->ec_locked = false;
   1111   1.1  riastrad 	splx(s);
   1112   1.1  riastrad 	percpu_putref(entropy_percpu);
   1113   1.1  riastrad 	extra[i++] = entropy_timer();
   1114   1.1  riastrad 
   1115   1.1  riastrad 	/*
   1116   1.1  riastrad 	 * Copy over statistics, and enter the per-CPU extract and the
   1117  1.19  riastrad 	 * extra timing into the temporary pool, under the global lock.
   1118   1.1  riastrad 	 */
   1119   1.1  riastrad 	mutex_enter(&E->lock);
   1120   1.1  riastrad 	extra[i++] = entropy_timer();
   1121  1.19  riastrad 	entpool_enter(pool, buf, sizeof buf);
   1122   1.1  riastrad 	explicit_memset(buf, 0, sizeof buf);
   1123   1.1  riastrad 	extra[i++] = entropy_timer();
   1124   1.1  riastrad 	KASSERT(i == __arraycount(extra));
   1125  1.19  riastrad 	entpool_enter(pool, extra, sizeof extra);
   1126   1.1  riastrad 	explicit_memset(extra, 0, sizeof extra);
   1127   1.1  riastrad 	mutex_exit(&E->lock);
   1128   1.1  riastrad }
   1129   1.1  riastrad 
   1130   1.1  riastrad /*
   1131   1.1  riastrad  * entropy_notify()
   1132   1.1  riastrad  *
   1133   1.1  riastrad  *	Caller just contributed entropy to the global pool.  Advance
   1134   1.1  riastrad  *	the entropy epoch and notify waiters.
   1135   1.1  riastrad  *
   1136   1.1  riastrad  *	Caller must hold the global entropy lock.  Except for the
   1137   1.1  riastrad  *	`sysctl -w kern.entropy.consolidate=1` trigger, the caller must
   1138   1.1  riastrad  *	have just have transitioned from partial entropy to full
   1139   1.1  riastrad  *	entropy -- E->needed should be zero now.
   1140   1.1  riastrad  */
   1141   1.1  riastrad static void
   1142   1.1  riastrad entropy_notify(void)
   1143   1.1  riastrad {
   1144  1.12  riastrad 	static const struct timeval interval = {.tv_sec = 60, .tv_usec = 0};
   1145  1.12  riastrad 	static struct timeval lasttime; /* serialized by E->lock */
   1146   1.1  riastrad 	unsigned epoch;
   1147   1.1  riastrad 
   1148   1.1  riastrad 	KASSERT(E->stage == ENTROPY_COLD || mutex_owned(&E->lock));
   1149   1.1  riastrad 
   1150   1.1  riastrad 	/*
   1151   1.1  riastrad 	 * If this is the first time, print a message to the console
   1152   1.1  riastrad 	 * that we're ready so operators can compare it to the timing
   1153   1.1  riastrad 	 * of other events.
   1154   1.1  riastrad 	 */
   1155  1.14  riastrad 	if (__predict_false(!rnd_initial_entropy) && E->needed == 0) {
   1156   1.1  riastrad 		printf("entropy: ready\n");
   1157  1.14  riastrad 		rnd_initial_entropy = 1;
   1158  1.14  riastrad 	}
   1159   1.1  riastrad 
   1160   1.1  riastrad 	/* Set the epoch; roll over from UINTMAX-1 to 1.  */
   1161  1.12  riastrad 	if (__predict_true(!atomic_load_relaxed(&entropy_depletion)) ||
   1162  1.12  riastrad 	    ratecheck(&lasttime, &interval)) {
   1163  1.12  riastrad 		epoch = E->epoch + 1;
   1164  1.12  riastrad 		if (epoch == 0 || epoch == (unsigned)-1)
   1165  1.12  riastrad 			epoch = 1;
   1166  1.12  riastrad 		atomic_store_relaxed(&E->epoch, epoch);
   1167  1.12  riastrad 	}
   1168   1.1  riastrad 
   1169   1.1  riastrad 	/* Notify waiters.  */
   1170   1.1  riastrad 	if (E->stage >= ENTROPY_WARM) {
   1171   1.1  riastrad 		cv_broadcast(&E->cv);
   1172   1.1  riastrad 		selnotify(&E->selq, POLLIN|POLLRDNORM, NOTE_SUBMIT);
   1173   1.1  riastrad 	}
   1174   1.1  riastrad 
   1175   1.1  riastrad 	/* Count another notification.  */
   1176   1.1  riastrad 	entropy_notify_evcnt.ev_count++;
   1177   1.1  riastrad }
   1178   1.1  riastrad 
   1179   1.1  riastrad /*
   1180  1.13  riastrad  * entropy_consolidate()
   1181  1.13  riastrad  *
   1182  1.13  riastrad  *	Trigger entropy consolidation and wait for it to complete.
   1183  1.13  riastrad  *
   1184  1.13  riastrad  *	This should be used sparingly, not periodically -- requiring
   1185  1.13  riastrad  *	conscious intervention by the operator or a clear policy
   1186  1.13  riastrad  *	decision.  Otherwise, the kernel will automatically consolidate
   1187  1.13  riastrad  *	when enough entropy has been gathered into per-CPU pools to
   1188  1.13  riastrad  *	transition to full entropy.
   1189  1.13  riastrad  */
   1190  1.13  riastrad void
   1191  1.13  riastrad entropy_consolidate(void)
   1192  1.13  riastrad {
   1193  1.13  riastrad 	uint64_t ticket;
   1194  1.13  riastrad 	int error;
   1195  1.13  riastrad 
   1196  1.13  riastrad 	KASSERT(E->stage == ENTROPY_HOT);
   1197  1.13  riastrad 
   1198  1.13  riastrad 	mutex_enter(&E->lock);
   1199  1.13  riastrad 	ticket = entropy_consolidate_evcnt.ev_count;
   1200  1.13  riastrad 	E->consolidate = true;
   1201  1.13  riastrad 	cv_broadcast(&E->cv);
   1202  1.13  riastrad 	while (ticket == entropy_consolidate_evcnt.ev_count) {
   1203  1.13  riastrad 		error = cv_wait_sig(&E->cv, &E->lock);
   1204  1.13  riastrad 		if (error)
   1205  1.13  riastrad 			break;
   1206  1.13  riastrad 	}
   1207  1.13  riastrad 	mutex_exit(&E->lock);
   1208  1.13  riastrad }
   1209  1.13  riastrad 
   1210  1.13  riastrad /*
   1211   1.1  riastrad  * sysctl -w kern.entropy.consolidate=1
   1212   1.1  riastrad  *
   1213   1.1  riastrad  *	Trigger entropy consolidation and wait for it to complete.
   1214  1.13  riastrad  *	Writable only by superuser.  This, writing to /dev/random, and
   1215  1.13  riastrad  *	ioctl(RNDADDDATA) are the only ways for the system to
   1216  1.13  riastrad  *	consolidate entropy if the operator knows something the kernel
   1217  1.13  riastrad  *	doesn't about how unpredictable the pending entropy pools are.
   1218   1.1  riastrad  */
   1219   1.1  riastrad static int
   1220   1.1  riastrad sysctl_entropy_consolidate(SYSCTLFN_ARGS)
   1221   1.1  riastrad {
   1222   1.1  riastrad 	struct sysctlnode node = *rnode;
   1223   1.1  riastrad 	int arg;
   1224   1.1  riastrad 	int error;
   1225   1.1  riastrad 
   1226   1.1  riastrad 	KASSERT(E->stage == ENTROPY_HOT);
   1227   1.1  riastrad 
   1228   1.1  riastrad 	node.sysctl_data = &arg;
   1229   1.1  riastrad 	error = sysctl_lookup(SYSCTLFN_CALL(&node));
   1230   1.1  riastrad 	if (error || newp == NULL)
   1231   1.1  riastrad 		return error;
   1232  1.13  riastrad 	if (arg)
   1233  1.13  riastrad 		entropy_consolidate();
   1234   1.1  riastrad 
   1235   1.1  riastrad 	return error;
   1236   1.1  riastrad }
   1237   1.1  riastrad 
   1238   1.1  riastrad /*
   1239  1.10  riastrad  * sysctl -w kern.entropy.gather=1
   1240  1.10  riastrad  *
   1241  1.10  riastrad  *	Trigger gathering entropy from all on-demand sources, and wait
   1242  1.10  riastrad  *	for synchronous sources (but not asynchronous sources) to
   1243  1.10  riastrad  *	complete.  Writable only by superuser.
   1244  1.10  riastrad  */
   1245  1.10  riastrad static int
   1246  1.10  riastrad sysctl_entropy_gather(SYSCTLFN_ARGS)
   1247  1.10  riastrad {
   1248  1.10  riastrad 	struct sysctlnode node = *rnode;
   1249  1.10  riastrad 	int arg;
   1250  1.10  riastrad 	int error;
   1251  1.10  riastrad 
   1252  1.10  riastrad 	KASSERT(E->stage == ENTROPY_HOT);
   1253  1.10  riastrad 
   1254  1.10  riastrad 	node.sysctl_data = &arg;
   1255  1.10  riastrad 	error = sysctl_lookup(SYSCTLFN_CALL(&node));
   1256  1.10  riastrad 	if (error || newp == NULL)
   1257  1.10  riastrad 		return error;
   1258  1.10  riastrad 	if (arg) {
   1259  1.10  riastrad 		mutex_enter(&E->lock);
   1260  1.10  riastrad 		entropy_request(ENTROPY_CAPACITY);
   1261  1.10  riastrad 		mutex_exit(&E->lock);
   1262  1.10  riastrad 	}
   1263  1.10  riastrad 
   1264  1.10  riastrad 	return 0;
   1265  1.10  riastrad }
   1266  1.10  riastrad 
   1267  1.10  riastrad /*
   1268   1.1  riastrad  * entropy_extract(buf, len, flags)
   1269   1.1  riastrad  *
   1270   1.1  riastrad  *	Extract len bytes from the global entropy pool into buf.
   1271   1.1  riastrad  *
   1272   1.1  riastrad  *	Flags may have:
   1273   1.1  riastrad  *
   1274   1.1  riastrad  *		ENTROPY_WAIT	Wait for entropy if not available yet.
   1275   1.1  riastrad  *		ENTROPY_SIG	Allow interruption by a signal during wait.
   1276  1.23  riastrad  *		ENTROPY_HARDFAIL Either fill the buffer with full entropy,
   1277  1.23  riastrad  *				or fail without filling it at all.
   1278   1.1  riastrad  *
   1279   1.1  riastrad  *	Return zero on success, or error on failure:
   1280   1.1  riastrad  *
   1281   1.1  riastrad  *		EWOULDBLOCK	No entropy and ENTROPY_WAIT not set.
   1282   1.1  riastrad  *		EINTR/ERESTART	No entropy, ENTROPY_SIG set, and interrupted.
   1283   1.1  riastrad  *
   1284   1.1  riastrad  *	If ENTROPY_WAIT is set, allowed only in thread context.  If
   1285   1.1  riastrad  *	ENTROPY_WAIT is not set, allowed up to IPL_VM.  (XXX That's
   1286   1.1  riastrad  *	awfully high...  Do we really need it in hard interrupts?  This
   1287   1.1  riastrad  *	arises from use of cprng_strong(9).)
   1288   1.1  riastrad  */
   1289   1.1  riastrad int
   1290   1.1  riastrad entropy_extract(void *buf, size_t len, int flags)
   1291   1.1  riastrad {
   1292   1.1  riastrad 	static const struct timeval interval = {.tv_sec = 60, .tv_usec = 0};
   1293   1.1  riastrad 	static struct timeval lasttime; /* serialized by E->lock */
   1294   1.1  riastrad 	int error;
   1295   1.1  riastrad 
   1296   1.1  riastrad 	if (ISSET(flags, ENTROPY_WAIT)) {
   1297   1.1  riastrad 		ASSERT_SLEEPABLE();
   1298   1.1  riastrad 		KASSERTMSG(E->stage >= ENTROPY_WARM,
   1299   1.1  riastrad 		    "can't wait for entropy until warm");
   1300   1.1  riastrad 	}
   1301   1.1  riastrad 
   1302  1.35  riastrad 	/* Refuse to operate in interrupt context.  */
   1303  1.35  riastrad 	KASSERT(!cpu_intr_p());
   1304  1.35  riastrad 
   1305   1.1  riastrad 	/* Acquire the global lock to get at the global pool.  */
   1306   1.1  riastrad 	if (E->stage >= ENTROPY_WARM)
   1307   1.1  riastrad 		mutex_enter(&E->lock);
   1308   1.1  riastrad 
   1309   1.1  riastrad 	/* Wait until there is enough entropy in the system.  */
   1310   1.1  riastrad 	error = 0;
   1311   1.1  riastrad 	while (E->needed) {
   1312   1.1  riastrad 		/* Ask for more, synchronously if possible.  */
   1313   1.1  riastrad 		entropy_request(len);
   1314   1.1  riastrad 
   1315   1.1  riastrad 		/* If we got enough, we're done.  */
   1316   1.1  riastrad 		if (E->needed == 0) {
   1317   1.1  riastrad 			KASSERT(error == 0);
   1318   1.1  riastrad 			break;
   1319   1.1  riastrad 		}
   1320   1.1  riastrad 
   1321   1.1  riastrad 		/* If not waiting, stop here.  */
   1322   1.1  riastrad 		if (!ISSET(flags, ENTROPY_WAIT)) {
   1323   1.1  riastrad 			error = EWOULDBLOCK;
   1324   1.1  riastrad 			break;
   1325   1.1  riastrad 		}
   1326   1.1  riastrad 
   1327   1.1  riastrad 		/* Wait for some entropy to come in and try again.  */
   1328   1.1  riastrad 		KASSERT(E->stage >= ENTROPY_WARM);
   1329  1.24      gson 		printf("entropy: pid %d (%s) blocking due to lack of entropy\n",
   1330  1.24      gson 		       curproc->p_pid, curproc->p_comm);
   1331  1.24      gson 
   1332   1.1  riastrad 		if (ISSET(flags, ENTROPY_SIG)) {
   1333   1.1  riastrad 			error = cv_wait_sig(&E->cv, &E->lock);
   1334   1.1  riastrad 			if (error)
   1335   1.1  riastrad 				break;
   1336   1.1  riastrad 		} else {
   1337   1.1  riastrad 			cv_wait(&E->cv, &E->lock);
   1338   1.1  riastrad 		}
   1339   1.1  riastrad 	}
   1340   1.1  riastrad 
   1341  1.23  riastrad 	/*
   1342  1.23  riastrad 	 * Count failure -- but fill the buffer nevertheless, unless
   1343  1.23  riastrad 	 * the caller specified ENTROPY_HARDFAIL.
   1344  1.23  riastrad 	 */
   1345  1.23  riastrad 	if (error) {
   1346  1.23  riastrad 		if (ISSET(flags, ENTROPY_HARDFAIL))
   1347  1.23  riastrad 			goto out;
   1348   1.1  riastrad 		entropy_extract_fail_evcnt.ev_count++;
   1349  1.23  riastrad 	}
   1350   1.1  riastrad 
   1351   1.1  riastrad 	/*
   1352   1.1  riastrad 	 * Report a warning if we have never yet reached full entropy.
   1353   1.1  riastrad 	 * This is the only case where we consider entropy to be
   1354   1.1  riastrad 	 * `depleted' without kern.entropy.depletion enabled -- when we
   1355   1.1  riastrad 	 * only have partial entropy, an adversary may be able to
   1356   1.1  riastrad 	 * narrow the state of the pool down to a small number of
   1357   1.1  riastrad 	 * possibilities; the output then enables them to confirm a
   1358   1.1  riastrad 	 * guess, reducing its entropy from the adversary's perspective
   1359   1.1  riastrad 	 * to zero.
   1360   1.1  riastrad 	 */
   1361   1.1  riastrad 	if (__predict_false(E->epoch == (unsigned)-1)) {
   1362   1.1  riastrad 		if (ratecheck(&lasttime, &interval))
   1363   1.1  riastrad 			printf("entropy: WARNING:"
   1364   1.1  riastrad 			    " extracting entropy too early\n");
   1365   1.1  riastrad 		atomic_store_relaxed(&E->needed, ENTROPY_CAPACITY*NBBY);
   1366   1.1  riastrad 	}
   1367   1.1  riastrad 
   1368   1.1  riastrad 	/* Extract data from the pool, and `deplete' if we're doing that.  */
   1369   1.1  riastrad 	entpool_extract(&E->pool, buf, len);
   1370   1.1  riastrad 	if (__predict_false(atomic_load_relaxed(&entropy_depletion)) &&
   1371   1.1  riastrad 	    error == 0) {
   1372   1.1  riastrad 		unsigned cost = MIN(len, ENTROPY_CAPACITY)*NBBY;
   1373   1.1  riastrad 
   1374   1.1  riastrad 		atomic_store_relaxed(&E->needed,
   1375   1.1  riastrad 		    E->needed + MIN(ENTROPY_CAPACITY*NBBY - E->needed, cost));
   1376   1.1  riastrad 		entropy_deplete_evcnt.ev_count++;
   1377   1.1  riastrad 	}
   1378   1.1  riastrad 
   1379  1.23  riastrad out:	/* Release the global lock and return the error.  */
   1380   1.1  riastrad 	if (E->stage >= ENTROPY_WARM)
   1381   1.1  riastrad 		mutex_exit(&E->lock);
   1382   1.1  riastrad 	return error;
   1383   1.1  riastrad }
   1384   1.1  riastrad 
   1385   1.1  riastrad /*
   1386   1.1  riastrad  * entropy_poll(events)
   1387   1.1  riastrad  *
   1388   1.1  riastrad  *	Return the subset of events ready, and if it is not all of
   1389   1.1  riastrad  *	events, record curlwp as waiting for entropy.
   1390   1.1  riastrad  */
   1391   1.1  riastrad int
   1392   1.1  riastrad entropy_poll(int events)
   1393   1.1  riastrad {
   1394   1.1  riastrad 	int revents = 0;
   1395   1.1  riastrad 
   1396   1.1  riastrad 	KASSERT(E->stage >= ENTROPY_WARM);
   1397   1.1  riastrad 
   1398   1.1  riastrad 	/* Always ready for writing.  */
   1399   1.1  riastrad 	revents |= events & (POLLOUT|POLLWRNORM);
   1400   1.1  riastrad 
   1401   1.1  riastrad 	/* Narrow it down to reads.  */
   1402   1.1  riastrad 	events &= POLLIN|POLLRDNORM;
   1403   1.1  riastrad 	if (events == 0)
   1404   1.1  riastrad 		return revents;
   1405   1.1  riastrad 
   1406   1.1  riastrad 	/*
   1407   1.1  riastrad 	 * If we have reached full entropy and we're not depleting
   1408   1.1  riastrad 	 * entropy, we are forever ready.
   1409   1.1  riastrad 	 */
   1410   1.1  riastrad 	if (__predict_true(atomic_load_relaxed(&E->needed) == 0) &&
   1411   1.1  riastrad 	    __predict_true(!atomic_load_relaxed(&entropy_depletion)))
   1412   1.1  riastrad 		return revents | events;
   1413   1.1  riastrad 
   1414   1.1  riastrad 	/*
   1415   1.1  riastrad 	 * Otherwise, check whether we need entropy under the lock.  If
   1416   1.1  riastrad 	 * we don't, we're ready; if we do, add ourselves to the queue.
   1417   1.1  riastrad 	 */
   1418   1.1  riastrad 	mutex_enter(&E->lock);
   1419   1.1  riastrad 	if (E->needed == 0)
   1420   1.1  riastrad 		revents |= events;
   1421   1.1  riastrad 	else
   1422   1.1  riastrad 		selrecord(curlwp, &E->selq);
   1423   1.1  riastrad 	mutex_exit(&E->lock);
   1424   1.1  riastrad 
   1425   1.1  riastrad 	return revents;
   1426   1.1  riastrad }
   1427   1.1  riastrad 
   1428   1.1  riastrad /*
   1429   1.1  riastrad  * filt_entropy_read_detach(kn)
   1430   1.1  riastrad  *
   1431   1.1  riastrad  *	struct filterops::f_detach callback for entropy read events:
   1432   1.1  riastrad  *	remove kn from the list of waiters.
   1433   1.1  riastrad  */
   1434   1.1  riastrad static void
   1435   1.1  riastrad filt_entropy_read_detach(struct knote *kn)
   1436   1.1  riastrad {
   1437   1.1  riastrad 
   1438   1.1  riastrad 	KASSERT(E->stage >= ENTROPY_WARM);
   1439   1.1  riastrad 
   1440   1.1  riastrad 	mutex_enter(&E->lock);
   1441  1.25   thorpej 	selremove_knote(&E->selq, kn);
   1442   1.1  riastrad 	mutex_exit(&E->lock);
   1443   1.1  riastrad }
   1444   1.1  riastrad 
   1445   1.1  riastrad /*
   1446   1.1  riastrad  * filt_entropy_read_event(kn, hint)
   1447   1.1  riastrad  *
   1448   1.1  riastrad  *	struct filterops::f_event callback for entropy read events:
   1449   1.1  riastrad  *	poll for entropy.  Caller must hold the global entropy lock if
   1450   1.1  riastrad  *	hint is NOTE_SUBMIT, and must not if hint is not NOTE_SUBMIT.
   1451   1.1  riastrad  */
   1452   1.1  riastrad static int
   1453   1.1  riastrad filt_entropy_read_event(struct knote *kn, long hint)
   1454   1.1  riastrad {
   1455   1.1  riastrad 	int ret;
   1456   1.1  riastrad 
   1457   1.1  riastrad 	KASSERT(E->stage >= ENTROPY_WARM);
   1458   1.1  riastrad 
   1459   1.1  riastrad 	/* Acquire the lock, if caller is outside entropy subsystem.  */
   1460   1.1  riastrad 	if (hint == NOTE_SUBMIT)
   1461   1.1  riastrad 		KASSERT(mutex_owned(&E->lock));
   1462   1.1  riastrad 	else
   1463   1.1  riastrad 		mutex_enter(&E->lock);
   1464   1.1  riastrad 
   1465   1.1  riastrad 	/*
   1466   1.1  riastrad 	 * If we still need entropy, can't read anything; if not, can
   1467   1.1  riastrad 	 * read arbitrarily much.
   1468   1.1  riastrad 	 */
   1469   1.1  riastrad 	if (E->needed != 0) {
   1470   1.1  riastrad 		ret = 0;
   1471   1.1  riastrad 	} else {
   1472   1.1  riastrad 		if (atomic_load_relaxed(&entropy_depletion))
   1473   1.1  riastrad 			kn->kn_data = ENTROPY_CAPACITY*NBBY;
   1474   1.1  riastrad 		else
   1475   1.1  riastrad 			kn->kn_data = MIN(INT64_MAX, SSIZE_MAX);
   1476   1.1  riastrad 		ret = 1;
   1477   1.1  riastrad 	}
   1478   1.1  riastrad 
   1479   1.1  riastrad 	/* Release the lock, if caller is outside entropy subsystem.  */
   1480   1.1  riastrad 	if (hint == NOTE_SUBMIT)
   1481   1.1  riastrad 		KASSERT(mutex_owned(&E->lock));
   1482   1.1  riastrad 	else
   1483   1.1  riastrad 		mutex_exit(&E->lock);
   1484   1.1  riastrad 
   1485   1.1  riastrad 	return ret;
   1486   1.1  riastrad }
   1487   1.1  riastrad 
   1488  1.33   thorpej /* XXX Makes sense only for /dev/u?random.  */
   1489   1.1  riastrad static const struct filterops entropy_read_filtops = {
   1490  1.33   thorpej 	.f_flags = FILTEROP_ISFD | FILTEROP_MPSAFE,
   1491   1.1  riastrad 	.f_attach = NULL,
   1492   1.1  riastrad 	.f_detach = filt_entropy_read_detach,
   1493   1.1  riastrad 	.f_event = filt_entropy_read_event,
   1494   1.1  riastrad };
   1495   1.1  riastrad 
   1496   1.1  riastrad /*
   1497   1.1  riastrad  * entropy_kqfilter(kn)
   1498   1.1  riastrad  *
   1499   1.1  riastrad  *	Register kn to receive entropy event notifications.  May be
   1500   1.1  riastrad  *	EVFILT_READ or EVFILT_WRITE; anything else yields EINVAL.
   1501   1.1  riastrad  */
   1502   1.1  riastrad int
   1503   1.1  riastrad entropy_kqfilter(struct knote *kn)
   1504   1.1  riastrad {
   1505   1.1  riastrad 
   1506   1.1  riastrad 	KASSERT(E->stage >= ENTROPY_WARM);
   1507   1.1  riastrad 
   1508   1.1  riastrad 	switch (kn->kn_filter) {
   1509   1.1  riastrad 	case EVFILT_READ:
   1510   1.1  riastrad 		/* Enter into the global select queue.  */
   1511   1.1  riastrad 		mutex_enter(&E->lock);
   1512   1.1  riastrad 		kn->kn_fop = &entropy_read_filtops;
   1513  1.25   thorpej 		selrecord_knote(&E->selq, kn);
   1514   1.1  riastrad 		mutex_exit(&E->lock);
   1515   1.1  riastrad 		return 0;
   1516   1.1  riastrad 	case EVFILT_WRITE:
   1517   1.1  riastrad 		/* Can always dump entropy into the system.  */
   1518   1.1  riastrad 		kn->kn_fop = &seltrue_filtops;
   1519   1.1  riastrad 		return 0;
   1520   1.1  riastrad 	default:
   1521   1.1  riastrad 		return EINVAL;
   1522   1.1  riastrad 	}
   1523   1.1  riastrad }
   1524   1.1  riastrad 
   1525   1.1  riastrad /*
   1526   1.1  riastrad  * rndsource_setcb(rs, get, getarg)
   1527   1.1  riastrad  *
   1528   1.1  riastrad  *	Set the request callback for the entropy source rs, if it can
   1529   1.1  riastrad  *	provide entropy on demand.  Must precede rnd_attach_source.
   1530   1.1  riastrad  */
   1531   1.1  riastrad void
   1532   1.1  riastrad rndsource_setcb(struct krndsource *rs, void (*get)(size_t, void *),
   1533   1.1  riastrad     void *getarg)
   1534   1.1  riastrad {
   1535   1.1  riastrad 
   1536   1.1  riastrad 	rs->get = get;
   1537   1.1  riastrad 	rs->getarg = getarg;
   1538   1.1  riastrad }
   1539   1.1  riastrad 
   1540   1.1  riastrad /*
   1541   1.1  riastrad  * rnd_attach_source(rs, name, type, flags)
   1542   1.1  riastrad  *
   1543   1.1  riastrad  *	Attach the entropy source rs.  Must be done after
   1544   1.1  riastrad  *	rndsource_setcb, if any, and before any calls to rnd_add_data.
   1545   1.1  riastrad  */
   1546   1.1  riastrad void
   1547   1.1  riastrad rnd_attach_source(struct krndsource *rs, const char *name, uint32_t type,
   1548   1.1  riastrad     uint32_t flags)
   1549   1.1  riastrad {
   1550   1.1  riastrad 	uint32_t extra[4];
   1551   1.1  riastrad 	unsigned i = 0;
   1552   1.1  riastrad 
   1553   1.1  riastrad 	/* Grab cycle counter to mix extra into the pool.  */
   1554   1.1  riastrad 	extra[i++] = entropy_timer();
   1555   1.1  riastrad 
   1556   1.1  riastrad 	/*
   1557   1.1  riastrad 	 * Apply some standard flags:
   1558   1.1  riastrad 	 *
   1559   1.1  riastrad 	 * - We do not bother with network devices by default, for
   1560   1.1  riastrad 	 *   hysterical raisins (perhaps: because it is often the case
   1561   1.1  riastrad 	 *   that an adversary can influence network packet timings).
   1562   1.1  riastrad 	 */
   1563   1.1  riastrad 	switch (type) {
   1564   1.1  riastrad 	case RND_TYPE_NET:
   1565   1.1  riastrad 		flags |= RND_FLAG_NO_COLLECT;
   1566   1.1  riastrad 		break;
   1567   1.1  riastrad 	}
   1568   1.1  riastrad 
   1569   1.1  riastrad 	/* Sanity-check the callback if RND_FLAG_HASCB is set.  */
   1570   1.1  riastrad 	KASSERT(!ISSET(flags, RND_FLAG_HASCB) || rs->get != NULL);
   1571   1.1  riastrad 
   1572   1.1  riastrad 	/* Initialize the random source.  */
   1573   1.1  riastrad 	memset(rs->name, 0, sizeof(rs->name)); /* paranoia */
   1574   1.1  riastrad 	strlcpy(rs->name, name, sizeof(rs->name));
   1575  1.28  riastrad 	memset(&rs->time_delta, 0, sizeof(rs->time_delta));
   1576  1.28  riastrad 	memset(&rs->value_delta, 0, sizeof(rs->value_delta));
   1577   1.9  riastrad 	rs->total = 0;
   1578   1.1  riastrad 	rs->type = type;
   1579   1.1  riastrad 	rs->flags = flags;
   1580   1.1  riastrad 	if (E->stage >= ENTROPY_WARM)
   1581   1.1  riastrad 		rs->state = percpu_alloc(sizeof(struct rndsource_cpu));
   1582   1.1  riastrad 	extra[i++] = entropy_timer();
   1583   1.1  riastrad 
   1584   1.1  riastrad 	/* Wire it into the global list of random sources.  */
   1585   1.1  riastrad 	if (E->stage >= ENTROPY_WARM)
   1586   1.1  riastrad 		mutex_enter(&E->lock);
   1587   1.1  riastrad 	LIST_INSERT_HEAD(&E->sources, rs, list);
   1588   1.1  riastrad 	if (E->stage >= ENTROPY_WARM)
   1589   1.1  riastrad 		mutex_exit(&E->lock);
   1590   1.1  riastrad 	extra[i++] = entropy_timer();
   1591   1.1  riastrad 
   1592   1.1  riastrad 	/* Request that it provide entropy ASAP, if we can.  */
   1593   1.1  riastrad 	if (ISSET(flags, RND_FLAG_HASCB))
   1594   1.1  riastrad 		(*rs->get)(ENTROPY_CAPACITY, rs->getarg);
   1595   1.1  riastrad 	extra[i++] = entropy_timer();
   1596   1.1  riastrad 
   1597   1.1  riastrad 	/* Mix the extra into the pool.  */
   1598   1.1  riastrad 	KASSERT(i == __arraycount(extra));
   1599   1.1  riastrad 	entropy_enter(extra, sizeof extra, 0);
   1600   1.1  riastrad 	explicit_memset(extra, 0, sizeof extra);
   1601   1.1  riastrad }
   1602   1.1  riastrad 
   1603   1.1  riastrad /*
   1604   1.1  riastrad  * rnd_detach_source(rs)
   1605   1.1  riastrad  *
   1606   1.1  riastrad  *	Detach the entropy source rs.  May sleep waiting for users to
   1607   1.1  riastrad  *	drain.  Further use is not allowed.
   1608   1.1  riastrad  */
   1609   1.1  riastrad void
   1610   1.1  riastrad rnd_detach_source(struct krndsource *rs)
   1611   1.1  riastrad {
   1612   1.1  riastrad 
   1613   1.1  riastrad 	/*
   1614   1.1  riastrad 	 * If we're cold (shouldn't happen, but hey), just remove it
   1615   1.1  riastrad 	 * from the list -- there's nothing allocated.
   1616   1.1  riastrad 	 */
   1617   1.1  riastrad 	if (E->stage == ENTROPY_COLD) {
   1618   1.1  riastrad 		LIST_REMOVE(rs, list);
   1619   1.1  riastrad 		return;
   1620   1.1  riastrad 	}
   1621   1.1  riastrad 
   1622   1.1  riastrad 	/* We may have to wait for entropy_request.  */
   1623   1.1  riastrad 	ASSERT_SLEEPABLE();
   1624   1.1  riastrad 
   1625   1.4  riastrad 	/* Wait until the source list is not in use, and remove it.  */
   1626   1.1  riastrad 	mutex_enter(&E->lock);
   1627   1.4  riastrad 	while (E->sourcelock)
   1628  1.27  riastrad 		cv_wait(&E->sourcelock_cv, &E->lock);
   1629   1.1  riastrad 	LIST_REMOVE(rs, list);
   1630   1.1  riastrad 	mutex_exit(&E->lock);
   1631   1.1  riastrad 
   1632   1.1  riastrad 	/* Free the per-CPU data.  */
   1633   1.1  riastrad 	percpu_free(rs->state, sizeof(struct rndsource_cpu));
   1634   1.1  riastrad }
   1635   1.1  riastrad 
   1636   1.1  riastrad /*
   1637   1.4  riastrad  * rnd_lock_sources()
   1638   1.4  riastrad  *
   1639   1.4  riastrad  *	Prevent changes to the list of rndsources while we iterate it.
   1640   1.4  riastrad  *	Interruptible.  Caller must hold the global entropy lock.  If
   1641   1.4  riastrad  *	successful, no rndsource will go away until rnd_unlock_sources
   1642   1.4  riastrad  *	even while the caller releases the global entropy lock.
   1643   1.4  riastrad  */
   1644   1.4  riastrad static int
   1645   1.4  riastrad rnd_lock_sources(void)
   1646   1.4  riastrad {
   1647   1.4  riastrad 	int error;
   1648   1.4  riastrad 
   1649   1.4  riastrad 	KASSERT(mutex_owned(&E->lock));
   1650   1.4  riastrad 
   1651   1.4  riastrad 	while (E->sourcelock) {
   1652  1.27  riastrad 		error = cv_wait_sig(&E->sourcelock_cv, &E->lock);
   1653   1.4  riastrad 		if (error)
   1654   1.4  riastrad 			return error;
   1655   1.4  riastrad 	}
   1656   1.4  riastrad 
   1657   1.4  riastrad 	E->sourcelock = curlwp;
   1658   1.4  riastrad 	return 0;
   1659   1.4  riastrad }
   1660   1.4  riastrad 
   1661   1.4  riastrad /*
   1662   1.4  riastrad  * rnd_trylock_sources()
   1663   1.4  riastrad  *
   1664   1.4  riastrad  *	Try to lock the list of sources, but if it's already locked,
   1665   1.4  riastrad  *	fail.  Caller must hold the global entropy lock.  If
   1666   1.4  riastrad  *	successful, no rndsource will go away until rnd_unlock_sources
   1667   1.4  riastrad  *	even while the caller releases the global entropy lock.
   1668   1.4  riastrad  */
   1669   1.4  riastrad static bool
   1670   1.4  riastrad rnd_trylock_sources(void)
   1671   1.4  riastrad {
   1672   1.4  riastrad 
   1673   1.4  riastrad 	KASSERT(E->stage == ENTROPY_COLD || mutex_owned(&E->lock));
   1674   1.4  riastrad 
   1675   1.4  riastrad 	if (E->sourcelock)
   1676   1.4  riastrad 		return false;
   1677  1.16  riastrad 	E->sourcelock = curlwp;
   1678   1.4  riastrad 	return true;
   1679   1.4  riastrad }
   1680   1.4  riastrad 
   1681   1.4  riastrad /*
   1682   1.4  riastrad  * rnd_unlock_sources()
   1683   1.4  riastrad  *
   1684   1.4  riastrad  *	Unlock the list of sources after rnd_lock_sources or
   1685   1.4  riastrad  *	rnd_trylock_sources.  Caller must hold the global entropy lock.
   1686   1.4  riastrad  */
   1687   1.4  riastrad static void
   1688   1.4  riastrad rnd_unlock_sources(void)
   1689   1.4  riastrad {
   1690   1.4  riastrad 
   1691   1.4  riastrad 	KASSERT(E->stage == ENTROPY_COLD || mutex_owned(&E->lock));
   1692   1.4  riastrad 
   1693  1.16  riastrad 	KASSERTMSG(E->sourcelock == curlwp, "lwp %p releasing lock held by %p",
   1694  1.16  riastrad 	    curlwp, E->sourcelock);
   1695   1.4  riastrad 	E->sourcelock = NULL;
   1696   1.4  riastrad 	if (E->stage >= ENTROPY_WARM)
   1697  1.27  riastrad 		cv_signal(&E->sourcelock_cv);
   1698   1.4  riastrad }
   1699   1.4  riastrad 
   1700   1.4  riastrad /*
   1701   1.4  riastrad  * rnd_sources_locked()
   1702   1.4  riastrad  *
   1703   1.4  riastrad  *	True if we hold the list of rndsources locked, for diagnostic
   1704   1.4  riastrad  *	assertions.
   1705   1.4  riastrad  */
   1706   1.7  riastrad static bool __diagused
   1707   1.4  riastrad rnd_sources_locked(void)
   1708   1.4  riastrad {
   1709   1.4  riastrad 
   1710  1.16  riastrad 	return E->sourcelock == curlwp;
   1711   1.4  riastrad }
   1712   1.4  riastrad 
   1713   1.4  riastrad /*
   1714   1.1  riastrad  * entropy_request(nbytes)
   1715   1.1  riastrad  *
   1716   1.1  riastrad  *	Request nbytes bytes of entropy from all sources in the system.
   1717   1.1  riastrad  *	OK if we overdo it.  Caller must hold the global entropy lock;
   1718   1.1  riastrad  *	will release and re-acquire it.
   1719   1.1  riastrad  */
   1720   1.1  riastrad static void
   1721   1.1  riastrad entropy_request(size_t nbytes)
   1722   1.1  riastrad {
   1723   1.4  riastrad 	struct krndsource *rs;
   1724   1.1  riastrad 
   1725   1.1  riastrad 	KASSERT(E->stage == ENTROPY_COLD || mutex_owned(&E->lock));
   1726   1.1  riastrad 
   1727   1.1  riastrad 	/*
   1728   1.1  riastrad 	 * If there is a request in progress, let it proceed.
   1729   1.1  riastrad 	 * Otherwise, note that a request is in progress to avoid
   1730   1.1  riastrad 	 * reentry and to block rnd_detach_source until we're done.
   1731   1.1  riastrad 	 */
   1732   1.4  riastrad 	if (!rnd_trylock_sources())
   1733   1.1  riastrad 		return;
   1734   1.1  riastrad 	entropy_request_evcnt.ev_count++;
   1735   1.1  riastrad 
   1736   1.1  riastrad 	/* Clamp to the maximum reasonable request.  */
   1737   1.1  riastrad 	nbytes = MIN(nbytes, ENTROPY_CAPACITY);
   1738   1.1  riastrad 
   1739   1.1  riastrad 	/* Walk the list of sources.  */
   1740   1.4  riastrad 	LIST_FOREACH(rs, &E->sources, list) {
   1741   1.1  riastrad 		/* Skip sources without callbacks.  */
   1742   1.1  riastrad 		if (!ISSET(rs->flags, RND_FLAG_HASCB))
   1743   1.1  riastrad 			continue;
   1744   1.1  riastrad 
   1745  1.22  riastrad 		/*
   1746  1.22  riastrad 		 * Skip sources that are disabled altogether -- we
   1747  1.22  riastrad 		 * would just ignore their samples anyway.
   1748  1.22  riastrad 		 */
   1749  1.22  riastrad 		if (ISSET(rs->flags, RND_FLAG_NO_COLLECT))
   1750  1.22  riastrad 			continue;
   1751  1.22  riastrad 
   1752   1.1  riastrad 		/* Drop the lock while we call the callback.  */
   1753   1.1  riastrad 		if (E->stage >= ENTROPY_WARM)
   1754   1.1  riastrad 			mutex_exit(&E->lock);
   1755   1.1  riastrad 		(*rs->get)(nbytes, rs->getarg);
   1756   1.1  riastrad 		if (E->stage >= ENTROPY_WARM)
   1757   1.1  riastrad 			mutex_enter(&E->lock);
   1758   1.1  riastrad 	}
   1759   1.1  riastrad 
   1760   1.1  riastrad 	/* Notify rnd_detach_source that the request is done.  */
   1761   1.4  riastrad 	rnd_unlock_sources();
   1762   1.1  riastrad }
   1763   1.1  riastrad 
   1764   1.1  riastrad /*
   1765   1.1  riastrad  * rnd_add_uint32(rs, value)
   1766   1.1  riastrad  *
   1767   1.1  riastrad  *	Enter 32 bits of data from an entropy source into the pool.
   1768   1.1  riastrad  *
   1769   1.1  riastrad  *	If rs is NULL, may not be called from interrupt context.
   1770   1.1  riastrad  *
   1771   1.1  riastrad  *	If rs is non-NULL, may be called from any context.  May drop
   1772   1.1  riastrad  *	data if called from interrupt context.
   1773   1.1  riastrad  */
   1774   1.1  riastrad void
   1775   1.1  riastrad rnd_add_uint32(struct krndsource *rs, uint32_t value)
   1776   1.1  riastrad {
   1777   1.1  riastrad 
   1778   1.1  riastrad 	rnd_add_data(rs, &value, sizeof value, 0);
   1779   1.1  riastrad }
   1780   1.1  riastrad 
   1781   1.1  riastrad void
   1782   1.1  riastrad _rnd_add_uint32(struct krndsource *rs, uint32_t value)
   1783   1.1  riastrad {
   1784   1.1  riastrad 
   1785   1.1  riastrad 	rnd_add_data(rs, &value, sizeof value, 0);
   1786   1.1  riastrad }
   1787   1.1  riastrad 
   1788   1.1  riastrad void
   1789   1.1  riastrad _rnd_add_uint64(struct krndsource *rs, uint64_t value)
   1790   1.1  riastrad {
   1791   1.1  riastrad 
   1792   1.1  riastrad 	rnd_add_data(rs, &value, sizeof value, 0);
   1793   1.1  riastrad }
   1794   1.1  riastrad 
   1795   1.1  riastrad /*
   1796   1.1  riastrad  * rnd_add_data(rs, buf, len, entropybits)
   1797   1.1  riastrad  *
   1798   1.1  riastrad  *	Enter data from an entropy source into the pool, with a
   1799   1.1  riastrad  *	driver's estimate of how much entropy the physical source of
   1800   1.1  riastrad  *	the data has.  If RND_FLAG_NO_ESTIMATE, we ignore the driver's
   1801   1.1  riastrad  *	estimate and treat it as zero.
   1802   1.1  riastrad  *
   1803   1.1  riastrad  *	If rs is NULL, may not be called from interrupt context.
   1804   1.1  riastrad  *
   1805   1.1  riastrad  *	If rs is non-NULL, may be called from any context.  May drop
   1806   1.1  riastrad  *	data if called from interrupt context.
   1807   1.1  riastrad  */
   1808   1.1  riastrad void
   1809   1.1  riastrad rnd_add_data(struct krndsource *rs, const void *buf, uint32_t len,
   1810   1.1  riastrad     uint32_t entropybits)
   1811   1.1  riastrad {
   1812   1.1  riastrad 	uint32_t extra;
   1813   1.1  riastrad 	uint32_t flags;
   1814   1.1  riastrad 
   1815   1.1  riastrad 	KASSERTMSG(howmany(entropybits, NBBY) <= len,
   1816   1.1  riastrad 	    "%s: impossible entropy rate:"
   1817   1.1  riastrad 	    " %"PRIu32" bits in %"PRIu32"-byte string",
   1818   1.1  riastrad 	    rs ? rs->name : "(anonymous)", entropybits, len);
   1819   1.1  riastrad 
   1820   1.1  riastrad 	/* If there's no rndsource, just enter the data and time now.  */
   1821   1.1  riastrad 	if (rs == NULL) {
   1822   1.1  riastrad 		entropy_enter(buf, len, entropybits);
   1823   1.1  riastrad 		extra = entropy_timer();
   1824   1.1  riastrad 		entropy_enter(&extra, sizeof extra, 0);
   1825   1.1  riastrad 		explicit_memset(&extra, 0, sizeof extra);
   1826   1.1  riastrad 		return;
   1827   1.1  riastrad 	}
   1828   1.1  riastrad 
   1829   1.1  riastrad 	/* Load a snapshot of the flags.  Ioctl may change them under us.  */
   1830   1.1  riastrad 	flags = atomic_load_relaxed(&rs->flags);
   1831   1.1  riastrad 
   1832   1.1  riastrad 	/*
   1833   1.1  riastrad 	 * Skip if:
   1834   1.1  riastrad 	 * - we're not collecting entropy, or
   1835   1.1  riastrad 	 * - the operator doesn't want to collect entropy from this, or
   1836   1.1  riastrad 	 * - neither data nor timings are being collected from this.
   1837   1.1  riastrad 	 */
   1838   1.1  riastrad 	if (!atomic_load_relaxed(&entropy_collection) ||
   1839   1.1  riastrad 	    ISSET(flags, RND_FLAG_NO_COLLECT) ||
   1840   1.1  riastrad 	    !ISSET(flags, RND_FLAG_COLLECT_VALUE|RND_FLAG_COLLECT_TIME))
   1841   1.1  riastrad 		return;
   1842   1.1  riastrad 
   1843   1.1  riastrad 	/* If asked, ignore the estimate.  */
   1844   1.1  riastrad 	if (ISSET(flags, RND_FLAG_NO_ESTIMATE))
   1845   1.1  riastrad 		entropybits = 0;
   1846   1.1  riastrad 
   1847   1.1  riastrad 	/* If we are collecting data, enter them.  */
   1848   1.1  riastrad 	if (ISSET(flags, RND_FLAG_COLLECT_VALUE))
   1849  1.28  riastrad 		rnd_add_data_1(rs, buf, len, entropybits,
   1850  1.28  riastrad 		    RND_FLAG_COLLECT_VALUE);
   1851   1.1  riastrad 
   1852   1.1  riastrad 	/* If we are collecting timings, enter one.  */
   1853   1.1  riastrad 	if (ISSET(flags, RND_FLAG_COLLECT_TIME)) {
   1854   1.1  riastrad 		extra = entropy_timer();
   1855  1.28  riastrad 		rnd_add_data_1(rs, &extra, sizeof extra, 0,
   1856  1.28  riastrad 		    RND_FLAG_COLLECT_TIME);
   1857   1.1  riastrad 	}
   1858   1.1  riastrad }
   1859   1.1  riastrad 
   1860  1.28  riastrad static unsigned
   1861  1.28  riastrad add_sat(unsigned a, unsigned b)
   1862  1.28  riastrad {
   1863  1.28  riastrad 	unsigned c = a + b;
   1864  1.28  riastrad 
   1865  1.28  riastrad 	return (c < a ? UINT_MAX : c);
   1866  1.28  riastrad }
   1867  1.28  riastrad 
   1868   1.1  riastrad /*
   1869  1.28  riastrad  * rnd_add_data_1(rs, buf, len, entropybits, flag)
   1870   1.1  riastrad  *
   1871   1.1  riastrad  *	Internal subroutine to call either entropy_enter_intr, if we're
   1872   1.1  riastrad  *	in interrupt context, or entropy_enter if not, and to count the
   1873   1.1  riastrad  *	entropy in an rndsource.
   1874   1.1  riastrad  */
   1875   1.1  riastrad static void
   1876   1.1  riastrad rnd_add_data_1(struct krndsource *rs, const void *buf, uint32_t len,
   1877  1.28  riastrad     uint32_t entropybits, uint32_t flag)
   1878   1.1  riastrad {
   1879   1.1  riastrad 	bool fullyused;
   1880   1.1  riastrad 
   1881   1.1  riastrad 	/*
   1882   1.1  riastrad 	 * If we're in interrupt context, use entropy_enter_intr and
   1883   1.1  riastrad 	 * take note of whether it consumed the full sample; if not,
   1884   1.1  riastrad 	 * use entropy_enter, which always consumes the full sample.
   1885   1.1  riastrad 	 */
   1886  1.16  riastrad 	if (curlwp && cpu_intr_p()) {
   1887   1.1  riastrad 		fullyused = entropy_enter_intr(buf, len, entropybits);
   1888   1.1  riastrad 	} else {
   1889   1.1  riastrad 		entropy_enter(buf, len, entropybits);
   1890   1.1  riastrad 		fullyused = true;
   1891   1.1  riastrad 	}
   1892   1.1  riastrad 
   1893   1.1  riastrad 	/*
   1894   1.1  riastrad 	 * If we used the full sample, note how many bits were
   1895   1.1  riastrad 	 * contributed from this source.
   1896   1.1  riastrad 	 */
   1897   1.1  riastrad 	if (fullyused) {
   1898  1.37  riastrad 		if (__predict_false(E->stage == ENTROPY_COLD)) {
   1899  1.28  riastrad 			rs->total = add_sat(rs->total, entropybits);
   1900  1.28  riastrad 			switch (flag) {
   1901  1.28  riastrad 			case RND_FLAG_COLLECT_TIME:
   1902  1.28  riastrad 				rs->time_delta.insamples =
   1903  1.28  riastrad 				    add_sat(rs->time_delta.insamples, 1);
   1904  1.28  riastrad 				break;
   1905  1.28  riastrad 			case RND_FLAG_COLLECT_VALUE:
   1906  1.28  riastrad 				rs->value_delta.insamples =
   1907  1.28  riastrad 				    add_sat(rs->value_delta.insamples, 1);
   1908  1.28  riastrad 				break;
   1909  1.28  riastrad 			}
   1910   1.1  riastrad 		} else {
   1911   1.1  riastrad 			struct rndsource_cpu *rc = percpu_getref(rs->state);
   1912   1.1  riastrad 
   1913  1.28  riastrad 			atomic_store_relaxed(&rc->rc_entropybits,
   1914  1.28  riastrad 			    add_sat(rc->rc_entropybits, entropybits));
   1915  1.28  riastrad 			switch (flag) {
   1916  1.28  riastrad 			case RND_FLAG_COLLECT_TIME:
   1917  1.28  riastrad 				atomic_store_relaxed(&rc->rc_timesamples,
   1918  1.28  riastrad 				    add_sat(rc->rc_timesamples, 1));
   1919  1.28  riastrad 				break;
   1920  1.28  riastrad 			case RND_FLAG_COLLECT_VALUE:
   1921  1.28  riastrad 				atomic_store_relaxed(&rc->rc_datasamples,
   1922  1.28  riastrad 				    add_sat(rc->rc_datasamples, 1));
   1923  1.28  riastrad 				break;
   1924  1.28  riastrad 			}
   1925   1.1  riastrad 			percpu_putref(rs->state);
   1926   1.1  riastrad 		}
   1927   1.1  riastrad 	}
   1928   1.1  riastrad }
   1929   1.1  riastrad 
   1930   1.1  riastrad /*
   1931   1.1  riastrad  * rnd_add_data_sync(rs, buf, len, entropybits)
   1932   1.1  riastrad  *
   1933   1.1  riastrad  *	Same as rnd_add_data.  Originally used in rndsource callbacks,
   1934   1.1  riastrad  *	to break an unnecessary cycle; no longer really needed.
   1935   1.1  riastrad  */
   1936   1.1  riastrad void
   1937   1.1  riastrad rnd_add_data_sync(struct krndsource *rs, const void *buf, uint32_t len,
   1938   1.1  riastrad     uint32_t entropybits)
   1939   1.1  riastrad {
   1940   1.1  riastrad 
   1941   1.1  riastrad 	rnd_add_data(rs, buf, len, entropybits);
   1942   1.1  riastrad }
   1943   1.1  riastrad 
   1944   1.1  riastrad /*
   1945   1.1  riastrad  * rndsource_entropybits(rs)
   1946   1.1  riastrad  *
   1947   1.1  riastrad  *	Return approximately the number of bits of entropy that have
   1948   1.1  riastrad  *	been contributed via rs so far.  Approximate if other CPUs may
   1949   1.1  riastrad  *	be calling rnd_add_data concurrently.
   1950   1.1  riastrad  */
   1951   1.1  riastrad static unsigned
   1952   1.1  riastrad rndsource_entropybits(struct krndsource *rs)
   1953   1.1  riastrad {
   1954   1.1  riastrad 	unsigned nbits = rs->total;
   1955   1.1  riastrad 
   1956   1.1  riastrad 	KASSERT(E->stage >= ENTROPY_WARM);
   1957   1.4  riastrad 	KASSERT(rnd_sources_locked());
   1958   1.1  riastrad 	percpu_foreach(rs->state, rndsource_entropybits_cpu, &nbits);
   1959   1.1  riastrad 	return nbits;
   1960   1.1  riastrad }
   1961   1.1  riastrad 
   1962   1.1  riastrad static void
   1963   1.1  riastrad rndsource_entropybits_cpu(void *ptr, void *cookie, struct cpu_info *ci)
   1964   1.1  riastrad {
   1965   1.1  riastrad 	struct rndsource_cpu *rc = ptr;
   1966   1.1  riastrad 	unsigned *nbitsp = cookie;
   1967   1.1  riastrad 	unsigned cpu_nbits;
   1968   1.1  riastrad 
   1969  1.28  riastrad 	cpu_nbits = atomic_load_relaxed(&rc->rc_entropybits);
   1970   1.1  riastrad 	*nbitsp += MIN(UINT_MAX - *nbitsp, cpu_nbits);
   1971   1.1  riastrad }
   1972   1.1  riastrad 
   1973   1.1  riastrad /*
   1974   1.1  riastrad  * rndsource_to_user(rs, urs)
   1975   1.1  riastrad  *
   1976   1.1  riastrad  *	Copy a description of rs out to urs for userland.
   1977   1.1  riastrad  */
   1978   1.1  riastrad static void
   1979   1.1  riastrad rndsource_to_user(struct krndsource *rs, rndsource_t *urs)
   1980   1.1  riastrad {
   1981   1.1  riastrad 
   1982   1.1  riastrad 	KASSERT(E->stage >= ENTROPY_WARM);
   1983   1.4  riastrad 	KASSERT(rnd_sources_locked());
   1984   1.1  riastrad 
   1985   1.1  riastrad 	/* Avoid kernel memory disclosure.  */
   1986   1.1  riastrad 	memset(urs, 0, sizeof(*urs));
   1987   1.1  riastrad 
   1988   1.1  riastrad 	CTASSERT(sizeof(urs->name) == sizeof(rs->name));
   1989   1.1  riastrad 	strlcpy(urs->name, rs->name, sizeof(urs->name));
   1990   1.1  riastrad 	urs->total = rndsource_entropybits(rs);
   1991   1.1  riastrad 	urs->type = rs->type;
   1992   1.1  riastrad 	urs->flags = atomic_load_relaxed(&rs->flags);
   1993   1.1  riastrad }
   1994   1.1  riastrad 
   1995   1.1  riastrad /*
   1996   1.1  riastrad  * rndsource_to_user_est(rs, urse)
   1997   1.1  riastrad  *
   1998   1.1  riastrad  *	Copy a description of rs and estimation statistics out to urse
   1999   1.1  riastrad  *	for userland.
   2000   1.1  riastrad  */
   2001   1.1  riastrad static void
   2002   1.1  riastrad rndsource_to_user_est(struct krndsource *rs, rndsource_est_t *urse)
   2003   1.1  riastrad {
   2004   1.1  riastrad 
   2005   1.1  riastrad 	KASSERT(E->stage >= ENTROPY_WARM);
   2006   1.4  riastrad 	KASSERT(rnd_sources_locked());
   2007   1.1  riastrad 
   2008   1.1  riastrad 	/* Avoid kernel memory disclosure.  */
   2009   1.1  riastrad 	memset(urse, 0, sizeof(*urse));
   2010   1.1  riastrad 
   2011   1.1  riastrad 	/* Copy out the rndsource description.  */
   2012   1.1  riastrad 	rndsource_to_user(rs, &urse->rt);
   2013   1.1  riastrad 
   2014  1.28  riastrad 	/* Gather the statistics.  */
   2015  1.28  riastrad 	urse->dt_samples = rs->time_delta.insamples;
   2016   1.1  riastrad 	urse->dt_total = 0;
   2017  1.28  riastrad 	urse->dv_samples = rs->value_delta.insamples;
   2018  1.28  riastrad 	urse->dv_total = urse->rt.total;
   2019  1.28  riastrad 	percpu_foreach(rs->state, rndsource_to_user_est_cpu, urse);
   2020  1.28  riastrad }
   2021  1.28  riastrad 
   2022  1.28  riastrad static void
   2023  1.28  riastrad rndsource_to_user_est_cpu(void *ptr, void *cookie, struct cpu_info *ci)
   2024  1.28  riastrad {
   2025  1.28  riastrad 	struct rndsource_cpu *rc = ptr;
   2026  1.28  riastrad 	rndsource_est_t *urse = cookie;
   2027  1.28  riastrad 
   2028  1.28  riastrad 	urse->dt_samples = add_sat(urse->dt_samples,
   2029  1.28  riastrad 	    atomic_load_relaxed(&rc->rc_timesamples));
   2030  1.28  riastrad 	urse->dv_samples = add_sat(urse->dv_samples,
   2031  1.28  riastrad 	    atomic_load_relaxed(&rc->rc_datasamples));
   2032   1.1  riastrad }
   2033   1.1  riastrad 
   2034   1.1  riastrad /*
   2035  1.21  riastrad  * entropy_reset_xc(arg1, arg2)
   2036  1.21  riastrad  *
   2037  1.21  riastrad  *	Reset the current CPU's pending entropy to zero.
   2038  1.21  riastrad  */
   2039  1.21  riastrad static void
   2040  1.21  riastrad entropy_reset_xc(void *arg1 __unused, void *arg2 __unused)
   2041  1.21  riastrad {
   2042  1.21  riastrad 	uint32_t extra = entropy_timer();
   2043  1.21  riastrad 	struct entropy_cpu *ec;
   2044  1.21  riastrad 	int s;
   2045  1.21  riastrad 
   2046  1.21  riastrad 	/*
   2047  1.21  riastrad 	 * Acquire the per-CPU state, blocking soft interrupts and
   2048  1.21  riastrad 	 * causing hard interrupts to drop samples on the floor.
   2049  1.21  riastrad 	 */
   2050  1.21  riastrad 	ec = percpu_getref(entropy_percpu);
   2051  1.21  riastrad 	s = splsoftserial();
   2052  1.21  riastrad 	KASSERT(!ec->ec_locked);
   2053  1.21  riastrad 	ec->ec_locked = true;
   2054  1.21  riastrad 	__insn_barrier();
   2055  1.21  riastrad 
   2056  1.21  riastrad 	/* Zero the pending count and enter a cycle count for fun.  */
   2057  1.21  riastrad 	ec->ec_pending = 0;
   2058  1.21  riastrad 	entpool_enter(ec->ec_pool, &extra, sizeof extra);
   2059  1.21  riastrad 
   2060  1.21  riastrad 	/* Release the per-CPU state.  */
   2061  1.21  riastrad 	KASSERT(ec->ec_locked);
   2062  1.21  riastrad 	__insn_barrier();
   2063  1.21  riastrad 	ec->ec_locked = false;
   2064  1.21  riastrad 	splx(s);
   2065  1.21  riastrad 	percpu_putref(entropy_percpu);
   2066  1.21  riastrad }
   2067  1.21  riastrad 
   2068  1.21  riastrad /*
   2069   1.1  riastrad  * entropy_ioctl(cmd, data)
   2070   1.1  riastrad  *
   2071   1.1  riastrad  *	Handle various /dev/random ioctl queries.
   2072   1.1  riastrad  */
   2073   1.1  riastrad int
   2074   1.1  riastrad entropy_ioctl(unsigned long cmd, void *data)
   2075   1.1  riastrad {
   2076   1.1  riastrad 	struct krndsource *rs;
   2077   1.1  riastrad 	bool privileged;
   2078   1.1  riastrad 	int error;
   2079   1.1  riastrad 
   2080   1.1  riastrad 	KASSERT(E->stage >= ENTROPY_WARM);
   2081   1.1  riastrad 
   2082   1.1  riastrad 	/* Verify user's authorization to perform the ioctl.  */
   2083   1.1  riastrad 	switch (cmd) {
   2084   1.1  riastrad 	case RNDGETENTCNT:
   2085   1.1  riastrad 	case RNDGETPOOLSTAT:
   2086   1.1  riastrad 	case RNDGETSRCNUM:
   2087   1.1  riastrad 	case RNDGETSRCNAME:
   2088   1.1  riastrad 	case RNDGETESTNUM:
   2089   1.1  riastrad 	case RNDGETESTNAME:
   2090  1.31  christos 		error = kauth_authorize_device(kauth_cred_get(),
   2091   1.1  riastrad 		    KAUTH_DEVICE_RND_GETPRIV, NULL, NULL, NULL, NULL);
   2092   1.1  riastrad 		break;
   2093   1.1  riastrad 	case RNDCTL:
   2094  1.31  christos 		error = kauth_authorize_device(kauth_cred_get(),
   2095   1.1  riastrad 		    KAUTH_DEVICE_RND_SETPRIV, NULL, NULL, NULL, NULL);
   2096   1.1  riastrad 		break;
   2097   1.1  riastrad 	case RNDADDDATA:
   2098  1.31  christos 		error = kauth_authorize_device(kauth_cred_get(),
   2099   1.1  riastrad 		    KAUTH_DEVICE_RND_ADDDATA, NULL, NULL, NULL, NULL);
   2100   1.1  riastrad 		/* Ascertain whether the user's inputs should be counted.  */
   2101  1.31  christos 		if (kauth_authorize_device(kauth_cred_get(),
   2102   1.1  riastrad 			KAUTH_DEVICE_RND_ADDDATA_ESTIMATE,
   2103   1.1  riastrad 			NULL, NULL, NULL, NULL) == 0)
   2104   1.1  riastrad 			privileged = true;
   2105   1.1  riastrad 		break;
   2106   1.1  riastrad 	default: {
   2107   1.1  riastrad 		/*
   2108   1.1  riastrad 		 * XXX Hack to avoid changing module ABI so this can be
   2109   1.1  riastrad 		 * pulled up.  Later, we can just remove the argument.
   2110   1.1  riastrad 		 */
   2111   1.1  riastrad 		static const struct fileops fops = {
   2112   1.1  riastrad 			.fo_ioctl = rnd_system_ioctl,
   2113   1.1  riastrad 		};
   2114   1.1  riastrad 		struct file f = {
   2115   1.1  riastrad 			.f_ops = &fops,
   2116   1.1  riastrad 		};
   2117   1.1  riastrad 		MODULE_HOOK_CALL(rnd_ioctl_50_hook, (&f, cmd, data),
   2118   1.1  riastrad 		    enosys(), error);
   2119   1.1  riastrad #if defined(_LP64)
   2120   1.1  riastrad 		if (error == ENOSYS)
   2121   1.1  riastrad 			MODULE_HOOK_CALL(rnd_ioctl32_50_hook, (&f, cmd, data),
   2122   1.1  riastrad 			    enosys(), error);
   2123   1.1  riastrad #endif
   2124   1.1  riastrad 		if (error == ENOSYS)
   2125   1.1  riastrad 			error = ENOTTY;
   2126   1.1  riastrad 		break;
   2127   1.1  riastrad 	}
   2128   1.1  riastrad 	}
   2129   1.1  riastrad 
   2130   1.1  riastrad 	/* If anything went wrong with authorization, stop here.  */
   2131   1.1  riastrad 	if (error)
   2132   1.1  riastrad 		return error;
   2133   1.1  riastrad 
   2134   1.1  riastrad 	/* Dispatch on the command.  */
   2135   1.1  riastrad 	switch (cmd) {
   2136   1.1  riastrad 	case RNDGETENTCNT: {	/* Get current entropy count in bits.  */
   2137   1.1  riastrad 		uint32_t *countp = data;
   2138   1.1  riastrad 
   2139   1.1  riastrad 		mutex_enter(&E->lock);
   2140   1.1  riastrad 		*countp = ENTROPY_CAPACITY*NBBY - E->needed;
   2141   1.1  riastrad 		mutex_exit(&E->lock);
   2142   1.1  riastrad 
   2143   1.1  riastrad 		break;
   2144   1.1  riastrad 	}
   2145   1.1  riastrad 	case RNDGETPOOLSTAT: {	/* Get entropy pool statistics.  */
   2146   1.1  riastrad 		rndpoolstat_t *pstat = data;
   2147   1.1  riastrad 
   2148   1.1  riastrad 		mutex_enter(&E->lock);
   2149   1.1  riastrad 
   2150   1.1  riastrad 		/* parameters */
   2151   1.1  riastrad 		pstat->poolsize = ENTPOOL_SIZE/sizeof(uint32_t); /* words */
   2152   1.1  riastrad 		pstat->threshold = ENTROPY_CAPACITY*1; /* bytes */
   2153   1.1  riastrad 		pstat->maxentropy = ENTROPY_CAPACITY*NBBY; /* bits */
   2154   1.1  riastrad 
   2155   1.1  riastrad 		/* state */
   2156   1.1  riastrad 		pstat->added = 0; /* XXX total entropy_enter count */
   2157   1.1  riastrad 		pstat->curentropy = ENTROPY_CAPACITY*NBBY - E->needed;
   2158   1.1  riastrad 		pstat->removed = 0; /* XXX total entropy_extract count */
   2159   1.1  riastrad 		pstat->discarded = 0; /* XXX bits of entropy beyond capacity */
   2160   1.1  riastrad 		pstat->generated = 0; /* XXX bits of data...fabricated? */
   2161   1.1  riastrad 
   2162   1.1  riastrad 		mutex_exit(&E->lock);
   2163   1.1  riastrad 		break;
   2164   1.1  riastrad 	}
   2165   1.1  riastrad 	case RNDGETSRCNUM: {	/* Get entropy sources by number.  */
   2166   1.1  riastrad 		rndstat_t *stat = data;
   2167   1.1  riastrad 		uint32_t start = 0, i = 0;
   2168   1.1  riastrad 
   2169   1.1  riastrad 		/* Skip if none requested; fail if too many requested.  */
   2170   1.1  riastrad 		if (stat->count == 0)
   2171   1.1  riastrad 			break;
   2172   1.1  riastrad 		if (stat->count > RND_MAXSTATCOUNT)
   2173   1.1  riastrad 			return EINVAL;
   2174   1.1  riastrad 
   2175   1.1  riastrad 		/*
   2176   1.1  riastrad 		 * Under the lock, find the first one, copy out as many
   2177   1.1  riastrad 		 * as requested, and report how many we copied out.
   2178   1.1  riastrad 		 */
   2179   1.1  riastrad 		mutex_enter(&E->lock);
   2180   1.4  riastrad 		error = rnd_lock_sources();
   2181   1.4  riastrad 		if (error) {
   2182   1.4  riastrad 			mutex_exit(&E->lock);
   2183   1.4  riastrad 			return error;
   2184   1.4  riastrad 		}
   2185   1.1  riastrad 		LIST_FOREACH(rs, &E->sources, list) {
   2186   1.1  riastrad 			if (start++ == stat->start)
   2187   1.1  riastrad 				break;
   2188   1.1  riastrad 		}
   2189   1.1  riastrad 		while (i < stat->count && rs != NULL) {
   2190   1.5  riastrad 			mutex_exit(&E->lock);
   2191   1.1  riastrad 			rndsource_to_user(rs, &stat->source[i++]);
   2192   1.5  riastrad 			mutex_enter(&E->lock);
   2193   1.1  riastrad 			rs = LIST_NEXT(rs, list);
   2194   1.1  riastrad 		}
   2195   1.1  riastrad 		KASSERT(i <= stat->count);
   2196   1.1  riastrad 		stat->count = i;
   2197   1.4  riastrad 		rnd_unlock_sources();
   2198   1.1  riastrad 		mutex_exit(&E->lock);
   2199   1.1  riastrad 		break;
   2200   1.1  riastrad 	}
   2201   1.1  riastrad 	case RNDGETESTNUM: {	/* Get sources and estimates by number.  */
   2202   1.1  riastrad 		rndstat_est_t *estat = data;
   2203   1.1  riastrad 		uint32_t start = 0, i = 0;
   2204   1.1  riastrad 
   2205   1.1  riastrad 		/* Skip if none requested; fail if too many requested.  */
   2206   1.1  riastrad 		if (estat->count == 0)
   2207   1.1  riastrad 			break;
   2208   1.1  riastrad 		if (estat->count > RND_MAXSTATCOUNT)
   2209   1.1  riastrad 			return EINVAL;
   2210   1.1  riastrad 
   2211   1.1  riastrad 		/*
   2212   1.1  riastrad 		 * Under the lock, find the first one, copy out as many
   2213   1.1  riastrad 		 * as requested, and report how many we copied out.
   2214   1.1  riastrad 		 */
   2215   1.1  riastrad 		mutex_enter(&E->lock);
   2216   1.4  riastrad 		error = rnd_lock_sources();
   2217   1.4  riastrad 		if (error) {
   2218   1.4  riastrad 			mutex_exit(&E->lock);
   2219   1.4  riastrad 			return error;
   2220   1.4  riastrad 		}
   2221   1.1  riastrad 		LIST_FOREACH(rs, &E->sources, list) {
   2222   1.1  riastrad 			if (start++ == estat->start)
   2223   1.1  riastrad 				break;
   2224   1.1  riastrad 		}
   2225   1.1  riastrad 		while (i < estat->count && rs != NULL) {
   2226   1.4  riastrad 			mutex_exit(&E->lock);
   2227   1.1  riastrad 			rndsource_to_user_est(rs, &estat->source[i++]);
   2228   1.4  riastrad 			mutex_enter(&E->lock);
   2229   1.1  riastrad 			rs = LIST_NEXT(rs, list);
   2230   1.1  riastrad 		}
   2231   1.1  riastrad 		KASSERT(i <= estat->count);
   2232   1.1  riastrad 		estat->count = i;
   2233   1.4  riastrad 		rnd_unlock_sources();
   2234   1.1  riastrad 		mutex_exit(&E->lock);
   2235   1.1  riastrad 		break;
   2236   1.1  riastrad 	}
   2237   1.1  riastrad 	case RNDGETSRCNAME: {	/* Get entropy sources by name.  */
   2238   1.1  riastrad 		rndstat_name_t *nstat = data;
   2239   1.1  riastrad 		const size_t n = sizeof(rs->name);
   2240   1.1  riastrad 
   2241   1.1  riastrad 		CTASSERT(sizeof(rs->name) == sizeof(nstat->name));
   2242   1.1  riastrad 
   2243   1.1  riastrad 		/*
   2244   1.1  riastrad 		 * Under the lock, search by name.  If found, copy it
   2245   1.1  riastrad 		 * out; if not found, fail with ENOENT.
   2246   1.1  riastrad 		 */
   2247   1.1  riastrad 		mutex_enter(&E->lock);
   2248   1.4  riastrad 		error = rnd_lock_sources();
   2249   1.4  riastrad 		if (error) {
   2250   1.4  riastrad 			mutex_exit(&E->lock);
   2251   1.4  riastrad 			return error;
   2252   1.4  riastrad 		}
   2253   1.1  riastrad 		LIST_FOREACH(rs, &E->sources, list) {
   2254   1.1  riastrad 			if (strncmp(rs->name, nstat->name, n) == 0)
   2255   1.1  riastrad 				break;
   2256   1.1  riastrad 		}
   2257   1.4  riastrad 		if (rs != NULL) {
   2258   1.4  riastrad 			mutex_exit(&E->lock);
   2259   1.1  riastrad 			rndsource_to_user(rs, &nstat->source);
   2260   1.4  riastrad 			mutex_enter(&E->lock);
   2261   1.4  riastrad 		} else {
   2262   1.1  riastrad 			error = ENOENT;
   2263   1.4  riastrad 		}
   2264   1.4  riastrad 		rnd_unlock_sources();
   2265   1.1  riastrad 		mutex_exit(&E->lock);
   2266   1.1  riastrad 		break;
   2267   1.1  riastrad 	}
   2268   1.1  riastrad 	case RNDGETESTNAME: {	/* Get sources and estimates by name.  */
   2269   1.1  riastrad 		rndstat_est_name_t *enstat = data;
   2270   1.1  riastrad 		const size_t n = sizeof(rs->name);
   2271   1.1  riastrad 
   2272   1.1  riastrad 		CTASSERT(sizeof(rs->name) == sizeof(enstat->name));
   2273   1.1  riastrad 
   2274   1.1  riastrad 		/*
   2275   1.1  riastrad 		 * Under the lock, search by name.  If found, copy it
   2276   1.1  riastrad 		 * out; if not found, fail with ENOENT.
   2277   1.1  riastrad 		 */
   2278   1.1  riastrad 		mutex_enter(&E->lock);
   2279   1.4  riastrad 		error = rnd_lock_sources();
   2280   1.4  riastrad 		if (error) {
   2281   1.4  riastrad 			mutex_exit(&E->lock);
   2282   1.4  riastrad 			return error;
   2283   1.4  riastrad 		}
   2284   1.1  riastrad 		LIST_FOREACH(rs, &E->sources, list) {
   2285   1.1  riastrad 			if (strncmp(rs->name, enstat->name, n) == 0)
   2286   1.1  riastrad 				break;
   2287   1.1  riastrad 		}
   2288   1.4  riastrad 		if (rs != NULL) {
   2289   1.4  riastrad 			mutex_exit(&E->lock);
   2290   1.1  riastrad 			rndsource_to_user_est(rs, &enstat->source);
   2291   1.4  riastrad 			mutex_enter(&E->lock);
   2292   1.4  riastrad 		} else {
   2293   1.1  riastrad 			error = ENOENT;
   2294   1.4  riastrad 		}
   2295   1.4  riastrad 		rnd_unlock_sources();
   2296   1.1  riastrad 		mutex_exit(&E->lock);
   2297   1.1  riastrad 		break;
   2298   1.1  riastrad 	}
   2299   1.1  riastrad 	case RNDCTL: {		/* Modify entropy source flags.  */
   2300   1.1  riastrad 		rndctl_t *rndctl = data;
   2301   1.1  riastrad 		const size_t n = sizeof(rs->name);
   2302  1.21  riastrad 		uint32_t resetflags = RND_FLAG_NO_ESTIMATE|RND_FLAG_NO_COLLECT;
   2303   1.1  riastrad 		uint32_t flags;
   2304  1.21  riastrad 		bool reset = false, request = false;
   2305   1.1  riastrad 
   2306   1.1  riastrad 		CTASSERT(sizeof(rs->name) == sizeof(rndctl->name));
   2307   1.1  riastrad 
   2308   1.1  riastrad 		/* Whitelist the flags that user can change.  */
   2309   1.1  riastrad 		rndctl->mask &= RND_FLAG_NO_ESTIMATE|RND_FLAG_NO_COLLECT;
   2310   1.1  riastrad 
   2311   1.1  riastrad 		/*
   2312   1.1  riastrad 		 * For each matching rndsource, either by type if
   2313   1.1  riastrad 		 * specified or by name if not, set the masked flags.
   2314   1.1  riastrad 		 */
   2315   1.1  riastrad 		mutex_enter(&E->lock);
   2316   1.1  riastrad 		LIST_FOREACH(rs, &E->sources, list) {
   2317   1.1  riastrad 			if (rndctl->type != 0xff) {
   2318   1.1  riastrad 				if (rs->type != rndctl->type)
   2319   1.1  riastrad 					continue;
   2320   1.1  riastrad 			} else {
   2321   1.1  riastrad 				if (strncmp(rs->name, rndctl->name, n) != 0)
   2322   1.1  riastrad 					continue;
   2323   1.1  riastrad 			}
   2324   1.1  riastrad 			flags = rs->flags & ~rndctl->mask;
   2325   1.1  riastrad 			flags |= rndctl->flags & rndctl->mask;
   2326  1.21  riastrad 			if ((rs->flags & resetflags) == 0 &&
   2327  1.21  riastrad 			    (flags & resetflags) != 0)
   2328  1.21  riastrad 				reset = true;
   2329  1.21  riastrad 			if ((rs->flags ^ flags) & resetflags)
   2330  1.21  riastrad 				request = true;
   2331   1.1  riastrad 			atomic_store_relaxed(&rs->flags, flags);
   2332   1.1  riastrad 		}
   2333   1.1  riastrad 		mutex_exit(&E->lock);
   2334  1.21  riastrad 
   2335  1.21  riastrad 		/*
   2336  1.21  riastrad 		 * If we disabled estimation or collection, nix all the
   2337  1.21  riastrad 		 * pending entropy and set needed to the maximum.
   2338  1.21  riastrad 		 */
   2339  1.21  riastrad 		if (reset) {
   2340  1.21  riastrad 			xc_broadcast(0, &entropy_reset_xc, NULL, NULL);
   2341  1.21  riastrad 			mutex_enter(&E->lock);
   2342  1.21  riastrad 			E->pending = 0;
   2343  1.21  riastrad 			atomic_store_relaxed(&E->needed,
   2344  1.21  riastrad 			    ENTROPY_CAPACITY*NBBY);
   2345  1.21  riastrad 			mutex_exit(&E->lock);
   2346  1.21  riastrad 		}
   2347  1.21  riastrad 
   2348  1.21  riastrad 		/*
   2349  1.21  riastrad 		 * If we changed any of the estimation or collection
   2350  1.21  riastrad 		 * flags, request new samples from everyone -- either
   2351  1.21  riastrad 		 * to make up for what we just lost, or to get new
   2352  1.21  riastrad 		 * samples from what we just added.
   2353  1.21  riastrad 		 */
   2354  1.21  riastrad 		if (request) {
   2355  1.21  riastrad 			mutex_enter(&E->lock);
   2356  1.21  riastrad 			entropy_request(ENTROPY_CAPACITY);
   2357  1.21  riastrad 			mutex_exit(&E->lock);
   2358  1.21  riastrad 		}
   2359   1.1  riastrad 		break;
   2360   1.1  riastrad 	}
   2361   1.1  riastrad 	case RNDADDDATA: {	/* Enter seed into entropy pool.  */
   2362   1.1  riastrad 		rnddata_t *rdata = data;
   2363   1.1  riastrad 		unsigned entropybits = 0;
   2364   1.1  riastrad 
   2365   1.1  riastrad 		if (!atomic_load_relaxed(&entropy_collection))
   2366   1.1  riastrad 			break;	/* thanks but no thanks */
   2367   1.1  riastrad 		if (rdata->len > MIN(sizeof(rdata->data), UINT32_MAX/NBBY))
   2368   1.1  riastrad 			return EINVAL;
   2369   1.1  riastrad 
   2370   1.1  riastrad 		/*
   2371   1.1  riastrad 		 * This ioctl serves as the userland alternative a
   2372   1.1  riastrad 		 * bootloader-provided seed -- typically furnished by
   2373   1.1  riastrad 		 * /etc/rc.d/random_seed.  We accept the user's entropy
   2374   1.1  riastrad 		 * claim only if
   2375   1.1  riastrad 		 *
   2376   1.1  riastrad 		 * (a) the user is privileged, and
   2377   1.1  riastrad 		 * (b) we have not entered a bootloader seed.
   2378   1.1  riastrad 		 *
   2379   1.1  riastrad 		 * under the assumption that the user may use this to
   2380   1.1  riastrad 		 * load a seed from disk that we have already loaded
   2381   1.1  riastrad 		 * from the bootloader, so we don't double-count it.
   2382   1.1  riastrad 		 */
   2383  1.11  riastrad 		if (privileged && rdata->entropy && rdata->len) {
   2384   1.1  riastrad 			mutex_enter(&E->lock);
   2385   1.1  riastrad 			if (!E->seeded) {
   2386   1.1  riastrad 				entropybits = MIN(rdata->entropy,
   2387   1.1  riastrad 				    MIN(rdata->len, ENTROPY_CAPACITY)*NBBY);
   2388   1.1  riastrad 				E->seeded = true;
   2389   1.1  riastrad 			}
   2390   1.1  riastrad 			mutex_exit(&E->lock);
   2391   1.1  riastrad 		}
   2392   1.1  riastrad 
   2393  1.13  riastrad 		/* Enter the data and consolidate entropy.  */
   2394   1.1  riastrad 		rnd_add_data(&seed_rndsource, rdata->data, rdata->len,
   2395   1.1  riastrad 		    entropybits);
   2396  1.13  riastrad 		entropy_consolidate();
   2397   1.1  riastrad 		break;
   2398   1.1  riastrad 	}
   2399   1.1  riastrad 	default:
   2400   1.1  riastrad 		error = ENOTTY;
   2401   1.1  riastrad 	}
   2402   1.1  riastrad 
   2403   1.1  riastrad 	/* Return any error that may have come up.  */
   2404   1.1  riastrad 	return error;
   2405   1.1  riastrad }
   2406   1.1  riastrad 
   2407   1.1  riastrad /* Legacy entry points */
   2408   1.1  riastrad 
   2409   1.1  riastrad void
   2410   1.1  riastrad rnd_seed(void *seed, size_t len)
   2411   1.1  riastrad {
   2412   1.1  riastrad 
   2413   1.1  riastrad 	if (len != sizeof(rndsave_t)) {
   2414   1.1  riastrad 		printf("entropy: invalid seed length: %zu,"
   2415   1.1  riastrad 		    " expected sizeof(rndsave_t) = %zu\n",
   2416   1.1  riastrad 		    len, sizeof(rndsave_t));
   2417   1.1  riastrad 		return;
   2418   1.1  riastrad 	}
   2419   1.1  riastrad 	entropy_seed(seed);
   2420   1.1  riastrad }
   2421   1.1  riastrad 
   2422   1.1  riastrad void
   2423   1.1  riastrad rnd_init(void)
   2424   1.1  riastrad {
   2425   1.1  riastrad 
   2426   1.1  riastrad 	entropy_init();
   2427   1.1  riastrad }
   2428   1.1  riastrad 
   2429   1.1  riastrad void
   2430   1.1  riastrad rnd_init_softint(void)
   2431   1.1  riastrad {
   2432   1.1  riastrad 
   2433   1.1  riastrad 	entropy_init_late();
   2434  1.38  riastrad 	entropy_bootrequest();
   2435   1.1  riastrad }
   2436   1.1  riastrad 
   2437   1.1  riastrad int
   2438   1.1  riastrad rnd_system_ioctl(struct file *fp, unsigned long cmd, void *data)
   2439   1.1  riastrad {
   2440   1.1  riastrad 
   2441   1.1  riastrad 	return entropy_ioctl(cmd, data);
   2442   1.1  riastrad }
   2443