1 /* $NetBSD: cprng_fast.c,v 1.20 2024/10/15 17:34:06 riastradh Exp $ */ 2 3 /*- 4 * Copyright (c) 2014 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Taylor R. Campbell. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 #include <sys/cdefs.h> 33 __KERNEL_RCSID(0, "$NetBSD: cprng_fast.c,v 1.20 2024/10/15 17:34:06 riastradh Exp $"); 34 35 #include <sys/types.h> 36 #include <sys/param.h> 37 #include <sys/bitops.h> 38 #include <sys/cprng.h> 39 #include <sys/cpu.h> 40 #include <sys/entropy.h> 41 #include <sys/evcnt.h> 42 #include <sys/intr.h> 43 #include <sys/kmem.h> 44 #include <sys/percpu.h> 45 46 #include <crypto/chacha/chacha.h> 47 48 #define CPRNG_FAST_SEED_BYTES CHACHA_STREAM_KEYBYTES 50 51 struct cprng_fast { 52 /* 128-bit vector unit generates 256 bytes at once */ 53 uint8_t buf[256]; 54 uint8_t key[CPRNG_FAST_SEED_BYTES]; 55 uint8_t nonce[CHACHA_STREAM_NONCEBYTES]; 56 unsigned i; 57 struct evcnt *reseed_evcnt; 58 unsigned epoch; 59 }; 60 61 static void cprng_fast_init_cpu(void *, void *, struct cpu_info *); 62 static void cprng_fast_schedule_reseed(struct cprng_fast *); 63 static void cprng_fast_intr(void *); 64 65 static void cprng_fast_seed(struct cprng_fast *, const void *); 66 static void cprng_fast_buf(struct cprng_fast *, void *, unsigned); 67 68 static void cprng_fast_buf_short(void *, size_t); 69 static void cprng_fast_buf_long(void *, size_t); 70 71 static percpu_t *cprng_fast_percpu __read_mostly; 72 static void *cprng_fast_softint __read_mostly; 73 74 void 75 cprng_fast_init(void) 76 { 77 78 cprng_fast_percpu = percpu_create(sizeof(struct cprng_fast), 79 cprng_fast_init_cpu, NULL, NULL); 80 cprng_fast_softint = softint_establish(SOFTINT_SERIAL|SOFTINT_MPSAFE, 81 &cprng_fast_intr, NULL); 82 } 83 84 static void 85 cprng_fast_init_cpu(void *p, void *arg __unused, struct cpu_info *ci) 86 { 87 struct cprng_fast *const cprng = p; 88 uint8_t seed[CPRNG_FAST_SEED_BYTES]; 89 90 cprng->epoch = entropy_epoch(); 91 cprng_strong(kern_cprng, seed, sizeof seed, 0); 92 cprng_fast_seed(cprng, seed); 93 (void)explicit_memset(seed, 0, sizeof seed); 94 95 cprng->reseed_evcnt = kmem_alloc(sizeof(*cprng->reseed_evcnt), 96 KM_SLEEP); 97 evcnt_attach_dynamic(cprng->reseed_evcnt, EVCNT_TYPE_MISC, NULL, 98 ci->ci_cpuname, "cprng_fast reseed"); 99 } 100 101 static int 103 cprng_fast_get(struct cprng_fast **cprngp) 104 { 105 struct cprng_fast *cprng; 106 int s; 107 108 *cprngp = cprng = percpu_getref(cprng_fast_percpu); 109 s = splvm(); 110 111 if (__predict_false(cprng->epoch != entropy_epoch())) 112 cprng_fast_schedule_reseed(cprng); 113 114 return s; 115 } 116 117 static void 118 cprng_fast_put(struct cprng_fast *cprng, int s) 119 { 120 121 KASSERT((cprng == percpu_getref(cprng_fast_percpu)) && 122 (percpu_putref(cprng_fast_percpu), true)); 123 splx(s); 124 percpu_putref(cprng_fast_percpu); 125 } 126 127 static void 128 cprng_fast_schedule_reseed(struct cprng_fast *cprng __unused) 129 { 130 131 softint_schedule(cprng_fast_softint); 132 } 133 134 static void 135 cprng_fast_intr(void *cookie __unused) 136 { 137 unsigned epoch = entropy_epoch(); 138 struct cprng_fast *cprng; 139 uint8_t seed[CPRNG_FAST_SEED_BYTES]; 140 int s; 141 142 cprng_strong(kern_cprng, seed, sizeof(seed), 0); 143 144 cprng = percpu_getref(cprng_fast_percpu); 145 s = splvm(); 146 cprng_fast_seed(cprng, seed); 147 cprng->epoch = epoch; 148 cprng->reseed_evcnt->ev_count++; 149 splx(s); 150 percpu_putref(cprng_fast_percpu); 151 152 explicit_memset(seed, 0, sizeof(seed)); 153 } 154 155 /* CPRNG algorithm */ 157 158 static void 159 cprng_fast_seed(struct cprng_fast *cprng, const void *seed) 160 { 161 162 (void)memset(cprng->buf, 0, sizeof cprng->buf); 163 (void)memcpy(cprng->key, seed, sizeof cprng->key); 164 (void)memset(cprng->nonce, 0, sizeof cprng->nonce); 165 cprng->i = sizeof cprng->buf; 166 } 167 168 static void 169 cprng_fast_buf(struct cprng_fast *cprng, void *buf, unsigned len) 170 { 171 uint8_t *p = buf; 172 unsigned n = len, n0; 173 174 KASSERT(cprng->i <= sizeof(cprng->buf)); 175 KASSERT(len <= sizeof(cprng->buf)); 176 177 n0 = MIN(n, sizeof(cprng->buf) - cprng->i); 178 memcpy(p, &cprng->buf[cprng->i], n0); 179 if ((n -= n0) == 0) { 180 cprng->i += n0; 181 KASSERT(cprng->i <= sizeof(cprng->buf)); 182 return; 183 } 184 p += n0; 185 le64enc(cprng->nonce, 1 + le64dec(cprng->nonce)); 186 chacha_stream(cprng->buf, sizeof(cprng->buf), 0, cprng->nonce, 187 cprng->key, 8); 188 memcpy(p, cprng->buf, n); 189 cprng->i = n; 190 } 191 192 /* Public API */ 194 195 static void 196 cprng_fast_buf_short(void *buf, size_t len) 197 { 198 struct cprng_fast *cprng; 199 int s; 200 201 KASSERT(len <= sizeof(cprng->buf)); 202 203 s = cprng_fast_get(&cprng); 204 cprng_fast_buf(cprng, buf, len); 205 cprng_fast_put(cprng, s); 206 } 207 208 static void 209 cprng_fast_buf_long(void *buf, size_t len) 210 { 211 uint8_t seed[CHACHA_STREAM_KEYBYTES]; 212 uint8_t nonce[CHACHA_STREAM_NONCEBYTES] = {0}; 213 214 CTASSERT(sizeof(seed) <= sizeof(((struct cprng_fast *)0)->buf)); 215 216 #if SIZE_MAX >= 0x3fffffffff 217 /* >=256 GB is not reasonable */ 218 KASSERT(len <= 0x3fffffffff); 219 #endif 220 221 cprng_fast_buf_short(seed, sizeof seed); 222 chacha_stream(buf, len, 0, nonce, seed, 8); 223 224 (void)explicit_memset(seed, 0, sizeof seed); 225 } 226 227 uint32_t 228 cprng_fast32(void) 229 { 230 uint32_t v; 231 232 cprng_fast_buf_short(&v, sizeof v); 233 234 return v; 235 } 236 237 uint64_t 238 cprng_fast64(void) 239 { 240 uint64_t v; 241 242 cprng_fast_buf_short(&v, sizeof v); 243 244 return v; 245 } 246 247 size_t 248 cprng_fast(void *buf, size_t len) 249 { 250 251 /* 252 * We don't want to hog the CPU, so we use the short version, 253 * to generate output without preemption, only if we can do it 254 * with at most one ChaCha call. 255 */ 256 if (len <= sizeof(((struct cprng_fast *)0)->buf)) 257 cprng_fast_buf_short(buf, len); 258 else 259 cprng_fast_buf_long(buf, len); 260 261 return len; /* hysterical raisins */ 262 } 263
Indexes created Sun Sep 21 20:09:37 GMT 2025