cprng_fast.c revision 1.16
1 1.16 riastrad /* $NetBSD: cprng_fast.c,v 1.16 2020/07/28 20:15:07 riastradh Exp $ */ 2 1.2 tls 3 1.2 tls /*- 4 1.2 tls * Copyright (c) 2014 The NetBSD Foundation, Inc. 5 1.2 tls * All rights reserved. 6 1.2 tls * 7 1.2 tls * This code is derived from software contributed to The NetBSD Foundation 8 1.2 tls * by Taylor R. Campbell. 9 1.2 tls * 10 1.2 tls * Redistribution and use in source and binary forms, with or without 11 1.2 tls * modification, are permitted provided that the following conditions 12 1.2 tls * are met: 13 1.2 tls * 1. Redistributions of source code must retain the above copyright 14 1.2 tls * notice, this list of conditions and the following disclaimer. 15 1.2 tls * 2. Redistributions in binary form must reproduce the above copyright 16 1.2 tls * notice, this list of conditions and the following disclaimer in the 17 1.2 tls * documentation and/or other materials provided with the distribution. 18 1.2 tls * 19 1.2 tls * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 1.2 tls * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 1.2 tls * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 1.2 tls * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 1.2 tls * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 1.2 tls * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 1.2 tls * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 1.2 tls * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 1.2 tls * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 1.2 tls * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 1.2 tls * POSSIBILITY OF SUCH DAMAGE. 30 1.2 tls */ 31 1.2 tls 32 1.2 tls #include <sys/cdefs.h> 33 1.16 riastrad __KERNEL_RCSID(0, "$NetBSD: cprng_fast.c,v 1.16 2020/07/28 20:15:07 riastradh Exp $"); 34 1.2 tls 35 1.2 tls #include <sys/types.h> 36 1.4 riastrad #include <sys/param.h> 37 1.2 tls #include <sys/bitops.h> 38 1.4 riastrad #include <sys/cprng.h> 39 1.2 tls #include <sys/cpu.h> 40 1.14 riastrad #include <sys/entropy.h> 41 1.15 riastrad #include <sys/evcnt.h> 42 1.2 tls #include <sys/intr.h> 43 1.15 riastrad #include <sys/kmem.h> 44 1.2 tls #include <sys/percpu.h> 45 1.2 tls 46 1.16 riastrad #include <crypto/chacha/chacha.h> 47 1.2 tls 48 1.16 riastrad #define CPRNG_FAST_SEED_BYTES CHACHA_STREAM_KEYBYTES 50 1.2 tls 51 1.16 riastrad struct cprng_fast { 52 1.16 riastrad /* 128-bit vector unit generates 256 bytes at once */ 53 1.16 riastrad uint8_t buf[256]; 54 1.16 riastrad uint8_t key[CPRNG_FAST_SEED_BYTES]; 55 1.16 riastrad uint8_t nonce[CHACHA_STREAM_NONCEBYTES]; 56 1.15 riastrad unsigned i; 57 1.14 riastrad struct evcnt *reseed_evcnt; 58 1.2 tls unsigned epoch; 59 1.2 tls }; 60 1.8 riastrad 61 1.2 tls static void cprng_fast_init_cpu(void *, void *, struct cpu_info *); 62 1.2 tls static void cprng_fast_schedule_reseed(struct cprng_fast *); 63 1.2 tls static void cprng_fast_intr(void *); 64 1.6 riastrad 65 1.2 tls static void cprng_fast_seed(struct cprng_fast *, const void *); 66 1.2 tls static void cprng_fast_buf(struct cprng_fast *, void *, unsigned); 67 1.2 tls 68 1.2 tls static void cprng_fast_buf_short(void *, size_t); 69 1.2 tls static void cprng_fast_buf_long(void *, size_t); 70 1.2 tls 71 1.2 tls static percpu_t *cprng_fast_percpu __read_mostly; 72 1.2 tls static void *cprng_fast_softint __read_mostly; 73 1.2 tls 74 1.2 tls void 75 1.2 tls cprng_fast_init(void) 76 1.2 tls { 77 1.15 riastrad 78 1.15 riastrad cprng_fast_percpu = percpu_create(sizeof(struct cprng_fast), 79 1.2 tls cprng_fast_init_cpu, NULL, NULL); 80 1.2 tls cprng_fast_softint = softint_establish(SOFTINT_SERIAL|SOFTINT_MPSAFE, 81 1.2 tls &cprng_fast_intr, NULL); 82 1.2 tls } 83 1.8 riastrad 84 1.15 riastrad static void 85 1.8 riastrad cprng_fast_init_cpu(void *p, void *arg __unused, struct cpu_info *ci) 86 1.8 riastrad { 87 1.8 riastrad struct cprng_fast *const cprng = p; 88 1.8 riastrad uint8_t seed[CPRNG_FAST_SEED_BYTES]; 89 1.14 riastrad 90 1.12 riastrad cprng->epoch = entropy_epoch(); 91 1.8 riastrad cprng_strong(kern_cprng, seed, sizeof seed, 0); 92 1.8 riastrad cprng_fast_seed(cprng, seed); 93 1.15 riastrad (void)explicit_memset(seed, 0, sizeof seed); 94 1.15 riastrad 95 1.15 riastrad cprng->reseed_evcnt = kmem_alloc(sizeof(*cprng->reseed_evcnt), 96 1.15 riastrad KM_SLEEP); 97 1.15 riastrad evcnt_attach_dynamic(cprng->reseed_evcnt, EVCNT_TYPE_MISC, NULL, 98 1.8 riastrad ci->ci_cpuname, "cprng_fast reseed"); 99 1.9 riastrad } 100 1.16 riastrad 101 1.2 tls static int 103 1.9 riastrad cprng_fast_get(struct cprng_fast **cprngp) 104 1.9 riastrad { 105 1.9 riastrad struct cprng_fast *cprng; 106 1.9 riastrad int s; 107 1.9 riastrad 108 1.2 tls *cprngp = cprng = percpu_getref(cprng_fast_percpu); 109 1.14 riastrad s = splvm(); 110 1.9 riastrad 111 1.9 riastrad if (__predict_false(cprng->epoch != entropy_epoch())) 112 1.9 riastrad cprng_fast_schedule_reseed(cprng); 113 1.2 tls 114 1.2 tls return s; 115 1.16 riastrad } 116 1.2 tls 117 1.2 tls static void 118 1.2 tls cprng_fast_put(struct cprng_fast *cprng, int s) 119 1.2 tls { 120 1.2 tls 121 1.2 tls KASSERT((cprng == percpu_getref(cprng_fast_percpu)) && 122 1.2 tls (percpu_putref(cprng_fast_percpu), true)); 123 1.2 tls splx(s); 124 1.9 riastrad percpu_putref(cprng_fast_percpu); 125 1.11 justin } 126 1.2 tls 127 1.2 tls static void 128 1.2 tls cprng_fast_schedule_reseed(struct cprng_fast *cprng __unused) 129 1.2 tls { 130 1.2 tls 131 1.2 tls softint_schedule(cprng_fast_softint); 132 1.2 tls } 133 1.2 tls 134 1.2 tls static void 135 1.14 riastrad cprng_fast_intr(void *cookie __unused) 136 1.2 tls { 137 1.2 tls unsigned epoch = entropy_epoch(); 138 1.7 riastrad struct cprng_fast *cprng; 139 1.2 tls uint8_t seed[CPRNG_FAST_SEED_BYTES]; 140 1.12 riastrad int s; 141 1.2 tls 142 1.2 tls cprng_strong(kern_cprng, seed, sizeof(seed), 0); 143 1.7 riastrad 144 1.2 tls cprng = percpu_getref(cprng_fast_percpu); 145 1.14 riastrad s = splvm(); 146 1.15 riastrad cprng_fast_seed(cprng, seed); 147 1.7 riastrad cprng->epoch = epoch; 148 1.2 tls cprng->reseed_evcnt->ev_count++; 149 1.2 tls splx(s); 150 1.2 tls percpu_putref(cprng_fast_percpu); 151 1.2 tls 152 1.2 tls explicit_memset(seed, 0, sizeof(seed)); 153 1.2 tls } 154 1.2 tls 155 1.6 riastrad /* CPRNG algorithm */ 157 1.2 tls 158 1.2 tls static void 159 1.16 riastrad cprng_fast_seed(struct cprng_fast *cprng, const void *seed) 160 1.2 tls { 161 1.2 tls 162 1.16 riastrad (void)memset(cprng->buf, 0, sizeof cprng->buf); 163 1.2 tls (void)memcpy(cprng->key, seed, sizeof cprng->key); 164 1.2 tls (void)memset(cprng->nonce, 0, sizeof cprng->nonce); 165 1.16 riastrad cprng->i = sizeof cprng->buf; 166 1.16 riastrad } 167 1.2 tls 168 1.16 riastrad static void 169 1.16 riastrad cprng_fast_buf(struct cprng_fast *cprng, void *buf, unsigned len) 170 1.16 riastrad { 171 1.16 riastrad uint8_t *p = buf; 172 1.16 riastrad unsigned n = len, n0; 173 1.2 tls 174 1.16 riastrad KASSERT(cprng->i <= sizeof(cprng->buf)); 175 1.16 riastrad KASSERT(len <= sizeof(cprng->buf)); 176 1.16 riastrad 177 1.16 riastrad n0 = MIN(n, sizeof(cprng->buf) - cprng->i); 178 1.16 riastrad memcpy(p, &cprng->buf[cprng->i], n0); 179 1.16 riastrad if ((n -= n0) == 0) { 180 1.2 tls cprng->i += n0; 181 1.16 riastrad KASSERT(cprng->i <= sizeof(cprng->buf)); 182 1.16 riastrad return; 183 1.16 riastrad } 184 1.16 riastrad p += n0; 185 1.16 riastrad le64enc(cprng->nonce, 1 + le64dec(cprng->nonce)); 186 1.16 riastrad chacha_stream(cprng->buf, sizeof(cprng->buf), 0, cprng->nonce, 187 1.2 tls cprng->key, 8); 188 1.16 riastrad memcpy(p, cprng->buf, n); 189 1.16 riastrad cprng->i = n; 190 1.2 tls } 191 1.16 riastrad 192 1.16 riastrad /* Public API */ 194 1.16 riastrad 195 1.16 riastrad static void 196 1.2 tls cprng_fast_buf_short(void *buf, size_t len) 197 1.16 riastrad { 198 1.10 riastrad struct cprng_fast *cprng; 199 1.16 riastrad int s; 200 1.16 riastrad 201 1.16 riastrad KASSERT(len <= sizeof(cprng->buf)); 202 1.2 tls 203 1.16 riastrad s = cprng_fast_get(&cprng); 204 1.2 tls cprng_fast_buf(cprng, buf, len); 205 1.16 riastrad cprng_fast_put(cprng, s); 206 1.2 tls } 207 1.16 riastrad 208 1.16 riastrad static void 209 1.2 tls cprng_fast_buf_long(void *buf, size_t len) 210 1.16 riastrad { 211 1.16 riastrad uint8_t seed[CHACHA_STREAM_KEYBYTES]; 212 1.16 riastrad uint8_t nonce[CHACHA_STREAM_NONCEBYTES] = {0}; 213 1.16 riastrad 214 1.16 riastrad CTASSERT(sizeof(seed) <= sizeof(((struct cprng_fast *)0)->buf)); 215 1.16 riastrad 216 1.2 tls #if SIZE_MAX >= 0x3fffffffff 217 1.16 riastrad /* >=256 GB is not reasonable */ 218 1.16 riastrad KASSERT(len <= 0x3fffffffff); 219 1.2 tls #endif 220 1.16 riastrad 221 1.2 tls cprng_fast_buf_short(seed, sizeof seed); 222 1.2 tls chacha_stream(buf, len, 0, nonce, seed, 8); 223 1.2 tls 224 1.2 tls (void)explicit_memset(seed, 0, sizeof seed); 225 1.2 tls } 226 1.2 tls 227 1.2 tls uint32_t 228 1.16 riastrad cprng_fast32(void) 229 1.2 tls { 230 1.2 tls uint32_t v; 231 1.2 tls 232 1.2 tls cprng_fast_buf_short(&v, sizeof v); 233 1.2 tls 234 1.2 tls return v; 235 1.2 tls } 236 1.16 riastrad 237 1.2 tls uint64_t 238 1.16 riastrad cprng_fast64(void) 239 1.2 tls { 240 1.16 riastrad uint64_t v; 241 1.2 tls 242 1.2 tls cprng_fast_buf_short(&v, sizeof v); 243 1.2 tls 244 1.2 tls return v; 245 1.2 tls } 246 1.2 tls 247 1.2 tls size_t 248 1.2 tls cprng_fast(void *buf, size_t len) 249 1.2 tls { 250 1.16 riastrad 251 1.2 tls /* 252 1.16 riastrad * We don't want to hog the CPU, so we use the short version, 253 1.2 tls * to generate output without preemption, only if we can do it 254 1.2 tls * with at most one ChaCha call. 255 1.2 tls */ 256 1.2 tls if (len <= sizeof(((struct cprng_fast *)0)->buf)) 257 1.16 riastrad cprng_fast_buf_short(buf, len); 258 1.2 tls else 259 cprng_fast_buf_long(buf, len); 260 261 return len; /* hysterical raisins */ 262 } 263
Indexes created Tue Sep 30 20:09:53 GMT 2025