1 /* $NetBSD: patch.c,v 1.53 2022/08/20 23:48:51 riastradh Exp $ */ 2 3 /*- 4 * Copyright (c) 2007, 2008, 2009 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Andrew Doran. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 /* 33 * Patch kernel code at boot time, depending on available CPU features. 34 */ 35 36 #include <sys/cdefs.h> 37 __KERNEL_RCSID(0, "$NetBSD: patch.c,v 1.53 2022/08/20 23:48:51 riastradh Exp $"); 38 39 #include "opt_lockdebug.h" 40 #ifdef i386 41 #include "opt_spldebug.h" 42 #endif 43 44 #include <sys/types.h> 45 #include <sys/systm.h> 46 47 #include <machine/cpu.h> 48 #include <machine/cpufunc.h> 49 #include <machine/specialreg.h> 50 #include <machine/frameasm.h> 51 52 #include <uvm/uvm.h> 53 #include <machine/pmap.h> 54 #include <machine/pmap_private.h> 55 56 #include <x86/bootspace.h> 57 #include <x86/cpuvar.h> 58 #include <x86/cputypes.h> 59 60 __link_set_decl(x86_hotpatch_descriptors, struct x86_hotpatch_descriptor); 61 62 struct x86_hotpatch_destination { 63 uint8_t name; 64 uint8_t size; 65 void *addr; 66 } __packed; 67 68 /* -------------------------------------------------------------------------- */ 69 70 /* CLAC instruction, part of SMAP. */ 71 extern uint8_t hp_clac, hp_clac_end; 72 static const struct x86_hotpatch_source hp_clac_source = { 73 .saddr = &hp_clac, 74 .eaddr = &hp_clac_end 75 }; 76 static const struct x86_hotpatch_descriptor hp_clac_desc = { 77 .name = HP_NAME_CLAC, 78 .nsrc = 1, 79 .srcs = { &hp_clac_source } 80 }; 81 __link_set_add_rodata(x86_hotpatch_descriptors, hp_clac_desc); 82 83 /* STAC instruction, part of SMAP. */ 84 extern uint8_t hp_stac, hp_stac_end; 85 static const struct x86_hotpatch_source hp_stac_source = { 86 .saddr = &hp_stac, 87 .eaddr = &hp_stac_end 88 }; 89 static const struct x86_hotpatch_descriptor hp_stac_desc = { 90 .name = HP_NAME_STAC, 91 .nsrc = 1, 92 .srcs = { &hp_stac_source } 93 }; 94 __link_set_add_rodata(x86_hotpatch_descriptors, hp_stac_desc); 95 96 /* Errata on certain AMD CPUs. */ 97 extern uint8_t hp_retfence, hp_retfence_end; 98 static const struct x86_hotpatch_source hp_retfence_source = { 99 .saddr = &hp_retfence, 100 .eaddr = &hp_retfence_end 101 }; 102 static const struct x86_hotpatch_descriptor hp_retfence_desc = { 103 .name = HP_NAME_RETFENCE, 104 .nsrc = 1, 105 .srcs = { &hp_retfence_source } 106 }; 107 __link_set_add_rodata(x86_hotpatch_descriptors, hp_retfence_desc); 108 109 /* No lock when on a single processor. */ 110 extern uint8_t hp_nolock, hp_nolock_end; 111 static const struct x86_hotpatch_source hp_nolock_source = { 112 .saddr = &hp_nolock, 113 .eaddr = &hp_nolock_end 114 }; 115 static const struct x86_hotpatch_descriptor hp_nolock_desc = { 116 .name = HP_NAME_NOLOCK, 117 .nsrc = 1, 118 .srcs = { &hp_nolock_source } 119 }; 120 __link_set_add_rodata(x86_hotpatch_descriptors, hp_nolock_desc); 121 122 #ifdef i386 123 /* CAS_64. */ 124 extern uint8_t _atomic_cas_cx8, _atomic_cas_cx8_end; 125 static const struct x86_hotpatch_source hp_cas_cx8_source = { 126 .saddr = &_atomic_cas_cx8, 127 .eaddr = &_atomic_cas_cx8_end 128 }; 129 static const struct x86_hotpatch_descriptor hp_cas_cx8_desc = { 130 .name = HP_NAME_CAS_64, 131 .nsrc = 1, 132 .srcs = { &hp_cas_cx8_source } 133 }; 134 __link_set_add_rodata(x86_hotpatch_descriptors, hp_cas_cx8_desc); 135 136 /* SPLLOWER. */ 137 extern uint8_t cx8_spllower, cx8_spllower_end; 138 static const struct x86_hotpatch_source hp_cx8_spllower_source = { 139 .saddr = &cx8_spllower, 140 .eaddr = &cx8_spllower_end 141 }; 142 static const struct x86_hotpatch_descriptor hp_cx8_spllower_desc = { 143 .name = HP_NAME_SPLLOWER, 144 .nsrc = 1, 145 .srcs = { &hp_cx8_spllower_source } 146 }; 147 __link_set_add_rodata(x86_hotpatch_descriptors, hp_cx8_spllower_desc); 148 149 /* MUTEX_EXIT. */ 150 #ifndef LOCKDEBUG 151 extern uint8_t i686_mutex_spin_exit, i686_mutex_spin_exit_end; 152 static const struct x86_hotpatch_source hp_i686_mutex_spin_exit_source = { 153 .saddr = &i686_mutex_spin_exit, 154 .eaddr = &i686_mutex_spin_exit_end 155 }; 156 static const struct x86_hotpatch_descriptor hp_i686_mutex_spin_exit_desc = { 157 .name = HP_NAME_MUTEX_EXIT, 158 .nsrc = 1, 159 .srcs = { &hp_i686_mutex_spin_exit_source } 160 }; 161 __link_set_add_rodata(x86_hotpatch_descriptors, hp_i686_mutex_spin_exit_desc); 162 #endif 163 #endif 164 165 /* -------------------------------------------------------------------------- */ 166 167 static inline void __unused 168 patchbytes(void *addr, const uint8_t *bytes, size_t size) 169 { 170 uint8_t *ptr = (uint8_t *)addr; 171 size_t i; 172 173 for (i = 0; i < size; i++) { 174 ptr[i] = bytes[i]; 175 } 176 } 177 178 /* 179 * Rules: each pointer accessed in this function MUST be read-only. 180 * 181 * Called from ASM only, prototype not public. 182 */ 183 int x86_hotpatch_apply(uint8_t, uint8_t); 184 int 185 __noubsan /* the local variables have unknown alignment to UBSan */ 186 x86_hotpatch_apply(uint8_t name, uint8_t sel) 187 { 188 struct x86_hotpatch_descriptor * const *iter; 189 const struct x86_hotpatch_descriptor *desc; 190 const struct x86_hotpatch_source *src; 191 const struct x86_hotpatch_destination *hps, *hpe, *hp; 192 extern char __rodata_hotpatch_start; 193 extern char __rodata_hotpatch_end; 194 const uint8_t *bytes; 195 bool found = false; 196 size_t size; 197 198 /* 199 * Find the descriptor, and perform some sanity checks. 200 */ 201 __link_set_foreach(iter, x86_hotpatch_descriptors) { 202 desc = *iter; 203 if (desc->name == name) { 204 found = true; 205 break; 206 } 207 } 208 if (!found) 209 return -1; 210 if (desc->nsrc > 2) 211 return -1; 212 if (sel >= desc->nsrc) 213 return -1; 214 215 /* 216 * Get the hotpatch source. 217 */ 218 src = desc->srcs[sel]; 219 bytes = src->saddr; 220 size = (size_t)src->eaddr - (size_t)src->saddr; 221 222 /* 223 * Apply the hotpatch on each registered destination. 224 */ 225 hps = (struct x86_hotpatch_destination *)&__rodata_hotpatch_start; 226 hpe = (struct x86_hotpatch_destination *)&__rodata_hotpatch_end; 227 for (hp = hps; hp < hpe; hp++) { 228 if (hp->name != name) { 229 continue; 230 } 231 if (hp->size != size) { 232 return -1; 233 } 234 patchbytes(hp->addr, bytes, size); 235 } 236 237 return 0; 238 } 239 240 #ifdef __x86_64__ 241 /* 242 * The CPU added the D bit on the text pages while we were writing to them. 243 * Remove that bit. Kinda annoying, but we can't avoid it. 244 */ 245 static void 246 remove_d_bit(void) 247 { 248 extern struct bootspace bootspace; 249 pt_entry_t pte; 250 vaddr_t va; 251 size_t i, n; 252 253 for (i = 0; i < BTSPACE_NSEGS; i++) { 254 if (bootspace.segs[i].type != BTSEG_TEXT) 255 continue; 256 va = bootspace.segs[i].va; 257 n = 0; 258 while (n < bootspace.segs[i].sz) { 259 if (L2_BASE[pl2_i(va)] & PTE_PS) { 260 pte = L2_BASE[pl2_i(va)] & ~PTE_D; 261 pmap_pte_set(&L2_BASE[pl2_i(va)], pte); 262 n += NBPD_L2; 263 va += NBPD_L2; 264 } else { 265 pte = L1_BASE[pl1_i(va)] & ~PTE_D; 266 pmap_pte_set(&L1_BASE[pl1_i(va)], pte); 267 n += NBPD_L1; 268 va += NBPD_L1; 269 } 270 } 271 } 272 273 tlbflushg(); 274 } 275 #else 276 #define remove_d_bit() __nothing 277 #endif 278 279 /* 280 * Interrupts disabled here. Called from ASM only, prototype not public. 281 */ 282 void x86_hotpatch_cleanup(int); 283 void 284 x86_hotpatch_cleanup(int retval) 285 { 286 if (retval != 0) { 287 panic("x86_hotpatch_apply failed"); 288 } 289 290 remove_d_bit(); 291 } 292 293 /* -------------------------------------------------------------------------- */ 294 295 void 296 x86_patch(bool early) 297 { 298 static bool first, second; 299 300 if (early) { 301 if (first) 302 return; 303 first = true; 304 } else { 305 if (second) 306 return; 307 second = true; 308 } 309 310 if (!early && ncpu == 1) { 311 #ifndef LOCKDEBUG 312 /* 313 * Uniprocessor: kill LOCK prefixes. 314 */ 315 x86_hotpatch(HP_NAME_NOLOCK, 0); 316 #endif 317 } 318 319 #ifdef i386 320 /* 321 * Patch early and late. Second time around the 'lock' prefix 322 * may be gone. 323 */ 324 if ((cpu_feature[0] & CPUID_CX8) != 0) { 325 x86_hotpatch(HP_NAME_CAS_64, 0); 326 } 327 328 #if !defined(SPLDEBUG) 329 if (!early && (cpu_feature[0] & CPUID_CX8) != 0) { 330 /* Faster splx(), mutex_spin_exit(). */ 331 x86_hotpatch(HP_NAME_SPLLOWER, 0); 332 #if !defined(LOCKDEBUG) 333 x86_hotpatch(HP_NAME_MUTEX_EXIT, 0); 334 #endif 335 } 336 #endif /* !SPLDEBUG */ 337 #endif /* i386 */ 338 339 /* 340 * On some Opteron revisions, locked operations erroneously 341 * allow memory references to be `bled' outside of critical 342 * sections. Apply workaround. 343 */ 344 if (cpu_vendor == CPUVENDOR_AMD && 345 (CPUID_TO_FAMILY(cpu_info_primary.ci_signature) == 0xe || 346 (CPUID_TO_FAMILY(cpu_info_primary.ci_signature) == 0xf && 347 CPUID_TO_EXTMODEL(cpu_info_primary.ci_signature) < 0x4))) { 348 x86_hotpatch(HP_NAME_RETFENCE, 0); 349 } 350 351 /* 352 * If SMAP is present then patch the prepared holes with clac/stac 353 * instructions. 354 */ 355 if (!early && cpu_feature[5] & CPUID_SEF_SMAP) { 356 KASSERT(rcr4() & CR4_SMAP); 357 358 x86_hotpatch(HP_NAME_CLAC, 0); 359 x86_hotpatch(HP_NAME_STAC, 0); 360 } 361 } 362
Indexes created Sun Sep 21 19:09:51 GMT 2025