1 /* $NetBSD: cpufunc.c,v 1.39 2026/11/03 06:52:27 skrll Exp $ */ 2 3 /* 4 * Copyright (c) 2017 Ryo Shimizu 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 17 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 18 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 19 * DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, 20 * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 21 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 22 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, 24 * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING 25 * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 26 * POSSIBILITY OF SUCH DAMAGE. 27 */ 28 29 #include "opt_multiprocessor.h" 30 31 #include <sys/cdefs.h> 32 __KERNEL_RCSID(0, "$NetBSD: cpufunc.c,v 1.39 2026/11/03 06:52:27 skrll Exp $"); 33 34 #include <sys/param.h> 35 #include <sys/types.h> 36 #include <sys/kmem.h> 37 #include <sys/cpu.h> 38 39 #include <uvm/uvm.h> 40 #include <uvm/uvm_page.h> 41 42 #include <arm/cpufunc.h> 43 44 u_int cputype; /* compat arm */ 45 u_int arm_dcache_align; /* compat arm */ 46 u_int arm_dcache_align_mask; /* compat arm */ 47 u_int arm_dcache_maxline; 48 49 u_int aarch64_cache_vindexsize; 50 u_int aarch64_cache_prefer_mask; 51 52 int aarch64_hafdbs_enabled __read_mostly; 53 int aarch64_pan_enabled __read_mostly; 54 int aarch64_pac_enabled __read_mostly; 55 56 static void __noasan 57 extract_cacheunit(int level, bool insn, int cachetype, 58 struct aarch64_cache_info *cacheinfo) 59 { 60 struct aarch64_cache_unit *cunit; 61 uint64_t ccsidr, mmfr2; 62 63 /* select and extract level N data cache */ 64 reg_csselr_el1_write(__SHIFTIN(level, CSSELR_LEVEL) | 65 __SHIFTIN(insn ? 1 : 0, CSSELR_IND)); 66 isb(); 67 68 ccsidr = reg_ccsidr_el1_read(); 69 mmfr2 = reg_id_aa64mmfr2_el1_read(); 70 71 if (insn) 72 cunit = &cacheinfo[level].icache; 73 else 74 cunit = &cacheinfo[level].dcache; 75 76 cunit->cache_type = cachetype; 77 78 switch (__SHIFTOUT(mmfr2, ID_AA64MMFR2_EL1_CCIDX)) { 79 case ID_AA64MMFR2_EL1_CCIDX_32BIT: 80 cunit->cache_line_size = 81 1 << (__SHIFTOUT(ccsidr, CCSIDR_LINESIZE) + 4); 82 cunit->cache_ways = __SHIFTOUT(ccsidr, CCSIDR_ASSOC) + 1; 83 cunit->cache_sets = __SHIFTOUT(ccsidr, CCSIDR_NUMSET) + 1; 84 break; 85 case ID_AA64MMFR2_EL1_CCIDX_64BIT: 86 cunit->cache_line_size = 87 1 << (__SHIFTOUT(ccsidr, CCSIDR64_LINESIZE) + 4); 88 cunit->cache_ways = __SHIFTOUT(ccsidr, CCSIDR64_ASSOC) + 1; 89 cunit->cache_sets = __SHIFTOUT(ccsidr, CCSIDR64_NUMSET) + 1; 90 break; 91 } 92 93 /* calc waysize and whole size */ 94 cunit->cache_way_size = cunit->cache_line_size * cunit->cache_sets; 95 cunit->cache_size = cunit->cache_way_size * cunit->cache_ways; 96 } 97 98 /* Must be called on each processor */ 99 void __noasan 100 aarch64_getcacheinfo(struct cpu_info *ci) 101 { 102 struct aarch64_cache_info * const cinfo = ci->ci_cacheinfo; 103 uint32_t clidr, ctr; 104 int level, cachetype; 105 106 /* 107 * CTR - Cache Type Register 108 */ 109 ctr = reg_ctr_el0_read(); 110 switch (__SHIFTOUT(ctr, CTR_EL0_L1IP_MASK)) { 111 case CTR_EL0_L1IP_VPIPT: 112 cachetype = CACHE_TYPE_VPIPT; 113 break; 114 case CTR_EL0_L1IP_AIVIVT: 115 cachetype = CACHE_TYPE_VIVT; 116 break; 117 case CTR_EL0_L1IP_VIPT: 118 cachetype = CACHE_TYPE_VIPT; 119 break; 120 case CTR_EL0_L1IP_PIPT: 121 cachetype = CACHE_TYPE_PIPT; 122 break; 123 } 124 125 /* 126 * CLIDR - Cache Level ID Register 127 * CSSELR - Cache Size Selection Register 128 * CCSIDR - CurrentCache Size ID Register (selected by CSSELR) 129 */ 130 131 /* L1, L2, L3, ..., L8 cache */ 132 for (level = 0, clidr = reg_clidr_el1_read(); 133 level < MAX_CACHE_LEVEL; level++, clidr >>= 3) { 134 135 int cacheable; 136 137 switch (clidr & 7) { 138 case CLIDR_TYPE_NOCACHE: 139 cacheable = CACHE_CACHEABLE_NONE; 140 break; 141 case CLIDR_TYPE_ICACHE: 142 cacheable = CACHE_CACHEABLE_ICACHE; 143 extract_cacheunit(level, true, cachetype, cinfo); 144 break; 145 case CLIDR_TYPE_DCACHE: 146 cacheable = CACHE_CACHEABLE_DCACHE; 147 extract_cacheunit(level, false, CACHE_TYPE_PIPT, cinfo); 148 break; 149 case CLIDR_TYPE_IDCACHE: 150 cacheable = CACHE_CACHEABLE_IDCACHE; 151 extract_cacheunit(level, true, cachetype, cinfo); 152 extract_cacheunit(level, false, CACHE_TYPE_PIPT, cinfo); 153 break; 154 case CLIDR_TYPE_UNIFIEDCACHE: 155 cacheable = CACHE_CACHEABLE_UNIFIED; 156 extract_cacheunit(level, false, CACHE_TYPE_PIPT, cinfo); 157 break; 158 default: 159 cacheable = CACHE_CACHEABLE_NONE; 160 break; 161 } 162 163 cinfo[level].cacheable = cacheable; 164 if (cacheable == CACHE_CACHEABLE_NONE) { 165 /* no more level */ 166 break; 167 } 168 169 /* 170 * L1 insn cachetype is CTR_EL0:L1IP, 171 * all other cachetype is PIPT. 172 */ 173 cachetype = CACHE_TYPE_PIPT; 174 } 175 } 176 177 void 178 aarch64_parsecacheinfo(struct cpu_info *ci) 179 { 180 struct aarch64_cache_info * const cinfo = ci->ci_cacheinfo; 181 struct aarch64_sysctl_cpu_id *id = &ci->ci_id; 182 const uint32_t ctr = id->ac_ctr; 183 u_int vindexsize; 184 185 /* remember maximum alignment */ 186 if (arm_dcache_maxline < __SHIFTOUT(ctr, CTR_EL0_DMIN_LINE)) { 187 arm_dcache_maxline = __SHIFTOUT(ctr, CTR_EL0_DMIN_LINE); 188 arm_dcache_align = sizeof(int) << arm_dcache_maxline; 189 arm_dcache_align_mask = arm_dcache_align - 1; 190 } 191 192 #ifdef MULTIPROCESSOR 193 if (coherency_unit < arm_dcache_align) 194 panic("coherency_unit %ld < %d; increase COHERENCY_UNIT", 195 coherency_unit, arm_dcache_align); 196 #endif 197 198 /* calculate L1 icache virtual index size */ 199 if ((cinfo[0].icache.cache_type == CACHE_TYPE_VIVT || 200 cinfo[0].icache.cache_type == CACHE_TYPE_VIPT) && 201 (cinfo[0].cacheable == CACHE_CACHEABLE_ICACHE || 202 cinfo[0].cacheable == CACHE_CACHEABLE_IDCACHE)) { 203 204 vindexsize = 205 cinfo[0].icache.cache_size / 206 cinfo[0].icache.cache_ways; 207 208 KASSERT(vindexsize != 0); 209 } else { 210 vindexsize = 0; 211 } 212 213 if (vindexsize > aarch64_cache_vindexsize) { 214 aarch64_cache_vindexsize = vindexsize; 215 aarch64_cache_prefer_mask = vindexsize - 1; 216 217 if (uvm.page_init_done) 218 uvm_page_recolor(vindexsize / PAGE_SIZE); 219 } 220 } 221 222 static int 223 prt_cache(device_t self, struct aarch64_cache_info *cinfo, int level) 224 { 225 struct aarch64_cache_unit *cunit; 226 int i; 227 const char *cacheable, *cachetype; 228 229 if (cinfo[level].cacheable == CACHE_CACHEABLE_NONE) 230 return -1; 231 232 for (i = 0; i < 2; i++) { 233 switch (cinfo[level].cacheable) { 234 case CACHE_CACHEABLE_ICACHE: 235 cunit = &cinfo[level].icache; 236 cacheable = "Instruction"; 237 break; 238 case CACHE_CACHEABLE_DCACHE: 239 cunit = &cinfo[level].dcache; 240 cacheable = "Data"; 241 break; 242 case CACHE_CACHEABLE_IDCACHE: 243 if (i == 0) { 244 cunit = &cinfo[level].icache; 245 cacheable = "Instruction"; 246 } else { 247 cunit = &cinfo[level].dcache; 248 cacheable = "Data"; 249 } 250 break; 251 case CACHE_CACHEABLE_UNIFIED: 252 cunit = &cinfo[level].dcache; 253 cacheable = "Unified"; 254 break; 255 default: 256 cunit = &cinfo[level].dcache; 257 cacheable = "*UNK*"; 258 break; 259 } 260 261 switch (cunit->cache_type) { 262 case CACHE_TYPE_VPIPT: 263 cachetype = "VPIPT"; 264 break; 265 case CACHE_TYPE_VIVT: 266 cachetype = "VIVT"; 267 break; 268 case CACHE_TYPE_VIPT: 269 cachetype = "VIPT"; 270 break; 271 case CACHE_TYPE_PIPT: 272 cachetype = "PIPT"; 273 break; 274 default: 275 cachetype = "*UNK*"; 276 break; 277 } 278 279 aprint_verbose_dev(self, 280 "L%d %uKB/%uB %u-way (%u set) %s %s cache\n", 281 level + 1, 282 cunit->cache_size / 1024, 283 cunit->cache_line_size, 284 cunit->cache_ways, 285 cunit->cache_sets, 286 cachetype, cacheable); 287 288 if (cinfo[level].cacheable != CACHE_CACHEABLE_IDCACHE) 289 break; 290 } 291 292 return 0; 293 } 294 295 void 296 aarch64_printcacheinfo(device_t dev, struct cpu_info *ci) 297 { 298 struct aarch64_cache_info * const cinfo = ci->ci_cacheinfo; 299 int level; 300 301 for (level = 0; level < MAX_CACHE_LEVEL; level++) 302 if (prt_cache(dev, cinfo, level) < 0) 303 break; 304 } 305 306 static inline void 307 ln_dcache_wb_all(int level, struct aarch64_cache_unit *cunit) 308 { 309 uint64_t x; 310 unsigned int set, way, setshift, wayshift; 311 312 setshift = ffs(cunit->cache_line_size) - 1; 313 wayshift = 32 - (ffs(cunit->cache_ways) - 1); 314 315 for (way = 0; way < cunit->cache_ways; way++) { 316 for (set = 0; set < cunit->cache_sets; set++) { 317 x = (way << wayshift) | (set << setshift) | 318 (level << 1); 319 __asm __volatile ("dc csw, %0" :: "r"(x)); 320 } 321 } 322 /* Complete all set/way operations for this cache level. */ 323 dsb(sy); 324 } 325 326 static inline void 327 ln_dcache_wbinv_all(int level, struct aarch64_cache_unit *cunit) 328 { 329 uint64_t x; 330 unsigned int set, way, setshift, wayshift; 331 332 setshift = ffs(cunit->cache_line_size) - 1; 333 wayshift = 32 - (ffs(cunit->cache_ways) - 1); 334 335 for (way = 0; way < cunit->cache_ways; way++) { 336 for (set = 0; set < cunit->cache_sets; set++) { 337 x = (way << wayshift) | (set << setshift) | 338 (level << 1); 339 __asm __volatile ("dc cisw, %0" :: "r"(x)); 340 } 341 } 342 /* Complete all set/way operations for this cache level. */ 343 dsb(sy); 344 } 345 346 static inline void 347 ln_dcache_inv_all(int level, struct aarch64_cache_unit *cunit) 348 { 349 uint64_t x; 350 unsigned int set, way, setshift, wayshift; 351 352 setshift = ffs(cunit->cache_line_size) - 1; 353 wayshift = 32 - (ffs(cunit->cache_ways) - 1); 354 355 for (way = 0; way < cunit->cache_ways; way++) { 356 for (set = 0; set < cunit->cache_sets; set++) { 357 x = (way << wayshift) | (set << setshift) | 358 (level << 1); 359 __asm __volatile ("dc isw, %0" :: "r"(x)); 360 } 361 } 362 /* Complete all set/way operations for this cache level. */ 363 dsb(sy); 364 } 365 366 void 367 aarch64_dcache_wbinv_all(void) 368 { 369 KASSERT(kpreempt_disabled()); 370 371 struct cpu_info * const ci = curcpu(); 372 struct aarch64_cache_info * const cinfo = ci->ci_cacheinfo; 373 int level; 374 375 for (level = 0; level < MAX_CACHE_LEVEL; level++) { 376 if (cinfo[level].cacheable == CACHE_CACHEABLE_NONE) 377 break; 378 379 dsb(ish); 380 ln_dcache_wbinv_all(level, &cinfo[level].dcache); 381 } 382 dsb(ish); 383 } 384 385 void 386 aarch64_dcache_inv_all(void) 387 { 388 KASSERT(kpreempt_disabled()); 389 390 struct cpu_info * const ci = curcpu(); 391 struct aarch64_cache_info * const cinfo = ci->ci_cacheinfo; 392 int level; 393 394 for (level = 0; level < MAX_CACHE_LEVEL; level++) { 395 if (cinfo[level].cacheable == CACHE_CACHEABLE_NONE) 396 break; 397 398 dsb(ish); 399 ln_dcache_inv_all(level, &cinfo[level].dcache); 400 } 401 dsb(ish); 402 } 403 404 void 405 aarch64_dcache_wb_all(void) 406 { 407 KASSERT(kpreempt_disabled()); 408 409 struct cpu_info * const ci = curcpu(); 410 struct aarch64_cache_info * const cinfo = ci->ci_cacheinfo; 411 int level; 412 413 for (level = 0; level < MAX_CACHE_LEVEL; level++) { 414 if (cinfo[level].cacheable == CACHE_CACHEABLE_NONE) 415 break; 416 417 dsb(ish); 418 ln_dcache_wb_all(level, &cinfo[level].dcache); 419 } 420 dsb(ish); 421 } 422 423 bool 424 aarch64_earlydevice_va_p(void) 425 { 426 /* This function may be called before enabling MMU, or mapping KVA */ 427 if ((reg_sctlr_el1_read() & SCTLR_M) == 0) 428 return false; 429 430 /* device mapping will be available after pmap_devmap_bootstrap() */ 431 if (!pmap_devmap_bootstrapped_p()) 432 return false; 433 434 return true; 435 } 436 437 int 438 set_cpufuncs(void) 439 { 440 // This is only called from the BP 441 442 return aarch64_setcpufuncs(&cpu_info_store[0]); 443 } 444 445 int 446 aarch64_setcpufuncs(struct cpu_info *ci) 447 { 448 const uint64_t ctr = reg_ctr_el0_read(); 449 const uint64_t clidr = reg_clidr_el1_read(); 450 451 /* install default functions */ 452 ci->ci_cpufuncs.cf_set_ttbr0 = aarch64_set_ttbr0; 453 ci->ci_cpufuncs.cf_icache_sync_range = aarch64_icache_sync_range; 454 455 /* 456 * install core/cluster specific functions 457 */ 458 459 /* Icache sync op */ 460 if (__SHIFTOUT(ctr, CTR_EL0_DIC) == 1) { 461 /* Icache invalidation to the PoU is not required */ 462 ci->ci_cpufuncs.cf_icache_sync_range = 463 aarch64_icache_barrier_range; 464 } else if (__SHIFTOUT(ctr, CTR_EL0_IDC) == 1 || 465 __SHIFTOUT(clidr, CLIDR_LOC) == 0 || 466 (__SHIFTOUT(clidr, CLIDR_LOUIS) == 0 && __SHIFTOUT(clidr, CLIDR_LOUU) == 0)) { 467 /* Dcache clean to the PoU is not required for Icache */ 468 ci->ci_cpufuncs.cf_icache_sync_range = 469 aarch64_icache_inv_range; 470 } 471 472 #ifdef CPU_THUNDERX 473 const uint32_t midr = reg_midr_el1_read(); 474 475 /* Cavium erratum 27456 */ 476 if ((midr == CPU_ID_THUNDERXP1d0) || 477 (midr == CPU_ID_THUNDERXP1d1) || 478 (midr == CPU_ID_THUNDERXP2d1) || 479 (midr == CPU_ID_THUNDERX81XXRX)) { 480 ci->ci_cpufuncs.cf_set_ttbr0 = aarch64_set_ttbr0_thunderx; 481 } 482 #endif 483 484 return 0; 485 } 486 487 void 488 aarch64_hafdbs_init(int primary) 489 { 490 #ifdef ARMV81_HAFDBS 491 uint64_t tcr; 492 int hafdbs; 493 494 hafdbs = __SHIFTOUT(reg_id_aa64mmfr1_el1_read(), 495 ID_AA64MMFR1_EL1_HAFDBS); 496 497 /* 498 * hafdbs 499 * 0:HAFDBS_NONE - no support for any hardware flags 500 * 1:HAFDBS_A - only hardware access flag supported 501 * 2:HAFDBS_AD - hardware access and modified flags supported. 502 */ 503 504 if (primary) { 505 /* CPU0 does the detection. */ 506 switch (hafdbs) { 507 case ID_AA64MMFR1_EL1_HAFDBS_NONE: 508 default: 509 aarch64_hafdbs_enabled = 0; 510 break; 511 case ID_AA64MMFR1_EL1_HAFDBS_A: 512 case ID_AA64MMFR1_EL1_HAFDBS_AD: 513 aarch64_hafdbs_enabled = hafdbs; 514 break; 515 } 516 } else { 517 /* 518 * The support status of HAFDBS on the primary CPU is different 519 * from that of the application processor. 520 * 521 * XXX: 522 * The correct way to do this is to disable it on all cores, 523 * or call pmap_fault_fixup() only on the unsupported cores, 524 * but for now, do panic(). 525 */ 526 if (aarch64_hafdbs_enabled != hafdbs) 527 panic("HAFDBS is supported (%d) on primary cpu, " 528 "but isn't equal (%d) on secondary cpu", 529 aarch64_hafdbs_enabled, hafdbs); 530 } 531 532 /* enable Hardware updates to Access flag and Dirty state */ 533 tcr = reg_tcr_el1_read(); 534 switch (hafdbs) { 535 case ID_AA64MMFR1_EL1_HAFDBS_NONE: 536 default: 537 break; 538 case ID_AA64MMFR1_EL1_HAFDBS_A: 539 /* enable only access */ 540 reg_tcr_el1_write(tcr | TCR_HA); 541 isb(); 542 break; 543 case ID_AA64MMFR1_EL1_HAFDBS_AD: 544 /* enable both access and dirty */ 545 reg_tcr_el1_write(tcr | TCR_HD | TCR_HA); 546 isb(); 547 break; 548 } 549 #endif 550 } 551 552 void 553 aarch64_pan_init(int primary) 554 { 555 #ifdef ARMV81_PAN 556 uint64_t reg, sctlr; 557 558 /* CPU0 does the detection. */ 559 if (primary) { 560 reg = reg_id_aa64mmfr1_el1_read(); 561 if (__SHIFTOUT(reg, ID_AA64MMFR1_EL1_PAN) != 562 ID_AA64MMFR1_EL1_PAN_NONE) 563 aarch64_pan_enabled = 1; 564 } 565 566 if (!aarch64_pan_enabled) 567 return; 568 569 /* 570 * On an exception to EL1, have the CPU set the PAN bit automatically. 571 * This ensures PAN is enabled each time the kernel is entered. 572 */ 573 sctlr = reg_sctlr_el1_read(); 574 sctlr &= ~SCTLR_SPAN; 575 reg_sctlr_el1_write(sctlr); 576 577 /* Set the PAN bit right now. */ 578 reg_pan_write(1); 579 #endif 580 } 581 582 /* 583 * In order to avoid inconsistencies with pointer authentication 584 * in this function itself, the caller must enable PAC according 585 * to the return value. 586 */ 587 int 588 aarch64_pac_init(int primary) 589 { 590 #ifdef ARMV83_PAC 591 uint64_t reg; 592 593 /* CPU0 does the detection. */ 594 if (primary) { 595 reg = reg_id_aa64isar1_el1_read(); 596 if (__SHIFTOUT(reg, ID_AA64ISAR1_EL1_APA) != 597 ID_AA64ISAR1_EL1_APA_NONE) 598 aarch64_pac_enabled = 1; 599 if (__SHIFTOUT(reg, ID_AA64ISAR1_EL1_API) != 600 ID_AA64ISAR1_EL1_API_NONE) 601 aarch64_pac_enabled = 1; 602 if (__SHIFTOUT(reg, ID_AA64ISAR1_EL1_GPA) != 603 ID_AA64ISAR1_EL1_GPA_NONE) 604 aarch64_pac_enabled = 1; 605 if (__SHIFTOUT(reg, ID_AA64ISAR1_EL1_GPI) != 606 ID_AA64ISAR1_EL1_GPI_NONE) 607 aarch64_pac_enabled = 1; 608 } 609 610 if (!aarch64_pac_enabled) 611 return -1; 612 613 /* Set the key. Curlwp here is the CPU's idlelwp. */ 614 reg_APIAKeyLo_EL1_write(curlwp->l_md.md_ia_kern[0]); 615 reg_APIAKeyHi_EL1_write(curlwp->l_md.md_ia_kern[1]); 616 617 return 0; 618 #else 619 return -1; 620 #endif 621 } 622
Indexes created Sun Nov 08 00:25:27 UTC 2026