1 /* $NetBSD: cpufunc.c,v 1.38 2026/05/31 09:04:28 skrll Exp $ */ 2 3 /* 4 * Copyright (c) 2017 Ryo Shimizu 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 17 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 18 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 19 * DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, 20 * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 21 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 22 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, 24 * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING 25 * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 26 * POSSIBILITY OF SUCH DAMAGE. 27 */ 28 29 #include "opt_multiprocessor.h" 30 31 #include <sys/cdefs.h> 32 __KERNEL_RCSID(0, "$NetBSD: cpufunc.c,v 1.38 2026/05/31 09:04:28 skrll Exp $"); 33 34 #include <sys/param.h> 35 #include <sys/types.h> 36 #include <sys/kmem.h> 37 #include <sys/cpu.h> 38 39 #include <uvm/uvm.h> 40 #include <uvm/uvm_page.h> 41 42 #include <arm/cpufunc.h> 43 44 u_int cputype; /* compat arm */ 45 u_int arm_dcache_align; /* compat arm */ 46 u_int arm_dcache_align_mask; /* compat arm */ 47 u_int arm_dcache_maxline; 48 49 u_int aarch64_cache_vindexsize; 50 u_int aarch64_cache_prefer_mask; 51 52 int aarch64_hafdbs_enabled __read_mostly; 53 int aarch64_pan_enabled __read_mostly; 54 int aarch64_pac_enabled __read_mostly; 55 56 static void __noasan 57 extract_cacheunit(int level, bool insn, int cachetype, 58 struct aarch64_cache_info *cacheinfo) 59 { 60 struct aarch64_cache_unit *cunit; 61 uint64_t ccsidr, mmfr2; 62 63 /* select and extract level N data cache */ 64 reg_csselr_el1_write(__SHIFTIN(level, CSSELR_LEVEL) | 65 __SHIFTIN(insn ? 1 : 0, CSSELR_IND)); 66 isb(); 67 68 ccsidr = reg_ccsidr_el1_read(); 69 mmfr2 = reg_id_aa64mmfr2_el1_read(); 70 71 if (insn) 72 cunit = &cacheinfo[level].icache; 73 else 74 cunit = &cacheinfo[level].dcache; 75 76 cunit->cache_type = cachetype; 77 78 switch (__SHIFTOUT(mmfr2, ID_AA64MMFR2_EL1_CCIDX)) { 79 case ID_AA64MMFR2_EL1_CCIDX_32BIT: 80 cunit->cache_line_size = 81 1 << (__SHIFTOUT(ccsidr, CCSIDR_LINESIZE) + 4); 82 cunit->cache_ways = __SHIFTOUT(ccsidr, CCSIDR_ASSOC) + 1; 83 cunit->cache_sets = __SHIFTOUT(ccsidr, CCSIDR_NUMSET) + 1; 84 break; 85 case ID_AA64MMFR2_EL1_CCIDX_64BIT: 86 cunit->cache_line_size = 87 1 << (__SHIFTOUT(ccsidr, CCSIDR64_LINESIZE) + 4); 88 cunit->cache_ways = __SHIFTOUT(ccsidr, CCSIDR64_ASSOC) + 1; 89 cunit->cache_sets = __SHIFTOUT(ccsidr, CCSIDR64_NUMSET) + 1; 90 break; 91 } 92 93 /* calc waysize and whole size */ 94 cunit->cache_way_size = cunit->cache_line_size * cunit->cache_sets; 95 cunit->cache_size = cunit->cache_way_size * cunit->cache_ways; 96 } 97 98 /* Must be called on each processor */ 99 void __noasan 100 aarch64_getcacheinfo(struct cpu_info *ci) 101 { 102 struct aarch64_cache_info * const cinfo = ci->ci_cacheinfo; 103 uint32_t clidr, ctr; 104 int level, cachetype; 105 106 /* 107 * CTR - Cache Type Register 108 */ 109 ctr = reg_ctr_el0_read(); 110 switch (__SHIFTOUT(ctr, CTR_EL0_L1IP_MASK)) { 111 case CTR_EL0_L1IP_VPIPT: 112 cachetype = CACHE_TYPE_VPIPT; 113 break; 114 case CTR_EL0_L1IP_AIVIVT: 115 cachetype = CACHE_TYPE_VIVT; 116 break; 117 case CTR_EL0_L1IP_VIPT: 118 cachetype = CACHE_TYPE_VIPT; 119 break; 120 case CTR_EL0_L1IP_PIPT: 121 cachetype = CACHE_TYPE_PIPT; 122 break; 123 } 124 125 /* 126 * CLIDR - Cache Level ID Register 127 * CSSELR - Cache Size Selection Register 128 * CCSIDR - CurrentCache Size ID Register (selected by CSSELR) 129 */ 130 131 /* L1, L2, L3, ..., L8 cache */ 132 for (level = 0, clidr = reg_clidr_el1_read(); 133 level < MAX_CACHE_LEVEL; level++, clidr >>= 3) { 134 135 int cacheable; 136 137 switch (clidr & 7) { 138 case CLIDR_TYPE_NOCACHE: 139 cacheable = CACHE_CACHEABLE_NONE; 140 break; 141 case CLIDR_TYPE_ICACHE: 142 cacheable = CACHE_CACHEABLE_ICACHE; 143 extract_cacheunit(level, true, cachetype, cinfo); 144 break; 145 case CLIDR_TYPE_DCACHE: 146 cacheable = CACHE_CACHEABLE_DCACHE; 147 extract_cacheunit(level, false, CACHE_TYPE_PIPT, cinfo); 148 break; 149 case CLIDR_TYPE_IDCACHE: 150 cacheable = CACHE_CACHEABLE_IDCACHE; 151 extract_cacheunit(level, true, cachetype, cinfo); 152 extract_cacheunit(level, false, CACHE_TYPE_PIPT, cinfo); 153 break; 154 case CLIDR_TYPE_UNIFIEDCACHE: 155 cacheable = CACHE_CACHEABLE_UNIFIED; 156 extract_cacheunit(level, false, CACHE_TYPE_PIPT, cinfo); 157 break; 158 default: 159 cacheable = CACHE_CACHEABLE_NONE; 160 break; 161 } 162 163 cinfo[level].cacheable = cacheable; 164 if (cacheable == CACHE_CACHEABLE_NONE) { 165 /* no more level */ 166 break; 167 } 168 169 /* 170 * L1 insn cachetype is CTR_EL0:L1IP, 171 * all other cachetype is PIPT. 172 */ 173 cachetype = CACHE_TYPE_PIPT; 174 } 175 } 176 177 void 178 aarch64_parsecacheinfo(struct cpu_info *ci) 179 { 180 struct aarch64_cache_info * const cinfo = ci->ci_cacheinfo; 181 struct aarch64_sysctl_cpu_id *id = &ci->ci_id; 182 const uint32_t ctr = id->ac_ctr; 183 u_int vindexsize; 184 185 /* remember maximum alignment */ 186 if (arm_dcache_maxline < __SHIFTOUT(ctr, CTR_EL0_DMIN_LINE)) { 187 arm_dcache_maxline = __SHIFTOUT(ctr, CTR_EL0_DMIN_LINE); 188 arm_dcache_align = sizeof(int) << arm_dcache_maxline; 189 arm_dcache_align_mask = arm_dcache_align - 1; 190 } 191 192 #ifdef MULTIPROCESSOR 193 if (coherency_unit < arm_dcache_align) 194 panic("coherency_unit %ld < %d; increase COHERENCY_UNIT", 195 coherency_unit, arm_dcache_align); 196 #endif 197 198 /* calculate L1 icache virtual index size */ 199 if ((cinfo[0].icache.cache_type == CACHE_TYPE_VIVT || 200 cinfo[0].icache.cache_type == CACHE_TYPE_VIPT) && 201 (cinfo[0].cacheable == CACHE_CACHEABLE_ICACHE || 202 cinfo[0].cacheable == CACHE_CACHEABLE_IDCACHE)) { 203 204 vindexsize = 205 cinfo[0].icache.cache_size / 206 cinfo[0].icache.cache_ways; 207 208 KASSERT(vindexsize != 0); 209 } else { 210 vindexsize = 0; 211 } 212 213 if (vindexsize > aarch64_cache_vindexsize) { 214 aarch64_cache_vindexsize = vindexsize; 215 aarch64_cache_prefer_mask = vindexsize - 1; 216 217 if (uvm.page_init_done) 218 uvm_page_recolor(vindexsize / PAGE_SIZE); 219 } 220 } 221 222 static int 223 prt_cache(device_t self, struct aarch64_cache_info *cinfo, int level) 224 { 225 struct aarch64_cache_unit *cunit; 226 int i; 227 const char *cacheable, *cachetype; 228 229 if (cinfo[level].cacheable == CACHE_CACHEABLE_NONE) 230 return -1; 231 232 for (i = 0; i < 2; i++) { 233 switch (cinfo[level].cacheable) { 234 case CACHE_CACHEABLE_ICACHE: 235 cunit = &cinfo[level].icache; 236 cacheable = "Instruction"; 237 break; 238 case CACHE_CACHEABLE_DCACHE: 239 cunit = &cinfo[level].dcache; 240 cacheable = "Data"; 241 break; 242 case CACHE_CACHEABLE_IDCACHE: 243 if (i == 0) { 244 cunit = &cinfo[level].icache; 245 cacheable = "Instruction"; 246 } else { 247 cunit = &cinfo[level].dcache; 248 cacheable = "Data"; 249 } 250 break; 251 case CACHE_CACHEABLE_UNIFIED: 252 cunit = &cinfo[level].dcache; 253 cacheable = "Unified"; 254 break; 255 default: 256 cunit = &cinfo[level].dcache; 257 cacheable = "*UNK*"; 258 break; 259 } 260 261 switch (cunit->cache_type) { 262 case CACHE_TYPE_VPIPT: 263 cachetype = "VPIPT"; 264 break; 265 case CACHE_TYPE_VIVT: 266 cachetype = "VIVT"; 267 break; 268 case CACHE_TYPE_VIPT: 269 cachetype = "VIPT"; 270 break; 271 case CACHE_TYPE_PIPT: 272 cachetype = "PIPT"; 273 break; 274 default: 275 cachetype = "*UNK*"; 276 break; 277 } 278 279 aprint_verbose_dev(self, 280 "L%d %uKB/%uB %u-way (%u set) %s %s cache\n", 281 level + 1, 282 cunit->cache_size / 1024, 283 cunit->cache_line_size, 284 cunit->cache_ways, 285 cunit->cache_sets, 286 cachetype, cacheable); 287 288 if (cinfo[level].cacheable != CACHE_CACHEABLE_IDCACHE) 289 break; 290 } 291 292 return 0; 293 } 294 295 void 296 aarch64_printcacheinfo(device_t dev, struct cpu_info *ci) 297 { 298 struct aarch64_cache_info * const cinfo = ci->ci_cacheinfo; 299 int level; 300 301 for (level = 0; level < MAX_CACHE_LEVEL; level++) 302 if (prt_cache(dev, cinfo, level) < 0) 303 break; 304 } 305 306 static inline void 307 ln_dcache_wb_all(int level, struct aarch64_cache_unit *cunit) 308 { 309 uint64_t x; 310 unsigned int set, way, setshift, wayshift; 311 312 setshift = ffs(cunit->cache_line_size) - 1; 313 wayshift = 32 - (ffs(cunit->cache_ways) - 1); 314 315 for (way = 0; way < cunit->cache_ways; way++) { 316 for (set = 0; set < cunit->cache_sets; set++) { 317 x = (way << wayshift) | (set << setshift) | 318 (level << 1); 319 __asm __volatile ("dc csw, %0; dsb sy" :: "r"(x)); 320 } 321 } 322 } 323 324 static inline void 325 ln_dcache_wbinv_all(int level, struct aarch64_cache_unit *cunit) 326 { 327 uint64_t x; 328 unsigned int set, way, setshift, wayshift; 329 330 setshift = ffs(cunit->cache_line_size) - 1; 331 wayshift = 32 - (ffs(cunit->cache_ways) - 1); 332 333 for (way = 0; way < cunit->cache_ways; way++) { 334 for (set = 0; set < cunit->cache_sets; set++) { 335 x = (way << wayshift) | (set << setshift) | 336 (level << 1); 337 __asm __volatile ("dc cisw, %0; dsb sy" :: "r"(x)); 338 } 339 } 340 } 341 342 static inline void 343 ln_dcache_inv_all(int level, struct aarch64_cache_unit *cunit) 344 { 345 uint64_t x; 346 unsigned int set, way, setshift, wayshift; 347 348 setshift = ffs(cunit->cache_line_size) - 1; 349 wayshift = 32 - (ffs(cunit->cache_ways) - 1); 350 351 for (way = 0; way < cunit->cache_ways; way++) { 352 for (set = 0; set < cunit->cache_sets; set++) { 353 x = (way << wayshift) | (set << setshift) | 354 (level << 1); 355 __asm __volatile ("dc isw, %0; dsb sy" :: "r"(x)); 356 } 357 } 358 } 359 360 void 361 aarch64_dcache_wbinv_all(void) 362 { 363 KASSERT(kpreempt_disabled()); 364 365 struct cpu_info * const ci = curcpu(); 366 struct aarch64_cache_info * const cinfo = ci->ci_cacheinfo; 367 int level; 368 369 for (level = 0; level < MAX_CACHE_LEVEL; level++) { 370 if (cinfo[level].cacheable == CACHE_CACHEABLE_NONE) 371 break; 372 373 dsb(ish); 374 ln_dcache_wbinv_all(level, &cinfo[level].dcache); 375 } 376 dsb(ish); 377 } 378 379 void 380 aarch64_dcache_inv_all(void) 381 { 382 KASSERT(kpreempt_disabled()); 383 384 struct cpu_info * const ci = curcpu(); 385 struct aarch64_cache_info * const cinfo = ci->ci_cacheinfo; 386 int level; 387 388 for (level = 0; level < MAX_CACHE_LEVEL; level++) { 389 if (cinfo[level].cacheable == CACHE_CACHEABLE_NONE) 390 break; 391 392 dsb(ish); 393 ln_dcache_inv_all(level, &cinfo[level].dcache); 394 } 395 dsb(ish); 396 } 397 398 void 399 aarch64_dcache_wb_all(void) 400 { 401 KASSERT(kpreempt_disabled()); 402 403 struct cpu_info * const ci = curcpu(); 404 struct aarch64_cache_info * const cinfo = ci->ci_cacheinfo; 405 int level; 406 407 for (level = 0; level < MAX_CACHE_LEVEL; level++) { 408 if (cinfo[level].cacheable == CACHE_CACHEABLE_NONE) 409 break; 410 411 dsb(ish); 412 ln_dcache_wb_all(level, &cinfo[level].dcache); 413 } 414 dsb(ish); 415 } 416 417 bool 418 aarch64_earlydevice_va_p(void) 419 { 420 /* This function may be called before enabling MMU, or mapping KVA */ 421 if ((reg_sctlr_el1_read() & SCTLR_M) == 0) 422 return false; 423 424 /* device mapping will be available after pmap_devmap_bootstrap() */ 425 if (!pmap_devmap_bootstrapped_p()) 426 return false; 427 428 return true; 429 } 430 431 int 432 set_cpufuncs(void) 433 { 434 // This is only called from the BP 435 436 return aarch64_setcpufuncs(&cpu_info_store[0]); 437 } 438 439 int 440 aarch64_setcpufuncs(struct cpu_info *ci) 441 { 442 const uint64_t ctr = reg_ctr_el0_read(); 443 const uint64_t clidr = reg_clidr_el1_read(); 444 445 /* install default functions */ 446 ci->ci_cpufuncs.cf_set_ttbr0 = aarch64_set_ttbr0; 447 ci->ci_cpufuncs.cf_icache_sync_range = aarch64_icache_sync_range; 448 449 /* 450 * install core/cluster specific functions 451 */ 452 453 /* Icache sync op */ 454 if (__SHIFTOUT(ctr, CTR_EL0_DIC) == 1) { 455 /* Icache invalidation to the PoU is not required */ 456 ci->ci_cpufuncs.cf_icache_sync_range = 457 aarch64_icache_barrier_range; 458 } else if (__SHIFTOUT(ctr, CTR_EL0_IDC) == 1 || 459 __SHIFTOUT(clidr, CLIDR_LOC) == 0 || 460 (__SHIFTOUT(clidr, CLIDR_LOUIS) == 0 && __SHIFTOUT(clidr, CLIDR_LOUU) == 0)) { 461 /* Dcache clean to the PoU is not required for Icache */ 462 ci->ci_cpufuncs.cf_icache_sync_range = 463 aarch64_icache_inv_range; 464 } 465 466 #ifdef CPU_THUNDERX 467 const uint32_t midr = reg_midr_el1_read(); 468 469 /* Cavium erratum 27456 */ 470 if ((midr == CPU_ID_THUNDERXP1d0) || 471 (midr == CPU_ID_THUNDERXP1d1) || 472 (midr == CPU_ID_THUNDERXP2d1) || 473 (midr == CPU_ID_THUNDERX81XXRX)) { 474 ci->ci_cpufuncs.cf_set_ttbr0 = aarch64_set_ttbr0_thunderx; 475 } 476 #endif 477 478 return 0; 479 } 480 481 void 482 aarch64_hafdbs_init(int primary) 483 { 484 #ifdef ARMV81_HAFDBS 485 uint64_t tcr; 486 int hafdbs; 487 488 hafdbs = __SHIFTOUT(reg_id_aa64mmfr1_el1_read(), 489 ID_AA64MMFR1_EL1_HAFDBS); 490 491 /* 492 * hafdbs 493 * 0:HAFDBS_NONE - no support for any hardware flags 494 * 1:HAFDBS_A - only hardware access flag supported 495 * 2:HAFDBS_AD - hardware access and modified flags supported. 496 */ 497 498 if (primary) { 499 /* CPU0 does the detection. */ 500 switch (hafdbs) { 501 case ID_AA64MMFR1_EL1_HAFDBS_NONE: 502 default: 503 aarch64_hafdbs_enabled = 0; 504 break; 505 case ID_AA64MMFR1_EL1_HAFDBS_A: 506 case ID_AA64MMFR1_EL1_HAFDBS_AD: 507 aarch64_hafdbs_enabled = hafdbs; 508 break; 509 } 510 } else { 511 /* 512 * The support status of HAFDBS on the primary CPU is different 513 * from that of the application processor. 514 * 515 * XXX: 516 * The correct way to do this is to disable it on all cores, 517 * or call pmap_fault_fixup() only on the unsupported cores, 518 * but for now, do panic(). 519 */ 520 if (aarch64_hafdbs_enabled != hafdbs) 521 panic("HAFDBS is supported (%d) on primary cpu, " 522 "but isn't equal (%d) on secondary cpu", 523 aarch64_hafdbs_enabled, hafdbs); 524 } 525 526 /* enable Hardware updates to Access flag and Dirty state */ 527 tcr = reg_tcr_el1_read(); 528 switch (hafdbs) { 529 case ID_AA64MMFR1_EL1_HAFDBS_NONE: 530 default: 531 break; 532 case ID_AA64MMFR1_EL1_HAFDBS_A: 533 /* enable only access */ 534 reg_tcr_el1_write(tcr | TCR_HA); 535 isb(); 536 break; 537 case ID_AA64MMFR1_EL1_HAFDBS_AD: 538 /* enable both access and dirty */ 539 reg_tcr_el1_write(tcr | TCR_HD | TCR_HA); 540 isb(); 541 break; 542 } 543 #endif 544 } 545 546 void 547 aarch64_pan_init(int primary) 548 { 549 #ifdef ARMV81_PAN 550 uint64_t reg, sctlr; 551 552 /* CPU0 does the detection. */ 553 if (primary) { 554 reg = reg_id_aa64mmfr1_el1_read(); 555 if (__SHIFTOUT(reg, ID_AA64MMFR1_EL1_PAN) != 556 ID_AA64MMFR1_EL1_PAN_NONE) 557 aarch64_pan_enabled = 1; 558 } 559 560 if (!aarch64_pan_enabled) 561 return; 562 563 /* 564 * On an exception to EL1, have the CPU set the PAN bit automatically. 565 * This ensures PAN is enabled each time the kernel is entered. 566 */ 567 sctlr = reg_sctlr_el1_read(); 568 sctlr &= ~SCTLR_SPAN; 569 reg_sctlr_el1_write(sctlr); 570 571 /* Set the PAN bit right now. */ 572 reg_pan_write(1); 573 #endif 574 } 575 576 /* 577 * In order to avoid inconsistencies with pointer authentication 578 * in this function itself, the caller must enable PAC according 579 * to the return value. 580 */ 581 int 582 aarch64_pac_init(int primary) 583 { 584 #ifdef ARMV83_PAC 585 uint64_t reg; 586 587 /* CPU0 does the detection. */ 588 if (primary) { 589 reg = reg_id_aa64isar1_el1_read(); 590 if (__SHIFTOUT(reg, ID_AA64ISAR1_EL1_APA) != 591 ID_AA64ISAR1_EL1_APA_NONE) 592 aarch64_pac_enabled = 1; 593 if (__SHIFTOUT(reg, ID_AA64ISAR1_EL1_API) != 594 ID_AA64ISAR1_EL1_API_NONE) 595 aarch64_pac_enabled = 1; 596 if (__SHIFTOUT(reg, ID_AA64ISAR1_EL1_GPA) != 597 ID_AA64ISAR1_EL1_GPA_NONE) 598 aarch64_pac_enabled = 1; 599 if (__SHIFTOUT(reg, ID_AA64ISAR1_EL1_GPI) != 600 ID_AA64ISAR1_EL1_GPI_NONE) 601 aarch64_pac_enabled = 1; 602 } 603 604 if (!aarch64_pac_enabled) 605 return -1; 606 607 /* Set the key. Curlwp here is the CPU's idlelwp. */ 608 reg_APIAKeyLo_EL1_write(curlwp->l_md.md_ia_kern[0]); 609 reg_APIAKeyHi_EL1_write(curlwp->l_md.md_ia_kern[1]); 610 611 return 0; 612 #else 613 return -1; 614 #endif 615 } 616
Indexes created Sun Oct 25 00:26:45 UTC 2026