1 /* $NetBSD: cpu.c,v 1.84 2026/03/29 11:15:03 skrll Exp $ */ 2 3 /* 4 * Copyright (c) 2017 Ryo Shimizu 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 17 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 18 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 19 * DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, 20 * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 21 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 22 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, 24 * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING 25 * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 26 * POSSIBILITY OF SUCH DAMAGE. 27 */ 28 29 #include <sys/cdefs.h> 30 __KERNEL_RCSID(1, "$NetBSD: cpu.c,v 1.84 2026/03/29 11:15:03 skrll Exp $"); 31 32 #include "locators.h" 33 #include "opt_arm_debug.h" 34 #include "opt_ddb.h" 35 #include "opt_fdt.h" 36 #include "opt_multiprocessor.h" 37 38 #include <sys/param.h> 39 #include <sys/atomic.h> 40 #include <sys/cpu.h> 41 #include <sys/device.h> 42 #include <sys/kmem.h> 43 #include <sys/reboot.h> 44 #include <sys/rndsource.h> 45 #include <sys/sdt.h> 46 #include <sys/sysctl.h> 47 #include <sys/systm.h> 48 49 #include <crypto/aes/aes_impl.h> 50 #include <crypto/aes/arch/arm/aes_armv8.h> 51 #include <crypto/aes/arch/arm/aes_neon.h> 52 #include <crypto/chacha/chacha_impl.h> 53 #include <crypto/chacha/arch/arm/chacha_neon.h> 54 55 #include <aarch64/armreg.h> 56 #include <aarch64/cpu.h> 57 #include <aarch64/cpu_counter.h> 58 #ifdef DDB 59 #include <aarch64/db_machdep.h> 60 #endif 61 #include <aarch64/machdep.h> 62 63 #include <arm/cpufunc.h> 64 #include <arm/cpuvar.h> 65 #include <arm/cpu_topology.h> 66 #ifdef FDT 67 #include <arm/fdt/arm_fdtvar.h> 68 #endif 69 70 #ifdef VERBOSE_INIT_ARM 71 #define VPRINTF(...) printf(__VA_ARGS__) 72 #else 73 #define VPRINTF(...) __nothing 74 #endif 75 76 void cpu_attach(device_t, cpuid_t); 77 void cpu_setup_id(struct cpu_info *); 78 79 static void identify_aarch64_model(uint32_t, char *, size_t); 80 static void cpu_identify(device_t self, struct cpu_info *); 81 static void cpu_identify1(device_t self, struct cpu_info *); 82 static void cpu_identify2(device_t self, struct cpu_info *); 83 static void cpu_init_counter(struct cpu_info *); 84 static void cpu_setup_sysctl(device_t, struct cpu_info *); 85 static void cpu_setup_rng(device_t, struct cpu_info *); 86 static void cpu_setup_aes(device_t, struct cpu_info *); 87 static void cpu_setup_chacha(device_t, struct cpu_info *); 88 89 #ifdef MULTIPROCESSOR 90 #define NCPUINFO MAXCPUS 91 #else 92 #define NCPUINFO 1 93 #endif /* MULTIPROCESSOR */ 94 95 /* 96 * Our exported cpu_info structs; these will be first used by the 97 * secondary cpus as part of cpu_mpstart and the hatching process. 98 */ 99 struct cpu_info cpu_info_store[NCPUINFO] = { 100 [0] = { 101 .ci_cpl = IPL_HIGH, 102 .ci_curlwp = &lwp0 103 } 104 }; 105 106 void 107 cpu_attach(device_t dv, cpuid_t id) 108 { 109 struct cpu_info *ci; 110 const int unit = device_unit(dv); 111 112 if (unit == 0) { 113 ci = curcpu(); 114 ci->ci_cpuid = id; 115 } else { 116 #ifdef MULTIPROCESSOR 117 if ((boothowto & RB_MD1) != 0) { 118 aprint_naive("\n"); 119 aprint_normal(": multiprocessor boot disabled\n"); 120 return; 121 } 122 123 KASSERT(unit < MAXCPUS); 124 ci = &cpu_info_store[unit]; 125 126 ci->ci_cpl = IPL_HIGH; 127 ci->ci_cpuid = id; 128 /* ci_id is stored by own cpus when hatching */ 129 130 cpu_info[ncpu] = ci; 131 if (cpu_hatched_p(unit) == 0) { 132 ci->ci_dev = dv; 133 device_set_private(dv, ci); 134 ci->ci_index = -1; 135 136 aprint_naive(": disabled\n"); 137 aprint_normal(": disabled (unresponsive)\n"); 138 return; 139 } 140 #else /* MULTIPROCESSOR */ 141 aprint_naive(": disabled\n"); 142 aprint_normal(": disabled (uniprocessor kernel)\n"); 143 return; 144 #endif /* MULTIPROCESSOR */ 145 } 146 147 ci->ci_dev = dv; 148 device_set_private(dv, ci); 149 150 ci->ci_kfpu_spl = -1; 151 152 arm_cpu_do_topology(ci); // XXXNH move this after mi_cpu_attach 153 cpu_identify(dv, ci); 154 155 cpu_setup_sysctl(dv, ci); 156 157 #ifdef MULTIPROCESSOR 158 if (unit != 0) { 159 mi_cpu_attach(ci); 160 pmap_tlb_info_attach(&pmap_tlb0_info, ci); 161 aarch64_parsecacheinfo(ci); 162 } 163 #endif /* MULTIPROCESSOR */ 164 165 fpu_attach(ci); 166 167 cpu_identify1(dv, ci); 168 aarch64_printcacheinfo(dv, ci); 169 cpu_identify2(dv, ci); 170 171 cpu_setup_rng(dv, ci); 172 173 if (unit != 0) { 174 return; 175 } 176 177 #ifdef DDB 178 db_machdep_init(ci); 179 #endif 180 181 cpu_init_counter(ci); 182 183 /* These currently only check the BP. */ 184 cpu_setup_aes(dv, ci); 185 cpu_setup_chacha(dv, ci); 186 187 cpu_rescan(dv, NULL, NULL); 188 } 189 190 int 191 cpu_rescan(device_t dv, const char *ifattr, const int *locators) 192 { 193 struct cpu_info *ci = device_private(dv); 194 195 if (ifattr_match(ifattr, "cpufeaturebus")) { 196 struct cpufeature_attach_args cfaa = { 197 .ci = ci, 198 }; 199 config_found(dv, &cfaa, NULL, CFARGS(.iattr = "cpufeaturebus")); 200 } 201 202 return 0; 203 } 204 205 void 206 cpu_childdetached(device_t dv, device_t child) 207 { 208 /* Nada */ 209 } 210 211 struct cpuidtab { 212 uint32_t cpu_partnum; 213 const char *cpu_name; 214 const char *cpu_vendor; 215 const char *cpu_architecture; 216 }; 217 218 #define CPU_PARTMASK (CPU_ID_IMPLEMENTOR_MASK | CPU_ID_PARTNO_MASK) 219 220 const struct cpuidtab cpuids[] = { 221 { CPU_ID_CORTEXA35R0 & CPU_PARTMASK, "Cortex-A35", "Arm", "v8-A" }, 222 { CPU_ID_CORTEXA53R0 & CPU_PARTMASK, "Cortex-A53", "Arm", "v8-A" }, 223 { CPU_ID_CORTEXA57R0 & CPU_PARTMASK, "Cortex-A57", "Arm", "v8-A" }, 224 { CPU_ID_CORTEXA55R1 & CPU_PARTMASK, "Cortex-A55", "Arm", "v8.2-A+" }, 225 { CPU_ID_CORTEXA65R0 & CPU_PARTMASK, "Cortex-A65", "Arm", "v8.2-A+" }, 226 { CPU_ID_CORTEXA72R0 & CPU_PARTMASK, "Cortex-A72", "Arm", "v8-A" }, 227 { CPU_ID_CORTEXA73R0 & CPU_PARTMASK, "Cortex-A73", "Arm", "v8-A" }, 228 { CPU_ID_CORTEXA75R2 & CPU_PARTMASK, "Cortex-A75", "Arm", "v8.2-A+" }, 229 { CPU_ID_CORTEXA76R3 & CPU_PARTMASK, "Cortex-A76", "Arm", "v8.2-A+" }, 230 { CPU_ID_CORTEXA76AER1 & CPU_PARTMASK, "Cortex-A76AE", "Arm", "v8.2-A+" }, 231 { CPU_ID_CORTEXA77R0 & CPU_PARTMASK, "Cortex-A77", "Arm", "v8.2-A+" }, 232 { CPU_ID_CORTEXA520R0 & CPU_PARTMASK, "Cortex-A520", "Arm", "v9.2-A" }, 233 { CPU_ID_CORTEXA710R2 & CPU_PARTMASK, "Cortex-A710", "Arm", "v9.0-A" }, 234 { CPU_ID_CORTEXA720R0 & CPU_PARTMASK, "Cortex-A720", "Arm", "v9.2-A" }, 235 { CPU_ID_NVIDIADENVER2 & CPU_PARTMASK, "Denver2", "NVIDIA", "v8-A" }, 236 { CPU_ID_EMAG8180 & CPU_PARTMASK, "eMAG", "Ampere", "v8-A" }, 237 { CPU_ID_NEOVERSEE1R1 & CPU_PARTMASK, "Neoverse E1", "Arm", "v8.2-A+" }, 238 { CPU_ID_NEOVERSEN1R3 & CPU_PARTMASK, "Neoverse N1", "Arm", "v8.2-A+" }, 239 { CPU_ID_NEOVERSEV1R1 & CPU_PARTMASK, "Neoverse V1", "Arm", "v8.4-A+" }, 240 { CPU_ID_NEOVERSEN2R0 & CPU_PARTMASK, "Neoverse N2", "Arm", "v9.0-A" }, 241 { CPU_ID_THUNDERXRX, "ThunderX", "Cavium", "v8-A" }, 242 { CPU_ID_THUNDERX81XXRX, "ThunderX CN81XX", "Cavium", "v8-A" }, 243 { CPU_ID_THUNDERX83XXRX, "ThunderX CN83XX", "Cavium", "v8-A" }, 244 { CPU_ID_THUNDERX2RX, "ThunderX2", "Marvell", "v8.1-A" }, 245 { CPU_ID_APPLE_M1_ICESTORM & CPU_PARTMASK, "M1 Icestorm", "Apple", "Apple Silicon" }, 246 { CPU_ID_APPLE_M1_FIRESTORM & CPU_PARTMASK, "M1 Firestorm", "Apple", "Apple Silicon" }, 247 { CPU_ID_AMPERE1 & CPU_PARTMASK, "Ampere-1", "Ampere", "v8.6-A+" }, 248 { CPU_ID_AMPERE1A & CPU_PARTMASK, "Ampere-1A", "Ampere", "v8.6-A+" }, 249 { CPU_ID_A64FX & CPU_PARTMASK, "A64FX", "Fujitsu", "v8.2-A+" }, 250 { CPU_ID_ORYON & CPU_PARTMASK, "Oryon", "Qualcomm", "v8.7-A+" }, 251 }; 252 253 static void 254 identify_aarch64_model(uint32_t cpuid, char *buf, size_t len) 255 { 256 int i; 257 uint32_t cpupart, variant, revision; 258 259 cpupart = cpuid & CPU_PARTMASK; 260 variant = __SHIFTOUT(cpuid, CPU_ID_VARIANT_MASK); 261 revision = __SHIFTOUT(cpuid, CPU_ID_REVISION_MASK); 262 263 for (i = 0; i < __arraycount(cpuids); i++) { 264 if (cpupart == cpuids[i].cpu_partnum) { 265 snprintf(buf, len, "%s %s r%dp%d (%s)", 266 cpuids[i].cpu_vendor, cpuids[i].cpu_name, 267 variant, revision, 268 cpuids[i].cpu_architecture); 269 return; 270 } 271 } 272 273 snprintf(buf, len, "unknown CPU (ID = 0x%08x)", cpuid); 274 } 275 276 static void 277 cpu_identify(device_t self, struct cpu_info *ci) 278 { 279 char model[128]; 280 const char *m; 281 282 identify_aarch64_model(ci->ci_id.ac_midr, model, sizeof(model)); 283 284 aprint_naive("\n"); 285 aprint_normal(": %s, id 0x%lx\n", model, ci->ci_cpuid); 286 aprint_normal_dev(ci->ci_dev, "package %u, core %u, smt %u, numa %u\n", 287 ci->ci_package_id, ci->ci_core_id, ci->ci_smt_id, ci->ci_numa_id); 288 289 if (ci->ci_index == 0) { 290 m = cpu_getmodel(); 291 if (m == NULL || *m == 0) 292 cpu_setmodel("%s", model); 293 294 if (CPU_ID_ERRATA_CAVIUM_THUNDERX_1_1_P(ci->ci_id.ac_midr)) 295 aprint_normal("WARNING: ThunderX Pass 1.1 detected.\n" 296 "This has known hardware bugs that may cause the " 297 "incorrect operation of atomic operations.\n"); 298 } 299 } 300 301 static void 302 cpu_identify1(device_t self, struct cpu_info *ci) 303 { 304 struct aarch64_sysctl_cpu_id *id = &ci->ci_id; 305 uint64_t sctlr = ci->ci_sctlr_el1; 306 307 if (sctlr & SCTLR_I) 308 aprint_verbose_dev(self, "IC enabled"); 309 else 310 aprint_verbose_dev(self, "IC disabled"); 311 312 if (sctlr & SCTLR_C) 313 aprint_verbose(", DC enabled"); 314 else 315 aprint_verbose(", DC disabled"); 316 317 if (sctlr & SCTLR_A) 318 aprint_verbose(", Alignment check enabled\n"); 319 else { 320 switch (sctlr & (SCTLR_SA | SCTLR_SA0)) { 321 case SCTLR_SA | SCTLR_SA0: 322 aprint_verbose( 323 ", EL0/EL1 stack Alignment check enabled\n"); 324 break; 325 case SCTLR_SA: 326 aprint_verbose(", EL1 stack Alignment check enabled\n"); 327 break; 328 case SCTLR_SA0: 329 aprint_verbose(", EL0 stack Alignment check enabled\n"); 330 break; 331 case 0: 332 aprint_verbose(", Alignment check disabled\n"); 333 break; 334 } 335 } 336 337 /* 338 * CTR - Cache Type Register 339 */ 340 const uint64_t ctr = id->ac_ctr; 341 const uint64_t clidr = id->ac_clidr; 342 aprint_verbose_dev(self, "Cache Writeback Granule %" PRIu64 "B," 343 " Exclusives Reservation Granule %" PRIu64 "B\n", 344 __SHIFTOUT(ctr, CTR_EL0_CWG_LINE) * 4, 345 __SHIFTOUT(ctr, CTR_EL0_ERG_LINE) * 4); 346 347 aprint_verbose_dev(self, "Dcache line %ld, Icache line %ld" 348 ", DIC=%lu, IDC=%lu, LoUU=%lu, LoC=%lu, LoUIS=%lu\n", 349 sizeof(int) << __SHIFTOUT(ctr, CTR_EL0_DMIN_LINE), 350 sizeof(int) << __SHIFTOUT(ctr, CTR_EL0_IMIN_LINE), 351 __SHIFTOUT(ctr, CTR_EL0_DIC), 352 __SHIFTOUT(ctr, CTR_EL0_IDC), 353 __SHIFTOUT(clidr, CLIDR_LOUU), 354 __SHIFTOUT(clidr, CLIDR_LOC), 355 __SHIFTOUT(clidr, CLIDR_LOUIS)); 356 } 357 358 359 /* 360 * identify vfp, etc. 361 */ 362 static void 363 cpu_identify2(device_t self, struct cpu_info *ci) 364 { 365 struct aarch64_sysctl_cpu_id * const id = &ci->ci_id; 366 367 aprint_debug_dev(self, "midr=0x%" PRIx64 " mpidr=0x%" PRIx64 "\n", 368 id->ac_midr, id->ac_mpidr); 369 aprint_verbose_dev(self, "revID=0x%" PRIx64, id->ac_revidr); 370 371 /* ID_AA64DFR0_EL1 */ 372 switch (__SHIFTOUT(id->ac_aa64dfr0, ID_AA64DFR0_EL1_PMUVER)) { 373 case ID_AA64DFR0_EL1_PMUVER_V3: 374 aprint_verbose(", PMUv3"); 375 break; 376 case ID_AA64DFR0_EL1_PMUVER_V3P1: 377 aprint_verbose(", PMUv3p1"); 378 break; 379 case ID_AA64DFR0_EL1_PMUVER_V3P4: 380 aprint_verbose(", PMUv3p4"); 381 break; 382 case ID_AA64DFR0_EL1_PMUVER_V3P5: 383 aprint_verbose(", PMUv3p5"); 384 break; 385 case ID_AA64DFR0_EL1_PMUVER_V3P7: 386 aprint_verbose(", PMUv3p7"); 387 break; 388 case ID_AA64DFR0_EL1_PMUVER_V3P8: 389 aprint_verbose(", PMUv3p8"); 390 break; 391 } 392 393 /* ID_AA64MMFR0_EL1 */ 394 switch (__SHIFTOUT(id->ac_aa64mmfr0, ID_AA64MMFR0_EL1_TGRAN4)) { 395 case ID_AA64MMFR0_EL1_TGRAN4_4KB: 396 aprint_verbose(", 4k table"); 397 break; 398 } 399 switch (__SHIFTOUT(id->ac_aa64mmfr0, ID_AA64MMFR0_EL1_TGRAN16)) { 400 case ID_AA64MMFR0_EL1_TGRAN16_16KB: 401 aprint_verbose(", 16k table"); 402 break; 403 } 404 switch (__SHIFTOUT(id->ac_aa64mmfr0, ID_AA64MMFR0_EL1_TGRAN64)) { 405 case ID_AA64MMFR0_EL1_TGRAN64_64KB: 406 aprint_verbose(", 64k table"); 407 break; 408 } 409 410 switch (__SHIFTOUT(id->ac_aa64mmfr0, ID_AA64MMFR0_EL1_ASIDBITS)) { 411 case ID_AA64MMFR0_EL1_ASIDBITS_8BIT: 412 aprint_verbose(", 8bit ASID"); 413 break; 414 case ID_AA64MMFR0_EL1_ASIDBITS_16BIT: 415 aprint_verbose(", 16bit ASID"); 416 break; 417 } 418 aprint_verbose("\n"); 419 420 aprint_verbose_dev(self, "auxID=0x%" PRIx64, ci->ci_id.ac_aa64isar0); 421 422 /* PFR0 */ 423 switch (__SHIFTOUT(id->ac_aa64pfr0, ID_AA64PFR0_EL1_CSV3)) { 424 case ID_AA64PFR0_EL1_CSV3_IMPL: 425 aprint_verbose(", CSV3"); 426 break; 427 } 428 switch (__SHIFTOUT(id->ac_aa64pfr0, ID_AA64PFR0_EL1_CSV2)) { 429 case ID_AA64PFR0_EL1_CSV2_IMPL: 430 aprint_verbose(", CSV2"); 431 break; 432 } 433 switch (__SHIFTOUT(id->ac_aa64pfr0, ID_AA64PFR0_EL1_GIC)) { 434 case ID_AA64PFR0_EL1_GIC_CPUIF_EN: 435 aprint_verbose(", GICv3"); 436 break; 437 } 438 switch (__SHIFTOUT(id->ac_aa64pfr0, ID_AA64PFR0_EL1_FP)) { 439 case ID_AA64PFR0_EL1_FP_NONE: 440 break; 441 default: 442 aprint_verbose(", FP"); 443 break; 444 } 445 446 /* ISAR0 */ 447 switch (__SHIFTOUT(id->ac_aa64isar0, ID_AA64ISAR0_EL1_CRC32)) { 448 case ID_AA64ISAR0_EL1_CRC32_CRC32X: 449 aprint_verbose(", CRC32"); 450 break; 451 } 452 switch (__SHIFTOUT(id->ac_aa64isar0, ID_AA64ISAR0_EL1_SHA1)) { 453 case ID_AA64ISAR0_EL1_SHA1_SHA1CPMHSU: 454 aprint_verbose(", SHA1"); 455 break; 456 } 457 switch (__SHIFTOUT(id->ac_aa64isar0, ID_AA64ISAR0_EL1_SHA2)) { 458 case ID_AA64ISAR0_EL1_SHA2_SHA256HSU: 459 aprint_verbose(", SHA256"); 460 break; 461 } 462 switch (__SHIFTOUT(id->ac_aa64isar0, ID_AA64ISAR0_EL1_AES)) { 463 case ID_AA64ISAR0_EL1_AES_AES: 464 aprint_verbose(", AES"); 465 break; 466 case ID_AA64ISAR0_EL1_AES_PMUL: 467 aprint_verbose(", AES+PMULL"); 468 break; 469 } 470 switch (__SHIFTOUT(id->ac_aa64isar0, ID_AA64ISAR0_EL1_RNDR)) { 471 case ID_AA64ISAR0_EL1_RNDR_RNDRRS: 472 aprint_verbose(", RNDRRS"); 473 break; 474 } 475 476 /* PFR0:DIT -- data-independent timing support */ 477 switch (__SHIFTOUT(id->ac_aa64pfr0, ID_AA64PFR0_EL1_DIT)) { 478 case ID_AA64PFR0_EL1_DIT_IMPL: 479 aprint_verbose(", DIT"); 480 break; 481 } 482 483 /* PFR0:AdvSIMD */ 484 switch (__SHIFTOUT(id->ac_aa64pfr0, ID_AA64PFR0_EL1_ADVSIMD)) { 485 case ID_AA64PFR0_EL1_ADV_SIMD_NONE: 486 break; 487 default: 488 aprint_verbose(", NEON"); 489 break; 490 } 491 492 /* MVFR0/MVFR1 */ 493 switch (__SHIFTOUT(id->ac_mvfr0, MVFR0_FPROUND)) { 494 case MVFR0_FPROUND_ALL: 495 aprint_verbose(", rounding"); 496 break; 497 } 498 switch (__SHIFTOUT(id->ac_mvfr0, MVFR0_FPTRAP)) { 499 case MVFR0_FPTRAP_TRAP: 500 aprint_verbose(", exceptions"); 501 break; 502 } 503 switch (__SHIFTOUT(id->ac_mvfr1, MVFR1_FPDNAN)) { 504 case MVFR1_FPDNAN_NAN: 505 aprint_verbose(", NaN propagation"); 506 break; 507 } 508 switch (__SHIFTOUT(id->ac_mvfr1, MVFR1_FPFTZ)) { 509 case MVFR1_FPFTZ_DENORMAL: 510 aprint_verbose(", denormals"); 511 break; 512 } 513 switch (__SHIFTOUT(id->ac_mvfr0, MVFR0_SIMDREG)) { 514 case MVFR0_SIMDREG_16x64: 515 aprint_verbose(", 16x64bitRegs"); 516 break; 517 case MVFR0_SIMDREG_32x64: 518 aprint_verbose(", 32x64bitRegs"); 519 break; 520 } 521 switch (__SHIFTOUT(id->ac_mvfr1, MVFR1_SIMDFMAC)) { 522 case MVFR1_SIMDFMAC_FMAC: 523 aprint_verbose(", Fused Multiply-Add"); 524 break; 525 } 526 527 aprint_verbose("\n"); 528 } 529 530 /* 531 * Enable the performance counter, then estimate frequency for 532 * the current PE and store the result in cpu_cc_freq. 533 */ 534 static void 535 cpu_init_counter(struct cpu_info *ci) 536 { 537 const uint64_t dfr0 = reg_id_aa64dfr0_el1_read(); 538 const u_int pmuver = __SHIFTOUT(dfr0, ID_AA64DFR0_EL1_PMUVER); 539 if (pmuver == ID_AA64DFR0_EL1_PMUVER_NONE) { 540 /* Performance Monitors Extension not implemented. */ 541 return; 542 } 543 if (pmuver == ID_AA64DFR0_EL1_PMUVER_IMPL) { 544 /* Non-standard Performance Monitors are not supported. */ 545 return; 546 } 547 548 reg_pmcr_el0_write(PMCR_E | PMCR_C | PMCR_LC); 549 reg_pmintenclr_el1_write(PMINTEN_C | PMINTEN_P); 550 reg_pmcntenset_el0_write(PMCNTEN_C); 551 552 const uint32_t prev = cpu_counter32(); 553 delay(100000); 554 ci->ci_data.cpu_cc_freq = (cpu_counter32() - prev) * 10; 555 } 556 557 /* 558 * Fill in this CPUs id data. Must be called on all cpus. 559 */ 560 void __noasan 561 cpu_setup_id(struct cpu_info *ci) 562 { 563 struct aarch64_sysctl_cpu_id *id = &ci->ci_id; 564 565 /* SCTLR - System Control Register */ 566 ci->ci_sctlr_el1 = reg_sctlr_el1_read(); 567 568 memset(id, 0, sizeof *id); 569 570 id->ac_midr = reg_midr_el1_read(); 571 id->ac_revidr = reg_revidr_el1_read(); 572 id->ac_mpidr = reg_mpidr_el1_read(); 573 574 id->ac_aa64dfr0 = reg_id_aa64dfr0_el1_read(); 575 id->ac_aa64dfr1 = reg_id_aa64dfr1_el1_read(); 576 577 id->ac_aa64isar0 = reg_id_aa64isar0_el1_read(); 578 id->ac_aa64isar1 = reg_id_aa64isar1_el1_read(); 579 580 id->ac_aa64mmfr0 = reg_id_aa64mmfr0_el1_read(); 581 id->ac_aa64mmfr1 = reg_id_aa64mmfr1_el1_read(); 582 id->ac_aa64mmfr2 = reg_id_aa64mmfr2_el1_read(); 583 584 id->ac_mvfr0 = reg_mvfr0_el1_read(); 585 id->ac_mvfr1 = reg_mvfr1_el1_read(); 586 id->ac_mvfr2 = reg_mvfr2_el1_read(); 587 588 id->ac_clidr = reg_clidr_el1_read(); 589 id->ac_ctr = reg_ctr_el0_read(); 590 591 /* Only in ARMv8.2. */ 592 id->ac_aa64zfr0 = 0 /* reg_id_aa64zfr0_el1_read() */; 593 594 id->ac_aa64pfr0 = reg_id_aa64pfr0_el1_read(); 595 id->ac_aa64pfr1 = reg_id_aa64pfr1_el1_read(); 596 } 597 598 /* 599 * setup the per-cpu sysctl tree. 600 */ 601 static void 602 cpu_setup_sysctl(device_t dv, struct cpu_info *ci) 603 { 604 const struct sysctlnode *cpunode = NULL; 605 606 sysctl_createv(NULL, 0, NULL, &cpunode, 607 CTLFLAG_PERMANENT, 608 CTLTYPE_NODE, device_xname(dv), NULL, 609 NULL, 0, NULL, 0, 610 CTL_MACHDEP, 611 CTL_CREATE, CTL_EOL); 612 613 if (cpunode == NULL) 614 return; 615 616 sysctl_createv(NULL, 0, &cpunode, NULL, 617 CTLFLAG_PERMANENT, 618 CTLTYPE_STRUCT, "cpu_id", NULL, 619 NULL, 0, &ci->ci_id, sizeof(ci->ci_id), 620 CTL_CREATE, CTL_EOL); 621 } 622 623 static struct krndsource rndrrs_source; 624 625 static void 626 rndrrs_get(size_t nbytes, void *cookie) 627 { 628 /* Entropy bits per data byte, wild-arse guess. */ 629 const unsigned bpb = 4; 630 size_t nbits = nbytes*NBBY; 631 uint64_t x; 632 int error, bound; 633 634 bound = curlwp_bind(); /* bind to CPU for rndrrs_fail evcnt */ 635 while (nbits) { 636 /* 637 * x := random 64-bit sample 638 * error := Z bit, set to 1 if sample is bad 639 * 640 * XXX This should be done by marking the function 641 * __attribute__((target("arch=armv8.5-a+rng"))) and 642 * using `mrs %0, rndrrs', but: 643 * 644 * (a) the version of gcc we use doesn't support that, 645 * and 646 * (b) clang doesn't seem to like `rndrrs' itself. 647 * 648 * So we use the numeric encoding for now. 649 */ 650 __asm __volatile("" 651 "mrs %0, s3_3_c2_c4_1\n" 652 "cset %w1, eq" 653 : "=r"(x), "=r"(error)); 654 if (error) { 655 DTRACE_PROBE(rndrrs_fail); 656 curcpu()->ci_rndrrs_fail.ev_count++; 657 break; 658 } 659 rnd_add_data_sync(&rndrrs_source, &x, sizeof(x), 660 bpb*sizeof(x)); 661 nbits -= MIN(nbits, bpb*sizeof(x)); 662 } 663 curlwp_bindx(bound); 664 665 explicit_memset(&x, 0, sizeof x); 666 } 667 668 /* 669 * setup the RNDRRS entropy source 670 */ 671 static void 672 cpu_setup_rng(device_t dv, struct cpu_info *ci) 673 { 674 struct aarch64_sysctl_cpu_id *id = &ci->ci_id; 675 676 /* Verify that it is supported. */ 677 switch (__SHIFTOUT(id->ac_aa64isar0, ID_AA64ISAR0_EL1_RNDR)) { 678 case ID_AA64ISAR0_EL1_RNDR_RNDRRS: 679 break; 680 default: 681 return; 682 } 683 684 /* Attach event counter for RNDRRS failure. */ 685 evcnt_attach_dynamic(&ci->ci_rndrrs_fail, EVCNT_TYPE_MISC, NULL, 686 ci->ci_cpuname, "rndrrs fail"); 687 688 /* 689 * On the primary CPU, attach random source -- this only 690 * happens once globally. 691 */ 692 if (!CPU_IS_PRIMARY(ci)) 693 return; 694 rndsource_setcb(&rndrrs_source, rndrrs_get, NULL); 695 rnd_attach_source(&rndrrs_source, "rndrrs", RND_TYPE_RNG, 696 RND_FLAG_DEFAULT|RND_FLAG_HASCB); 697 } 698 699 /* 700 * setup the AES implementation 701 */ 702 static void 703 cpu_setup_aes(device_t dv, struct cpu_info *ci) 704 { 705 struct aarch64_sysctl_cpu_id *id = &ci->ci_id; 706 707 /* Check for ARMv8.0-AES support. */ 708 switch (__SHIFTOUT(id->ac_aa64isar0, ID_AA64ISAR0_EL1_AES)) { 709 case ID_AA64ISAR0_EL1_AES_AES: 710 case ID_AA64ISAR0_EL1_AES_PMUL: 711 aes_md_init(&aes_armv8_impl); 712 return; 713 default: 714 break; 715 } 716 717 /* Failing that, check for SIMD support. */ 718 switch (__SHIFTOUT(id->ac_aa64pfr0, ID_AA64PFR0_EL1_ADVSIMD)) { 719 case ID_AA64PFR0_EL1_ADV_SIMD_IMPL: 720 aes_md_init(&aes_neon_impl); 721 return; 722 default: 723 break; 724 } 725 } 726 727 /* 728 * setup the ChaCha implementation 729 */ 730 static void 731 cpu_setup_chacha(device_t dv, struct cpu_info *ci) 732 { 733 struct aarch64_sysctl_cpu_id *id = &ci->ci_id; 734 735 /* Check for SIMD support. */ 736 switch (__SHIFTOUT(id->ac_aa64pfr0, ID_AA64PFR0_EL1_ADVSIMD)) { 737 case ID_AA64PFR0_EL1_ADV_SIMD_IMPL: 738 chacha_md_init(&chacha_neon_impl); 739 return; 740 default: 741 break; 742 } 743 } 744 745 #ifdef MULTIPROCESSOR 746 /* 747 * Initialise a secondary processor. 748 * 749 * printf isn't available as kmutex(9) relies on curcpu which isn't setup yet. 750 * 751 */ 752 void __noasan 753 cpu_init_secondary_processor(int cpuindex) 754 { 755 struct cpu_info * ci = &cpu_info_store[cpuindex]; 756 struct aarch64_sysctl_cpu_id *id = &ci->ci_id; 757 758 aarch64_setcpufuncs(ci); 759 760 /* Sets ci->ci_{sctlr,midr,mpidr}, etc */ 761 cpu_setup_id(ci); 762 763 arm_cpu_topology_set(ci, id->ac_mpidr); 764 aarch64_getcacheinfo(ci); 765 766 cpu_set_hatched(cpuindex); 767 768 /* 769 * return to assembly to wait for cpu_boot_secondary_processors 770 */ 771 } 772 773 774 /* 775 * When we are called, the MMU and caches are on and we are running on the stack 776 * of the idlelwp for this cpu. 777 */ 778 void 779 cpu_hatch(struct cpu_info *ci) 780 { 781 KASSERT(curcpu() == ci); 782 KASSERT((reg_tcr_el1_read() & TCR_EPD0) != 0); 783 784 #ifdef DDB 785 db_machdep_cpu_init(); 786 #endif 787 788 cpu_init_counter(ci); 789 790 intr_cpu_init(ci); 791 792 #ifdef FDT 793 arm_fdt_cpu_hatch(ci); 794 #endif 795 796 /* 797 * clear my bit of arm_cpu_mbox to tell cpu_boot_secondary_processors(). 798 * there are cpu0,1,2,3, and if cpu2 is unresponsive, 799 * ci_index are each cpu0=0, cpu1=1, cpu2=undef, cpu3=2. 800 * therefore we have to use device_unit instead of ci_index for mbox. 801 */ 802 803 cpu_clr_mbox(device_unit(ci->ci_dev)); 804 } 805 #endif /* MULTIPROCESSOR */ 806
Indexes created Sun Oct 25 00:26:45 UTC 2026