1 /* $NetBSD: gicv3.c,v 1.56 2025/09/04 02:17:44 rin Exp $ */ 2 3 /*- 4 * Copyright (c) 2018 Jared McNeill <jmcneill (at) invisible.ca> 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 17 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 18 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 19 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 20 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 21 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 22 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 23 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 24 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 * SUCH DAMAGE. 27 */ 28 29 #include "opt_multiprocessor.h" 30 31 #define _INTR_PRIVATE 32 33 #include <sys/cdefs.h> 34 __KERNEL_RCSID(0, "$NetBSD: gicv3.c,v 1.56 2025/09/04 02:17:44 rin Exp $"); 35 36 #include <sys/param.h> 37 #include <sys/kernel.h> 38 #include <sys/bus.h> 39 #include <sys/device.h> 40 #include <sys/intr.h> 41 #include <sys/systm.h> 42 #include <sys/cpu.h> 43 #include <sys/vmem.h> 44 #include <sys/kmem.h> 45 #include <sys/atomic.h> 46 47 #include <machine/cpufunc.h> 48 49 #include <arm/locore.h> 50 #include <arm/armreg.h> 51 52 #include <arm/cortex/gicv3.h> 53 #include <arm/cortex/gic_reg.h> 54 55 #define PICTOSOFTC(pic) \ 56 container_of(pic, struct gicv3_softc, sc_pic) 57 #define LPITOSOFTC(lpi) \ 58 container_of(lpi, struct gicv3_softc, sc_lpi) 59 60 #define IPL_TO_PRIORITY(sc, ipl) (((0xffU - (ipl)) << (sc)->sc_priority_shift) & 0xffU) 61 #define IPL_TO_PMR(sc, ipl) (((0xffU - (ipl)) << (sc)->sc_pmr_shift) & 0xffU) 62 #define IPL_TO_HWPL(ipl) ((ipl) >= IPL_VM ? (ipl) : IPL_NONE) 63 64 #define GIC_SUPPORTS_1OFN(sc) (((sc)->sc_gicd_typer & GICD_TYPER_No1N) == 0) 65 66 #define GIC_PRIO_SHIFT_NS 4 67 #define GIC_PRIO_SHIFT_S 3 68 69 /* 70 * Set to true if you want to use 1 of N interrupt distribution for SPIs 71 * when available. Disabled by default because it causes issues with the 72 * USB stack. 73 */ 74 bool gicv3_use_1ofn = false; 75 76 static struct gicv3_softc *gicv3_softc; 77 78 static inline uint32_t 79 gicd_read_4(struct gicv3_softc *sc, bus_size_t reg) 80 { 81 return bus_space_read_4(sc->sc_bst, sc->sc_bsh_d, reg); 82 } 83 84 static inline void 85 gicd_write_4(struct gicv3_softc *sc, bus_size_t reg, uint32_t val) 86 { 87 bus_space_write_4(sc->sc_bst, sc->sc_bsh_d, reg, val); 88 } 89 90 #ifdef MULTIPROCESSOR 91 static inline uint64_t 92 gicd_read_8(struct gicv3_softc *sc, bus_size_t reg) 93 { 94 return bus_space_read_8(sc->sc_bst, sc->sc_bsh_d, reg); 95 } 96 #endif 97 98 static inline void 99 gicd_write_8(struct gicv3_softc *sc, bus_size_t reg, uint64_t val) 100 { 101 bus_space_write_8(sc->sc_bst, sc->sc_bsh_d, reg, val); 102 } 103 104 static inline uint32_t 105 gicr_read_4(struct gicv3_softc *sc, u_int index, bus_size_t reg) 106 { 107 KASSERT(index < sc->sc_bsh_r_count); 108 return bus_space_read_4(sc->sc_bst, sc->sc_bsh_r[index], reg); 109 } 110 111 static inline void 112 gicr_write_4(struct gicv3_softc *sc, u_int index, bus_size_t reg, uint32_t val) 113 { 114 KASSERT(index < sc->sc_bsh_r_count); 115 bus_space_write_4(sc->sc_bst, sc->sc_bsh_r[index], reg, val); 116 } 117 118 static inline uint64_t 119 gicr_read_8(struct gicv3_softc *sc, u_int index, bus_size_t reg) 120 { 121 KASSERT(index < sc->sc_bsh_r_count); 122 return bus_space_read_8(sc->sc_bst, sc->sc_bsh_r[index], reg); 123 } 124 125 static inline void 126 gicr_write_8(struct gicv3_softc *sc, u_int index, bus_size_t reg, uint64_t val) 127 { 128 KASSERT(index < sc->sc_bsh_r_count); 129 bus_space_write_8(sc->sc_bst, sc->sc_bsh_r[index], reg, val); 130 } 131 132 static void 133 gicv3_unblock_irqs(struct pic_softc *pic, size_t irqbase, uint32_t mask) 134 { 135 struct gicv3_softc * const sc = PICTOSOFTC(pic); 136 struct cpu_info * const ci = curcpu(); 137 const u_int group = irqbase / 32; 138 139 if (group == 0) { 140 atomic_or_32(&sc->sc_enabled_sgippi, mask); 141 gicr_write_4(sc, ci->ci_gic_redist, GICR_ISENABLER0, mask); 142 while (gicr_read_4(sc, ci->ci_gic_redist, GICR_CTLR) & GICR_CTLR_RWP) 143 ; 144 } else { 145 gicd_write_4(sc, GICD_ISENABLERn(group), mask); 146 while (gicd_read_4(sc, GICD_CTRL) & GICD_CTRL_RWP) 147 ; 148 } 149 } 150 151 static void 152 gicv3_block_irqs(struct pic_softc *pic, size_t irqbase, uint32_t mask) 153 { 154 struct gicv3_softc * const sc = PICTOSOFTC(pic); 155 struct cpu_info * const ci = curcpu(); 156 const u_int group = irqbase / 32; 157 158 if (group == 0) { 159 atomic_and_32(&sc->sc_enabled_sgippi, ~mask); 160 gicr_write_4(sc, ci->ci_gic_redist, GICR_ICENABLER0, mask); 161 while (gicr_read_4(sc, ci->ci_gic_redist, GICR_CTLR) & GICR_CTLR_RWP) 162 ; 163 } else { 164 gicd_write_4(sc, GICD_ICENABLERn(group), mask); 165 while (gicd_read_4(sc, GICD_CTRL) & GICD_CTRL_RWP) 166 ; 167 } 168 } 169 170 static void 171 gicv3_establish_irq(struct pic_softc *pic, struct intrsource *is) 172 { 173 struct gicv3_softc * const sc = PICTOSOFTC(pic); 174 const u_int group = is->is_irq / 32; 175 uint32_t ipriority, icfg; 176 uint64_t irouter; 177 u_int n; 178 179 const u_int ipriority_val = IPL_TO_PRIORITY(sc, is->is_ipl); 180 const u_int ipriority_shift = (is->is_irq & 0x3) * 8; 181 const u_int icfg_shift = (is->is_irq & 0xf) * 2; 182 183 if (group == 0) { 184 /* SGIs and PPIs are per-CPU and always MP-safe */ 185 is->is_mpsafe = true; 186 is->is_percpu = true; 187 188 /* Update interrupt configuration and priority on all redistributors */ 189 for (n = 0; n < sc->sc_bsh_r_count; n++) { 190 icfg = gicr_read_4(sc, n, GICR_ICFGRn(is->is_irq / 16)); 191 if (is->is_type == IST_LEVEL) 192 icfg &= ~(0x2U << icfg_shift); 193 if (is->is_type == IST_EDGE) 194 icfg |= (0x2U << icfg_shift); 195 gicr_write_4(sc, n, GICR_ICFGRn(is->is_irq / 16), icfg); 196 197 ipriority = gicr_read_4(sc, n, GICR_IPRIORITYRn(is->is_irq / 4)); 198 ipriority &= ~(0xffU << ipriority_shift); 199 ipriority |= (ipriority_val << ipriority_shift); 200 gicr_write_4(sc, n, GICR_IPRIORITYRn(is->is_irq / 4), ipriority); 201 } 202 } else { 203 /* 204 * If 1 of N SPI routing is supported, route MP-safe interrupts to all 205 * participating PEs. Otherwise, just route to the primary PE. 206 */ 207 if (is->is_mpsafe && GIC_SUPPORTS_1OFN(sc) && gicv3_use_1ofn) { 208 irouter = GICD_IROUTER_Interrupt_Routing_mode; 209 } else { 210 irouter = sc->sc_irouter[0]; 211 } 212 gicd_write_8(sc, GICD_IROUTER(is->is_irq), irouter); 213 214 /* Update interrupt configuration */ 215 icfg = gicd_read_4(sc, GICD_ICFGRn(is->is_irq / 16)); 216 if (is->is_type == IST_LEVEL) 217 icfg &= ~(0x2U << icfg_shift); 218 if (is->is_type == IST_EDGE) 219 icfg |= (0x2U << icfg_shift); 220 gicd_write_4(sc, GICD_ICFGRn(is->is_irq / 16), icfg); 221 222 /* Update interrupt priority */ 223 ipriority = gicd_read_4(sc, GICD_IPRIORITYRn(is->is_irq / 4)); 224 ipriority &= ~(0xffU << ipriority_shift); 225 ipriority |= (ipriority_val << ipriority_shift); 226 gicd_write_4(sc, GICD_IPRIORITYRn(is->is_irq / 4), ipriority); 227 } 228 } 229 230 static void 231 gicv3_set_priority(struct pic_softc *pic, int ipl) 232 { 233 struct gicv3_softc * const sc = PICTOSOFTC(pic); 234 struct cpu_info * const ci = curcpu(); 235 const int hwpl = IPL_TO_HWPL(ipl); 236 237 while (hwpl < ci->ci_hwpl) { 238 /* Lowering priority mask */ 239 ci->ci_hwpl = hwpl; 240 __insn_barrier(); 241 icc_pmr_write(IPL_TO_PMR(sc, hwpl)); 242 } 243 __insn_barrier(); 244 ci->ci_cpl = ipl; 245 } 246 247 static void 248 gicv3_dist_enable(struct gicv3_softc *sc) 249 { 250 uint32_t gicd_ctrl; 251 u_int n; 252 253 /* Disable the distributor */ 254 gicd_ctrl = gicd_read_4(sc, GICD_CTRL); 255 gicd_ctrl &= ~(GICD_CTRL_EnableGrp1A | GICD_CTRL_ARE_NS); 256 gicd_write_4(sc, GICD_CTRL, gicd_ctrl); 257 258 /* Wait for register write to complete */ 259 while (gicd_read_4(sc, GICD_CTRL) & GICD_CTRL_RWP) 260 ; 261 262 /* Clear all INTID enable bits */ 263 for (n = 32; n < sc->sc_pic.pic_maxsources; n += 32) 264 gicd_write_4(sc, GICD_ICENABLERn(n / 32), ~0); 265 266 /* Set default priorities to lowest */ 267 for (n = 32; n < sc->sc_pic.pic_maxsources; n += 4) 268 gicd_write_4(sc, GICD_IPRIORITYRn(n / 4), ~0); 269 270 /* Set all interrupts to G1NS */ 271 for (n = 32; n < sc->sc_pic.pic_maxsources; n += 32) { 272 gicd_write_4(sc, GICD_IGROUPRn(n / 32), ~0); 273 gicd_write_4(sc, GICD_IGRPMODRn(n / 32), 0); 274 } 275 276 /* Set all interrupts level-sensitive by default */ 277 for (n = 32; n < sc->sc_pic.pic_maxsources; n += 16) 278 gicd_write_4(sc, GICD_ICFGRn(n / 16), 0); 279 280 /* Wait for register writes to complete */ 281 while (gicd_read_4(sc, GICD_CTRL) & GICD_CTRL_RWP) 282 ; 283 284 /* Enable Affinity routing and G1NS interrupts */ 285 gicd_ctrl = GICD_CTRL_EnableGrp1A | GICD_CTRL_ARE_NS; 286 gicd_write_4(sc, GICD_CTRL, gicd_ctrl); 287 } 288 289 static void 290 gicv3_redist_enable(struct gicv3_softc *sc, struct cpu_info *ci) 291 { 292 uint32_t icfg; 293 u_int n, o; 294 295 /* Clear INTID enable bits */ 296 gicr_write_4(sc, ci->ci_gic_redist, GICR_ICENABLER0, ~0); 297 298 /* Wait for register write to complete */ 299 while (gicr_read_4(sc, ci->ci_gic_redist, GICR_CTLR) & GICR_CTLR_RWP) 300 ; 301 302 /* Set default priorities */ 303 for (n = 0; n < 32; n += 4) { 304 uint32_t priority = 0; 305 size_t byte_shift = 0; 306 for (o = 0; o < 4; o++, byte_shift += 8) { 307 struct intrsource * const is = sc->sc_pic.pic_sources[n + o]; 308 if (is == NULL) 309 priority |= (0xffU << byte_shift); 310 else { 311 const u_int ipriority_val = IPL_TO_PRIORITY(sc, is->is_ipl); 312 priority |= ipriority_val << byte_shift; 313 } 314 } 315 gicr_write_4(sc, ci->ci_gic_redist, GICR_IPRIORITYRn(n / 4), priority); 316 } 317 318 /* Set all interrupts to G1NS */ 319 gicr_write_4(sc, ci->ci_gic_redist, GICR_IGROUPR0, ~0); 320 gicr_write_4(sc, ci->ci_gic_redist, GICR_IGRPMODR0, 0); 321 322 /* Restore PPI configs */ 323 for (n = 0, icfg = 0; n < 16; n++) { 324 struct intrsource * const is = sc->sc_pic.pic_sources[16 + n]; 325 if (is != NULL && is->is_type == IST_EDGE) 326 icfg |= (0x2U << (n * 2)); 327 } 328 gicr_write_4(sc, ci->ci_gic_redist, GICR_ICFGRn(1), icfg); 329 330 /* Restore current enable bits */ 331 gicr_write_4(sc, ci->ci_gic_redist, GICR_ISENABLER0, sc->sc_enabled_sgippi); 332 333 /* Wait for register write to complete */ 334 while (gicr_read_4(sc, ci->ci_gic_redist, GICR_CTLR) & GICR_CTLR_RWP) 335 ; 336 } 337 338 static uint64_t 339 gicv3_cpu_identity(void) 340 { 341 u_int aff3, aff2, aff1, aff0; 342 343 const register_t mpidr = cpu_mpidr_aff_read(); 344 aff0 = __SHIFTOUT(mpidr, MPIDR_AFF0); 345 aff1 = __SHIFTOUT(mpidr, MPIDR_AFF1); 346 aff2 = __SHIFTOUT(mpidr, MPIDR_AFF2); 347 aff3 = __SHIFTOUT(mpidr, MPIDR_AFF3); 348 349 return __SHIFTIN(aff0, GICR_TYPER_Affinity_Value_Aff0) | 350 __SHIFTIN(aff1, GICR_TYPER_Affinity_Value_Aff1) | 351 __SHIFTIN(aff2, GICR_TYPER_Affinity_Value_Aff2) | 352 __SHIFTIN(aff3, GICR_TYPER_Affinity_Value_Aff3); 353 } 354 355 static u_int 356 gicv3_find_redist(struct gicv3_softc *sc) 357 { 358 uint64_t gicr_typer; 359 u_int n; 360 361 const uint64_t cpu_identity = gicv3_cpu_identity(); 362 363 for (n = 0; n < sc->sc_bsh_r_count; n++) { 364 gicr_typer = gicr_read_8(sc, n, GICR_TYPER); 365 if ((gicr_typer & GICR_TYPER_Affinity_Value) == cpu_identity) 366 return n; 367 } 368 369 const u_int aff0 = __SHIFTOUT(cpu_identity, GICR_TYPER_Affinity_Value_Aff0); 370 const u_int aff1 = __SHIFTOUT(cpu_identity, GICR_TYPER_Affinity_Value_Aff1); 371 const u_int aff2 = __SHIFTOUT(cpu_identity, GICR_TYPER_Affinity_Value_Aff2); 372 const u_int aff3 = __SHIFTOUT(cpu_identity, GICR_TYPER_Affinity_Value_Aff3); 373 374 panic("%s: could not find GICv3 redistributor for cpu %d.%d.%d.%d", 375 cpu_name(curcpu()), aff3, aff2, aff1, aff0); 376 } 377 378 static uint64_t 379 gicv3_sgir(struct gicv3_softc *sc) 380 { 381 const uint64_t cpu_identity = gicv3_cpu_identity(); 382 383 const u_int aff0 = __SHIFTOUT(cpu_identity, GICR_TYPER_Affinity_Value_Aff0); 384 const u_int aff1 = __SHIFTOUT(cpu_identity, GICR_TYPER_Affinity_Value_Aff1); 385 const u_int aff2 = __SHIFTOUT(cpu_identity, GICR_TYPER_Affinity_Value_Aff2); 386 const u_int aff3 = __SHIFTOUT(cpu_identity, GICR_TYPER_Affinity_Value_Aff3); 387 388 return __SHIFTIN(__BIT(aff0), ICC_SGIR_EL1_TargetList) | 389 __SHIFTIN(aff1, ICC_SGIR_EL1_Aff1) | 390 __SHIFTIN(aff2, ICC_SGIR_EL1_Aff2) | 391 __SHIFTIN(aff3, ICC_SGIR_EL1_Aff3); 392 } 393 394 static void 395 gicv3_cpu_init(struct pic_softc *pic, struct cpu_info *ci) 396 { 397 struct gicv3_softc * const sc = PICTOSOFTC(pic); 398 uint32_t icc_sre, icc_ctlr, gicr_waker; 399 400 evcnt_attach_dynamic(&ci->ci_intr_preempt, EVCNT_TYPE_MISC, NULL, 401 ci->ci_cpuname, "intr preempt"); 402 403 ci->ci_gic_redist = gicv3_find_redist(sc); 404 ci->ci_gic_sgir = gicv3_sgir(sc); 405 406 /* Enable System register access and disable IRQ/FIQ bypass */ 407 icc_sre = ICC_SRE_EL1_SRE | ICC_SRE_EL1_DFB | ICC_SRE_EL1_DIB; 408 icc_sre_write(icc_sre); 409 410 /* Mark the connected PE as being awake */ 411 gicr_waker = gicr_read_4(sc, ci->ci_gic_redist, GICR_WAKER); 412 gicr_waker &= ~GICR_WAKER_ProcessorSleep; 413 gicr_write_4(sc, ci->ci_gic_redist, GICR_WAKER, gicr_waker); 414 while (gicr_read_4(sc, ci->ci_gic_redist, GICR_WAKER) & GICR_WAKER_ChildrenAsleep) 415 ; 416 417 /* Set initial priority mask */ 418 ci->ci_hwpl = IPL_HIGH; 419 icc_pmr_write(IPL_TO_PMR(sc, IPL_HIGH)); 420 421 /* Set the binary point field to the minimum value */ 422 icc_bpr1_write(0); 423 424 /* Enable group 1 interrupt signaling */ 425 icc_igrpen1_write(ICC_IGRPEN_EL1_Enable); 426 427 /* Set EOI mode */ 428 icc_ctlr = icc_ctlr_read(); 429 icc_ctlr &= ~ICC_CTLR_EL1_EOImode; 430 icc_ctlr_write(icc_ctlr); 431 432 /* Enable redistributor */ 433 gicv3_redist_enable(sc, ci); 434 435 /* Allow IRQ exceptions */ 436 ENABLE_INTERRUPT(); 437 } 438 439 #ifdef MULTIPROCESSOR 440 static void 441 gicv3_ipi_send(struct pic_softc *pic, const kcpuset_t *kcp, u_long ipi) 442 { 443 struct cpu_info *ci; 444 uint64_t sgir; 445 446 sgir = __SHIFTIN(ipi, ICC_SGIR_EL1_INTID); 447 if (kcp == NULL) { 448 /* Interrupts routed to all PEs, excluding "self" */ 449 if (ncpu == 1) 450 return; 451 sgir |= ICC_SGIR_EL1_IRM; 452 } else { 453 /* Interrupt to exactly one PE */ 454 ci = cpu_lookup(kcpuset_ffs(kcp) - 1); 455 if (ci == curcpu()) 456 return; 457 sgir |= ci->ci_gic_sgir; 458 } 459 icc_sgi1r_write(sgir); 460 isb(); 461 } 462 463 static void 464 gicv3_get_affinity(struct pic_softc *pic, size_t irq, kcpuset_t *affinity) 465 { 466 struct gicv3_softc * const sc = PICTOSOFTC(pic); 467 const size_t group = irq / 32; 468 int n; 469 470 kcpuset_zero(affinity); 471 if (group == 0) { 472 /* All CPUs are targets for group 0 (SGI/PPI) */ 473 for (n = 0; n < ncpu; n++) { 474 kcpuset_set(affinity, n); 475 } 476 } else { 477 /* Find distributor targets (SPI) */ 478 const uint64_t irouter = gicd_read_8(sc, GICD_IROUTER(irq)); 479 for (n = 0; n < ncpu; n++) { 480 if (irouter == GICD_IROUTER_Interrupt_Routing_mode || 481 irouter == sc->sc_irouter[n]) 482 kcpuset_set(affinity, n); 483 } 484 } 485 } 486 487 static int 488 gicv3_set_affinity(struct pic_softc *pic, size_t irq, const kcpuset_t *affinity) 489 { 490 struct gicv3_softc * const sc = PICTOSOFTC(pic); 491 const size_t group = irq / 32; 492 uint64_t irouter; 493 494 if (group == 0) 495 return EINVAL; 496 497 const int set = kcpuset_countset(affinity); 498 if (set == 1) { 499 irouter = sc->sc_irouter[kcpuset_ffs(affinity) - 1]; 500 } else if (set == ncpu && GIC_SUPPORTS_1OFN(sc) && gicv3_use_1ofn) { 501 irouter = GICD_IROUTER_Interrupt_Routing_mode; 502 } else { 503 return EINVAL; 504 } 505 506 gicd_write_8(sc, GICD_IROUTER(irq), irouter); 507 508 return 0; 509 } 510 #endif 511 512 static const struct pic_ops gicv3_picops = { 513 .pic_unblock_irqs = gicv3_unblock_irqs, 514 .pic_block_irqs = gicv3_block_irqs, 515 .pic_establish_irq = gicv3_establish_irq, 516 .pic_set_priority = gicv3_set_priority, 517 #ifdef MULTIPROCESSOR 518 .pic_cpu_init = gicv3_cpu_init, 519 .pic_ipi_send = gicv3_ipi_send, 520 .pic_get_affinity = gicv3_get_affinity, 521 .pic_set_affinity = gicv3_set_affinity, 522 #endif 523 }; 524 525 static void 526 gicv3_dcache_wb_range(vaddr_t va, vsize_t len) 527 { 528 cpu_dcache_wb_range(va, len); 529 dsb(sy); 530 } 531 532 static void 533 gicv3_lpi_unblock_irqs(struct pic_softc *pic, size_t irqbase, uint32_t mask) 534 { 535 struct gicv3_softc * const sc = LPITOSOFTC(pic); 536 int bit; 537 538 while ((bit = ffs(mask)) != 0) { 539 sc->sc_lpiconf.base[irqbase + bit - 1] |= GIC_LPICONF_Enable; 540 if (sc->sc_lpiconf_flush) 541 gicv3_dcache_wb_range((vaddr_t)&sc->sc_lpiconf.base[irqbase + bit - 1], 1); 542 mask &= ~__BIT(bit - 1); 543 } 544 545 if (!sc->sc_lpiconf_flush) 546 dsb(ishst); 547 } 548 549 static void 550 gicv3_lpi_block_irqs(struct pic_softc *pic, size_t irqbase, uint32_t mask) 551 { 552 struct gicv3_softc * const sc = LPITOSOFTC(pic); 553 int bit; 554 555 while ((bit = ffs(mask)) != 0) { 556 sc->sc_lpiconf.base[irqbase + bit - 1] &= ~GIC_LPICONF_Enable; 557 if (sc->sc_lpiconf_flush) 558 gicv3_dcache_wb_range((vaddr_t)&sc->sc_lpiconf.base[irqbase + bit - 1], 1); 559 mask &= ~__BIT(bit - 1); 560 } 561 562 if (!sc->sc_lpiconf_flush) 563 dsb(ishst); 564 } 565 566 static void 567 gicv3_lpi_establish_irq(struct pic_softc *pic, struct intrsource *is) 568 { 569 struct gicv3_softc * const sc = LPITOSOFTC(pic); 570 571 sc->sc_lpiconf.base[is->is_irq] = IPL_TO_PRIORITY(sc, is->is_ipl) | GIC_LPICONF_Res1; 572 573 if (sc->sc_lpiconf_flush) 574 gicv3_dcache_wb_range((vaddr_t)&sc->sc_lpiconf.base[is->is_irq], 1); 575 else 576 dsb(ishst); 577 } 578 579 static void 580 gicv3_lpi_cpu_init(struct pic_softc *pic, struct cpu_info *ci) 581 { 582 struct gicv3_softc * const sc = LPITOSOFTC(pic); 583 struct gicv3_lpi_callback *cb; 584 uint64_t propbase, pendbase; 585 uint32_t ctlr; 586 587 /* If physical LPIs are not supported on this redistributor, just return. */ 588 const uint64_t typer = gicr_read_8(sc, ci->ci_gic_redist, GICR_TYPER); 589 if ((typer & GICR_TYPER_PLPIS) == 0) 590 return; 591 592 /* Interrupt target address for this CPU, used by ITS when GITS_TYPER.PTA == 0 */ 593 sc->sc_processor_id[cpu_index(ci)] = __SHIFTOUT(typer, GICR_TYPER_Processor_Number); 594 595 /* Disable LPIs before making changes */ 596 ctlr = gicr_read_4(sc, ci->ci_gic_redist, GICR_CTLR); 597 ctlr &= ~GICR_CTLR_Enable_LPIs; 598 gicr_write_4(sc, ci->ci_gic_redist, GICR_CTLR, ctlr); 599 dsb(sy); 600 601 /* Setup the LPI configuration table */ 602 propbase = sc->sc_lpiconf.segs[0].ds_addr | 603 __SHIFTIN(ffs(pic->pic_maxsources) - 1, GICR_PROPBASER_IDbits) | 604 __SHIFTIN(GICR_Shareability_IS, GICR_PROPBASER_Shareability) | 605 __SHIFTIN(GICR_Cache_NORMAL_RA_WA_WB, GICR_PROPBASER_InnerCache); 606 gicr_write_8(sc, ci->ci_gic_redist, GICR_PROPBASER, propbase); 607 propbase = gicr_read_8(sc, ci->ci_gic_redist, GICR_PROPBASER); 608 if (__SHIFTOUT(propbase, GICR_PROPBASER_Shareability) != GICR_Shareability_IS) { 609 if (__SHIFTOUT(propbase, GICR_PROPBASER_Shareability) == GICR_Shareability_NS) { 610 propbase &= ~GICR_PROPBASER_Shareability; 611 propbase |= __SHIFTIN(GICR_Shareability_NS, GICR_PROPBASER_Shareability); 612 propbase &= ~GICR_PROPBASER_InnerCache; 613 propbase |= __SHIFTIN(GICR_Cache_NORMAL_NC, GICR_PROPBASER_InnerCache); 614 gicr_write_8(sc, ci->ci_gic_redist, GICR_PROPBASER, propbase); 615 } 616 sc->sc_lpiconf_flush = true; 617 } 618 619 /* Setup the LPI pending table */ 620 pendbase = sc->sc_lpipend[cpu_index(ci)].segs[0].ds_addr | 621 __SHIFTIN(GICR_Shareability_IS, GICR_PENDBASER_Shareability) | 622 __SHIFTIN(GICR_Cache_NORMAL_RA_WA_WB, GICR_PENDBASER_InnerCache); 623 gicr_write_8(sc, ci->ci_gic_redist, GICR_PENDBASER, pendbase); 624 pendbase = gicr_read_8(sc, ci->ci_gic_redist, GICR_PENDBASER); 625 if (__SHIFTOUT(pendbase, GICR_PENDBASER_Shareability) == GICR_Shareability_NS) { 626 pendbase &= ~GICR_PENDBASER_Shareability; 627 pendbase |= __SHIFTIN(GICR_Shareability_NS, GICR_PENDBASER_Shareability); 628 pendbase &= ~GICR_PENDBASER_InnerCache; 629 pendbase |= __SHIFTIN(GICR_Cache_NORMAL_NC, GICR_PENDBASER_InnerCache); 630 gicr_write_8(sc, ci->ci_gic_redist, GICR_PENDBASER, pendbase); 631 } 632 633 /* Enable LPIs */ 634 ctlr = gicr_read_4(sc, ci->ci_gic_redist, GICR_CTLR); 635 ctlr |= GICR_CTLR_Enable_LPIs; 636 gicr_write_4(sc, ci->ci_gic_redist, GICR_CTLR, ctlr); 637 dsb(sy); 638 639 /* Setup ITS if present */ 640 LIST_FOREACH(cb, &sc->sc_lpi_callbacks, list) 641 cb->cpu_init(cb->priv, ci); 642 } 643 644 #ifdef MULTIPROCESSOR 645 static void 646 gicv3_lpi_get_affinity(struct pic_softc *pic, size_t irq, kcpuset_t *affinity) 647 { 648 struct gicv3_softc * const sc = LPITOSOFTC(pic); 649 struct gicv3_lpi_callback *cb; 650 651 kcpuset_zero(affinity); 652 LIST_FOREACH(cb, &sc->sc_lpi_callbacks, list) 653 cb->get_affinity(cb->priv, irq, affinity); 654 } 655 656 static int 657 gicv3_lpi_set_affinity(struct pic_softc *pic, size_t irq, const kcpuset_t *affinity) 658 { 659 struct gicv3_softc * const sc = LPITOSOFTC(pic); 660 struct gicv3_lpi_callback *cb; 661 int error = EINVAL; 662 663 LIST_FOREACH(cb, &sc->sc_lpi_callbacks, list) { 664 error = cb->set_affinity(cb->priv, irq, affinity); 665 if (error != EPASSTHROUGH) 666 return error; 667 } 668 669 return EINVAL; 670 } 671 #endif 672 673 static const struct pic_ops gicv3_lpiops = { 674 .pic_unblock_irqs = gicv3_lpi_unblock_irqs, 675 .pic_block_irqs = gicv3_lpi_block_irqs, 676 .pic_establish_irq = gicv3_lpi_establish_irq, 677 #ifdef MULTIPROCESSOR 678 .pic_cpu_init = gicv3_lpi_cpu_init, 679 .pic_get_affinity = gicv3_lpi_get_affinity, 680 .pic_set_affinity = gicv3_lpi_set_affinity, 681 #endif 682 }; 683 684 void 685 gicv3_dma_alloc(struct gicv3_softc *sc, struct gicv3_dma *dma, bus_size_t len, bus_size_t align) 686 { 687 int nsegs, error; 688 689 dma->len = len; 690 error = bus_dmamem_alloc(sc->sc_dmat, dma->len, align, 0, dma->segs, 1, &nsegs, BUS_DMA_WAITOK); 691 if (error) 692 panic("bus_dmamem_alloc failed: %d", error); 693 error = bus_dmamem_map(sc->sc_dmat, dma->segs, nsegs, len, (void **)&dma->base, BUS_DMA_WAITOK); 694 if (error) 695 panic("bus_dmamem_map failed: %d", error); 696 error = bus_dmamap_create(sc->sc_dmat, len, 1, len, 0, BUS_DMA_WAITOK, &dma->map); 697 if (error) 698 panic("bus_dmamap_create failed: %d", error); 699 error = bus_dmamap_load(sc->sc_dmat, dma->map, dma->base, dma->len, NULL, BUS_DMA_WAITOK); 700 if (error) 701 panic("bus_dmamap_load failed: %d", error); 702 703 memset(dma->base, 0, dma->len); 704 bus_dmamap_sync(sc->sc_dmat, dma->map, 0, dma->len, BUS_DMASYNC_PREWRITE); 705 } 706 707 static void 708 gicv3_lpi_init(struct gicv3_softc *sc) 709 { 710 /* 711 * Allocate LPI configuration table 712 */ 713 gicv3_dma_alloc(sc, &sc->sc_lpiconf, sc->sc_lpi.pic_maxsources, 0x1000); 714 KASSERT((sc->sc_lpiconf.segs[0].ds_addr & ~GICR_PROPBASER_Physical_Address) == 0); 715 716 /* 717 * Allocate LPI pending tables 718 */ 719 const bus_size_t lpipend_sz = (8192 + sc->sc_lpi.pic_maxsources) / NBBY; 720 for (int cpuindex = 0; cpuindex < ncpu; cpuindex++) { 721 gicv3_dma_alloc(sc, &sc->sc_lpipend[cpuindex], lpipend_sz, 0x10000); 722 KASSERT((sc->sc_lpipend[cpuindex].segs[0].ds_addr & ~GICR_PENDBASER_Physical_Address) == 0); 723 } 724 } 725 726 void 727 gicv3_irq_handler(void *frame) 728 { 729 struct cpu_info * const ci = curcpu(); 730 struct gicv3_softc * const sc = gicv3_softc; 731 struct pic_softc *pic; 732 const int oldipl = ci->ci_cpl; 733 const int oldhwpl = IPL_TO_HWPL(oldipl); 734 735 ci->ci_data.cpu_nintr++; 736 737 if (ci->ci_hwpl != oldhwpl) { 738 ci->ci_hwpl = oldhwpl; 739 icc_pmr_write(IPL_TO_PMR(sc, oldhwpl)); 740 if (oldhwpl == IPL_HIGH) { 741 return; 742 } 743 } 744 745 for (;;) { 746 const uint32_t iar = icc_iar1_read(); 747 dsb(sy); 748 const uint32_t irq = __SHIFTOUT(iar, ICC_IAR_INTID); 749 if (irq == ICC_IAR_INTID_SPURIOUS) 750 break; 751 752 pic = irq >= GIC_LPI_BASE ? &sc->sc_lpi : &sc->sc_pic; 753 if (irq - pic->pic_irqbase >= pic->pic_maxsources) 754 continue; 755 756 struct intrsource * const is = pic->pic_sources[irq - pic->pic_irqbase]; 757 KASSERT(is != NULL); 758 759 const bool early_eoi = irq < GIC_LPI_BASE && is->is_type == IST_EDGE; 760 761 const int ipl = is->is_ipl; 762 if (__predict_false(ipl < ci->ci_cpl)) { 763 pic_do_pending_ints(I32_bit, ipl, frame); 764 } else if (ci->ci_cpl != ipl) { 765 KASSERT(ipl >= IPL_VM); 766 icc_pmr_write(IPL_TO_PMR(sc, ipl)); 767 ci->ci_hwpl = ci->ci_cpl = ipl; 768 } 769 770 if (early_eoi) { 771 icc_eoi1r_write(iar); 772 isb(); 773 } 774 775 const int64_t nintr = ci->ci_data.cpu_nintr; 776 777 ENABLE_INTERRUPT(); 778 pic_dispatch(is, frame); 779 DISABLE_INTERRUPT(); 780 781 if (nintr != ci->ci_data.cpu_nintr) 782 ci->ci_intr_preempt.ev_count++; 783 784 if (!early_eoi) { 785 icc_eoi1r_write(iar); 786 isb(); 787 } 788 } 789 790 pic_do_pending_ints(I32_bit, oldipl, frame); 791 } 792 793 static bool 794 gicv3_cpuif_is_nonsecure(struct gicv3_softc *sc) 795 { 796 /* 797 * Write 0 to bit7 and see if it sticks. This is only possible if 798 * we have a non-secure view of the PMR register. 799 */ 800 const uint32_t opmr = icc_pmr_read(); 801 icc_pmr_write(0); 802 const uint32_t npmr = icc_pmr_read(); 803 icc_pmr_write(opmr); 804 805 return (npmr & GICC_PMR_NONSECURE) == 0; 806 } 807 808 static bool 809 gicv3_dist_is_nonsecure(struct gicv3_softc *sc) 810 { 811 const uint32_t gicd_ctrl = gicd_read_4(sc, GICD_CTRL); 812 813 /* 814 * If security is enabled, we have a non-secure view of the IPRIORITYRn 815 * registers and LPI configuration priority fields. 816 */ 817 return (gicd_ctrl & GICD_CTRL_DS) == 0; 818 } 819 820 /* 821 * Rockchip RK3399 provides a different view of int priority registers 822 * depending on which firmware is in use. This is hard to detect in 823 * a way that could possibly break other boards, so only do this 824 * detection if we know we are on a RK3399 SoC. 825 */ 826 static void 827 gicv3_quirk_rockchip_rk3399(struct gicv3_softc *sc) 828 { 829 /* Detect the number of supported PMR bits */ 830 icc_pmr_write(0xff); 831 const uint8_t pmrbits = icc_pmr_read(); 832 833 /* Detect the number of supported IPRIORITYRn bits */ 834 const uint32_t oiprio = gicd_read_4(sc, GICD_IPRIORITYRn(8)); 835 gicd_write_4(sc, GICD_IPRIORITYRn(8), oiprio | 0xff); 836 const uint8_t pribits = gicd_read_4(sc, GICD_IPRIORITYRn(8)) & 0xff; 837 gicd_write_4(sc, GICD_IPRIORITYRn(8), oiprio); 838 839 /* 840 * If we see fewer PMR bits than IPRIORITYRn bits here, it means 841 * we have a secure view of IPRIORITYRn (this is not supposed to 842 * happen!). 843 */ 844 if (pmrbits < pribits) { 845 aprint_verbose_dev(sc->sc_dev, 846 "buggy RK3399 firmware detected; applying workaround\n"); 847 sc->sc_priority_shift = GIC_PRIO_SHIFT_S; 848 } 849 } 850 851 int 852 gicv3_init(struct gicv3_softc *sc) 853 { 854 CPU_INFO_ITERATOR cii; 855 struct cpu_info *ci; 856 857 KASSERT(CPU_IS_PRIMARY(curcpu())); 858 859 LIST_INIT(&sc->sc_lpi_callbacks); 860 861 /* Store route to CPU for SPIs */ 862 sc->sc_irouter = kmem_zalloc(sizeof(*sc->sc_irouter) * ncpu, KM_SLEEP); 863 for (CPU_INFO_FOREACH(cii, ci)) { 864 KASSERT(cpu_index(ci) < ncpu); 865 sc->sc_irouter[cpu_index(ci)] = ci->ci_cpuid; 866 } 867 868 sc->sc_gicd_typer = gicd_read_4(sc, GICD_TYPER); 869 870 /* 871 * We don't always have a consistent view of priorities between the 872 * CPU interface (ICC_PMR_EL1) and the GICD/GICR registers. Detect 873 * if we are making secure or non-secure accesses to each, and adjust 874 * the values that we write to each accordingly. 875 */ 876 const bool dist_ns = gicv3_dist_is_nonsecure(sc); 877 sc->sc_priority_shift = dist_ns ? GIC_PRIO_SHIFT_NS : GIC_PRIO_SHIFT_S; 878 const bool cpuif_ns = gicv3_cpuif_is_nonsecure(sc); 879 sc->sc_pmr_shift = cpuif_ns ? GIC_PRIO_SHIFT_NS : GIC_PRIO_SHIFT_S; 880 881 if ((sc->sc_quirks & GICV3_QUIRK_RK3399) != 0) 882 gicv3_quirk_rockchip_rk3399(sc); 883 884 aprint_verbose_dev(sc->sc_dev, 885 "iidr 0x%08x, cpuif %ssecure, dist %ssecure, " 886 "priority shift %d, pmr shift %d, quirks %#x\n", 887 gicd_read_4(sc, GICD_IIDR), 888 cpuif_ns ? "non-" : "", 889 dist_ns ? "non-" : "", 890 sc->sc_priority_shift, 891 sc->sc_pmr_shift, 892 sc->sc_quirks); 893 894 sc->sc_pic.pic_ops = &gicv3_picops; 895 sc->sc_pic.pic_maxsources = GICD_TYPER_LINES(sc->sc_gicd_typer); 896 snprintf(sc->sc_pic.pic_name, sizeof(sc->sc_pic.pic_name), "gicv3"); 897 #ifdef MULTIPROCESSOR 898 sc->sc_pic.pic_cpus = kcpuset_running; 899 #endif 900 pic_add(&sc->sc_pic, 0); 901 902 if ((sc->sc_gicd_typer & GICD_TYPER_LPIS) != 0) { 903 sc->sc_lpipend = kmem_zalloc(sizeof(*sc->sc_lpipend) * ncpu, KM_SLEEP); 904 sc->sc_processor_id = kmem_zalloc(sizeof(*sc->sc_processor_id) * ncpu, KM_SLEEP); 905 906 sc->sc_lpi.pic_ops = &gicv3_lpiops; 907 sc->sc_lpi.pic_maxsources = 8192; /* Min. required by GICv3 spec */ 908 snprintf(sc->sc_lpi.pic_name, sizeof(sc->sc_lpi.pic_name), "gicv3-lpi"); 909 pic_add(&sc->sc_lpi, GIC_LPI_BASE); 910 911 sc->sc_lpi_pool = vmem_create("gicv3-lpi", 0, sc->sc_lpi.pic_maxsources, 912 1, NULL, NULL, NULL, 0, VM_SLEEP, IPL_HIGH); 913 if (sc->sc_lpi_pool == NULL) 914 panic("failed to create gicv3 lpi pool\n"); 915 916 gicv3_lpi_init(sc); 917 } 918 919 KASSERT(gicv3_softc == NULL); 920 gicv3_softc = sc; 921 922 for (int i = 0; i < sc->sc_bsh_r_count; i++) { 923 const uint64_t gicr_typer = gicr_read_8(sc, i, GICR_TYPER); 924 const u_int aff0 = __SHIFTOUT(gicr_typer, GICR_TYPER_Affinity_Value_Aff0); 925 const u_int aff1 = __SHIFTOUT(gicr_typer, GICR_TYPER_Affinity_Value_Aff1); 926 const u_int aff2 = __SHIFTOUT(gicr_typer, GICR_TYPER_Affinity_Value_Aff2); 927 const u_int aff3 = __SHIFTOUT(gicr_typer, GICR_TYPER_Affinity_Value_Aff3); 928 929 aprint_debug_dev(sc->sc_dev, "redist %d: cpu %d.%d.%d.%d\n", 930 i, aff3, aff2, aff1, aff0); 931 } 932 933 gicv3_dist_enable(sc); 934 935 gicv3_cpu_init(&sc->sc_pic, curcpu()); 936 if ((sc->sc_gicd_typer & GICD_TYPER_LPIS) != 0) 937 gicv3_lpi_cpu_init(&sc->sc_lpi, curcpu()); 938 939 #ifdef MULTIPROCESSOR 940 intr_establish_xname(IPI_AST, IPL_VM, IST_MPSAFE | IST_EDGE, pic_ipi_ast, (void *)-1, "IPI ast"); 941 intr_establish_xname(IPI_XCALL, IPL_HIGH, IST_MPSAFE | IST_EDGE, pic_ipi_xcall, (void *)-1, "IPI xcall"); 942 intr_establish_xname(IPI_GENERIC, IPL_HIGH, IST_MPSAFE | IST_EDGE, pic_ipi_generic, (void *)-1, "IPI generic"); 943 intr_establish_xname(IPI_NOP, IPL_VM, IST_MPSAFE | IST_EDGE, pic_ipi_nop, (void *)-1, "IPI nop"); 944 intr_establish_xname(IPI_SHOOTDOWN, IPL_SCHED, IST_MPSAFE | IST_EDGE, pic_ipi_shootdown, (void *)-1, "IPI shootdown"); 945 #ifdef DDB 946 intr_establish_xname(IPI_DDB, IPL_HIGH, IST_MPSAFE | IST_EDGE, pic_ipi_ddb, NULL, "IPI ddb"); 947 #endif 948 #ifdef __HAVE_PREEMPTION 949 intr_establish_xname(IPI_KPREEMPT, IPL_VM, IST_MPSAFE | IST_EDGE, pic_ipi_kpreempt, (void *)-1, "IPI kpreempt"); 950 #endif 951 #endif 952 953 return 0; 954 } 955
Indexes created Sat Sep 20 22:09:52 GMT 2025