1 /* $NetBSD: gicv3.c,v 1.59 2026/09/13 15:53:44 jmcneill Exp $ */ 2 3 /*- 4 * Copyright (c) 2018-2026 Jared McNeill <jmcneill (at) invisible.ca> 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 17 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 18 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 19 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 20 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 21 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 22 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 23 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 24 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 * SUCH DAMAGE. 27 */ 28 29 #include "opt_multiprocessor.h" 30 31 #define _INTR_PRIVATE 32 33 #include <sys/cdefs.h> 34 __KERNEL_RCSID(0, "$NetBSD: gicv3.c,v 1.59 2026/09/13 15:53:44 jmcneill Exp $"); 35 36 #include <sys/param.h> 37 #include <sys/kernel.h> 38 #include <sys/bus.h> 39 #include <sys/device.h> 40 #include <sys/intr.h> 41 #include <sys/systm.h> 42 #include <sys/cpu.h> 43 #include <sys/vmem.h> 44 #include <sys/kmem.h> 45 #include <sys/atomic.h> 46 47 #include <machine/cpufunc.h> 48 49 #include <arm/locore.h> 50 #include <arm/armreg.h> 51 52 #include <arm/cortex/gicv3.h> 53 #include <arm/cortex/gic_reg.h> 54 55 #define PICTOSOFTC(pic) \ 56 container_of(pic, struct gicv3_softc, sc_pic) 57 #define LPITOSOFTC(lpi) \ 58 container_of(lpi, struct gicv3_softc, sc_lpi) 59 60 #define IPL_TO_PRIORITY(sc, ipl) (((0xffU - (ipl)) << (sc)->sc_priority_shift) & 0xffU) 61 #define IPL_TO_PMR(sc, ipl) (((0xffU - (ipl)) << (sc)->sc_pmr_shift) & 0xffU) 62 #define IPL_TO_HWPL(ipl) ((ipl) >= IPL_VM ? (ipl) : IPL_NONE) 63 64 #define GIC_SUPPORTS_1OFN(sc) (((sc)->sc_gicd_typer & GICD_TYPER_No1N) == 0) 65 66 #define GIC_PRIO_SHIFT_NS 4 67 #define GIC_PRIO_SHIFT_S 3 68 69 /* 70 * Set to true if you want to use 1 of N interrupt distribution for SPIs 71 * when available. Disabled by default because it causes issues with the 72 * USB stack. 73 */ 74 bool gicv3_use_1ofn = false; 75 76 static struct gicv3_softc *gicv3_softc; 77 78 static inline uint32_t 79 gicd_read_4(struct gicv3_softc *sc, bus_size_t reg) 80 { 81 return bus_space_read_4(sc->sc_bst, sc->sc_bsh_d, reg); 82 } 83 84 static inline void 85 gicd_write_4(struct gicv3_softc *sc, bus_size_t reg, uint32_t val) 86 { 87 bus_space_write_4(sc->sc_bst, sc->sc_bsh_d, reg, val); 88 } 89 90 #ifdef MULTIPROCESSOR 91 static inline uint64_t 92 gicd_read_8(struct gicv3_softc *sc, bus_size_t reg) 93 { 94 return bus_space_read_8(sc->sc_bst, sc->sc_bsh_d, reg); 95 } 96 #endif 97 98 static inline void 99 gicd_write_8(struct gicv3_softc *sc, bus_size_t reg, uint64_t val) 100 { 101 bus_space_write_8(sc->sc_bst, sc->sc_bsh_d, reg, val); 102 } 103 104 static inline uint32_t 105 gicr_read_4(struct gicv3_softc *sc, u_int index, bus_size_t reg) 106 { 107 KASSERT(index < sc->sc_bsh_r_count); 108 return bus_space_read_4(sc->sc_bst, sc->sc_bsh_r[index], reg); 109 } 110 111 static inline void 112 gicr_write_4(struct gicv3_softc *sc, u_int index, bus_size_t reg, uint32_t val) 113 { 114 KASSERT(index < sc->sc_bsh_r_count); 115 bus_space_write_4(sc->sc_bst, sc->sc_bsh_r[index], reg, val); 116 } 117 118 static inline uint64_t 119 gicr_read_8(struct gicv3_softc *sc, u_int index, bus_size_t reg) 120 { 121 KASSERT(index < sc->sc_bsh_r_count); 122 return bus_space_read_8(sc->sc_bst, sc->sc_bsh_r[index], reg); 123 } 124 125 static inline void 126 gicr_write_8(struct gicv3_softc *sc, u_int index, bus_size_t reg, uint64_t val) 127 { 128 KASSERT(index < sc->sc_bsh_r_count); 129 bus_space_write_8(sc->sc_bst, sc->sc_bsh_r[index], reg, val); 130 } 131 132 static void 133 gicv3_unblock_irqs(struct pic_softc *pic, size_t irqbase, uint32_t mask) 134 { 135 struct gicv3_softc * const sc = PICTOSOFTC(pic); 136 struct cpu_info * const ci = curcpu(); 137 const u_int group = irqbase / 32; 138 139 if (group == 0) { 140 atomic_or_32(&sc->sc_enabled_sgippi, mask); 141 gicr_write_4(sc, ci->ci_gic_redist, GICR_ISENABLER0, mask); 142 } else { 143 gicd_write_4(sc, GICD_ISENABLERn(group), mask); 144 } 145 } 146 147 static void 148 gicv3_block_irqs(struct pic_softc *pic, size_t irqbase, uint32_t mask) 149 { 150 struct gicv3_softc * const sc = PICTOSOFTC(pic); 151 struct cpu_info * const ci = curcpu(); 152 const u_int group = irqbase / 32; 153 154 if (group == 0) { 155 atomic_and_32(&sc->sc_enabled_sgippi, ~mask); 156 gicr_write_4(sc, ci->ci_gic_redist, GICR_ICENABLER0, mask); 157 while (gicr_read_4(sc, ci->ci_gic_redist, GICR_CTLR) & GICR_CTLR_RWP) 158 ; 159 } else { 160 gicd_write_4(sc, GICD_ICENABLERn(group), mask); 161 while (gicd_read_4(sc, GICD_CTRL) & GICD_CTRL_RWP) 162 ; 163 } 164 } 165 166 static void 167 gicv3_establish_irq(struct pic_softc *pic, struct intrsource *is) 168 { 169 struct gicv3_softc * const sc = PICTOSOFTC(pic); 170 const u_int group = is->is_irq / 32; 171 uint32_t ipriority, icfg; 172 uint64_t irouter; 173 u_int n; 174 175 const u_int ipriority_val = IPL_TO_PRIORITY(sc, is->is_ipl); 176 const u_int ipriority_shift = (is->is_irq & 0x3) * 8; 177 const u_int icfg_shift = (is->is_irq & 0xf) * 2; 178 179 if (group == 0) { 180 /* SGIs and PPIs are per-CPU and always MP-safe */ 181 is->is_mpsafe = true; 182 is->is_percpu = true; 183 184 /* Update interrupt configuration and priority on all redistributors */ 185 for (n = 0; n < sc->sc_bsh_r_count; n++) { 186 icfg = gicr_read_4(sc, n, GICR_ICFGRn(is->is_irq / 16)); 187 if (is->is_type == IST_LEVEL) 188 icfg &= ~(0x2U << icfg_shift); 189 if (is->is_type == IST_EDGE) 190 icfg |= (0x2U << icfg_shift); 191 gicr_write_4(sc, n, GICR_ICFGRn(is->is_irq / 16), icfg); 192 193 ipriority = gicr_read_4(sc, n, GICR_IPRIORITYRn(is->is_irq / 4)); 194 ipriority &= ~(0xffU << ipriority_shift); 195 ipriority |= (ipriority_val << ipriority_shift); 196 gicr_write_4(sc, n, GICR_IPRIORITYRn(is->is_irq / 4), ipriority); 197 } 198 } else { 199 /* 200 * If 1 of N SPI routing is supported, route MP-safe interrupts to all 201 * participating PEs. Otherwise, just route to the primary PE. 202 */ 203 if (is->is_mpsafe && GIC_SUPPORTS_1OFN(sc) && gicv3_use_1ofn) { 204 irouter = GICD_IROUTER_Interrupt_Routing_mode; 205 } else { 206 irouter = sc->sc_irouter[0]; 207 } 208 gicd_write_8(sc, GICD_IROUTER(is->is_irq), irouter); 209 210 /* Update interrupt configuration */ 211 icfg = gicd_read_4(sc, GICD_ICFGRn(is->is_irq / 16)); 212 if (is->is_type == IST_LEVEL) 213 icfg &= ~(0x2U << icfg_shift); 214 if (is->is_type == IST_EDGE) 215 icfg |= (0x2U << icfg_shift); 216 gicd_write_4(sc, GICD_ICFGRn(is->is_irq / 16), icfg); 217 218 /* Update interrupt priority */ 219 ipriority = gicd_read_4(sc, GICD_IPRIORITYRn(is->is_irq / 4)); 220 ipriority &= ~(0xffU << ipriority_shift); 221 ipriority |= (ipriority_val << ipriority_shift); 222 gicd_write_4(sc, GICD_IPRIORITYRn(is->is_irq / 4), ipriority); 223 } 224 } 225 226 static void 227 gicv3_set_priority(struct pic_softc *pic, int ipl) 228 { 229 struct gicv3_softc * const sc = PICTOSOFTC(pic); 230 struct cpu_info * const ci = curcpu(); 231 const int hwpl = IPL_TO_HWPL(ipl); 232 233 while (hwpl < ci->ci_hwpl) { 234 /* Lowering priority mask */ 235 ci->ci_hwpl = hwpl; 236 __insn_barrier(); 237 icc_pmr_write(IPL_TO_PMR(sc, hwpl)); 238 } 239 __insn_barrier(); 240 ci->ci_cpl = ipl; 241 } 242 243 static void 244 gicv3_dist_enable(struct gicv3_softc *sc) 245 { 246 uint32_t gicd_ctrl; 247 u_int n; 248 249 /* Disable the distributor */ 250 gicd_ctrl = gicd_read_4(sc, GICD_CTRL); 251 gicd_ctrl &= ~(GICD_CTRL_EnableGrp1A | GICD_CTRL_ARE_NS); 252 gicd_write_4(sc, GICD_CTRL, gicd_ctrl); 253 254 /* Wait for register write to complete */ 255 while (gicd_read_4(sc, GICD_CTRL) & GICD_CTRL_RWP) 256 ; 257 258 /* Clear all INTID enable bits */ 259 for (n = 32; n < sc->sc_pic.pic_maxsources; n += 32) 260 gicd_write_4(sc, GICD_ICENABLERn(n / 32), ~0); 261 262 /* Set default priorities to lowest */ 263 for (n = 32; n < sc->sc_pic.pic_maxsources; n += 4) 264 gicd_write_4(sc, GICD_IPRIORITYRn(n / 4), ~0); 265 266 /* Set all interrupts to G1NS */ 267 for (n = 32; n < sc->sc_pic.pic_maxsources; n += 32) { 268 gicd_write_4(sc, GICD_IGROUPRn(n / 32), ~0); 269 gicd_write_4(sc, GICD_IGRPMODRn(n / 32), 0); 270 } 271 272 /* Set all interrupts level-sensitive by default */ 273 for (n = 32; n < sc->sc_pic.pic_maxsources; n += 16) 274 gicd_write_4(sc, GICD_ICFGRn(n / 16), 0); 275 276 /* Wait for register writes to complete */ 277 while (gicd_read_4(sc, GICD_CTRL) & GICD_CTRL_RWP) 278 ; 279 280 /* Enable Affinity routing and G1NS interrupts */ 281 gicd_ctrl = GICD_CTRL_EnableGrp1A | GICD_CTRL_ARE_NS; 282 gicd_write_4(sc, GICD_CTRL, gicd_ctrl); 283 } 284 285 static void 286 gicv3_redist_enable(struct gicv3_softc *sc, struct cpu_info *ci) 287 { 288 uint32_t icfg; 289 u_int n, o; 290 291 /* Clear INTID enable bits */ 292 gicr_write_4(sc, ci->ci_gic_redist, GICR_ICENABLER0, ~0); 293 294 /* Wait for register write to complete */ 295 while (gicr_read_4(sc, ci->ci_gic_redist, GICR_CTLR) & GICR_CTLR_RWP) 296 ; 297 298 /* Set default priorities */ 299 for (n = 0; n < 32; n += 4) { 300 uint32_t priority = 0; 301 size_t byte_shift = 0; 302 for (o = 0; o < 4; o++, byte_shift += 8) { 303 struct intrsource * const is = sc->sc_pic.pic_sources[n + o]; 304 if (is == NULL) 305 priority |= (0xffU << byte_shift); 306 else { 307 const u_int ipriority_val = IPL_TO_PRIORITY(sc, is->is_ipl); 308 priority |= ipriority_val << byte_shift; 309 } 310 } 311 gicr_write_4(sc, ci->ci_gic_redist, GICR_IPRIORITYRn(n / 4), priority); 312 } 313 314 /* Set all interrupts to G1NS */ 315 gicr_write_4(sc, ci->ci_gic_redist, GICR_IGROUPR0, ~0); 316 gicr_write_4(sc, ci->ci_gic_redist, GICR_IGRPMODR0, 0); 317 318 /* Restore PPI configs */ 319 for (n = 0, icfg = 0; n < 16; n++) { 320 struct intrsource * const is = sc->sc_pic.pic_sources[16 + n]; 321 if (is != NULL && is->is_type == IST_EDGE) 322 icfg |= (0x2U << (n * 2)); 323 } 324 gicr_write_4(sc, ci->ci_gic_redist, GICR_ICFGRn(1), icfg); 325 326 /* Restore current enable bits */ 327 gicr_write_4(sc, ci->ci_gic_redist, GICR_ISENABLER0, sc->sc_enabled_sgippi); 328 329 /* Wait for register write to complete */ 330 while (gicr_read_4(sc, ci->ci_gic_redist, GICR_CTLR) & GICR_CTLR_RWP) 331 ; 332 } 333 334 static uint64_t 335 gicv3_cpu_identity(void) 336 { 337 u_int aff3, aff2, aff1, aff0; 338 339 const register_t mpidr = cpu_mpidr_aff_read(); 340 aff0 = __SHIFTOUT(mpidr, MPIDR_AFF0); 341 aff1 = __SHIFTOUT(mpidr, MPIDR_AFF1); 342 aff2 = __SHIFTOUT(mpidr, MPIDR_AFF2); 343 aff3 = __SHIFTOUT(mpidr, MPIDR_AFF3); 344 345 return __SHIFTIN(aff0, GICR_TYPER_Affinity_Value_Aff0) | 346 __SHIFTIN(aff1, GICR_TYPER_Affinity_Value_Aff1) | 347 __SHIFTIN(aff2, GICR_TYPER_Affinity_Value_Aff2) | 348 __SHIFTIN(aff3, GICR_TYPER_Affinity_Value_Aff3); 349 } 350 351 static u_int 352 gicv3_find_redist(struct gicv3_softc *sc) 353 { 354 uint64_t gicr_typer; 355 u_int n; 356 357 const uint64_t cpu_identity = gicv3_cpu_identity(); 358 359 for (n = 0; n < sc->sc_bsh_r_count; n++) { 360 gicr_typer = gicr_read_8(sc, n, GICR_TYPER); 361 if ((gicr_typer & GICR_TYPER_Affinity_Value) == cpu_identity) 362 return n; 363 } 364 365 const u_int aff0 = __SHIFTOUT(cpu_identity, GICR_TYPER_Affinity_Value_Aff0); 366 const u_int aff1 = __SHIFTOUT(cpu_identity, GICR_TYPER_Affinity_Value_Aff1); 367 const u_int aff2 = __SHIFTOUT(cpu_identity, GICR_TYPER_Affinity_Value_Aff2); 368 const u_int aff3 = __SHIFTOUT(cpu_identity, GICR_TYPER_Affinity_Value_Aff3); 369 370 panic("%s: could not find GICv3 redistributor for cpu %d.%d.%d.%d", 371 cpu_name(curcpu()), aff3, aff2, aff1, aff0); 372 } 373 374 static uint64_t 375 gicv3_sgir(struct gicv3_softc *sc) 376 { 377 const uint64_t cpu_identity = gicv3_cpu_identity(); 378 379 const u_int aff0 = __SHIFTOUT(cpu_identity, GICR_TYPER_Affinity_Value_Aff0); 380 const u_int aff1 = __SHIFTOUT(cpu_identity, GICR_TYPER_Affinity_Value_Aff1); 381 const u_int aff2 = __SHIFTOUT(cpu_identity, GICR_TYPER_Affinity_Value_Aff2); 382 const u_int aff3 = __SHIFTOUT(cpu_identity, GICR_TYPER_Affinity_Value_Aff3); 383 384 return __SHIFTIN(__BIT(aff0), ICC_SGIR_EL1_TargetList) | 385 __SHIFTIN(aff1, ICC_SGIR_EL1_Aff1) | 386 __SHIFTIN(aff2, ICC_SGIR_EL1_Aff2) | 387 __SHIFTIN(aff3, ICC_SGIR_EL1_Aff3); 388 } 389 390 static void 391 gicv3_cpu_init(struct pic_softc *pic, struct cpu_info *ci) 392 { 393 struct gicv3_softc * const sc = PICTOSOFTC(pic); 394 uint32_t icc_sre, icc_ctlr, gicr_waker; 395 396 evcnt_attach_dynamic(&ci->ci_intr_preempt, EVCNT_TYPE_MISC, NULL, 397 ci->ci_cpuname, "intr preempt"); 398 399 ci->ci_gic_redist = gicv3_find_redist(sc); 400 ci->ci_gic_sgir = gicv3_sgir(sc); 401 402 /* Enable System register access and disable IRQ/FIQ bypass */ 403 icc_sre = ICC_SRE_EL1_SRE | ICC_SRE_EL1_DFB | ICC_SRE_EL1_DIB; 404 icc_sre_write(icc_sre); 405 406 /* Mark the connected PE as being awake */ 407 gicr_waker = gicr_read_4(sc, ci->ci_gic_redist, GICR_WAKER); 408 gicr_waker &= ~GICR_WAKER_ProcessorSleep; 409 gicr_write_4(sc, ci->ci_gic_redist, GICR_WAKER, gicr_waker); 410 while (gicr_read_4(sc, ci->ci_gic_redist, GICR_WAKER) & GICR_WAKER_ChildrenAsleep) 411 ; 412 413 /* Set initial priority mask */ 414 ci->ci_hwpl = IPL_HIGH; 415 icc_pmr_write(IPL_TO_PMR(sc, IPL_HIGH)); 416 417 /* Set the binary point field to the minimum value */ 418 icc_bpr1_write(0); 419 420 /* Enable group 1 interrupt signaling */ 421 icc_igrpen1_write(ICC_IGRPEN_EL1_Enable); 422 423 /* Set EOI mode */ 424 icc_ctlr = icc_ctlr_read(); 425 icc_ctlr &= ~ICC_CTLR_EL1_EOImode; 426 icc_ctlr_write(icc_ctlr); 427 428 /* Enable redistributor */ 429 gicv3_redist_enable(sc, ci); 430 431 /* Allow IRQ exceptions */ 432 ENABLE_INTERRUPT(); 433 } 434 435 #ifdef MULTIPROCESSOR 436 static void 437 gicv3_ipi_send(struct pic_softc *pic, const kcpuset_t *kcp, u_long ipi) 438 { 439 struct cpu_info *ci; 440 uint64_t sgir; 441 442 sgir = __SHIFTIN(ipi, ICC_SGIR_EL1_INTID); 443 if (kcp == NULL) { 444 /* Interrupts routed to all PEs, excluding "self" */ 445 if (ncpu == 1) 446 return; 447 sgir |= ICC_SGIR_EL1_IRM; 448 } else { 449 /* Interrupt to exactly one PE */ 450 ci = cpu_lookup(kcpuset_ffs(kcp) - 1); 451 if (ci == curcpu()) 452 return; 453 sgir |= ci->ci_gic_sgir; 454 } 455 icc_sgi1r_write(sgir); 456 isb(); 457 } 458 459 static void 460 gicv3_get_affinity(struct pic_softc *pic, size_t irq, kcpuset_t *affinity) 461 { 462 struct gicv3_softc * const sc = PICTOSOFTC(pic); 463 const size_t group = irq / 32; 464 int n; 465 466 kcpuset_zero(affinity); 467 if (group == 0) { 468 /* All CPUs are targets for group 0 (SGI/PPI) */ 469 for (n = 0; n < ncpu; n++) { 470 kcpuset_set(affinity, n); 471 } 472 } else { 473 /* Find distributor targets (SPI) */ 474 const uint64_t irouter = gicd_read_8(sc, GICD_IROUTER(irq)); 475 for (n = 0; n < ncpu; n++) { 476 if (irouter == GICD_IROUTER_Interrupt_Routing_mode || 477 irouter == sc->sc_irouter[n]) 478 kcpuset_set(affinity, n); 479 } 480 } 481 } 482 483 static int 484 gicv3_set_affinity(struct pic_softc *pic, size_t irq, const kcpuset_t *affinity) 485 { 486 struct gicv3_softc * const sc = PICTOSOFTC(pic); 487 const size_t group = irq / 32; 488 uint64_t irouter; 489 490 if (group == 0) 491 return EINVAL; 492 493 const int set = kcpuset_countset(affinity); 494 if (set == 1) { 495 irouter = sc->sc_irouter[kcpuset_ffs(affinity) - 1]; 496 } else if (set == ncpu && GIC_SUPPORTS_1OFN(sc) && gicv3_use_1ofn) { 497 irouter = GICD_IROUTER_Interrupt_Routing_mode; 498 } else { 499 return EINVAL; 500 } 501 502 gicd_write_8(sc, GICD_IROUTER(irq), irouter); 503 504 return 0; 505 } 506 #endif 507 508 static const struct pic_ops gicv3_picops = { 509 .pic_unblock_irqs = gicv3_unblock_irqs, 510 .pic_block_irqs = gicv3_block_irqs, 511 .pic_establish_irq = gicv3_establish_irq, 512 .pic_set_priority = gicv3_set_priority, 513 #ifdef MULTIPROCESSOR 514 .pic_cpu_init = gicv3_cpu_init, 515 .pic_ipi_send = gicv3_ipi_send, 516 .pic_get_affinity = gicv3_get_affinity, 517 .pic_set_affinity = gicv3_set_affinity, 518 #endif 519 }; 520 521 static void 522 gicv3_dcache_wb_range(vaddr_t va, vsize_t len) 523 { 524 cpu_dcache_wb_range(va, len); 525 dsb(sy); 526 } 527 528 static void 529 gicv3_lpi_unblock_irqs(struct pic_softc *pic, size_t irqbase, uint32_t mask) 530 { 531 struct gicv3_softc * const sc = LPITOSOFTC(pic); 532 int bit; 533 534 while ((bit = ffs(mask)) != 0) { 535 sc->sc_lpiconf.base[irqbase + bit - 1] |= GIC_LPICONF_Enable; 536 if (sc->sc_lpiconf_flush) 537 gicv3_dcache_wb_range((vaddr_t)&sc->sc_lpiconf.base[irqbase + bit - 1], 1); 538 mask &= ~__BIT(bit - 1); 539 } 540 541 if (!sc->sc_lpiconf_flush) 542 dsb(ishst); 543 } 544 545 static void 546 gicv3_lpi_block_irqs(struct pic_softc *pic, size_t irqbase, uint32_t mask) 547 { 548 struct gicv3_softc * const sc = LPITOSOFTC(pic); 549 int bit; 550 551 while ((bit = ffs(mask)) != 0) { 552 sc->sc_lpiconf.base[irqbase + bit - 1] &= ~GIC_LPICONF_Enable; 553 if (sc->sc_lpiconf_flush) 554 gicv3_dcache_wb_range((vaddr_t)&sc->sc_lpiconf.base[irqbase + bit - 1], 1); 555 mask &= ~__BIT(bit - 1); 556 } 557 558 if (!sc->sc_lpiconf_flush) 559 dsb(ishst); 560 } 561 562 static void 563 gicv3_lpi_establish_irq(struct pic_softc *pic, struct intrsource *is) 564 { 565 struct gicv3_softc * const sc = LPITOSOFTC(pic); 566 567 sc->sc_lpiconf.base[is->is_irq] = IPL_TO_PRIORITY(sc, is->is_ipl) | GIC_LPICONF_Res1; 568 569 if (sc->sc_lpiconf_flush) 570 gicv3_dcache_wb_range((vaddr_t)&sc->sc_lpiconf.base[is->is_irq], 1); 571 else 572 dsb(ishst); 573 } 574 575 static void 576 gicv3_lpi_cpu_init(struct pic_softc *pic, struct cpu_info *ci) 577 { 578 struct gicv3_softc * const sc = LPITOSOFTC(pic); 579 struct gicv3_lpi_callback *cb; 580 uint64_t propbase, pendbase; 581 uint32_t ctlr; 582 583 /* If physical LPIs are not supported on this redistributor, just return. */ 584 const uint64_t typer = gicr_read_8(sc, ci->ci_gic_redist, GICR_TYPER); 585 if ((typer & GICR_TYPER_PLPIS) == 0) 586 return; 587 588 /* Interrupt target address for this CPU, used by ITS when GITS_TYPER.PTA == 0 */ 589 sc->sc_processor_id[cpu_index(ci)] = __SHIFTOUT(typer, GICR_TYPER_Processor_Number); 590 591 /* Disable LPIs before making changes */ 592 ctlr = gicr_read_4(sc, ci->ci_gic_redist, GICR_CTLR); 593 ctlr &= ~GICR_CTLR_Enable_LPIs; 594 gicr_write_4(sc, ci->ci_gic_redist, GICR_CTLR, ctlr); 595 dsb(sy); 596 597 /* Setup the LPI configuration table */ 598 propbase = sc->sc_lpiconf.segs[0].ds_addr | 599 __SHIFTIN(ffs(pic->pic_maxsources) - 1, GICR_PROPBASER_IDbits) | 600 __SHIFTIN(GICR_Shareability_IS, GICR_PROPBASER_Shareability) | 601 __SHIFTIN(GICR_Cache_NORMAL_RA_WA_WB, GICR_PROPBASER_InnerCache); 602 gicr_write_8(sc, ci->ci_gic_redist, GICR_PROPBASER, propbase); 603 propbase = gicr_read_8(sc, ci->ci_gic_redist, GICR_PROPBASER); 604 if (__SHIFTOUT(propbase, GICR_PROPBASER_Shareability) != GICR_Shareability_IS) { 605 if (__SHIFTOUT(propbase, GICR_PROPBASER_Shareability) == GICR_Shareability_NS) { 606 propbase &= ~GICR_PROPBASER_Shareability; 607 propbase |= __SHIFTIN(GICR_Shareability_NS, GICR_PROPBASER_Shareability); 608 propbase &= ~GICR_PROPBASER_InnerCache; 609 propbase |= __SHIFTIN(GICR_Cache_NORMAL_NC, GICR_PROPBASER_InnerCache); 610 gicr_write_8(sc, ci->ci_gic_redist, GICR_PROPBASER, propbase); 611 } 612 sc->sc_lpiconf_flush = true; 613 } 614 615 /* Setup the LPI pending table */ 616 pendbase = sc->sc_lpipend[cpu_index(ci)].segs[0].ds_addr | 617 __SHIFTIN(GICR_Shareability_IS, GICR_PENDBASER_Shareability) | 618 __SHIFTIN(GICR_Cache_NORMAL_RA_WA_WB, GICR_PENDBASER_InnerCache); 619 gicr_write_8(sc, ci->ci_gic_redist, GICR_PENDBASER, pendbase); 620 pendbase = gicr_read_8(sc, ci->ci_gic_redist, GICR_PENDBASER); 621 if (__SHIFTOUT(pendbase, GICR_PENDBASER_Shareability) == GICR_Shareability_NS) { 622 pendbase &= ~GICR_PENDBASER_Shareability; 623 pendbase |= __SHIFTIN(GICR_Shareability_NS, GICR_PENDBASER_Shareability); 624 pendbase &= ~GICR_PENDBASER_InnerCache; 625 pendbase |= __SHIFTIN(GICR_Cache_NORMAL_NC, GICR_PENDBASER_InnerCache); 626 gicr_write_8(sc, ci->ci_gic_redist, GICR_PENDBASER, pendbase); 627 } 628 629 /* Enable LPIs */ 630 ctlr = gicr_read_4(sc, ci->ci_gic_redist, GICR_CTLR); 631 ctlr |= GICR_CTLR_Enable_LPIs; 632 gicr_write_4(sc, ci->ci_gic_redist, GICR_CTLR, ctlr); 633 dsb(sy); 634 635 /* Setup ITS if present */ 636 LIST_FOREACH(cb, &sc->sc_lpi_callbacks, list) 637 cb->cpu_init(cb->priv, ci); 638 } 639 640 #ifdef MULTIPROCESSOR 641 static void 642 gicv3_lpi_get_affinity(struct pic_softc *pic, size_t irq, kcpuset_t *affinity) 643 { 644 struct gicv3_softc * const sc = LPITOSOFTC(pic); 645 struct gicv3_lpi_callback *cb; 646 647 kcpuset_zero(affinity); 648 LIST_FOREACH(cb, &sc->sc_lpi_callbacks, list) 649 cb->get_affinity(cb->priv, irq, affinity); 650 } 651 652 static int 653 gicv3_lpi_set_affinity(struct pic_softc *pic, size_t irq, const kcpuset_t *affinity) 654 { 655 struct gicv3_softc * const sc = LPITOSOFTC(pic); 656 struct gicv3_lpi_callback *cb; 657 int error = EINVAL; 658 659 LIST_FOREACH(cb, &sc->sc_lpi_callbacks, list) { 660 error = cb->set_affinity(cb->priv, irq, affinity); 661 if (error != EPASSTHROUGH) 662 return error; 663 } 664 665 return EINVAL; 666 } 667 #endif 668 669 static const struct pic_ops gicv3_lpiops = { 670 .pic_unblock_irqs = gicv3_lpi_unblock_irqs, 671 .pic_block_irqs = gicv3_lpi_block_irqs, 672 .pic_establish_irq = gicv3_lpi_establish_irq, 673 #ifdef MULTIPROCESSOR 674 .pic_cpu_init = gicv3_lpi_cpu_init, 675 .pic_get_affinity = gicv3_lpi_get_affinity, 676 .pic_set_affinity = gicv3_lpi_set_affinity, 677 #endif 678 }; 679 680 void 681 gicv3_dma_alloc(struct gicv3_softc *sc, struct gicv3_dma *dma, bus_size_t len, bus_size_t align) 682 { 683 int nsegs, error; 684 685 dma->len = len; 686 error = bus_dmamem_alloc(sc->sc_dmat, dma->len, align, 0, dma->segs, 1, &nsegs, BUS_DMA_WAITOK); 687 if (error) 688 panic("bus_dmamem_alloc failed: %d", error); 689 error = bus_dmamem_map(sc->sc_dmat, dma->segs, nsegs, len, (void **)&dma->base, BUS_DMA_WAITOK); 690 if (error) 691 panic("bus_dmamem_map failed: %d", error); 692 error = bus_dmamap_create(sc->sc_dmat, len, 1, len, 0, BUS_DMA_WAITOK, &dma->map); 693 if (error) 694 panic("bus_dmamap_create failed: %d", error); 695 error = bus_dmamap_load(sc->sc_dmat, dma->map, dma->base, dma->len, NULL, BUS_DMA_WAITOK); 696 if (error) 697 panic("bus_dmamap_load failed: %d", error); 698 699 memset(dma->base, 0, dma->len); 700 bus_dmamap_sync(sc->sc_dmat, dma->map, 0, dma->len, BUS_DMASYNC_PREWRITE); 701 } 702 703 static void 704 gicv3_lpi_init(struct gicv3_softc *sc) 705 { 706 /* 707 * Allocate LPI configuration table 708 */ 709 gicv3_dma_alloc(sc, &sc->sc_lpiconf, sc->sc_lpi.pic_maxsources, 0x1000); 710 KASSERT((sc->sc_lpiconf.segs[0].ds_addr & ~GICR_PROPBASER_Physical_Address) == 0); 711 712 /* 713 * Allocate LPI pending tables 714 */ 715 const bus_size_t lpipend_sz = (8192 + sc->sc_lpi.pic_maxsources) / NBBY; 716 for (int cpuindex = 0; cpuindex < ncpu; cpuindex++) { 717 gicv3_dma_alloc(sc, &sc->sc_lpipend[cpuindex], lpipend_sz, 0x10000); 718 KASSERT((sc->sc_lpipend[cpuindex].segs[0].ds_addr & ~GICR_PENDBASER_Physical_Address) == 0); 719 } 720 } 721 722 void 723 gicv3_irq_handler(void *frame) 724 { 725 struct cpu_info * const ci = curcpu(); 726 struct gicv3_softc * const sc = gicv3_softc; 727 struct pic_softc *pic; 728 const int oldipl = ci->ci_cpl; 729 const int oldhwpl = IPL_TO_HWPL(oldipl); 730 731 ci->ci_data.cpu_nintr++; 732 733 if (ci->ci_hwpl != oldhwpl) { 734 ci->ci_hwpl = oldhwpl; 735 icc_pmr_write(IPL_TO_PMR(sc, oldhwpl)); 736 if (oldhwpl == IPL_HIGH) { 737 return; 738 } 739 } 740 741 for (;;) { 742 const uint32_t iar = icc_iar1_read(); 743 dsb(sy); 744 const uint32_t irq = __SHIFTOUT(iar, ICC_IAR_INTID); 745 if (irq == ICC_IAR_INTID_SPURIOUS) 746 break; 747 748 pic = irq >= GIC_LPI_BASE ? &sc->sc_lpi : &sc->sc_pic; 749 if (irq - pic->pic_irqbase >= pic->pic_maxsources) 750 continue; 751 752 struct intrsource * const is = pic->pic_sources[irq - pic->pic_irqbase]; 753 KASSERT(is != NULL); 754 755 const bool early_eoi = irq < GIC_LPI_BASE && is->is_type == IST_EDGE; 756 757 const int ipl = is->is_ipl; 758 if (__predict_false(ipl < ci->ci_cpl)) { 759 pic_do_pending_ints(I32_bit, ipl, frame); 760 } else if (ci->ci_cpl != ipl) { 761 KASSERT(ipl >= IPL_VM); 762 icc_pmr_write(IPL_TO_PMR(sc, ipl)); 763 ci->ci_hwpl = ci->ci_cpl = ipl; 764 } 765 766 if (early_eoi) { 767 icc_eoi1r_write(iar); 768 isb(); 769 } 770 771 const int64_t nintr = ci->ci_data.cpu_nintr; 772 773 ENABLE_INTERRUPT(); 774 pic_dispatch(is, frame); 775 DISABLE_INTERRUPT(); 776 777 if (nintr != ci->ci_data.cpu_nintr) 778 ci->ci_intr_preempt.ev_count++; 779 780 if (!early_eoi) { 781 icc_eoi1r_write(iar); 782 isb(); 783 } 784 } 785 786 pic_do_pending_ints(I32_bit, oldipl, frame); 787 } 788 789 static bool 790 gicv3_cpuif_is_nonsecure(struct gicv3_softc *sc) 791 { 792 /* 793 * Write 0 to bit7 and see if it sticks. This is only possible if 794 * we have a non-secure view of the PMR register. 795 */ 796 const uint32_t opmr = icc_pmr_read(); 797 icc_pmr_write(0); 798 const uint32_t npmr = icc_pmr_read(); 799 icc_pmr_write(opmr); 800 801 return (npmr & GICC_PMR_NONSECURE) == 0; 802 } 803 804 static bool 805 gicv3_dist_is_nonsecure(struct gicv3_softc *sc) 806 { 807 const uint32_t gicd_ctrl = gicd_read_4(sc, GICD_CTRL); 808 809 /* 810 * If security is enabled, we have a non-secure view of the IPRIORITYRn 811 * registers and LPI configuration priority fields. 812 */ 813 return (gicd_ctrl & GICD_CTRL_DS) == 0; 814 } 815 816 /* 817 * Rockchip RK3399 provides a different view of int priority registers 818 * depending on which firmware is in use. This is hard to detect in 819 * a way that could possibly break other boards, so only do this 820 * detection if we know we are on a RK3399 SoC. 821 */ 822 static void 823 gicv3_quirk_rockchip_rk3399(struct gicv3_softc *sc) 824 { 825 /* Detect the number of supported PMR bits */ 826 icc_pmr_write(0xff); 827 const uint8_t pmrbits = icc_pmr_read(); 828 829 /* Detect the number of supported IPRIORITYRn bits */ 830 const uint32_t oiprio = gicd_read_4(sc, GICD_IPRIORITYRn(8)); 831 gicd_write_4(sc, GICD_IPRIORITYRn(8), oiprio | 0xff); 832 const uint8_t pribits = gicd_read_4(sc, GICD_IPRIORITYRn(8)) & 0xff; 833 gicd_write_4(sc, GICD_IPRIORITYRn(8), oiprio); 834 835 /* 836 * If we see fewer PMR bits than IPRIORITYRn bits here, it means 837 * we have a secure view of IPRIORITYRn (this is not supposed to 838 * happen!). 839 */ 840 if (pmrbits < pribits) { 841 aprint_verbose_dev(sc->sc_dev, 842 "buggy RK3399 firmware detected; applying workaround\n"); 843 sc->sc_priority_shift = GIC_PRIO_SHIFT_S; 844 } 845 } 846 847 int 848 gicv3_init(struct gicv3_softc *sc) 849 { 850 CPU_INFO_ITERATOR cii; 851 struct cpu_info *ci; 852 853 KASSERT(CPU_IS_PRIMARY(curcpu())); 854 855 LIST_INIT(&sc->sc_lpi_callbacks); 856 857 /* Store route to CPU for SPIs */ 858 sc->sc_irouter = kmem_zalloc(sizeof(*sc->sc_irouter) * ncpu, KM_SLEEP); 859 for (CPU_INFO_FOREACH(cii, ci)) { 860 KASSERT(cpu_index(ci) < ncpu); 861 sc->sc_irouter[cpu_index(ci)] = ci->ci_cpuid; 862 } 863 864 sc->sc_gicd_typer = gicd_read_4(sc, GICD_TYPER); 865 866 /* 867 * We don't always have a consistent view of priorities between the 868 * CPU interface (ICC_PMR_EL1) and the GICD/GICR registers. Detect 869 * if we are making secure or non-secure accesses to each, and adjust 870 * the values that we write to each accordingly. 871 */ 872 const bool dist_ns = gicv3_dist_is_nonsecure(sc); 873 sc->sc_priority_shift = dist_ns ? GIC_PRIO_SHIFT_NS : GIC_PRIO_SHIFT_S; 874 const bool cpuif_ns = gicv3_cpuif_is_nonsecure(sc); 875 sc->sc_pmr_shift = cpuif_ns ? GIC_PRIO_SHIFT_NS : GIC_PRIO_SHIFT_S; 876 877 if ((sc->sc_quirks & GICV3_QUIRK_RK3399) != 0) 878 gicv3_quirk_rockchip_rk3399(sc); 879 880 aprint_verbose_dev(sc->sc_dev, 881 "iidr 0x%08x, cpuif %ssecure, dist %ssecure, " 882 "priority shift %d, pmr shift %d, quirks %#x\n", 883 gicd_read_4(sc, GICD_IIDR), 884 cpuif_ns ? "non-" : "", 885 dist_ns ? "non-" : "", 886 sc->sc_priority_shift, 887 sc->sc_pmr_shift, 888 sc->sc_quirks); 889 890 sc->sc_pic.pic_ops = &gicv3_picops; 891 sc->sc_pic.pic_maxsources = GICD_TYPER_LINES(sc->sc_gicd_typer); 892 snprintf(sc->sc_pic.pic_name, sizeof(sc->sc_pic.pic_name), "gicv3"); 893 #ifdef MULTIPROCESSOR 894 sc->sc_pic.pic_cpus = kcpuset_running; 895 #endif 896 pic_add(&sc->sc_pic, 0); 897 898 if ((sc->sc_gicd_typer & GICD_TYPER_LPIS) != 0) { 899 sc->sc_lpipend = kmem_zalloc(sizeof(*sc->sc_lpipend) * ncpu, KM_SLEEP); 900 sc->sc_processor_id = kmem_zalloc(sizeof(*sc->sc_processor_id) * ncpu, KM_SLEEP); 901 902 sc->sc_lpi.pic_ops = &gicv3_lpiops; 903 sc->sc_lpi.pic_maxsources = 8192; /* Min. required by GICv3 spec */ 904 snprintf(sc->sc_lpi.pic_name, sizeof(sc->sc_lpi.pic_name), "gicv3-lpi"); 905 pic_add(&sc->sc_lpi, GIC_LPI_BASE); 906 907 sc->sc_lpi_pool = vmem_create("gicv3-lpi", 0, sc->sc_lpi.pic_maxsources, 908 1, NULL, NULL, NULL, 0, VM_SLEEP, IPL_HIGH); 909 if (sc->sc_lpi_pool == NULL) 910 panic("failed to create gicv3 lpi pool\n"); 911 912 gicv3_lpi_init(sc); 913 } 914 915 KASSERT(gicv3_softc == NULL); 916 gicv3_softc = sc; 917 918 for (int i = 0; i < sc->sc_bsh_r_count; i++) { 919 const uint64_t gicr_typer = gicr_read_8(sc, i, GICR_TYPER); 920 const u_int aff0 = __SHIFTOUT(gicr_typer, GICR_TYPER_Affinity_Value_Aff0); 921 const u_int aff1 = __SHIFTOUT(gicr_typer, GICR_TYPER_Affinity_Value_Aff1); 922 const u_int aff2 = __SHIFTOUT(gicr_typer, GICR_TYPER_Affinity_Value_Aff2); 923 const u_int aff3 = __SHIFTOUT(gicr_typer, GICR_TYPER_Affinity_Value_Aff3); 924 925 aprint_debug_dev(sc->sc_dev, "redist %d: cpu %d.%d.%d.%d\n", 926 i, aff3, aff2, aff1, aff0); 927 } 928 929 gicv3_dist_enable(sc); 930 931 gicv3_cpu_init(&sc->sc_pic, curcpu()); 932 if ((sc->sc_gicd_typer & GICD_TYPER_LPIS) != 0) 933 gicv3_lpi_cpu_init(&sc->sc_lpi, curcpu()); 934 935 #ifdef MULTIPROCESSOR 936 intr_establish_xname(IPI_AST, IPL_VM, IST_MPSAFE | IST_EDGE, pic_ipi_ast, (void *)-1, "IPI ast"); 937 intr_establish_xname(IPI_XCALL, IPL_HIGH, IST_MPSAFE | IST_EDGE, pic_ipi_xcall, (void *)-1, "IPI xcall"); 938 intr_establish_xname(IPI_GENERIC, IPL_HIGH, IST_MPSAFE | IST_EDGE, pic_ipi_generic, (void *)-1, "IPI generic"); 939 intr_establish_xname(IPI_NOP, IPL_VM, IST_MPSAFE | IST_EDGE, pic_ipi_nop, (void *)-1, "IPI nop"); 940 intr_establish_xname(IPI_SHOOTDOWN, IPL_SCHED, IST_MPSAFE | IST_EDGE, pic_ipi_shootdown, (void *)-1, "IPI shootdown"); 941 #ifdef DDB 942 intr_establish_xname(IPI_DDB, IPL_HIGH, IST_MPSAFE | IST_EDGE, pic_ipi_ddb, NULL, "IPI ddb"); 943 #endif 944 #ifdef __HAVE_PREEMPTION 945 intr_establish_xname(IPI_KPREEMPT, IPL_VM, IST_MPSAFE | IST_EDGE, pic_ipi_kpreempt, (void *)-1, "IPI kpreempt"); 946 #endif 947 #endif 948 949 return 0; 950 } 951
Indexes created Sun Oct 25 00:26:45 UTC 2026