gicv3.c revision 1.32.2.2
1 /* $NetBSD: gicv3.c,v 1.32.2.2 2021/01/03 16:34:51 thorpej Exp $ */ 2 3 /*- 4 * Copyright (c) 2018 Jared McNeill <jmcneill (at) invisible.ca> 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 17 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 18 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 19 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 20 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 21 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 22 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 23 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 24 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 * SUCH DAMAGE. 27 */ 28 29 #include "opt_multiprocessor.h" 30 31 #define _INTR_PRIVATE 32 33 #include <sys/cdefs.h> 34 __KERNEL_RCSID(0, "$NetBSD: gicv3.c,v 1.32.2.2 2021/01/03 16:34:51 thorpej Exp $"); 35 36 #include <sys/param.h> 37 #include <sys/kernel.h> 38 #include <sys/bus.h> 39 #include <sys/device.h> 40 #include <sys/intr.h> 41 #include <sys/systm.h> 42 #include <sys/cpu.h> 43 #include <sys/vmem.h> 44 #include <sys/atomic.h> 45 46 #include <machine/cpufunc.h> 47 48 #include <arm/locore.h> 49 #include <arm/armreg.h> 50 51 #include <arm/cortex/gicv3.h> 52 #include <arm/cortex/gic_reg.h> 53 54 #define PICTOSOFTC(pic) \ 55 ((void *)((uintptr_t)(pic) - offsetof(struct gicv3_softc, sc_pic))) 56 #define LPITOSOFTC(lpi) \ 57 ((void *)((uintptr_t)(lpi) - offsetof(struct gicv3_softc, sc_lpi))) 58 59 #define IPL_TO_PRIORITY(sc, ipl) (((0xff - (ipl)) << (sc)->sc_priority_shift) & 0xff) 60 #define IPL_TO_PMR(sc, ipl) (((0xff - (ipl)) << (sc)->sc_pmr_shift) & 0xff) 61 62 #define GIC_SUPPORTS_1OFN(sc) (((sc)->sc_gicd_typer & GICD_TYPER_No1N) == 0) 63 64 #define GIC_PRIO_SHIFT_NS 4 65 #define GIC_PRIO_SHIFT_S 3 66 67 static struct gicv3_softc *gicv3_softc; 68 69 static inline uint32_t 70 gicd_read_4(struct gicv3_softc *sc, bus_size_t reg) 71 { 72 return bus_space_read_4(sc->sc_bst, sc->sc_bsh_d, reg); 73 } 74 75 static inline void 76 gicd_write_4(struct gicv3_softc *sc, bus_size_t reg, uint32_t val) 77 { 78 bus_space_write_4(sc->sc_bst, sc->sc_bsh_d, reg, val); 79 } 80 81 static inline uint64_t 82 gicd_read_8(struct gicv3_softc *sc, bus_size_t reg) 83 { 84 return bus_space_read_8(sc->sc_bst, sc->sc_bsh_d, reg); 85 } 86 87 static inline void 88 gicd_write_8(struct gicv3_softc *sc, bus_size_t reg, uint64_t val) 89 { 90 bus_space_write_8(sc->sc_bst, sc->sc_bsh_d, reg, val); 91 } 92 93 static inline uint32_t 94 gicr_read_4(struct gicv3_softc *sc, u_int index, bus_size_t reg) 95 { 96 KASSERT(index < sc->sc_bsh_r_count); 97 return bus_space_read_4(sc->sc_bst, sc->sc_bsh_r[index], reg); 98 } 99 100 static inline void 101 gicr_write_4(struct gicv3_softc *sc, u_int index, bus_size_t reg, uint32_t val) 102 { 103 KASSERT(index < sc->sc_bsh_r_count); 104 bus_space_write_4(sc->sc_bst, sc->sc_bsh_r[index], reg, val); 105 } 106 107 static inline uint64_t 108 gicr_read_8(struct gicv3_softc *sc, u_int index, bus_size_t reg) 109 { 110 KASSERT(index < sc->sc_bsh_r_count); 111 return bus_space_read_8(sc->sc_bst, sc->sc_bsh_r[index], reg); 112 } 113 114 static inline void 115 gicr_write_8(struct gicv3_softc *sc, u_int index, bus_size_t reg, uint64_t val) 116 { 117 KASSERT(index < sc->sc_bsh_r_count); 118 bus_space_write_8(sc->sc_bst, sc->sc_bsh_r[index], reg, val); 119 } 120 121 static void 122 gicv3_unblock_irqs(struct pic_softc *pic, size_t irqbase, uint32_t mask) 123 { 124 struct gicv3_softc * const sc = PICTOSOFTC(pic); 125 struct cpu_info * const ci = curcpu(); 126 const u_int group = irqbase / 32; 127 128 if (group == 0) { 129 atomic_or_32(&sc->sc_enabled_sgippi, mask); 130 gicr_write_4(sc, ci->ci_gic_redist, GICR_ISENABLER0, mask); 131 while (gicr_read_4(sc, ci->ci_gic_redist, GICR_CTLR) & GICR_CTLR_RWP) 132 ; 133 } else { 134 gicd_write_4(sc, GICD_ISENABLERn(group), mask); 135 while (gicd_read_4(sc, GICD_CTRL) & GICD_CTRL_RWP) 136 ; 137 } 138 } 139 140 static void 141 gicv3_block_irqs(struct pic_softc *pic, size_t irqbase, uint32_t mask) 142 { 143 struct gicv3_softc * const sc = PICTOSOFTC(pic); 144 struct cpu_info * const ci = curcpu(); 145 const u_int group = irqbase / 32; 146 147 if (group == 0) { 148 atomic_and_32(&sc->sc_enabled_sgippi, ~mask); 149 gicr_write_4(sc, ci->ci_gic_redist, GICR_ICENABLER0, mask); 150 while (gicr_read_4(sc, ci->ci_gic_redist, GICR_CTLR) & GICR_CTLR_RWP) 151 ; 152 } else { 153 gicd_write_4(sc, GICD_ICENABLERn(group), mask); 154 while (gicd_read_4(sc, GICD_CTRL) & GICD_CTRL_RWP) 155 ; 156 } 157 } 158 159 static void 160 gicv3_establish_irq(struct pic_softc *pic, struct intrsource *is) 161 { 162 struct gicv3_softc * const sc = PICTOSOFTC(pic); 163 const u_int group = is->is_irq / 32; 164 uint32_t ipriority, icfg; 165 uint64_t irouter; 166 u_int n; 167 168 const u_int ipriority_val = IPL_TO_PRIORITY(sc, is->is_ipl); 169 const u_int ipriority_shift = (is->is_irq & 0x3) * 8; 170 const u_int icfg_shift = (is->is_irq & 0xf) * 2; 171 172 if (group == 0) { 173 /* SGIs and PPIs are always MP-safe */ 174 is->is_mpsafe = true; 175 176 /* Update interrupt configuration and priority on all redistributors */ 177 for (n = 0; n < sc->sc_bsh_r_count; n++) { 178 icfg = gicr_read_4(sc, n, GICR_ICFGRn(is->is_irq / 16)); 179 if (is->is_type == IST_LEVEL) 180 icfg &= ~(0x2 << icfg_shift); 181 if (is->is_type == IST_EDGE) 182 icfg |= (0x2 << icfg_shift); 183 gicr_write_4(sc, n, GICR_ICFGRn(is->is_irq / 16), icfg); 184 185 ipriority = gicr_read_4(sc, n, GICR_IPRIORITYRn(is->is_irq / 4)); 186 ipriority &= ~(0xffU << ipriority_shift); 187 ipriority |= (ipriority_val << ipriority_shift); 188 gicr_write_4(sc, n, GICR_IPRIORITYRn(is->is_irq / 4), ipriority); 189 } 190 } else { 191 /* 192 * If 1 of N SPI routing is supported, route MP-safe interrupts to all 193 * participating PEs. Otherwise, just route to the primary PE. 194 */ 195 if (is->is_mpsafe && GIC_SUPPORTS_1OFN(sc)) { 196 irouter = GICD_IROUTER_Interrupt_Routing_mode; 197 } else { 198 irouter = sc->sc_irouter[0]; 199 } 200 gicd_write_8(sc, GICD_IROUTER(is->is_irq), irouter); 201 202 /* Update interrupt configuration */ 203 icfg = gicd_read_4(sc, GICD_ICFGRn(is->is_irq / 16)); 204 if (is->is_type == IST_LEVEL) 205 icfg &= ~(0x2 << icfg_shift); 206 if (is->is_type == IST_EDGE) 207 icfg |= (0x2 << icfg_shift); 208 gicd_write_4(sc, GICD_ICFGRn(is->is_irq / 16), icfg); 209 210 /* Update interrupt priority */ 211 ipriority = gicd_read_4(sc, GICD_IPRIORITYRn(is->is_irq / 4)); 212 ipriority &= ~(0xffU << ipriority_shift); 213 ipriority |= (ipriority_val << ipriority_shift); 214 gicd_write_4(sc, GICD_IPRIORITYRn(is->is_irq / 4), ipriority); 215 } 216 } 217 218 static void 219 gicv3_set_priority(struct pic_softc *pic, int ipl) 220 { 221 struct gicv3_softc * const sc = PICTOSOFTC(pic); 222 223 icc_pmr_write(IPL_TO_PMR(sc, ipl)); 224 } 225 226 static void 227 gicv3_dist_enable(struct gicv3_softc *sc) 228 { 229 uint32_t gicd_ctrl; 230 u_int n; 231 232 /* Disable the distributor */ 233 gicd_ctrl = gicd_read_4(sc, GICD_CTRL); 234 gicd_ctrl &= ~(GICD_CTRL_EnableGrp1A | GICD_CTRL_ARE_NS); 235 gicd_write_4(sc, GICD_CTRL, gicd_ctrl); 236 237 /* Wait for register write to complete */ 238 while (gicd_read_4(sc, GICD_CTRL) & GICD_CTRL_RWP) 239 ; 240 241 /* Clear all INTID enable bits */ 242 for (n = 32; n < sc->sc_pic.pic_maxsources; n += 32) 243 gicd_write_4(sc, GICD_ICENABLERn(n / 32), ~0); 244 245 /* Set default priorities to lowest */ 246 for (n = 32; n < sc->sc_pic.pic_maxsources; n += 4) 247 gicd_write_4(sc, GICD_IPRIORITYRn(n / 4), ~0); 248 249 /* Set all interrupts to G1NS */ 250 for (n = 32; n < sc->sc_pic.pic_maxsources; n += 32) { 251 gicd_write_4(sc, GICD_IGROUPRn(n / 32), ~0); 252 gicd_write_4(sc, GICD_IGRPMODRn(n / 32), 0); 253 } 254 255 /* Set all interrupts level-sensitive by default */ 256 for (n = 32; n < sc->sc_pic.pic_maxsources; n += 16) 257 gicd_write_4(sc, GICD_ICFGRn(n / 16), 0); 258 259 /* Wait for register writes to complete */ 260 while (gicd_read_4(sc, GICD_CTRL) & GICD_CTRL_RWP) 261 ; 262 263 /* Enable Affinity routing and G1NS interrupts */ 264 gicd_ctrl = GICD_CTRL_EnableGrp1A | GICD_CTRL_ARE_NS; 265 gicd_write_4(sc, GICD_CTRL, gicd_ctrl); 266 } 267 268 static void 269 gicv3_redist_enable(struct gicv3_softc *sc, struct cpu_info *ci) 270 { 271 uint32_t icfg; 272 u_int n, o; 273 274 /* Clear INTID enable bits */ 275 gicr_write_4(sc, ci->ci_gic_redist, GICR_ICENABLER0, ~0); 276 277 /* Wait for register write to complete */ 278 while (gicr_read_4(sc, ci->ci_gic_redist, GICR_CTLR) & GICR_CTLR_RWP) 279 ; 280 281 /* Set default priorities */ 282 for (n = 0; n < 32; n += 4) { 283 uint32_t priority = 0; 284 size_t byte_shift = 0; 285 for (o = 0; o < 4; o++, byte_shift += 8) { 286 struct intrsource * const is = sc->sc_pic.pic_sources[n + o]; 287 if (is == NULL) 288 priority |= (0xffU << byte_shift); 289 else { 290 const u_int ipriority_val = IPL_TO_PRIORITY(sc, is->is_ipl); 291 priority |= ipriority_val << byte_shift; 292 } 293 } 294 gicr_write_4(sc, ci->ci_gic_redist, GICR_IPRIORITYRn(n / 4), priority); 295 } 296 297 /* Set all interrupts to G1NS */ 298 gicr_write_4(sc, ci->ci_gic_redist, GICR_IGROUPR0, ~0); 299 gicr_write_4(sc, ci->ci_gic_redist, GICR_IGRPMODR0, 0); 300 301 /* Restore PPI configs */ 302 for (n = 0, icfg = 0; n < 16; n++) { 303 struct intrsource * const is = sc->sc_pic.pic_sources[16 + n]; 304 if (is != NULL && is->is_type == IST_EDGE) 305 icfg |= (0x2 << (n * 2)); 306 } 307 gicr_write_4(sc, ci->ci_gic_redist, GICR_ICFGRn(1), icfg); 308 309 /* Restore current enable bits */ 310 gicr_write_4(sc, ci->ci_gic_redist, GICR_ISENABLER0, sc->sc_enabled_sgippi); 311 312 /* Wait for register write to complete */ 313 while (gicr_read_4(sc, ci->ci_gic_redist, GICR_CTLR) & GICR_CTLR_RWP) 314 ; 315 } 316 317 static uint64_t 318 gicv3_cpu_identity(void) 319 { 320 u_int aff3, aff2, aff1, aff0; 321 322 const register_t mpidr = cpu_mpidr_aff_read(); 323 aff0 = __SHIFTOUT(mpidr, MPIDR_AFF0); 324 aff1 = __SHIFTOUT(mpidr, MPIDR_AFF1); 325 aff2 = __SHIFTOUT(mpidr, MPIDR_AFF2); 326 aff3 = __SHIFTOUT(mpidr, MPIDR_AFF3); 327 328 return __SHIFTIN(aff0, GICR_TYPER_Affinity_Value_Aff0) | 329 __SHIFTIN(aff1, GICR_TYPER_Affinity_Value_Aff1) | 330 __SHIFTIN(aff2, GICR_TYPER_Affinity_Value_Aff2) | 331 __SHIFTIN(aff3, GICR_TYPER_Affinity_Value_Aff3); 332 } 333 334 static u_int 335 gicv3_find_redist(struct gicv3_softc *sc) 336 { 337 uint64_t gicr_typer; 338 u_int n; 339 340 const uint64_t cpu_identity = gicv3_cpu_identity(); 341 342 for (n = 0; n < sc->sc_bsh_r_count; n++) { 343 gicr_typer = gicr_read_8(sc, n, GICR_TYPER); 344 if ((gicr_typer & GICR_TYPER_Affinity_Value) == cpu_identity) 345 return n; 346 } 347 348 const u_int aff0 = __SHIFTOUT(cpu_identity, GICR_TYPER_Affinity_Value_Aff0); 349 const u_int aff1 = __SHIFTOUT(cpu_identity, GICR_TYPER_Affinity_Value_Aff1); 350 const u_int aff2 = __SHIFTOUT(cpu_identity, GICR_TYPER_Affinity_Value_Aff2); 351 const u_int aff3 = __SHIFTOUT(cpu_identity, GICR_TYPER_Affinity_Value_Aff3); 352 353 panic("%s: could not find GICv3 redistributor for cpu %d.%d.%d.%d", 354 cpu_name(curcpu()), aff3, aff2, aff1, aff0); 355 } 356 357 static uint64_t 358 gicv3_sgir(struct gicv3_softc *sc) 359 { 360 const uint64_t cpu_identity = gicv3_cpu_identity(); 361 362 const u_int aff0 = __SHIFTOUT(cpu_identity, GICR_TYPER_Affinity_Value_Aff0); 363 const u_int aff1 = __SHIFTOUT(cpu_identity, GICR_TYPER_Affinity_Value_Aff1); 364 const u_int aff2 = __SHIFTOUT(cpu_identity, GICR_TYPER_Affinity_Value_Aff2); 365 const u_int aff3 = __SHIFTOUT(cpu_identity, GICR_TYPER_Affinity_Value_Aff3); 366 367 return __SHIFTIN(__BIT(aff0), ICC_SGIR_EL1_TargetList) | 368 __SHIFTIN(aff1, ICC_SGIR_EL1_Aff1) | 369 __SHIFTIN(aff2, ICC_SGIR_EL1_Aff2) | 370 __SHIFTIN(aff3, ICC_SGIR_EL1_Aff3); 371 } 372 373 static void 374 gicv3_cpu_init(struct pic_softc *pic, struct cpu_info *ci) 375 { 376 struct gicv3_softc * const sc = PICTOSOFTC(pic); 377 uint32_t icc_sre, icc_ctlr, gicr_waker; 378 379 evcnt_attach_dynamic(&ci->ci_intr_preempt, EVCNT_TYPE_MISC, NULL, 380 ci->ci_cpuname, "intr preempt"); 381 382 ci->ci_gic_redist = gicv3_find_redist(sc); 383 ci->ci_gic_sgir = gicv3_sgir(sc); 384 385 /* Store route to CPU for SPIs */ 386 const uint64_t cpu_identity = gicv3_cpu_identity(); 387 const u_int aff0 = __SHIFTOUT(cpu_identity, GICR_TYPER_Affinity_Value_Aff0); 388 const u_int aff1 = __SHIFTOUT(cpu_identity, GICR_TYPER_Affinity_Value_Aff1); 389 const u_int aff2 = __SHIFTOUT(cpu_identity, GICR_TYPER_Affinity_Value_Aff2); 390 const u_int aff3 = __SHIFTOUT(cpu_identity, GICR_TYPER_Affinity_Value_Aff3); 391 sc->sc_irouter[cpu_index(ci)] = 392 __SHIFTIN(aff0, GICD_IROUTER_Aff0) | 393 __SHIFTIN(aff1, GICD_IROUTER_Aff1) | 394 __SHIFTIN(aff2, GICD_IROUTER_Aff2) | 395 __SHIFTIN(aff3, GICD_IROUTER_Aff3); 396 397 /* Enable System register access and disable IRQ/FIQ bypass */ 398 icc_sre = ICC_SRE_EL1_SRE | ICC_SRE_EL1_DFB | ICC_SRE_EL1_DIB; 399 icc_sre_write(icc_sre); 400 401 /* Mark the connected PE as being awake */ 402 gicr_waker = gicr_read_4(sc, ci->ci_gic_redist, GICR_WAKER); 403 gicr_waker &= ~GICR_WAKER_ProcessorSleep; 404 gicr_write_4(sc, ci->ci_gic_redist, GICR_WAKER, gicr_waker); 405 while (gicr_read_4(sc, ci->ci_gic_redist, GICR_WAKER) & GICR_WAKER_ChildrenAsleep) 406 ; 407 408 /* Set initial priority mask */ 409 gicv3_set_priority(pic, IPL_HIGH); 410 411 /* Set the binary point field to the minimum value */ 412 icc_bpr1_write(0); 413 414 /* Enable group 1 interrupt signaling */ 415 icc_igrpen1_write(ICC_IGRPEN_EL1_Enable); 416 417 /* Set EOI mode */ 418 icc_ctlr = icc_ctlr_read(); 419 icc_ctlr &= ~ICC_CTLR_EL1_EOImode; 420 icc_ctlr_write(icc_ctlr); 421 422 /* Enable redistributor */ 423 gicv3_redist_enable(sc, ci); 424 425 /* Allow IRQ exceptions */ 426 cpsie(I32_bit); 427 } 428 429 #ifdef MULTIPROCESSOR 430 static void 431 gicv3_ipi_send(struct pic_softc *pic, const kcpuset_t *kcp, u_long ipi) 432 { 433 struct cpu_info *ci; 434 uint64_t sgir; 435 436 sgir = __SHIFTIN(ipi, ICC_SGIR_EL1_INTID); 437 if (kcp == NULL) { 438 /* Interrupts routed to all PEs, excluding "self" */ 439 if (ncpu == 1) 440 return; 441 sgir |= ICC_SGIR_EL1_IRM; 442 } else { 443 /* Interrupt to exactly one PE */ 444 ci = cpu_lookup(kcpuset_ffs(kcp) - 1); 445 if (ci == curcpu()) 446 return; 447 sgir |= ci->ci_gic_sgir; 448 } 449 icc_sgi1r_write(sgir); 450 isb(); 451 } 452 453 static void 454 gicv3_get_affinity(struct pic_softc *pic, size_t irq, kcpuset_t *affinity) 455 { 456 struct gicv3_softc * const sc = PICTOSOFTC(pic); 457 const size_t group = irq / 32; 458 int n; 459 460 kcpuset_zero(affinity); 461 if (group == 0) { 462 /* All CPUs are targets for group 0 (SGI/PPI) */ 463 for (n = 0; n < ncpu; n++) { 464 if (sc->sc_irouter[n] != UINT64_MAX) 465 kcpuset_set(affinity, n); 466 } 467 } else { 468 /* Find distributor targets (SPI) */ 469 const uint64_t irouter = gicd_read_8(sc, GICD_IROUTER(irq)); 470 for (n = 0; n < ncpu; n++) { 471 if (irouter == GICD_IROUTER_Interrupt_Routing_mode || 472 irouter == sc->sc_irouter[n]) 473 kcpuset_set(affinity, n); 474 } 475 } 476 } 477 478 static int 479 gicv3_set_affinity(struct pic_softc *pic, size_t irq, const kcpuset_t *affinity) 480 { 481 struct gicv3_softc * const sc = PICTOSOFTC(pic); 482 const size_t group = irq / 32; 483 uint64_t irouter; 484 485 if (group == 0) 486 return EINVAL; 487 488 const int set = kcpuset_countset(affinity); 489 if (set == 1) { 490 irouter = sc->sc_irouter[kcpuset_ffs(affinity) - 1]; 491 } else if (set == ncpu && GIC_SUPPORTS_1OFN(sc)) { 492 irouter = GICD_IROUTER_Interrupt_Routing_mode; 493 } else { 494 return EINVAL; 495 } 496 497 gicd_write_8(sc, GICD_IROUTER(irq), irouter); 498 499 return 0; 500 } 501 #endif 502 503 static const struct pic_ops gicv3_picops = { 504 .pic_unblock_irqs = gicv3_unblock_irqs, 505 .pic_block_irqs = gicv3_block_irqs, 506 .pic_establish_irq = gicv3_establish_irq, 507 .pic_set_priority = gicv3_set_priority, 508 #ifdef MULTIPROCESSOR 509 .pic_cpu_init = gicv3_cpu_init, 510 .pic_ipi_send = gicv3_ipi_send, 511 .pic_get_affinity = gicv3_get_affinity, 512 .pic_set_affinity = gicv3_set_affinity, 513 #endif 514 }; 515 516 static void 517 gicv3_dcache_wb_range(vaddr_t va, vsize_t len) 518 { 519 cpu_dcache_wb_range(va, len); 520 dsb(sy); 521 } 522 523 static void 524 gicv3_lpi_unblock_irqs(struct pic_softc *pic, size_t irqbase, uint32_t mask) 525 { 526 struct gicv3_softc * const sc = LPITOSOFTC(pic); 527 int bit; 528 529 while ((bit = ffs(mask)) != 0) { 530 sc->sc_lpiconf.base[irqbase + bit - 1] |= GIC_LPICONF_Enable; 531 if (sc->sc_lpiconf_flush) 532 gicv3_dcache_wb_range((vaddr_t)&sc->sc_lpiconf.base[irqbase + bit - 1], 1); 533 mask &= ~__BIT(bit - 1); 534 } 535 536 if (!sc->sc_lpiconf_flush) 537 dsb(ishst); 538 } 539 540 static void 541 gicv3_lpi_block_irqs(struct pic_softc *pic, size_t irqbase, uint32_t mask) 542 { 543 struct gicv3_softc * const sc = LPITOSOFTC(pic); 544 int bit; 545 546 while ((bit = ffs(mask)) != 0) { 547 sc->sc_lpiconf.base[irqbase + bit - 1] &= ~GIC_LPICONF_Enable; 548 if (sc->sc_lpiconf_flush) 549 gicv3_dcache_wb_range((vaddr_t)&sc->sc_lpiconf.base[irqbase + bit - 1], 1); 550 mask &= ~__BIT(bit - 1); 551 } 552 553 if (!sc->sc_lpiconf_flush) 554 dsb(ishst); 555 } 556 557 static void 558 gicv3_lpi_establish_irq(struct pic_softc *pic, struct intrsource *is) 559 { 560 struct gicv3_softc * const sc = LPITOSOFTC(pic); 561 562 sc->sc_lpiconf.base[is->is_irq] = IPL_TO_PRIORITY(sc, is->is_ipl) | GIC_LPICONF_Res1; 563 564 if (sc->sc_lpiconf_flush) 565 gicv3_dcache_wb_range((vaddr_t)&sc->sc_lpiconf.base[is->is_irq], 1); 566 else 567 dsb(ishst); 568 } 569 570 static void 571 gicv3_lpi_cpu_init(struct pic_softc *pic, struct cpu_info *ci) 572 { 573 struct gicv3_softc * const sc = LPITOSOFTC(pic); 574 struct gicv3_lpi_callback *cb; 575 uint64_t propbase, pendbase; 576 uint32_t ctlr; 577 578 /* If physical LPIs are not supported on this redistributor, just return. */ 579 const uint64_t typer = gicr_read_8(sc, ci->ci_gic_redist, GICR_TYPER); 580 if ((typer & GICR_TYPER_PLPIS) == 0) 581 return; 582 583 /* Interrupt target address for this CPU, used by ITS when GITS_TYPER.PTA == 0 */ 584 sc->sc_processor_id[cpu_index(ci)] = __SHIFTOUT(typer, GICR_TYPER_Processor_Number); 585 586 /* Disable LPIs before making changes */ 587 ctlr = gicr_read_4(sc, ci->ci_gic_redist, GICR_CTLR); 588 ctlr &= ~GICR_CTLR_Enable_LPIs; 589 gicr_write_4(sc, ci->ci_gic_redist, GICR_CTLR, ctlr); 590 dsb(sy); 591 592 /* Setup the LPI configuration table */ 593 propbase = sc->sc_lpiconf.segs[0].ds_addr | 594 __SHIFTIN(ffs(pic->pic_maxsources) - 1, GICR_PROPBASER_IDbits) | 595 __SHIFTIN(GICR_Shareability_IS, GICR_PROPBASER_Shareability) | 596 __SHIFTIN(GICR_Cache_NORMAL_RA_WA_WB, GICR_PROPBASER_InnerCache); 597 gicr_write_8(sc, ci->ci_gic_redist, GICR_PROPBASER, propbase); 598 propbase = gicr_read_8(sc, ci->ci_gic_redist, GICR_PROPBASER); 599 if (__SHIFTOUT(propbase, GICR_PROPBASER_Shareability) != GICR_Shareability_IS) { 600 if (__SHIFTOUT(propbase, GICR_PROPBASER_Shareability) == GICR_Shareability_NS) { 601 propbase &= ~GICR_PROPBASER_Shareability; 602 propbase |= __SHIFTIN(GICR_Shareability_NS, GICR_PROPBASER_Shareability); 603 propbase &= ~GICR_PROPBASER_InnerCache; 604 propbase |= __SHIFTIN(GICR_Cache_NORMAL_NC, GICR_PROPBASER_InnerCache); 605 gicr_write_8(sc, ci->ci_gic_redist, GICR_PROPBASER, propbase); 606 } 607 sc->sc_lpiconf_flush = true; 608 } 609 610 /* Setup the LPI pending table */ 611 pendbase = sc->sc_lpipend[cpu_index(ci)].segs[0].ds_addr | 612 __SHIFTIN(GICR_Shareability_IS, GICR_PENDBASER_Shareability) | 613 __SHIFTIN(GICR_Cache_NORMAL_RA_WA_WB, GICR_PENDBASER_InnerCache); 614 gicr_write_8(sc, ci->ci_gic_redist, GICR_PENDBASER, pendbase); 615 pendbase = gicr_read_8(sc, ci->ci_gic_redist, GICR_PENDBASER); 616 if (__SHIFTOUT(pendbase, GICR_PENDBASER_Shareability) == GICR_Shareability_NS) { 617 pendbase &= ~GICR_PENDBASER_Shareability; 618 pendbase |= __SHIFTIN(GICR_Shareability_NS, GICR_PENDBASER_Shareability); 619 pendbase &= ~GICR_PENDBASER_InnerCache; 620 pendbase |= __SHIFTIN(GICR_Cache_NORMAL_NC, GICR_PENDBASER_InnerCache); 621 gicr_write_8(sc, ci->ci_gic_redist, GICR_PENDBASER, pendbase); 622 } 623 624 /* Enable LPIs */ 625 ctlr = gicr_read_4(sc, ci->ci_gic_redist, GICR_CTLR); 626 ctlr |= GICR_CTLR_Enable_LPIs; 627 gicr_write_4(sc, ci->ci_gic_redist, GICR_CTLR, ctlr); 628 dsb(sy); 629 630 /* Setup ITS if present */ 631 LIST_FOREACH(cb, &sc->sc_lpi_callbacks, list) 632 cb->cpu_init(cb->priv, ci); 633 } 634 635 #ifdef MULTIPROCESSOR 636 static void 637 gicv3_lpi_get_affinity(struct pic_softc *pic, size_t irq, kcpuset_t *affinity) 638 { 639 struct gicv3_softc * const sc = LPITOSOFTC(pic); 640 struct gicv3_lpi_callback *cb; 641 642 kcpuset_zero(affinity); 643 LIST_FOREACH(cb, &sc->sc_lpi_callbacks, list) 644 cb->get_affinity(cb->priv, irq, affinity); 645 } 646 647 static int 648 gicv3_lpi_set_affinity(struct pic_softc *pic, size_t irq, const kcpuset_t *affinity) 649 { 650 struct gicv3_softc * const sc = LPITOSOFTC(pic); 651 struct gicv3_lpi_callback *cb; 652 int error = EINVAL; 653 654 LIST_FOREACH(cb, &sc->sc_lpi_callbacks, list) { 655 error = cb->set_affinity(cb->priv, irq, affinity); 656 if (error != EPASSTHROUGH) 657 return error; 658 } 659 660 return EINVAL; 661 } 662 #endif 663 664 static const struct pic_ops gicv3_lpiops = { 665 .pic_unblock_irqs = gicv3_lpi_unblock_irqs, 666 .pic_block_irqs = gicv3_lpi_block_irqs, 667 .pic_establish_irq = gicv3_lpi_establish_irq, 668 #ifdef MULTIPROCESSOR 669 .pic_cpu_init = gicv3_lpi_cpu_init, 670 .pic_get_affinity = gicv3_lpi_get_affinity, 671 .pic_set_affinity = gicv3_lpi_set_affinity, 672 #endif 673 }; 674 675 void 676 gicv3_dma_alloc(struct gicv3_softc *sc, struct gicv3_dma *dma, bus_size_t len, bus_size_t align) 677 { 678 int nsegs, error; 679 680 dma->len = len; 681 error = bus_dmamem_alloc(sc->sc_dmat, dma->len, align, 0, dma->segs, 1, &nsegs, BUS_DMA_WAITOK); 682 if (error) 683 panic("bus_dmamem_alloc failed: %d", error); 684 error = bus_dmamem_map(sc->sc_dmat, dma->segs, nsegs, len, (void **)&dma->base, BUS_DMA_WAITOK); 685 if (error) 686 panic("bus_dmamem_map failed: %d", error); 687 error = bus_dmamap_create(sc->sc_dmat, len, 1, len, 0, BUS_DMA_WAITOK, &dma->map); 688 if (error) 689 panic("bus_dmamap_create failed: %d", error); 690 error = bus_dmamap_load(sc->sc_dmat, dma->map, dma->base, dma->len, NULL, BUS_DMA_WAITOK); 691 if (error) 692 panic("bus_dmamap_load failed: %d", error); 693 694 memset(dma->base, 0, dma->len); 695 bus_dmamap_sync(sc->sc_dmat, dma->map, 0, dma->len, BUS_DMASYNC_PREWRITE); 696 } 697 698 static void 699 gicv3_lpi_init(struct gicv3_softc *sc) 700 { 701 /* 702 * Allocate LPI configuration table 703 */ 704 gicv3_dma_alloc(sc, &sc->sc_lpiconf, sc->sc_lpi.pic_maxsources, 0x1000); 705 KASSERT((sc->sc_lpiconf.segs[0].ds_addr & ~GICR_PROPBASER_Physical_Address) == 0); 706 707 /* 708 * Allocate LPI pending tables 709 */ 710 const bus_size_t lpipend_sz = (8192 + sc->sc_lpi.pic_maxsources) / NBBY; 711 for (int cpuindex = 0; cpuindex < ncpu; cpuindex++) { 712 gicv3_dma_alloc(sc, &sc->sc_lpipend[cpuindex], lpipend_sz, 0x10000); 713 KASSERT((sc->sc_lpipend[cpuindex].segs[0].ds_addr & ~GICR_PENDBASER_Physical_Address) == 0); 714 } 715 } 716 717 void 718 gicv3_irq_handler(void *frame) 719 { 720 struct cpu_info * const ci = curcpu(); 721 struct gicv3_softc * const sc = gicv3_softc; 722 struct pic_softc *pic; 723 const int oldipl = ci->ci_cpl; 724 725 ci->ci_data.cpu_nintr++; 726 727 for (;;) { 728 const uint32_t iar = icc_iar1_read(); 729 dsb(sy); 730 const uint32_t irq = __SHIFTOUT(iar, ICC_IAR_INTID); 731 if (irq == ICC_IAR_INTID_SPURIOUS) 732 break; 733 734 pic = irq >= GIC_LPI_BASE ? &sc->sc_lpi : &sc->sc_pic; 735 if (irq - pic->pic_irqbase >= pic->pic_maxsources) 736 continue; 737 738 struct intrsource * const is = pic->pic_sources[irq - pic->pic_irqbase]; 739 KASSERT(is != NULL); 740 741 const bool early_eoi = irq < GIC_LPI_BASE && is->is_type == IST_EDGE; 742 743 const int ipl = is->is_ipl; 744 if (__predict_false(ipl < ci->ci_cpl)) { 745 pic_do_pending_ints(I32_bit, ipl, frame); 746 } else if (ci->ci_cpl != ipl) { 747 gicv3_set_priority(pic, ipl); 748 ci->ci_cpl = ipl; 749 } 750 751 if (early_eoi) { 752 icc_eoi1r_write(iar); 753 isb(); 754 } 755 756 const int64_t nintr = ci->ci_data.cpu_nintr; 757 758 cpsie(I32_bit); 759 pic_dispatch(is, frame); 760 cpsid(I32_bit); 761 762 if (nintr != ci->ci_data.cpu_nintr) 763 ci->ci_intr_preempt.ev_count++; 764 765 if (!early_eoi) { 766 icc_eoi1r_write(iar); 767 isb(); 768 } 769 } 770 771 pic_do_pending_ints(I32_bit, oldipl, frame); 772 } 773 774 static bool 775 gicv3_cpuif_is_nonsecure(struct gicv3_softc *sc) 776 { 777 /* 778 * Write 0 to bit7 and see if it sticks. This is only possible if 779 * we have a non-secure view of the PMR register. 780 */ 781 const uint32_t opmr = icc_pmr_read(); 782 icc_pmr_write(0); 783 const uint32_t npmr = icc_pmr_read(); 784 icc_pmr_write(opmr); 785 786 return (npmr & GICC_PMR_NONSECURE) == 0; 787 } 788 789 static bool 790 gicv3_dist_is_nonsecure(struct gicv3_softc *sc) 791 { 792 const uint32_t gicd_ctrl = gicd_read_4(sc, GICD_CTRL); 793 794 /* 795 * If security is enabled, we have a non-secure view of the IPRIORITYRn 796 * registers and LPI configuration priority fields. 797 */ 798 return (gicd_ctrl & GICD_CTRL_DS) == 0; 799 } 800 801 /* 802 * Rockchip RK3399 provides a different view of int priority registers 803 * depending on which firmware is in use. This is hard to detect in 804 * a way that could possibly break other boards, so only do this 805 * detection if we know we are on a RK3399 SoC. 806 */ 807 static void 808 gicv3_quirk_rockchip_rk3399(struct gicv3_softc *sc) 809 { 810 /* Detect the number of supported PMR bits */ 811 icc_pmr_write(0xff); 812 const uint8_t pmrbits = icc_pmr_read(); 813 814 /* Detect the number of supported IPRIORITYRn bits */ 815 const uint32_t oiprio = gicd_read_4(sc, GICD_IPRIORITYRn(8)); 816 gicd_write_4(sc, GICD_IPRIORITYRn(8), oiprio | 0xff); 817 const uint8_t pribits = gicd_read_4(sc, GICD_IPRIORITYRn(8)) & 0xff; 818 gicd_write_4(sc, GICD_IPRIORITYRn(8), oiprio); 819 820 /* 821 * If we see fewer PMR bits than IPRIORITYRn bits here, it means 822 * we have a secure view of IPRIORITYRn (this is not supposed to 823 * happen!). 824 */ 825 if (pmrbits < pribits) { 826 aprint_verbose_dev(sc->sc_dev, 827 "buggy RK3399 firmware detected; applying workaround\n"); 828 sc->sc_priority_shift = GIC_PRIO_SHIFT_S; 829 } 830 } 831 832 int 833 gicv3_init(struct gicv3_softc *sc) 834 { 835 int n; 836 837 KASSERT(CPU_IS_PRIMARY(curcpu())); 838 839 LIST_INIT(&sc->sc_lpi_callbacks); 840 841 for (n = 0; n < MAXCPUS; n++) 842 sc->sc_irouter[n] = UINT64_MAX; 843 844 sc->sc_gicd_typer = gicd_read_4(sc, GICD_TYPER); 845 846 /* 847 * We don't always have a consistent view of priorities between the 848 * CPU interface (ICC_PMR_EL1) and the GICD/GICR registers. Detect 849 * if we are making secure or non-secure accesses to each, and adjust 850 * the values that we write to each accordingly. 851 */ 852 const bool dist_ns = gicv3_dist_is_nonsecure(sc); 853 sc->sc_priority_shift = dist_ns ? GIC_PRIO_SHIFT_NS : GIC_PRIO_SHIFT_S; 854 const bool cpuif_ns = gicv3_cpuif_is_nonsecure(sc); 855 sc->sc_pmr_shift = cpuif_ns ? GIC_PRIO_SHIFT_NS : GIC_PRIO_SHIFT_S; 856 857 if ((sc->sc_quirks & GICV3_QUIRK_RK3399) != 0) 858 gicv3_quirk_rockchip_rk3399(sc); 859 860 aprint_verbose_dev(sc->sc_dev, 861 "iidr 0x%08x, cpuif %ssecure, dist %ssecure, " 862 "priority shift %d, pmr shift %d, quirks %#x\n", 863 gicd_read_4(sc, GICD_IIDR), 864 cpuif_ns ? "non-" : "", 865 dist_ns ? "non-" : "", 866 sc->sc_priority_shift, 867 sc->sc_pmr_shift, 868 sc->sc_quirks); 869 870 sc->sc_pic.pic_ops = &gicv3_picops; 871 sc->sc_pic.pic_maxsources = GICD_TYPER_LINES(sc->sc_gicd_typer); 872 snprintf(sc->sc_pic.pic_name, sizeof(sc->sc_pic.pic_name), "gicv3"); 873 #ifdef MULTIPROCESSOR 874 sc->sc_pic.pic_cpus = kcpuset_running; 875 #endif 876 pic_add(&sc->sc_pic, 0); 877 878 if ((sc->sc_gicd_typer & GICD_TYPER_LPIS) != 0) { 879 sc->sc_lpi.pic_ops = &gicv3_lpiops; 880 sc->sc_lpi.pic_maxsources = 8192; /* Min. required by GICv3 spec */ 881 snprintf(sc->sc_lpi.pic_name, sizeof(sc->sc_lpi.pic_name), "gicv3-lpi"); 882 pic_add(&sc->sc_lpi, GIC_LPI_BASE); 883 884 sc->sc_lpi_pool = vmem_create("gicv3-lpi", 0, sc->sc_lpi.pic_maxsources, 885 1, NULL, NULL, NULL, 0, VM_SLEEP, IPL_HIGH); 886 if (sc->sc_lpi_pool == NULL) 887 panic("failed to create gicv3 lpi pool\n"); 888 889 gicv3_lpi_init(sc); 890 } 891 892 KASSERT(gicv3_softc == NULL); 893 gicv3_softc = sc; 894 895 for (int i = 0; i < sc->sc_bsh_r_count; i++) { 896 const uint64_t gicr_typer = gicr_read_8(sc, i, GICR_TYPER); 897 const u_int aff0 = __SHIFTOUT(gicr_typer, GICR_TYPER_Affinity_Value_Aff0); 898 const u_int aff1 = __SHIFTOUT(gicr_typer, GICR_TYPER_Affinity_Value_Aff1); 899 const u_int aff2 = __SHIFTOUT(gicr_typer, GICR_TYPER_Affinity_Value_Aff2); 900 const u_int aff3 = __SHIFTOUT(gicr_typer, GICR_TYPER_Affinity_Value_Aff3); 901 902 aprint_debug_dev(sc->sc_dev, "redist %d: cpu %d.%d.%d.%d\n", 903 i, aff3, aff2, aff1, aff0); 904 } 905 906 gicv3_dist_enable(sc); 907 908 gicv3_cpu_init(&sc->sc_pic, curcpu()); 909 if ((sc->sc_gicd_typer & GICD_TYPER_LPIS) != 0) 910 gicv3_lpi_cpu_init(&sc->sc_lpi, curcpu()); 911 912 #ifdef MULTIPROCESSOR 913 intr_establish_xname(IPI_AST, IPL_VM, IST_MPSAFE | IST_EDGE, pic_ipi_ast, (void *)-1, "IPI ast"); 914 intr_establish_xname(IPI_XCALL, IPL_HIGH, IST_MPSAFE | IST_EDGE, pic_ipi_xcall, (void *)-1, "IPI xcall"); 915 intr_establish_xname(IPI_GENERIC, IPL_HIGH, IST_MPSAFE | IST_EDGE, pic_ipi_generic, (void *)-1, "IPI generic"); 916 intr_establish_xname(IPI_NOP, IPL_VM, IST_MPSAFE | IST_EDGE, pic_ipi_nop, (void *)-1, "IPI nop"); 917 intr_establish_xname(IPI_SHOOTDOWN, IPL_SCHED, IST_MPSAFE | IST_EDGE, pic_ipi_shootdown, (void *)-1, "IPI shootdown"); 918 #ifdef DDB 919 intr_establish_xname(IPI_DDB, IPL_HIGH, IST_MPSAFE | IST_EDGE, pic_ipi_ddb, NULL, "IPI ddb"); 920 #endif 921 #ifdef __HAVE_PREEMPTION 922 intr_establish_xname(IPI_KPREEMPT, IPL_VM, IST_MPSAFE | IST_EDGE, pic_ipi_kpreempt, (void *)-1, "IPI kpreempt"); 923 #endif 924 #endif 925 926 return 0; 927 } 928
Indexes created Thu Oct 02 10:09:58 GMT 2025