gicv3.c revision 1.26
1 /* $NetBSD: gicv3.c,v 1.26 2020/10/30 18:54:36 skrll Exp $ */ 2 3 /*- 4 * Copyright (c) 2018 Jared McNeill <jmcneill (at) invisible.ca> 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 17 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 18 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 19 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 20 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 21 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 22 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 23 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 24 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 * SUCH DAMAGE. 27 */ 28 29 #include "opt_multiprocessor.h" 30 31 #define _INTR_PRIVATE 32 33 #include <sys/cdefs.h> 34 __KERNEL_RCSID(0, "$NetBSD: gicv3.c,v 1.26 2020/10/30 18:54:36 skrll Exp $"); 35 36 #include <sys/param.h> 37 #include <sys/kernel.h> 38 #include <sys/bus.h> 39 #include <sys/device.h> 40 #include <sys/intr.h> 41 #include <sys/systm.h> 42 #include <sys/cpu.h> 43 #include <sys/vmem.h> 44 45 #include <machine/cpufunc.h> 46 47 #include <arm/locore.h> 48 #include <arm/armreg.h> 49 50 #include <arm/cortex/gicv3.h> 51 #include <arm/cortex/gic_reg.h> 52 53 #define PICTOSOFTC(pic) \ 54 ((void *)((uintptr_t)(pic) - offsetof(struct gicv3_softc, sc_pic))) 55 #define LPITOSOFTC(lpi) \ 56 ((void *)((uintptr_t)(lpi) - offsetof(struct gicv3_softc, sc_lpi))) 57 58 #define IPL_TO_PRIORITY(sc, ipl) (((0xff - (ipl)) << (sc)->sc_priority_shift) & 0xff) 59 #define IPL_TO_PMR(sc, ipl) (((0xff - (ipl)) << (sc)->sc_pmr_shift) & 0xff) 60 #define IPL_TO_LPIPRIO(sc, ipl) (((0xff - (ipl)) << 4) & 0xff) 61 62 static struct gicv3_softc *gicv3_softc; 63 64 static inline uint32_t 65 gicd_read_4(struct gicv3_softc *sc, bus_size_t reg) 66 { 67 return bus_space_read_4(sc->sc_bst, sc->sc_bsh_d, reg); 68 } 69 70 static inline void 71 gicd_write_4(struct gicv3_softc *sc, bus_size_t reg, uint32_t val) 72 { 73 bus_space_write_4(sc->sc_bst, sc->sc_bsh_d, reg, val); 74 } 75 76 static inline uint64_t 77 gicd_read_8(struct gicv3_softc *sc, bus_size_t reg) 78 { 79 return bus_space_read_8(sc->sc_bst, sc->sc_bsh_d, reg); 80 } 81 82 static inline void 83 gicd_write_8(struct gicv3_softc *sc, bus_size_t reg, uint64_t val) 84 { 85 bus_space_write_8(sc->sc_bst, sc->sc_bsh_d, reg, val); 86 } 87 88 static inline uint32_t 89 gicr_read_4(struct gicv3_softc *sc, u_int index, bus_size_t reg) 90 { 91 KASSERT(index < sc->sc_bsh_r_count); 92 return bus_space_read_4(sc->sc_bst, sc->sc_bsh_r[index], reg); 93 } 94 95 static inline void 96 gicr_write_4(struct gicv3_softc *sc, u_int index, bus_size_t reg, uint32_t val) 97 { 98 KASSERT(index < sc->sc_bsh_r_count); 99 bus_space_write_4(sc->sc_bst, sc->sc_bsh_r[index], reg, val); 100 } 101 102 static inline uint64_t 103 gicr_read_8(struct gicv3_softc *sc, u_int index, bus_size_t reg) 104 { 105 KASSERT(index < sc->sc_bsh_r_count); 106 return bus_space_read_8(sc->sc_bst, sc->sc_bsh_r[index], reg); 107 } 108 109 static inline void 110 gicr_write_8(struct gicv3_softc *sc, u_int index, bus_size_t reg, uint64_t val) 111 { 112 KASSERT(index < sc->sc_bsh_r_count); 113 bus_space_write_8(sc->sc_bst, sc->sc_bsh_r[index], reg, val); 114 } 115 116 static void 117 gicv3_unblock_irqs(struct pic_softc *pic, size_t irqbase, uint32_t mask) 118 { 119 struct gicv3_softc * const sc = PICTOSOFTC(pic); 120 struct cpu_info * const ci = curcpu(); 121 const u_int group = irqbase / 32; 122 123 if (group == 0) { 124 sc->sc_enabled_sgippi |= mask; 125 gicr_write_4(sc, ci->ci_gic_redist, GICR_ISENABLER0, mask); 126 while (gicr_read_4(sc, ci->ci_gic_redist, GICR_CTLR) & GICR_CTLR_RWP) 127 ; 128 } else { 129 gicd_write_4(sc, GICD_ISENABLERn(group), mask); 130 while (gicd_read_4(sc, GICD_CTRL) & GICD_CTRL_RWP) 131 ; 132 } 133 } 134 135 static void 136 gicv3_block_irqs(struct pic_softc *pic, size_t irqbase, uint32_t mask) 137 { 138 struct gicv3_softc * const sc = PICTOSOFTC(pic); 139 struct cpu_info * const ci = curcpu(); 140 const u_int group = irqbase / 32; 141 142 if (group == 0) { 143 sc->sc_enabled_sgippi &= ~mask; 144 gicr_write_4(sc, ci->ci_gic_redist, GICR_ICENABLER0, mask); 145 while (gicr_read_4(sc, ci->ci_gic_redist, GICR_CTLR) & GICR_CTLR_RWP) 146 ; 147 } else { 148 gicd_write_4(sc, GICD_ICENABLERn(group), mask); 149 while (gicd_read_4(sc, GICD_CTRL) & GICD_CTRL_RWP) 150 ; 151 } 152 } 153 154 static void 155 gicv3_establish_irq(struct pic_softc *pic, struct intrsource *is) 156 { 157 struct gicv3_softc * const sc = PICTOSOFTC(pic); 158 const u_int group = is->is_irq / 32; 159 uint32_t ipriority, icfg; 160 uint64_t irouter; 161 u_int n; 162 163 const u_int ipriority_val = IPL_TO_PRIORITY(sc, is->is_ipl); 164 const u_int ipriority_shift = (is->is_irq & 0x3) * 8; 165 const u_int icfg_shift = (is->is_irq & 0xf) * 2; 166 167 if (group == 0) { 168 /* SGIs and PPIs are always MP-safe */ 169 is->is_mpsafe = true; 170 171 /* Update interrupt configuration and priority on all redistributors */ 172 for (n = 0; n < sc->sc_bsh_r_count; n++) { 173 icfg = gicr_read_4(sc, n, GICR_ICFGRn(is->is_irq / 16)); 174 if (is->is_type == IST_LEVEL) 175 icfg &= ~(0x2 << icfg_shift); 176 if (is->is_type == IST_EDGE) 177 icfg |= (0x2 << icfg_shift); 178 gicr_write_4(sc, n, GICR_ICFGRn(is->is_irq / 16), icfg); 179 180 ipriority = gicr_read_4(sc, n, GICR_IPRIORITYRn(is->is_irq / 4)); 181 ipriority &= ~(0xffU << ipriority_shift); 182 ipriority |= (ipriority_val << ipriority_shift); 183 gicr_write_4(sc, n, GICR_IPRIORITYRn(is->is_irq / 4), ipriority); 184 } 185 } else { 186 if (is->is_mpsafe) { 187 /* Route MP-safe interrupts to all participating PEs */ 188 irouter = GICD_IROUTER_Interrupt_Routing_mode; 189 } else { 190 /* Route non-MP-safe interrupts to the primary PE only */ 191 irouter = sc->sc_irouter[0]; 192 } 193 gicd_write_8(sc, GICD_IROUTER(is->is_irq), irouter); 194 195 /* Update interrupt configuration */ 196 icfg = gicd_read_4(sc, GICD_ICFGRn(is->is_irq / 16)); 197 if (is->is_type == IST_LEVEL) 198 icfg &= ~(0x2 << icfg_shift); 199 if (is->is_type == IST_EDGE) 200 icfg |= (0x2 << icfg_shift); 201 gicd_write_4(sc, GICD_ICFGRn(is->is_irq / 16), icfg); 202 203 /* Update interrupt priority */ 204 ipriority = gicd_read_4(sc, GICD_IPRIORITYRn(is->is_irq / 4)); 205 ipriority &= ~(0xffU << ipriority_shift); 206 ipriority |= (ipriority_val << ipriority_shift); 207 gicd_write_4(sc, GICD_IPRIORITYRn(is->is_irq / 4), ipriority); 208 } 209 } 210 211 static void 212 gicv3_set_priority(struct pic_softc *pic, int ipl) 213 { 214 struct gicv3_softc * const sc = PICTOSOFTC(pic); 215 216 icc_pmr_write(IPL_TO_PMR(sc, ipl)); 217 isb(); 218 } 219 220 static void 221 gicv3_dist_enable(struct gicv3_softc *sc) 222 { 223 uint32_t gicd_ctrl; 224 u_int n; 225 226 /* Disable the distributor */ 227 gicd_write_4(sc, GICD_CTRL, 0); 228 229 /* Wait for register write to complete */ 230 while (gicd_read_4(sc, GICD_CTRL) & GICD_CTRL_RWP) 231 ; 232 233 /* Clear all INTID enable bits */ 234 for (n = 32; n < sc->sc_pic.pic_maxsources; n += 32) 235 gicd_write_4(sc, GICD_ICENABLERn(n / 32), ~0); 236 237 /* Set default priorities to lowest */ 238 for (n = 32; n < sc->sc_pic.pic_maxsources; n += 4) 239 gicd_write_4(sc, GICD_IPRIORITYRn(n / 4), ~0); 240 241 /* Set all interrupts to G1NS */ 242 for (n = 32; n < sc->sc_pic.pic_maxsources; n += 32) { 243 gicd_write_4(sc, GICD_IGROUPRn(n / 32), ~0); 244 gicd_write_4(sc, GICD_IGRPMODRn(n / 32), 0); 245 } 246 247 /* Set all interrupts level-sensitive by default */ 248 for (n = 32; n < sc->sc_pic.pic_maxsources; n += 16) 249 gicd_write_4(sc, GICD_ICFGRn(n / 16), 0); 250 251 /* Wait for register writes to complete */ 252 while (gicd_read_4(sc, GICD_CTRL) & GICD_CTRL_RWP) 253 ; 254 255 /* Enable Affinity routing and G1NS interrupts */ 256 gicd_ctrl = GICD_CTRL_EnableGrp1A | GICD_CTRL_ARE_NS; 257 gicd_write_4(sc, GICD_CTRL, gicd_ctrl); 258 } 259 260 static void 261 gicv3_redist_enable(struct gicv3_softc *sc, struct cpu_info *ci) 262 { 263 uint32_t icfg; 264 u_int n, o; 265 266 /* Clear INTID enable bits */ 267 gicr_write_4(sc, ci->ci_gic_redist, GICR_ICENABLER0, ~0); 268 269 /* Wait for register write to complete */ 270 while (gicr_read_4(sc, ci->ci_gic_redist, GICR_CTLR) & GICR_CTLR_RWP) 271 ; 272 273 /* Set default priorities */ 274 for (n = 0; n < 32; n += 4) { 275 uint32_t priority = 0; 276 size_t byte_shift = 0; 277 for (o = 0; o < 4; o++, byte_shift += 8) { 278 struct intrsource * const is = sc->sc_pic.pic_sources[n + o]; 279 if (is == NULL) 280 priority |= (0xffU << byte_shift); 281 else { 282 const u_int ipriority_val = IPL_TO_PRIORITY(sc, is->is_ipl); 283 priority |= ipriority_val << byte_shift; 284 } 285 } 286 gicr_write_4(sc, ci->ci_gic_redist, GICR_IPRIORITYRn(n / 4), priority); 287 } 288 289 /* Set all interrupts to G1NS */ 290 gicr_write_4(sc, ci->ci_gic_redist, GICR_IGROUPR0, ~0); 291 gicr_write_4(sc, ci->ci_gic_redist, GICR_IGRPMODR0, 0); 292 293 /* Restore PPI configs */ 294 for (n = 0, icfg = 0; n < 16; n++) { 295 struct intrsource * const is = sc->sc_pic.pic_sources[16 + n]; 296 if (is != NULL && is->is_type == IST_EDGE) 297 icfg |= (0x2 << (n * 2)); 298 } 299 gicr_write_4(sc, ci->ci_gic_redist, GICR_ICFGRn(1), icfg); 300 301 /* Restore current enable bits */ 302 gicr_write_4(sc, ci->ci_gic_redist, GICR_ISENABLER0, sc->sc_enabled_sgippi); 303 304 /* Wait for register write to complete */ 305 while (gicr_read_4(sc, ci->ci_gic_redist, GICR_CTLR) & GICR_CTLR_RWP) 306 ; 307 } 308 309 static uint64_t 310 gicv3_cpu_identity(void) 311 { 312 u_int aff3, aff2, aff1, aff0; 313 314 const register_t mpidr = cpu_mpidr_aff_read(); 315 aff0 = __SHIFTOUT(mpidr, MPIDR_AFF0); 316 aff1 = __SHIFTOUT(mpidr, MPIDR_AFF1); 317 aff2 = __SHIFTOUT(mpidr, MPIDR_AFF2); 318 aff3 = __SHIFTOUT(mpidr, MPIDR_AFF3); 319 320 return __SHIFTIN(aff0, GICR_TYPER_Affinity_Value_Aff0) | 321 __SHIFTIN(aff1, GICR_TYPER_Affinity_Value_Aff1) | 322 __SHIFTIN(aff2, GICR_TYPER_Affinity_Value_Aff2) | 323 __SHIFTIN(aff3, GICR_TYPER_Affinity_Value_Aff3); 324 } 325 326 static u_int 327 gicv3_find_redist(struct gicv3_softc *sc) 328 { 329 uint64_t gicr_typer; 330 u_int n; 331 332 const uint64_t cpu_identity = gicv3_cpu_identity(); 333 334 for (n = 0; n < sc->sc_bsh_r_count; n++) { 335 gicr_typer = gicr_read_8(sc, n, GICR_TYPER); 336 if ((gicr_typer & GICR_TYPER_Affinity_Value) == cpu_identity) 337 return n; 338 } 339 340 const u_int aff0 = __SHIFTOUT(cpu_identity, GICR_TYPER_Affinity_Value_Aff0); 341 const u_int aff1 = __SHIFTOUT(cpu_identity, GICR_TYPER_Affinity_Value_Aff1); 342 const u_int aff2 = __SHIFTOUT(cpu_identity, GICR_TYPER_Affinity_Value_Aff2); 343 const u_int aff3 = __SHIFTOUT(cpu_identity, GICR_TYPER_Affinity_Value_Aff3); 344 345 panic("%s: could not find GICv3 redistributor for cpu %d.%d.%d.%d", 346 cpu_name(curcpu()), aff3, aff2, aff1, aff0); 347 } 348 349 static uint64_t 350 gicv3_sgir(struct gicv3_softc *sc) 351 { 352 const uint64_t cpu_identity = gicv3_cpu_identity(); 353 354 const u_int aff0 = __SHIFTOUT(cpu_identity, GICR_TYPER_Affinity_Value_Aff0); 355 const u_int aff1 = __SHIFTOUT(cpu_identity, GICR_TYPER_Affinity_Value_Aff1); 356 const u_int aff2 = __SHIFTOUT(cpu_identity, GICR_TYPER_Affinity_Value_Aff2); 357 const u_int aff3 = __SHIFTOUT(cpu_identity, GICR_TYPER_Affinity_Value_Aff3); 358 359 return __SHIFTIN(__BIT(aff0), ICC_SGIR_EL1_TargetList) | 360 __SHIFTIN(aff1, ICC_SGIR_EL1_Aff1) | 361 __SHIFTIN(aff2, ICC_SGIR_EL1_Aff2) | 362 __SHIFTIN(aff3, ICC_SGIR_EL1_Aff3); 363 } 364 365 static void 366 gicv3_cpu_init(struct pic_softc *pic, struct cpu_info *ci) 367 { 368 struct gicv3_softc * const sc = PICTOSOFTC(pic); 369 uint32_t icc_sre, icc_ctlr, gicr_waker; 370 371 ci->ci_gic_redist = gicv3_find_redist(sc); 372 ci->ci_gic_sgir = gicv3_sgir(sc); 373 374 /* Store route to CPU for SPIs */ 375 const uint64_t cpu_identity = gicv3_cpu_identity(); 376 const u_int aff0 = __SHIFTOUT(cpu_identity, GICR_TYPER_Affinity_Value_Aff0); 377 const u_int aff1 = __SHIFTOUT(cpu_identity, GICR_TYPER_Affinity_Value_Aff1); 378 const u_int aff2 = __SHIFTOUT(cpu_identity, GICR_TYPER_Affinity_Value_Aff2); 379 const u_int aff3 = __SHIFTOUT(cpu_identity, GICR_TYPER_Affinity_Value_Aff3); 380 sc->sc_irouter[cpu_index(ci)] = 381 __SHIFTIN(aff0, GICD_IROUTER_Aff0) | 382 __SHIFTIN(aff1, GICD_IROUTER_Aff1) | 383 __SHIFTIN(aff2, GICD_IROUTER_Aff2) | 384 __SHIFTIN(aff3, GICD_IROUTER_Aff3); 385 386 /* Enable System register access and disable IRQ/FIQ bypass */ 387 icc_sre = ICC_SRE_EL1_SRE | ICC_SRE_EL1_DFB | ICC_SRE_EL1_DIB; 388 icc_sre_write(icc_sre); 389 390 /* Mark the connected PE as being awake */ 391 gicr_waker = gicr_read_4(sc, ci->ci_gic_redist, GICR_WAKER); 392 gicr_waker &= ~GICR_WAKER_ProcessorSleep; 393 gicr_write_4(sc, ci->ci_gic_redist, GICR_WAKER, gicr_waker); 394 while (gicr_read_4(sc, ci->ci_gic_redist, GICR_WAKER) & GICR_WAKER_ChildrenAsleep) 395 ; 396 397 /* Set initial priority mask */ 398 gicv3_set_priority(pic, IPL_HIGH); 399 400 /* Set the binary point field to the minimum value */ 401 icc_bpr1_write(0); 402 403 /* Enable group 1 interrupt signaling */ 404 icc_igrpen1_write(ICC_IGRPEN_EL1_Enable); 405 406 /* Set EOI mode */ 407 icc_ctlr = icc_ctlr_read(); 408 icc_ctlr &= ~ICC_CTLR_EL1_EOImode; 409 icc_ctlr_write(icc_ctlr); 410 411 /* Enable redistributor */ 412 gicv3_redist_enable(sc, ci); 413 414 /* Allow IRQ exceptions */ 415 cpsie(I32_bit); 416 } 417 418 #ifdef MULTIPROCESSOR 419 static void 420 gicv3_ipi_send(struct pic_softc *pic, const kcpuset_t *kcp, u_long ipi) 421 { 422 CPU_INFO_ITERATOR cii; 423 struct cpu_info *ci; 424 uint64_t intid, aff, targets; 425 426 intid = __SHIFTIN(ipi, ICC_SGIR_EL1_INTID); 427 if (kcp == NULL) { 428 /* Interrupts routed to all PEs, excluding "self" */ 429 if (ncpu == 1) 430 return; 431 icc_sgi1r_write(intid | ICC_SGIR_EL1_IRM); 432 } else { 433 /* Interrupts routed to specific PEs */ 434 aff = 0; 435 targets = 0; 436 for (CPU_INFO_FOREACH(cii, ci)) { 437 if (!kcpuset_isset(kcp, cpu_index(ci))) 438 continue; 439 if ((ci->ci_gic_sgir & ICC_SGIR_EL1_Aff) != aff) { 440 if (targets != 0) { 441 icc_sgi1r_write(intid | aff | targets); 442 isb(); 443 targets = 0; 444 } 445 aff = (ci->ci_gic_sgir & ICC_SGIR_EL1_Aff); 446 } 447 targets |= (ci->ci_gic_sgir & ICC_SGIR_EL1_TargetList); 448 } 449 if (targets != 0) { 450 icc_sgi1r_write(intid | aff | targets); 451 isb(); 452 } 453 } 454 } 455 456 static void 457 gicv3_get_affinity(struct pic_softc *pic, size_t irq, kcpuset_t *affinity) 458 { 459 struct gicv3_softc * const sc = PICTOSOFTC(pic); 460 const size_t group = irq / 32; 461 int n; 462 463 kcpuset_zero(affinity); 464 if (group == 0) { 465 /* All CPUs are targets for group 0 (SGI/PPI) */ 466 for (n = 0; n < ncpu; n++) { 467 if (sc->sc_irouter[n] != UINT64_MAX) 468 kcpuset_set(affinity, n); 469 } 470 } else { 471 /* Find distributor targets (SPI) */ 472 const uint64_t irouter = gicd_read_8(sc, GICD_IROUTER(irq)); 473 for (n = 0; n < ncpu; n++) { 474 if (irouter == GICD_IROUTER_Interrupt_Routing_mode || 475 irouter == sc->sc_irouter[n]) 476 kcpuset_set(affinity, n); 477 } 478 } 479 } 480 481 static int 482 gicv3_set_affinity(struct pic_softc *pic, size_t irq, const kcpuset_t *affinity) 483 { 484 struct gicv3_softc * const sc = PICTOSOFTC(pic); 485 const size_t group = irq / 32; 486 uint64_t irouter; 487 488 if (group == 0) 489 return EINVAL; 490 491 const int set = kcpuset_countset(affinity); 492 if (set == ncpu) 493 irouter = GICD_IROUTER_Interrupt_Routing_mode; 494 else if (set == 1) 495 irouter = sc->sc_irouter[kcpuset_ffs(affinity) - 1]; 496 else 497 return EINVAL; 498 499 gicd_write_8(sc, GICD_IROUTER(irq), irouter); 500 501 return 0; 502 } 503 #endif 504 505 static const struct pic_ops gicv3_picops = { 506 .pic_unblock_irqs = gicv3_unblock_irqs, 507 .pic_block_irqs = gicv3_block_irqs, 508 .pic_establish_irq = gicv3_establish_irq, 509 .pic_set_priority = gicv3_set_priority, 510 #ifdef MULTIPROCESSOR 511 .pic_cpu_init = gicv3_cpu_init, 512 .pic_ipi_send = gicv3_ipi_send, 513 .pic_get_affinity = gicv3_get_affinity, 514 .pic_set_affinity = gicv3_set_affinity, 515 #endif 516 }; 517 518 static void 519 gicv3_lpi_unblock_irqs(struct pic_softc *pic, size_t irqbase, uint32_t mask) 520 { 521 struct gicv3_softc * const sc = LPITOSOFTC(pic); 522 int bit; 523 524 while ((bit = ffs(mask)) != 0) { 525 sc->sc_lpiconf.base[irqbase + bit - 1] |= GIC_LPICONF_Enable; 526 if (sc->sc_lpiconf_flush) 527 cpu_dcache_wb_range((vaddr_t)&sc->sc_lpiconf.base[irqbase + bit - 1], 1); 528 mask &= ~__BIT(bit - 1); 529 } 530 531 if (!sc->sc_lpiconf_flush) 532 dsb(ishst); 533 } 534 535 static void 536 gicv3_lpi_block_irqs(struct pic_softc *pic, size_t irqbase, uint32_t mask) 537 { 538 struct gicv3_softc * const sc = LPITOSOFTC(pic); 539 int bit; 540 541 while ((bit = ffs(mask)) != 0) { 542 sc->sc_lpiconf.base[irqbase + bit - 1] &= ~GIC_LPICONF_Enable; 543 if (sc->sc_lpiconf_flush) 544 cpu_dcache_wb_range((vaddr_t)&sc->sc_lpiconf.base[irqbase + bit - 1], 1); 545 mask &= ~__BIT(bit - 1); 546 } 547 548 if (!sc->sc_lpiconf_flush) 549 dsb(ishst); 550 } 551 552 static void 553 gicv3_lpi_establish_irq(struct pic_softc *pic, struct intrsource *is) 554 { 555 struct gicv3_softc * const sc = LPITOSOFTC(pic); 556 557 sc->sc_lpiconf.base[is->is_irq] = IPL_TO_LPIPRIO(sc, is->is_ipl) | GIC_LPICONF_Res1; 558 559 if (sc->sc_lpiconf_flush) 560 cpu_dcache_wb_range((vaddr_t)&sc->sc_lpiconf.base[is->is_irq], 1); 561 else 562 dsb(ishst); 563 } 564 565 static void 566 gicv3_lpi_cpu_init(struct pic_softc *pic, struct cpu_info *ci) 567 { 568 struct gicv3_softc * const sc = LPITOSOFTC(pic); 569 struct gicv3_lpi_callback *cb; 570 uint64_t propbase, pendbase; 571 uint32_t ctlr; 572 573 /* If physical LPIs are not supported on this redistributor, just return. */ 574 const uint64_t typer = gicr_read_8(sc, ci->ci_gic_redist, GICR_TYPER); 575 if ((typer & GICR_TYPER_PLPIS) == 0) 576 return; 577 578 /* Interrupt target address for this CPU, used by ITS when GITS_TYPER.PTA == 0 */ 579 sc->sc_processor_id[cpu_index(ci)] = __SHIFTOUT(typer, GICR_TYPER_Processor_Number); 580 581 /* Disable LPIs before making changes */ 582 ctlr = gicr_read_4(sc, ci->ci_gic_redist, GICR_CTLR); 583 ctlr &= ~GICR_CTLR_Enable_LPIs; 584 gicr_write_4(sc, ci->ci_gic_redist, GICR_CTLR, ctlr); 585 dsb(sy); 586 587 /* Setup the LPI configuration table */ 588 propbase = sc->sc_lpiconf.segs[0].ds_addr | 589 __SHIFTIN(ffs(pic->pic_maxsources) - 1, GICR_PROPBASER_IDbits) | 590 __SHIFTIN(GICR_Shareability_IS, GICR_PROPBASER_Shareability) | 591 __SHIFTIN(GICR_Cache_NORMAL_RA_WA_WB, GICR_PROPBASER_InnerCache); 592 gicr_write_8(sc, ci->ci_gic_redist, GICR_PROPBASER, propbase); 593 propbase = gicr_read_8(sc, ci->ci_gic_redist, GICR_PROPBASER); 594 if (__SHIFTOUT(propbase, GICR_PROPBASER_Shareability) != GICR_Shareability_IS) { 595 if (__SHIFTOUT(propbase, GICR_PROPBASER_Shareability) == GICR_Shareability_NS) { 596 propbase &= ~GICR_PROPBASER_Shareability; 597 propbase |= __SHIFTIN(GICR_Shareability_NS, GICR_PROPBASER_Shareability); 598 propbase &= ~GICR_PROPBASER_InnerCache; 599 propbase |= __SHIFTIN(GICR_Cache_NORMAL_NC, GICR_PROPBASER_InnerCache); 600 gicr_write_8(sc, ci->ci_gic_redist, GICR_PROPBASER, propbase); 601 } 602 sc->sc_lpiconf_flush = true; 603 } 604 605 /* Setup the LPI pending table */ 606 pendbase = sc->sc_lpipend[cpu_index(ci)].segs[0].ds_addr | 607 __SHIFTIN(GICR_Shareability_IS, GICR_PENDBASER_Shareability) | 608 __SHIFTIN(GICR_Cache_NORMAL_RA_WA_WB, GICR_PENDBASER_InnerCache); 609 gicr_write_8(sc, ci->ci_gic_redist, GICR_PENDBASER, pendbase); 610 pendbase = gicr_read_8(sc, ci->ci_gic_redist, GICR_PENDBASER); 611 if (__SHIFTOUT(pendbase, GICR_PENDBASER_Shareability) == GICR_Shareability_NS) { 612 pendbase &= ~GICR_PENDBASER_Shareability; 613 pendbase |= __SHIFTIN(GICR_Shareability_NS, GICR_PENDBASER_Shareability); 614 pendbase &= ~GICR_PENDBASER_InnerCache; 615 pendbase |= __SHIFTIN(GICR_Cache_NORMAL_NC, GICR_PENDBASER_InnerCache); 616 gicr_write_8(sc, ci->ci_gic_redist, GICR_PENDBASER, pendbase); 617 } 618 619 /* Enable LPIs */ 620 ctlr = gicr_read_4(sc, ci->ci_gic_redist, GICR_CTLR); 621 ctlr |= GICR_CTLR_Enable_LPIs; 622 gicr_write_4(sc, ci->ci_gic_redist, GICR_CTLR, ctlr); 623 dsb(sy); 624 625 /* Setup ITS if present */ 626 LIST_FOREACH(cb, &sc->sc_lpi_callbacks, list) 627 cb->cpu_init(cb->priv, ci); 628 } 629 630 #ifdef MULTIPROCESSOR 631 static void 632 gicv3_lpi_get_affinity(struct pic_softc *pic, size_t irq, kcpuset_t *affinity) 633 { 634 struct gicv3_softc * const sc = LPITOSOFTC(pic); 635 struct gicv3_lpi_callback *cb; 636 637 kcpuset_zero(affinity); 638 LIST_FOREACH(cb, &sc->sc_lpi_callbacks, list) 639 cb->get_affinity(cb->priv, irq, affinity); 640 } 641 642 static int 643 gicv3_lpi_set_affinity(struct pic_softc *pic, size_t irq, const kcpuset_t *affinity) 644 { 645 struct gicv3_softc * const sc = LPITOSOFTC(pic); 646 struct gicv3_lpi_callback *cb; 647 int error = EINVAL; 648 649 LIST_FOREACH(cb, &sc->sc_lpi_callbacks, list) { 650 error = cb->set_affinity(cb->priv, irq, affinity); 651 if (error != EPASSTHROUGH) 652 return error; 653 } 654 655 return EINVAL; 656 } 657 #endif 658 659 static const struct pic_ops gicv3_lpiops = { 660 .pic_unblock_irqs = gicv3_lpi_unblock_irqs, 661 .pic_block_irqs = gicv3_lpi_block_irqs, 662 .pic_establish_irq = gicv3_lpi_establish_irq, 663 #ifdef MULTIPROCESSOR 664 .pic_cpu_init = gicv3_lpi_cpu_init, 665 .pic_get_affinity = gicv3_lpi_get_affinity, 666 .pic_set_affinity = gicv3_lpi_set_affinity, 667 #endif 668 }; 669 670 void 671 gicv3_dma_alloc(struct gicv3_softc *sc, struct gicv3_dma *dma, bus_size_t len, bus_size_t align) 672 { 673 int nsegs, error; 674 675 dma->len = len; 676 error = bus_dmamem_alloc(sc->sc_dmat, dma->len, align, 0, dma->segs, 1, &nsegs, BUS_DMA_WAITOK); 677 if (error) 678 panic("bus_dmamem_alloc failed: %d", error); 679 error = bus_dmamem_map(sc->sc_dmat, dma->segs, nsegs, len, (void **)&dma->base, BUS_DMA_WAITOK); 680 if (error) 681 panic("bus_dmamem_map failed: %d", error); 682 error = bus_dmamap_create(sc->sc_dmat, len, 1, len, 0, BUS_DMA_WAITOK, &dma->map); 683 if (error) 684 panic("bus_dmamap_create failed: %d", error); 685 error = bus_dmamap_load(sc->sc_dmat, dma->map, dma->base, dma->len, NULL, BUS_DMA_WAITOK); 686 if (error) 687 panic("bus_dmamap_load failed: %d", error); 688 689 memset(dma->base, 0, dma->len); 690 bus_dmamap_sync(sc->sc_dmat, dma->map, 0, dma->len, BUS_DMASYNC_PREWRITE); 691 } 692 693 static void 694 gicv3_lpi_init(struct gicv3_softc *sc) 695 { 696 /* 697 * Allocate LPI configuration table 698 */ 699 gicv3_dma_alloc(sc, &sc->sc_lpiconf, sc->sc_lpi.pic_maxsources, 0x1000); 700 KASSERT((sc->sc_lpiconf.segs[0].ds_addr & ~GICR_PROPBASER_Physical_Address) == 0); 701 702 /* 703 * Allocate LPI pending tables 704 */ 705 const bus_size_t lpipend_sz = (8192 + sc->sc_lpi.pic_maxsources) / NBBY; 706 for (int cpuindex = 0; cpuindex < ncpu; cpuindex++) { 707 gicv3_dma_alloc(sc, &sc->sc_lpipend[cpuindex], lpipend_sz, 0x10000); 708 KASSERT((sc->sc_lpipend[cpuindex].segs[0].ds_addr & ~GICR_PENDBASER_Physical_Address) == 0); 709 } 710 } 711 712 void 713 gicv3_irq_handler(void *frame) 714 { 715 struct cpu_info * const ci = curcpu(); 716 struct gicv3_softc * const sc = gicv3_softc; 717 struct pic_softc *pic; 718 const int oldipl = ci->ci_cpl; 719 720 ci->ci_data.cpu_nintr++; 721 722 for (;;) { 723 const uint32_t iar = icc_iar1_read(); 724 dsb(sy); 725 const uint32_t irq = __SHIFTOUT(iar, ICC_IAR_INTID); 726 if (irq == ICC_IAR_INTID_SPURIOUS) 727 break; 728 729 pic = irq >= GIC_LPI_BASE ? &sc->sc_lpi : &sc->sc_pic; 730 if (irq - pic->pic_irqbase >= pic->pic_maxsources) 731 continue; 732 733 struct intrsource * const is = pic->pic_sources[irq - pic->pic_irqbase]; 734 KASSERT(is != NULL); 735 736 const bool early_eoi = irq < GIC_LPI_BASE && is->is_type == IST_EDGE; 737 738 const int ipl = is->is_ipl; 739 if (__predict_false(ipl < ci->ci_cpl)) { 740 pic_do_pending_ints(I32_bit, ipl, frame); 741 } else { 742 gicv3_set_priority(pic, ipl); 743 ci->ci_cpl = ipl; 744 } 745 746 if (early_eoi) { 747 icc_eoi1r_write(iar); 748 isb(); 749 } 750 751 cpsie(I32_bit); 752 pic_dispatch(is, frame); 753 cpsid(I32_bit); 754 755 if (!early_eoi) { 756 icc_eoi1r_write(iar); 757 isb(); 758 } 759 } 760 761 pic_do_pending_ints(I32_bit, oldipl, frame); 762 } 763 764 static int 765 gicv3_detect_pmr_bits(struct gicv3_softc *sc) 766 { 767 const uint32_t opmr = icc_pmr_read(); 768 icc_pmr_write(0xbf); 769 const uint32_t npmr = icc_pmr_read(); 770 icc_pmr_write(opmr); 771 772 return NBBY - (ffs(npmr) - 1); 773 } 774 775 static int 776 gicv3_detect_ipriority_bits(struct gicv3_softc *sc) 777 { 778 const uint32_t oipriorityr = gicd_read_4(sc, GICD_IPRIORITYRn(8)); 779 gicd_write_4(sc, GICD_IPRIORITYRn(8), oipriorityr | 0xff); 780 const uint32_t nipriorityr = gicd_read_4(sc, GICD_IPRIORITYRn(8)); 781 gicd_write_4(sc, GICD_IPRIORITYRn(8), oipriorityr); 782 783 return NBBY - (ffs(nipriorityr & 0xff) - 1); 784 } 785 786 int 787 gicv3_init(struct gicv3_softc *sc) 788 { 789 const uint32_t gicd_typer = gicd_read_4(sc, GICD_TYPER); 790 const uint32_t gicd_ctrl = gicd_read_4(sc, GICD_CTRL); 791 int n; 792 793 KASSERT(CPU_IS_PRIMARY(curcpu())); 794 795 LIST_INIT(&sc->sc_lpi_callbacks); 796 797 for (n = 0; n < MAXCPUS; n++) 798 sc->sc_irouter[n] = UINT64_MAX; 799 800 sc->sc_priority_shift = 4; 801 sc->sc_pmr_shift = 4; 802 803 if ((gicd_ctrl & GICD_CTRL_DS) == 0) { 804 const int pmr_bits = gicv3_detect_pmr_bits(sc); 805 const int ipriority_bits = gicv3_detect_ipriority_bits(sc); 806 807 if (ipriority_bits != pmr_bits) 808 --sc->sc_priority_shift; 809 810 aprint_verbose_dev(sc->sc_dev, "%d pmr bits, %d ipriority bits\n", 811 pmr_bits, ipriority_bits); 812 } else { 813 aprint_verbose_dev(sc->sc_dev, "security disabled\n"); 814 } 815 816 aprint_verbose_dev(sc->sc_dev, "priority shift %d, pmr shift %d\n", 817 sc->sc_priority_shift, sc->sc_pmr_shift); 818 819 sc->sc_pic.pic_ops = &gicv3_picops; 820 sc->sc_pic.pic_maxsources = GICD_TYPER_LINES(gicd_typer); 821 snprintf(sc->sc_pic.pic_name, sizeof(sc->sc_pic.pic_name), "gicv3"); 822 #ifdef MULTIPROCESSOR 823 sc->sc_pic.pic_cpus = kcpuset_running; 824 #endif 825 pic_add(&sc->sc_pic, 0); 826 827 if ((gicd_typer & GICD_TYPER_LPIS) != 0) { 828 sc->sc_lpi.pic_ops = &gicv3_lpiops; 829 sc->sc_lpi.pic_maxsources = 8192; /* Min. required by GICv3 spec */ 830 snprintf(sc->sc_lpi.pic_name, sizeof(sc->sc_lpi.pic_name), "gicv3-lpi"); 831 pic_add(&sc->sc_lpi, GIC_LPI_BASE); 832 833 sc->sc_lpi_pool = vmem_create("gicv3-lpi", 0, sc->sc_lpi.pic_maxsources, 834 1, NULL, NULL, NULL, 0, VM_SLEEP, IPL_HIGH); 835 if (sc->sc_lpi_pool == NULL) 836 panic("failed to create gicv3 lpi pool\n"); 837 838 gicv3_lpi_init(sc); 839 } 840 841 KASSERT(gicv3_softc == NULL); 842 gicv3_softc = sc; 843 844 for (int i = 0; i < sc->sc_bsh_r_count; i++) { 845 const uint64_t gicr_typer = gicr_read_8(sc, i, GICR_TYPER); 846 const u_int aff0 = __SHIFTOUT(gicr_typer, GICR_TYPER_Affinity_Value_Aff0); 847 const u_int aff1 = __SHIFTOUT(gicr_typer, GICR_TYPER_Affinity_Value_Aff1); 848 const u_int aff2 = __SHIFTOUT(gicr_typer, GICR_TYPER_Affinity_Value_Aff2); 849 const u_int aff3 = __SHIFTOUT(gicr_typer, GICR_TYPER_Affinity_Value_Aff3); 850 851 aprint_debug_dev(sc->sc_dev, "redist %d: cpu %d.%d.%d.%d\n", 852 i, aff3, aff2, aff1, aff0); 853 } 854 855 gicv3_dist_enable(sc); 856 857 gicv3_cpu_init(&sc->sc_pic, curcpu()); 858 if ((gicd_typer & GICD_TYPER_LPIS) != 0) 859 gicv3_lpi_cpu_init(&sc->sc_lpi, curcpu()); 860 861 #ifdef __HAVE_PIC_FAST_SOFTINTS 862 intr_establish_xname(SOFTINT_BIO, IPL_SOFTBIO, IST_MPSAFE | IST_EDGE, pic_handle_softint, (void *)SOFTINT_BIO, "softint bio"); 863 intr_establish_xname(SOFTINT_CLOCK, IPL_SOFTCLOCK, IST_MPSAFE | IST_EDGE, pic_handle_softint, (void *)SOFTINT_CLOCK, "softint clock"); 864 intr_establish_xname(SOFTINT_NET, IPL_SOFTNET, IST_MPSAFE | IST_EDGE, pic_handle_softint, (void *)SOFTINT_NET, "softint net"); 865 intr_establish_xname(SOFTINT_SERIAL, IPL_SOFTSERIAL, IST_MPSAFE | IST_EDGE, pic_handle_softint, (void *)SOFTINT_SERIAL, "softint serial"); 866 #endif 867 868 #ifdef MULTIPROCESSOR 869 intr_establish_xname(IPI_AST, IPL_VM, IST_MPSAFE | IST_EDGE, pic_ipi_ast, (void *)-1, "IPI ast"); 870 intr_establish_xname(IPI_XCALL, IPL_HIGH, IST_MPSAFE | IST_EDGE, pic_ipi_xcall, (void *)-1, "IPI xcall"); 871 intr_establish_xname(IPI_GENERIC, IPL_HIGH, IST_MPSAFE | IST_EDGE, pic_ipi_generic, (void *)-1, "IPI generic"); 872 intr_establish_xname(IPI_NOP, IPL_VM, IST_MPSAFE | IST_EDGE, pic_ipi_nop, (void *)-1, "IPI nop"); 873 intr_establish_xname(IPI_SHOOTDOWN, IPL_SCHED, IST_MPSAFE | IST_EDGE, pic_ipi_shootdown, (void *)-1, "IPI shootdown"); 874 #ifdef DDB 875 intr_establish_xname(IPI_DDB, IPL_HIGH, IST_MPSAFE | IST_EDGE, pic_ipi_ddb, NULL, "IPI ddb"); 876 #endif 877 #ifdef __HAVE_PREEMPTION 878 intr_establish_xname(IPI_KPREEMPT, IPL_VM, IST_MPSAFE | IST_EDGE, pic_ipi_kpreempt, (void *)-1, "IPI kpreempt"); 879 #endif 880 #endif 881 882 return 0; 883 } 884
Indexes created Mon Sep 22 13:09:51 GMT 2025