gicv3.c revision 1.7
1 /* $NetBSD: gicv3.c,v 1.7 2018/11/10 11:46:31 jmcneill Exp $ */ 2 3 /*- 4 * Copyright (c) 2018 Jared McNeill <jmcneill (at) invisible.ca> 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 17 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 18 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 19 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 20 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 21 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 22 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 23 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 24 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 * SUCH DAMAGE. 27 */ 28 29 #include "opt_multiprocessor.h" 30 31 #define _INTR_PRIVATE 32 33 #include <sys/cdefs.h> 34 __KERNEL_RCSID(0, "$NetBSD: gicv3.c,v 1.7 2018/11/10 11:46:31 jmcneill Exp $"); 35 36 #include <sys/param.h> 37 #include <sys/kernel.h> 38 #include <sys/bus.h> 39 #include <sys/device.h> 40 #include <sys/intr.h> 41 #include <sys/systm.h> 42 #include <sys/cpu.h> 43 44 #include <arm/locore.h> 45 #include <arm/armreg.h> 46 47 #include <arm/cortex/gicv3.h> 48 #include <arm/cortex/gic_reg.h> 49 50 #define PICTOSOFTC(pic) \ 51 ((void *)((uintptr_t)(pic) - offsetof(struct gicv3_softc, sc_pic))) 52 #define LPITOSOFTC(lpi) \ 53 ((void *)((uintptr_t)(lpi) - offsetof(struct gicv3_softc, sc_lpi))) 54 55 #define IPL_TO_PRIORITY(ipl) ((IPL_HIGH - (ipl)) << 4) 56 57 static struct gicv3_softc *gicv3_softc; 58 59 static inline uint32_t 60 gicd_read_4(struct gicv3_softc *sc, bus_size_t reg) 61 { 62 return bus_space_read_4(sc->sc_bst, sc->sc_bsh_d, reg); 63 } 64 65 static inline void 66 gicd_write_4(struct gicv3_softc *sc, bus_size_t reg, uint32_t val) 67 { 68 bus_space_write_4(sc->sc_bst, sc->sc_bsh_d, reg, val); 69 } 70 71 static inline uint64_t 72 gicd_read_8(struct gicv3_softc *sc, bus_size_t reg) 73 { 74 return bus_space_read_8(sc->sc_bst, sc->sc_bsh_d, reg); 75 } 76 77 static inline void 78 gicd_write_8(struct gicv3_softc *sc, bus_size_t reg, uint64_t val) 79 { 80 bus_space_write_8(sc->sc_bst, sc->sc_bsh_d, reg, val); 81 } 82 83 static inline uint32_t 84 gicr_read_4(struct gicv3_softc *sc, u_int index, bus_size_t reg) 85 { 86 KASSERT(index < sc->sc_bsh_r_count); 87 return bus_space_read_4(sc->sc_bst, sc->sc_bsh_r[index], reg); 88 } 89 90 static inline void 91 gicr_write_4(struct gicv3_softc *sc, u_int index, bus_size_t reg, uint32_t val) 92 { 93 KASSERT(index < sc->sc_bsh_r_count); 94 bus_space_write_4(sc->sc_bst, sc->sc_bsh_r[index], reg, val); 95 } 96 97 static inline uint64_t 98 gicr_read_8(struct gicv3_softc *sc, u_int index, bus_size_t reg) 99 { 100 KASSERT(index < sc->sc_bsh_r_count); 101 return bus_space_read_8(sc->sc_bst, sc->sc_bsh_r[index], reg); 102 } 103 104 static inline void 105 gicr_write_8(struct gicv3_softc *sc, u_int index, bus_size_t reg, uint64_t val) 106 { 107 KASSERT(index < sc->sc_bsh_r_count); 108 bus_space_write_8(sc->sc_bst, sc->sc_bsh_r[index], reg, val); 109 } 110 111 static void 112 gicv3_unblock_irqs(struct pic_softc *pic, size_t irqbase, uint32_t mask) 113 { 114 struct gicv3_softc * const sc = PICTOSOFTC(pic); 115 struct cpu_info * const ci = curcpu(); 116 const u_int group = irqbase / 32; 117 118 if (group == 0) { 119 sc->sc_enabled_sgippi |= mask; 120 gicr_write_4(sc, ci->ci_gic_redist, GICR_ISENABLER0, mask); 121 while (gicr_read_4(sc, ci->ci_gic_redist, GICR_CTLR) & GICR_CTLR_RWP) 122 ; 123 } else { 124 gicd_write_4(sc, GICD_ISENABLERn(group), mask); 125 while (gicd_read_4(sc, GICD_CTRL) & GICD_CTRL_RWP) 126 ; 127 } 128 } 129 130 static void 131 gicv3_block_irqs(struct pic_softc *pic, size_t irqbase, uint32_t mask) 132 { 133 struct gicv3_softc * const sc = PICTOSOFTC(pic); 134 struct cpu_info * const ci = curcpu(); 135 const u_int group = irqbase / 32; 136 137 if (group == 0) { 138 sc->sc_enabled_sgippi &= ~mask; 139 gicr_write_4(sc, ci->ci_gic_redist, GICR_ICENABLER0, mask); 140 while (gicr_read_4(sc, ci->ci_gic_redist, GICR_CTLR) & GICR_CTLR_RWP) 141 ; 142 } else { 143 gicd_write_4(sc, GICD_ICENABLERn(group), mask); 144 while (gicd_read_4(sc, GICD_CTRL) & GICD_CTRL_RWP) 145 ; 146 } 147 } 148 149 static void 150 gicv3_establish_irq(struct pic_softc *pic, struct intrsource *is) 151 { 152 struct gicv3_softc * const sc = PICTOSOFTC(pic); 153 const u_int group = is->is_irq / 32; 154 uint32_t ipriority, icfg; 155 uint64_t irouter; 156 u_int n; 157 158 const u_int ipriority_val = 0x80 | IPL_TO_PRIORITY(is->is_ipl); 159 const u_int ipriority_shift = (is->is_irq & 0x3) * 8; 160 const u_int icfg_shift = (is->is_irq & 0xf) * 2; 161 162 if (group == 0) { 163 /* SGIs and PPIs are always MP-safe */ 164 is->is_mpsafe = true; 165 166 /* Update interrupt configuration and priority on all redistributors */ 167 for (n = 0; n < sc->sc_bsh_r_count; n++) { 168 icfg = gicr_read_4(sc, n, GICR_ICFGRn(is->is_irq / 16)); 169 if (is->is_type == IST_LEVEL) 170 icfg &= ~(0x2 << icfg_shift); 171 if (is->is_type == IST_EDGE) 172 icfg |= (0x2 << icfg_shift); 173 gicr_write_4(sc, n, GICR_ICFGRn(is->is_irq / 16), icfg); 174 175 ipriority = gicr_read_4(sc, n, GICR_IPRIORITYRn(is->is_irq / 4)); 176 ipriority &= ~(0xff << ipriority_shift); 177 ipriority |= (ipriority_val << ipriority_shift); 178 gicr_write_4(sc, n, GICR_IPRIORITYRn(is->is_irq / 4), ipriority); 179 } 180 } else { 181 if (is->is_mpsafe) { 182 /* Route MP-safe interrupts to all participating PEs */ 183 irouter = GICD_IROUTER_Interrupt_Routing_mode; 184 } else { 185 /* Route non-MP-safe interrupts to the primary PE only */ 186 irouter = sc->sc_irouter[0]; 187 } 188 gicd_write_8(sc, GICD_IROUTER(is->is_irq), irouter); 189 190 /* Update interrupt configuration */ 191 icfg = gicd_read_4(sc, GICD_ICFGRn(is->is_irq / 16)); 192 if (is->is_type == IST_LEVEL) 193 icfg &= ~(0x2 << icfg_shift); 194 if (is->is_type == IST_EDGE) 195 icfg |= (0x2 << icfg_shift); 196 gicd_write_4(sc, GICD_ICFGRn(is->is_irq / 16), icfg); 197 198 /* Update interrupt priority */ 199 ipriority = gicd_read_4(sc, GICD_IPRIORITYRn(is->is_irq / 4)); 200 ipriority &= ~(0xff << ipriority_shift); 201 ipriority |= (ipriority_val << ipriority_shift); 202 gicd_write_4(sc, GICD_IPRIORITYRn(is->is_irq / 4), ipriority); 203 } 204 } 205 206 static void 207 gicv3_set_priority(struct pic_softc *pic, int ipl) 208 { 209 icc_pmr_write(IPL_TO_PRIORITY(ipl) << 1); 210 } 211 212 static void 213 gicv3_dist_enable(struct gicv3_softc *sc) 214 { 215 uint32_t gicd_ctrl; 216 u_int n; 217 218 /* Disable the distributor */ 219 gicd_write_4(sc, GICD_CTRL, 0); 220 221 /* Wait for register write to complete */ 222 while (gicd_read_4(sc, GICD_CTRL) & GICD_CTRL_RWP) 223 ; 224 225 /* Clear all INTID enable bits */ 226 for (n = 32; n < sc->sc_pic.pic_maxsources; n += 32) 227 gicd_write_4(sc, GICD_ICENABLERn(n / 32), ~0); 228 229 /* Set default priorities to lowest */ 230 for (n = 32; n < sc->sc_pic.pic_maxsources; n += 4) 231 gicd_write_4(sc, GICD_IPRIORITYRn(n / 4), ~0); 232 233 /* Set all interrupts to G1NS */ 234 for (n = 32; n < sc->sc_pic.pic_maxsources; n += 32) { 235 gicd_write_4(sc, GICD_IGROUPRn(n / 32), ~0); 236 gicd_write_4(sc, GICD_IGRPMODRn(n / 32), 0); 237 } 238 239 /* Set all interrupts level-sensitive by default */ 240 for (n = 32; n < sc->sc_pic.pic_maxsources; n += 16) 241 gicd_write_4(sc, GICD_ICFGRn(n / 16), 0); 242 243 /* Wait for register writes to complete */ 244 while (gicd_read_4(sc, GICD_CTRL) & GICD_CTRL_RWP) 245 ; 246 247 /* Enable Affinity routing and G1NS interrupts */ 248 gicd_ctrl = GICD_CTRL_EnableGrp1NS | GICD_CTRL_Enable | GICD_CTRL_ARE_NS; 249 gicd_write_4(sc, GICD_CTRL, gicd_ctrl); 250 } 251 252 static void 253 gicv3_redist_enable(struct gicv3_softc *sc, struct cpu_info *ci) 254 { 255 uint32_t icfg; 256 u_int n, o; 257 258 /* Clear INTID enable bits */ 259 gicr_write_4(sc, ci->ci_gic_redist, GICR_ICENABLER0, ~0); 260 261 /* Wait for register write to complete */ 262 while (gicr_read_4(sc, ci->ci_gic_redist, GICR_CTLR) & GICR_CTLR_RWP) 263 ; 264 265 /* Set default priorities */ 266 for (n = 0; n < 32; n += 4) { 267 uint32_t priority = 0; 268 size_t byte_shift = 0; 269 for (o = 0; o < 4; o++, byte_shift += 8) { 270 struct intrsource * const is = sc->sc_pic.pic_sources[n + o]; 271 if (is == NULL) 272 priority |= 0xff << byte_shift; 273 else { 274 const u_int ipriority_val = 0x80 | IPL_TO_PRIORITY(is->is_ipl); 275 priority |= ipriority_val << byte_shift; 276 } 277 } 278 gicr_write_4(sc, ci->ci_gic_redist, GICR_IPRIORITYRn(n / 4), priority); 279 } 280 281 /* Set all interrupts to G1NS */ 282 gicr_write_4(sc, ci->ci_gic_redist, GICR_IGROUPR0, ~0); 283 gicr_write_4(sc, ci->ci_gic_redist, GICR_IGRPMODR0, 0); 284 285 /* Restore PPI configs */ 286 for (n = 0, icfg = 0; n < 16; n++) { 287 struct intrsource * const is = sc->sc_pic.pic_sources[16 + n]; 288 if (is != NULL && is->is_type == IST_EDGE) 289 icfg |= (0x2 << (n * 2)); 290 } 291 gicr_write_4(sc, ci->ci_gic_redist, GICR_ICFGRn(1), icfg); 292 293 /* Restore current enable bits */ 294 gicr_write_4(sc, ci->ci_gic_redist, GICR_ISENABLER0, sc->sc_enabled_sgippi); 295 296 /* Wait for register write to complete */ 297 while (gicr_read_4(sc, ci->ci_gic_redist, GICR_CTLR) & GICR_CTLR_RWP) 298 ; 299 } 300 301 static uint64_t 302 gicv3_cpu_identity(void) 303 { 304 u_int aff3, aff2, aff1, aff0; 305 306 #ifdef __aarch64__ 307 const register_t mpidr = reg_mpidr_el1_read(); 308 aff0 = __SHIFTOUT(mpidr, MPIDR_AFF0); 309 aff1 = __SHIFTOUT(mpidr, MPIDR_AFF1); 310 aff2 = __SHIFTOUT(mpidr, MPIDR_AFF2); 311 aff3 = __SHIFTOUT(mpidr, MPIDR_AFF3); 312 #else 313 const register_t mpidr = armreg_mpidr_read(); 314 aff0 = __SHIFTOUT(mpidr, MPIDR_AFF0); 315 aff1 = __SHIFTOUT(mpidr, MPIDR_AFF1); 316 aff2 = __SHIFTOUT(mpidr, MPIDR_AFF2); 317 aff3 = 0; 318 #endif 319 320 return __SHIFTIN(aff0, GICR_TYPER_Affinity_Value_Aff0) | 321 __SHIFTIN(aff1, GICR_TYPER_Affinity_Value_Aff1) | 322 __SHIFTIN(aff2, GICR_TYPER_Affinity_Value_Aff2) | 323 __SHIFTIN(aff3, GICR_TYPER_Affinity_Value_Aff3); 324 } 325 326 static u_int 327 gicv3_find_redist(struct gicv3_softc *sc) 328 { 329 uint64_t gicr_typer; 330 u_int n; 331 332 const uint64_t cpu_identity = gicv3_cpu_identity(); 333 334 for (n = 0; n < sc->sc_bsh_r_count; n++) { 335 gicr_typer = gicr_read_8(sc, n, GICR_TYPER); 336 if ((gicr_typer & GICR_TYPER_Affinity_Value) == cpu_identity) 337 return n; 338 } 339 340 const u_int aff0 = __SHIFTOUT(cpu_identity, GICR_TYPER_Affinity_Value_Aff0); 341 const u_int aff1 = __SHIFTOUT(cpu_identity, GICR_TYPER_Affinity_Value_Aff1); 342 const u_int aff2 = __SHIFTOUT(cpu_identity, GICR_TYPER_Affinity_Value_Aff2); 343 const u_int aff3 = __SHIFTOUT(cpu_identity, GICR_TYPER_Affinity_Value_Aff3); 344 345 panic("%s: could not find GICv3 redistributor for cpu %d.%d.%d.%d", 346 cpu_name(curcpu()), aff3, aff2, aff1, aff0); 347 } 348 349 static uint64_t 350 gicv3_sgir(struct gicv3_softc *sc) 351 { 352 const uint64_t cpu_identity = gicv3_cpu_identity(); 353 354 const u_int aff0 = __SHIFTOUT(cpu_identity, GICR_TYPER_Affinity_Value_Aff0); 355 const u_int aff1 = __SHIFTOUT(cpu_identity, GICR_TYPER_Affinity_Value_Aff1); 356 const u_int aff2 = __SHIFTOUT(cpu_identity, GICR_TYPER_Affinity_Value_Aff2); 357 const u_int aff3 = __SHIFTOUT(cpu_identity, GICR_TYPER_Affinity_Value_Aff3); 358 359 return __SHIFTIN(__BIT(aff0), ICC_SGIR_EL1_TargetList) | 360 __SHIFTIN(aff1, ICC_SGIR_EL1_Aff1) | 361 __SHIFTIN(aff2, ICC_SGIR_EL1_Aff2) | 362 __SHIFTIN(aff3, ICC_SGIR_EL1_Aff3); 363 } 364 365 static void 366 gicv3_cpu_init(struct pic_softc *pic, struct cpu_info *ci) 367 { 368 struct gicv3_softc * const sc = PICTOSOFTC(pic); 369 uint32_t icc_sre, icc_ctlr, gicr_waker; 370 371 ci->ci_gic_redist = gicv3_find_redist(sc); 372 ci->ci_gic_sgir = gicv3_sgir(sc); 373 374 /* Store route to CPU for SPIs */ 375 const uint64_t cpu_identity = gicv3_cpu_identity(); 376 const u_int aff0 = __SHIFTOUT(cpu_identity, GICR_TYPER_Affinity_Value_Aff0); 377 const u_int aff1 = __SHIFTOUT(cpu_identity, GICR_TYPER_Affinity_Value_Aff1); 378 const u_int aff2 = __SHIFTOUT(cpu_identity, GICR_TYPER_Affinity_Value_Aff2); 379 const u_int aff3 = __SHIFTOUT(cpu_identity, GICR_TYPER_Affinity_Value_Aff3); 380 sc->sc_irouter[cpu_index(ci)] = 381 __SHIFTIN(aff0, GICD_IROUTER_Aff0) | 382 __SHIFTIN(aff1, GICD_IROUTER_Aff1) | 383 __SHIFTIN(aff2, GICD_IROUTER_Aff2) | 384 __SHIFTIN(aff3, GICD_IROUTER_Aff3); 385 386 /* Enable System register access and disable IRQ/FIQ bypass */ 387 icc_sre = ICC_SRE_EL1_SRE | ICC_SRE_EL1_DFB | ICC_SRE_EL1_DIB; 388 icc_sre_write(icc_sre); 389 390 /* Mark the connected PE as being awake */ 391 gicr_waker = gicr_read_4(sc, ci->ci_gic_redist, GICR_WAKER); 392 gicr_waker &= ~GICR_WAKER_ProcessorSleep; 393 gicr_write_4(sc, ci->ci_gic_redist, GICR_WAKER, gicr_waker); 394 while (gicr_read_4(sc, ci->ci_gic_redist, GICR_WAKER) & GICR_WAKER_ChildrenAsleep) 395 ; 396 397 /* Set initial priority mask */ 398 gicv3_set_priority(pic, IPL_HIGH); 399 400 /* Disable preemption */ 401 const uint32_t icc_bpr = __SHIFTIN(0x7, ICC_BPR_EL1_BinaryPoint); 402 icc_bpr1_write(icc_bpr); 403 404 /* Enable group 1 interrupt signaling */ 405 icc_igrpen1_write(ICC_IGRPEN_EL1_Enable); 406 407 /* Set EOI mode */ 408 icc_ctlr = icc_ctlr_read(); 409 icc_ctlr &= ~ICC_CTLR_EL1_EOImode; 410 icc_ctlr_write(icc_ctlr); 411 412 /* Enable redistributor */ 413 gicv3_redist_enable(sc, ci); 414 415 /* Allow IRQ exceptions */ 416 cpsie(I32_bit); 417 } 418 419 #ifdef MULTIPROCESSOR 420 static void 421 gicv3_ipi_send(struct pic_softc *pic, const kcpuset_t *kcp, u_long ipi) 422 { 423 CPU_INFO_ITERATOR cii; 424 struct cpu_info *ci; 425 uint64_t intid, aff, targets; 426 427 intid = __SHIFTIN(ipi, ICC_SGIR_EL1_INTID); 428 if (kcp == NULL) { 429 /* Interrupts routed to all PEs, excluding "self" */ 430 if (ncpu == 1) 431 return; 432 icc_sgi1r_write(intid | ICC_SGIR_EL1_IRM); 433 } else { 434 /* Interrupts routed to specific PEs */ 435 aff = 0; 436 targets = 0; 437 for (CPU_INFO_FOREACH(cii, ci)) { 438 if (!kcpuset_isset(kcp, cpu_index(ci))) 439 continue; 440 if ((ci->ci_gic_sgir & ICC_SGIR_EL1_Aff) != aff) { 441 if (targets != 0) { 442 icc_sgi1r_write(intid | aff | targets); 443 targets = 0; 444 } 445 aff = (ci->ci_gic_sgir & ICC_SGIR_EL1_Aff); 446 } 447 targets |= (ci->ci_gic_sgir & ICC_SGIR_EL1_TargetList); 448 } 449 if (targets != 0) 450 icc_sgi1r_write(intid | aff | targets); 451 } 452 } 453 454 static void 455 gicv3_get_affinity(struct pic_softc *pic, size_t irq, kcpuset_t *affinity) 456 { 457 struct gicv3_softc * const sc = PICTOSOFTC(pic); 458 const size_t group = irq / 32; 459 int n; 460 461 kcpuset_zero(affinity); 462 if (group == 0) { 463 /* All CPUs are targets for group 0 (SGI/PPI) */ 464 for (n = 0; n < ncpu; n++) { 465 if (sc->sc_irouter[n] != UINT64_MAX) 466 kcpuset_set(affinity, n); 467 } 468 } else { 469 /* Find distributor targets (SPI) */ 470 const uint64_t irouter = gicd_read_8(sc, GICD_IROUTER(irq)); 471 for (n = 0; n < ncpu; n++) { 472 if (irouter == GICD_IROUTER_Interrupt_Routing_mode || 473 irouter == sc->sc_irouter[n]) 474 kcpuset_set(affinity, n); 475 } 476 } 477 } 478 479 static int 480 gicv3_set_affinity(struct pic_softc *pic, size_t irq, const kcpuset_t *affinity) 481 { 482 struct gicv3_softc * const sc = PICTOSOFTC(pic); 483 const size_t group = irq / 32; 484 uint64_t irouter; 485 486 if (group == 0) 487 return EINVAL; 488 489 const int set = kcpuset_countset(affinity); 490 if (set == ncpu) 491 irouter = GICD_IROUTER_Interrupt_Routing_mode; 492 else if (set == 1) 493 irouter = sc->sc_irouter[kcpuset_ffs(affinity)]; 494 else 495 return EINVAL; 496 497 gicd_write_8(sc, GICD_IROUTER(irq), irouter); 498 499 return 0; 500 } 501 #endif 502 503 static const struct pic_ops gicv3_picops = { 504 .pic_unblock_irqs = gicv3_unblock_irqs, 505 .pic_block_irqs = gicv3_block_irqs, 506 .pic_establish_irq = gicv3_establish_irq, 507 .pic_set_priority = gicv3_set_priority, 508 #ifdef MULTIPROCESSOR 509 .pic_cpu_init = gicv3_cpu_init, 510 .pic_ipi_send = gicv3_ipi_send, 511 .pic_get_affinity = gicv3_get_affinity, 512 .pic_set_affinity = gicv3_set_affinity, 513 #endif 514 }; 515 516 static void 517 gicv3_lpi_unblock_irqs(struct pic_softc *pic, size_t irqbase, uint32_t mask) 518 { 519 struct gicv3_softc * const sc = LPITOSOFTC(pic); 520 int bit; 521 522 while ((bit = ffs(mask)) != 0) { 523 sc->sc_lpiconf.base[irqbase + bit - 1] |= GIC_LPICONF_Enable; 524 mask &= ~__BIT(bit - 1); 525 } 526 527 bus_dmamap_sync(sc->sc_dmat, sc->sc_lpiconf.map, irqbase, 32, BUS_DMASYNC_PREWRITE); 528 } 529 530 static void 531 gicv3_lpi_block_irqs(struct pic_softc *pic, size_t irqbase, uint32_t mask) 532 { 533 struct gicv3_softc * const sc = LPITOSOFTC(pic); 534 const u_int off = irqbase - pic->pic_irqbase; 535 int bit; 536 537 while ((bit = ffs(mask)) != 0) { 538 sc->sc_lpiconf.base[off + bit - 1] &= ~GIC_LPICONF_Enable; 539 mask &= ~__BIT(bit - 1); 540 } 541 542 bus_dmamap_sync(sc->sc_dmat, sc->sc_lpiconf.map, off, 32, BUS_DMASYNC_PREWRITE); 543 } 544 545 static void 546 gicv3_lpi_establish_irq(struct pic_softc *pic, struct intrsource *is) 547 { 548 struct gicv3_softc * const sc = LPITOSOFTC(pic); 549 550 sc->sc_lpiconf.base[is->is_irq] = IPL_TO_PRIORITY(is->is_ipl) | GIC_LPICONF_Res1; 551 552 bus_dmamap_sync(sc->sc_dmat, sc->sc_lpiconf.map, is->is_irq, 1, BUS_DMASYNC_PREWRITE); 553 } 554 555 static void 556 gicv3_lpi_cpu_init(struct pic_softc *pic, struct cpu_info *ci) 557 { 558 struct gicv3_softc * const sc = LPITOSOFTC(pic); 559 struct gicv3_lpi_callback *cb; 560 uint32_t ctlr; 561 562 /* If physical LPIs are not supported on this redistributor, just return. */ 563 const uint64_t typer = gicr_read_8(sc, ci->ci_gic_redist, GICR_TYPER); 564 if ((typer & GICR_TYPER_PLPIS) == 0) 565 return; 566 567 /* Interrupt target address for this CPU, used by ITS when GITS_TYPER.PTA == 0 */ 568 sc->sc_processor_id[cpu_index(ci)] = __SHIFTOUT(typer, GICR_TYPER_Processor_Number); 569 570 /* Disable LPIs before making changes */ 571 ctlr = gicr_read_4(sc, ci->ci_gic_redist, GICR_CTLR); 572 ctlr &= ~GICR_CTLR_Enable_LPIs; 573 gicr_write_4(sc, ci->ci_gic_redist, GICR_CTLR, ctlr); 574 arm_dsb(); 575 576 /* Setup the LPI configuration table */ 577 const uint64_t propbase = sc->sc_lpiconf.segs[0].ds_addr | 578 __SHIFTIN(ffs(pic->pic_maxsources) - 1, GICR_PROPBASER_IDbits) | 579 __SHIFTIN(GICR_Shareability_NS, GICR_PROPBASER_Shareability) | 580 __SHIFTIN(GICR_Cache_NORMAL_NC, GICR_PROPBASER_InnerCache); 581 gicr_write_8(sc, ci->ci_gic_redist, GICR_PROPBASER, propbase); 582 583 /* Setup the LPI pending table */ 584 const uint64_t pendbase = sc->sc_lpipend[cpu_index(ci)].segs[0].ds_addr | 585 __SHIFTIN(GICR_Shareability_NS, GICR_PENDBASER_Shareability) | 586 __SHIFTIN(GICR_Cache_NORMAL_NC, GICR_PENDBASER_InnerCache) | 587 GICR_PENDBASER_PTZ; 588 gicr_write_8(sc, ci->ci_gic_redist, GICR_PENDBASER, pendbase); 589 590 /* Enable LPIs */ 591 ctlr = gicr_read_4(sc, ci->ci_gic_redist, GICR_CTLR); 592 ctlr |= GICR_CTLR_Enable_LPIs; 593 gicr_write_4(sc, ci->ci_gic_redist, GICR_CTLR, ctlr); 594 arm_dsb(); 595 596 /* Setup ITS if present */ 597 LIST_FOREACH(cb, &sc->sc_lpi_callbacks, list) 598 cb->cpu_init(cb->priv, ci); 599 } 600 601 #ifdef MULTIPROCESSOR 602 static void 603 gicv3_lpi_get_affinity(struct pic_softc *pic, size_t irq, kcpuset_t *affinity) 604 { 605 struct gicv3_softc * const sc = LPITOSOFTC(pic); 606 struct gicv3_lpi_callback *cb; 607 608 LIST_FOREACH(cb, &sc->sc_lpi_callbacks, list) 609 cb->get_affinity(cb->priv, irq, affinity); 610 } 611 612 static int 613 gicv3_lpi_set_affinity(struct pic_softc *pic, size_t irq, const kcpuset_t *affinity) 614 { 615 struct gicv3_softc * const sc = LPITOSOFTC(pic); 616 struct gicv3_lpi_callback *cb; 617 int error = EINVAL; 618 619 LIST_FOREACH(cb, &sc->sc_lpi_callbacks, list) { 620 error = cb->set_affinity(cb->priv, irq, affinity); 621 if (error) 622 return error; 623 } 624 625 return error; 626 } 627 #endif 628 629 static const struct pic_ops gicv3_lpiops = { 630 .pic_unblock_irqs = gicv3_lpi_unblock_irqs, 631 .pic_block_irqs = gicv3_lpi_block_irqs, 632 .pic_establish_irq = gicv3_lpi_establish_irq, 633 #ifdef MULTIPROCESSOR 634 .pic_cpu_init = gicv3_lpi_cpu_init, 635 .pic_get_affinity = gicv3_lpi_get_affinity, 636 .pic_set_affinity = gicv3_lpi_set_affinity, 637 #endif 638 }; 639 640 void 641 gicv3_dma_alloc(struct gicv3_softc *sc, struct gicv3_dma *dma, bus_size_t len, bus_size_t align) 642 { 643 int nsegs, error; 644 645 dma->len = len; 646 error = bus_dmamem_alloc(sc->sc_dmat, dma->len, align, 0, dma->segs, 1, &nsegs, BUS_DMA_WAITOK); 647 if (error) 648 panic("bus_dmamem_alloc failed: %d", error); 649 error = bus_dmamem_map(sc->sc_dmat, dma->segs, nsegs, len, (void **)&dma->base, BUS_DMA_WAITOK); 650 if (error) 651 panic("bus_dmamem_map failed: %d", error); 652 error = bus_dmamap_create(sc->sc_dmat, len, 1, len, 0, BUS_DMA_WAITOK, &dma->map); 653 if (error) 654 panic("bus_dmamap_create failed: %d", error); 655 error = bus_dmamap_load(sc->sc_dmat, dma->map, dma->base, dma->len, NULL, BUS_DMA_WAITOK); 656 if (error) 657 panic("bus_dmamap_load failed: %d", error); 658 659 memset(dma->base, 0, dma->len); 660 bus_dmamap_sync(sc->sc_dmat, dma->map, 0, dma->len, BUS_DMASYNC_PREWRITE); 661 } 662 663 static void 664 gicv3_lpi_init(struct gicv3_softc *sc) 665 { 666 /* 667 * Allocate LPI configuration table 668 */ 669 gicv3_dma_alloc(sc, &sc->sc_lpiconf, sc->sc_lpi.pic_maxsources, 0x1000); 670 KASSERT((sc->sc_lpiconf.segs[0].ds_addr & ~GICR_PROPBASER_Physical_Address) == 0); 671 672 /* 673 * Allocate LPI pending tables 674 */ 675 const bus_size_t lpipend_sz = (sc->sc_lpi.pic_maxsources + sc->sc_lpi.pic_irqbase) / NBBY; 676 for (int cpuindex = 0; cpuindex < MAXCPUS; cpuindex++) { 677 gicv3_dma_alloc(sc, &sc->sc_lpipend[cpuindex], lpipend_sz, 0x10000); 678 KASSERT((sc->sc_lpipend[cpuindex].segs[0].ds_addr & ~GICR_PENDBASER_Physical_Address) == 0); 679 } 680 } 681 682 void 683 gicv3_irq_handler(void *frame) 684 { 685 struct cpu_info * const ci = curcpu(); 686 struct gicv3_softc * const sc = gicv3_softc; 687 struct pic_softc *pic; 688 const int oldipl = ci->ci_cpl; 689 690 ci->ci_data.cpu_nintr++; 691 692 for (;;) { 693 const uint32_t iar = icc_iar1_read(); 694 const uint32_t irq = __SHIFTOUT(iar, ICC_IAR_INTID); 695 if (irq == ICC_IAR_INTID_SPURIOUS) 696 break; 697 698 pic = irq >= GIC_LPI_BASE ? &sc->sc_lpi : &sc->sc_pic; 699 if (irq - pic->pic_irqbase >= pic->pic_maxsources) 700 continue; 701 702 struct intrsource * const is = pic->pic_sources[irq - pic->pic_irqbase]; 703 KASSERT(is != NULL); 704 705 const int ipl = is->is_ipl; 706 if (ci->ci_cpl < ipl) 707 pic_set_priority(ci, ipl); 708 709 cpsie(I32_bit); 710 pic_dispatch(is, frame); 711 cpsid(I32_bit); 712 713 icc_eoi1r_write(iar); 714 } 715 716 if (ci->ci_cpl != oldipl) 717 pic_set_priority(ci, oldipl); 718 } 719 720 int 721 gicv3_init(struct gicv3_softc *sc) 722 { 723 const uint32_t gicd_typer = gicd_read_4(sc, GICD_TYPER); 724 int n; 725 726 KASSERT(CPU_IS_PRIMARY(curcpu())); 727 728 LIST_INIT(&sc->sc_lpi_callbacks); 729 730 for (n = 0; n < MAXCPUS; n++) 731 sc->sc_irouter[n] = UINT64_MAX; 732 733 sc->sc_pic.pic_ops = &gicv3_picops; 734 sc->sc_pic.pic_maxsources = GICD_TYPER_LINES(gicd_typer); 735 snprintf(sc->sc_pic.pic_name, sizeof(sc->sc_pic.pic_name), "gicv3"); 736 #ifdef MULTIPROCESSOR 737 sc->sc_pic.pic_cpus = kcpuset_running; 738 #endif 739 pic_add(&sc->sc_pic, 0); 740 741 if ((gicd_typer & GICD_TYPER_LPIS) != 0) { 742 sc->sc_lpi.pic_ops = &gicv3_lpiops; 743 sc->sc_lpi.pic_maxsources = 8192; /* Min. required by GICv3 spec */ 744 snprintf(sc->sc_lpi.pic_name, sizeof(sc->sc_lpi.pic_name), "gicv3-lpi"); 745 pic_add(&sc->sc_lpi, GIC_LPI_BASE); 746 747 gicv3_lpi_init(sc); 748 } 749 750 KASSERT(gicv3_softc == NULL); 751 gicv3_softc = sc; 752 753 for (int i = 0; i < sc->sc_bsh_r_count; i++) { 754 const uint64_t gicr_typer = gicr_read_8(sc, i, GICR_TYPER); 755 const u_int aff0 = __SHIFTOUT(gicr_typer, GICR_TYPER_Affinity_Value_Aff0); 756 const u_int aff1 = __SHIFTOUT(gicr_typer, GICR_TYPER_Affinity_Value_Aff1); 757 const u_int aff2 = __SHIFTOUT(gicr_typer, GICR_TYPER_Affinity_Value_Aff2); 758 const u_int aff3 = __SHIFTOUT(gicr_typer, GICR_TYPER_Affinity_Value_Aff3); 759 760 aprint_debug_dev(sc->sc_dev, "redist %d: cpu %d.%d.%d.%d\n", 761 i, aff3, aff2, aff1, aff0); 762 } 763 764 gicv3_dist_enable(sc); 765 766 gicv3_cpu_init(&sc->sc_pic, curcpu()); 767 if ((gicd_typer & GICD_TYPER_LPIS) != 0) 768 gicv3_lpi_cpu_init(&sc->sc_lpi, curcpu()); 769 770 #ifdef __HAVE_PIC_FAST_SOFTINTS 771 intr_establish(SOFTINT_BIO, IPL_SOFTBIO, IST_MPSAFE | IST_EDGE, pic_handle_softint, (void *)SOFTINT_BIO); 772 intr_establish(SOFTINT_CLOCK, IPL_SOFTCLOCK, IST_MPSAFE | IST_EDGE, pic_handle_softint, (void *)SOFTINT_CLOCK); 773 intr_establish(SOFTINT_NET, IPL_SOFTNET, IST_MPSAFE | IST_EDGE, pic_handle_softint, (void *)SOFTINT_NET); 774 intr_establish(SOFTINT_SERIAL, IPL_SOFTSERIAL, IST_MPSAFE | IST_EDGE, pic_handle_softint, (void *)SOFTINT_SERIAL); 775 #endif 776 777 #ifdef MULTIPROCESSOR 778 intr_establish(IPI_AST, IPL_VM, IST_MPSAFE | IST_EDGE, pic_ipi_ast, (void *)-1); 779 intr_establish(IPI_XCALL, IPL_HIGH, IST_MPSAFE | IST_EDGE, pic_ipi_xcall, (void *)-1); 780 intr_establish(IPI_GENERIC, IPL_HIGH, IST_MPSAFE | IST_EDGE, pic_ipi_generic, (void *)-1); 781 intr_establish(IPI_NOP, IPL_VM, IST_MPSAFE | IST_EDGE, pic_ipi_nop, (void *)-1); 782 intr_establish(IPI_SHOOTDOWN, IPL_SCHED, IST_MPSAFE | IST_EDGE, pic_ipi_shootdown, (void *)-1); 783 #ifdef DDB 784 intr_establish(IPI_DDB, IPL_HIGH, IST_MPSAFE | IST_EDGE, pic_ipi_ddb, NULL); 785 #endif 786 #ifdef __HAVE_PREEMPTION 787 intr_establish(IPI_KPREEMPT, IPL_VM, IST_MPSAFE | IST_EDGE, pic_ipi_kpreempt, (void *)-1); 788 #endif 789 #endif 790 791 return 0; 792 } 793
Indexes created Sun Sep 21 20:09:37 GMT 2025