gicv3.c revision 1.2.2.4
1 1.2.2.4 pgoyette /* $NetBSD: gicv3.c,v 1.2.2.4 2018/11/26 01:52:18 pgoyette Exp $ */ 2 1.2.2.2 pgoyette 3 1.2.2.2 pgoyette /*- 4 1.2.2.2 pgoyette * Copyright (c) 2018 Jared McNeill <jmcneill (at) invisible.ca> 5 1.2.2.2 pgoyette * All rights reserved. 6 1.2.2.2 pgoyette * 7 1.2.2.2 pgoyette * Redistribution and use in source and binary forms, with or without 8 1.2.2.2 pgoyette * modification, are permitted provided that the following conditions 9 1.2.2.2 pgoyette * are met: 10 1.2.2.2 pgoyette * 1. Redistributions of source code must retain the above copyright 11 1.2.2.2 pgoyette * notice, this list of conditions and the following disclaimer. 12 1.2.2.2 pgoyette * 2. Redistributions in binary form must reproduce the above copyright 13 1.2.2.2 pgoyette * notice, this list of conditions and the following disclaimer in the 14 1.2.2.2 pgoyette * documentation and/or other materials provided with the distribution. 15 1.2.2.2 pgoyette * 16 1.2.2.2 pgoyette * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 17 1.2.2.2 pgoyette * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 18 1.2.2.2 pgoyette * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 19 1.2.2.2 pgoyette * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 20 1.2.2.2 pgoyette * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 21 1.2.2.2 pgoyette * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 22 1.2.2.2 pgoyette * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 23 1.2.2.2 pgoyette * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 24 1.2.2.2 pgoyette * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 1.2.2.2 pgoyette * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 1.2.2.2 pgoyette * SUCH DAMAGE. 27 1.2.2.2 pgoyette */ 28 1.2.2.2 pgoyette 29 1.2.2.2 pgoyette #include "opt_multiprocessor.h" 30 1.2.2.2 pgoyette 31 1.2.2.2 pgoyette #define _INTR_PRIVATE 32 1.2.2.2 pgoyette 33 1.2.2.2 pgoyette #include <sys/cdefs.h> 34 1.2.2.4 pgoyette __KERNEL_RCSID(0, "$NetBSD: gicv3.c,v 1.2.2.4 2018/11/26 01:52:18 pgoyette Exp $"); 35 1.2.2.2 pgoyette 36 1.2.2.2 pgoyette #include <sys/param.h> 37 1.2.2.2 pgoyette #include <sys/kernel.h> 38 1.2.2.2 pgoyette #include <sys/bus.h> 39 1.2.2.2 pgoyette #include <sys/device.h> 40 1.2.2.2 pgoyette #include <sys/intr.h> 41 1.2.2.2 pgoyette #include <sys/systm.h> 42 1.2.2.2 pgoyette #include <sys/cpu.h> 43 1.2.2.2 pgoyette 44 1.2.2.2 pgoyette #include <arm/locore.h> 45 1.2.2.2 pgoyette #include <arm/armreg.h> 46 1.2.2.2 pgoyette 47 1.2.2.2 pgoyette #include <arm/cortex/gicv3.h> 48 1.2.2.2 pgoyette #include <arm/cortex/gic_reg.h> 49 1.2.2.2 pgoyette 50 1.2.2.2 pgoyette #define PICTOSOFTC(pic) \ 51 1.2.2.2 pgoyette ((void *)((uintptr_t)(pic) - offsetof(struct gicv3_softc, sc_pic))) 52 1.2.2.4 pgoyette #define LPITOSOFTC(lpi) \ 53 1.2.2.4 pgoyette ((void *)((uintptr_t)(lpi) - offsetof(struct gicv3_softc, sc_lpi))) 54 1.2.2.2 pgoyette 55 1.2.2.4 pgoyette #define IPL_TO_PRIORITY(ipl) ((IPL_HIGH - (ipl)) << 4) 56 1.2.2.2 pgoyette 57 1.2.2.2 pgoyette static struct gicv3_softc *gicv3_softc; 58 1.2.2.2 pgoyette 59 1.2.2.2 pgoyette static inline uint32_t 60 1.2.2.2 pgoyette gicd_read_4(struct gicv3_softc *sc, bus_size_t reg) 61 1.2.2.2 pgoyette { 62 1.2.2.2 pgoyette return bus_space_read_4(sc->sc_bst, sc->sc_bsh_d, reg); 63 1.2.2.2 pgoyette } 64 1.2.2.2 pgoyette 65 1.2.2.2 pgoyette static inline void 66 1.2.2.2 pgoyette gicd_write_4(struct gicv3_softc *sc, bus_size_t reg, uint32_t val) 67 1.2.2.2 pgoyette { 68 1.2.2.2 pgoyette bus_space_write_4(sc->sc_bst, sc->sc_bsh_d, reg, val); 69 1.2.2.2 pgoyette } 70 1.2.2.2 pgoyette 71 1.2.2.4 pgoyette static inline uint64_t 72 1.2.2.4 pgoyette gicd_read_8(struct gicv3_softc *sc, bus_size_t reg) 73 1.2.2.4 pgoyette { 74 1.2.2.4 pgoyette return bus_space_read_8(sc->sc_bst, sc->sc_bsh_d, reg); 75 1.2.2.4 pgoyette } 76 1.2.2.4 pgoyette 77 1.2.2.2 pgoyette static inline void 78 1.2.2.2 pgoyette gicd_write_8(struct gicv3_softc *sc, bus_size_t reg, uint64_t val) 79 1.2.2.2 pgoyette { 80 1.2.2.2 pgoyette bus_space_write_8(sc->sc_bst, sc->sc_bsh_d, reg, val); 81 1.2.2.2 pgoyette } 82 1.2.2.2 pgoyette 83 1.2.2.2 pgoyette static inline uint32_t 84 1.2.2.2 pgoyette gicr_read_4(struct gicv3_softc *sc, u_int index, bus_size_t reg) 85 1.2.2.2 pgoyette { 86 1.2.2.2 pgoyette KASSERT(index < sc->sc_bsh_r_count); 87 1.2.2.2 pgoyette return bus_space_read_4(sc->sc_bst, sc->sc_bsh_r[index], reg); 88 1.2.2.2 pgoyette } 89 1.2.2.2 pgoyette 90 1.2.2.2 pgoyette static inline void 91 1.2.2.2 pgoyette gicr_write_4(struct gicv3_softc *sc, u_int index, bus_size_t reg, uint32_t val) 92 1.2.2.2 pgoyette { 93 1.2.2.2 pgoyette KASSERT(index < sc->sc_bsh_r_count); 94 1.2.2.2 pgoyette bus_space_write_4(sc->sc_bst, sc->sc_bsh_r[index], reg, val); 95 1.2.2.2 pgoyette } 96 1.2.2.2 pgoyette 97 1.2.2.2 pgoyette static inline uint64_t 98 1.2.2.2 pgoyette gicr_read_8(struct gicv3_softc *sc, u_int index, bus_size_t reg) 99 1.2.2.2 pgoyette { 100 1.2.2.2 pgoyette KASSERT(index < sc->sc_bsh_r_count); 101 1.2.2.2 pgoyette return bus_space_read_8(sc->sc_bst, sc->sc_bsh_r[index], reg); 102 1.2.2.2 pgoyette } 103 1.2.2.2 pgoyette 104 1.2.2.2 pgoyette static inline void 105 1.2.2.2 pgoyette gicr_write_8(struct gicv3_softc *sc, u_int index, bus_size_t reg, uint64_t val) 106 1.2.2.2 pgoyette { 107 1.2.2.2 pgoyette KASSERT(index < sc->sc_bsh_r_count); 108 1.2.2.2 pgoyette bus_space_write_8(sc->sc_bst, sc->sc_bsh_r[index], reg, val); 109 1.2.2.2 pgoyette } 110 1.2.2.2 pgoyette 111 1.2.2.2 pgoyette static void 112 1.2.2.2 pgoyette gicv3_unblock_irqs(struct pic_softc *pic, size_t irqbase, uint32_t mask) 113 1.2.2.2 pgoyette { 114 1.2.2.2 pgoyette struct gicv3_softc * const sc = PICTOSOFTC(pic); 115 1.2.2.2 pgoyette struct cpu_info * const ci = curcpu(); 116 1.2.2.2 pgoyette const u_int group = irqbase / 32; 117 1.2.2.2 pgoyette 118 1.2.2.2 pgoyette if (group == 0) { 119 1.2.2.2 pgoyette sc->sc_enabled_sgippi |= mask; 120 1.2.2.2 pgoyette gicr_write_4(sc, ci->ci_gic_redist, GICR_ISENABLER0, mask); 121 1.2.2.4 pgoyette while (gicr_read_4(sc, ci->ci_gic_redist, GICR_CTLR) & GICR_CTLR_RWP) 122 1.2.2.2 pgoyette ; 123 1.2.2.2 pgoyette } else { 124 1.2.2.2 pgoyette gicd_write_4(sc, GICD_ISENABLERn(group), mask); 125 1.2.2.2 pgoyette while (gicd_read_4(sc, GICD_CTRL) & GICD_CTRL_RWP) 126 1.2.2.2 pgoyette ; 127 1.2.2.2 pgoyette } 128 1.2.2.2 pgoyette } 129 1.2.2.2 pgoyette 130 1.2.2.2 pgoyette static void 131 1.2.2.2 pgoyette gicv3_block_irqs(struct pic_softc *pic, size_t irqbase, uint32_t mask) 132 1.2.2.2 pgoyette { 133 1.2.2.2 pgoyette struct gicv3_softc * const sc = PICTOSOFTC(pic); 134 1.2.2.2 pgoyette struct cpu_info * const ci = curcpu(); 135 1.2.2.2 pgoyette const u_int group = irqbase / 32; 136 1.2.2.2 pgoyette 137 1.2.2.2 pgoyette if (group == 0) { 138 1.2.2.2 pgoyette sc->sc_enabled_sgippi &= ~mask; 139 1.2.2.2 pgoyette gicr_write_4(sc, ci->ci_gic_redist, GICR_ICENABLER0, mask); 140 1.2.2.4 pgoyette while (gicr_read_4(sc, ci->ci_gic_redist, GICR_CTLR) & GICR_CTLR_RWP) 141 1.2.2.2 pgoyette ; 142 1.2.2.2 pgoyette } else { 143 1.2.2.2 pgoyette gicd_write_4(sc, GICD_ICENABLERn(group), mask); 144 1.2.2.2 pgoyette while (gicd_read_4(sc, GICD_CTRL) & GICD_CTRL_RWP) 145 1.2.2.2 pgoyette ; 146 1.2.2.2 pgoyette } 147 1.2.2.2 pgoyette } 148 1.2.2.2 pgoyette 149 1.2.2.2 pgoyette static void 150 1.2.2.2 pgoyette gicv3_establish_irq(struct pic_softc *pic, struct intrsource *is) 151 1.2.2.2 pgoyette { 152 1.2.2.2 pgoyette struct gicv3_softc * const sc = PICTOSOFTC(pic); 153 1.2.2.2 pgoyette const u_int group = is->is_irq / 32; 154 1.2.2.2 pgoyette uint32_t ipriority, icfg; 155 1.2.2.2 pgoyette uint64_t irouter; 156 1.2.2.2 pgoyette u_int n; 157 1.2.2.2 pgoyette 158 1.2.2.4 pgoyette const u_int ipriority_val = 0x80 | IPL_TO_PRIORITY(is->is_ipl); 159 1.2.2.2 pgoyette const u_int ipriority_shift = (is->is_irq & 0x3) * 8; 160 1.2.2.2 pgoyette const u_int icfg_shift = (is->is_irq & 0xf) * 2; 161 1.2.2.2 pgoyette 162 1.2.2.2 pgoyette if (group == 0) { 163 1.2.2.2 pgoyette /* SGIs and PPIs are always MP-safe */ 164 1.2.2.2 pgoyette is->is_mpsafe = true; 165 1.2.2.2 pgoyette 166 1.2.2.2 pgoyette /* Update interrupt configuration and priority on all redistributors */ 167 1.2.2.2 pgoyette for (n = 0; n < sc->sc_bsh_r_count; n++) { 168 1.2.2.2 pgoyette icfg = gicr_read_4(sc, n, GICR_ICFGRn(is->is_irq / 16)); 169 1.2.2.2 pgoyette if (is->is_type == IST_LEVEL) 170 1.2.2.2 pgoyette icfg &= ~(0x2 << icfg_shift); 171 1.2.2.2 pgoyette if (is->is_type == IST_EDGE) 172 1.2.2.2 pgoyette icfg |= (0x2 << icfg_shift); 173 1.2.2.2 pgoyette gicr_write_4(sc, n, GICR_ICFGRn(is->is_irq / 16), icfg); 174 1.2.2.2 pgoyette 175 1.2.2.2 pgoyette ipriority = gicr_read_4(sc, n, GICR_IPRIORITYRn(is->is_irq / 4)); 176 1.2.2.2 pgoyette ipriority &= ~(0xff << ipriority_shift); 177 1.2.2.2 pgoyette ipriority |= (ipriority_val << ipriority_shift); 178 1.2.2.2 pgoyette gicr_write_4(sc, n, GICR_IPRIORITYRn(is->is_irq / 4), ipriority); 179 1.2.2.2 pgoyette } 180 1.2.2.2 pgoyette } else { 181 1.2.2.2 pgoyette if (is->is_mpsafe) { 182 1.2.2.2 pgoyette /* Route MP-safe interrupts to all participating PEs */ 183 1.2.2.2 pgoyette irouter = GICD_IROUTER_Interrupt_Routing_mode; 184 1.2.2.2 pgoyette } else { 185 1.2.2.2 pgoyette /* Route non-MP-safe interrupts to the primary PE only */ 186 1.2.2.4 pgoyette irouter = sc->sc_irouter[0]; 187 1.2.2.2 pgoyette } 188 1.2.2.2 pgoyette gicd_write_8(sc, GICD_IROUTER(is->is_irq), irouter); 189 1.2.2.2 pgoyette 190 1.2.2.2 pgoyette /* Update interrupt configuration */ 191 1.2.2.2 pgoyette icfg = gicd_read_4(sc, GICD_ICFGRn(is->is_irq / 16)); 192 1.2.2.2 pgoyette if (is->is_type == IST_LEVEL) 193 1.2.2.2 pgoyette icfg &= ~(0x2 << icfg_shift); 194 1.2.2.2 pgoyette if (is->is_type == IST_EDGE) 195 1.2.2.2 pgoyette icfg |= (0x2 << icfg_shift); 196 1.2.2.2 pgoyette gicd_write_4(sc, GICD_ICFGRn(is->is_irq / 16), icfg); 197 1.2.2.2 pgoyette 198 1.2.2.2 pgoyette /* Update interrupt priority */ 199 1.2.2.2 pgoyette ipriority = gicd_read_4(sc, GICD_IPRIORITYRn(is->is_irq / 4)); 200 1.2.2.2 pgoyette ipriority &= ~(0xff << ipriority_shift); 201 1.2.2.2 pgoyette ipriority |= (ipriority_val << ipriority_shift); 202 1.2.2.2 pgoyette gicd_write_4(sc, GICD_IPRIORITYRn(is->is_irq / 4), ipriority); 203 1.2.2.2 pgoyette } 204 1.2.2.2 pgoyette } 205 1.2.2.2 pgoyette 206 1.2.2.2 pgoyette static void 207 1.2.2.2 pgoyette gicv3_set_priority(struct pic_softc *pic, int ipl) 208 1.2.2.2 pgoyette { 209 1.2.2.4 pgoyette icc_pmr_write(IPL_TO_PRIORITY(ipl) << 1); 210 1.2.2.2 pgoyette } 211 1.2.2.2 pgoyette 212 1.2.2.2 pgoyette static void 213 1.2.2.2 pgoyette gicv3_dist_enable(struct gicv3_softc *sc) 214 1.2.2.2 pgoyette { 215 1.2.2.2 pgoyette uint32_t gicd_ctrl; 216 1.2.2.2 pgoyette u_int n; 217 1.2.2.2 pgoyette 218 1.2.2.2 pgoyette /* Disable the distributor */ 219 1.2.2.2 pgoyette gicd_write_4(sc, GICD_CTRL, 0); 220 1.2.2.2 pgoyette 221 1.2.2.2 pgoyette /* Wait for register write to complete */ 222 1.2.2.2 pgoyette while (gicd_read_4(sc, GICD_CTRL) & GICD_CTRL_RWP) 223 1.2.2.2 pgoyette ; 224 1.2.2.2 pgoyette 225 1.2.2.2 pgoyette /* Clear all INTID enable bits */ 226 1.2.2.2 pgoyette for (n = 32; n < sc->sc_pic.pic_maxsources; n += 32) 227 1.2.2.2 pgoyette gicd_write_4(sc, GICD_ICENABLERn(n / 32), ~0); 228 1.2.2.2 pgoyette 229 1.2.2.2 pgoyette /* Set default priorities to lowest */ 230 1.2.2.2 pgoyette for (n = 32; n < sc->sc_pic.pic_maxsources; n += 4) 231 1.2.2.2 pgoyette gicd_write_4(sc, GICD_IPRIORITYRn(n / 4), ~0); 232 1.2.2.2 pgoyette 233 1.2.2.2 pgoyette /* Set all interrupts to G1NS */ 234 1.2.2.2 pgoyette for (n = 32; n < sc->sc_pic.pic_maxsources; n += 32) { 235 1.2.2.2 pgoyette gicd_write_4(sc, GICD_IGROUPRn(n / 32), ~0); 236 1.2.2.2 pgoyette gicd_write_4(sc, GICD_IGRPMODRn(n / 32), 0); 237 1.2.2.2 pgoyette } 238 1.2.2.2 pgoyette 239 1.2.2.2 pgoyette /* Set all interrupts level-sensitive by default */ 240 1.2.2.2 pgoyette for (n = 32; n < sc->sc_pic.pic_maxsources; n += 16) 241 1.2.2.2 pgoyette gicd_write_4(sc, GICD_ICFGRn(n / 16), 0); 242 1.2.2.2 pgoyette 243 1.2.2.2 pgoyette /* Wait for register writes to complete */ 244 1.2.2.2 pgoyette while (gicd_read_4(sc, GICD_CTRL) & GICD_CTRL_RWP) 245 1.2.2.2 pgoyette ; 246 1.2.2.2 pgoyette 247 1.2.2.2 pgoyette /* Enable Affinity routing and G1NS interrupts */ 248 1.2.2.4 pgoyette gicd_ctrl = GICD_CTRL_EnableGrp1A | GICD_CTRL_Enable | GICD_CTRL_ARE_NS; 249 1.2.2.2 pgoyette gicd_write_4(sc, GICD_CTRL, gicd_ctrl); 250 1.2.2.2 pgoyette } 251 1.2.2.2 pgoyette 252 1.2.2.2 pgoyette static void 253 1.2.2.2 pgoyette gicv3_redist_enable(struct gicv3_softc *sc, struct cpu_info *ci) 254 1.2.2.2 pgoyette { 255 1.2.2.2 pgoyette uint32_t icfg; 256 1.2.2.2 pgoyette u_int n, o; 257 1.2.2.2 pgoyette 258 1.2.2.2 pgoyette /* Clear INTID enable bits */ 259 1.2.2.2 pgoyette gicr_write_4(sc, ci->ci_gic_redist, GICR_ICENABLER0, ~0); 260 1.2.2.2 pgoyette 261 1.2.2.2 pgoyette /* Wait for register write to complete */ 262 1.2.2.4 pgoyette while (gicr_read_4(sc, ci->ci_gic_redist, GICR_CTLR) & GICR_CTLR_RWP) 263 1.2.2.2 pgoyette ; 264 1.2.2.2 pgoyette 265 1.2.2.2 pgoyette /* Set default priorities */ 266 1.2.2.2 pgoyette for (n = 0; n < 32; n += 4) { 267 1.2.2.2 pgoyette uint32_t priority = 0; 268 1.2.2.2 pgoyette size_t byte_shift = 0; 269 1.2.2.2 pgoyette for (o = 0; o < 4; o++, byte_shift += 8) { 270 1.2.2.2 pgoyette struct intrsource * const is = sc->sc_pic.pic_sources[n + o]; 271 1.2.2.2 pgoyette if (is == NULL) 272 1.2.2.2 pgoyette priority |= 0xff << byte_shift; 273 1.2.2.2 pgoyette else { 274 1.2.2.2 pgoyette const u_int ipriority_val = 0x80 | IPL_TO_PRIORITY(is->is_ipl); 275 1.2.2.2 pgoyette priority |= ipriority_val << byte_shift; 276 1.2.2.2 pgoyette } 277 1.2.2.2 pgoyette } 278 1.2.2.2 pgoyette gicr_write_4(sc, ci->ci_gic_redist, GICR_IPRIORITYRn(n / 4), priority); 279 1.2.2.2 pgoyette } 280 1.2.2.2 pgoyette 281 1.2.2.2 pgoyette /* Set all interrupts to G1NS */ 282 1.2.2.2 pgoyette gicr_write_4(sc, ci->ci_gic_redist, GICR_IGROUPR0, ~0); 283 1.2.2.2 pgoyette gicr_write_4(sc, ci->ci_gic_redist, GICR_IGRPMODR0, 0); 284 1.2.2.2 pgoyette 285 1.2.2.2 pgoyette /* Restore PPI configs */ 286 1.2.2.2 pgoyette for (n = 0, icfg = 0; n < 16; n++) { 287 1.2.2.2 pgoyette struct intrsource * const is = sc->sc_pic.pic_sources[16 + n]; 288 1.2.2.2 pgoyette if (is != NULL && is->is_type == IST_EDGE) 289 1.2.2.2 pgoyette icfg |= (0x2 << (n * 2)); 290 1.2.2.2 pgoyette } 291 1.2.2.2 pgoyette gicr_write_4(sc, ci->ci_gic_redist, GICR_ICFGRn(1), icfg); 292 1.2.2.2 pgoyette 293 1.2.2.2 pgoyette /* Restore current enable bits */ 294 1.2.2.2 pgoyette gicr_write_4(sc, ci->ci_gic_redist, GICR_ISENABLER0, sc->sc_enabled_sgippi); 295 1.2.2.2 pgoyette 296 1.2.2.2 pgoyette /* Wait for register write to complete */ 297 1.2.2.4 pgoyette while (gicr_read_4(sc, ci->ci_gic_redist, GICR_CTLR) & GICR_CTLR_RWP) 298 1.2.2.2 pgoyette ; 299 1.2.2.2 pgoyette } 300 1.2.2.2 pgoyette 301 1.2.2.2 pgoyette static uint64_t 302 1.2.2.2 pgoyette gicv3_cpu_identity(void) 303 1.2.2.2 pgoyette { 304 1.2.2.2 pgoyette u_int aff3, aff2, aff1, aff0; 305 1.2.2.2 pgoyette 306 1.2.2.2 pgoyette #ifdef __aarch64__ 307 1.2.2.2 pgoyette const register_t mpidr = reg_mpidr_el1_read(); 308 1.2.2.2 pgoyette aff0 = __SHIFTOUT(mpidr, MPIDR_AFF0); 309 1.2.2.2 pgoyette aff1 = __SHIFTOUT(mpidr, MPIDR_AFF1); 310 1.2.2.2 pgoyette aff2 = __SHIFTOUT(mpidr, MPIDR_AFF2); 311 1.2.2.2 pgoyette aff3 = __SHIFTOUT(mpidr, MPIDR_AFF3); 312 1.2.2.2 pgoyette #else 313 1.2.2.2 pgoyette const register_t mpidr = armreg_mpidr_read(); 314 1.2.2.2 pgoyette aff0 = __SHIFTOUT(mpidr, MPIDR_AFF0); 315 1.2.2.2 pgoyette aff1 = __SHIFTOUT(mpidr, MPIDR_AFF1); 316 1.2.2.2 pgoyette aff2 = __SHIFTOUT(mpidr, MPIDR_AFF2); 317 1.2.2.2 pgoyette aff3 = 0; 318 1.2.2.2 pgoyette #endif 319 1.2.2.2 pgoyette 320 1.2.2.2 pgoyette return __SHIFTIN(aff0, GICR_TYPER_Affinity_Value_Aff0) | 321 1.2.2.2 pgoyette __SHIFTIN(aff1, GICR_TYPER_Affinity_Value_Aff1) | 322 1.2.2.2 pgoyette __SHIFTIN(aff2, GICR_TYPER_Affinity_Value_Aff2) | 323 1.2.2.2 pgoyette __SHIFTIN(aff3, GICR_TYPER_Affinity_Value_Aff3); 324 1.2.2.2 pgoyette } 325 1.2.2.2 pgoyette 326 1.2.2.2 pgoyette static u_int 327 1.2.2.2 pgoyette gicv3_find_redist(struct gicv3_softc *sc) 328 1.2.2.2 pgoyette { 329 1.2.2.2 pgoyette uint64_t gicr_typer; 330 1.2.2.2 pgoyette u_int n; 331 1.2.2.2 pgoyette 332 1.2.2.2 pgoyette const uint64_t cpu_identity = gicv3_cpu_identity(); 333 1.2.2.2 pgoyette 334 1.2.2.2 pgoyette for (n = 0; n < sc->sc_bsh_r_count; n++) { 335 1.2.2.2 pgoyette gicr_typer = gicr_read_8(sc, n, GICR_TYPER); 336 1.2.2.2 pgoyette if ((gicr_typer & GICR_TYPER_Affinity_Value) == cpu_identity) 337 1.2.2.2 pgoyette return n; 338 1.2.2.2 pgoyette } 339 1.2.2.2 pgoyette 340 1.2.2.2 pgoyette const u_int aff0 = __SHIFTOUT(cpu_identity, GICR_TYPER_Affinity_Value_Aff0); 341 1.2.2.2 pgoyette const u_int aff1 = __SHIFTOUT(cpu_identity, GICR_TYPER_Affinity_Value_Aff1); 342 1.2.2.2 pgoyette const u_int aff2 = __SHIFTOUT(cpu_identity, GICR_TYPER_Affinity_Value_Aff2); 343 1.2.2.2 pgoyette const u_int aff3 = __SHIFTOUT(cpu_identity, GICR_TYPER_Affinity_Value_Aff3); 344 1.2.2.2 pgoyette 345 1.2.2.2 pgoyette panic("%s: could not find GICv3 redistributor for cpu %d.%d.%d.%d", 346 1.2.2.2 pgoyette cpu_name(curcpu()), aff3, aff2, aff1, aff0); 347 1.2.2.2 pgoyette } 348 1.2.2.2 pgoyette 349 1.2.2.2 pgoyette static uint64_t 350 1.2.2.2 pgoyette gicv3_sgir(struct gicv3_softc *sc) 351 1.2.2.2 pgoyette { 352 1.2.2.2 pgoyette const uint64_t cpu_identity = gicv3_cpu_identity(); 353 1.2.2.2 pgoyette 354 1.2.2.2 pgoyette const u_int aff0 = __SHIFTOUT(cpu_identity, GICR_TYPER_Affinity_Value_Aff0); 355 1.2.2.2 pgoyette const u_int aff1 = __SHIFTOUT(cpu_identity, GICR_TYPER_Affinity_Value_Aff1); 356 1.2.2.2 pgoyette const u_int aff2 = __SHIFTOUT(cpu_identity, GICR_TYPER_Affinity_Value_Aff2); 357 1.2.2.2 pgoyette const u_int aff3 = __SHIFTOUT(cpu_identity, GICR_TYPER_Affinity_Value_Aff3); 358 1.2.2.2 pgoyette 359 1.2.2.2 pgoyette return __SHIFTIN(__BIT(aff0), ICC_SGIR_EL1_TargetList) | 360 1.2.2.2 pgoyette __SHIFTIN(aff1, ICC_SGIR_EL1_Aff1) | 361 1.2.2.2 pgoyette __SHIFTIN(aff2, ICC_SGIR_EL1_Aff2) | 362 1.2.2.2 pgoyette __SHIFTIN(aff3, ICC_SGIR_EL1_Aff3); 363 1.2.2.2 pgoyette } 364 1.2.2.2 pgoyette 365 1.2.2.2 pgoyette static void 366 1.2.2.2 pgoyette gicv3_cpu_init(struct pic_softc *pic, struct cpu_info *ci) 367 1.2.2.2 pgoyette { 368 1.2.2.2 pgoyette struct gicv3_softc * const sc = PICTOSOFTC(pic); 369 1.2.2.2 pgoyette uint32_t icc_sre, icc_ctlr, gicr_waker; 370 1.2.2.2 pgoyette 371 1.2.2.2 pgoyette ci->ci_gic_redist = gicv3_find_redist(sc); 372 1.2.2.2 pgoyette ci->ci_gic_sgir = gicv3_sgir(sc); 373 1.2.2.2 pgoyette 374 1.2.2.4 pgoyette /* Store route to CPU for SPIs */ 375 1.2.2.4 pgoyette const uint64_t cpu_identity = gicv3_cpu_identity(); 376 1.2.2.4 pgoyette const u_int aff0 = __SHIFTOUT(cpu_identity, GICR_TYPER_Affinity_Value_Aff0); 377 1.2.2.4 pgoyette const u_int aff1 = __SHIFTOUT(cpu_identity, GICR_TYPER_Affinity_Value_Aff1); 378 1.2.2.4 pgoyette const u_int aff2 = __SHIFTOUT(cpu_identity, GICR_TYPER_Affinity_Value_Aff2); 379 1.2.2.4 pgoyette const u_int aff3 = __SHIFTOUT(cpu_identity, GICR_TYPER_Affinity_Value_Aff3); 380 1.2.2.4 pgoyette sc->sc_irouter[cpu_index(ci)] = 381 1.2.2.4 pgoyette __SHIFTIN(aff0, GICD_IROUTER_Aff0) | 382 1.2.2.4 pgoyette __SHIFTIN(aff1, GICD_IROUTER_Aff1) | 383 1.2.2.4 pgoyette __SHIFTIN(aff2, GICD_IROUTER_Aff2) | 384 1.2.2.4 pgoyette __SHIFTIN(aff3, GICD_IROUTER_Aff3); 385 1.2.2.2 pgoyette 386 1.2.2.2 pgoyette /* Enable System register access and disable IRQ/FIQ bypass */ 387 1.2.2.2 pgoyette icc_sre = ICC_SRE_EL1_SRE | ICC_SRE_EL1_DFB | ICC_SRE_EL1_DIB; 388 1.2.2.2 pgoyette icc_sre_write(icc_sre); 389 1.2.2.2 pgoyette 390 1.2.2.2 pgoyette /* Mark the connected PE as being awake */ 391 1.2.2.2 pgoyette gicr_waker = gicr_read_4(sc, ci->ci_gic_redist, GICR_WAKER); 392 1.2.2.2 pgoyette gicr_waker &= ~GICR_WAKER_ProcessorSleep; 393 1.2.2.2 pgoyette gicr_write_4(sc, ci->ci_gic_redist, GICR_WAKER, gicr_waker); 394 1.2.2.2 pgoyette while (gicr_read_4(sc, ci->ci_gic_redist, GICR_WAKER) & GICR_WAKER_ChildrenAsleep) 395 1.2.2.2 pgoyette ; 396 1.2.2.2 pgoyette 397 1.2.2.2 pgoyette /* Set initial priority mask */ 398 1.2.2.4 pgoyette gicv3_set_priority(pic, IPL_HIGH); 399 1.2.2.2 pgoyette 400 1.2.2.4 pgoyette /* Set the binary point field to the minimum value */ 401 1.2.2.4 pgoyette icc_bpr1_write(0); 402 1.2.2.2 pgoyette 403 1.2.2.2 pgoyette /* Enable group 1 interrupt signaling */ 404 1.2.2.2 pgoyette icc_igrpen1_write(ICC_IGRPEN_EL1_Enable); 405 1.2.2.2 pgoyette 406 1.2.2.2 pgoyette /* Set EOI mode */ 407 1.2.2.2 pgoyette icc_ctlr = icc_ctlr_read(); 408 1.2.2.2 pgoyette icc_ctlr &= ~ICC_CTLR_EL1_EOImode; 409 1.2.2.2 pgoyette icc_ctlr_write(icc_ctlr); 410 1.2.2.2 pgoyette 411 1.2.2.2 pgoyette /* Enable redistributor */ 412 1.2.2.2 pgoyette gicv3_redist_enable(sc, ci); 413 1.2.2.2 pgoyette 414 1.2.2.2 pgoyette /* Allow IRQ exceptions */ 415 1.2.2.2 pgoyette cpsie(I32_bit); 416 1.2.2.2 pgoyette } 417 1.2.2.2 pgoyette 418 1.2.2.2 pgoyette #ifdef MULTIPROCESSOR 419 1.2.2.2 pgoyette static void 420 1.2.2.2 pgoyette gicv3_ipi_send(struct pic_softc *pic, const kcpuset_t *kcp, u_long ipi) 421 1.2.2.2 pgoyette { 422 1.2.2.2 pgoyette CPU_INFO_ITERATOR cii; 423 1.2.2.2 pgoyette struct cpu_info *ci; 424 1.2.2.2 pgoyette uint64_t intid, aff, targets; 425 1.2.2.2 pgoyette 426 1.2.2.2 pgoyette intid = __SHIFTIN(ipi, ICC_SGIR_EL1_INTID); 427 1.2.2.2 pgoyette if (kcp == NULL) { 428 1.2.2.2 pgoyette /* Interrupts routed to all PEs, excluding "self" */ 429 1.2.2.2 pgoyette if (ncpu == 1) 430 1.2.2.2 pgoyette return; 431 1.2.2.2 pgoyette icc_sgi1r_write(intid | ICC_SGIR_EL1_IRM); 432 1.2.2.2 pgoyette } else { 433 1.2.2.2 pgoyette /* Interrupts routed to specific PEs */ 434 1.2.2.2 pgoyette aff = 0; 435 1.2.2.2 pgoyette targets = 0; 436 1.2.2.2 pgoyette for (CPU_INFO_FOREACH(cii, ci)) { 437 1.2.2.2 pgoyette if (!kcpuset_isset(kcp, cpu_index(ci))) 438 1.2.2.2 pgoyette continue; 439 1.2.2.2 pgoyette if ((ci->ci_gic_sgir & ICC_SGIR_EL1_Aff) != aff) { 440 1.2.2.2 pgoyette if (targets != 0) { 441 1.2.2.2 pgoyette icc_sgi1r_write(intid | aff | targets); 442 1.2.2.2 pgoyette targets = 0; 443 1.2.2.2 pgoyette } 444 1.2.2.2 pgoyette aff = (ci->ci_gic_sgir & ICC_SGIR_EL1_Aff); 445 1.2.2.2 pgoyette } 446 1.2.2.2 pgoyette targets |= (ci->ci_gic_sgir & ICC_SGIR_EL1_TargetList); 447 1.2.2.2 pgoyette } 448 1.2.2.2 pgoyette if (targets != 0) 449 1.2.2.2 pgoyette icc_sgi1r_write(intid | aff | targets); 450 1.2.2.2 pgoyette } 451 1.2.2.2 pgoyette } 452 1.2.2.4 pgoyette 453 1.2.2.4 pgoyette static void 454 1.2.2.4 pgoyette gicv3_get_affinity(struct pic_softc *pic, size_t irq, kcpuset_t *affinity) 455 1.2.2.4 pgoyette { 456 1.2.2.4 pgoyette struct gicv3_softc * const sc = PICTOSOFTC(pic); 457 1.2.2.4 pgoyette const size_t group = irq / 32; 458 1.2.2.4 pgoyette int n; 459 1.2.2.4 pgoyette 460 1.2.2.4 pgoyette kcpuset_zero(affinity); 461 1.2.2.4 pgoyette if (group == 0) { 462 1.2.2.4 pgoyette /* All CPUs are targets for group 0 (SGI/PPI) */ 463 1.2.2.4 pgoyette for (n = 0; n < ncpu; n++) { 464 1.2.2.4 pgoyette if (sc->sc_irouter[n] != UINT64_MAX) 465 1.2.2.4 pgoyette kcpuset_set(affinity, n); 466 1.2.2.4 pgoyette } 467 1.2.2.4 pgoyette } else { 468 1.2.2.4 pgoyette /* Find distributor targets (SPI) */ 469 1.2.2.4 pgoyette const uint64_t irouter = gicd_read_8(sc, GICD_IROUTER(irq)); 470 1.2.2.4 pgoyette for (n = 0; n < ncpu; n++) { 471 1.2.2.4 pgoyette if (irouter == GICD_IROUTER_Interrupt_Routing_mode || 472 1.2.2.4 pgoyette irouter == sc->sc_irouter[n]) 473 1.2.2.4 pgoyette kcpuset_set(affinity, n); 474 1.2.2.4 pgoyette } 475 1.2.2.4 pgoyette } 476 1.2.2.4 pgoyette } 477 1.2.2.4 pgoyette 478 1.2.2.4 pgoyette static int 479 1.2.2.4 pgoyette gicv3_set_affinity(struct pic_softc *pic, size_t irq, const kcpuset_t *affinity) 480 1.2.2.4 pgoyette { 481 1.2.2.4 pgoyette struct gicv3_softc * const sc = PICTOSOFTC(pic); 482 1.2.2.4 pgoyette const size_t group = irq / 32; 483 1.2.2.4 pgoyette uint64_t irouter; 484 1.2.2.4 pgoyette 485 1.2.2.4 pgoyette if (group == 0) 486 1.2.2.4 pgoyette return EINVAL; 487 1.2.2.4 pgoyette 488 1.2.2.4 pgoyette const int set = kcpuset_countset(affinity); 489 1.2.2.4 pgoyette if (set == ncpu) 490 1.2.2.4 pgoyette irouter = GICD_IROUTER_Interrupt_Routing_mode; 491 1.2.2.4 pgoyette else if (set == 1) 492 1.2.2.4 pgoyette irouter = sc->sc_irouter[kcpuset_ffs(affinity) - 1]; 493 1.2.2.4 pgoyette else 494 1.2.2.4 pgoyette return EINVAL; 495 1.2.2.4 pgoyette 496 1.2.2.4 pgoyette gicd_write_8(sc, GICD_IROUTER(irq), irouter); 497 1.2.2.4 pgoyette 498 1.2.2.4 pgoyette return 0; 499 1.2.2.4 pgoyette } 500 1.2.2.2 pgoyette #endif 501 1.2.2.2 pgoyette 502 1.2.2.2 pgoyette static const struct pic_ops gicv3_picops = { 503 1.2.2.2 pgoyette .pic_unblock_irqs = gicv3_unblock_irqs, 504 1.2.2.2 pgoyette .pic_block_irqs = gicv3_block_irqs, 505 1.2.2.2 pgoyette .pic_establish_irq = gicv3_establish_irq, 506 1.2.2.2 pgoyette .pic_set_priority = gicv3_set_priority, 507 1.2.2.2 pgoyette #ifdef MULTIPROCESSOR 508 1.2.2.2 pgoyette .pic_cpu_init = gicv3_cpu_init, 509 1.2.2.2 pgoyette .pic_ipi_send = gicv3_ipi_send, 510 1.2.2.4 pgoyette .pic_get_affinity = gicv3_get_affinity, 511 1.2.2.4 pgoyette .pic_set_affinity = gicv3_set_affinity, 512 1.2.2.4 pgoyette #endif 513 1.2.2.4 pgoyette }; 514 1.2.2.4 pgoyette 515 1.2.2.4 pgoyette static void 516 1.2.2.4 pgoyette gicv3_lpi_unblock_irqs(struct pic_softc *pic, size_t irqbase, uint32_t mask) 517 1.2.2.4 pgoyette { 518 1.2.2.4 pgoyette struct gicv3_softc * const sc = LPITOSOFTC(pic); 519 1.2.2.4 pgoyette int bit; 520 1.2.2.4 pgoyette 521 1.2.2.4 pgoyette while ((bit = ffs(mask)) != 0) { 522 1.2.2.4 pgoyette sc->sc_lpiconf.base[irqbase + bit - 1] |= GIC_LPICONF_Enable; 523 1.2.2.4 pgoyette mask &= ~__BIT(bit - 1); 524 1.2.2.4 pgoyette } 525 1.2.2.4 pgoyette 526 1.2.2.4 pgoyette bus_dmamap_sync(sc->sc_dmat, sc->sc_lpiconf.map, irqbase, 32, BUS_DMASYNC_PREWRITE); 527 1.2.2.4 pgoyette } 528 1.2.2.4 pgoyette 529 1.2.2.4 pgoyette static void 530 1.2.2.4 pgoyette gicv3_lpi_block_irqs(struct pic_softc *pic, size_t irqbase, uint32_t mask) 531 1.2.2.4 pgoyette { 532 1.2.2.4 pgoyette struct gicv3_softc * const sc = LPITOSOFTC(pic); 533 1.2.2.4 pgoyette int bit; 534 1.2.2.4 pgoyette 535 1.2.2.4 pgoyette while ((bit = ffs(mask)) != 0) { 536 1.2.2.4 pgoyette sc->sc_lpiconf.base[irqbase + bit - 1] &= ~GIC_LPICONF_Enable; 537 1.2.2.4 pgoyette mask &= ~__BIT(bit - 1); 538 1.2.2.4 pgoyette } 539 1.2.2.4 pgoyette 540 1.2.2.4 pgoyette bus_dmamap_sync(sc->sc_dmat, sc->sc_lpiconf.map, irqbase, 32, BUS_DMASYNC_PREWRITE); 541 1.2.2.4 pgoyette } 542 1.2.2.4 pgoyette 543 1.2.2.4 pgoyette static void 544 1.2.2.4 pgoyette gicv3_lpi_establish_irq(struct pic_softc *pic, struct intrsource *is) 545 1.2.2.4 pgoyette { 546 1.2.2.4 pgoyette struct gicv3_softc * const sc = LPITOSOFTC(pic); 547 1.2.2.4 pgoyette 548 1.2.2.4 pgoyette sc->sc_lpiconf.base[is->is_irq] = 0x80 | IPL_TO_PRIORITY(is->is_ipl) | GIC_LPICONF_Res1; 549 1.2.2.4 pgoyette 550 1.2.2.4 pgoyette bus_dmamap_sync(sc->sc_dmat, sc->sc_lpiconf.map, is->is_irq, 1, BUS_DMASYNC_PREWRITE); 551 1.2.2.4 pgoyette } 552 1.2.2.4 pgoyette 553 1.2.2.4 pgoyette static void 554 1.2.2.4 pgoyette gicv3_lpi_cpu_init(struct pic_softc *pic, struct cpu_info *ci) 555 1.2.2.4 pgoyette { 556 1.2.2.4 pgoyette struct gicv3_softc * const sc = LPITOSOFTC(pic); 557 1.2.2.4 pgoyette struct gicv3_lpi_callback *cb; 558 1.2.2.4 pgoyette uint32_t ctlr; 559 1.2.2.4 pgoyette 560 1.2.2.4 pgoyette /* If physical LPIs are not supported on this redistributor, just return. */ 561 1.2.2.4 pgoyette const uint64_t typer = gicr_read_8(sc, ci->ci_gic_redist, GICR_TYPER); 562 1.2.2.4 pgoyette if ((typer & GICR_TYPER_PLPIS) == 0) 563 1.2.2.4 pgoyette return; 564 1.2.2.4 pgoyette 565 1.2.2.4 pgoyette /* Interrupt target address for this CPU, used by ITS when GITS_TYPER.PTA == 0 */ 566 1.2.2.4 pgoyette sc->sc_processor_id[cpu_index(ci)] = __SHIFTOUT(typer, GICR_TYPER_Processor_Number); 567 1.2.2.4 pgoyette 568 1.2.2.4 pgoyette /* Disable LPIs before making changes */ 569 1.2.2.4 pgoyette ctlr = gicr_read_4(sc, ci->ci_gic_redist, GICR_CTLR); 570 1.2.2.4 pgoyette ctlr &= ~GICR_CTLR_Enable_LPIs; 571 1.2.2.4 pgoyette gicr_write_4(sc, ci->ci_gic_redist, GICR_CTLR, ctlr); 572 1.2.2.4 pgoyette arm_dsb(); 573 1.2.2.4 pgoyette 574 1.2.2.4 pgoyette /* Setup the LPI configuration table */ 575 1.2.2.4 pgoyette const uint64_t propbase = sc->sc_lpiconf.segs[0].ds_addr | 576 1.2.2.4 pgoyette __SHIFTIN(ffs(pic->pic_maxsources) - 1, GICR_PROPBASER_IDbits) | 577 1.2.2.4 pgoyette __SHIFTIN(GICR_Shareability_NS, GICR_PROPBASER_Shareability) | 578 1.2.2.4 pgoyette __SHIFTIN(GICR_Cache_NORMAL_NC, GICR_PROPBASER_InnerCache); 579 1.2.2.4 pgoyette gicr_write_8(sc, ci->ci_gic_redist, GICR_PROPBASER, propbase); 580 1.2.2.4 pgoyette 581 1.2.2.4 pgoyette /* Setup the LPI pending table */ 582 1.2.2.4 pgoyette const uint64_t pendbase = sc->sc_lpipend[cpu_index(ci)].segs[0].ds_addr | 583 1.2.2.4 pgoyette __SHIFTIN(GICR_Shareability_NS, GICR_PENDBASER_Shareability) | 584 1.2.2.4 pgoyette __SHIFTIN(GICR_Cache_NORMAL_NC, GICR_PENDBASER_InnerCache) | 585 1.2.2.4 pgoyette GICR_PENDBASER_PTZ; 586 1.2.2.4 pgoyette gicr_write_8(sc, ci->ci_gic_redist, GICR_PENDBASER, pendbase); 587 1.2.2.4 pgoyette 588 1.2.2.4 pgoyette /* Enable LPIs */ 589 1.2.2.4 pgoyette ctlr = gicr_read_4(sc, ci->ci_gic_redist, GICR_CTLR); 590 1.2.2.4 pgoyette ctlr |= GICR_CTLR_Enable_LPIs; 591 1.2.2.4 pgoyette gicr_write_4(sc, ci->ci_gic_redist, GICR_CTLR, ctlr); 592 1.2.2.4 pgoyette arm_dsb(); 593 1.2.2.4 pgoyette 594 1.2.2.4 pgoyette /* Setup ITS if present */ 595 1.2.2.4 pgoyette LIST_FOREACH(cb, &sc->sc_lpi_callbacks, list) 596 1.2.2.4 pgoyette cb->cpu_init(cb->priv, ci); 597 1.2.2.4 pgoyette } 598 1.2.2.4 pgoyette 599 1.2.2.4 pgoyette #ifdef MULTIPROCESSOR 600 1.2.2.4 pgoyette static void 601 1.2.2.4 pgoyette gicv3_lpi_get_affinity(struct pic_softc *pic, size_t irq, kcpuset_t *affinity) 602 1.2.2.4 pgoyette { 603 1.2.2.4 pgoyette struct gicv3_softc * const sc = LPITOSOFTC(pic); 604 1.2.2.4 pgoyette struct gicv3_lpi_callback *cb; 605 1.2.2.4 pgoyette 606 1.2.2.4 pgoyette LIST_FOREACH(cb, &sc->sc_lpi_callbacks, list) 607 1.2.2.4 pgoyette cb->get_affinity(cb->priv, irq, affinity); 608 1.2.2.4 pgoyette } 609 1.2.2.4 pgoyette 610 1.2.2.4 pgoyette static int 611 1.2.2.4 pgoyette gicv3_lpi_set_affinity(struct pic_softc *pic, size_t irq, const kcpuset_t *affinity) 612 1.2.2.4 pgoyette { 613 1.2.2.4 pgoyette struct gicv3_softc * const sc = LPITOSOFTC(pic); 614 1.2.2.4 pgoyette struct gicv3_lpi_callback *cb; 615 1.2.2.4 pgoyette int error = EINVAL; 616 1.2.2.4 pgoyette 617 1.2.2.4 pgoyette LIST_FOREACH(cb, &sc->sc_lpi_callbacks, list) { 618 1.2.2.4 pgoyette error = cb->set_affinity(cb->priv, irq, affinity); 619 1.2.2.4 pgoyette if (error) 620 1.2.2.4 pgoyette return error; 621 1.2.2.4 pgoyette } 622 1.2.2.4 pgoyette 623 1.2.2.4 pgoyette return error; 624 1.2.2.4 pgoyette } 625 1.2.2.4 pgoyette #endif 626 1.2.2.4 pgoyette 627 1.2.2.4 pgoyette static const struct pic_ops gicv3_lpiops = { 628 1.2.2.4 pgoyette .pic_unblock_irqs = gicv3_lpi_unblock_irqs, 629 1.2.2.4 pgoyette .pic_block_irqs = gicv3_lpi_block_irqs, 630 1.2.2.4 pgoyette .pic_establish_irq = gicv3_lpi_establish_irq, 631 1.2.2.4 pgoyette #ifdef MULTIPROCESSOR 632 1.2.2.4 pgoyette .pic_cpu_init = gicv3_lpi_cpu_init, 633 1.2.2.4 pgoyette .pic_get_affinity = gicv3_lpi_get_affinity, 634 1.2.2.4 pgoyette .pic_set_affinity = gicv3_lpi_set_affinity, 635 1.2.2.2 pgoyette #endif 636 1.2.2.2 pgoyette }; 637 1.2.2.2 pgoyette 638 1.2.2.2 pgoyette void 639 1.2.2.4 pgoyette gicv3_dma_alloc(struct gicv3_softc *sc, struct gicv3_dma *dma, bus_size_t len, bus_size_t align) 640 1.2.2.4 pgoyette { 641 1.2.2.4 pgoyette int nsegs, error; 642 1.2.2.4 pgoyette 643 1.2.2.4 pgoyette dma->len = len; 644 1.2.2.4 pgoyette error = bus_dmamem_alloc(sc->sc_dmat, dma->len, align, 0, dma->segs, 1, &nsegs, BUS_DMA_WAITOK); 645 1.2.2.4 pgoyette if (error) 646 1.2.2.4 pgoyette panic("bus_dmamem_alloc failed: %d", error); 647 1.2.2.4 pgoyette error = bus_dmamem_map(sc->sc_dmat, dma->segs, nsegs, len, (void **)&dma->base, BUS_DMA_WAITOK); 648 1.2.2.4 pgoyette if (error) 649 1.2.2.4 pgoyette panic("bus_dmamem_map failed: %d", error); 650 1.2.2.4 pgoyette error = bus_dmamap_create(sc->sc_dmat, len, 1, len, 0, BUS_DMA_WAITOK, &dma->map); 651 1.2.2.4 pgoyette if (error) 652 1.2.2.4 pgoyette panic("bus_dmamap_create failed: %d", error); 653 1.2.2.4 pgoyette error = bus_dmamap_load(sc->sc_dmat, dma->map, dma->base, dma->len, NULL, BUS_DMA_WAITOK); 654 1.2.2.4 pgoyette if (error) 655 1.2.2.4 pgoyette panic("bus_dmamap_load failed: %d", error); 656 1.2.2.4 pgoyette 657 1.2.2.4 pgoyette memset(dma->base, 0, dma->len); 658 1.2.2.4 pgoyette bus_dmamap_sync(sc->sc_dmat, dma->map, 0, dma->len, BUS_DMASYNC_PREWRITE); 659 1.2.2.4 pgoyette } 660 1.2.2.4 pgoyette 661 1.2.2.4 pgoyette static void 662 1.2.2.4 pgoyette gicv3_lpi_init(struct gicv3_softc *sc) 663 1.2.2.4 pgoyette { 664 1.2.2.4 pgoyette /* 665 1.2.2.4 pgoyette * Allocate LPI configuration table 666 1.2.2.4 pgoyette */ 667 1.2.2.4 pgoyette gicv3_dma_alloc(sc, &sc->sc_lpiconf, sc->sc_lpi.pic_maxsources, 0x1000); 668 1.2.2.4 pgoyette KASSERT((sc->sc_lpiconf.segs[0].ds_addr & ~GICR_PROPBASER_Physical_Address) == 0); 669 1.2.2.4 pgoyette 670 1.2.2.4 pgoyette /* 671 1.2.2.4 pgoyette * Allocate LPI pending tables 672 1.2.2.4 pgoyette */ 673 1.2.2.4 pgoyette const bus_size_t lpipend_sz = sc->sc_lpi.pic_maxsources / NBBY; 674 1.2.2.4 pgoyette for (int cpuindex = 0; cpuindex < ncpu; cpuindex++) { 675 1.2.2.4 pgoyette gicv3_dma_alloc(sc, &sc->sc_lpipend[cpuindex], lpipend_sz, 0x10000); 676 1.2.2.4 pgoyette KASSERT((sc->sc_lpipend[cpuindex].segs[0].ds_addr & ~GICR_PENDBASER_Physical_Address) == 0); 677 1.2.2.4 pgoyette } 678 1.2.2.4 pgoyette } 679 1.2.2.4 pgoyette 680 1.2.2.4 pgoyette void 681 1.2.2.2 pgoyette gicv3_irq_handler(void *frame) 682 1.2.2.2 pgoyette { 683 1.2.2.2 pgoyette struct cpu_info * const ci = curcpu(); 684 1.2.2.2 pgoyette struct gicv3_softc * const sc = gicv3_softc; 685 1.2.2.4 pgoyette struct pic_softc *pic; 686 1.2.2.2 pgoyette const int oldipl = ci->ci_cpl; 687 1.2.2.2 pgoyette 688 1.2.2.2 pgoyette ci->ci_data.cpu_nintr++; 689 1.2.2.2 pgoyette 690 1.2.2.2 pgoyette for (;;) { 691 1.2.2.2 pgoyette const uint32_t iar = icc_iar1_read(); 692 1.2.2.2 pgoyette const uint32_t irq = __SHIFTOUT(iar, ICC_IAR_INTID); 693 1.2.2.2 pgoyette if (irq == ICC_IAR_INTID_SPURIOUS) 694 1.2.2.2 pgoyette break; 695 1.2.2.2 pgoyette 696 1.2.2.4 pgoyette pic = irq >= GIC_LPI_BASE ? &sc->sc_lpi : &sc->sc_pic; 697 1.2.2.4 pgoyette if (irq - pic->pic_irqbase >= pic->pic_maxsources) 698 1.2.2.2 pgoyette continue; 699 1.2.2.2 pgoyette 700 1.2.2.4 pgoyette struct intrsource * const is = pic->pic_sources[irq - pic->pic_irqbase]; 701 1.2.2.2 pgoyette KASSERT(is != NULL); 702 1.2.2.2 pgoyette 703 1.2.2.2 pgoyette const int ipl = is->is_ipl; 704 1.2.2.2 pgoyette if (ci->ci_cpl < ipl) 705 1.2.2.2 pgoyette pic_set_priority(ci, ipl); 706 1.2.2.2 pgoyette 707 1.2.2.2 pgoyette cpsie(I32_bit); 708 1.2.2.2 pgoyette pic_dispatch(is, frame); 709 1.2.2.2 pgoyette cpsid(I32_bit); 710 1.2.2.2 pgoyette 711 1.2.2.2 pgoyette icc_eoi1r_write(iar); 712 1.2.2.2 pgoyette } 713 1.2.2.2 pgoyette 714 1.2.2.2 pgoyette if (ci->ci_cpl != oldipl) 715 1.2.2.2 pgoyette pic_set_priority(ci, oldipl); 716 1.2.2.2 pgoyette } 717 1.2.2.2 pgoyette 718 1.2.2.2 pgoyette int 719 1.2.2.2 pgoyette gicv3_init(struct gicv3_softc *sc) 720 1.2.2.2 pgoyette { 721 1.2.2.2 pgoyette const uint32_t gicd_typer = gicd_read_4(sc, GICD_TYPER); 722 1.2.2.4 pgoyette int n; 723 1.2.2.2 pgoyette 724 1.2.2.2 pgoyette KASSERT(CPU_IS_PRIMARY(curcpu())); 725 1.2.2.2 pgoyette 726 1.2.2.4 pgoyette LIST_INIT(&sc->sc_lpi_callbacks); 727 1.2.2.4 pgoyette 728 1.2.2.4 pgoyette for (n = 0; n < MAXCPUS; n++) 729 1.2.2.4 pgoyette sc->sc_irouter[n] = UINT64_MAX; 730 1.2.2.4 pgoyette 731 1.2.2.2 pgoyette sc->sc_pic.pic_ops = &gicv3_picops; 732 1.2.2.2 pgoyette sc->sc_pic.pic_maxsources = GICD_TYPER_LINES(gicd_typer); 733 1.2.2.2 pgoyette snprintf(sc->sc_pic.pic_name, sizeof(sc->sc_pic.pic_name), "gicv3"); 734 1.2.2.2 pgoyette #ifdef MULTIPROCESSOR 735 1.2.2.2 pgoyette sc->sc_pic.pic_cpus = kcpuset_running; 736 1.2.2.2 pgoyette #endif 737 1.2.2.2 pgoyette pic_add(&sc->sc_pic, 0); 738 1.2.2.2 pgoyette 739 1.2.2.4 pgoyette if ((gicd_typer & GICD_TYPER_LPIS) != 0) { 740 1.2.2.4 pgoyette sc->sc_lpi.pic_ops = &gicv3_lpiops; 741 1.2.2.4 pgoyette sc->sc_lpi.pic_maxsources = 8192; /* Min. required by GICv3 spec */ 742 1.2.2.4 pgoyette snprintf(sc->sc_lpi.pic_name, sizeof(sc->sc_lpi.pic_name), "gicv3-lpi"); 743 1.2.2.4 pgoyette pic_add(&sc->sc_lpi, GIC_LPI_BASE); 744 1.2.2.4 pgoyette 745 1.2.2.4 pgoyette gicv3_lpi_init(sc); 746 1.2.2.4 pgoyette } 747 1.2.2.4 pgoyette 748 1.2.2.2 pgoyette KASSERT(gicv3_softc == NULL); 749 1.2.2.2 pgoyette gicv3_softc = sc; 750 1.2.2.2 pgoyette 751 1.2.2.2 pgoyette for (int i = 0; i < sc->sc_bsh_r_count; i++) { 752 1.2.2.2 pgoyette const uint64_t gicr_typer = gicr_read_8(sc, i, GICR_TYPER); 753 1.2.2.2 pgoyette const u_int aff0 = __SHIFTOUT(gicr_typer, GICR_TYPER_Affinity_Value_Aff0); 754 1.2.2.2 pgoyette const u_int aff1 = __SHIFTOUT(gicr_typer, GICR_TYPER_Affinity_Value_Aff1); 755 1.2.2.2 pgoyette const u_int aff2 = __SHIFTOUT(gicr_typer, GICR_TYPER_Affinity_Value_Aff2); 756 1.2.2.2 pgoyette const u_int aff3 = __SHIFTOUT(gicr_typer, GICR_TYPER_Affinity_Value_Aff3); 757 1.2.2.2 pgoyette 758 1.2.2.2 pgoyette aprint_debug_dev(sc->sc_dev, "redist %d: cpu %d.%d.%d.%d\n", 759 1.2.2.2 pgoyette i, aff3, aff2, aff1, aff0); 760 1.2.2.2 pgoyette } 761 1.2.2.2 pgoyette 762 1.2.2.2 pgoyette gicv3_dist_enable(sc); 763 1.2.2.2 pgoyette 764 1.2.2.2 pgoyette gicv3_cpu_init(&sc->sc_pic, curcpu()); 765 1.2.2.4 pgoyette if ((gicd_typer & GICD_TYPER_LPIS) != 0) 766 1.2.2.4 pgoyette gicv3_lpi_cpu_init(&sc->sc_lpi, curcpu()); 767 1.2.2.2 pgoyette 768 1.2.2.2 pgoyette #ifdef __HAVE_PIC_FAST_SOFTINTS 769 1.2.2.4 pgoyette intr_establish_xname(SOFTINT_BIO, IPL_SOFTBIO, IST_MPSAFE | IST_EDGE, pic_handle_softint, (void *)SOFTINT_BIO, "softint bio"); 770 1.2.2.4 pgoyette intr_establish_xname(SOFTINT_CLOCK, IPL_SOFTCLOCK, IST_MPSAFE | IST_EDGE, pic_handle_softint, (void *)SOFTINT_CLOCK, "softint clock"); 771 1.2.2.4 pgoyette intr_establish_xname(SOFTINT_NET, IPL_SOFTNET, IST_MPSAFE | IST_EDGE, pic_handle_softint, (void *)SOFTINT_NET, "softint net"); 772 1.2.2.4 pgoyette intr_establish_xname(SOFTINT_SERIAL, IPL_SOFTSERIAL, IST_MPSAFE | IST_EDGE, pic_handle_softint, (void *)SOFTINT_SERIAL, "softint serial"); 773 1.2.2.2 pgoyette #endif 774 1.2.2.2 pgoyette 775 1.2.2.2 pgoyette #ifdef MULTIPROCESSOR 776 1.2.2.4 pgoyette intr_establish_xname(IPI_AST, IPL_VM, IST_MPSAFE | IST_EDGE, pic_ipi_ast, (void *)-1, "IPI ast"); 777 1.2.2.4 pgoyette intr_establish_xname(IPI_XCALL, IPL_HIGH, IST_MPSAFE | IST_EDGE, pic_ipi_xcall, (void *)-1, "IPI xcall"); 778 1.2.2.4 pgoyette intr_establish_xname(IPI_GENERIC, IPL_HIGH, IST_MPSAFE | IST_EDGE, pic_ipi_generic, (void *)-1, "IPI generic"); 779 1.2.2.4 pgoyette intr_establish_xname(IPI_NOP, IPL_VM, IST_MPSAFE | IST_EDGE, pic_ipi_nop, (void *)-1, "IPI nop"); 780 1.2.2.4 pgoyette intr_establish_xname(IPI_SHOOTDOWN, IPL_SCHED, IST_MPSAFE | IST_EDGE, pic_ipi_shootdown, (void *)-1, "IPI shootdown"); 781 1.2.2.2 pgoyette #ifdef DDB 782 1.2.2.4 pgoyette intr_establish_xname(IPI_DDB, IPL_HIGH, IST_MPSAFE | IST_EDGE, pic_ipi_ddb, NULL, "IPI ddb"); 783 1.2.2.2 pgoyette #endif 784 1.2.2.2 pgoyette #ifdef __HAVE_PREEMPTION 785 1.2.2.4 pgoyette intr_establish_xname(IPI_KPREEMPT, IPL_VM, IST_MPSAFE | IST_EDGE, pic_ipi_kpreempt, (void *)-1, "IPI kpreempt"); 786 1.2.2.2 pgoyette #endif 787 1.2.2.2 pgoyette #endif 788 1.2.2.2 pgoyette 789 1.2.2.2 pgoyette return 0; 790 1.2.2.2 pgoyette } 791
Indexes created Thu Oct 02 14:10:14 GMT 2025