Home | History | Annotate | Line # | Download | only in cortex
gicv3.c revision 1.11
      1  1.11  jmcneill /* $NetBSD: gicv3.c,v 1.11 2018/11/17 00:17:54 jmcneill Exp $ */
      2   1.1  jmcneill 
      3   1.1  jmcneill /*-
      4   1.1  jmcneill  * Copyright (c) 2018 Jared McNeill <jmcneill (at) invisible.ca>
      5   1.1  jmcneill  * All rights reserved.
      6   1.1  jmcneill  *
      7   1.1  jmcneill  * Redistribution and use in source and binary forms, with or without
      8   1.1  jmcneill  * modification, are permitted provided that the following conditions
      9   1.1  jmcneill  * are met:
     10   1.1  jmcneill  * 1. Redistributions of source code must retain the above copyright
     11   1.1  jmcneill  *    notice, this list of conditions and the following disclaimer.
     12   1.1  jmcneill  * 2. Redistributions in binary form must reproduce the above copyright
     13   1.1  jmcneill  *    notice, this list of conditions and the following disclaimer in the
     14   1.1  jmcneill  *    documentation and/or other materials provided with the distribution.
     15   1.1  jmcneill  *
     16   1.1  jmcneill  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
     17   1.1  jmcneill  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
     18   1.1  jmcneill  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
     19   1.1  jmcneill  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
     20   1.1  jmcneill  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
     21   1.1  jmcneill  * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
     22   1.1  jmcneill  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
     23   1.1  jmcneill  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
     24   1.1  jmcneill  * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
     25   1.1  jmcneill  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
     26   1.1  jmcneill  * SUCH DAMAGE.
     27   1.1  jmcneill  */
     28   1.1  jmcneill 
     29   1.1  jmcneill #include "opt_multiprocessor.h"
     30   1.1  jmcneill 
     31   1.1  jmcneill #define	_INTR_PRIVATE
     32   1.1  jmcneill 
     33   1.1  jmcneill #include <sys/cdefs.h>
     34  1.11  jmcneill __KERNEL_RCSID(0, "$NetBSD: gicv3.c,v 1.11 2018/11/17 00:17:54 jmcneill Exp $");
     35   1.1  jmcneill 
     36   1.1  jmcneill #include <sys/param.h>
     37   1.1  jmcneill #include <sys/kernel.h>
     38   1.1  jmcneill #include <sys/bus.h>
     39   1.1  jmcneill #include <sys/device.h>
     40   1.1  jmcneill #include <sys/intr.h>
     41   1.1  jmcneill #include <sys/systm.h>
     42   1.1  jmcneill #include <sys/cpu.h>
     43   1.1  jmcneill 
     44   1.1  jmcneill #include <arm/locore.h>
     45   1.1  jmcneill #include <arm/armreg.h>
     46   1.1  jmcneill 
     47   1.1  jmcneill #include <arm/cortex/gicv3.h>
     48   1.1  jmcneill #include <arm/cortex/gic_reg.h>
     49   1.1  jmcneill 
     50   1.1  jmcneill #define	PICTOSOFTC(pic)	\
     51   1.1  jmcneill 	((void *)((uintptr_t)(pic) - offsetof(struct gicv3_softc, sc_pic)))
     52   1.5  jmcneill #define	LPITOSOFTC(lpi) \
     53   1.5  jmcneill 	((void *)((uintptr_t)(lpi) - offsetof(struct gicv3_softc, sc_lpi)))
     54   1.1  jmcneill 
     55   1.4  jmcneill #define	IPL_TO_PRIORITY(ipl)	((IPL_HIGH - (ipl)) << 4)
     56   1.1  jmcneill 
     57   1.1  jmcneill static struct gicv3_softc *gicv3_softc;
     58   1.1  jmcneill 
     59   1.1  jmcneill static inline uint32_t
     60   1.1  jmcneill gicd_read_4(struct gicv3_softc *sc, bus_size_t reg)
     61   1.1  jmcneill {
     62   1.1  jmcneill 	return bus_space_read_4(sc->sc_bst, sc->sc_bsh_d, reg);
     63   1.1  jmcneill }
     64   1.1  jmcneill 
     65   1.1  jmcneill static inline void
     66   1.1  jmcneill gicd_write_4(struct gicv3_softc *sc, bus_size_t reg, uint32_t val)
     67   1.1  jmcneill {
     68   1.1  jmcneill 	bus_space_write_4(sc->sc_bst, sc->sc_bsh_d, reg, val);
     69   1.1  jmcneill }
     70   1.1  jmcneill 
     71   1.6  jmcneill static inline uint64_t
     72   1.6  jmcneill gicd_read_8(struct gicv3_softc *sc, bus_size_t reg)
     73   1.6  jmcneill {
     74   1.6  jmcneill 	return bus_space_read_8(sc->sc_bst, sc->sc_bsh_d, reg);
     75   1.6  jmcneill }
     76   1.6  jmcneill 
     77   1.1  jmcneill static inline void
     78   1.1  jmcneill gicd_write_8(struct gicv3_softc *sc, bus_size_t reg, uint64_t val)
     79   1.1  jmcneill {
     80   1.1  jmcneill 	bus_space_write_8(sc->sc_bst, sc->sc_bsh_d, reg, val);
     81   1.1  jmcneill }
     82   1.1  jmcneill 
     83   1.1  jmcneill static inline uint32_t
     84   1.1  jmcneill gicr_read_4(struct gicv3_softc *sc, u_int index, bus_size_t reg)
     85   1.1  jmcneill {
     86   1.1  jmcneill 	KASSERT(index < sc->sc_bsh_r_count);
     87   1.1  jmcneill 	return bus_space_read_4(sc->sc_bst, sc->sc_bsh_r[index], reg);
     88   1.1  jmcneill }
     89   1.1  jmcneill 
     90   1.1  jmcneill static inline void
     91   1.1  jmcneill gicr_write_4(struct gicv3_softc *sc, u_int index, bus_size_t reg, uint32_t val)
     92   1.1  jmcneill {
     93   1.1  jmcneill 	KASSERT(index < sc->sc_bsh_r_count);
     94   1.1  jmcneill 	bus_space_write_4(sc->sc_bst, sc->sc_bsh_r[index], reg, val);
     95   1.1  jmcneill }
     96   1.1  jmcneill 
     97   1.1  jmcneill static inline uint64_t
     98   1.1  jmcneill gicr_read_8(struct gicv3_softc *sc, u_int index, bus_size_t reg)
     99   1.1  jmcneill {
    100   1.1  jmcneill 	KASSERT(index < sc->sc_bsh_r_count);
    101   1.1  jmcneill 	return bus_space_read_8(sc->sc_bst, sc->sc_bsh_r[index], reg);
    102   1.1  jmcneill }
    103   1.1  jmcneill 
    104   1.1  jmcneill static inline void
    105   1.1  jmcneill gicr_write_8(struct gicv3_softc *sc, u_int index, bus_size_t reg, uint64_t val)
    106   1.1  jmcneill {
    107   1.1  jmcneill 	KASSERT(index < sc->sc_bsh_r_count);
    108   1.1  jmcneill 	bus_space_write_8(sc->sc_bst, sc->sc_bsh_r[index], reg, val);
    109   1.1  jmcneill }
    110   1.1  jmcneill 
    111   1.1  jmcneill static void
    112   1.1  jmcneill gicv3_unblock_irqs(struct pic_softc *pic, size_t irqbase, uint32_t mask)
    113   1.1  jmcneill {
    114   1.1  jmcneill 	struct gicv3_softc * const sc = PICTOSOFTC(pic);
    115   1.1  jmcneill 	struct cpu_info * const ci = curcpu();
    116   1.1  jmcneill 	const u_int group = irqbase / 32;
    117   1.1  jmcneill 
    118   1.1  jmcneill 	if (group == 0) {
    119   1.1  jmcneill 		sc->sc_enabled_sgippi |= mask;
    120   1.1  jmcneill 		gicr_write_4(sc, ci->ci_gic_redist, GICR_ISENABLER0, mask);
    121   1.5  jmcneill 		while (gicr_read_4(sc, ci->ci_gic_redist, GICR_CTLR) & GICR_CTLR_RWP)
    122   1.1  jmcneill 			;
    123   1.1  jmcneill 	} else {
    124   1.1  jmcneill 		gicd_write_4(sc, GICD_ISENABLERn(group), mask);
    125   1.1  jmcneill 		while (gicd_read_4(sc, GICD_CTRL) & GICD_CTRL_RWP)
    126   1.1  jmcneill 			;
    127   1.1  jmcneill 	}
    128   1.1  jmcneill }
    129   1.1  jmcneill 
    130   1.1  jmcneill static void
    131   1.1  jmcneill gicv3_block_irqs(struct pic_softc *pic, size_t irqbase, uint32_t mask)
    132   1.1  jmcneill {
    133   1.1  jmcneill 	struct gicv3_softc * const sc = PICTOSOFTC(pic);
    134   1.1  jmcneill 	struct cpu_info * const ci = curcpu();
    135   1.1  jmcneill 	const u_int group = irqbase / 32;
    136   1.1  jmcneill 
    137   1.1  jmcneill 	if (group == 0) {
    138   1.1  jmcneill 		sc->sc_enabled_sgippi &= ~mask;
    139   1.1  jmcneill 		gicr_write_4(sc, ci->ci_gic_redist, GICR_ICENABLER0, mask);
    140   1.5  jmcneill 		while (gicr_read_4(sc, ci->ci_gic_redist, GICR_CTLR) & GICR_CTLR_RWP)
    141   1.1  jmcneill 			;
    142   1.1  jmcneill 	} else {
    143   1.1  jmcneill 		gicd_write_4(sc, GICD_ICENABLERn(group), mask);
    144   1.1  jmcneill 		while (gicd_read_4(sc, GICD_CTRL) & GICD_CTRL_RWP)
    145   1.1  jmcneill 			;
    146   1.1  jmcneill 	}
    147   1.1  jmcneill }
    148   1.1  jmcneill 
    149   1.1  jmcneill static void
    150   1.1  jmcneill gicv3_establish_irq(struct pic_softc *pic, struct intrsource *is)
    151   1.1  jmcneill {
    152   1.1  jmcneill 	struct gicv3_softc * const sc = PICTOSOFTC(pic);
    153   1.1  jmcneill 	const u_int group = is->is_irq / 32;
    154   1.1  jmcneill 	uint32_t ipriority, icfg;
    155   1.1  jmcneill 	uint64_t irouter;
    156   1.1  jmcneill 	u_int n;
    157   1.1  jmcneill 
    158   1.4  jmcneill 	const u_int ipriority_val = 0x80 | IPL_TO_PRIORITY(is->is_ipl);
    159   1.1  jmcneill 	const u_int ipriority_shift = (is->is_irq & 0x3) * 8;
    160   1.1  jmcneill 	const u_int icfg_shift = (is->is_irq & 0xf) * 2;
    161   1.1  jmcneill 
    162   1.1  jmcneill 	if (group == 0) {
    163   1.1  jmcneill 		/* SGIs and PPIs are always MP-safe */
    164   1.1  jmcneill 		is->is_mpsafe = true;
    165   1.1  jmcneill 
    166   1.1  jmcneill 		/* Update interrupt configuration and priority on all redistributors */
    167   1.1  jmcneill 		for (n = 0; n < sc->sc_bsh_r_count; n++) {
    168   1.1  jmcneill 			icfg = gicr_read_4(sc, n, GICR_ICFGRn(is->is_irq / 16));
    169   1.1  jmcneill 			if (is->is_type == IST_LEVEL)
    170   1.1  jmcneill 				icfg &= ~(0x2 << icfg_shift);
    171   1.1  jmcneill 			if (is->is_type == IST_EDGE)
    172   1.1  jmcneill 				icfg |= (0x2 << icfg_shift);
    173   1.1  jmcneill 			gicr_write_4(sc, n, GICR_ICFGRn(is->is_irq / 16), icfg);
    174   1.1  jmcneill 
    175   1.1  jmcneill 			ipriority = gicr_read_4(sc, n, GICR_IPRIORITYRn(is->is_irq / 4));
    176   1.1  jmcneill 			ipriority &= ~(0xff << ipriority_shift);
    177   1.2  jmcneill 			ipriority |= (ipriority_val << ipriority_shift);
    178   1.1  jmcneill 			gicr_write_4(sc, n, GICR_IPRIORITYRn(is->is_irq / 4), ipriority);
    179   1.1  jmcneill 		}
    180   1.1  jmcneill 	} else {
    181   1.1  jmcneill 		if (is->is_mpsafe) {
    182   1.1  jmcneill 			/* Route MP-safe interrupts to all participating PEs */
    183   1.1  jmcneill 			irouter = GICD_IROUTER_Interrupt_Routing_mode;
    184   1.1  jmcneill 		} else {
    185   1.1  jmcneill 			/* Route non-MP-safe interrupts to the primary PE only */
    186   1.6  jmcneill 			irouter = sc->sc_irouter[0];
    187   1.1  jmcneill 		}
    188   1.1  jmcneill 		gicd_write_8(sc, GICD_IROUTER(is->is_irq), irouter);
    189   1.1  jmcneill 
    190   1.1  jmcneill 		/* Update interrupt configuration */
    191   1.1  jmcneill 		icfg = gicd_read_4(sc, GICD_ICFGRn(is->is_irq / 16));
    192   1.1  jmcneill 		if (is->is_type == IST_LEVEL)
    193   1.1  jmcneill 			icfg &= ~(0x2 << icfg_shift);
    194   1.1  jmcneill 		if (is->is_type == IST_EDGE)
    195   1.1  jmcneill 			icfg |= (0x2 << icfg_shift);
    196   1.1  jmcneill 		gicd_write_4(sc, GICD_ICFGRn(is->is_irq / 16), icfg);
    197   1.1  jmcneill 
    198   1.1  jmcneill 		/* Update interrupt priority */
    199   1.1  jmcneill 		ipriority = gicd_read_4(sc, GICD_IPRIORITYRn(is->is_irq / 4));
    200   1.1  jmcneill 		ipriority &= ~(0xff << ipriority_shift);
    201   1.2  jmcneill 		ipriority |= (ipriority_val << ipriority_shift);
    202   1.1  jmcneill 		gicd_write_4(sc, GICD_IPRIORITYRn(is->is_irq / 4), ipriority);
    203   1.1  jmcneill 	}
    204   1.1  jmcneill }
    205   1.1  jmcneill 
    206   1.1  jmcneill static void
    207   1.1  jmcneill gicv3_set_priority(struct pic_softc *pic, int ipl)
    208   1.1  jmcneill {
    209   1.4  jmcneill 	icc_pmr_write(IPL_TO_PRIORITY(ipl) << 1);
    210   1.1  jmcneill }
    211   1.1  jmcneill 
    212   1.1  jmcneill static void
    213   1.1  jmcneill gicv3_dist_enable(struct gicv3_softc *sc)
    214   1.1  jmcneill {
    215   1.1  jmcneill 	uint32_t gicd_ctrl;
    216   1.1  jmcneill 	u_int n;
    217   1.1  jmcneill 
    218   1.1  jmcneill 	/* Disable the distributor */
    219   1.1  jmcneill 	gicd_write_4(sc, GICD_CTRL, 0);
    220   1.1  jmcneill 
    221   1.1  jmcneill 	/* Wait for register write to complete */
    222   1.1  jmcneill 	while (gicd_read_4(sc, GICD_CTRL) & GICD_CTRL_RWP)
    223   1.1  jmcneill 		;
    224   1.1  jmcneill 
    225   1.1  jmcneill 	/* Clear all INTID enable bits */
    226   1.1  jmcneill 	for (n = 32; n < sc->sc_pic.pic_maxsources; n += 32)
    227   1.1  jmcneill 		gicd_write_4(sc, GICD_ICENABLERn(n / 32), ~0);
    228   1.1  jmcneill 
    229   1.1  jmcneill 	/* Set default priorities to lowest */
    230   1.1  jmcneill 	for (n = 32; n < sc->sc_pic.pic_maxsources; n += 4)
    231   1.1  jmcneill 		gicd_write_4(sc, GICD_IPRIORITYRn(n / 4), ~0);
    232   1.1  jmcneill 
    233   1.1  jmcneill 	/* Set all interrupts to G1NS */
    234   1.1  jmcneill 	for (n = 32; n < sc->sc_pic.pic_maxsources; n += 32) {
    235   1.1  jmcneill 		gicd_write_4(sc, GICD_IGROUPRn(n / 32), ~0);
    236   1.1  jmcneill 		gicd_write_4(sc, GICD_IGRPMODRn(n / 32), 0);
    237   1.1  jmcneill 	}
    238   1.1  jmcneill 
    239   1.1  jmcneill 	/* Set all interrupts level-sensitive by default */
    240   1.1  jmcneill 	for (n = 32; n < sc->sc_pic.pic_maxsources; n += 16)
    241   1.1  jmcneill 		gicd_write_4(sc, GICD_ICFGRn(n / 16), 0);
    242   1.1  jmcneill 
    243   1.1  jmcneill 	/* Wait for register writes to complete */
    244   1.1  jmcneill 	while (gicd_read_4(sc, GICD_CTRL) & GICD_CTRL_RWP)
    245   1.1  jmcneill 		;
    246   1.1  jmcneill 
    247   1.1  jmcneill 	/* Enable Affinity routing and G1NS interrupts */
    248   1.9  jmcneill 	gicd_ctrl = GICD_CTRL_EnableGrp1A | GICD_CTRL_Enable | GICD_CTRL_ARE_NS;
    249   1.1  jmcneill 	gicd_write_4(sc, GICD_CTRL, gicd_ctrl);
    250   1.1  jmcneill }
    251   1.1  jmcneill 
    252   1.1  jmcneill static void
    253   1.1  jmcneill gicv3_redist_enable(struct gicv3_softc *sc, struct cpu_info *ci)
    254   1.1  jmcneill {
    255   1.1  jmcneill 	uint32_t icfg;
    256   1.1  jmcneill 	u_int n, o;
    257   1.1  jmcneill 
    258   1.1  jmcneill 	/* Clear INTID enable bits */
    259   1.1  jmcneill 	gicr_write_4(sc, ci->ci_gic_redist, GICR_ICENABLER0, ~0);
    260   1.1  jmcneill 
    261   1.1  jmcneill 	/* Wait for register write to complete */
    262   1.5  jmcneill 	while (gicr_read_4(sc, ci->ci_gic_redist, GICR_CTLR) & GICR_CTLR_RWP)
    263   1.1  jmcneill 		;
    264   1.1  jmcneill 
    265   1.1  jmcneill 	/* Set default priorities */
    266   1.1  jmcneill 	for (n = 0; n < 32; n += 4) {
    267   1.1  jmcneill 		uint32_t priority = 0;
    268   1.1  jmcneill 		size_t byte_shift = 0;
    269   1.1  jmcneill 		for (o = 0; o < 4; o++, byte_shift += 8) {
    270   1.1  jmcneill 			struct intrsource * const is = sc->sc_pic.pic_sources[n + o];
    271   1.1  jmcneill 			if (is == NULL)
    272   1.1  jmcneill 				priority |= 0xff << byte_shift;
    273   1.2  jmcneill 			else {
    274   1.2  jmcneill 				const u_int ipriority_val = 0x80 | IPL_TO_PRIORITY(is->is_ipl);
    275   1.2  jmcneill 				priority |= ipriority_val << byte_shift;
    276   1.2  jmcneill 			}
    277   1.1  jmcneill 		}
    278   1.1  jmcneill 		gicr_write_4(sc, ci->ci_gic_redist, GICR_IPRIORITYRn(n / 4), priority);
    279   1.1  jmcneill 	}
    280   1.1  jmcneill 
    281   1.1  jmcneill 	/* Set all interrupts to G1NS */
    282   1.1  jmcneill 	gicr_write_4(sc, ci->ci_gic_redist, GICR_IGROUPR0, ~0);
    283   1.1  jmcneill 	gicr_write_4(sc, ci->ci_gic_redist, GICR_IGRPMODR0, 0);
    284   1.1  jmcneill 
    285   1.1  jmcneill 	/* Restore PPI configs */
    286   1.1  jmcneill 	for (n = 0, icfg = 0; n < 16; n++) {
    287   1.1  jmcneill 		struct intrsource * const is = sc->sc_pic.pic_sources[16 + n];
    288   1.1  jmcneill 		if (is != NULL && is->is_type == IST_EDGE)
    289   1.1  jmcneill 			icfg |= (0x2 << (n * 2));
    290   1.1  jmcneill 	}
    291   1.1  jmcneill 	gicr_write_4(sc, ci->ci_gic_redist, GICR_ICFGRn(1), icfg);
    292   1.1  jmcneill 
    293   1.1  jmcneill 	/* Restore current enable bits */
    294   1.1  jmcneill 	gicr_write_4(sc, ci->ci_gic_redist, GICR_ISENABLER0, sc->sc_enabled_sgippi);
    295   1.1  jmcneill 
    296   1.1  jmcneill 	/* Wait for register write to complete */
    297   1.5  jmcneill 	while (gicr_read_4(sc, ci->ci_gic_redist, GICR_CTLR) & GICR_CTLR_RWP)
    298   1.1  jmcneill 		;
    299   1.1  jmcneill }
    300   1.1  jmcneill 
    301   1.1  jmcneill static uint64_t
    302   1.1  jmcneill gicv3_cpu_identity(void)
    303   1.1  jmcneill {
    304   1.1  jmcneill 	u_int aff3, aff2, aff1, aff0;
    305   1.1  jmcneill 
    306   1.1  jmcneill #ifdef __aarch64__
    307   1.1  jmcneill 	const register_t mpidr = reg_mpidr_el1_read();
    308   1.1  jmcneill 	aff0 = __SHIFTOUT(mpidr, MPIDR_AFF0);
    309   1.1  jmcneill 	aff1 = __SHIFTOUT(mpidr, MPIDR_AFF1);
    310   1.1  jmcneill 	aff2 = __SHIFTOUT(mpidr, MPIDR_AFF2);
    311   1.1  jmcneill 	aff3 = __SHIFTOUT(mpidr, MPIDR_AFF3);
    312   1.1  jmcneill #else
    313   1.1  jmcneill 	const register_t mpidr = armreg_mpidr_read();
    314   1.1  jmcneill 	aff0 = __SHIFTOUT(mpidr, MPIDR_AFF0);
    315   1.1  jmcneill 	aff1 = __SHIFTOUT(mpidr, MPIDR_AFF1);
    316   1.1  jmcneill 	aff2 = __SHIFTOUT(mpidr, MPIDR_AFF2);
    317   1.1  jmcneill 	aff3 = 0;
    318   1.1  jmcneill #endif
    319   1.1  jmcneill 
    320   1.1  jmcneill 	return __SHIFTIN(aff0, GICR_TYPER_Affinity_Value_Aff0) |
    321   1.1  jmcneill 	       __SHIFTIN(aff1, GICR_TYPER_Affinity_Value_Aff1) |
    322   1.1  jmcneill 	       __SHIFTIN(aff2, GICR_TYPER_Affinity_Value_Aff2) |
    323   1.1  jmcneill 	       __SHIFTIN(aff3, GICR_TYPER_Affinity_Value_Aff3);
    324   1.1  jmcneill }
    325   1.1  jmcneill 
    326   1.1  jmcneill static u_int
    327   1.1  jmcneill gicv3_find_redist(struct gicv3_softc *sc)
    328   1.1  jmcneill {
    329   1.1  jmcneill 	uint64_t gicr_typer;
    330   1.1  jmcneill 	u_int n;
    331   1.1  jmcneill 
    332   1.1  jmcneill 	const uint64_t cpu_identity = gicv3_cpu_identity();
    333   1.1  jmcneill 
    334   1.1  jmcneill 	for (n = 0; n < sc->sc_bsh_r_count; n++) {
    335   1.1  jmcneill 		gicr_typer = gicr_read_8(sc, n, GICR_TYPER);
    336   1.1  jmcneill 		if ((gicr_typer & GICR_TYPER_Affinity_Value) == cpu_identity)
    337   1.1  jmcneill 			return n;
    338   1.1  jmcneill 	}
    339   1.1  jmcneill 
    340   1.1  jmcneill 	const u_int aff0 = __SHIFTOUT(cpu_identity, GICR_TYPER_Affinity_Value_Aff0);
    341   1.1  jmcneill 	const u_int aff1 = __SHIFTOUT(cpu_identity, GICR_TYPER_Affinity_Value_Aff1);
    342   1.1  jmcneill 	const u_int aff2 = __SHIFTOUT(cpu_identity, GICR_TYPER_Affinity_Value_Aff2);
    343   1.1  jmcneill 	const u_int aff3 = __SHIFTOUT(cpu_identity, GICR_TYPER_Affinity_Value_Aff3);
    344   1.1  jmcneill 
    345   1.1  jmcneill 	panic("%s: could not find GICv3 redistributor for cpu %d.%d.%d.%d",
    346   1.1  jmcneill 	    cpu_name(curcpu()), aff3, aff2, aff1, aff0);
    347   1.1  jmcneill }
    348   1.1  jmcneill 
    349   1.1  jmcneill static uint64_t
    350   1.1  jmcneill gicv3_sgir(struct gicv3_softc *sc)
    351   1.1  jmcneill {
    352   1.1  jmcneill 	const uint64_t cpu_identity = gicv3_cpu_identity();
    353   1.1  jmcneill 
    354   1.1  jmcneill 	const u_int aff0 = __SHIFTOUT(cpu_identity, GICR_TYPER_Affinity_Value_Aff0);
    355   1.1  jmcneill 	const u_int aff1 = __SHIFTOUT(cpu_identity, GICR_TYPER_Affinity_Value_Aff1);
    356   1.1  jmcneill 	const u_int aff2 = __SHIFTOUT(cpu_identity, GICR_TYPER_Affinity_Value_Aff2);
    357   1.1  jmcneill 	const u_int aff3 = __SHIFTOUT(cpu_identity, GICR_TYPER_Affinity_Value_Aff3);
    358   1.1  jmcneill 
    359   1.1  jmcneill 	return __SHIFTIN(__BIT(aff0), ICC_SGIR_EL1_TargetList) |
    360   1.1  jmcneill 	       __SHIFTIN(aff1, ICC_SGIR_EL1_Aff1) |
    361   1.1  jmcneill 	       __SHIFTIN(aff2, ICC_SGIR_EL1_Aff2) |
    362   1.1  jmcneill 	       __SHIFTIN(aff3, ICC_SGIR_EL1_Aff3);
    363   1.1  jmcneill }
    364   1.1  jmcneill 
    365   1.1  jmcneill static void
    366   1.1  jmcneill gicv3_cpu_init(struct pic_softc *pic, struct cpu_info *ci)
    367   1.1  jmcneill {
    368   1.1  jmcneill 	struct gicv3_softc * const sc = PICTOSOFTC(pic);
    369   1.1  jmcneill 	uint32_t icc_sre, icc_ctlr, gicr_waker;
    370   1.1  jmcneill 
    371   1.1  jmcneill 	ci->ci_gic_redist = gicv3_find_redist(sc);
    372   1.1  jmcneill 	ci->ci_gic_sgir = gicv3_sgir(sc);
    373   1.1  jmcneill 
    374   1.6  jmcneill 	/* Store route to CPU for SPIs */
    375   1.6  jmcneill 	const uint64_t cpu_identity = gicv3_cpu_identity();
    376   1.6  jmcneill 	const u_int aff0 = __SHIFTOUT(cpu_identity, GICR_TYPER_Affinity_Value_Aff0);
    377   1.6  jmcneill 	const u_int aff1 = __SHIFTOUT(cpu_identity, GICR_TYPER_Affinity_Value_Aff1);
    378   1.6  jmcneill 	const u_int aff2 = __SHIFTOUT(cpu_identity, GICR_TYPER_Affinity_Value_Aff2);
    379   1.6  jmcneill 	const u_int aff3 = __SHIFTOUT(cpu_identity, GICR_TYPER_Affinity_Value_Aff3);
    380   1.6  jmcneill 	sc->sc_irouter[cpu_index(ci)] =
    381   1.6  jmcneill 	    __SHIFTIN(aff0, GICD_IROUTER_Aff0) |
    382   1.6  jmcneill 	    __SHIFTIN(aff1, GICD_IROUTER_Aff1) |
    383   1.6  jmcneill 	    __SHIFTIN(aff2, GICD_IROUTER_Aff2) |
    384   1.6  jmcneill 	    __SHIFTIN(aff3, GICD_IROUTER_Aff3);
    385   1.1  jmcneill 
    386   1.1  jmcneill 	/* Enable System register access and disable IRQ/FIQ bypass */
    387   1.1  jmcneill 	icc_sre = ICC_SRE_EL1_SRE | ICC_SRE_EL1_DFB | ICC_SRE_EL1_DIB;
    388   1.1  jmcneill 	icc_sre_write(icc_sre);
    389   1.1  jmcneill 
    390   1.1  jmcneill 	/* Mark the connected PE as being awake */
    391   1.1  jmcneill 	gicr_waker = gicr_read_4(sc, ci->ci_gic_redist, GICR_WAKER);
    392   1.1  jmcneill 	gicr_waker &= ~GICR_WAKER_ProcessorSleep;
    393   1.1  jmcneill 	gicr_write_4(sc, ci->ci_gic_redist, GICR_WAKER, gicr_waker);
    394   1.1  jmcneill 	while (gicr_read_4(sc, ci->ci_gic_redist, GICR_WAKER) & GICR_WAKER_ChildrenAsleep)
    395   1.1  jmcneill 		;
    396   1.1  jmcneill 
    397   1.1  jmcneill 	/* Set initial priority mask */
    398   1.4  jmcneill 	gicv3_set_priority(pic, IPL_HIGH);
    399   1.1  jmcneill 
    400  1.10  jmcneill 	/* Set the binary point field to the minimum value */
    401  1.10  jmcneill 	icc_bpr1_write(0);
    402   1.1  jmcneill 
    403   1.1  jmcneill 	/* Enable group 1 interrupt signaling */
    404   1.1  jmcneill 	icc_igrpen1_write(ICC_IGRPEN_EL1_Enable);
    405   1.1  jmcneill 
    406   1.1  jmcneill 	/* Set EOI mode */
    407   1.1  jmcneill 	icc_ctlr = icc_ctlr_read();
    408   1.1  jmcneill 	icc_ctlr &= ~ICC_CTLR_EL1_EOImode;
    409   1.1  jmcneill 	icc_ctlr_write(icc_ctlr);
    410   1.1  jmcneill 
    411   1.1  jmcneill 	/* Enable redistributor */
    412   1.1  jmcneill 	gicv3_redist_enable(sc, ci);
    413   1.1  jmcneill 
    414   1.1  jmcneill 	/* Allow IRQ exceptions */
    415   1.1  jmcneill 	cpsie(I32_bit);
    416   1.1  jmcneill }
    417   1.1  jmcneill 
    418   1.1  jmcneill #ifdef MULTIPROCESSOR
    419   1.1  jmcneill static void
    420   1.1  jmcneill gicv3_ipi_send(struct pic_softc *pic, const kcpuset_t *kcp, u_long ipi)
    421   1.1  jmcneill {
    422   1.1  jmcneill 	CPU_INFO_ITERATOR cii;
    423   1.1  jmcneill 	struct cpu_info *ci;
    424   1.1  jmcneill 	uint64_t intid, aff, targets;
    425   1.1  jmcneill 
    426   1.1  jmcneill 	intid = __SHIFTIN(ipi, ICC_SGIR_EL1_INTID);
    427   1.1  jmcneill 	if (kcp == NULL) {
    428   1.1  jmcneill 		/* Interrupts routed to all PEs, excluding "self" */
    429   1.1  jmcneill 		if (ncpu == 1)
    430   1.1  jmcneill 			return;
    431   1.1  jmcneill 		icc_sgi1r_write(intid | ICC_SGIR_EL1_IRM);
    432   1.1  jmcneill 	} else {
    433   1.1  jmcneill 		/* Interrupts routed to specific PEs */
    434   1.1  jmcneill 		aff = 0;
    435   1.1  jmcneill 		targets = 0;
    436   1.1  jmcneill 		for (CPU_INFO_FOREACH(cii, ci)) {
    437   1.2  jmcneill 			if (!kcpuset_isset(kcp, cpu_index(ci)))
    438   1.2  jmcneill 				continue;
    439   1.1  jmcneill 			if ((ci->ci_gic_sgir & ICC_SGIR_EL1_Aff) != aff) {
    440   1.1  jmcneill 				if (targets != 0) {
    441   1.1  jmcneill 					icc_sgi1r_write(intid | aff | targets);
    442   1.1  jmcneill 					targets = 0;
    443   1.1  jmcneill 				}
    444   1.1  jmcneill 				aff = (ci->ci_gic_sgir & ICC_SGIR_EL1_Aff);
    445   1.1  jmcneill 			}
    446   1.1  jmcneill 			targets |= (ci->ci_gic_sgir & ICC_SGIR_EL1_TargetList);
    447   1.1  jmcneill 		}
    448   1.1  jmcneill 		if (targets != 0)
    449   1.1  jmcneill 			icc_sgi1r_write(intid | aff | targets);
    450   1.1  jmcneill 	}
    451   1.1  jmcneill }
    452   1.6  jmcneill 
    453   1.6  jmcneill static void
    454   1.6  jmcneill gicv3_get_affinity(struct pic_softc *pic, size_t irq, kcpuset_t *affinity)
    455   1.6  jmcneill {
    456   1.6  jmcneill 	struct gicv3_softc * const sc = PICTOSOFTC(pic);
    457   1.6  jmcneill 	const size_t group = irq / 32;
    458   1.6  jmcneill 	int n;
    459   1.6  jmcneill 
    460   1.6  jmcneill 	kcpuset_zero(affinity);
    461   1.6  jmcneill 	if (group == 0) {
    462   1.6  jmcneill 		/* All CPUs are targets for group 0 (SGI/PPI) */
    463   1.6  jmcneill 		for (n = 0; n < ncpu; n++) {
    464   1.6  jmcneill 			if (sc->sc_irouter[n] != UINT64_MAX)
    465   1.6  jmcneill 				kcpuset_set(affinity, n);
    466   1.6  jmcneill 		}
    467   1.6  jmcneill 	} else {
    468   1.6  jmcneill 		/* Find distributor targets (SPI) */
    469   1.6  jmcneill 		const uint64_t irouter = gicd_read_8(sc, GICD_IROUTER(irq));
    470   1.6  jmcneill 		for (n = 0; n < ncpu; n++) {
    471   1.6  jmcneill 			if (irouter == GICD_IROUTER_Interrupt_Routing_mode ||
    472   1.6  jmcneill 			    irouter == sc->sc_irouter[n])
    473   1.6  jmcneill 				kcpuset_set(affinity, n);
    474   1.6  jmcneill 		}
    475   1.6  jmcneill 	}
    476   1.6  jmcneill }
    477   1.6  jmcneill 
    478   1.6  jmcneill static int
    479   1.6  jmcneill gicv3_set_affinity(struct pic_softc *pic, size_t irq, const kcpuset_t *affinity)
    480   1.6  jmcneill {
    481   1.6  jmcneill 	struct gicv3_softc * const sc = PICTOSOFTC(pic);
    482   1.6  jmcneill 	const size_t group = irq / 32;
    483   1.6  jmcneill 	uint64_t irouter;
    484   1.6  jmcneill 
    485   1.6  jmcneill 	if (group == 0)
    486   1.6  jmcneill 		return EINVAL;
    487   1.6  jmcneill 
    488   1.6  jmcneill 	const int set = kcpuset_countset(affinity);
    489   1.6  jmcneill 	if (set == ncpu)
    490   1.6  jmcneill 		irouter = GICD_IROUTER_Interrupt_Routing_mode;
    491   1.6  jmcneill 	else if (set == 1)
    492   1.6  jmcneill 		irouter = sc->sc_irouter[kcpuset_ffs(affinity)];
    493   1.6  jmcneill 	else
    494   1.6  jmcneill 		return EINVAL;
    495   1.6  jmcneill 
    496   1.6  jmcneill 	gicd_write_8(sc, GICD_IROUTER(irq), irouter);
    497   1.6  jmcneill 
    498   1.6  jmcneill 	return 0;
    499   1.6  jmcneill }
    500   1.1  jmcneill #endif
    501   1.1  jmcneill 
    502   1.1  jmcneill static const struct pic_ops gicv3_picops = {
    503   1.1  jmcneill 	.pic_unblock_irqs = gicv3_unblock_irqs,
    504   1.1  jmcneill 	.pic_block_irqs = gicv3_block_irqs,
    505   1.1  jmcneill 	.pic_establish_irq = gicv3_establish_irq,
    506   1.1  jmcneill 	.pic_set_priority = gicv3_set_priority,
    507   1.1  jmcneill #ifdef MULTIPROCESSOR
    508   1.1  jmcneill 	.pic_cpu_init = gicv3_cpu_init,
    509   1.1  jmcneill 	.pic_ipi_send = gicv3_ipi_send,
    510   1.6  jmcneill 	.pic_get_affinity = gicv3_get_affinity,
    511   1.6  jmcneill 	.pic_set_affinity = gicv3_set_affinity,
    512   1.1  jmcneill #endif
    513   1.1  jmcneill };
    514   1.1  jmcneill 
    515   1.5  jmcneill static void
    516   1.5  jmcneill gicv3_lpi_unblock_irqs(struct pic_softc *pic, size_t irqbase, uint32_t mask)
    517   1.5  jmcneill {
    518   1.5  jmcneill 	struct gicv3_softc * const sc = LPITOSOFTC(pic);
    519   1.5  jmcneill 	int bit;
    520   1.5  jmcneill 
    521   1.5  jmcneill 	while ((bit = ffs(mask)) != 0) {
    522   1.5  jmcneill 		sc->sc_lpiconf.base[irqbase + bit - 1] |= GIC_LPICONF_Enable;
    523   1.5  jmcneill 		mask &= ~__BIT(bit - 1);
    524   1.5  jmcneill 	}
    525   1.5  jmcneill 
    526   1.5  jmcneill 	bus_dmamap_sync(sc->sc_dmat, sc->sc_lpiconf.map, irqbase, 32, BUS_DMASYNC_PREWRITE);
    527   1.5  jmcneill }
    528   1.5  jmcneill 
    529   1.5  jmcneill static void
    530   1.5  jmcneill gicv3_lpi_block_irqs(struct pic_softc *pic, size_t irqbase, uint32_t mask)
    531   1.5  jmcneill {
    532   1.5  jmcneill 	struct gicv3_softc * const sc = LPITOSOFTC(pic);
    533   1.5  jmcneill 	const u_int off = irqbase - pic->pic_irqbase;
    534   1.5  jmcneill 	int bit;
    535   1.5  jmcneill 
    536   1.5  jmcneill 	while ((bit = ffs(mask)) != 0) {
    537   1.5  jmcneill 		sc->sc_lpiconf.base[off + bit - 1] &= ~GIC_LPICONF_Enable;
    538   1.5  jmcneill 		mask &= ~__BIT(bit - 1);
    539   1.5  jmcneill 	}
    540   1.5  jmcneill 
    541   1.5  jmcneill 	bus_dmamap_sync(sc->sc_dmat, sc->sc_lpiconf.map, off, 32, BUS_DMASYNC_PREWRITE);
    542   1.5  jmcneill }
    543   1.5  jmcneill 
    544   1.5  jmcneill static void
    545   1.5  jmcneill gicv3_lpi_establish_irq(struct pic_softc *pic, struct intrsource *is)
    546   1.5  jmcneill {
    547   1.5  jmcneill 	struct gicv3_softc * const sc = LPITOSOFTC(pic);
    548   1.5  jmcneill 
    549   1.5  jmcneill 	sc->sc_lpiconf.base[is->is_irq] = IPL_TO_PRIORITY(is->is_ipl) | GIC_LPICONF_Res1;
    550   1.5  jmcneill 
    551   1.5  jmcneill 	bus_dmamap_sync(sc->sc_dmat, sc->sc_lpiconf.map, is->is_irq, 1, BUS_DMASYNC_PREWRITE);
    552   1.5  jmcneill }
    553   1.5  jmcneill 
    554   1.5  jmcneill static void
    555   1.5  jmcneill gicv3_lpi_cpu_init(struct pic_softc *pic, struct cpu_info *ci)
    556   1.5  jmcneill {
    557   1.5  jmcneill 	struct gicv3_softc * const sc = LPITOSOFTC(pic);
    558   1.7  jmcneill 	struct gicv3_lpi_callback *cb;
    559   1.5  jmcneill 	uint32_t ctlr;
    560   1.5  jmcneill 
    561   1.5  jmcneill 	/* If physical LPIs are not supported on this redistributor, just return. */
    562   1.5  jmcneill 	const uint64_t typer = gicr_read_8(sc, ci->ci_gic_redist, GICR_TYPER);
    563   1.5  jmcneill 	if ((typer & GICR_TYPER_PLPIS) == 0)
    564   1.5  jmcneill 		return;
    565   1.5  jmcneill 
    566   1.5  jmcneill 	/* Interrupt target address for this CPU, used by ITS when GITS_TYPER.PTA == 0 */
    567   1.5  jmcneill 	sc->sc_processor_id[cpu_index(ci)] = __SHIFTOUT(typer, GICR_TYPER_Processor_Number);
    568   1.5  jmcneill 
    569   1.5  jmcneill 	/* Disable LPIs before making changes */
    570   1.5  jmcneill 	ctlr = gicr_read_4(sc, ci->ci_gic_redist, GICR_CTLR);
    571   1.5  jmcneill 	ctlr &= ~GICR_CTLR_Enable_LPIs;
    572   1.5  jmcneill 	gicr_write_4(sc, ci->ci_gic_redist, GICR_CTLR, ctlr);
    573   1.5  jmcneill 	arm_dsb();
    574   1.5  jmcneill 
    575   1.5  jmcneill 	/* Setup the LPI configuration table */
    576   1.5  jmcneill 	const uint64_t propbase = sc->sc_lpiconf.segs[0].ds_addr |
    577   1.5  jmcneill 	    __SHIFTIN(ffs(pic->pic_maxsources) - 1, GICR_PROPBASER_IDbits) |
    578   1.5  jmcneill 	    __SHIFTIN(GICR_Shareability_NS, GICR_PROPBASER_Shareability) |
    579   1.5  jmcneill 	    __SHIFTIN(GICR_Cache_NORMAL_NC, GICR_PROPBASER_InnerCache);
    580   1.5  jmcneill 	gicr_write_8(sc, ci->ci_gic_redist, GICR_PROPBASER, propbase);
    581   1.5  jmcneill 
    582   1.5  jmcneill 	/* Setup the LPI pending table */
    583   1.5  jmcneill 	const uint64_t pendbase = sc->sc_lpipend[cpu_index(ci)].segs[0].ds_addr |
    584   1.5  jmcneill 	    __SHIFTIN(GICR_Shareability_NS, GICR_PENDBASER_Shareability) |
    585   1.5  jmcneill 	    __SHIFTIN(GICR_Cache_NORMAL_NC, GICR_PENDBASER_InnerCache) |
    586   1.5  jmcneill 	    GICR_PENDBASER_PTZ;
    587   1.5  jmcneill 	gicr_write_8(sc, ci->ci_gic_redist, GICR_PENDBASER, pendbase);
    588   1.5  jmcneill 
    589   1.5  jmcneill 	/* Enable LPIs */
    590   1.5  jmcneill 	ctlr = gicr_read_4(sc, ci->ci_gic_redist, GICR_CTLR);
    591   1.5  jmcneill 	ctlr |= GICR_CTLR_Enable_LPIs;
    592   1.5  jmcneill 	gicr_write_4(sc, ci->ci_gic_redist, GICR_CTLR, ctlr);
    593   1.5  jmcneill 	arm_dsb();
    594   1.5  jmcneill 
    595   1.5  jmcneill 	/* Setup ITS if present */
    596   1.7  jmcneill 	LIST_FOREACH(cb, &sc->sc_lpi_callbacks, list)
    597   1.7  jmcneill 		cb->cpu_init(cb->priv, ci);
    598   1.5  jmcneill }
    599   1.5  jmcneill 
    600   1.7  jmcneill #ifdef MULTIPROCESSOR
    601   1.7  jmcneill static void
    602   1.7  jmcneill gicv3_lpi_get_affinity(struct pic_softc *pic, size_t irq, kcpuset_t *affinity)
    603   1.7  jmcneill {
    604   1.7  jmcneill 	struct gicv3_softc * const sc = LPITOSOFTC(pic);
    605   1.7  jmcneill 	struct gicv3_lpi_callback *cb;
    606   1.7  jmcneill 
    607   1.7  jmcneill 	LIST_FOREACH(cb, &sc->sc_lpi_callbacks, list)
    608   1.7  jmcneill 		cb->get_affinity(cb->priv, irq, affinity);
    609   1.7  jmcneill }
    610   1.7  jmcneill 
    611   1.7  jmcneill static int
    612   1.7  jmcneill gicv3_lpi_set_affinity(struct pic_softc *pic, size_t irq, const kcpuset_t *affinity)
    613   1.7  jmcneill {
    614   1.7  jmcneill 	struct gicv3_softc * const sc = LPITOSOFTC(pic);
    615   1.7  jmcneill 	struct gicv3_lpi_callback *cb;
    616   1.7  jmcneill 	int error = EINVAL;
    617   1.7  jmcneill 
    618   1.7  jmcneill 	LIST_FOREACH(cb, &sc->sc_lpi_callbacks, list) {
    619   1.7  jmcneill 		error = cb->set_affinity(cb->priv, irq, affinity);
    620   1.7  jmcneill 		if (error)
    621   1.7  jmcneill 			return error;
    622   1.7  jmcneill 	}
    623   1.7  jmcneill 
    624   1.7  jmcneill 	return error;
    625   1.7  jmcneill }
    626   1.7  jmcneill #endif
    627   1.7  jmcneill 
    628   1.5  jmcneill static const struct pic_ops gicv3_lpiops = {
    629   1.5  jmcneill 	.pic_unblock_irqs = gicv3_lpi_unblock_irqs,
    630   1.5  jmcneill 	.pic_block_irqs = gicv3_lpi_block_irqs,
    631   1.5  jmcneill 	.pic_establish_irq = gicv3_lpi_establish_irq,
    632   1.5  jmcneill #ifdef MULTIPROCESSOR
    633   1.5  jmcneill 	.pic_cpu_init = gicv3_lpi_cpu_init,
    634   1.7  jmcneill 	.pic_get_affinity = gicv3_lpi_get_affinity,
    635   1.7  jmcneill 	.pic_set_affinity = gicv3_lpi_set_affinity,
    636   1.5  jmcneill #endif
    637   1.5  jmcneill };
    638   1.5  jmcneill 
    639   1.5  jmcneill void
    640   1.5  jmcneill gicv3_dma_alloc(struct gicv3_softc *sc, struct gicv3_dma *dma, bus_size_t len, bus_size_t align)
    641   1.5  jmcneill {
    642   1.5  jmcneill 	int nsegs, error;
    643   1.5  jmcneill 
    644   1.5  jmcneill 	dma->len = len;
    645   1.5  jmcneill 	error = bus_dmamem_alloc(sc->sc_dmat, dma->len, align, 0, dma->segs, 1, &nsegs, BUS_DMA_WAITOK);
    646   1.5  jmcneill 	if (error)
    647   1.5  jmcneill 		panic("bus_dmamem_alloc failed: %d", error);
    648   1.5  jmcneill 	error = bus_dmamem_map(sc->sc_dmat, dma->segs, nsegs, len, (void **)&dma->base, BUS_DMA_WAITOK);
    649   1.5  jmcneill 	if (error)
    650   1.5  jmcneill 		panic("bus_dmamem_map failed: %d", error);
    651   1.5  jmcneill 	error = bus_dmamap_create(sc->sc_dmat, len, 1, len, 0, BUS_DMA_WAITOK, &dma->map);
    652   1.5  jmcneill 	if (error)
    653   1.5  jmcneill 		panic("bus_dmamap_create failed: %d", error);
    654   1.5  jmcneill 	error = bus_dmamap_load(sc->sc_dmat, dma->map, dma->base, dma->len, NULL, BUS_DMA_WAITOK);
    655   1.5  jmcneill 	if (error)
    656   1.5  jmcneill 		panic("bus_dmamap_load failed: %d", error);
    657   1.5  jmcneill 
    658   1.5  jmcneill 	memset(dma->base, 0, dma->len);
    659   1.5  jmcneill 	bus_dmamap_sync(sc->sc_dmat, dma->map, 0, dma->len, BUS_DMASYNC_PREWRITE);
    660   1.5  jmcneill }
    661   1.5  jmcneill 
    662   1.5  jmcneill static void
    663   1.5  jmcneill gicv3_lpi_init(struct gicv3_softc *sc)
    664   1.5  jmcneill {
    665   1.5  jmcneill 	/*
    666   1.5  jmcneill 	 * Allocate LPI configuration table
    667   1.5  jmcneill 	 */
    668   1.5  jmcneill 	gicv3_dma_alloc(sc, &sc->sc_lpiconf, sc->sc_lpi.pic_maxsources, 0x1000);
    669   1.5  jmcneill 	KASSERT((sc->sc_lpiconf.segs[0].ds_addr & ~GICR_PROPBASER_Physical_Address) == 0);
    670   1.5  jmcneill 
    671   1.5  jmcneill 	/*
    672   1.5  jmcneill 	 * Allocate LPI pending tables
    673   1.5  jmcneill 	 */
    674   1.5  jmcneill 	const bus_size_t lpipend_sz = (sc->sc_lpi.pic_maxsources + sc->sc_lpi.pic_irqbase) / NBBY;
    675   1.8  jmcneill 	for (int cpuindex = 0; cpuindex < ncpu; cpuindex++) {
    676   1.5  jmcneill 		gicv3_dma_alloc(sc, &sc->sc_lpipend[cpuindex], lpipend_sz, 0x10000);
    677   1.5  jmcneill 		KASSERT((sc->sc_lpipend[cpuindex].segs[0].ds_addr & ~GICR_PENDBASER_Physical_Address) == 0);
    678   1.5  jmcneill 	}
    679   1.5  jmcneill }
    680   1.5  jmcneill 
    681   1.1  jmcneill void
    682   1.1  jmcneill gicv3_irq_handler(void *frame)
    683   1.1  jmcneill {
    684   1.1  jmcneill 	struct cpu_info * const ci = curcpu();
    685   1.1  jmcneill 	struct gicv3_softc * const sc = gicv3_softc;
    686   1.5  jmcneill 	struct pic_softc *pic;
    687   1.1  jmcneill 	const int oldipl = ci->ci_cpl;
    688   1.1  jmcneill 
    689   1.1  jmcneill 	ci->ci_data.cpu_nintr++;
    690   1.1  jmcneill 
    691   1.1  jmcneill 	for (;;) {
    692   1.1  jmcneill 		const uint32_t iar = icc_iar1_read();
    693   1.1  jmcneill 		const uint32_t irq = __SHIFTOUT(iar, ICC_IAR_INTID);
    694   1.1  jmcneill 		if (irq == ICC_IAR_INTID_SPURIOUS)
    695   1.1  jmcneill 			break;
    696   1.1  jmcneill 
    697   1.5  jmcneill 		pic = irq >= GIC_LPI_BASE ? &sc->sc_lpi : &sc->sc_pic;
    698   1.5  jmcneill 		if (irq - pic->pic_irqbase >= pic->pic_maxsources)
    699   1.1  jmcneill 			continue;
    700   1.1  jmcneill 
    701   1.5  jmcneill 		struct intrsource * const is = pic->pic_sources[irq - pic->pic_irqbase];
    702   1.1  jmcneill 		KASSERT(is != NULL);
    703   1.1  jmcneill 
    704   1.1  jmcneill 		const int ipl = is->is_ipl;
    705   1.2  jmcneill 		if (ci->ci_cpl < ipl)
    706   1.1  jmcneill 			pic_set_priority(ci, ipl);
    707   1.1  jmcneill 
    708   1.1  jmcneill 		cpsie(I32_bit);
    709   1.1  jmcneill 		pic_dispatch(is, frame);
    710   1.1  jmcneill 		cpsid(I32_bit);
    711   1.1  jmcneill 
    712   1.1  jmcneill 		icc_eoi1r_write(iar);
    713   1.1  jmcneill 	}
    714   1.1  jmcneill 
    715   1.1  jmcneill 	if (ci->ci_cpl != oldipl)
    716   1.1  jmcneill 		pic_set_priority(ci, oldipl);
    717   1.1  jmcneill }
    718   1.1  jmcneill 
    719   1.1  jmcneill int
    720   1.1  jmcneill gicv3_init(struct gicv3_softc *sc)
    721   1.1  jmcneill {
    722   1.1  jmcneill 	const uint32_t gicd_typer = gicd_read_4(sc, GICD_TYPER);
    723   1.6  jmcneill 	int n;
    724   1.1  jmcneill 
    725   1.1  jmcneill 	KASSERT(CPU_IS_PRIMARY(curcpu()));
    726   1.1  jmcneill 
    727   1.7  jmcneill 	LIST_INIT(&sc->sc_lpi_callbacks);
    728   1.5  jmcneill 
    729   1.6  jmcneill 	for (n = 0; n < MAXCPUS; n++)
    730   1.6  jmcneill 		sc->sc_irouter[n] = UINT64_MAX;
    731   1.6  jmcneill 
    732   1.1  jmcneill 	sc->sc_pic.pic_ops = &gicv3_picops;
    733   1.1  jmcneill 	sc->sc_pic.pic_maxsources = GICD_TYPER_LINES(gicd_typer);
    734   1.1  jmcneill 	snprintf(sc->sc_pic.pic_name, sizeof(sc->sc_pic.pic_name), "gicv3");
    735   1.1  jmcneill #ifdef MULTIPROCESSOR
    736   1.1  jmcneill 	sc->sc_pic.pic_cpus = kcpuset_running;
    737   1.1  jmcneill #endif
    738   1.1  jmcneill 	pic_add(&sc->sc_pic, 0);
    739   1.1  jmcneill 
    740   1.5  jmcneill 	if ((gicd_typer & GICD_TYPER_LPIS) != 0) {
    741   1.5  jmcneill 		sc->sc_lpi.pic_ops = &gicv3_lpiops;
    742   1.5  jmcneill 		sc->sc_lpi.pic_maxsources = 8192;	/* Min. required by GICv3 spec */
    743   1.5  jmcneill 		snprintf(sc->sc_lpi.pic_name, sizeof(sc->sc_lpi.pic_name), "gicv3-lpi");
    744   1.5  jmcneill 		pic_add(&sc->sc_lpi, GIC_LPI_BASE);
    745   1.5  jmcneill 
    746   1.5  jmcneill 		gicv3_lpi_init(sc);
    747   1.5  jmcneill 	}
    748   1.5  jmcneill 
    749   1.1  jmcneill 	KASSERT(gicv3_softc == NULL);
    750   1.1  jmcneill 	gicv3_softc = sc;
    751   1.1  jmcneill 
    752   1.1  jmcneill 	for (int i = 0; i < sc->sc_bsh_r_count; i++) {
    753   1.1  jmcneill 		const uint64_t gicr_typer = gicr_read_8(sc, i, GICR_TYPER);
    754   1.1  jmcneill 		const u_int aff0 = __SHIFTOUT(gicr_typer, GICR_TYPER_Affinity_Value_Aff0);
    755   1.1  jmcneill 		const u_int aff1 = __SHIFTOUT(gicr_typer, GICR_TYPER_Affinity_Value_Aff1);
    756   1.1  jmcneill 		const u_int aff2 = __SHIFTOUT(gicr_typer, GICR_TYPER_Affinity_Value_Aff2);
    757   1.1  jmcneill 		const u_int aff3 = __SHIFTOUT(gicr_typer, GICR_TYPER_Affinity_Value_Aff3);
    758   1.1  jmcneill 
    759   1.1  jmcneill 		aprint_debug_dev(sc->sc_dev, "redist %d: cpu %d.%d.%d.%d\n",
    760   1.1  jmcneill 		    i, aff3, aff2, aff1, aff0);
    761   1.1  jmcneill 	}
    762   1.1  jmcneill 
    763   1.1  jmcneill 	gicv3_dist_enable(sc);
    764   1.1  jmcneill 
    765   1.1  jmcneill 	gicv3_cpu_init(&sc->sc_pic, curcpu());
    766   1.5  jmcneill 	if ((gicd_typer & GICD_TYPER_LPIS) != 0)
    767   1.5  jmcneill 		gicv3_lpi_cpu_init(&sc->sc_lpi, curcpu());
    768   1.1  jmcneill 
    769   1.1  jmcneill #ifdef __HAVE_PIC_FAST_SOFTINTS
    770  1.11  jmcneill 	intr_establish_xname(SOFTINT_BIO, IPL_SOFTBIO, IST_MPSAFE | IST_EDGE, pic_handle_softint, (void *)SOFTINT_BIO, "softint bio");
    771  1.11  jmcneill 	intr_establish_xname(SOFTINT_CLOCK, IPL_SOFTCLOCK, IST_MPSAFE | IST_EDGE, pic_handle_softint, (void *)SOFTINT_CLOCK, "softint clock");
    772  1.11  jmcneill 	intr_establish_xname(SOFTINT_NET, IPL_SOFTNET, IST_MPSAFE | IST_EDGE, pic_handle_softint, (void *)SOFTINT_NET, "softint net");
    773  1.11  jmcneill 	intr_establish_xname(SOFTINT_SERIAL, IPL_SOFTSERIAL, IST_MPSAFE | IST_EDGE, pic_handle_softint, (void *)SOFTINT_SERIAL, "softint serial");
    774   1.1  jmcneill #endif
    775   1.1  jmcneill 
    776   1.1  jmcneill #ifdef MULTIPROCESSOR
    777  1.11  jmcneill 	intr_establish_xname(IPI_AST, IPL_VM, IST_MPSAFE | IST_EDGE, pic_ipi_ast, (void *)-1, "IPI ast");
    778  1.11  jmcneill 	intr_establish_xname(IPI_XCALL, IPL_HIGH, IST_MPSAFE | IST_EDGE, pic_ipi_xcall, (void *)-1, "IPI xcall");
    779  1.11  jmcneill 	intr_establish_xname(IPI_GENERIC, IPL_HIGH, IST_MPSAFE | IST_EDGE, pic_ipi_generic, (void *)-1, "IPI generic");
    780  1.11  jmcneill 	intr_establish_xname(IPI_NOP, IPL_VM, IST_MPSAFE | IST_EDGE, pic_ipi_nop, (void *)-1, "IPI nop");
    781  1.11  jmcneill 	intr_establish_xname(IPI_SHOOTDOWN, IPL_SCHED, IST_MPSAFE | IST_EDGE, pic_ipi_shootdown, (void *)-1, "IPI shootdown");
    782   1.1  jmcneill #ifdef DDB
    783  1.11  jmcneill 	intr_establish_xname(IPI_DDB, IPL_HIGH, IST_MPSAFE | IST_EDGE, pic_ipi_ddb, NULL, "IPI ddb");
    784   1.1  jmcneill #endif
    785   1.1  jmcneill #ifdef __HAVE_PREEMPTION
    786  1.11  jmcneill 	intr_establish_xname(IPI_KPREEMPT, IPL_VM, IST_MPSAFE | IST_EDGE, pic_ipi_kpreempt, (void *)-1, "IPI kpreempt");
    787   1.1  jmcneill #endif
    788   1.1  jmcneill #endif
    789   1.1  jmcneill 
    790   1.1  jmcneill 	return 0;
    791   1.1  jmcneill }
    792