Home | History | Annotate | Line # | Download | only in cortex
gicv3.c revision 1.19
      1  1.19  jmcneill /* $NetBSD: gicv3.c,v 1.19 2019/06/26 23:00:09 jmcneill Exp $ */
      2   1.1  jmcneill 
      3   1.1  jmcneill /*-
      4   1.1  jmcneill  * Copyright (c) 2018 Jared McNeill <jmcneill (at) invisible.ca>
      5   1.1  jmcneill  * All rights reserved.
      6   1.1  jmcneill  *
      7   1.1  jmcneill  * Redistribution and use in source and binary forms, with or without
      8   1.1  jmcneill  * modification, are permitted provided that the following conditions
      9   1.1  jmcneill  * are met:
     10   1.1  jmcneill  * 1. Redistributions of source code must retain the above copyright
     11   1.1  jmcneill  *    notice, this list of conditions and the following disclaimer.
     12   1.1  jmcneill  * 2. Redistributions in binary form must reproduce the above copyright
     13   1.1  jmcneill  *    notice, this list of conditions and the following disclaimer in the
     14   1.1  jmcneill  *    documentation and/or other materials provided with the distribution.
     15   1.1  jmcneill  *
     16   1.1  jmcneill  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
     17   1.1  jmcneill  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
     18   1.1  jmcneill  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
     19   1.1  jmcneill  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
     20   1.1  jmcneill  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
     21   1.1  jmcneill  * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
     22   1.1  jmcneill  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
     23   1.1  jmcneill  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
     24   1.1  jmcneill  * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
     25   1.1  jmcneill  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
     26   1.1  jmcneill  * SUCH DAMAGE.
     27   1.1  jmcneill  */
     28   1.1  jmcneill 
     29   1.1  jmcneill #include "opt_multiprocessor.h"
     30   1.1  jmcneill 
     31   1.1  jmcneill #define	_INTR_PRIVATE
     32   1.1  jmcneill 
     33   1.1  jmcneill #include <sys/cdefs.h>
     34  1.19  jmcneill __KERNEL_RCSID(0, "$NetBSD: gicv3.c,v 1.19 2019/06/26 23:00:09 jmcneill Exp $");
     35   1.1  jmcneill 
     36   1.1  jmcneill #include <sys/param.h>
     37   1.1  jmcneill #include <sys/kernel.h>
     38   1.1  jmcneill #include <sys/bus.h>
     39   1.1  jmcneill #include <sys/device.h>
     40   1.1  jmcneill #include <sys/intr.h>
     41   1.1  jmcneill #include <sys/systm.h>
     42   1.1  jmcneill #include <sys/cpu.h>
     43   1.1  jmcneill 
     44   1.1  jmcneill #include <arm/locore.h>
     45   1.1  jmcneill #include <arm/armreg.h>
     46   1.1  jmcneill 
     47   1.1  jmcneill #include <arm/cortex/gicv3.h>
     48   1.1  jmcneill #include <arm/cortex/gic_reg.h>
     49   1.1  jmcneill 
     50   1.1  jmcneill #define	PICTOSOFTC(pic)	\
     51   1.1  jmcneill 	((void *)((uintptr_t)(pic) - offsetof(struct gicv3_softc, sc_pic)))
     52   1.5  jmcneill #define	LPITOSOFTC(lpi) \
     53   1.5  jmcneill 	((void *)((uintptr_t)(lpi) - offsetof(struct gicv3_softc, sc_lpi)))
     54   1.1  jmcneill 
     55  1.18  jmcneill #define	IPL_TO_PRIORITY(sc, ipl)	(((0xff - (ipl)) << (sc)->sc_priority_shift) & 0xff)
     56  1.18  jmcneill #define	IPL_TO_PMR(sc, ipl)		(((0xff - (ipl)) << (sc)->sc_pmr_shift) & 0xff)
     57  1.18  jmcneill #define	IPL_TO_LPIPRIO(sc, ipl)		(((0xff - (ipl)) << 4) & 0xff)
     58   1.1  jmcneill 
     59   1.1  jmcneill static struct gicv3_softc *gicv3_softc;
     60   1.1  jmcneill 
     61   1.1  jmcneill static inline uint32_t
     62   1.1  jmcneill gicd_read_4(struct gicv3_softc *sc, bus_size_t reg)
     63   1.1  jmcneill {
     64   1.1  jmcneill 	return bus_space_read_4(sc->sc_bst, sc->sc_bsh_d, reg);
     65   1.1  jmcneill }
     66   1.1  jmcneill 
     67   1.1  jmcneill static inline void
     68   1.1  jmcneill gicd_write_4(struct gicv3_softc *sc, bus_size_t reg, uint32_t val)
     69   1.1  jmcneill {
     70   1.1  jmcneill 	bus_space_write_4(sc->sc_bst, sc->sc_bsh_d, reg, val);
     71   1.1  jmcneill }
     72   1.1  jmcneill 
     73   1.6  jmcneill static inline uint64_t
     74   1.6  jmcneill gicd_read_8(struct gicv3_softc *sc, bus_size_t reg)
     75   1.6  jmcneill {
     76   1.6  jmcneill 	return bus_space_read_8(sc->sc_bst, sc->sc_bsh_d, reg);
     77   1.6  jmcneill }
     78   1.6  jmcneill 
     79   1.1  jmcneill static inline void
     80   1.1  jmcneill gicd_write_8(struct gicv3_softc *sc, bus_size_t reg, uint64_t val)
     81   1.1  jmcneill {
     82   1.1  jmcneill 	bus_space_write_8(sc->sc_bst, sc->sc_bsh_d, reg, val);
     83   1.1  jmcneill }
     84   1.1  jmcneill 
     85   1.1  jmcneill static inline uint32_t
     86   1.1  jmcneill gicr_read_4(struct gicv3_softc *sc, u_int index, bus_size_t reg)
     87   1.1  jmcneill {
     88   1.1  jmcneill 	KASSERT(index < sc->sc_bsh_r_count);
     89   1.1  jmcneill 	return bus_space_read_4(sc->sc_bst, sc->sc_bsh_r[index], reg);
     90   1.1  jmcneill }
     91   1.1  jmcneill 
     92   1.1  jmcneill static inline void
     93   1.1  jmcneill gicr_write_4(struct gicv3_softc *sc, u_int index, bus_size_t reg, uint32_t val)
     94   1.1  jmcneill {
     95   1.1  jmcneill 	KASSERT(index < sc->sc_bsh_r_count);
     96   1.1  jmcneill 	bus_space_write_4(sc->sc_bst, sc->sc_bsh_r[index], reg, val);
     97   1.1  jmcneill }
     98   1.1  jmcneill 
     99   1.1  jmcneill static inline uint64_t
    100   1.1  jmcneill gicr_read_8(struct gicv3_softc *sc, u_int index, bus_size_t reg)
    101   1.1  jmcneill {
    102   1.1  jmcneill 	KASSERT(index < sc->sc_bsh_r_count);
    103   1.1  jmcneill 	return bus_space_read_8(sc->sc_bst, sc->sc_bsh_r[index], reg);
    104   1.1  jmcneill }
    105   1.1  jmcneill 
    106   1.1  jmcneill static inline void
    107   1.1  jmcneill gicr_write_8(struct gicv3_softc *sc, u_int index, bus_size_t reg, uint64_t val)
    108   1.1  jmcneill {
    109   1.1  jmcneill 	KASSERT(index < sc->sc_bsh_r_count);
    110   1.1  jmcneill 	bus_space_write_8(sc->sc_bst, sc->sc_bsh_r[index], reg, val);
    111   1.1  jmcneill }
    112   1.1  jmcneill 
    113   1.1  jmcneill static void
    114   1.1  jmcneill gicv3_unblock_irqs(struct pic_softc *pic, size_t irqbase, uint32_t mask)
    115   1.1  jmcneill {
    116   1.1  jmcneill 	struct gicv3_softc * const sc = PICTOSOFTC(pic);
    117   1.1  jmcneill 	struct cpu_info * const ci = curcpu();
    118   1.1  jmcneill 	const u_int group = irqbase / 32;
    119   1.1  jmcneill 
    120   1.1  jmcneill 	if (group == 0) {
    121   1.1  jmcneill 		sc->sc_enabled_sgippi |= mask;
    122   1.1  jmcneill 		gicr_write_4(sc, ci->ci_gic_redist, GICR_ISENABLER0, mask);
    123   1.5  jmcneill 		while (gicr_read_4(sc, ci->ci_gic_redist, GICR_CTLR) & GICR_CTLR_RWP)
    124   1.1  jmcneill 			;
    125   1.1  jmcneill 	} else {
    126   1.1  jmcneill 		gicd_write_4(sc, GICD_ISENABLERn(group), mask);
    127   1.1  jmcneill 		while (gicd_read_4(sc, GICD_CTRL) & GICD_CTRL_RWP)
    128   1.1  jmcneill 			;
    129   1.1  jmcneill 	}
    130   1.1  jmcneill }
    131   1.1  jmcneill 
    132   1.1  jmcneill static void
    133   1.1  jmcneill gicv3_block_irqs(struct pic_softc *pic, size_t irqbase, uint32_t mask)
    134   1.1  jmcneill {
    135   1.1  jmcneill 	struct gicv3_softc * const sc = PICTOSOFTC(pic);
    136   1.1  jmcneill 	struct cpu_info * const ci = curcpu();
    137   1.1  jmcneill 	const u_int group = irqbase / 32;
    138   1.1  jmcneill 
    139   1.1  jmcneill 	if (group == 0) {
    140   1.1  jmcneill 		sc->sc_enabled_sgippi &= ~mask;
    141   1.1  jmcneill 		gicr_write_4(sc, ci->ci_gic_redist, GICR_ICENABLER0, mask);
    142   1.5  jmcneill 		while (gicr_read_4(sc, ci->ci_gic_redist, GICR_CTLR) & GICR_CTLR_RWP)
    143   1.1  jmcneill 			;
    144   1.1  jmcneill 	} else {
    145   1.1  jmcneill 		gicd_write_4(sc, GICD_ICENABLERn(group), mask);
    146   1.1  jmcneill 		while (gicd_read_4(sc, GICD_CTRL) & GICD_CTRL_RWP)
    147   1.1  jmcneill 			;
    148   1.1  jmcneill 	}
    149   1.1  jmcneill }
    150   1.1  jmcneill 
    151   1.1  jmcneill static void
    152   1.1  jmcneill gicv3_establish_irq(struct pic_softc *pic, struct intrsource *is)
    153   1.1  jmcneill {
    154   1.1  jmcneill 	struct gicv3_softc * const sc = PICTOSOFTC(pic);
    155   1.1  jmcneill 	const u_int group = is->is_irq / 32;
    156   1.1  jmcneill 	uint32_t ipriority, icfg;
    157   1.1  jmcneill 	uint64_t irouter;
    158   1.1  jmcneill 	u_int n;
    159   1.1  jmcneill 
    160  1.18  jmcneill 	const u_int ipriority_val = IPL_TO_PRIORITY(sc, is->is_ipl);
    161   1.1  jmcneill 	const u_int ipriority_shift = (is->is_irq & 0x3) * 8;
    162   1.1  jmcneill 	const u_int icfg_shift = (is->is_irq & 0xf) * 2;
    163   1.1  jmcneill 
    164   1.1  jmcneill 	if (group == 0) {
    165   1.1  jmcneill 		/* SGIs and PPIs are always MP-safe */
    166   1.1  jmcneill 		is->is_mpsafe = true;
    167   1.1  jmcneill 
    168   1.1  jmcneill 		/* Update interrupt configuration and priority on all redistributors */
    169   1.1  jmcneill 		for (n = 0; n < sc->sc_bsh_r_count; n++) {
    170   1.1  jmcneill 			icfg = gicr_read_4(sc, n, GICR_ICFGRn(is->is_irq / 16));
    171   1.1  jmcneill 			if (is->is_type == IST_LEVEL)
    172   1.1  jmcneill 				icfg &= ~(0x2 << icfg_shift);
    173   1.1  jmcneill 			if (is->is_type == IST_EDGE)
    174   1.1  jmcneill 				icfg |= (0x2 << icfg_shift);
    175   1.1  jmcneill 			gicr_write_4(sc, n, GICR_ICFGRn(is->is_irq / 16), icfg);
    176   1.1  jmcneill 
    177   1.1  jmcneill 			ipriority = gicr_read_4(sc, n, GICR_IPRIORITYRn(is->is_irq / 4));
    178   1.1  jmcneill 			ipriority &= ~(0xff << ipriority_shift);
    179   1.2  jmcneill 			ipriority |= (ipriority_val << ipriority_shift);
    180   1.1  jmcneill 			gicr_write_4(sc, n, GICR_IPRIORITYRn(is->is_irq / 4), ipriority);
    181   1.1  jmcneill 		}
    182   1.1  jmcneill 	} else {
    183   1.1  jmcneill 		if (is->is_mpsafe) {
    184   1.1  jmcneill 			/* Route MP-safe interrupts to all participating PEs */
    185   1.1  jmcneill 			irouter = GICD_IROUTER_Interrupt_Routing_mode;
    186   1.1  jmcneill 		} else {
    187   1.1  jmcneill 			/* Route non-MP-safe interrupts to the primary PE only */
    188   1.6  jmcneill 			irouter = sc->sc_irouter[0];
    189   1.1  jmcneill 		}
    190   1.1  jmcneill 		gicd_write_8(sc, GICD_IROUTER(is->is_irq), irouter);
    191   1.1  jmcneill 
    192   1.1  jmcneill 		/* Update interrupt configuration */
    193   1.1  jmcneill 		icfg = gicd_read_4(sc, GICD_ICFGRn(is->is_irq / 16));
    194   1.1  jmcneill 		if (is->is_type == IST_LEVEL)
    195   1.1  jmcneill 			icfg &= ~(0x2 << icfg_shift);
    196   1.1  jmcneill 		if (is->is_type == IST_EDGE)
    197   1.1  jmcneill 			icfg |= (0x2 << icfg_shift);
    198   1.1  jmcneill 		gicd_write_4(sc, GICD_ICFGRn(is->is_irq / 16), icfg);
    199   1.1  jmcneill 
    200   1.1  jmcneill 		/* Update interrupt priority */
    201   1.1  jmcneill 		ipriority = gicd_read_4(sc, GICD_IPRIORITYRn(is->is_irq / 4));
    202   1.1  jmcneill 		ipriority &= ~(0xff << ipriority_shift);
    203   1.2  jmcneill 		ipriority |= (ipriority_val << ipriority_shift);
    204   1.1  jmcneill 		gicd_write_4(sc, GICD_IPRIORITYRn(is->is_irq / 4), ipriority);
    205   1.1  jmcneill 	}
    206   1.1  jmcneill }
    207   1.1  jmcneill 
    208   1.1  jmcneill static void
    209   1.1  jmcneill gicv3_set_priority(struct pic_softc *pic, int ipl)
    210   1.1  jmcneill {
    211  1.18  jmcneill 	struct gicv3_softc * const sc = PICTOSOFTC(pic);
    212  1.18  jmcneill 
    213  1.18  jmcneill 	icc_pmr_write(IPL_TO_PMR(sc, ipl));
    214   1.1  jmcneill }
    215   1.1  jmcneill 
    216   1.1  jmcneill static void
    217   1.1  jmcneill gicv3_dist_enable(struct gicv3_softc *sc)
    218   1.1  jmcneill {
    219   1.1  jmcneill 	uint32_t gicd_ctrl;
    220   1.1  jmcneill 	u_int n;
    221   1.1  jmcneill 
    222   1.1  jmcneill 	/* Disable the distributor */
    223   1.1  jmcneill 	gicd_write_4(sc, GICD_CTRL, 0);
    224   1.1  jmcneill 
    225   1.1  jmcneill 	/* Wait for register write to complete */
    226   1.1  jmcneill 	while (gicd_read_4(sc, GICD_CTRL) & GICD_CTRL_RWP)
    227   1.1  jmcneill 		;
    228   1.1  jmcneill 
    229   1.1  jmcneill 	/* Clear all INTID enable bits */
    230   1.1  jmcneill 	for (n = 32; n < sc->sc_pic.pic_maxsources; n += 32)
    231   1.1  jmcneill 		gicd_write_4(sc, GICD_ICENABLERn(n / 32), ~0);
    232   1.1  jmcneill 
    233   1.1  jmcneill 	/* Set default priorities to lowest */
    234   1.1  jmcneill 	for (n = 32; n < sc->sc_pic.pic_maxsources; n += 4)
    235   1.1  jmcneill 		gicd_write_4(sc, GICD_IPRIORITYRn(n / 4), ~0);
    236   1.1  jmcneill 
    237   1.1  jmcneill 	/* Set all interrupts to G1NS */
    238   1.1  jmcneill 	for (n = 32; n < sc->sc_pic.pic_maxsources; n += 32) {
    239   1.1  jmcneill 		gicd_write_4(sc, GICD_IGROUPRn(n / 32), ~0);
    240   1.1  jmcneill 		gicd_write_4(sc, GICD_IGRPMODRn(n / 32), 0);
    241   1.1  jmcneill 	}
    242   1.1  jmcneill 
    243   1.1  jmcneill 	/* Set all interrupts level-sensitive by default */
    244   1.1  jmcneill 	for (n = 32; n < sc->sc_pic.pic_maxsources; n += 16)
    245   1.1  jmcneill 		gicd_write_4(sc, GICD_ICFGRn(n / 16), 0);
    246   1.1  jmcneill 
    247   1.1  jmcneill 	/* Wait for register writes to complete */
    248   1.1  jmcneill 	while (gicd_read_4(sc, GICD_CTRL) & GICD_CTRL_RWP)
    249   1.1  jmcneill 		;
    250   1.1  jmcneill 
    251   1.1  jmcneill 	/* Enable Affinity routing and G1NS interrupts */
    252  1.19  jmcneill 	gicd_ctrl = GICD_CTRL_EnableGrp1A | GICD_CTRL_ARE_NS;
    253   1.1  jmcneill 	gicd_write_4(sc, GICD_CTRL, gicd_ctrl);
    254   1.1  jmcneill }
    255   1.1  jmcneill 
    256   1.1  jmcneill static void
    257   1.1  jmcneill gicv3_redist_enable(struct gicv3_softc *sc, struct cpu_info *ci)
    258   1.1  jmcneill {
    259   1.1  jmcneill 	uint32_t icfg;
    260   1.1  jmcneill 	u_int n, o;
    261   1.1  jmcneill 
    262   1.1  jmcneill 	/* Clear INTID enable bits */
    263   1.1  jmcneill 	gicr_write_4(sc, ci->ci_gic_redist, GICR_ICENABLER0, ~0);
    264   1.1  jmcneill 
    265   1.1  jmcneill 	/* Wait for register write to complete */
    266   1.5  jmcneill 	while (gicr_read_4(sc, ci->ci_gic_redist, GICR_CTLR) & GICR_CTLR_RWP)
    267   1.1  jmcneill 		;
    268   1.1  jmcneill 
    269   1.1  jmcneill 	/* Set default priorities */
    270   1.1  jmcneill 	for (n = 0; n < 32; n += 4) {
    271   1.1  jmcneill 		uint32_t priority = 0;
    272   1.1  jmcneill 		size_t byte_shift = 0;
    273   1.1  jmcneill 		for (o = 0; o < 4; o++, byte_shift += 8) {
    274   1.1  jmcneill 			struct intrsource * const is = sc->sc_pic.pic_sources[n + o];
    275   1.1  jmcneill 			if (is == NULL)
    276   1.1  jmcneill 				priority |= 0xff << byte_shift;
    277   1.2  jmcneill 			else {
    278  1.18  jmcneill 				const u_int ipriority_val = IPL_TO_PRIORITY(sc, is->is_ipl);
    279   1.2  jmcneill 				priority |= ipriority_val << byte_shift;
    280   1.2  jmcneill 			}
    281   1.1  jmcneill 		}
    282   1.1  jmcneill 		gicr_write_4(sc, ci->ci_gic_redist, GICR_IPRIORITYRn(n / 4), priority);
    283   1.1  jmcneill 	}
    284   1.1  jmcneill 
    285   1.1  jmcneill 	/* Set all interrupts to G1NS */
    286   1.1  jmcneill 	gicr_write_4(sc, ci->ci_gic_redist, GICR_IGROUPR0, ~0);
    287   1.1  jmcneill 	gicr_write_4(sc, ci->ci_gic_redist, GICR_IGRPMODR0, 0);
    288   1.1  jmcneill 
    289   1.1  jmcneill 	/* Restore PPI configs */
    290   1.1  jmcneill 	for (n = 0, icfg = 0; n < 16; n++) {
    291   1.1  jmcneill 		struct intrsource * const is = sc->sc_pic.pic_sources[16 + n];
    292   1.1  jmcneill 		if (is != NULL && is->is_type == IST_EDGE)
    293   1.1  jmcneill 			icfg |= (0x2 << (n * 2));
    294   1.1  jmcneill 	}
    295   1.1  jmcneill 	gicr_write_4(sc, ci->ci_gic_redist, GICR_ICFGRn(1), icfg);
    296   1.1  jmcneill 
    297   1.1  jmcneill 	/* Restore current enable bits */
    298   1.1  jmcneill 	gicr_write_4(sc, ci->ci_gic_redist, GICR_ISENABLER0, sc->sc_enabled_sgippi);
    299   1.1  jmcneill 
    300   1.1  jmcneill 	/* Wait for register write to complete */
    301   1.5  jmcneill 	while (gicr_read_4(sc, ci->ci_gic_redist, GICR_CTLR) & GICR_CTLR_RWP)
    302   1.1  jmcneill 		;
    303   1.1  jmcneill }
    304   1.1  jmcneill 
    305   1.1  jmcneill static uint64_t
    306   1.1  jmcneill gicv3_cpu_identity(void)
    307   1.1  jmcneill {
    308   1.1  jmcneill 	u_int aff3, aff2, aff1, aff0;
    309   1.1  jmcneill 
    310  1.18  jmcneill 	const register_t mpidr = cpu_mpidr_aff_read();
    311   1.1  jmcneill 	aff0 = __SHIFTOUT(mpidr, MPIDR_AFF0);
    312   1.1  jmcneill 	aff1 = __SHIFTOUT(mpidr, MPIDR_AFF1);
    313   1.1  jmcneill 	aff2 = __SHIFTOUT(mpidr, MPIDR_AFF2);
    314   1.1  jmcneill 	aff3 = __SHIFTOUT(mpidr, MPIDR_AFF3);
    315   1.1  jmcneill 
    316   1.1  jmcneill 	return __SHIFTIN(aff0, GICR_TYPER_Affinity_Value_Aff0) |
    317   1.1  jmcneill 	       __SHIFTIN(aff1, GICR_TYPER_Affinity_Value_Aff1) |
    318   1.1  jmcneill 	       __SHIFTIN(aff2, GICR_TYPER_Affinity_Value_Aff2) |
    319   1.1  jmcneill 	       __SHIFTIN(aff3, GICR_TYPER_Affinity_Value_Aff3);
    320   1.1  jmcneill }
    321   1.1  jmcneill 
    322   1.1  jmcneill static u_int
    323   1.1  jmcneill gicv3_find_redist(struct gicv3_softc *sc)
    324   1.1  jmcneill {
    325   1.1  jmcneill 	uint64_t gicr_typer;
    326   1.1  jmcneill 	u_int n;
    327   1.1  jmcneill 
    328   1.1  jmcneill 	const uint64_t cpu_identity = gicv3_cpu_identity();
    329   1.1  jmcneill 
    330   1.1  jmcneill 	for (n = 0; n < sc->sc_bsh_r_count; n++) {
    331   1.1  jmcneill 		gicr_typer = gicr_read_8(sc, n, GICR_TYPER);
    332   1.1  jmcneill 		if ((gicr_typer & GICR_TYPER_Affinity_Value) == cpu_identity)
    333   1.1  jmcneill 			return n;
    334   1.1  jmcneill 	}
    335   1.1  jmcneill 
    336   1.1  jmcneill 	const u_int aff0 = __SHIFTOUT(cpu_identity, GICR_TYPER_Affinity_Value_Aff0);
    337   1.1  jmcneill 	const u_int aff1 = __SHIFTOUT(cpu_identity, GICR_TYPER_Affinity_Value_Aff1);
    338   1.1  jmcneill 	const u_int aff2 = __SHIFTOUT(cpu_identity, GICR_TYPER_Affinity_Value_Aff2);
    339   1.1  jmcneill 	const u_int aff3 = __SHIFTOUT(cpu_identity, GICR_TYPER_Affinity_Value_Aff3);
    340   1.1  jmcneill 
    341   1.1  jmcneill 	panic("%s: could not find GICv3 redistributor for cpu %d.%d.%d.%d",
    342   1.1  jmcneill 	    cpu_name(curcpu()), aff3, aff2, aff1, aff0);
    343   1.1  jmcneill }
    344   1.1  jmcneill 
    345   1.1  jmcneill static uint64_t
    346   1.1  jmcneill gicv3_sgir(struct gicv3_softc *sc)
    347   1.1  jmcneill {
    348   1.1  jmcneill 	const uint64_t cpu_identity = gicv3_cpu_identity();
    349   1.1  jmcneill 
    350   1.1  jmcneill 	const u_int aff0 = __SHIFTOUT(cpu_identity, GICR_TYPER_Affinity_Value_Aff0);
    351   1.1  jmcneill 	const u_int aff1 = __SHIFTOUT(cpu_identity, GICR_TYPER_Affinity_Value_Aff1);
    352   1.1  jmcneill 	const u_int aff2 = __SHIFTOUT(cpu_identity, GICR_TYPER_Affinity_Value_Aff2);
    353   1.1  jmcneill 	const u_int aff3 = __SHIFTOUT(cpu_identity, GICR_TYPER_Affinity_Value_Aff3);
    354   1.1  jmcneill 
    355   1.1  jmcneill 	return __SHIFTIN(__BIT(aff0), ICC_SGIR_EL1_TargetList) |
    356   1.1  jmcneill 	       __SHIFTIN(aff1, ICC_SGIR_EL1_Aff1) |
    357   1.1  jmcneill 	       __SHIFTIN(aff2, ICC_SGIR_EL1_Aff2) |
    358   1.1  jmcneill 	       __SHIFTIN(aff3, ICC_SGIR_EL1_Aff3);
    359   1.1  jmcneill }
    360   1.1  jmcneill 
    361   1.1  jmcneill static void
    362   1.1  jmcneill gicv3_cpu_init(struct pic_softc *pic, struct cpu_info *ci)
    363   1.1  jmcneill {
    364   1.1  jmcneill 	struct gicv3_softc * const sc = PICTOSOFTC(pic);
    365   1.1  jmcneill 	uint32_t icc_sre, icc_ctlr, gicr_waker;
    366   1.1  jmcneill 
    367   1.1  jmcneill 	ci->ci_gic_redist = gicv3_find_redist(sc);
    368   1.1  jmcneill 	ci->ci_gic_sgir = gicv3_sgir(sc);
    369   1.1  jmcneill 
    370   1.6  jmcneill 	/* Store route to CPU for SPIs */
    371   1.6  jmcneill 	const uint64_t cpu_identity = gicv3_cpu_identity();
    372   1.6  jmcneill 	const u_int aff0 = __SHIFTOUT(cpu_identity, GICR_TYPER_Affinity_Value_Aff0);
    373   1.6  jmcneill 	const u_int aff1 = __SHIFTOUT(cpu_identity, GICR_TYPER_Affinity_Value_Aff1);
    374   1.6  jmcneill 	const u_int aff2 = __SHIFTOUT(cpu_identity, GICR_TYPER_Affinity_Value_Aff2);
    375   1.6  jmcneill 	const u_int aff3 = __SHIFTOUT(cpu_identity, GICR_TYPER_Affinity_Value_Aff3);
    376   1.6  jmcneill 	sc->sc_irouter[cpu_index(ci)] =
    377   1.6  jmcneill 	    __SHIFTIN(aff0, GICD_IROUTER_Aff0) |
    378   1.6  jmcneill 	    __SHIFTIN(aff1, GICD_IROUTER_Aff1) |
    379   1.6  jmcneill 	    __SHIFTIN(aff2, GICD_IROUTER_Aff2) |
    380   1.6  jmcneill 	    __SHIFTIN(aff3, GICD_IROUTER_Aff3);
    381   1.1  jmcneill 
    382   1.1  jmcneill 	/* Enable System register access and disable IRQ/FIQ bypass */
    383   1.1  jmcneill 	icc_sre = ICC_SRE_EL1_SRE | ICC_SRE_EL1_DFB | ICC_SRE_EL1_DIB;
    384   1.1  jmcneill 	icc_sre_write(icc_sre);
    385   1.1  jmcneill 
    386   1.1  jmcneill 	/* Mark the connected PE as being awake */
    387   1.1  jmcneill 	gicr_waker = gicr_read_4(sc, ci->ci_gic_redist, GICR_WAKER);
    388   1.1  jmcneill 	gicr_waker &= ~GICR_WAKER_ProcessorSleep;
    389   1.1  jmcneill 	gicr_write_4(sc, ci->ci_gic_redist, GICR_WAKER, gicr_waker);
    390   1.1  jmcneill 	while (gicr_read_4(sc, ci->ci_gic_redist, GICR_WAKER) & GICR_WAKER_ChildrenAsleep)
    391   1.1  jmcneill 		;
    392   1.1  jmcneill 
    393   1.1  jmcneill 	/* Set initial priority mask */
    394   1.4  jmcneill 	gicv3_set_priority(pic, IPL_HIGH);
    395   1.1  jmcneill 
    396  1.10  jmcneill 	/* Set the binary point field to the minimum value */
    397  1.10  jmcneill 	icc_bpr1_write(0);
    398   1.1  jmcneill 
    399   1.1  jmcneill 	/* Enable group 1 interrupt signaling */
    400   1.1  jmcneill 	icc_igrpen1_write(ICC_IGRPEN_EL1_Enable);
    401   1.1  jmcneill 
    402   1.1  jmcneill 	/* Set EOI mode */
    403   1.1  jmcneill 	icc_ctlr = icc_ctlr_read();
    404   1.1  jmcneill 	icc_ctlr &= ~ICC_CTLR_EL1_EOImode;
    405   1.1  jmcneill 	icc_ctlr_write(icc_ctlr);
    406   1.1  jmcneill 
    407   1.1  jmcneill 	/* Enable redistributor */
    408   1.1  jmcneill 	gicv3_redist_enable(sc, ci);
    409   1.1  jmcneill 
    410   1.1  jmcneill 	/* Allow IRQ exceptions */
    411   1.1  jmcneill 	cpsie(I32_bit);
    412   1.1  jmcneill }
    413   1.1  jmcneill 
    414   1.1  jmcneill #ifdef MULTIPROCESSOR
    415   1.1  jmcneill static void
    416   1.1  jmcneill gicv3_ipi_send(struct pic_softc *pic, const kcpuset_t *kcp, u_long ipi)
    417   1.1  jmcneill {
    418   1.1  jmcneill 	CPU_INFO_ITERATOR cii;
    419   1.1  jmcneill 	struct cpu_info *ci;
    420   1.1  jmcneill 	uint64_t intid, aff, targets;
    421   1.1  jmcneill 
    422   1.1  jmcneill 	intid = __SHIFTIN(ipi, ICC_SGIR_EL1_INTID);
    423   1.1  jmcneill 	if (kcp == NULL) {
    424   1.1  jmcneill 		/* Interrupts routed to all PEs, excluding "self" */
    425   1.1  jmcneill 		if (ncpu == 1)
    426   1.1  jmcneill 			return;
    427   1.1  jmcneill 		icc_sgi1r_write(intid | ICC_SGIR_EL1_IRM);
    428   1.1  jmcneill 	} else {
    429   1.1  jmcneill 		/* Interrupts routed to specific PEs */
    430   1.1  jmcneill 		aff = 0;
    431   1.1  jmcneill 		targets = 0;
    432   1.1  jmcneill 		for (CPU_INFO_FOREACH(cii, ci)) {
    433   1.2  jmcneill 			if (!kcpuset_isset(kcp, cpu_index(ci)))
    434   1.2  jmcneill 				continue;
    435   1.1  jmcneill 			if ((ci->ci_gic_sgir & ICC_SGIR_EL1_Aff) != aff) {
    436   1.1  jmcneill 				if (targets != 0) {
    437   1.1  jmcneill 					icc_sgi1r_write(intid | aff | targets);
    438   1.1  jmcneill 					targets = 0;
    439   1.1  jmcneill 				}
    440   1.1  jmcneill 				aff = (ci->ci_gic_sgir & ICC_SGIR_EL1_Aff);
    441   1.1  jmcneill 			}
    442   1.1  jmcneill 			targets |= (ci->ci_gic_sgir & ICC_SGIR_EL1_TargetList);
    443   1.1  jmcneill 		}
    444   1.1  jmcneill 		if (targets != 0)
    445   1.1  jmcneill 			icc_sgi1r_write(intid | aff | targets);
    446   1.1  jmcneill 	}
    447   1.1  jmcneill }
    448   1.6  jmcneill 
    449   1.6  jmcneill static void
    450   1.6  jmcneill gicv3_get_affinity(struct pic_softc *pic, size_t irq, kcpuset_t *affinity)
    451   1.6  jmcneill {
    452   1.6  jmcneill 	struct gicv3_softc * const sc = PICTOSOFTC(pic);
    453   1.6  jmcneill 	const size_t group = irq / 32;
    454   1.6  jmcneill 	int n;
    455   1.6  jmcneill 
    456   1.6  jmcneill 	kcpuset_zero(affinity);
    457   1.6  jmcneill 	if (group == 0) {
    458   1.6  jmcneill 		/* All CPUs are targets for group 0 (SGI/PPI) */
    459   1.6  jmcneill 		for (n = 0; n < ncpu; n++) {
    460   1.6  jmcneill 			if (sc->sc_irouter[n] != UINT64_MAX)
    461   1.6  jmcneill 				kcpuset_set(affinity, n);
    462   1.6  jmcneill 		}
    463   1.6  jmcneill 	} else {
    464   1.6  jmcneill 		/* Find distributor targets (SPI) */
    465   1.6  jmcneill 		const uint64_t irouter = gicd_read_8(sc, GICD_IROUTER(irq));
    466   1.6  jmcneill 		for (n = 0; n < ncpu; n++) {
    467   1.6  jmcneill 			if (irouter == GICD_IROUTER_Interrupt_Routing_mode ||
    468   1.6  jmcneill 			    irouter == sc->sc_irouter[n])
    469   1.6  jmcneill 				kcpuset_set(affinity, n);
    470   1.6  jmcneill 		}
    471   1.6  jmcneill 	}
    472   1.6  jmcneill }
    473   1.6  jmcneill 
    474   1.6  jmcneill static int
    475   1.6  jmcneill gicv3_set_affinity(struct pic_softc *pic, size_t irq, const kcpuset_t *affinity)
    476   1.6  jmcneill {
    477   1.6  jmcneill 	struct gicv3_softc * const sc = PICTOSOFTC(pic);
    478   1.6  jmcneill 	const size_t group = irq / 32;
    479   1.6  jmcneill 	uint64_t irouter;
    480   1.6  jmcneill 
    481   1.6  jmcneill 	if (group == 0)
    482   1.6  jmcneill 		return EINVAL;
    483   1.6  jmcneill 
    484   1.6  jmcneill 	const int set = kcpuset_countset(affinity);
    485   1.6  jmcneill 	if (set == ncpu)
    486   1.6  jmcneill 		irouter = GICD_IROUTER_Interrupt_Routing_mode;
    487   1.6  jmcneill 	else if (set == 1)
    488  1.12  jmcneill 		irouter = sc->sc_irouter[kcpuset_ffs(affinity) - 1];
    489   1.6  jmcneill 	else
    490   1.6  jmcneill 		return EINVAL;
    491   1.6  jmcneill 
    492   1.6  jmcneill 	gicd_write_8(sc, GICD_IROUTER(irq), irouter);
    493   1.6  jmcneill 
    494   1.6  jmcneill 	return 0;
    495   1.6  jmcneill }
    496   1.1  jmcneill #endif
    497   1.1  jmcneill 
    498   1.1  jmcneill static const struct pic_ops gicv3_picops = {
    499   1.1  jmcneill 	.pic_unblock_irqs = gicv3_unblock_irqs,
    500   1.1  jmcneill 	.pic_block_irqs = gicv3_block_irqs,
    501   1.1  jmcneill 	.pic_establish_irq = gicv3_establish_irq,
    502   1.1  jmcneill 	.pic_set_priority = gicv3_set_priority,
    503   1.1  jmcneill #ifdef MULTIPROCESSOR
    504   1.1  jmcneill 	.pic_cpu_init = gicv3_cpu_init,
    505   1.1  jmcneill 	.pic_ipi_send = gicv3_ipi_send,
    506   1.6  jmcneill 	.pic_get_affinity = gicv3_get_affinity,
    507   1.6  jmcneill 	.pic_set_affinity = gicv3_set_affinity,
    508   1.1  jmcneill #endif
    509   1.1  jmcneill };
    510   1.1  jmcneill 
    511   1.5  jmcneill static void
    512   1.5  jmcneill gicv3_lpi_unblock_irqs(struct pic_softc *pic, size_t irqbase, uint32_t mask)
    513   1.5  jmcneill {
    514   1.5  jmcneill 	struct gicv3_softc * const sc = LPITOSOFTC(pic);
    515   1.5  jmcneill 	int bit;
    516   1.5  jmcneill 
    517   1.5  jmcneill 	while ((bit = ffs(mask)) != 0) {
    518   1.5  jmcneill 		sc->sc_lpiconf.base[irqbase + bit - 1] |= GIC_LPICONF_Enable;
    519   1.5  jmcneill 		mask &= ~__BIT(bit - 1);
    520   1.5  jmcneill 	}
    521   1.5  jmcneill 
    522   1.5  jmcneill 	bus_dmamap_sync(sc->sc_dmat, sc->sc_lpiconf.map, irqbase, 32, BUS_DMASYNC_PREWRITE);
    523   1.5  jmcneill }
    524   1.5  jmcneill 
    525   1.5  jmcneill static void
    526   1.5  jmcneill gicv3_lpi_block_irqs(struct pic_softc *pic, size_t irqbase, uint32_t mask)
    527   1.5  jmcneill {
    528   1.5  jmcneill 	struct gicv3_softc * const sc = LPITOSOFTC(pic);
    529   1.5  jmcneill 	int bit;
    530   1.5  jmcneill 
    531   1.5  jmcneill 	while ((bit = ffs(mask)) != 0) {
    532  1.13  jmcneill 		sc->sc_lpiconf.base[irqbase + bit - 1] &= ~GIC_LPICONF_Enable;
    533   1.5  jmcneill 		mask &= ~__BIT(bit - 1);
    534   1.5  jmcneill 	}
    535   1.5  jmcneill 
    536  1.13  jmcneill 	bus_dmamap_sync(sc->sc_dmat, sc->sc_lpiconf.map, irqbase, 32, BUS_DMASYNC_PREWRITE);
    537   1.5  jmcneill }
    538   1.5  jmcneill 
    539   1.5  jmcneill static void
    540   1.5  jmcneill gicv3_lpi_establish_irq(struct pic_softc *pic, struct intrsource *is)
    541   1.5  jmcneill {
    542   1.5  jmcneill 	struct gicv3_softc * const sc = LPITOSOFTC(pic);
    543   1.5  jmcneill 
    544  1.18  jmcneill 	sc->sc_lpiconf.base[is->is_irq] = IPL_TO_LPIPRIO(sc, is->is_ipl) | GIC_LPICONF_Res1;
    545   1.5  jmcneill 
    546   1.5  jmcneill 	bus_dmamap_sync(sc->sc_dmat, sc->sc_lpiconf.map, is->is_irq, 1, BUS_DMASYNC_PREWRITE);
    547   1.5  jmcneill }
    548   1.5  jmcneill 
    549   1.5  jmcneill static void
    550   1.5  jmcneill gicv3_lpi_cpu_init(struct pic_softc *pic, struct cpu_info *ci)
    551   1.5  jmcneill {
    552   1.5  jmcneill 	struct gicv3_softc * const sc = LPITOSOFTC(pic);
    553   1.7  jmcneill 	struct gicv3_lpi_callback *cb;
    554   1.5  jmcneill 	uint32_t ctlr;
    555   1.5  jmcneill 
    556   1.5  jmcneill 	/* If physical LPIs are not supported on this redistributor, just return. */
    557   1.5  jmcneill 	const uint64_t typer = gicr_read_8(sc, ci->ci_gic_redist, GICR_TYPER);
    558   1.5  jmcneill 	if ((typer & GICR_TYPER_PLPIS) == 0)
    559   1.5  jmcneill 		return;
    560   1.5  jmcneill 
    561   1.5  jmcneill 	/* Interrupt target address for this CPU, used by ITS when GITS_TYPER.PTA == 0 */
    562   1.5  jmcneill 	sc->sc_processor_id[cpu_index(ci)] = __SHIFTOUT(typer, GICR_TYPER_Processor_Number);
    563   1.5  jmcneill 
    564   1.5  jmcneill 	/* Disable LPIs before making changes */
    565   1.5  jmcneill 	ctlr = gicr_read_4(sc, ci->ci_gic_redist, GICR_CTLR);
    566   1.5  jmcneill 	ctlr &= ~GICR_CTLR_Enable_LPIs;
    567   1.5  jmcneill 	gicr_write_4(sc, ci->ci_gic_redist, GICR_CTLR, ctlr);
    568   1.5  jmcneill 	arm_dsb();
    569   1.5  jmcneill 
    570   1.5  jmcneill 	/* Setup the LPI configuration table */
    571   1.5  jmcneill 	const uint64_t propbase = sc->sc_lpiconf.segs[0].ds_addr |
    572   1.5  jmcneill 	    __SHIFTIN(ffs(pic->pic_maxsources) - 1, GICR_PROPBASER_IDbits) |
    573   1.5  jmcneill 	    __SHIFTIN(GICR_Shareability_NS, GICR_PROPBASER_Shareability) |
    574   1.5  jmcneill 	    __SHIFTIN(GICR_Cache_NORMAL_NC, GICR_PROPBASER_InnerCache);
    575   1.5  jmcneill 	gicr_write_8(sc, ci->ci_gic_redist, GICR_PROPBASER, propbase);
    576   1.5  jmcneill 
    577   1.5  jmcneill 	/* Setup the LPI pending table */
    578   1.5  jmcneill 	const uint64_t pendbase = sc->sc_lpipend[cpu_index(ci)].segs[0].ds_addr |
    579   1.5  jmcneill 	    __SHIFTIN(GICR_Shareability_NS, GICR_PENDBASER_Shareability) |
    580   1.5  jmcneill 	    __SHIFTIN(GICR_Cache_NORMAL_NC, GICR_PENDBASER_InnerCache) |
    581   1.5  jmcneill 	    GICR_PENDBASER_PTZ;
    582   1.5  jmcneill 	gicr_write_8(sc, ci->ci_gic_redist, GICR_PENDBASER, pendbase);
    583   1.5  jmcneill 
    584   1.5  jmcneill 	/* Enable LPIs */
    585   1.5  jmcneill 	ctlr = gicr_read_4(sc, ci->ci_gic_redist, GICR_CTLR);
    586   1.5  jmcneill 	ctlr |= GICR_CTLR_Enable_LPIs;
    587   1.5  jmcneill 	gicr_write_4(sc, ci->ci_gic_redist, GICR_CTLR, ctlr);
    588   1.5  jmcneill 	arm_dsb();
    589   1.5  jmcneill 
    590   1.5  jmcneill 	/* Setup ITS if present */
    591   1.7  jmcneill 	LIST_FOREACH(cb, &sc->sc_lpi_callbacks, list)
    592   1.7  jmcneill 		cb->cpu_init(cb->priv, ci);
    593   1.5  jmcneill }
    594   1.5  jmcneill 
    595   1.7  jmcneill #ifdef MULTIPROCESSOR
    596   1.7  jmcneill static void
    597   1.7  jmcneill gicv3_lpi_get_affinity(struct pic_softc *pic, size_t irq, kcpuset_t *affinity)
    598   1.7  jmcneill {
    599   1.7  jmcneill 	struct gicv3_softc * const sc = LPITOSOFTC(pic);
    600   1.7  jmcneill 	struct gicv3_lpi_callback *cb;
    601   1.7  jmcneill 
    602   1.7  jmcneill 	LIST_FOREACH(cb, &sc->sc_lpi_callbacks, list)
    603   1.7  jmcneill 		cb->get_affinity(cb->priv, irq, affinity);
    604   1.7  jmcneill }
    605   1.7  jmcneill 
    606   1.7  jmcneill static int
    607   1.7  jmcneill gicv3_lpi_set_affinity(struct pic_softc *pic, size_t irq, const kcpuset_t *affinity)
    608   1.7  jmcneill {
    609   1.7  jmcneill 	struct gicv3_softc * const sc = LPITOSOFTC(pic);
    610   1.7  jmcneill 	struct gicv3_lpi_callback *cb;
    611   1.7  jmcneill 	int error = EINVAL;
    612   1.7  jmcneill 
    613   1.7  jmcneill 	LIST_FOREACH(cb, &sc->sc_lpi_callbacks, list) {
    614   1.7  jmcneill 		error = cb->set_affinity(cb->priv, irq, affinity);
    615   1.7  jmcneill 		if (error)
    616   1.7  jmcneill 			return error;
    617   1.7  jmcneill 	}
    618   1.7  jmcneill 
    619   1.7  jmcneill 	return error;
    620   1.7  jmcneill }
    621   1.7  jmcneill #endif
    622   1.7  jmcneill 
    623   1.5  jmcneill static const struct pic_ops gicv3_lpiops = {
    624   1.5  jmcneill 	.pic_unblock_irqs = gicv3_lpi_unblock_irqs,
    625   1.5  jmcneill 	.pic_block_irqs = gicv3_lpi_block_irqs,
    626   1.5  jmcneill 	.pic_establish_irq = gicv3_lpi_establish_irq,
    627   1.5  jmcneill #ifdef MULTIPROCESSOR
    628   1.5  jmcneill 	.pic_cpu_init = gicv3_lpi_cpu_init,
    629   1.7  jmcneill 	.pic_get_affinity = gicv3_lpi_get_affinity,
    630   1.7  jmcneill 	.pic_set_affinity = gicv3_lpi_set_affinity,
    631   1.5  jmcneill #endif
    632   1.5  jmcneill };
    633   1.5  jmcneill 
    634   1.5  jmcneill void
    635   1.5  jmcneill gicv3_dma_alloc(struct gicv3_softc *sc, struct gicv3_dma *dma, bus_size_t len, bus_size_t align)
    636   1.5  jmcneill {
    637   1.5  jmcneill 	int nsegs, error;
    638   1.5  jmcneill 
    639   1.5  jmcneill 	dma->len = len;
    640   1.5  jmcneill 	error = bus_dmamem_alloc(sc->sc_dmat, dma->len, align, 0, dma->segs, 1, &nsegs, BUS_DMA_WAITOK);
    641   1.5  jmcneill 	if (error)
    642   1.5  jmcneill 		panic("bus_dmamem_alloc failed: %d", error);
    643   1.5  jmcneill 	error = bus_dmamem_map(sc->sc_dmat, dma->segs, nsegs, len, (void **)&dma->base, BUS_DMA_WAITOK);
    644   1.5  jmcneill 	if (error)
    645   1.5  jmcneill 		panic("bus_dmamem_map failed: %d", error);
    646   1.5  jmcneill 	error = bus_dmamap_create(sc->sc_dmat, len, 1, len, 0, BUS_DMA_WAITOK, &dma->map);
    647   1.5  jmcneill 	if (error)
    648   1.5  jmcneill 		panic("bus_dmamap_create failed: %d", error);
    649   1.5  jmcneill 	error = bus_dmamap_load(sc->sc_dmat, dma->map, dma->base, dma->len, NULL, BUS_DMA_WAITOK);
    650   1.5  jmcneill 	if (error)
    651   1.5  jmcneill 		panic("bus_dmamap_load failed: %d", error);
    652   1.5  jmcneill 
    653   1.5  jmcneill 	memset(dma->base, 0, dma->len);
    654   1.5  jmcneill 	bus_dmamap_sync(sc->sc_dmat, dma->map, 0, dma->len, BUS_DMASYNC_PREWRITE);
    655   1.5  jmcneill }
    656   1.5  jmcneill 
    657   1.5  jmcneill static void
    658   1.5  jmcneill gicv3_lpi_init(struct gicv3_softc *sc)
    659   1.5  jmcneill {
    660   1.5  jmcneill 	/*
    661   1.5  jmcneill 	 * Allocate LPI configuration table
    662   1.5  jmcneill 	 */
    663   1.5  jmcneill 	gicv3_dma_alloc(sc, &sc->sc_lpiconf, sc->sc_lpi.pic_maxsources, 0x1000);
    664   1.5  jmcneill 	KASSERT((sc->sc_lpiconf.segs[0].ds_addr & ~GICR_PROPBASER_Physical_Address) == 0);
    665   1.5  jmcneill 
    666   1.5  jmcneill 	/*
    667   1.5  jmcneill 	 * Allocate LPI pending tables
    668   1.5  jmcneill 	 */
    669  1.13  jmcneill 	const bus_size_t lpipend_sz = sc->sc_lpi.pic_maxsources / NBBY;
    670   1.8  jmcneill 	for (int cpuindex = 0; cpuindex < ncpu; cpuindex++) {
    671   1.5  jmcneill 		gicv3_dma_alloc(sc, &sc->sc_lpipend[cpuindex], lpipend_sz, 0x10000);
    672   1.5  jmcneill 		KASSERT((sc->sc_lpipend[cpuindex].segs[0].ds_addr & ~GICR_PENDBASER_Physical_Address) == 0);
    673   1.5  jmcneill 	}
    674   1.5  jmcneill }
    675   1.5  jmcneill 
    676   1.1  jmcneill void
    677   1.1  jmcneill gicv3_irq_handler(void *frame)
    678   1.1  jmcneill {
    679   1.1  jmcneill 	struct cpu_info * const ci = curcpu();
    680   1.1  jmcneill 	struct gicv3_softc * const sc = gicv3_softc;
    681   1.5  jmcneill 	struct pic_softc *pic;
    682   1.1  jmcneill 	const int oldipl = ci->ci_cpl;
    683   1.1  jmcneill 
    684   1.1  jmcneill 	ci->ci_data.cpu_nintr++;
    685   1.1  jmcneill 
    686   1.1  jmcneill 	for (;;) {
    687   1.1  jmcneill 		const uint32_t iar = icc_iar1_read();
    688   1.1  jmcneill 		const uint32_t irq = __SHIFTOUT(iar, ICC_IAR_INTID);
    689   1.1  jmcneill 		if (irq == ICC_IAR_INTID_SPURIOUS)
    690   1.1  jmcneill 			break;
    691   1.1  jmcneill 
    692   1.5  jmcneill 		pic = irq >= GIC_LPI_BASE ? &sc->sc_lpi : &sc->sc_pic;
    693   1.5  jmcneill 		if (irq - pic->pic_irqbase >= pic->pic_maxsources)
    694   1.1  jmcneill 			continue;
    695   1.1  jmcneill 
    696   1.5  jmcneill 		struct intrsource * const is = pic->pic_sources[irq - pic->pic_irqbase];
    697   1.1  jmcneill 		KASSERT(is != NULL);
    698   1.1  jmcneill 
    699   1.1  jmcneill 		const int ipl = is->is_ipl;
    700   1.2  jmcneill 		if (ci->ci_cpl < ipl)
    701   1.1  jmcneill 			pic_set_priority(ci, ipl);
    702   1.1  jmcneill 
    703   1.1  jmcneill 		cpsie(I32_bit);
    704   1.1  jmcneill 		pic_dispatch(is, frame);
    705   1.1  jmcneill 		cpsid(I32_bit);
    706   1.1  jmcneill 
    707   1.1  jmcneill 		icc_eoi1r_write(iar);
    708   1.1  jmcneill 	}
    709   1.1  jmcneill 
    710   1.1  jmcneill 	if (ci->ci_cpl != oldipl)
    711   1.1  jmcneill 		pic_set_priority(ci, oldipl);
    712   1.1  jmcneill }
    713   1.1  jmcneill 
    714  1.19  jmcneill static int
    715  1.19  jmcneill gicv3_detect_pmr_bits(struct gicv3_softc *sc)
    716  1.19  jmcneill {
    717  1.19  jmcneill 	const uint32_t opmr = icc_pmr_read();
    718  1.19  jmcneill 	icc_pmr_write(0xff);
    719  1.19  jmcneill 	const uint32_t npmr = icc_pmr_read();
    720  1.19  jmcneill 	icc_pmr_write(opmr);
    721  1.19  jmcneill 
    722  1.19  jmcneill 	return NBBY - (ffs(npmr) - 1);
    723  1.19  jmcneill }
    724  1.19  jmcneill 
    725  1.19  jmcneill static int
    726  1.19  jmcneill gicv3_detect_ipriority_bits(struct gicv3_softc *sc)
    727  1.19  jmcneill {
    728  1.19  jmcneill 	const uint32_t oipriorityr = gicd_read_4(sc, GICD_IPRIORITYRn(8));
    729  1.19  jmcneill 	gicd_write_4(sc, GICD_IPRIORITYRn(8), oipriorityr | 0xff);
    730  1.19  jmcneill 	const uint32_t nipriorityr = gicd_read_4(sc, GICD_IPRIORITYRn(8));
    731  1.19  jmcneill 	gicd_write_4(sc, GICD_IPRIORITYRn(8), oipriorityr);
    732  1.19  jmcneill 
    733  1.19  jmcneill 	return NBBY - (ffs(nipriorityr & 0xff) - 1);
    734  1.19  jmcneill }
    735  1.19  jmcneill 
    736   1.1  jmcneill int
    737   1.1  jmcneill gicv3_init(struct gicv3_softc *sc)
    738   1.1  jmcneill {
    739   1.1  jmcneill 	const uint32_t gicd_typer = gicd_read_4(sc, GICD_TYPER);
    740  1.18  jmcneill 	const uint32_t gicd_ctrl = gicd_read_4(sc, GICD_CTRL);
    741   1.6  jmcneill 	int n;
    742   1.1  jmcneill 
    743   1.1  jmcneill 	KASSERT(CPU_IS_PRIMARY(curcpu()));
    744   1.1  jmcneill 
    745   1.7  jmcneill 	LIST_INIT(&sc->sc_lpi_callbacks);
    746   1.5  jmcneill 
    747   1.6  jmcneill 	for (n = 0; n < MAXCPUS; n++)
    748   1.6  jmcneill 		sc->sc_irouter[n] = UINT64_MAX;
    749   1.6  jmcneill 
    750  1.18  jmcneill 	sc->sc_priority_shift = 4;
    751  1.19  jmcneill 	sc->sc_pmr_shift = 4;
    752  1.18  jmcneill 
    753  1.18  jmcneill 	if ((gicd_ctrl & GICD_CTRL_DS) == 0) {
    754  1.19  jmcneill 		const int pmr_bits = gicv3_detect_pmr_bits(sc);
    755  1.19  jmcneill 		const int ipriority_bits = gicv3_detect_ipriority_bits(sc);
    756  1.19  jmcneill 
    757  1.19  jmcneill 		if (ipriority_bits != pmr_bits)
    758  1.19  jmcneill 			--sc->sc_priority_shift;
    759  1.19  jmcneill 
    760  1.19  jmcneill 		aprint_verbose_dev(sc->sc_dev, "%d pmr bits, %d ipriority bits\n",
    761  1.19  jmcneill 		    pmr_bits, ipriority_bits);
    762  1.19  jmcneill 	} else {
    763  1.19  jmcneill 		aprint_verbose_dev(sc->sc_dev, "security disabled\n");
    764  1.18  jmcneill 	}
    765  1.19  jmcneill 
    766  1.18  jmcneill 	aprint_verbose_dev(sc->sc_dev, "priority shift %d, pmr shift %d\n",
    767  1.18  jmcneill 	    sc->sc_priority_shift, sc->sc_pmr_shift);
    768  1.18  jmcneill 
    769   1.1  jmcneill 	sc->sc_pic.pic_ops = &gicv3_picops;
    770   1.1  jmcneill 	sc->sc_pic.pic_maxsources = GICD_TYPER_LINES(gicd_typer);
    771   1.1  jmcneill 	snprintf(sc->sc_pic.pic_name, sizeof(sc->sc_pic.pic_name), "gicv3");
    772   1.1  jmcneill #ifdef MULTIPROCESSOR
    773   1.1  jmcneill 	sc->sc_pic.pic_cpus = kcpuset_running;
    774   1.1  jmcneill #endif
    775   1.1  jmcneill 	pic_add(&sc->sc_pic, 0);
    776   1.1  jmcneill 
    777   1.5  jmcneill 	if ((gicd_typer & GICD_TYPER_LPIS) != 0) {
    778   1.5  jmcneill 		sc->sc_lpi.pic_ops = &gicv3_lpiops;
    779   1.5  jmcneill 		sc->sc_lpi.pic_maxsources = 8192;	/* Min. required by GICv3 spec */
    780   1.5  jmcneill 		snprintf(sc->sc_lpi.pic_name, sizeof(sc->sc_lpi.pic_name), "gicv3-lpi");
    781   1.5  jmcneill 		pic_add(&sc->sc_lpi, GIC_LPI_BASE);
    782   1.5  jmcneill 
    783   1.5  jmcneill 		gicv3_lpi_init(sc);
    784   1.5  jmcneill 	}
    785   1.5  jmcneill 
    786   1.1  jmcneill 	KASSERT(gicv3_softc == NULL);
    787   1.1  jmcneill 	gicv3_softc = sc;
    788   1.1  jmcneill 
    789   1.1  jmcneill 	for (int i = 0; i < sc->sc_bsh_r_count; i++) {
    790   1.1  jmcneill 		const uint64_t gicr_typer = gicr_read_8(sc, i, GICR_TYPER);
    791   1.1  jmcneill 		const u_int aff0 = __SHIFTOUT(gicr_typer, GICR_TYPER_Affinity_Value_Aff0);
    792   1.1  jmcneill 		const u_int aff1 = __SHIFTOUT(gicr_typer, GICR_TYPER_Affinity_Value_Aff1);
    793   1.1  jmcneill 		const u_int aff2 = __SHIFTOUT(gicr_typer, GICR_TYPER_Affinity_Value_Aff2);
    794   1.1  jmcneill 		const u_int aff3 = __SHIFTOUT(gicr_typer, GICR_TYPER_Affinity_Value_Aff3);
    795   1.1  jmcneill 
    796   1.1  jmcneill 		aprint_debug_dev(sc->sc_dev, "redist %d: cpu %d.%d.%d.%d\n",
    797   1.1  jmcneill 		    i, aff3, aff2, aff1, aff0);
    798   1.1  jmcneill 	}
    799   1.1  jmcneill 
    800   1.1  jmcneill 	gicv3_dist_enable(sc);
    801   1.1  jmcneill 
    802   1.1  jmcneill 	gicv3_cpu_init(&sc->sc_pic, curcpu());
    803   1.5  jmcneill 	if ((gicd_typer & GICD_TYPER_LPIS) != 0)
    804   1.5  jmcneill 		gicv3_lpi_cpu_init(&sc->sc_lpi, curcpu());
    805   1.1  jmcneill 
    806   1.1  jmcneill #ifdef __HAVE_PIC_FAST_SOFTINTS
    807  1.11  jmcneill 	intr_establish_xname(SOFTINT_BIO, IPL_SOFTBIO, IST_MPSAFE | IST_EDGE, pic_handle_softint, (void *)SOFTINT_BIO, "softint bio");
    808  1.11  jmcneill 	intr_establish_xname(SOFTINT_CLOCK, IPL_SOFTCLOCK, IST_MPSAFE | IST_EDGE, pic_handle_softint, (void *)SOFTINT_CLOCK, "softint clock");
    809  1.11  jmcneill 	intr_establish_xname(SOFTINT_NET, IPL_SOFTNET, IST_MPSAFE | IST_EDGE, pic_handle_softint, (void *)SOFTINT_NET, "softint net");
    810  1.11  jmcneill 	intr_establish_xname(SOFTINT_SERIAL, IPL_SOFTSERIAL, IST_MPSAFE | IST_EDGE, pic_handle_softint, (void *)SOFTINT_SERIAL, "softint serial");
    811   1.1  jmcneill #endif
    812   1.1  jmcneill 
    813   1.1  jmcneill #ifdef MULTIPROCESSOR
    814  1.11  jmcneill 	intr_establish_xname(IPI_AST, IPL_VM, IST_MPSAFE | IST_EDGE, pic_ipi_ast, (void *)-1, "IPI ast");
    815  1.11  jmcneill 	intr_establish_xname(IPI_XCALL, IPL_HIGH, IST_MPSAFE | IST_EDGE, pic_ipi_xcall, (void *)-1, "IPI xcall");
    816  1.11  jmcneill 	intr_establish_xname(IPI_GENERIC, IPL_HIGH, IST_MPSAFE | IST_EDGE, pic_ipi_generic, (void *)-1, "IPI generic");
    817  1.11  jmcneill 	intr_establish_xname(IPI_NOP, IPL_VM, IST_MPSAFE | IST_EDGE, pic_ipi_nop, (void *)-1, "IPI nop");
    818  1.11  jmcneill 	intr_establish_xname(IPI_SHOOTDOWN, IPL_SCHED, IST_MPSAFE | IST_EDGE, pic_ipi_shootdown, (void *)-1, "IPI shootdown");
    819   1.1  jmcneill #ifdef DDB
    820  1.11  jmcneill 	intr_establish_xname(IPI_DDB, IPL_HIGH, IST_MPSAFE | IST_EDGE, pic_ipi_ddb, NULL, "IPI ddb");
    821   1.1  jmcneill #endif
    822   1.1  jmcneill #ifdef __HAVE_PREEMPTION
    823  1.11  jmcneill 	intr_establish_xname(IPI_KPREEMPT, IPL_VM, IST_MPSAFE | IST_EDGE, pic_ipi_kpreempt, (void *)-1, "IPI kpreempt");
    824   1.1  jmcneill #endif
    825   1.1  jmcneill #endif
    826   1.1  jmcneill 
    827   1.1  jmcneill 	return 0;
    828   1.1  jmcneill }
    829