Home | History | Annotate | Line # | Download | only in cortex
gicv3.c revision 1.41
      1  1.41       ryo /* $NetBSD: gicv3.c,v 1.41 2021/02/09 17:44:01 ryo Exp $ */
      2   1.1  jmcneill 
      3   1.1  jmcneill /*-
      4   1.1  jmcneill  * Copyright (c) 2018 Jared McNeill <jmcneill (at) invisible.ca>
      5   1.1  jmcneill  * All rights reserved.
      6   1.1  jmcneill  *
      7   1.1  jmcneill  * Redistribution and use in source and binary forms, with or without
      8   1.1  jmcneill  * modification, are permitted provided that the following conditions
      9   1.1  jmcneill  * are met:
     10   1.1  jmcneill  * 1. Redistributions of source code must retain the above copyright
     11   1.1  jmcneill  *    notice, this list of conditions and the following disclaimer.
     12   1.1  jmcneill  * 2. Redistributions in binary form must reproduce the above copyright
     13   1.1  jmcneill  *    notice, this list of conditions and the following disclaimer in the
     14   1.1  jmcneill  *    documentation and/or other materials provided with the distribution.
     15   1.1  jmcneill  *
     16   1.1  jmcneill  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
     17   1.1  jmcneill  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
     18   1.1  jmcneill  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
     19   1.1  jmcneill  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
     20   1.1  jmcneill  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
     21   1.1  jmcneill  * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
     22   1.1  jmcneill  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
     23   1.1  jmcneill  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
     24   1.1  jmcneill  * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
     25   1.1  jmcneill  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
     26   1.1  jmcneill  * SUCH DAMAGE.
     27   1.1  jmcneill  */
     28   1.1  jmcneill 
     29   1.1  jmcneill #include "opt_multiprocessor.h"
     30   1.1  jmcneill 
     31   1.1  jmcneill #define	_INTR_PRIVATE
     32   1.1  jmcneill 
     33   1.1  jmcneill #include <sys/cdefs.h>
     34  1.41       ryo __KERNEL_RCSID(0, "$NetBSD: gicv3.c,v 1.41 2021/02/09 17:44:01 ryo Exp $");
     35   1.1  jmcneill 
     36   1.1  jmcneill #include <sys/param.h>
     37   1.1  jmcneill #include <sys/kernel.h>
     38   1.1  jmcneill #include <sys/bus.h>
     39   1.1  jmcneill #include <sys/device.h>
     40   1.1  jmcneill #include <sys/intr.h>
     41   1.1  jmcneill #include <sys/systm.h>
     42   1.1  jmcneill #include <sys/cpu.h>
     43  1.23  jmcneill #include <sys/vmem.h>
     44  1.39  jmcneill #include <sys/kmem.h>
     45  1.32  jmcneill #include <sys/atomic.h>
     46   1.1  jmcneill 
     47  1.20  jmcneill #include <machine/cpufunc.h>
     48  1.20  jmcneill 
     49   1.1  jmcneill #include <arm/locore.h>
     50   1.1  jmcneill #include <arm/armreg.h>
     51   1.1  jmcneill 
     52   1.1  jmcneill #include <arm/cortex/gicv3.h>
     53   1.1  jmcneill #include <arm/cortex/gic_reg.h>
     54   1.1  jmcneill 
     55   1.1  jmcneill #define	PICTOSOFTC(pic)	\
     56   1.1  jmcneill 	((void *)((uintptr_t)(pic) - offsetof(struct gicv3_softc, sc_pic)))
     57   1.5  jmcneill #define	LPITOSOFTC(lpi) \
     58   1.5  jmcneill 	((void *)((uintptr_t)(lpi) - offsetof(struct gicv3_softc, sc_lpi)))
     59   1.1  jmcneill 
     60  1.18  jmcneill #define	IPL_TO_PRIORITY(sc, ipl)	(((0xff - (ipl)) << (sc)->sc_priority_shift) & 0xff)
     61  1.18  jmcneill #define	IPL_TO_PMR(sc, ipl)		(((0xff - (ipl)) << (sc)->sc_pmr_shift) & 0xff)
     62  1.35  jmcneill 
     63  1.36  jmcneill #define	GIC_SUPPORTS_1OFN(sc)		(((sc)->sc_gicd_typer & GICD_TYPER_No1N) == 0)
     64  1.36  jmcneill 
     65  1.35  jmcneill #define	GIC_PRIO_SHIFT_NS		4
     66  1.35  jmcneill #define	GIC_PRIO_SHIFT_S		3
     67   1.1  jmcneill 
     68   1.1  jmcneill static struct gicv3_softc *gicv3_softc;
     69   1.1  jmcneill 
     70   1.1  jmcneill static inline uint32_t
     71   1.1  jmcneill gicd_read_4(struct gicv3_softc *sc, bus_size_t reg)
     72   1.1  jmcneill {
     73   1.1  jmcneill 	return bus_space_read_4(sc->sc_bst, sc->sc_bsh_d, reg);
     74   1.1  jmcneill }
     75   1.1  jmcneill 
     76   1.1  jmcneill static inline void
     77   1.1  jmcneill gicd_write_4(struct gicv3_softc *sc, bus_size_t reg, uint32_t val)
     78   1.1  jmcneill {
     79   1.1  jmcneill 	bus_space_write_4(sc->sc_bst, sc->sc_bsh_d, reg, val);
     80   1.1  jmcneill }
     81   1.1  jmcneill 
     82  1.41       ryo #ifdef MULTIPROCESSOR
     83   1.6  jmcneill static inline uint64_t
     84   1.6  jmcneill gicd_read_8(struct gicv3_softc *sc, bus_size_t reg)
     85   1.6  jmcneill {
     86   1.6  jmcneill 	return bus_space_read_8(sc->sc_bst, sc->sc_bsh_d, reg);
     87   1.6  jmcneill }
     88  1.41       ryo #endif
     89   1.6  jmcneill 
     90   1.1  jmcneill static inline void
     91   1.1  jmcneill gicd_write_8(struct gicv3_softc *sc, bus_size_t reg, uint64_t val)
     92   1.1  jmcneill {
     93   1.1  jmcneill 	bus_space_write_8(sc->sc_bst, sc->sc_bsh_d, reg, val);
     94   1.1  jmcneill }
     95   1.1  jmcneill 
     96   1.1  jmcneill static inline uint32_t
     97   1.1  jmcneill gicr_read_4(struct gicv3_softc *sc, u_int index, bus_size_t reg)
     98   1.1  jmcneill {
     99   1.1  jmcneill 	KASSERT(index < sc->sc_bsh_r_count);
    100   1.1  jmcneill 	return bus_space_read_4(sc->sc_bst, sc->sc_bsh_r[index], reg);
    101   1.1  jmcneill }
    102   1.1  jmcneill 
    103   1.1  jmcneill static inline void
    104   1.1  jmcneill gicr_write_4(struct gicv3_softc *sc, u_int index, bus_size_t reg, uint32_t val)
    105   1.1  jmcneill {
    106   1.1  jmcneill 	KASSERT(index < sc->sc_bsh_r_count);
    107   1.1  jmcneill 	bus_space_write_4(sc->sc_bst, sc->sc_bsh_r[index], reg, val);
    108   1.1  jmcneill }
    109   1.1  jmcneill 
    110   1.1  jmcneill static inline uint64_t
    111   1.1  jmcneill gicr_read_8(struct gicv3_softc *sc, u_int index, bus_size_t reg)
    112   1.1  jmcneill {
    113   1.1  jmcneill 	KASSERT(index < sc->sc_bsh_r_count);
    114   1.1  jmcneill 	return bus_space_read_8(sc->sc_bst, sc->sc_bsh_r[index], reg);
    115   1.1  jmcneill }
    116   1.1  jmcneill 
    117   1.1  jmcneill static inline void
    118   1.1  jmcneill gicr_write_8(struct gicv3_softc *sc, u_int index, bus_size_t reg, uint64_t val)
    119   1.1  jmcneill {
    120   1.1  jmcneill 	KASSERT(index < sc->sc_bsh_r_count);
    121   1.1  jmcneill 	bus_space_write_8(sc->sc_bst, sc->sc_bsh_r[index], reg, val);
    122   1.1  jmcneill }
    123   1.1  jmcneill 
    124   1.1  jmcneill static void
    125   1.1  jmcneill gicv3_unblock_irqs(struct pic_softc *pic, size_t irqbase, uint32_t mask)
    126   1.1  jmcneill {
    127   1.1  jmcneill 	struct gicv3_softc * const sc = PICTOSOFTC(pic);
    128   1.1  jmcneill 	struct cpu_info * const ci = curcpu();
    129   1.1  jmcneill 	const u_int group = irqbase / 32;
    130   1.1  jmcneill 
    131   1.1  jmcneill 	if (group == 0) {
    132  1.32  jmcneill 		atomic_or_32(&sc->sc_enabled_sgippi, mask);
    133   1.1  jmcneill 		gicr_write_4(sc, ci->ci_gic_redist, GICR_ISENABLER0, mask);
    134   1.5  jmcneill 		while (gicr_read_4(sc, ci->ci_gic_redist, GICR_CTLR) & GICR_CTLR_RWP)
    135   1.1  jmcneill 			;
    136   1.1  jmcneill 	} else {
    137   1.1  jmcneill 		gicd_write_4(sc, GICD_ISENABLERn(group), mask);
    138   1.1  jmcneill 		while (gicd_read_4(sc, GICD_CTRL) & GICD_CTRL_RWP)
    139   1.1  jmcneill 			;
    140   1.1  jmcneill 	}
    141   1.1  jmcneill }
    142   1.1  jmcneill 
    143   1.1  jmcneill static void
    144   1.1  jmcneill gicv3_block_irqs(struct pic_softc *pic, size_t irqbase, uint32_t mask)
    145   1.1  jmcneill {
    146   1.1  jmcneill 	struct gicv3_softc * const sc = PICTOSOFTC(pic);
    147   1.1  jmcneill 	struct cpu_info * const ci = curcpu();
    148   1.1  jmcneill 	const u_int group = irqbase / 32;
    149   1.1  jmcneill 
    150   1.1  jmcneill 	if (group == 0) {
    151  1.32  jmcneill 		atomic_and_32(&sc->sc_enabled_sgippi, ~mask);
    152   1.1  jmcneill 		gicr_write_4(sc, ci->ci_gic_redist, GICR_ICENABLER0, mask);
    153   1.5  jmcneill 		while (gicr_read_4(sc, ci->ci_gic_redist, GICR_CTLR) & GICR_CTLR_RWP)
    154   1.1  jmcneill 			;
    155   1.1  jmcneill 	} else {
    156   1.1  jmcneill 		gicd_write_4(sc, GICD_ICENABLERn(group), mask);
    157   1.1  jmcneill 		while (gicd_read_4(sc, GICD_CTRL) & GICD_CTRL_RWP)
    158   1.1  jmcneill 			;
    159   1.1  jmcneill 	}
    160   1.1  jmcneill }
    161   1.1  jmcneill 
    162   1.1  jmcneill static void
    163   1.1  jmcneill gicv3_establish_irq(struct pic_softc *pic, struct intrsource *is)
    164   1.1  jmcneill {
    165   1.1  jmcneill 	struct gicv3_softc * const sc = PICTOSOFTC(pic);
    166   1.1  jmcneill 	const u_int group = is->is_irq / 32;
    167   1.1  jmcneill 	uint32_t ipriority, icfg;
    168   1.1  jmcneill 	uint64_t irouter;
    169   1.1  jmcneill 	u_int n;
    170   1.1  jmcneill 
    171  1.18  jmcneill 	const u_int ipriority_val = IPL_TO_PRIORITY(sc, is->is_ipl);
    172   1.1  jmcneill 	const u_int ipriority_shift = (is->is_irq & 0x3) * 8;
    173   1.1  jmcneill 	const u_int icfg_shift = (is->is_irq & 0xf) * 2;
    174   1.1  jmcneill 
    175   1.1  jmcneill 	if (group == 0) {
    176   1.1  jmcneill 		/* SGIs and PPIs are always MP-safe */
    177   1.1  jmcneill 		is->is_mpsafe = true;
    178   1.1  jmcneill 
    179   1.1  jmcneill 		/* Update interrupt configuration and priority on all redistributors */
    180   1.1  jmcneill 		for (n = 0; n < sc->sc_bsh_r_count; n++) {
    181   1.1  jmcneill 			icfg = gicr_read_4(sc, n, GICR_ICFGRn(is->is_irq / 16));
    182   1.1  jmcneill 			if (is->is_type == IST_LEVEL)
    183   1.1  jmcneill 				icfg &= ~(0x2 << icfg_shift);
    184   1.1  jmcneill 			if (is->is_type == IST_EDGE)
    185   1.1  jmcneill 				icfg |= (0x2 << icfg_shift);
    186   1.1  jmcneill 			gicr_write_4(sc, n, GICR_ICFGRn(is->is_irq / 16), icfg);
    187   1.1  jmcneill 
    188   1.1  jmcneill 			ipriority = gicr_read_4(sc, n, GICR_IPRIORITYRn(is->is_irq / 4));
    189  1.25  jmcneill 			ipriority &= ~(0xffU << ipriority_shift);
    190   1.2  jmcneill 			ipriority |= (ipriority_val << ipriority_shift);
    191   1.1  jmcneill 			gicr_write_4(sc, n, GICR_IPRIORITYRn(is->is_irq / 4), ipriority);
    192   1.1  jmcneill 		}
    193   1.1  jmcneill 	} else {
    194  1.36  jmcneill 		/*
    195  1.36  jmcneill 		 * If 1 of N SPI routing is supported, route MP-safe interrupts to all
    196  1.36  jmcneill 		 * participating PEs. Otherwise, just route to the primary PE.
    197  1.36  jmcneill 		 */
    198  1.36  jmcneill 		if (is->is_mpsafe && GIC_SUPPORTS_1OFN(sc)) {
    199   1.1  jmcneill 			irouter = GICD_IROUTER_Interrupt_Routing_mode;
    200   1.1  jmcneill 		} else {
    201   1.6  jmcneill 			irouter = sc->sc_irouter[0];
    202   1.1  jmcneill 		}
    203   1.1  jmcneill 		gicd_write_8(sc, GICD_IROUTER(is->is_irq), irouter);
    204   1.1  jmcneill 
    205   1.1  jmcneill 		/* Update interrupt configuration */
    206   1.1  jmcneill 		icfg = gicd_read_4(sc, GICD_ICFGRn(is->is_irq / 16));
    207   1.1  jmcneill 		if (is->is_type == IST_LEVEL)
    208   1.1  jmcneill 			icfg &= ~(0x2 << icfg_shift);
    209   1.1  jmcneill 		if (is->is_type == IST_EDGE)
    210   1.1  jmcneill 			icfg |= (0x2 << icfg_shift);
    211   1.1  jmcneill 		gicd_write_4(sc, GICD_ICFGRn(is->is_irq / 16), icfg);
    212   1.1  jmcneill 
    213   1.1  jmcneill 		/* Update interrupt priority */
    214   1.1  jmcneill 		ipriority = gicd_read_4(sc, GICD_IPRIORITYRn(is->is_irq / 4));
    215  1.25  jmcneill 		ipriority &= ~(0xffU << ipriority_shift);
    216   1.2  jmcneill 		ipriority |= (ipriority_val << ipriority_shift);
    217   1.1  jmcneill 		gicd_write_4(sc, GICD_IPRIORITYRn(is->is_irq / 4), ipriority);
    218   1.1  jmcneill 	}
    219   1.1  jmcneill }
    220   1.1  jmcneill 
    221   1.1  jmcneill static void
    222   1.1  jmcneill gicv3_set_priority(struct pic_softc *pic, int ipl)
    223   1.1  jmcneill {
    224  1.18  jmcneill 	struct gicv3_softc * const sc = PICTOSOFTC(pic);
    225  1.40  jmcneill 	const uint8_t curpmr = icc_pmr_read();
    226  1.40  jmcneill 	const uint8_t newpmr = IPL_TO_PMR(sc, ipl);
    227  1.18  jmcneill 
    228  1.40  jmcneill 	if (newpmr > curpmr) {
    229  1.40  jmcneill 		/* Lowering priority mask */
    230  1.40  jmcneill 		icc_pmr_write(newpmr);
    231  1.40  jmcneill 	}
    232   1.1  jmcneill }
    233   1.1  jmcneill 
    234   1.1  jmcneill static void
    235   1.1  jmcneill gicv3_dist_enable(struct gicv3_softc *sc)
    236   1.1  jmcneill {
    237   1.1  jmcneill 	uint32_t gicd_ctrl;
    238   1.1  jmcneill 	u_int n;
    239   1.1  jmcneill 
    240   1.1  jmcneill 	/* Disable the distributor */
    241  1.35  jmcneill 	gicd_ctrl = gicd_read_4(sc, GICD_CTRL);
    242  1.35  jmcneill 	gicd_ctrl &= ~(GICD_CTRL_EnableGrp1A | GICD_CTRL_ARE_NS);
    243  1.35  jmcneill 	gicd_write_4(sc, GICD_CTRL, gicd_ctrl);
    244   1.1  jmcneill 
    245   1.1  jmcneill 	/* Wait for register write to complete */
    246   1.1  jmcneill 	while (gicd_read_4(sc, GICD_CTRL) & GICD_CTRL_RWP)
    247   1.1  jmcneill 		;
    248   1.1  jmcneill 
    249   1.1  jmcneill 	/* Clear all INTID enable bits */
    250   1.1  jmcneill 	for (n = 32; n < sc->sc_pic.pic_maxsources; n += 32)
    251   1.1  jmcneill 		gicd_write_4(sc, GICD_ICENABLERn(n / 32), ~0);
    252   1.1  jmcneill 
    253   1.1  jmcneill 	/* Set default priorities to lowest */
    254   1.1  jmcneill 	for (n = 32; n < sc->sc_pic.pic_maxsources; n += 4)
    255   1.1  jmcneill 		gicd_write_4(sc, GICD_IPRIORITYRn(n / 4), ~0);
    256   1.1  jmcneill 
    257   1.1  jmcneill 	/* Set all interrupts to G1NS */
    258   1.1  jmcneill 	for (n = 32; n < sc->sc_pic.pic_maxsources; n += 32) {
    259   1.1  jmcneill 		gicd_write_4(sc, GICD_IGROUPRn(n / 32), ~0);
    260   1.1  jmcneill 		gicd_write_4(sc, GICD_IGRPMODRn(n / 32), 0);
    261   1.1  jmcneill 	}
    262   1.1  jmcneill 
    263   1.1  jmcneill 	/* Set all interrupts level-sensitive by default */
    264   1.1  jmcneill 	for (n = 32; n < sc->sc_pic.pic_maxsources; n += 16)
    265   1.1  jmcneill 		gicd_write_4(sc, GICD_ICFGRn(n / 16), 0);
    266   1.1  jmcneill 
    267   1.1  jmcneill 	/* Wait for register writes to complete */
    268   1.1  jmcneill 	while (gicd_read_4(sc, GICD_CTRL) & GICD_CTRL_RWP)
    269   1.1  jmcneill 		;
    270   1.1  jmcneill 
    271   1.1  jmcneill 	/* Enable Affinity routing and G1NS interrupts */
    272  1.19  jmcneill 	gicd_ctrl = GICD_CTRL_EnableGrp1A | GICD_CTRL_ARE_NS;
    273   1.1  jmcneill 	gicd_write_4(sc, GICD_CTRL, gicd_ctrl);
    274   1.1  jmcneill }
    275   1.1  jmcneill 
    276   1.1  jmcneill static void
    277   1.1  jmcneill gicv3_redist_enable(struct gicv3_softc *sc, struct cpu_info *ci)
    278   1.1  jmcneill {
    279   1.1  jmcneill 	uint32_t icfg;
    280   1.1  jmcneill 	u_int n, o;
    281   1.1  jmcneill 
    282   1.1  jmcneill 	/* Clear INTID enable bits */
    283   1.1  jmcneill 	gicr_write_4(sc, ci->ci_gic_redist, GICR_ICENABLER0, ~0);
    284   1.1  jmcneill 
    285   1.1  jmcneill 	/* Wait for register write to complete */
    286   1.5  jmcneill 	while (gicr_read_4(sc, ci->ci_gic_redist, GICR_CTLR) & GICR_CTLR_RWP)
    287   1.1  jmcneill 		;
    288   1.1  jmcneill 
    289   1.1  jmcneill 	/* Set default priorities */
    290   1.1  jmcneill 	for (n = 0; n < 32; n += 4) {
    291   1.1  jmcneill 		uint32_t priority = 0;
    292   1.1  jmcneill 		size_t byte_shift = 0;
    293   1.1  jmcneill 		for (o = 0; o < 4; o++, byte_shift += 8) {
    294   1.1  jmcneill 			struct intrsource * const is = sc->sc_pic.pic_sources[n + o];
    295   1.1  jmcneill 			if (is == NULL)
    296  1.25  jmcneill 				priority |= (0xffU << byte_shift);
    297   1.2  jmcneill 			else {
    298  1.18  jmcneill 				const u_int ipriority_val = IPL_TO_PRIORITY(sc, is->is_ipl);
    299   1.2  jmcneill 				priority |= ipriority_val << byte_shift;
    300   1.2  jmcneill 			}
    301   1.1  jmcneill 		}
    302   1.1  jmcneill 		gicr_write_4(sc, ci->ci_gic_redist, GICR_IPRIORITYRn(n / 4), priority);
    303   1.1  jmcneill 	}
    304   1.1  jmcneill 
    305   1.1  jmcneill 	/* Set all interrupts to G1NS */
    306   1.1  jmcneill 	gicr_write_4(sc, ci->ci_gic_redist, GICR_IGROUPR0, ~0);
    307   1.1  jmcneill 	gicr_write_4(sc, ci->ci_gic_redist, GICR_IGRPMODR0, 0);
    308   1.1  jmcneill 
    309   1.1  jmcneill 	/* Restore PPI configs */
    310   1.1  jmcneill 	for (n = 0, icfg = 0; n < 16; n++) {
    311   1.1  jmcneill 		struct intrsource * const is = sc->sc_pic.pic_sources[16 + n];
    312   1.1  jmcneill 		if (is != NULL && is->is_type == IST_EDGE)
    313   1.1  jmcneill 			icfg |= (0x2 << (n * 2));
    314   1.1  jmcneill 	}
    315   1.1  jmcneill 	gicr_write_4(sc, ci->ci_gic_redist, GICR_ICFGRn(1), icfg);
    316   1.1  jmcneill 
    317   1.1  jmcneill 	/* Restore current enable bits */
    318   1.1  jmcneill 	gicr_write_4(sc, ci->ci_gic_redist, GICR_ISENABLER0, sc->sc_enabled_sgippi);
    319   1.1  jmcneill 
    320   1.1  jmcneill 	/* Wait for register write to complete */
    321   1.5  jmcneill 	while (gicr_read_4(sc, ci->ci_gic_redist, GICR_CTLR) & GICR_CTLR_RWP)
    322   1.1  jmcneill 		;
    323   1.1  jmcneill }
    324   1.1  jmcneill 
    325   1.1  jmcneill static uint64_t
    326   1.1  jmcneill gicv3_cpu_identity(void)
    327   1.1  jmcneill {
    328   1.1  jmcneill 	u_int aff3, aff2, aff1, aff0;
    329   1.1  jmcneill 
    330  1.18  jmcneill 	const register_t mpidr = cpu_mpidr_aff_read();
    331   1.1  jmcneill 	aff0 = __SHIFTOUT(mpidr, MPIDR_AFF0);
    332   1.1  jmcneill 	aff1 = __SHIFTOUT(mpidr, MPIDR_AFF1);
    333   1.1  jmcneill 	aff2 = __SHIFTOUT(mpidr, MPIDR_AFF2);
    334   1.1  jmcneill 	aff3 = __SHIFTOUT(mpidr, MPIDR_AFF3);
    335   1.1  jmcneill 
    336   1.1  jmcneill 	return __SHIFTIN(aff0, GICR_TYPER_Affinity_Value_Aff0) |
    337   1.1  jmcneill 	       __SHIFTIN(aff1, GICR_TYPER_Affinity_Value_Aff1) |
    338   1.1  jmcneill 	       __SHIFTIN(aff2, GICR_TYPER_Affinity_Value_Aff2) |
    339   1.1  jmcneill 	       __SHIFTIN(aff3, GICR_TYPER_Affinity_Value_Aff3);
    340   1.1  jmcneill }
    341   1.1  jmcneill 
    342   1.1  jmcneill static u_int
    343   1.1  jmcneill gicv3_find_redist(struct gicv3_softc *sc)
    344   1.1  jmcneill {
    345   1.1  jmcneill 	uint64_t gicr_typer;
    346   1.1  jmcneill 	u_int n;
    347   1.1  jmcneill 
    348   1.1  jmcneill 	const uint64_t cpu_identity = gicv3_cpu_identity();
    349   1.1  jmcneill 
    350   1.1  jmcneill 	for (n = 0; n < sc->sc_bsh_r_count; n++) {
    351   1.1  jmcneill 		gicr_typer = gicr_read_8(sc, n, GICR_TYPER);
    352   1.1  jmcneill 		if ((gicr_typer & GICR_TYPER_Affinity_Value) == cpu_identity)
    353   1.1  jmcneill 			return n;
    354   1.1  jmcneill 	}
    355   1.1  jmcneill 
    356   1.1  jmcneill 	const u_int aff0 = __SHIFTOUT(cpu_identity, GICR_TYPER_Affinity_Value_Aff0);
    357   1.1  jmcneill 	const u_int aff1 = __SHIFTOUT(cpu_identity, GICR_TYPER_Affinity_Value_Aff1);
    358   1.1  jmcneill 	const u_int aff2 = __SHIFTOUT(cpu_identity, GICR_TYPER_Affinity_Value_Aff2);
    359   1.1  jmcneill 	const u_int aff3 = __SHIFTOUT(cpu_identity, GICR_TYPER_Affinity_Value_Aff3);
    360   1.1  jmcneill 
    361   1.1  jmcneill 	panic("%s: could not find GICv3 redistributor for cpu %d.%d.%d.%d",
    362   1.1  jmcneill 	    cpu_name(curcpu()), aff3, aff2, aff1, aff0);
    363   1.1  jmcneill }
    364   1.1  jmcneill 
    365   1.1  jmcneill static uint64_t
    366   1.1  jmcneill gicv3_sgir(struct gicv3_softc *sc)
    367   1.1  jmcneill {
    368  1.22     skrll 	const uint64_t cpu_identity = gicv3_cpu_identity();
    369   1.1  jmcneill 
    370   1.1  jmcneill 	const u_int aff0 = __SHIFTOUT(cpu_identity, GICR_TYPER_Affinity_Value_Aff0);
    371   1.1  jmcneill 	const u_int aff1 = __SHIFTOUT(cpu_identity, GICR_TYPER_Affinity_Value_Aff1);
    372   1.1  jmcneill 	const u_int aff2 = __SHIFTOUT(cpu_identity, GICR_TYPER_Affinity_Value_Aff2);
    373   1.1  jmcneill 	const u_int aff3 = __SHIFTOUT(cpu_identity, GICR_TYPER_Affinity_Value_Aff3);
    374   1.1  jmcneill 
    375   1.1  jmcneill 	return __SHIFTIN(__BIT(aff0), ICC_SGIR_EL1_TargetList) |
    376   1.1  jmcneill 	       __SHIFTIN(aff1, ICC_SGIR_EL1_Aff1) |
    377   1.1  jmcneill 	       __SHIFTIN(aff2, ICC_SGIR_EL1_Aff2) |
    378  1.22     skrll 	       __SHIFTIN(aff3, ICC_SGIR_EL1_Aff3);
    379   1.1  jmcneill }
    380   1.1  jmcneill 
    381   1.1  jmcneill static void
    382   1.1  jmcneill gicv3_cpu_init(struct pic_softc *pic, struct cpu_info *ci)
    383   1.1  jmcneill {
    384   1.1  jmcneill 	struct gicv3_softc * const sc = PICTOSOFTC(pic);
    385   1.1  jmcneill 	uint32_t icc_sre, icc_ctlr, gicr_waker;
    386   1.1  jmcneill 
    387  1.33  jmcneill 	evcnt_attach_dynamic(&ci->ci_intr_preempt, EVCNT_TYPE_MISC, NULL,
    388  1.33  jmcneill 	    ci->ci_cpuname, "intr preempt");
    389  1.33  jmcneill 
    390   1.1  jmcneill 	ci->ci_gic_redist = gicv3_find_redist(sc);
    391   1.1  jmcneill 	ci->ci_gic_sgir = gicv3_sgir(sc);
    392   1.1  jmcneill 
    393   1.6  jmcneill 	/* Store route to CPU for SPIs */
    394   1.6  jmcneill 	const uint64_t cpu_identity = gicv3_cpu_identity();
    395   1.6  jmcneill 	const u_int aff0 = __SHIFTOUT(cpu_identity, GICR_TYPER_Affinity_Value_Aff0);
    396   1.6  jmcneill 	const u_int aff1 = __SHIFTOUT(cpu_identity, GICR_TYPER_Affinity_Value_Aff1);
    397   1.6  jmcneill 	const u_int aff2 = __SHIFTOUT(cpu_identity, GICR_TYPER_Affinity_Value_Aff2);
    398   1.6  jmcneill 	const u_int aff3 = __SHIFTOUT(cpu_identity, GICR_TYPER_Affinity_Value_Aff3);
    399   1.6  jmcneill 	sc->sc_irouter[cpu_index(ci)] =
    400   1.6  jmcneill 	    __SHIFTIN(aff0, GICD_IROUTER_Aff0) |
    401   1.6  jmcneill 	    __SHIFTIN(aff1, GICD_IROUTER_Aff1) |
    402   1.6  jmcneill 	    __SHIFTIN(aff2, GICD_IROUTER_Aff2) |
    403   1.6  jmcneill 	    __SHIFTIN(aff3, GICD_IROUTER_Aff3);
    404   1.1  jmcneill 
    405   1.1  jmcneill 	/* Enable System register access and disable IRQ/FIQ bypass */
    406   1.1  jmcneill 	icc_sre = ICC_SRE_EL1_SRE | ICC_SRE_EL1_DFB | ICC_SRE_EL1_DIB;
    407   1.1  jmcneill 	icc_sre_write(icc_sre);
    408   1.1  jmcneill 
    409   1.1  jmcneill 	/* Mark the connected PE as being awake */
    410   1.1  jmcneill 	gicr_waker = gicr_read_4(sc, ci->ci_gic_redist, GICR_WAKER);
    411   1.1  jmcneill 	gicr_waker &= ~GICR_WAKER_ProcessorSleep;
    412   1.1  jmcneill 	gicr_write_4(sc, ci->ci_gic_redist, GICR_WAKER, gicr_waker);
    413   1.1  jmcneill 	while (gicr_read_4(sc, ci->ci_gic_redist, GICR_WAKER) & GICR_WAKER_ChildrenAsleep)
    414   1.1  jmcneill 		;
    415   1.1  jmcneill 
    416   1.1  jmcneill 	/* Set initial priority mask */
    417  1.40  jmcneill 	icc_pmr_write(IPL_TO_PMR(sc, IPL_HIGH));
    418   1.1  jmcneill 
    419  1.10  jmcneill 	/* Set the binary point field to the minimum value */
    420  1.10  jmcneill 	icc_bpr1_write(0);
    421   1.1  jmcneill 
    422   1.1  jmcneill 	/* Enable group 1 interrupt signaling */
    423   1.1  jmcneill 	icc_igrpen1_write(ICC_IGRPEN_EL1_Enable);
    424   1.1  jmcneill 
    425   1.1  jmcneill 	/* Set EOI mode */
    426   1.1  jmcneill 	icc_ctlr = icc_ctlr_read();
    427   1.1  jmcneill 	icc_ctlr &= ~ICC_CTLR_EL1_EOImode;
    428   1.1  jmcneill 	icc_ctlr_write(icc_ctlr);
    429   1.1  jmcneill 
    430   1.1  jmcneill 	/* Enable redistributor */
    431   1.1  jmcneill 	gicv3_redist_enable(sc, ci);
    432   1.1  jmcneill 
    433   1.1  jmcneill 	/* Allow IRQ exceptions */
    434  1.40  jmcneill 	ENABLE_INTERRUPT();
    435   1.1  jmcneill }
    436   1.1  jmcneill 
    437   1.1  jmcneill #ifdef MULTIPROCESSOR
    438   1.1  jmcneill static void
    439   1.1  jmcneill gicv3_ipi_send(struct pic_softc *pic, const kcpuset_t *kcp, u_long ipi)
    440   1.1  jmcneill {
    441   1.1  jmcneill 	struct cpu_info *ci;
    442  1.27  jmcneill 	uint64_t sgir;
    443   1.1  jmcneill 
    444  1.27  jmcneill 	sgir = __SHIFTIN(ipi, ICC_SGIR_EL1_INTID);
    445   1.1  jmcneill 	if (kcp == NULL) {
    446   1.1  jmcneill 		/* Interrupts routed to all PEs, excluding "self" */
    447   1.1  jmcneill 		if (ncpu == 1)
    448   1.1  jmcneill 			return;
    449  1.27  jmcneill 		sgir |= ICC_SGIR_EL1_IRM;
    450   1.1  jmcneill 	} else {
    451  1.27  jmcneill 		/* Interrupt to exactly one PE */
    452  1.27  jmcneill 		ci = cpu_lookup(kcpuset_ffs(kcp) - 1);
    453  1.27  jmcneill 		if (ci == curcpu())
    454  1.27  jmcneill 			return;
    455  1.27  jmcneill 		sgir |= ci->ci_gic_sgir;
    456   1.1  jmcneill 	}
    457  1.27  jmcneill 	icc_sgi1r_write(sgir);
    458  1.30  jmcneill 	isb();
    459   1.1  jmcneill }
    460   1.6  jmcneill 
    461   1.6  jmcneill static void
    462   1.6  jmcneill gicv3_get_affinity(struct pic_softc *pic, size_t irq, kcpuset_t *affinity)
    463   1.6  jmcneill {
    464   1.6  jmcneill 	struct gicv3_softc * const sc = PICTOSOFTC(pic);
    465   1.6  jmcneill 	const size_t group = irq / 32;
    466   1.6  jmcneill 	int n;
    467   1.6  jmcneill 
    468   1.6  jmcneill 	kcpuset_zero(affinity);
    469   1.6  jmcneill 	if (group == 0) {
    470   1.6  jmcneill 		/* All CPUs are targets for group 0 (SGI/PPI) */
    471   1.6  jmcneill 		for (n = 0; n < ncpu; n++) {
    472   1.6  jmcneill 			if (sc->sc_irouter[n] != UINT64_MAX)
    473   1.6  jmcneill 				kcpuset_set(affinity, n);
    474   1.6  jmcneill 		}
    475   1.6  jmcneill 	} else {
    476   1.6  jmcneill 		/* Find distributor targets (SPI) */
    477   1.6  jmcneill 		const uint64_t irouter = gicd_read_8(sc, GICD_IROUTER(irq));
    478   1.6  jmcneill 		for (n = 0; n < ncpu; n++) {
    479   1.6  jmcneill 			if (irouter == GICD_IROUTER_Interrupt_Routing_mode ||
    480   1.6  jmcneill 			    irouter == sc->sc_irouter[n])
    481   1.6  jmcneill 				kcpuset_set(affinity, n);
    482   1.6  jmcneill 		}
    483   1.6  jmcneill 	}
    484   1.6  jmcneill }
    485   1.6  jmcneill 
    486   1.6  jmcneill static int
    487   1.6  jmcneill gicv3_set_affinity(struct pic_softc *pic, size_t irq, const kcpuset_t *affinity)
    488   1.6  jmcneill {
    489   1.6  jmcneill 	struct gicv3_softc * const sc = PICTOSOFTC(pic);
    490   1.6  jmcneill 	const size_t group = irq / 32;
    491   1.6  jmcneill 	uint64_t irouter;
    492   1.6  jmcneill 
    493   1.6  jmcneill 	if (group == 0)
    494   1.6  jmcneill 		return EINVAL;
    495   1.6  jmcneill 
    496   1.6  jmcneill 	const int set = kcpuset_countset(affinity);
    497  1.36  jmcneill 	if (set == 1) {
    498  1.36  jmcneill 		irouter = sc->sc_irouter[kcpuset_ffs(affinity) - 1];
    499  1.36  jmcneill 	} else if (set == ncpu && GIC_SUPPORTS_1OFN(sc)) {
    500   1.6  jmcneill 		irouter = GICD_IROUTER_Interrupt_Routing_mode;
    501  1.36  jmcneill 	} else {
    502   1.6  jmcneill 		return EINVAL;
    503  1.36  jmcneill 	}
    504   1.6  jmcneill 
    505   1.6  jmcneill 	gicd_write_8(sc, GICD_IROUTER(irq), irouter);
    506   1.6  jmcneill 
    507   1.6  jmcneill 	return 0;
    508   1.6  jmcneill }
    509   1.1  jmcneill #endif
    510   1.1  jmcneill 
    511   1.1  jmcneill static const struct pic_ops gicv3_picops = {
    512   1.1  jmcneill 	.pic_unblock_irqs = gicv3_unblock_irqs,
    513   1.1  jmcneill 	.pic_block_irqs = gicv3_block_irqs,
    514   1.1  jmcneill 	.pic_establish_irq = gicv3_establish_irq,
    515   1.1  jmcneill 	.pic_set_priority = gicv3_set_priority,
    516   1.1  jmcneill #ifdef MULTIPROCESSOR
    517   1.1  jmcneill 	.pic_cpu_init = gicv3_cpu_init,
    518   1.1  jmcneill 	.pic_ipi_send = gicv3_ipi_send,
    519   1.6  jmcneill 	.pic_get_affinity = gicv3_get_affinity,
    520   1.6  jmcneill 	.pic_set_affinity = gicv3_set_affinity,
    521   1.1  jmcneill #endif
    522   1.1  jmcneill };
    523   1.1  jmcneill 
    524   1.5  jmcneill static void
    525  1.38  jmcneill gicv3_dcache_wb_range(vaddr_t va, vsize_t len)
    526  1.38  jmcneill {
    527  1.38  jmcneill 	cpu_dcache_wb_range(va, len);
    528  1.38  jmcneill 	dsb(sy);
    529  1.38  jmcneill }
    530  1.38  jmcneill 
    531  1.38  jmcneill static void
    532   1.5  jmcneill gicv3_lpi_unblock_irqs(struct pic_softc *pic, size_t irqbase, uint32_t mask)
    533   1.5  jmcneill {
    534   1.5  jmcneill 	struct gicv3_softc * const sc = LPITOSOFTC(pic);
    535   1.5  jmcneill 	int bit;
    536   1.5  jmcneill 
    537   1.5  jmcneill 	while ((bit = ffs(mask)) != 0) {
    538   1.5  jmcneill 		sc->sc_lpiconf.base[irqbase + bit - 1] |= GIC_LPICONF_Enable;
    539  1.20  jmcneill 		if (sc->sc_lpiconf_flush)
    540  1.38  jmcneill 			gicv3_dcache_wb_range((vaddr_t)&sc->sc_lpiconf.base[irqbase + bit - 1], 1);
    541   1.5  jmcneill 		mask &= ~__BIT(bit - 1);
    542   1.5  jmcneill 	}
    543   1.5  jmcneill 
    544  1.20  jmcneill 	if (!sc->sc_lpiconf_flush)
    545  1.26     skrll 		dsb(ishst);
    546   1.5  jmcneill }
    547   1.5  jmcneill 
    548   1.5  jmcneill static void
    549   1.5  jmcneill gicv3_lpi_block_irqs(struct pic_softc *pic, size_t irqbase, uint32_t mask)
    550   1.5  jmcneill {
    551   1.5  jmcneill 	struct gicv3_softc * const sc = LPITOSOFTC(pic);
    552   1.5  jmcneill 	int bit;
    553   1.5  jmcneill 
    554   1.5  jmcneill 	while ((bit = ffs(mask)) != 0) {
    555  1.13  jmcneill 		sc->sc_lpiconf.base[irqbase + bit - 1] &= ~GIC_LPICONF_Enable;
    556  1.20  jmcneill 		if (sc->sc_lpiconf_flush)
    557  1.38  jmcneill 			gicv3_dcache_wb_range((vaddr_t)&sc->sc_lpiconf.base[irqbase + bit - 1], 1);
    558   1.5  jmcneill 		mask &= ~__BIT(bit - 1);
    559   1.5  jmcneill 	}
    560   1.5  jmcneill 
    561  1.20  jmcneill 	if (!sc->sc_lpiconf_flush)
    562  1.26     skrll 		dsb(ishst);
    563   1.5  jmcneill }
    564   1.5  jmcneill 
    565   1.5  jmcneill static void
    566   1.5  jmcneill gicv3_lpi_establish_irq(struct pic_softc *pic, struct intrsource *is)
    567   1.5  jmcneill {
    568   1.5  jmcneill 	struct gicv3_softc * const sc = LPITOSOFTC(pic);
    569   1.5  jmcneill 
    570  1.35  jmcneill 	sc->sc_lpiconf.base[is->is_irq] = IPL_TO_PRIORITY(sc, is->is_ipl) | GIC_LPICONF_Res1;
    571   1.5  jmcneill 
    572  1.20  jmcneill 	if (sc->sc_lpiconf_flush)
    573  1.38  jmcneill 		gicv3_dcache_wb_range((vaddr_t)&sc->sc_lpiconf.base[is->is_irq], 1);
    574  1.20  jmcneill 	else
    575  1.26     skrll 		dsb(ishst);
    576   1.5  jmcneill }
    577   1.5  jmcneill 
    578   1.5  jmcneill static void
    579   1.5  jmcneill gicv3_lpi_cpu_init(struct pic_softc *pic, struct cpu_info *ci)
    580   1.5  jmcneill {
    581   1.5  jmcneill 	struct gicv3_softc * const sc = LPITOSOFTC(pic);
    582   1.7  jmcneill 	struct gicv3_lpi_callback *cb;
    583  1.20  jmcneill 	uint64_t propbase, pendbase;
    584   1.5  jmcneill 	uint32_t ctlr;
    585   1.5  jmcneill 
    586   1.5  jmcneill 	/* If physical LPIs are not supported on this redistributor, just return. */
    587   1.5  jmcneill 	const uint64_t typer = gicr_read_8(sc, ci->ci_gic_redist, GICR_TYPER);
    588   1.5  jmcneill 	if ((typer & GICR_TYPER_PLPIS) == 0)
    589   1.5  jmcneill 		return;
    590   1.5  jmcneill 
    591   1.5  jmcneill 	/* Interrupt target address for this CPU, used by ITS when GITS_TYPER.PTA == 0 */
    592   1.5  jmcneill 	sc->sc_processor_id[cpu_index(ci)] = __SHIFTOUT(typer, GICR_TYPER_Processor_Number);
    593   1.5  jmcneill 
    594   1.5  jmcneill 	/* Disable LPIs before making changes */
    595   1.5  jmcneill 	ctlr = gicr_read_4(sc, ci->ci_gic_redist, GICR_CTLR);
    596   1.5  jmcneill 	ctlr &= ~GICR_CTLR_Enable_LPIs;
    597   1.5  jmcneill 	gicr_write_4(sc, ci->ci_gic_redist, GICR_CTLR, ctlr);
    598  1.26     skrll 	dsb(sy);
    599   1.5  jmcneill 
    600   1.5  jmcneill 	/* Setup the LPI configuration table */
    601  1.20  jmcneill 	propbase = sc->sc_lpiconf.segs[0].ds_addr |
    602   1.5  jmcneill 	    __SHIFTIN(ffs(pic->pic_maxsources) - 1, GICR_PROPBASER_IDbits) |
    603  1.20  jmcneill 	    __SHIFTIN(GICR_Shareability_IS, GICR_PROPBASER_Shareability) |
    604  1.20  jmcneill 	    __SHIFTIN(GICR_Cache_NORMAL_RA_WA_WB, GICR_PROPBASER_InnerCache);
    605   1.5  jmcneill 	gicr_write_8(sc, ci->ci_gic_redist, GICR_PROPBASER, propbase);
    606  1.20  jmcneill 	propbase = gicr_read_8(sc, ci->ci_gic_redist, GICR_PROPBASER);
    607  1.20  jmcneill 	if (__SHIFTOUT(propbase, GICR_PROPBASER_Shareability) != GICR_Shareability_IS) {
    608  1.20  jmcneill 		if (__SHIFTOUT(propbase, GICR_PROPBASER_Shareability) == GICR_Shareability_NS) {
    609  1.20  jmcneill 			propbase &= ~GICR_PROPBASER_Shareability;
    610  1.20  jmcneill 			propbase |= __SHIFTIN(GICR_Shareability_NS, GICR_PROPBASER_Shareability);
    611  1.20  jmcneill 			propbase &= ~GICR_PROPBASER_InnerCache;
    612  1.20  jmcneill 			propbase |= __SHIFTIN(GICR_Cache_NORMAL_NC, GICR_PROPBASER_InnerCache);
    613  1.20  jmcneill 			gicr_write_8(sc, ci->ci_gic_redist, GICR_PROPBASER, propbase);
    614  1.20  jmcneill 		}
    615  1.20  jmcneill 		sc->sc_lpiconf_flush = true;
    616  1.20  jmcneill 	}
    617   1.5  jmcneill 
    618   1.5  jmcneill 	/* Setup the LPI pending table */
    619  1.20  jmcneill 	pendbase = sc->sc_lpipend[cpu_index(ci)].segs[0].ds_addr |
    620  1.20  jmcneill 	    __SHIFTIN(GICR_Shareability_IS, GICR_PENDBASER_Shareability) |
    621  1.20  jmcneill 	    __SHIFTIN(GICR_Cache_NORMAL_RA_WA_WB, GICR_PENDBASER_InnerCache);
    622   1.5  jmcneill 	gicr_write_8(sc, ci->ci_gic_redist, GICR_PENDBASER, pendbase);
    623  1.20  jmcneill 	pendbase = gicr_read_8(sc, ci->ci_gic_redist, GICR_PENDBASER);
    624  1.20  jmcneill 	if (__SHIFTOUT(pendbase, GICR_PENDBASER_Shareability) == GICR_Shareability_NS) {
    625  1.20  jmcneill 		pendbase &= ~GICR_PENDBASER_Shareability;
    626  1.20  jmcneill 		pendbase |= __SHIFTIN(GICR_Shareability_NS, GICR_PENDBASER_Shareability);
    627  1.20  jmcneill 		pendbase &= ~GICR_PENDBASER_InnerCache;
    628  1.20  jmcneill 		pendbase |= __SHIFTIN(GICR_Cache_NORMAL_NC, GICR_PENDBASER_InnerCache);
    629  1.20  jmcneill 		gicr_write_8(sc, ci->ci_gic_redist, GICR_PENDBASER, pendbase);
    630  1.20  jmcneill 	}
    631   1.5  jmcneill 
    632   1.5  jmcneill 	/* Enable LPIs */
    633   1.5  jmcneill 	ctlr = gicr_read_4(sc, ci->ci_gic_redist, GICR_CTLR);
    634   1.5  jmcneill 	ctlr |= GICR_CTLR_Enable_LPIs;
    635   1.5  jmcneill 	gicr_write_4(sc, ci->ci_gic_redist, GICR_CTLR, ctlr);
    636  1.26     skrll 	dsb(sy);
    637   1.5  jmcneill 
    638   1.5  jmcneill 	/* Setup ITS if present */
    639   1.7  jmcneill 	LIST_FOREACH(cb, &sc->sc_lpi_callbacks, list)
    640   1.7  jmcneill 		cb->cpu_init(cb->priv, ci);
    641   1.5  jmcneill }
    642   1.5  jmcneill 
    643   1.7  jmcneill #ifdef MULTIPROCESSOR
    644   1.7  jmcneill static void
    645   1.7  jmcneill gicv3_lpi_get_affinity(struct pic_softc *pic, size_t irq, kcpuset_t *affinity)
    646   1.7  jmcneill {
    647   1.7  jmcneill 	struct gicv3_softc * const sc = LPITOSOFTC(pic);
    648   1.7  jmcneill 	struct gicv3_lpi_callback *cb;
    649   1.7  jmcneill 
    650  1.24  jmcneill 	kcpuset_zero(affinity);
    651   1.7  jmcneill 	LIST_FOREACH(cb, &sc->sc_lpi_callbacks, list)
    652   1.7  jmcneill 		cb->get_affinity(cb->priv, irq, affinity);
    653   1.7  jmcneill }
    654   1.7  jmcneill 
    655   1.7  jmcneill static int
    656   1.7  jmcneill gicv3_lpi_set_affinity(struct pic_softc *pic, size_t irq, const kcpuset_t *affinity)
    657   1.7  jmcneill {
    658   1.7  jmcneill 	struct gicv3_softc * const sc = LPITOSOFTC(pic);
    659   1.7  jmcneill 	struct gicv3_lpi_callback *cb;
    660   1.7  jmcneill 	int error = EINVAL;
    661   1.7  jmcneill 
    662   1.7  jmcneill 	LIST_FOREACH(cb, &sc->sc_lpi_callbacks, list) {
    663   1.7  jmcneill 		error = cb->set_affinity(cb->priv, irq, affinity);
    664  1.24  jmcneill 		if (error != EPASSTHROUGH)
    665   1.7  jmcneill 			return error;
    666   1.7  jmcneill 	}
    667   1.7  jmcneill 
    668  1.24  jmcneill 	return EINVAL;
    669   1.7  jmcneill }
    670   1.7  jmcneill #endif
    671   1.7  jmcneill 
    672   1.5  jmcneill static const struct pic_ops gicv3_lpiops = {
    673   1.5  jmcneill 	.pic_unblock_irqs = gicv3_lpi_unblock_irqs,
    674   1.5  jmcneill 	.pic_block_irqs = gicv3_lpi_block_irqs,
    675   1.5  jmcneill 	.pic_establish_irq = gicv3_lpi_establish_irq,
    676   1.5  jmcneill #ifdef MULTIPROCESSOR
    677   1.5  jmcneill 	.pic_cpu_init = gicv3_lpi_cpu_init,
    678   1.7  jmcneill 	.pic_get_affinity = gicv3_lpi_get_affinity,
    679   1.7  jmcneill 	.pic_set_affinity = gicv3_lpi_set_affinity,
    680   1.5  jmcneill #endif
    681   1.5  jmcneill };
    682   1.5  jmcneill 
    683   1.5  jmcneill void
    684   1.5  jmcneill gicv3_dma_alloc(struct gicv3_softc *sc, struct gicv3_dma *dma, bus_size_t len, bus_size_t align)
    685   1.5  jmcneill {
    686   1.5  jmcneill 	int nsegs, error;
    687   1.5  jmcneill 
    688   1.5  jmcneill 	dma->len = len;
    689   1.5  jmcneill 	error = bus_dmamem_alloc(sc->sc_dmat, dma->len, align, 0, dma->segs, 1, &nsegs, BUS_DMA_WAITOK);
    690   1.5  jmcneill 	if (error)
    691   1.5  jmcneill 		panic("bus_dmamem_alloc failed: %d", error);
    692   1.5  jmcneill 	error = bus_dmamem_map(sc->sc_dmat, dma->segs, nsegs, len, (void **)&dma->base, BUS_DMA_WAITOK);
    693   1.5  jmcneill 	if (error)
    694   1.5  jmcneill 		panic("bus_dmamem_map failed: %d", error);
    695   1.5  jmcneill 	error = bus_dmamap_create(sc->sc_dmat, len, 1, len, 0, BUS_DMA_WAITOK, &dma->map);
    696   1.5  jmcneill 	if (error)
    697   1.5  jmcneill 		panic("bus_dmamap_create failed: %d", error);
    698   1.5  jmcneill 	error = bus_dmamap_load(sc->sc_dmat, dma->map, dma->base, dma->len, NULL, BUS_DMA_WAITOK);
    699   1.5  jmcneill 	if (error)
    700   1.5  jmcneill 		panic("bus_dmamap_load failed: %d", error);
    701   1.5  jmcneill 
    702   1.5  jmcneill 	memset(dma->base, 0, dma->len);
    703   1.5  jmcneill 	bus_dmamap_sync(sc->sc_dmat, dma->map, 0, dma->len, BUS_DMASYNC_PREWRITE);
    704   1.5  jmcneill }
    705   1.5  jmcneill 
    706   1.5  jmcneill static void
    707   1.5  jmcneill gicv3_lpi_init(struct gicv3_softc *sc)
    708   1.5  jmcneill {
    709   1.5  jmcneill 	/*
    710   1.5  jmcneill 	 * Allocate LPI configuration table
    711   1.5  jmcneill 	 */
    712   1.5  jmcneill 	gicv3_dma_alloc(sc, &sc->sc_lpiconf, sc->sc_lpi.pic_maxsources, 0x1000);
    713   1.5  jmcneill 	KASSERT((sc->sc_lpiconf.segs[0].ds_addr & ~GICR_PROPBASER_Physical_Address) == 0);
    714   1.5  jmcneill 
    715   1.5  jmcneill 	/*
    716   1.5  jmcneill 	 * Allocate LPI pending tables
    717   1.5  jmcneill 	 */
    718  1.20  jmcneill 	const bus_size_t lpipend_sz = (8192 + sc->sc_lpi.pic_maxsources) / NBBY;
    719   1.8  jmcneill 	for (int cpuindex = 0; cpuindex < ncpu; cpuindex++) {
    720   1.5  jmcneill 		gicv3_dma_alloc(sc, &sc->sc_lpipend[cpuindex], lpipend_sz, 0x10000);
    721   1.5  jmcneill 		KASSERT((sc->sc_lpipend[cpuindex].segs[0].ds_addr & ~GICR_PENDBASER_Physical_Address) == 0);
    722   1.5  jmcneill 	}
    723   1.5  jmcneill }
    724   1.5  jmcneill 
    725   1.1  jmcneill void
    726   1.1  jmcneill gicv3_irq_handler(void *frame)
    727   1.1  jmcneill {
    728   1.1  jmcneill 	struct cpu_info * const ci = curcpu();
    729   1.1  jmcneill 	struct gicv3_softc * const sc = gicv3_softc;
    730   1.5  jmcneill 	struct pic_softc *pic;
    731   1.1  jmcneill 	const int oldipl = ci->ci_cpl;
    732  1.40  jmcneill 	const uint8_t pmr = IPL_TO_PMR(sc, oldipl);
    733   1.1  jmcneill 
    734   1.1  jmcneill 	ci->ci_data.cpu_nintr++;
    735   1.1  jmcneill 
    736  1.40  jmcneill 	if (icc_pmr_read() != pmr) {
    737  1.40  jmcneill 		icc_pmr_write(pmr);
    738  1.40  jmcneill 	}
    739  1.40  jmcneill 
    740   1.1  jmcneill 	for (;;) {
    741   1.1  jmcneill 		const uint32_t iar = icc_iar1_read();
    742  1.26     skrll 		dsb(sy);
    743   1.1  jmcneill 		const uint32_t irq = __SHIFTOUT(iar, ICC_IAR_INTID);
    744   1.1  jmcneill 		if (irq == ICC_IAR_INTID_SPURIOUS)
    745   1.1  jmcneill 			break;
    746   1.1  jmcneill 
    747   1.5  jmcneill 		pic = irq >= GIC_LPI_BASE ? &sc->sc_lpi : &sc->sc_pic;
    748   1.5  jmcneill 		if (irq - pic->pic_irqbase >= pic->pic_maxsources)
    749   1.1  jmcneill 			continue;
    750   1.1  jmcneill 
    751   1.5  jmcneill 		struct intrsource * const is = pic->pic_sources[irq - pic->pic_irqbase];
    752   1.1  jmcneill 		KASSERT(is != NULL);
    753   1.1  jmcneill 
    754  1.21  jmcneill 		const bool early_eoi = irq < GIC_LPI_BASE && is->is_type == IST_EDGE;
    755  1.21  jmcneill 
    756   1.1  jmcneill 		const int ipl = is->is_ipl;
    757  1.21  jmcneill 		if (__predict_false(ipl < ci->ci_cpl)) {
    758  1.21  jmcneill 			pic_do_pending_ints(I32_bit, ipl, frame);
    759  1.28  jmcneill 		} else if (ci->ci_cpl != ipl) {
    760  1.40  jmcneill 			icc_pmr_write(IPL_TO_PMR(sc, ipl));
    761  1.21  jmcneill 			ci->ci_cpl = ipl;
    762  1.21  jmcneill 		}
    763  1.21  jmcneill 
    764  1.21  jmcneill 		if (early_eoi) {
    765  1.21  jmcneill 			icc_eoi1r_write(iar);
    766  1.26     skrll 			isb();
    767  1.21  jmcneill 		}
    768   1.1  jmcneill 
    769  1.33  jmcneill 		const int64_t nintr = ci->ci_data.cpu_nintr;
    770  1.33  jmcneill 
    771  1.40  jmcneill 		ENABLE_INTERRUPT();
    772   1.1  jmcneill 		pic_dispatch(is, frame);
    773  1.40  jmcneill 		DISABLE_INTERRUPT();
    774   1.1  jmcneill 
    775  1.33  jmcneill 		if (nintr != ci->ci_data.cpu_nintr)
    776  1.33  jmcneill 			ci->ci_intr_preempt.ev_count++;
    777  1.33  jmcneill 
    778  1.21  jmcneill 		if (!early_eoi) {
    779  1.21  jmcneill 			icc_eoi1r_write(iar);
    780  1.26     skrll 			isb();
    781  1.21  jmcneill 		}
    782   1.1  jmcneill 	}
    783   1.1  jmcneill 
    784  1.21  jmcneill 	pic_do_pending_ints(I32_bit, oldipl, frame);
    785   1.1  jmcneill }
    786   1.1  jmcneill 
    787  1.34  jmcneill static bool
    788  1.35  jmcneill gicv3_cpuif_is_nonsecure(struct gicv3_softc *sc)
    789  1.34  jmcneill {
    790  1.35  jmcneill 	/*
    791  1.35  jmcneill 	 * Write 0 to bit7 and see if it sticks. This is only possible if
    792  1.35  jmcneill 	 * we have a non-secure view of the PMR register.
    793  1.35  jmcneill 	 */
    794  1.35  jmcneill 	const uint32_t opmr = icc_pmr_read();
    795  1.35  jmcneill 	icc_pmr_write(0);
    796  1.35  jmcneill 	const uint32_t npmr = icc_pmr_read();
    797  1.35  jmcneill 	icc_pmr_write(opmr);
    798  1.34  jmcneill 
    799  1.35  jmcneill 	return (npmr & GICC_PMR_NONSECURE) == 0;
    800  1.34  jmcneill }
    801  1.34  jmcneill 
    802  1.35  jmcneill static bool
    803  1.35  jmcneill gicv3_dist_is_nonsecure(struct gicv3_softc *sc)
    804  1.19  jmcneill {
    805  1.35  jmcneill 	const uint32_t gicd_ctrl = gicd_read_4(sc, GICD_CTRL);
    806  1.19  jmcneill 
    807  1.35  jmcneill 	/*
    808  1.35  jmcneill 	 * If security is enabled, we have a non-secure view of the IPRIORITYRn
    809  1.35  jmcneill 	 * registers and LPI configuration priority fields.
    810  1.35  jmcneill 	 */
    811  1.35  jmcneill 	return (gicd_ctrl & GICD_CTRL_DS) == 0;
    812  1.19  jmcneill }
    813  1.19  jmcneill 
    814  1.35  jmcneill /*
    815  1.35  jmcneill  * Rockchip RK3399 provides a different view of int priority registers
    816  1.35  jmcneill  * depending on which firmware is in use. This is hard to detect in
    817  1.35  jmcneill  * a way that could possibly break other boards, so only do this
    818  1.35  jmcneill  * detection if we know we are on a RK3399 SoC.
    819  1.35  jmcneill  */
    820  1.35  jmcneill static void
    821  1.35  jmcneill gicv3_quirk_rockchip_rk3399(struct gicv3_softc *sc)
    822  1.19  jmcneill {
    823  1.35  jmcneill 	/* Detect the number of supported PMR bits */
    824  1.35  jmcneill 	icc_pmr_write(0xff);
    825  1.35  jmcneill 	const uint8_t pmrbits = icc_pmr_read();
    826  1.19  jmcneill 
    827  1.35  jmcneill 	/* Detect the number of supported IPRIORITYRn bits */
    828  1.35  jmcneill 	const uint32_t oiprio = gicd_read_4(sc, GICD_IPRIORITYRn(8));
    829  1.35  jmcneill 	gicd_write_4(sc, GICD_IPRIORITYRn(8), oiprio | 0xff);
    830  1.35  jmcneill 	const uint8_t pribits = gicd_read_4(sc, GICD_IPRIORITYRn(8)) & 0xff;
    831  1.35  jmcneill 	gicd_write_4(sc, GICD_IPRIORITYRn(8), oiprio);
    832  1.35  jmcneill 
    833  1.35  jmcneill 	/*
    834  1.35  jmcneill 	 * If we see fewer PMR bits than IPRIORITYRn bits here, it means
    835  1.35  jmcneill 	 * we have a secure view of IPRIORITYRn (this is not supposed to
    836  1.35  jmcneill 	 * happen!).
    837  1.35  jmcneill 	 */
    838  1.35  jmcneill 	if (pmrbits < pribits) {
    839  1.35  jmcneill 		aprint_verbose_dev(sc->sc_dev,
    840  1.35  jmcneill 		    "buggy RK3399 firmware detected; applying workaround\n");
    841  1.35  jmcneill 		sc->sc_priority_shift = GIC_PRIO_SHIFT_S;
    842  1.35  jmcneill 	}
    843  1.19  jmcneill }
    844  1.19  jmcneill 
    845   1.1  jmcneill int
    846   1.1  jmcneill gicv3_init(struct gicv3_softc *sc)
    847   1.1  jmcneill {
    848   1.6  jmcneill 	int n;
    849   1.1  jmcneill 
    850   1.1  jmcneill 	KASSERT(CPU_IS_PRIMARY(curcpu()));
    851   1.1  jmcneill 
    852   1.7  jmcneill 	LIST_INIT(&sc->sc_lpi_callbacks);
    853   1.5  jmcneill 
    854  1.39  jmcneill 	sc->sc_irouter = kmem_zalloc(sizeof(*sc->sc_irouter) * ncpu, KM_SLEEP);
    855  1.39  jmcneill 	for (n = 0; n < ncpu; n++)
    856   1.6  jmcneill 		sc->sc_irouter[n] = UINT64_MAX;
    857   1.6  jmcneill 
    858  1.36  jmcneill 	sc->sc_gicd_typer = gicd_read_4(sc, GICD_TYPER);
    859  1.36  jmcneill 
    860  1.35  jmcneill 	/*
    861  1.37  jmcneill 	 * We don't always have a consistent view of priorities between the
    862  1.35  jmcneill 	 * CPU interface (ICC_PMR_EL1) and the GICD/GICR registers. Detect
    863  1.35  jmcneill 	 * if we are making secure or non-secure accesses to each, and adjust
    864  1.35  jmcneill 	 * the values that we write to each accordingly.
    865  1.35  jmcneill 	 */
    866  1.35  jmcneill 	const bool dist_ns = gicv3_dist_is_nonsecure(sc);
    867  1.35  jmcneill 	sc->sc_priority_shift = dist_ns ? GIC_PRIO_SHIFT_NS : GIC_PRIO_SHIFT_S;
    868  1.35  jmcneill 	const bool cpuif_ns = gicv3_cpuif_is_nonsecure(sc);
    869  1.35  jmcneill 	sc->sc_pmr_shift = cpuif_ns ? GIC_PRIO_SHIFT_NS : GIC_PRIO_SHIFT_S;
    870  1.34  jmcneill 
    871  1.35  jmcneill 	if ((sc->sc_quirks & GICV3_QUIRK_RK3399) != 0)
    872  1.35  jmcneill 		gicv3_quirk_rockchip_rk3399(sc);
    873  1.19  jmcneill 
    874  1.34  jmcneill 	aprint_verbose_dev(sc->sc_dev,
    875  1.35  jmcneill 	    "iidr 0x%08x, cpuif %ssecure, dist %ssecure, "
    876  1.35  jmcneill 	    "priority shift %d, pmr shift %d, quirks %#x\n",
    877  1.35  jmcneill 	    gicd_read_4(sc, GICD_IIDR),
    878  1.35  jmcneill 	    cpuif_ns ? "non-" : "",
    879  1.35  jmcneill 	    dist_ns ? "non-" : "",
    880  1.35  jmcneill 	    sc->sc_priority_shift,
    881  1.35  jmcneill 	    sc->sc_pmr_shift,
    882  1.35  jmcneill 	    sc->sc_quirks);
    883  1.18  jmcneill 
    884   1.1  jmcneill 	sc->sc_pic.pic_ops = &gicv3_picops;
    885  1.36  jmcneill 	sc->sc_pic.pic_maxsources = GICD_TYPER_LINES(sc->sc_gicd_typer);
    886   1.1  jmcneill 	snprintf(sc->sc_pic.pic_name, sizeof(sc->sc_pic.pic_name), "gicv3");
    887   1.1  jmcneill #ifdef MULTIPROCESSOR
    888   1.1  jmcneill 	sc->sc_pic.pic_cpus = kcpuset_running;
    889   1.1  jmcneill #endif
    890   1.1  jmcneill 	pic_add(&sc->sc_pic, 0);
    891   1.1  jmcneill 
    892  1.36  jmcneill 	if ((sc->sc_gicd_typer & GICD_TYPER_LPIS) != 0) {
    893  1.39  jmcneill 		sc->sc_lpipend = kmem_zalloc(sizeof(*sc->sc_lpipend) * ncpu, KM_SLEEP);
    894  1.39  jmcneill 		sc->sc_processor_id = kmem_zalloc(sizeof(*sc->sc_processor_id) * ncpu, KM_SLEEP);
    895  1.39  jmcneill 
    896   1.5  jmcneill 		sc->sc_lpi.pic_ops = &gicv3_lpiops;
    897   1.5  jmcneill 		sc->sc_lpi.pic_maxsources = 8192;	/* Min. required by GICv3 spec */
    898   1.5  jmcneill 		snprintf(sc->sc_lpi.pic_name, sizeof(sc->sc_lpi.pic_name), "gicv3-lpi");
    899   1.5  jmcneill 		pic_add(&sc->sc_lpi, GIC_LPI_BASE);
    900   1.5  jmcneill 
    901  1.23  jmcneill 		sc->sc_lpi_pool = vmem_create("gicv3-lpi", 0, sc->sc_lpi.pic_maxsources,
    902  1.23  jmcneill 		    1, NULL, NULL, NULL, 0, VM_SLEEP, IPL_HIGH);
    903  1.23  jmcneill 		if (sc->sc_lpi_pool == NULL)
    904  1.23  jmcneill 			panic("failed to create gicv3 lpi pool\n");
    905  1.23  jmcneill 
    906   1.5  jmcneill 		gicv3_lpi_init(sc);
    907   1.5  jmcneill 	}
    908   1.5  jmcneill 
    909   1.1  jmcneill 	KASSERT(gicv3_softc == NULL);
    910   1.1  jmcneill 	gicv3_softc = sc;
    911   1.1  jmcneill 
    912   1.1  jmcneill 	for (int i = 0; i < sc->sc_bsh_r_count; i++) {
    913   1.1  jmcneill 		const uint64_t gicr_typer = gicr_read_8(sc, i, GICR_TYPER);
    914   1.1  jmcneill 		const u_int aff0 = __SHIFTOUT(gicr_typer, GICR_TYPER_Affinity_Value_Aff0);
    915   1.1  jmcneill 		const u_int aff1 = __SHIFTOUT(gicr_typer, GICR_TYPER_Affinity_Value_Aff1);
    916   1.1  jmcneill 		const u_int aff2 = __SHIFTOUT(gicr_typer, GICR_TYPER_Affinity_Value_Aff2);
    917   1.1  jmcneill 		const u_int aff3 = __SHIFTOUT(gicr_typer, GICR_TYPER_Affinity_Value_Aff3);
    918   1.1  jmcneill 
    919   1.1  jmcneill 		aprint_debug_dev(sc->sc_dev, "redist %d: cpu %d.%d.%d.%d\n",
    920   1.1  jmcneill 		    i, aff3, aff2, aff1, aff0);
    921   1.1  jmcneill 	}
    922   1.1  jmcneill 
    923   1.1  jmcneill 	gicv3_dist_enable(sc);
    924   1.1  jmcneill 
    925   1.1  jmcneill 	gicv3_cpu_init(&sc->sc_pic, curcpu());
    926  1.36  jmcneill 	if ((sc->sc_gicd_typer & GICD_TYPER_LPIS) != 0)
    927   1.5  jmcneill 		gicv3_lpi_cpu_init(&sc->sc_lpi, curcpu());
    928   1.1  jmcneill 
    929   1.1  jmcneill #ifdef MULTIPROCESSOR
    930  1.11  jmcneill 	intr_establish_xname(IPI_AST, IPL_VM, IST_MPSAFE | IST_EDGE, pic_ipi_ast, (void *)-1, "IPI ast");
    931  1.11  jmcneill 	intr_establish_xname(IPI_XCALL, IPL_HIGH, IST_MPSAFE | IST_EDGE, pic_ipi_xcall, (void *)-1, "IPI xcall");
    932  1.11  jmcneill 	intr_establish_xname(IPI_GENERIC, IPL_HIGH, IST_MPSAFE | IST_EDGE, pic_ipi_generic, (void *)-1, "IPI generic");
    933  1.11  jmcneill 	intr_establish_xname(IPI_NOP, IPL_VM, IST_MPSAFE | IST_EDGE, pic_ipi_nop, (void *)-1, "IPI nop");
    934  1.11  jmcneill 	intr_establish_xname(IPI_SHOOTDOWN, IPL_SCHED, IST_MPSAFE | IST_EDGE, pic_ipi_shootdown, (void *)-1, "IPI shootdown");
    935   1.1  jmcneill #ifdef DDB
    936  1.11  jmcneill 	intr_establish_xname(IPI_DDB, IPL_HIGH, IST_MPSAFE | IST_EDGE, pic_ipi_ddb, NULL, "IPI ddb");
    937   1.1  jmcneill #endif
    938   1.1  jmcneill #ifdef __HAVE_PREEMPTION
    939  1.11  jmcneill 	intr_establish_xname(IPI_KPREEMPT, IPL_VM, IST_MPSAFE | IST_EDGE, pic_ipi_kpreempt, (void *)-1, "IPI kpreempt");
    940   1.1  jmcneill #endif
    941   1.1  jmcneill #endif
    942   1.1  jmcneill 
    943   1.1  jmcneill 	return 0;
    944   1.1  jmcneill }
    945