Home | History | Annotate | Line # | Download | only in pci
virtio_pci.c revision 1.2.2.2
      1  1.2.2.2  pgoyette /* $NetBSD: virtio_pci.c,v 1.2.2.2 2019/01/18 08:50:42 pgoyette Exp $ */
      2      1.1    cherry 
      3      1.1    cherry /*
      4      1.1    cherry  * Copyright (c) 2010 Minoura Makoto.
      5      1.1    cherry  * All rights reserved.
      6      1.1    cherry  *
      7      1.1    cherry  * Redistribution and use in source and binary forms, with or without
      8      1.1    cherry  * modification, are permitted provided that the following conditions
      9      1.1    cherry  * are met:
     10      1.1    cherry  * 1. Redistributions of source code must retain the above copyright
     11      1.1    cherry  *    notice, this list of conditions and the following disclaimer.
     12      1.1    cherry  * 2. Redistributions in binary form must reproduce the above copyright
     13      1.1    cherry  *    notice, this list of conditions and the following disclaimer in the
     14      1.1    cherry  *    documentation and/or other materials provided with the distribution.
     15      1.1    cherry  *
     16      1.1    cherry  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
     17      1.1    cherry  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
     18      1.1    cherry  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
     19      1.1    cherry  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
     20      1.1    cherry  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
     21      1.1    cherry  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
     22      1.1    cherry  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
     23      1.1    cherry  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
     24      1.1    cherry  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
     25      1.1    cherry  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
     26      1.1    cherry  */
     27      1.1    cherry 
     28      1.1    cherry #include <sys/cdefs.h>
     29  1.2.2.2  pgoyette __KERNEL_RCSID(0, "$NetBSD: virtio_pci.c,v 1.2.2.2 2019/01/18 08:50:42 pgoyette Exp $");
     30      1.1    cherry 
     31      1.1    cherry #include <sys/param.h>
     32      1.1    cherry #include <sys/systm.h>
     33  1.2.2.1  pgoyette #include <sys/kmem.h>
     34  1.2.2.1  pgoyette #include <sys/module.h>
     35  1.2.2.2  pgoyette #include <sys/interrupt.h>
     36      1.1    cherry 
     37      1.1    cherry #include <sys/device.h>
     38      1.1    cherry 
     39      1.1    cherry #include <dev/pci/pcidevs.h>
     40      1.1    cherry #include <dev/pci/pcireg.h>
     41      1.1    cherry #include <dev/pci/pcivar.h>
     42      1.1    cherry 
     43      1.1    cherry #define VIRTIO_PRIVATE
     44      1.1    cherry 
     45      1.1    cherry #include <dev/pci/virtioreg.h> /* XXX: move to non-pci */
     46      1.1    cherry #include <dev/pci/virtiovar.h> /* XXX: move to non-pci */
     47      1.1    cherry 
     48  1.2.2.1  pgoyette static int	virtio_pci_match(device_t, cfdata_t, void *);
     49  1.2.2.1  pgoyette static void	virtio_pci_attach(device_t, device_t, void *);
     50  1.2.2.1  pgoyette static int	virtio_pci_rescan(device_t, const char *, const int *);
     51  1.2.2.1  pgoyette static int	virtio_pci_detach(device_t, int);
     52  1.2.2.1  pgoyette 
     53  1.2.2.1  pgoyette struct virtio_pci_softc {
     54  1.2.2.1  pgoyette 	struct virtio_softc	sc_sc;
     55  1.2.2.1  pgoyette 	bus_space_tag_t		sc_iot;
     56  1.2.2.1  pgoyette 	bus_space_handle_t	sc_ioh;
     57  1.2.2.1  pgoyette 	bus_size_t		sc_iosize;
     58  1.2.2.1  pgoyette 	struct pci_attach_args	sc_pa;
     59  1.2.2.1  pgoyette 	pci_intr_handle_t	*sc_ihp;
     60  1.2.2.1  pgoyette 	void			**sc_ihs;
     61  1.2.2.1  pgoyette 	int			sc_ihs_num;
     62  1.2.2.1  pgoyette 	int			sc_config_offset;
     63  1.2.2.1  pgoyette };
     64  1.2.2.1  pgoyette 
     65  1.2.2.1  pgoyette static void	virtio_pci_kick(struct virtio_softc *, uint16_t);
     66  1.2.2.1  pgoyette static uint8_t	virtio_pci_read_device_config_1(struct virtio_softc *, int);
     67  1.2.2.1  pgoyette static uint16_t	virtio_pci_read_device_config_2(struct virtio_softc *, int);
     68  1.2.2.1  pgoyette static uint32_t	virtio_pci_read_device_config_4(struct virtio_softc *, int);
     69  1.2.2.1  pgoyette static uint64_t	virtio_pci_read_device_config_8(struct virtio_softc *, int);
     70  1.2.2.1  pgoyette static void 	virtio_pci_write_device_config_1(struct virtio_softc *, int, uint8_t);
     71  1.2.2.1  pgoyette static void	virtio_pci_write_device_config_2(struct virtio_softc *, int, uint16_t);
     72  1.2.2.1  pgoyette static void	virtio_pci_write_device_config_4(struct virtio_softc *, int, uint32_t);
     73  1.2.2.1  pgoyette static void	virtio_pci_write_device_config_8(struct virtio_softc *, int, uint64_t);
     74  1.2.2.1  pgoyette static uint16_t	virtio_pci_read_queue_size(struct virtio_softc *, uint16_t);
     75  1.2.2.1  pgoyette static void	virtio_pci_setup_queue(struct virtio_softc *, uint16_t, uint32_t);
     76  1.2.2.1  pgoyette static void	virtio_pci_set_status(struct virtio_softc *, int);
     77  1.2.2.1  pgoyette static uint32_t	virtio_pci_negotiate_features(struct virtio_softc *, uint32_t);
     78  1.2.2.1  pgoyette static int	virtio_pci_setup_interrupts(struct virtio_softc *);
     79  1.2.2.1  pgoyette static void	virtio_pci_free_interrupts(struct virtio_softc *);
     80  1.2.2.1  pgoyette 
     81  1.2.2.1  pgoyette static int	virtio_pci_intr(void *arg);
     82  1.2.2.1  pgoyette static int	virtio_pci_msix_queue_intr(void *);
     83  1.2.2.2  pgoyette static int	virtio_pci_msix_vq_intr(void *);
     84  1.2.2.1  pgoyette static int	virtio_pci_msix_config_intr(void *);
     85  1.2.2.1  pgoyette static int	virtio_pci_setup_msix_vectors(struct virtio_softc *);
     86  1.2.2.1  pgoyette static int	virtio_pci_setup_msix_interrupts(struct virtio_softc *,
     87  1.2.2.1  pgoyette 		    struct pci_attach_args *);
     88  1.2.2.1  pgoyette static int	virtio_pci_setup_intx_interrupt(struct virtio_softc *,
     89  1.2.2.1  pgoyette 		    struct pci_attach_args *);
     90  1.2.2.1  pgoyette 
     91  1.2.2.1  pgoyette #define VIRTIO_MSIX_CONFIG_VECTOR_INDEX	0
     92  1.2.2.1  pgoyette #define VIRTIO_MSIX_QUEUE_VECTOR_INDEX	1
     93  1.2.2.1  pgoyette 
     94  1.2.2.1  pgoyette /* we use the legacy virtio spec, so the PCI registers are host native
     95  1.2.2.1  pgoyette  * byte order, not PCI (i.e. LE) byte order */
     96  1.2.2.1  pgoyette #if BYTE_ORDER == BIG_ENDIAN
     97  1.2.2.1  pgoyette #define REG_HI_OFF      0
     98  1.2.2.1  pgoyette #define REG_LO_OFF      4
     99  1.2.2.1  pgoyette #ifndef __BUS_SPACE_HAS_STREAM_METHODS
    100  1.2.2.1  pgoyette #define bus_space_read_stream_1 bus_space_read_1
    101  1.2.2.1  pgoyette #define bus_space_write_stream_1 bus_space_write_1
    102  1.2.2.1  pgoyette static inline uint16_t
    103  1.2.2.1  pgoyette bus_space_read_stream_2(bus_space_tag_t t, bus_space_handle_t h,
    104  1.2.2.1  pgoyette     bus_size_t o)
    105  1.2.2.1  pgoyette {
    106  1.2.2.1  pgoyette 	return le16toh(bus_space_read_2(t, h, o));
    107  1.2.2.1  pgoyette }
    108  1.2.2.1  pgoyette static inline void
    109  1.2.2.1  pgoyette bus_space_write_stream_2(bus_space_tag_t t, bus_space_handle_t h,
    110  1.2.2.1  pgoyette     bus_size_t o, uint16_t v)
    111  1.2.2.1  pgoyette {
    112  1.2.2.1  pgoyette 	bus_space_write_2(t, h, o, htole16(v));
    113  1.2.2.1  pgoyette }
    114  1.2.2.1  pgoyette static inline uint32_t
    115  1.2.2.1  pgoyette bus_space_read_stream_4(bus_space_tag_t t, bus_space_handle_t h,
    116  1.2.2.1  pgoyette     bus_size_t o)
    117  1.2.2.1  pgoyette {
    118  1.2.2.1  pgoyette 	return le32toh(bus_space_read_4(t, h, o));
    119  1.2.2.1  pgoyette }
    120  1.2.2.1  pgoyette static inline void
    121  1.2.2.1  pgoyette bus_space_write_stream_4(bus_space_tag_t t, bus_space_handle_t h,
    122  1.2.2.1  pgoyette     bus_size_t o, uint32_t v)
    123  1.2.2.1  pgoyette {
    124  1.2.2.1  pgoyette 	bus_space_write_4(t, h, o, htole32(v));
    125  1.2.2.1  pgoyette }
    126  1.2.2.1  pgoyette #endif
    127  1.2.2.1  pgoyette #else
    128  1.2.2.1  pgoyette #define REG_HI_OFF	4
    129  1.2.2.1  pgoyette #define REG_LO_OFF	0
    130  1.2.2.1  pgoyette #ifndef __BUS_SPACE_HAS_STREAM_METHODS
    131  1.2.2.1  pgoyette #define bus_space_read_stream_1 bus_space_read_1
    132  1.2.2.1  pgoyette #define bus_space_read_stream_2 bus_space_read_2
    133  1.2.2.1  pgoyette #define bus_space_read_stream_4 bus_space_read_4
    134  1.2.2.1  pgoyette #define bus_space_write_stream_1 bus_space_write_1
    135  1.2.2.1  pgoyette #define bus_space_write_stream_2 bus_space_write_2
    136  1.2.2.1  pgoyette #define bus_space_write_stream_4 bus_space_write_4
    137  1.2.2.1  pgoyette #endif
    138  1.2.2.1  pgoyette #endif
    139  1.2.2.1  pgoyette 
    140      1.1    cherry 
    141      1.1    cherry static const char *virtio_device_name[] = {
    142      1.1    cherry 	"Unknown (0)",			/* 0 */
    143      1.1    cherry 	"Network",			/* 1 */
    144      1.1    cherry 	"Block",			/* 2 */
    145      1.1    cherry 	"Console",			/* 3 */
    146      1.1    cherry 	"Entropy",			/* 4 */
    147      1.1    cherry 	"Memory Balloon",		/* 5 */
    148      1.1    cherry 	"I/O Memory",			/* 6 */
    149      1.1    cherry 	"Remote Processor Messaging",	/* 7 */
    150      1.1    cherry 	"SCSI",				/* 8 */
    151      1.1    cherry 	"9P Transport",			/* 9 */
    152      1.1    cherry 	"mac80211 wlan",		/* 10 */
    153      1.1    cherry };
    154      1.1    cherry #define NDEVNAMES	__arraycount(virtio_device_name)
    155      1.1    cherry 
    156  1.2.2.1  pgoyette CFATTACH_DECL3_NEW(virtio_pci, sizeof(struct virtio_pci_softc),
    157  1.2.2.1  pgoyette     virtio_pci_match, virtio_pci_attach, virtio_pci_detach, NULL,
    158  1.2.2.1  pgoyette     virtio_pci_rescan, NULL, DVF_DETACH_SHUTDOWN);
    159  1.2.2.1  pgoyette 
    160  1.2.2.1  pgoyette static const struct virtio_ops virtio_pci_ops = {
    161  1.2.2.1  pgoyette 	.kick = virtio_pci_kick,
    162  1.2.2.1  pgoyette 	.read_dev_cfg_1 = virtio_pci_read_device_config_1,
    163  1.2.2.1  pgoyette 	.read_dev_cfg_2 = virtio_pci_read_device_config_2,
    164  1.2.2.1  pgoyette 	.read_dev_cfg_4 = virtio_pci_read_device_config_4,
    165  1.2.2.1  pgoyette 	.read_dev_cfg_8 = virtio_pci_read_device_config_8,
    166  1.2.2.1  pgoyette 	.write_dev_cfg_1 = virtio_pci_write_device_config_1,
    167  1.2.2.1  pgoyette 	.write_dev_cfg_2 = virtio_pci_write_device_config_2,
    168  1.2.2.1  pgoyette 	.write_dev_cfg_4 = virtio_pci_write_device_config_4,
    169  1.2.2.1  pgoyette 	.write_dev_cfg_8 = virtio_pci_write_device_config_8,
    170  1.2.2.1  pgoyette 	.read_queue_size = virtio_pci_read_queue_size,
    171  1.2.2.1  pgoyette 	.setup_queue = virtio_pci_setup_queue,
    172  1.2.2.1  pgoyette 	.set_status = virtio_pci_set_status,
    173  1.2.2.1  pgoyette 	.neg_features = virtio_pci_negotiate_features,
    174  1.2.2.1  pgoyette 	.setup_interrupts = virtio_pci_setup_interrupts,
    175  1.2.2.1  pgoyette 	.free_interrupts = virtio_pci_free_interrupts,
    176  1.2.2.1  pgoyette };
    177      1.1    cherry 
    178      1.1    cherry static int
    179  1.2.2.1  pgoyette virtio_pci_match(device_t parent, cfdata_t match, void *aux)
    180      1.1    cherry {
    181      1.1    cherry 	struct pci_attach_args *pa;
    182      1.1    cherry 
    183      1.1    cherry 	pa = (struct pci_attach_args *)aux;
    184      1.1    cherry 	switch (PCI_VENDOR(pa->pa_id)) {
    185      1.1    cherry 	case PCI_VENDOR_QUMRANET:
    186      1.1    cherry 		if ((PCI_PRODUCT_QUMRANET_VIRTIO_1000 <=
    187      1.1    cherry 		     PCI_PRODUCT(pa->pa_id)) &&
    188      1.1    cherry 		    (PCI_PRODUCT(pa->pa_id) <=
    189      1.1    cherry 		     PCI_PRODUCT_QUMRANET_VIRTIO_103F))
    190      1.1    cherry 			return 1;
    191      1.1    cherry 		break;
    192      1.1    cherry 	}
    193      1.1    cherry 
    194      1.1    cherry 	return 0;
    195      1.1    cherry }
    196      1.1    cherry 
    197      1.1    cherry static void
    198  1.2.2.1  pgoyette virtio_pci_attach(device_t parent, device_t self, void *aux)
    199      1.1    cherry {
    200  1.2.2.1  pgoyette 	struct virtio_pci_softc * const psc = device_private(self);
    201  1.2.2.1  pgoyette 	struct virtio_softc * const sc = &psc->sc_sc;
    202      1.1    cherry 	struct pci_attach_args *pa = (struct pci_attach_args *)aux;
    203      1.1    cherry 	pci_chipset_tag_t pc = pa->pa_pc;
    204      1.1    cherry 	pcitag_t tag = pa->pa_tag;
    205      1.1    cherry 	int revision;
    206      1.1    cherry 	pcireg_t id;
    207      1.2       uwe 	pcireg_t csr;
    208      1.1    cherry 
    209      1.1    cherry 	revision = PCI_REVISION(pa->pa_class);
    210      1.1    cherry 	if (revision != 0) {
    211      1.1    cherry 		aprint_normal(": unknown revision 0x%02x; giving up\n",
    212      1.1    cherry 			      revision);
    213      1.1    cherry 		return;
    214      1.1    cherry 	}
    215      1.1    cherry 	aprint_normal("\n");
    216      1.1    cherry 	aprint_naive("\n");
    217      1.1    cherry 
    218      1.1    cherry 	/* subsystem ID shows what I am */
    219      1.1    cherry 	id = pci_conf_read(pc, tag, PCI_SUBSYS_ID_REG);
    220      1.1    cherry 	aprint_normal_dev(self, "Virtio %s Device (rev. 0x%02x)\n",
    221      1.1    cherry 			  (PCI_SUBSYS_ID(id) < NDEVNAMES?
    222      1.1    cherry 			   virtio_device_name[PCI_SUBSYS_ID(id)] : "Unknown"),
    223      1.1    cherry 			  revision);
    224      1.1    cherry 
    225      1.2       uwe 	csr = pci_conf_read(pc, tag, PCI_COMMAND_STATUS_REG);
    226      1.2       uwe 	csr |= PCI_COMMAND_MASTER_ENABLE | PCI_COMMAND_IO_ENABLE;
    227      1.2       uwe 	pci_conf_write(pc, tag, PCI_COMMAND_STATUS_REG, csr);
    228      1.2       uwe 
    229      1.1    cherry 	sc->sc_dev = self;
    230  1.2.2.1  pgoyette 	sc->sc_ops = &virtio_pci_ops;
    231  1.2.2.1  pgoyette 	psc->sc_pa = *pa;
    232  1.2.2.1  pgoyette 	psc->sc_iot = pa->pa_iot;
    233      1.1    cherry 	if (pci_dma64_available(pa))
    234      1.1    cherry 		sc->sc_dmat = pa->pa_dmat64;
    235      1.1    cherry 	else
    236      1.1    cherry 		sc->sc_dmat = pa->pa_dmat;
    237  1.2.2.1  pgoyette 	psc->sc_config_offset = VIRTIO_CONFIG_DEVICE_CONFIG_NOMSI;
    238      1.1    cherry 
    239      1.1    cherry 	if (pci_mapreg_map(pa, PCI_MAPREG_START, PCI_MAPREG_TYPE_IO, 0,
    240  1.2.2.1  pgoyette 			   &psc->sc_iot, &psc->sc_ioh, NULL, &psc->sc_iosize)) {
    241      1.1    cherry 		aprint_error_dev(self, "can't map i/o space\n");
    242      1.1    cherry 		return;
    243      1.1    cherry 	}
    244      1.1    cherry 
    245      1.1    cherry 	virtio_device_reset(sc);
    246      1.1    cherry 	virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_ACK);
    247      1.1    cherry 	virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_DRIVER);
    248      1.1    cherry 
    249      1.1    cherry 	sc->sc_childdevid = PCI_SUBSYS_ID(id);
    250      1.1    cherry 	sc->sc_child = NULL;
    251  1.2.2.1  pgoyette 	virtio_pci_rescan(self, "virtio", 0);
    252      1.1    cherry 	return;
    253      1.1    cherry }
    254      1.1    cherry 
    255      1.1    cherry /* ARGSUSED */
    256      1.1    cherry static int
    257  1.2.2.1  pgoyette virtio_pci_rescan(device_t self, const char *attr, const int *scan_flags)
    258      1.1    cherry {
    259  1.2.2.1  pgoyette 	struct virtio_pci_softc * const psc = device_private(self);
    260  1.2.2.1  pgoyette 	struct virtio_softc * const sc = &psc->sc_sc;
    261      1.1    cherry 	struct virtio_attach_args va;
    262      1.1    cherry 
    263      1.1    cherry 	if (sc->sc_child)	/* Child already attached? */
    264      1.1    cherry 		return 0;
    265      1.1    cherry 
    266      1.1    cherry 	memset(&va, 0, sizeof(va));
    267      1.1    cherry 	va.sc_childdevid = sc->sc_childdevid;
    268      1.1    cherry 
    269      1.1    cherry 	config_found_ia(self, attr, &va, NULL);
    270      1.1    cherry 
    271      1.1    cherry 	if (sc->sc_child == NULL) {
    272      1.1    cherry 		aprint_error_dev(self,
    273      1.1    cherry 				 "no matching child driver; not configured\n");
    274      1.1    cherry 		return 0;
    275      1.1    cherry 	}
    276  1.2.2.1  pgoyette 
    277      1.1    cherry 	if (sc->sc_child == VIRTIO_CHILD_FAILED) {
    278      1.1    cherry 		aprint_error_dev(self,
    279      1.1    cherry 				 "virtio configuration failed\n");
    280      1.1    cherry 		return 0;
    281      1.1    cherry 	}
    282      1.1    cherry 
    283      1.1    cherry 	/*
    284      1.1    cherry 	 * Make sure child drivers initialize interrupts via call
    285      1.1    cherry 	 * to virtio_child_attach_finish().
    286      1.1    cherry 	 */
    287  1.2.2.1  pgoyette 	KASSERT(psc->sc_ihs_num != 0);
    288      1.1    cherry 
    289      1.1    cherry 	return 0;
    290      1.1    cherry }
    291      1.1    cherry 
    292      1.1    cherry 
    293      1.1    cherry static int
    294  1.2.2.1  pgoyette virtio_pci_detach(device_t self, int flags)
    295      1.1    cherry {
    296  1.2.2.1  pgoyette 	struct virtio_pci_softc * const psc = device_private(self);
    297  1.2.2.1  pgoyette 	struct virtio_softc * const sc = &psc->sc_sc;
    298      1.1    cherry 	int r;
    299      1.1    cherry 
    300      1.1    cherry 	if (sc->sc_child != NULL) {
    301      1.1    cherry 		r = config_detach(sc->sc_child, flags);
    302      1.1    cherry 		if (r)
    303      1.1    cherry 			return r;
    304      1.1    cherry 	}
    305      1.1    cherry 
    306      1.1    cherry 	/* Check that child detached properly */
    307      1.1    cherry 	KASSERT(sc->sc_child == NULL);
    308      1.1    cherry 	KASSERT(sc->sc_vqs == NULL);
    309  1.2.2.1  pgoyette 	KASSERT(psc->sc_ihs_num == 0);
    310  1.2.2.1  pgoyette 
    311  1.2.2.1  pgoyette 	if (psc->sc_iosize)
    312  1.2.2.1  pgoyette 		bus_space_unmap(psc->sc_iot, psc->sc_ioh, psc->sc_iosize);
    313  1.2.2.1  pgoyette 	psc->sc_iosize = 0;
    314  1.2.2.1  pgoyette 
    315  1.2.2.1  pgoyette 	return 0;
    316  1.2.2.1  pgoyette }
    317  1.2.2.1  pgoyette 
    318  1.2.2.1  pgoyette static void
    319  1.2.2.1  pgoyette virtio_pci_kick(struct virtio_softc *sc, uint16_t idx)
    320  1.2.2.1  pgoyette {
    321  1.2.2.1  pgoyette 	struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc;
    322  1.2.2.1  pgoyette 
    323  1.2.2.1  pgoyette 	bus_space_write_stream_2(psc->sc_iot, psc->sc_ioh,
    324  1.2.2.1  pgoyette 	    VIRTIO_CONFIG_QUEUE_NOTIFY, idx);
    325  1.2.2.1  pgoyette }
    326  1.2.2.1  pgoyette 
    327  1.2.2.1  pgoyette static uint8_t
    328  1.2.2.1  pgoyette virtio_pci_read_device_config_1(struct virtio_softc *sc, int index)
    329  1.2.2.1  pgoyette {
    330  1.2.2.1  pgoyette 	struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc;
    331  1.2.2.1  pgoyette 	return bus_space_read_stream_1(psc->sc_iot, psc->sc_ioh,
    332  1.2.2.1  pgoyette 	    psc->sc_config_offset + index);
    333  1.2.2.1  pgoyette }
    334  1.2.2.1  pgoyette 
    335  1.2.2.1  pgoyette static uint16_t
    336  1.2.2.1  pgoyette virtio_pci_read_device_config_2(struct virtio_softc *sc, int index)
    337  1.2.2.1  pgoyette {
    338  1.2.2.1  pgoyette 	struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc;
    339  1.2.2.1  pgoyette 	return bus_space_read_stream_2(psc->sc_iot, psc->sc_ioh,
    340  1.2.2.1  pgoyette 	    psc->sc_config_offset + index);
    341  1.2.2.1  pgoyette }
    342  1.2.2.1  pgoyette 
    343  1.2.2.1  pgoyette static uint32_t
    344  1.2.2.1  pgoyette virtio_pci_read_device_config_4(struct virtio_softc *sc, int index)
    345  1.2.2.1  pgoyette {
    346  1.2.2.1  pgoyette 	struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc;
    347  1.2.2.1  pgoyette 	return bus_space_read_stream_4(psc->sc_iot, psc->sc_ioh,
    348  1.2.2.1  pgoyette 	    psc->sc_config_offset + index);
    349  1.2.2.1  pgoyette }
    350  1.2.2.1  pgoyette 
    351  1.2.2.1  pgoyette static uint64_t
    352  1.2.2.1  pgoyette virtio_pci_read_device_config_8(struct virtio_softc *sc, int index)
    353  1.2.2.1  pgoyette {
    354  1.2.2.1  pgoyette 	struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc;
    355  1.2.2.1  pgoyette 	uint64_t r;
    356  1.2.2.1  pgoyette 
    357  1.2.2.1  pgoyette 	r = bus_space_read_stream_4(psc->sc_iot, psc->sc_ioh,
    358  1.2.2.1  pgoyette 	    psc->sc_config_offset + index + REG_HI_OFF);
    359  1.2.2.1  pgoyette 	r <<= 32;
    360  1.2.2.1  pgoyette 	r |= bus_space_read_stream_4(psc->sc_iot, psc->sc_ioh,
    361  1.2.2.1  pgoyette 	    psc->sc_config_offset + index + REG_LO_OFF);
    362  1.2.2.1  pgoyette 
    363  1.2.2.1  pgoyette 	return r;
    364  1.2.2.1  pgoyette }
    365  1.2.2.1  pgoyette 
    366  1.2.2.1  pgoyette static void
    367  1.2.2.1  pgoyette virtio_pci_write_device_config_1(struct virtio_softc *sc, int index,
    368  1.2.2.1  pgoyette     uint8_t value)
    369  1.2.2.1  pgoyette {
    370  1.2.2.1  pgoyette 	struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc;
    371  1.2.2.1  pgoyette 
    372  1.2.2.1  pgoyette 	bus_space_write_stream_1(psc->sc_iot, psc->sc_ioh,
    373  1.2.2.1  pgoyette 	    psc->sc_config_offset + index, value);
    374  1.2.2.1  pgoyette }
    375  1.2.2.1  pgoyette 
    376  1.2.2.1  pgoyette static void
    377  1.2.2.1  pgoyette virtio_pci_write_device_config_2(struct virtio_softc *sc, int index,
    378  1.2.2.1  pgoyette     uint16_t value)
    379  1.2.2.1  pgoyette {
    380  1.2.2.1  pgoyette 	struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc;
    381  1.2.2.1  pgoyette 
    382  1.2.2.1  pgoyette 	bus_space_write_stream_2(psc->sc_iot, psc->sc_ioh,
    383  1.2.2.1  pgoyette 	    psc->sc_config_offset + index, value);
    384  1.2.2.1  pgoyette }
    385  1.2.2.1  pgoyette 
    386  1.2.2.1  pgoyette static void
    387  1.2.2.1  pgoyette virtio_pci_write_device_config_4(struct virtio_softc *sc, int index,
    388  1.2.2.1  pgoyette     uint32_t value)
    389  1.2.2.1  pgoyette {
    390  1.2.2.1  pgoyette 	struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc;
    391  1.2.2.1  pgoyette 
    392  1.2.2.1  pgoyette 	bus_space_write_stream_4(psc->sc_iot, psc->sc_ioh,
    393  1.2.2.1  pgoyette 	    psc->sc_config_offset + index, value);
    394  1.2.2.1  pgoyette }
    395  1.2.2.1  pgoyette 
    396  1.2.2.1  pgoyette static void
    397  1.2.2.1  pgoyette virtio_pci_write_device_config_8(struct virtio_softc *sc, int index,
    398  1.2.2.1  pgoyette     uint64_t value)
    399  1.2.2.1  pgoyette {
    400  1.2.2.1  pgoyette 	struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc;
    401  1.2.2.1  pgoyette 
    402  1.2.2.1  pgoyette 	bus_space_write_stream_4(psc->sc_iot, psc->sc_ioh,
    403  1.2.2.1  pgoyette 	    psc->sc_config_offset + index + REG_LO_OFF,
    404  1.2.2.1  pgoyette 	    value & 0xffffffff);
    405  1.2.2.1  pgoyette 	bus_space_write_stream_4(psc->sc_iot, psc->sc_ioh,
    406  1.2.2.1  pgoyette 	    psc->sc_config_offset + index + REG_HI_OFF,
    407  1.2.2.1  pgoyette 	    value >> 32);
    408  1.2.2.1  pgoyette }
    409  1.2.2.1  pgoyette 
    410  1.2.2.1  pgoyette static uint16_t
    411  1.2.2.1  pgoyette virtio_pci_read_queue_size(struct virtio_softc *sc, uint16_t idx)
    412  1.2.2.1  pgoyette {
    413  1.2.2.1  pgoyette 	struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc;
    414  1.2.2.1  pgoyette 
    415  1.2.2.1  pgoyette 	bus_space_write_stream_2(psc->sc_iot, psc->sc_ioh,
    416  1.2.2.1  pgoyette 	    VIRTIO_CONFIG_QUEUE_SELECT, idx);
    417  1.2.2.1  pgoyette 	return bus_space_read_stream_2(psc->sc_iot, psc->sc_ioh,
    418  1.2.2.1  pgoyette 	    VIRTIO_CONFIG_QUEUE_SIZE);
    419  1.2.2.1  pgoyette }
    420  1.2.2.1  pgoyette 
    421  1.2.2.1  pgoyette static void
    422  1.2.2.1  pgoyette virtio_pci_setup_queue(struct virtio_softc *sc, uint16_t idx, uint32_t addr)
    423  1.2.2.1  pgoyette {
    424  1.2.2.1  pgoyette 	struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc;
    425  1.2.2.1  pgoyette 
    426  1.2.2.1  pgoyette 	bus_space_write_stream_2(psc->sc_iot, psc->sc_ioh,
    427  1.2.2.1  pgoyette 	    VIRTIO_CONFIG_QUEUE_SELECT, idx);
    428  1.2.2.1  pgoyette 	bus_space_write_stream_4(psc->sc_iot, psc->sc_ioh,
    429  1.2.2.1  pgoyette 	    VIRTIO_CONFIG_QUEUE_ADDRESS, addr);
    430  1.2.2.1  pgoyette 
    431  1.2.2.1  pgoyette 	if (psc->sc_ihs_num > 1) {
    432  1.2.2.1  pgoyette 		int vec = VIRTIO_MSIX_QUEUE_VECTOR_INDEX;
    433  1.2.2.2  pgoyette 		if (sc->sc_child_mq)
    434  1.2.2.1  pgoyette 			vec += idx;
    435  1.2.2.1  pgoyette 		bus_space_write_stream_2(psc->sc_iot, psc->sc_ioh,
    436  1.2.2.1  pgoyette 		    VIRTIO_CONFIG_MSI_QUEUE_VECTOR, vec);
    437  1.2.2.1  pgoyette 	}
    438  1.2.2.1  pgoyette }
    439  1.2.2.1  pgoyette 
    440  1.2.2.1  pgoyette static void
    441  1.2.2.1  pgoyette virtio_pci_set_status(struct virtio_softc *sc, int status)
    442  1.2.2.1  pgoyette {
    443  1.2.2.1  pgoyette 	struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc;
    444  1.2.2.1  pgoyette 	int old = 0;
    445  1.2.2.1  pgoyette 
    446  1.2.2.1  pgoyette 	if (status != 0) {
    447  1.2.2.1  pgoyette 	    old = bus_space_read_stream_1(psc->sc_iot, psc->sc_ioh,
    448  1.2.2.1  pgoyette 		VIRTIO_CONFIG_DEVICE_STATUS);
    449  1.2.2.1  pgoyette 	}
    450  1.2.2.1  pgoyette 	bus_space_write_stream_1(psc->sc_iot, psc->sc_ioh,
    451  1.2.2.1  pgoyette 	    VIRTIO_CONFIG_DEVICE_STATUS, status|old);
    452  1.2.2.1  pgoyette }
    453  1.2.2.1  pgoyette 
    454  1.2.2.1  pgoyette static uint32_t
    455  1.2.2.1  pgoyette virtio_pci_negotiate_features(struct virtio_softc *sc, uint32_t guest_features)
    456  1.2.2.1  pgoyette {
    457  1.2.2.1  pgoyette 	struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc;
    458  1.2.2.1  pgoyette 	uint32_t r;
    459  1.2.2.1  pgoyette 
    460  1.2.2.1  pgoyette 	r = bus_space_read_stream_4(psc->sc_iot, psc->sc_ioh,
    461  1.2.2.1  pgoyette 	    VIRTIO_CONFIG_DEVICE_FEATURES);
    462  1.2.2.1  pgoyette 	r &= guest_features;
    463  1.2.2.1  pgoyette 	bus_space_write_stream_4(psc->sc_iot, psc->sc_ioh,
    464  1.2.2.1  pgoyette 	    VIRTIO_CONFIG_GUEST_FEATURES, r);
    465  1.2.2.1  pgoyette 
    466  1.2.2.1  pgoyette 	return r;
    467  1.2.2.1  pgoyette }
    468  1.2.2.1  pgoyette 
    469  1.2.2.1  pgoyette 
    470  1.2.2.1  pgoyette static int
    471  1.2.2.1  pgoyette virtio_pci_setup_msix_vectors(struct virtio_softc *sc)
    472  1.2.2.1  pgoyette {
    473  1.2.2.1  pgoyette 	struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc;
    474  1.2.2.1  pgoyette 	int offset, vector, ret, qid;
    475  1.2.2.1  pgoyette 
    476  1.2.2.1  pgoyette 	offset = VIRTIO_CONFIG_MSI_CONFIG_VECTOR;
    477  1.2.2.1  pgoyette 	vector = VIRTIO_MSIX_CONFIG_VECTOR_INDEX;
    478      1.1    cherry 
    479  1.2.2.1  pgoyette 	bus_space_write_stream_2(psc->sc_iot, psc->sc_ioh, offset, vector);
    480  1.2.2.1  pgoyette 	ret = bus_space_read_stream_2(psc->sc_iot, psc->sc_ioh, offset);
    481  1.2.2.1  pgoyette 	aprint_debug_dev(sc->sc_dev, "expected=%d, actual=%d\n",
    482  1.2.2.1  pgoyette 	    vector, ret);
    483  1.2.2.1  pgoyette 	if (ret != vector)
    484  1.2.2.1  pgoyette 		return -1;
    485  1.2.2.1  pgoyette 
    486  1.2.2.1  pgoyette 	for (qid = 0; qid < sc->sc_nvqs; qid++) {
    487  1.2.2.1  pgoyette 		offset = VIRTIO_CONFIG_QUEUE_SELECT;
    488  1.2.2.1  pgoyette 		bus_space_write_stream_2(psc->sc_iot, psc->sc_ioh, offset, qid);
    489  1.2.2.1  pgoyette 
    490  1.2.2.1  pgoyette 		offset = VIRTIO_CONFIG_MSI_QUEUE_VECTOR;
    491  1.2.2.1  pgoyette 		vector = VIRTIO_MSIX_QUEUE_VECTOR_INDEX;
    492  1.2.2.1  pgoyette 
    493  1.2.2.2  pgoyette 		if (sc->sc_child_mq)
    494  1.2.2.2  pgoyette 			vector += qid;
    495  1.2.2.2  pgoyette 
    496  1.2.2.1  pgoyette 		bus_space_write_stream_2(psc->sc_iot, psc->sc_ioh, offset, vector);
    497  1.2.2.1  pgoyette 		ret = bus_space_read_stream_2(psc->sc_iot, psc->sc_ioh, offset);
    498  1.2.2.1  pgoyette 		aprint_debug_dev(sc->sc_dev, "expected=%d, actual=%d\n",
    499  1.2.2.1  pgoyette 		    vector, ret);
    500  1.2.2.1  pgoyette 		if (ret != vector)
    501  1.2.2.1  pgoyette 			return -1;
    502  1.2.2.1  pgoyette 	}
    503      1.1    cherry 
    504      1.1    cherry 	return 0;
    505      1.1    cherry }
    506  1.2.2.1  pgoyette 
    507  1.2.2.1  pgoyette static int
    508  1.2.2.1  pgoyette virtio_pci_setup_msix_interrupts(struct virtio_softc *sc,
    509  1.2.2.1  pgoyette     struct pci_attach_args *pa)
    510  1.2.2.1  pgoyette {
    511  1.2.2.1  pgoyette 	struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc;
    512  1.2.2.1  pgoyette 	device_t self = sc->sc_dev;
    513  1.2.2.1  pgoyette 	pci_chipset_tag_t pc = pa->pa_pc;
    514  1.2.2.1  pgoyette 	char intrbuf[PCI_INTRSTR_LEN];
    515  1.2.2.2  pgoyette 	char intr_xname[INTRDEVNAMEBUF];
    516  1.2.2.1  pgoyette 	char const *intrstr;
    517  1.2.2.2  pgoyette 	int idx, qid, n;
    518  1.2.2.1  pgoyette 
    519  1.2.2.1  pgoyette 	idx = VIRTIO_MSIX_CONFIG_VECTOR_INDEX;
    520  1.2.2.1  pgoyette 	if (sc->sc_flags & VIRTIO_F_PCI_INTR_MPSAFE)
    521  1.2.2.1  pgoyette 		pci_intr_setattr(pc, &psc->sc_ihp[idx], PCI_INTR_MPSAFE, true);
    522  1.2.2.1  pgoyette 
    523  1.2.2.2  pgoyette 	snprintf(intr_xname, sizeof(intr_xname), "%s config",
    524  1.2.2.2  pgoyette 	    device_xname(sc->sc_dev));
    525  1.2.2.2  pgoyette 
    526  1.2.2.1  pgoyette 	psc->sc_ihs[idx] = pci_intr_establish_xname(pc, psc->sc_ihp[idx],
    527  1.2.2.2  pgoyette 	    sc->sc_ipl, virtio_pci_msix_config_intr, sc, intr_xname);
    528  1.2.2.1  pgoyette 	if (psc->sc_ihs[idx] == NULL) {
    529  1.2.2.1  pgoyette 		aprint_error_dev(self, "couldn't establish MSI-X for config\n");
    530  1.2.2.1  pgoyette 		goto error;
    531  1.2.2.1  pgoyette 	}
    532  1.2.2.1  pgoyette 
    533  1.2.2.1  pgoyette 	idx = VIRTIO_MSIX_QUEUE_VECTOR_INDEX;
    534  1.2.2.2  pgoyette 	if (sc->sc_child_mq) {
    535  1.2.2.2  pgoyette 		for (qid = 0; qid < sc->sc_nvqs; qid++) {
    536  1.2.2.2  pgoyette 			n = idx + qid;
    537  1.2.2.2  pgoyette 
    538  1.2.2.2  pgoyette 			snprintf(intr_xname, sizeof(intr_xname), "%s vq#%d",
    539  1.2.2.2  pgoyette 			    device_xname(sc->sc_dev), qid);
    540  1.2.2.2  pgoyette 
    541  1.2.2.2  pgoyette 			if (sc->sc_flags & VIRTIO_F_PCI_INTR_MPSAFE) {
    542  1.2.2.2  pgoyette 				pci_intr_setattr(pc, &psc->sc_ihp[n],
    543  1.2.2.2  pgoyette 				    PCI_INTR_MPSAFE, true);
    544  1.2.2.2  pgoyette 			}
    545  1.2.2.2  pgoyette 
    546  1.2.2.2  pgoyette 			psc->sc_ihs[n] = pci_intr_establish_xname(pc, psc->sc_ihp[n],
    547  1.2.2.2  pgoyette 			    sc->sc_ipl, virtio_pci_msix_vq_intr, &sc->sc_vqs[qid],
    548  1.2.2.2  pgoyette 			    intr_xname);
    549  1.2.2.2  pgoyette 			if (psc->sc_ihs[n] == NULL) {
    550  1.2.2.2  pgoyette 				aprint_error_dev(self, "couldn't establish MSI-X for a vq\n");
    551  1.2.2.2  pgoyette 				goto error;
    552  1.2.2.2  pgoyette 			}
    553  1.2.2.2  pgoyette 		}
    554  1.2.2.2  pgoyette 	} else {
    555  1.2.2.2  pgoyette 		if (sc->sc_flags & VIRTIO_F_PCI_INTR_MPSAFE)
    556  1.2.2.2  pgoyette 			pci_intr_setattr(pc, &psc->sc_ihp[idx], PCI_INTR_MPSAFE, true);
    557  1.2.2.1  pgoyette 
    558  1.2.2.2  pgoyette 		snprintf(intr_xname, sizeof(intr_xname), "%s queues",
    559  1.2.2.2  pgoyette 		    device_xname(sc->sc_dev));
    560  1.2.2.2  pgoyette 		psc->sc_ihs[idx] = pci_intr_establish_xname(pc, psc->sc_ihp[idx],
    561  1.2.2.2  pgoyette 		    sc->sc_ipl, virtio_pci_msix_queue_intr, sc, intr_xname);
    562  1.2.2.2  pgoyette 		if (psc->sc_ihs[idx] == NULL) {
    563  1.2.2.2  pgoyette 			aprint_error_dev(self, "couldn't establish MSI-X for queues\n");
    564  1.2.2.2  pgoyette 			goto error;
    565  1.2.2.2  pgoyette 		}
    566  1.2.2.1  pgoyette 	}
    567  1.2.2.1  pgoyette 
    568  1.2.2.1  pgoyette 	if (virtio_pci_setup_msix_vectors(sc) != 0) {
    569  1.2.2.1  pgoyette 		aprint_error_dev(self, "couldn't setup MSI-X vectors\n");
    570  1.2.2.1  pgoyette 		goto error;
    571  1.2.2.1  pgoyette 	}
    572  1.2.2.1  pgoyette 
    573  1.2.2.1  pgoyette 	idx = VIRTIO_MSIX_CONFIG_VECTOR_INDEX;
    574  1.2.2.1  pgoyette 	intrstr = pci_intr_string(pc, psc->sc_ihp[idx], intrbuf, sizeof(intrbuf));
    575  1.2.2.1  pgoyette 	aprint_normal_dev(self, "config interrupting at %s\n", intrstr);
    576  1.2.2.1  pgoyette 	idx = VIRTIO_MSIX_QUEUE_VECTOR_INDEX;
    577  1.2.2.2  pgoyette 	if (sc->sc_child_mq) {
    578  1.2.2.2  pgoyette 		kcpuset_t *affinity;
    579  1.2.2.2  pgoyette 		int affinity_to, r;
    580  1.2.2.2  pgoyette 
    581  1.2.2.2  pgoyette 		kcpuset_create(&affinity, false);
    582  1.2.2.2  pgoyette 
    583  1.2.2.2  pgoyette 		for (qid = 0; qid < sc->sc_nvqs; qid++) {
    584  1.2.2.2  pgoyette 			n = idx + qid;
    585  1.2.2.2  pgoyette 			affinity_to = (qid / 2) % ncpu;
    586  1.2.2.2  pgoyette 
    587  1.2.2.2  pgoyette 			intrstr = pci_intr_string(pc, psc->sc_ihp[n],
    588  1.2.2.2  pgoyette 			    intrbuf, sizeof(intrbuf));
    589  1.2.2.2  pgoyette 
    590  1.2.2.2  pgoyette 			kcpuset_zero(affinity);
    591  1.2.2.2  pgoyette 			kcpuset_set(affinity, affinity_to);
    592  1.2.2.2  pgoyette 			r = interrupt_distribute(psc->sc_ihs[n], affinity, NULL);
    593  1.2.2.2  pgoyette 			if (r == 0) {
    594  1.2.2.2  pgoyette 				aprint_normal_dev(self,
    595  1.2.2.2  pgoyette 				    "for vq #%d interrupting at %s affinity to %u\n",
    596  1.2.2.2  pgoyette 				    qid, intrstr, affinity_to);
    597  1.2.2.2  pgoyette 			} else {
    598  1.2.2.2  pgoyette 				aprint_normal_dev(self,
    599  1.2.2.2  pgoyette 				    "for vq #%d interrupting at %s\n",
    600  1.2.2.2  pgoyette 				    qid, intrstr);
    601  1.2.2.2  pgoyette 			}
    602  1.2.2.2  pgoyette 		}
    603  1.2.2.2  pgoyette 
    604  1.2.2.2  pgoyette 		kcpuset_destroy(affinity);
    605  1.2.2.2  pgoyette 	} else {
    606  1.2.2.2  pgoyette 		intrstr = pci_intr_string(pc, psc->sc_ihp[idx], intrbuf, sizeof(intrbuf));
    607  1.2.2.2  pgoyette 		aprint_normal_dev(self, "queues interrupting at %s\n", intrstr);
    608  1.2.2.2  pgoyette 	}
    609  1.2.2.1  pgoyette 
    610  1.2.2.1  pgoyette 	return 0;
    611  1.2.2.1  pgoyette 
    612  1.2.2.1  pgoyette error:
    613  1.2.2.1  pgoyette 	idx = VIRTIO_MSIX_CONFIG_VECTOR_INDEX;
    614  1.2.2.1  pgoyette 	if (psc->sc_ihs[idx] != NULL)
    615  1.2.2.1  pgoyette 		pci_intr_disestablish(psc->sc_pa.pa_pc, psc->sc_ihs[idx]);
    616  1.2.2.1  pgoyette 	idx = VIRTIO_MSIX_QUEUE_VECTOR_INDEX;
    617  1.2.2.2  pgoyette 	if (sc->sc_child_mq) {
    618  1.2.2.2  pgoyette 		for (qid = 0; qid < sc->sc_nvqs; qid++) {
    619  1.2.2.2  pgoyette 			n = idx + qid;
    620  1.2.2.2  pgoyette 			if (psc->sc_ihs[n] == NULL)
    621  1.2.2.2  pgoyette 				continue;
    622  1.2.2.2  pgoyette 			pci_intr_disestablish(psc->sc_pa.pa_pc, psc->sc_ihs[n]);
    623  1.2.2.2  pgoyette 		}
    624  1.2.2.2  pgoyette 
    625  1.2.2.2  pgoyette 	} else {
    626  1.2.2.2  pgoyette 		if (psc->sc_ihs[idx] != NULL)
    627  1.2.2.2  pgoyette 			pci_intr_disestablish(psc->sc_pa.pa_pc, psc->sc_ihs[idx]);
    628  1.2.2.2  pgoyette 	}
    629  1.2.2.1  pgoyette 
    630  1.2.2.1  pgoyette 	return -1;
    631  1.2.2.1  pgoyette }
    632  1.2.2.1  pgoyette 
    633  1.2.2.1  pgoyette static int
    634  1.2.2.1  pgoyette virtio_pci_setup_intx_interrupt(struct virtio_softc *sc,
    635  1.2.2.1  pgoyette     struct pci_attach_args *pa)
    636  1.2.2.1  pgoyette {
    637  1.2.2.1  pgoyette 	struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc;
    638  1.2.2.1  pgoyette 	device_t self = sc->sc_dev;
    639  1.2.2.1  pgoyette 	pci_chipset_tag_t pc = pa->pa_pc;
    640  1.2.2.1  pgoyette 	char intrbuf[PCI_INTRSTR_LEN];
    641  1.2.2.1  pgoyette 	char const *intrstr;
    642  1.2.2.1  pgoyette 
    643  1.2.2.1  pgoyette 	if (sc->sc_flags & VIRTIO_F_PCI_INTR_MPSAFE)
    644  1.2.2.1  pgoyette 		pci_intr_setattr(pc, &psc->sc_ihp[0], PCI_INTR_MPSAFE, true);
    645  1.2.2.1  pgoyette 
    646  1.2.2.1  pgoyette 	psc->sc_ihs[0] = pci_intr_establish_xname(pc, psc->sc_ihp[0],
    647  1.2.2.1  pgoyette 	    sc->sc_ipl, virtio_pci_intr, sc, device_xname(sc->sc_dev));
    648  1.2.2.1  pgoyette 	if (psc->sc_ihs[0] == NULL) {
    649  1.2.2.1  pgoyette 		aprint_error_dev(self, "couldn't establish INTx\n");
    650  1.2.2.1  pgoyette 		return -1;
    651  1.2.2.1  pgoyette 	}
    652  1.2.2.1  pgoyette 
    653  1.2.2.1  pgoyette 	intrstr = pci_intr_string(pc, psc->sc_ihp[0], intrbuf, sizeof(intrbuf));
    654  1.2.2.1  pgoyette 	aprint_normal_dev(self, "interrupting at %s\n", intrstr);
    655  1.2.2.1  pgoyette 
    656  1.2.2.1  pgoyette 	return 0;
    657  1.2.2.1  pgoyette }
    658  1.2.2.1  pgoyette 
    659  1.2.2.1  pgoyette static int
    660  1.2.2.1  pgoyette virtio_pci_setup_interrupts(struct virtio_softc *sc)
    661  1.2.2.1  pgoyette {
    662  1.2.2.1  pgoyette 	struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc;
    663  1.2.2.1  pgoyette 	device_t self = sc->sc_dev;
    664  1.2.2.1  pgoyette 	pci_chipset_tag_t pc = psc->sc_pa.pa_pc;
    665  1.2.2.1  pgoyette 	int error;
    666  1.2.2.1  pgoyette 	int nmsix;
    667  1.2.2.1  pgoyette 	int counts[PCI_INTR_TYPE_SIZE];
    668  1.2.2.1  pgoyette 	pci_intr_type_t max_type;
    669  1.2.2.1  pgoyette 
    670  1.2.2.1  pgoyette 	nmsix = pci_msix_count(psc->sc_pa.pa_pc, psc->sc_pa.pa_tag);
    671  1.2.2.1  pgoyette 	aprint_debug_dev(self, "pci_msix_count=%d\n", nmsix);
    672  1.2.2.1  pgoyette 
    673  1.2.2.1  pgoyette 	/* We need at least two: one for config and the other for queues */
    674  1.2.2.1  pgoyette 	if ((sc->sc_flags & VIRTIO_F_PCI_INTR_MSIX) == 0 || nmsix < 2) {
    675  1.2.2.1  pgoyette 		/* Try INTx only */
    676  1.2.2.1  pgoyette 		max_type = PCI_INTR_TYPE_INTX;
    677  1.2.2.1  pgoyette 		counts[PCI_INTR_TYPE_INTX] = 1;
    678  1.2.2.1  pgoyette 	} else {
    679  1.2.2.1  pgoyette 		/* Try MSI-X first and INTx second */
    680  1.2.2.2  pgoyette 		if (sc->sc_child_mq &&
    681  1.2.2.2  pgoyette 		    sc->sc_nvqs > (nmsix - VIRTIO_MSIX_QUEUE_VECTOR_INDEX)) {
    682  1.2.2.2  pgoyette 			nmsix = 2;
    683  1.2.2.2  pgoyette 			sc->sc_child_mq = false;
    684  1.2.2.2  pgoyette 		}
    685  1.2.2.2  pgoyette 
    686  1.2.2.1  pgoyette 		max_type = PCI_INTR_TYPE_MSIX;
    687  1.2.2.2  pgoyette 		counts[PCI_INTR_TYPE_MSIX] = nmsix;
    688  1.2.2.1  pgoyette 		counts[PCI_INTR_TYPE_MSI] = 0;
    689  1.2.2.1  pgoyette 		counts[PCI_INTR_TYPE_INTX] = 1;
    690  1.2.2.1  pgoyette 	}
    691  1.2.2.1  pgoyette 
    692  1.2.2.1  pgoyette retry:
    693  1.2.2.1  pgoyette 	error = pci_intr_alloc(&psc->sc_pa, &psc->sc_ihp, counts, max_type);
    694  1.2.2.1  pgoyette 	if (error != 0) {
    695  1.2.2.1  pgoyette 		aprint_error_dev(self, "couldn't map interrupt\n");
    696  1.2.2.1  pgoyette 		return -1;
    697  1.2.2.1  pgoyette 	}
    698  1.2.2.1  pgoyette 
    699  1.2.2.1  pgoyette 	if (pci_intr_type(pc, psc->sc_ihp[0]) == PCI_INTR_TYPE_MSIX) {
    700  1.2.2.2  pgoyette 		psc->sc_ihs = kmem_alloc(sizeof(*psc->sc_ihs) * nmsix,
    701  1.2.2.1  pgoyette 		    KM_SLEEP);
    702  1.2.2.1  pgoyette 
    703  1.2.2.1  pgoyette 		error = virtio_pci_setup_msix_interrupts(sc, &psc->sc_pa);
    704  1.2.2.1  pgoyette 		if (error != 0) {
    705  1.2.2.2  pgoyette 			kmem_free(psc->sc_ihs, sizeof(*psc->sc_ihs) * nmsix);
    706  1.2.2.2  pgoyette 			pci_intr_release(pc, psc->sc_ihp, nmsix);
    707  1.2.2.1  pgoyette 
    708  1.2.2.1  pgoyette 			/* Retry INTx */
    709  1.2.2.1  pgoyette 			max_type = PCI_INTR_TYPE_INTX;
    710  1.2.2.1  pgoyette 			counts[PCI_INTR_TYPE_INTX] = 1;
    711  1.2.2.1  pgoyette 			goto retry;
    712  1.2.2.1  pgoyette 		}
    713  1.2.2.1  pgoyette 
    714  1.2.2.2  pgoyette 		psc->sc_ihs_num = nmsix;
    715  1.2.2.1  pgoyette 		psc->sc_config_offset = VIRTIO_CONFIG_DEVICE_CONFIG_MSI;
    716  1.2.2.1  pgoyette 	} else if (pci_intr_type(pc, psc->sc_ihp[0]) == PCI_INTR_TYPE_INTX) {
    717  1.2.2.1  pgoyette 		psc->sc_ihs = kmem_alloc(sizeof(*psc->sc_ihs) * 1,
    718  1.2.2.1  pgoyette 		    KM_SLEEP);
    719  1.2.2.1  pgoyette 
    720  1.2.2.1  pgoyette 		error = virtio_pci_setup_intx_interrupt(sc, &psc->sc_pa);
    721  1.2.2.1  pgoyette 		if (error != 0) {
    722  1.2.2.1  pgoyette 			kmem_free(psc->sc_ihs, sizeof(*psc->sc_ihs) * 1);
    723  1.2.2.1  pgoyette 			pci_intr_release(pc, psc->sc_ihp, 1);
    724  1.2.2.1  pgoyette 			return -1;
    725  1.2.2.1  pgoyette 		}
    726  1.2.2.1  pgoyette 
    727  1.2.2.1  pgoyette 		psc->sc_ihs_num = 1;
    728  1.2.2.1  pgoyette 		psc->sc_config_offset = VIRTIO_CONFIG_DEVICE_CONFIG_NOMSI;
    729  1.2.2.1  pgoyette 	}
    730  1.2.2.1  pgoyette 
    731  1.2.2.1  pgoyette 	return 0;
    732  1.2.2.1  pgoyette }
    733  1.2.2.1  pgoyette 
    734  1.2.2.1  pgoyette static void
    735  1.2.2.1  pgoyette virtio_pci_free_interrupts(struct virtio_softc *sc)
    736  1.2.2.1  pgoyette {
    737  1.2.2.1  pgoyette 	struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc;
    738  1.2.2.1  pgoyette 
    739  1.2.2.1  pgoyette 	for (int i = 0; i < psc->sc_ihs_num; i++) {
    740  1.2.2.1  pgoyette 		if (psc->sc_ihs[i] == NULL)
    741  1.2.2.1  pgoyette 			continue;
    742  1.2.2.1  pgoyette 		pci_intr_disestablish(psc->sc_pa.pa_pc, psc->sc_ihs[i]);
    743  1.2.2.1  pgoyette 		psc->sc_ihs[i] = NULL;
    744  1.2.2.1  pgoyette 	}
    745  1.2.2.1  pgoyette 
    746  1.2.2.1  pgoyette 	if (psc->sc_ihs_num > 0)
    747  1.2.2.1  pgoyette 		pci_intr_release(psc->sc_pa.pa_pc, psc->sc_ihp, psc->sc_ihs_num);
    748  1.2.2.1  pgoyette 
    749  1.2.2.1  pgoyette 	if (psc->sc_ihs != NULL) {
    750  1.2.2.1  pgoyette 		kmem_free(psc->sc_ihs, sizeof(*psc->sc_ihs) * psc->sc_ihs_num);
    751  1.2.2.1  pgoyette 		psc->sc_ihs = NULL;
    752  1.2.2.1  pgoyette 	}
    753  1.2.2.1  pgoyette 	psc->sc_ihs_num = 0;
    754  1.2.2.1  pgoyette }
    755  1.2.2.1  pgoyette 
    756  1.2.2.1  pgoyette /*
    757  1.2.2.1  pgoyette  * Interrupt handler.
    758  1.2.2.1  pgoyette  */
    759  1.2.2.1  pgoyette static int
    760  1.2.2.1  pgoyette virtio_pci_intr(void *arg)
    761  1.2.2.1  pgoyette {
    762  1.2.2.1  pgoyette 	struct virtio_softc *sc = arg;
    763  1.2.2.1  pgoyette 	struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc;
    764  1.2.2.1  pgoyette 	int isr, r = 0;
    765  1.2.2.1  pgoyette 
    766  1.2.2.1  pgoyette 	/* check and ack the interrupt */
    767  1.2.2.1  pgoyette 	isr = bus_space_read_stream_1(psc->sc_iot, psc->sc_ioh,
    768  1.2.2.1  pgoyette 			       VIRTIO_CONFIG_ISR_STATUS);
    769  1.2.2.1  pgoyette 	if (isr == 0)
    770  1.2.2.1  pgoyette 		return 0;
    771  1.2.2.1  pgoyette 	if ((isr & VIRTIO_CONFIG_ISR_CONFIG_CHANGE) &&
    772  1.2.2.1  pgoyette 	    (sc->sc_config_change != NULL))
    773  1.2.2.1  pgoyette 		r = (sc->sc_config_change)(sc);
    774  1.2.2.1  pgoyette 	if (sc->sc_intrhand != NULL) {
    775  1.2.2.1  pgoyette 		if (sc->sc_soft_ih != NULL)
    776  1.2.2.1  pgoyette 			softint_schedule(sc->sc_soft_ih);
    777  1.2.2.1  pgoyette 		else
    778  1.2.2.1  pgoyette 			r |= (sc->sc_intrhand)(sc);
    779  1.2.2.1  pgoyette 	}
    780  1.2.2.1  pgoyette 
    781  1.2.2.1  pgoyette 	return r;
    782  1.2.2.1  pgoyette }
    783  1.2.2.1  pgoyette 
    784  1.2.2.1  pgoyette static int
    785  1.2.2.1  pgoyette virtio_pci_msix_queue_intr(void *arg)
    786  1.2.2.1  pgoyette {
    787  1.2.2.1  pgoyette 	struct virtio_softc *sc = arg;
    788  1.2.2.1  pgoyette 	int r = 0;
    789  1.2.2.1  pgoyette 
    790  1.2.2.1  pgoyette 	if (sc->sc_intrhand != NULL) {
    791  1.2.2.1  pgoyette 		if (sc->sc_soft_ih != NULL)
    792  1.2.2.1  pgoyette 			softint_schedule(sc->sc_soft_ih);
    793  1.2.2.1  pgoyette 		else
    794  1.2.2.1  pgoyette 			r |= (sc->sc_intrhand)(sc);
    795  1.2.2.1  pgoyette 	}
    796  1.2.2.1  pgoyette 
    797  1.2.2.1  pgoyette 	return r;
    798  1.2.2.1  pgoyette }
    799  1.2.2.1  pgoyette 
    800  1.2.2.1  pgoyette static int
    801  1.2.2.2  pgoyette virtio_pci_msix_vq_intr(void *arg)
    802  1.2.2.2  pgoyette {
    803  1.2.2.2  pgoyette 	struct virtqueue *vq = arg;
    804  1.2.2.2  pgoyette 	int r = 0;
    805  1.2.2.2  pgoyette 
    806  1.2.2.2  pgoyette 	if (vq->vq_intrhand != NULL) {
    807  1.2.2.2  pgoyette 		if (vq->vq_soft_ih)
    808  1.2.2.2  pgoyette 			softint_schedule(vq->vq_soft_ih);
    809  1.2.2.2  pgoyette 		else
    810  1.2.2.2  pgoyette 			r |= vq->vq_intrhand(vq);
    811  1.2.2.2  pgoyette 	}
    812  1.2.2.2  pgoyette 
    813  1.2.2.2  pgoyette 	return r;
    814  1.2.2.2  pgoyette }
    815  1.2.2.2  pgoyette 
    816  1.2.2.2  pgoyette static int
    817  1.2.2.1  pgoyette virtio_pci_msix_config_intr(void *arg)
    818  1.2.2.1  pgoyette {
    819  1.2.2.1  pgoyette 	struct virtio_softc *sc = arg;
    820  1.2.2.1  pgoyette 	int r = 0;
    821  1.2.2.1  pgoyette 
    822  1.2.2.1  pgoyette 	if (sc->sc_config_change != NULL)
    823  1.2.2.1  pgoyette 		r = (sc->sc_config_change)(sc);
    824  1.2.2.1  pgoyette 	return r;
    825  1.2.2.1  pgoyette }
    826  1.2.2.1  pgoyette 
    827  1.2.2.1  pgoyette MODULE(MODULE_CLASS_DRIVER, virtio_pci, "pci,virtio");
    828  1.2.2.1  pgoyette 
    829  1.2.2.1  pgoyette #ifdef _MODULE
    830  1.2.2.1  pgoyette #include "ioconf.c"
    831  1.2.2.1  pgoyette #endif
    832  1.2.2.1  pgoyette 
    833  1.2.2.1  pgoyette static int
    834  1.2.2.1  pgoyette virtio_pci_modcmd(modcmd_t cmd, void *opaque)
    835  1.2.2.1  pgoyette {
    836  1.2.2.1  pgoyette 	int error = 0;
    837  1.2.2.1  pgoyette 
    838  1.2.2.1  pgoyette #ifdef _MODULE
    839  1.2.2.1  pgoyette 	switch (cmd) {
    840  1.2.2.1  pgoyette 	case MODULE_CMD_INIT:
    841  1.2.2.1  pgoyette 		error = config_init_component(cfdriver_ioconf_virtio_pci,
    842  1.2.2.1  pgoyette 		    cfattach_ioconf_virtio_pci, cfdata_ioconf_virtio_pci);
    843  1.2.2.1  pgoyette 		break;
    844  1.2.2.1  pgoyette 	case MODULE_CMD_FINI:
    845  1.2.2.1  pgoyette 		error = config_fini_component(cfdriver_ioconf_virtio_pci,
    846  1.2.2.1  pgoyette 		    cfattach_ioconf_virtio_pci, cfdata_ioconf_virtio_pci);
    847  1.2.2.1  pgoyette 		break;
    848  1.2.2.1  pgoyette 	default:
    849  1.2.2.1  pgoyette 		error = ENOTTY;
    850  1.2.2.1  pgoyette 		break;
    851  1.2.2.1  pgoyette 	}
    852  1.2.2.1  pgoyette #endif
    853  1.2.2.1  pgoyette 
    854  1.2.2.1  pgoyette 	return error;
    855  1.2.2.1  pgoyette }
    856  1.2.2.1  pgoyette 
    857