Home | History | Annotate | Line # | Download | only in pci
virtio_pci.c revision 1.7.4.1
      1  1.7.4.1    martin /* $NetBSD: virtio_pci.c,v 1.7.4.1 2020/09/20 10:03:11 martin Exp $ */
      2      1.1    cherry 
      3      1.1    cherry /*
      4      1.1    cherry  * Copyright (c) 2010 Minoura Makoto.
      5      1.1    cherry  * All rights reserved.
      6      1.1    cherry  *
      7      1.1    cherry  * Redistribution and use in source and binary forms, with or without
      8      1.1    cherry  * modification, are permitted provided that the following conditions
      9      1.1    cherry  * are met:
     10      1.1    cherry  * 1. Redistributions of source code must retain the above copyright
     11      1.1    cherry  *    notice, this list of conditions and the following disclaimer.
     12      1.1    cherry  * 2. Redistributions in binary form must reproduce the above copyright
     13      1.1    cherry  *    notice, this list of conditions and the following disclaimer in the
     14      1.1    cherry  *    documentation and/or other materials provided with the distribution.
     15      1.1    cherry  *
     16      1.1    cherry  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
     17      1.1    cherry  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
     18      1.1    cherry  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
     19      1.1    cherry  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
     20      1.1    cherry  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
     21      1.1    cherry  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
     22      1.1    cherry  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
     23      1.1    cherry  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
     24      1.1    cherry  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
     25      1.1    cherry  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
     26      1.1    cherry  */
     27      1.1    cherry 
     28      1.1    cherry #include <sys/cdefs.h>
     29  1.7.4.1    martin __KERNEL_RCSID(0, "$NetBSD: virtio_pci.c,v 1.7.4.1 2020/09/20 10:03:11 martin Exp $");
     30      1.1    cherry 
     31      1.1    cherry #include <sys/param.h>
     32      1.1    cherry #include <sys/systm.h>
     33      1.4  jakllsch #include <sys/kmem.h>
     34      1.5  jakllsch #include <sys/module.h>
     35      1.6  yamaguch #include <sys/interrupt.h>
     36      1.1    cherry 
     37      1.1    cherry #include <sys/device.h>
     38      1.1    cherry 
     39      1.1    cherry #include <dev/pci/pcidevs.h>
     40      1.1    cherry #include <dev/pci/pcireg.h>
     41      1.1    cherry #include <dev/pci/pcivar.h>
     42      1.1    cherry 
     43      1.1    cherry #define VIRTIO_PRIVATE
     44      1.1    cherry 
     45      1.1    cherry #include <dev/pci/virtioreg.h> /* XXX: move to non-pci */
     46      1.1    cherry #include <dev/pci/virtiovar.h> /* XXX: move to non-pci */
     47      1.1    cherry 
     48      1.4  jakllsch static int	virtio_pci_match(device_t, cfdata_t, void *);
     49      1.4  jakllsch static void	virtio_pci_attach(device_t, device_t, void *);
     50      1.4  jakllsch static int	virtio_pci_rescan(device_t, const char *, const int *);
     51      1.4  jakllsch static int	virtio_pci_detach(device_t, int);
     52      1.4  jakllsch 
     53      1.4  jakllsch struct virtio_pci_softc {
     54      1.4  jakllsch 	struct virtio_softc	sc_sc;
     55      1.4  jakllsch 	bus_space_tag_t		sc_iot;
     56      1.4  jakllsch 	bus_space_handle_t	sc_ioh;
     57      1.4  jakllsch 	bus_size_t		sc_iosize;
     58      1.4  jakllsch 	struct pci_attach_args	sc_pa;
     59      1.4  jakllsch 	pci_intr_handle_t	*sc_ihp;
     60      1.4  jakllsch 	void			**sc_ihs;
     61      1.4  jakllsch 	int			sc_ihs_num;
     62      1.4  jakllsch 	int			sc_config_offset;
     63      1.4  jakllsch };
     64      1.4  jakllsch 
     65      1.4  jakllsch static void	virtio_pci_kick(struct virtio_softc *, uint16_t);
     66      1.4  jakllsch static uint8_t	virtio_pci_read_device_config_1(struct virtio_softc *, int);
     67      1.4  jakllsch static uint16_t	virtio_pci_read_device_config_2(struct virtio_softc *, int);
     68      1.4  jakllsch static uint32_t	virtio_pci_read_device_config_4(struct virtio_softc *, int);
     69      1.4  jakllsch static uint64_t	virtio_pci_read_device_config_8(struct virtio_softc *, int);
     70      1.4  jakllsch static void 	virtio_pci_write_device_config_1(struct virtio_softc *, int, uint8_t);
     71      1.4  jakllsch static void	virtio_pci_write_device_config_2(struct virtio_softc *, int, uint16_t);
     72      1.4  jakllsch static void	virtio_pci_write_device_config_4(struct virtio_softc *, int, uint32_t);
     73      1.4  jakllsch static void	virtio_pci_write_device_config_8(struct virtio_softc *, int, uint64_t);
     74      1.4  jakllsch static uint16_t	virtio_pci_read_queue_size(struct virtio_softc *, uint16_t);
     75      1.4  jakllsch static void	virtio_pci_setup_queue(struct virtio_softc *, uint16_t, uint32_t);
     76      1.4  jakllsch static void	virtio_pci_set_status(struct virtio_softc *, int);
     77      1.4  jakllsch static uint32_t	virtio_pci_negotiate_features(struct virtio_softc *, uint32_t);
     78      1.4  jakllsch static int	virtio_pci_setup_interrupts(struct virtio_softc *);
     79      1.4  jakllsch static void	virtio_pci_free_interrupts(struct virtio_softc *);
     80      1.4  jakllsch 
     81      1.4  jakllsch static int	virtio_pci_intr(void *arg);
     82      1.4  jakllsch static int	virtio_pci_msix_queue_intr(void *);
     83      1.6  yamaguch static int	virtio_pci_msix_vq_intr(void *);
     84      1.4  jakllsch static int	virtio_pci_msix_config_intr(void *);
     85      1.4  jakllsch static int	virtio_pci_setup_msix_vectors(struct virtio_softc *);
     86      1.4  jakllsch static int	virtio_pci_setup_msix_interrupts(struct virtio_softc *,
     87      1.4  jakllsch 		    struct pci_attach_args *);
     88      1.4  jakllsch static int	virtio_pci_setup_intx_interrupt(struct virtio_softc *,
     89      1.4  jakllsch 		    struct pci_attach_args *);
     90      1.4  jakllsch 
     91      1.4  jakllsch #define VIRTIO_MSIX_CONFIG_VECTOR_INDEX	0
     92      1.4  jakllsch #define VIRTIO_MSIX_QUEUE_VECTOR_INDEX	1
     93      1.4  jakllsch 
     94      1.4  jakllsch /* we use the legacy virtio spec, so the PCI registers are host native
     95      1.4  jakllsch  * byte order, not PCI (i.e. LE) byte order */
     96      1.4  jakllsch #if BYTE_ORDER == BIG_ENDIAN
     97      1.4  jakllsch #define REG_HI_OFF      0
     98      1.4  jakllsch #define REG_LO_OFF      4
     99      1.4  jakllsch #ifndef __BUS_SPACE_HAS_STREAM_METHODS
    100      1.4  jakllsch #define bus_space_read_stream_1 bus_space_read_1
    101      1.4  jakllsch #define bus_space_write_stream_1 bus_space_write_1
    102      1.4  jakllsch static inline uint16_t
    103      1.4  jakllsch bus_space_read_stream_2(bus_space_tag_t t, bus_space_handle_t h,
    104      1.4  jakllsch     bus_size_t o)
    105      1.4  jakllsch {
    106      1.4  jakllsch 	return le16toh(bus_space_read_2(t, h, o));
    107      1.4  jakllsch }
    108      1.4  jakllsch static inline void
    109      1.4  jakllsch bus_space_write_stream_2(bus_space_tag_t t, bus_space_handle_t h,
    110      1.4  jakllsch     bus_size_t o, uint16_t v)
    111      1.4  jakllsch {
    112      1.4  jakllsch 	bus_space_write_2(t, h, o, htole16(v));
    113      1.4  jakllsch }
    114      1.4  jakllsch static inline uint32_t
    115      1.4  jakllsch bus_space_read_stream_4(bus_space_tag_t t, bus_space_handle_t h,
    116      1.4  jakllsch     bus_size_t o)
    117      1.4  jakllsch {
    118      1.4  jakllsch 	return le32toh(bus_space_read_4(t, h, o));
    119      1.4  jakllsch }
    120      1.4  jakllsch static inline void
    121      1.4  jakllsch bus_space_write_stream_4(bus_space_tag_t t, bus_space_handle_t h,
    122      1.4  jakllsch     bus_size_t o, uint32_t v)
    123      1.4  jakllsch {
    124      1.4  jakllsch 	bus_space_write_4(t, h, o, htole32(v));
    125      1.4  jakllsch }
    126      1.4  jakllsch #endif
    127      1.4  jakllsch #else
    128      1.4  jakllsch #define REG_HI_OFF	4
    129      1.4  jakllsch #define REG_LO_OFF	0
    130      1.4  jakllsch #ifndef __BUS_SPACE_HAS_STREAM_METHODS
    131      1.4  jakllsch #define bus_space_read_stream_1 bus_space_read_1
    132      1.4  jakllsch #define bus_space_read_stream_2 bus_space_read_2
    133      1.4  jakllsch #define bus_space_read_stream_4 bus_space_read_4
    134      1.4  jakllsch #define bus_space_write_stream_1 bus_space_write_1
    135      1.4  jakllsch #define bus_space_write_stream_2 bus_space_write_2
    136      1.4  jakllsch #define bus_space_write_stream_4 bus_space_write_4
    137      1.4  jakllsch #endif
    138      1.4  jakllsch #endif
    139      1.4  jakllsch 
    140      1.1    cherry 
    141      1.1    cherry static const char *virtio_device_name[] = {
    142      1.1    cherry 	"Unknown (0)",			/* 0 */
    143      1.1    cherry 	"Network",			/* 1 */
    144      1.1    cherry 	"Block",			/* 2 */
    145      1.1    cherry 	"Console",			/* 3 */
    146      1.1    cherry 	"Entropy",			/* 4 */
    147      1.1    cherry 	"Memory Balloon",		/* 5 */
    148      1.1    cherry 	"I/O Memory",			/* 6 */
    149      1.1    cherry 	"Remote Processor Messaging",	/* 7 */
    150      1.1    cherry 	"SCSI",				/* 8 */
    151      1.1    cherry 	"9P Transport",			/* 9 */
    152      1.1    cherry 	"mac80211 wlan",		/* 10 */
    153      1.1    cherry };
    154      1.1    cherry #define NDEVNAMES	__arraycount(virtio_device_name)
    155      1.1    cherry 
    156      1.4  jakllsch CFATTACH_DECL3_NEW(virtio_pci, sizeof(struct virtio_pci_softc),
    157      1.4  jakllsch     virtio_pci_match, virtio_pci_attach, virtio_pci_detach, NULL,
    158      1.4  jakllsch     virtio_pci_rescan, NULL, DVF_DETACH_SHUTDOWN);
    159      1.4  jakllsch 
    160      1.4  jakllsch static const struct virtio_ops virtio_pci_ops = {
    161      1.4  jakllsch 	.kick = virtio_pci_kick,
    162      1.4  jakllsch 	.read_dev_cfg_1 = virtio_pci_read_device_config_1,
    163      1.4  jakllsch 	.read_dev_cfg_2 = virtio_pci_read_device_config_2,
    164      1.4  jakllsch 	.read_dev_cfg_4 = virtio_pci_read_device_config_4,
    165      1.4  jakllsch 	.read_dev_cfg_8 = virtio_pci_read_device_config_8,
    166      1.4  jakllsch 	.write_dev_cfg_1 = virtio_pci_write_device_config_1,
    167      1.4  jakllsch 	.write_dev_cfg_2 = virtio_pci_write_device_config_2,
    168      1.4  jakllsch 	.write_dev_cfg_4 = virtio_pci_write_device_config_4,
    169      1.4  jakllsch 	.write_dev_cfg_8 = virtio_pci_write_device_config_8,
    170      1.4  jakllsch 	.read_queue_size = virtio_pci_read_queue_size,
    171      1.4  jakllsch 	.setup_queue = virtio_pci_setup_queue,
    172      1.4  jakllsch 	.set_status = virtio_pci_set_status,
    173      1.4  jakllsch 	.neg_features = virtio_pci_negotiate_features,
    174      1.4  jakllsch 	.setup_interrupts = virtio_pci_setup_interrupts,
    175      1.4  jakllsch 	.free_interrupts = virtio_pci_free_interrupts,
    176      1.4  jakllsch };
    177      1.1    cherry 
    178      1.1    cherry static int
    179      1.4  jakllsch virtio_pci_match(device_t parent, cfdata_t match, void *aux)
    180      1.1    cherry {
    181      1.1    cherry 	struct pci_attach_args *pa;
    182      1.1    cherry 
    183      1.1    cherry 	pa = (struct pci_attach_args *)aux;
    184      1.1    cherry 	switch (PCI_VENDOR(pa->pa_id)) {
    185      1.1    cherry 	case PCI_VENDOR_QUMRANET:
    186      1.1    cherry 		if ((PCI_PRODUCT_QUMRANET_VIRTIO_1000 <=
    187      1.1    cherry 		     PCI_PRODUCT(pa->pa_id)) &&
    188      1.1    cherry 		    (PCI_PRODUCT(pa->pa_id) <=
    189      1.1    cherry 		     PCI_PRODUCT_QUMRANET_VIRTIO_103F))
    190      1.1    cherry 			return 1;
    191      1.1    cherry 		break;
    192      1.1    cherry 	}
    193      1.1    cherry 
    194      1.1    cherry 	return 0;
    195      1.1    cherry }
    196      1.1    cherry 
    197      1.1    cherry static void
    198      1.4  jakllsch virtio_pci_attach(device_t parent, device_t self, void *aux)
    199      1.1    cherry {
    200      1.4  jakllsch 	struct virtio_pci_softc * const psc = device_private(self);
    201      1.4  jakllsch 	struct virtio_softc * const sc = &psc->sc_sc;
    202      1.1    cherry 	struct pci_attach_args *pa = (struct pci_attach_args *)aux;
    203      1.1    cherry 	pci_chipset_tag_t pc = pa->pa_pc;
    204      1.1    cherry 	pcitag_t tag = pa->pa_tag;
    205      1.1    cherry 	int revision;
    206      1.1    cherry 	pcireg_t id;
    207      1.2       uwe 	pcireg_t csr;
    208      1.1    cherry 
    209      1.1    cherry 	revision = PCI_REVISION(pa->pa_class);
    210      1.1    cherry 	if (revision != 0) {
    211      1.1    cherry 		aprint_normal(": unknown revision 0x%02x; giving up\n",
    212      1.1    cherry 			      revision);
    213      1.1    cherry 		return;
    214      1.1    cherry 	}
    215      1.1    cherry 	aprint_normal("\n");
    216      1.1    cherry 	aprint_naive("\n");
    217      1.1    cherry 
    218      1.1    cherry 	/* subsystem ID shows what I am */
    219      1.1    cherry 	id = pci_conf_read(pc, tag, PCI_SUBSYS_ID_REG);
    220      1.1    cherry 	aprint_normal_dev(self, "Virtio %s Device (rev. 0x%02x)\n",
    221      1.1    cherry 			  (PCI_SUBSYS_ID(id) < NDEVNAMES?
    222      1.1    cherry 			   virtio_device_name[PCI_SUBSYS_ID(id)] : "Unknown"),
    223      1.1    cherry 			  revision);
    224      1.1    cherry 
    225      1.2       uwe 	csr = pci_conf_read(pc, tag, PCI_COMMAND_STATUS_REG);
    226      1.2       uwe 	csr |= PCI_COMMAND_MASTER_ENABLE | PCI_COMMAND_IO_ENABLE;
    227      1.2       uwe 	pci_conf_write(pc, tag, PCI_COMMAND_STATUS_REG, csr);
    228      1.2       uwe 
    229      1.1    cherry 	sc->sc_dev = self;
    230      1.4  jakllsch 	sc->sc_ops = &virtio_pci_ops;
    231      1.4  jakllsch 	psc->sc_pa = *pa;
    232      1.4  jakllsch 	psc->sc_iot = pa->pa_iot;
    233      1.1    cherry 	if (pci_dma64_available(pa))
    234      1.1    cherry 		sc->sc_dmat = pa->pa_dmat64;
    235      1.1    cherry 	else
    236      1.1    cherry 		sc->sc_dmat = pa->pa_dmat;
    237      1.4  jakllsch 	psc->sc_config_offset = VIRTIO_CONFIG_DEVICE_CONFIG_NOMSI;
    238      1.1    cherry 
    239      1.1    cherry 	if (pci_mapreg_map(pa, PCI_MAPREG_START, PCI_MAPREG_TYPE_IO, 0,
    240      1.4  jakllsch 			   &psc->sc_iot, &psc->sc_ioh, NULL, &psc->sc_iosize)) {
    241      1.1    cherry 		aprint_error_dev(self, "can't map i/o space\n");
    242      1.1    cherry 		return;
    243      1.1    cherry 	}
    244      1.1    cherry 
    245      1.1    cherry 	virtio_device_reset(sc);
    246      1.1    cherry 	virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_ACK);
    247      1.1    cherry 	virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_DRIVER);
    248      1.1    cherry 
    249      1.1    cherry 	sc->sc_childdevid = PCI_SUBSYS_ID(id);
    250      1.1    cherry 	sc->sc_child = NULL;
    251      1.4  jakllsch 	virtio_pci_rescan(self, "virtio", 0);
    252      1.1    cherry 	return;
    253      1.1    cherry }
    254      1.1    cherry 
    255      1.1    cherry /* ARGSUSED */
    256      1.1    cherry static int
    257      1.4  jakllsch virtio_pci_rescan(device_t self, const char *attr, const int *scan_flags)
    258      1.1    cherry {
    259      1.4  jakllsch 	struct virtio_pci_softc * const psc = device_private(self);
    260      1.4  jakllsch 	struct virtio_softc * const sc = &psc->sc_sc;
    261      1.1    cherry 	struct virtio_attach_args va;
    262      1.1    cherry 
    263      1.1    cherry 	if (sc->sc_child)	/* Child already attached? */
    264      1.1    cherry 		return 0;
    265      1.1    cherry 
    266      1.1    cherry 	memset(&va, 0, sizeof(va));
    267      1.1    cherry 	va.sc_childdevid = sc->sc_childdevid;
    268      1.1    cherry 
    269      1.1    cherry 	config_found_ia(self, attr, &va, NULL);
    270      1.1    cherry 
    271      1.1    cherry 	if (sc->sc_child == NULL) {
    272      1.1    cherry 		aprint_error_dev(self,
    273      1.1    cherry 				 "no matching child driver; not configured\n");
    274      1.1    cherry 		return 0;
    275      1.1    cherry 	}
    276      1.4  jakllsch 
    277      1.1    cherry 	if (sc->sc_child == VIRTIO_CHILD_FAILED) {
    278      1.1    cherry 		aprint_error_dev(self,
    279      1.1    cherry 				 "virtio configuration failed\n");
    280      1.1    cherry 		return 0;
    281      1.1    cherry 	}
    282      1.1    cherry 
    283      1.1    cherry 	/*
    284      1.1    cherry 	 * Make sure child drivers initialize interrupts via call
    285      1.1    cherry 	 * to virtio_child_attach_finish().
    286      1.1    cherry 	 */
    287      1.4  jakllsch 	KASSERT(psc->sc_ihs_num != 0);
    288      1.1    cherry 
    289      1.1    cherry 	return 0;
    290      1.1    cherry }
    291      1.1    cherry 
    292      1.1    cherry 
    293      1.1    cherry static int
    294      1.4  jakllsch virtio_pci_detach(device_t self, int flags)
    295      1.1    cherry {
    296      1.4  jakllsch 	struct virtio_pci_softc * const psc = device_private(self);
    297      1.4  jakllsch 	struct virtio_softc * const sc = &psc->sc_sc;
    298      1.1    cherry 	int r;
    299      1.1    cherry 
    300      1.1    cherry 	if (sc->sc_child != NULL) {
    301      1.1    cherry 		r = config_detach(sc->sc_child, flags);
    302      1.1    cherry 		if (r)
    303      1.1    cherry 			return r;
    304      1.1    cherry 	}
    305      1.1    cherry 
    306      1.1    cherry 	/* Check that child detached properly */
    307      1.1    cherry 	KASSERT(sc->sc_child == NULL);
    308      1.1    cherry 	KASSERT(sc->sc_vqs == NULL);
    309      1.4  jakllsch 	KASSERT(psc->sc_ihs_num == 0);
    310      1.1    cherry 
    311      1.4  jakllsch 	if (psc->sc_iosize)
    312      1.4  jakllsch 		bus_space_unmap(psc->sc_iot, psc->sc_ioh, psc->sc_iosize);
    313      1.4  jakllsch 	psc->sc_iosize = 0;
    314      1.1    cherry 
    315      1.1    cherry 	return 0;
    316      1.1    cherry }
    317      1.4  jakllsch 
    318      1.4  jakllsch static void
    319      1.4  jakllsch virtio_pci_kick(struct virtio_softc *sc, uint16_t idx)
    320      1.4  jakllsch {
    321      1.4  jakllsch 	struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc;
    322      1.4  jakllsch 
    323      1.4  jakllsch 	bus_space_write_stream_2(psc->sc_iot, psc->sc_ioh,
    324      1.4  jakllsch 	    VIRTIO_CONFIG_QUEUE_NOTIFY, idx);
    325      1.4  jakllsch }
    326      1.4  jakllsch 
    327      1.4  jakllsch static uint8_t
    328      1.4  jakllsch virtio_pci_read_device_config_1(struct virtio_softc *sc, int index)
    329      1.4  jakllsch {
    330      1.4  jakllsch 	struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc;
    331      1.4  jakllsch 	return bus_space_read_stream_1(psc->sc_iot, psc->sc_ioh,
    332      1.4  jakllsch 	    psc->sc_config_offset + index);
    333      1.4  jakllsch }
    334      1.4  jakllsch 
    335      1.4  jakllsch static uint16_t
    336      1.4  jakllsch virtio_pci_read_device_config_2(struct virtio_softc *sc, int index)
    337      1.4  jakllsch {
    338      1.4  jakllsch 	struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc;
    339      1.4  jakllsch 	return bus_space_read_stream_2(psc->sc_iot, psc->sc_ioh,
    340      1.4  jakllsch 	    psc->sc_config_offset + index);
    341      1.4  jakllsch }
    342      1.4  jakllsch 
    343      1.4  jakllsch static uint32_t
    344      1.4  jakllsch virtio_pci_read_device_config_4(struct virtio_softc *sc, int index)
    345      1.4  jakllsch {
    346      1.4  jakllsch 	struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc;
    347      1.4  jakllsch 	return bus_space_read_stream_4(psc->sc_iot, psc->sc_ioh,
    348      1.4  jakllsch 	    psc->sc_config_offset + index);
    349      1.4  jakllsch }
    350      1.4  jakllsch 
    351      1.4  jakllsch static uint64_t
    352      1.4  jakllsch virtio_pci_read_device_config_8(struct virtio_softc *sc, int index)
    353      1.4  jakllsch {
    354      1.4  jakllsch 	struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc;
    355      1.4  jakllsch 	uint64_t r;
    356      1.4  jakllsch 
    357      1.4  jakllsch 	r = bus_space_read_stream_4(psc->sc_iot, psc->sc_ioh,
    358      1.4  jakllsch 	    psc->sc_config_offset + index + REG_HI_OFF);
    359      1.4  jakllsch 	r <<= 32;
    360      1.4  jakllsch 	r |= bus_space_read_stream_4(psc->sc_iot, psc->sc_ioh,
    361      1.4  jakllsch 	    psc->sc_config_offset + index + REG_LO_OFF);
    362      1.4  jakllsch 
    363      1.4  jakllsch 	return r;
    364      1.4  jakllsch }
    365      1.4  jakllsch 
    366      1.4  jakllsch static void
    367      1.4  jakllsch virtio_pci_write_device_config_1(struct virtio_softc *sc, int index,
    368      1.4  jakllsch     uint8_t value)
    369      1.4  jakllsch {
    370      1.4  jakllsch 	struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc;
    371      1.4  jakllsch 
    372      1.4  jakllsch 	bus_space_write_stream_1(psc->sc_iot, psc->sc_ioh,
    373      1.4  jakllsch 	    psc->sc_config_offset + index, value);
    374      1.4  jakllsch }
    375      1.4  jakllsch 
    376      1.4  jakllsch static void
    377      1.4  jakllsch virtio_pci_write_device_config_2(struct virtio_softc *sc, int index,
    378      1.4  jakllsch     uint16_t value)
    379      1.4  jakllsch {
    380      1.4  jakllsch 	struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc;
    381      1.4  jakllsch 
    382      1.4  jakllsch 	bus_space_write_stream_2(psc->sc_iot, psc->sc_ioh,
    383      1.4  jakllsch 	    psc->sc_config_offset + index, value);
    384      1.4  jakllsch }
    385      1.4  jakllsch 
    386      1.4  jakllsch static void
    387      1.4  jakllsch virtio_pci_write_device_config_4(struct virtio_softc *sc, int index,
    388      1.4  jakllsch     uint32_t value)
    389      1.4  jakllsch {
    390      1.4  jakllsch 	struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc;
    391      1.4  jakllsch 
    392      1.4  jakllsch 	bus_space_write_stream_4(psc->sc_iot, psc->sc_ioh,
    393      1.4  jakllsch 	    psc->sc_config_offset + index, value);
    394      1.4  jakllsch }
    395      1.4  jakllsch 
    396      1.4  jakllsch static void
    397      1.4  jakllsch virtio_pci_write_device_config_8(struct virtio_softc *sc, int index,
    398      1.4  jakllsch     uint64_t value)
    399      1.4  jakllsch {
    400      1.4  jakllsch 	struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc;
    401      1.4  jakllsch 
    402      1.4  jakllsch 	bus_space_write_stream_4(psc->sc_iot, psc->sc_ioh,
    403      1.4  jakllsch 	    psc->sc_config_offset + index + REG_LO_OFF,
    404      1.4  jakllsch 	    value & 0xffffffff);
    405      1.4  jakllsch 	bus_space_write_stream_4(psc->sc_iot, psc->sc_ioh,
    406      1.4  jakllsch 	    psc->sc_config_offset + index + REG_HI_OFF,
    407      1.4  jakllsch 	    value >> 32);
    408      1.4  jakllsch }
    409      1.4  jakllsch 
    410      1.4  jakllsch static uint16_t
    411      1.4  jakllsch virtio_pci_read_queue_size(struct virtio_softc *sc, uint16_t idx)
    412      1.4  jakllsch {
    413      1.4  jakllsch 	struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc;
    414      1.4  jakllsch 
    415      1.4  jakllsch 	bus_space_write_stream_2(psc->sc_iot, psc->sc_ioh,
    416      1.4  jakllsch 	    VIRTIO_CONFIG_QUEUE_SELECT, idx);
    417      1.4  jakllsch 	return bus_space_read_stream_2(psc->sc_iot, psc->sc_ioh,
    418      1.4  jakllsch 	    VIRTIO_CONFIG_QUEUE_SIZE);
    419      1.4  jakllsch }
    420      1.4  jakllsch 
    421      1.4  jakllsch static void
    422      1.4  jakllsch virtio_pci_setup_queue(struct virtio_softc *sc, uint16_t idx, uint32_t addr)
    423      1.4  jakllsch {
    424      1.4  jakllsch 	struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc;
    425      1.4  jakllsch 
    426      1.4  jakllsch 	bus_space_write_stream_2(psc->sc_iot, psc->sc_ioh,
    427      1.4  jakllsch 	    VIRTIO_CONFIG_QUEUE_SELECT, idx);
    428      1.4  jakllsch 	bus_space_write_stream_4(psc->sc_iot, psc->sc_ioh,
    429      1.4  jakllsch 	    VIRTIO_CONFIG_QUEUE_ADDRESS, addr);
    430      1.4  jakllsch 
    431      1.4  jakllsch 	if (psc->sc_ihs_num > 1) {
    432      1.4  jakllsch 		int vec = VIRTIO_MSIX_QUEUE_VECTOR_INDEX;
    433      1.6  yamaguch 		if (sc->sc_child_mq)
    434      1.4  jakllsch 			vec += idx;
    435      1.4  jakllsch 		bus_space_write_stream_2(psc->sc_iot, psc->sc_ioh,
    436      1.4  jakllsch 		    VIRTIO_CONFIG_MSI_QUEUE_VECTOR, vec);
    437      1.4  jakllsch 	}
    438      1.4  jakllsch }
    439      1.4  jakllsch 
    440      1.4  jakllsch static void
    441      1.4  jakllsch virtio_pci_set_status(struct virtio_softc *sc, int status)
    442      1.4  jakllsch {
    443      1.4  jakllsch 	struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc;
    444      1.4  jakllsch 	int old = 0;
    445      1.4  jakllsch 
    446      1.4  jakllsch 	if (status != 0) {
    447      1.4  jakllsch 	    old = bus_space_read_stream_1(psc->sc_iot, psc->sc_ioh,
    448      1.4  jakllsch 		VIRTIO_CONFIG_DEVICE_STATUS);
    449      1.4  jakllsch 	}
    450      1.4  jakllsch 	bus_space_write_stream_1(psc->sc_iot, psc->sc_ioh,
    451      1.4  jakllsch 	    VIRTIO_CONFIG_DEVICE_STATUS, status|old);
    452      1.4  jakllsch }
    453      1.4  jakllsch 
    454      1.4  jakllsch static uint32_t
    455      1.4  jakllsch virtio_pci_negotiate_features(struct virtio_softc *sc, uint32_t guest_features)
    456      1.4  jakllsch {
    457      1.4  jakllsch 	struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc;
    458      1.4  jakllsch 	uint32_t r;
    459      1.4  jakllsch 
    460      1.4  jakllsch 	r = bus_space_read_stream_4(psc->sc_iot, psc->sc_ioh,
    461      1.4  jakllsch 	    VIRTIO_CONFIG_DEVICE_FEATURES);
    462      1.4  jakllsch 	r &= guest_features;
    463      1.4  jakllsch 	bus_space_write_stream_4(psc->sc_iot, psc->sc_ioh,
    464      1.4  jakllsch 	    VIRTIO_CONFIG_GUEST_FEATURES, r);
    465      1.4  jakllsch 
    466      1.4  jakllsch 	return r;
    467      1.4  jakllsch }
    468      1.4  jakllsch 
    469      1.4  jakllsch 
    470      1.4  jakllsch static int
    471      1.4  jakllsch virtio_pci_setup_msix_vectors(struct virtio_softc *sc)
    472      1.4  jakllsch {
    473      1.4  jakllsch 	struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc;
    474      1.4  jakllsch 	int offset, vector, ret, qid;
    475      1.4  jakllsch 
    476      1.4  jakllsch 	offset = VIRTIO_CONFIG_MSI_CONFIG_VECTOR;
    477      1.4  jakllsch 	vector = VIRTIO_MSIX_CONFIG_VECTOR_INDEX;
    478      1.4  jakllsch 
    479      1.4  jakllsch 	bus_space_write_stream_2(psc->sc_iot, psc->sc_ioh, offset, vector);
    480      1.4  jakllsch 	ret = bus_space_read_stream_2(psc->sc_iot, psc->sc_ioh, offset);
    481      1.4  jakllsch 	aprint_debug_dev(sc->sc_dev, "expected=%d, actual=%d\n",
    482      1.4  jakllsch 	    vector, ret);
    483      1.4  jakllsch 	if (ret != vector)
    484      1.4  jakllsch 		return -1;
    485      1.4  jakllsch 
    486      1.4  jakllsch 	for (qid = 0; qid < sc->sc_nvqs; qid++) {
    487      1.4  jakllsch 		offset = VIRTIO_CONFIG_QUEUE_SELECT;
    488      1.4  jakllsch 		bus_space_write_stream_2(psc->sc_iot, psc->sc_ioh, offset, qid);
    489      1.4  jakllsch 
    490      1.4  jakllsch 		offset = VIRTIO_CONFIG_MSI_QUEUE_VECTOR;
    491      1.4  jakllsch 		vector = VIRTIO_MSIX_QUEUE_VECTOR_INDEX;
    492      1.4  jakllsch 
    493      1.6  yamaguch 		if (sc->sc_child_mq)
    494      1.6  yamaguch 			vector += qid;
    495      1.6  yamaguch 
    496      1.4  jakllsch 		bus_space_write_stream_2(psc->sc_iot, psc->sc_ioh, offset, vector);
    497      1.4  jakllsch 		ret = bus_space_read_stream_2(psc->sc_iot, psc->sc_ioh, offset);
    498      1.4  jakllsch 		aprint_debug_dev(sc->sc_dev, "expected=%d, actual=%d\n",
    499      1.4  jakllsch 		    vector, ret);
    500      1.4  jakllsch 		if (ret != vector)
    501      1.4  jakllsch 			return -1;
    502      1.4  jakllsch 	}
    503      1.4  jakllsch 
    504      1.4  jakllsch 	return 0;
    505      1.4  jakllsch }
    506      1.4  jakllsch 
    507      1.4  jakllsch static int
    508      1.4  jakllsch virtio_pci_setup_msix_interrupts(struct virtio_softc *sc,
    509      1.4  jakllsch     struct pci_attach_args *pa)
    510      1.4  jakllsch {
    511      1.4  jakllsch 	struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc;
    512      1.4  jakllsch 	device_t self = sc->sc_dev;
    513      1.4  jakllsch 	pci_chipset_tag_t pc = pa->pa_pc;
    514      1.4  jakllsch 	char intrbuf[PCI_INTRSTR_LEN];
    515      1.6  yamaguch 	char intr_xname[INTRDEVNAMEBUF];
    516      1.4  jakllsch 	char const *intrstr;
    517      1.6  yamaguch 	int idx, qid, n;
    518      1.4  jakllsch 
    519      1.4  jakllsch 	idx = VIRTIO_MSIX_CONFIG_VECTOR_INDEX;
    520      1.4  jakllsch 	if (sc->sc_flags & VIRTIO_F_PCI_INTR_MPSAFE)
    521      1.4  jakllsch 		pci_intr_setattr(pc, &psc->sc_ihp[idx], PCI_INTR_MPSAFE, true);
    522      1.4  jakllsch 
    523      1.6  yamaguch 	snprintf(intr_xname, sizeof(intr_xname), "%s config",
    524      1.6  yamaguch 	    device_xname(sc->sc_dev));
    525      1.6  yamaguch 
    526      1.4  jakllsch 	psc->sc_ihs[idx] = pci_intr_establish_xname(pc, psc->sc_ihp[idx],
    527      1.6  yamaguch 	    sc->sc_ipl, virtio_pci_msix_config_intr, sc, intr_xname);
    528      1.4  jakllsch 	if (psc->sc_ihs[idx] == NULL) {
    529      1.4  jakllsch 		aprint_error_dev(self, "couldn't establish MSI-X for config\n");
    530      1.4  jakllsch 		goto error;
    531      1.4  jakllsch 	}
    532      1.4  jakllsch 
    533      1.4  jakllsch 	idx = VIRTIO_MSIX_QUEUE_VECTOR_INDEX;
    534      1.6  yamaguch 	if (sc->sc_child_mq) {
    535      1.6  yamaguch 		for (qid = 0; qid < sc->sc_nvqs; qid++) {
    536      1.6  yamaguch 			n = idx + qid;
    537      1.6  yamaguch 
    538      1.6  yamaguch 			snprintf(intr_xname, sizeof(intr_xname), "%s vq#%d",
    539      1.6  yamaguch 			    device_xname(sc->sc_dev), qid);
    540      1.6  yamaguch 
    541      1.6  yamaguch 			if (sc->sc_flags & VIRTIO_F_PCI_INTR_MPSAFE) {
    542      1.6  yamaguch 				pci_intr_setattr(pc, &psc->sc_ihp[n],
    543      1.6  yamaguch 				    PCI_INTR_MPSAFE, true);
    544      1.6  yamaguch 			}
    545      1.6  yamaguch 
    546      1.6  yamaguch 			psc->sc_ihs[n] = pci_intr_establish_xname(pc, psc->sc_ihp[n],
    547      1.6  yamaguch 			    sc->sc_ipl, virtio_pci_msix_vq_intr, &sc->sc_vqs[qid],
    548      1.6  yamaguch 			    intr_xname);
    549      1.6  yamaguch 			if (psc->sc_ihs[n] == NULL) {
    550      1.6  yamaguch 				aprint_error_dev(self, "couldn't establish MSI-X for a vq\n");
    551      1.6  yamaguch 				goto error;
    552      1.6  yamaguch 			}
    553      1.6  yamaguch 		}
    554      1.6  yamaguch 	} else {
    555      1.6  yamaguch 		if (sc->sc_flags & VIRTIO_F_PCI_INTR_MPSAFE)
    556      1.6  yamaguch 			pci_intr_setattr(pc, &psc->sc_ihp[idx], PCI_INTR_MPSAFE, true);
    557      1.4  jakllsch 
    558      1.6  yamaguch 		snprintf(intr_xname, sizeof(intr_xname), "%s queues",
    559      1.6  yamaguch 		    device_xname(sc->sc_dev));
    560      1.6  yamaguch 		psc->sc_ihs[idx] = pci_intr_establish_xname(pc, psc->sc_ihp[idx],
    561      1.6  yamaguch 		    sc->sc_ipl, virtio_pci_msix_queue_intr, sc, intr_xname);
    562      1.6  yamaguch 		if (psc->sc_ihs[idx] == NULL) {
    563      1.6  yamaguch 			aprint_error_dev(self, "couldn't establish MSI-X for queues\n");
    564      1.6  yamaguch 			goto error;
    565      1.6  yamaguch 		}
    566      1.4  jakllsch 	}
    567      1.4  jakllsch 
    568      1.4  jakllsch 	if (virtio_pci_setup_msix_vectors(sc) != 0) {
    569      1.4  jakllsch 		aprint_error_dev(self, "couldn't setup MSI-X vectors\n");
    570      1.4  jakllsch 		goto error;
    571      1.4  jakllsch 	}
    572      1.4  jakllsch 
    573      1.4  jakllsch 	idx = VIRTIO_MSIX_CONFIG_VECTOR_INDEX;
    574      1.4  jakllsch 	intrstr = pci_intr_string(pc, psc->sc_ihp[idx], intrbuf, sizeof(intrbuf));
    575      1.4  jakllsch 	aprint_normal_dev(self, "config interrupting at %s\n", intrstr);
    576      1.4  jakllsch 	idx = VIRTIO_MSIX_QUEUE_VECTOR_INDEX;
    577      1.6  yamaguch 	if (sc->sc_child_mq) {
    578      1.6  yamaguch 		kcpuset_t *affinity;
    579      1.6  yamaguch 		int affinity_to, r;
    580      1.6  yamaguch 
    581      1.6  yamaguch 		kcpuset_create(&affinity, false);
    582      1.6  yamaguch 
    583      1.6  yamaguch 		for (qid = 0; qid < sc->sc_nvqs; qid++) {
    584      1.6  yamaguch 			n = idx + qid;
    585      1.6  yamaguch 			affinity_to = (qid / 2) % ncpu;
    586      1.6  yamaguch 
    587      1.6  yamaguch 			intrstr = pci_intr_string(pc, psc->sc_ihp[n],
    588      1.6  yamaguch 			    intrbuf, sizeof(intrbuf));
    589      1.6  yamaguch 
    590      1.6  yamaguch 			kcpuset_zero(affinity);
    591      1.6  yamaguch 			kcpuset_set(affinity, affinity_to);
    592      1.6  yamaguch 			r = interrupt_distribute(psc->sc_ihs[n], affinity, NULL);
    593      1.6  yamaguch 			if (r == 0) {
    594      1.6  yamaguch 				aprint_normal_dev(self,
    595      1.6  yamaguch 				    "for vq #%d interrupting at %s affinity to %u\n",
    596      1.6  yamaguch 				    qid, intrstr, affinity_to);
    597      1.6  yamaguch 			} else {
    598      1.6  yamaguch 				aprint_normal_dev(self,
    599      1.6  yamaguch 				    "for vq #%d interrupting at %s\n",
    600      1.6  yamaguch 				    qid, intrstr);
    601      1.6  yamaguch 			}
    602      1.6  yamaguch 		}
    603      1.6  yamaguch 
    604      1.6  yamaguch 		kcpuset_destroy(affinity);
    605      1.6  yamaguch 	} else {
    606      1.6  yamaguch 		intrstr = pci_intr_string(pc, psc->sc_ihp[idx], intrbuf, sizeof(intrbuf));
    607      1.6  yamaguch 		aprint_normal_dev(self, "queues interrupting at %s\n", intrstr);
    608      1.6  yamaguch 	}
    609      1.4  jakllsch 
    610      1.4  jakllsch 	return 0;
    611      1.4  jakllsch 
    612      1.4  jakllsch error:
    613      1.4  jakllsch 	idx = VIRTIO_MSIX_CONFIG_VECTOR_INDEX;
    614      1.4  jakllsch 	if (psc->sc_ihs[idx] != NULL)
    615      1.4  jakllsch 		pci_intr_disestablish(psc->sc_pa.pa_pc, psc->sc_ihs[idx]);
    616      1.4  jakllsch 	idx = VIRTIO_MSIX_QUEUE_VECTOR_INDEX;
    617      1.6  yamaguch 	if (sc->sc_child_mq) {
    618      1.6  yamaguch 		for (qid = 0; qid < sc->sc_nvqs; qid++) {
    619      1.6  yamaguch 			n = idx + qid;
    620      1.6  yamaguch 			if (psc->sc_ihs[n] == NULL)
    621      1.6  yamaguch 				continue;
    622      1.6  yamaguch 			pci_intr_disestablish(psc->sc_pa.pa_pc, psc->sc_ihs[n]);
    623      1.6  yamaguch 		}
    624      1.6  yamaguch 
    625      1.6  yamaguch 	} else {
    626      1.6  yamaguch 		if (psc->sc_ihs[idx] != NULL)
    627      1.6  yamaguch 			pci_intr_disestablish(psc->sc_pa.pa_pc, psc->sc_ihs[idx]);
    628      1.6  yamaguch 	}
    629      1.4  jakllsch 
    630      1.4  jakllsch 	return -1;
    631      1.4  jakllsch }
    632      1.4  jakllsch 
    633      1.4  jakllsch static int
    634      1.4  jakllsch virtio_pci_setup_intx_interrupt(struct virtio_softc *sc,
    635      1.4  jakllsch     struct pci_attach_args *pa)
    636      1.4  jakllsch {
    637      1.4  jakllsch 	struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc;
    638      1.4  jakllsch 	device_t self = sc->sc_dev;
    639      1.4  jakllsch 	pci_chipset_tag_t pc = pa->pa_pc;
    640      1.4  jakllsch 	char intrbuf[PCI_INTRSTR_LEN];
    641      1.4  jakllsch 	char const *intrstr;
    642      1.4  jakllsch 
    643      1.4  jakllsch 	if (sc->sc_flags & VIRTIO_F_PCI_INTR_MPSAFE)
    644      1.4  jakllsch 		pci_intr_setattr(pc, &psc->sc_ihp[0], PCI_INTR_MPSAFE, true);
    645      1.4  jakllsch 
    646      1.4  jakllsch 	psc->sc_ihs[0] = pci_intr_establish_xname(pc, psc->sc_ihp[0],
    647      1.4  jakllsch 	    sc->sc_ipl, virtio_pci_intr, sc, device_xname(sc->sc_dev));
    648      1.4  jakllsch 	if (psc->sc_ihs[0] == NULL) {
    649      1.4  jakllsch 		aprint_error_dev(self, "couldn't establish INTx\n");
    650      1.4  jakllsch 		return -1;
    651      1.4  jakllsch 	}
    652      1.4  jakllsch 
    653      1.4  jakllsch 	intrstr = pci_intr_string(pc, psc->sc_ihp[0], intrbuf, sizeof(intrbuf));
    654      1.4  jakllsch 	aprint_normal_dev(self, "interrupting at %s\n", intrstr);
    655      1.4  jakllsch 
    656      1.4  jakllsch 	return 0;
    657      1.4  jakllsch }
    658      1.4  jakllsch 
    659      1.4  jakllsch static int
    660      1.4  jakllsch virtio_pci_setup_interrupts(struct virtio_softc *sc)
    661      1.4  jakllsch {
    662      1.4  jakllsch 	struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc;
    663      1.4  jakllsch 	device_t self = sc->sc_dev;
    664      1.4  jakllsch 	pci_chipset_tag_t pc = psc->sc_pa.pa_pc;
    665      1.4  jakllsch 	int error;
    666      1.4  jakllsch 	int nmsix;
    667      1.4  jakllsch 	int counts[PCI_INTR_TYPE_SIZE];
    668      1.4  jakllsch 	pci_intr_type_t max_type;
    669      1.4  jakllsch 
    670      1.4  jakllsch 	nmsix = pci_msix_count(psc->sc_pa.pa_pc, psc->sc_pa.pa_tag);
    671      1.4  jakllsch 	aprint_debug_dev(self, "pci_msix_count=%d\n", nmsix);
    672      1.4  jakllsch 
    673      1.4  jakllsch 	/* We need at least two: one for config and the other for queues */
    674      1.4  jakllsch 	if ((sc->sc_flags & VIRTIO_F_PCI_INTR_MSIX) == 0 || nmsix < 2) {
    675      1.4  jakllsch 		/* Try INTx only */
    676      1.4  jakllsch 		max_type = PCI_INTR_TYPE_INTX;
    677      1.4  jakllsch 		counts[PCI_INTR_TYPE_INTX] = 1;
    678      1.4  jakllsch 	} else {
    679      1.4  jakllsch 		/* Try MSI-X first and INTx second */
    680      1.6  yamaguch 		if (sc->sc_child_mq &&
    681      1.6  yamaguch 		    sc->sc_nvqs > (nmsix - VIRTIO_MSIX_QUEUE_VECTOR_INDEX)) {
    682      1.6  yamaguch 			nmsix = 2;
    683      1.6  yamaguch 			sc->sc_child_mq = false;
    684      1.6  yamaguch 		}
    685      1.6  yamaguch 
    686      1.4  jakllsch 		max_type = PCI_INTR_TYPE_MSIX;
    687      1.6  yamaguch 		counts[PCI_INTR_TYPE_MSIX] = nmsix;
    688      1.4  jakllsch 		counts[PCI_INTR_TYPE_MSI] = 0;
    689      1.4  jakllsch 		counts[PCI_INTR_TYPE_INTX] = 1;
    690      1.4  jakllsch 	}
    691      1.4  jakllsch 
    692      1.4  jakllsch retry:
    693      1.4  jakllsch 	error = pci_intr_alloc(&psc->sc_pa, &psc->sc_ihp, counts, max_type);
    694      1.4  jakllsch 	if (error != 0) {
    695      1.4  jakllsch 		aprint_error_dev(self, "couldn't map interrupt\n");
    696      1.4  jakllsch 		return -1;
    697      1.4  jakllsch 	}
    698      1.4  jakllsch 
    699      1.4  jakllsch 	if (pci_intr_type(pc, psc->sc_ihp[0]) == PCI_INTR_TYPE_MSIX) {
    700  1.7.4.1    martin 		psc->sc_ihs = kmem_zalloc(sizeof(*psc->sc_ihs) * nmsix,
    701      1.4  jakllsch 		    KM_SLEEP);
    702      1.4  jakllsch 
    703      1.4  jakllsch 		error = virtio_pci_setup_msix_interrupts(sc, &psc->sc_pa);
    704      1.4  jakllsch 		if (error != 0) {
    705      1.6  yamaguch 			kmem_free(psc->sc_ihs, sizeof(*psc->sc_ihs) * nmsix);
    706      1.6  yamaguch 			pci_intr_release(pc, psc->sc_ihp, nmsix);
    707      1.4  jakllsch 
    708      1.4  jakllsch 			/* Retry INTx */
    709      1.4  jakllsch 			max_type = PCI_INTR_TYPE_INTX;
    710      1.4  jakllsch 			counts[PCI_INTR_TYPE_INTX] = 1;
    711      1.4  jakllsch 			goto retry;
    712      1.4  jakllsch 		}
    713      1.4  jakllsch 
    714      1.6  yamaguch 		psc->sc_ihs_num = nmsix;
    715      1.4  jakllsch 		psc->sc_config_offset = VIRTIO_CONFIG_DEVICE_CONFIG_MSI;
    716      1.4  jakllsch 	} else if (pci_intr_type(pc, psc->sc_ihp[0]) == PCI_INTR_TYPE_INTX) {
    717  1.7.4.1    martin 		psc->sc_ihs = kmem_zalloc(sizeof(*psc->sc_ihs) * 1,
    718      1.4  jakllsch 		    KM_SLEEP);
    719      1.4  jakllsch 
    720      1.4  jakllsch 		error = virtio_pci_setup_intx_interrupt(sc, &psc->sc_pa);
    721      1.4  jakllsch 		if (error != 0) {
    722      1.4  jakllsch 			kmem_free(psc->sc_ihs, sizeof(*psc->sc_ihs) * 1);
    723      1.4  jakllsch 			pci_intr_release(pc, psc->sc_ihp, 1);
    724      1.4  jakllsch 			return -1;
    725      1.4  jakllsch 		}
    726      1.4  jakllsch 
    727      1.4  jakllsch 		psc->sc_ihs_num = 1;
    728      1.4  jakllsch 		psc->sc_config_offset = VIRTIO_CONFIG_DEVICE_CONFIG_NOMSI;
    729      1.4  jakllsch 	}
    730      1.4  jakllsch 
    731      1.4  jakllsch 	return 0;
    732      1.4  jakllsch }
    733      1.4  jakllsch 
    734      1.4  jakllsch static void
    735      1.4  jakllsch virtio_pci_free_interrupts(struct virtio_softc *sc)
    736      1.4  jakllsch {
    737      1.4  jakllsch 	struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc;
    738      1.4  jakllsch 
    739      1.4  jakllsch 	for (int i = 0; i < psc->sc_ihs_num; i++) {
    740      1.4  jakllsch 		if (psc->sc_ihs[i] == NULL)
    741      1.4  jakllsch 			continue;
    742      1.4  jakllsch 		pci_intr_disestablish(psc->sc_pa.pa_pc, psc->sc_ihs[i]);
    743      1.4  jakllsch 		psc->sc_ihs[i] = NULL;
    744      1.4  jakllsch 	}
    745      1.4  jakllsch 
    746      1.4  jakllsch 	if (psc->sc_ihs_num > 0)
    747      1.4  jakllsch 		pci_intr_release(psc->sc_pa.pa_pc, psc->sc_ihp, psc->sc_ihs_num);
    748      1.4  jakllsch 
    749      1.4  jakllsch 	if (psc->sc_ihs != NULL) {
    750      1.4  jakllsch 		kmem_free(psc->sc_ihs, sizeof(*psc->sc_ihs) * psc->sc_ihs_num);
    751      1.4  jakllsch 		psc->sc_ihs = NULL;
    752      1.4  jakllsch 	}
    753      1.4  jakllsch 	psc->sc_ihs_num = 0;
    754      1.4  jakllsch }
    755      1.4  jakllsch 
    756      1.4  jakllsch /*
    757      1.4  jakllsch  * Interrupt handler.
    758      1.4  jakllsch  */
    759      1.4  jakllsch static int
    760      1.4  jakllsch virtio_pci_intr(void *arg)
    761      1.4  jakllsch {
    762      1.4  jakllsch 	struct virtio_softc *sc = arg;
    763      1.4  jakllsch 	struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc;
    764      1.4  jakllsch 	int isr, r = 0;
    765      1.4  jakllsch 
    766      1.4  jakllsch 	/* check and ack the interrupt */
    767      1.4  jakllsch 	isr = bus_space_read_stream_1(psc->sc_iot, psc->sc_ioh,
    768      1.4  jakllsch 			       VIRTIO_CONFIG_ISR_STATUS);
    769      1.4  jakllsch 	if (isr == 0)
    770      1.4  jakllsch 		return 0;
    771      1.4  jakllsch 	if ((isr & VIRTIO_CONFIG_ISR_CONFIG_CHANGE) &&
    772      1.4  jakllsch 	    (sc->sc_config_change != NULL))
    773      1.4  jakllsch 		r = (sc->sc_config_change)(sc);
    774      1.4  jakllsch 	if (sc->sc_intrhand != NULL) {
    775      1.4  jakllsch 		if (sc->sc_soft_ih != NULL)
    776      1.4  jakllsch 			softint_schedule(sc->sc_soft_ih);
    777      1.4  jakllsch 		else
    778      1.4  jakllsch 			r |= (sc->sc_intrhand)(sc);
    779      1.4  jakllsch 	}
    780      1.4  jakllsch 
    781      1.4  jakllsch 	return r;
    782      1.4  jakllsch }
    783      1.4  jakllsch 
    784      1.4  jakllsch static int
    785      1.4  jakllsch virtio_pci_msix_queue_intr(void *arg)
    786      1.4  jakllsch {
    787      1.4  jakllsch 	struct virtio_softc *sc = arg;
    788      1.4  jakllsch 	int r = 0;
    789      1.4  jakllsch 
    790      1.4  jakllsch 	if (sc->sc_intrhand != NULL) {
    791      1.4  jakllsch 		if (sc->sc_soft_ih != NULL)
    792      1.4  jakllsch 			softint_schedule(sc->sc_soft_ih);
    793      1.4  jakllsch 		else
    794      1.4  jakllsch 			r |= (sc->sc_intrhand)(sc);
    795      1.4  jakllsch 	}
    796      1.4  jakllsch 
    797      1.4  jakllsch 	return r;
    798      1.4  jakllsch }
    799      1.4  jakllsch 
    800      1.4  jakllsch static int
    801      1.6  yamaguch virtio_pci_msix_vq_intr(void *arg)
    802      1.6  yamaguch {
    803      1.6  yamaguch 	struct virtqueue *vq = arg;
    804      1.6  yamaguch 	int r = 0;
    805      1.6  yamaguch 
    806      1.6  yamaguch 	if (vq->vq_intrhand != NULL) {
    807      1.6  yamaguch 		if (vq->vq_soft_ih)
    808      1.6  yamaguch 			softint_schedule(vq->vq_soft_ih);
    809      1.6  yamaguch 		else
    810      1.6  yamaguch 			r |= vq->vq_intrhand(vq);
    811      1.6  yamaguch 	}
    812      1.6  yamaguch 
    813      1.6  yamaguch 	return r;
    814      1.6  yamaguch }
    815      1.6  yamaguch 
    816      1.6  yamaguch static int
    817      1.4  jakllsch virtio_pci_msix_config_intr(void *arg)
    818      1.4  jakllsch {
    819      1.4  jakllsch 	struct virtio_softc *sc = arg;
    820      1.4  jakllsch 	int r = 0;
    821      1.4  jakllsch 
    822      1.4  jakllsch 	if (sc->sc_config_change != NULL)
    823      1.4  jakllsch 		r = (sc->sc_config_change)(sc);
    824      1.4  jakllsch 	return r;
    825      1.4  jakllsch }
    826      1.5  jakllsch 
    827      1.5  jakllsch MODULE(MODULE_CLASS_DRIVER, virtio_pci, "pci,virtio");
    828      1.5  jakllsch 
    829      1.5  jakllsch #ifdef _MODULE
    830      1.5  jakllsch #include "ioconf.c"
    831      1.5  jakllsch #endif
    832      1.5  jakllsch 
    833      1.5  jakllsch static int
    834      1.5  jakllsch virtio_pci_modcmd(modcmd_t cmd, void *opaque)
    835      1.5  jakllsch {
    836      1.5  jakllsch 	int error = 0;
    837      1.5  jakllsch 
    838      1.5  jakllsch #ifdef _MODULE
    839      1.5  jakllsch 	switch (cmd) {
    840      1.5  jakllsch 	case MODULE_CMD_INIT:
    841      1.5  jakllsch 		error = config_init_component(cfdriver_ioconf_virtio_pci,
    842      1.5  jakllsch 		    cfattach_ioconf_virtio_pci, cfdata_ioconf_virtio_pci);
    843      1.5  jakllsch 		break;
    844      1.5  jakllsch 	case MODULE_CMD_FINI:
    845      1.5  jakllsch 		error = config_fini_component(cfdriver_ioconf_virtio_pci,
    846      1.5  jakllsch 		    cfattach_ioconf_virtio_pci, cfdata_ioconf_virtio_pci);
    847      1.5  jakllsch 		break;
    848      1.5  jakllsch 	default:
    849      1.5  jakllsch 		error = ENOTTY;
    850      1.5  jakllsch 		break;
    851      1.5  jakllsch 	}
    852      1.5  jakllsch #endif
    853      1.5  jakllsch 
    854      1.5  jakllsch 	return error;
    855      1.5  jakllsch }
    856