Home | History | Annotate | Line # | Download | only in pci
virtio_pci.c revision 1.2.2.1
      1  1.2.2.1  pgoyette /* $NetBSD: virtio_pci.c,v 1.2.2.1 2018/06/25 07:26:01 pgoyette Exp $ */
      2      1.1    cherry 
      3      1.1    cherry /*
      4      1.1    cherry  * Copyright (c) 2010 Minoura Makoto.
      5      1.1    cherry  * All rights reserved.
      6      1.1    cherry  *
      7      1.1    cherry  * Redistribution and use in source and binary forms, with or without
      8      1.1    cherry  * modification, are permitted provided that the following conditions
      9      1.1    cherry  * are met:
     10      1.1    cherry  * 1. Redistributions of source code must retain the above copyright
     11      1.1    cherry  *    notice, this list of conditions and the following disclaimer.
     12      1.1    cherry  * 2. Redistributions in binary form must reproduce the above copyright
     13      1.1    cherry  *    notice, this list of conditions and the following disclaimer in the
     14      1.1    cherry  *    documentation and/or other materials provided with the distribution.
     15      1.1    cherry  *
     16      1.1    cherry  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
     17      1.1    cherry  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
     18      1.1    cherry  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
     19      1.1    cherry  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
     20      1.1    cherry  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
     21      1.1    cherry  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
     22      1.1    cherry  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
     23      1.1    cherry  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
     24      1.1    cherry  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
     25      1.1    cherry  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
     26      1.1    cherry  */
     27      1.1    cherry 
     28      1.1    cherry #include <sys/cdefs.h>
     29  1.2.2.1  pgoyette __KERNEL_RCSID(0, "$NetBSD: virtio_pci.c,v 1.2.2.1 2018/06/25 07:26:01 pgoyette Exp $");
     30      1.1    cherry 
     31      1.1    cherry #include <sys/param.h>
     32      1.1    cherry #include <sys/systm.h>
     33  1.2.2.1  pgoyette #include <sys/kmem.h>
     34  1.2.2.1  pgoyette #include <sys/module.h>
     35      1.1    cherry 
     36      1.1    cherry #include <sys/device.h>
     37      1.1    cherry 
     38      1.1    cherry #include <dev/pci/pcidevs.h>
     39      1.1    cherry #include <dev/pci/pcireg.h>
     40      1.1    cherry #include <dev/pci/pcivar.h>
     41      1.1    cherry 
     42      1.1    cherry #define VIRTIO_PRIVATE
     43      1.1    cherry 
     44      1.1    cherry #include <dev/pci/virtioreg.h> /* XXX: move to non-pci */
     45      1.1    cherry #include <dev/pci/virtiovar.h> /* XXX: move to non-pci */
     46      1.1    cherry 
     47  1.2.2.1  pgoyette static int	virtio_pci_match(device_t, cfdata_t, void *);
     48  1.2.2.1  pgoyette static void	virtio_pci_attach(device_t, device_t, void *);
     49  1.2.2.1  pgoyette static int	virtio_pci_rescan(device_t, const char *, const int *);
     50  1.2.2.1  pgoyette static int	virtio_pci_detach(device_t, int);
     51  1.2.2.1  pgoyette 
     52  1.2.2.1  pgoyette struct virtio_pci_softc {
     53  1.2.2.1  pgoyette 	struct virtio_softc	sc_sc;
     54  1.2.2.1  pgoyette 	bus_space_tag_t		sc_iot;
     55  1.2.2.1  pgoyette 	bus_space_handle_t	sc_ioh;
     56  1.2.2.1  pgoyette 	bus_size_t		sc_iosize;
     57  1.2.2.1  pgoyette 	struct pci_attach_args	sc_pa;
     58  1.2.2.1  pgoyette 	pci_intr_handle_t	*sc_ihp;
     59  1.2.2.1  pgoyette 	void			**sc_ihs;
     60  1.2.2.1  pgoyette 	int			sc_ihs_num;
     61  1.2.2.1  pgoyette 	int			sc_config_offset;
     62  1.2.2.1  pgoyette };
     63  1.2.2.1  pgoyette 
     64  1.2.2.1  pgoyette static void	virtio_pci_kick(struct virtio_softc *, uint16_t);
     65  1.2.2.1  pgoyette static uint8_t	virtio_pci_read_device_config_1(struct virtio_softc *, int);
     66  1.2.2.1  pgoyette static uint16_t	virtio_pci_read_device_config_2(struct virtio_softc *, int);
     67  1.2.2.1  pgoyette static uint32_t	virtio_pci_read_device_config_4(struct virtio_softc *, int);
     68  1.2.2.1  pgoyette static uint64_t	virtio_pci_read_device_config_8(struct virtio_softc *, int);
     69  1.2.2.1  pgoyette static void 	virtio_pci_write_device_config_1(struct virtio_softc *, int, uint8_t);
     70  1.2.2.1  pgoyette static void	virtio_pci_write_device_config_2(struct virtio_softc *, int, uint16_t);
     71  1.2.2.1  pgoyette static void	virtio_pci_write_device_config_4(struct virtio_softc *, int, uint32_t);
     72  1.2.2.1  pgoyette static void	virtio_pci_write_device_config_8(struct virtio_softc *, int, uint64_t);
     73  1.2.2.1  pgoyette static uint16_t	virtio_pci_read_queue_size(struct virtio_softc *, uint16_t);
     74  1.2.2.1  pgoyette static void	virtio_pci_setup_queue(struct virtio_softc *, uint16_t, uint32_t);
     75  1.2.2.1  pgoyette static void	virtio_pci_set_status(struct virtio_softc *, int);
     76  1.2.2.1  pgoyette static uint32_t	virtio_pci_negotiate_features(struct virtio_softc *, uint32_t);
     77  1.2.2.1  pgoyette static int	virtio_pci_setup_interrupts(struct virtio_softc *);
     78  1.2.2.1  pgoyette static void	virtio_pci_free_interrupts(struct virtio_softc *);
     79  1.2.2.1  pgoyette 
     80  1.2.2.1  pgoyette static int	virtio_pci_intr(void *arg);
     81  1.2.2.1  pgoyette static int	virtio_pci_msix_queue_intr(void *);
     82  1.2.2.1  pgoyette static int	virtio_pci_msix_config_intr(void *);
     83  1.2.2.1  pgoyette static int	virtio_pci_setup_msix_vectors(struct virtio_softc *);
     84  1.2.2.1  pgoyette static int	virtio_pci_setup_msix_interrupts(struct virtio_softc *,
     85  1.2.2.1  pgoyette 		    struct pci_attach_args *);
     86  1.2.2.1  pgoyette static int	virtio_pci_setup_intx_interrupt(struct virtio_softc *,
     87  1.2.2.1  pgoyette 		    struct pci_attach_args *);
     88  1.2.2.1  pgoyette 
     89  1.2.2.1  pgoyette #define VIRTIO_MSIX_CONFIG_VECTOR_INDEX	0
     90  1.2.2.1  pgoyette #define VIRTIO_MSIX_QUEUE_VECTOR_INDEX	1
     91  1.2.2.1  pgoyette 
     92  1.2.2.1  pgoyette /* we use the legacy virtio spec, so the PCI registers are host native
     93  1.2.2.1  pgoyette  * byte order, not PCI (i.e. LE) byte order */
     94  1.2.2.1  pgoyette #if BYTE_ORDER == BIG_ENDIAN
     95  1.2.2.1  pgoyette #define REG_HI_OFF      0
     96  1.2.2.1  pgoyette #define REG_LO_OFF      4
     97  1.2.2.1  pgoyette #ifndef __BUS_SPACE_HAS_STREAM_METHODS
     98  1.2.2.1  pgoyette #define bus_space_read_stream_1 bus_space_read_1
     99  1.2.2.1  pgoyette #define bus_space_write_stream_1 bus_space_write_1
    100  1.2.2.1  pgoyette static inline uint16_t
    101  1.2.2.1  pgoyette bus_space_read_stream_2(bus_space_tag_t t, bus_space_handle_t h,
    102  1.2.2.1  pgoyette     bus_size_t o)
    103  1.2.2.1  pgoyette {
    104  1.2.2.1  pgoyette 	return le16toh(bus_space_read_2(t, h, o));
    105  1.2.2.1  pgoyette }
    106  1.2.2.1  pgoyette static inline void
    107  1.2.2.1  pgoyette bus_space_write_stream_2(bus_space_tag_t t, bus_space_handle_t h,
    108  1.2.2.1  pgoyette     bus_size_t o, uint16_t v)
    109  1.2.2.1  pgoyette {
    110  1.2.2.1  pgoyette 	bus_space_write_2(t, h, o, htole16(v));
    111  1.2.2.1  pgoyette }
    112  1.2.2.1  pgoyette static inline uint32_t
    113  1.2.2.1  pgoyette bus_space_read_stream_4(bus_space_tag_t t, bus_space_handle_t h,
    114  1.2.2.1  pgoyette     bus_size_t o)
    115  1.2.2.1  pgoyette {
    116  1.2.2.1  pgoyette 	return le32toh(bus_space_read_4(t, h, o));
    117  1.2.2.1  pgoyette }
    118  1.2.2.1  pgoyette static inline void
    119  1.2.2.1  pgoyette bus_space_write_stream_4(bus_space_tag_t t, bus_space_handle_t h,
    120  1.2.2.1  pgoyette     bus_size_t o, uint32_t v)
    121  1.2.2.1  pgoyette {
    122  1.2.2.1  pgoyette 	bus_space_write_4(t, h, o, htole32(v));
    123  1.2.2.1  pgoyette }
    124  1.2.2.1  pgoyette #endif
    125  1.2.2.1  pgoyette #else
    126  1.2.2.1  pgoyette #define REG_HI_OFF	4
    127  1.2.2.1  pgoyette #define REG_LO_OFF	0
    128  1.2.2.1  pgoyette #ifndef __BUS_SPACE_HAS_STREAM_METHODS
    129  1.2.2.1  pgoyette #define bus_space_read_stream_1 bus_space_read_1
    130  1.2.2.1  pgoyette #define bus_space_read_stream_2 bus_space_read_2
    131  1.2.2.1  pgoyette #define bus_space_read_stream_4 bus_space_read_4
    132  1.2.2.1  pgoyette #define bus_space_write_stream_1 bus_space_write_1
    133  1.2.2.1  pgoyette #define bus_space_write_stream_2 bus_space_write_2
    134  1.2.2.1  pgoyette #define bus_space_write_stream_4 bus_space_write_4
    135  1.2.2.1  pgoyette #endif
    136  1.2.2.1  pgoyette #endif
    137  1.2.2.1  pgoyette 
    138      1.1    cherry 
    139      1.1    cherry static const char *virtio_device_name[] = {
    140      1.1    cherry 	"Unknown (0)",			/* 0 */
    141      1.1    cherry 	"Network",			/* 1 */
    142      1.1    cherry 	"Block",			/* 2 */
    143      1.1    cherry 	"Console",			/* 3 */
    144      1.1    cherry 	"Entropy",			/* 4 */
    145      1.1    cherry 	"Memory Balloon",		/* 5 */
    146      1.1    cherry 	"I/O Memory",			/* 6 */
    147      1.1    cherry 	"Remote Processor Messaging",	/* 7 */
    148      1.1    cherry 	"SCSI",				/* 8 */
    149      1.1    cherry 	"9P Transport",			/* 9 */
    150      1.1    cherry 	"mac80211 wlan",		/* 10 */
    151      1.1    cherry };
    152      1.1    cherry #define NDEVNAMES	__arraycount(virtio_device_name)
    153      1.1    cherry 
    154  1.2.2.1  pgoyette CFATTACH_DECL3_NEW(virtio_pci, sizeof(struct virtio_pci_softc),
    155  1.2.2.1  pgoyette     virtio_pci_match, virtio_pci_attach, virtio_pci_detach, NULL,
    156  1.2.2.1  pgoyette     virtio_pci_rescan, NULL, DVF_DETACH_SHUTDOWN);
    157  1.2.2.1  pgoyette 
    158  1.2.2.1  pgoyette static const struct virtio_ops virtio_pci_ops = {
    159  1.2.2.1  pgoyette 	.kick = virtio_pci_kick,
    160  1.2.2.1  pgoyette 	.read_dev_cfg_1 = virtio_pci_read_device_config_1,
    161  1.2.2.1  pgoyette 	.read_dev_cfg_2 = virtio_pci_read_device_config_2,
    162  1.2.2.1  pgoyette 	.read_dev_cfg_4 = virtio_pci_read_device_config_4,
    163  1.2.2.1  pgoyette 	.read_dev_cfg_8 = virtio_pci_read_device_config_8,
    164  1.2.2.1  pgoyette 	.write_dev_cfg_1 = virtio_pci_write_device_config_1,
    165  1.2.2.1  pgoyette 	.write_dev_cfg_2 = virtio_pci_write_device_config_2,
    166  1.2.2.1  pgoyette 	.write_dev_cfg_4 = virtio_pci_write_device_config_4,
    167  1.2.2.1  pgoyette 	.write_dev_cfg_8 = virtio_pci_write_device_config_8,
    168  1.2.2.1  pgoyette 	.read_queue_size = virtio_pci_read_queue_size,
    169  1.2.2.1  pgoyette 	.setup_queue = virtio_pci_setup_queue,
    170  1.2.2.1  pgoyette 	.set_status = virtio_pci_set_status,
    171  1.2.2.1  pgoyette 	.neg_features = virtio_pci_negotiate_features,
    172  1.2.2.1  pgoyette 	.setup_interrupts = virtio_pci_setup_interrupts,
    173  1.2.2.1  pgoyette 	.free_interrupts = virtio_pci_free_interrupts,
    174  1.2.2.1  pgoyette };
    175      1.1    cherry 
    176      1.1    cherry static int
    177  1.2.2.1  pgoyette virtio_pci_match(device_t parent, cfdata_t match, void *aux)
    178      1.1    cherry {
    179      1.1    cherry 	struct pci_attach_args *pa;
    180      1.1    cherry 
    181      1.1    cherry 	pa = (struct pci_attach_args *)aux;
    182      1.1    cherry 	switch (PCI_VENDOR(pa->pa_id)) {
    183      1.1    cherry 	case PCI_VENDOR_QUMRANET:
    184      1.1    cherry 		if ((PCI_PRODUCT_QUMRANET_VIRTIO_1000 <=
    185      1.1    cherry 		     PCI_PRODUCT(pa->pa_id)) &&
    186      1.1    cherry 		    (PCI_PRODUCT(pa->pa_id) <=
    187      1.1    cherry 		     PCI_PRODUCT_QUMRANET_VIRTIO_103F))
    188      1.1    cherry 			return 1;
    189      1.1    cherry 		break;
    190      1.1    cherry 	}
    191      1.1    cherry 
    192      1.1    cherry 	return 0;
    193      1.1    cherry }
    194      1.1    cherry 
    195      1.1    cherry static void
    196  1.2.2.1  pgoyette virtio_pci_attach(device_t parent, device_t self, void *aux)
    197      1.1    cherry {
    198  1.2.2.1  pgoyette 	struct virtio_pci_softc * const psc = device_private(self);
    199  1.2.2.1  pgoyette 	struct virtio_softc * const sc = &psc->sc_sc;
    200      1.1    cherry 	struct pci_attach_args *pa = (struct pci_attach_args *)aux;
    201      1.1    cherry 	pci_chipset_tag_t pc = pa->pa_pc;
    202      1.1    cherry 	pcitag_t tag = pa->pa_tag;
    203      1.1    cherry 	int revision;
    204      1.1    cherry 	pcireg_t id;
    205      1.2       uwe 	pcireg_t csr;
    206      1.1    cherry 
    207      1.1    cherry 	revision = PCI_REVISION(pa->pa_class);
    208      1.1    cherry 	if (revision != 0) {
    209      1.1    cherry 		aprint_normal(": unknown revision 0x%02x; giving up\n",
    210      1.1    cherry 			      revision);
    211      1.1    cherry 		return;
    212      1.1    cherry 	}
    213      1.1    cherry 	aprint_normal("\n");
    214      1.1    cherry 	aprint_naive("\n");
    215      1.1    cherry 
    216      1.1    cherry 	/* subsystem ID shows what I am */
    217      1.1    cherry 	id = pci_conf_read(pc, tag, PCI_SUBSYS_ID_REG);
    218      1.1    cherry 	aprint_normal_dev(self, "Virtio %s Device (rev. 0x%02x)\n",
    219      1.1    cherry 			  (PCI_SUBSYS_ID(id) < NDEVNAMES?
    220      1.1    cherry 			   virtio_device_name[PCI_SUBSYS_ID(id)] : "Unknown"),
    221      1.1    cherry 			  revision);
    222      1.1    cherry 
    223      1.2       uwe 	csr = pci_conf_read(pc, tag, PCI_COMMAND_STATUS_REG);
    224      1.2       uwe 	csr |= PCI_COMMAND_MASTER_ENABLE | PCI_COMMAND_IO_ENABLE;
    225      1.2       uwe 	pci_conf_write(pc, tag, PCI_COMMAND_STATUS_REG, csr);
    226      1.2       uwe 
    227      1.1    cherry 	sc->sc_dev = self;
    228  1.2.2.1  pgoyette 	sc->sc_ops = &virtio_pci_ops;
    229  1.2.2.1  pgoyette 	psc->sc_pa = *pa;
    230  1.2.2.1  pgoyette 	psc->sc_iot = pa->pa_iot;
    231      1.1    cherry 	if (pci_dma64_available(pa))
    232      1.1    cherry 		sc->sc_dmat = pa->pa_dmat64;
    233      1.1    cherry 	else
    234      1.1    cherry 		sc->sc_dmat = pa->pa_dmat;
    235  1.2.2.1  pgoyette 	psc->sc_config_offset = VIRTIO_CONFIG_DEVICE_CONFIG_NOMSI;
    236      1.1    cherry 
    237      1.1    cherry 	if (pci_mapreg_map(pa, PCI_MAPREG_START, PCI_MAPREG_TYPE_IO, 0,
    238  1.2.2.1  pgoyette 			   &psc->sc_iot, &psc->sc_ioh, NULL, &psc->sc_iosize)) {
    239      1.1    cherry 		aprint_error_dev(self, "can't map i/o space\n");
    240      1.1    cherry 		return;
    241      1.1    cherry 	}
    242      1.1    cherry 
    243      1.1    cherry 	virtio_device_reset(sc);
    244      1.1    cherry 	virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_ACK);
    245      1.1    cherry 	virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_DRIVER);
    246      1.1    cherry 
    247      1.1    cherry 	sc->sc_childdevid = PCI_SUBSYS_ID(id);
    248      1.1    cherry 	sc->sc_child = NULL;
    249  1.2.2.1  pgoyette 	virtio_pci_rescan(self, "virtio", 0);
    250      1.1    cherry 	return;
    251      1.1    cherry }
    252      1.1    cherry 
    253      1.1    cherry /* ARGSUSED */
    254      1.1    cherry static int
    255  1.2.2.1  pgoyette virtio_pci_rescan(device_t self, const char *attr, const int *scan_flags)
    256      1.1    cherry {
    257  1.2.2.1  pgoyette 	struct virtio_pci_softc * const psc = device_private(self);
    258  1.2.2.1  pgoyette 	struct virtio_softc * const sc = &psc->sc_sc;
    259      1.1    cherry 	struct virtio_attach_args va;
    260      1.1    cherry 
    261      1.1    cherry 	if (sc->sc_child)	/* Child already attached? */
    262      1.1    cherry 		return 0;
    263      1.1    cherry 
    264      1.1    cherry 	memset(&va, 0, sizeof(va));
    265      1.1    cherry 	va.sc_childdevid = sc->sc_childdevid;
    266      1.1    cherry 
    267      1.1    cherry 	config_found_ia(self, attr, &va, NULL);
    268      1.1    cherry 
    269      1.1    cherry 	if (sc->sc_child == NULL) {
    270      1.1    cherry 		aprint_error_dev(self,
    271      1.1    cherry 				 "no matching child driver; not configured\n");
    272      1.1    cherry 		return 0;
    273      1.1    cherry 	}
    274  1.2.2.1  pgoyette 
    275      1.1    cherry 	if (sc->sc_child == VIRTIO_CHILD_FAILED) {
    276      1.1    cherry 		aprint_error_dev(self,
    277      1.1    cherry 				 "virtio configuration failed\n");
    278      1.1    cherry 		return 0;
    279      1.1    cherry 	}
    280      1.1    cherry 
    281      1.1    cherry 	/*
    282      1.1    cherry 	 * Make sure child drivers initialize interrupts via call
    283      1.1    cherry 	 * to virtio_child_attach_finish().
    284      1.1    cherry 	 */
    285  1.2.2.1  pgoyette 	KASSERT(psc->sc_ihs_num != 0);
    286      1.1    cherry 
    287      1.1    cherry 	return 0;
    288      1.1    cherry }
    289      1.1    cherry 
    290      1.1    cherry 
    291      1.1    cherry static int
    292  1.2.2.1  pgoyette virtio_pci_detach(device_t self, int flags)
    293      1.1    cherry {
    294  1.2.2.1  pgoyette 	struct virtio_pci_softc * const psc = device_private(self);
    295  1.2.2.1  pgoyette 	struct virtio_softc * const sc = &psc->sc_sc;
    296      1.1    cherry 	int r;
    297      1.1    cherry 
    298      1.1    cherry 	if (sc->sc_child != NULL) {
    299      1.1    cherry 		r = config_detach(sc->sc_child, flags);
    300      1.1    cherry 		if (r)
    301      1.1    cherry 			return r;
    302      1.1    cherry 	}
    303      1.1    cherry 
    304      1.1    cherry 	/* Check that child detached properly */
    305      1.1    cherry 	KASSERT(sc->sc_child == NULL);
    306      1.1    cherry 	KASSERT(sc->sc_vqs == NULL);
    307  1.2.2.1  pgoyette 	KASSERT(psc->sc_ihs_num == 0);
    308  1.2.2.1  pgoyette 
    309  1.2.2.1  pgoyette 	if (psc->sc_iosize)
    310  1.2.2.1  pgoyette 		bus_space_unmap(psc->sc_iot, psc->sc_ioh, psc->sc_iosize);
    311  1.2.2.1  pgoyette 	psc->sc_iosize = 0;
    312  1.2.2.1  pgoyette 
    313  1.2.2.1  pgoyette 	return 0;
    314  1.2.2.1  pgoyette }
    315  1.2.2.1  pgoyette 
    316  1.2.2.1  pgoyette static void
    317  1.2.2.1  pgoyette virtio_pci_kick(struct virtio_softc *sc, uint16_t idx)
    318  1.2.2.1  pgoyette {
    319  1.2.2.1  pgoyette 	struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc;
    320  1.2.2.1  pgoyette 
    321  1.2.2.1  pgoyette 	bus_space_write_stream_2(psc->sc_iot, psc->sc_ioh,
    322  1.2.2.1  pgoyette 	    VIRTIO_CONFIG_QUEUE_NOTIFY, idx);
    323  1.2.2.1  pgoyette }
    324  1.2.2.1  pgoyette 
    325  1.2.2.1  pgoyette static uint8_t
    326  1.2.2.1  pgoyette virtio_pci_read_device_config_1(struct virtio_softc *sc, int index)
    327  1.2.2.1  pgoyette {
    328  1.2.2.1  pgoyette 	struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc;
    329  1.2.2.1  pgoyette 	return bus_space_read_stream_1(psc->sc_iot, psc->sc_ioh,
    330  1.2.2.1  pgoyette 	    psc->sc_config_offset + index);
    331  1.2.2.1  pgoyette }
    332  1.2.2.1  pgoyette 
    333  1.2.2.1  pgoyette static uint16_t
    334  1.2.2.1  pgoyette virtio_pci_read_device_config_2(struct virtio_softc *sc, int index)
    335  1.2.2.1  pgoyette {
    336  1.2.2.1  pgoyette 	struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc;
    337  1.2.2.1  pgoyette 	return bus_space_read_stream_2(psc->sc_iot, psc->sc_ioh,
    338  1.2.2.1  pgoyette 	    psc->sc_config_offset + index);
    339  1.2.2.1  pgoyette }
    340  1.2.2.1  pgoyette 
    341  1.2.2.1  pgoyette static uint32_t
    342  1.2.2.1  pgoyette virtio_pci_read_device_config_4(struct virtio_softc *sc, int index)
    343  1.2.2.1  pgoyette {
    344  1.2.2.1  pgoyette 	struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc;
    345  1.2.2.1  pgoyette 	return bus_space_read_stream_4(psc->sc_iot, psc->sc_ioh,
    346  1.2.2.1  pgoyette 	    psc->sc_config_offset + index);
    347  1.2.2.1  pgoyette }
    348  1.2.2.1  pgoyette 
    349  1.2.2.1  pgoyette static uint64_t
    350  1.2.2.1  pgoyette virtio_pci_read_device_config_8(struct virtio_softc *sc, int index)
    351  1.2.2.1  pgoyette {
    352  1.2.2.1  pgoyette 	struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc;
    353  1.2.2.1  pgoyette 	uint64_t r;
    354  1.2.2.1  pgoyette 
    355  1.2.2.1  pgoyette 	r = bus_space_read_stream_4(psc->sc_iot, psc->sc_ioh,
    356  1.2.2.1  pgoyette 	    psc->sc_config_offset + index + REG_HI_OFF);
    357  1.2.2.1  pgoyette 	r <<= 32;
    358  1.2.2.1  pgoyette 	r |= bus_space_read_stream_4(psc->sc_iot, psc->sc_ioh,
    359  1.2.2.1  pgoyette 	    psc->sc_config_offset + index + REG_LO_OFF);
    360  1.2.2.1  pgoyette 
    361  1.2.2.1  pgoyette 	return r;
    362  1.2.2.1  pgoyette }
    363  1.2.2.1  pgoyette 
    364  1.2.2.1  pgoyette static void
    365  1.2.2.1  pgoyette virtio_pci_write_device_config_1(struct virtio_softc *sc, int index,
    366  1.2.2.1  pgoyette     uint8_t value)
    367  1.2.2.1  pgoyette {
    368  1.2.2.1  pgoyette 	struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc;
    369  1.2.2.1  pgoyette 
    370  1.2.2.1  pgoyette 	bus_space_write_stream_1(psc->sc_iot, psc->sc_ioh,
    371  1.2.2.1  pgoyette 	    psc->sc_config_offset + index, value);
    372  1.2.2.1  pgoyette }
    373  1.2.2.1  pgoyette 
    374  1.2.2.1  pgoyette static void
    375  1.2.2.1  pgoyette virtio_pci_write_device_config_2(struct virtio_softc *sc, int index,
    376  1.2.2.1  pgoyette     uint16_t value)
    377  1.2.2.1  pgoyette {
    378  1.2.2.1  pgoyette 	struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc;
    379  1.2.2.1  pgoyette 
    380  1.2.2.1  pgoyette 	bus_space_write_stream_2(psc->sc_iot, psc->sc_ioh,
    381  1.2.2.1  pgoyette 	    psc->sc_config_offset + index, value);
    382  1.2.2.1  pgoyette }
    383  1.2.2.1  pgoyette 
    384  1.2.2.1  pgoyette static void
    385  1.2.2.1  pgoyette virtio_pci_write_device_config_4(struct virtio_softc *sc, int index,
    386  1.2.2.1  pgoyette     uint32_t value)
    387  1.2.2.1  pgoyette {
    388  1.2.2.1  pgoyette 	struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc;
    389  1.2.2.1  pgoyette 
    390  1.2.2.1  pgoyette 	bus_space_write_stream_4(psc->sc_iot, psc->sc_ioh,
    391  1.2.2.1  pgoyette 	    psc->sc_config_offset + index, value);
    392  1.2.2.1  pgoyette }
    393  1.2.2.1  pgoyette 
    394  1.2.2.1  pgoyette static void
    395  1.2.2.1  pgoyette virtio_pci_write_device_config_8(struct virtio_softc *sc, int index,
    396  1.2.2.1  pgoyette     uint64_t value)
    397  1.2.2.1  pgoyette {
    398  1.2.2.1  pgoyette 	struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc;
    399  1.2.2.1  pgoyette 
    400  1.2.2.1  pgoyette 	bus_space_write_stream_4(psc->sc_iot, psc->sc_ioh,
    401  1.2.2.1  pgoyette 	    psc->sc_config_offset + index + REG_LO_OFF,
    402  1.2.2.1  pgoyette 	    value & 0xffffffff);
    403  1.2.2.1  pgoyette 	bus_space_write_stream_4(psc->sc_iot, psc->sc_ioh,
    404  1.2.2.1  pgoyette 	    psc->sc_config_offset + index + REG_HI_OFF,
    405  1.2.2.1  pgoyette 	    value >> 32);
    406  1.2.2.1  pgoyette }
    407  1.2.2.1  pgoyette 
    408  1.2.2.1  pgoyette static uint16_t
    409  1.2.2.1  pgoyette virtio_pci_read_queue_size(struct virtio_softc *sc, uint16_t idx)
    410  1.2.2.1  pgoyette {
    411  1.2.2.1  pgoyette 	struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc;
    412  1.2.2.1  pgoyette 
    413  1.2.2.1  pgoyette 	bus_space_write_stream_2(psc->sc_iot, psc->sc_ioh,
    414  1.2.2.1  pgoyette 	    VIRTIO_CONFIG_QUEUE_SELECT, idx);
    415  1.2.2.1  pgoyette 	return bus_space_read_stream_2(psc->sc_iot, psc->sc_ioh,
    416  1.2.2.1  pgoyette 	    VIRTIO_CONFIG_QUEUE_SIZE);
    417  1.2.2.1  pgoyette }
    418  1.2.2.1  pgoyette 
    419  1.2.2.1  pgoyette static void
    420  1.2.2.1  pgoyette virtio_pci_setup_queue(struct virtio_softc *sc, uint16_t idx, uint32_t addr)
    421  1.2.2.1  pgoyette {
    422  1.2.2.1  pgoyette 	struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc;
    423  1.2.2.1  pgoyette 
    424  1.2.2.1  pgoyette 	bus_space_write_stream_2(psc->sc_iot, psc->sc_ioh,
    425  1.2.2.1  pgoyette 	    VIRTIO_CONFIG_QUEUE_SELECT, idx);
    426  1.2.2.1  pgoyette 	bus_space_write_stream_4(psc->sc_iot, psc->sc_ioh,
    427  1.2.2.1  pgoyette 	    VIRTIO_CONFIG_QUEUE_ADDRESS, addr);
    428  1.2.2.1  pgoyette 
    429  1.2.2.1  pgoyette 	if (psc->sc_ihs_num > 1) {
    430  1.2.2.1  pgoyette 		int vec = VIRTIO_MSIX_QUEUE_VECTOR_INDEX;
    431  1.2.2.1  pgoyette 		if (false) /* (for per-vq vectors) */
    432  1.2.2.1  pgoyette 			vec += idx;
    433  1.2.2.1  pgoyette 		bus_space_write_stream_2(psc->sc_iot, psc->sc_ioh,
    434  1.2.2.1  pgoyette 		    VIRTIO_CONFIG_MSI_QUEUE_VECTOR, vec);
    435  1.2.2.1  pgoyette 	}
    436  1.2.2.1  pgoyette }
    437  1.2.2.1  pgoyette 
    438  1.2.2.1  pgoyette static void
    439  1.2.2.1  pgoyette virtio_pci_set_status(struct virtio_softc *sc, int status)
    440  1.2.2.1  pgoyette {
    441  1.2.2.1  pgoyette 	struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc;
    442  1.2.2.1  pgoyette 	int old = 0;
    443  1.2.2.1  pgoyette 
    444  1.2.2.1  pgoyette 	if (status != 0) {
    445  1.2.2.1  pgoyette 	    old = bus_space_read_stream_1(psc->sc_iot, psc->sc_ioh,
    446  1.2.2.1  pgoyette 		VIRTIO_CONFIG_DEVICE_STATUS);
    447  1.2.2.1  pgoyette 	}
    448  1.2.2.1  pgoyette 	bus_space_write_stream_1(psc->sc_iot, psc->sc_ioh,
    449  1.2.2.1  pgoyette 	    VIRTIO_CONFIG_DEVICE_STATUS, status|old);
    450  1.2.2.1  pgoyette }
    451  1.2.2.1  pgoyette 
    452  1.2.2.1  pgoyette static uint32_t
    453  1.2.2.1  pgoyette virtio_pci_negotiate_features(struct virtio_softc *sc, uint32_t guest_features)
    454  1.2.2.1  pgoyette {
    455  1.2.2.1  pgoyette 	struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc;
    456  1.2.2.1  pgoyette 	uint32_t r;
    457  1.2.2.1  pgoyette 
    458  1.2.2.1  pgoyette 	r = bus_space_read_stream_4(psc->sc_iot, psc->sc_ioh,
    459  1.2.2.1  pgoyette 	    VIRTIO_CONFIG_DEVICE_FEATURES);
    460  1.2.2.1  pgoyette 	r &= guest_features;
    461  1.2.2.1  pgoyette 	bus_space_write_stream_4(psc->sc_iot, psc->sc_ioh,
    462  1.2.2.1  pgoyette 	    VIRTIO_CONFIG_GUEST_FEATURES, r);
    463  1.2.2.1  pgoyette 
    464  1.2.2.1  pgoyette 	return r;
    465  1.2.2.1  pgoyette }
    466  1.2.2.1  pgoyette 
    467  1.2.2.1  pgoyette 
    468  1.2.2.1  pgoyette static int
    469  1.2.2.1  pgoyette virtio_pci_setup_msix_vectors(struct virtio_softc *sc)
    470  1.2.2.1  pgoyette {
    471  1.2.2.1  pgoyette 	struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc;
    472  1.2.2.1  pgoyette 	int offset, vector, ret, qid;
    473  1.2.2.1  pgoyette 
    474  1.2.2.1  pgoyette 	offset = VIRTIO_CONFIG_MSI_CONFIG_VECTOR;
    475  1.2.2.1  pgoyette 	vector = VIRTIO_MSIX_CONFIG_VECTOR_INDEX;
    476      1.1    cherry 
    477  1.2.2.1  pgoyette 	bus_space_write_stream_2(psc->sc_iot, psc->sc_ioh, offset, vector);
    478  1.2.2.1  pgoyette 	ret = bus_space_read_stream_2(psc->sc_iot, psc->sc_ioh, offset);
    479  1.2.2.1  pgoyette 	aprint_debug_dev(sc->sc_dev, "expected=%d, actual=%d\n",
    480  1.2.2.1  pgoyette 	    vector, ret);
    481  1.2.2.1  pgoyette 	if (ret != vector)
    482  1.2.2.1  pgoyette 		return -1;
    483  1.2.2.1  pgoyette 
    484  1.2.2.1  pgoyette 	for (qid = 0; qid < sc->sc_nvqs; qid++) {
    485  1.2.2.1  pgoyette 		offset = VIRTIO_CONFIG_QUEUE_SELECT;
    486  1.2.2.1  pgoyette 		bus_space_write_stream_2(psc->sc_iot, psc->sc_ioh, offset, qid);
    487  1.2.2.1  pgoyette 
    488  1.2.2.1  pgoyette 		offset = VIRTIO_CONFIG_MSI_QUEUE_VECTOR;
    489  1.2.2.1  pgoyette 		vector = VIRTIO_MSIX_QUEUE_VECTOR_INDEX;
    490  1.2.2.1  pgoyette 
    491  1.2.2.1  pgoyette 		bus_space_write_stream_2(psc->sc_iot, psc->sc_ioh, offset, vector);
    492  1.2.2.1  pgoyette 		ret = bus_space_read_stream_2(psc->sc_iot, psc->sc_ioh, offset);
    493  1.2.2.1  pgoyette 		aprint_debug_dev(sc->sc_dev, "expected=%d, actual=%d\n",
    494  1.2.2.1  pgoyette 		    vector, ret);
    495  1.2.2.1  pgoyette 		if (ret != vector)
    496  1.2.2.1  pgoyette 			return -1;
    497  1.2.2.1  pgoyette 	}
    498      1.1    cherry 
    499      1.1    cherry 	return 0;
    500      1.1    cherry }
    501  1.2.2.1  pgoyette 
    502  1.2.2.1  pgoyette static int
    503  1.2.2.1  pgoyette virtio_pci_setup_msix_interrupts(struct virtio_softc *sc,
    504  1.2.2.1  pgoyette     struct pci_attach_args *pa)
    505  1.2.2.1  pgoyette {
    506  1.2.2.1  pgoyette 	struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc;
    507  1.2.2.1  pgoyette 	device_t self = sc->sc_dev;
    508  1.2.2.1  pgoyette 	pci_chipset_tag_t pc = pa->pa_pc;
    509  1.2.2.1  pgoyette 	char intrbuf[PCI_INTRSTR_LEN];
    510  1.2.2.1  pgoyette 	char const *intrstr;
    511  1.2.2.1  pgoyette 	int idx;
    512  1.2.2.1  pgoyette 
    513  1.2.2.1  pgoyette 	idx = VIRTIO_MSIX_CONFIG_VECTOR_INDEX;
    514  1.2.2.1  pgoyette 	if (sc->sc_flags & VIRTIO_F_PCI_INTR_MPSAFE)
    515  1.2.2.1  pgoyette 		pci_intr_setattr(pc, &psc->sc_ihp[idx], PCI_INTR_MPSAFE, true);
    516  1.2.2.1  pgoyette 
    517  1.2.2.1  pgoyette 	psc->sc_ihs[idx] = pci_intr_establish_xname(pc, psc->sc_ihp[idx],
    518  1.2.2.1  pgoyette 	    sc->sc_ipl, virtio_pci_msix_config_intr, sc, device_xname(sc->sc_dev));
    519  1.2.2.1  pgoyette 	if (psc->sc_ihs[idx] == NULL) {
    520  1.2.2.1  pgoyette 		aprint_error_dev(self, "couldn't establish MSI-X for config\n");
    521  1.2.2.1  pgoyette 		goto error;
    522  1.2.2.1  pgoyette 	}
    523  1.2.2.1  pgoyette 
    524  1.2.2.1  pgoyette 	idx = VIRTIO_MSIX_QUEUE_VECTOR_INDEX;
    525  1.2.2.1  pgoyette 	if (sc->sc_flags & VIRTIO_F_PCI_INTR_MPSAFE)
    526  1.2.2.1  pgoyette 		pci_intr_setattr(pc, &psc->sc_ihp[idx], PCI_INTR_MPSAFE, true);
    527  1.2.2.1  pgoyette 
    528  1.2.2.1  pgoyette 	psc->sc_ihs[idx] = pci_intr_establish_xname(pc, psc->sc_ihp[idx],
    529  1.2.2.1  pgoyette 	    sc->sc_ipl, virtio_pci_msix_queue_intr, sc, device_xname(sc->sc_dev));
    530  1.2.2.1  pgoyette 	if (psc->sc_ihs[idx] == NULL) {
    531  1.2.2.1  pgoyette 		aprint_error_dev(self, "couldn't establish MSI-X for queues\n");
    532  1.2.2.1  pgoyette 		goto error;
    533  1.2.2.1  pgoyette 	}
    534  1.2.2.1  pgoyette 
    535  1.2.2.1  pgoyette 	if (virtio_pci_setup_msix_vectors(sc) != 0) {
    536  1.2.2.1  pgoyette 		aprint_error_dev(self, "couldn't setup MSI-X vectors\n");
    537  1.2.2.1  pgoyette 		goto error;
    538  1.2.2.1  pgoyette 	}
    539  1.2.2.1  pgoyette 
    540  1.2.2.1  pgoyette 	idx = VIRTIO_MSIX_CONFIG_VECTOR_INDEX;
    541  1.2.2.1  pgoyette 	intrstr = pci_intr_string(pc, psc->sc_ihp[idx], intrbuf, sizeof(intrbuf));
    542  1.2.2.1  pgoyette 	aprint_normal_dev(self, "config interrupting at %s\n", intrstr);
    543  1.2.2.1  pgoyette 	idx = VIRTIO_MSIX_QUEUE_VECTOR_INDEX;
    544  1.2.2.1  pgoyette 	intrstr = pci_intr_string(pc, psc->sc_ihp[idx], intrbuf, sizeof(intrbuf));
    545  1.2.2.1  pgoyette 	aprint_normal_dev(self, "queues interrupting at %s\n", intrstr);
    546  1.2.2.1  pgoyette 
    547  1.2.2.1  pgoyette 	return 0;
    548  1.2.2.1  pgoyette 
    549  1.2.2.1  pgoyette error:
    550  1.2.2.1  pgoyette 	idx = VIRTIO_MSIX_CONFIG_VECTOR_INDEX;
    551  1.2.2.1  pgoyette 	if (psc->sc_ihs[idx] != NULL)
    552  1.2.2.1  pgoyette 		pci_intr_disestablish(psc->sc_pa.pa_pc, psc->sc_ihs[idx]);
    553  1.2.2.1  pgoyette 	idx = VIRTIO_MSIX_QUEUE_VECTOR_INDEX;
    554  1.2.2.1  pgoyette 	if (psc->sc_ihs[idx] != NULL)
    555  1.2.2.1  pgoyette 		pci_intr_disestablish(psc->sc_pa.pa_pc, psc->sc_ihs[idx]);
    556  1.2.2.1  pgoyette 
    557  1.2.2.1  pgoyette 	return -1;
    558  1.2.2.1  pgoyette }
    559  1.2.2.1  pgoyette 
    560  1.2.2.1  pgoyette static int
    561  1.2.2.1  pgoyette virtio_pci_setup_intx_interrupt(struct virtio_softc *sc,
    562  1.2.2.1  pgoyette     struct pci_attach_args *pa)
    563  1.2.2.1  pgoyette {
    564  1.2.2.1  pgoyette 	struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc;
    565  1.2.2.1  pgoyette 	device_t self = sc->sc_dev;
    566  1.2.2.1  pgoyette 	pci_chipset_tag_t pc = pa->pa_pc;
    567  1.2.2.1  pgoyette 	char intrbuf[PCI_INTRSTR_LEN];
    568  1.2.2.1  pgoyette 	char const *intrstr;
    569  1.2.2.1  pgoyette 
    570  1.2.2.1  pgoyette 	if (sc->sc_flags & VIRTIO_F_PCI_INTR_MPSAFE)
    571  1.2.2.1  pgoyette 		pci_intr_setattr(pc, &psc->sc_ihp[0], PCI_INTR_MPSAFE, true);
    572  1.2.2.1  pgoyette 
    573  1.2.2.1  pgoyette 	psc->sc_ihs[0] = pci_intr_establish_xname(pc, psc->sc_ihp[0],
    574  1.2.2.1  pgoyette 	    sc->sc_ipl, virtio_pci_intr, sc, device_xname(sc->sc_dev));
    575  1.2.2.1  pgoyette 	if (psc->sc_ihs[0] == NULL) {
    576  1.2.2.1  pgoyette 		aprint_error_dev(self, "couldn't establish INTx\n");
    577  1.2.2.1  pgoyette 		return -1;
    578  1.2.2.1  pgoyette 	}
    579  1.2.2.1  pgoyette 
    580  1.2.2.1  pgoyette 	intrstr = pci_intr_string(pc, psc->sc_ihp[0], intrbuf, sizeof(intrbuf));
    581  1.2.2.1  pgoyette 	aprint_normal_dev(self, "interrupting at %s\n", intrstr);
    582  1.2.2.1  pgoyette 
    583  1.2.2.1  pgoyette 	return 0;
    584  1.2.2.1  pgoyette }
    585  1.2.2.1  pgoyette 
    586  1.2.2.1  pgoyette static int
    587  1.2.2.1  pgoyette virtio_pci_setup_interrupts(struct virtio_softc *sc)
    588  1.2.2.1  pgoyette {
    589  1.2.2.1  pgoyette 	struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc;
    590  1.2.2.1  pgoyette 	device_t self = sc->sc_dev;
    591  1.2.2.1  pgoyette 	pci_chipset_tag_t pc = psc->sc_pa.pa_pc;
    592  1.2.2.1  pgoyette 	int error;
    593  1.2.2.1  pgoyette 	int nmsix;
    594  1.2.2.1  pgoyette 	int counts[PCI_INTR_TYPE_SIZE];
    595  1.2.2.1  pgoyette 	pci_intr_type_t max_type;
    596  1.2.2.1  pgoyette 
    597  1.2.2.1  pgoyette 	nmsix = pci_msix_count(psc->sc_pa.pa_pc, psc->sc_pa.pa_tag);
    598  1.2.2.1  pgoyette 	aprint_debug_dev(self, "pci_msix_count=%d\n", nmsix);
    599  1.2.2.1  pgoyette 
    600  1.2.2.1  pgoyette 	/* We need at least two: one for config and the other for queues */
    601  1.2.2.1  pgoyette 	if ((sc->sc_flags & VIRTIO_F_PCI_INTR_MSIX) == 0 || nmsix < 2) {
    602  1.2.2.1  pgoyette 		/* Try INTx only */
    603  1.2.2.1  pgoyette 		max_type = PCI_INTR_TYPE_INTX;
    604  1.2.2.1  pgoyette 		counts[PCI_INTR_TYPE_INTX] = 1;
    605  1.2.2.1  pgoyette 	} else {
    606  1.2.2.1  pgoyette 		/* Try MSI-X first and INTx second */
    607  1.2.2.1  pgoyette 		max_type = PCI_INTR_TYPE_MSIX;
    608  1.2.2.1  pgoyette 		counts[PCI_INTR_TYPE_MSIX] = 2;
    609  1.2.2.1  pgoyette 		counts[PCI_INTR_TYPE_MSI] = 0;
    610  1.2.2.1  pgoyette 		counts[PCI_INTR_TYPE_INTX] = 1;
    611  1.2.2.1  pgoyette 	}
    612  1.2.2.1  pgoyette 
    613  1.2.2.1  pgoyette retry:
    614  1.2.2.1  pgoyette 	error = pci_intr_alloc(&psc->sc_pa, &psc->sc_ihp, counts, max_type);
    615  1.2.2.1  pgoyette 	if (error != 0) {
    616  1.2.2.1  pgoyette 		aprint_error_dev(self, "couldn't map interrupt\n");
    617  1.2.2.1  pgoyette 		return -1;
    618  1.2.2.1  pgoyette 	}
    619  1.2.2.1  pgoyette 
    620  1.2.2.1  pgoyette 	if (pci_intr_type(pc, psc->sc_ihp[0]) == PCI_INTR_TYPE_MSIX) {
    621  1.2.2.1  pgoyette 		psc->sc_ihs = kmem_alloc(sizeof(*psc->sc_ihs) * 2,
    622  1.2.2.1  pgoyette 		    KM_SLEEP);
    623  1.2.2.1  pgoyette 
    624  1.2.2.1  pgoyette 		error = virtio_pci_setup_msix_interrupts(sc, &psc->sc_pa);
    625  1.2.2.1  pgoyette 		if (error != 0) {
    626  1.2.2.1  pgoyette 			kmem_free(psc->sc_ihs, sizeof(*psc->sc_ihs) * 2);
    627  1.2.2.1  pgoyette 			pci_intr_release(pc, psc->sc_ihp, 2);
    628  1.2.2.1  pgoyette 
    629  1.2.2.1  pgoyette 			/* Retry INTx */
    630  1.2.2.1  pgoyette 			max_type = PCI_INTR_TYPE_INTX;
    631  1.2.2.1  pgoyette 			counts[PCI_INTR_TYPE_INTX] = 1;
    632  1.2.2.1  pgoyette 			goto retry;
    633  1.2.2.1  pgoyette 		}
    634  1.2.2.1  pgoyette 
    635  1.2.2.1  pgoyette 		psc->sc_ihs_num = 2;
    636  1.2.2.1  pgoyette 		psc->sc_config_offset = VIRTIO_CONFIG_DEVICE_CONFIG_MSI;
    637  1.2.2.1  pgoyette 	} else if (pci_intr_type(pc, psc->sc_ihp[0]) == PCI_INTR_TYPE_INTX) {
    638  1.2.2.1  pgoyette 		psc->sc_ihs = kmem_alloc(sizeof(*psc->sc_ihs) * 1,
    639  1.2.2.1  pgoyette 		    KM_SLEEP);
    640  1.2.2.1  pgoyette 
    641  1.2.2.1  pgoyette 		error = virtio_pci_setup_intx_interrupt(sc, &psc->sc_pa);
    642  1.2.2.1  pgoyette 		if (error != 0) {
    643  1.2.2.1  pgoyette 			kmem_free(psc->sc_ihs, sizeof(*psc->sc_ihs) * 1);
    644  1.2.2.1  pgoyette 			pci_intr_release(pc, psc->sc_ihp, 1);
    645  1.2.2.1  pgoyette 			return -1;
    646  1.2.2.1  pgoyette 		}
    647  1.2.2.1  pgoyette 
    648  1.2.2.1  pgoyette 		psc->sc_ihs_num = 1;
    649  1.2.2.1  pgoyette 		psc->sc_config_offset = VIRTIO_CONFIG_DEVICE_CONFIG_NOMSI;
    650  1.2.2.1  pgoyette 	}
    651  1.2.2.1  pgoyette 
    652  1.2.2.1  pgoyette 	return 0;
    653  1.2.2.1  pgoyette }
    654  1.2.2.1  pgoyette 
    655  1.2.2.1  pgoyette static void
    656  1.2.2.1  pgoyette virtio_pci_free_interrupts(struct virtio_softc *sc)
    657  1.2.2.1  pgoyette {
    658  1.2.2.1  pgoyette 	struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc;
    659  1.2.2.1  pgoyette 
    660  1.2.2.1  pgoyette 	for (int i = 0; i < psc->sc_ihs_num; i++) {
    661  1.2.2.1  pgoyette 		if (psc->sc_ihs[i] == NULL)
    662  1.2.2.1  pgoyette 			continue;
    663  1.2.2.1  pgoyette 		pci_intr_disestablish(psc->sc_pa.pa_pc, psc->sc_ihs[i]);
    664  1.2.2.1  pgoyette 		psc->sc_ihs[i] = NULL;
    665  1.2.2.1  pgoyette 	}
    666  1.2.2.1  pgoyette 
    667  1.2.2.1  pgoyette 	if (psc->sc_ihs_num > 0)
    668  1.2.2.1  pgoyette 		pci_intr_release(psc->sc_pa.pa_pc, psc->sc_ihp, psc->sc_ihs_num);
    669  1.2.2.1  pgoyette 
    670  1.2.2.1  pgoyette 	if (psc->sc_ihs != NULL) {
    671  1.2.2.1  pgoyette 		kmem_free(psc->sc_ihs, sizeof(*psc->sc_ihs) * psc->sc_ihs_num);
    672  1.2.2.1  pgoyette 		psc->sc_ihs = NULL;
    673  1.2.2.1  pgoyette 	}
    674  1.2.2.1  pgoyette 	psc->sc_ihs_num = 0;
    675  1.2.2.1  pgoyette }
    676  1.2.2.1  pgoyette 
    677  1.2.2.1  pgoyette /*
    678  1.2.2.1  pgoyette  * Interrupt handler.
    679  1.2.2.1  pgoyette  */
    680  1.2.2.1  pgoyette static int
    681  1.2.2.1  pgoyette virtio_pci_intr(void *arg)
    682  1.2.2.1  pgoyette {
    683  1.2.2.1  pgoyette 	struct virtio_softc *sc = arg;
    684  1.2.2.1  pgoyette 	struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc;
    685  1.2.2.1  pgoyette 	int isr, r = 0;
    686  1.2.2.1  pgoyette 
    687  1.2.2.1  pgoyette 	/* check and ack the interrupt */
    688  1.2.2.1  pgoyette 	isr = bus_space_read_stream_1(psc->sc_iot, psc->sc_ioh,
    689  1.2.2.1  pgoyette 			       VIRTIO_CONFIG_ISR_STATUS);
    690  1.2.2.1  pgoyette 	if (isr == 0)
    691  1.2.2.1  pgoyette 		return 0;
    692  1.2.2.1  pgoyette 	if ((isr & VIRTIO_CONFIG_ISR_CONFIG_CHANGE) &&
    693  1.2.2.1  pgoyette 	    (sc->sc_config_change != NULL))
    694  1.2.2.1  pgoyette 		r = (sc->sc_config_change)(sc);
    695  1.2.2.1  pgoyette 	if (sc->sc_intrhand != NULL) {
    696  1.2.2.1  pgoyette 		if (sc->sc_soft_ih != NULL)
    697  1.2.2.1  pgoyette 			softint_schedule(sc->sc_soft_ih);
    698  1.2.2.1  pgoyette 		else
    699  1.2.2.1  pgoyette 			r |= (sc->sc_intrhand)(sc);
    700  1.2.2.1  pgoyette 	}
    701  1.2.2.1  pgoyette 
    702  1.2.2.1  pgoyette 	return r;
    703  1.2.2.1  pgoyette }
    704  1.2.2.1  pgoyette 
    705  1.2.2.1  pgoyette static int
    706  1.2.2.1  pgoyette virtio_pci_msix_queue_intr(void *arg)
    707  1.2.2.1  pgoyette {
    708  1.2.2.1  pgoyette 	struct virtio_softc *sc = arg;
    709  1.2.2.1  pgoyette 	int r = 0;
    710  1.2.2.1  pgoyette 
    711  1.2.2.1  pgoyette 	if (sc->sc_intrhand != NULL) {
    712  1.2.2.1  pgoyette 		if (sc->sc_soft_ih != NULL)
    713  1.2.2.1  pgoyette 			softint_schedule(sc->sc_soft_ih);
    714  1.2.2.1  pgoyette 		else
    715  1.2.2.1  pgoyette 			r |= (sc->sc_intrhand)(sc);
    716  1.2.2.1  pgoyette 	}
    717  1.2.2.1  pgoyette 
    718  1.2.2.1  pgoyette 	return r;
    719  1.2.2.1  pgoyette }
    720  1.2.2.1  pgoyette 
    721  1.2.2.1  pgoyette static int
    722  1.2.2.1  pgoyette virtio_pci_msix_config_intr(void *arg)
    723  1.2.2.1  pgoyette {
    724  1.2.2.1  pgoyette 	struct virtio_softc *sc = arg;
    725  1.2.2.1  pgoyette 	int r = 0;
    726  1.2.2.1  pgoyette 
    727  1.2.2.1  pgoyette 	if (sc->sc_config_change != NULL)
    728  1.2.2.1  pgoyette 		r = (sc->sc_config_change)(sc);
    729  1.2.2.1  pgoyette 	return r;
    730  1.2.2.1  pgoyette }
    731  1.2.2.1  pgoyette 
    732  1.2.2.1  pgoyette MODULE(MODULE_CLASS_DRIVER, virtio_pci, "pci,virtio");
    733  1.2.2.1  pgoyette 
    734  1.2.2.1  pgoyette #ifdef _MODULE
    735  1.2.2.1  pgoyette #include "ioconf.c"
    736  1.2.2.1  pgoyette #endif
    737  1.2.2.1  pgoyette 
    738  1.2.2.1  pgoyette static int
    739  1.2.2.1  pgoyette virtio_pci_modcmd(modcmd_t cmd, void *opaque)
    740  1.2.2.1  pgoyette {
    741  1.2.2.1  pgoyette 	int error = 0;
    742  1.2.2.1  pgoyette 
    743  1.2.2.1  pgoyette #ifdef _MODULE
    744  1.2.2.1  pgoyette 	switch (cmd) {
    745  1.2.2.1  pgoyette 	case MODULE_CMD_INIT:
    746  1.2.2.1  pgoyette 		error = config_init_component(cfdriver_ioconf_virtio_pci,
    747  1.2.2.1  pgoyette 		    cfattach_ioconf_virtio_pci, cfdata_ioconf_virtio_pci);
    748  1.2.2.1  pgoyette 		break;
    749  1.2.2.1  pgoyette 	case MODULE_CMD_FINI:
    750  1.2.2.1  pgoyette 		error = config_fini_component(cfdriver_ioconf_virtio_pci,
    751  1.2.2.1  pgoyette 		    cfattach_ioconf_virtio_pci, cfdata_ioconf_virtio_pci);
    752  1.2.2.1  pgoyette 		break;
    753  1.2.2.1  pgoyette 	default:
    754  1.2.2.1  pgoyette 		error = ENOTTY;
    755  1.2.2.1  pgoyette 		break;
    756  1.2.2.1  pgoyette 	}
    757  1.2.2.1  pgoyette #endif
    758  1.2.2.1  pgoyette 
    759  1.2.2.1  pgoyette 	return error;
    760  1.2.2.1  pgoyette }
    761  1.2.2.1  pgoyette 
    762