Home | History | Annotate | Line # | Download | only in pci
virtio_pci.c revision 1.14.2.1
      1  1.14.2.1   thorpej /* $NetBSD: virtio_pci.c,v 1.14.2.1 2021/04/03 22:28:49 thorpej Exp $ */
      2       1.1    cherry 
      3       1.1    cherry /*
      4  1.14.2.1   thorpej  * Copyright (c) 2020 The NetBSD Foundation, Inc.
      5  1.14.2.1   thorpej  * Copyright (c) 2012 Stefan Fritsch.
      6       1.1    cherry  * Copyright (c) 2010 Minoura Makoto.
      7       1.1    cherry  * All rights reserved.
      8       1.1    cherry  *
      9       1.1    cherry  * Redistribution and use in source and binary forms, with or without
     10       1.1    cherry  * modification, are permitted provided that the following conditions
     11       1.1    cherry  * are met:
     12       1.1    cherry  * 1. Redistributions of source code must retain the above copyright
     13       1.1    cherry  *    notice, this list of conditions and the following disclaimer.
     14       1.1    cherry  * 2. Redistributions in binary form must reproduce the above copyright
     15       1.1    cherry  *    notice, this list of conditions and the following disclaimer in the
     16       1.1    cherry  *    documentation and/or other materials provided with the distribution.
     17       1.1    cherry  *
     18       1.1    cherry  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
     19       1.1    cherry  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
     20       1.1    cherry  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
     21       1.1    cherry  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
     22       1.1    cherry  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
     23       1.1    cherry  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
     24       1.1    cherry  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
     25       1.1    cherry  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
     26       1.1    cherry  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
     27       1.1    cherry  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
     28       1.1    cherry  */
     29       1.1    cherry 
     30       1.1    cherry #include <sys/cdefs.h>
     31  1.14.2.1   thorpej __KERNEL_RCSID(0, "$NetBSD: virtio_pci.c,v 1.14.2.1 2021/04/03 22:28:49 thorpej Exp $");
     32       1.1    cherry 
     33       1.1    cherry #include <sys/param.h>
     34       1.1    cherry #include <sys/systm.h>
     35       1.4  jakllsch #include <sys/kmem.h>
     36       1.5  jakllsch #include <sys/module.h>
     37  1.14.2.1   thorpej #include <sys/endian.h>
     38       1.6  yamaguch #include <sys/interrupt.h>
     39       1.1    cherry 
     40       1.1    cherry #include <sys/device.h>
     41       1.1    cherry 
     42       1.1    cherry #include <dev/pci/pcidevs.h>
     43       1.1    cherry #include <dev/pci/pcireg.h>
     44       1.1    cherry #include <dev/pci/pcivar.h>
     45       1.1    cherry 
     46       1.1    cherry #include <dev/pci/virtioreg.h> /* XXX: move to non-pci */
     47  1.14.2.1   thorpej #include <dev/pci/virtio_pcireg.h>
     48  1.14.2.1   thorpej 
     49  1.14.2.1   thorpej #define VIRTIO_PRIVATE
     50       1.1    cherry #include <dev/pci/virtiovar.h> /* XXX: move to non-pci */
     51       1.1    cherry 
     52  1.14.2.1   thorpej 
     53       1.4  jakllsch static int	virtio_pci_match(device_t, cfdata_t, void *);
     54       1.4  jakllsch static void	virtio_pci_attach(device_t, device_t, void *);
     55       1.4  jakllsch static int	virtio_pci_rescan(device_t, const char *, const int *);
     56       1.4  jakllsch static int	virtio_pci_detach(device_t, int);
     57       1.4  jakllsch 
     58  1.14.2.1   thorpej 
     59  1.14.2.1   thorpej #define NMAPREG		((PCI_MAPREG_END - PCI_MAPREG_START) / \
     60  1.14.2.1   thorpej 				sizeof(pcireg_t))
     61       1.4  jakllsch struct virtio_pci_softc {
     62       1.4  jakllsch 	struct virtio_softc	sc_sc;
     63  1.14.2.1   thorpej 
     64  1.14.2.1   thorpej 	/* IO space */
     65       1.4  jakllsch 	bus_space_tag_t		sc_iot;
     66       1.4  jakllsch 	bus_space_handle_t	sc_ioh;
     67       1.4  jakllsch 	bus_size_t		sc_iosize;
     68  1.14.2.1   thorpej 	bus_size_t		sc_mapped_iosize;
     69  1.14.2.1   thorpej 
     70  1.14.2.1   thorpej 	/* BARs */
     71  1.14.2.1   thorpej 	bus_space_tag_t		sc_bars_iot[NMAPREG];
     72  1.14.2.1   thorpej 	bus_space_handle_t	sc_bars_ioh[NMAPREG];
     73  1.14.2.1   thorpej 	bus_size_t		sc_bars_iosize[NMAPREG];
     74  1.14.2.1   thorpej 
     75  1.14.2.1   thorpej 	/* notify space */
     76  1.14.2.1   thorpej 	bus_space_tag_t		sc_notify_iot;
     77  1.14.2.1   thorpej 	bus_space_handle_t	sc_notify_ioh;
     78  1.14.2.1   thorpej 	bus_size_t		sc_notify_iosize;
     79  1.14.2.1   thorpej 	uint32_t		sc_notify_off_multiplier;
     80  1.14.2.1   thorpej 
     81  1.14.2.1   thorpej 	/* isr space */
     82  1.14.2.1   thorpej 	bus_space_tag_t		sc_isr_iot;
     83  1.14.2.1   thorpej 	bus_space_handle_t	sc_isr_ioh;
     84  1.14.2.1   thorpej 	bus_size_t		sc_isr_iosize;
     85  1.14.2.1   thorpej 
     86  1.14.2.1   thorpej 	/* generic */
     87       1.4  jakllsch 	struct pci_attach_args	sc_pa;
     88       1.4  jakllsch 	pci_intr_handle_t	*sc_ihp;
     89       1.4  jakllsch 	void			**sc_ihs;
     90       1.4  jakllsch 	int			sc_ihs_num;
     91  1.14.2.1   thorpej 	int			sc_devcfg_offset;	/* for 0.9 */
     92       1.4  jakllsch };
     93       1.4  jakllsch 
     94  1.14.2.1   thorpej static int	virtio_pci_attach_09(device_t, void *);
     95  1.14.2.1   thorpej static void	virtio_pci_kick_09(struct virtio_softc *, uint16_t);
     96  1.14.2.1   thorpej static uint16_t	virtio_pci_read_queue_size_09(struct virtio_softc *, uint16_t);
     97  1.14.2.1   thorpej static void	virtio_pci_setup_queue_09(struct virtio_softc *, uint16_t, uint64_t);
     98  1.14.2.1   thorpej static void	virtio_pci_set_status_09(struct virtio_softc *, int);
     99  1.14.2.1   thorpej static void	virtio_pci_negotiate_features_09(struct virtio_softc *, uint64_t);
    100  1.14.2.1   thorpej 
    101  1.14.2.1   thorpej static int	virtio_pci_attach_10(device_t, void *);
    102  1.14.2.1   thorpej static void	virtio_pci_kick_10(struct virtio_softc *, uint16_t);
    103  1.14.2.1   thorpej static uint16_t	virtio_pci_read_queue_size_10(struct virtio_softc *, uint16_t);
    104  1.14.2.1   thorpej static void	virtio_pci_setup_queue_10(struct virtio_softc *, uint16_t, uint64_t);
    105  1.14.2.1   thorpej static void	virtio_pci_set_status_10(struct virtio_softc *, int);
    106  1.14.2.1   thorpej static void	virtio_pci_negotiate_features_10(struct virtio_softc *, uint64_t);
    107  1.14.2.1   thorpej static int	virtio_pci_find_cap(struct virtio_pci_softc *psc, int cfg_type, void *buf, int buflen);
    108  1.14.2.1   thorpej 
    109       1.4  jakllsch static int	virtio_pci_setup_interrupts(struct virtio_softc *);
    110       1.4  jakllsch static void	virtio_pci_free_interrupts(struct virtio_softc *);
    111  1.14.2.1   thorpej static int	virtio_pci_adjust_config_region(struct virtio_pci_softc *psc);
    112       1.4  jakllsch static int	virtio_pci_intr(void *arg);
    113       1.4  jakllsch static int	virtio_pci_msix_queue_intr(void *);
    114       1.4  jakllsch static int	virtio_pci_msix_config_intr(void *);
    115  1.14.2.1   thorpej static int	virtio_pci_setup_msix_vectors_09(struct virtio_softc *);
    116  1.14.2.1   thorpej static int	virtio_pci_setup_msix_vectors_10(struct virtio_softc *);
    117       1.4  jakllsch static int	virtio_pci_setup_msix_interrupts(struct virtio_softc *,
    118       1.4  jakllsch 		    struct pci_attach_args *);
    119       1.4  jakllsch static int	virtio_pci_setup_intx_interrupt(struct virtio_softc *,
    120       1.4  jakllsch 		    struct pci_attach_args *);
    121       1.4  jakllsch 
    122       1.4  jakllsch #define VIRTIO_MSIX_CONFIG_VECTOR_INDEX	0
    123       1.4  jakllsch #define VIRTIO_MSIX_QUEUE_VECTOR_INDEX	1
    124       1.4  jakllsch 
    125  1.14.2.1   thorpej /*
    126  1.14.2.1   thorpej  * When using PCI attached virtio on aarch64-eb under Qemu, the IO space
    127  1.14.2.1   thorpej  * suddenly read BIG_ENDIAN where it should stay LITTLE_ENDIAN. The data read
    128  1.14.2.1   thorpej  * 1 byte at a time seem OK but reading bigger lengths result in swapped
    129  1.14.2.1   thorpej  * endian. This is most notable on reading 8 byters since we can't use
    130  1.14.2.1   thorpej  * bus_space_{read,write}_8().
    131  1.14.2.1   thorpej  */
    132       1.4  jakllsch 
    133  1.14.2.1   thorpej #if defined(__aarch64__) && BYTE_ORDER == BIG_ENDIAN
    134  1.14.2.1   thorpej #	define READ_ENDIAN_09	BIG_ENDIAN	/* should be LITTLE_ENDIAN */
    135  1.14.2.1   thorpej #	define READ_ENDIAN_10	BIG_ENDIAN
    136  1.14.2.1   thorpej #	define STRUCT_ENDIAN_09	BIG_ENDIAN
    137  1.14.2.1   thorpej #	define STRUCT_ENDIAN_10	LITTLE_ENDIAN
    138  1.14.2.1   thorpej #elif BYTE_ORDER == BIG_ENDIAN
    139  1.14.2.1   thorpej #	define READ_ENDIAN_09	LITTLE_ENDIAN
    140  1.14.2.1   thorpej #	define READ_ENDIAN_10	BIG_ENDIAN
    141  1.14.2.1   thorpej #	define STRUCT_ENDIAN_09	BIG_ENDIAN
    142  1.14.2.1   thorpej #	define STRUCT_ENDIAN_10	LITTLE_ENDIAN
    143  1.14.2.1   thorpej #else /* little endian */
    144  1.14.2.1   thorpej #	define READ_ENDIAN_09	LITTLE_ENDIAN
    145  1.14.2.1   thorpej #	define READ_ENDIAN_10	LITTLE_ENDIAN
    146  1.14.2.1   thorpej #	define STRUCT_ENDIAN_09	LITTLE_ENDIAN
    147  1.14.2.1   thorpej #	define STRUCT_ENDIAN_10	LITTLE_ENDIAN
    148  1.14.2.1   thorpej #endif
    149       1.1    cherry 
    150       1.1    cherry 
    151       1.4  jakllsch CFATTACH_DECL3_NEW(virtio_pci, sizeof(struct virtio_pci_softc),
    152       1.4  jakllsch     virtio_pci_match, virtio_pci_attach, virtio_pci_detach, NULL,
    153       1.4  jakllsch     virtio_pci_rescan, NULL, DVF_DETACH_SHUTDOWN);
    154       1.4  jakllsch 
    155  1.14.2.1   thorpej static const struct virtio_ops virtio_pci_ops_09 = {
    156  1.14.2.1   thorpej 	.kick = virtio_pci_kick_09,
    157  1.14.2.1   thorpej 	.read_queue_size = virtio_pci_read_queue_size_09,
    158  1.14.2.1   thorpej 	.setup_queue = virtio_pci_setup_queue_09,
    159  1.14.2.1   thorpej 	.set_status = virtio_pci_set_status_09,
    160  1.14.2.1   thorpej 	.neg_features = virtio_pci_negotiate_features_09,
    161  1.14.2.1   thorpej 	.setup_interrupts = virtio_pci_setup_interrupts,
    162  1.14.2.1   thorpej 	.free_interrupts = virtio_pci_free_interrupts,
    163  1.14.2.1   thorpej };
    164  1.14.2.1   thorpej 
    165  1.14.2.1   thorpej static const struct virtio_ops virtio_pci_ops_10 = {
    166  1.14.2.1   thorpej 	.kick = virtio_pci_kick_10,
    167  1.14.2.1   thorpej 	.read_queue_size = virtio_pci_read_queue_size_10,
    168  1.14.2.1   thorpej 	.setup_queue = virtio_pci_setup_queue_10,
    169  1.14.2.1   thorpej 	.set_status = virtio_pci_set_status_10,
    170  1.14.2.1   thorpej 	.neg_features = virtio_pci_negotiate_features_10,
    171       1.4  jakllsch 	.setup_interrupts = virtio_pci_setup_interrupts,
    172       1.4  jakllsch 	.free_interrupts = virtio_pci_free_interrupts,
    173       1.4  jakllsch };
    174       1.1    cherry 
    175       1.1    cherry static int
    176       1.4  jakllsch virtio_pci_match(device_t parent, cfdata_t match, void *aux)
    177       1.1    cherry {
    178       1.1    cherry 	struct pci_attach_args *pa;
    179       1.1    cherry 
    180       1.1    cherry 	pa = (struct pci_attach_args *)aux;
    181       1.1    cherry 	switch (PCI_VENDOR(pa->pa_id)) {
    182       1.1    cherry 	case PCI_VENDOR_QUMRANET:
    183  1.14.2.1   thorpej 		if (((PCI_PRODUCT_QUMRANET_VIRTIO_1000 <=
    184  1.14.2.1   thorpej 		      PCI_PRODUCT(pa->pa_id)) &&
    185  1.14.2.1   thorpej 		     (PCI_PRODUCT(pa->pa_id) <=
    186  1.14.2.1   thorpej 		      PCI_PRODUCT_QUMRANET_VIRTIO_103F)) &&
    187  1.14.2.1   thorpej 	              PCI_REVISION(pa->pa_class) == 0)
    188  1.14.2.1   thorpej 			return 1;
    189  1.14.2.1   thorpej 		if (((PCI_PRODUCT_QUMRANET_VIRTIO_1040 <=
    190  1.14.2.1   thorpej 		      PCI_PRODUCT(pa->pa_id)) &&
    191  1.14.2.1   thorpej 		     (PCI_PRODUCT(pa->pa_id) <=
    192  1.14.2.1   thorpej 		      PCI_PRODUCT_QUMRANET_VIRTIO_107F)) &&
    193  1.14.2.1   thorpej 		      PCI_REVISION(pa->pa_class) == 1)
    194       1.1    cherry 			return 1;
    195       1.1    cherry 		break;
    196       1.1    cherry 	}
    197       1.1    cherry 
    198       1.1    cherry 	return 0;
    199       1.1    cherry }
    200       1.1    cherry 
    201       1.1    cherry static void
    202       1.4  jakllsch virtio_pci_attach(device_t parent, device_t self, void *aux)
    203       1.1    cherry {
    204       1.4  jakllsch 	struct virtio_pci_softc * const psc = device_private(self);
    205       1.4  jakllsch 	struct virtio_softc * const sc = &psc->sc_sc;
    206       1.1    cherry 	struct pci_attach_args *pa = (struct pci_attach_args *)aux;
    207       1.1    cherry 	pci_chipset_tag_t pc = pa->pa_pc;
    208       1.1    cherry 	pcitag_t tag = pa->pa_tag;
    209       1.1    cherry 	int revision;
    210  1.14.2.1   thorpej 	int ret;
    211       1.1    cherry 	pcireg_t id;
    212       1.2       uwe 	pcireg_t csr;
    213       1.1    cherry 
    214       1.1    cherry 	revision = PCI_REVISION(pa->pa_class);
    215  1.14.2.1   thorpej 	switch (revision) {
    216  1.14.2.1   thorpej 	case 0:
    217  1.14.2.1   thorpej 		/* subsystem ID shows what I am */
    218  1.14.2.1   thorpej 		id = PCI_SUBSYS_ID(pci_conf_read(pc, tag, PCI_SUBSYS_ID_REG));
    219  1.14.2.1   thorpej 		break;
    220  1.14.2.1   thorpej 	case 1:
    221  1.14.2.1   thorpej 		/* pci product number shows what I am */
    222  1.14.2.1   thorpej 		id = PCI_PRODUCT(pa->pa_id) - PCI_PRODUCT_QUMRANET_VIRTIO_1040;
    223  1.14.2.1   thorpej 		break;
    224  1.14.2.1   thorpej 	default:
    225       1.1    cherry 		aprint_normal(": unknown revision 0x%02x; giving up\n",
    226       1.1    cherry 			      revision);
    227       1.1    cherry 		return;
    228       1.1    cherry 	}
    229  1.14.2.1   thorpej 
    230       1.1    cherry 	aprint_normal("\n");
    231       1.1    cherry 	aprint_naive("\n");
    232  1.14.2.1   thorpej 	virtio_print_device_type(self, id, revision);
    233       1.1    cherry 
    234       1.2       uwe 	csr = pci_conf_read(pc, tag, PCI_COMMAND_STATUS_REG);
    235       1.2       uwe 	csr |= PCI_COMMAND_MASTER_ENABLE | PCI_COMMAND_IO_ENABLE;
    236       1.2       uwe 	pci_conf_write(pc, tag, PCI_COMMAND_STATUS_REG, csr);
    237       1.2       uwe 
    238       1.1    cherry 	sc->sc_dev = self;
    239       1.4  jakllsch 	psc->sc_pa = *pa;
    240       1.4  jakllsch 	psc->sc_iot = pa->pa_iot;
    241  1.14.2.1   thorpej 
    242  1.14.2.1   thorpej 	sc->sc_dmat = pa->pa_dmat;
    243       1.1    cherry 	if (pci_dma64_available(pa))
    244       1.1    cherry 		sc->sc_dmat = pa->pa_dmat64;
    245       1.1    cherry 
    246  1.14.2.1   thorpej 	/* attach is dependent on revision */
    247  1.14.2.1   thorpej 	ret = 0;
    248  1.14.2.1   thorpej 	if (revision == 1) {
    249  1.14.2.1   thorpej 		/* try to attach 1.0 */
    250  1.14.2.1   thorpej 		ret = virtio_pci_attach_10(self, aux);
    251  1.14.2.1   thorpej 	}
    252  1.14.2.1   thorpej 	if (ret == 0 && revision == 0) {
    253  1.14.2.1   thorpej 		/* revision 0 means 0.9 only or both 0.9 and 1.0 */
    254  1.14.2.1   thorpej 		ret = virtio_pci_attach_09(self, aux);
    255  1.14.2.1   thorpej 	}
    256  1.14.2.1   thorpej 	if (ret) {
    257  1.14.2.1   thorpej 		aprint_error_dev(self, "cannot attach (%d)\n", ret);
    258       1.1    cherry 		return;
    259       1.1    cherry 	}
    260  1.14.2.1   thorpej 	KASSERT(sc->sc_ops);
    261  1.14.2.1   thorpej 
    262  1.14.2.1   thorpej 	/* preset config region */
    263  1.14.2.1   thorpej 	psc->sc_devcfg_offset = VIRTIO_CONFIG_DEVICE_CONFIG_NOMSI;
    264  1.14.2.1   thorpej 	if (virtio_pci_adjust_config_region(psc))
    265  1.14.2.1   thorpej 		return;
    266       1.1    cherry 
    267  1.14.2.1   thorpej 	/* generic */
    268       1.1    cherry 	virtio_device_reset(sc);
    269       1.1    cherry 	virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_ACK);
    270       1.1    cherry 	virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_DRIVER);
    271       1.1    cherry 
    272  1.14.2.1   thorpej 	sc->sc_childdevid = id;
    273       1.1    cherry 	sc->sc_child = NULL;
    274       1.4  jakllsch 	virtio_pci_rescan(self, "virtio", 0);
    275       1.1    cherry 	return;
    276       1.1    cherry }
    277       1.1    cherry 
    278       1.1    cherry /* ARGSUSED */
    279       1.1    cherry static int
    280       1.4  jakllsch virtio_pci_rescan(device_t self, const char *attr, const int *scan_flags)
    281       1.1    cherry {
    282       1.4  jakllsch 	struct virtio_pci_softc * const psc = device_private(self);
    283       1.4  jakllsch 	struct virtio_softc * const sc = &psc->sc_sc;
    284       1.1    cherry 	struct virtio_attach_args va;
    285       1.1    cherry 
    286       1.1    cherry 	if (sc->sc_child)	/* Child already attached? */
    287       1.1    cherry 		return 0;
    288       1.1    cherry 
    289       1.1    cherry 	memset(&va, 0, sizeof(va));
    290       1.1    cherry 	va.sc_childdevid = sc->sc_childdevid;
    291       1.1    cherry 
    292       1.1    cherry 	config_found_ia(self, attr, &va, NULL);
    293       1.1    cherry 
    294  1.14.2.1   thorpej 	if (virtio_attach_failed(sc))
    295       1.1    cherry 		return 0;
    296       1.1    cherry 
    297       1.1    cherry 	return 0;
    298       1.1    cherry }
    299       1.1    cherry 
    300       1.1    cherry 
    301       1.1    cherry static int
    302       1.4  jakllsch virtio_pci_detach(device_t self, int flags)
    303       1.1    cherry {
    304       1.4  jakllsch 	struct virtio_pci_softc * const psc = device_private(self);
    305       1.4  jakllsch 	struct virtio_softc * const sc = &psc->sc_sc;
    306       1.1    cherry 	int r;
    307       1.1    cherry 
    308       1.1    cherry 	if (sc->sc_child != NULL) {
    309       1.1    cherry 		r = config_detach(sc->sc_child, flags);
    310       1.1    cherry 		if (r)
    311       1.1    cherry 			return r;
    312       1.1    cherry 	}
    313       1.1    cherry 
    314       1.1    cherry 	/* Check that child detached properly */
    315       1.1    cherry 	KASSERT(sc->sc_child == NULL);
    316       1.1    cherry 	KASSERT(sc->sc_vqs == NULL);
    317       1.4  jakllsch 	KASSERT(psc->sc_ihs_num == 0);
    318       1.1    cherry 
    319       1.4  jakllsch 	if (psc->sc_iosize)
    320  1.14.2.1   thorpej 		bus_space_unmap(psc->sc_iot, psc->sc_ioh,
    321  1.14.2.1   thorpej 			psc->sc_mapped_iosize);
    322       1.4  jakllsch 	psc->sc_iosize = 0;
    323       1.1    cherry 
    324       1.1    cherry 	return 0;
    325       1.1    cherry }
    326       1.4  jakllsch 
    327  1.14.2.1   thorpej 
    328  1.14.2.1   thorpej static int
    329  1.14.2.1   thorpej virtio_pci_attach_09(device_t self, void *aux)
    330  1.14.2.1   thorpej 	//struct virtio_pci_softc *psc, struct pci_attach_args *pa)
    331       1.4  jakllsch {
    332  1.14.2.1   thorpej 	struct virtio_pci_softc * const psc = device_private(self);
    333  1.14.2.1   thorpej 	struct pci_attach_args *pa = (struct pci_attach_args *)aux;
    334  1.14.2.1   thorpej 	struct virtio_softc * const sc = &psc->sc_sc;
    335  1.14.2.1   thorpej //	pci_chipset_tag_t pc = pa->pa_pc;
    336  1.14.2.1   thorpej //	pcitag_t tag = pa->pa_tag;
    337  1.14.2.1   thorpej 
    338  1.14.2.1   thorpej 	/* complete IO region */
    339  1.14.2.1   thorpej 	if (pci_mapreg_map(pa, PCI_MAPREG_START, PCI_MAPREG_TYPE_IO, 0,
    340  1.14.2.1   thorpej 			   &psc->sc_iot, &psc->sc_ioh, NULL, &psc->sc_iosize)) {
    341  1.14.2.1   thorpej 		aprint_error_dev(self, "can't map i/o space\n");
    342  1.14.2.1   thorpej 		return EIO;
    343  1.14.2.1   thorpej 	}
    344  1.14.2.1   thorpej 	psc->sc_mapped_iosize = psc->sc_iosize;
    345       1.4  jakllsch 
    346  1.14.2.1   thorpej 	/* queue space */
    347  1.14.2.1   thorpej 	if (bus_space_subregion(psc->sc_iot, psc->sc_ioh,
    348  1.14.2.1   thorpej 			VIRTIO_CONFIG_QUEUE_NOTIFY, 2, &psc->sc_notify_ioh)) {
    349  1.14.2.1   thorpej 		aprint_error_dev(self, "can't map notify i/o space\n");
    350  1.14.2.1   thorpej 		return EIO;
    351  1.14.2.1   thorpej 	}
    352  1.14.2.1   thorpej 	psc->sc_notify_iosize = 2;
    353  1.14.2.1   thorpej 	psc->sc_notify_iot = psc->sc_iot;
    354  1.14.2.1   thorpej 
    355  1.14.2.1   thorpej 	/* ISR space */
    356  1.14.2.1   thorpej 	if (bus_space_subregion(psc->sc_iot, psc->sc_ioh,
    357  1.14.2.1   thorpej 			VIRTIO_CONFIG_ISR_STATUS, 1, &psc->sc_isr_ioh)) {
    358  1.14.2.1   thorpej 		aprint_error_dev(self, "can't map isr i/o space\n");
    359  1.14.2.1   thorpej 		return EIO;
    360  1.14.2.1   thorpej 	}
    361  1.14.2.1   thorpej 	psc->sc_isr_iosize = 1;
    362  1.14.2.1   thorpej 	psc->sc_isr_iot = psc->sc_iot;
    363  1.14.2.1   thorpej 
    364  1.14.2.1   thorpej 	/* set our version 0.9 ops */
    365  1.14.2.1   thorpej 	sc->sc_ops = &virtio_pci_ops_09;
    366  1.14.2.1   thorpej 	sc->sc_bus_endian    = READ_ENDIAN_09;
    367  1.14.2.1   thorpej 	sc->sc_struct_endian = STRUCT_ENDIAN_09;
    368  1.14.2.1   thorpej 	return 0;
    369       1.4  jakllsch }
    370       1.4  jakllsch 
    371  1.14.2.1   thorpej 
    372  1.14.2.1   thorpej static int
    373  1.14.2.1   thorpej virtio_pci_attach_10(device_t self, void *aux)
    374       1.4  jakllsch {
    375  1.14.2.1   thorpej 	struct virtio_pci_softc * const psc = device_private(self);
    376  1.14.2.1   thorpej 	struct pci_attach_args *pa = (struct pci_attach_args *)aux;
    377  1.14.2.1   thorpej 	struct virtio_softc * const sc = &psc->sc_sc;
    378  1.14.2.1   thorpej 	pci_chipset_tag_t pc = pa->pa_pc;
    379  1.14.2.1   thorpej 	pcitag_t tag = pa->pa_tag;
    380  1.14.2.1   thorpej 
    381  1.14.2.1   thorpej 	struct virtio_pci_cap common, isr, device;
    382  1.14.2.1   thorpej 	struct virtio_pci_notify_cap notify;
    383  1.14.2.1   thorpej 	int have_device_cfg = 0;
    384  1.14.2.1   thorpej 	bus_size_t bars[NMAPREG] = { 0 };
    385  1.14.2.1   thorpej 	int bars_idx[NMAPREG] = { 0 };
    386  1.14.2.1   thorpej 	struct virtio_pci_cap *caps[] = { &common, &isr, &device, &notify.cap };
    387  1.14.2.1   thorpej 	int i, j, ret = 0;
    388  1.14.2.1   thorpej 
    389  1.14.2.1   thorpej 	if (virtio_pci_find_cap(psc, VIRTIO_PCI_CAP_COMMON_CFG,
    390  1.14.2.1   thorpej 			&common, sizeof(common)))
    391  1.14.2.1   thorpej 		return ENODEV;
    392  1.14.2.1   thorpej 	if (virtio_pci_find_cap(psc, VIRTIO_PCI_CAP_NOTIFY_CFG,
    393  1.14.2.1   thorpej 			&notify, sizeof(notify)))
    394  1.14.2.1   thorpej 		return ENODEV;
    395  1.14.2.1   thorpej 	if (virtio_pci_find_cap(psc, VIRTIO_PCI_CAP_ISR_CFG,
    396  1.14.2.1   thorpej 			&isr, sizeof(isr)))
    397  1.14.2.1   thorpej 		return ENODEV;
    398  1.14.2.1   thorpej 	if (virtio_pci_find_cap(psc, VIRTIO_PCI_CAP_DEVICE_CFG,
    399  1.14.2.1   thorpej 			&device, sizeof(device)))
    400  1.14.2.1   thorpej 		memset(&device, 0, sizeof(device));
    401  1.14.2.1   thorpej 	else
    402  1.14.2.1   thorpej 		have_device_cfg = 1;
    403  1.14.2.1   thorpej 
    404  1.14.2.1   thorpej 	/* Figure out which bars we need to map */
    405  1.14.2.1   thorpej 	for (i = 0; i < __arraycount(caps); i++) {
    406  1.14.2.1   thorpej 		int bar = caps[i]->bar;
    407  1.14.2.1   thorpej 		bus_size_t len = caps[i]->offset + caps[i]->length;
    408  1.14.2.1   thorpej 		if (caps[i]->length == 0)
    409  1.14.2.1   thorpej 			continue;
    410  1.14.2.1   thorpej 		if (bars[bar] < len)
    411  1.14.2.1   thorpej 			bars[bar] = len;
    412  1.14.2.1   thorpej 	}
    413  1.14.2.1   thorpej 
    414  1.14.2.1   thorpej 	for (i = j = 0; i < __arraycount(bars); i++) {
    415  1.14.2.1   thorpej 		int reg;
    416  1.14.2.1   thorpej 		pcireg_t type;
    417  1.14.2.1   thorpej 		if (bars[i] == 0)
    418  1.14.2.1   thorpej 			continue;
    419  1.14.2.1   thorpej 		reg = PCI_MAPREG_START + i * 4;
    420  1.14.2.1   thorpej 		type = pci_mapreg_type(pc, tag, reg);
    421  1.14.2.1   thorpej 		if (pci_mapreg_map(pa, reg, type, 0,
    422  1.14.2.1   thorpej 				&psc->sc_bars_iot[j], &psc->sc_bars_ioh[j],
    423  1.14.2.1   thorpej 				NULL, &psc->sc_bars_iosize[j])) {
    424  1.14.2.1   thorpej 			aprint_error_dev(self, "can't map bar %u \n", i);
    425  1.14.2.1   thorpej 			ret = EIO;
    426  1.14.2.1   thorpej 			goto err;
    427  1.14.2.1   thorpej 		}
    428  1.14.2.1   thorpej 		aprint_debug_dev(self,
    429  1.14.2.1   thorpej 		    "bar[%d]: iot %p, size 0x%" PRIxBUSSIZE "\n",
    430  1.14.2.1   thorpej 		    j, psc->sc_bars_iot[j], psc->sc_bars_iosize[j]);
    431  1.14.2.1   thorpej 		bars_idx[i] = j;
    432  1.14.2.1   thorpej 		j++;
    433  1.14.2.1   thorpej 	}
    434  1.14.2.1   thorpej 
    435  1.14.2.1   thorpej 	i = bars_idx[notify.cap.bar];
    436  1.14.2.1   thorpej 	if (bus_space_subregion(psc->sc_bars_iot[i], psc->sc_bars_ioh[i],
    437  1.14.2.1   thorpej 			notify.cap.offset, notify.cap.length,
    438  1.14.2.1   thorpej 			&psc->sc_notify_ioh)) {
    439  1.14.2.1   thorpej 		aprint_error_dev(self, "can't map notify i/o space\n");
    440  1.14.2.1   thorpej 		ret = EIO;
    441  1.14.2.1   thorpej 		goto err;
    442  1.14.2.1   thorpej 	}
    443  1.14.2.1   thorpej 	psc->sc_notify_iosize = notify.cap.length;
    444  1.14.2.1   thorpej 	psc->sc_notify_iot = psc->sc_bars_iot[i];
    445  1.14.2.1   thorpej 	psc->sc_notify_off_multiplier = le32toh(notify.notify_off_multiplier);
    446  1.14.2.1   thorpej 
    447  1.14.2.1   thorpej 	if (have_device_cfg) {
    448  1.14.2.1   thorpej 		i = bars_idx[device.bar];
    449  1.14.2.1   thorpej 		if (bus_space_subregion(psc->sc_bars_iot[i], psc->sc_bars_ioh[i],
    450  1.14.2.1   thorpej 				device.offset, device.length,
    451  1.14.2.1   thorpej 				&sc->sc_devcfg_ioh)) {
    452  1.14.2.1   thorpej 			aprint_error_dev(self, "can't map devcfg i/o space\n");
    453  1.14.2.1   thorpej 			ret = EIO;
    454  1.14.2.1   thorpej 			goto err;
    455  1.14.2.1   thorpej 		}
    456  1.14.2.1   thorpej 		aprint_debug_dev(self,
    457  1.14.2.1   thorpej 			"device.offset = 0x%x, device.length = 0x%x\n",
    458  1.14.2.1   thorpej 			device.offset, device.length);
    459  1.14.2.1   thorpej 		sc->sc_devcfg_iosize = device.length;
    460  1.14.2.1   thorpej 		sc->sc_devcfg_iot = psc->sc_bars_iot[i];
    461  1.14.2.1   thorpej 	}
    462  1.14.2.1   thorpej 
    463  1.14.2.1   thorpej 	i = bars_idx[isr.bar];
    464  1.14.2.1   thorpej 	if (bus_space_subregion(psc->sc_bars_iot[i], psc->sc_bars_ioh[i],
    465  1.14.2.1   thorpej 			isr.offset, isr.length, &psc->sc_isr_ioh)) {
    466  1.14.2.1   thorpej 		aprint_error_dev(self, "can't map isr i/o space\n");
    467  1.14.2.1   thorpej 		ret = EIO;
    468  1.14.2.1   thorpej 		goto err;
    469  1.14.2.1   thorpej 	}
    470  1.14.2.1   thorpej 	psc->sc_isr_iosize = isr.length;
    471  1.14.2.1   thorpej 	psc->sc_isr_iot = psc->sc_bars_iot[i];
    472  1.14.2.1   thorpej 
    473  1.14.2.1   thorpej 	i = bars_idx[common.bar];
    474  1.14.2.1   thorpej 	if (bus_space_subregion(psc->sc_bars_iot[i], psc->sc_bars_ioh[i],
    475  1.14.2.1   thorpej 			common.offset, common.length, &psc->sc_ioh)) {
    476  1.14.2.1   thorpej 		aprint_error_dev(self, "can't map common i/o space\n");
    477  1.14.2.1   thorpej 		ret = EIO;
    478  1.14.2.1   thorpej 		goto err;
    479  1.14.2.1   thorpej 	}
    480  1.14.2.1   thorpej 	psc->sc_iosize = common.length;
    481  1.14.2.1   thorpej 	psc->sc_iot = psc->sc_bars_iot[i];
    482  1.14.2.1   thorpej 	psc->sc_mapped_iosize = psc->sc_bars_iosize[i];
    483  1.14.2.1   thorpej 
    484  1.14.2.1   thorpej 	psc->sc_sc.sc_version_1 = 1;
    485  1.14.2.1   thorpej 
    486  1.14.2.1   thorpej 	/* set our version 1.0 ops */
    487  1.14.2.1   thorpej 	sc->sc_ops = &virtio_pci_ops_10;
    488  1.14.2.1   thorpej 	sc->sc_bus_endian    = READ_ENDIAN_10;
    489  1.14.2.1   thorpej 	sc->sc_struct_endian = STRUCT_ENDIAN_10;
    490  1.14.2.1   thorpej 	return 0;
    491  1.14.2.1   thorpej 
    492  1.14.2.1   thorpej err:
    493  1.14.2.1   thorpej 	/* undo our pci_mapreg_map()s */
    494  1.14.2.1   thorpej 	for (i = 0; i < __arraycount(bars); i++) {
    495  1.14.2.1   thorpej 		if (psc->sc_bars_iosize[i] == 0)
    496  1.14.2.1   thorpej 			continue;
    497  1.14.2.1   thorpej 		bus_space_unmap(psc->sc_bars_iot[i], psc->sc_bars_ioh[i],
    498  1.14.2.1   thorpej 				psc->sc_bars_iosize[i]);
    499  1.14.2.1   thorpej 	}
    500  1.14.2.1   thorpej 	return ret;
    501       1.4  jakllsch }
    502       1.4  jakllsch 
    503  1.14.2.1   thorpej /* v1.0 attach helper */
    504  1.14.2.1   thorpej static int
    505  1.14.2.1   thorpej virtio_pci_find_cap(struct virtio_pci_softc *psc, int cfg_type, void *buf, int buflen)
    506       1.4  jakllsch {
    507  1.14.2.1   thorpej 	device_t self = psc->sc_sc.sc_dev;
    508  1.14.2.1   thorpej 	pci_chipset_tag_t pc = psc->sc_pa.pa_pc;
    509  1.14.2.1   thorpej 	pcitag_t tag = psc->sc_pa.pa_tag;
    510  1.14.2.1   thorpej 	unsigned int offset, i, len;
    511  1.14.2.1   thorpej 	union {
    512  1.14.2.1   thorpej 		pcireg_t reg[8];
    513  1.14.2.1   thorpej 		struct virtio_pci_cap vcap;
    514  1.14.2.1   thorpej 	} *v = buf;
    515  1.14.2.1   thorpej 
    516  1.14.2.1   thorpej 	if (buflen < sizeof(struct virtio_pci_cap))
    517  1.14.2.1   thorpej 		return ERANGE;
    518  1.14.2.1   thorpej 
    519  1.14.2.1   thorpej 	if (!pci_get_capability(pc, tag, PCI_CAP_VENDSPEC, &offset, &v->reg[0]))
    520  1.14.2.1   thorpej 		return ENOENT;
    521  1.14.2.1   thorpej 
    522  1.14.2.1   thorpej 	do {
    523  1.14.2.1   thorpej 		for (i = 0; i < 4; i++)
    524  1.14.2.1   thorpej 			v->reg[i] =
    525  1.14.2.1   thorpej 				le32toh(pci_conf_read(pc, tag, offset + i * 4));
    526  1.14.2.1   thorpej 		if (v->vcap.cfg_type == cfg_type)
    527  1.14.2.1   thorpej 			break;
    528  1.14.2.1   thorpej 		offset = v->vcap.cap_next;
    529  1.14.2.1   thorpej 	} while (offset != 0);
    530  1.14.2.1   thorpej 
    531  1.14.2.1   thorpej 	if (offset == 0)
    532  1.14.2.1   thorpej 		return ENOENT;
    533  1.14.2.1   thorpej 
    534  1.14.2.1   thorpej 	if (v->vcap.cap_len > sizeof(struct virtio_pci_cap)) {
    535  1.14.2.1   thorpej 		len = roundup(v->vcap.cap_len, sizeof(pcireg_t));
    536  1.14.2.1   thorpej 		if (len > buflen) {
    537  1.14.2.1   thorpej 			aprint_error_dev(self, "%s cap too large\n", __func__);
    538  1.14.2.1   thorpej 			return ERANGE;
    539  1.14.2.1   thorpej 		}
    540  1.14.2.1   thorpej 		for (i = 4; i < len / sizeof(pcireg_t);  i++)
    541  1.14.2.1   thorpej 			v->reg[i] =
    542  1.14.2.1   thorpej 				le32toh(pci_conf_read(pc, tag, offset + i * 4));
    543  1.14.2.1   thorpej 	}
    544  1.14.2.1   thorpej 
    545  1.14.2.1   thorpej 	/* endian fixup */
    546  1.14.2.1   thorpej 	v->vcap.offset = le32toh(v->vcap.offset);
    547  1.14.2.1   thorpej 	v->vcap.length = le32toh(v->vcap.length);
    548  1.14.2.1   thorpej 	return 0;
    549       1.4  jakllsch }
    550       1.4  jakllsch 
    551  1.14.2.1   thorpej 
    552  1.14.2.1   thorpej /* -------------------------------------
    553  1.14.2.1   thorpej  * Version 0.9 support
    554  1.14.2.1   thorpej  * -------------------------------------*/
    555  1.14.2.1   thorpej 
    556  1.14.2.1   thorpej static void
    557  1.14.2.1   thorpej virtio_pci_kick_09(struct virtio_softc *sc, uint16_t idx)
    558       1.4  jakllsch {
    559       1.4  jakllsch 	struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc;
    560  1.14.2.1   thorpej 
    561  1.14.2.1   thorpej 	bus_space_write_2(psc->sc_notify_iot, psc->sc_notify_ioh, 0, idx);
    562       1.4  jakllsch }
    563       1.4  jakllsch 
    564  1.14.2.1   thorpej /* only applicable for v 0.9 but also called for 1.0 */
    565  1.14.2.1   thorpej static int
    566  1.14.2.1   thorpej virtio_pci_adjust_config_region(struct virtio_pci_softc *psc)
    567       1.4  jakllsch {
    568  1.14.2.1   thorpej 	struct virtio_softc * const sc = (struct virtio_softc *) psc;
    569  1.14.2.1   thorpej 	device_t self = psc->sc_sc.sc_dev;
    570  1.14.2.1   thorpej 
    571  1.14.2.1   thorpej 	if (psc->sc_sc.sc_version_1)
    572  1.14.2.1   thorpej 		return 0;
    573       1.4  jakllsch 
    574  1.14.2.1   thorpej 	sc->sc_devcfg_iosize = psc->sc_iosize - psc->sc_devcfg_offset;
    575  1.14.2.1   thorpej 	sc->sc_devcfg_iot = psc->sc_iot;
    576  1.14.2.1   thorpej 	if (bus_space_subregion(psc->sc_iot, psc->sc_ioh,
    577  1.14.2.1   thorpej 			psc->sc_devcfg_offset, sc->sc_devcfg_iosize,
    578  1.14.2.1   thorpej 			&sc->sc_devcfg_ioh)) {
    579  1.14.2.1   thorpej 		aprint_error_dev(self, "can't map config i/o space\n");
    580  1.14.2.1   thorpej 		return EIO;
    581  1.14.2.1   thorpej 	}
    582       1.4  jakllsch 
    583  1.14.2.1   thorpej 	return 0;
    584  1.14.2.1   thorpej }
    585  1.14.2.1   thorpej 
    586  1.14.2.1   thorpej static uint16_t
    587  1.14.2.1   thorpej virtio_pci_read_queue_size_09(struct virtio_softc *sc, uint16_t idx)
    588  1.14.2.1   thorpej {
    589  1.14.2.1   thorpej 	struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc;
    590  1.14.2.1   thorpej 
    591  1.14.2.1   thorpej 	bus_space_write_2(psc->sc_iot, psc->sc_ioh,
    592  1.14.2.1   thorpej 	    VIRTIO_CONFIG_QUEUE_SELECT, idx);
    593  1.14.2.1   thorpej 	return bus_space_read_2(psc->sc_iot, psc->sc_ioh,
    594  1.14.2.1   thorpej 	    VIRTIO_CONFIG_QUEUE_SIZE);
    595       1.4  jakllsch }
    596       1.4  jakllsch 
    597       1.4  jakllsch static void
    598  1.14.2.1   thorpej virtio_pci_setup_queue_09(struct virtio_softc *sc, uint16_t idx, uint64_t addr)
    599       1.4  jakllsch {
    600       1.4  jakllsch 	struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc;
    601       1.4  jakllsch 
    602  1.14.2.1   thorpej 	bus_space_write_2(psc->sc_iot, psc->sc_ioh,
    603  1.14.2.1   thorpej 	    VIRTIO_CONFIG_QUEUE_SELECT, idx);
    604  1.14.2.1   thorpej 	bus_space_write_4(psc->sc_iot, psc->sc_ioh,
    605  1.14.2.1   thorpej 	    VIRTIO_CONFIG_QUEUE_ADDRESS, addr / VIRTIO_PAGE_SIZE);
    606  1.14.2.1   thorpej 
    607  1.14.2.1   thorpej 	if (psc->sc_ihs_num > 1) {
    608  1.14.2.1   thorpej 		int vec = VIRTIO_MSIX_QUEUE_VECTOR_INDEX;
    609  1.14.2.1   thorpej 		if (sc->sc_child_mq)
    610  1.14.2.1   thorpej 			vec += idx;
    611  1.14.2.1   thorpej 		bus_space_write_2(psc->sc_iot, psc->sc_ioh,
    612  1.14.2.1   thorpej 		    VIRTIO_CONFIG_MSI_QUEUE_VECTOR, vec);
    613  1.14.2.1   thorpej 	}
    614       1.4  jakllsch }
    615       1.4  jakllsch 
    616       1.4  jakllsch static void
    617  1.14.2.1   thorpej virtio_pci_set_status_09(struct virtio_softc *sc, int status)
    618       1.4  jakllsch {
    619       1.4  jakllsch 	struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc;
    620  1.14.2.1   thorpej 	int old = 0;
    621       1.4  jakllsch 
    622  1.14.2.1   thorpej 	if (status != 0) {
    623  1.14.2.1   thorpej 	    old = bus_space_read_1(psc->sc_iot, psc->sc_ioh,
    624  1.14.2.1   thorpej 		VIRTIO_CONFIG_DEVICE_STATUS);
    625  1.14.2.1   thorpej 	}
    626  1.14.2.1   thorpej 	bus_space_write_1(psc->sc_iot, psc->sc_ioh,
    627  1.14.2.1   thorpej 	    VIRTIO_CONFIG_DEVICE_STATUS, status|old);
    628       1.4  jakllsch }
    629       1.4  jakllsch 
    630       1.4  jakllsch static void
    631  1.14.2.1   thorpej virtio_pci_negotiate_features_09(struct virtio_softc *sc, uint64_t guest_features)
    632       1.4  jakllsch {
    633       1.4  jakllsch 	struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc;
    634  1.14.2.1   thorpej 	uint32_t r;
    635  1.14.2.1   thorpej 
    636  1.14.2.1   thorpej 	r = bus_space_read_4(psc->sc_iot, psc->sc_ioh,
    637  1.14.2.1   thorpej 	    VIRTIO_CONFIG_DEVICE_FEATURES);
    638  1.14.2.1   thorpej 
    639  1.14.2.1   thorpej 	r &= guest_features;
    640  1.14.2.1   thorpej 
    641  1.14.2.1   thorpej 	bus_space_write_4(psc->sc_iot, psc->sc_ioh,
    642  1.14.2.1   thorpej 	    VIRTIO_CONFIG_GUEST_FEATURES, r);
    643       1.4  jakllsch 
    644  1.14.2.1   thorpej 	sc->sc_active_features = r;
    645       1.4  jakllsch }
    646       1.4  jakllsch 
    647  1.14.2.1   thorpej /* -------------------------------------
    648  1.14.2.1   thorpej  * Version 1.0 support
    649  1.14.2.1   thorpej  * -------------------------------------*/
    650  1.14.2.1   thorpej 
    651       1.4  jakllsch static void
    652  1.14.2.1   thorpej virtio_pci_kick_10(struct virtio_softc *sc, uint16_t idx)
    653       1.4  jakllsch {
    654       1.4  jakllsch 	struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc;
    655  1.14.2.1   thorpej 	unsigned offset = sc->sc_vqs[idx].vq_notify_off *
    656  1.14.2.1   thorpej 		psc->sc_notify_off_multiplier;
    657       1.4  jakllsch 
    658  1.14.2.1   thorpej 	bus_space_write_2(psc->sc_notify_iot, psc->sc_notify_ioh, offset, idx);
    659       1.4  jakllsch }
    660       1.4  jakllsch 
    661  1.14.2.1   thorpej 
    662       1.4  jakllsch static uint16_t
    663  1.14.2.1   thorpej virtio_pci_read_queue_size_10(struct virtio_softc *sc, uint16_t idx)
    664       1.4  jakllsch {
    665       1.4  jakllsch 	struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc;
    666  1.14.2.1   thorpej 	bus_space_tag_t	   iot = psc->sc_iot;
    667  1.14.2.1   thorpej 	bus_space_handle_t ioh = psc->sc_ioh;
    668       1.4  jakllsch 
    669  1.14.2.1   thorpej 	bus_space_write_2(iot, ioh, VIRTIO_CONFIG1_QUEUE_SELECT, idx);
    670  1.14.2.1   thorpej 	return bus_space_read_2(iot, ioh, VIRTIO_CONFIG1_QUEUE_SIZE);
    671  1.14.2.1   thorpej }
    672  1.14.2.1   thorpej 
    673  1.14.2.1   thorpej /*
    674  1.14.2.1   thorpej  * By definition little endian only in v1.0 and 8 byters are allowed to be
    675  1.14.2.1   thorpej  * written as two 4 byters
    676  1.14.2.1   thorpej  *
    677  1.14.2.1   thorpej  * This is not a general purpose function that can be used in any
    678  1.14.2.1   thorpej  * driver. Virtio specifically allows the 8 byte bus transaction
    679  1.14.2.1   thorpej  * to be split into two 4 byte transactions. Do not copy/use it
    680  1.14.2.1   thorpej  * in other device drivers unless you know that the device accepts it.
    681  1.14.2.1   thorpej  */
    682  1.14.2.1   thorpej static __inline void
    683  1.14.2.1   thorpej virtio_pci_bus_space_write_8(bus_space_tag_t iot, bus_space_handle_t ioh,
    684  1.14.2.1   thorpej      bus_size_t offset, uint64_t value)
    685  1.14.2.1   thorpej {
    686  1.14.2.1   thorpej #if defined(__HAVE_BUS_SPACE_8)
    687  1.14.2.1   thorpej 	bus_space_write_8(iot, ioh, offset, value);
    688  1.14.2.1   thorpej #elif _QUAD_HIGHWORD
    689  1.14.2.1   thorpej 	bus_space_write_4(iot, ioh, offset, BUS_ADDR_LO32(value));
    690  1.14.2.1   thorpej 	bus_space_write_4(iot, ioh, offset + 4, BUS_ADDR_HI32(value));
    691  1.14.2.1   thorpej #else
    692  1.14.2.1   thorpej 	bus_space_write_4(iot, ioh, offset, BUS_ADDR_HI32(value));
    693  1.14.2.1   thorpej 	bus_space_write_4(iot, ioh, offset + 4, BUS_ADDR_LO32(value));
    694  1.14.2.1   thorpej #endif
    695       1.4  jakllsch }
    696       1.4  jakllsch 
    697       1.4  jakllsch static void
    698  1.14.2.1   thorpej virtio_pci_setup_queue_10(struct virtio_softc *sc, uint16_t idx, uint64_t addr)
    699       1.4  jakllsch {
    700       1.4  jakllsch 	struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc;
    701  1.14.2.1   thorpej 	struct virtqueue *vq = &sc->sc_vqs[idx];
    702  1.14.2.1   thorpej 	bus_space_tag_t	   iot = psc->sc_iot;
    703  1.14.2.1   thorpej 	bus_space_handle_t ioh = psc->sc_ioh;
    704  1.14.2.1   thorpej 	KASSERT(vq->vq_index == idx);
    705  1.14.2.1   thorpej 
    706  1.14.2.1   thorpej 	bus_space_write_2(iot, ioh, VIRTIO_CONFIG1_QUEUE_SELECT, vq->vq_index);
    707  1.14.2.1   thorpej 	if (addr == 0) {
    708  1.14.2.1   thorpej 		bus_space_write_2(iot, ioh, VIRTIO_CONFIG1_QUEUE_ENABLE, 0);
    709  1.14.2.1   thorpej 		virtio_pci_bus_space_write_8(iot, ioh,
    710  1.14.2.1   thorpej 		    VIRTIO_CONFIG1_QUEUE_DESC,   0);
    711  1.14.2.1   thorpej 		virtio_pci_bus_space_write_8(iot, ioh,
    712  1.14.2.1   thorpej 		    VIRTIO_CONFIG1_QUEUE_AVAIL,  0);
    713  1.14.2.1   thorpej 		virtio_pci_bus_space_write_8(iot, ioh,
    714  1.14.2.1   thorpej 		    VIRTIO_CONFIG1_QUEUE_USED,   0);
    715  1.14.2.1   thorpej 	} else {
    716  1.14.2.1   thorpej 		virtio_pci_bus_space_write_8(iot, ioh,
    717  1.14.2.1   thorpej 			VIRTIO_CONFIG1_QUEUE_DESC, addr);
    718  1.14.2.1   thorpej 		virtio_pci_bus_space_write_8(iot, ioh,
    719  1.14.2.1   thorpej 			VIRTIO_CONFIG1_QUEUE_AVAIL, addr + vq->vq_availoffset);
    720  1.14.2.1   thorpej 		virtio_pci_bus_space_write_8(iot, ioh,
    721  1.14.2.1   thorpej 			VIRTIO_CONFIG1_QUEUE_USED, addr + vq->vq_usedoffset);
    722  1.14.2.1   thorpej 		bus_space_write_2(iot, ioh,
    723  1.14.2.1   thorpej 			VIRTIO_CONFIG1_QUEUE_ENABLE, 1);
    724  1.14.2.1   thorpej 		vq->vq_notify_off = bus_space_read_2(iot, ioh,
    725  1.14.2.1   thorpej 			VIRTIO_CONFIG1_QUEUE_NOTIFY_OFF);
    726  1.14.2.1   thorpej 	}
    727       1.4  jakllsch 
    728       1.4  jakllsch 	if (psc->sc_ihs_num > 1) {
    729       1.4  jakllsch 		int vec = VIRTIO_MSIX_QUEUE_VECTOR_INDEX;
    730       1.6  yamaguch 		if (sc->sc_child_mq)
    731       1.4  jakllsch 			vec += idx;
    732  1.14.2.1   thorpej 		bus_space_write_2(iot, ioh,
    733  1.14.2.1   thorpej 			VIRTIO_CONFIG1_QUEUE_MSIX_VECTOR, vec);
    734       1.4  jakllsch 	}
    735       1.4  jakllsch }
    736       1.4  jakllsch 
    737       1.4  jakllsch static void
    738  1.14.2.1   thorpej virtio_pci_set_status_10(struct virtio_softc *sc, int status)
    739       1.4  jakllsch {
    740       1.4  jakllsch 	struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc;
    741  1.14.2.1   thorpej 	bus_space_tag_t	   iot = psc->sc_iot;
    742  1.14.2.1   thorpej 	bus_space_handle_t ioh = psc->sc_ioh;
    743       1.4  jakllsch 	int old = 0;
    744       1.4  jakllsch 
    745  1.14.2.1   thorpej 	if (status)
    746  1.14.2.1   thorpej 		old = bus_space_read_1(iot, ioh, VIRTIO_CONFIG1_DEVICE_STATUS);
    747  1.14.2.1   thorpej 	bus_space_write_1(iot, ioh, VIRTIO_CONFIG1_DEVICE_STATUS, status | old);
    748       1.4  jakllsch }
    749       1.4  jakllsch 
    750  1.14.2.1   thorpej void
    751  1.14.2.1   thorpej virtio_pci_negotiate_features_10(struct virtio_softc *sc, uint64_t guest_features)
    752       1.4  jakllsch {
    753       1.4  jakllsch 	struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc;
    754  1.14.2.1   thorpej 	device_t self          =  sc->sc_dev;
    755  1.14.2.1   thorpej 	bus_space_tag_t	   iot = psc->sc_iot;
    756  1.14.2.1   thorpej 	bus_space_handle_t ioh = psc->sc_ioh;
    757  1.14.2.1   thorpej 	uint64_t host, negotiated, device_status;
    758  1.14.2.1   thorpej 
    759  1.14.2.1   thorpej 	guest_features |= VIRTIO_F_VERSION_1;
    760  1.14.2.1   thorpej 	/* notify on empty is 0.9 only */
    761  1.14.2.1   thorpej 	guest_features &= ~VIRTIO_F_NOTIFY_ON_EMPTY;
    762  1.14.2.1   thorpej 	sc->sc_active_features = 0;
    763  1.14.2.1   thorpej 
    764  1.14.2.1   thorpej 	bus_space_write_4(iot, ioh, VIRTIO_CONFIG1_DEVICE_FEATURE_SELECT, 0);
    765  1.14.2.1   thorpej 	host = bus_space_read_4(iot, ioh, VIRTIO_CONFIG1_DEVICE_FEATURE);
    766  1.14.2.1   thorpej 	bus_space_write_4(iot, ioh, VIRTIO_CONFIG1_DEVICE_FEATURE_SELECT, 1);
    767  1.14.2.1   thorpej 	host |= (uint64_t)
    768  1.14.2.1   thorpej 		bus_space_read_4(iot, ioh, VIRTIO_CONFIG1_DEVICE_FEATURE) << 32;
    769  1.14.2.1   thorpej 
    770  1.14.2.1   thorpej 	negotiated = host & guest_features;
    771  1.14.2.1   thorpej 
    772  1.14.2.1   thorpej 	bus_space_write_4(iot, ioh, VIRTIO_CONFIG1_DRIVER_FEATURE_SELECT, 0);
    773  1.14.2.1   thorpej 	bus_space_write_4(iot, ioh, VIRTIO_CONFIG1_DRIVER_FEATURE,
    774  1.14.2.1   thorpej 			negotiated & 0xffffffff);
    775  1.14.2.1   thorpej 	bus_space_write_4(iot, ioh, VIRTIO_CONFIG1_DRIVER_FEATURE_SELECT, 1);
    776  1.14.2.1   thorpej 	bus_space_write_4(iot, ioh, VIRTIO_CONFIG1_DRIVER_FEATURE,
    777  1.14.2.1   thorpej 			negotiated >> 32);
    778  1.14.2.1   thorpej 	virtio_pci_set_status_10(sc, VIRTIO_CONFIG_DEVICE_STATUS_FEATURES_OK);
    779  1.14.2.1   thorpej 
    780  1.14.2.1   thorpej 	device_status = bus_space_read_1(iot, ioh, VIRTIO_CONFIG1_DEVICE_STATUS);
    781  1.14.2.1   thorpej 	if ((device_status & VIRTIO_CONFIG_DEVICE_STATUS_FEATURES_OK) == 0) {
    782  1.14.2.1   thorpej 		aprint_error_dev(self, "feature negotiation failed\n");
    783  1.14.2.1   thorpej 		bus_space_write_1(iot, ioh, VIRTIO_CONFIG1_DEVICE_STATUS,
    784  1.14.2.1   thorpej 				VIRTIO_CONFIG_DEVICE_STATUS_FAILED);
    785  1.14.2.1   thorpej 		return;
    786  1.14.2.1   thorpej 	}
    787       1.4  jakllsch 
    788  1.14.2.1   thorpej 	if ((negotiated & VIRTIO_F_VERSION_1) == 0) {
    789  1.14.2.1   thorpej 		aprint_error_dev(self, "host rejected version 1\n");
    790  1.14.2.1   thorpej 		bus_space_write_1(iot, ioh, VIRTIO_CONFIG1_DEVICE_STATUS,
    791  1.14.2.1   thorpej 				VIRTIO_CONFIG_DEVICE_STATUS_FAILED);
    792  1.14.2.1   thorpej 		return;
    793  1.14.2.1   thorpej 	}
    794       1.4  jakllsch 
    795  1.14.2.1   thorpej 	sc->sc_active_features = negotiated;
    796  1.14.2.1   thorpej 	return;
    797       1.4  jakllsch }
    798       1.4  jakllsch 
    799       1.4  jakllsch 
    800  1.14.2.1   thorpej /* -------------------------------------
    801  1.14.2.1   thorpej  * Generic PCI interrupt code
    802  1.14.2.1   thorpej  * -------------------------------------*/
    803  1.14.2.1   thorpej 
    804       1.4  jakllsch static int
    805  1.14.2.1   thorpej virtio_pci_setup_msix_vectors_10(struct virtio_softc *sc)
    806       1.4  jakllsch {
    807       1.4  jakllsch 	struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc;
    808  1.14.2.1   thorpej 	device_t self          =  sc->sc_dev;
    809  1.14.2.1   thorpej 	bus_space_tag_t	   iot = psc->sc_iot;
    810  1.14.2.1   thorpej 	bus_space_handle_t ioh = psc->sc_ioh;
    811  1.14.2.1   thorpej 	int vector, ret, qid;
    812  1.14.2.1   thorpej 
    813  1.14.2.1   thorpej 	vector = VIRTIO_MSIX_CONFIG_VECTOR_INDEX;
    814  1.14.2.1   thorpej 	bus_space_write_2(iot, ioh,
    815  1.14.2.1   thorpej 		VIRTIO_CONFIG1_CONFIG_MSIX_VECTOR, vector);
    816  1.14.2.1   thorpej 	ret = bus_space_read_2(iot, ioh, VIRTIO_CONFIG1_CONFIG_MSIX_VECTOR);
    817  1.14.2.1   thorpej 	if (ret != vector) {
    818  1.14.2.1   thorpej 		aprint_error_dev(self, "can't set config msix vector\n");
    819  1.14.2.1   thorpej 		return -1;
    820  1.14.2.1   thorpej 	}
    821  1.14.2.1   thorpej 
    822  1.14.2.1   thorpej 	for (qid = 0; qid < sc->sc_nvqs; qid++) {
    823  1.14.2.1   thorpej 		vector = VIRTIO_MSIX_QUEUE_VECTOR_INDEX;
    824  1.14.2.1   thorpej 
    825  1.14.2.1   thorpej 		if (sc->sc_child_mq)
    826  1.14.2.1   thorpej 			vector += qid;
    827  1.14.2.1   thorpej 		bus_space_write_2(iot, ioh, VIRTIO_CONFIG1_QUEUE_SELECT, qid);
    828  1.14.2.1   thorpej 		bus_space_write_2(iot, ioh, VIRTIO_CONFIG1_QUEUE_MSIX_VECTOR,
    829  1.14.2.1   thorpej 			vector);
    830  1.14.2.1   thorpej 		ret = bus_space_read_2(iot, ioh,
    831  1.14.2.1   thorpej 			VIRTIO_CONFIG1_QUEUE_MSIX_VECTOR);
    832  1.14.2.1   thorpej 		if (ret != vector) {
    833  1.14.2.1   thorpej 			aprint_error_dev(self, "can't set queue %d "
    834  1.14.2.1   thorpej 				"msix vector\n", qid);
    835  1.14.2.1   thorpej 			return -1;
    836  1.14.2.1   thorpej 		}
    837  1.14.2.1   thorpej 	}
    838  1.14.2.1   thorpej 
    839  1.14.2.1   thorpej 	return 0;
    840  1.14.2.1   thorpej }
    841  1.14.2.1   thorpej 
    842  1.14.2.1   thorpej static int
    843  1.14.2.1   thorpej virtio_pci_setup_msix_vectors_09(struct virtio_softc *sc)
    844  1.14.2.1   thorpej {
    845  1.14.2.1   thorpej 	struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc;
    846  1.14.2.1   thorpej 	device_t self = sc->sc_dev;
    847       1.4  jakllsch 	int offset, vector, ret, qid;
    848       1.4  jakllsch 
    849       1.4  jakllsch 	offset = VIRTIO_CONFIG_MSI_CONFIG_VECTOR;
    850       1.4  jakllsch 	vector = VIRTIO_MSIX_CONFIG_VECTOR_INDEX;
    851       1.4  jakllsch 
    852  1.14.2.1   thorpej 	bus_space_write_2(psc->sc_iot, psc->sc_ioh, offset, vector);
    853  1.14.2.1   thorpej 	ret = bus_space_read_2(psc->sc_iot, psc->sc_ioh, offset);
    854       1.4  jakllsch 	aprint_debug_dev(sc->sc_dev, "expected=%d, actual=%d\n",
    855       1.4  jakllsch 	    vector, ret);
    856  1.14.2.1   thorpej 	if (ret != vector) {
    857  1.14.2.1   thorpej 		aprint_error_dev(self, "can't set config msix vector\n");
    858       1.4  jakllsch 		return -1;
    859  1.14.2.1   thorpej 	}
    860       1.4  jakllsch 
    861       1.4  jakllsch 	for (qid = 0; qid < sc->sc_nvqs; qid++) {
    862       1.4  jakllsch 		offset = VIRTIO_CONFIG_QUEUE_SELECT;
    863  1.14.2.1   thorpej 		bus_space_write_2(psc->sc_iot, psc->sc_ioh, offset, qid);
    864       1.4  jakllsch 
    865       1.4  jakllsch 		offset = VIRTIO_CONFIG_MSI_QUEUE_VECTOR;
    866       1.4  jakllsch 		vector = VIRTIO_MSIX_QUEUE_VECTOR_INDEX;
    867       1.4  jakllsch 
    868       1.6  yamaguch 		if (sc->sc_child_mq)
    869       1.6  yamaguch 			vector += qid;
    870       1.6  yamaguch 
    871  1.14.2.1   thorpej 		bus_space_write_2(psc->sc_iot, psc->sc_ioh, offset, vector);
    872  1.14.2.1   thorpej 		ret = bus_space_read_2(psc->sc_iot, psc->sc_ioh, offset);
    873       1.4  jakllsch 		aprint_debug_dev(sc->sc_dev, "expected=%d, actual=%d\n",
    874       1.4  jakllsch 		    vector, ret);
    875  1.14.2.1   thorpej 		if (ret != vector) {
    876  1.14.2.1   thorpej 			aprint_error_dev(self, "can't set queue %d "
    877  1.14.2.1   thorpej 				"msix vector\n", qid);
    878       1.4  jakllsch 			return -1;
    879  1.14.2.1   thorpej 		}
    880       1.4  jakllsch 	}
    881       1.4  jakllsch 
    882       1.4  jakllsch 	return 0;
    883       1.4  jakllsch }
    884       1.4  jakllsch 
    885       1.4  jakllsch static int
    886       1.4  jakllsch virtio_pci_setup_msix_interrupts(struct virtio_softc *sc,
    887       1.4  jakllsch     struct pci_attach_args *pa)
    888       1.4  jakllsch {
    889       1.4  jakllsch 	struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc;
    890       1.4  jakllsch 	device_t self = sc->sc_dev;
    891       1.4  jakllsch 	pci_chipset_tag_t pc = pa->pa_pc;
    892       1.9  yamaguch 	struct virtqueue *vq;
    893       1.4  jakllsch 	char intrbuf[PCI_INTRSTR_LEN];
    894       1.6  yamaguch 	char intr_xname[INTRDEVNAMEBUF];
    895       1.4  jakllsch 	char const *intrstr;
    896       1.6  yamaguch 	int idx, qid, n;
    897  1.14.2.1   thorpej 	int ret;
    898       1.4  jakllsch 
    899       1.4  jakllsch 	idx = VIRTIO_MSIX_CONFIG_VECTOR_INDEX;
    900  1.14.2.1   thorpej 	if (sc->sc_flags & VIRTIO_F_INTR_MPSAFE)
    901       1.4  jakllsch 		pci_intr_setattr(pc, &psc->sc_ihp[idx], PCI_INTR_MPSAFE, true);
    902       1.4  jakllsch 
    903       1.6  yamaguch 	snprintf(intr_xname, sizeof(intr_xname), "%s config",
    904       1.6  yamaguch 	    device_xname(sc->sc_dev));
    905       1.6  yamaguch 
    906       1.4  jakllsch 	psc->sc_ihs[idx] = pci_intr_establish_xname(pc, psc->sc_ihp[idx],
    907       1.6  yamaguch 	    sc->sc_ipl, virtio_pci_msix_config_intr, sc, intr_xname);
    908       1.4  jakllsch 	if (psc->sc_ihs[idx] == NULL) {
    909       1.4  jakllsch 		aprint_error_dev(self, "couldn't establish MSI-X for config\n");
    910       1.4  jakllsch 		goto error;
    911       1.4  jakllsch 	}
    912       1.4  jakllsch 
    913       1.4  jakllsch 	idx = VIRTIO_MSIX_QUEUE_VECTOR_INDEX;
    914       1.6  yamaguch 	if (sc->sc_child_mq) {
    915       1.6  yamaguch 		for (qid = 0; qid < sc->sc_nvqs; qid++) {
    916       1.6  yamaguch 			n = idx + qid;
    917       1.9  yamaguch 			vq = &sc->sc_vqs[qid];
    918       1.6  yamaguch 
    919       1.6  yamaguch 			snprintf(intr_xname, sizeof(intr_xname), "%s vq#%d",
    920       1.6  yamaguch 			    device_xname(sc->sc_dev), qid);
    921       1.6  yamaguch 
    922  1.14.2.1   thorpej 			if (sc->sc_flags & VIRTIO_F_INTR_MPSAFE) {
    923       1.6  yamaguch 				pci_intr_setattr(pc, &psc->sc_ihp[n],
    924       1.6  yamaguch 				    PCI_INTR_MPSAFE, true);
    925       1.6  yamaguch 			}
    926       1.6  yamaguch 
    927       1.6  yamaguch 			psc->sc_ihs[n] = pci_intr_establish_xname(pc, psc->sc_ihp[n],
    928      1.10  yamaguch 			    sc->sc_ipl, vq->vq_intrhand, vq->vq_intrhand_arg, intr_xname);
    929       1.6  yamaguch 			if (psc->sc_ihs[n] == NULL) {
    930       1.6  yamaguch 				aprint_error_dev(self, "couldn't establish MSI-X for a vq\n");
    931       1.6  yamaguch 				goto error;
    932       1.6  yamaguch 			}
    933       1.6  yamaguch 		}
    934       1.6  yamaguch 	} else {
    935  1.14.2.1   thorpej 		if (sc->sc_flags & VIRTIO_F_INTR_MPSAFE)
    936       1.6  yamaguch 			pci_intr_setattr(pc, &psc->sc_ihp[idx], PCI_INTR_MPSAFE, true);
    937       1.4  jakllsch 
    938       1.6  yamaguch 		snprintf(intr_xname, sizeof(intr_xname), "%s queues",
    939       1.6  yamaguch 		    device_xname(sc->sc_dev));
    940       1.6  yamaguch 		psc->sc_ihs[idx] = pci_intr_establish_xname(pc, psc->sc_ihp[idx],
    941       1.6  yamaguch 		    sc->sc_ipl, virtio_pci_msix_queue_intr, sc, intr_xname);
    942       1.6  yamaguch 		if (psc->sc_ihs[idx] == NULL) {
    943       1.6  yamaguch 			aprint_error_dev(self, "couldn't establish MSI-X for queues\n");
    944       1.6  yamaguch 			goto error;
    945       1.6  yamaguch 		}
    946       1.4  jakllsch 	}
    947       1.4  jakllsch 
    948  1.14.2.1   thorpej 	if (sc->sc_version_1) {
    949  1.14.2.1   thorpej 		ret = virtio_pci_setup_msix_vectors_10(sc);
    950  1.14.2.1   thorpej 	} else {
    951  1.14.2.1   thorpej 		ret = virtio_pci_setup_msix_vectors_09(sc);
    952  1.14.2.1   thorpej 	}
    953  1.14.2.1   thorpej 	if (ret) {
    954       1.4  jakllsch 		aprint_error_dev(self, "couldn't setup MSI-X vectors\n");
    955       1.4  jakllsch 		goto error;
    956       1.4  jakllsch 	}
    957       1.4  jakllsch 
    958       1.4  jakllsch 	idx = VIRTIO_MSIX_CONFIG_VECTOR_INDEX;
    959       1.4  jakllsch 	intrstr = pci_intr_string(pc, psc->sc_ihp[idx], intrbuf, sizeof(intrbuf));
    960       1.4  jakllsch 	aprint_normal_dev(self, "config interrupting at %s\n", intrstr);
    961       1.4  jakllsch 	idx = VIRTIO_MSIX_QUEUE_VECTOR_INDEX;
    962       1.6  yamaguch 	if (sc->sc_child_mq) {
    963       1.6  yamaguch 		kcpuset_t *affinity;
    964       1.6  yamaguch 		int affinity_to, r;
    965       1.6  yamaguch 
    966       1.6  yamaguch 		kcpuset_create(&affinity, false);
    967       1.6  yamaguch 
    968       1.6  yamaguch 		for (qid = 0; qid < sc->sc_nvqs; qid++) {
    969       1.6  yamaguch 			n = idx + qid;
    970       1.6  yamaguch 			affinity_to = (qid / 2) % ncpu;
    971       1.6  yamaguch 
    972       1.6  yamaguch 			intrstr = pci_intr_string(pc, psc->sc_ihp[n],
    973       1.6  yamaguch 			    intrbuf, sizeof(intrbuf));
    974       1.6  yamaguch 
    975       1.6  yamaguch 			kcpuset_zero(affinity);
    976       1.6  yamaguch 			kcpuset_set(affinity, affinity_to);
    977       1.6  yamaguch 			r = interrupt_distribute(psc->sc_ihs[n], affinity, NULL);
    978       1.6  yamaguch 			if (r == 0) {
    979       1.6  yamaguch 				aprint_normal_dev(self,
    980       1.6  yamaguch 				    "for vq #%d interrupting at %s affinity to %u\n",
    981       1.6  yamaguch 				    qid, intrstr, affinity_to);
    982       1.6  yamaguch 			} else {
    983       1.6  yamaguch 				aprint_normal_dev(self,
    984       1.6  yamaguch 				    "for vq #%d interrupting at %s\n",
    985       1.6  yamaguch 				    qid, intrstr);
    986       1.6  yamaguch 			}
    987       1.6  yamaguch 		}
    988       1.6  yamaguch 
    989       1.6  yamaguch 		kcpuset_destroy(affinity);
    990       1.6  yamaguch 	} else {
    991       1.6  yamaguch 		intrstr = pci_intr_string(pc, psc->sc_ihp[idx], intrbuf, sizeof(intrbuf));
    992       1.6  yamaguch 		aprint_normal_dev(self, "queues interrupting at %s\n", intrstr);
    993       1.6  yamaguch 	}
    994       1.4  jakllsch 
    995       1.4  jakllsch 	return 0;
    996       1.4  jakllsch 
    997       1.4  jakllsch error:
    998       1.4  jakllsch 	idx = VIRTIO_MSIX_CONFIG_VECTOR_INDEX;
    999       1.4  jakllsch 	if (psc->sc_ihs[idx] != NULL)
   1000       1.4  jakllsch 		pci_intr_disestablish(psc->sc_pa.pa_pc, psc->sc_ihs[idx]);
   1001       1.4  jakllsch 	idx = VIRTIO_MSIX_QUEUE_VECTOR_INDEX;
   1002       1.6  yamaguch 	if (sc->sc_child_mq) {
   1003       1.6  yamaguch 		for (qid = 0; qid < sc->sc_nvqs; qid++) {
   1004       1.6  yamaguch 			n = idx + qid;
   1005       1.6  yamaguch 			if (psc->sc_ihs[n] == NULL)
   1006       1.6  yamaguch 				continue;
   1007       1.6  yamaguch 			pci_intr_disestablish(psc->sc_pa.pa_pc, psc->sc_ihs[n]);
   1008       1.6  yamaguch 		}
   1009       1.6  yamaguch 
   1010       1.6  yamaguch 	} else {
   1011       1.6  yamaguch 		if (psc->sc_ihs[idx] != NULL)
   1012       1.6  yamaguch 			pci_intr_disestablish(psc->sc_pa.pa_pc, psc->sc_ihs[idx]);
   1013       1.6  yamaguch 	}
   1014       1.4  jakllsch 
   1015       1.4  jakllsch 	return -1;
   1016       1.4  jakllsch }
   1017       1.4  jakllsch 
   1018       1.4  jakllsch static int
   1019       1.4  jakllsch virtio_pci_setup_intx_interrupt(struct virtio_softc *sc,
   1020       1.4  jakllsch     struct pci_attach_args *pa)
   1021       1.4  jakllsch {
   1022       1.4  jakllsch 	struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc;
   1023       1.4  jakllsch 	device_t self = sc->sc_dev;
   1024       1.4  jakllsch 	pci_chipset_tag_t pc = pa->pa_pc;
   1025       1.4  jakllsch 	char intrbuf[PCI_INTRSTR_LEN];
   1026       1.4  jakllsch 	char const *intrstr;
   1027       1.4  jakllsch 
   1028  1.14.2.1   thorpej 	if (sc->sc_flags & VIRTIO_F_INTR_MPSAFE)
   1029       1.4  jakllsch 		pci_intr_setattr(pc, &psc->sc_ihp[0], PCI_INTR_MPSAFE, true);
   1030       1.4  jakllsch 
   1031       1.4  jakllsch 	psc->sc_ihs[0] = pci_intr_establish_xname(pc, psc->sc_ihp[0],
   1032       1.4  jakllsch 	    sc->sc_ipl, virtio_pci_intr, sc, device_xname(sc->sc_dev));
   1033       1.4  jakllsch 	if (psc->sc_ihs[0] == NULL) {
   1034       1.4  jakllsch 		aprint_error_dev(self, "couldn't establish INTx\n");
   1035       1.4  jakllsch 		return -1;
   1036       1.4  jakllsch 	}
   1037       1.4  jakllsch 
   1038       1.4  jakllsch 	intrstr = pci_intr_string(pc, psc->sc_ihp[0], intrbuf, sizeof(intrbuf));
   1039       1.4  jakllsch 	aprint_normal_dev(self, "interrupting at %s\n", intrstr);
   1040       1.4  jakllsch 
   1041       1.4  jakllsch 	return 0;
   1042       1.4  jakllsch }
   1043       1.4  jakllsch 
   1044       1.4  jakllsch static int
   1045       1.4  jakllsch virtio_pci_setup_interrupts(struct virtio_softc *sc)
   1046       1.4  jakllsch {
   1047       1.4  jakllsch 	struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc;
   1048       1.4  jakllsch 	device_t self = sc->sc_dev;
   1049       1.4  jakllsch 	pci_chipset_tag_t pc = psc->sc_pa.pa_pc;
   1050      1.13  jakllsch 	pcitag_t tag = psc->sc_pa.pa_tag;
   1051       1.4  jakllsch 	int error;
   1052       1.4  jakllsch 	int nmsix;
   1053      1.13  jakllsch 	int off;
   1054       1.4  jakllsch 	int counts[PCI_INTR_TYPE_SIZE];
   1055       1.4  jakllsch 	pci_intr_type_t max_type;
   1056      1.13  jakllsch 	pcireg_t ctl;
   1057       1.4  jakllsch 
   1058       1.4  jakllsch 	nmsix = pci_msix_count(psc->sc_pa.pa_pc, psc->sc_pa.pa_tag);
   1059       1.4  jakllsch 	aprint_debug_dev(self, "pci_msix_count=%d\n", nmsix);
   1060       1.4  jakllsch 
   1061       1.4  jakllsch 	/* We need at least two: one for config and the other for queues */
   1062  1.14.2.1   thorpej 	if ((sc->sc_flags & VIRTIO_F_INTR_MSIX) == 0 || nmsix < 2) {
   1063       1.4  jakllsch 		/* Try INTx only */
   1064       1.4  jakllsch 		max_type = PCI_INTR_TYPE_INTX;
   1065       1.4  jakllsch 		counts[PCI_INTR_TYPE_INTX] = 1;
   1066       1.4  jakllsch 	} else {
   1067       1.4  jakllsch 		/* Try MSI-X first and INTx second */
   1068      1.11  yamaguch 		if (sc->sc_nvqs + VIRTIO_MSIX_QUEUE_VECTOR_INDEX <= nmsix) {
   1069      1.11  yamaguch 			nmsix = sc->sc_nvqs + VIRTIO_MSIX_QUEUE_VECTOR_INDEX;
   1070      1.11  yamaguch 		} else {
   1071      1.11  yamaguch 			sc->sc_child_mq = false;
   1072      1.11  yamaguch 		}
   1073      1.11  yamaguch 
   1074      1.11  yamaguch 		if (sc->sc_child_mq == false) {
   1075       1.6  yamaguch 			nmsix = 2;
   1076       1.6  yamaguch 		}
   1077       1.6  yamaguch 
   1078       1.4  jakllsch 		max_type = PCI_INTR_TYPE_MSIX;
   1079       1.6  yamaguch 		counts[PCI_INTR_TYPE_MSIX] = nmsix;
   1080       1.4  jakllsch 		counts[PCI_INTR_TYPE_MSI] = 0;
   1081       1.4  jakllsch 		counts[PCI_INTR_TYPE_INTX] = 1;
   1082       1.4  jakllsch 	}
   1083       1.4  jakllsch 
   1084       1.4  jakllsch retry:
   1085       1.4  jakllsch 	error = pci_intr_alloc(&psc->sc_pa, &psc->sc_ihp, counts, max_type);
   1086       1.4  jakllsch 	if (error != 0) {
   1087       1.4  jakllsch 		aprint_error_dev(self, "couldn't map interrupt\n");
   1088       1.4  jakllsch 		return -1;
   1089       1.4  jakllsch 	}
   1090       1.4  jakllsch 
   1091       1.4  jakllsch 	if (pci_intr_type(pc, psc->sc_ihp[0]) == PCI_INTR_TYPE_MSIX) {
   1092      1.12  jakllsch 		psc->sc_ihs = kmem_zalloc(sizeof(*psc->sc_ihs) * nmsix,
   1093       1.4  jakllsch 		    KM_SLEEP);
   1094       1.4  jakllsch 
   1095       1.4  jakllsch 		error = virtio_pci_setup_msix_interrupts(sc, &psc->sc_pa);
   1096       1.4  jakllsch 		if (error != 0) {
   1097       1.6  yamaguch 			kmem_free(psc->sc_ihs, sizeof(*psc->sc_ihs) * nmsix);
   1098       1.6  yamaguch 			pci_intr_release(pc, psc->sc_ihp, nmsix);
   1099       1.4  jakllsch 
   1100       1.4  jakllsch 			/* Retry INTx */
   1101       1.4  jakllsch 			max_type = PCI_INTR_TYPE_INTX;
   1102       1.4  jakllsch 			counts[PCI_INTR_TYPE_INTX] = 1;
   1103       1.4  jakllsch 			goto retry;
   1104       1.4  jakllsch 		}
   1105       1.4  jakllsch 
   1106       1.6  yamaguch 		psc->sc_ihs_num = nmsix;
   1107  1.14.2.1   thorpej 		psc->sc_devcfg_offset = VIRTIO_CONFIG_DEVICE_CONFIG_MSI;
   1108  1.14.2.1   thorpej 		virtio_pci_adjust_config_region(psc);
   1109       1.4  jakllsch 	} else if (pci_intr_type(pc, psc->sc_ihp[0]) == PCI_INTR_TYPE_INTX) {
   1110      1.12  jakllsch 		psc->sc_ihs = kmem_zalloc(sizeof(*psc->sc_ihs) * 1,
   1111       1.4  jakllsch 		    KM_SLEEP);
   1112       1.4  jakllsch 
   1113       1.4  jakllsch 		error = virtio_pci_setup_intx_interrupt(sc, &psc->sc_pa);
   1114       1.4  jakllsch 		if (error != 0) {
   1115       1.4  jakllsch 			kmem_free(psc->sc_ihs, sizeof(*psc->sc_ihs) * 1);
   1116       1.4  jakllsch 			pci_intr_release(pc, psc->sc_ihp, 1);
   1117       1.4  jakllsch 			return -1;
   1118       1.4  jakllsch 		}
   1119       1.4  jakllsch 
   1120       1.4  jakllsch 		psc->sc_ihs_num = 1;
   1121  1.14.2.1   thorpej 		psc->sc_devcfg_offset = VIRTIO_CONFIG_DEVICE_CONFIG_NOMSI;
   1122  1.14.2.1   thorpej 		virtio_pci_adjust_config_region(psc);
   1123      1.13  jakllsch 
   1124      1.14  jakllsch 		error = pci_get_capability(pc, tag, PCI_CAP_MSIX, &off, NULL);
   1125      1.13  jakllsch 		if (error != 0) {
   1126      1.13  jakllsch 			ctl = pci_conf_read(pc, tag, off + PCI_MSIX_CTL);
   1127      1.13  jakllsch 			ctl &= ~PCI_MSIX_CTL_ENABLE;
   1128      1.13  jakllsch 			pci_conf_write(pc, tag, off + PCI_MSIX_CTL, ctl);
   1129      1.13  jakllsch 		}
   1130       1.4  jakllsch 	}
   1131       1.4  jakllsch 
   1132       1.4  jakllsch 	return 0;
   1133       1.4  jakllsch }
   1134       1.4  jakllsch 
   1135       1.4  jakllsch static void
   1136       1.4  jakllsch virtio_pci_free_interrupts(struct virtio_softc *sc)
   1137       1.4  jakllsch {
   1138       1.4  jakllsch 	struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc;
   1139       1.4  jakllsch 
   1140       1.4  jakllsch 	for (int i = 0; i < psc->sc_ihs_num; i++) {
   1141       1.4  jakllsch 		if (psc->sc_ihs[i] == NULL)
   1142       1.4  jakllsch 			continue;
   1143       1.4  jakllsch 		pci_intr_disestablish(psc->sc_pa.pa_pc, psc->sc_ihs[i]);
   1144       1.4  jakllsch 		psc->sc_ihs[i] = NULL;
   1145       1.4  jakllsch 	}
   1146       1.4  jakllsch 
   1147       1.4  jakllsch 	if (psc->sc_ihs_num > 0)
   1148       1.4  jakllsch 		pci_intr_release(psc->sc_pa.pa_pc, psc->sc_ihp, psc->sc_ihs_num);
   1149       1.4  jakllsch 
   1150       1.4  jakllsch 	if (psc->sc_ihs != NULL) {
   1151       1.4  jakllsch 		kmem_free(psc->sc_ihs, sizeof(*psc->sc_ihs) * psc->sc_ihs_num);
   1152       1.4  jakllsch 		psc->sc_ihs = NULL;
   1153       1.4  jakllsch 	}
   1154       1.4  jakllsch 	psc->sc_ihs_num = 0;
   1155       1.4  jakllsch }
   1156       1.4  jakllsch 
   1157       1.4  jakllsch /*
   1158       1.4  jakllsch  * Interrupt handler.
   1159       1.4  jakllsch  */
   1160       1.4  jakllsch static int
   1161       1.4  jakllsch virtio_pci_intr(void *arg)
   1162       1.4  jakllsch {
   1163       1.4  jakllsch 	struct virtio_softc *sc = arg;
   1164       1.4  jakllsch 	struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc;
   1165       1.4  jakllsch 	int isr, r = 0;
   1166       1.4  jakllsch 
   1167       1.4  jakllsch 	/* check and ack the interrupt */
   1168  1.14.2.1   thorpej 	isr = bus_space_read_1(psc->sc_isr_iot, psc->sc_isr_ioh, 0);
   1169       1.4  jakllsch 	if (isr == 0)
   1170       1.4  jakllsch 		return 0;
   1171       1.4  jakllsch 	if ((isr & VIRTIO_CONFIG_ISR_CONFIG_CHANGE) &&
   1172       1.4  jakllsch 	    (sc->sc_config_change != NULL))
   1173       1.4  jakllsch 		r = (sc->sc_config_change)(sc);
   1174       1.4  jakllsch 	if (sc->sc_intrhand != NULL) {
   1175       1.4  jakllsch 		if (sc->sc_soft_ih != NULL)
   1176       1.4  jakllsch 			softint_schedule(sc->sc_soft_ih);
   1177       1.4  jakllsch 		else
   1178       1.4  jakllsch 			r |= (sc->sc_intrhand)(sc);
   1179       1.4  jakllsch 	}
   1180       1.4  jakllsch 
   1181       1.4  jakllsch 	return r;
   1182       1.4  jakllsch }
   1183       1.4  jakllsch 
   1184       1.4  jakllsch static int
   1185       1.4  jakllsch virtio_pci_msix_queue_intr(void *arg)
   1186       1.4  jakllsch {
   1187       1.4  jakllsch 	struct virtio_softc *sc = arg;
   1188       1.4  jakllsch 	int r = 0;
   1189       1.4  jakllsch 
   1190       1.4  jakllsch 	if (sc->sc_intrhand != NULL) {
   1191       1.4  jakllsch 		if (sc->sc_soft_ih != NULL)
   1192       1.4  jakllsch 			softint_schedule(sc->sc_soft_ih);
   1193       1.4  jakllsch 		else
   1194       1.4  jakllsch 			r |= (sc->sc_intrhand)(sc);
   1195       1.4  jakllsch 	}
   1196       1.4  jakllsch 
   1197       1.4  jakllsch 	return r;
   1198       1.4  jakllsch }
   1199       1.4  jakllsch 
   1200       1.4  jakllsch static int
   1201       1.4  jakllsch virtio_pci_msix_config_intr(void *arg)
   1202       1.4  jakllsch {
   1203       1.4  jakllsch 	struct virtio_softc *sc = arg;
   1204       1.4  jakllsch 	int r = 0;
   1205       1.4  jakllsch 
   1206       1.4  jakllsch 	if (sc->sc_config_change != NULL)
   1207       1.4  jakllsch 		r = (sc->sc_config_change)(sc);
   1208       1.4  jakllsch 	return r;
   1209       1.4  jakllsch }
   1210       1.5  jakllsch 
   1211       1.5  jakllsch MODULE(MODULE_CLASS_DRIVER, virtio_pci, "pci,virtio");
   1212       1.5  jakllsch 
   1213       1.5  jakllsch #ifdef _MODULE
   1214       1.5  jakllsch #include "ioconf.c"
   1215       1.5  jakllsch #endif
   1216       1.5  jakllsch 
   1217       1.5  jakllsch static int
   1218       1.5  jakllsch virtio_pci_modcmd(modcmd_t cmd, void *opaque)
   1219       1.5  jakllsch {
   1220       1.5  jakllsch 	int error = 0;
   1221       1.5  jakllsch 
   1222       1.5  jakllsch #ifdef _MODULE
   1223       1.5  jakllsch 	switch (cmd) {
   1224       1.5  jakllsch 	case MODULE_CMD_INIT:
   1225       1.5  jakllsch 		error = config_init_component(cfdriver_ioconf_virtio_pci,
   1226       1.5  jakllsch 		    cfattach_ioconf_virtio_pci, cfdata_ioconf_virtio_pci);
   1227       1.5  jakllsch 		break;
   1228       1.5  jakllsch 	case MODULE_CMD_FINI:
   1229       1.5  jakllsch 		error = config_fini_component(cfdriver_ioconf_virtio_pci,
   1230       1.5  jakllsch 		    cfattach_ioconf_virtio_pci, cfdata_ioconf_virtio_pci);
   1231       1.5  jakllsch 		break;
   1232       1.5  jakllsch 	default:
   1233       1.5  jakllsch 		error = ENOTTY;
   1234       1.5  jakllsch 		break;
   1235       1.5  jakllsch 	}
   1236       1.5  jakllsch #endif
   1237       1.5  jakllsch 
   1238       1.5  jakllsch 	return error;
   1239       1.5  jakllsch }
   1240