Home | History | Annotate | Line # | Download | only in pci
virtio_pci.c revision 1.53
      1 /* $NetBSD: virtio_pci.c,v 1.53 2024/06/25 14:55:09 riastradh Exp $ */
      2 
      3 /*
      4  * Copyright (c) 2020 The NetBSD Foundation, Inc.
      5  * Copyright (c) 2012 Stefan Fritsch.
      6  * Copyright (c) 2010 Minoura Makoto.
      7  * All rights reserved.
      8  *
      9  * Redistribution and use in source and binary forms, with or without
     10  * modification, are permitted provided that the following conditions
     11  * are met:
     12  * 1. Redistributions of source code must retain the above copyright
     13  *    notice, this list of conditions and the following disclaimer.
     14  * 2. Redistributions in binary form must reproduce the above copyright
     15  *    notice, this list of conditions and the following disclaimer in the
     16  *    documentation and/or other materials provided with the distribution.
     17  *
     18  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
     19  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
     20  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
     21  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
     22  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
     23  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
     24  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
     25  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
     26  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
     27  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
     28  */
     29 
     30 #include <sys/cdefs.h>
     31 __KERNEL_RCSID(0, "$NetBSD: virtio_pci.c,v 1.53 2024/06/25 14:55:09 riastradh Exp $");
     32 
     33 #include <sys/param.h>
     34 #include <sys/types.h>
     35 
     36 #include <sys/device.h>
     37 #include <sys/endian.h>
     38 #include <sys/interrupt.h>
     39 #include <sys/kmem.h>
     40 #include <sys/module.h>
     41 #include <sys/syslog.h>
     42 #include <sys/systm.h>
     43 
     44 #include <dev/pci/pcidevs.h>
     45 #include <dev/pci/pcireg.h>
     46 #include <dev/pci/pcivar.h>
     47 
     48 #include <dev/pci/virtioreg.h> /* XXX: move to non-pci */
     49 #include <dev/pci/virtio_pcireg.h>
     50 
     51 #define VIRTIO_PRIVATE
     52 #include <dev/pci/virtiovar.h> /* XXX: move to non-pci */
     53 
     54 #if defined(__alpha__) || defined(__sparc64__)
     55 /*
     56  * XXX VIRTIO_F_ACCESS_PLATFORM is required for standard PCI DMA
     57  * XXX to work on these platforms, at least by Qemu.
     58  * XXX
     59  * XXX Generalize this later.
     60  */
     61 #define	__NEED_VIRTIO_F_ACCESS_PLATFORM
     62 #endif /* __alpha__ || __sparc64__ */
     63 
     64 #define VIRTIO_PCI_LOG(_sc, _use_log, _fmt, _args...)	\
     65 do {							\
     66 	if ((_use_log)) {				\
     67 		log(LOG_DEBUG, "%s: " _fmt,		\
     68 		    device_xname((_sc)->sc_dev),	\
     69 		    ##_args);				\
     70 	} else {					\
     71 		aprint_error_dev((_sc)->sc_dev,		\
     72 		    _fmt, ##_args);			\
     73 	}						\
     74 } while(0)
     75 
     76 static int	virtio_pci_match(device_t, cfdata_t, void *);
     77 static void	virtio_pci_attach(device_t, device_t, void *);
     78 static int	virtio_pci_rescan(device_t, const char *, const int *);
     79 static int	virtio_pci_detach(device_t, int);
     80 
     81 #define NMAPREG		((PCI_MAPREG_END - PCI_MAPREG_START) / \
     82 				sizeof(pcireg_t))
     83 struct virtio_pci_softc {
     84 	struct virtio_softc	sc_sc;
     85 	bool			sc_intr_pervq;
     86 
     87 	/* IO space */
     88 	bus_space_tag_t		sc_iot;
     89 	bus_space_handle_t	sc_ioh;
     90 	bus_size_t		sc_iosize;
     91 
     92 	/* BARs */
     93 	bus_space_tag_t		sc_bars_iot[NMAPREG];
     94 	bus_space_handle_t	sc_bars_ioh[NMAPREG];
     95 	bus_size_t		sc_bars_iosize[NMAPREG];
     96 
     97 	/* notify space */
     98 	bus_space_tag_t		sc_notify_iot;
     99 	bus_space_handle_t	sc_notify_ioh;
    100 	bus_size_t		sc_notify_iosize;
    101 	uint32_t		sc_notify_off_multiplier;
    102 
    103 	/* isr space */
    104 	bus_space_tag_t		sc_isr_iot;
    105 	bus_space_handle_t	sc_isr_ioh;
    106 	bus_size_t		sc_isr_iosize;
    107 
    108 	/* generic */
    109 	struct pci_attach_args	sc_pa;
    110 	pci_intr_handle_t	*sc_ihp;
    111 	void			**sc_ihs;
    112 	int			sc_ihs_num;
    113 	int			sc_devcfg_offset;	/* for 0.9 */
    114 };
    115 
    116 static int	virtio_pci_attach_09(device_t, void *);
    117 static void	virtio_pci_kick_09(struct virtio_softc *, uint16_t);
    118 static uint16_t	virtio_pci_read_queue_size_09(struct virtio_softc *, uint16_t);
    119 static void	virtio_pci_setup_queue_09(struct virtio_softc *, uint16_t,
    120 		    uint64_t);
    121 static void	virtio_pci_set_status_09(struct virtio_softc *, int);
    122 static void	virtio_pci_negotiate_features_09(struct virtio_softc *,
    123 		    uint64_t);
    124 
    125 static int	virtio_pci_attach_10(device_t, void *);
    126 static void	virtio_pci_kick_10(struct virtio_softc *, uint16_t);
    127 static uint16_t	virtio_pci_read_queue_size_10(struct virtio_softc *, uint16_t);
    128 static void	virtio_pci_setup_queue_10(struct virtio_softc *, uint16_t,
    129 		    uint64_t);
    130 static void	virtio_pci_set_status_10(struct virtio_softc *, int);
    131 static void	virtio_pci_negotiate_features_10(struct virtio_softc *,
    132 		    uint64_t);
    133 static int	virtio_pci_find_cap(struct virtio_pci_softc *, int, void *,
    134 		    int);
    135 
    136 static int	virtio_pci_alloc_interrupts(struct virtio_softc *);
    137 static void	virtio_pci_free_interrupts(struct virtio_softc *);
    138 static int	virtio_pci_adjust_config_region(struct virtio_pci_softc *);
    139 static int	virtio_pci_intr(void *);
    140 static int	virtio_pci_msix_queue_intr(void *);
    141 static int	virtio_pci_msix_config_intr(void *);
    142 static int	virtio_pci_setup_interrupts_09(struct virtio_softc *, int);
    143 static int	virtio_pci_setup_interrupts_10(struct virtio_softc *, int);
    144 static int	virtio_pci_establish_msix_interrupts(struct virtio_softc *,
    145 		    const struct pci_attach_args *);
    146 static int	virtio_pci_establish_intx_interrupt(struct virtio_softc *,
    147 		    const struct pci_attach_args *);
    148 static bool	virtio_pci_msix_enabled(struct virtio_pci_softc *);
    149 
    150 #define VIRTIO_MSIX_CONFIG_VECTOR_INDEX	0
    151 #define VIRTIO_MSIX_QUEUE_VECTOR_INDEX	1
    152 
    153 /*
    154  * For big-endian aarch64/armv7 on QEMU (and most real HW), only CPU cores
    155  * are running in big-endian mode, with all peripheral being configured to
    156  * little-endian mode. Their default bus_space(9) functions forcibly swap
    157  * byte-order. This guarantees that PIO'ed data from pci(4), e.g., are
    158  * correctly handled by bus_space(9), while DMA'ed ones should be swapped
    159  * by hand, in violation of virtio(4) specifications.
    160  */
    161 
    162 #if (defined(__aarch64__) || defined(__arm__)) && BYTE_ORDER == BIG_ENDIAN
    163 #	define READ_ENDIAN_09	BIG_ENDIAN
    164 #	define READ_ENDIAN_10	BIG_ENDIAN
    165 #	define STRUCT_ENDIAN_09	BIG_ENDIAN
    166 #	define STRUCT_ENDIAN_10	LITTLE_ENDIAN
    167 #elif BYTE_ORDER == BIG_ENDIAN
    168 #	define READ_ENDIAN_09	LITTLE_ENDIAN
    169 #	define READ_ENDIAN_10	BIG_ENDIAN
    170 #	define STRUCT_ENDIAN_09	BIG_ENDIAN
    171 #	define STRUCT_ENDIAN_10	LITTLE_ENDIAN
    172 #else /* little endian */
    173 #	define READ_ENDIAN_09	LITTLE_ENDIAN
    174 #	define READ_ENDIAN_10	LITTLE_ENDIAN
    175 #	define STRUCT_ENDIAN_09	LITTLE_ENDIAN
    176 #	define STRUCT_ENDIAN_10	LITTLE_ENDIAN
    177 #endif
    178 
    179 CFATTACH_DECL3_NEW(virtio_pci, sizeof(struct virtio_pci_softc),
    180     virtio_pci_match, virtio_pci_attach, virtio_pci_detach, NULL,
    181     virtio_pci_rescan, NULL, 0);
    182 
    183 static const struct virtio_ops virtio_pci_ops_09 = {
    184 	.kick = virtio_pci_kick_09,
    185 	.read_queue_size = virtio_pci_read_queue_size_09,
    186 	.setup_queue = virtio_pci_setup_queue_09,
    187 	.set_status = virtio_pci_set_status_09,
    188 	.neg_features = virtio_pci_negotiate_features_09,
    189 	.alloc_interrupts = virtio_pci_alloc_interrupts,
    190 	.free_interrupts = virtio_pci_free_interrupts,
    191 	.setup_interrupts = virtio_pci_setup_interrupts_09,
    192 };
    193 
    194 static const struct virtio_ops virtio_pci_ops_10 = {
    195 	.kick = virtio_pci_kick_10,
    196 	.read_queue_size = virtio_pci_read_queue_size_10,
    197 	.setup_queue = virtio_pci_setup_queue_10,
    198 	.set_status = virtio_pci_set_status_10,
    199 	.neg_features = virtio_pci_negotiate_features_10,
    200 	.alloc_interrupts = virtio_pci_alloc_interrupts,
    201 	.free_interrupts = virtio_pci_free_interrupts,
    202 	.setup_interrupts = virtio_pci_setup_interrupts_10,
    203 };
    204 
    205 static int
    206 virtio_pci_match(device_t parent, cfdata_t match, void *aux)
    207 {
    208 	const struct pci_attach_args * const pa = aux;
    209 
    210 	switch (PCI_VENDOR(pa->pa_id)) {
    211 	case PCI_VENDOR_QUMRANET:
    212 		/* Transitional devices MUST have a PCI Revision ID of 0. */
    213 		if (((PCI_PRODUCT_QUMRANET_VIRTIO_1000 <=
    214 			    PCI_PRODUCT(pa->pa_id)) &&
    215 			(PCI_PRODUCT(pa->pa_id) <=
    216 			    PCI_PRODUCT_QUMRANET_VIRTIO_103F)) &&
    217 		    PCI_REVISION(pa->pa_class) == 0)
    218 			return 1;
    219 		/*
    220 		 * Non-transitional devices SHOULD have a PCI Revision
    221 		 * ID of 1 or higher.  Drivers MUST match any PCI
    222 		 * Revision ID value.
    223 		 */
    224 		if (((PCI_PRODUCT_QUMRANET_VIRTIO_1040 <=
    225 			    PCI_PRODUCT(pa->pa_id)) &&
    226 			(PCI_PRODUCT(pa->pa_id) <=
    227 			    PCI_PRODUCT_QUMRANET_VIRTIO_107F)) &&
    228 		    /* XXX: TODO */
    229 		    PCI_REVISION(pa->pa_class) == 1)
    230 			return 1;
    231 		break;
    232 	}
    233 
    234 	return 0;
    235 }
    236 
    237 static void
    238 virtio_pci_attach(device_t parent, device_t self, void *aux)
    239 {
    240 	struct virtio_pci_softc * const psc = device_private(self);
    241 	struct virtio_softc * const sc = &psc->sc_sc;
    242 	const struct pci_attach_args * const pa = aux;
    243 	pci_chipset_tag_t pc = pa->pa_pc;
    244 	pcitag_t tag = pa->pa_tag;
    245 	int revision;
    246 	int ret;
    247 	pcireg_t id;
    248 	pcireg_t csr;
    249 
    250 	revision = PCI_REVISION(pa->pa_class);
    251 	switch (revision) {
    252 	case 0:
    253 		/* subsystem ID shows what I am */
    254 		id = PCI_SUBSYS_ID(pci_conf_read(pc, tag, PCI_SUBSYS_ID_REG));
    255 		break;
    256 	case 1:
    257 		/* pci product number shows what I am */
    258 		id = PCI_PRODUCT(pa->pa_id) - PCI_PRODUCT_QUMRANET_VIRTIO_1040;
    259 		break;
    260 	default:
    261 		aprint_normal(": unknown revision 0x%02x; giving up\n",
    262 		    revision);
    263 		return;
    264 	}
    265 
    266 	aprint_normal("\n");
    267 	aprint_naive("\n");
    268 	virtio_print_device_type(self, id, revision);
    269 
    270 	csr = pci_conf_read(pc, tag, PCI_COMMAND_STATUS_REG);
    271 	csr |= PCI_COMMAND_MASTER_ENABLE | PCI_COMMAND_IO_ENABLE;
    272 	pci_conf_write(pc, tag, PCI_COMMAND_STATUS_REG, csr);
    273 
    274 	sc->sc_dev = self;
    275 	psc->sc_pa = *pa;
    276 	psc->sc_iot = pa->pa_iot;
    277 
    278 	sc->sc_dmat = pa->pa_dmat;
    279 	if (pci_dma64_available(pa))
    280 		sc->sc_dmat = pa->pa_dmat64;
    281 
    282 	/* attach is dependent on revision */
    283 	ret = 0;
    284 	if (revision == 1) {
    285 		/* try to attach 1.0 */
    286 		ret = virtio_pci_attach_10(self, aux);
    287 	}
    288 	if (ret == 0 && revision == 0) {
    289 		/*
    290 		 * revision 0 means 0.9 only or both 0.9 and 1.0.  The
    291 		 * latter are so-called "Transitional Devices".  For
    292 		 * those devices, we want to use the 1.0 interface if
    293 		 * possible.
    294 		 *
    295 		 * XXX Currently only on platforms that require 1.0
    296 		 * XXX features, such as VIRTIO_F_ACCESS_PLATFORM.
    297 		 */
    298 #ifdef __NEED_VIRTIO_F_ACCESS_PLATFORM
    299 		/* First, try to attach 1.0 */
    300 		ret = virtio_pci_attach_10(self, aux);
    301 		if (ret != 0) {
    302 			aprint_error_dev(self,
    303 			    "VirtIO 1.0 error = %d, falling back to 0.9\n",
    304 			    ret);
    305 			/* Fall back to 0.9. */
    306 			ret = virtio_pci_attach_09(self, aux);
    307 		}
    308 #else
    309 		ret = virtio_pci_attach_09(self, aux);
    310 #endif /* __NEED_VIRTIO_F_ACCESS_PLATFORM */
    311 	}
    312 	if (ret) {
    313 		aprint_error_dev(self, "cannot attach (%d)\n", ret);
    314 		return;
    315 	}
    316 	KASSERT(sc->sc_ops);
    317 
    318 	/* preset config region */
    319 	psc->sc_devcfg_offset = VIRTIO_CONFIG_DEVICE_CONFIG_NOMSI;
    320 	if (virtio_pci_adjust_config_region(psc))
    321 		return;
    322 
    323 	/* generic */
    324 	virtio_device_reset(sc);
    325 	virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_ACK);
    326 	virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_DRIVER);
    327 
    328 	sc->sc_childdevid = id;
    329 	sc->sc_child = NULL;
    330 	virtio_pci_rescan(self, NULL, NULL);
    331 	return;
    332 }
    333 
    334 /* ARGSUSED */
    335 static int
    336 virtio_pci_rescan(device_t self, const char *ifattr, const int *locs)
    337 {
    338 	struct virtio_pci_softc * const psc = device_private(self);
    339 	struct virtio_softc * const sc = &psc->sc_sc;
    340 	struct virtio_attach_args va;
    341 
    342 	if (sc->sc_child)	/* Child already attached? */
    343 		return 0;
    344 
    345 	memset(&va, 0, sizeof(va));
    346 	va.sc_childdevid = sc->sc_childdevid;
    347 
    348 	config_found(self, &va, NULL, CFARGS_NONE);
    349 
    350 	if (virtio_attach_failed(sc))
    351 		return 0;
    352 
    353 	return 0;
    354 }
    355 
    356 static int
    357 virtio_pci_detach(device_t self, int flags)
    358 {
    359 	struct virtio_pci_softc * const psc = device_private(self);
    360 	struct virtio_softc * const sc = &psc->sc_sc;
    361 	unsigned i;
    362 	int r;
    363 
    364 	r = config_detach_children(self, flags);
    365 	if (r != 0)
    366 		return r;
    367 
    368 	/* Check that child never attached, or detached properly */
    369 	KASSERT(sc->sc_child == NULL);
    370 	KASSERT(sc->sc_vqs == NULL);
    371 	KASSERT(psc->sc_ihs_num == 0);
    372 
    373 	if (sc->sc_version_1) {
    374 		for (i = 0; i < __arraycount(psc->sc_bars_iot); i++) {
    375 			if (psc->sc_bars_iosize[i] == 0)
    376 				continue;
    377 			bus_space_unmap(psc->sc_bars_iot[i],
    378 			    psc->sc_bars_ioh[i], psc->sc_bars_iosize[i]);
    379 			psc->sc_bars_iosize[i] = 0;
    380 		}
    381 	} else {
    382 		if (psc->sc_iosize) {
    383 			bus_space_unmap(psc->sc_iot, psc->sc_ioh,
    384 			    psc->sc_iosize);
    385 			psc->sc_iosize = 0;
    386 		}
    387 	}
    388 
    389 	return 0;
    390 }
    391 
    392 static int
    393 virtio_pci_attach_09(device_t self, void *aux)
    394 {
    395 	struct virtio_pci_softc * const psc = device_private(self);
    396 	const struct pci_attach_args * const pa = aux;
    397 	struct virtio_softc * const sc = &psc->sc_sc;
    398 
    399 	/* complete IO region */
    400 	if (pci_mapreg_map(pa, PCI_MAPREG_START, PCI_MAPREG_TYPE_IO, 0,
    401 		&psc->sc_iot, &psc->sc_ioh, NULL, &psc->sc_iosize)) {
    402 		aprint_error_dev(self, "can't map i/o space\n");
    403 		return EIO;
    404 	}
    405 
    406 	/* queue space */
    407 	if (bus_space_subregion(psc->sc_iot, psc->sc_ioh,
    408 		VIRTIO_CONFIG_QUEUE_NOTIFY, 2, &psc->sc_notify_ioh)) {
    409 		aprint_error_dev(self, "can't map notify i/o space\n");
    410 		return EIO;
    411 	}
    412 	psc->sc_notify_iosize = 2;
    413 	psc->sc_notify_iot = psc->sc_iot;
    414 
    415 	/* ISR space */
    416 	if (bus_space_subregion(psc->sc_iot, psc->sc_ioh,
    417 		VIRTIO_CONFIG_ISR_STATUS, 1, &psc->sc_isr_ioh)) {
    418 		aprint_error_dev(self, "can't map isr i/o space\n");
    419 		return EIO;
    420 	}
    421 	psc->sc_isr_iosize = 1;
    422 	psc->sc_isr_iot = psc->sc_iot;
    423 
    424 	/* set our version 0.9 ops */
    425 	sc->sc_ops = &virtio_pci_ops_09;
    426 	sc->sc_bus_endian = READ_ENDIAN_09;
    427 	sc->sc_struct_endian = STRUCT_ENDIAN_09;
    428 	return 0;
    429 }
    430 
    431 static int
    432 virtio_pci_attach_10(device_t self, void *aux)
    433 {
    434 	struct virtio_pci_softc * const psc = device_private(self);
    435 	const struct pci_attach_args * const pa = aux;
    436 	struct virtio_softc * const sc = &psc->sc_sc;
    437 	const pci_chipset_tag_t pc = pa->pa_pc;
    438 	const pcitag_t tag = pa->pa_tag;
    439 
    440 	struct virtio_pci_cap common, isr, device;
    441 	struct virtio_pci_notify_cap notify;
    442 	int have_device_cfg = 0;
    443 	bus_size_t bars[NMAPREG] = { 0 };
    444 	int bars_idx[NMAPREG] = { 0 };
    445 	struct virtio_pci_cap * const caps[] =
    446 	    { &common, &isr, &device, &notify.cap };
    447 	int i, j, ret = 0;
    448 
    449 	if (virtio_pci_find_cap(psc, VIRTIO_PCI_CAP_COMMON_CFG,
    450 		&common, sizeof(common)))
    451 		return ENODEV;
    452 	if (virtio_pci_find_cap(psc, VIRTIO_PCI_CAP_NOTIFY_CFG,
    453 		&notify, sizeof(notify)))
    454 		return ENODEV;
    455 	if (virtio_pci_find_cap(psc, VIRTIO_PCI_CAP_ISR_CFG,
    456 		&isr, sizeof(isr)))
    457 		return ENODEV;
    458 	if (virtio_pci_find_cap(psc, VIRTIO_PCI_CAP_DEVICE_CFG,
    459 		&device, sizeof(device)))
    460 		memset(&device, 0, sizeof(device));
    461 	else
    462 		have_device_cfg = 1;
    463 
    464 	/* Figure out which bars we need to map */
    465 	for (i = 0; i < __arraycount(caps); i++) {
    466 		int bar = caps[i]->bar;
    467 		bus_size_t len = caps[i]->offset + caps[i]->length;
    468 
    469 		if (caps[i]->length == 0)
    470 			continue;
    471 		if (bars[bar] < len)
    472 			bars[bar] = len;
    473 	}
    474 
    475 	for (i = j = 0; i < __arraycount(bars); i++) {
    476 		int reg;
    477 		pcireg_t type;
    478 
    479 		if (bars[i] == 0)
    480 			continue;
    481 		reg = PCI_BAR(i);
    482 		type = pci_mapreg_type(pc, tag, reg);
    483 		if (pci_mapreg_map(pa, reg, type, 0,
    484 			&psc->sc_bars_iot[j], &psc->sc_bars_ioh[j],
    485 			NULL, &psc->sc_bars_iosize[j])) {
    486 			aprint_error_dev(self, "can't map bar %u \n", i);
    487 			ret = EIO;
    488 			goto err;
    489 		}
    490 		aprint_debug_dev(self,
    491 		    "bar[%d]: iot %p, size 0x%" PRIxBUSSIZE "\n",
    492 		    j, psc->sc_bars_iot[j], psc->sc_bars_iosize[j]);
    493 		bars_idx[i] = j;
    494 		j++;
    495 	}
    496 
    497 	i = bars_idx[notify.cap.bar];
    498 	if (bus_space_subregion(psc->sc_bars_iot[i], psc->sc_bars_ioh[i],
    499 		notify.cap.offset, notify.cap.length, &psc->sc_notify_ioh)) {
    500 		aprint_error_dev(self, "can't map notify i/o space\n");
    501 		ret = EIO;
    502 		goto err;
    503 	}
    504 	psc->sc_notify_iosize = notify.cap.length;
    505 	psc->sc_notify_iot = psc->sc_bars_iot[i];
    506 	psc->sc_notify_off_multiplier = le32toh(notify.notify_off_multiplier);
    507 
    508 	if (have_device_cfg) {
    509 		i = bars_idx[device.bar];
    510 		if (bus_space_subregion(psc->sc_bars_iot[i],
    511 			psc->sc_bars_ioh[i], device.offset, device.length,
    512 			&sc->sc_devcfg_ioh)) {
    513 			aprint_error_dev(self, "can't map devcfg i/o space\n");
    514 			ret = EIO;
    515 			goto err;
    516 		}
    517 		aprint_debug_dev(self,
    518 		    "device.offset = 0x%x, device.length = 0x%x\n",
    519 		    device.offset, device.length);
    520 		sc->sc_devcfg_iosize = device.length;
    521 		sc->sc_devcfg_iot = psc->sc_bars_iot[i];
    522 	}
    523 
    524 	i = bars_idx[isr.bar];
    525 	if (bus_space_subregion(psc->sc_bars_iot[i], psc->sc_bars_ioh[i],
    526 		isr.offset, isr.length, &psc->sc_isr_ioh)) {
    527 		aprint_error_dev(self, "can't map isr i/o space\n");
    528 		ret = EIO;
    529 		goto err;
    530 	}
    531 	psc->sc_isr_iosize = isr.length;
    532 	psc->sc_isr_iot = psc->sc_bars_iot[i];
    533 
    534 	i = bars_idx[common.bar];
    535 	if (bus_space_subregion(psc->sc_bars_iot[i], psc->sc_bars_ioh[i],
    536 		common.offset, common.length, &psc->sc_ioh)) {
    537 		aprint_error_dev(self, "can't map common i/o space\n");
    538 		ret = EIO;
    539 		goto err;
    540 	}
    541 	psc->sc_iosize = common.length;
    542 	psc->sc_iot = psc->sc_bars_iot[i];
    543 
    544 	psc->sc_sc.sc_version_1 = 1;
    545 
    546 	/* set our version 1.0 ops */
    547 	sc->sc_ops = &virtio_pci_ops_10;
    548 	sc->sc_bus_endian = READ_ENDIAN_10;
    549 	sc->sc_struct_endian = STRUCT_ENDIAN_10;
    550 	return 0;
    551 
    552 err:
    553 	/* undo our pci_mapreg_map()s */
    554 	for (i = 0; i < __arraycount(bars); i++) {
    555 		if (psc->sc_bars_iosize[i] == 0)
    556 			continue;
    557 		bus_space_unmap(psc->sc_bars_iot[i], psc->sc_bars_ioh[i],
    558 		    psc->sc_bars_iosize[i]);
    559 		psc->sc_bars_iosize[i] = 0;
    560 	}
    561 	return ret;
    562 }
    563 
    564 /* v1.0 attach helper */
    565 static int
    566 virtio_pci_find_cap(struct virtio_pci_softc *psc, int cfg_type, void *buf,
    567     int buflen)
    568 {
    569 	device_t self = psc->sc_sc.sc_dev;
    570 	pci_chipset_tag_t pc = psc->sc_pa.pa_pc;
    571 	pcitag_t tag = psc->sc_pa.pa_tag;
    572 	unsigned int offset, i, len;
    573 	union {
    574 		pcireg_t reg[8];
    575 		struct virtio_pci_cap vcap;
    576 	} *v = buf;
    577 
    578 	if (buflen < sizeof(struct virtio_pci_cap))
    579 		return ERANGE;
    580 
    581 	if (!pci_get_capability(pc, tag, PCI_CAP_VENDSPEC, &offset,
    582 		&v->reg[0]))
    583 		return ENOENT;
    584 
    585 	do {
    586 		for (i = 0; i < 4; i++)
    587 			v->reg[i] =
    588 			    le32toh(pci_conf_read(pc, tag, offset + i * 4));
    589 		if (v->vcap.cfg_type == cfg_type)
    590 			break;
    591 		offset = v->vcap.cap_next;
    592 	} while (offset != 0);
    593 
    594 	if (offset == 0)
    595 		return ENOENT;
    596 
    597 	if (v->vcap.cap_len > sizeof(struct virtio_pci_cap)) {
    598 		len = roundup(v->vcap.cap_len, sizeof(pcireg_t));
    599 		if (len > buflen) {
    600 			aprint_error_dev(self, "%s cap too large\n", __func__);
    601 			return ERANGE;
    602 		}
    603 		for (i = 4; i < len / sizeof(pcireg_t);  i++)
    604 			v->reg[i] =
    605 			    le32toh(pci_conf_read(pc, tag, offset + i * 4));
    606 	}
    607 
    608 	/* endian fixup */
    609 	v->vcap.offset = le32toh(v->vcap.offset);
    610 	v->vcap.length = le32toh(v->vcap.length);
    611 	return 0;
    612 }
    613 
    614 /* -------------------------------------
    615  * Version 0.9 support
    616  * -------------------------------------*/
    617 
    618 static void
    619 virtio_pci_kick_09(struct virtio_softc *sc, uint16_t idx)
    620 {
    621 	struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc;
    622 
    623 	bus_space_write_2(psc->sc_notify_iot, psc->sc_notify_ioh, 0, idx);
    624 }
    625 
    626 /* only applicable for v 0.9 but also called for 1.0 */
    627 static int
    628 virtio_pci_adjust_config_region(struct virtio_pci_softc *psc)
    629 {
    630 	struct virtio_softc * const sc = &psc->sc_sc;
    631 	device_t self = sc->sc_dev;
    632 
    633 	if (psc->sc_sc.sc_version_1)
    634 		return 0;
    635 
    636 	sc->sc_devcfg_iosize = psc->sc_iosize - psc->sc_devcfg_offset;
    637 	sc->sc_devcfg_iot = psc->sc_iot;
    638 	if (bus_space_subregion(psc->sc_iot, psc->sc_ioh,
    639 		psc->sc_devcfg_offset, sc->sc_devcfg_iosize,
    640 		&sc->sc_devcfg_ioh)) {
    641 		aprint_error_dev(self, "can't map config i/o space\n");
    642 		return EIO;
    643 	}
    644 
    645 	return 0;
    646 }
    647 
    648 static uint16_t
    649 virtio_pci_read_queue_size_09(struct virtio_softc *sc, uint16_t idx)
    650 {
    651 	struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc;
    652 
    653 	bus_space_write_2(psc->sc_iot, psc->sc_ioh,
    654 	    VIRTIO_CONFIG_QUEUE_SELECT, idx);
    655 	return bus_space_read_2(psc->sc_iot, psc->sc_ioh,
    656 	    VIRTIO_CONFIG_QUEUE_SIZE);
    657 }
    658 
    659 static void
    660 virtio_pci_setup_queue_09(struct virtio_softc *sc, uint16_t idx, uint64_t addr)
    661 {
    662 	struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc;
    663 
    664 	bus_space_write_2(psc->sc_iot, psc->sc_ioh,
    665 	    VIRTIO_CONFIG_QUEUE_SELECT, idx);
    666 	bus_space_write_4(psc->sc_iot, psc->sc_ioh,
    667 	    VIRTIO_CONFIG_QUEUE_ADDRESS, addr / VIRTIO_PAGE_SIZE);
    668 
    669 	if (psc->sc_ihs_num > 1) {
    670 		int vec = VIRTIO_MSIX_QUEUE_VECTOR_INDEX;
    671 		if (psc->sc_intr_pervq)
    672 			vec += idx;
    673 		bus_space_write_2(psc->sc_iot, psc->sc_ioh,
    674 		    VIRTIO_CONFIG_MSI_QUEUE_VECTOR, vec);
    675 	}
    676 }
    677 
    678 static void
    679 virtio_pci_set_status_09(struct virtio_softc *sc, int status)
    680 {
    681 	struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc;
    682 	int old = 0;
    683 
    684 	if (status != 0) {
    685 		old = bus_space_read_1(psc->sc_iot, psc->sc_ioh,
    686 		    VIRTIO_CONFIG_DEVICE_STATUS);
    687 	}
    688 	bus_space_write_1(psc->sc_iot, psc->sc_ioh,
    689 	    VIRTIO_CONFIG_DEVICE_STATUS, status|old);
    690 }
    691 
    692 static void
    693 virtio_pci_negotiate_features_09(struct virtio_softc *sc,
    694     uint64_t guest_features)
    695 {
    696 	struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc;
    697 	uint32_t r;
    698 
    699 	r = bus_space_read_4(psc->sc_iot, psc->sc_ioh,
    700 	    VIRTIO_CONFIG_DEVICE_FEATURES);
    701 
    702 	r &= guest_features;
    703 
    704 	bus_space_write_4(psc->sc_iot, psc->sc_ioh,
    705 	    VIRTIO_CONFIG_GUEST_FEATURES, r);
    706 
    707 	sc->sc_active_features = r;
    708 }
    709 
    710 /* -------------------------------------
    711  * Version 1.0 support
    712  * -------------------------------------*/
    713 
    714 static void
    715 virtio_pci_kick_10(struct virtio_softc *sc, uint16_t idx)
    716 {
    717 	struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc;
    718 	unsigned offset = sc->sc_vqs[idx].vq_notify_off *
    719 	    psc->sc_notify_off_multiplier;
    720 
    721 	bus_space_write_2(psc->sc_notify_iot, psc->sc_notify_ioh, offset, idx);
    722 }
    723 
    724 static uint16_t
    725 virtio_pci_read_queue_size_10(struct virtio_softc *sc, uint16_t idx)
    726 {
    727 	struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc;
    728 	bus_space_tag_t	   iot = psc->sc_iot;
    729 	bus_space_handle_t ioh = psc->sc_ioh;
    730 
    731 	bus_space_write_2(iot, ioh, VIRTIO_CONFIG1_QUEUE_SELECT, idx);
    732 	return bus_space_read_2(iot, ioh, VIRTIO_CONFIG1_QUEUE_SIZE);
    733 }
    734 
    735 /*
    736  * By definition little endian only in v1.0.  NB: "MAY" in the text
    737  * below refers to "independently" (i.e. the order of accesses) not
    738  * "32-bit" (which is restricted by the earlier "MUST").
    739  *
    740  * 4.1.3.1 Driver Requirements: PCI Device Layout
    741  *
    742  * For device configuration access, the driver MUST use ... 32-bit
    743  * wide and aligned accesses for ... 64-bit wide fields.  For 64-bit
    744  * fields, the driver MAY access each of the high and low 32-bit parts
    745  * of the field independently.
    746  */
    747 static __inline void
    748 virtio_pci_bus_space_write_8(bus_space_tag_t iot, bus_space_handle_t ioh,
    749     bus_size_t offset, uint64_t value)
    750 {
    751 #if _QUAD_HIGHWORD
    752 	bus_space_write_4(iot, ioh, offset, BUS_ADDR_LO32(value));
    753 	bus_space_write_4(iot, ioh, offset + 4, BUS_ADDR_HI32(value));
    754 #else
    755 	bus_space_write_4(iot, ioh, offset, BUS_ADDR_HI32(value));
    756 	bus_space_write_4(iot, ioh, offset + 4, BUS_ADDR_LO32(value));
    757 #endif
    758 }
    759 
    760 static void
    761 virtio_pci_setup_queue_10(struct virtio_softc *sc, uint16_t idx, uint64_t addr)
    762 {
    763 	struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc;
    764 	struct virtqueue *vq = &sc->sc_vqs[idx];
    765 	bus_space_tag_t iot = psc->sc_iot;
    766 	bus_space_handle_t ioh = psc->sc_ioh;
    767 	KASSERT(vq->vq_index == idx);
    768 
    769 	bus_space_write_2(iot, ioh, VIRTIO_CONFIG1_QUEUE_SELECT, vq->vq_index);
    770 	if (addr == 0) {
    771 		bus_space_write_2(iot, ioh, VIRTIO_CONFIG1_QUEUE_ENABLE, 0);
    772 		virtio_pci_bus_space_write_8(iot, ioh,
    773 		    VIRTIO_CONFIG1_QUEUE_DESC, 0);
    774 		virtio_pci_bus_space_write_8(iot, ioh,
    775 		    VIRTIO_CONFIG1_QUEUE_AVAIL, 0);
    776 		virtio_pci_bus_space_write_8(iot, ioh,
    777 		    VIRTIO_CONFIG1_QUEUE_USED, 0);
    778 	} else {
    779 		virtio_pci_bus_space_write_8(iot, ioh,
    780 		    VIRTIO_CONFIG1_QUEUE_DESC, addr);
    781 		virtio_pci_bus_space_write_8(iot, ioh,
    782 		    VIRTIO_CONFIG1_QUEUE_AVAIL, addr + vq->vq_availoffset);
    783 		virtio_pci_bus_space_write_8(iot, ioh,
    784 		    VIRTIO_CONFIG1_QUEUE_USED, addr + vq->vq_usedoffset);
    785 		bus_space_write_2(iot, ioh,
    786 		    VIRTIO_CONFIG1_QUEUE_ENABLE, 1);
    787 		vq->vq_notify_off = bus_space_read_2(iot, ioh,
    788 		    VIRTIO_CONFIG1_QUEUE_NOTIFY_OFF);
    789 	}
    790 
    791 	if (psc->sc_ihs_num > 1) {
    792 		int vec = VIRTIO_MSIX_QUEUE_VECTOR_INDEX;
    793 		if (psc->sc_intr_pervq)
    794 			vec += idx;
    795 		bus_space_write_2(iot, ioh,
    796 		    VIRTIO_CONFIG1_QUEUE_MSIX_VECTOR, vec);
    797 	}
    798 }
    799 
    800 static void
    801 virtio_pci_set_status_10(struct virtio_softc *sc, int status)
    802 {
    803 	struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc;
    804 	bus_space_tag_t	   iot = psc->sc_iot;
    805 	bus_space_handle_t ioh = psc->sc_ioh;
    806 	int old = 0;
    807 
    808 	if (status)
    809 		old = bus_space_read_1(iot, ioh, VIRTIO_CONFIG1_DEVICE_STATUS);
    810 	bus_space_write_1(iot, ioh, VIRTIO_CONFIG1_DEVICE_STATUS,
    811 	    status | old);
    812 }
    813 
    814 void
    815 virtio_pci_negotiate_features_10(struct virtio_softc *sc,
    816     uint64_t guest_features)
    817 {
    818 	struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc;
    819 	device_t self          =  sc->sc_dev;
    820 	bus_space_tag_t	   iot = psc->sc_iot;
    821 	bus_space_handle_t ioh = psc->sc_ioh;
    822 	uint64_t host, negotiated, device_status;
    823 
    824 	guest_features |= VIRTIO_F_VERSION_1;
    825 #ifdef __NEED_VIRTIO_F_ACCESS_PLATFORM
    826 	/* XXX This could use some work. */
    827 	guest_features |= VIRTIO_F_ACCESS_PLATFORM;
    828 #endif /* __NEED_VIRTIO_F_ACCESS_PLATFORM */
    829 	/* notify on empty is 0.9 only */
    830 	guest_features &= ~VIRTIO_F_NOTIFY_ON_EMPTY;
    831 	sc->sc_active_features = 0;
    832 
    833 	bus_space_write_4(iot, ioh, VIRTIO_CONFIG1_DEVICE_FEATURE_SELECT, 0);
    834 	host = bus_space_read_4(iot, ioh, VIRTIO_CONFIG1_DEVICE_FEATURE);
    835 	bus_space_write_4(iot, ioh, VIRTIO_CONFIG1_DEVICE_FEATURE_SELECT, 1);
    836 	host |= (uint64_t)bus_space_read_4(iot, ioh,
    837 	    VIRTIO_CONFIG1_DEVICE_FEATURE) << 32;
    838 
    839 	negotiated = host & guest_features;
    840 
    841 	bus_space_write_4(iot, ioh, VIRTIO_CONFIG1_DRIVER_FEATURE_SELECT, 0);
    842 	bus_space_write_4(iot, ioh, VIRTIO_CONFIG1_DRIVER_FEATURE,
    843 	    negotiated & 0xffffffff);
    844 	bus_space_write_4(iot, ioh, VIRTIO_CONFIG1_DRIVER_FEATURE_SELECT, 1);
    845 	bus_space_write_4(iot, ioh, VIRTIO_CONFIG1_DRIVER_FEATURE,
    846 	    negotiated >> 32);
    847 	virtio_pci_set_status_10(sc, VIRTIO_CONFIG_DEVICE_STATUS_FEATURES_OK);
    848 
    849 	device_status = bus_space_read_1(iot, ioh,
    850 	    VIRTIO_CONFIG1_DEVICE_STATUS);
    851 	if ((device_status & VIRTIO_CONFIG_DEVICE_STATUS_FEATURES_OK) == 0) {
    852 		aprint_error_dev(self, "feature negotiation failed\n");
    853 		bus_space_write_1(iot, ioh, VIRTIO_CONFIG1_DEVICE_STATUS,
    854 		    VIRTIO_CONFIG_DEVICE_STATUS_FAILED);
    855 		return;
    856 	}
    857 
    858 	if ((negotiated & VIRTIO_F_VERSION_1) == 0) {
    859 		aprint_error_dev(self, "host rejected version 1\n");
    860 		bus_space_write_1(iot, ioh, VIRTIO_CONFIG1_DEVICE_STATUS,
    861 		    VIRTIO_CONFIG_DEVICE_STATUS_FAILED);
    862 		return;
    863 	}
    864 
    865 	sc->sc_active_features = negotiated;
    866 	return;
    867 }
    868 
    869 /* -------------------------------------
    870  * Generic PCI interrupt code
    871  * -------------------------------------*/
    872 
    873 static int
    874 virtio_pci_setup_interrupts_10(struct virtio_softc *sc, int reinit)
    875 {
    876 	struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc;
    877 	bus_space_tag_t	   iot = psc->sc_iot;
    878 	bus_space_handle_t ioh = psc->sc_ioh;
    879 	int vector, ret, qid;
    880 
    881 	if (!virtio_pci_msix_enabled(psc))
    882 		return 0;
    883 
    884 	vector = VIRTIO_MSIX_CONFIG_VECTOR_INDEX;
    885 	bus_space_write_2(iot, ioh, VIRTIO_CONFIG1_CONFIG_MSIX_VECTOR, vector);
    886 	ret = bus_space_read_2(iot, ioh, VIRTIO_CONFIG1_CONFIG_MSIX_VECTOR);
    887 	if (ret != vector) {
    888 		VIRTIO_PCI_LOG(sc, reinit, "can't set config msix vector\n");
    889 		return -1;
    890 	}
    891 
    892 	for (qid = 0; qid < sc->sc_nvqs; qid++) {
    893 		vector = VIRTIO_MSIX_QUEUE_VECTOR_INDEX;
    894 
    895 		if (psc->sc_intr_pervq)
    896 			vector += qid;
    897 		bus_space_write_2(iot, ioh, VIRTIO_CONFIG1_QUEUE_SELECT, qid);
    898 		bus_space_write_2(iot, ioh, VIRTIO_CONFIG1_QUEUE_MSIX_VECTOR,
    899 		    vector);
    900 		ret = bus_space_read_2(iot, ioh,
    901 		    VIRTIO_CONFIG1_QUEUE_MSIX_VECTOR);
    902 		if (ret != vector) {
    903 			VIRTIO_PCI_LOG(sc, reinit, "can't set queue %d "
    904 			    "msix vector\n", qid);
    905 			return -1;
    906 		}
    907 	}
    908 
    909 	return 0;
    910 }
    911 
    912 static int
    913 virtio_pci_setup_interrupts_09(struct virtio_softc *sc, int reinit)
    914 {
    915 	struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc;
    916 	int offset, vector, ret, qid;
    917 
    918 	if (!virtio_pci_msix_enabled(psc))
    919 		return 0;
    920 
    921 	offset = VIRTIO_CONFIG_MSI_CONFIG_VECTOR;
    922 	vector = VIRTIO_MSIX_CONFIG_VECTOR_INDEX;
    923 
    924 	bus_space_write_2(psc->sc_iot, psc->sc_ioh, offset, vector);
    925 	ret = bus_space_read_2(psc->sc_iot, psc->sc_ioh, offset);
    926 	if (ret != vector) {
    927 		aprint_debug_dev(sc->sc_dev, "%s: expected=%d, actual=%d\n",
    928 		    __func__, vector, ret);
    929 		VIRTIO_PCI_LOG(sc, reinit,
    930 		    "can't set config msix vector\n");
    931 		return -1;
    932 	}
    933 
    934 	for (qid = 0; qid < sc->sc_nvqs; qid++) {
    935 		offset = VIRTIO_CONFIG_QUEUE_SELECT;
    936 		bus_space_write_2(psc->sc_iot, psc->sc_ioh, offset, qid);
    937 
    938 		offset = VIRTIO_CONFIG_MSI_QUEUE_VECTOR;
    939 		vector = VIRTIO_MSIX_QUEUE_VECTOR_INDEX;
    940 
    941 		if (psc->sc_intr_pervq)
    942 			vector += qid;
    943 
    944 		bus_space_write_2(psc->sc_iot, psc->sc_ioh, offset, vector);
    945 		ret = bus_space_read_2(psc->sc_iot, psc->sc_ioh, offset);
    946 		if (ret != vector) {
    947 			aprint_debug_dev(sc->sc_dev, "%s[qid=%d]:"
    948 			    " expected=%d, actual=%d\n",
    949 			    __func__, qid, vector, ret);
    950 			VIRTIO_PCI_LOG(sc, reinit, "can't set queue %d "
    951 			    "msix vector\n", qid);
    952 			return -1;
    953 		}
    954 	}
    955 
    956 	return 0;
    957 }
    958 
    959 static int
    960 virtio_pci_establish_msix_interrupts(struct virtio_softc *sc,
    961     const struct pci_attach_args *pa)
    962 {
    963 	struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc;
    964 	device_t self = sc->sc_dev;
    965 	pci_chipset_tag_t pc = pa->pa_pc;
    966 	struct virtqueue *vq;
    967 	char intrbuf[PCI_INTRSTR_LEN];
    968 	char intr_xname[INTRDEVNAMEBUF];
    969 	char const *intrstr;
    970 	int idx, qid, n;
    971 
    972 	idx = VIRTIO_MSIX_CONFIG_VECTOR_INDEX;
    973 	if (sc->sc_flags & VIRTIO_F_INTR_MPSAFE)
    974 		pci_intr_setattr(pc, &psc->sc_ihp[idx], PCI_INTR_MPSAFE, true);
    975 
    976 	snprintf(intr_xname, sizeof(intr_xname), "%s config",
    977 	    device_xname(sc->sc_dev));
    978 
    979 	psc->sc_ihs[idx] = pci_intr_establish_xname(pc, psc->sc_ihp[idx],
    980 	    sc->sc_ipl, virtio_pci_msix_config_intr, sc, intr_xname);
    981 	if (psc->sc_ihs[idx] == NULL) {
    982 		aprint_error_dev(self,
    983 		    "couldn't establish MSI-X for config\n");
    984 		goto error;
    985 	}
    986 
    987 	idx = VIRTIO_MSIX_QUEUE_VECTOR_INDEX;
    988 	if (psc->sc_intr_pervq) {
    989 		for (qid = 0; qid < sc->sc_nvqs; qid++) {
    990 			n = idx + qid;
    991 			vq = &sc->sc_vqs[qid];
    992 
    993 			snprintf(intr_xname, sizeof(intr_xname), "%s vq#%d",
    994 			    device_xname(sc->sc_dev), qid);
    995 
    996 			if (sc->sc_flags & VIRTIO_F_INTR_MPSAFE) {
    997 				pci_intr_setattr(pc, &psc->sc_ihp[n],
    998 				    PCI_INTR_MPSAFE, true);
    999 			}
   1000 
   1001 			psc->sc_ihs[n] = pci_intr_establish_xname(pc,
   1002 			    psc->sc_ihp[n], sc->sc_ipl,
   1003 			    vq->vq_intrhand, vq->vq_intrhand_arg, intr_xname);
   1004 			if (psc->sc_ihs[n] == NULL) {
   1005 				aprint_error_dev(self,
   1006 				    "couldn't establish MSI-X for a vq\n");
   1007 				goto error;
   1008 			}
   1009 		}
   1010 	} else {
   1011 		if (sc->sc_flags & VIRTIO_F_INTR_MPSAFE) {
   1012 			pci_intr_setattr(pc, &psc->sc_ihp[idx],
   1013 			    PCI_INTR_MPSAFE, true);
   1014 		}
   1015 
   1016 		snprintf(intr_xname, sizeof(intr_xname), "%s queues",
   1017 		    device_xname(sc->sc_dev));
   1018 		psc->sc_ihs[idx] = pci_intr_establish_xname(pc,
   1019 		    psc->sc_ihp[idx], sc->sc_ipl,
   1020 		    virtio_pci_msix_queue_intr, sc, intr_xname);
   1021 		if (psc->sc_ihs[idx] == NULL) {
   1022 			aprint_error_dev(self,
   1023 			    "couldn't establish MSI-X for queues\n");
   1024 			goto error;
   1025 		}
   1026 	}
   1027 
   1028 	idx = VIRTIO_MSIX_CONFIG_VECTOR_INDEX;
   1029 	intrstr = pci_intr_string(pc, psc->sc_ihp[idx], intrbuf,
   1030 	    sizeof(intrbuf));
   1031 	aprint_normal_dev(self, "config interrupting at %s\n", intrstr);
   1032 	idx = VIRTIO_MSIX_QUEUE_VECTOR_INDEX;
   1033 	if (psc->sc_intr_pervq) {
   1034 		kcpuset_t *affinity;
   1035 		int affinity_to, r;
   1036 
   1037 		kcpuset_create(&affinity, false);
   1038 
   1039 		for (qid = 0; qid < sc->sc_nvqs; qid++) {
   1040 			n = idx + qid;
   1041 			affinity_to = (qid / 2) % ncpu;
   1042 
   1043 			intrstr = pci_intr_string(pc, psc->sc_ihp[n],
   1044 			    intrbuf, sizeof(intrbuf));
   1045 
   1046 			kcpuset_zero(affinity);
   1047 			kcpuset_set(affinity, affinity_to);
   1048 			r = interrupt_distribute(psc->sc_ihs[n], affinity,
   1049 			    NULL);
   1050 			if (r == 0) {
   1051 				aprint_normal_dev(self,
   1052 				    "for vq #%d interrupting at %s"
   1053 				    " affinity to %u\n",
   1054 				    qid, intrstr, affinity_to);
   1055 			} else {
   1056 				aprint_normal_dev(self,
   1057 				    "for vq #%d interrupting at %s\n",
   1058 				    qid, intrstr);
   1059 			}
   1060 		}
   1061 
   1062 		kcpuset_destroy(affinity);
   1063 	} else {
   1064 		intrstr = pci_intr_string(pc, psc->sc_ihp[idx], intrbuf,
   1065 		    sizeof(intrbuf));
   1066 		aprint_normal_dev(self, "queues interrupting at %s\n",
   1067 		    intrstr);
   1068 	}
   1069 
   1070 	return 0;
   1071 
   1072 error:
   1073 	idx = VIRTIO_MSIX_CONFIG_VECTOR_INDEX;
   1074 	if (psc->sc_ihs[idx] != NULL)
   1075 		pci_intr_disestablish(psc->sc_pa.pa_pc, psc->sc_ihs[idx]);
   1076 	idx = VIRTIO_MSIX_QUEUE_VECTOR_INDEX;
   1077 	if (psc->sc_intr_pervq) {
   1078 		for (qid = 0; qid < sc->sc_nvqs; qid++) {
   1079 			n = idx + qid;
   1080 			if (psc->sc_ihs[n] == NULL)
   1081 				continue;
   1082 			pci_intr_disestablish(psc->sc_pa.pa_pc,
   1083 			    psc->sc_ihs[n]);
   1084 		}
   1085 
   1086 	} else {
   1087 		if (psc->sc_ihs[idx] != NULL) {
   1088 			pci_intr_disestablish(psc->sc_pa.pa_pc,
   1089 			    psc->sc_ihs[idx]);
   1090 		}
   1091 	}
   1092 
   1093 	return -1;
   1094 }
   1095 
   1096 static int
   1097 virtio_pci_establish_intx_interrupt(struct virtio_softc *sc,
   1098     const struct pci_attach_args *pa)
   1099 {
   1100 	struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc;
   1101 	device_t self = sc->sc_dev;
   1102 	pci_chipset_tag_t pc = pa->pa_pc;
   1103 	char intrbuf[PCI_INTRSTR_LEN];
   1104 	char const *intrstr;
   1105 
   1106 	if (sc->sc_flags & VIRTIO_F_INTR_MPSAFE)
   1107 		pci_intr_setattr(pc, &psc->sc_ihp[0], PCI_INTR_MPSAFE, true);
   1108 
   1109 	psc->sc_ihs[0] = pci_intr_establish_xname(pc, psc->sc_ihp[0],
   1110 	    sc->sc_ipl, virtio_pci_intr, sc, device_xname(sc->sc_dev));
   1111 	if (psc->sc_ihs[0] == NULL) {
   1112 		aprint_error_dev(self, "couldn't establish INTx\n");
   1113 		return -1;
   1114 	}
   1115 
   1116 	intrstr = pci_intr_string(pc, psc->sc_ihp[0], intrbuf,
   1117 	    sizeof(intrbuf));
   1118 	aprint_normal_dev(self, "interrupting at %s\n", intrstr);
   1119 
   1120 	return 0;
   1121 }
   1122 
   1123 static int
   1124 virtio_pci_alloc_interrupts(struct virtio_softc *sc)
   1125 {
   1126 	struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc;
   1127 	device_t self = sc->sc_dev;
   1128 	pci_chipset_tag_t pc = psc->sc_pa.pa_pc;
   1129 	pcitag_t tag = psc->sc_pa.pa_tag;
   1130 	int error;
   1131 	int nmsix;
   1132 	int off;
   1133 	int counts[PCI_INTR_TYPE_SIZE];
   1134 	pci_intr_type_t max_type;
   1135 	pcireg_t ctl;
   1136 
   1137 	nmsix = pci_msix_count(psc->sc_pa.pa_pc, psc->sc_pa.pa_tag);
   1138 	aprint_debug_dev(self, "pci_msix_count=%d\n", nmsix);
   1139 
   1140 	/* We need at least two: one for config and the other for queues */
   1141 	if ((sc->sc_flags & VIRTIO_F_INTR_MSIX) == 0 || nmsix < 2) {
   1142 		/* Try INTx only */
   1143 		max_type = PCI_INTR_TYPE_INTX;
   1144 		counts[PCI_INTR_TYPE_INTX] = 1;
   1145 	} else {
   1146 		/* Try MSI-X first and INTx second */
   1147 		if (ISSET(sc->sc_flags, VIRTIO_F_INTR_PERVQ) &&
   1148 		    sc->sc_nvqs + VIRTIO_MSIX_QUEUE_VECTOR_INDEX <= nmsix) {
   1149 			nmsix = sc->sc_nvqs + VIRTIO_MSIX_QUEUE_VECTOR_INDEX;
   1150 		} else {
   1151 			nmsix = 2;
   1152 		}
   1153 
   1154 		max_type = PCI_INTR_TYPE_MSIX;
   1155 		counts[PCI_INTR_TYPE_MSIX] = nmsix;
   1156 		counts[PCI_INTR_TYPE_MSI] = 0;
   1157 		counts[PCI_INTR_TYPE_INTX] = 1;
   1158 	}
   1159 
   1160 retry:
   1161 	error = pci_intr_alloc(&psc->sc_pa, &psc->sc_ihp, counts, max_type);
   1162 	if (error != 0) {
   1163 		aprint_error_dev(self, "couldn't map interrupt\n");
   1164 		return -1;
   1165 	}
   1166 
   1167 	if (pci_intr_type(pc, psc->sc_ihp[0]) == PCI_INTR_TYPE_MSIX) {
   1168 		psc->sc_intr_pervq = nmsix > 2 ? true : false;
   1169 		psc->sc_ihs = kmem_zalloc(sizeof(*psc->sc_ihs) * nmsix,
   1170 		    KM_SLEEP);
   1171 
   1172 		error = virtio_pci_establish_msix_interrupts(sc, &psc->sc_pa);
   1173 		if (error != 0) {
   1174 			kmem_free(psc->sc_ihs, sizeof(*psc->sc_ihs) * nmsix);
   1175 			pci_intr_release(pc, psc->sc_ihp, nmsix);
   1176 
   1177 			/* Retry INTx */
   1178 			max_type = PCI_INTR_TYPE_INTX;
   1179 			counts[PCI_INTR_TYPE_INTX] = 1;
   1180 			goto retry;
   1181 		}
   1182 
   1183 		psc->sc_ihs_num = nmsix;
   1184 		psc->sc_devcfg_offset = VIRTIO_CONFIG_DEVICE_CONFIG_MSI;
   1185 		virtio_pci_adjust_config_region(psc);
   1186 	} else if (pci_intr_type(pc, psc->sc_ihp[0]) == PCI_INTR_TYPE_INTX) {
   1187 		psc->sc_intr_pervq = false;
   1188 		psc->sc_ihs = kmem_zalloc(sizeof(*psc->sc_ihs) * 1,
   1189 		    KM_SLEEP);
   1190 
   1191 		error = virtio_pci_establish_intx_interrupt(sc, &psc->sc_pa);
   1192 		if (error != 0) {
   1193 			kmem_free(psc->sc_ihs, sizeof(*psc->sc_ihs) * 1);
   1194 			pci_intr_release(pc, psc->sc_ihp, 1);
   1195 			return -1;
   1196 		}
   1197 
   1198 		psc->sc_ihs_num = 1;
   1199 		psc->sc_devcfg_offset = VIRTIO_CONFIG_DEVICE_CONFIG_NOMSI;
   1200 		virtio_pci_adjust_config_region(psc);
   1201 
   1202 		error = pci_get_capability(pc, tag, PCI_CAP_MSIX, &off, NULL);
   1203 		if (error != 0) {
   1204 			ctl = pci_conf_read(pc, tag, off + PCI_MSIX_CTL);
   1205 			ctl &= ~PCI_MSIX_CTL_ENABLE;
   1206 			pci_conf_write(pc, tag, off + PCI_MSIX_CTL, ctl);
   1207 		}
   1208 	}
   1209 
   1210 	if (!psc->sc_intr_pervq)
   1211 		CLR(sc->sc_flags, VIRTIO_F_INTR_PERVQ);
   1212 	return 0;
   1213 }
   1214 
   1215 static void
   1216 virtio_pci_free_interrupts(struct virtio_softc *sc)
   1217 {
   1218 	struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc;
   1219 
   1220 	for (int i = 0; i < psc->sc_ihs_num; i++) {
   1221 		if (psc->sc_ihs[i] == NULL)
   1222 			continue;
   1223 		pci_intr_disestablish(psc->sc_pa.pa_pc, psc->sc_ihs[i]);
   1224 		psc->sc_ihs[i] = NULL;
   1225 	}
   1226 
   1227 	if (psc->sc_ihs_num > 0) {
   1228 		pci_intr_release(psc->sc_pa.pa_pc, psc->sc_ihp,
   1229 		    psc->sc_ihs_num);
   1230 	}
   1231 
   1232 	if (psc->sc_ihs != NULL) {
   1233 		kmem_free(psc->sc_ihs, sizeof(*psc->sc_ihs) * psc->sc_ihs_num);
   1234 		psc->sc_ihs = NULL;
   1235 	}
   1236 	psc->sc_ihs_num = 0;
   1237 }
   1238 
   1239 static bool
   1240 virtio_pci_msix_enabled(struct virtio_pci_softc *psc)
   1241 {
   1242 	pci_chipset_tag_t pc = psc->sc_pa.pa_pc;
   1243 
   1244 	if (pci_intr_type(pc, psc->sc_ihp[0]) == PCI_INTR_TYPE_MSIX)
   1245 		return true;
   1246 
   1247 	return false;
   1248 }
   1249 
   1250 /*
   1251  * Interrupt handler.
   1252  */
   1253 static int
   1254 virtio_pci_intr(void *arg)
   1255 {
   1256 	struct virtio_softc *sc = arg;
   1257 	struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc;
   1258 	int isr, r = 0;
   1259 
   1260 	/* check and ack the interrupt */
   1261 	isr = bus_space_read_1(psc->sc_isr_iot, psc->sc_isr_ioh, 0);
   1262 	if (isr == 0)
   1263 		return 0;
   1264 	if ((isr & VIRTIO_CONFIG_ISR_CONFIG_CHANGE) &&
   1265 	    (sc->sc_config_change != NULL))
   1266 		r = (sc->sc_config_change)(sc);
   1267 	if (sc->sc_intrhand != NULL) {
   1268 		if (sc->sc_soft_ih != NULL)
   1269 			softint_schedule(sc->sc_soft_ih);
   1270 		else
   1271 			r |= (sc->sc_intrhand)(sc);
   1272 	}
   1273 
   1274 	return r;
   1275 }
   1276 
   1277 static int
   1278 virtio_pci_msix_queue_intr(void *arg)
   1279 {
   1280 	struct virtio_softc *sc = arg;
   1281 	int r = 0;
   1282 
   1283 	if (sc->sc_intrhand != NULL) {
   1284 		if (sc->sc_soft_ih != NULL)
   1285 			softint_schedule(sc->sc_soft_ih);
   1286 		else
   1287 			r |= (sc->sc_intrhand)(sc);
   1288 	}
   1289 
   1290 	return r;
   1291 }
   1292 
   1293 static int
   1294 virtio_pci_msix_config_intr(void *arg)
   1295 {
   1296 	struct virtio_softc *sc = arg;
   1297 	int r = 0;
   1298 
   1299 	if (sc->sc_config_change != NULL)
   1300 		r = (sc->sc_config_change)(sc);
   1301 	return r;
   1302 }
   1303 
   1304 MODULE(MODULE_CLASS_DRIVER, virtio_pci, "pci,virtio");
   1305 
   1306 #ifdef _MODULE
   1307 #include "ioconf.c"
   1308 #endif
   1309 
   1310 static int
   1311 virtio_pci_modcmd(modcmd_t cmd, void *opaque)
   1312 {
   1313 	int error = 0;
   1314 
   1315 #ifdef _MODULE
   1316 	switch (cmd) {
   1317 	case MODULE_CMD_INIT:
   1318 		error = config_init_component(cfdriver_ioconf_virtio_pci,
   1319 		    cfattach_ioconf_virtio_pci, cfdata_ioconf_virtio_pci);
   1320 		break;
   1321 	case MODULE_CMD_FINI:
   1322 		error = config_fini_component(cfdriver_ioconf_virtio_pci,
   1323 		    cfattach_ioconf_virtio_pci, cfdata_ioconf_virtio_pci);
   1324 		break;
   1325 	default:
   1326 		error = ENOTTY;
   1327 		break;
   1328 	}
   1329 #endif
   1330 
   1331 	return error;
   1332 }
   1333