Home | History | Annotate | Line # | Download | only in pci
virtio_pci.c revision 1.33
      1 /* $NetBSD: virtio_pci.c,v 1.33 2021/10/28 01:36:43 yamaguchi Exp $ */
      2 
      3 /*
      4  * Copyright (c) 2020 The NetBSD Foundation, Inc.
      5  * Copyright (c) 2012 Stefan Fritsch.
      6  * Copyright (c) 2010 Minoura Makoto.
      7  * All rights reserved.
      8  *
      9  * Redistribution and use in source and binary forms, with or without
     10  * modification, are permitted provided that the following conditions
     11  * are met:
     12  * 1. Redistributions of source code must retain the above copyright
     13  *    notice, this list of conditions and the following disclaimer.
     14  * 2. Redistributions in binary form must reproduce the above copyright
     15  *    notice, this list of conditions and the following disclaimer in the
     16  *    documentation and/or other materials provided with the distribution.
     17  *
     18  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
     19  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
     20  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
     21  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
     22  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
     23  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
     24  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
     25  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
     26  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
     27  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
     28  */
     29 
     30 #include <sys/cdefs.h>
     31 __KERNEL_RCSID(0, "$NetBSD: virtio_pci.c,v 1.33 2021/10/28 01:36:43 yamaguchi Exp $");
     32 
     33 #include <sys/param.h>
     34 #include <sys/systm.h>
     35 #include <sys/kmem.h>
     36 #include <sys/module.h>
     37 #include <sys/endian.h>
     38 #include <sys/interrupt.h>
     39 #include <sys/syslog.h>
     40 
     41 #include <sys/device.h>
     42 
     43 #include <dev/pci/pcidevs.h>
     44 #include <dev/pci/pcireg.h>
     45 #include <dev/pci/pcivar.h>
     46 
     47 #include <dev/pci/virtioreg.h> /* XXX: move to non-pci */
     48 #include <dev/pci/virtio_pcireg.h>
     49 
     50 #define VIRTIO_PRIVATE
     51 #include <dev/pci/virtiovar.h> /* XXX: move to non-pci */
     52 
     53 
     54 #define VIRTIO_PCI_LOG(_sc, _use_log, _fmt, _args...)	\
     55 do {							\
     56 	if ((_use_log)) {				\
     57 		log(LOG_DEBUG, "%s: " _fmt,		\
     58 		    device_xname((_sc)->sc_dev),	\
     59 		    ##_args);				\
     60 	} else {					\
     61 		aprint_error_dev((_sc)->sc_dev,		\
     62 		    _fmt, ##_args);			\
     63 	}						\
     64 } while(0)
     65 
     66 static int	virtio_pci_match(device_t, cfdata_t, void *);
     67 static void	virtio_pci_attach(device_t, device_t, void *);
     68 static int	virtio_pci_rescan(device_t, const char *, const int *);
     69 static int	virtio_pci_detach(device_t, int);
     70 
     71 
     72 #define NMAPREG		((PCI_MAPREG_END - PCI_MAPREG_START) / \
     73 				sizeof(pcireg_t))
     74 struct virtio_pci_softc {
     75 	struct virtio_softc	sc_sc;
     76 
     77 	/* IO space */
     78 	bus_space_tag_t		sc_iot;
     79 	bus_space_handle_t	sc_ioh;
     80 	bus_size_t		sc_iosize;
     81 	bus_size_t		sc_mapped_iosize;
     82 
     83 	/* BARs */
     84 	bus_space_tag_t		sc_bars_iot[NMAPREG];
     85 	bus_space_handle_t	sc_bars_ioh[NMAPREG];
     86 	bus_size_t		sc_bars_iosize[NMAPREG];
     87 
     88 	/* notify space */
     89 	bus_space_tag_t		sc_notify_iot;
     90 	bus_space_handle_t	sc_notify_ioh;
     91 	bus_size_t		sc_notify_iosize;
     92 	uint32_t		sc_notify_off_multiplier;
     93 
     94 	/* isr space */
     95 	bus_space_tag_t		sc_isr_iot;
     96 	bus_space_handle_t	sc_isr_ioh;
     97 	bus_size_t		sc_isr_iosize;
     98 
     99 	/* generic */
    100 	struct pci_attach_args	sc_pa;
    101 	pci_intr_handle_t	*sc_ihp;
    102 	void			**sc_ihs;
    103 	int			sc_ihs_num;
    104 	int			sc_devcfg_offset;	/* for 0.9 */
    105 };
    106 
    107 static int	virtio_pci_attach_09(device_t, void *);
    108 static void	virtio_pci_kick_09(struct virtio_softc *, uint16_t);
    109 static uint16_t	virtio_pci_read_queue_size_09(struct virtio_softc *, uint16_t);
    110 static void	virtio_pci_setup_queue_09(struct virtio_softc *, uint16_t, uint64_t);
    111 static void	virtio_pci_set_status_09(struct virtio_softc *, int);
    112 static void	virtio_pci_negotiate_features_09(struct virtio_softc *, uint64_t);
    113 
    114 static int	virtio_pci_attach_10(device_t, void *);
    115 static void	virtio_pci_kick_10(struct virtio_softc *, uint16_t);
    116 static uint16_t	virtio_pci_read_queue_size_10(struct virtio_softc *, uint16_t);
    117 static void	virtio_pci_setup_queue_10(struct virtio_softc *, uint16_t, uint64_t);
    118 static void	virtio_pci_set_status_10(struct virtio_softc *, int);
    119 static void	virtio_pci_negotiate_features_10(struct virtio_softc *, uint64_t);
    120 static int	virtio_pci_find_cap(struct virtio_pci_softc *psc, int cfg_type, void *buf, int buflen);
    121 
    122 static int	virtio_pci_alloc_interrupts(struct virtio_softc *);
    123 static void	virtio_pci_free_interrupts(struct virtio_softc *);
    124 static int	virtio_pci_adjust_config_region(struct virtio_pci_softc *psc);
    125 static int	virtio_pci_intr(void *arg);
    126 static int	virtio_pci_msix_queue_intr(void *);
    127 static int	virtio_pci_msix_config_intr(void *);
    128 static int	virtio_pci_setup_interrupts_09(struct virtio_softc *, int);
    129 static int	virtio_pci_setup_interrupts_10(struct virtio_softc *, int);
    130 static int	virtio_pci_establish_msix_interrupts(struct virtio_softc *,
    131 		    struct pci_attach_args *);
    132 static int	virtio_pci_establish_intx_interrupt(struct virtio_softc *,
    133 		    struct pci_attach_args *);
    134 static bool	virtio_pci_msix_enabled(struct virtio_pci_softc *);
    135 
    136 #define VIRTIO_MSIX_CONFIG_VECTOR_INDEX	0
    137 #define VIRTIO_MSIX_QUEUE_VECTOR_INDEX	1
    138 
    139 /*
    140  * When using PCI attached virtio on aarch64-eb under Qemu, the IO space
    141  * suddenly read BIG_ENDIAN where it should stay LITTLE_ENDIAN. The data read
    142  * 1 byte at a time seem OK but reading bigger lengths result in swapped
    143  * endian. This is most notable on reading 8 byters since we can't use
    144  * bus_space_{read,write}_8().
    145  */
    146 
    147 #if defined(__aarch64__) && BYTE_ORDER == BIG_ENDIAN
    148 #	define READ_ENDIAN_09	BIG_ENDIAN	/* should be LITTLE_ENDIAN */
    149 #	define READ_ENDIAN_10	BIG_ENDIAN
    150 #	define STRUCT_ENDIAN_09	BIG_ENDIAN
    151 #	define STRUCT_ENDIAN_10	LITTLE_ENDIAN
    152 #elif BYTE_ORDER == BIG_ENDIAN
    153 #	define READ_ENDIAN_09	LITTLE_ENDIAN
    154 #	define READ_ENDIAN_10	BIG_ENDIAN
    155 #	define STRUCT_ENDIAN_09	BIG_ENDIAN
    156 #	define STRUCT_ENDIAN_10	LITTLE_ENDIAN
    157 #else /* little endian */
    158 #	define READ_ENDIAN_09	LITTLE_ENDIAN
    159 #	define READ_ENDIAN_10	LITTLE_ENDIAN
    160 #	define STRUCT_ENDIAN_09	LITTLE_ENDIAN
    161 #	define STRUCT_ENDIAN_10	LITTLE_ENDIAN
    162 #endif
    163 
    164 
    165 CFATTACH_DECL3_NEW(virtio_pci, sizeof(struct virtio_pci_softc),
    166     virtio_pci_match, virtio_pci_attach, virtio_pci_detach, NULL,
    167     virtio_pci_rescan, NULL, DVF_DETACH_SHUTDOWN);
    168 
    169 static const struct virtio_ops virtio_pci_ops_09 = {
    170 	.kick = virtio_pci_kick_09,
    171 	.read_queue_size = virtio_pci_read_queue_size_09,
    172 	.setup_queue = virtio_pci_setup_queue_09,
    173 	.set_status = virtio_pci_set_status_09,
    174 	.neg_features = virtio_pci_negotiate_features_09,
    175 	.alloc_interrupts = virtio_pci_alloc_interrupts,
    176 	.free_interrupts = virtio_pci_free_interrupts,
    177 	.setup_interrupts = virtio_pci_setup_interrupts_09,
    178 };
    179 
    180 static const struct virtio_ops virtio_pci_ops_10 = {
    181 	.kick = virtio_pci_kick_10,
    182 	.read_queue_size = virtio_pci_read_queue_size_10,
    183 	.setup_queue = virtio_pci_setup_queue_10,
    184 	.set_status = virtio_pci_set_status_10,
    185 	.neg_features = virtio_pci_negotiate_features_10,
    186 	.alloc_interrupts = virtio_pci_alloc_interrupts,
    187 	.free_interrupts = virtio_pci_free_interrupts,
    188 	.setup_interrupts = virtio_pci_setup_interrupts_10,
    189 };
    190 
    191 static int
    192 virtio_pci_match(device_t parent, cfdata_t match, void *aux)
    193 {
    194 	struct pci_attach_args *pa;
    195 
    196 	pa = (struct pci_attach_args *)aux;
    197 	switch (PCI_VENDOR(pa->pa_id)) {
    198 	case PCI_VENDOR_QUMRANET:
    199 		if (((PCI_PRODUCT_QUMRANET_VIRTIO_1000 <=
    200 		      PCI_PRODUCT(pa->pa_id)) &&
    201 		     (PCI_PRODUCT(pa->pa_id) <=
    202 		      PCI_PRODUCT_QUMRANET_VIRTIO_103F)) &&
    203 	              PCI_REVISION(pa->pa_class) == 0)
    204 			return 1;
    205 		if (((PCI_PRODUCT_QUMRANET_VIRTIO_1040 <=
    206 		      PCI_PRODUCT(pa->pa_id)) &&
    207 		     (PCI_PRODUCT(pa->pa_id) <=
    208 		      PCI_PRODUCT_QUMRANET_VIRTIO_107F)) &&
    209 		      PCI_REVISION(pa->pa_class) == 1)
    210 			return 1;
    211 		break;
    212 	}
    213 
    214 	return 0;
    215 }
    216 
    217 static void
    218 virtio_pci_attach(device_t parent, device_t self, void *aux)
    219 {
    220 	struct virtio_pci_softc * const psc = device_private(self);
    221 	struct virtio_softc * const sc = &psc->sc_sc;
    222 	struct pci_attach_args *pa = (struct pci_attach_args *)aux;
    223 	pci_chipset_tag_t pc = pa->pa_pc;
    224 	pcitag_t tag = pa->pa_tag;
    225 	int revision;
    226 	int ret;
    227 	pcireg_t id;
    228 	pcireg_t csr;
    229 
    230 	revision = PCI_REVISION(pa->pa_class);
    231 	switch (revision) {
    232 	case 0:
    233 		/* subsystem ID shows what I am */
    234 		id = PCI_SUBSYS_ID(pci_conf_read(pc, tag, PCI_SUBSYS_ID_REG));
    235 		break;
    236 	case 1:
    237 		/* pci product number shows what I am */
    238 		id = PCI_PRODUCT(pa->pa_id) - PCI_PRODUCT_QUMRANET_VIRTIO_1040;
    239 		break;
    240 	default:
    241 		aprint_normal(": unknown revision 0x%02x; giving up\n",
    242 			      revision);
    243 		return;
    244 	}
    245 
    246 	aprint_normal("\n");
    247 	aprint_naive("\n");
    248 	virtio_print_device_type(self, id, revision);
    249 
    250 	csr = pci_conf_read(pc, tag, PCI_COMMAND_STATUS_REG);
    251 	csr |= PCI_COMMAND_MASTER_ENABLE | PCI_COMMAND_IO_ENABLE;
    252 	pci_conf_write(pc, tag, PCI_COMMAND_STATUS_REG, csr);
    253 
    254 	sc->sc_dev = self;
    255 	psc->sc_pa = *pa;
    256 	psc->sc_iot = pa->pa_iot;
    257 
    258 	sc->sc_dmat = pa->pa_dmat;
    259 	if (pci_dma64_available(pa))
    260 		sc->sc_dmat = pa->pa_dmat64;
    261 
    262 	/* attach is dependent on revision */
    263 	ret = 0;
    264 	if (revision == 1) {
    265 		/* try to attach 1.0 */
    266 		ret = virtio_pci_attach_10(self, aux);
    267 	}
    268 	if (ret == 0 && revision == 0) {
    269 		/* revision 0 means 0.9 only or both 0.9 and 1.0 */
    270 		ret = virtio_pci_attach_09(self, aux);
    271 	}
    272 	if (ret) {
    273 		aprint_error_dev(self, "cannot attach (%d)\n", ret);
    274 		return;
    275 	}
    276 	KASSERT(sc->sc_ops);
    277 
    278 	/* preset config region */
    279 	psc->sc_devcfg_offset = VIRTIO_CONFIG_DEVICE_CONFIG_NOMSI;
    280 	if (virtio_pci_adjust_config_region(psc))
    281 		return;
    282 
    283 	/* generic */
    284 	virtio_device_reset(sc);
    285 	virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_ACK);
    286 	virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_DRIVER);
    287 
    288 	sc->sc_childdevid = id;
    289 	sc->sc_child = NULL;
    290 	virtio_pci_rescan(self, NULL, NULL);
    291 	return;
    292 }
    293 
    294 /* ARGSUSED */
    295 static int
    296 virtio_pci_rescan(device_t self, const char *ifattr, const int *locs)
    297 {
    298 	struct virtio_pci_softc * const psc = device_private(self);
    299 	struct virtio_softc * const sc = &psc->sc_sc;
    300 	struct virtio_attach_args va;
    301 
    302 	if (sc->sc_child)	/* Child already attached? */
    303 		return 0;
    304 
    305 	memset(&va, 0, sizeof(va));
    306 	va.sc_childdevid = sc->sc_childdevid;
    307 
    308 	config_found(self, &va, NULL, CFARGS_NONE);
    309 
    310 	if (virtio_attach_failed(sc))
    311 		return 0;
    312 
    313 	return 0;
    314 }
    315 
    316 
    317 static int
    318 virtio_pci_detach(device_t self, int flags)
    319 {
    320 	struct virtio_pci_softc * const psc = device_private(self);
    321 	struct virtio_softc * const sc = &psc->sc_sc;
    322 	int r;
    323 
    324 	if (sc->sc_child != NULL) {
    325 		r = config_detach(sc->sc_child, flags);
    326 		if (r)
    327 			return r;
    328 	}
    329 
    330 	/* Check that child detached properly */
    331 	KASSERT(sc->sc_child == NULL);
    332 	KASSERT(sc->sc_vqs == NULL);
    333 	KASSERT(psc->sc_ihs_num == 0);
    334 
    335 	if (psc->sc_iosize)
    336 		bus_space_unmap(psc->sc_iot, psc->sc_ioh,
    337 			psc->sc_mapped_iosize);
    338 	psc->sc_iosize = 0;
    339 
    340 	return 0;
    341 }
    342 
    343 
    344 static int
    345 virtio_pci_attach_09(device_t self, void *aux)
    346 	//struct virtio_pci_softc *psc, struct pci_attach_args *pa)
    347 {
    348 	struct virtio_pci_softc * const psc = device_private(self);
    349 	struct pci_attach_args *pa = (struct pci_attach_args *)aux;
    350 	struct virtio_softc * const sc = &psc->sc_sc;
    351 //	pci_chipset_tag_t pc = pa->pa_pc;
    352 //	pcitag_t tag = pa->pa_tag;
    353 
    354 	/* complete IO region */
    355 	if (pci_mapreg_map(pa, PCI_MAPREG_START, PCI_MAPREG_TYPE_IO, 0,
    356 			   &psc->sc_iot, &psc->sc_ioh, NULL, &psc->sc_iosize)) {
    357 		aprint_error_dev(self, "can't map i/o space\n");
    358 		return EIO;
    359 	}
    360 	psc->sc_mapped_iosize = psc->sc_iosize;
    361 
    362 	/* queue space */
    363 	if (bus_space_subregion(psc->sc_iot, psc->sc_ioh,
    364 			VIRTIO_CONFIG_QUEUE_NOTIFY, 2, &psc->sc_notify_ioh)) {
    365 		aprint_error_dev(self, "can't map notify i/o space\n");
    366 		return EIO;
    367 	}
    368 	psc->sc_notify_iosize = 2;
    369 	psc->sc_notify_iot = psc->sc_iot;
    370 
    371 	/* ISR space */
    372 	if (bus_space_subregion(psc->sc_iot, psc->sc_ioh,
    373 			VIRTIO_CONFIG_ISR_STATUS, 1, &psc->sc_isr_ioh)) {
    374 		aprint_error_dev(self, "can't map isr i/o space\n");
    375 		return EIO;
    376 	}
    377 	psc->sc_isr_iosize = 1;
    378 	psc->sc_isr_iot = psc->sc_iot;
    379 
    380 	/* set our version 0.9 ops */
    381 	sc->sc_ops = &virtio_pci_ops_09;
    382 	sc->sc_bus_endian    = READ_ENDIAN_09;
    383 	sc->sc_struct_endian = STRUCT_ENDIAN_09;
    384 	return 0;
    385 }
    386 
    387 
    388 static int
    389 virtio_pci_attach_10(device_t self, void *aux)
    390 {
    391 	struct virtio_pci_softc * const psc = device_private(self);
    392 	struct pci_attach_args *pa = (struct pci_attach_args *)aux;
    393 	struct virtio_softc * const sc = &psc->sc_sc;
    394 	pci_chipset_tag_t pc = pa->pa_pc;
    395 	pcitag_t tag = pa->pa_tag;
    396 
    397 	struct virtio_pci_cap common, isr, device;
    398 	struct virtio_pci_notify_cap notify;
    399 	int have_device_cfg = 0;
    400 	bus_size_t bars[NMAPREG] = { 0 };
    401 	int bars_idx[NMAPREG] = { 0 };
    402 	struct virtio_pci_cap *caps[] = { &common, &isr, &device, &notify.cap };
    403 	int i, j, ret = 0;
    404 
    405 	if (virtio_pci_find_cap(psc, VIRTIO_PCI_CAP_COMMON_CFG,
    406 			&common, sizeof(common)))
    407 		return ENODEV;
    408 	if (virtio_pci_find_cap(psc, VIRTIO_PCI_CAP_NOTIFY_CFG,
    409 			&notify, sizeof(notify)))
    410 		return ENODEV;
    411 	if (virtio_pci_find_cap(psc, VIRTIO_PCI_CAP_ISR_CFG,
    412 			&isr, sizeof(isr)))
    413 		return ENODEV;
    414 	if (virtio_pci_find_cap(psc, VIRTIO_PCI_CAP_DEVICE_CFG,
    415 			&device, sizeof(device)))
    416 		memset(&device, 0, sizeof(device));
    417 	else
    418 		have_device_cfg = 1;
    419 
    420 	/* Figure out which bars we need to map */
    421 	for (i = 0; i < __arraycount(caps); i++) {
    422 		int bar = caps[i]->bar;
    423 		bus_size_t len = caps[i]->offset + caps[i]->length;
    424 		if (caps[i]->length == 0)
    425 			continue;
    426 		if (bars[bar] < len)
    427 			bars[bar] = len;
    428 	}
    429 
    430 	for (i = j = 0; i < __arraycount(bars); i++) {
    431 		int reg;
    432 		pcireg_t type;
    433 		if (bars[i] == 0)
    434 			continue;
    435 		reg = PCI_MAPREG_START + i * 4;
    436 		type = pci_mapreg_type(pc, tag, reg);
    437 		if (pci_mapreg_map(pa, reg, type, 0,
    438 				&psc->sc_bars_iot[j], &psc->sc_bars_ioh[j],
    439 				NULL, &psc->sc_bars_iosize[j])) {
    440 			aprint_error_dev(self, "can't map bar %u \n", i);
    441 			ret = EIO;
    442 			goto err;
    443 		}
    444 		aprint_debug_dev(self,
    445 		    "bar[%d]: iot %p, size 0x%" PRIxBUSSIZE "\n",
    446 		    j, psc->sc_bars_iot[j], psc->sc_bars_iosize[j]);
    447 		bars_idx[i] = j;
    448 		j++;
    449 	}
    450 
    451 	i = bars_idx[notify.cap.bar];
    452 	if (bus_space_subregion(psc->sc_bars_iot[i], psc->sc_bars_ioh[i],
    453 			notify.cap.offset, notify.cap.length,
    454 			&psc->sc_notify_ioh)) {
    455 		aprint_error_dev(self, "can't map notify i/o space\n");
    456 		ret = EIO;
    457 		goto err;
    458 	}
    459 	psc->sc_notify_iosize = notify.cap.length;
    460 	psc->sc_notify_iot = psc->sc_bars_iot[i];
    461 	psc->sc_notify_off_multiplier = le32toh(notify.notify_off_multiplier);
    462 
    463 	if (have_device_cfg) {
    464 		i = bars_idx[device.bar];
    465 		if (bus_space_subregion(psc->sc_bars_iot[i], psc->sc_bars_ioh[i],
    466 				device.offset, device.length,
    467 				&sc->sc_devcfg_ioh)) {
    468 			aprint_error_dev(self, "can't map devcfg i/o space\n");
    469 			ret = EIO;
    470 			goto err;
    471 		}
    472 		aprint_debug_dev(self,
    473 			"device.offset = 0x%x, device.length = 0x%x\n",
    474 			device.offset, device.length);
    475 		sc->sc_devcfg_iosize = device.length;
    476 		sc->sc_devcfg_iot = psc->sc_bars_iot[i];
    477 	}
    478 
    479 	i = bars_idx[isr.bar];
    480 	if (bus_space_subregion(psc->sc_bars_iot[i], psc->sc_bars_ioh[i],
    481 			isr.offset, isr.length, &psc->sc_isr_ioh)) {
    482 		aprint_error_dev(self, "can't map isr i/o space\n");
    483 		ret = EIO;
    484 		goto err;
    485 	}
    486 	psc->sc_isr_iosize = isr.length;
    487 	psc->sc_isr_iot = psc->sc_bars_iot[i];
    488 
    489 	i = bars_idx[common.bar];
    490 	if (bus_space_subregion(psc->sc_bars_iot[i], psc->sc_bars_ioh[i],
    491 			common.offset, common.length, &psc->sc_ioh)) {
    492 		aprint_error_dev(self, "can't map common i/o space\n");
    493 		ret = EIO;
    494 		goto err;
    495 	}
    496 	psc->sc_iosize = common.length;
    497 	psc->sc_iot = psc->sc_bars_iot[i];
    498 	psc->sc_mapped_iosize = psc->sc_bars_iosize[i];
    499 
    500 	psc->sc_sc.sc_version_1 = 1;
    501 
    502 	/* set our version 1.0 ops */
    503 	sc->sc_ops = &virtio_pci_ops_10;
    504 	sc->sc_bus_endian    = READ_ENDIAN_10;
    505 	sc->sc_struct_endian = STRUCT_ENDIAN_10;
    506 	return 0;
    507 
    508 err:
    509 	/* undo our pci_mapreg_map()s */
    510 	for (i = 0; i < __arraycount(bars); i++) {
    511 		if (psc->sc_bars_iosize[i] == 0)
    512 			continue;
    513 		bus_space_unmap(psc->sc_bars_iot[i], psc->sc_bars_ioh[i],
    514 				psc->sc_bars_iosize[i]);
    515 	}
    516 	return ret;
    517 }
    518 
    519 /* v1.0 attach helper */
    520 static int
    521 virtio_pci_find_cap(struct virtio_pci_softc *psc, int cfg_type, void *buf, int buflen)
    522 {
    523 	device_t self = psc->sc_sc.sc_dev;
    524 	pci_chipset_tag_t pc = psc->sc_pa.pa_pc;
    525 	pcitag_t tag = psc->sc_pa.pa_tag;
    526 	unsigned int offset, i, len;
    527 	union {
    528 		pcireg_t reg[8];
    529 		struct virtio_pci_cap vcap;
    530 	} *v = buf;
    531 
    532 	if (buflen < sizeof(struct virtio_pci_cap))
    533 		return ERANGE;
    534 
    535 	if (!pci_get_capability(pc, tag, PCI_CAP_VENDSPEC, &offset, &v->reg[0]))
    536 		return ENOENT;
    537 
    538 	do {
    539 		for (i = 0; i < 4; i++)
    540 			v->reg[i] =
    541 				le32toh(pci_conf_read(pc, tag, offset + i * 4));
    542 		if (v->vcap.cfg_type == cfg_type)
    543 			break;
    544 		offset = v->vcap.cap_next;
    545 	} while (offset != 0);
    546 
    547 	if (offset == 0)
    548 		return ENOENT;
    549 
    550 	if (v->vcap.cap_len > sizeof(struct virtio_pci_cap)) {
    551 		len = roundup(v->vcap.cap_len, sizeof(pcireg_t));
    552 		if (len > buflen) {
    553 			aprint_error_dev(self, "%s cap too large\n", __func__);
    554 			return ERANGE;
    555 		}
    556 		for (i = 4; i < len / sizeof(pcireg_t);  i++)
    557 			v->reg[i] =
    558 				le32toh(pci_conf_read(pc, tag, offset + i * 4));
    559 	}
    560 
    561 	/* endian fixup */
    562 	v->vcap.offset = le32toh(v->vcap.offset);
    563 	v->vcap.length = le32toh(v->vcap.length);
    564 	return 0;
    565 }
    566 
    567 
    568 /* -------------------------------------
    569  * Version 0.9 support
    570  * -------------------------------------*/
    571 
    572 static void
    573 virtio_pci_kick_09(struct virtio_softc *sc, uint16_t idx)
    574 {
    575 	struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc;
    576 
    577 	bus_space_write_2(psc->sc_notify_iot, psc->sc_notify_ioh, 0, idx);
    578 }
    579 
    580 /* only applicable for v 0.9 but also called for 1.0 */
    581 static int
    582 virtio_pci_adjust_config_region(struct virtio_pci_softc *psc)
    583 {
    584 	struct virtio_softc * const sc = (struct virtio_softc *) psc;
    585 	device_t self = psc->sc_sc.sc_dev;
    586 
    587 	if (psc->sc_sc.sc_version_1)
    588 		return 0;
    589 
    590 	sc->sc_devcfg_iosize = psc->sc_iosize - psc->sc_devcfg_offset;
    591 	sc->sc_devcfg_iot = psc->sc_iot;
    592 	if (bus_space_subregion(psc->sc_iot, psc->sc_ioh,
    593 			psc->sc_devcfg_offset, sc->sc_devcfg_iosize,
    594 			&sc->sc_devcfg_ioh)) {
    595 		aprint_error_dev(self, "can't map config i/o space\n");
    596 		return EIO;
    597 	}
    598 
    599 	return 0;
    600 }
    601 
    602 static uint16_t
    603 virtio_pci_read_queue_size_09(struct virtio_softc *sc, uint16_t idx)
    604 {
    605 	struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc;
    606 
    607 	bus_space_write_2(psc->sc_iot, psc->sc_ioh,
    608 	    VIRTIO_CONFIG_QUEUE_SELECT, idx);
    609 	return bus_space_read_2(psc->sc_iot, psc->sc_ioh,
    610 	    VIRTIO_CONFIG_QUEUE_SIZE);
    611 }
    612 
    613 static void
    614 virtio_pci_setup_queue_09(struct virtio_softc *sc, uint16_t idx, uint64_t addr)
    615 {
    616 	struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc;
    617 
    618 	bus_space_write_2(psc->sc_iot, psc->sc_ioh,
    619 	    VIRTIO_CONFIG_QUEUE_SELECT, idx);
    620 	bus_space_write_4(psc->sc_iot, psc->sc_ioh,
    621 	    VIRTIO_CONFIG_QUEUE_ADDRESS, addr / VIRTIO_PAGE_SIZE);
    622 
    623 	if (psc->sc_ihs_num > 1) {
    624 		int vec = VIRTIO_MSIX_QUEUE_VECTOR_INDEX;
    625 		if (sc->sc_child_mq)
    626 			vec += idx;
    627 		bus_space_write_2(psc->sc_iot, psc->sc_ioh,
    628 		    VIRTIO_CONFIG_MSI_QUEUE_VECTOR, vec);
    629 	}
    630 }
    631 
    632 static void
    633 virtio_pci_set_status_09(struct virtio_softc *sc, int status)
    634 {
    635 	struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc;
    636 	int old = 0;
    637 
    638 	if (status != 0) {
    639 	    old = bus_space_read_1(psc->sc_iot, psc->sc_ioh,
    640 		VIRTIO_CONFIG_DEVICE_STATUS);
    641 	}
    642 	bus_space_write_1(psc->sc_iot, psc->sc_ioh,
    643 	    VIRTIO_CONFIG_DEVICE_STATUS, status|old);
    644 }
    645 
    646 static void
    647 virtio_pci_negotiate_features_09(struct virtio_softc *sc, uint64_t guest_features)
    648 {
    649 	struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc;
    650 	uint32_t r;
    651 
    652 	r = bus_space_read_4(psc->sc_iot, psc->sc_ioh,
    653 	    VIRTIO_CONFIG_DEVICE_FEATURES);
    654 
    655 	r &= guest_features;
    656 
    657 	bus_space_write_4(psc->sc_iot, psc->sc_ioh,
    658 	    VIRTIO_CONFIG_GUEST_FEATURES, r);
    659 
    660 	sc->sc_active_features = r;
    661 }
    662 
    663 /* -------------------------------------
    664  * Version 1.0 support
    665  * -------------------------------------*/
    666 
    667 static void
    668 virtio_pci_kick_10(struct virtio_softc *sc, uint16_t idx)
    669 {
    670 	struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc;
    671 	unsigned offset = sc->sc_vqs[idx].vq_notify_off *
    672 		psc->sc_notify_off_multiplier;
    673 
    674 	bus_space_write_2(psc->sc_notify_iot, psc->sc_notify_ioh, offset, idx);
    675 }
    676 
    677 
    678 static uint16_t
    679 virtio_pci_read_queue_size_10(struct virtio_softc *sc, uint16_t idx)
    680 {
    681 	struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc;
    682 	bus_space_tag_t	   iot = psc->sc_iot;
    683 	bus_space_handle_t ioh = psc->sc_ioh;
    684 
    685 	bus_space_write_2(iot, ioh, VIRTIO_CONFIG1_QUEUE_SELECT, idx);
    686 	return bus_space_read_2(iot, ioh, VIRTIO_CONFIG1_QUEUE_SIZE);
    687 }
    688 
    689 /*
    690  * By definition little endian only in v1.0 and 8 byters are allowed to be
    691  * written as two 4 byters
    692  *
    693  * This is not a general purpose function that can be used in any
    694  * driver. Virtio specifically allows the 8 byte bus transaction
    695  * to be split into two 4 byte transactions. Do not copy/use it
    696  * in other device drivers unless you know that the device accepts it.
    697  */
    698 static __inline void
    699 virtio_pci_bus_space_write_8(bus_space_tag_t iot, bus_space_handle_t ioh,
    700      bus_size_t offset, uint64_t value)
    701 {
    702 #if defined(__HAVE_BUS_SPACE_8)
    703 	bus_space_write_8(iot, ioh, offset, value);
    704 #elif _QUAD_HIGHWORD
    705 	bus_space_write_4(iot, ioh, offset, BUS_ADDR_LO32(value));
    706 	bus_space_write_4(iot, ioh, offset + 4, BUS_ADDR_HI32(value));
    707 #else
    708 	bus_space_write_4(iot, ioh, offset, BUS_ADDR_HI32(value));
    709 	bus_space_write_4(iot, ioh, offset + 4, BUS_ADDR_LO32(value));
    710 #endif
    711 }
    712 
    713 static void
    714 virtio_pci_setup_queue_10(struct virtio_softc *sc, uint16_t idx, uint64_t addr)
    715 {
    716 	struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc;
    717 	struct virtqueue *vq = &sc->sc_vqs[idx];
    718 	bus_space_tag_t	   iot = psc->sc_iot;
    719 	bus_space_handle_t ioh = psc->sc_ioh;
    720 	KASSERT(vq->vq_index == idx);
    721 
    722 	bus_space_write_2(iot, ioh, VIRTIO_CONFIG1_QUEUE_SELECT, vq->vq_index);
    723 	if (addr == 0) {
    724 		bus_space_write_2(iot, ioh, VIRTIO_CONFIG1_QUEUE_ENABLE, 0);
    725 		virtio_pci_bus_space_write_8(iot, ioh,
    726 		    VIRTIO_CONFIG1_QUEUE_DESC,   0);
    727 		virtio_pci_bus_space_write_8(iot, ioh,
    728 		    VIRTIO_CONFIG1_QUEUE_AVAIL,  0);
    729 		virtio_pci_bus_space_write_8(iot, ioh,
    730 		    VIRTIO_CONFIG1_QUEUE_USED,   0);
    731 	} else {
    732 		virtio_pci_bus_space_write_8(iot, ioh,
    733 			VIRTIO_CONFIG1_QUEUE_DESC, addr);
    734 		virtio_pci_bus_space_write_8(iot, ioh,
    735 			VIRTIO_CONFIG1_QUEUE_AVAIL, addr + vq->vq_availoffset);
    736 		virtio_pci_bus_space_write_8(iot, ioh,
    737 			VIRTIO_CONFIG1_QUEUE_USED, addr + vq->vq_usedoffset);
    738 		bus_space_write_2(iot, ioh,
    739 			VIRTIO_CONFIG1_QUEUE_ENABLE, 1);
    740 		vq->vq_notify_off = bus_space_read_2(iot, ioh,
    741 			VIRTIO_CONFIG1_QUEUE_NOTIFY_OFF);
    742 	}
    743 
    744 	if (psc->sc_ihs_num > 1) {
    745 		int vec = VIRTIO_MSIX_QUEUE_VECTOR_INDEX;
    746 		if (sc->sc_child_mq)
    747 			vec += idx;
    748 		bus_space_write_2(iot, ioh,
    749 			VIRTIO_CONFIG1_QUEUE_MSIX_VECTOR, vec);
    750 	}
    751 }
    752 
    753 static void
    754 virtio_pci_set_status_10(struct virtio_softc *sc, int status)
    755 {
    756 	struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc;
    757 	bus_space_tag_t	   iot = psc->sc_iot;
    758 	bus_space_handle_t ioh = psc->sc_ioh;
    759 	int old = 0;
    760 
    761 	if (status)
    762 		old = bus_space_read_1(iot, ioh, VIRTIO_CONFIG1_DEVICE_STATUS);
    763 	bus_space_write_1(iot, ioh, VIRTIO_CONFIG1_DEVICE_STATUS, status | old);
    764 }
    765 
    766 void
    767 virtio_pci_negotiate_features_10(struct virtio_softc *sc, uint64_t guest_features)
    768 {
    769 	struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc;
    770 	device_t self          =  sc->sc_dev;
    771 	bus_space_tag_t	   iot = psc->sc_iot;
    772 	bus_space_handle_t ioh = psc->sc_ioh;
    773 	uint64_t host, negotiated, device_status;
    774 
    775 	guest_features |= VIRTIO_F_VERSION_1;
    776 	/* notify on empty is 0.9 only */
    777 	guest_features &= ~VIRTIO_F_NOTIFY_ON_EMPTY;
    778 	sc->sc_active_features = 0;
    779 
    780 	bus_space_write_4(iot, ioh, VIRTIO_CONFIG1_DEVICE_FEATURE_SELECT, 0);
    781 	host = bus_space_read_4(iot, ioh, VIRTIO_CONFIG1_DEVICE_FEATURE);
    782 	bus_space_write_4(iot, ioh, VIRTIO_CONFIG1_DEVICE_FEATURE_SELECT, 1);
    783 	host |= (uint64_t)
    784 		bus_space_read_4(iot, ioh, VIRTIO_CONFIG1_DEVICE_FEATURE) << 32;
    785 
    786 	negotiated = host & guest_features;
    787 
    788 	bus_space_write_4(iot, ioh, VIRTIO_CONFIG1_DRIVER_FEATURE_SELECT, 0);
    789 	bus_space_write_4(iot, ioh, VIRTIO_CONFIG1_DRIVER_FEATURE,
    790 			negotiated & 0xffffffff);
    791 	bus_space_write_4(iot, ioh, VIRTIO_CONFIG1_DRIVER_FEATURE_SELECT, 1);
    792 	bus_space_write_4(iot, ioh, VIRTIO_CONFIG1_DRIVER_FEATURE,
    793 			negotiated >> 32);
    794 	virtio_pci_set_status_10(sc, VIRTIO_CONFIG_DEVICE_STATUS_FEATURES_OK);
    795 
    796 	device_status = bus_space_read_1(iot, ioh, VIRTIO_CONFIG1_DEVICE_STATUS);
    797 	if ((device_status & VIRTIO_CONFIG_DEVICE_STATUS_FEATURES_OK) == 0) {
    798 		aprint_error_dev(self, "feature negotiation failed\n");
    799 		bus_space_write_1(iot, ioh, VIRTIO_CONFIG1_DEVICE_STATUS,
    800 				VIRTIO_CONFIG_DEVICE_STATUS_FAILED);
    801 		return;
    802 	}
    803 
    804 	if ((negotiated & VIRTIO_F_VERSION_1) == 0) {
    805 		aprint_error_dev(self, "host rejected version 1\n");
    806 		bus_space_write_1(iot, ioh, VIRTIO_CONFIG1_DEVICE_STATUS,
    807 				VIRTIO_CONFIG_DEVICE_STATUS_FAILED);
    808 		return;
    809 	}
    810 
    811 	sc->sc_active_features = negotiated;
    812 	return;
    813 }
    814 
    815 
    816 /* -------------------------------------
    817  * Generic PCI interrupt code
    818  * -------------------------------------*/
    819 
    820 static int
    821 virtio_pci_setup_interrupts_10(struct virtio_softc *sc, int reinit)
    822 {
    823 	struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc;
    824 	bus_space_tag_t	   iot = psc->sc_iot;
    825 	bus_space_handle_t ioh = psc->sc_ioh;
    826 	int vector, ret, qid;
    827 
    828 	if (!virtio_pci_msix_enabled(psc))
    829 		return 0;
    830 
    831 	vector = VIRTIO_MSIX_CONFIG_VECTOR_INDEX;
    832 	bus_space_write_2(iot, ioh,
    833 		VIRTIO_CONFIG1_CONFIG_MSIX_VECTOR, vector);
    834 	ret = bus_space_read_2(iot, ioh, VIRTIO_CONFIG1_CONFIG_MSIX_VECTOR);
    835 	if (ret != vector) {
    836 		VIRTIO_PCI_LOG(sc, reinit,
    837 		    "can't set config msix vector\n");
    838 		return -1;
    839 	}
    840 
    841 	for (qid = 0; qid < sc->sc_nvqs; qid++) {
    842 		vector = VIRTIO_MSIX_QUEUE_VECTOR_INDEX;
    843 
    844 		if (sc->sc_child_mq)
    845 			vector += qid;
    846 		bus_space_write_2(iot, ioh, VIRTIO_CONFIG1_QUEUE_SELECT, qid);
    847 		bus_space_write_2(iot, ioh, VIRTIO_CONFIG1_QUEUE_MSIX_VECTOR,
    848 			vector);
    849 		ret = bus_space_read_2(iot, ioh,
    850 			VIRTIO_CONFIG1_QUEUE_MSIX_VECTOR);
    851 		if (ret != vector) {
    852 			VIRTIO_PCI_LOG(sc, reinit, "can't set queue %d "
    853 			    "msix vector\n", qid);
    854 			return -1;
    855 		}
    856 	}
    857 
    858 	return 0;
    859 }
    860 
    861 static int
    862 virtio_pci_setup_interrupts_09(struct virtio_softc *sc, int reinit)
    863 {
    864 	struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc;
    865 	int offset, vector, ret, qid;
    866 
    867 	if (!virtio_pci_msix_enabled(psc))
    868 		return 0;
    869 
    870 	offset = VIRTIO_CONFIG_MSI_CONFIG_VECTOR;
    871 	vector = VIRTIO_MSIX_CONFIG_VECTOR_INDEX;
    872 
    873 	bus_space_write_2(psc->sc_iot, psc->sc_ioh, offset, vector);
    874 	ret = bus_space_read_2(psc->sc_iot, psc->sc_ioh, offset);
    875 	aprint_debug_dev(sc->sc_dev, "expected=%d, actual=%d\n",
    876 	    vector, ret);
    877 	if (ret != vector) {
    878 		VIRTIO_PCI_LOG(sc, reinit,
    879 		    "can't set config msix vector\n");
    880 		return -1;
    881 	}
    882 
    883 	for (qid = 0; qid < sc->sc_nvqs; qid++) {
    884 		offset = VIRTIO_CONFIG_QUEUE_SELECT;
    885 		bus_space_write_2(psc->sc_iot, psc->sc_ioh, offset, qid);
    886 
    887 		offset = VIRTIO_CONFIG_MSI_QUEUE_VECTOR;
    888 		vector = VIRTIO_MSIX_QUEUE_VECTOR_INDEX;
    889 
    890 		if (sc->sc_child_mq)
    891 			vector += qid;
    892 
    893 		bus_space_write_2(psc->sc_iot, psc->sc_ioh, offset, vector);
    894 		ret = bus_space_read_2(psc->sc_iot, psc->sc_ioh, offset);
    895 		aprint_debug_dev(sc->sc_dev, "expected=%d, actual=%d\n",
    896 		    vector, ret);
    897 		if (ret != vector) {
    898 			VIRTIO_PCI_LOG(sc, reinit, "can't set queue %d "
    899 			    "msix vector\n", qid);
    900 			return -1;
    901 		}
    902 	}
    903 
    904 	return 0;
    905 }
    906 
    907 static int
    908 virtio_pci_establish_msix_interrupts(struct virtio_softc *sc,
    909     struct pci_attach_args *pa)
    910 {
    911 	struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc;
    912 	device_t self = sc->sc_dev;
    913 	pci_chipset_tag_t pc = pa->pa_pc;
    914 	struct virtqueue *vq;
    915 	char intrbuf[PCI_INTRSTR_LEN];
    916 	char intr_xname[INTRDEVNAMEBUF];
    917 	char const *intrstr;
    918 	int idx, qid, n;
    919 
    920 	idx = VIRTIO_MSIX_CONFIG_VECTOR_INDEX;
    921 	if (sc->sc_flags & VIRTIO_F_INTR_MPSAFE)
    922 		pci_intr_setattr(pc, &psc->sc_ihp[idx], PCI_INTR_MPSAFE, true);
    923 
    924 	snprintf(intr_xname, sizeof(intr_xname), "%s config",
    925 	    device_xname(sc->sc_dev));
    926 
    927 	psc->sc_ihs[idx] = pci_intr_establish_xname(pc, psc->sc_ihp[idx],
    928 	    sc->sc_ipl, virtio_pci_msix_config_intr, sc, intr_xname);
    929 	if (psc->sc_ihs[idx] == NULL) {
    930 		aprint_error_dev(self, "couldn't establish MSI-X for config\n");
    931 		goto error;
    932 	}
    933 
    934 	idx = VIRTIO_MSIX_QUEUE_VECTOR_INDEX;
    935 	if (sc->sc_child_mq) {
    936 		for (qid = 0; qid < sc->sc_nvqs; qid++) {
    937 			n = idx + qid;
    938 			vq = &sc->sc_vqs[qid];
    939 
    940 			snprintf(intr_xname, sizeof(intr_xname), "%s vq#%d",
    941 			    device_xname(sc->sc_dev), qid);
    942 
    943 			if (sc->sc_flags & VIRTIO_F_INTR_MPSAFE) {
    944 				pci_intr_setattr(pc, &psc->sc_ihp[n],
    945 				    PCI_INTR_MPSAFE, true);
    946 			}
    947 
    948 			psc->sc_ihs[n] = pci_intr_establish_xname(pc, psc->sc_ihp[n],
    949 			    sc->sc_ipl, vq->vq_intrhand, vq->vq_intrhand_arg, intr_xname);
    950 			if (psc->sc_ihs[n] == NULL) {
    951 				aprint_error_dev(self, "couldn't establish MSI-X for a vq\n");
    952 				goto error;
    953 			}
    954 		}
    955 	} else {
    956 		if (sc->sc_flags & VIRTIO_F_INTR_MPSAFE)
    957 			pci_intr_setattr(pc, &psc->sc_ihp[idx], PCI_INTR_MPSAFE, true);
    958 
    959 		snprintf(intr_xname, sizeof(intr_xname), "%s queues",
    960 		    device_xname(sc->sc_dev));
    961 		psc->sc_ihs[idx] = pci_intr_establish_xname(pc, psc->sc_ihp[idx],
    962 		    sc->sc_ipl, virtio_pci_msix_queue_intr, sc, intr_xname);
    963 		if (psc->sc_ihs[idx] == NULL) {
    964 			aprint_error_dev(self, "couldn't establish MSI-X for queues\n");
    965 			goto error;
    966 		}
    967 	}
    968 
    969 	idx = VIRTIO_MSIX_CONFIG_VECTOR_INDEX;
    970 	intrstr = pci_intr_string(pc, psc->sc_ihp[idx], intrbuf, sizeof(intrbuf));
    971 	aprint_normal_dev(self, "config interrupting at %s\n", intrstr);
    972 	idx = VIRTIO_MSIX_QUEUE_VECTOR_INDEX;
    973 	if (sc->sc_child_mq) {
    974 		kcpuset_t *affinity;
    975 		int affinity_to, r;
    976 
    977 		kcpuset_create(&affinity, false);
    978 
    979 		for (qid = 0; qid < sc->sc_nvqs; qid++) {
    980 			n = idx + qid;
    981 			affinity_to = (qid / 2) % ncpu;
    982 
    983 			intrstr = pci_intr_string(pc, psc->sc_ihp[n],
    984 			    intrbuf, sizeof(intrbuf));
    985 
    986 			kcpuset_zero(affinity);
    987 			kcpuset_set(affinity, affinity_to);
    988 			r = interrupt_distribute(psc->sc_ihs[n], affinity, NULL);
    989 			if (r == 0) {
    990 				aprint_normal_dev(self,
    991 				    "for vq #%d interrupting at %s affinity to %u\n",
    992 				    qid, intrstr, affinity_to);
    993 			} else {
    994 				aprint_normal_dev(self,
    995 				    "for vq #%d interrupting at %s\n",
    996 				    qid, intrstr);
    997 			}
    998 		}
    999 
   1000 		kcpuset_destroy(affinity);
   1001 	} else {
   1002 		intrstr = pci_intr_string(pc, psc->sc_ihp[idx], intrbuf, sizeof(intrbuf));
   1003 		aprint_normal_dev(self, "queues interrupting at %s\n", intrstr);
   1004 	}
   1005 
   1006 	return 0;
   1007 
   1008 error:
   1009 	idx = VIRTIO_MSIX_CONFIG_VECTOR_INDEX;
   1010 	if (psc->sc_ihs[idx] != NULL)
   1011 		pci_intr_disestablish(psc->sc_pa.pa_pc, psc->sc_ihs[idx]);
   1012 	idx = VIRTIO_MSIX_QUEUE_VECTOR_INDEX;
   1013 	if (sc->sc_child_mq) {
   1014 		for (qid = 0; qid < sc->sc_nvqs; qid++) {
   1015 			n = idx + qid;
   1016 			if (psc->sc_ihs[n] == NULL)
   1017 				continue;
   1018 			pci_intr_disestablish(psc->sc_pa.pa_pc, psc->sc_ihs[n]);
   1019 		}
   1020 
   1021 	} else {
   1022 		if (psc->sc_ihs[idx] != NULL)
   1023 			pci_intr_disestablish(psc->sc_pa.pa_pc, psc->sc_ihs[idx]);
   1024 	}
   1025 
   1026 	return -1;
   1027 }
   1028 
   1029 static int
   1030 virtio_pci_establish_intx_interrupt(struct virtio_softc *sc,
   1031     struct pci_attach_args *pa)
   1032 {
   1033 	struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc;
   1034 	device_t self = sc->sc_dev;
   1035 	pci_chipset_tag_t pc = pa->pa_pc;
   1036 	char intrbuf[PCI_INTRSTR_LEN];
   1037 	char const *intrstr;
   1038 
   1039 	if (sc->sc_flags & VIRTIO_F_INTR_MPSAFE)
   1040 		pci_intr_setattr(pc, &psc->sc_ihp[0], PCI_INTR_MPSAFE, true);
   1041 
   1042 	psc->sc_ihs[0] = pci_intr_establish_xname(pc, psc->sc_ihp[0],
   1043 	    sc->sc_ipl, virtio_pci_intr, sc, device_xname(sc->sc_dev));
   1044 	if (psc->sc_ihs[0] == NULL) {
   1045 		aprint_error_dev(self, "couldn't establish INTx\n");
   1046 		return -1;
   1047 	}
   1048 
   1049 	intrstr = pci_intr_string(pc, psc->sc_ihp[0], intrbuf, sizeof(intrbuf));
   1050 	aprint_normal_dev(self, "interrupting at %s\n", intrstr);
   1051 
   1052 	return 0;
   1053 }
   1054 
   1055 static int
   1056 virtio_pci_alloc_interrupts(struct virtio_softc *sc)
   1057 {
   1058 	struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc;
   1059 	device_t self = sc->sc_dev;
   1060 	pci_chipset_tag_t pc = psc->sc_pa.pa_pc;
   1061 	pcitag_t tag = psc->sc_pa.pa_tag;
   1062 	int error;
   1063 	int nmsix;
   1064 	int off;
   1065 	int counts[PCI_INTR_TYPE_SIZE];
   1066 	pci_intr_type_t max_type;
   1067 	pcireg_t ctl;
   1068 
   1069 	nmsix = pci_msix_count(psc->sc_pa.pa_pc, psc->sc_pa.pa_tag);
   1070 	aprint_debug_dev(self, "pci_msix_count=%d\n", nmsix);
   1071 
   1072 	/* We need at least two: one for config and the other for queues */
   1073 	if ((sc->sc_flags & VIRTIO_F_INTR_MSIX) == 0 || nmsix < 2) {
   1074 		/* Try INTx only */
   1075 		max_type = PCI_INTR_TYPE_INTX;
   1076 		counts[PCI_INTR_TYPE_INTX] = 1;
   1077 	} else {
   1078 		/* Try MSI-X first and INTx second */
   1079 		if (sc->sc_nvqs + VIRTIO_MSIX_QUEUE_VECTOR_INDEX <= nmsix) {
   1080 			nmsix = sc->sc_nvqs + VIRTIO_MSIX_QUEUE_VECTOR_INDEX;
   1081 		} else {
   1082 			sc->sc_child_mq = false;
   1083 		}
   1084 
   1085 		if (sc->sc_child_mq == false) {
   1086 			nmsix = 2;
   1087 		}
   1088 
   1089 		max_type = PCI_INTR_TYPE_MSIX;
   1090 		counts[PCI_INTR_TYPE_MSIX] = nmsix;
   1091 		counts[PCI_INTR_TYPE_MSI] = 0;
   1092 		counts[PCI_INTR_TYPE_INTX] = 1;
   1093 	}
   1094 
   1095 retry:
   1096 	error = pci_intr_alloc(&psc->sc_pa, &psc->sc_ihp, counts, max_type);
   1097 	if (error != 0) {
   1098 		aprint_error_dev(self, "couldn't map interrupt\n");
   1099 		return -1;
   1100 	}
   1101 
   1102 	if (pci_intr_type(pc, psc->sc_ihp[0]) == PCI_INTR_TYPE_MSIX) {
   1103 		psc->sc_ihs = kmem_zalloc(sizeof(*psc->sc_ihs) * nmsix,
   1104 		    KM_SLEEP);
   1105 
   1106 		error = virtio_pci_establish_msix_interrupts(sc, &psc->sc_pa);
   1107 		if (error != 0) {
   1108 			kmem_free(psc->sc_ihs, sizeof(*psc->sc_ihs) * nmsix);
   1109 			pci_intr_release(pc, psc->sc_ihp, nmsix);
   1110 
   1111 			/* Retry INTx */
   1112 			max_type = PCI_INTR_TYPE_INTX;
   1113 			counts[PCI_INTR_TYPE_INTX] = 1;
   1114 			goto retry;
   1115 		}
   1116 
   1117 		psc->sc_ihs_num = nmsix;
   1118 		psc->sc_devcfg_offset = VIRTIO_CONFIG_DEVICE_CONFIG_MSI;
   1119 		virtio_pci_adjust_config_region(psc);
   1120 	} else if (pci_intr_type(pc, psc->sc_ihp[0]) == PCI_INTR_TYPE_INTX) {
   1121 		psc->sc_ihs = kmem_zalloc(sizeof(*psc->sc_ihs) * 1,
   1122 		    KM_SLEEP);
   1123 
   1124 		error = virtio_pci_establish_intx_interrupt(sc, &psc->sc_pa);
   1125 		if (error != 0) {
   1126 			kmem_free(psc->sc_ihs, sizeof(*psc->sc_ihs) * 1);
   1127 			pci_intr_release(pc, psc->sc_ihp, 1);
   1128 			return -1;
   1129 		}
   1130 
   1131 		psc->sc_ihs_num = 1;
   1132 		psc->sc_devcfg_offset = VIRTIO_CONFIG_DEVICE_CONFIG_NOMSI;
   1133 		virtio_pci_adjust_config_region(psc);
   1134 
   1135 		error = pci_get_capability(pc, tag, PCI_CAP_MSIX, &off, NULL);
   1136 		if (error != 0) {
   1137 			ctl = pci_conf_read(pc, tag, off + PCI_MSIX_CTL);
   1138 			ctl &= ~PCI_MSIX_CTL_ENABLE;
   1139 			pci_conf_write(pc, tag, off + PCI_MSIX_CTL, ctl);
   1140 		}
   1141 	}
   1142 
   1143 	return 0;
   1144 }
   1145 
   1146 static void
   1147 virtio_pci_free_interrupts(struct virtio_softc *sc)
   1148 {
   1149 	struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc;
   1150 
   1151 	for (int i = 0; i < psc->sc_ihs_num; i++) {
   1152 		if (psc->sc_ihs[i] == NULL)
   1153 			continue;
   1154 		pci_intr_disestablish(psc->sc_pa.pa_pc, psc->sc_ihs[i]);
   1155 		psc->sc_ihs[i] = NULL;
   1156 	}
   1157 
   1158 	if (psc->sc_ihs_num > 0)
   1159 		pci_intr_release(psc->sc_pa.pa_pc, psc->sc_ihp, psc->sc_ihs_num);
   1160 
   1161 	if (psc->sc_ihs != NULL) {
   1162 		kmem_free(psc->sc_ihs, sizeof(*psc->sc_ihs) * psc->sc_ihs_num);
   1163 		psc->sc_ihs = NULL;
   1164 	}
   1165 	psc->sc_ihs_num = 0;
   1166 }
   1167 
   1168 static bool
   1169 virtio_pci_msix_enabled(struct virtio_pci_softc *psc)
   1170 {
   1171 	pci_chipset_tag_t pc = psc->sc_pa.pa_pc;
   1172 
   1173 	if (pci_intr_type(pc, psc->sc_ihp[0]) == PCI_INTR_TYPE_MSIX)
   1174 		return true;
   1175 
   1176 	return false;
   1177 }
   1178 
   1179 /*
   1180  * Interrupt handler.
   1181  */
   1182 static int
   1183 virtio_pci_intr(void *arg)
   1184 {
   1185 	struct virtio_softc *sc = arg;
   1186 	struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc;
   1187 	int isr, r = 0;
   1188 
   1189 	/* check and ack the interrupt */
   1190 	isr = bus_space_read_1(psc->sc_isr_iot, psc->sc_isr_ioh, 0);
   1191 	if (isr == 0)
   1192 		return 0;
   1193 	if ((isr & VIRTIO_CONFIG_ISR_CONFIG_CHANGE) &&
   1194 	    (sc->sc_config_change != NULL))
   1195 		r = (sc->sc_config_change)(sc);
   1196 	if (sc->sc_intrhand != NULL) {
   1197 		if (sc->sc_soft_ih != NULL)
   1198 			softint_schedule(sc->sc_soft_ih);
   1199 		else
   1200 			r |= (sc->sc_intrhand)(sc);
   1201 	}
   1202 
   1203 	return r;
   1204 }
   1205 
   1206 static int
   1207 virtio_pci_msix_queue_intr(void *arg)
   1208 {
   1209 	struct virtio_softc *sc = arg;
   1210 	int r = 0;
   1211 
   1212 	if (sc->sc_intrhand != NULL) {
   1213 		if (sc->sc_soft_ih != NULL)
   1214 			softint_schedule(sc->sc_soft_ih);
   1215 		else
   1216 			r |= (sc->sc_intrhand)(sc);
   1217 	}
   1218 
   1219 	return r;
   1220 }
   1221 
   1222 static int
   1223 virtio_pci_msix_config_intr(void *arg)
   1224 {
   1225 	struct virtio_softc *sc = arg;
   1226 	int r = 0;
   1227 
   1228 	if (sc->sc_config_change != NULL)
   1229 		r = (sc->sc_config_change)(sc);
   1230 	return r;
   1231 }
   1232 
   1233 MODULE(MODULE_CLASS_DRIVER, virtio_pci, "pci,virtio");
   1234 
   1235 #ifdef _MODULE
   1236 #include "ioconf.c"
   1237 #endif
   1238 
   1239 static int
   1240 virtio_pci_modcmd(modcmd_t cmd, void *opaque)
   1241 {
   1242 	int error = 0;
   1243 
   1244 #ifdef _MODULE
   1245 	switch (cmd) {
   1246 	case MODULE_CMD_INIT:
   1247 		error = config_init_component(cfdriver_ioconf_virtio_pci,
   1248 		    cfattach_ioconf_virtio_pci, cfdata_ioconf_virtio_pci);
   1249 		break;
   1250 	case MODULE_CMD_FINI:
   1251 		error = config_fini_component(cfdriver_ioconf_virtio_pci,
   1252 		    cfattach_ioconf_virtio_pci, cfdata_ioconf_virtio_pci);
   1253 		break;
   1254 	default:
   1255 		error = ENOTTY;
   1256 		break;
   1257 	}
   1258 #endif
   1259 
   1260 	return error;
   1261 }
   1262