virtio_pci.c revision 1.33
1 /* $NetBSD: virtio_pci.c,v 1.33 2021/10/28 01:36:43 yamaguchi Exp $ */ 2 3 /* 4 * Copyright (c) 2020 The NetBSD Foundation, Inc. 5 * Copyright (c) 2012 Stefan Fritsch. 6 * Copyright (c) 2010 Minoura Makoto. 7 * All rights reserved. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 19 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 20 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 21 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 22 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 23 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 27 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 */ 29 30 #include <sys/cdefs.h> 31 __KERNEL_RCSID(0, "$NetBSD: virtio_pci.c,v 1.33 2021/10/28 01:36:43 yamaguchi Exp $"); 32 33 #include <sys/param.h> 34 #include <sys/systm.h> 35 #include <sys/kmem.h> 36 #include <sys/module.h> 37 #include <sys/endian.h> 38 #include <sys/interrupt.h> 39 #include <sys/syslog.h> 40 41 #include <sys/device.h> 42 43 #include <dev/pci/pcidevs.h> 44 #include <dev/pci/pcireg.h> 45 #include <dev/pci/pcivar.h> 46 47 #include <dev/pci/virtioreg.h> /* XXX: move to non-pci */ 48 #include <dev/pci/virtio_pcireg.h> 49 50 #define VIRTIO_PRIVATE 51 #include <dev/pci/virtiovar.h> /* XXX: move to non-pci */ 52 53 54 #define VIRTIO_PCI_LOG(_sc, _use_log, _fmt, _args...) \ 55 do { \ 56 if ((_use_log)) { \ 57 log(LOG_DEBUG, "%s: " _fmt, \ 58 device_xname((_sc)->sc_dev), \ 59 ##_args); \ 60 } else { \ 61 aprint_error_dev((_sc)->sc_dev, \ 62 _fmt, ##_args); \ 63 } \ 64 } while(0) 65 66 static int virtio_pci_match(device_t, cfdata_t, void *); 67 static void virtio_pci_attach(device_t, device_t, void *); 68 static int virtio_pci_rescan(device_t, const char *, const int *); 69 static int virtio_pci_detach(device_t, int); 70 71 72 #define NMAPREG ((PCI_MAPREG_END - PCI_MAPREG_START) / \ 73 sizeof(pcireg_t)) 74 struct virtio_pci_softc { 75 struct virtio_softc sc_sc; 76 77 /* IO space */ 78 bus_space_tag_t sc_iot; 79 bus_space_handle_t sc_ioh; 80 bus_size_t sc_iosize; 81 bus_size_t sc_mapped_iosize; 82 83 /* BARs */ 84 bus_space_tag_t sc_bars_iot[NMAPREG]; 85 bus_space_handle_t sc_bars_ioh[NMAPREG]; 86 bus_size_t sc_bars_iosize[NMAPREG]; 87 88 /* notify space */ 89 bus_space_tag_t sc_notify_iot; 90 bus_space_handle_t sc_notify_ioh; 91 bus_size_t sc_notify_iosize; 92 uint32_t sc_notify_off_multiplier; 93 94 /* isr space */ 95 bus_space_tag_t sc_isr_iot; 96 bus_space_handle_t sc_isr_ioh; 97 bus_size_t sc_isr_iosize; 98 99 /* generic */ 100 struct pci_attach_args sc_pa; 101 pci_intr_handle_t *sc_ihp; 102 void **sc_ihs; 103 int sc_ihs_num; 104 int sc_devcfg_offset; /* for 0.9 */ 105 }; 106 107 static int virtio_pci_attach_09(device_t, void *); 108 static void virtio_pci_kick_09(struct virtio_softc *, uint16_t); 109 static uint16_t virtio_pci_read_queue_size_09(struct virtio_softc *, uint16_t); 110 static void virtio_pci_setup_queue_09(struct virtio_softc *, uint16_t, uint64_t); 111 static void virtio_pci_set_status_09(struct virtio_softc *, int); 112 static void virtio_pci_negotiate_features_09(struct virtio_softc *, uint64_t); 113 114 static int virtio_pci_attach_10(device_t, void *); 115 static void virtio_pci_kick_10(struct virtio_softc *, uint16_t); 116 static uint16_t virtio_pci_read_queue_size_10(struct virtio_softc *, uint16_t); 117 static void virtio_pci_setup_queue_10(struct virtio_softc *, uint16_t, uint64_t); 118 static void virtio_pci_set_status_10(struct virtio_softc *, int); 119 static void virtio_pci_negotiate_features_10(struct virtio_softc *, uint64_t); 120 static int virtio_pci_find_cap(struct virtio_pci_softc *psc, int cfg_type, void *buf, int buflen); 121 122 static int virtio_pci_alloc_interrupts(struct virtio_softc *); 123 static void virtio_pci_free_interrupts(struct virtio_softc *); 124 static int virtio_pci_adjust_config_region(struct virtio_pci_softc *psc); 125 static int virtio_pci_intr(void *arg); 126 static int virtio_pci_msix_queue_intr(void *); 127 static int virtio_pci_msix_config_intr(void *); 128 static int virtio_pci_setup_interrupts_09(struct virtio_softc *, int); 129 static int virtio_pci_setup_interrupts_10(struct virtio_softc *, int); 130 static int virtio_pci_establish_msix_interrupts(struct virtio_softc *, 131 struct pci_attach_args *); 132 static int virtio_pci_establish_intx_interrupt(struct virtio_softc *, 133 struct pci_attach_args *); 134 static bool virtio_pci_msix_enabled(struct virtio_pci_softc *); 135 136 #define VIRTIO_MSIX_CONFIG_VECTOR_INDEX 0 137 #define VIRTIO_MSIX_QUEUE_VECTOR_INDEX 1 138 139 /* 140 * When using PCI attached virtio on aarch64-eb under Qemu, the IO space 141 * suddenly read BIG_ENDIAN where it should stay LITTLE_ENDIAN. The data read 142 * 1 byte at a time seem OK but reading bigger lengths result in swapped 143 * endian. This is most notable on reading 8 byters since we can't use 144 * bus_space_{read,write}_8(). 145 */ 146 147 #if defined(__aarch64__) && BYTE_ORDER == BIG_ENDIAN 148 # define READ_ENDIAN_09 BIG_ENDIAN /* should be LITTLE_ENDIAN */ 149 # define READ_ENDIAN_10 BIG_ENDIAN 150 # define STRUCT_ENDIAN_09 BIG_ENDIAN 151 # define STRUCT_ENDIAN_10 LITTLE_ENDIAN 152 #elif BYTE_ORDER == BIG_ENDIAN 153 # define READ_ENDIAN_09 LITTLE_ENDIAN 154 # define READ_ENDIAN_10 BIG_ENDIAN 155 # define STRUCT_ENDIAN_09 BIG_ENDIAN 156 # define STRUCT_ENDIAN_10 LITTLE_ENDIAN 157 #else /* little endian */ 158 # define READ_ENDIAN_09 LITTLE_ENDIAN 159 # define READ_ENDIAN_10 LITTLE_ENDIAN 160 # define STRUCT_ENDIAN_09 LITTLE_ENDIAN 161 # define STRUCT_ENDIAN_10 LITTLE_ENDIAN 162 #endif 163 164 165 CFATTACH_DECL3_NEW(virtio_pci, sizeof(struct virtio_pci_softc), 166 virtio_pci_match, virtio_pci_attach, virtio_pci_detach, NULL, 167 virtio_pci_rescan, NULL, DVF_DETACH_SHUTDOWN); 168 169 static const struct virtio_ops virtio_pci_ops_09 = { 170 .kick = virtio_pci_kick_09, 171 .read_queue_size = virtio_pci_read_queue_size_09, 172 .setup_queue = virtio_pci_setup_queue_09, 173 .set_status = virtio_pci_set_status_09, 174 .neg_features = virtio_pci_negotiate_features_09, 175 .alloc_interrupts = virtio_pci_alloc_interrupts, 176 .free_interrupts = virtio_pci_free_interrupts, 177 .setup_interrupts = virtio_pci_setup_interrupts_09, 178 }; 179 180 static const struct virtio_ops virtio_pci_ops_10 = { 181 .kick = virtio_pci_kick_10, 182 .read_queue_size = virtio_pci_read_queue_size_10, 183 .setup_queue = virtio_pci_setup_queue_10, 184 .set_status = virtio_pci_set_status_10, 185 .neg_features = virtio_pci_negotiate_features_10, 186 .alloc_interrupts = virtio_pci_alloc_interrupts, 187 .free_interrupts = virtio_pci_free_interrupts, 188 .setup_interrupts = virtio_pci_setup_interrupts_10, 189 }; 190 191 static int 192 virtio_pci_match(device_t parent, cfdata_t match, void *aux) 193 { 194 struct pci_attach_args *pa; 195 196 pa = (struct pci_attach_args *)aux; 197 switch (PCI_VENDOR(pa->pa_id)) { 198 case PCI_VENDOR_QUMRANET: 199 if (((PCI_PRODUCT_QUMRANET_VIRTIO_1000 <= 200 PCI_PRODUCT(pa->pa_id)) && 201 (PCI_PRODUCT(pa->pa_id) <= 202 PCI_PRODUCT_QUMRANET_VIRTIO_103F)) && 203 PCI_REVISION(pa->pa_class) == 0) 204 return 1; 205 if (((PCI_PRODUCT_QUMRANET_VIRTIO_1040 <= 206 PCI_PRODUCT(pa->pa_id)) && 207 (PCI_PRODUCT(pa->pa_id) <= 208 PCI_PRODUCT_QUMRANET_VIRTIO_107F)) && 209 PCI_REVISION(pa->pa_class) == 1) 210 return 1; 211 break; 212 } 213 214 return 0; 215 } 216 217 static void 218 virtio_pci_attach(device_t parent, device_t self, void *aux) 219 { 220 struct virtio_pci_softc * const psc = device_private(self); 221 struct virtio_softc * const sc = &psc->sc_sc; 222 struct pci_attach_args *pa = (struct pci_attach_args *)aux; 223 pci_chipset_tag_t pc = pa->pa_pc; 224 pcitag_t tag = pa->pa_tag; 225 int revision; 226 int ret; 227 pcireg_t id; 228 pcireg_t csr; 229 230 revision = PCI_REVISION(pa->pa_class); 231 switch (revision) { 232 case 0: 233 /* subsystem ID shows what I am */ 234 id = PCI_SUBSYS_ID(pci_conf_read(pc, tag, PCI_SUBSYS_ID_REG)); 235 break; 236 case 1: 237 /* pci product number shows what I am */ 238 id = PCI_PRODUCT(pa->pa_id) - PCI_PRODUCT_QUMRANET_VIRTIO_1040; 239 break; 240 default: 241 aprint_normal(": unknown revision 0x%02x; giving up\n", 242 revision); 243 return; 244 } 245 246 aprint_normal("\n"); 247 aprint_naive("\n"); 248 virtio_print_device_type(self, id, revision); 249 250 csr = pci_conf_read(pc, tag, PCI_COMMAND_STATUS_REG); 251 csr |= PCI_COMMAND_MASTER_ENABLE | PCI_COMMAND_IO_ENABLE; 252 pci_conf_write(pc, tag, PCI_COMMAND_STATUS_REG, csr); 253 254 sc->sc_dev = self; 255 psc->sc_pa = *pa; 256 psc->sc_iot = pa->pa_iot; 257 258 sc->sc_dmat = pa->pa_dmat; 259 if (pci_dma64_available(pa)) 260 sc->sc_dmat = pa->pa_dmat64; 261 262 /* attach is dependent on revision */ 263 ret = 0; 264 if (revision == 1) { 265 /* try to attach 1.0 */ 266 ret = virtio_pci_attach_10(self, aux); 267 } 268 if (ret == 0 && revision == 0) { 269 /* revision 0 means 0.9 only or both 0.9 and 1.0 */ 270 ret = virtio_pci_attach_09(self, aux); 271 } 272 if (ret) { 273 aprint_error_dev(self, "cannot attach (%d)\n", ret); 274 return; 275 } 276 KASSERT(sc->sc_ops); 277 278 /* preset config region */ 279 psc->sc_devcfg_offset = VIRTIO_CONFIG_DEVICE_CONFIG_NOMSI; 280 if (virtio_pci_adjust_config_region(psc)) 281 return; 282 283 /* generic */ 284 virtio_device_reset(sc); 285 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_ACK); 286 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_DRIVER); 287 288 sc->sc_childdevid = id; 289 sc->sc_child = NULL; 290 virtio_pci_rescan(self, NULL, NULL); 291 return; 292 } 293 294 /* ARGSUSED */ 295 static int 296 virtio_pci_rescan(device_t self, const char *ifattr, const int *locs) 297 { 298 struct virtio_pci_softc * const psc = device_private(self); 299 struct virtio_softc * const sc = &psc->sc_sc; 300 struct virtio_attach_args va; 301 302 if (sc->sc_child) /* Child already attached? */ 303 return 0; 304 305 memset(&va, 0, sizeof(va)); 306 va.sc_childdevid = sc->sc_childdevid; 307 308 config_found(self, &va, NULL, CFARGS_NONE); 309 310 if (virtio_attach_failed(sc)) 311 return 0; 312 313 return 0; 314 } 315 316 317 static int 318 virtio_pci_detach(device_t self, int flags) 319 { 320 struct virtio_pci_softc * const psc = device_private(self); 321 struct virtio_softc * const sc = &psc->sc_sc; 322 int r; 323 324 if (sc->sc_child != NULL) { 325 r = config_detach(sc->sc_child, flags); 326 if (r) 327 return r; 328 } 329 330 /* Check that child detached properly */ 331 KASSERT(sc->sc_child == NULL); 332 KASSERT(sc->sc_vqs == NULL); 333 KASSERT(psc->sc_ihs_num == 0); 334 335 if (psc->sc_iosize) 336 bus_space_unmap(psc->sc_iot, psc->sc_ioh, 337 psc->sc_mapped_iosize); 338 psc->sc_iosize = 0; 339 340 return 0; 341 } 342 343 344 static int 345 virtio_pci_attach_09(device_t self, void *aux) 346 //struct virtio_pci_softc *psc, struct pci_attach_args *pa) 347 { 348 struct virtio_pci_softc * const psc = device_private(self); 349 struct pci_attach_args *pa = (struct pci_attach_args *)aux; 350 struct virtio_softc * const sc = &psc->sc_sc; 351 // pci_chipset_tag_t pc = pa->pa_pc; 352 // pcitag_t tag = pa->pa_tag; 353 354 /* complete IO region */ 355 if (pci_mapreg_map(pa, PCI_MAPREG_START, PCI_MAPREG_TYPE_IO, 0, 356 &psc->sc_iot, &psc->sc_ioh, NULL, &psc->sc_iosize)) { 357 aprint_error_dev(self, "can't map i/o space\n"); 358 return EIO; 359 } 360 psc->sc_mapped_iosize = psc->sc_iosize; 361 362 /* queue space */ 363 if (bus_space_subregion(psc->sc_iot, psc->sc_ioh, 364 VIRTIO_CONFIG_QUEUE_NOTIFY, 2, &psc->sc_notify_ioh)) { 365 aprint_error_dev(self, "can't map notify i/o space\n"); 366 return EIO; 367 } 368 psc->sc_notify_iosize = 2; 369 psc->sc_notify_iot = psc->sc_iot; 370 371 /* ISR space */ 372 if (bus_space_subregion(psc->sc_iot, psc->sc_ioh, 373 VIRTIO_CONFIG_ISR_STATUS, 1, &psc->sc_isr_ioh)) { 374 aprint_error_dev(self, "can't map isr i/o space\n"); 375 return EIO; 376 } 377 psc->sc_isr_iosize = 1; 378 psc->sc_isr_iot = psc->sc_iot; 379 380 /* set our version 0.9 ops */ 381 sc->sc_ops = &virtio_pci_ops_09; 382 sc->sc_bus_endian = READ_ENDIAN_09; 383 sc->sc_struct_endian = STRUCT_ENDIAN_09; 384 return 0; 385 } 386 387 388 static int 389 virtio_pci_attach_10(device_t self, void *aux) 390 { 391 struct virtio_pci_softc * const psc = device_private(self); 392 struct pci_attach_args *pa = (struct pci_attach_args *)aux; 393 struct virtio_softc * const sc = &psc->sc_sc; 394 pci_chipset_tag_t pc = pa->pa_pc; 395 pcitag_t tag = pa->pa_tag; 396 397 struct virtio_pci_cap common, isr, device; 398 struct virtio_pci_notify_cap notify; 399 int have_device_cfg = 0; 400 bus_size_t bars[NMAPREG] = { 0 }; 401 int bars_idx[NMAPREG] = { 0 }; 402 struct virtio_pci_cap *caps[] = { &common, &isr, &device, ¬ify.cap }; 403 int i, j, ret = 0; 404 405 if (virtio_pci_find_cap(psc, VIRTIO_PCI_CAP_COMMON_CFG, 406 &common, sizeof(common))) 407 return ENODEV; 408 if (virtio_pci_find_cap(psc, VIRTIO_PCI_CAP_NOTIFY_CFG, 409 ¬ify, sizeof(notify))) 410 return ENODEV; 411 if (virtio_pci_find_cap(psc, VIRTIO_PCI_CAP_ISR_CFG, 412 &isr, sizeof(isr))) 413 return ENODEV; 414 if (virtio_pci_find_cap(psc, VIRTIO_PCI_CAP_DEVICE_CFG, 415 &device, sizeof(device))) 416 memset(&device, 0, sizeof(device)); 417 else 418 have_device_cfg = 1; 419 420 /* Figure out which bars we need to map */ 421 for (i = 0; i < __arraycount(caps); i++) { 422 int bar = caps[i]->bar; 423 bus_size_t len = caps[i]->offset + caps[i]->length; 424 if (caps[i]->length == 0) 425 continue; 426 if (bars[bar] < len) 427 bars[bar] = len; 428 } 429 430 for (i = j = 0; i < __arraycount(bars); i++) { 431 int reg; 432 pcireg_t type; 433 if (bars[i] == 0) 434 continue; 435 reg = PCI_MAPREG_START + i * 4; 436 type = pci_mapreg_type(pc, tag, reg); 437 if (pci_mapreg_map(pa, reg, type, 0, 438 &psc->sc_bars_iot[j], &psc->sc_bars_ioh[j], 439 NULL, &psc->sc_bars_iosize[j])) { 440 aprint_error_dev(self, "can't map bar %u \n", i); 441 ret = EIO; 442 goto err; 443 } 444 aprint_debug_dev(self, 445 "bar[%d]: iot %p, size 0x%" PRIxBUSSIZE "\n", 446 j, psc->sc_bars_iot[j], psc->sc_bars_iosize[j]); 447 bars_idx[i] = j; 448 j++; 449 } 450 451 i = bars_idx[notify.cap.bar]; 452 if (bus_space_subregion(psc->sc_bars_iot[i], psc->sc_bars_ioh[i], 453 notify.cap.offset, notify.cap.length, 454 &psc->sc_notify_ioh)) { 455 aprint_error_dev(self, "can't map notify i/o space\n"); 456 ret = EIO; 457 goto err; 458 } 459 psc->sc_notify_iosize = notify.cap.length; 460 psc->sc_notify_iot = psc->sc_bars_iot[i]; 461 psc->sc_notify_off_multiplier = le32toh(notify.notify_off_multiplier); 462 463 if (have_device_cfg) { 464 i = bars_idx[device.bar]; 465 if (bus_space_subregion(psc->sc_bars_iot[i], psc->sc_bars_ioh[i], 466 device.offset, device.length, 467 &sc->sc_devcfg_ioh)) { 468 aprint_error_dev(self, "can't map devcfg i/o space\n"); 469 ret = EIO; 470 goto err; 471 } 472 aprint_debug_dev(self, 473 "device.offset = 0x%x, device.length = 0x%x\n", 474 device.offset, device.length); 475 sc->sc_devcfg_iosize = device.length; 476 sc->sc_devcfg_iot = psc->sc_bars_iot[i]; 477 } 478 479 i = bars_idx[isr.bar]; 480 if (bus_space_subregion(psc->sc_bars_iot[i], psc->sc_bars_ioh[i], 481 isr.offset, isr.length, &psc->sc_isr_ioh)) { 482 aprint_error_dev(self, "can't map isr i/o space\n"); 483 ret = EIO; 484 goto err; 485 } 486 psc->sc_isr_iosize = isr.length; 487 psc->sc_isr_iot = psc->sc_bars_iot[i]; 488 489 i = bars_idx[common.bar]; 490 if (bus_space_subregion(psc->sc_bars_iot[i], psc->sc_bars_ioh[i], 491 common.offset, common.length, &psc->sc_ioh)) { 492 aprint_error_dev(self, "can't map common i/o space\n"); 493 ret = EIO; 494 goto err; 495 } 496 psc->sc_iosize = common.length; 497 psc->sc_iot = psc->sc_bars_iot[i]; 498 psc->sc_mapped_iosize = psc->sc_bars_iosize[i]; 499 500 psc->sc_sc.sc_version_1 = 1; 501 502 /* set our version 1.0 ops */ 503 sc->sc_ops = &virtio_pci_ops_10; 504 sc->sc_bus_endian = READ_ENDIAN_10; 505 sc->sc_struct_endian = STRUCT_ENDIAN_10; 506 return 0; 507 508 err: 509 /* undo our pci_mapreg_map()s */ 510 for (i = 0; i < __arraycount(bars); i++) { 511 if (psc->sc_bars_iosize[i] == 0) 512 continue; 513 bus_space_unmap(psc->sc_bars_iot[i], psc->sc_bars_ioh[i], 514 psc->sc_bars_iosize[i]); 515 } 516 return ret; 517 } 518 519 /* v1.0 attach helper */ 520 static int 521 virtio_pci_find_cap(struct virtio_pci_softc *psc, int cfg_type, void *buf, int buflen) 522 { 523 device_t self = psc->sc_sc.sc_dev; 524 pci_chipset_tag_t pc = psc->sc_pa.pa_pc; 525 pcitag_t tag = psc->sc_pa.pa_tag; 526 unsigned int offset, i, len; 527 union { 528 pcireg_t reg[8]; 529 struct virtio_pci_cap vcap; 530 } *v = buf; 531 532 if (buflen < sizeof(struct virtio_pci_cap)) 533 return ERANGE; 534 535 if (!pci_get_capability(pc, tag, PCI_CAP_VENDSPEC, &offset, &v->reg[0])) 536 return ENOENT; 537 538 do { 539 for (i = 0; i < 4; i++) 540 v->reg[i] = 541 le32toh(pci_conf_read(pc, tag, offset + i * 4)); 542 if (v->vcap.cfg_type == cfg_type) 543 break; 544 offset = v->vcap.cap_next; 545 } while (offset != 0); 546 547 if (offset == 0) 548 return ENOENT; 549 550 if (v->vcap.cap_len > sizeof(struct virtio_pci_cap)) { 551 len = roundup(v->vcap.cap_len, sizeof(pcireg_t)); 552 if (len > buflen) { 553 aprint_error_dev(self, "%s cap too large\n", __func__); 554 return ERANGE; 555 } 556 for (i = 4; i < len / sizeof(pcireg_t); i++) 557 v->reg[i] = 558 le32toh(pci_conf_read(pc, tag, offset + i * 4)); 559 } 560 561 /* endian fixup */ 562 v->vcap.offset = le32toh(v->vcap.offset); 563 v->vcap.length = le32toh(v->vcap.length); 564 return 0; 565 } 566 567 568 /* ------------------------------------- 569 * Version 0.9 support 570 * -------------------------------------*/ 571 572 static void 573 virtio_pci_kick_09(struct virtio_softc *sc, uint16_t idx) 574 { 575 struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc; 576 577 bus_space_write_2(psc->sc_notify_iot, psc->sc_notify_ioh, 0, idx); 578 } 579 580 /* only applicable for v 0.9 but also called for 1.0 */ 581 static int 582 virtio_pci_adjust_config_region(struct virtio_pci_softc *psc) 583 { 584 struct virtio_softc * const sc = (struct virtio_softc *) psc; 585 device_t self = psc->sc_sc.sc_dev; 586 587 if (psc->sc_sc.sc_version_1) 588 return 0; 589 590 sc->sc_devcfg_iosize = psc->sc_iosize - psc->sc_devcfg_offset; 591 sc->sc_devcfg_iot = psc->sc_iot; 592 if (bus_space_subregion(psc->sc_iot, psc->sc_ioh, 593 psc->sc_devcfg_offset, sc->sc_devcfg_iosize, 594 &sc->sc_devcfg_ioh)) { 595 aprint_error_dev(self, "can't map config i/o space\n"); 596 return EIO; 597 } 598 599 return 0; 600 } 601 602 static uint16_t 603 virtio_pci_read_queue_size_09(struct virtio_softc *sc, uint16_t idx) 604 { 605 struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc; 606 607 bus_space_write_2(psc->sc_iot, psc->sc_ioh, 608 VIRTIO_CONFIG_QUEUE_SELECT, idx); 609 return bus_space_read_2(psc->sc_iot, psc->sc_ioh, 610 VIRTIO_CONFIG_QUEUE_SIZE); 611 } 612 613 static void 614 virtio_pci_setup_queue_09(struct virtio_softc *sc, uint16_t idx, uint64_t addr) 615 { 616 struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc; 617 618 bus_space_write_2(psc->sc_iot, psc->sc_ioh, 619 VIRTIO_CONFIG_QUEUE_SELECT, idx); 620 bus_space_write_4(psc->sc_iot, psc->sc_ioh, 621 VIRTIO_CONFIG_QUEUE_ADDRESS, addr / VIRTIO_PAGE_SIZE); 622 623 if (psc->sc_ihs_num > 1) { 624 int vec = VIRTIO_MSIX_QUEUE_VECTOR_INDEX; 625 if (sc->sc_child_mq) 626 vec += idx; 627 bus_space_write_2(psc->sc_iot, psc->sc_ioh, 628 VIRTIO_CONFIG_MSI_QUEUE_VECTOR, vec); 629 } 630 } 631 632 static void 633 virtio_pci_set_status_09(struct virtio_softc *sc, int status) 634 { 635 struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc; 636 int old = 0; 637 638 if (status != 0) { 639 old = bus_space_read_1(psc->sc_iot, psc->sc_ioh, 640 VIRTIO_CONFIG_DEVICE_STATUS); 641 } 642 bus_space_write_1(psc->sc_iot, psc->sc_ioh, 643 VIRTIO_CONFIG_DEVICE_STATUS, status|old); 644 } 645 646 static void 647 virtio_pci_negotiate_features_09(struct virtio_softc *sc, uint64_t guest_features) 648 { 649 struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc; 650 uint32_t r; 651 652 r = bus_space_read_4(psc->sc_iot, psc->sc_ioh, 653 VIRTIO_CONFIG_DEVICE_FEATURES); 654 655 r &= guest_features; 656 657 bus_space_write_4(psc->sc_iot, psc->sc_ioh, 658 VIRTIO_CONFIG_GUEST_FEATURES, r); 659 660 sc->sc_active_features = r; 661 } 662 663 /* ------------------------------------- 664 * Version 1.0 support 665 * -------------------------------------*/ 666 667 static void 668 virtio_pci_kick_10(struct virtio_softc *sc, uint16_t idx) 669 { 670 struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc; 671 unsigned offset = sc->sc_vqs[idx].vq_notify_off * 672 psc->sc_notify_off_multiplier; 673 674 bus_space_write_2(psc->sc_notify_iot, psc->sc_notify_ioh, offset, idx); 675 } 676 677 678 static uint16_t 679 virtio_pci_read_queue_size_10(struct virtio_softc *sc, uint16_t idx) 680 { 681 struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc; 682 bus_space_tag_t iot = psc->sc_iot; 683 bus_space_handle_t ioh = psc->sc_ioh; 684 685 bus_space_write_2(iot, ioh, VIRTIO_CONFIG1_QUEUE_SELECT, idx); 686 return bus_space_read_2(iot, ioh, VIRTIO_CONFIG1_QUEUE_SIZE); 687 } 688 689 /* 690 * By definition little endian only in v1.0 and 8 byters are allowed to be 691 * written as two 4 byters 692 * 693 * This is not a general purpose function that can be used in any 694 * driver. Virtio specifically allows the 8 byte bus transaction 695 * to be split into two 4 byte transactions. Do not copy/use it 696 * in other device drivers unless you know that the device accepts it. 697 */ 698 static __inline void 699 virtio_pci_bus_space_write_8(bus_space_tag_t iot, bus_space_handle_t ioh, 700 bus_size_t offset, uint64_t value) 701 { 702 #if defined(__HAVE_BUS_SPACE_8) 703 bus_space_write_8(iot, ioh, offset, value); 704 #elif _QUAD_HIGHWORD 705 bus_space_write_4(iot, ioh, offset, BUS_ADDR_LO32(value)); 706 bus_space_write_4(iot, ioh, offset + 4, BUS_ADDR_HI32(value)); 707 #else 708 bus_space_write_4(iot, ioh, offset, BUS_ADDR_HI32(value)); 709 bus_space_write_4(iot, ioh, offset + 4, BUS_ADDR_LO32(value)); 710 #endif 711 } 712 713 static void 714 virtio_pci_setup_queue_10(struct virtio_softc *sc, uint16_t idx, uint64_t addr) 715 { 716 struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc; 717 struct virtqueue *vq = &sc->sc_vqs[idx]; 718 bus_space_tag_t iot = psc->sc_iot; 719 bus_space_handle_t ioh = psc->sc_ioh; 720 KASSERT(vq->vq_index == idx); 721 722 bus_space_write_2(iot, ioh, VIRTIO_CONFIG1_QUEUE_SELECT, vq->vq_index); 723 if (addr == 0) { 724 bus_space_write_2(iot, ioh, VIRTIO_CONFIG1_QUEUE_ENABLE, 0); 725 virtio_pci_bus_space_write_8(iot, ioh, 726 VIRTIO_CONFIG1_QUEUE_DESC, 0); 727 virtio_pci_bus_space_write_8(iot, ioh, 728 VIRTIO_CONFIG1_QUEUE_AVAIL, 0); 729 virtio_pci_bus_space_write_8(iot, ioh, 730 VIRTIO_CONFIG1_QUEUE_USED, 0); 731 } else { 732 virtio_pci_bus_space_write_8(iot, ioh, 733 VIRTIO_CONFIG1_QUEUE_DESC, addr); 734 virtio_pci_bus_space_write_8(iot, ioh, 735 VIRTIO_CONFIG1_QUEUE_AVAIL, addr + vq->vq_availoffset); 736 virtio_pci_bus_space_write_8(iot, ioh, 737 VIRTIO_CONFIG1_QUEUE_USED, addr + vq->vq_usedoffset); 738 bus_space_write_2(iot, ioh, 739 VIRTIO_CONFIG1_QUEUE_ENABLE, 1); 740 vq->vq_notify_off = bus_space_read_2(iot, ioh, 741 VIRTIO_CONFIG1_QUEUE_NOTIFY_OFF); 742 } 743 744 if (psc->sc_ihs_num > 1) { 745 int vec = VIRTIO_MSIX_QUEUE_VECTOR_INDEX; 746 if (sc->sc_child_mq) 747 vec += idx; 748 bus_space_write_2(iot, ioh, 749 VIRTIO_CONFIG1_QUEUE_MSIX_VECTOR, vec); 750 } 751 } 752 753 static void 754 virtio_pci_set_status_10(struct virtio_softc *sc, int status) 755 { 756 struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc; 757 bus_space_tag_t iot = psc->sc_iot; 758 bus_space_handle_t ioh = psc->sc_ioh; 759 int old = 0; 760 761 if (status) 762 old = bus_space_read_1(iot, ioh, VIRTIO_CONFIG1_DEVICE_STATUS); 763 bus_space_write_1(iot, ioh, VIRTIO_CONFIG1_DEVICE_STATUS, status | old); 764 } 765 766 void 767 virtio_pci_negotiate_features_10(struct virtio_softc *sc, uint64_t guest_features) 768 { 769 struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc; 770 device_t self = sc->sc_dev; 771 bus_space_tag_t iot = psc->sc_iot; 772 bus_space_handle_t ioh = psc->sc_ioh; 773 uint64_t host, negotiated, device_status; 774 775 guest_features |= VIRTIO_F_VERSION_1; 776 /* notify on empty is 0.9 only */ 777 guest_features &= ~VIRTIO_F_NOTIFY_ON_EMPTY; 778 sc->sc_active_features = 0; 779 780 bus_space_write_4(iot, ioh, VIRTIO_CONFIG1_DEVICE_FEATURE_SELECT, 0); 781 host = bus_space_read_4(iot, ioh, VIRTIO_CONFIG1_DEVICE_FEATURE); 782 bus_space_write_4(iot, ioh, VIRTIO_CONFIG1_DEVICE_FEATURE_SELECT, 1); 783 host |= (uint64_t) 784 bus_space_read_4(iot, ioh, VIRTIO_CONFIG1_DEVICE_FEATURE) << 32; 785 786 negotiated = host & guest_features; 787 788 bus_space_write_4(iot, ioh, VIRTIO_CONFIG1_DRIVER_FEATURE_SELECT, 0); 789 bus_space_write_4(iot, ioh, VIRTIO_CONFIG1_DRIVER_FEATURE, 790 negotiated & 0xffffffff); 791 bus_space_write_4(iot, ioh, VIRTIO_CONFIG1_DRIVER_FEATURE_SELECT, 1); 792 bus_space_write_4(iot, ioh, VIRTIO_CONFIG1_DRIVER_FEATURE, 793 negotiated >> 32); 794 virtio_pci_set_status_10(sc, VIRTIO_CONFIG_DEVICE_STATUS_FEATURES_OK); 795 796 device_status = bus_space_read_1(iot, ioh, VIRTIO_CONFIG1_DEVICE_STATUS); 797 if ((device_status & VIRTIO_CONFIG_DEVICE_STATUS_FEATURES_OK) == 0) { 798 aprint_error_dev(self, "feature negotiation failed\n"); 799 bus_space_write_1(iot, ioh, VIRTIO_CONFIG1_DEVICE_STATUS, 800 VIRTIO_CONFIG_DEVICE_STATUS_FAILED); 801 return; 802 } 803 804 if ((negotiated & VIRTIO_F_VERSION_1) == 0) { 805 aprint_error_dev(self, "host rejected version 1\n"); 806 bus_space_write_1(iot, ioh, VIRTIO_CONFIG1_DEVICE_STATUS, 807 VIRTIO_CONFIG_DEVICE_STATUS_FAILED); 808 return; 809 } 810 811 sc->sc_active_features = negotiated; 812 return; 813 } 814 815 816 /* ------------------------------------- 817 * Generic PCI interrupt code 818 * -------------------------------------*/ 819 820 static int 821 virtio_pci_setup_interrupts_10(struct virtio_softc *sc, int reinit) 822 { 823 struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc; 824 bus_space_tag_t iot = psc->sc_iot; 825 bus_space_handle_t ioh = psc->sc_ioh; 826 int vector, ret, qid; 827 828 if (!virtio_pci_msix_enabled(psc)) 829 return 0; 830 831 vector = VIRTIO_MSIX_CONFIG_VECTOR_INDEX; 832 bus_space_write_2(iot, ioh, 833 VIRTIO_CONFIG1_CONFIG_MSIX_VECTOR, vector); 834 ret = bus_space_read_2(iot, ioh, VIRTIO_CONFIG1_CONFIG_MSIX_VECTOR); 835 if (ret != vector) { 836 VIRTIO_PCI_LOG(sc, reinit, 837 "can't set config msix vector\n"); 838 return -1; 839 } 840 841 for (qid = 0; qid < sc->sc_nvqs; qid++) { 842 vector = VIRTIO_MSIX_QUEUE_VECTOR_INDEX; 843 844 if (sc->sc_child_mq) 845 vector += qid; 846 bus_space_write_2(iot, ioh, VIRTIO_CONFIG1_QUEUE_SELECT, qid); 847 bus_space_write_2(iot, ioh, VIRTIO_CONFIG1_QUEUE_MSIX_VECTOR, 848 vector); 849 ret = bus_space_read_2(iot, ioh, 850 VIRTIO_CONFIG1_QUEUE_MSIX_VECTOR); 851 if (ret != vector) { 852 VIRTIO_PCI_LOG(sc, reinit, "can't set queue %d " 853 "msix vector\n", qid); 854 return -1; 855 } 856 } 857 858 return 0; 859 } 860 861 static int 862 virtio_pci_setup_interrupts_09(struct virtio_softc *sc, int reinit) 863 { 864 struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc; 865 int offset, vector, ret, qid; 866 867 if (!virtio_pci_msix_enabled(psc)) 868 return 0; 869 870 offset = VIRTIO_CONFIG_MSI_CONFIG_VECTOR; 871 vector = VIRTIO_MSIX_CONFIG_VECTOR_INDEX; 872 873 bus_space_write_2(psc->sc_iot, psc->sc_ioh, offset, vector); 874 ret = bus_space_read_2(psc->sc_iot, psc->sc_ioh, offset); 875 aprint_debug_dev(sc->sc_dev, "expected=%d, actual=%d\n", 876 vector, ret); 877 if (ret != vector) { 878 VIRTIO_PCI_LOG(sc, reinit, 879 "can't set config msix vector\n"); 880 return -1; 881 } 882 883 for (qid = 0; qid < sc->sc_nvqs; qid++) { 884 offset = VIRTIO_CONFIG_QUEUE_SELECT; 885 bus_space_write_2(psc->sc_iot, psc->sc_ioh, offset, qid); 886 887 offset = VIRTIO_CONFIG_MSI_QUEUE_VECTOR; 888 vector = VIRTIO_MSIX_QUEUE_VECTOR_INDEX; 889 890 if (sc->sc_child_mq) 891 vector += qid; 892 893 bus_space_write_2(psc->sc_iot, psc->sc_ioh, offset, vector); 894 ret = bus_space_read_2(psc->sc_iot, psc->sc_ioh, offset); 895 aprint_debug_dev(sc->sc_dev, "expected=%d, actual=%d\n", 896 vector, ret); 897 if (ret != vector) { 898 VIRTIO_PCI_LOG(sc, reinit, "can't set queue %d " 899 "msix vector\n", qid); 900 return -1; 901 } 902 } 903 904 return 0; 905 } 906 907 static int 908 virtio_pci_establish_msix_interrupts(struct virtio_softc *sc, 909 struct pci_attach_args *pa) 910 { 911 struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc; 912 device_t self = sc->sc_dev; 913 pci_chipset_tag_t pc = pa->pa_pc; 914 struct virtqueue *vq; 915 char intrbuf[PCI_INTRSTR_LEN]; 916 char intr_xname[INTRDEVNAMEBUF]; 917 char const *intrstr; 918 int idx, qid, n; 919 920 idx = VIRTIO_MSIX_CONFIG_VECTOR_INDEX; 921 if (sc->sc_flags & VIRTIO_F_INTR_MPSAFE) 922 pci_intr_setattr(pc, &psc->sc_ihp[idx], PCI_INTR_MPSAFE, true); 923 924 snprintf(intr_xname, sizeof(intr_xname), "%s config", 925 device_xname(sc->sc_dev)); 926 927 psc->sc_ihs[idx] = pci_intr_establish_xname(pc, psc->sc_ihp[idx], 928 sc->sc_ipl, virtio_pci_msix_config_intr, sc, intr_xname); 929 if (psc->sc_ihs[idx] == NULL) { 930 aprint_error_dev(self, "couldn't establish MSI-X for config\n"); 931 goto error; 932 } 933 934 idx = VIRTIO_MSIX_QUEUE_VECTOR_INDEX; 935 if (sc->sc_child_mq) { 936 for (qid = 0; qid < sc->sc_nvqs; qid++) { 937 n = idx + qid; 938 vq = &sc->sc_vqs[qid]; 939 940 snprintf(intr_xname, sizeof(intr_xname), "%s vq#%d", 941 device_xname(sc->sc_dev), qid); 942 943 if (sc->sc_flags & VIRTIO_F_INTR_MPSAFE) { 944 pci_intr_setattr(pc, &psc->sc_ihp[n], 945 PCI_INTR_MPSAFE, true); 946 } 947 948 psc->sc_ihs[n] = pci_intr_establish_xname(pc, psc->sc_ihp[n], 949 sc->sc_ipl, vq->vq_intrhand, vq->vq_intrhand_arg, intr_xname); 950 if (psc->sc_ihs[n] == NULL) { 951 aprint_error_dev(self, "couldn't establish MSI-X for a vq\n"); 952 goto error; 953 } 954 } 955 } else { 956 if (sc->sc_flags & VIRTIO_F_INTR_MPSAFE) 957 pci_intr_setattr(pc, &psc->sc_ihp[idx], PCI_INTR_MPSAFE, true); 958 959 snprintf(intr_xname, sizeof(intr_xname), "%s queues", 960 device_xname(sc->sc_dev)); 961 psc->sc_ihs[idx] = pci_intr_establish_xname(pc, psc->sc_ihp[idx], 962 sc->sc_ipl, virtio_pci_msix_queue_intr, sc, intr_xname); 963 if (psc->sc_ihs[idx] == NULL) { 964 aprint_error_dev(self, "couldn't establish MSI-X for queues\n"); 965 goto error; 966 } 967 } 968 969 idx = VIRTIO_MSIX_CONFIG_VECTOR_INDEX; 970 intrstr = pci_intr_string(pc, psc->sc_ihp[idx], intrbuf, sizeof(intrbuf)); 971 aprint_normal_dev(self, "config interrupting at %s\n", intrstr); 972 idx = VIRTIO_MSIX_QUEUE_VECTOR_INDEX; 973 if (sc->sc_child_mq) { 974 kcpuset_t *affinity; 975 int affinity_to, r; 976 977 kcpuset_create(&affinity, false); 978 979 for (qid = 0; qid < sc->sc_nvqs; qid++) { 980 n = idx + qid; 981 affinity_to = (qid / 2) % ncpu; 982 983 intrstr = pci_intr_string(pc, psc->sc_ihp[n], 984 intrbuf, sizeof(intrbuf)); 985 986 kcpuset_zero(affinity); 987 kcpuset_set(affinity, affinity_to); 988 r = interrupt_distribute(psc->sc_ihs[n], affinity, NULL); 989 if (r == 0) { 990 aprint_normal_dev(self, 991 "for vq #%d interrupting at %s affinity to %u\n", 992 qid, intrstr, affinity_to); 993 } else { 994 aprint_normal_dev(self, 995 "for vq #%d interrupting at %s\n", 996 qid, intrstr); 997 } 998 } 999 1000 kcpuset_destroy(affinity); 1001 } else { 1002 intrstr = pci_intr_string(pc, psc->sc_ihp[idx], intrbuf, sizeof(intrbuf)); 1003 aprint_normal_dev(self, "queues interrupting at %s\n", intrstr); 1004 } 1005 1006 return 0; 1007 1008 error: 1009 idx = VIRTIO_MSIX_CONFIG_VECTOR_INDEX; 1010 if (psc->sc_ihs[idx] != NULL) 1011 pci_intr_disestablish(psc->sc_pa.pa_pc, psc->sc_ihs[idx]); 1012 idx = VIRTIO_MSIX_QUEUE_VECTOR_INDEX; 1013 if (sc->sc_child_mq) { 1014 for (qid = 0; qid < sc->sc_nvqs; qid++) { 1015 n = idx + qid; 1016 if (psc->sc_ihs[n] == NULL) 1017 continue; 1018 pci_intr_disestablish(psc->sc_pa.pa_pc, psc->sc_ihs[n]); 1019 } 1020 1021 } else { 1022 if (psc->sc_ihs[idx] != NULL) 1023 pci_intr_disestablish(psc->sc_pa.pa_pc, psc->sc_ihs[idx]); 1024 } 1025 1026 return -1; 1027 } 1028 1029 static int 1030 virtio_pci_establish_intx_interrupt(struct virtio_softc *sc, 1031 struct pci_attach_args *pa) 1032 { 1033 struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc; 1034 device_t self = sc->sc_dev; 1035 pci_chipset_tag_t pc = pa->pa_pc; 1036 char intrbuf[PCI_INTRSTR_LEN]; 1037 char const *intrstr; 1038 1039 if (sc->sc_flags & VIRTIO_F_INTR_MPSAFE) 1040 pci_intr_setattr(pc, &psc->sc_ihp[0], PCI_INTR_MPSAFE, true); 1041 1042 psc->sc_ihs[0] = pci_intr_establish_xname(pc, psc->sc_ihp[0], 1043 sc->sc_ipl, virtio_pci_intr, sc, device_xname(sc->sc_dev)); 1044 if (psc->sc_ihs[0] == NULL) { 1045 aprint_error_dev(self, "couldn't establish INTx\n"); 1046 return -1; 1047 } 1048 1049 intrstr = pci_intr_string(pc, psc->sc_ihp[0], intrbuf, sizeof(intrbuf)); 1050 aprint_normal_dev(self, "interrupting at %s\n", intrstr); 1051 1052 return 0; 1053 } 1054 1055 static int 1056 virtio_pci_alloc_interrupts(struct virtio_softc *sc) 1057 { 1058 struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc; 1059 device_t self = sc->sc_dev; 1060 pci_chipset_tag_t pc = psc->sc_pa.pa_pc; 1061 pcitag_t tag = psc->sc_pa.pa_tag; 1062 int error; 1063 int nmsix; 1064 int off; 1065 int counts[PCI_INTR_TYPE_SIZE]; 1066 pci_intr_type_t max_type; 1067 pcireg_t ctl; 1068 1069 nmsix = pci_msix_count(psc->sc_pa.pa_pc, psc->sc_pa.pa_tag); 1070 aprint_debug_dev(self, "pci_msix_count=%d\n", nmsix); 1071 1072 /* We need at least two: one for config and the other for queues */ 1073 if ((sc->sc_flags & VIRTIO_F_INTR_MSIX) == 0 || nmsix < 2) { 1074 /* Try INTx only */ 1075 max_type = PCI_INTR_TYPE_INTX; 1076 counts[PCI_INTR_TYPE_INTX] = 1; 1077 } else { 1078 /* Try MSI-X first and INTx second */ 1079 if (sc->sc_nvqs + VIRTIO_MSIX_QUEUE_VECTOR_INDEX <= nmsix) { 1080 nmsix = sc->sc_nvqs + VIRTIO_MSIX_QUEUE_VECTOR_INDEX; 1081 } else { 1082 sc->sc_child_mq = false; 1083 } 1084 1085 if (sc->sc_child_mq == false) { 1086 nmsix = 2; 1087 } 1088 1089 max_type = PCI_INTR_TYPE_MSIX; 1090 counts[PCI_INTR_TYPE_MSIX] = nmsix; 1091 counts[PCI_INTR_TYPE_MSI] = 0; 1092 counts[PCI_INTR_TYPE_INTX] = 1; 1093 } 1094 1095 retry: 1096 error = pci_intr_alloc(&psc->sc_pa, &psc->sc_ihp, counts, max_type); 1097 if (error != 0) { 1098 aprint_error_dev(self, "couldn't map interrupt\n"); 1099 return -1; 1100 } 1101 1102 if (pci_intr_type(pc, psc->sc_ihp[0]) == PCI_INTR_TYPE_MSIX) { 1103 psc->sc_ihs = kmem_zalloc(sizeof(*psc->sc_ihs) * nmsix, 1104 KM_SLEEP); 1105 1106 error = virtio_pci_establish_msix_interrupts(sc, &psc->sc_pa); 1107 if (error != 0) { 1108 kmem_free(psc->sc_ihs, sizeof(*psc->sc_ihs) * nmsix); 1109 pci_intr_release(pc, psc->sc_ihp, nmsix); 1110 1111 /* Retry INTx */ 1112 max_type = PCI_INTR_TYPE_INTX; 1113 counts[PCI_INTR_TYPE_INTX] = 1; 1114 goto retry; 1115 } 1116 1117 psc->sc_ihs_num = nmsix; 1118 psc->sc_devcfg_offset = VIRTIO_CONFIG_DEVICE_CONFIG_MSI; 1119 virtio_pci_adjust_config_region(psc); 1120 } else if (pci_intr_type(pc, psc->sc_ihp[0]) == PCI_INTR_TYPE_INTX) { 1121 psc->sc_ihs = kmem_zalloc(sizeof(*psc->sc_ihs) * 1, 1122 KM_SLEEP); 1123 1124 error = virtio_pci_establish_intx_interrupt(sc, &psc->sc_pa); 1125 if (error != 0) { 1126 kmem_free(psc->sc_ihs, sizeof(*psc->sc_ihs) * 1); 1127 pci_intr_release(pc, psc->sc_ihp, 1); 1128 return -1; 1129 } 1130 1131 psc->sc_ihs_num = 1; 1132 psc->sc_devcfg_offset = VIRTIO_CONFIG_DEVICE_CONFIG_NOMSI; 1133 virtio_pci_adjust_config_region(psc); 1134 1135 error = pci_get_capability(pc, tag, PCI_CAP_MSIX, &off, NULL); 1136 if (error != 0) { 1137 ctl = pci_conf_read(pc, tag, off + PCI_MSIX_CTL); 1138 ctl &= ~PCI_MSIX_CTL_ENABLE; 1139 pci_conf_write(pc, tag, off + PCI_MSIX_CTL, ctl); 1140 } 1141 } 1142 1143 return 0; 1144 } 1145 1146 static void 1147 virtio_pci_free_interrupts(struct virtio_softc *sc) 1148 { 1149 struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc; 1150 1151 for (int i = 0; i < psc->sc_ihs_num; i++) { 1152 if (psc->sc_ihs[i] == NULL) 1153 continue; 1154 pci_intr_disestablish(psc->sc_pa.pa_pc, psc->sc_ihs[i]); 1155 psc->sc_ihs[i] = NULL; 1156 } 1157 1158 if (psc->sc_ihs_num > 0) 1159 pci_intr_release(psc->sc_pa.pa_pc, psc->sc_ihp, psc->sc_ihs_num); 1160 1161 if (psc->sc_ihs != NULL) { 1162 kmem_free(psc->sc_ihs, sizeof(*psc->sc_ihs) * psc->sc_ihs_num); 1163 psc->sc_ihs = NULL; 1164 } 1165 psc->sc_ihs_num = 0; 1166 } 1167 1168 static bool 1169 virtio_pci_msix_enabled(struct virtio_pci_softc *psc) 1170 { 1171 pci_chipset_tag_t pc = psc->sc_pa.pa_pc; 1172 1173 if (pci_intr_type(pc, psc->sc_ihp[0]) == PCI_INTR_TYPE_MSIX) 1174 return true; 1175 1176 return false; 1177 } 1178 1179 /* 1180 * Interrupt handler. 1181 */ 1182 static int 1183 virtio_pci_intr(void *arg) 1184 { 1185 struct virtio_softc *sc = arg; 1186 struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc; 1187 int isr, r = 0; 1188 1189 /* check and ack the interrupt */ 1190 isr = bus_space_read_1(psc->sc_isr_iot, psc->sc_isr_ioh, 0); 1191 if (isr == 0) 1192 return 0; 1193 if ((isr & VIRTIO_CONFIG_ISR_CONFIG_CHANGE) && 1194 (sc->sc_config_change != NULL)) 1195 r = (sc->sc_config_change)(sc); 1196 if (sc->sc_intrhand != NULL) { 1197 if (sc->sc_soft_ih != NULL) 1198 softint_schedule(sc->sc_soft_ih); 1199 else 1200 r |= (sc->sc_intrhand)(sc); 1201 } 1202 1203 return r; 1204 } 1205 1206 static int 1207 virtio_pci_msix_queue_intr(void *arg) 1208 { 1209 struct virtio_softc *sc = arg; 1210 int r = 0; 1211 1212 if (sc->sc_intrhand != NULL) { 1213 if (sc->sc_soft_ih != NULL) 1214 softint_schedule(sc->sc_soft_ih); 1215 else 1216 r |= (sc->sc_intrhand)(sc); 1217 } 1218 1219 return r; 1220 } 1221 1222 static int 1223 virtio_pci_msix_config_intr(void *arg) 1224 { 1225 struct virtio_softc *sc = arg; 1226 int r = 0; 1227 1228 if (sc->sc_config_change != NULL) 1229 r = (sc->sc_config_change)(sc); 1230 return r; 1231 } 1232 1233 MODULE(MODULE_CLASS_DRIVER, virtio_pci, "pci,virtio"); 1234 1235 #ifdef _MODULE 1236 #include "ioconf.c" 1237 #endif 1238 1239 static int 1240 virtio_pci_modcmd(modcmd_t cmd, void *opaque) 1241 { 1242 int error = 0; 1243 1244 #ifdef _MODULE 1245 switch (cmd) { 1246 case MODULE_CMD_INIT: 1247 error = config_init_component(cfdriver_ioconf_virtio_pci, 1248 cfattach_ioconf_virtio_pci, cfdata_ioconf_virtio_pci); 1249 break; 1250 case MODULE_CMD_FINI: 1251 error = config_fini_component(cfdriver_ioconf_virtio_pci, 1252 cfattach_ioconf_virtio_pci, cfdata_ioconf_virtio_pci); 1253 break; 1254 default: 1255 error = ENOTTY; 1256 break; 1257 } 1258 #endif 1259 1260 return error; 1261 } 1262
Indexes created Tue Sep 23 02:09:52 GMT 2025