1 /* $NetBSD: virtio_pci.c,v 1.56 2026/05/13 02:40:56 riastradh Exp $ */ 2 3 /* 4 * Copyright (c) 2020 The NetBSD Foundation, Inc. 5 * Copyright (c) 2012 Stefan Fritsch. 6 * Copyright (c) 2010 Minoura Makoto. 7 * All rights reserved. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 19 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 20 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 21 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 22 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 23 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 27 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 */ 29 30 #include <sys/cdefs.h> 31 __KERNEL_RCSID(0, "$NetBSD: virtio_pci.c,v 1.56 2026/05/13 02:40:56 riastradh Exp $"); 32 33 #include <sys/param.h> 34 #include <sys/types.h> 35 36 #include <sys/device.h> 37 #include <sys/endian.h> 38 #include <sys/interrupt.h> 39 #include <sys/kmem.h> 40 #include <sys/module.h> 41 #include <sys/syslog.h> 42 #include <sys/systm.h> 43 44 #include <dev/pci/pcidevs.h> 45 #include <dev/pci/pcireg.h> 46 #include <dev/pci/pcivar.h> 47 48 #include <dev/pci/virtioreg.h> /* XXX: move to non-pci */ 49 #include <dev/pci/virtio_pcireg.h> 50 51 #define VIRTIO_PRIVATE 52 #include <dev/pci/virtiovar.h> /* XXX: move to non-pci */ 53 54 #if defined(__alpha__) || defined(__sparc64__) 55 /* 56 * XXX VIRTIO_F_ACCESS_PLATFORM is required for standard PCI DMA 57 * XXX to work on these platforms, at least by Qemu. 58 * XXX 59 * XXX Generalize this later. 60 */ 61 #define __NEED_VIRTIO_F_ACCESS_PLATFORM 62 #endif /* __alpha__ || __sparc64__ */ 63 64 #define VIRTIO_PCI_LOG(_sc, _use_log, _fmt, _args...) \ 65 do { \ 66 if ((_use_log)) { \ 67 log(LOG_DEBUG, "%s: " _fmt, \ 68 device_xname((_sc)->sc_dev), \ 69 ##_args); \ 70 } else { \ 71 aprint_error_dev((_sc)->sc_dev, \ 72 _fmt, ##_args); \ 73 } \ 74 } while(0) 75 76 static int virtio_pci_match(device_t, cfdata_t, void *); 77 static void virtio_pci_attach(device_t, device_t, void *); 78 static int virtio_pci_rescan(device_t, const char *, const int *); 79 static int virtio_pci_detach(device_t, int); 80 81 #define NMAPREG ((PCI_MAPREG_END - PCI_MAPREG_START) / \ 82 sizeof(pcireg_t)) 83 struct virtio_pci_softc { 84 struct virtio_softc sc_sc; 85 bool sc_intr_pervq; 86 87 /* IO space */ 88 bus_space_tag_t sc_iot; 89 bus_space_handle_t sc_ioh; 90 bus_size_t sc_iosize; 91 92 /* BARs */ 93 bus_space_tag_t sc_bars_iot[NMAPREG]; 94 bus_space_handle_t sc_bars_ioh[NMAPREG]; 95 bus_size_t sc_bars_iosize[NMAPREG]; 96 97 /* notify space */ 98 bus_space_tag_t sc_notify_iot; 99 bus_space_handle_t sc_notify_ioh; 100 bus_size_t sc_notify_iosize; 101 uint32_t sc_notify_off_multiplier; 102 103 /* isr space */ 104 bus_space_tag_t sc_isr_iot; 105 bus_space_handle_t sc_isr_ioh; 106 bus_size_t sc_isr_iosize; 107 108 /* generic */ 109 struct pci_attach_args sc_pa; 110 pci_intr_handle_t *sc_ihp; 111 void **sc_ihs; 112 int sc_ihs_num; 113 int sc_devcfg_offset; /* for 0.9 */ 114 }; 115 116 static int virtio_pci_attach_09(device_t, void *); 117 static void virtio_pci_kick_09(struct virtio_softc *, uint16_t); 118 static uint16_t virtio_pci_read_queue_size_09(struct virtio_softc *, uint16_t); 119 static void virtio_pci_setup_queue_09(struct virtio_softc *, uint16_t, 120 uint64_t); 121 static void virtio_pci_set_status_09(struct virtio_softc *, int); 122 static void virtio_pci_negotiate_features_09(struct virtio_softc *, 123 uint64_t); 124 125 static int virtio_pci_attach_10(device_t, void *); 126 static void virtio_pci_kick_10(struct virtio_softc *, uint16_t); 127 static uint16_t virtio_pci_read_queue_size_10(struct virtio_softc *, uint16_t); 128 static void virtio_pci_setup_queue_10(struct virtio_softc *, uint16_t, 129 uint64_t); 130 static void virtio_pci_set_status_10(struct virtio_softc *, int); 131 static void virtio_pci_negotiate_features_10(struct virtio_softc *, 132 uint64_t); 133 static int virtio_pci_find_cap(struct virtio_pci_softc *, int, void *, 134 int); 135 136 static int virtio_pci_alloc_interrupts(struct virtio_softc *); 137 static void virtio_pci_free_interrupts(struct virtio_softc *); 138 static int virtio_pci_adjust_config_region(struct virtio_pci_softc *); 139 static int virtio_pci_intr(void *); 140 static int virtio_pci_msix_queue_intr(void *); 141 static int virtio_pci_msix_config_intr(void *); 142 static int virtio_pci_setup_interrupts_09(struct virtio_softc *, int); 143 static int virtio_pci_setup_interrupts_10(struct virtio_softc *, int); 144 static int virtio_pci_establish_msix_interrupts(struct virtio_softc *, 145 const struct pci_attach_args *); 146 static int virtio_pci_establish_intx_interrupt(struct virtio_softc *, 147 const struct pci_attach_args *); 148 static bool virtio_pci_msix_enabled(struct virtio_pci_softc *); 149 150 #define VIRTIO_MSIX_CONFIG_VECTOR_INDEX 0 151 #define VIRTIO_MSIX_QUEUE_VECTOR_INDEX 1 152 153 /* 154 * For big-endian aarch64/armv7 on QEMU (and most real HW), only CPU cores 155 * are running in big-endian mode, with all peripheral being configured to 156 * little-endian mode. Their default bus_space(9) functions forcibly swap 157 * byte-order. This guarantees that PIO'ed data from pci(4), e.g., are 158 * correctly handled by bus_space(9), while DMA'ed ones should be swapped 159 * by hand, in violation of virtio(4) specifications. 160 */ 161 162 #if (defined(__aarch64__) || defined(__arm__)) && BYTE_ORDER == BIG_ENDIAN 163 # define READ_ENDIAN_09 BIG_ENDIAN 164 # define READ_ENDIAN_10 BIG_ENDIAN 165 # define STRUCT_ENDIAN_09 BIG_ENDIAN 166 # define STRUCT_ENDIAN_10 LITTLE_ENDIAN 167 #elif BYTE_ORDER == BIG_ENDIAN 168 # define READ_ENDIAN_09 LITTLE_ENDIAN 169 # define READ_ENDIAN_10 BIG_ENDIAN 170 # define STRUCT_ENDIAN_09 BIG_ENDIAN 171 # define STRUCT_ENDIAN_10 LITTLE_ENDIAN 172 #else /* little endian */ 173 # define READ_ENDIAN_09 LITTLE_ENDIAN 174 # define READ_ENDIAN_10 LITTLE_ENDIAN 175 # define STRUCT_ENDIAN_09 LITTLE_ENDIAN 176 # define STRUCT_ENDIAN_10 LITTLE_ENDIAN 177 #endif 178 179 CFATTACH_DECL3_NEW(virtio_pci, sizeof(struct virtio_pci_softc), 180 virtio_pci_match, virtio_pci_attach, virtio_pci_detach, NULL, 181 virtio_pci_rescan, NULL, 0); 182 183 static const struct virtio_ops virtio_pci_ops_09 = { 184 .kick = virtio_pci_kick_09, 185 .read_queue_size = virtio_pci_read_queue_size_09, 186 .setup_queue = virtio_pci_setup_queue_09, 187 .set_status = virtio_pci_set_status_09, 188 .neg_features = virtio_pci_negotiate_features_09, 189 .alloc_interrupts = virtio_pci_alloc_interrupts, 190 .free_interrupts = virtio_pci_free_interrupts, 191 .setup_interrupts = virtio_pci_setup_interrupts_09, 192 }; 193 194 static const struct virtio_ops virtio_pci_ops_10 = { 195 .kick = virtio_pci_kick_10, 196 .read_queue_size = virtio_pci_read_queue_size_10, 197 .setup_queue = virtio_pci_setup_queue_10, 198 .set_status = virtio_pci_set_status_10, 199 .neg_features = virtio_pci_negotiate_features_10, 200 .alloc_interrupts = virtio_pci_alloc_interrupts, 201 .free_interrupts = virtio_pci_free_interrupts, 202 .setup_interrupts = virtio_pci_setup_interrupts_10, 203 }; 204 205 static int 206 virtio_pci_match(device_t parent, cfdata_t match, void *aux) 207 { 208 const struct pci_attach_args * const pa = aux; 209 210 switch (PCI_VENDOR(pa->pa_id)) { 211 case PCI_VENDOR_QUMRANET: 212 /* Transitional devices MUST have a PCI Revision ID of 0. */ 213 if (((PCI_PRODUCT_QUMRANET_VIRTIO_1000 <= 214 PCI_PRODUCT(pa->pa_id)) && 215 (PCI_PRODUCT(pa->pa_id) <= 216 PCI_PRODUCT_QUMRANET_VIRTIO_103F)) && 217 PCI_REVISION(pa->pa_class) == 0) 218 return 1; 219 /* 220 * Non-transitional devices SHOULD have a PCI Revision 221 * ID of 1 or higher. Drivers MUST match any PCI 222 * Revision ID value. 223 */ 224 if (((PCI_PRODUCT_QUMRANET_VIRTIO_1040 <= 225 PCI_PRODUCT(pa->pa_id)) && 226 (PCI_PRODUCT(pa->pa_id) <= 227 PCI_PRODUCT_QUMRANET_VIRTIO_107F)) && 228 /* XXX: TODO */ 229 PCI_REVISION(pa->pa_class) == 1) 230 return 1; 231 break; 232 } 233 234 return 0; 235 } 236 237 static void 238 virtio_pci_attach(device_t parent, device_t self, void *aux) 239 { 240 struct virtio_pci_softc * const psc = device_private(self); 241 struct virtio_softc * const sc = &psc->sc_sc; 242 const struct pci_attach_args * const pa = aux; 243 pci_chipset_tag_t pc = pa->pa_pc; 244 pcitag_t tag = pa->pa_tag; 245 int revision; 246 int ret; 247 pcireg_t id; 248 pcireg_t csr; 249 250 revision = PCI_REVISION(pa->pa_class); 251 switch (revision) { 252 case 0: 253 /* subsystem ID shows what I am */ 254 id = PCI_SUBSYS_ID(pci_conf_read(pc, tag, PCI_SUBSYS_ID_REG)); 255 break; 256 case 1: 257 /* pci product number shows what I am */ 258 id = PCI_PRODUCT(pa->pa_id) - PCI_PRODUCT_QUMRANET_VIRTIO_1040; 259 break; 260 default: 261 aprint_normal(": unknown revision 0x%02x; giving up\n", 262 revision); 263 return; 264 } 265 266 aprint_normal("\n"); 267 aprint_naive("\n"); 268 virtio_print_device_type(self, id, revision); 269 270 csr = pci_conf_read(pc, tag, PCI_COMMAND_STATUS_REG); 271 csr |= PCI_COMMAND_MASTER_ENABLE | PCI_COMMAND_IO_ENABLE; 272 pci_conf_write(pc, tag, PCI_COMMAND_STATUS_REG, csr); 273 274 sc->sc_dev = self; 275 psc->sc_pa = *pa; 276 psc->sc_iot = pa->pa_iot; 277 278 sc->sc_dmat = pa->pa_dmat; 279 if (pci_dma64_available(pa)) 280 sc->sc_dmat = pa->pa_dmat64; 281 282 /* attach is dependent on revision */ 283 ret = 0; 284 if (revision == 1) { 285 /* try to attach 1.0 */ 286 ret = virtio_pci_attach_10(self, aux); 287 } 288 if (ret == 0 && revision == 0) { 289 /* 290 * revision 0 means 0.9 only or both 0.9 and 1.0. The 291 * latter are so-called "Transitional Devices". For 292 * those devices, we want to use the 1.0 interface if 293 * possible. 294 * 295 * XXX Currently only on platforms that require 1.0 296 * XXX features, such as VIRTIO_F_ACCESS_PLATFORM. 297 */ 298 #ifdef __NEED_VIRTIO_F_ACCESS_PLATFORM 299 /* First, try to attach 1.0 */ 300 ret = virtio_pci_attach_10(self, aux); 301 if (ret != 0) { 302 aprint_error_dev(self, 303 "VirtIO 1.0 error = %d, falling back to 0.9\n", 304 ret); 305 /* Fall back to 0.9. */ 306 ret = virtio_pci_attach_09(self, aux); 307 } 308 #else 309 ret = virtio_pci_attach_09(self, aux); 310 #endif /* __NEED_VIRTIO_F_ACCESS_PLATFORM */ 311 } 312 if (ret) { 313 aprint_error_dev(self, "cannot attach (%d)\n", ret); 314 return; 315 } 316 KASSERT(sc->sc_ops); 317 318 /* preset config region */ 319 psc->sc_devcfg_offset = VIRTIO_CONFIG_DEVICE_CONFIG_NOMSI; 320 if (virtio_pci_adjust_config_region(psc)) 321 return; 322 323 /* generic */ 324 virtio_device_reset(sc); 325 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_ACK); 326 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_DRIVER); 327 328 sc->sc_childdevid = id; 329 sc->sc_child = NULL; 330 virtio_pci_rescan(self, NULL, NULL); 331 return; 332 } 333 334 /* ARGSUSED */ 335 static int 336 virtio_pci_rescan(device_t self, const char *ifattr, const int *locs) 337 { 338 struct virtio_pci_softc * const psc = device_private(self); 339 struct virtio_softc * const sc = &psc->sc_sc; 340 struct virtio_attach_args va; 341 342 if (sc->sc_child) /* Child already attached? */ 343 return 0; 344 345 memset(&va, 0, sizeof(va)); 346 va.sc_childdevid = sc->sc_childdevid; 347 348 config_found(self, &va, NULL, CFARGS_NONE); 349 350 if (virtio_attach_failed(sc)) 351 return 0; 352 353 return 0; 354 } 355 356 static int 357 virtio_pci_detach(device_t self, int flags) 358 { 359 struct virtio_pci_softc * const psc = device_private(self); 360 struct virtio_softc * const sc = &psc->sc_sc; 361 unsigned i; 362 int r; 363 364 r = config_detach_children(self, flags); 365 if (r != 0) 366 return r; 367 368 /* Check that child never attached, or detached properly */ 369 KASSERT(sc->sc_child == NULL); 370 KASSERT(sc->sc_vqs == NULL); 371 KASSERT(psc->sc_ihs_num == 0); 372 373 if (sc->sc_version_1) { 374 for (i = 0; i < __arraycount(psc->sc_bars_iot); i++) { 375 if (psc->sc_bars_iosize[i] == 0) 376 continue; 377 bus_space_unmap(psc->sc_bars_iot[i], 378 psc->sc_bars_ioh[i], psc->sc_bars_iosize[i]); 379 psc->sc_bars_iosize[i] = 0; 380 } 381 } else { 382 if (psc->sc_iosize) { 383 bus_space_unmap(psc->sc_iot, psc->sc_ioh, 384 psc->sc_iosize); 385 psc->sc_iosize = 0; 386 } 387 } 388 389 return 0; 390 } 391 392 static int 393 virtio_pci_attach_09(device_t self, void *aux) 394 { 395 struct virtio_pci_softc * const psc = device_private(self); 396 const struct pci_attach_args * const pa = aux; 397 struct virtio_softc * const sc = &psc->sc_sc; 398 pcireg_t bar_type; 399 400 /* complete BAR0 region */ 401 bar_type = pci_mapreg_type(pa->pa_pc, pa->pa_tag, PCI_MAPREG_START); 402 if (pci_mapreg_map(pa, PCI_MAPREG_START, bar_type, 0, 403 &psc->sc_iot, &psc->sc_ioh, NULL, &psc->sc_iosize)) { 404 aprint_error_dev(self, "can't map BAR0 (%s)\n", 405 bar_type == PCI_MAPREG_TYPE_IO ? "I/O" : "MEM"); 406 return EIO; 407 } 408 409 /* queue space */ 410 if (bus_space_subregion(psc->sc_iot, psc->sc_ioh, 411 VIRTIO_CONFIG_QUEUE_NOTIFY, 2, &psc->sc_notify_ioh)) { 412 aprint_error_dev(self, "can't map notify i/o space\n"); 413 return EIO; 414 } 415 psc->sc_notify_iosize = 2; 416 psc->sc_notify_iot = psc->sc_iot; 417 418 /* ISR space */ 419 if (bus_space_subregion(psc->sc_iot, psc->sc_ioh, 420 VIRTIO_CONFIG_ISR_STATUS, 1, &psc->sc_isr_ioh)) { 421 aprint_error_dev(self, "can't map isr i/o space\n"); 422 return EIO; 423 } 424 psc->sc_isr_iosize = 1; 425 psc->sc_isr_iot = psc->sc_iot; 426 427 /* set our version 0.9 ops */ 428 sc->sc_ops = &virtio_pci_ops_09; 429 sc->sc_bus_endian = READ_ENDIAN_09; 430 sc->sc_struct_endian = STRUCT_ENDIAN_09; 431 return 0; 432 } 433 434 static int 435 virtio_pci_attach_10(device_t self, void *aux) 436 { 437 struct virtio_pci_softc * const psc = device_private(self); 438 const struct pci_attach_args * const pa = aux; 439 struct virtio_softc * const sc = &psc->sc_sc; 440 const pci_chipset_tag_t pc = pa->pa_pc; 441 const pcitag_t tag = pa->pa_tag; 442 443 struct virtio_pci_cap common, isr, device; 444 struct virtio_pci_notify_cap notify; 445 int have_device_cfg = 0; 446 bus_size_t bars[NMAPREG] = { 0 }; 447 int bars_idx[NMAPREG] = { 0 }; 448 struct virtio_pci_cap * const caps[] = 449 { &common, &isr, &device, ¬ify.cap }; 450 int i, j, ret = 0; 451 452 if (virtio_pci_find_cap(psc, VIRTIO_PCI_CAP_COMMON_CFG, 453 &common, sizeof(common))) 454 return ENODEV; 455 if (virtio_pci_find_cap(psc, VIRTIO_PCI_CAP_NOTIFY_CFG, 456 ¬ify, sizeof(notify))) 457 return ENODEV; 458 if (virtio_pci_find_cap(psc, VIRTIO_PCI_CAP_ISR_CFG, 459 &isr, sizeof(isr))) 460 return ENODEV; 461 if (virtio_pci_find_cap(psc, VIRTIO_PCI_CAP_DEVICE_CFG, 462 &device, sizeof(device))) 463 memset(&device, 0, sizeof(device)); 464 else 465 have_device_cfg = 1; 466 467 /* Figure out which bars we need to map */ 468 for (i = 0; i < __arraycount(caps); i++) { 469 int bar = caps[i]->bar; 470 bus_size_t len = caps[i]->offset + caps[i]->length; 471 472 if (caps[i]->length == 0) 473 continue; 474 if (bars[bar] < len) 475 bars[bar] = len; 476 } 477 478 for (i = j = 0; i < __arraycount(bars); i++) { 479 int reg; 480 pcireg_t type; 481 482 if (bars[i] == 0) 483 continue; 484 reg = PCI_BAR(i); 485 type = pci_mapreg_type(pc, tag, reg); 486 if (pci_mapreg_map(pa, reg, type, 0, 487 &psc->sc_bars_iot[j], &psc->sc_bars_ioh[j], 488 NULL, &psc->sc_bars_iosize[j])) { 489 aprint_error_dev(self, "can't map bar %u \n", i); 490 ret = EIO; 491 goto err; 492 } 493 aprint_debug_dev(self, 494 "bar[%d]: iot %p, size 0x%" PRIxBUSSIZE "\n", 495 j, psc->sc_bars_iot[j], psc->sc_bars_iosize[j]); 496 bars_idx[i] = j; 497 j++; 498 } 499 500 i = bars_idx[notify.cap.bar]; 501 if (bus_space_subregion(psc->sc_bars_iot[i], psc->sc_bars_ioh[i], 502 notify.cap.offset, notify.cap.length, &psc->sc_notify_ioh)) { 503 aprint_error_dev(self, "can't map notify i/o space\n"); 504 ret = EIO; 505 goto err; 506 } 507 psc->sc_notify_iosize = notify.cap.length; 508 psc->sc_notify_iot = psc->sc_bars_iot[i]; 509 psc->sc_notify_off_multiplier = le32toh(notify.notify_off_multiplier); 510 511 if (have_device_cfg) { 512 i = bars_idx[device.bar]; 513 if (bus_space_subregion(psc->sc_bars_iot[i], 514 psc->sc_bars_ioh[i], device.offset, device.length, 515 &sc->sc_devcfg_ioh)) { 516 aprint_error_dev(self, "can't map devcfg i/o space\n"); 517 ret = EIO; 518 goto err; 519 } 520 aprint_debug_dev(self, 521 "device.offset = 0x%x, device.length = 0x%x\n", 522 device.offset, device.length); 523 sc->sc_devcfg_iosize = device.length; 524 sc->sc_devcfg_iot = psc->sc_bars_iot[i]; 525 } 526 527 i = bars_idx[isr.bar]; 528 if (bus_space_subregion(psc->sc_bars_iot[i], psc->sc_bars_ioh[i], 529 isr.offset, isr.length, &psc->sc_isr_ioh)) { 530 aprint_error_dev(self, "can't map isr i/o space\n"); 531 ret = EIO; 532 goto err; 533 } 534 psc->sc_isr_iosize = isr.length; 535 psc->sc_isr_iot = psc->sc_bars_iot[i]; 536 537 i = bars_idx[common.bar]; 538 if (bus_space_subregion(psc->sc_bars_iot[i], psc->sc_bars_ioh[i], 539 common.offset, common.length, &psc->sc_ioh)) { 540 aprint_error_dev(self, "can't map common i/o space\n"); 541 ret = EIO; 542 goto err; 543 } 544 psc->sc_iosize = common.length; 545 psc->sc_iot = psc->sc_bars_iot[i]; 546 547 psc->sc_sc.sc_version_1 = 1; 548 549 /* set our version 1.0 ops */ 550 sc->sc_ops = &virtio_pci_ops_10; 551 sc->sc_bus_endian = READ_ENDIAN_10; 552 sc->sc_struct_endian = STRUCT_ENDIAN_10; 553 return 0; 554 555 err: 556 /* undo our pci_mapreg_map()s */ 557 for (i = 0; i < __arraycount(bars); i++) { 558 if (psc->sc_bars_iosize[i] == 0) 559 continue; 560 bus_space_unmap(psc->sc_bars_iot[i], psc->sc_bars_ioh[i], 561 psc->sc_bars_iosize[i]); 562 psc->sc_bars_iosize[i] = 0; 563 } 564 return ret; 565 } 566 567 /* v1.0 attach helper */ 568 static int 569 virtio_pci_find_cap(struct virtio_pci_softc *psc, int cfg_type, void *buf, 570 int buflen) 571 { 572 device_t self = psc->sc_sc.sc_dev; 573 pci_chipset_tag_t pc = psc->sc_pa.pa_pc; 574 pcitag_t tag = psc->sc_pa.pa_tag; 575 unsigned int offset, i, len; 576 union { 577 pcireg_t reg[8]; 578 struct virtio_pci_cap vcap; 579 } *v = buf; 580 581 if (buflen < sizeof(struct virtio_pci_cap)) 582 return ERANGE; 583 584 if (!pci_get_capability(pc, tag, PCI_CAP_VENDSPEC, &offset, 585 &v->reg[0])) 586 return ENOENT; 587 588 do { 589 for (i = 0; i < 4; i++) 590 v->reg[i] = 591 le32toh(pci_conf_read(pc, tag, offset + i * 4)); 592 if (v->vcap.cfg_type == cfg_type) 593 break; 594 offset = v->vcap.cap_next; 595 } while (offset != 0); 596 597 if (offset == 0) 598 return ENOENT; 599 600 if (v->vcap.cap_len > sizeof(struct virtio_pci_cap)) { 601 len = roundup(v->vcap.cap_len, sizeof(pcireg_t)); 602 if (len > buflen) { 603 aprint_error_dev(self, "%s cap too large\n", __func__); 604 return ERANGE; 605 } 606 for (i = 4; i < len / sizeof(pcireg_t); i++) 607 v->reg[i] = 608 le32toh(pci_conf_read(pc, tag, offset + i * 4)); 609 } 610 611 /* endian fixup */ 612 v->vcap.offset = le32toh(v->vcap.offset); 613 v->vcap.length = le32toh(v->vcap.length); 614 return 0; 615 } 616 617 /* ------------------------------------- 618 * Version 0.9 support 619 * -------------------------------------*/ 620 621 static void 622 virtio_pci_kick_09(struct virtio_softc *sc, uint16_t idx) 623 { 624 struct virtio_pci_softc * const psc = container_of(sc, 625 struct virtio_pci_softc, sc_sc); 626 627 bus_space_write_2(psc->sc_notify_iot, psc->sc_notify_ioh, 0, idx); 628 } 629 630 /* only applicable for v 0.9 but also called for 1.0 */ 631 static int 632 virtio_pci_adjust_config_region(struct virtio_pci_softc *psc) 633 { 634 struct virtio_softc * const sc = &psc->sc_sc; 635 device_t self = sc->sc_dev; 636 637 if (psc->sc_sc.sc_version_1) 638 return 0; 639 640 sc->sc_devcfg_iosize = psc->sc_iosize - psc->sc_devcfg_offset; 641 sc->sc_devcfg_iot = psc->sc_iot; 642 if (bus_space_subregion(psc->sc_iot, psc->sc_ioh, 643 psc->sc_devcfg_offset, sc->sc_devcfg_iosize, 644 &sc->sc_devcfg_ioh)) { 645 aprint_error_dev(self, "can't map config i/o space\n"); 646 return EIO; 647 } 648 649 return 0; 650 } 651 652 static uint16_t 653 virtio_pci_read_queue_size_09(struct virtio_softc *sc, uint16_t idx) 654 { 655 struct virtio_pci_softc * const psc = container_of(sc, 656 struct virtio_pci_softc, sc_sc); 657 658 bus_space_write_2(psc->sc_iot, psc->sc_ioh, 659 VIRTIO_CONFIG_QUEUE_SELECT, idx); 660 return bus_space_read_2(psc->sc_iot, psc->sc_ioh, 661 VIRTIO_CONFIG_QUEUE_SIZE); 662 } 663 664 static void 665 virtio_pci_setup_queue_09(struct virtio_softc *sc, uint16_t idx, uint64_t addr) 666 { 667 struct virtio_pci_softc * const psc = container_of(sc, 668 struct virtio_pci_softc, sc_sc); 669 670 bus_space_write_2(psc->sc_iot, psc->sc_ioh, 671 VIRTIO_CONFIG_QUEUE_SELECT, idx); 672 bus_space_write_4(psc->sc_iot, psc->sc_ioh, 673 VIRTIO_CONFIG_QUEUE_ADDRESS, addr / VIRTIO_PAGE_SIZE); 674 675 if (psc->sc_ihs_num > 1) { 676 int vec = VIRTIO_MSIX_QUEUE_VECTOR_INDEX; 677 if (psc->sc_intr_pervq) 678 vec += idx; 679 bus_space_write_2(psc->sc_iot, psc->sc_ioh, 680 VIRTIO_CONFIG_MSI_QUEUE_VECTOR, vec); 681 } 682 } 683 684 static void 685 virtio_pci_set_status_09(struct virtio_softc *sc, int status) 686 { 687 struct virtio_pci_softc * const psc = container_of(sc, 688 struct virtio_pci_softc, sc_sc); 689 int old = 0; 690 691 if (status != 0) { 692 old = bus_space_read_1(psc->sc_iot, psc->sc_ioh, 693 VIRTIO_CONFIG_DEVICE_STATUS); 694 } 695 bus_space_write_1(psc->sc_iot, psc->sc_ioh, 696 VIRTIO_CONFIG_DEVICE_STATUS, status|old); 697 } 698 699 static void 700 virtio_pci_negotiate_features_09(struct virtio_softc *sc, 701 uint64_t guest_features) 702 { 703 struct virtio_pci_softc * const psc = container_of(sc, 704 struct virtio_pci_softc, sc_sc); 705 uint32_t r; 706 707 r = bus_space_read_4(psc->sc_iot, psc->sc_ioh, 708 VIRTIO_CONFIG_DEVICE_FEATURES); 709 710 r &= guest_features; 711 712 bus_space_write_4(psc->sc_iot, psc->sc_ioh, 713 VIRTIO_CONFIG_GUEST_FEATURES, r); 714 715 sc->sc_active_features = r; 716 } 717 718 /* ------------------------------------- 719 * Version 1.0 support 720 * -------------------------------------*/ 721 722 static void 723 virtio_pci_kick_10(struct virtio_softc *sc, uint16_t idx) 724 { 725 struct virtio_pci_softc * const psc = container_of(sc, 726 struct virtio_pci_softc, sc_sc); 727 unsigned offset = sc->sc_vqs[idx].vq_notify_off * 728 psc->sc_notify_off_multiplier; 729 730 bus_space_write_2(psc->sc_notify_iot, psc->sc_notify_ioh, offset, idx); 731 } 732 733 static uint16_t 734 virtio_pci_read_queue_size_10(struct virtio_softc *sc, uint16_t idx) 735 { 736 struct virtio_pci_softc * const psc = container_of(sc, 737 struct virtio_pci_softc, sc_sc); 738 bus_space_tag_t iot = psc->sc_iot; 739 bus_space_handle_t ioh = psc->sc_ioh; 740 741 bus_space_write_2(iot, ioh, VIRTIO_CONFIG1_QUEUE_SELECT, idx); 742 return bus_space_read_2(iot, ioh, VIRTIO_CONFIG1_QUEUE_SIZE); 743 } 744 745 /* 746 * By definition little endian only in v1.0. NB: "MAY" in the text 747 * below refers to "independently" (i.e. the order of accesses) not 748 * "32-bit" (which is restricted by the earlier "MUST"). 749 * 750 * 4.1.3.1 Driver Requirements: PCI Device Layout 751 * 752 * For device configuration access, the driver MUST use ... 32-bit 753 * wide and aligned accesses for ... 64-bit wide fields. For 64-bit 754 * fields, the driver MAY access each of the high and low 32-bit parts 755 * of the field independently. 756 */ 757 static __inline void 758 virtio_pci_bus_space_write_8(bus_space_tag_t iot, bus_space_handle_t ioh, 759 bus_size_t offset, uint64_t value) 760 { 761 bus_space_write_4(iot, ioh, offset, BUS_ADDR_LO32(value)); 762 bus_space_write_4(iot, ioh, offset + 4, BUS_ADDR_HI32(value)); 763 } 764 765 static void 766 virtio_pci_setup_queue_10(struct virtio_softc *sc, uint16_t idx, uint64_t addr) 767 { 768 struct virtio_pci_softc * const psc = container_of(sc, 769 struct virtio_pci_softc, sc_sc); 770 struct virtqueue *vq = &sc->sc_vqs[idx]; 771 bus_space_tag_t iot = psc->sc_iot; 772 bus_space_handle_t ioh = psc->sc_ioh; 773 KASSERT(vq->vq_index == idx); 774 775 bus_space_write_2(iot, ioh, VIRTIO_CONFIG1_QUEUE_SELECT, vq->vq_index); 776 if (addr == 0) { 777 bus_space_write_2(iot, ioh, VIRTIO_CONFIG1_QUEUE_ENABLE, 0); 778 virtio_pci_bus_space_write_8(iot, ioh, 779 VIRTIO_CONFIG1_QUEUE_DESC, 0); 780 virtio_pci_bus_space_write_8(iot, ioh, 781 VIRTIO_CONFIG1_QUEUE_AVAIL, 0); 782 virtio_pci_bus_space_write_8(iot, ioh, 783 VIRTIO_CONFIG1_QUEUE_USED, 0); 784 } else { 785 virtio_pci_bus_space_write_8(iot, ioh, 786 VIRTIO_CONFIG1_QUEUE_DESC, addr); 787 virtio_pci_bus_space_write_8(iot, ioh, 788 VIRTIO_CONFIG1_QUEUE_AVAIL, addr + vq->vq_availoffset); 789 virtio_pci_bus_space_write_8(iot, ioh, 790 VIRTIO_CONFIG1_QUEUE_USED, addr + vq->vq_usedoffset); 791 bus_space_write_2(iot, ioh, 792 VIRTIO_CONFIG1_QUEUE_ENABLE, 1); 793 vq->vq_notify_off = bus_space_read_2(iot, ioh, 794 VIRTIO_CONFIG1_QUEUE_NOTIFY_OFF); 795 } 796 797 if (psc->sc_ihs_num > 1) { 798 int vec = VIRTIO_MSIX_QUEUE_VECTOR_INDEX; 799 if (psc->sc_intr_pervq) 800 vec += idx; 801 bus_space_write_2(iot, ioh, 802 VIRTIO_CONFIG1_QUEUE_MSIX_VECTOR, vec); 803 } 804 } 805 806 static void 807 virtio_pci_set_status_10(struct virtio_softc *sc, int status) 808 { 809 struct virtio_pci_softc * const psc = container_of(sc, 810 struct virtio_pci_softc, sc_sc); 811 bus_space_tag_t iot = psc->sc_iot; 812 bus_space_handle_t ioh = psc->sc_ioh; 813 int old = 0; 814 815 if (status) 816 old = bus_space_read_1(iot, ioh, VIRTIO_CONFIG1_DEVICE_STATUS); 817 bus_space_write_1(iot, ioh, VIRTIO_CONFIG1_DEVICE_STATUS, 818 status | old); 819 } 820 821 void 822 virtio_pci_negotiate_features_10(struct virtio_softc *sc, 823 uint64_t guest_features) 824 { 825 struct virtio_pci_softc * const psc = container_of(sc, 826 struct virtio_pci_softc, sc_sc); 827 device_t self = sc->sc_dev; 828 bus_space_tag_t iot = psc->sc_iot; 829 bus_space_handle_t ioh = psc->sc_ioh; 830 uint64_t host, negotiated, device_status; 831 832 guest_features |= VIRTIO_F_VERSION_1; 833 #ifdef __NEED_VIRTIO_F_ACCESS_PLATFORM 834 /* XXX This could use some work. */ 835 guest_features |= VIRTIO_F_ACCESS_PLATFORM; 836 #endif /* __NEED_VIRTIO_F_ACCESS_PLATFORM */ 837 /* notify on empty is 0.9 only */ 838 guest_features &= ~VIRTIO_F_NOTIFY_ON_EMPTY; 839 sc->sc_active_features = 0; 840 841 bus_space_write_4(iot, ioh, VIRTIO_CONFIG1_DEVICE_FEATURE_SELECT, 0); 842 host = bus_space_read_4(iot, ioh, VIRTIO_CONFIG1_DEVICE_FEATURE); 843 bus_space_write_4(iot, ioh, VIRTIO_CONFIG1_DEVICE_FEATURE_SELECT, 1); 844 host |= (uint64_t)bus_space_read_4(iot, ioh, 845 VIRTIO_CONFIG1_DEVICE_FEATURE) << 32; 846 847 negotiated = host & guest_features; 848 849 bus_space_write_4(iot, ioh, VIRTIO_CONFIG1_DRIVER_FEATURE_SELECT, 0); 850 bus_space_write_4(iot, ioh, VIRTIO_CONFIG1_DRIVER_FEATURE, 851 negotiated & 0xffffffff); 852 bus_space_write_4(iot, ioh, VIRTIO_CONFIG1_DRIVER_FEATURE_SELECT, 1); 853 bus_space_write_4(iot, ioh, VIRTIO_CONFIG1_DRIVER_FEATURE, 854 negotiated >> 32); 855 virtio_pci_set_status_10(sc, VIRTIO_CONFIG_DEVICE_STATUS_FEATURES_OK); 856 857 device_status = bus_space_read_1(iot, ioh, 858 VIRTIO_CONFIG1_DEVICE_STATUS); 859 if ((device_status & VIRTIO_CONFIG_DEVICE_STATUS_FEATURES_OK) == 0) { 860 aprint_error_dev(self, "feature negotiation failed\n"); 861 bus_space_write_1(iot, ioh, VIRTIO_CONFIG1_DEVICE_STATUS, 862 VIRTIO_CONFIG_DEVICE_STATUS_FAILED); 863 return; 864 } 865 866 if ((negotiated & VIRTIO_F_VERSION_1) == 0) { 867 aprint_error_dev(self, "host rejected version 1\n"); 868 bus_space_write_1(iot, ioh, VIRTIO_CONFIG1_DEVICE_STATUS, 869 VIRTIO_CONFIG_DEVICE_STATUS_FAILED); 870 return; 871 } 872 873 sc->sc_active_features = negotiated; 874 return; 875 } 876 877 /* ------------------------------------- 878 * Generic PCI interrupt code 879 * -------------------------------------*/ 880 881 static int 882 virtio_pci_setup_interrupts_10(struct virtio_softc *sc, int reinit) 883 { 884 struct virtio_pci_softc * const psc = container_of(sc, 885 struct virtio_pci_softc, sc_sc); 886 bus_space_tag_t iot = psc->sc_iot; 887 bus_space_handle_t ioh = psc->sc_ioh; 888 int vector, ret, qid; 889 890 if (!virtio_pci_msix_enabled(psc)) 891 return 0; 892 893 vector = VIRTIO_MSIX_CONFIG_VECTOR_INDEX; 894 bus_space_write_2(iot, ioh, VIRTIO_CONFIG1_CONFIG_MSIX_VECTOR, vector); 895 ret = bus_space_read_2(iot, ioh, VIRTIO_CONFIG1_CONFIG_MSIX_VECTOR); 896 if (ret != vector) { 897 VIRTIO_PCI_LOG(sc, reinit, "can't set config msix vector\n"); 898 return -1; 899 } 900 901 for (qid = 0; qid < sc->sc_nvqs; qid++) { 902 vector = VIRTIO_MSIX_QUEUE_VECTOR_INDEX; 903 904 if (psc->sc_intr_pervq) 905 vector += qid; 906 bus_space_write_2(iot, ioh, VIRTIO_CONFIG1_QUEUE_SELECT, qid); 907 bus_space_write_2(iot, ioh, VIRTIO_CONFIG1_QUEUE_MSIX_VECTOR, 908 vector); 909 ret = bus_space_read_2(iot, ioh, 910 VIRTIO_CONFIG1_QUEUE_MSIX_VECTOR); 911 if (ret != vector) { 912 VIRTIO_PCI_LOG(sc, reinit, "can't set queue %d " 913 "msix vector\n", qid); 914 return -1; 915 } 916 } 917 918 return 0; 919 } 920 921 static int 922 virtio_pci_setup_interrupts_09(struct virtio_softc *sc, int reinit) 923 { 924 struct virtio_pci_softc * const psc = container_of(sc, 925 struct virtio_pci_softc, sc_sc); 926 int offset, vector, ret, qid; 927 928 if (!virtio_pci_msix_enabled(psc)) 929 return 0; 930 931 offset = VIRTIO_CONFIG_MSI_CONFIG_VECTOR; 932 vector = VIRTIO_MSIX_CONFIG_VECTOR_INDEX; 933 934 bus_space_write_2(psc->sc_iot, psc->sc_ioh, offset, vector); 935 ret = bus_space_read_2(psc->sc_iot, psc->sc_ioh, offset); 936 if (ret != vector) { 937 aprint_debug_dev(sc->sc_dev, "%s: expected=%d, actual=%d\n", 938 __func__, vector, ret); 939 VIRTIO_PCI_LOG(sc, reinit, 940 "can't set config msix vector\n"); 941 return -1; 942 } 943 944 for (qid = 0; qid < sc->sc_nvqs; qid++) { 945 offset = VIRTIO_CONFIG_QUEUE_SELECT; 946 bus_space_write_2(psc->sc_iot, psc->sc_ioh, offset, qid); 947 948 offset = VIRTIO_CONFIG_MSI_QUEUE_VECTOR; 949 vector = VIRTIO_MSIX_QUEUE_VECTOR_INDEX; 950 951 if (psc->sc_intr_pervq) 952 vector += qid; 953 954 bus_space_write_2(psc->sc_iot, psc->sc_ioh, offset, vector); 955 ret = bus_space_read_2(psc->sc_iot, psc->sc_ioh, offset); 956 if (ret != vector) { 957 aprint_debug_dev(sc->sc_dev, "%s[qid=%d]:" 958 " expected=%d, actual=%d\n", 959 __func__, qid, vector, ret); 960 VIRTIO_PCI_LOG(sc, reinit, "can't set queue %d " 961 "msix vector\n", qid); 962 return -1; 963 } 964 } 965 966 return 0; 967 } 968 969 static int 970 virtio_pci_establish_msix_interrupts(struct virtio_softc *sc, 971 const struct pci_attach_args *pa) 972 { 973 struct virtio_pci_softc * const psc = container_of(sc, 974 struct virtio_pci_softc, sc_sc); 975 device_t self = sc->sc_dev; 976 pci_chipset_tag_t pc = pa->pa_pc; 977 struct virtqueue *vq; 978 char intrbuf[PCI_INTRSTR_LEN]; 979 char intr_xname[INTRDEVNAMEBUF]; 980 char const *intrstr; 981 int idx, qid, n; 982 983 idx = VIRTIO_MSIX_CONFIG_VECTOR_INDEX; 984 if (sc->sc_flags & VIRTIO_F_INTR_MPSAFE) 985 pci_intr_setattr(pc, &psc->sc_ihp[idx], PCI_INTR_MPSAFE, true); 986 987 snprintf(intr_xname, sizeof(intr_xname), "%s config", 988 device_xname(sc->sc_dev)); 989 990 psc->sc_ihs[idx] = pci_intr_establish_xname(pc, psc->sc_ihp[idx], 991 sc->sc_ipl, virtio_pci_msix_config_intr, sc, intr_xname); 992 if (psc->sc_ihs[idx] == NULL) { 993 aprint_error_dev(self, 994 "couldn't establish MSI-X for config\n"); 995 goto error; 996 } 997 998 idx = VIRTIO_MSIX_QUEUE_VECTOR_INDEX; 999 if (psc->sc_intr_pervq) { 1000 for (qid = 0; qid < sc->sc_nvqs; qid++) { 1001 n = idx + qid; 1002 vq = &sc->sc_vqs[qid]; 1003 1004 snprintf(intr_xname, sizeof(intr_xname), "%s vq#%d", 1005 device_xname(sc->sc_dev), qid); 1006 1007 if (sc->sc_flags & VIRTIO_F_INTR_MPSAFE) { 1008 pci_intr_setattr(pc, &psc->sc_ihp[n], 1009 PCI_INTR_MPSAFE, true); 1010 } 1011 1012 psc->sc_ihs[n] = pci_intr_establish_xname(pc, 1013 psc->sc_ihp[n], sc->sc_ipl, 1014 vq->vq_intrhand, vq->vq_intrhand_arg, intr_xname); 1015 if (psc->sc_ihs[n] == NULL) { 1016 aprint_error_dev(self, 1017 "couldn't establish MSI-X for a vq\n"); 1018 goto error; 1019 } 1020 } 1021 } else { 1022 if (sc->sc_flags & VIRTIO_F_INTR_MPSAFE) { 1023 pci_intr_setattr(pc, &psc->sc_ihp[idx], 1024 PCI_INTR_MPSAFE, true); 1025 } 1026 1027 snprintf(intr_xname, sizeof(intr_xname), "%s queues", 1028 device_xname(sc->sc_dev)); 1029 psc->sc_ihs[idx] = pci_intr_establish_xname(pc, 1030 psc->sc_ihp[idx], sc->sc_ipl, 1031 virtio_pci_msix_queue_intr, sc, intr_xname); 1032 if (psc->sc_ihs[idx] == NULL) { 1033 aprint_error_dev(self, 1034 "couldn't establish MSI-X for queues\n"); 1035 goto error; 1036 } 1037 } 1038 1039 idx = VIRTIO_MSIX_CONFIG_VECTOR_INDEX; 1040 intrstr = pci_intr_string(pc, psc->sc_ihp[idx], intrbuf, 1041 sizeof(intrbuf)); 1042 aprint_normal_dev(self, "config interrupting at %s\n", intrstr); 1043 idx = VIRTIO_MSIX_QUEUE_VECTOR_INDEX; 1044 if (psc->sc_intr_pervq) { 1045 kcpuset_t *affinity; 1046 int affinity_to, r; 1047 1048 kcpuset_create(&affinity, false); 1049 1050 for (qid = 0; qid < sc->sc_nvqs; qid++) { 1051 n = idx + qid; 1052 affinity_to = (qid / 2) % ncpu; 1053 1054 intrstr = pci_intr_string(pc, psc->sc_ihp[n], 1055 intrbuf, sizeof(intrbuf)); 1056 1057 kcpuset_zero(affinity); 1058 kcpuset_set(affinity, affinity_to); 1059 r = interrupt_distribute(psc->sc_ihs[n], affinity, 1060 NULL); 1061 if (r == 0) { 1062 aprint_normal_dev(self, 1063 "for vq #%d interrupting at %s" 1064 " affinity to %u\n", 1065 qid, intrstr, affinity_to); 1066 } else { 1067 aprint_normal_dev(self, 1068 "for vq #%d interrupting at %s\n", 1069 qid, intrstr); 1070 } 1071 } 1072 1073 kcpuset_destroy(affinity); 1074 } else { 1075 intrstr = pci_intr_string(pc, psc->sc_ihp[idx], intrbuf, 1076 sizeof(intrbuf)); 1077 aprint_normal_dev(self, "queues interrupting at %s\n", 1078 intrstr); 1079 } 1080 1081 return 0; 1082 1083 error: 1084 idx = VIRTIO_MSIX_CONFIG_VECTOR_INDEX; 1085 if (psc->sc_ihs[idx] != NULL) 1086 pci_intr_disestablish(psc->sc_pa.pa_pc, psc->sc_ihs[idx]); 1087 idx = VIRTIO_MSIX_QUEUE_VECTOR_INDEX; 1088 if (psc->sc_intr_pervq) { 1089 for (qid = 0; qid < sc->sc_nvqs; qid++) { 1090 n = idx + qid; 1091 if (psc->sc_ihs[n] == NULL) 1092 continue; 1093 pci_intr_disestablish(psc->sc_pa.pa_pc, 1094 psc->sc_ihs[n]); 1095 } 1096 1097 } else { 1098 if (psc->sc_ihs[idx] != NULL) { 1099 pci_intr_disestablish(psc->sc_pa.pa_pc, 1100 psc->sc_ihs[idx]); 1101 } 1102 } 1103 1104 return -1; 1105 } 1106 1107 static int 1108 virtio_pci_establish_intx_interrupt(struct virtio_softc *sc, 1109 const struct pci_attach_args *pa) 1110 { 1111 struct virtio_pci_softc * const psc = container_of(sc, 1112 struct virtio_pci_softc, sc_sc); 1113 device_t self = sc->sc_dev; 1114 pci_chipset_tag_t pc = pa->pa_pc; 1115 char intrbuf[PCI_INTRSTR_LEN]; 1116 char const *intrstr; 1117 1118 if (sc->sc_flags & VIRTIO_F_INTR_MPSAFE) 1119 pci_intr_setattr(pc, &psc->sc_ihp[0], PCI_INTR_MPSAFE, true); 1120 1121 psc->sc_ihs[0] = pci_intr_establish_xname(pc, psc->sc_ihp[0], 1122 sc->sc_ipl, virtio_pci_intr, sc, device_xname(sc->sc_dev)); 1123 if (psc->sc_ihs[0] == NULL) { 1124 aprint_error_dev(self, "couldn't establish INTx\n"); 1125 return -1; 1126 } 1127 1128 intrstr = pci_intr_string(pc, psc->sc_ihp[0], intrbuf, 1129 sizeof(intrbuf)); 1130 aprint_normal_dev(self, "interrupting at %s\n", intrstr); 1131 1132 return 0; 1133 } 1134 1135 static int 1136 virtio_pci_alloc_interrupts(struct virtio_softc *sc) 1137 { 1138 struct virtio_pci_softc * const psc = container_of(sc, 1139 struct virtio_pci_softc, sc_sc); 1140 device_t self = sc->sc_dev; 1141 pci_chipset_tag_t pc = psc->sc_pa.pa_pc; 1142 pcitag_t tag = psc->sc_pa.pa_tag; 1143 int error; 1144 int nmsix; 1145 int off; 1146 int counts[PCI_INTR_TYPE_SIZE]; 1147 pci_intr_type_t max_type; 1148 pcireg_t ctl; 1149 1150 nmsix = pci_msix_count(psc->sc_pa.pa_pc, psc->sc_pa.pa_tag); 1151 aprint_debug_dev(self, "pci_msix_count=%d\n", nmsix); 1152 1153 /* We need at least two: one for config and the other for queues */ 1154 if ((sc->sc_flags & VIRTIO_F_INTR_MSIX) == 0 || nmsix < 2) { 1155 /* Try INTx only */ 1156 max_type = PCI_INTR_TYPE_INTX; 1157 counts[PCI_INTR_TYPE_INTX] = 1; 1158 } else { 1159 /* Try MSI-X first and INTx second */ 1160 if (ISSET(sc->sc_flags, VIRTIO_F_INTR_PERVQ) && 1161 sc->sc_nvqs + VIRTIO_MSIX_QUEUE_VECTOR_INDEX <= nmsix) { 1162 nmsix = sc->sc_nvqs + VIRTIO_MSIX_QUEUE_VECTOR_INDEX; 1163 } else { 1164 nmsix = 2; 1165 } 1166 1167 max_type = PCI_INTR_TYPE_MSIX; 1168 counts[PCI_INTR_TYPE_MSIX] = nmsix; 1169 counts[PCI_INTR_TYPE_MSI] = 0; 1170 counts[PCI_INTR_TYPE_INTX] = 1; 1171 } 1172 1173 retry: 1174 error = pci_intr_alloc(&psc->sc_pa, &psc->sc_ihp, counts, max_type); 1175 if (error != 0) { 1176 aprint_error_dev(self, "couldn't map interrupt\n"); 1177 return -1; 1178 } 1179 1180 if (pci_intr_type(pc, psc->sc_ihp[0]) == PCI_INTR_TYPE_MSIX) { 1181 psc->sc_intr_pervq = nmsix > 2 ? true : false; 1182 psc->sc_ihs = kmem_zalloc(sizeof(*psc->sc_ihs) * nmsix, 1183 KM_SLEEP); 1184 1185 error = virtio_pci_establish_msix_interrupts(sc, &psc->sc_pa); 1186 if (error != 0) { 1187 kmem_free(psc->sc_ihs, sizeof(*psc->sc_ihs) * nmsix); 1188 pci_intr_release(pc, psc->sc_ihp, nmsix); 1189 1190 /* Retry INTx */ 1191 max_type = PCI_INTR_TYPE_INTX; 1192 counts[PCI_INTR_TYPE_INTX] = 1; 1193 goto retry; 1194 } 1195 1196 psc->sc_ihs_num = nmsix; 1197 psc->sc_devcfg_offset = VIRTIO_CONFIG_DEVICE_CONFIG_MSI; 1198 virtio_pci_adjust_config_region(psc); 1199 } else if (pci_intr_type(pc, psc->sc_ihp[0]) == PCI_INTR_TYPE_INTX) { 1200 psc->sc_intr_pervq = false; 1201 psc->sc_ihs = kmem_zalloc(sizeof(*psc->sc_ihs) * 1, 1202 KM_SLEEP); 1203 1204 error = virtio_pci_establish_intx_interrupt(sc, &psc->sc_pa); 1205 if (error != 0) { 1206 kmem_free(psc->sc_ihs, sizeof(*psc->sc_ihs) * 1); 1207 pci_intr_release(pc, psc->sc_ihp, 1); 1208 return -1; 1209 } 1210 1211 psc->sc_ihs_num = 1; 1212 psc->sc_devcfg_offset = VIRTIO_CONFIG_DEVICE_CONFIG_NOMSI; 1213 virtio_pci_adjust_config_region(psc); 1214 1215 error = pci_get_capability(pc, tag, PCI_CAP_MSIX, &off, NULL); 1216 if (error != 0) { 1217 ctl = pci_conf_read(pc, tag, off + PCI_MSIX_CTL); 1218 ctl &= ~PCI_MSIX_CTL_ENABLE; 1219 pci_conf_write(pc, tag, off + PCI_MSIX_CTL, ctl); 1220 } 1221 } 1222 1223 if (!psc->sc_intr_pervq) 1224 CLR(sc->sc_flags, VIRTIO_F_INTR_PERVQ); 1225 return 0; 1226 } 1227 1228 static void 1229 virtio_pci_free_interrupts(struct virtio_softc *sc) 1230 { 1231 struct virtio_pci_softc * const psc = container_of(sc, 1232 struct virtio_pci_softc, sc_sc); 1233 1234 for (int i = 0; i < psc->sc_ihs_num; i++) { 1235 if (psc->sc_ihs[i] == NULL) 1236 continue; 1237 pci_intr_disestablish(psc->sc_pa.pa_pc, psc->sc_ihs[i]); 1238 psc->sc_ihs[i] = NULL; 1239 } 1240 1241 if (psc->sc_ihs_num > 0) { 1242 pci_intr_release(psc->sc_pa.pa_pc, psc->sc_ihp, 1243 psc->sc_ihs_num); 1244 } 1245 1246 if (psc->sc_ihs != NULL) { 1247 kmem_free(psc->sc_ihs, sizeof(*psc->sc_ihs) * psc->sc_ihs_num); 1248 psc->sc_ihs = NULL; 1249 } 1250 psc->sc_ihs_num = 0; 1251 } 1252 1253 static bool 1254 virtio_pci_msix_enabled(struct virtio_pci_softc *psc) 1255 { 1256 pci_chipset_tag_t pc = psc->sc_pa.pa_pc; 1257 1258 if (pci_intr_type(pc, psc->sc_ihp[0]) == PCI_INTR_TYPE_MSIX) 1259 return true; 1260 1261 return false; 1262 } 1263 1264 /* 1265 * Interrupt handler. 1266 */ 1267 static int 1268 virtio_pci_intr(void *arg) 1269 { 1270 struct virtio_softc *sc = arg; 1271 struct virtio_pci_softc * const psc = container_of(sc, 1272 struct virtio_pci_softc, sc_sc); 1273 int isr, r = 0; 1274 1275 /* check and ack the interrupt */ 1276 isr = bus_space_read_1(psc->sc_isr_iot, psc->sc_isr_ioh, 0); 1277 if (isr == 0) 1278 return 0; 1279 if ((isr & VIRTIO_CONFIG_ISR_CONFIG_CHANGE) && 1280 (sc->sc_config_change != NULL)) 1281 r = (sc->sc_config_change)(sc); 1282 if (sc->sc_intrhand != NULL) { 1283 if (sc->sc_soft_ih != NULL) 1284 softint_schedule(sc->sc_soft_ih); 1285 else 1286 r |= (sc->sc_intrhand)(sc); 1287 } 1288 1289 return r; 1290 } 1291 1292 static int 1293 virtio_pci_msix_queue_intr(void *arg) 1294 { 1295 struct virtio_softc *sc = arg; 1296 int r = 0; 1297 1298 if (sc->sc_intrhand != NULL) { 1299 if (sc->sc_soft_ih != NULL) 1300 softint_schedule(sc->sc_soft_ih); 1301 else 1302 r |= (sc->sc_intrhand)(sc); 1303 } 1304 1305 return r; 1306 } 1307 1308 static int 1309 virtio_pci_msix_config_intr(void *arg) 1310 { 1311 struct virtio_softc *sc = arg; 1312 int r = 0; 1313 1314 if (sc->sc_config_change != NULL) 1315 r = (sc->sc_config_change)(sc); 1316 return r; 1317 } 1318 1319 MODULE(MODULE_CLASS_DRIVER, virtio_pci, "pci,virtio"); 1320 1321 #ifdef _MODULE 1322 #include "ioconf.c" 1323 #endif 1324 1325 static int 1326 virtio_pci_modcmd(modcmd_t cmd, void *opaque) 1327 { 1328 int error = 0; 1329 1330 #ifdef _MODULE 1331 switch (cmd) { 1332 case MODULE_CMD_INIT: 1333 error = config_init_component(cfdriver_ioconf_virtio_pci, 1334 cfattach_ioconf_virtio_pci, cfdata_ioconf_virtio_pci); 1335 break; 1336 case MODULE_CMD_FINI: 1337 error = config_fini_component(cfdriver_ioconf_virtio_pci, 1338 cfattach_ioconf_virtio_pci, cfdata_ioconf_virtio_pci); 1339 break; 1340 default: 1341 error = ENOTTY; 1342 break; 1343 } 1344 #endif 1345 1346 return error; 1347 } 1348
Indexes created Sun Oct 25 00:26:45 UTC 2026