virtio_pci.c revision 1.51
1 /* $NetBSD: virtio_pci.c,v 1.51 2024/06/25 14:54:03 riastradh Exp $ */ 2 3 /* 4 * Copyright (c) 2020 The NetBSD Foundation, Inc. 5 * Copyright (c) 2012 Stefan Fritsch. 6 * Copyright (c) 2010 Minoura Makoto. 7 * All rights reserved. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 19 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 20 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 21 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 22 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 23 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 27 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 */ 29 30 #include <sys/cdefs.h> 31 __KERNEL_RCSID(0, "$NetBSD: virtio_pci.c,v 1.51 2024/06/25 14:54:03 riastradh Exp $"); 32 33 #include <sys/param.h> 34 #include <sys/types.h> 35 36 #include <sys/device.h> 37 #include <sys/endian.h> 38 #include <sys/interrupt.h> 39 #include <sys/kmem.h> 40 #include <sys/module.h> 41 #include <sys/syslog.h> 42 #include <sys/systm.h> 43 44 #include <dev/pci/pcidevs.h> 45 #include <dev/pci/pcireg.h> 46 #include <dev/pci/pcivar.h> 47 48 #include <dev/pci/virtioreg.h> /* XXX: move to non-pci */ 49 #include <dev/pci/virtio_pcireg.h> 50 51 #define VIRTIO_PRIVATE 52 #include <dev/pci/virtiovar.h> /* XXX: move to non-pci */ 53 54 #if defined(__alpha__) || defined(__sparc64__) 55 /* 56 * XXX VIRTIO_F_ACCESS_PLATFORM is required for standard PCI DMA 57 * XXX to work on these platforms, at least by Qemu. 58 * XXX 59 * XXX Generalize this later. 60 */ 61 #define __NEED_VIRTIO_F_ACCESS_PLATFORM 62 #endif /* __alpha__ || __sparc64__ */ 63 64 #define VIRTIO_PCI_LOG(_sc, _use_log, _fmt, _args...) \ 65 do { \ 66 if ((_use_log)) { \ 67 log(LOG_DEBUG, "%s: " _fmt, \ 68 device_xname((_sc)->sc_dev), \ 69 ##_args); \ 70 } else { \ 71 aprint_error_dev((_sc)->sc_dev, \ 72 _fmt, ##_args); \ 73 } \ 74 } while(0) 75 76 static int virtio_pci_match(device_t, cfdata_t, void *); 77 static void virtio_pci_attach(device_t, device_t, void *); 78 static int virtio_pci_rescan(device_t, const char *, const int *); 79 static int virtio_pci_detach(device_t, int); 80 81 #define NMAPREG ((PCI_MAPREG_END - PCI_MAPREG_START) / \ 82 sizeof(pcireg_t)) 83 struct virtio_pci_softc { 84 struct virtio_softc sc_sc; 85 bool sc_intr_pervq; 86 87 /* IO space */ 88 bus_space_tag_t sc_iot; 89 bus_space_handle_t sc_ioh; 90 bus_size_t sc_iosize; 91 92 /* BARs */ 93 bus_space_tag_t sc_bars_iot[NMAPREG]; 94 bus_space_handle_t sc_bars_ioh[NMAPREG]; 95 bus_size_t sc_bars_iosize[NMAPREG]; 96 97 /* notify space */ 98 bus_space_tag_t sc_notify_iot; 99 bus_space_handle_t sc_notify_ioh; 100 bus_size_t sc_notify_iosize; 101 uint32_t sc_notify_off_multiplier; 102 103 /* isr space */ 104 bus_space_tag_t sc_isr_iot; 105 bus_space_handle_t sc_isr_ioh; 106 bus_size_t sc_isr_iosize; 107 108 /* generic */ 109 struct pci_attach_args sc_pa; 110 pci_intr_handle_t *sc_ihp; 111 void **sc_ihs; 112 int sc_ihs_num; 113 int sc_devcfg_offset; /* for 0.9 */ 114 }; 115 116 static int virtio_pci_attach_09(device_t, void *); 117 static void virtio_pci_kick_09(struct virtio_softc *, uint16_t); 118 static uint16_t virtio_pci_read_queue_size_09(struct virtio_softc *, uint16_t); 119 static void virtio_pci_setup_queue_09(struct virtio_softc *, uint16_t, uint64_t); 120 static void virtio_pci_set_status_09(struct virtio_softc *, int); 121 static void virtio_pci_negotiate_features_09(struct virtio_softc *, uint64_t); 122 123 static int virtio_pci_attach_10(device_t, void *); 124 static void virtio_pci_kick_10(struct virtio_softc *, uint16_t); 125 static uint16_t virtio_pci_read_queue_size_10(struct virtio_softc *, uint16_t); 126 static void virtio_pci_setup_queue_10(struct virtio_softc *, uint16_t, uint64_t); 127 static void virtio_pci_set_status_10(struct virtio_softc *, int); 128 static void virtio_pci_negotiate_features_10(struct virtio_softc *, uint64_t); 129 static int virtio_pci_find_cap(struct virtio_pci_softc *psc, int cfg_type, void *buf, int buflen); 130 131 static int virtio_pci_alloc_interrupts(struct virtio_softc *); 132 static void virtio_pci_free_interrupts(struct virtio_softc *); 133 static int virtio_pci_adjust_config_region(struct virtio_pci_softc *psc); 134 static int virtio_pci_intr(void *arg); 135 static int virtio_pci_msix_queue_intr(void *); 136 static int virtio_pci_msix_config_intr(void *); 137 static int virtio_pci_setup_interrupts_09(struct virtio_softc *, int); 138 static int virtio_pci_setup_interrupts_10(struct virtio_softc *, int); 139 static int virtio_pci_establish_msix_interrupts(struct virtio_softc *, 140 struct pci_attach_args *); 141 static int virtio_pci_establish_intx_interrupt(struct virtio_softc *, 142 struct pci_attach_args *); 143 static bool virtio_pci_msix_enabled(struct virtio_pci_softc *); 144 145 #define VIRTIO_MSIX_CONFIG_VECTOR_INDEX 0 146 #define VIRTIO_MSIX_QUEUE_VECTOR_INDEX 1 147 148 /* 149 * For big-endian aarch64/armv7 on QEMU (and most real HW), only CPU cores 150 * are running in big-endian mode, with all peripheral being configured to 151 * little-endian mode. Their default bus_space(9) functions forcibly swap 152 * byte-order. This guarantees that PIO'ed data from pci(4), e.g., are 153 * correctly handled by bus_space(9), while DMA'ed ones should be swapped 154 * by hand, in violation of virtio(4) specifications. 155 */ 156 157 #if (defined(__aarch64__) || defined(__arm__)) && BYTE_ORDER == BIG_ENDIAN 158 # define READ_ENDIAN_09 BIG_ENDIAN 159 # define READ_ENDIAN_10 BIG_ENDIAN 160 # define STRUCT_ENDIAN_09 BIG_ENDIAN 161 # define STRUCT_ENDIAN_10 LITTLE_ENDIAN 162 #elif BYTE_ORDER == BIG_ENDIAN 163 # define READ_ENDIAN_09 LITTLE_ENDIAN 164 # define READ_ENDIAN_10 BIG_ENDIAN 165 # define STRUCT_ENDIAN_09 BIG_ENDIAN 166 # define STRUCT_ENDIAN_10 LITTLE_ENDIAN 167 #else /* little endian */ 168 # define READ_ENDIAN_09 LITTLE_ENDIAN 169 # define READ_ENDIAN_10 LITTLE_ENDIAN 170 # define STRUCT_ENDIAN_09 LITTLE_ENDIAN 171 # define STRUCT_ENDIAN_10 LITTLE_ENDIAN 172 #endif 173 174 CFATTACH_DECL3_NEW(virtio_pci, sizeof(struct virtio_pci_softc), 175 virtio_pci_match, virtio_pci_attach, virtio_pci_detach, NULL, 176 virtio_pci_rescan, NULL, 0); 177 178 static const struct virtio_ops virtio_pci_ops_09 = { 179 .kick = virtio_pci_kick_09, 180 .read_queue_size = virtio_pci_read_queue_size_09, 181 .setup_queue = virtio_pci_setup_queue_09, 182 .set_status = virtio_pci_set_status_09, 183 .neg_features = virtio_pci_negotiate_features_09, 184 .alloc_interrupts = virtio_pci_alloc_interrupts, 185 .free_interrupts = virtio_pci_free_interrupts, 186 .setup_interrupts = virtio_pci_setup_interrupts_09, 187 }; 188 189 static const struct virtio_ops virtio_pci_ops_10 = { 190 .kick = virtio_pci_kick_10, 191 .read_queue_size = virtio_pci_read_queue_size_10, 192 .setup_queue = virtio_pci_setup_queue_10, 193 .set_status = virtio_pci_set_status_10, 194 .neg_features = virtio_pci_negotiate_features_10, 195 .alloc_interrupts = virtio_pci_alloc_interrupts, 196 .free_interrupts = virtio_pci_free_interrupts, 197 .setup_interrupts = virtio_pci_setup_interrupts_10, 198 }; 199 200 static int 201 virtio_pci_match(device_t parent, cfdata_t match, void *aux) 202 { 203 struct pci_attach_args *pa; 204 205 pa = (struct pci_attach_args *)aux; 206 switch (PCI_VENDOR(pa->pa_id)) { 207 case PCI_VENDOR_QUMRANET: 208 /* Transitional devices MUST have a PCI Revision ID of 0. */ 209 if (((PCI_PRODUCT_QUMRANET_VIRTIO_1000 <= 210 PCI_PRODUCT(pa->pa_id)) && 211 (PCI_PRODUCT(pa->pa_id) <= 212 PCI_PRODUCT_QUMRANET_VIRTIO_103F)) && 213 PCI_REVISION(pa->pa_class) == 0) 214 return 1; 215 /* 216 * Non-transitional devices SHOULD have a PCI Revision 217 * ID of 1 or higher. Drivers MUST match any PCI 218 * Revision ID value. 219 */ 220 if (((PCI_PRODUCT_QUMRANET_VIRTIO_1040 <= 221 PCI_PRODUCT(pa->pa_id)) && 222 (PCI_PRODUCT(pa->pa_id) <= 223 PCI_PRODUCT_QUMRANET_VIRTIO_107F)) && 224 /* XXX: TODO */ 225 PCI_REVISION(pa->pa_class) == 1) 226 return 1; 227 break; 228 } 229 230 return 0; 231 } 232 233 static void 234 virtio_pci_attach(device_t parent, device_t self, void *aux) 235 { 236 struct virtio_pci_softc * const psc = device_private(self); 237 struct virtio_softc * const sc = &psc->sc_sc; 238 struct pci_attach_args *pa = (struct pci_attach_args *)aux; 239 pci_chipset_tag_t pc = pa->pa_pc; 240 pcitag_t tag = pa->pa_tag; 241 int revision; 242 int ret; 243 pcireg_t id; 244 pcireg_t csr; 245 246 revision = PCI_REVISION(pa->pa_class); 247 switch (revision) { 248 case 0: 249 /* subsystem ID shows what I am */ 250 id = PCI_SUBSYS_ID(pci_conf_read(pc, tag, PCI_SUBSYS_ID_REG)); 251 break; 252 case 1: 253 /* pci product number shows what I am */ 254 id = PCI_PRODUCT(pa->pa_id) - PCI_PRODUCT_QUMRANET_VIRTIO_1040; 255 break; 256 default: 257 aprint_normal(": unknown revision 0x%02x; giving up\n", 258 revision); 259 return; 260 } 261 262 aprint_normal("\n"); 263 aprint_naive("\n"); 264 virtio_print_device_type(self, id, revision); 265 266 csr = pci_conf_read(pc, tag, PCI_COMMAND_STATUS_REG); 267 csr |= PCI_COMMAND_MASTER_ENABLE | PCI_COMMAND_IO_ENABLE; 268 pci_conf_write(pc, tag, PCI_COMMAND_STATUS_REG, csr); 269 270 sc->sc_dev = self; 271 psc->sc_pa = *pa; 272 psc->sc_iot = pa->pa_iot; 273 274 sc->sc_dmat = pa->pa_dmat; 275 if (pci_dma64_available(pa)) 276 sc->sc_dmat = pa->pa_dmat64; 277 278 /* attach is dependent on revision */ 279 ret = 0; 280 if (revision == 1) { 281 /* try to attach 1.0 */ 282 ret = virtio_pci_attach_10(self, aux); 283 } 284 if (ret == 0 && revision == 0) { 285 /* 286 * revision 0 means 0.9 only or both 0.9 and 1.0. The 287 * latter are so-called "Transitional Devices". For 288 * those devices, we want to use the 1.0 interface if 289 * possible. 290 * 291 * XXX Currently only on platforms that require 1.0 292 * XXX features, such as VIRTIO_F_ACCESS_PLATFORM. 293 */ 294 #ifdef __NEED_VIRTIO_F_ACCESS_PLATFORM 295 /* First, try to attach 1.0 */ 296 ret = virtio_pci_attach_10(self, aux); 297 if (ret != 0) { 298 aprint_error_dev(self, 299 "VirtIO 1.0 error = %d, falling back to 0.9\n", 300 ret); 301 /* Fall back to 0.9. */ 302 ret = virtio_pci_attach_09(self, aux); 303 } 304 #else 305 ret = virtio_pci_attach_09(self, aux); 306 #endif /* __NEED_VIRTIO_F_ACCESS_PLATFORM */ 307 } 308 if (ret) { 309 aprint_error_dev(self, "cannot attach (%d)\n", ret); 310 return; 311 } 312 KASSERT(sc->sc_ops); 313 314 /* preset config region */ 315 psc->sc_devcfg_offset = VIRTIO_CONFIG_DEVICE_CONFIG_NOMSI; 316 if (virtio_pci_adjust_config_region(psc)) 317 return; 318 319 /* generic */ 320 virtio_device_reset(sc); 321 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_ACK); 322 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_DRIVER); 323 324 sc->sc_childdevid = id; 325 sc->sc_child = NULL; 326 virtio_pci_rescan(self, NULL, NULL); 327 return; 328 } 329 330 /* ARGSUSED */ 331 static int 332 virtio_pci_rescan(device_t self, const char *ifattr, const int *locs) 333 { 334 struct virtio_pci_softc * const psc = device_private(self); 335 struct virtio_softc * const sc = &psc->sc_sc; 336 struct virtio_attach_args va; 337 338 if (sc->sc_child) /* Child already attached? */ 339 return 0; 340 341 memset(&va, 0, sizeof(va)); 342 va.sc_childdevid = sc->sc_childdevid; 343 344 config_found(self, &va, NULL, CFARGS_NONE); 345 346 if (virtio_attach_failed(sc)) 347 return 0; 348 349 return 0; 350 } 351 352 static int 353 virtio_pci_detach(device_t self, int flags) 354 { 355 struct virtio_pci_softc * const psc = device_private(self); 356 struct virtio_softc * const sc = &psc->sc_sc; 357 unsigned i; 358 int r; 359 360 r = config_detach_children(self, flags); 361 if (r != 0) 362 return r; 363 364 /* Check that child never attached, or detached properly */ 365 KASSERT(sc->sc_child == NULL); 366 KASSERT(sc->sc_vqs == NULL); 367 KASSERT(psc->sc_ihs_num == 0); 368 369 if (sc->sc_version_1) { 370 for (i = 0; i < __arraycount(psc->sc_bars_iot); i++) { 371 if (psc->sc_bars_iosize[i] == 0) 372 continue; 373 bus_space_unmap(psc->sc_bars_iot[i], 374 psc->sc_bars_ioh[i], psc->sc_bars_iosize[i]); 375 psc->sc_bars_iosize[i] = 0; 376 } 377 } else { 378 if (psc->sc_iosize) { 379 bus_space_unmap(psc->sc_iot, psc->sc_ioh, 380 psc->sc_iosize); 381 psc->sc_iosize = 0; 382 } 383 } 384 385 return 0; 386 } 387 388 static int 389 virtio_pci_attach_09(device_t self, void *aux) 390 { 391 struct virtio_pci_softc * const psc = device_private(self); 392 struct pci_attach_args *pa = (struct pci_attach_args *)aux; 393 struct virtio_softc * const sc = &psc->sc_sc; 394 395 /* complete IO region */ 396 if (pci_mapreg_map(pa, PCI_MAPREG_START, PCI_MAPREG_TYPE_IO, 0, 397 &psc->sc_iot, &psc->sc_ioh, NULL, &psc->sc_iosize)) { 398 aprint_error_dev(self, "can't map i/o space\n"); 399 return EIO; 400 } 401 402 /* queue space */ 403 if (bus_space_subregion(psc->sc_iot, psc->sc_ioh, 404 VIRTIO_CONFIG_QUEUE_NOTIFY, 2, &psc->sc_notify_ioh)) { 405 aprint_error_dev(self, "can't map notify i/o space\n"); 406 return EIO; 407 } 408 psc->sc_notify_iosize = 2; 409 psc->sc_notify_iot = psc->sc_iot; 410 411 /* ISR space */ 412 if (bus_space_subregion(psc->sc_iot, psc->sc_ioh, 413 VIRTIO_CONFIG_ISR_STATUS, 1, &psc->sc_isr_ioh)) { 414 aprint_error_dev(self, "can't map isr i/o space\n"); 415 return EIO; 416 } 417 psc->sc_isr_iosize = 1; 418 psc->sc_isr_iot = psc->sc_iot; 419 420 /* set our version 0.9 ops */ 421 sc->sc_ops = &virtio_pci_ops_09; 422 sc->sc_bus_endian = READ_ENDIAN_09; 423 sc->sc_struct_endian = STRUCT_ENDIAN_09; 424 return 0; 425 } 426 427 static int 428 virtio_pci_attach_10(device_t self, void *aux) 429 { 430 struct virtio_pci_softc * const psc = device_private(self); 431 struct pci_attach_args *pa = (struct pci_attach_args *)aux; 432 struct virtio_softc * const sc = &psc->sc_sc; 433 pci_chipset_tag_t pc = pa->pa_pc; 434 pcitag_t tag = pa->pa_tag; 435 436 struct virtio_pci_cap common, isr, device; 437 struct virtio_pci_notify_cap notify; 438 int have_device_cfg = 0; 439 bus_size_t bars[NMAPREG] = { 0 }; 440 int bars_idx[NMAPREG] = { 0 }; 441 struct virtio_pci_cap *caps[] = { &common, &isr, &device, ¬ify.cap }; 442 int i, j, ret = 0; 443 444 if (virtio_pci_find_cap(psc, VIRTIO_PCI_CAP_COMMON_CFG, 445 &common, sizeof(common))) 446 return ENODEV; 447 if (virtio_pci_find_cap(psc, VIRTIO_PCI_CAP_NOTIFY_CFG, 448 ¬ify, sizeof(notify))) 449 return ENODEV; 450 if (virtio_pci_find_cap(psc, VIRTIO_PCI_CAP_ISR_CFG, 451 &isr, sizeof(isr))) 452 return ENODEV; 453 if (virtio_pci_find_cap(psc, VIRTIO_PCI_CAP_DEVICE_CFG, 454 &device, sizeof(device))) 455 memset(&device, 0, sizeof(device)); 456 else 457 have_device_cfg = 1; 458 459 /* Figure out which bars we need to map */ 460 for (i = 0; i < __arraycount(caps); i++) { 461 int bar = caps[i]->bar; 462 bus_size_t len = caps[i]->offset + caps[i]->length; 463 if (caps[i]->length == 0) 464 continue; 465 if (bars[bar] < len) 466 bars[bar] = len; 467 } 468 469 for (i = j = 0; i < __arraycount(bars); i++) { 470 int reg; 471 pcireg_t type; 472 if (bars[i] == 0) 473 continue; 474 reg = PCI_BAR(i); 475 type = pci_mapreg_type(pc, tag, reg); 476 if (pci_mapreg_map(pa, reg, type, 0, 477 &psc->sc_bars_iot[j], &psc->sc_bars_ioh[j], 478 NULL, &psc->sc_bars_iosize[j])) { 479 aprint_error_dev(self, "can't map bar %u \n", i); 480 ret = EIO; 481 goto err; 482 } 483 aprint_debug_dev(self, 484 "bar[%d]: iot %p, size 0x%" PRIxBUSSIZE "\n", 485 j, psc->sc_bars_iot[j], psc->sc_bars_iosize[j]); 486 bars_idx[i] = j; 487 j++; 488 } 489 490 i = bars_idx[notify.cap.bar]; 491 if (bus_space_subregion(psc->sc_bars_iot[i], psc->sc_bars_ioh[i], 492 notify.cap.offset, notify.cap.length, 493 &psc->sc_notify_ioh)) { 494 aprint_error_dev(self, "can't map notify i/o space\n"); 495 ret = EIO; 496 goto err; 497 } 498 psc->sc_notify_iosize = notify.cap.length; 499 psc->sc_notify_iot = psc->sc_bars_iot[i]; 500 psc->sc_notify_off_multiplier = le32toh(notify.notify_off_multiplier); 501 502 if (have_device_cfg) { 503 i = bars_idx[device.bar]; 504 if (bus_space_subregion(psc->sc_bars_iot[i], psc->sc_bars_ioh[i], 505 device.offset, device.length, 506 &sc->sc_devcfg_ioh)) { 507 aprint_error_dev(self, "can't map devcfg i/o space\n"); 508 ret = EIO; 509 goto err; 510 } 511 aprint_debug_dev(self, 512 "device.offset = 0x%x, device.length = 0x%x\n", 513 device.offset, device.length); 514 sc->sc_devcfg_iosize = device.length; 515 sc->sc_devcfg_iot = psc->sc_bars_iot[i]; 516 } 517 518 i = bars_idx[isr.bar]; 519 if (bus_space_subregion(psc->sc_bars_iot[i], psc->sc_bars_ioh[i], 520 isr.offset, isr.length, &psc->sc_isr_ioh)) { 521 aprint_error_dev(self, "can't map isr i/o space\n"); 522 ret = EIO; 523 goto err; 524 } 525 psc->sc_isr_iosize = isr.length; 526 psc->sc_isr_iot = psc->sc_bars_iot[i]; 527 528 i = bars_idx[common.bar]; 529 if (bus_space_subregion(psc->sc_bars_iot[i], psc->sc_bars_ioh[i], 530 common.offset, common.length, &psc->sc_ioh)) { 531 aprint_error_dev(self, "can't map common i/o space\n"); 532 ret = EIO; 533 goto err; 534 } 535 psc->sc_iosize = common.length; 536 psc->sc_iot = psc->sc_bars_iot[i]; 537 538 psc->sc_sc.sc_version_1 = 1; 539 540 /* set our version 1.0 ops */ 541 sc->sc_ops = &virtio_pci_ops_10; 542 sc->sc_bus_endian = READ_ENDIAN_10; 543 sc->sc_struct_endian = STRUCT_ENDIAN_10; 544 return 0; 545 546 err: 547 /* undo our pci_mapreg_map()s */ 548 for (i = 0; i < __arraycount(bars); i++) { 549 if (psc->sc_bars_iosize[i] == 0) 550 continue; 551 bus_space_unmap(psc->sc_bars_iot[i], psc->sc_bars_ioh[i], 552 psc->sc_bars_iosize[i]); 553 psc->sc_bars_iosize[i] = 0; 554 } 555 return ret; 556 } 557 558 /* v1.0 attach helper */ 559 static int 560 virtio_pci_find_cap(struct virtio_pci_softc *psc, int cfg_type, void *buf, int buflen) 561 { 562 device_t self = psc->sc_sc.sc_dev; 563 pci_chipset_tag_t pc = psc->sc_pa.pa_pc; 564 pcitag_t tag = psc->sc_pa.pa_tag; 565 unsigned int offset, i, len; 566 union { 567 pcireg_t reg[8]; 568 struct virtio_pci_cap vcap; 569 } *v = buf; 570 571 if (buflen < sizeof(struct virtio_pci_cap)) 572 return ERANGE; 573 574 if (!pci_get_capability(pc, tag, PCI_CAP_VENDSPEC, &offset, &v->reg[0])) 575 return ENOENT; 576 577 do { 578 for (i = 0; i < 4; i++) 579 v->reg[i] = 580 le32toh(pci_conf_read(pc, tag, offset + i * 4)); 581 if (v->vcap.cfg_type == cfg_type) 582 break; 583 offset = v->vcap.cap_next; 584 } while (offset != 0); 585 586 if (offset == 0) 587 return ENOENT; 588 589 if (v->vcap.cap_len > sizeof(struct virtio_pci_cap)) { 590 len = roundup(v->vcap.cap_len, sizeof(pcireg_t)); 591 if (len > buflen) { 592 aprint_error_dev(self, "%s cap too large\n", __func__); 593 return ERANGE; 594 } 595 for (i = 4; i < len / sizeof(pcireg_t); i++) 596 v->reg[i] = 597 le32toh(pci_conf_read(pc, tag, offset + i * 4)); 598 } 599 600 /* endian fixup */ 601 v->vcap.offset = le32toh(v->vcap.offset); 602 v->vcap.length = le32toh(v->vcap.length); 603 return 0; 604 } 605 606 /* ------------------------------------- 607 * Version 0.9 support 608 * -------------------------------------*/ 609 610 static void 611 virtio_pci_kick_09(struct virtio_softc *sc, uint16_t idx) 612 { 613 struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc; 614 615 bus_space_write_2(psc->sc_notify_iot, psc->sc_notify_ioh, 0, idx); 616 } 617 618 /* only applicable for v 0.9 but also called for 1.0 */ 619 static int 620 virtio_pci_adjust_config_region(struct virtio_pci_softc *psc) 621 { 622 struct virtio_softc * const sc = &psc->sc_sc; 623 device_t self = sc->sc_dev; 624 625 if (psc->sc_sc.sc_version_1) 626 return 0; 627 628 sc->sc_devcfg_iosize = psc->sc_iosize - psc->sc_devcfg_offset; 629 sc->sc_devcfg_iot = psc->sc_iot; 630 if (bus_space_subregion(psc->sc_iot, psc->sc_ioh, 631 psc->sc_devcfg_offset, sc->sc_devcfg_iosize, 632 &sc->sc_devcfg_ioh)) { 633 aprint_error_dev(self, "can't map config i/o space\n"); 634 return EIO; 635 } 636 637 return 0; 638 } 639 640 static uint16_t 641 virtio_pci_read_queue_size_09(struct virtio_softc *sc, uint16_t idx) 642 { 643 struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc; 644 645 bus_space_write_2(psc->sc_iot, psc->sc_ioh, 646 VIRTIO_CONFIG_QUEUE_SELECT, idx); 647 return bus_space_read_2(psc->sc_iot, psc->sc_ioh, 648 VIRTIO_CONFIG_QUEUE_SIZE); 649 } 650 651 static void 652 virtio_pci_setup_queue_09(struct virtio_softc *sc, uint16_t idx, uint64_t addr) 653 { 654 struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc; 655 656 bus_space_write_2(psc->sc_iot, psc->sc_ioh, 657 VIRTIO_CONFIG_QUEUE_SELECT, idx); 658 bus_space_write_4(psc->sc_iot, psc->sc_ioh, 659 VIRTIO_CONFIG_QUEUE_ADDRESS, addr / VIRTIO_PAGE_SIZE); 660 661 if (psc->sc_ihs_num > 1) { 662 int vec = VIRTIO_MSIX_QUEUE_VECTOR_INDEX; 663 if (psc->sc_intr_pervq) 664 vec += idx; 665 bus_space_write_2(psc->sc_iot, psc->sc_ioh, 666 VIRTIO_CONFIG_MSI_QUEUE_VECTOR, vec); 667 } 668 } 669 670 static void 671 virtio_pci_set_status_09(struct virtio_softc *sc, int status) 672 { 673 struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc; 674 int old = 0; 675 676 if (status != 0) { 677 old = bus_space_read_1(psc->sc_iot, psc->sc_ioh, 678 VIRTIO_CONFIG_DEVICE_STATUS); 679 } 680 bus_space_write_1(psc->sc_iot, psc->sc_ioh, 681 VIRTIO_CONFIG_DEVICE_STATUS, status|old); 682 } 683 684 static void 685 virtio_pci_negotiate_features_09(struct virtio_softc *sc, uint64_t guest_features) 686 { 687 struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc; 688 uint32_t r; 689 690 r = bus_space_read_4(psc->sc_iot, psc->sc_ioh, 691 VIRTIO_CONFIG_DEVICE_FEATURES); 692 693 r &= guest_features; 694 695 bus_space_write_4(psc->sc_iot, psc->sc_ioh, 696 VIRTIO_CONFIG_GUEST_FEATURES, r); 697 698 sc->sc_active_features = r; 699 } 700 701 /* ------------------------------------- 702 * Version 1.0 support 703 * -------------------------------------*/ 704 705 static void 706 virtio_pci_kick_10(struct virtio_softc *sc, uint16_t idx) 707 { 708 struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc; 709 unsigned offset = sc->sc_vqs[idx].vq_notify_off * 710 psc->sc_notify_off_multiplier; 711 712 bus_space_write_2(psc->sc_notify_iot, psc->sc_notify_ioh, offset, idx); 713 } 714 715 static uint16_t 716 virtio_pci_read_queue_size_10(struct virtio_softc *sc, uint16_t idx) 717 { 718 struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc; 719 bus_space_tag_t iot = psc->sc_iot; 720 bus_space_handle_t ioh = psc->sc_ioh; 721 722 bus_space_write_2(iot, ioh, VIRTIO_CONFIG1_QUEUE_SELECT, idx); 723 return bus_space_read_2(iot, ioh, VIRTIO_CONFIG1_QUEUE_SIZE); 724 } 725 726 /* 727 * By definition little endian only in v1.0. NB: "MAY" in the text 728 * below refers to "independently" (i.e. the order of accesses) not 729 * "32-bit" (which is restricted by the earlier "MUST"). 730 * 731 * 4.1.3.1 Driver Requirements: PCI Device Layout 732 * 733 * For device configuration access, the driver MUST use ... 32-bit 734 * wide and aligned accesses for ... 64-bit wide fields. For 64-bit 735 * fields, the driver MAY access each of the high and low 32-bit parts 736 * of the field independently. 737 */ 738 static __inline void 739 virtio_pci_bus_space_write_8(bus_space_tag_t iot, bus_space_handle_t ioh, 740 bus_size_t offset, uint64_t value) 741 { 742 #if _QUAD_HIGHWORD 743 bus_space_write_4(iot, ioh, offset, BUS_ADDR_LO32(value)); 744 bus_space_write_4(iot, ioh, offset + 4, BUS_ADDR_HI32(value)); 745 #else 746 bus_space_write_4(iot, ioh, offset, BUS_ADDR_HI32(value)); 747 bus_space_write_4(iot, ioh, offset + 4, BUS_ADDR_LO32(value)); 748 #endif 749 } 750 751 static void 752 virtio_pci_setup_queue_10(struct virtio_softc *sc, uint16_t idx, uint64_t addr) 753 { 754 struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc; 755 struct virtqueue *vq = &sc->sc_vqs[idx]; 756 bus_space_tag_t iot = psc->sc_iot; 757 bus_space_handle_t ioh = psc->sc_ioh; 758 KASSERT(vq->vq_index == idx); 759 760 bus_space_write_2(iot, ioh, VIRTIO_CONFIG1_QUEUE_SELECT, vq->vq_index); 761 if (addr == 0) { 762 bus_space_write_2(iot, ioh, VIRTIO_CONFIG1_QUEUE_ENABLE, 0); 763 virtio_pci_bus_space_write_8(iot, ioh, 764 VIRTIO_CONFIG1_QUEUE_DESC, 0); 765 virtio_pci_bus_space_write_8(iot, ioh, 766 VIRTIO_CONFIG1_QUEUE_AVAIL, 0); 767 virtio_pci_bus_space_write_8(iot, ioh, 768 VIRTIO_CONFIG1_QUEUE_USED, 0); 769 } else { 770 virtio_pci_bus_space_write_8(iot, ioh, 771 VIRTIO_CONFIG1_QUEUE_DESC, addr); 772 virtio_pci_bus_space_write_8(iot, ioh, 773 VIRTIO_CONFIG1_QUEUE_AVAIL, addr + vq->vq_availoffset); 774 virtio_pci_bus_space_write_8(iot, ioh, 775 VIRTIO_CONFIG1_QUEUE_USED, addr + vq->vq_usedoffset); 776 bus_space_write_2(iot, ioh, 777 VIRTIO_CONFIG1_QUEUE_ENABLE, 1); 778 vq->vq_notify_off = bus_space_read_2(iot, ioh, 779 VIRTIO_CONFIG1_QUEUE_NOTIFY_OFF); 780 } 781 782 if (psc->sc_ihs_num > 1) { 783 int vec = VIRTIO_MSIX_QUEUE_VECTOR_INDEX; 784 if (psc->sc_intr_pervq) 785 vec += idx; 786 bus_space_write_2(iot, ioh, 787 VIRTIO_CONFIG1_QUEUE_MSIX_VECTOR, vec); 788 } 789 } 790 791 static void 792 virtio_pci_set_status_10(struct virtio_softc *sc, int status) 793 { 794 struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc; 795 bus_space_tag_t iot = psc->sc_iot; 796 bus_space_handle_t ioh = psc->sc_ioh; 797 int old = 0; 798 799 if (status) 800 old = bus_space_read_1(iot, ioh, VIRTIO_CONFIG1_DEVICE_STATUS); 801 bus_space_write_1(iot, ioh, VIRTIO_CONFIG1_DEVICE_STATUS, status | old); 802 } 803 804 void 805 virtio_pci_negotiate_features_10(struct virtio_softc *sc, uint64_t guest_features) 806 { 807 struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc; 808 device_t self = sc->sc_dev; 809 bus_space_tag_t iot = psc->sc_iot; 810 bus_space_handle_t ioh = psc->sc_ioh; 811 uint64_t host, negotiated, device_status; 812 813 guest_features |= VIRTIO_F_VERSION_1; 814 #ifdef __NEED_VIRTIO_F_ACCESS_PLATFORM 815 /* XXX This could use some work. */ 816 guest_features |= VIRTIO_F_ACCESS_PLATFORM; 817 #endif /* __NEED_VIRTIO_F_ACCESS_PLATFORM */ 818 /* notify on empty is 0.9 only */ 819 guest_features &= ~VIRTIO_F_NOTIFY_ON_EMPTY; 820 sc->sc_active_features = 0; 821 822 bus_space_write_4(iot, ioh, VIRTIO_CONFIG1_DEVICE_FEATURE_SELECT, 0); 823 host = bus_space_read_4(iot, ioh, VIRTIO_CONFIG1_DEVICE_FEATURE); 824 bus_space_write_4(iot, ioh, VIRTIO_CONFIG1_DEVICE_FEATURE_SELECT, 1); 825 host |= (uint64_t) 826 bus_space_read_4(iot, ioh, VIRTIO_CONFIG1_DEVICE_FEATURE) << 32; 827 828 negotiated = host & guest_features; 829 830 bus_space_write_4(iot, ioh, VIRTIO_CONFIG1_DRIVER_FEATURE_SELECT, 0); 831 bus_space_write_4(iot, ioh, VIRTIO_CONFIG1_DRIVER_FEATURE, 832 negotiated & 0xffffffff); 833 bus_space_write_4(iot, ioh, VIRTIO_CONFIG1_DRIVER_FEATURE_SELECT, 1); 834 bus_space_write_4(iot, ioh, VIRTIO_CONFIG1_DRIVER_FEATURE, 835 negotiated >> 32); 836 virtio_pci_set_status_10(sc, VIRTIO_CONFIG_DEVICE_STATUS_FEATURES_OK); 837 838 device_status = bus_space_read_1(iot, ioh, VIRTIO_CONFIG1_DEVICE_STATUS); 839 if ((device_status & VIRTIO_CONFIG_DEVICE_STATUS_FEATURES_OK) == 0) { 840 aprint_error_dev(self, "feature negotiation failed\n"); 841 bus_space_write_1(iot, ioh, VIRTIO_CONFIG1_DEVICE_STATUS, 842 VIRTIO_CONFIG_DEVICE_STATUS_FAILED); 843 return; 844 } 845 846 if ((negotiated & VIRTIO_F_VERSION_1) == 0) { 847 aprint_error_dev(self, "host rejected version 1\n"); 848 bus_space_write_1(iot, ioh, VIRTIO_CONFIG1_DEVICE_STATUS, 849 VIRTIO_CONFIG_DEVICE_STATUS_FAILED); 850 return; 851 } 852 853 sc->sc_active_features = negotiated; 854 return; 855 } 856 857 /* ------------------------------------- 858 * Generic PCI interrupt code 859 * -------------------------------------*/ 860 861 static int 862 virtio_pci_setup_interrupts_10(struct virtio_softc *sc, int reinit) 863 { 864 struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc; 865 bus_space_tag_t iot = psc->sc_iot; 866 bus_space_handle_t ioh = psc->sc_ioh; 867 int vector, ret, qid; 868 869 if (!virtio_pci_msix_enabled(psc)) 870 return 0; 871 872 vector = VIRTIO_MSIX_CONFIG_VECTOR_INDEX; 873 bus_space_write_2(iot, ioh, 874 VIRTIO_CONFIG1_CONFIG_MSIX_VECTOR, vector); 875 ret = bus_space_read_2(iot, ioh, VIRTIO_CONFIG1_CONFIG_MSIX_VECTOR); 876 if (ret != vector) { 877 VIRTIO_PCI_LOG(sc, reinit, 878 "can't set config msix vector\n"); 879 return -1; 880 } 881 882 for (qid = 0; qid < sc->sc_nvqs; qid++) { 883 vector = VIRTIO_MSIX_QUEUE_VECTOR_INDEX; 884 885 if (psc->sc_intr_pervq) 886 vector += qid; 887 bus_space_write_2(iot, ioh, VIRTIO_CONFIG1_QUEUE_SELECT, qid); 888 bus_space_write_2(iot, ioh, VIRTIO_CONFIG1_QUEUE_MSIX_VECTOR, 889 vector); 890 ret = bus_space_read_2(iot, ioh, 891 VIRTIO_CONFIG1_QUEUE_MSIX_VECTOR); 892 if (ret != vector) { 893 VIRTIO_PCI_LOG(sc, reinit, "can't set queue %d " 894 "msix vector\n", qid); 895 return -1; 896 } 897 } 898 899 return 0; 900 } 901 902 static int 903 virtio_pci_setup_interrupts_09(struct virtio_softc *sc, int reinit) 904 { 905 struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc; 906 int offset, vector, ret, qid; 907 908 if (!virtio_pci_msix_enabled(psc)) 909 return 0; 910 911 offset = VIRTIO_CONFIG_MSI_CONFIG_VECTOR; 912 vector = VIRTIO_MSIX_CONFIG_VECTOR_INDEX; 913 914 bus_space_write_2(psc->sc_iot, psc->sc_ioh, offset, vector); 915 ret = bus_space_read_2(psc->sc_iot, psc->sc_ioh, offset); 916 if (ret != vector) { 917 aprint_debug_dev(sc->sc_dev, "%s: expected=%d, actual=%d\n", 918 __func__, vector, ret); 919 VIRTIO_PCI_LOG(sc, reinit, 920 "can't set config msix vector\n"); 921 return -1; 922 } 923 924 for (qid = 0; qid < sc->sc_nvqs; qid++) { 925 offset = VIRTIO_CONFIG_QUEUE_SELECT; 926 bus_space_write_2(psc->sc_iot, psc->sc_ioh, offset, qid); 927 928 offset = VIRTIO_CONFIG_MSI_QUEUE_VECTOR; 929 vector = VIRTIO_MSIX_QUEUE_VECTOR_INDEX; 930 931 if (psc->sc_intr_pervq) 932 vector += qid; 933 934 bus_space_write_2(psc->sc_iot, psc->sc_ioh, offset, vector); 935 ret = bus_space_read_2(psc->sc_iot, psc->sc_ioh, offset); 936 if (ret != vector) { 937 aprint_debug_dev(sc->sc_dev, "%s[qid=%d]:" 938 " expected=%d, actual=%d\n", 939 __func__, qid, vector, ret); 940 VIRTIO_PCI_LOG(sc, reinit, "can't set queue %d " 941 "msix vector\n", qid); 942 return -1; 943 } 944 } 945 946 return 0; 947 } 948 949 static int 950 virtio_pci_establish_msix_interrupts(struct virtio_softc *sc, 951 struct pci_attach_args *pa) 952 { 953 struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc; 954 device_t self = sc->sc_dev; 955 pci_chipset_tag_t pc = pa->pa_pc; 956 struct virtqueue *vq; 957 char intrbuf[PCI_INTRSTR_LEN]; 958 char intr_xname[INTRDEVNAMEBUF]; 959 char const *intrstr; 960 int idx, qid, n; 961 962 idx = VIRTIO_MSIX_CONFIG_VECTOR_INDEX; 963 if (sc->sc_flags & VIRTIO_F_INTR_MPSAFE) 964 pci_intr_setattr(pc, &psc->sc_ihp[idx], PCI_INTR_MPSAFE, true); 965 966 snprintf(intr_xname, sizeof(intr_xname), "%s config", 967 device_xname(sc->sc_dev)); 968 969 psc->sc_ihs[idx] = pci_intr_establish_xname(pc, psc->sc_ihp[idx], 970 sc->sc_ipl, virtio_pci_msix_config_intr, sc, intr_xname); 971 if (psc->sc_ihs[idx] == NULL) { 972 aprint_error_dev(self, "couldn't establish MSI-X for config\n"); 973 goto error; 974 } 975 976 idx = VIRTIO_MSIX_QUEUE_VECTOR_INDEX; 977 if (psc->sc_intr_pervq) { 978 for (qid = 0; qid < sc->sc_nvqs; qid++) { 979 n = idx + qid; 980 vq = &sc->sc_vqs[qid]; 981 982 snprintf(intr_xname, sizeof(intr_xname), "%s vq#%d", 983 device_xname(sc->sc_dev), qid); 984 985 if (sc->sc_flags & VIRTIO_F_INTR_MPSAFE) { 986 pci_intr_setattr(pc, &psc->sc_ihp[n], 987 PCI_INTR_MPSAFE, true); 988 } 989 990 psc->sc_ihs[n] = pci_intr_establish_xname(pc, psc->sc_ihp[n], 991 sc->sc_ipl, vq->vq_intrhand, vq->vq_intrhand_arg, intr_xname); 992 if (psc->sc_ihs[n] == NULL) { 993 aprint_error_dev(self, "couldn't establish MSI-X for a vq\n"); 994 goto error; 995 } 996 } 997 } else { 998 if (sc->sc_flags & VIRTIO_F_INTR_MPSAFE) 999 pci_intr_setattr(pc, &psc->sc_ihp[idx], PCI_INTR_MPSAFE, true); 1000 1001 snprintf(intr_xname, sizeof(intr_xname), "%s queues", 1002 device_xname(sc->sc_dev)); 1003 psc->sc_ihs[idx] = pci_intr_establish_xname(pc, psc->sc_ihp[idx], 1004 sc->sc_ipl, virtio_pci_msix_queue_intr, sc, intr_xname); 1005 if (psc->sc_ihs[idx] == NULL) { 1006 aprint_error_dev(self, "couldn't establish MSI-X for queues\n"); 1007 goto error; 1008 } 1009 } 1010 1011 idx = VIRTIO_MSIX_CONFIG_VECTOR_INDEX; 1012 intrstr = pci_intr_string(pc, psc->sc_ihp[idx], intrbuf, sizeof(intrbuf)); 1013 aprint_normal_dev(self, "config interrupting at %s\n", intrstr); 1014 idx = VIRTIO_MSIX_QUEUE_VECTOR_INDEX; 1015 if (psc->sc_intr_pervq) { 1016 kcpuset_t *affinity; 1017 int affinity_to, r; 1018 1019 kcpuset_create(&affinity, false); 1020 1021 for (qid = 0; qid < sc->sc_nvqs; qid++) { 1022 n = idx + qid; 1023 affinity_to = (qid / 2) % ncpu; 1024 1025 intrstr = pci_intr_string(pc, psc->sc_ihp[n], 1026 intrbuf, sizeof(intrbuf)); 1027 1028 kcpuset_zero(affinity); 1029 kcpuset_set(affinity, affinity_to); 1030 r = interrupt_distribute(psc->sc_ihs[n], affinity, NULL); 1031 if (r == 0) { 1032 aprint_normal_dev(self, 1033 "for vq #%d interrupting at %s affinity to %u\n", 1034 qid, intrstr, affinity_to); 1035 } else { 1036 aprint_normal_dev(self, 1037 "for vq #%d interrupting at %s\n", 1038 qid, intrstr); 1039 } 1040 } 1041 1042 kcpuset_destroy(affinity); 1043 } else { 1044 intrstr = pci_intr_string(pc, psc->sc_ihp[idx], intrbuf, sizeof(intrbuf)); 1045 aprint_normal_dev(self, "queues interrupting at %s\n", intrstr); 1046 } 1047 1048 return 0; 1049 1050 error: 1051 idx = VIRTIO_MSIX_CONFIG_VECTOR_INDEX; 1052 if (psc->sc_ihs[idx] != NULL) 1053 pci_intr_disestablish(psc->sc_pa.pa_pc, psc->sc_ihs[idx]); 1054 idx = VIRTIO_MSIX_QUEUE_VECTOR_INDEX; 1055 if (psc->sc_intr_pervq) { 1056 for (qid = 0; qid < sc->sc_nvqs; qid++) { 1057 n = idx + qid; 1058 if (psc->sc_ihs[n] == NULL) 1059 continue; 1060 pci_intr_disestablish(psc->sc_pa.pa_pc, psc->sc_ihs[n]); 1061 } 1062 1063 } else { 1064 if (psc->sc_ihs[idx] != NULL) 1065 pci_intr_disestablish(psc->sc_pa.pa_pc, psc->sc_ihs[idx]); 1066 } 1067 1068 return -1; 1069 } 1070 1071 static int 1072 virtio_pci_establish_intx_interrupt(struct virtio_softc *sc, 1073 struct pci_attach_args *pa) 1074 { 1075 struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc; 1076 device_t self = sc->sc_dev; 1077 pci_chipset_tag_t pc = pa->pa_pc; 1078 char intrbuf[PCI_INTRSTR_LEN]; 1079 char const *intrstr; 1080 1081 if (sc->sc_flags & VIRTIO_F_INTR_MPSAFE) 1082 pci_intr_setattr(pc, &psc->sc_ihp[0], PCI_INTR_MPSAFE, true); 1083 1084 psc->sc_ihs[0] = pci_intr_establish_xname(pc, psc->sc_ihp[0], 1085 sc->sc_ipl, virtio_pci_intr, sc, device_xname(sc->sc_dev)); 1086 if (psc->sc_ihs[0] == NULL) { 1087 aprint_error_dev(self, "couldn't establish INTx\n"); 1088 return -1; 1089 } 1090 1091 intrstr = pci_intr_string(pc, psc->sc_ihp[0], intrbuf, sizeof(intrbuf)); 1092 aprint_normal_dev(self, "interrupting at %s\n", intrstr); 1093 1094 return 0; 1095 } 1096 1097 static int 1098 virtio_pci_alloc_interrupts(struct virtio_softc *sc) 1099 { 1100 struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc; 1101 device_t self = sc->sc_dev; 1102 pci_chipset_tag_t pc = psc->sc_pa.pa_pc; 1103 pcitag_t tag = psc->sc_pa.pa_tag; 1104 int error; 1105 int nmsix; 1106 int off; 1107 int counts[PCI_INTR_TYPE_SIZE]; 1108 pci_intr_type_t max_type; 1109 pcireg_t ctl; 1110 1111 nmsix = pci_msix_count(psc->sc_pa.pa_pc, psc->sc_pa.pa_tag); 1112 aprint_debug_dev(self, "pci_msix_count=%d\n", nmsix); 1113 1114 /* We need at least two: one for config and the other for queues */ 1115 if ((sc->sc_flags & VIRTIO_F_INTR_MSIX) == 0 || nmsix < 2) { 1116 /* Try INTx only */ 1117 max_type = PCI_INTR_TYPE_INTX; 1118 counts[PCI_INTR_TYPE_INTX] = 1; 1119 } else { 1120 /* Try MSI-X first and INTx second */ 1121 if (ISSET(sc->sc_flags, VIRTIO_F_INTR_PERVQ) && 1122 sc->sc_nvqs + VIRTIO_MSIX_QUEUE_VECTOR_INDEX <= nmsix) { 1123 nmsix = sc->sc_nvqs + VIRTIO_MSIX_QUEUE_VECTOR_INDEX; 1124 } else { 1125 nmsix = 2; 1126 } 1127 1128 max_type = PCI_INTR_TYPE_MSIX; 1129 counts[PCI_INTR_TYPE_MSIX] = nmsix; 1130 counts[PCI_INTR_TYPE_MSI] = 0; 1131 counts[PCI_INTR_TYPE_INTX] = 1; 1132 } 1133 1134 retry: 1135 error = pci_intr_alloc(&psc->sc_pa, &psc->sc_ihp, counts, max_type); 1136 if (error != 0) { 1137 aprint_error_dev(self, "couldn't map interrupt\n"); 1138 return -1; 1139 } 1140 1141 if (pci_intr_type(pc, psc->sc_ihp[0]) == PCI_INTR_TYPE_MSIX) { 1142 psc->sc_intr_pervq = nmsix > 2 ? true : false; 1143 psc->sc_ihs = kmem_zalloc(sizeof(*psc->sc_ihs) * nmsix, 1144 KM_SLEEP); 1145 1146 error = virtio_pci_establish_msix_interrupts(sc, &psc->sc_pa); 1147 if (error != 0) { 1148 kmem_free(psc->sc_ihs, sizeof(*psc->sc_ihs) * nmsix); 1149 pci_intr_release(pc, psc->sc_ihp, nmsix); 1150 1151 /* Retry INTx */ 1152 max_type = PCI_INTR_TYPE_INTX; 1153 counts[PCI_INTR_TYPE_INTX] = 1; 1154 goto retry; 1155 } 1156 1157 psc->sc_ihs_num = nmsix; 1158 psc->sc_devcfg_offset = VIRTIO_CONFIG_DEVICE_CONFIG_MSI; 1159 virtio_pci_adjust_config_region(psc); 1160 } else if (pci_intr_type(pc, psc->sc_ihp[0]) == PCI_INTR_TYPE_INTX) { 1161 psc->sc_intr_pervq = false; 1162 psc->sc_ihs = kmem_zalloc(sizeof(*psc->sc_ihs) * 1, 1163 KM_SLEEP); 1164 1165 error = virtio_pci_establish_intx_interrupt(sc, &psc->sc_pa); 1166 if (error != 0) { 1167 kmem_free(psc->sc_ihs, sizeof(*psc->sc_ihs) * 1); 1168 pci_intr_release(pc, psc->sc_ihp, 1); 1169 return -1; 1170 } 1171 1172 psc->sc_ihs_num = 1; 1173 psc->sc_devcfg_offset = VIRTIO_CONFIG_DEVICE_CONFIG_NOMSI; 1174 virtio_pci_adjust_config_region(psc); 1175 1176 error = pci_get_capability(pc, tag, PCI_CAP_MSIX, &off, NULL); 1177 if (error != 0) { 1178 ctl = pci_conf_read(pc, tag, off + PCI_MSIX_CTL); 1179 ctl &= ~PCI_MSIX_CTL_ENABLE; 1180 pci_conf_write(pc, tag, off + PCI_MSIX_CTL, ctl); 1181 } 1182 } 1183 1184 if (!psc->sc_intr_pervq) 1185 CLR(sc->sc_flags, VIRTIO_F_INTR_PERVQ); 1186 return 0; 1187 } 1188 1189 static void 1190 virtio_pci_free_interrupts(struct virtio_softc *sc) 1191 { 1192 struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc; 1193 1194 for (int i = 0; i < psc->sc_ihs_num; i++) { 1195 if (psc->sc_ihs[i] == NULL) 1196 continue; 1197 pci_intr_disestablish(psc->sc_pa.pa_pc, psc->sc_ihs[i]); 1198 psc->sc_ihs[i] = NULL; 1199 } 1200 1201 if (psc->sc_ihs_num > 0) 1202 pci_intr_release(psc->sc_pa.pa_pc, psc->sc_ihp, psc->sc_ihs_num); 1203 1204 if (psc->sc_ihs != NULL) { 1205 kmem_free(psc->sc_ihs, sizeof(*psc->sc_ihs) * psc->sc_ihs_num); 1206 psc->sc_ihs = NULL; 1207 } 1208 psc->sc_ihs_num = 0; 1209 } 1210 1211 static bool 1212 virtio_pci_msix_enabled(struct virtio_pci_softc *psc) 1213 { 1214 pci_chipset_tag_t pc = psc->sc_pa.pa_pc; 1215 1216 if (pci_intr_type(pc, psc->sc_ihp[0]) == PCI_INTR_TYPE_MSIX) 1217 return true; 1218 1219 return false; 1220 } 1221 1222 /* 1223 * Interrupt handler. 1224 */ 1225 static int 1226 virtio_pci_intr(void *arg) 1227 { 1228 struct virtio_softc *sc = arg; 1229 struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc; 1230 int isr, r = 0; 1231 1232 /* check and ack the interrupt */ 1233 isr = bus_space_read_1(psc->sc_isr_iot, psc->sc_isr_ioh, 0); 1234 if (isr == 0) 1235 return 0; 1236 if ((isr & VIRTIO_CONFIG_ISR_CONFIG_CHANGE) && 1237 (sc->sc_config_change != NULL)) 1238 r = (sc->sc_config_change)(sc); 1239 if (sc->sc_intrhand != NULL) { 1240 if (sc->sc_soft_ih != NULL) 1241 softint_schedule(sc->sc_soft_ih); 1242 else 1243 r |= (sc->sc_intrhand)(sc); 1244 } 1245 1246 return r; 1247 } 1248 1249 static int 1250 virtio_pci_msix_queue_intr(void *arg) 1251 { 1252 struct virtio_softc *sc = arg; 1253 int r = 0; 1254 1255 if (sc->sc_intrhand != NULL) { 1256 if (sc->sc_soft_ih != NULL) 1257 softint_schedule(sc->sc_soft_ih); 1258 else 1259 r |= (sc->sc_intrhand)(sc); 1260 } 1261 1262 return r; 1263 } 1264 1265 static int 1266 virtio_pci_msix_config_intr(void *arg) 1267 { 1268 struct virtio_softc *sc = arg; 1269 int r = 0; 1270 1271 if (sc->sc_config_change != NULL) 1272 r = (sc->sc_config_change)(sc); 1273 return r; 1274 } 1275 1276 MODULE(MODULE_CLASS_DRIVER, virtio_pci, "pci,virtio"); 1277 1278 #ifdef _MODULE 1279 #include "ioconf.c" 1280 #endif 1281 1282 static int 1283 virtio_pci_modcmd(modcmd_t cmd, void *opaque) 1284 { 1285 int error = 0; 1286 1287 #ifdef _MODULE 1288 switch (cmd) { 1289 case MODULE_CMD_INIT: 1290 error = config_init_component(cfdriver_ioconf_virtio_pci, 1291 cfattach_ioconf_virtio_pci, cfdata_ioconf_virtio_pci); 1292 break; 1293 case MODULE_CMD_FINI: 1294 error = config_fini_component(cfdriver_ioconf_virtio_pci, 1295 cfattach_ioconf_virtio_pci, cfdata_ioconf_virtio_pci); 1296 break; 1297 default: 1298 error = ENOTTY; 1299 break; 1300 } 1301 #endif 1302 1303 return error; 1304 } 1305
Indexes created Tue Sep 30 20:09:53 GMT 2025