virtio.c revision 1.63.2.6
1 /* $NetBSD: virtio.c,v 1.63.2.6 2024/10/02 18:20:48 martin Exp $ */ 2 3 /* 4 * Copyright (c) 2020 The NetBSD Foundation, Inc. 5 * Copyright (c) 2012 Stefan Fritsch, Alexander Fiveg. 6 * Copyright (c) 2010 Minoura Makoto. 7 * All rights reserved. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 19 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 20 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 21 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 22 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 23 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 27 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 */ 29 30 #include <sys/cdefs.h> 31 __KERNEL_RCSID(0, "$NetBSD: virtio.c,v 1.63.2.6 2024/10/02 18:20:48 martin Exp $"); 32 33 #include <sys/param.h> 34 #include <sys/systm.h> 35 #include <sys/kernel.h> 36 #include <sys/atomic.h> 37 #include <sys/bus.h> 38 #include <sys/device.h> 39 #include <sys/kmem.h> 40 #include <sys/module.h> 41 42 #define VIRTIO_PRIVATE 43 44 #include <dev/pci/virtioreg.h> /* XXX: move to non-pci */ 45 #include <dev/pci/virtiovar.h> /* XXX: move to non-pci */ 46 47 #define MINSEG_INDIRECT 2 /* use indirect if nsegs >= this value */ 48 49 /* 50 * The maximum descriptor size is 2^15. Use that value as the end of 51 * descriptor chain terminator since it will never be a valid index 52 * in the descriptor table. 53 */ 54 #define VRING_DESC_CHAIN_END 32768 55 56 /* incomplete list */ 57 static const char *virtio_device_name[] = { 58 "unknown (0)", /* 0 */ 59 "network", /* 1 */ 60 "block", /* 2 */ 61 "console", /* 3 */ 62 "entropy", /* 4 */ 63 "memory balloon", /* 5 */ 64 "I/O memory", /* 6 */ 65 "remote processor messaging", /* 7 */ 66 "SCSI", /* 8 */ 67 "9P transport", /* 9 */ 68 }; 69 #define NDEVNAMES __arraycount(virtio_device_name) 70 71 static void virtio_reset_vq(struct virtio_softc *, 72 struct virtqueue *); 73 74 void 75 virtio_set_status(struct virtio_softc *sc, int status) 76 { 77 sc->sc_ops->set_status(sc, status); 78 } 79 80 /* 81 * Reset the device. 82 */ 83 /* 84 * To reset the device to a known state, do following: 85 * virtio_reset(sc); // this will stop the device activity 86 * <dequeue finished requests>; // virtio_dequeue() still can be called 87 * <revoke pending requests in the vqs if any>; 88 * virtio_reinit_start(sc); // dequeue prohibited 89 * newfeatures = virtio_negotiate_features(sc, requestedfeatures); 90 * <some other initialization>; 91 * virtio_reinit_end(sc); // device activated; enqueue allowed 92 * Once attached, feature negotiation can only be allowed after virtio_reset. 93 */ 94 void 95 virtio_reset(struct virtio_softc *sc) 96 { 97 virtio_device_reset(sc); 98 } 99 100 int 101 virtio_reinit_start(struct virtio_softc *sc) 102 { 103 int i, r; 104 105 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_ACK); 106 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_DRIVER); 107 for (i = 0; i < sc->sc_nvqs; i++) { 108 int n; 109 struct virtqueue *vq = &sc->sc_vqs[i]; 110 n = sc->sc_ops->read_queue_size(sc, vq->vq_index); 111 if (n == 0) /* vq disappeared */ 112 continue; 113 if (n != vq->vq_num) { 114 panic("%s: virtqueue size changed, vq index %d\n", 115 device_xname(sc->sc_dev), 116 vq->vq_index); 117 } 118 virtio_reset_vq(sc, vq); 119 sc->sc_ops->setup_queue(sc, vq->vq_index, 120 vq->vq_dmamap->dm_segs[0].ds_addr); 121 } 122 123 r = sc->sc_ops->setup_interrupts(sc, 1); 124 if (r != 0) 125 goto fail; 126 127 return 0; 128 129 fail: 130 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_FAILED); 131 132 return 1; 133 } 134 135 void 136 virtio_reinit_end(struct virtio_softc *sc) 137 { 138 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_DRIVER_OK); 139 } 140 141 /* 142 * Feature negotiation. 143 */ 144 void 145 virtio_negotiate_features(struct virtio_softc *sc, uint64_t guest_features) 146 { 147 if (!(device_cfdata(sc->sc_dev)->cf_flags & 1) && 148 !(device_cfdata(sc->sc_child)->cf_flags & 1)) /* XXX */ 149 guest_features |= VIRTIO_F_RING_INDIRECT_DESC; 150 sc->sc_ops->neg_features(sc, guest_features); 151 if (sc->sc_active_features & VIRTIO_F_RING_INDIRECT_DESC) 152 sc->sc_indirect = true; 153 else 154 sc->sc_indirect = false; 155 } 156 157 158 /* 159 * Device configuration registers readers/writers 160 */ 161 #if 0 162 #define DPRINTFR(n, fmt, val, index, num) \ 163 printf("\n%s (", n); \ 164 for (int i = 0; i < num; i++) \ 165 printf("%02x ", bus_space_read_1(sc->sc_devcfg_iot, sc->sc_devcfg_ioh, index+i)); \ 166 printf(") -> "); printf(fmt, val); printf("\n"); 167 #define DPRINTFR2(n, fmt, val_s, val_n) \ 168 printf("%s ", n); \ 169 printf("\n stream "); printf(fmt, val_s); printf(" norm "); printf(fmt, val_n); printf("\n"); 170 #else 171 #define DPRINTFR(n, fmt, val, index, num) 172 #define DPRINTFR2(n, fmt, val_s, val_n) 173 #endif 174 175 176 uint8_t 177 virtio_read_device_config_1(struct virtio_softc *sc, int index) 178 { 179 bus_space_tag_t iot = sc->sc_devcfg_iot; 180 bus_space_handle_t ioh = sc->sc_devcfg_ioh; 181 uint8_t val; 182 183 val = bus_space_read_1(iot, ioh, index); 184 185 DPRINTFR("read_1", "%02x", val, index, 1); 186 return val; 187 } 188 189 uint16_t 190 virtio_read_device_config_2(struct virtio_softc *sc, int index) 191 { 192 bus_space_tag_t iot = sc->sc_devcfg_iot; 193 bus_space_handle_t ioh = sc->sc_devcfg_ioh; 194 uint16_t val; 195 196 val = bus_space_read_2(iot, ioh, index); 197 if (BYTE_ORDER != sc->sc_bus_endian) 198 val = bswap16(val); 199 200 DPRINTFR("read_2", "%04x", val, index, 2); 201 DPRINTFR2("read_2", "%04x", 202 bus_space_read_stream_2(sc->sc_devcfg_iot, sc->sc_devcfg_ioh, 203 index), 204 bus_space_read_2(sc->sc_devcfg_iot, sc->sc_devcfg_ioh, index)); 205 return val; 206 } 207 208 uint32_t 209 virtio_read_device_config_4(struct virtio_softc *sc, int index) 210 { 211 bus_space_tag_t iot = sc->sc_devcfg_iot; 212 bus_space_handle_t ioh = sc->sc_devcfg_ioh; 213 uint32_t val; 214 215 val = bus_space_read_4(iot, ioh, index); 216 if (BYTE_ORDER != sc->sc_bus_endian) 217 val = bswap32(val); 218 219 DPRINTFR("read_4", "%08x", val, index, 4); 220 DPRINTFR2("read_4", "%08x", 221 bus_space_read_stream_4(sc->sc_devcfg_iot, sc->sc_devcfg_ioh, 222 index), 223 bus_space_read_4(sc->sc_devcfg_iot, sc->sc_devcfg_ioh, index)); 224 return val; 225 } 226 227 /* 228 * The Virtio spec explicitly tells that reading and writing 8 bytes are not 229 * considered atomic and no triggers may be connected to reading or writing 230 * it. We access it using two 32 reads. See virtio spec 4.1.3.1. 231 */ 232 uint64_t 233 virtio_read_device_config_8(struct virtio_softc *sc, int index) 234 { 235 bus_space_tag_t iot = sc->sc_devcfg_iot; 236 bus_space_handle_t ioh = sc->sc_devcfg_ioh; 237 union { 238 uint64_t u64; 239 uint32_t l[2]; 240 } v; 241 uint64_t val; 242 243 v.l[0] = bus_space_read_4(iot, ioh, index); 244 v.l[1] = bus_space_read_4(iot, ioh, index + 4); 245 if (sc->sc_bus_endian != sc->sc_struct_endian) { 246 v.l[0] = bswap32(v.l[0]); 247 v.l[1] = bswap32(v.l[1]); 248 } 249 val = v.u64; 250 251 if (BYTE_ORDER != sc->sc_struct_endian) 252 val = bswap64(val); 253 254 DPRINTFR("read_8", "%08"PRIx64, val, index, 8); 255 DPRINTFR2("read_8 low ", "%08x", 256 bus_space_read_stream_4(sc->sc_devcfg_iot, sc->sc_devcfg_ioh, 257 index), 258 bus_space_read_4(sc->sc_devcfg_iot, sc->sc_devcfg_ioh, index)); 259 DPRINTFR2("read_8 high ", "%08x", 260 bus_space_read_stream_4(sc->sc_devcfg_iot, sc->sc_devcfg_ioh, 261 index + 4), 262 bus_space_read_4(sc->sc_devcfg_iot, sc->sc_devcfg_ioh, index + 4)); 263 return val; 264 } 265 266 /* 267 * In the older virtio spec, device config registers are host endian. On newer 268 * they are little endian. Some newer devices however explicitly specify their 269 * register to always be little endian. These functions cater for these. 270 */ 271 uint16_t 272 virtio_read_device_config_le_2(struct virtio_softc *sc, int index) 273 { 274 bus_space_tag_t iot = sc->sc_devcfg_iot; 275 bus_space_handle_t ioh = sc->sc_devcfg_ioh; 276 uint16_t val; 277 278 val = bus_space_read_2(iot, ioh, index); 279 #if !defined(__aarch64__) && !defined(__arm__) 280 /* 281 * For big-endian aarch64/armv7, bus endian is always LSB, but 282 * byte-order is automatically swapped by bus_space(9) (see also 283 * comments in virtio_pci.c). Therefore, no need to swap here. 284 */ 285 if (sc->sc_bus_endian != LITTLE_ENDIAN) 286 val = bswap16(val); 287 #endif 288 289 DPRINTFR("read_le_2", "%04x", val, index, 2); 290 DPRINTFR2("read_le_2", "%04x", 291 bus_space_read_stream_2(sc->sc_devcfg_iot, sc->sc_devcfg_ioh, 0), 292 bus_space_read_2(sc->sc_devcfg_iot, sc->sc_devcfg_ioh, 0)); 293 return val; 294 } 295 296 uint32_t 297 virtio_read_device_config_le_4(struct virtio_softc *sc, int index) 298 { 299 bus_space_tag_t iot = sc->sc_devcfg_iot; 300 bus_space_handle_t ioh = sc->sc_devcfg_ioh; 301 uint32_t val; 302 303 val = bus_space_read_4(iot, ioh, index); 304 #if !defined(__aarch64__) && !defined(__arm__) 305 /* See virtio_read_device_config_le_2() above. */ 306 if (sc->sc_bus_endian != LITTLE_ENDIAN) 307 val = bswap32(val); 308 #endif 309 310 DPRINTFR("read_le_4", "%08x", val, index, 4); 311 DPRINTFR2("read_le_4", "%08x", 312 bus_space_read_stream_4(sc->sc_devcfg_iot, sc->sc_devcfg_ioh, 0), 313 bus_space_read_4(sc->sc_devcfg_iot, sc->sc_devcfg_ioh, 0)); 314 return val; 315 } 316 317 void 318 virtio_write_device_config_1(struct virtio_softc *sc, int index, uint8_t value) 319 { 320 bus_space_tag_t iot = sc->sc_devcfg_iot; 321 bus_space_handle_t ioh = sc->sc_devcfg_ioh; 322 323 bus_space_write_1(iot, ioh, index, value); 324 } 325 326 void 327 virtio_write_device_config_2(struct virtio_softc *sc, int index, 328 uint16_t value) 329 { 330 bus_space_tag_t iot = sc->sc_devcfg_iot; 331 bus_space_handle_t ioh = sc->sc_devcfg_ioh; 332 333 if (BYTE_ORDER != sc->sc_bus_endian) 334 value = bswap16(value); 335 bus_space_write_2(iot, ioh, index, value); 336 } 337 338 void 339 virtio_write_device_config_4(struct virtio_softc *sc, int index, 340 uint32_t value) 341 { 342 bus_space_tag_t iot = sc->sc_devcfg_iot; 343 bus_space_handle_t ioh = sc->sc_devcfg_ioh; 344 345 if (BYTE_ORDER != sc->sc_bus_endian) 346 value = bswap32(value); 347 bus_space_write_4(iot, ioh, index, value); 348 } 349 350 /* 351 * The Virtio spec explicitly tells that reading and writing 8 bytes are not 352 * considered atomic and no triggers may be connected to reading or writing 353 * it. We access it using two 32 bit writes. For good measure it is stated to 354 * always write lsb first just in case of a hypervisor bug. See See virtio 355 * spec 4.1.3.1. 356 */ 357 void 358 virtio_write_device_config_8(struct virtio_softc *sc, int index, 359 uint64_t value) 360 { 361 bus_space_tag_t iot = sc->sc_devcfg_iot; 362 bus_space_handle_t ioh = sc->sc_devcfg_ioh; 363 union { 364 uint64_t u64; 365 uint32_t l[2]; 366 } v; 367 368 if (BYTE_ORDER != sc->sc_struct_endian) 369 value = bswap64(value); 370 371 v.u64 = value; 372 if (sc->sc_bus_endian != sc->sc_struct_endian) { 373 v.l[0] = bswap32(v.l[0]); 374 v.l[1] = bswap32(v.l[1]); 375 } 376 377 if (sc->sc_struct_endian == LITTLE_ENDIAN) { 378 bus_space_write_4(iot, ioh, index, v.l[0]); 379 bus_space_write_4(iot, ioh, index + 4, v.l[1]); 380 } else { 381 bus_space_write_4(iot, ioh, index + 4, v.l[1]); 382 bus_space_write_4(iot, ioh, index, v.l[0]); 383 } 384 } 385 386 /* 387 * In the older virtio spec, device config registers are host endian. On newer 388 * they are little endian. Some newer devices however explicitly specify their 389 * register to always be little endian. These functions cater for these. 390 */ 391 void 392 virtio_write_device_config_le_2(struct virtio_softc *sc, int index, 393 uint16_t value) 394 { 395 bus_space_tag_t iot = sc->sc_devcfg_iot; 396 bus_space_handle_t ioh = sc->sc_devcfg_ioh; 397 398 if (sc->sc_bus_endian != LITTLE_ENDIAN) 399 value = bswap16(value); 400 bus_space_write_2(iot, ioh, index, value); 401 } 402 403 void 404 virtio_write_device_config_le_4(struct virtio_softc *sc, int index, 405 uint32_t value) 406 { 407 bus_space_tag_t iot = sc->sc_devcfg_iot; 408 bus_space_handle_t ioh = sc->sc_devcfg_ioh; 409 410 if (sc->sc_bus_endian != LITTLE_ENDIAN) 411 value = bswap32(value); 412 bus_space_write_4(iot, ioh, index, value); 413 } 414 415 416 /* 417 * data structures endian helpers 418 */ 419 uint16_t 420 virtio_rw16(struct virtio_softc *sc, uint16_t val) 421 { 422 KASSERT(sc); 423 return BYTE_ORDER != sc->sc_struct_endian ? bswap16(val) : val; 424 } 425 426 uint32_t 427 virtio_rw32(struct virtio_softc *sc, uint32_t val) 428 { 429 KASSERT(sc); 430 return BYTE_ORDER != sc->sc_struct_endian ? bswap32(val) : val; 431 } 432 433 uint64_t 434 virtio_rw64(struct virtio_softc *sc, uint64_t val) 435 { 436 KASSERT(sc); 437 return BYTE_ORDER != sc->sc_struct_endian ? bswap64(val) : val; 438 } 439 440 441 /* 442 * Interrupt handler. 443 */ 444 static void 445 virtio_soft_intr(void *arg) 446 { 447 struct virtio_softc *sc = arg; 448 449 KASSERT(sc->sc_intrhand != NULL); 450 451 (*sc->sc_intrhand)(sc); 452 } 453 454 /* set to vq->vq_intrhand in virtio_init_vq_vqdone() */ 455 static int 456 virtio_vq_done(void *xvq) 457 { 458 struct virtqueue *vq = xvq; 459 460 return vq->vq_done(vq); 461 } 462 463 static int 464 virtio_vq_intr(struct virtio_softc *sc) 465 { 466 struct virtqueue *vq; 467 int i, r = 0; 468 469 for (i = 0; i < sc->sc_nvqs; i++) { 470 vq = &sc->sc_vqs[i]; 471 if (virtio_vq_is_enqueued(sc, vq) == 1) { 472 r |= (*vq->vq_intrhand)(vq->vq_intrhand_arg); 473 } 474 } 475 476 return r; 477 } 478 479 /* 480 * dmamap sync operations for a virtqueue. 481 */ 482 static inline void 483 vq_sync_descs(struct virtio_softc *sc, struct virtqueue *vq, int ops) 484 { 485 486 /* availoffset == sizeof(vring_desc) * vq_num */ 487 bus_dmamap_sync(sc->sc_dmat, vq->vq_dmamap, 0, vq->vq_availoffset, 488 ops); 489 } 490 491 static inline void 492 vq_sync_aring_all(struct virtio_softc *sc, struct virtqueue *vq, int ops) 493 { 494 uint16_t hdrlen = offsetof(struct vring_avail, ring); 495 size_t payloadlen = vq->vq_num * sizeof(uint16_t); 496 size_t usedlen = 0; 497 498 if (sc->sc_active_features & VIRTIO_F_RING_EVENT_IDX) 499 usedlen = sizeof(uint16_t); 500 bus_dmamap_sync(sc->sc_dmat, vq->vq_dmamap, 501 vq->vq_availoffset, hdrlen + payloadlen + usedlen, ops); 502 } 503 504 static inline void 505 vq_sync_aring_header(struct virtio_softc *sc, struct virtqueue *vq, int ops) 506 { 507 uint16_t hdrlen = offsetof(struct vring_avail, ring); 508 509 bus_dmamap_sync(sc->sc_dmat, vq->vq_dmamap, 510 vq->vq_availoffset, hdrlen, ops); 511 } 512 513 static inline void 514 vq_sync_aring_payload(struct virtio_softc *sc, struct virtqueue *vq, int ops) 515 { 516 uint16_t hdrlen = offsetof(struct vring_avail, ring); 517 size_t payloadlen = vq->vq_num * sizeof(uint16_t); 518 519 bus_dmamap_sync(sc->sc_dmat, vq->vq_dmamap, 520 vq->vq_availoffset + hdrlen, payloadlen, ops); 521 } 522 523 static inline void 524 vq_sync_aring_used(struct virtio_softc *sc, struct virtqueue *vq, int ops) 525 { 526 uint16_t hdrlen = offsetof(struct vring_avail, ring); 527 size_t payloadlen = vq->vq_num * sizeof(uint16_t); 528 size_t usedlen = sizeof(uint16_t); 529 530 if ((sc->sc_active_features & VIRTIO_F_RING_EVENT_IDX) == 0) 531 return; 532 bus_dmamap_sync(sc->sc_dmat, vq->vq_dmamap, 533 vq->vq_availoffset + hdrlen + payloadlen, usedlen, ops); 534 } 535 536 static inline void 537 vq_sync_uring_all(struct virtio_softc *sc, struct virtqueue *vq, int ops) 538 { 539 uint16_t hdrlen = offsetof(struct vring_used, ring); 540 size_t payloadlen = vq->vq_num * sizeof(struct vring_used_elem); 541 size_t availlen = 0; 542 543 if (sc->sc_active_features & VIRTIO_F_RING_EVENT_IDX) 544 availlen = sizeof(uint16_t); 545 bus_dmamap_sync(sc->sc_dmat, vq->vq_dmamap, 546 vq->vq_usedoffset, hdrlen + payloadlen + availlen, ops); 547 } 548 549 static inline void 550 vq_sync_uring_header(struct virtio_softc *sc, struct virtqueue *vq, int ops) 551 { 552 uint16_t hdrlen = offsetof(struct vring_used, ring); 553 554 bus_dmamap_sync(sc->sc_dmat, vq->vq_dmamap, 555 vq->vq_usedoffset, hdrlen, ops); 556 } 557 558 static inline void 559 vq_sync_uring_payload(struct virtio_softc *sc, struct virtqueue *vq, int ops) 560 { 561 uint16_t hdrlen = offsetof(struct vring_used, ring); 562 size_t payloadlen = vq->vq_num * sizeof(struct vring_used_elem); 563 564 bus_dmamap_sync(sc->sc_dmat, vq->vq_dmamap, 565 vq->vq_usedoffset + hdrlen, payloadlen, ops); 566 } 567 568 static inline void 569 vq_sync_uring_avail(struct virtio_softc *sc, struct virtqueue *vq, int ops) 570 { 571 uint16_t hdrlen = offsetof(struct vring_used, ring); 572 size_t payloadlen = vq->vq_num * sizeof(struct vring_used_elem); 573 size_t availlen = sizeof(uint16_t); 574 575 if ((sc->sc_active_features & VIRTIO_F_RING_EVENT_IDX) == 0) 576 return; 577 bus_dmamap_sync(sc->sc_dmat, vq->vq_dmamap, 578 vq->vq_usedoffset + hdrlen + payloadlen, availlen, ops); 579 } 580 581 static inline void 582 vq_sync_indirect(struct virtio_softc *sc, struct virtqueue *vq, int slot, 583 int ops) 584 { 585 int offset = vq->vq_indirectoffset + 586 sizeof(struct vring_desc) * vq->vq_maxnsegs * slot; 587 588 bus_dmamap_sync(sc->sc_dmat, vq->vq_dmamap, 589 offset, sizeof(struct vring_desc) * vq->vq_maxnsegs, ops); 590 } 591 592 bool 593 virtio_vq_is_enqueued(struct virtio_softc *sc, struct virtqueue *vq) 594 { 595 596 if (vq->vq_queued) { 597 vq->vq_queued = 0; 598 vq_sync_aring_all(sc, vq, BUS_DMASYNC_POSTWRITE); 599 } 600 601 vq_sync_uring_header(sc, vq, BUS_DMASYNC_POSTREAD); 602 if (vq->vq_used_idx == virtio_rw16(sc, vq->vq_used->idx)) 603 return 0; 604 vq_sync_uring_payload(sc, vq, BUS_DMASYNC_POSTREAD); 605 return 1; 606 } 607 608 /* 609 * Increase the event index in order to delay interrupts. 610 */ 611 int 612 virtio_postpone_intr(struct virtio_softc *sc, struct virtqueue *vq, 613 uint16_t nslots) 614 { 615 uint16_t idx, nused; 616 617 idx = vq->vq_used_idx + nslots; 618 619 /* set the new event index: avail_ring->used_event = idx */ 620 *vq->vq_used_event = virtio_rw16(sc, idx); 621 vq_sync_aring_used(vq->vq_owner, vq, BUS_DMASYNC_PREWRITE); 622 vq->vq_queued++; 623 624 nused = (uint16_t) 625 (virtio_rw16(sc, vq->vq_used->idx) - vq->vq_used_idx); 626 KASSERT(nused <= vq->vq_num); 627 628 return nslots < nused; 629 } 630 631 /* 632 * Postpone interrupt until 3/4 of the available descriptors have been 633 * consumed. 634 */ 635 int 636 virtio_postpone_intr_smart(struct virtio_softc *sc, struct virtqueue *vq) 637 { 638 uint16_t nslots; 639 640 nslots = (uint16_t) 641 (virtio_rw16(sc, vq->vq_avail->idx) - vq->vq_used_idx) * 3 / 4; 642 643 return virtio_postpone_intr(sc, vq, nslots); 644 } 645 646 /* 647 * Postpone interrupt until all of the available descriptors have been 648 * consumed. 649 */ 650 int 651 virtio_postpone_intr_far(struct virtio_softc *sc, struct virtqueue *vq) 652 { 653 uint16_t nslots; 654 655 nslots = (uint16_t) 656 (virtio_rw16(sc, vq->vq_avail->idx) - vq->vq_used_idx); 657 658 return virtio_postpone_intr(sc, vq, nslots); 659 } 660 661 /* 662 * Start/stop vq interrupt. No guarantee. 663 */ 664 void 665 virtio_stop_vq_intr(struct virtio_softc *sc, struct virtqueue *vq) 666 { 667 668 if (sc->sc_active_features & VIRTIO_F_RING_EVENT_IDX) { 669 /* 670 * No way to disable the interrupt completely with 671 * RingEventIdx. Instead advance used_event by half the 672 * possible value. This won't happen soon and is far enough in 673 * the past to not trigger a spurious interrupt. 674 */ 675 *vq->vq_used_event = virtio_rw16(sc, vq->vq_used_idx + 0x8000); 676 vq_sync_aring_used(sc, vq, BUS_DMASYNC_PREWRITE); 677 } else { 678 vq->vq_avail->flags |= 679 virtio_rw16(sc, VRING_AVAIL_F_NO_INTERRUPT); 680 vq_sync_aring_header(sc, vq, BUS_DMASYNC_PREWRITE); 681 } 682 vq->vq_queued++; 683 } 684 685 int 686 virtio_start_vq_intr(struct virtio_softc *sc, struct virtqueue *vq) 687 { 688 689 if (sc->sc_active_features & VIRTIO_F_RING_EVENT_IDX) { 690 /* 691 * If event index feature is negotiated, enabling interrupts 692 * is done through setting the latest consumed index in the 693 * used_event field 694 */ 695 *vq->vq_used_event = virtio_rw16(sc, vq->vq_used_idx); 696 vq_sync_aring_used(sc, vq, BUS_DMASYNC_PREWRITE); 697 } else { 698 vq->vq_avail->flags &= 699 ~virtio_rw16(sc, VRING_AVAIL_F_NO_INTERRUPT); 700 vq_sync_aring_header(sc, vq, BUS_DMASYNC_PREWRITE); 701 } 702 vq->vq_queued++; 703 704 vq_sync_uring_header(sc, vq, BUS_DMASYNC_POSTREAD); 705 if (vq->vq_used_idx == virtio_rw16(sc, vq->vq_used->idx)) 706 return 0; 707 vq_sync_uring_payload(sc, vq, BUS_DMASYNC_POSTREAD); 708 return 1; 709 } 710 711 /* 712 * Initialize vq structure. 713 */ 714 /* 715 * Reset virtqueue parameters 716 */ 717 static void 718 virtio_reset_vq(struct virtio_softc *sc, struct virtqueue *vq) 719 { 720 struct vring_desc *vds; 721 int i, j; 722 int vq_size = vq->vq_num; 723 724 memset(vq->vq_vaddr, 0, vq->vq_bytesize); 725 726 /* build the descriptor chain for free slot management */ 727 vds = vq->vq_desc; 728 for (i = 0; i < vq_size - 1; i++) { 729 vds[i].next = virtio_rw16(sc, i + 1); 730 } 731 vds[i].next = virtio_rw16(sc, VRING_DESC_CHAIN_END); 732 vq->vq_free_idx = 0; 733 734 /* build the indirect descriptor chain */ 735 if (vq->vq_indirect != NULL) { 736 struct vring_desc *vd; 737 738 for (i = 0; i < vq_size; i++) { 739 vd = vq->vq_indirect; 740 vd += vq->vq_maxnsegs * i; 741 for (j = 0; j < vq->vq_maxnsegs - 1; j++) { 742 vd[j].next = virtio_rw16(sc, j + 1); 743 } 744 } 745 } 746 747 /* enqueue/dequeue status */ 748 vq->vq_avail_idx = 0; 749 vq->vq_used_idx = 0; 750 vq->vq_queued = 0; 751 vq_sync_uring_all(sc, vq, BUS_DMASYNC_PREREAD); 752 vq->vq_queued++; 753 } 754 755 /* Initialize vq */ 756 void 757 virtio_init_vq_vqdone(struct virtio_softc *sc, struct virtqueue *vq, 758 int index, int (*vq_done)(struct virtqueue *)) 759 { 760 761 virtio_init_vq(sc, vq, index, virtio_vq_done, vq); 762 vq->vq_done = vq_done; 763 } 764 765 void 766 virtio_init_vq(struct virtio_softc *sc, struct virtqueue *vq, int index, 767 int (*func)(void *), void *arg) 768 { 769 770 memset(vq, 0, sizeof(*vq)); 771 772 vq->vq_owner = sc; 773 vq->vq_num = sc->sc_ops->read_queue_size(sc, index); 774 vq->vq_index = index; 775 vq->vq_intrhand = func; 776 vq->vq_intrhand_arg = arg; 777 } 778 779 /* 780 * Allocate/free a vq. 781 */ 782 int 783 virtio_alloc_vq(struct virtio_softc *sc, struct virtqueue *vq, 784 int maxsegsize, int maxnsegs, const char *name) 785 { 786 bus_size_t size_desc, size_avail, size_used, size_indirect; 787 bus_size_t allocsize = 0, size_desc_avail; 788 int rsegs, r, hdrlen; 789 unsigned int vq_num; 790 #define VIRTQUEUE_ALIGN(n) roundup(n, VIRTIO_PAGE_SIZE) 791 792 vq_num = vq->vq_num; 793 794 if (vq_num == 0) { 795 aprint_error_dev(sc->sc_dev, 796 "virtqueue not exist, index %d for %s\n", 797 vq->vq_index, name); 798 goto err; 799 } 800 801 hdrlen = sc->sc_active_features & VIRTIO_F_RING_EVENT_IDX ? 3 : 2; 802 803 size_desc = sizeof(vq->vq_desc[0]) * vq_num; 804 size_avail = sizeof(uint16_t) * hdrlen 805 + sizeof(vq->vq_avail[0].ring[0]) * vq_num; 806 size_used = sizeof(uint16_t) *hdrlen 807 + sizeof(vq->vq_used[0].ring[0]) * vq_num; 808 size_indirect = (sc->sc_indirect && maxnsegs >= MINSEG_INDIRECT) ? 809 sizeof(struct vring_desc) * maxnsegs * vq_num : 0; 810 811 size_desc_avail = VIRTQUEUE_ALIGN(size_desc + size_avail); 812 size_used = VIRTQUEUE_ALIGN(size_used); 813 814 allocsize = size_desc_avail + size_used + size_indirect; 815 816 /* alloc and map the memory */ 817 r = bus_dmamem_alloc(sc->sc_dmat, allocsize, VIRTIO_PAGE_SIZE, 0, 818 &vq->vq_segs[0], 1, &rsegs, BUS_DMA_WAITOK); 819 if (r != 0) { 820 aprint_error_dev(sc->sc_dev, 821 "virtqueue %d for %s allocation failed, " 822 "error code %d\n", vq->vq_index, name, r); 823 goto err; 824 } 825 826 r = bus_dmamem_map(sc->sc_dmat, &vq->vq_segs[0], rsegs, allocsize, 827 &vq->vq_vaddr, BUS_DMA_WAITOK); 828 if (r != 0) { 829 aprint_error_dev(sc->sc_dev, 830 "virtqueue %d for %s map failed, " 831 "error code %d\n", vq->vq_index, name, r); 832 goto err; 833 } 834 835 r = bus_dmamap_create(sc->sc_dmat, allocsize, 1, allocsize, 0, 836 BUS_DMA_WAITOK, &vq->vq_dmamap); 837 if (r != 0) { 838 aprint_error_dev(sc->sc_dev, 839 "virtqueue %d for %s dmamap creation failed, " 840 "error code %d\n", vq->vq_index, name, r); 841 goto err; 842 } 843 844 r = bus_dmamap_load(sc->sc_dmat, vq->vq_dmamap, 845 vq->vq_vaddr, allocsize, NULL, BUS_DMA_WAITOK); 846 if (r != 0) { 847 aprint_error_dev(sc->sc_dev, 848 "virtqueue %d for %s dmamap load failed, " 849 "error code %d\n", vq->vq_index, name, r); 850 goto err; 851 } 852 853 vq->vq_bytesize = allocsize; 854 vq->vq_maxsegsize = maxsegsize; 855 vq->vq_maxnsegs = maxnsegs; 856 857 #define VIRTIO_PTR(base, offset) (void *)((intptr_t)(base) + (offset)) 858 /* initialize vring pointers */ 859 vq->vq_desc = VIRTIO_PTR(vq->vq_vaddr, 0); 860 vq->vq_availoffset = size_desc; 861 vq->vq_avail = VIRTIO_PTR(vq->vq_vaddr, vq->vq_availoffset); 862 vq->vq_used_event = VIRTIO_PTR(vq->vq_avail, 863 offsetof(struct vring_avail, ring[vq_num])); 864 vq->vq_usedoffset = size_desc_avail; 865 vq->vq_used = VIRTIO_PTR(vq->vq_vaddr, vq->vq_usedoffset); 866 vq->vq_avail_event = VIRTIO_PTR(vq->vq_used, 867 offsetof(struct vring_used, ring[vq_num])); 868 869 if (size_indirect > 0) { 870 vq->vq_indirectoffset = size_desc_avail + size_used; 871 vq->vq_indirect = VIRTIO_PTR(vq->vq_vaddr, 872 vq->vq_indirectoffset); 873 } 874 #undef VIRTIO_PTR 875 876 vq->vq_descx = kmem_zalloc(sizeof(vq->vq_descx[0]) * vq_num, 877 KM_SLEEP); 878 879 mutex_init(&vq->vq_freedesc_lock, MUTEX_SPIN, sc->sc_ipl); 880 mutex_init(&vq->vq_aring_lock, MUTEX_SPIN, sc->sc_ipl); 881 mutex_init(&vq->vq_uring_lock, MUTEX_SPIN, sc->sc_ipl); 882 883 virtio_reset_vq(sc, vq); 884 885 aprint_verbose_dev(sc->sc_dev, 886 "allocated %" PRIuBUSSIZE " byte for virtqueue %d for %s, " 887 "size %d\n", allocsize, vq->vq_index, name, vq_num); 888 if (size_indirect > 0) 889 aprint_verbose_dev(sc->sc_dev, 890 "using %" PRIuBUSSIZE " byte (%d entries) indirect " 891 "descriptors\n", size_indirect, maxnsegs * vq_num); 892 893 return 0; 894 895 err: 896 sc->sc_ops->setup_queue(sc, vq->vq_index, 0); 897 if (vq->vq_dmamap) 898 bus_dmamap_destroy(sc->sc_dmat, vq->vq_dmamap); 899 if (vq->vq_vaddr) 900 bus_dmamem_unmap(sc->sc_dmat, vq->vq_vaddr, allocsize); 901 if (vq->vq_segs[0].ds_addr) 902 bus_dmamem_free(sc->sc_dmat, &vq->vq_segs[0], 1); 903 memset(vq, 0, sizeof(*vq)); 904 905 return -1; 906 } 907 908 int 909 virtio_free_vq(struct virtio_softc *sc, struct virtqueue *vq) 910 { 911 uint16_t s; 912 size_t i; 913 914 if (vq->vq_vaddr == NULL) 915 return 0; 916 917 /* device must be already deactivated */ 918 /* confirm the vq is empty */ 919 s = vq->vq_free_idx; 920 i = 0; 921 while (s != virtio_rw16(sc, VRING_DESC_CHAIN_END)) { 922 s = vq->vq_desc[s].next; 923 i++; 924 } 925 if (i != vq->vq_num) { 926 printf("%s: freeing non-empty vq, index %d\n", 927 device_xname(sc->sc_dev), vq->vq_index); 928 return EBUSY; 929 } 930 931 /* tell device that there's no virtqueue any longer */ 932 sc->sc_ops->setup_queue(sc, vq->vq_index, 0); 933 934 vq_sync_aring_all(sc, vq, BUS_DMASYNC_POSTWRITE); 935 936 kmem_free(vq->vq_descx, sizeof(vq->vq_descx[0]) * vq->vq_num); 937 bus_dmamap_unload(sc->sc_dmat, vq->vq_dmamap); 938 bus_dmamap_destroy(sc->sc_dmat, vq->vq_dmamap); 939 bus_dmamem_unmap(sc->sc_dmat, vq->vq_vaddr, vq->vq_bytesize); 940 bus_dmamem_free(sc->sc_dmat, &vq->vq_segs[0], 1); 941 mutex_destroy(&vq->vq_freedesc_lock); 942 mutex_destroy(&vq->vq_uring_lock); 943 mutex_destroy(&vq->vq_aring_lock); 944 memset(vq, 0, sizeof(*vq)); 945 946 return 0; 947 } 948 949 /* 950 * Free descriptor management. 951 */ 952 static int 953 vq_alloc_slot_locked(struct virtio_softc *sc, struct virtqueue *vq, 954 size_t nslots) 955 { 956 struct vring_desc *vd; 957 uint16_t head, tail; 958 size_t i; 959 960 KASSERT(mutex_owned(&vq->vq_freedesc_lock)); 961 962 head = tail = virtio_rw16(sc, vq->vq_free_idx); 963 for (i = 0; i < nslots - 1; i++) { 964 if (tail == VRING_DESC_CHAIN_END) 965 return VRING_DESC_CHAIN_END; 966 967 vd = &vq->vq_desc[tail]; 968 vd->flags = virtio_rw16(sc, VRING_DESC_F_NEXT); 969 tail = virtio_rw16(sc, vd->next); 970 } 971 972 if (tail == VRING_DESC_CHAIN_END) 973 return VRING_DESC_CHAIN_END; 974 975 vd = &vq->vq_desc[tail]; 976 vd->flags = virtio_rw16(sc, 0); 977 vq->vq_free_idx = vd->next; 978 979 return head; 980 } 981 static uint16_t 982 vq_alloc_slot(struct virtio_softc *sc, struct virtqueue *vq, size_t nslots) 983 { 984 uint16_t rv; 985 986 mutex_enter(&vq->vq_freedesc_lock); 987 rv = vq_alloc_slot_locked(sc, vq, nslots); 988 mutex_exit(&vq->vq_freedesc_lock); 989 990 return rv; 991 } 992 993 static void 994 vq_free_slot(struct virtio_softc *sc, struct virtqueue *vq, uint16_t slot) 995 { 996 struct vring_desc *vd; 997 uint16_t s; 998 999 mutex_enter(&vq->vq_freedesc_lock); 1000 vd = &vq->vq_desc[slot]; 1001 while ((vd->flags & virtio_rw16(sc, VRING_DESC_F_NEXT)) != 0) { 1002 s = virtio_rw16(sc, vd->next); 1003 vd = &vq->vq_desc[s]; 1004 } 1005 vd->next = vq->vq_free_idx; 1006 vq->vq_free_idx = virtio_rw16(sc, slot); 1007 mutex_exit(&vq->vq_freedesc_lock); 1008 } 1009 1010 /* 1011 * Enqueue several dmamaps as a single request. 1012 */ 1013 /* 1014 * Typical usage: 1015 * <queue size> number of followings are stored in arrays 1016 * - command blocks (in dmamem) should be pre-allocated and mapped 1017 * - dmamaps for command blocks should be pre-allocated and loaded 1018 * - dmamaps for payload should be pre-allocated 1019 * r = virtio_enqueue_prep(sc, vq, &slot); // allocate a slot 1020 * if (r) // currently 0 or EAGAIN 1021 * return r; 1022 * r = bus_dmamap_load(dmat, dmamap_payload[slot], data, count, ..); 1023 * if (r) { 1024 * virtio_enqueue_abort(sc, vq, slot); 1025 * return r; 1026 * } 1027 * r = virtio_enqueue_reserve(sc, vq, slot, 1028 * dmamap_payload[slot]->dm_nsegs + 1); 1029 * // ^ +1 for command 1030 * if (r) { // currently 0 or EAGAIN 1031 * bus_dmamap_unload(dmat, dmamap_payload[slot]); 1032 * return r; // do not call abort() 1033 * } 1034 * <setup and prepare commands> 1035 * bus_dmamap_sync(dmat, dmamap_cmd[slot],... BUS_DMASYNC_PREWRITE); 1036 * bus_dmamap_sync(dmat, dmamap_payload[slot],...); 1037 * virtio_enqueue(sc, vq, slot, dmamap_cmd[slot], false); 1038 * virtio_enqueue(sc, vq, slot, dmamap_payload[slot], iswrite); 1039 * virtio_enqueue_commit(sc, vq, slot, true); 1040 */ 1041 1042 /* 1043 * enqueue_prep: allocate a slot number 1044 */ 1045 int 1046 virtio_enqueue_prep(struct virtio_softc *sc, struct virtqueue *vq, int *slotp) 1047 { 1048 uint16_t slot; 1049 1050 KASSERT(sc->sc_child_state == VIRTIO_CHILD_ATTACH_FINISHED); 1051 KASSERT(slotp != NULL); 1052 1053 slot = vq_alloc_slot(sc, vq, 1); 1054 if (slot == VRING_DESC_CHAIN_END) 1055 return EAGAIN; 1056 1057 *slotp = slot; 1058 1059 return 0; 1060 } 1061 1062 /* 1063 * enqueue_reserve: allocate remaining slots and build the descriptor chain. 1064 */ 1065 int 1066 virtio_enqueue_reserve(struct virtio_softc *sc, struct virtqueue *vq, 1067 int slot, int nsegs) 1068 { 1069 struct vring_desc *vd; 1070 struct vring_desc_extra *vdx; 1071 int i; 1072 1073 KASSERT(1 <= nsegs); 1074 KASSERT(nsegs <= vq->vq_num); 1075 1076 vdx = &vq->vq_descx[slot]; 1077 vd = &vq->vq_desc[slot]; 1078 1079 KASSERT((vd->flags & virtio_rw16(sc, VRING_DESC_F_NEXT)) == 0); 1080 1081 if ((vq->vq_indirect != NULL) && 1082 (nsegs >= MINSEG_INDIRECT) && 1083 (nsegs <= vq->vq_maxnsegs)) 1084 vdx->use_indirect = true; 1085 else 1086 vdx->use_indirect = false; 1087 1088 if (vdx->use_indirect) { 1089 uint64_t addr; 1090 1091 addr = vq->vq_dmamap->dm_segs[0].ds_addr 1092 + vq->vq_indirectoffset; 1093 addr += sizeof(struct vring_desc) 1094 * vq->vq_maxnsegs * slot; 1095 1096 vd->addr = virtio_rw64(sc, addr); 1097 vd->len = virtio_rw32(sc, sizeof(struct vring_desc) * nsegs); 1098 vd->flags = virtio_rw16(sc, VRING_DESC_F_INDIRECT); 1099 1100 vd = &vq->vq_indirect[vq->vq_maxnsegs * slot]; 1101 vdx->desc_base = vd; 1102 vdx->desc_free_idx = 0; 1103 1104 for (i = 0; i < nsegs - 1; i++) { 1105 vd[i].flags = virtio_rw16(sc, VRING_DESC_F_NEXT); 1106 } 1107 vd[i].flags = virtio_rw16(sc, 0); 1108 } else { 1109 if (nsegs > 1) { 1110 uint16_t s; 1111 1112 s = vq_alloc_slot(sc, vq, nsegs - 1); 1113 if (s == VRING_DESC_CHAIN_END) { 1114 vq_free_slot(sc, vq, slot); 1115 return EAGAIN; 1116 } 1117 vd->next = virtio_rw16(sc, s); 1118 vd->flags = virtio_rw16(sc, VRING_DESC_F_NEXT); 1119 } 1120 1121 vdx->desc_base = &vq->vq_desc[0]; 1122 vdx->desc_free_idx = slot; 1123 } 1124 1125 return 0; 1126 } 1127 1128 /* 1129 * enqueue: enqueue a single dmamap. 1130 */ 1131 int 1132 virtio_enqueue(struct virtio_softc *sc, struct virtqueue *vq, int slot, 1133 bus_dmamap_t dmamap, bool write) 1134 { 1135 struct vring_desc *vds; 1136 struct vring_desc_extra *vdx; 1137 uint16_t s; 1138 int i; 1139 1140 KASSERT(dmamap->dm_nsegs > 0); 1141 1142 vdx = &vq->vq_descx[slot]; 1143 vds = vdx->desc_base; 1144 s = vdx->desc_free_idx; 1145 1146 KASSERT(vds != NULL); 1147 1148 for (i = 0; i < dmamap->dm_nsegs; i++) { 1149 KASSERT(s != VRING_DESC_CHAIN_END); 1150 1151 vds[s].addr = virtio_rw64(sc, dmamap->dm_segs[i].ds_addr); 1152 vds[s].len = virtio_rw32(sc, dmamap->dm_segs[i].ds_len); 1153 if (!write) 1154 vds[s].flags |= virtio_rw16(sc, VRING_DESC_F_WRITE); 1155 1156 if ((vds[s].flags & virtio_rw16(sc, VRING_DESC_F_NEXT)) == 0) { 1157 s = VRING_DESC_CHAIN_END; 1158 } else { 1159 s = virtio_rw16(sc, vds[s].next); 1160 } 1161 } 1162 1163 vdx->desc_free_idx = s; 1164 1165 return 0; 1166 } 1167 1168 int 1169 virtio_enqueue_p(struct virtio_softc *sc, struct virtqueue *vq, int slot, 1170 bus_dmamap_t dmamap, bus_addr_t start, bus_size_t len, 1171 bool write) 1172 { 1173 struct vring_desc_extra *vdx; 1174 struct vring_desc *vds; 1175 uint16_t s; 1176 1177 vdx = &vq->vq_descx[slot]; 1178 vds = vdx->desc_base; 1179 s = vdx->desc_free_idx; 1180 1181 KASSERT(s != VRING_DESC_CHAIN_END); 1182 KASSERT(vds != NULL); 1183 KASSERT(dmamap->dm_nsegs == 1); /* XXX */ 1184 KASSERT(dmamap->dm_segs[0].ds_len > start); 1185 KASSERT(dmamap->dm_segs[0].ds_len >= start + len); 1186 1187 vds[s].addr = virtio_rw64(sc, dmamap->dm_segs[0].ds_addr + start); 1188 vds[s].len = virtio_rw32(sc, len); 1189 if (!write) 1190 vds[s].flags |= virtio_rw16(sc, VRING_DESC_F_WRITE); 1191 1192 if ((vds[s].flags & virtio_rw16(sc, VRING_DESC_F_NEXT)) == 0) { 1193 s = VRING_DESC_CHAIN_END; 1194 } else { 1195 s = virtio_rw16(sc, vds[s].next); 1196 } 1197 1198 vdx->desc_free_idx = s; 1199 1200 return 0; 1201 } 1202 1203 /* 1204 * enqueue_commit: add it to the aring. 1205 */ 1206 int 1207 virtio_enqueue_commit(struct virtio_softc *sc, struct virtqueue *vq, int slot, 1208 bool notifynow) 1209 { 1210 1211 if (slot < 0) { 1212 mutex_enter(&vq->vq_aring_lock); 1213 goto notify; 1214 } 1215 1216 vq_sync_descs(sc, vq, BUS_DMASYNC_PREWRITE); 1217 if (vq->vq_descx[slot].use_indirect) 1218 vq_sync_indirect(sc, vq, slot, BUS_DMASYNC_PREWRITE); 1219 1220 mutex_enter(&vq->vq_aring_lock); 1221 vq->vq_avail->ring[(vq->vq_avail_idx++) % vq->vq_num] = 1222 virtio_rw16(sc, slot); 1223 1224 notify: 1225 if (notifynow) { 1226 uint16_t o, n, t; 1227 uint16_t flags; 1228 1229 o = virtio_rw16(sc, vq->vq_avail->idx) - 1; 1230 n = vq->vq_avail_idx; 1231 1232 /* 1233 * Prepare for `device->CPU' (host->guest) transfer 1234 * into the buffer. This must happen before we commit 1235 * the vq->vq_avail->idx update to ensure we're not 1236 * still using the buffer in case program-prior loads 1237 * or stores in it get delayed past the store to 1238 * vq->vq_avail->idx. 1239 */ 1240 vq_sync_uring_all(sc, vq, BUS_DMASYNC_PREREAD); 1241 1242 /* ensure payload is published, then avail idx */ 1243 vq_sync_aring_payload(sc, vq, BUS_DMASYNC_PREWRITE); 1244 vq->vq_avail->idx = virtio_rw16(sc, vq->vq_avail_idx); 1245 vq_sync_aring_header(sc, vq, BUS_DMASYNC_PREWRITE); 1246 vq->vq_queued++; 1247 1248 if (sc->sc_active_features & VIRTIO_F_RING_EVENT_IDX) { 1249 vq_sync_uring_avail(sc, vq, BUS_DMASYNC_POSTREAD); 1250 t = virtio_rw16(sc, *vq->vq_avail_event) + 1; 1251 if ((uint16_t) (n - t) < (uint16_t) (n - o)) 1252 sc->sc_ops->kick(sc, vq->vq_index); 1253 } else { 1254 vq_sync_uring_header(sc, vq, BUS_DMASYNC_POSTREAD); 1255 flags = virtio_rw16(sc, vq->vq_used->flags); 1256 if (!(flags & VRING_USED_F_NO_NOTIFY)) 1257 sc->sc_ops->kick(sc, vq->vq_index); 1258 } 1259 } 1260 mutex_exit(&vq->vq_aring_lock); 1261 1262 return 0; 1263 } 1264 1265 /* 1266 * enqueue_abort: rollback. 1267 */ 1268 int 1269 virtio_enqueue_abort(struct virtio_softc *sc, struct virtqueue *vq, int slot) 1270 { 1271 struct vring_desc_extra *vdx; 1272 1273 vdx = &vq->vq_descx[slot]; 1274 vdx->desc_free_idx = VRING_DESC_CHAIN_END; 1275 vdx->desc_base = NULL; 1276 1277 vq_free_slot(sc, vq, slot); 1278 1279 return 0; 1280 } 1281 1282 /* 1283 * Dequeue a request. 1284 */ 1285 /* 1286 * dequeue: dequeue a request from uring; dmamap_sync for uring is 1287 * already done in the interrupt handler. 1288 */ 1289 int 1290 virtio_dequeue(struct virtio_softc *sc, struct virtqueue *vq, 1291 int *slotp, int *lenp) 1292 { 1293 uint16_t slot, usedidx; 1294 1295 if (vq->vq_used_idx == virtio_rw16(sc, vq->vq_used->idx)) 1296 return ENOENT; 1297 mutex_enter(&vq->vq_uring_lock); 1298 usedidx = vq->vq_used_idx++; 1299 mutex_exit(&vq->vq_uring_lock); 1300 usedidx %= vq->vq_num; 1301 slot = virtio_rw32(sc, vq->vq_used->ring[usedidx].id); 1302 1303 if (vq->vq_descx[slot].use_indirect) 1304 vq_sync_indirect(sc, vq, slot, BUS_DMASYNC_POSTWRITE); 1305 1306 if (slotp) 1307 *slotp = slot; 1308 if (lenp) 1309 *lenp = virtio_rw32(sc, vq->vq_used->ring[usedidx].len); 1310 1311 return 0; 1312 } 1313 1314 /* 1315 * dequeue_commit: complete dequeue; the slot is recycled for future use. 1316 * if you forget to call this the slot will be leaked. 1317 */ 1318 int 1319 virtio_dequeue_commit(struct virtio_softc *sc, struct virtqueue *vq, int slot) 1320 { 1321 struct vring_desc_extra *vdx; 1322 1323 vdx = &vq->vq_descx[slot]; 1324 vdx->desc_base = NULL; 1325 vdx->desc_free_idx = VRING_DESC_CHAIN_END; 1326 1327 vq_free_slot(sc, vq, slot); 1328 1329 return 0; 1330 } 1331 1332 /* 1333 * Attach a child, fill all the members. 1334 */ 1335 void 1336 virtio_child_attach_start(struct virtio_softc *sc, device_t child, int ipl, 1337 uint64_t req_features, const char *feat_bits) 1338 { 1339 char buf[1024]; 1340 1341 KASSERT(sc->sc_child == NULL); 1342 KASSERT(sc->sc_child_state == VIRTIO_NO_CHILD); 1343 1344 sc->sc_child = child; 1345 sc->sc_ipl = ipl; 1346 1347 virtio_negotiate_features(sc, req_features); 1348 snprintb(buf, sizeof(buf), feat_bits, sc->sc_active_features); 1349 aprint_normal(": features: %s\n", buf); 1350 aprint_naive("\n"); 1351 } 1352 1353 int 1354 virtio_child_attach_finish(struct virtio_softc *sc, 1355 struct virtqueue *vqs, size_t nvqs, 1356 virtio_callback config_change, 1357 int req_flags) 1358 { 1359 size_t i; 1360 int r; 1361 1362 #ifdef DIAGNOSTIC 1363 KASSERT(nvqs > 0); 1364 #define VIRTIO_ASSERT_FLAGS (VIRTIO_F_INTR_SOFTINT | VIRTIO_F_INTR_PERVQ) 1365 KASSERT((req_flags & VIRTIO_ASSERT_FLAGS) != VIRTIO_ASSERT_FLAGS); 1366 #undef VIRTIO_ASSERT_FLAGS 1367 1368 for (i = 0; i < nvqs; i++){ 1369 KASSERT(vqs[i].vq_index == i); 1370 KASSERT(vqs[i].vq_intrhand != NULL); 1371 KASSERT(vqs[i].vq_done == NULL || 1372 vqs[i].vq_intrhand == virtio_vq_done); 1373 } 1374 #endif 1375 1376 1377 sc->sc_vqs = vqs; 1378 sc->sc_nvqs = nvqs; 1379 sc->sc_config_change = config_change; 1380 sc->sc_intrhand = virtio_vq_intr; 1381 sc->sc_flags = req_flags; 1382 1383 /* set the vq address */ 1384 for (i = 0; i < nvqs; i++) { 1385 sc->sc_ops->setup_queue(sc, vqs[i].vq_index, 1386 vqs[i].vq_dmamap->dm_segs[0].ds_addr); 1387 } 1388 1389 r = sc->sc_ops->alloc_interrupts(sc); 1390 if (r != 0) { 1391 aprint_error_dev(sc->sc_dev, 1392 "failed to allocate interrupts\n"); 1393 goto fail; 1394 } 1395 1396 r = sc->sc_ops->setup_interrupts(sc, 0); 1397 if (r != 0) { 1398 aprint_error_dev(sc->sc_dev, "failed to setup interrupts\n"); 1399 goto fail; 1400 } 1401 1402 KASSERT(sc->sc_soft_ih == NULL); 1403 if (sc->sc_flags & VIRTIO_F_INTR_SOFTINT) { 1404 u_int flags = SOFTINT_NET; 1405 if (sc->sc_flags & VIRTIO_F_INTR_MPSAFE) 1406 flags |= SOFTINT_MPSAFE; 1407 1408 sc->sc_soft_ih = softint_establish(flags, virtio_soft_intr, 1409 sc); 1410 if (sc->sc_soft_ih == NULL) { 1411 sc->sc_ops->free_interrupts(sc); 1412 aprint_error_dev(sc->sc_dev, 1413 "failed to establish soft interrupt\n"); 1414 goto fail; 1415 } 1416 } 1417 1418 sc->sc_child_state = VIRTIO_CHILD_ATTACH_FINISHED; 1419 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_DRIVER_OK); 1420 return 0; 1421 1422 fail: 1423 if (sc->sc_soft_ih) { 1424 softint_disestablish(sc->sc_soft_ih); 1425 sc->sc_soft_ih = NULL; 1426 } 1427 1428 sc->sc_ops->free_interrupts(sc); 1429 1430 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_FAILED); 1431 return 1; 1432 } 1433 1434 void 1435 virtio_child_detach(struct virtio_softc *sc) 1436 { 1437 1438 /* already detached */ 1439 if (sc->sc_child == NULL) 1440 return; 1441 1442 1443 virtio_device_reset(sc); 1444 1445 sc->sc_ops->free_interrupts(sc); 1446 1447 if (sc->sc_soft_ih) { 1448 softint_disestablish(sc->sc_soft_ih); 1449 sc->sc_soft_ih = NULL; 1450 } 1451 1452 sc->sc_vqs = NULL; 1453 sc->sc_child = NULL; 1454 } 1455 1456 void 1457 virtio_child_attach_failed(struct virtio_softc *sc) 1458 { 1459 virtio_child_detach(sc); 1460 1461 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_FAILED); 1462 1463 sc->sc_child_state = VIRTIO_CHILD_ATTACH_FAILED; 1464 } 1465 1466 bus_dma_tag_t 1467 virtio_dmat(struct virtio_softc *sc) 1468 { 1469 return sc->sc_dmat; 1470 } 1471 1472 device_t 1473 virtio_child(struct virtio_softc *sc) 1474 { 1475 return sc->sc_child; 1476 } 1477 1478 int 1479 virtio_intrhand(struct virtio_softc *sc) 1480 { 1481 return (*sc->sc_intrhand)(sc); 1482 } 1483 1484 uint64_t 1485 virtio_features(struct virtio_softc *sc) 1486 { 1487 return sc->sc_active_features; 1488 } 1489 1490 int 1491 virtio_attach_failed(struct virtio_softc *sc) 1492 { 1493 device_t self = sc->sc_dev; 1494 1495 /* no error if its not connected, but its failed */ 1496 if (sc->sc_childdevid == 0) 1497 return 1; 1498 1499 if (sc->sc_child == NULL) { 1500 switch (sc->sc_child_state) { 1501 case VIRTIO_CHILD_ATTACH_FAILED: 1502 aprint_error_dev(self, 1503 "virtio configuration failed\n"); 1504 break; 1505 case VIRTIO_NO_CHILD: 1506 aprint_error_dev(self, 1507 "no matching child driver; not configured\n"); 1508 break; 1509 default: 1510 /* sanity check */ 1511 aprint_error_dev(self, 1512 "virtio internal error, " 1513 "child driver is not configured\n"); 1514 break; 1515 } 1516 1517 return 1; 1518 } 1519 1520 /* sanity check */ 1521 if (sc->sc_child_state != VIRTIO_CHILD_ATTACH_FINISHED) { 1522 aprint_error_dev(self, "virtio internal error, child driver " 1523 "signaled OK but didn't initialize interrupts\n"); 1524 return 1; 1525 } 1526 1527 return 0; 1528 } 1529 1530 void 1531 virtio_print_device_type(device_t self, int id, int revision) 1532 { 1533 aprint_normal_dev(self, "%s device (id %d, rev. 0x%02x)\n", 1534 (id < NDEVNAMES ? virtio_device_name[id] : "Unknown"), 1535 id, 1536 revision); 1537 } 1538 1539 1540 MODULE(MODULE_CLASS_DRIVER, virtio, NULL); 1541 1542 #ifdef _MODULE 1543 #include "ioconf.c" 1544 #endif 1545 1546 static int 1547 virtio_modcmd(modcmd_t cmd, void *opaque) 1548 { 1549 int error = 0; 1550 1551 #ifdef _MODULE 1552 switch (cmd) { 1553 case MODULE_CMD_INIT: 1554 error = config_init_component(cfdriver_ioconf_virtio, 1555 cfattach_ioconf_virtio, cfdata_ioconf_virtio); 1556 break; 1557 case MODULE_CMD_FINI: 1558 error = config_fini_component(cfdriver_ioconf_virtio, 1559 cfattach_ioconf_virtio, cfdata_ioconf_virtio); 1560 break; 1561 default: 1562 error = ENOTTY; 1563 break; 1564 } 1565 #endif 1566 1567 return error; 1568 } 1569
Indexes created Thu Oct 02 14:10:14 GMT 2025