ld_virtio.c revision 1.24
1 /* $NetBSD: ld_virtio.c,v 1.24 2018/07/12 12:48:50 jakllsch Exp $ */ 2 3 /* 4 * Copyright (c) 2010 Minoura Makoto. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 17 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 18 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 19 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 20 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 21 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 22 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 23 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 24 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 25 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 */ 27 28 #include <sys/cdefs.h> 29 __KERNEL_RCSID(0, "$NetBSD: ld_virtio.c,v 1.24 2018/07/12 12:48:50 jakllsch Exp $"); 30 31 #include <sys/param.h> 32 #include <sys/systm.h> 33 #include <sys/kernel.h> 34 #include <sys/buf.h> 35 #include <sys/bufq.h> 36 #include <sys/bus.h> 37 #include <sys/device.h> 38 #include <sys/disk.h> 39 #include <sys/mutex.h> 40 #include <sys/module.h> 41 #include <sys/sysctl.h> 42 43 #include <dev/ldvar.h> 44 #include <dev/pci/virtioreg.h> 45 #include <dev/pci/virtiovar.h> 46 47 #include "ioconf.h" 48 49 /* 50 * ld_virtioreg: 51 */ 52 /* Configuration registers */ 53 #define VIRTIO_BLK_CONFIG_CAPACITY 0 /* 64bit */ 54 #define VIRTIO_BLK_CONFIG_SIZE_MAX 8 /* 32bit */ 55 #define VIRTIO_BLK_CONFIG_SEG_MAX 12 /* 32bit */ 56 #define VIRTIO_BLK_CONFIG_GEOMETRY_C 16 /* 16bit */ 57 #define VIRTIO_BLK_CONFIG_GEOMETRY_H 18 /* 8bit */ 58 #define VIRTIO_BLK_CONFIG_GEOMETRY_S 19 /* 8bit */ 59 #define VIRTIO_BLK_CONFIG_BLK_SIZE 20 /* 32bit */ 60 #define VIRTIO_BLK_CONFIG_WRITEBACK 32 /* 8bit */ 61 62 /* Feature bits */ 63 #define VIRTIO_BLK_F_BARRIER (1<<0) 64 #define VIRTIO_BLK_F_SIZE_MAX (1<<1) 65 #define VIRTIO_BLK_F_SEG_MAX (1<<2) 66 #define VIRTIO_BLK_F_GEOMETRY (1<<4) 67 #define VIRTIO_BLK_F_RO (1<<5) 68 #define VIRTIO_BLK_F_BLK_SIZE (1<<6) 69 #define VIRTIO_BLK_F_SCSI (1<<7) 70 #define VIRTIO_BLK_F_FLUSH (1<<9) 71 #define VIRTIO_BLK_F_TOPOLOGY (1<<10) 72 #define VIRTIO_BLK_F_CONFIG_WCE (1<<11) 73 74 /* 75 * Each block request uses at least two segments - one for the header 76 * and one for the status. 77 */ 78 #define VIRTIO_BLK_MIN_SEGMENTS 2 79 80 #define VIRTIO_BLK_FLAG_BITS \ 81 VIRTIO_COMMON_FLAG_BITS \ 82 "\x0c""CONFIG_WCE" \ 83 "\x0b""TOPOLOGY" \ 84 "\x0a""FLUSH" \ 85 "\x08""SCSI" \ 86 "\x07""BLK_SIZE" \ 87 "\x06""RO" \ 88 "\x05""GEOMETRY" \ 89 "\x03""SEG_MAX" \ 90 "\x02""SIZE_MAX" \ 91 "\x01""BARRIER" 92 93 /* Command */ 94 #define VIRTIO_BLK_T_IN 0 95 #define VIRTIO_BLK_T_OUT 1 96 #define VIRTIO_BLK_T_FLUSH 4 97 #define VIRTIO_BLK_T_BARRIER 0x80000000 98 99 /* Sector */ 100 #define VIRTIO_BLK_BSIZE 512 101 102 /* Status */ 103 #define VIRTIO_BLK_S_OK 0 104 #define VIRTIO_BLK_S_IOERR 1 105 #define VIRTIO_BLK_S_UNSUPP 2 106 107 /* Request header structure */ 108 struct virtio_blk_req_hdr { 109 uint32_t type; /* VIRTIO_BLK_T_* */ 110 uint32_t ioprio; 111 uint64_t sector; 112 } __packed; 113 /* payload and 1 byte status follows */ 114 115 116 /* 117 * ld_virtiovar: 118 */ 119 struct virtio_blk_req { 120 struct virtio_blk_req_hdr vr_hdr; 121 uint8_t vr_status; 122 struct buf *vr_bp; 123 #define DUMMY_VR_BP ((void *)1) 124 bus_dmamap_t vr_cmdsts; 125 bus_dmamap_t vr_payload; 126 }; 127 128 struct ld_virtio_softc { 129 struct ld_softc sc_ld; 130 device_t sc_dev; 131 132 struct virtio_softc *sc_virtio; 133 struct virtqueue sc_vq; 134 135 struct virtio_blk_req *sc_reqs; 136 bus_dma_segment_t sc_reqs_seg; 137 138 int sc_readonly; 139 140 enum { 141 SYNC_FREE, SYNC_BUSY, SYNC_DONE 142 } sc_sync_use; 143 kcondvar_t sc_sync_wait; 144 kmutex_t sc_sync_wait_lock; 145 uint8_t sc_sync_status; 146 }; 147 148 int ld_virtio_notify_threshold = 0; 149 150 static void 151 setup_sysctl(void) 152 { 153 static bool done; 154 155 if (done) 156 return; 157 158 done = true; 159 160 sysctl_createv(NULL, 0, NULL, NULL, 161 CTLFLAG_READWRITE, CTLTYPE_INT, "ld_virtio_notify_threshold", 162 SYSCTL_DESCR(""), 163 NULL, 0, &ld_virtio_notify_threshold, 0, 164 CTL_HW, CTL_CREATE, CTL_EOL); 165 } 166 167 static int ld_virtio_match(device_t, cfdata_t, void *); 168 static void ld_virtio_attach(device_t, device_t, void *); 169 static int ld_virtio_detach(device_t, int); 170 171 CFATTACH_DECL_NEW(ld_virtio, sizeof(struct ld_virtio_softc), 172 ld_virtio_match, ld_virtio_attach, ld_virtio_detach, NULL); 173 174 static int 175 ld_virtio_match(device_t parent, cfdata_t match, void *aux) 176 { 177 struct virtio_attach_args *va = aux; 178 179 if (va->sc_childdevid == PCI_PRODUCT_VIRTIO_BLOCK) 180 return 1; 181 182 return 0; 183 } 184 185 static int ld_virtio_vq_done(struct virtqueue *); 186 static int ld_virtio_dump(struct ld_softc *, void *, int, int); 187 static int ld_virtio_start(struct ld_softc *, struct buf *); 188 static int ld_virtio_ioctl(struct ld_softc *, u_long, void *, int32_t, bool); 189 190 static int 191 ld_virtio_alloc_reqs(struct ld_virtio_softc *sc, int qsize) 192 { 193 int allocsize, r, rsegs, i; 194 struct ld_softc *ld = &sc->sc_ld; 195 void *vaddr; 196 197 allocsize = sizeof(struct virtio_blk_req) * qsize; 198 r = bus_dmamem_alloc(virtio_dmat(sc->sc_virtio), allocsize, 0, 0, 199 &sc->sc_reqs_seg, 1, &rsegs, BUS_DMA_NOWAIT); 200 if (r != 0) { 201 aprint_error_dev(sc->sc_dev, 202 "DMA memory allocation failed, size %d, " 203 "error code %d\n", allocsize, r); 204 goto err_none; 205 } 206 r = bus_dmamem_map(virtio_dmat(sc->sc_virtio), 207 &sc->sc_reqs_seg, 1, allocsize, 208 &vaddr, BUS_DMA_NOWAIT); 209 if (r != 0) { 210 aprint_error_dev(sc->sc_dev, 211 "DMA memory map failed, " 212 "error code %d\n", r); 213 goto err_dmamem_alloc; 214 } 215 sc->sc_reqs = vaddr; 216 memset(vaddr, 0, allocsize); 217 for (i = 0; i < qsize; i++) { 218 struct virtio_blk_req *vr = &sc->sc_reqs[i]; 219 r = bus_dmamap_create(virtio_dmat(sc->sc_virtio), 220 offsetof(struct virtio_blk_req, vr_bp), 221 1, 222 offsetof(struct virtio_blk_req, vr_bp), 223 0, 224 BUS_DMA_NOWAIT|BUS_DMA_ALLOCNOW, 225 &vr->vr_cmdsts); 226 if (r != 0) { 227 aprint_error_dev(sc->sc_dev, 228 "command dmamap creation failed, " 229 "error code %d\n", r); 230 goto err_reqs; 231 } 232 r = bus_dmamap_load(virtio_dmat(sc->sc_virtio), vr->vr_cmdsts, 233 &vr->vr_hdr, 234 offsetof(struct virtio_blk_req, vr_bp), 235 NULL, BUS_DMA_NOWAIT); 236 if (r != 0) { 237 aprint_error_dev(sc->sc_dev, 238 "command dmamap load failed, " 239 "error code %d\n", r); 240 goto err_reqs; 241 } 242 r = bus_dmamap_create(virtio_dmat(sc->sc_virtio), 243 ld->sc_maxxfer, 244 (ld->sc_maxxfer / NBPG) + 245 VIRTIO_BLK_MIN_SEGMENTS, 246 ld->sc_maxxfer, 247 0, 248 BUS_DMA_NOWAIT|BUS_DMA_ALLOCNOW, 249 &vr->vr_payload); 250 if (r != 0) { 251 aprint_error_dev(sc->sc_dev, 252 "payload dmamap creation failed, " 253 "error code %d\n", r); 254 goto err_reqs; 255 } 256 } 257 return 0; 258 259 err_reqs: 260 for (i = 0; i < qsize; i++) { 261 struct virtio_blk_req *vr = &sc->sc_reqs[i]; 262 if (vr->vr_cmdsts) { 263 bus_dmamap_destroy(virtio_dmat(sc->sc_virtio), 264 vr->vr_cmdsts); 265 vr->vr_cmdsts = 0; 266 } 267 if (vr->vr_payload) { 268 bus_dmamap_destroy(virtio_dmat(sc->sc_virtio), 269 vr->vr_payload); 270 vr->vr_payload = 0; 271 } 272 } 273 bus_dmamem_unmap(virtio_dmat(sc->sc_virtio), sc->sc_reqs, allocsize); 274 err_dmamem_alloc: 275 bus_dmamem_free(virtio_dmat(sc->sc_virtio), &sc->sc_reqs_seg, 1); 276 err_none: 277 return -1; 278 } 279 280 static void 281 ld_virtio_attach(device_t parent, device_t self, void *aux) 282 { 283 struct ld_virtio_softc *sc = device_private(self); 284 struct ld_softc *ld = &sc->sc_ld; 285 struct virtio_softc *vsc = device_private(parent); 286 uint32_t features; 287 int qsize, maxxfersize, maxnsegs; 288 289 if (virtio_child(vsc) != NULL) { 290 aprint_normal(": child already attached for %s; " 291 "something wrong...\n", device_xname(parent)); 292 return; 293 } 294 295 sc->sc_dev = self; 296 sc->sc_virtio = vsc; 297 298 virtio_child_attach_start(vsc, self, IPL_BIO, &sc->sc_vq, 299 NULL, virtio_vq_intr, VIRTIO_F_PCI_INTR_MPSAFE|VIRTIO_F_PCI_INTR_MSIX, 300 (VIRTIO_BLK_F_SIZE_MAX | VIRTIO_BLK_F_SEG_MAX | 301 VIRTIO_BLK_F_GEOMETRY | VIRTIO_BLK_F_RO | VIRTIO_BLK_F_BLK_SIZE | 302 VIRTIO_BLK_F_FLUSH | VIRTIO_BLK_F_CONFIG_WCE), 303 VIRTIO_BLK_FLAG_BITS); 304 305 features = virtio_features(vsc); 306 307 if (features & VIRTIO_BLK_F_RO) 308 sc->sc_readonly = 1; 309 else 310 sc->sc_readonly = 0; 311 312 if (features & VIRTIO_BLK_F_BLK_SIZE) { 313 ld->sc_secsize = virtio_read_device_config_4(vsc, 314 VIRTIO_BLK_CONFIG_BLK_SIZE); 315 } else 316 ld->sc_secsize = VIRTIO_BLK_BSIZE; 317 318 /* At least genfs_io assumes maxxfer == MAXPHYS. */ 319 if (features & VIRTIO_BLK_F_SIZE_MAX) { 320 maxxfersize = virtio_read_device_config_4(vsc, 321 VIRTIO_BLK_CONFIG_SIZE_MAX); 322 if (maxxfersize < MAXPHYS) { 323 aprint_error_dev(sc->sc_dev, 324 "Too small SIZE_MAX %dK minimum is %dK\n", 325 maxxfersize / 1024, MAXPHYS / 1024); 326 // goto err; 327 maxxfersize = MAXPHYS; 328 } else if (maxxfersize > MAXPHYS) { 329 aprint_normal_dev(sc->sc_dev, 330 "Clip SEG_MAX from %dK to %dK\n", 331 maxxfersize / 1024, 332 MAXPHYS / 1024); 333 maxxfersize = MAXPHYS; 334 } 335 } else 336 maxxfersize = MAXPHYS; 337 338 if (features & VIRTIO_BLK_F_SEG_MAX) { 339 maxnsegs = virtio_read_device_config_4(vsc, 340 VIRTIO_BLK_CONFIG_SEG_MAX); 341 if (maxnsegs < VIRTIO_BLK_MIN_SEGMENTS) { 342 aprint_error_dev(sc->sc_dev, 343 "Too small SEG_MAX %d minimum is %d\n", 344 maxnsegs, VIRTIO_BLK_MIN_SEGMENTS); 345 maxnsegs = maxxfersize / NBPG; 346 // goto err; 347 } 348 } else 349 maxnsegs = maxxfersize / NBPG; 350 351 /* 2 for the minimum size */ 352 maxnsegs += VIRTIO_BLK_MIN_SEGMENTS; 353 354 if (virtio_alloc_vq(vsc, &sc->sc_vq, 0, maxxfersize, maxnsegs, 355 "I/O request") != 0) { 356 goto err; 357 } 358 qsize = sc->sc_vq.vq_num; 359 sc->sc_vq.vq_done = ld_virtio_vq_done; 360 361 if (virtio_child_attach_finish(vsc) != 0) 362 goto err; 363 364 ld->sc_dv = self; 365 ld->sc_secperunit = virtio_read_device_config_8(vsc, 366 VIRTIO_BLK_CONFIG_CAPACITY) / (ld->sc_secsize / VIRTIO_BLK_BSIZE); 367 ld->sc_maxxfer = maxxfersize; 368 if (features & VIRTIO_BLK_F_GEOMETRY) { 369 ld->sc_ncylinders = virtio_read_device_config_2(vsc, 370 VIRTIO_BLK_CONFIG_GEOMETRY_C); 371 ld->sc_nheads = virtio_read_device_config_1(vsc, 372 VIRTIO_BLK_CONFIG_GEOMETRY_H); 373 ld->sc_nsectors = virtio_read_device_config_1(vsc, 374 VIRTIO_BLK_CONFIG_GEOMETRY_S); 375 } 376 ld->sc_maxqueuecnt = qsize - 1; /* reserve slot for dumps, flushes */ 377 378 if (ld_virtio_alloc_reqs(sc, qsize) < 0) 379 goto err; 380 381 cv_init(&sc->sc_sync_wait, "vblksync"); 382 mutex_init(&sc->sc_sync_wait_lock, MUTEX_DEFAULT, IPL_BIO); 383 sc->sc_sync_use = SYNC_FREE; 384 385 ld->sc_dump = ld_virtio_dump; 386 ld->sc_start = ld_virtio_start; 387 ld->sc_ioctl = ld_virtio_ioctl; 388 389 ld->sc_flags = LDF_ENABLED | LDF_MPSAFE; 390 ldattach(ld, BUFQ_DISK_DEFAULT_STRAT); 391 392 setup_sysctl(); 393 394 return; 395 396 err: 397 virtio_child_attach_failed(vsc); 398 return; 399 } 400 401 static int 402 ld_virtio_start(struct ld_softc *ld, struct buf *bp) 403 { 404 /* splbio */ 405 struct ld_virtio_softc *sc = device_private(ld->sc_dv); 406 struct virtio_softc *vsc = sc->sc_virtio; 407 struct virtqueue *vq = &sc->sc_vq; 408 struct virtio_blk_req *vr; 409 int r; 410 int isread = (bp->b_flags & B_READ); 411 int slot; 412 413 if (sc->sc_readonly && !isread) 414 return EIO; 415 416 r = virtio_enqueue_prep(vsc, vq, &slot); 417 if (r != 0) 418 return r; 419 420 vr = &sc->sc_reqs[slot]; 421 KASSERT(vr->vr_bp == NULL); 422 423 r = bus_dmamap_load(virtio_dmat(vsc), vr->vr_payload, 424 bp->b_data, bp->b_bcount, NULL, 425 ((isread?BUS_DMA_READ:BUS_DMA_WRITE) 426 |BUS_DMA_NOWAIT)); 427 if (r != 0) { 428 aprint_error_dev(sc->sc_dev, 429 "payload dmamap failed, error code %d\n", r); 430 virtio_enqueue_abort(vsc, vq, slot); 431 return r; 432 } 433 434 r = virtio_enqueue_reserve(vsc, vq, slot, vr->vr_payload->dm_nsegs + 435 VIRTIO_BLK_MIN_SEGMENTS); 436 if (r != 0) { 437 bus_dmamap_unload(virtio_dmat(vsc), vr->vr_payload); 438 return r; 439 } 440 441 vr->vr_bp = bp; 442 vr->vr_hdr.type = isread?VIRTIO_BLK_T_IN:VIRTIO_BLK_T_OUT; 443 vr->vr_hdr.ioprio = 0; 444 vr->vr_hdr.sector = bp->b_rawblkno * sc->sc_ld.sc_secsize / 445 VIRTIO_BLK_BSIZE; 446 447 bus_dmamap_sync(virtio_dmat(vsc), vr->vr_cmdsts, 448 0, sizeof(struct virtio_blk_req_hdr), 449 BUS_DMASYNC_PREWRITE); 450 bus_dmamap_sync(virtio_dmat(vsc), vr->vr_payload, 451 0, bp->b_bcount, 452 isread?BUS_DMASYNC_PREREAD:BUS_DMASYNC_PREWRITE); 453 bus_dmamap_sync(virtio_dmat(vsc), vr->vr_cmdsts, 454 offsetof(struct virtio_blk_req, vr_status), 455 sizeof(uint8_t), 456 BUS_DMASYNC_PREREAD); 457 458 virtio_enqueue_p(vsc, vq, slot, vr->vr_cmdsts, 459 0, sizeof(struct virtio_blk_req_hdr), 460 true); 461 virtio_enqueue(vsc, vq, slot, vr->vr_payload, !isread); 462 virtio_enqueue_p(vsc, vq, slot, vr->vr_cmdsts, 463 offsetof(struct virtio_blk_req, vr_status), 464 sizeof(uint8_t), 465 false); 466 const bool notify = !dk_strategy_pending(&sc->sc_ld.sc_dksc) || 467 sc->sc_ld.sc_queuecnt + 1 >= sc->sc_ld.sc_maxqueuecnt || 468 sc->sc_ld.sc_queuecnt + 1 >= (sc->sc_ld.sc_maxqueuecnt * ld_virtio_notify_threshold / 1000); 469 virtio_enqueue_commit(vsc, vq, slot, notify); 470 471 return 0; 472 } 473 474 static void 475 ld_virtio_vq_done1(struct ld_virtio_softc *sc, struct virtio_softc *vsc, 476 struct virtqueue *vq, int slot) 477 { 478 struct virtio_blk_req *vr = &sc->sc_reqs[slot]; 479 struct buf *bp = vr->vr_bp; 480 481 vr->vr_bp = NULL; 482 483 bus_dmamap_sync(virtio_dmat(vsc), vr->vr_cmdsts, 484 0, sizeof(struct virtio_blk_req_hdr), 485 BUS_DMASYNC_POSTWRITE); 486 bus_dmamap_sync(virtio_dmat(vsc), vr->vr_cmdsts, 487 sizeof(struct virtio_blk_req_hdr), sizeof(uint8_t), 488 BUS_DMASYNC_POSTREAD); 489 if (bp == DUMMY_VR_BP) { 490 mutex_enter(&sc->sc_sync_wait_lock); 491 sc->sc_sync_status = vr->vr_status; 492 sc->sc_sync_use = SYNC_DONE; 493 cv_signal(&sc->sc_sync_wait); 494 mutex_exit(&sc->sc_sync_wait_lock); 495 virtio_dequeue_commit(vsc, vq, slot); 496 return; 497 } 498 bus_dmamap_sync(virtio_dmat(vsc), vr->vr_payload, 499 0, bp->b_bcount, 500 (bp->b_flags & B_READ)?BUS_DMASYNC_POSTREAD 501 :BUS_DMASYNC_POSTWRITE); 502 bus_dmamap_unload(virtio_dmat(vsc), vr->vr_payload); 503 504 if (vr->vr_status != VIRTIO_BLK_S_OK) { 505 bp->b_error = EIO; 506 bp->b_resid = bp->b_bcount; 507 } else { 508 bp->b_error = 0; 509 bp->b_resid = 0; 510 } 511 512 virtio_dequeue_commit(vsc, vq, slot); 513 514 lddone(&sc->sc_ld, bp); 515 } 516 517 static int 518 ld_virtio_vq_done(struct virtqueue *vq) 519 { 520 struct virtio_softc *vsc = vq->vq_owner; 521 struct ld_virtio_softc *sc = device_private(virtio_child(vsc)); 522 int r = 0; 523 int slot; 524 525 again: 526 if (virtio_dequeue(vsc, vq, &slot, NULL)) 527 return r; 528 r = 1; 529 530 ld_virtio_vq_done1(sc, vsc, vq, slot); 531 goto again; 532 } 533 534 static int 535 ld_virtio_dump(struct ld_softc *ld, void *data, int blkno, int blkcnt) 536 { 537 struct ld_virtio_softc *sc = device_private(ld->sc_dv); 538 struct virtio_softc *vsc = sc->sc_virtio; 539 struct virtqueue *vq = &sc->sc_vq; 540 struct virtio_blk_req *vr; 541 int slot, r; 542 543 if (sc->sc_readonly) 544 return EIO; 545 546 r = virtio_enqueue_prep(vsc, vq, &slot); 547 if (r != 0) { 548 if (r == EAGAIN) { /* no free slot; dequeue first */ 549 delay(100); 550 ld_virtio_vq_done(vq); 551 r = virtio_enqueue_prep(vsc, vq, &slot); 552 if (r != 0) 553 return r; 554 } 555 return r; 556 } 557 vr = &sc->sc_reqs[slot]; 558 r = bus_dmamap_load(virtio_dmat(vsc), vr->vr_payload, 559 data, blkcnt*ld->sc_secsize, NULL, 560 BUS_DMA_WRITE|BUS_DMA_NOWAIT); 561 if (r != 0) 562 return r; 563 564 r = virtio_enqueue_reserve(vsc, vq, slot, vr->vr_payload->dm_nsegs + 565 VIRTIO_BLK_MIN_SEGMENTS); 566 if (r != 0) { 567 bus_dmamap_unload(virtio_dmat(vsc), vr->vr_payload); 568 return r; 569 } 570 571 vr->vr_bp = (void*)0xdeadbeef; 572 vr->vr_hdr.type = VIRTIO_BLK_T_OUT; 573 vr->vr_hdr.ioprio = 0; 574 vr->vr_hdr.sector = (daddr_t) blkno * ld->sc_secsize / 575 VIRTIO_BLK_BSIZE; 576 577 bus_dmamap_sync(virtio_dmat(vsc), vr->vr_cmdsts, 578 0, sizeof(struct virtio_blk_req_hdr), 579 BUS_DMASYNC_PREWRITE); 580 bus_dmamap_sync(virtio_dmat(vsc), vr->vr_payload, 581 0, blkcnt*ld->sc_secsize, 582 BUS_DMASYNC_PREWRITE); 583 bus_dmamap_sync(virtio_dmat(vsc), vr->vr_cmdsts, 584 offsetof(struct virtio_blk_req, vr_status), 585 sizeof(uint8_t), 586 BUS_DMASYNC_PREREAD); 587 588 virtio_enqueue_p(vsc, vq, slot, vr->vr_cmdsts, 589 0, sizeof(struct virtio_blk_req_hdr), 590 true); 591 virtio_enqueue(vsc, vq, slot, vr->vr_payload, true); 592 virtio_enqueue_p(vsc, vq, slot, vr->vr_cmdsts, 593 offsetof(struct virtio_blk_req, vr_status), 594 sizeof(uint8_t), 595 false); 596 virtio_enqueue_commit(vsc, vq, slot, true); 597 598 for ( ; ; ) { 599 int dslot; 600 601 r = virtio_dequeue(vsc, vq, &dslot, NULL); 602 if (r != 0) 603 continue; 604 if (dslot != slot) { 605 ld_virtio_vq_done1(sc, vsc, vq, dslot); 606 continue; 607 } else 608 break; 609 } 610 611 bus_dmamap_sync(virtio_dmat(vsc), vr->vr_cmdsts, 612 0, sizeof(struct virtio_blk_req_hdr), 613 BUS_DMASYNC_POSTWRITE); 614 bus_dmamap_sync(virtio_dmat(vsc), vr->vr_payload, 615 0, blkcnt*ld->sc_secsize, 616 BUS_DMASYNC_POSTWRITE); 617 bus_dmamap_sync(virtio_dmat(vsc), vr->vr_cmdsts, 618 offsetof(struct virtio_blk_req, vr_status), 619 sizeof(uint8_t), 620 BUS_DMASYNC_POSTREAD); 621 if (vr->vr_status == VIRTIO_BLK_S_OK) 622 r = 0; 623 else 624 r = EIO; 625 virtio_dequeue_commit(vsc, vq, slot); 626 627 return r; 628 } 629 630 static int 631 ld_virtio_detach(device_t self, int flags) 632 { 633 struct ld_virtio_softc *sc = device_private(self); 634 struct ld_softc *ld = &sc->sc_ld; 635 bus_dma_tag_t dmat = virtio_dmat(sc->sc_virtio); 636 int r, i, qsize; 637 638 qsize = sc->sc_vq.vq_num; 639 r = ldbegindetach(ld, flags); 640 if (r != 0) 641 return r; 642 virtio_reset(sc->sc_virtio); 643 virtio_free_vq(sc->sc_virtio, &sc->sc_vq); 644 645 for (i = 0; i < qsize; i++) { 646 bus_dmamap_destroy(dmat, 647 sc->sc_reqs[i].vr_cmdsts); 648 bus_dmamap_destroy(dmat, 649 sc->sc_reqs[i].vr_payload); 650 } 651 bus_dmamem_unmap(dmat, sc->sc_reqs, 652 sizeof(struct virtio_blk_req) * qsize); 653 bus_dmamem_free(dmat, &sc->sc_reqs_seg, 1); 654 655 ldenddetach(ld); 656 657 cv_destroy(&sc->sc_sync_wait); 658 mutex_destroy(&sc->sc_sync_wait_lock); 659 660 virtio_child_detach(sc->sc_virtio); 661 662 return 0; 663 } 664 665 static int 666 ld_virtio_flush(struct ld_softc *ld, bool poll) 667 { 668 struct ld_virtio_softc * const sc = device_private(ld->sc_dv); 669 struct virtio_softc * const vsc = sc->sc_virtio; 670 const uint32_t features = virtio_features(vsc); 671 struct virtqueue *vq = &sc->sc_vq; 672 struct virtio_blk_req *vr; 673 int slot; 674 int r; 675 676 if ((features & VIRTIO_BLK_F_FLUSH) == 0) 677 return 0; 678 679 mutex_enter(&sc->sc_sync_wait_lock); 680 while (sc->sc_sync_use != SYNC_FREE) { 681 if (poll) { 682 mutex_exit(&sc->sc_sync_wait_lock); 683 ld_virtio_vq_done(vq); 684 mutex_enter(&sc->sc_sync_wait_lock); 685 continue; 686 } 687 cv_wait(&sc->sc_sync_wait, &sc->sc_sync_wait_lock); 688 } 689 sc->sc_sync_use = SYNC_BUSY; 690 mutex_exit(&sc->sc_sync_wait_lock); 691 692 r = virtio_enqueue_prep(vsc, vq, &slot); 693 if (r != 0) { 694 return r; 695 } 696 697 vr = &sc->sc_reqs[slot]; 698 KASSERT(vr->vr_bp == NULL); 699 700 r = virtio_enqueue_reserve(vsc, vq, slot, VIRTIO_BLK_MIN_SEGMENTS); 701 if (r != 0) { 702 return r; 703 } 704 705 vr->vr_bp = DUMMY_VR_BP; 706 vr->vr_hdr.type = VIRTIO_BLK_T_FLUSH; 707 vr->vr_hdr.ioprio = 0; 708 vr->vr_hdr.sector = 0; 709 710 bus_dmamap_sync(virtio_dmat(vsc), vr->vr_cmdsts, 711 0, sizeof(struct virtio_blk_req_hdr), 712 BUS_DMASYNC_PREWRITE); 713 bus_dmamap_sync(virtio_dmat(vsc), vr->vr_cmdsts, 714 offsetof(struct virtio_blk_req, vr_status), 715 sizeof(uint8_t), 716 BUS_DMASYNC_PREREAD); 717 718 virtio_enqueue_p(vsc, vq, slot, vr->vr_cmdsts, 719 0, sizeof(struct virtio_blk_req_hdr), 720 true); 721 virtio_enqueue_p(vsc, vq, slot, vr->vr_cmdsts, 722 offsetof(struct virtio_blk_req, vr_status), 723 sizeof(uint8_t), 724 false); 725 virtio_enqueue_commit(vsc, vq, slot, true); 726 727 mutex_enter(&sc->sc_sync_wait_lock); 728 while (sc->sc_sync_use != SYNC_DONE) { 729 if (poll) { 730 mutex_exit(&sc->sc_sync_wait_lock); 731 ld_virtio_vq_done(vq); 732 mutex_enter(&sc->sc_sync_wait_lock); 733 continue; 734 } 735 cv_wait(&sc->sc_sync_wait, &sc->sc_sync_wait_lock); 736 } 737 738 if (sc->sc_sync_status == VIRTIO_BLK_S_OK) 739 r = 0; 740 else 741 r = EIO; 742 743 sc->sc_sync_use = SYNC_FREE; 744 cv_signal(&sc->sc_sync_wait); 745 mutex_exit(&sc->sc_sync_wait_lock); 746 747 return r; 748 } 749 750 static int 751 ld_virtio_getcache(struct ld_softc *ld, int *bitsp) 752 { 753 struct ld_virtio_softc * const sc = device_private(ld->sc_dv); 754 struct virtio_softc * const vsc = sc->sc_virtio; 755 const uint32_t features = virtio_features(vsc); 756 757 *bitsp = DKCACHE_READ; 758 if ((features & VIRTIO_BLK_F_CONFIG_WCE) != 0) 759 *bitsp |= DKCACHE_WCHANGE; 760 if (virtio_read_device_config_1(vsc, 761 VIRTIO_BLK_CONFIG_WRITEBACK) != 0x00) 762 *bitsp |= DKCACHE_WRITE; 763 764 return 0; 765 } 766 767 static int 768 ld_virtio_setcache(struct ld_softc *ld, int bits) 769 { 770 struct ld_virtio_softc * const sc = device_private(ld->sc_dv); 771 struct virtio_softc * const vsc = sc->sc_virtio; 772 const uint8_t wce = (bits & DKCACHE_WRITE) ? 0x01 : 0x00; 773 774 virtio_write_device_config_1(vsc, 775 VIRTIO_BLK_CONFIG_WRITEBACK, wce); 776 if (virtio_read_device_config_1(vsc, 777 VIRTIO_BLK_CONFIG_WRITEBACK) != wce) 778 return EIO; 779 780 return 0; 781 } 782 783 static int 784 ld_virtio_ioctl(struct ld_softc *ld, u_long cmd, void *addr, int32_t flag, bool poll) 785 { 786 int error; 787 788 switch (cmd) { 789 case DIOCCACHESYNC: 790 error = ld_virtio_flush(ld, poll); 791 break; 792 793 case DIOCGCACHE: 794 error = ld_virtio_getcache(ld, (int *)addr); 795 break; 796 797 case DIOCSCACHE: 798 error = ld_virtio_setcache(ld, *(int *)addr); 799 break; 800 801 default: 802 error = EPASSTHROUGH; 803 break; 804 } 805 806 return error; 807 } 808 809 MODULE(MODULE_CLASS_DRIVER, ld_virtio, "ld,virtio"); 810 811 #ifdef _MODULE 812 /* 813 * XXX Don't allow ioconf.c to redefine the "struct cfdriver ld_cd" 814 * XXX it will be defined in the common-code module 815 */ 816 #undef CFDRIVER_DECL 817 #define CFDRIVER_DECL(name, class, attr) 818 #include "ioconf.c" 819 #endif 820 821 static int 822 ld_virtio_modcmd(modcmd_t cmd, void *opaque) 823 { 824 #ifdef _MODULE 825 /* 826 * We ignore the cfdriver_vec[] that ioconf provides, since 827 * the cfdrivers are attached already. 828 */ 829 static struct cfdriver * const no_cfdriver_vec[] = { NULL }; 830 #endif 831 int error = 0; 832 833 #ifdef _MODULE 834 switch (cmd) { 835 case MODULE_CMD_INIT: 836 error = config_init_component(no_cfdriver_vec, 837 cfattach_ioconf_ld_virtio, cfdata_ioconf_ld_virtio); 838 break; 839 case MODULE_CMD_FINI: 840 error = config_fini_component(no_cfdriver_vec, 841 cfattach_ioconf_ld_virtio, cfdata_ioconf_ld_virtio); 842 break; 843 default: 844 error = ENOTTY; 845 break; 846 } 847 #endif 848 849 return error; 850 } 851
Indexes created Mon Sep 29 18:09:42 GMT 2025