ld_virtio.c revision 1.32
1 /* $NetBSD: ld_virtio.c,v 1.32 2023/03/23 03:55:11 yamaguchi Exp $ */ 2 3 /* 4 * Copyright (c) 2010 Minoura Makoto. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 17 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 18 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 19 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 20 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 21 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 22 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 23 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 24 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 25 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 */ 27 28 #include <sys/cdefs.h> 29 __KERNEL_RCSID(0, "$NetBSD: ld_virtio.c,v 1.32 2023/03/23 03:55:11 yamaguchi Exp $"); 30 31 #include <sys/param.h> 32 #include <sys/systm.h> 33 #include <sys/kernel.h> 34 #include <sys/buf.h> 35 #include <sys/bufq.h> 36 #include <sys/bus.h> 37 #include <sys/device.h> 38 #include <sys/disk.h> 39 #include <sys/mutex.h> 40 #include <sys/module.h> 41 42 #include <dev/ldvar.h> 43 #include <dev/pci/virtioreg.h> 44 #include <dev/pci/virtiovar.h> 45 46 #include "ioconf.h" 47 48 /* 49 * ld_virtioreg: 50 */ 51 /* Configuration registers */ 52 #define VIRTIO_BLK_CONFIG_CAPACITY 0 /* 64bit */ 53 #define VIRTIO_BLK_CONFIG_SIZE_MAX 8 /* 32bit */ 54 #define VIRTIO_BLK_CONFIG_SEG_MAX 12 /* 32bit */ 55 #define VIRTIO_BLK_CONFIG_GEOMETRY_C 16 /* 16bit */ 56 #define VIRTIO_BLK_CONFIG_GEOMETRY_H 18 /* 8bit */ 57 #define VIRTIO_BLK_CONFIG_GEOMETRY_S 19 /* 8bit */ 58 #define VIRTIO_BLK_CONFIG_BLK_SIZE 20 /* 32bit */ 59 #define VIRTIO_BLK_CONFIG_WRITEBACK 32 /* 8bit */ 60 61 /* Feature bits */ 62 #define VIRTIO_BLK_F_BARRIER (1<<0) 63 #define VIRTIO_BLK_F_SIZE_MAX (1<<1) 64 #define VIRTIO_BLK_F_SEG_MAX (1<<2) 65 #define VIRTIO_BLK_F_GEOMETRY (1<<4) 66 #define VIRTIO_BLK_F_RO (1<<5) 67 #define VIRTIO_BLK_F_BLK_SIZE (1<<6) 68 #define VIRTIO_BLK_F_SCSI (1<<7) 69 #define VIRTIO_BLK_F_FLUSH (1<<9) 70 #define VIRTIO_BLK_F_TOPOLOGY (1<<10) 71 #define VIRTIO_BLK_F_CONFIG_WCE (1<<11) 72 73 /* 74 * Each block request uses at least two segments - one for the header 75 * and one for the status. 76 */ 77 #define VIRTIO_BLK_MIN_SEGMENTS 2 78 79 #define VIRTIO_BLK_FLAG_BITS \ 80 VIRTIO_COMMON_FLAG_BITS \ 81 "b\x0b" "CONFIG_WCE\0" \ 82 "b\x0a" "TOPOLOGY\0" \ 83 "b\x09" "FLUSH\0" \ 84 "b\x07" "SCSI\0" \ 85 "b\x06" "BLK_SIZE\0" \ 86 "b\x05" "RO\0" \ 87 "b\x04" "GEOMETRY\0" \ 88 "b\x02" "SEG_MAX\0" \ 89 "b\x01" "SIZE_MAX\0" \ 90 "b\x00" "BARRIER\0" 91 92 /* Command */ 93 #define VIRTIO_BLK_T_IN 0 94 #define VIRTIO_BLK_T_OUT 1 95 #define VIRTIO_BLK_T_FLUSH 4 96 #define VIRTIO_BLK_T_BARRIER 0x80000000 97 98 /* Sector */ 99 #define VIRTIO_BLK_BSIZE 512 100 101 /* Status */ 102 #define VIRTIO_BLK_S_OK 0 103 #define VIRTIO_BLK_S_IOERR 1 104 #define VIRTIO_BLK_S_UNSUPP 2 105 106 /* Request header structure */ 107 struct virtio_blk_req_hdr { 108 uint32_t type; /* VIRTIO_BLK_T_* */ 109 uint32_t ioprio; 110 uint64_t sector; 111 } __packed; 112 /* payload and 1 byte status follows */ 113 114 115 /* 116 * ld_virtiovar: 117 */ 118 struct virtio_blk_req { 119 struct virtio_blk_req_hdr vr_hdr; 120 uint8_t vr_status; 121 struct buf *vr_bp; 122 #define DUMMY_VR_BP ((void *)1) 123 bus_dmamap_t vr_cmdsts; 124 bus_dmamap_t vr_payload; 125 }; 126 127 struct ld_virtio_softc { 128 struct ld_softc sc_ld; 129 device_t sc_dev; 130 131 struct virtio_softc *sc_virtio; 132 struct virtqueue sc_vq; 133 134 struct virtio_blk_req *sc_reqs; 135 bus_dma_segment_t sc_reqs_seg; 136 137 int sc_readonly; 138 139 enum { 140 SYNC_FREE, SYNC_BUSY, SYNC_DONE 141 } sc_sync_use; 142 kcondvar_t sc_sync_wait; 143 kmutex_t sc_sync_wait_lock; 144 uint8_t sc_sync_status; 145 }; 146 147 static int ld_virtio_match(device_t, cfdata_t, void *); 148 static void ld_virtio_attach(device_t, device_t, void *); 149 static int ld_virtio_detach(device_t, int); 150 151 CFATTACH_DECL_NEW(ld_virtio, sizeof(struct ld_virtio_softc), 152 ld_virtio_match, ld_virtio_attach, ld_virtio_detach, NULL); 153 154 static int 155 ld_virtio_match(device_t parent, cfdata_t match, void *aux) 156 { 157 struct virtio_attach_args *va = aux; 158 159 if (va->sc_childdevid == VIRTIO_DEVICE_ID_BLOCK) 160 return 1; 161 162 return 0; 163 } 164 165 static int ld_virtio_vq_done(struct virtqueue *); 166 static int ld_virtio_dump(struct ld_softc *, void *, int, int); 167 static int ld_virtio_start(struct ld_softc *, struct buf *); 168 static int ld_virtio_ioctl(struct ld_softc *, u_long, void *, int32_t, bool); 169 170 static int 171 ld_virtio_alloc_reqs(struct ld_virtio_softc *sc, int qsize) 172 { 173 int allocsize, r, rsegs, i; 174 struct ld_softc *ld = &sc->sc_ld; 175 void *vaddr; 176 177 allocsize = sizeof(struct virtio_blk_req) * qsize; 178 r = bus_dmamem_alloc(virtio_dmat(sc->sc_virtio), allocsize, 0, 0, 179 &sc->sc_reqs_seg, 1, &rsegs, BUS_DMA_WAITOK); 180 if (r != 0) { 181 aprint_error_dev(sc->sc_dev, 182 "DMA memory allocation failed, size %d, " 183 "error code %d\n", allocsize, r); 184 goto err_none; 185 } 186 r = bus_dmamem_map(virtio_dmat(sc->sc_virtio), 187 &sc->sc_reqs_seg, 1, allocsize, 188 &vaddr, BUS_DMA_WAITOK); 189 if (r != 0) { 190 aprint_error_dev(sc->sc_dev, 191 "DMA memory map failed, " 192 "error code %d\n", r); 193 goto err_dmamem_alloc; 194 } 195 sc->sc_reqs = vaddr; 196 memset(vaddr, 0, allocsize); 197 for (i = 0; i < qsize; i++) { 198 struct virtio_blk_req *vr = &sc->sc_reqs[i]; 199 r = bus_dmamap_create(virtio_dmat(sc->sc_virtio), 200 offsetof(struct virtio_blk_req, vr_bp), 201 1, 202 offsetof(struct virtio_blk_req, vr_bp), 203 0, 204 BUS_DMA_WAITOK|BUS_DMA_ALLOCNOW, 205 &vr->vr_cmdsts); 206 if (r != 0) { 207 aprint_error_dev(sc->sc_dev, 208 "command dmamap creation failed, " 209 "error code %d\n", r); 210 goto err_reqs; 211 } 212 r = bus_dmamap_load(virtio_dmat(sc->sc_virtio), vr->vr_cmdsts, 213 &vr->vr_hdr, 214 offsetof(struct virtio_blk_req, vr_bp), 215 NULL, BUS_DMA_WAITOK); 216 if (r != 0) { 217 aprint_error_dev(sc->sc_dev, 218 "command dmamap load failed, " 219 "error code %d\n", r); 220 goto err_reqs; 221 } 222 r = bus_dmamap_create(virtio_dmat(sc->sc_virtio), 223 ld->sc_maxxfer, 224 (ld->sc_maxxfer / NBPG) + 225 VIRTIO_BLK_MIN_SEGMENTS, 226 ld->sc_maxxfer, 227 0, 228 BUS_DMA_WAITOK|BUS_DMA_ALLOCNOW, 229 &vr->vr_payload); 230 if (r != 0) { 231 aprint_error_dev(sc->sc_dev, 232 "payload dmamap creation failed, " 233 "error code %d\n", r); 234 goto err_reqs; 235 } 236 } 237 return 0; 238 239 err_reqs: 240 for (i = 0; i < qsize; i++) { 241 struct virtio_blk_req *vr = &sc->sc_reqs[i]; 242 if (vr->vr_cmdsts) { 243 bus_dmamap_destroy(virtio_dmat(sc->sc_virtio), 244 vr->vr_cmdsts); 245 vr->vr_cmdsts = 0; 246 } 247 if (vr->vr_payload) { 248 bus_dmamap_destroy(virtio_dmat(sc->sc_virtio), 249 vr->vr_payload); 250 vr->vr_payload = 0; 251 } 252 } 253 bus_dmamem_unmap(virtio_dmat(sc->sc_virtio), sc->sc_reqs, allocsize); 254 err_dmamem_alloc: 255 bus_dmamem_free(virtio_dmat(sc->sc_virtio), &sc->sc_reqs_seg, 1); 256 err_none: 257 return -1; 258 } 259 260 static void 261 ld_virtio_attach(device_t parent, device_t self, void *aux) 262 { 263 struct ld_virtio_softc *sc = device_private(self); 264 struct ld_softc *ld = &sc->sc_ld; 265 struct virtio_softc *vsc = device_private(parent); 266 uint64_t features; 267 int qsize, maxxfersize, maxnsegs; 268 269 if (virtio_child(vsc) != NULL) { 270 aprint_normal(": child already attached for %s; " 271 "something wrong...\n", device_xname(parent)); 272 return; 273 } 274 275 sc->sc_dev = self; 276 sc->sc_virtio = vsc; 277 278 virtio_child_attach_start(vsc, self, IPL_BIO, 279 (VIRTIO_BLK_F_SIZE_MAX | VIRTIO_BLK_F_SEG_MAX | 280 VIRTIO_BLK_F_GEOMETRY | VIRTIO_BLK_F_RO | VIRTIO_BLK_F_BLK_SIZE | 281 VIRTIO_BLK_F_FLUSH | VIRTIO_BLK_F_CONFIG_WCE), 282 VIRTIO_BLK_FLAG_BITS); 283 284 features = virtio_features(vsc); 285 if (features == 0) 286 goto err; 287 288 if (features & VIRTIO_BLK_F_RO) 289 sc->sc_readonly = 1; 290 else 291 sc->sc_readonly = 0; 292 293 if (features & VIRTIO_BLK_F_BLK_SIZE) { 294 ld->sc_secsize = virtio_read_device_config_4(vsc, 295 VIRTIO_BLK_CONFIG_BLK_SIZE); 296 } else 297 ld->sc_secsize = VIRTIO_BLK_BSIZE; 298 299 /* At least genfs_io assumes maxxfer == MAXPHYS. */ 300 if (features & VIRTIO_BLK_F_SIZE_MAX) { 301 maxxfersize = virtio_read_device_config_4(vsc, 302 VIRTIO_BLK_CONFIG_SIZE_MAX); 303 if (maxxfersize < MAXPHYS) { 304 aprint_error_dev(sc->sc_dev, 305 "Too small SIZE_MAX %dK minimum is %dK\n", 306 maxxfersize / 1024, MAXPHYS / 1024); 307 // goto err; 308 maxxfersize = MAXPHYS; 309 } else if (maxxfersize > MAXPHYS) { 310 aprint_normal_dev(sc->sc_dev, 311 "Clip SEG_MAX from %dK to %dK\n", 312 maxxfersize / 1024, 313 MAXPHYS / 1024); 314 maxxfersize = MAXPHYS; 315 } 316 } else 317 maxxfersize = MAXPHYS; 318 319 if (features & VIRTIO_BLK_F_SEG_MAX) { 320 maxnsegs = virtio_read_device_config_4(vsc, 321 VIRTIO_BLK_CONFIG_SEG_MAX); 322 if (maxnsegs < VIRTIO_BLK_MIN_SEGMENTS) { 323 aprint_error_dev(sc->sc_dev, 324 "Too small SEG_MAX %d minimum is %d\n", 325 maxnsegs, VIRTIO_BLK_MIN_SEGMENTS); 326 maxnsegs = maxxfersize / NBPG; 327 // goto err; 328 } 329 } else 330 maxnsegs = maxxfersize / NBPG; 331 332 /* 2 for the minimum size */ 333 maxnsegs += VIRTIO_BLK_MIN_SEGMENTS; 334 335 virtio_init_vq_vqdone(vsc, &sc->sc_vq, 0, 336 ld_virtio_vq_done); 337 338 if (virtio_alloc_vq(vsc, &sc->sc_vq, maxxfersize, maxnsegs, 339 "I/O request") != 0) { 340 goto err; 341 } 342 qsize = sc->sc_vq.vq_num; 343 344 if (virtio_child_attach_finish(vsc, &sc->sc_vq, 1, 345 NULL, VIRTIO_F_INTR_MSIX) != 0) 346 goto err; 347 348 ld->sc_dv = self; 349 ld->sc_secperunit = virtio_read_device_config_8(vsc, 350 VIRTIO_BLK_CONFIG_CAPACITY) / (ld->sc_secsize / VIRTIO_BLK_BSIZE); 351 ld->sc_maxxfer = maxxfersize; 352 if (features & VIRTIO_BLK_F_GEOMETRY) { 353 ld->sc_ncylinders = virtio_read_device_config_2(vsc, 354 VIRTIO_BLK_CONFIG_GEOMETRY_C); 355 ld->sc_nheads = virtio_read_device_config_1(vsc, 356 VIRTIO_BLK_CONFIG_GEOMETRY_H); 357 ld->sc_nsectors = virtio_read_device_config_1(vsc, 358 VIRTIO_BLK_CONFIG_GEOMETRY_S); 359 } 360 ld->sc_maxqueuecnt = qsize - 1; /* reserve slot for dumps, flushes */ 361 362 if (ld_virtio_alloc_reqs(sc, qsize) < 0) 363 goto err; 364 365 cv_init(&sc->sc_sync_wait, "vblksync"); 366 mutex_init(&sc->sc_sync_wait_lock, MUTEX_DEFAULT, IPL_BIO); 367 sc->sc_sync_use = SYNC_FREE; 368 369 ld->sc_dump = ld_virtio_dump; 370 ld->sc_start = ld_virtio_start; 371 ld->sc_ioctl = ld_virtio_ioctl; 372 373 ld->sc_flags = LDF_ENABLED | LDF_MPSAFE; 374 ldattach(ld, BUFQ_DISK_DEFAULT_STRAT); 375 376 return; 377 378 err: 379 virtio_child_attach_failed(vsc); 380 return; 381 } 382 383 static int 384 ld_virtio_start(struct ld_softc *ld, struct buf *bp) 385 { 386 /* splbio */ 387 struct ld_virtio_softc *sc = device_private(ld->sc_dv); 388 struct virtio_softc *vsc = sc->sc_virtio; 389 struct virtqueue *vq = &sc->sc_vq; 390 struct virtio_blk_req *vr; 391 int r; 392 int isread = (bp->b_flags & B_READ); 393 int slot; 394 395 if (sc->sc_readonly && !isread) 396 return EIO; 397 398 r = virtio_enqueue_prep(vsc, vq, &slot); 399 if (r != 0) 400 return r; 401 402 vr = &sc->sc_reqs[slot]; 403 KASSERT(vr->vr_bp == NULL); 404 405 r = bus_dmamap_load(virtio_dmat(vsc), vr->vr_payload, 406 bp->b_data, bp->b_bcount, NULL, 407 ((isread?BUS_DMA_READ:BUS_DMA_WRITE) 408 |BUS_DMA_NOWAIT)); 409 if (r != 0) { 410 aprint_error_dev(sc->sc_dev, 411 "payload dmamap failed, error code %d\n", r); 412 virtio_enqueue_abort(vsc, vq, slot); 413 return r; 414 } 415 416 r = virtio_enqueue_reserve(vsc, vq, slot, vr->vr_payload->dm_nsegs + 417 VIRTIO_BLK_MIN_SEGMENTS); 418 if (r != 0) { 419 bus_dmamap_unload(virtio_dmat(vsc), vr->vr_payload); 420 return r; 421 } 422 423 vr->vr_bp = bp; 424 vr->vr_hdr.type = virtio_rw32(vsc, 425 isread ? VIRTIO_BLK_T_IN : VIRTIO_BLK_T_OUT); 426 vr->vr_hdr.ioprio = virtio_rw32(vsc, 0); 427 vr->vr_hdr.sector = virtio_rw64(vsc, 428 bp->b_rawblkno * sc->sc_ld.sc_secsize / 429 VIRTIO_BLK_BSIZE); 430 431 bus_dmamap_sync(virtio_dmat(vsc), vr->vr_cmdsts, 432 0, sizeof(struct virtio_blk_req_hdr), 433 BUS_DMASYNC_PREWRITE); 434 bus_dmamap_sync(virtio_dmat(vsc), vr->vr_payload, 435 0, bp->b_bcount, 436 isread?BUS_DMASYNC_PREREAD:BUS_DMASYNC_PREWRITE); 437 bus_dmamap_sync(virtio_dmat(vsc), vr->vr_cmdsts, 438 offsetof(struct virtio_blk_req, vr_status), 439 sizeof(uint8_t), 440 BUS_DMASYNC_PREREAD); 441 442 virtio_enqueue_p(vsc, vq, slot, vr->vr_cmdsts, 443 0, sizeof(struct virtio_blk_req_hdr), 444 true); 445 virtio_enqueue(vsc, vq, slot, vr->vr_payload, !isread); 446 virtio_enqueue_p(vsc, vq, slot, vr->vr_cmdsts, 447 offsetof(struct virtio_blk_req, vr_status), 448 sizeof(uint8_t), 449 false); 450 virtio_enqueue_commit(vsc, vq, slot, true); 451 452 return 0; 453 } 454 455 static void 456 ld_virtio_vq_done1(struct ld_virtio_softc *sc, struct virtio_softc *vsc, 457 struct virtqueue *vq, int slot) 458 { 459 struct virtio_blk_req *vr = &sc->sc_reqs[slot]; 460 struct buf *bp = vr->vr_bp; 461 462 vr->vr_bp = NULL; 463 464 bus_dmamap_sync(virtio_dmat(vsc), vr->vr_cmdsts, 465 0, sizeof(struct virtio_blk_req_hdr), 466 BUS_DMASYNC_POSTWRITE); 467 bus_dmamap_sync(virtio_dmat(vsc), vr->vr_cmdsts, 468 sizeof(struct virtio_blk_req_hdr), sizeof(uint8_t), 469 BUS_DMASYNC_POSTREAD); 470 if (bp == DUMMY_VR_BP) { 471 mutex_enter(&sc->sc_sync_wait_lock); 472 sc->sc_sync_status = vr->vr_status; 473 sc->sc_sync_use = SYNC_DONE; 474 cv_broadcast(&sc->sc_sync_wait); 475 mutex_exit(&sc->sc_sync_wait_lock); 476 virtio_dequeue_commit(vsc, vq, slot); 477 return; 478 } 479 bus_dmamap_sync(virtio_dmat(vsc), vr->vr_payload, 480 0, bp->b_bcount, 481 (bp->b_flags & B_READ)?BUS_DMASYNC_POSTREAD 482 :BUS_DMASYNC_POSTWRITE); 483 bus_dmamap_unload(virtio_dmat(vsc), vr->vr_payload); 484 485 if (vr->vr_status != VIRTIO_BLK_S_OK) { 486 bp->b_error = EIO; 487 bp->b_resid = bp->b_bcount; 488 } else { 489 bp->b_error = 0; 490 bp->b_resid = 0; 491 } 492 493 virtio_dequeue_commit(vsc, vq, slot); 494 495 lddone(&sc->sc_ld, bp); 496 } 497 498 static int 499 ld_virtio_vq_done(struct virtqueue *vq) 500 { 501 struct virtio_softc *vsc = vq->vq_owner; 502 struct ld_virtio_softc *sc = device_private(virtio_child(vsc)); 503 int r = 0; 504 int slot; 505 506 again: 507 if (virtio_dequeue(vsc, vq, &slot, NULL)) 508 return r; 509 r = 1; 510 511 ld_virtio_vq_done1(sc, vsc, vq, slot); 512 goto again; 513 } 514 515 static int 516 ld_virtio_dump(struct ld_softc *ld, void *data, int blkno, int blkcnt) 517 { 518 struct ld_virtio_softc *sc = device_private(ld->sc_dv); 519 struct virtio_softc *vsc = sc->sc_virtio; 520 struct virtqueue *vq = &sc->sc_vq; 521 struct virtio_blk_req *vr; 522 int slot, r; 523 524 if (sc->sc_readonly) 525 return EIO; 526 527 r = virtio_enqueue_prep(vsc, vq, &slot); 528 if (r != 0) { 529 if (r == EAGAIN) { /* no free slot; dequeue first */ 530 delay(100); 531 ld_virtio_vq_done(vq); 532 r = virtio_enqueue_prep(vsc, vq, &slot); 533 if (r != 0) 534 return r; 535 } 536 return r; 537 } 538 vr = &sc->sc_reqs[slot]; 539 r = bus_dmamap_load(virtio_dmat(vsc), vr->vr_payload, 540 data, blkcnt*ld->sc_secsize, NULL, 541 BUS_DMA_WRITE|BUS_DMA_NOWAIT); 542 if (r != 0) 543 return r; 544 545 r = virtio_enqueue_reserve(vsc, vq, slot, vr->vr_payload->dm_nsegs + 546 VIRTIO_BLK_MIN_SEGMENTS); 547 if (r != 0) { 548 bus_dmamap_unload(virtio_dmat(vsc), vr->vr_payload); 549 return r; 550 } 551 552 vr->vr_bp = (void*)0xdeadbeef; 553 vr->vr_hdr.type = virtio_rw32(vsc, VIRTIO_BLK_T_OUT); 554 vr->vr_hdr.ioprio = virtio_rw32(vsc, 0); 555 vr->vr_hdr.sector = virtio_rw64(vsc, 556 (daddr_t) blkno * ld->sc_secsize / 557 VIRTIO_BLK_BSIZE); 558 559 bus_dmamap_sync(virtio_dmat(vsc), vr->vr_cmdsts, 560 0, sizeof(struct virtio_blk_req_hdr), 561 BUS_DMASYNC_PREWRITE); 562 bus_dmamap_sync(virtio_dmat(vsc), vr->vr_payload, 563 0, blkcnt*ld->sc_secsize, 564 BUS_DMASYNC_PREWRITE); 565 bus_dmamap_sync(virtio_dmat(vsc), vr->vr_cmdsts, 566 offsetof(struct virtio_blk_req, vr_status), 567 sizeof(uint8_t), 568 BUS_DMASYNC_PREREAD); 569 570 virtio_enqueue_p(vsc, vq, slot, vr->vr_cmdsts, 571 0, sizeof(struct virtio_blk_req_hdr), 572 true); 573 virtio_enqueue(vsc, vq, slot, vr->vr_payload, true); 574 virtio_enqueue_p(vsc, vq, slot, vr->vr_cmdsts, 575 offsetof(struct virtio_blk_req, vr_status), 576 sizeof(uint8_t), 577 false); 578 virtio_enqueue_commit(vsc, vq, slot, true); 579 580 for ( ; ; ) { 581 int dslot; 582 583 r = virtio_dequeue(vsc, vq, &dslot, NULL); 584 if (r != 0) 585 continue; 586 if (dslot != slot) { 587 ld_virtio_vq_done1(sc, vsc, vq, dslot); 588 continue; 589 } else 590 break; 591 } 592 593 bus_dmamap_sync(virtio_dmat(vsc), vr->vr_cmdsts, 594 0, sizeof(struct virtio_blk_req_hdr), 595 BUS_DMASYNC_POSTWRITE); 596 bus_dmamap_sync(virtio_dmat(vsc), vr->vr_payload, 597 0, blkcnt*ld->sc_secsize, 598 BUS_DMASYNC_POSTWRITE); 599 bus_dmamap_sync(virtio_dmat(vsc), vr->vr_cmdsts, 600 offsetof(struct virtio_blk_req, vr_status), 601 sizeof(uint8_t), 602 BUS_DMASYNC_POSTREAD); 603 if (vr->vr_status == VIRTIO_BLK_S_OK) 604 r = 0; 605 else 606 r = EIO; 607 virtio_dequeue_commit(vsc, vq, slot); 608 609 return r; 610 } 611 612 static int 613 ld_virtio_detach(device_t self, int flags) 614 { 615 struct ld_virtio_softc *sc = device_private(self); 616 struct ld_softc *ld = &sc->sc_ld; 617 bus_dma_tag_t dmat = virtio_dmat(sc->sc_virtio); 618 int r, i, qsize; 619 620 qsize = sc->sc_vq.vq_num; 621 r = ldbegindetach(ld, flags); 622 if (r != 0) 623 return r; 624 virtio_reset(sc->sc_virtio); 625 virtio_free_vq(sc->sc_virtio, &sc->sc_vq); 626 627 for (i = 0; i < qsize; i++) { 628 bus_dmamap_destroy(dmat, 629 sc->sc_reqs[i].vr_cmdsts); 630 bus_dmamap_destroy(dmat, 631 sc->sc_reqs[i].vr_payload); 632 } 633 bus_dmamem_unmap(dmat, sc->sc_reqs, 634 sizeof(struct virtio_blk_req) * qsize); 635 bus_dmamem_free(dmat, &sc->sc_reqs_seg, 1); 636 637 ldenddetach(ld); 638 639 cv_destroy(&sc->sc_sync_wait); 640 mutex_destroy(&sc->sc_sync_wait_lock); 641 642 virtio_child_detach(sc->sc_virtio); 643 644 return 0; 645 } 646 647 static int 648 ld_virtio_flush(struct ld_softc *ld, bool poll) 649 { 650 struct ld_virtio_softc * const sc = device_private(ld->sc_dv); 651 struct virtio_softc * const vsc = sc->sc_virtio; 652 const uint64_t features = virtio_features(vsc); 653 struct virtqueue *vq = &sc->sc_vq; 654 struct virtio_blk_req *vr; 655 int slot; 656 int r; 657 658 if ((features & VIRTIO_BLK_F_FLUSH) == 0) 659 return 0; 660 661 mutex_enter(&sc->sc_sync_wait_lock); 662 while (sc->sc_sync_use != SYNC_FREE) { 663 if (poll) { 664 mutex_exit(&sc->sc_sync_wait_lock); 665 ld_virtio_vq_done(vq); 666 mutex_enter(&sc->sc_sync_wait_lock); 667 continue; 668 } 669 cv_wait(&sc->sc_sync_wait, &sc->sc_sync_wait_lock); 670 } 671 sc->sc_sync_use = SYNC_BUSY; 672 mutex_exit(&sc->sc_sync_wait_lock); 673 674 r = virtio_enqueue_prep(vsc, vq, &slot); 675 if (r != 0) { 676 return r; 677 } 678 679 vr = &sc->sc_reqs[slot]; 680 KASSERT(vr->vr_bp == NULL); 681 682 r = virtio_enqueue_reserve(vsc, vq, slot, VIRTIO_BLK_MIN_SEGMENTS); 683 if (r != 0) { 684 return r; 685 } 686 687 vr->vr_bp = DUMMY_VR_BP; 688 vr->vr_hdr.type = virtio_rw32(vsc, VIRTIO_BLK_T_FLUSH); 689 vr->vr_hdr.ioprio = virtio_rw32(vsc, 0); 690 vr->vr_hdr.sector = virtio_rw64(vsc, 0); 691 692 bus_dmamap_sync(virtio_dmat(vsc), vr->vr_cmdsts, 693 0, sizeof(struct virtio_blk_req_hdr), 694 BUS_DMASYNC_PREWRITE); 695 bus_dmamap_sync(virtio_dmat(vsc), vr->vr_cmdsts, 696 offsetof(struct virtio_blk_req, vr_status), 697 sizeof(uint8_t), 698 BUS_DMASYNC_PREREAD); 699 700 virtio_enqueue_p(vsc, vq, slot, vr->vr_cmdsts, 701 0, sizeof(struct virtio_blk_req_hdr), 702 true); 703 virtio_enqueue_p(vsc, vq, slot, vr->vr_cmdsts, 704 offsetof(struct virtio_blk_req, vr_status), 705 sizeof(uint8_t), 706 false); 707 virtio_enqueue_commit(vsc, vq, slot, true); 708 709 mutex_enter(&sc->sc_sync_wait_lock); 710 while (sc->sc_sync_use != SYNC_DONE) { 711 if (poll) { 712 mutex_exit(&sc->sc_sync_wait_lock); 713 ld_virtio_vq_done(vq); 714 mutex_enter(&sc->sc_sync_wait_lock); 715 continue; 716 } 717 cv_wait(&sc->sc_sync_wait, &sc->sc_sync_wait_lock); 718 } 719 720 if (sc->sc_sync_status == VIRTIO_BLK_S_OK) 721 r = 0; 722 else 723 r = EIO; 724 725 sc->sc_sync_use = SYNC_FREE; 726 cv_broadcast(&sc->sc_sync_wait); 727 mutex_exit(&sc->sc_sync_wait_lock); 728 729 return r; 730 } 731 732 static int 733 ld_virtio_getcache(struct ld_softc *ld, int *bitsp) 734 { 735 struct ld_virtio_softc * const sc = device_private(ld->sc_dv); 736 struct virtio_softc * const vsc = sc->sc_virtio; 737 const uint64_t features = virtio_features(vsc); 738 739 *bitsp = DKCACHE_READ; 740 if ((features & VIRTIO_BLK_F_CONFIG_WCE) != 0) 741 *bitsp |= DKCACHE_WCHANGE; 742 if (virtio_read_device_config_1(vsc, 743 VIRTIO_BLK_CONFIG_WRITEBACK) != 0x00) 744 *bitsp |= DKCACHE_WRITE; 745 746 return 0; 747 } 748 749 static int 750 ld_virtio_setcache(struct ld_softc *ld, int bits) 751 { 752 struct ld_virtio_softc * const sc = device_private(ld->sc_dv); 753 struct virtio_softc * const vsc = sc->sc_virtio; 754 const uint8_t wce = (bits & DKCACHE_WRITE) ? 0x01 : 0x00; 755 756 virtio_write_device_config_1(vsc, 757 VIRTIO_BLK_CONFIG_WRITEBACK, wce); 758 if (virtio_read_device_config_1(vsc, 759 VIRTIO_BLK_CONFIG_WRITEBACK) != wce) 760 return EIO; 761 762 return 0; 763 } 764 765 static int 766 ld_virtio_ioctl(struct ld_softc *ld, u_long cmd, void *addr, int32_t flag, bool poll) 767 { 768 int error; 769 770 switch (cmd) { 771 case DIOCCACHESYNC: 772 error = ld_virtio_flush(ld, poll); 773 break; 774 775 case DIOCGCACHE: 776 error = ld_virtio_getcache(ld, (int *)addr); 777 break; 778 779 case DIOCSCACHE: 780 error = ld_virtio_setcache(ld, *(int *)addr); 781 break; 782 783 default: 784 error = EPASSTHROUGH; 785 break; 786 } 787 788 return error; 789 } 790 791 MODULE(MODULE_CLASS_DRIVER, ld_virtio, "ld,virtio"); 792 793 #ifdef _MODULE 794 /* 795 * XXX Don't allow ioconf.c to redefine the "struct cfdriver ld_cd" 796 * XXX it will be defined in the common-code module 797 */ 798 #undef CFDRIVER_DECL 799 #define CFDRIVER_DECL(name, class, attr) 800 #include "ioconf.c" 801 #endif 802 803 static int 804 ld_virtio_modcmd(modcmd_t cmd, void *opaque) 805 { 806 #ifdef _MODULE 807 /* 808 * We ignore the cfdriver_vec[] that ioconf provides, since 809 * the cfdrivers are attached already. 810 */ 811 static struct cfdriver * const no_cfdriver_vec[] = { NULL }; 812 #endif 813 int error = 0; 814 815 #ifdef _MODULE 816 switch (cmd) { 817 case MODULE_CMD_INIT: 818 error = config_init_component(no_cfdriver_vec, 819 cfattach_ioconf_ld_virtio, cfdata_ioconf_ld_virtio); 820 break; 821 case MODULE_CMD_FINI: 822 error = config_fini_component(no_cfdriver_vec, 823 cfattach_ioconf_ld_virtio, cfdata_ioconf_ld_virtio); 824 break; 825 default: 826 error = ENOTTY; 827 break; 828 } 829 #endif 830 831 return error; 832 } 833
Indexes created Sun Sep 28 07:09:56 GMT 2025