1 /* $NetBSD: xbdback_xenbus.c,v 1.111 2026/10/04 06:46:55 mlelstv Exp $ */ 2 3 /* 4 * Copyright (c) 2006,2024 Manuel Bouyer. 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 16 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 17 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 18 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 19 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 20 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 21 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 22 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 23 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 24 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25 * 26 */ 27 28 #include <sys/cdefs.h> 29 __KERNEL_RCSID(0, "$NetBSD: xbdback_xenbus.c,v 1.111 2026/10/04 06:46:55 mlelstv Exp $"); 30 31 #include <sys/buf.h> 32 #include <sys/condvar.h> 33 #include <sys/conf.h> 34 #include <sys/disk.h> 35 #include <sys/device.h> 36 #include <sys/fcntl.h> 37 #include <sys/kauth.h> 38 #include <sys/kernel.h> 39 #include <sys/kmem.h> 40 #include <sys/kthread.h> 41 #include <sys/mutex.h> 42 #include <sys/param.h> 43 #include <sys/queue.h> 44 #include <sys/systm.h> 45 #include <sys/time.h> 46 #include <sys/types.h> 47 #include <sys/vnode.h> 48 49 #include <xen/intr.h> 50 #include <xen/hypervisor.h> 51 #include <xen/xen.h> 52 #include <xen/xen_shm.h> 53 #include <xen/evtchn.h> 54 #include <xen/xenbus.h> 55 #include <xen/xenring.h> 56 #include <xen/include/public/io/protocols.h> 57 58 /* #define XENDEBUG_VBD */ 59 #ifdef XENDEBUG_VBD 60 #define XENPRINTF(x) printf x 61 #else 62 #define XENPRINTF(x) 63 #endif 64 65 #define BLKIF_RING_SIZE __CONST_RING_SIZE(blkif, PAGE_SIZE) 66 67 /* 68 * Backend block device driver for Xen 69 */ 70 71 /* Values are expressed in 512-byte sectors */ 72 #define VBD_BSIZE 512 73 #define VBD_MAXSECT ((PAGE_SIZE / VBD_BSIZE) - 1) 74 75 #define VBD_VA_SIZE MAXPHYS 76 #define VBD_MAX_INDIRECT_SEGMENTS (VBD_VA_SIZE >> PAGE_SHIFT) 77 78 CTASSERT(XENSHM_MAX_PAGES_PER_REQUEST >= VBD_MAX_INDIRECT_SEGMENTS); 79 80 struct xbdback_instance; 81 82 /* 83 * status of a xbdback instance: 84 * WAITING: xbdback instance is connected, waiting for requests 85 * RUN: xbdi thread must be woken up, I/Os have to be processed 86 * DISCONNECTING: the instance is closing, no more I/Os can be scheduled 87 * DISCONNECTED: no I/Os, no ring, the thread should terminate. 88 */ 89 typedef enum {WAITING, RUN, DISCONNECTING, DISCONNECTED} xbdback_state_t; 90 91 /* 92 * Each xbdback instance is managed by a single thread that handles all 93 * the I/O processing. As there are a variety of conditions that can block, 94 * everything will be done in a sort of continuation-passing style. 95 * 96 * When the execution has to block to delay processing, for example to 97 * allow system to recover because of memory shortage (via shared memory 98 * callback), the return value of a continuation can be set to NULL. In that 99 * case, the thread will go back to sleeping and wait for the proper 100 * condition before it starts processing requests again from where it left. 101 * Continuation state is "stored" in the xbdback instance (xbdi_cont), 102 * and should only be manipulated by the instance thread. 103 * If a continuation has to be restarted from a specific point, 104 * the callback and argument can be stored in xbdi_cont_restart and 105 * xbdi_cont_restart_obj 106 * 107 * 108 * As xbdback(4) has to handle different sort of asynchronous events (Xen 109 * event channels, biointr() soft interrupts, xenbus commands), the xbdi_lock 110 * mutex is used to protect specific elements of the xbdback instance from 111 * concurrent access: thread status and ring access (when pushing responses). 112 * 113 * Here's how the call graph is supposed to be for a single I/O: 114 * 115 * xbdback_co_main() 116 * | --> xbdback_co_cache_flush() 117 * | | | 118 * | | -> xbdback_co_do_io() or NULL 119 * xbdback_co_main_loop()-| 120 * | |-> xbdback_co_main_done2() or NULL 121 * | | 122 * | --> xbdback_co_main_incr() -> xbdback_co_main_loop() 123 * | 124 * xbdback_co_io() -> xbdback_co_main_incr() -> xbdback_co_main_loop() 125 * | 126 * xbdback_co_io_gotio() -> xbdback_co_main_incr() -> xbdback_co_main_loop() 127 * | 128 * xbdback_co_do_io() 129 * | 130 * xbdback_co_main_incr() -> xbdback_co_main_loop() 131 */ 132 typedef void *(* xbdback_cont_t)(struct xbdback_instance *, void *); 133 134 enum xbdi_proto { 135 XBDIP_NATIVE, 136 XBDIP_32, 137 XBDIP_64 138 }; 139 140 struct xbdback_va { 141 SLIST_ENTRY(xbdback_va) xv_next; 142 vaddr_t xv_vaddr; 143 }; 144 145 /* 146 * For each I/O operation associated with one of those requests, an 147 * xbdback_io is allocated from a pool. It may correspond to multiple 148 * Xen disk requests, or parts of them, if several arrive at once that 149 * can be coalesced. 150 */ 151 struct xbdback_io { 152 SLIST_ENTRY(xbdback_io) xio_next; 153 /* The instance pointer is duplicated for convenience. */ 154 struct xbdback_instance *xio_xbdi; /* our xbd instance */ 155 /* _request state: track requests fetched from ring */ 156 blkif_request_t xio_xen_req; 157 /* array of segments[VBD_MAX_INDIRECT_SEGMENTS] allocated separately */ 158 struct blkif_request_segment *xio_seg; 159 bus_dmamap_t xio_seg_dmamap; 160 /* internal states */ 161 union { 162 struct { 163 struct buf xio_buf; /* our I/O */ 164 /* the virtual address to map the request at */ 165 vaddr_t xio_vaddr; 166 struct xbdback_va *xio_xv; 167 vaddr_t xio_start_offset; /* I/O start offset */ 168 /* grants to map */ 169 grant_ref_t xio_gref[VBD_MAX_INDIRECT_SEGMENTS]; 170 /* grants release */ 171 grant_handle_t xio_gh[VBD_MAX_INDIRECT_SEGMENTS]; 172 bool xio_need_bounce; /* request is not contiguous */ 173 } xio_rw; 174 } u; 175 }; 176 #define xio_buf u.xio_rw.xio_buf 177 #define xio_vaddr u.xio_rw.xio_vaddr 178 #define xio_start_offset u.xio_rw.xio_start_offset 179 #define xio_xv u.xio_rw.xio_xv 180 #define xio_gref u.xio_rw.xio_gref 181 #define xio_gh u.xio_rw.xio_gh 182 #define xio_need_bounce u.xio_rw.xio_need_bounce 183 184 /* we keep the xbdback instances in a linked list */ 185 struct xbdback_instance { 186 SLIST_ENTRY(xbdback_instance) next; 187 struct xenbus_device *xbdi_xbusd; /* our xenstore entry */ 188 struct xenbus_watch xbdi_watch; /* to watch our store */ 189 domid_t xbdi_domid; /* attached to this domain */ 190 uint32_t xbdi_handle; /* domain-specific handle */ 191 char xbdi_name[16]; /* name of this instance */ 192 /* mutex that protects concurrent access to the xbdback instance */ 193 kmutex_t xbdi_lock; 194 kcondvar_t xbdi_cv; /* wait channel for thread work */ 195 xbdback_state_t xbdi_status; /* thread's status */ 196 /* context and KVA for mapping transfers */ 197 struct xbdback_io xbdi_io[BLKIF_RING_SIZE]; 198 SLIST_HEAD(, xbdback_io) xbdi_io_free; 199 struct xbdback_va xbdi_va[BLKIF_RING_SIZE]; 200 SLIST_HEAD(, xbdback_va) xbdi_va_free; 201 /* segments structure allocated in page-aligned chunks */ 202 struct blkif_request_segment *xbdi_segs; 203 /* bounce buffer in case a transfer is not contiguous */ 204 vaddr_t xbdi_bouncebuf; 205 int xbdi_bouncebuf_use; /* is bounce buffer in use? */ 206 /* backing device parameters */ 207 dev_t xbdi_dev; 208 const struct bdevsw *xbdi_bdevsw; /* pointer to the device's bdevsw */ 209 struct vnode *xbdi_vp; 210 uint64_t xbdi_size; 211 unsigned xbdi_secsize; 212 bool xbdi_ro; /* is device read-only ? */ 213 /* parameters for the communication */ 214 unsigned int xbdi_evtchn; 215 struct intrhand *xbdi_ih; 216 /* private parameters for communication */ 217 blkif_back_ring_proto_t xbdi_ring; 218 enum xbdi_proto xbdi_proto; 219 grant_handle_t xbdi_ring_handle; /* to unmap the ring */ 220 vaddr_t xbdi_ring_va; /* to unmap the ring */ 221 /* disconnection must be postponed until all I/O is done */ 222 int xbdi_refcnt; 223 /* 224 * State for I/O processing/coalescing follows; this has to 225 * live here instead of on the stack because of the 226 * continuation-ness (see above). 227 */ 228 RING_IDX xbdi_req_prod; /* limit on request indices */ 229 xbdback_cont_t xbdi_cont; 230 /* if not NULL, will restart here after thread wakes up */ 231 xbdback_cont_t xbdi_cont_restart; 232 void *xbdi_cont_restart_obj; 233 /* other state */ 234 uint xbdi_pendingreqs; /* number of I/O in fly */ 235 struct timeval xbdi_lasterr_time; /* error time tracking */ 236 }; 237 /* Manipulation of the above reference count. */ 238 #define xbdi_get(xbdip) \ 239 do { \ 240 KASSERT(mutex_owned(&xbdip->xbdi_lock)); \ 241 (xbdip)->xbdi_refcnt++; \ 242 } while (0) 243 244 #define xbdi_put(xbdip) \ 245 do { \ 246 KASSERT(mutex_owned(&xbdip->xbdi_lock)); \ 247 if (--((xbdip)->xbdi_refcnt) == 0) \ 248 xbdback_finish_disconnect(xbdip); \ 249 } while (0) 250 251 static SLIST_HEAD(, xbdback_instance) xbdback_instances; 252 static kmutex_t xbdback_lock; 253 254 /* Interval between reports of I/O errors from frontend */ 255 static const struct timeval xbdback_err_intvl = { 1, 0 }; 256 257 void xbdbackattach(int); 258 static int xbdback_xenbus_create(struct xenbus_device *); 259 static int xbdback_xenbus_destroy(void *); 260 static void xbdback_frontend_changed(void *, XenbusState); 261 static void xbdback_backend_changed(struct xenbus_watch *, 262 const char **, unsigned int); 263 static int xbdback_evthandler(void *); 264 265 static int xbdback_connect(struct xbdback_instance *); 266 static void xbdback_disconnect(struct xbdback_instance *); 267 static void xbdback_finish_disconnect(struct xbdback_instance *); 268 269 static bool xbdif_lookup(domid_t, uint32_t); 270 271 static void *xbdback_co_main(struct xbdback_instance *, void *); 272 static void *xbdback_co_main_loop(struct xbdback_instance *, void *); 273 static void *xbdback_co_main_incr(struct xbdback_instance *, void *); 274 static void *xbdback_co_main_done2(struct xbdback_instance *, void *); 275 276 static void *xbdback_co_cache_flush(struct xbdback_instance *, void *); 277 278 static void *xbdback_co_io(struct xbdback_instance *, void *); 279 static void *xbdback_co_io_gotio(struct xbdback_instance *, void *); 280 281 static void *xbdback_co_do_io(struct xbdback_instance *, void *); 282 283 static void xbdback_io_error(struct xbdback_io *, int); 284 static void xbdback_iodone(struct buf *); 285 static void xbdback_iodone_locked(struct xbdback_instance *, 286 struct xbdback_io *, struct buf *); 287 static void xbdback_send_reply(struct xbdback_instance *, uint64_t , int , int); 288 289 static int xbdback_map_shm(struct xbdback_io *); 290 static void xbdback_unmap_shm(struct xbdback_io *); 291 292 static struct xbdback_io *xbdback_io_get(struct xbdback_instance *); 293 static void xbdback_io_put(struct xbdback_instance *, struct xbdback_io *); 294 static void xbdback_thread(void *); 295 static void xbdback_wakeup_thread(struct xbdback_instance *); 296 static void xbdback_trampoline(struct xbdback_instance *, void *); 297 298 static struct xenbus_backend_driver xbd_backend_driver = { 299 .xbakd_create = xbdback_xenbus_create, 300 .xbakd_type = "vbd" 301 }; 302 303 void 304 xbdbackattach(int n) 305 { 306 XENPRINTF(("xbdbackattach\n")); 307 308 /* 309 * initialize the backend driver, register the control message handler 310 * and send driver up message. 311 */ 312 SLIST_INIT(&xbdback_instances); 313 mutex_init(&xbdback_lock, MUTEX_DEFAULT, IPL_NONE); 314 315 xenbus_backend_register(&xbd_backend_driver); 316 } 317 318 static int 319 xbdback_xenbus_create(struct xenbus_device *xbusd) 320 { 321 struct xbdback_instance *xbdi; 322 long domid, handle; 323 int error, i; 324 int segalloc = 0; 325 char *ep; 326 327 if ((error = xenbus_read_ul(NULL, xbusd->xbusd_path, 328 "frontend-id", &domid, 10)) != 0) { 329 aprint_error("xbdback: can't read %s/frontend-id: %d\n", 330 xbusd->xbusd_path, error); 331 return error; 332 } 333 334 /* 335 * get handle: this is the last component of the path; which is 336 * a decimal number. $path/dev contains the device name, which is not 337 * appropriate. 338 */ 339 for (i = strlen(xbusd->xbusd_path); i > 0; i--) { 340 if (xbusd->xbusd_path[i] == '/') 341 break; 342 } 343 if (i == 0) { 344 aprint_error("xbdback: can't parse %s\n", 345 xbusd->xbusd_path); 346 return EFTYPE; 347 } 348 handle = strtoul(&xbusd->xbusd_path[i+1], &ep, 10); 349 if (*ep != '\0') { 350 aprint_error("xbdback: can't parse %s\n", 351 xbusd->xbusd_path); 352 return EFTYPE; 353 } 354 355 xbdi = kmem_zalloc(sizeof(*xbdi), KM_SLEEP); 356 357 xbdi->xbdi_domid = domid; 358 xbdi->xbdi_handle = handle; 359 snprintf(xbdi->xbdi_name, sizeof(xbdi->xbdi_name), "xbdb%di%d", 360 xbdi->xbdi_domid, xbdi->xbdi_handle); 361 362 mutex_enter(&xbdback_lock); 363 if (xbdif_lookup(domid, handle)) { 364 mutex_exit(&xbdback_lock); 365 kmem_free(xbdi, sizeof(*xbdi)); 366 return EEXIST; 367 } 368 SLIST_INSERT_HEAD(&xbdback_instances, xbdi, next); 369 mutex_exit(&xbdback_lock); 370 371 /* initialize status and reference counter */ 372 xbdi->xbdi_status = DISCONNECTED; 373 374 mutex_init(&xbdi->xbdi_lock, MUTEX_DEFAULT, IPL_BIO); 375 cv_init(&xbdi->xbdi_cv, xbdi->xbdi_name); 376 377 mutex_enter(&xbdi->xbdi_lock); 378 xbdi_get(xbdi); 379 mutex_exit(&xbdi->xbdi_lock); 380 381 xbusd->xbusd_u.b.b_cookie = xbdi; 382 xbusd->xbusd_u.b.b_detach = xbdback_xenbus_destroy; 383 xbusd->xbusd_otherend_changed = xbdback_frontend_changed; 384 xbdi->xbdi_xbusd = xbusd; 385 386 SLIST_INIT(&xbdi->xbdi_va_free); 387 for (i = 0; i < BLKIF_RING_SIZE; i++) { 388 xbdi->xbdi_va[i].xv_vaddr = uvm_km_alloc(kernel_map, 389 VBD_VA_SIZE, 0, UVM_KMF_VAONLY|UVM_KMF_WAITVA); 390 SLIST_INSERT_HEAD(&xbdi->xbdi_va_free, &xbdi->xbdi_va[i], 391 xv_next); 392 } 393 394 /* 395 * allocate page-aligned memory for segments, so that for each 396 * xbdback_io its segments are in a single page. 397 * avoid a page boundary within a block of VBD_MAX_INDIRECT_SEGMENTS 398 * segments. 399 */ 400 401 CTASSERT(PAGE_SIZE % (sizeof(struct blkif_request_segment) * VBD_MAX_INDIRECT_SEGMENTS) == 0); 402 CTASSERT(PAGE_SIZE / (sizeof(struct blkif_request_segment) * VBD_MAX_INDIRECT_SEGMENTS) >= 1); 403 404 xbdi->xbdi_segs = (void *)uvm_km_alloc(kernel_map, round_page( 405 sizeof(struct blkif_request_segment) * VBD_MAX_INDIRECT_SEGMENTS * BLKIF_RING_SIZE), 406 PAGE_SIZE, UVM_KMF_WIRED | UVM_KMF_WAITVA); 407 408 SLIST_INIT(&xbdi->xbdi_io_free); 409 for (i = 0; i < BLKIF_RING_SIZE; i++) { 410 struct xbdback_io *xbd_io = &xbdi->xbdi_io[i]; 411 xbd_io->xio_seg = 412 &xbdi->xbdi_segs[i * VBD_MAX_INDIRECT_SEGMENTS]; 413 error = bus_dmamap_create(xbdi->xbdi_xbusd->xbusd_dmat, 414 PAGE_SIZE, 1, PAGE_SIZE, PAGE_SIZE, 415 BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, 416 &xbd_io->xio_seg_dmamap); 417 if (error != 0) { 418 printf("%s: can't create dma map for indirect segments %d\n", 419 xbdi->xbdi_name, i); 420 goto fail; 421 } 422 error = bus_dmamap_load(xbdi->xbdi_xbusd->xbusd_dmat, 423 xbd_io->xio_seg_dmamap, xbd_io->xio_seg, 424 sizeof(struct blkif_request_segment) * VBD_MAX_INDIRECT_SEGMENTS, 425 NULL, BUS_DMA_WAITOK); 426 if (error != 0) { 427 printf("%s: can't load dma map for indirect segments %d @%p (%d, %zu)\n", 428 xbdi->xbdi_name, i, xbd_io->xio_seg, error, sizeof(xbd_io->xio_seg)); 429 bus_dmamap_destroy(xbdi->xbdi_xbusd->xbusd_dmat, 430 xbd_io->xio_seg_dmamap); 431 goto fail; 432 } 433 KASSERT(xbd_io->xio_seg_dmamap->dm_nsegs == 1); 434 segalloc = i; 435 SLIST_INSERT_HEAD(&xbdi->xbdi_io_free, xbd_io, xio_next); 436 } 437 438 error = xenbus_watch_path2(xbusd, xbusd->xbusd_path, "physical-device", 439 &xbdi->xbdi_watch, xbdback_backend_changed); 440 if (error) { 441 printf("failed to watch on %s/physical-device: %d\n", 442 xbusd->xbusd_path, error); 443 goto fail; 444 } 445 xbdi->xbdi_watch.xbw_dev = xbusd; 446 error = xenbus_switch_state(xbusd, NULL, XenbusStateInitWait); 447 if (error) { 448 printf("failed to switch state on %s: %d\n", 449 xbusd->xbusd_path, error); 450 goto fail2; 451 } 452 453 xbdi->xbdi_bouncebuf = uvm_km_alloc(kernel_map, MAXPHYS, PAGE_SIZE, 454 UVM_KMF_WIRED | UVM_KMF_WAITVA); 455 return 0; 456 fail2: 457 unregister_xenbus_watch(&xbdi->xbdi_watch); 458 fail: 459 for (i = 0; i < segalloc; i++) { 460 struct xbdback_io *xbd_io = &xbdi->xbdi_io[i]; 461 bus_dmamap_unload(xbdi->xbdi_xbusd->xbusd_dmat, 462 xbd_io->xio_seg_dmamap); 463 bus_dmamap_destroy(xbdi->xbdi_xbusd->xbusd_dmat, 464 xbd_io->xio_seg_dmamap); 465 } 466 mutex_enter(&xbdback_lock); 467 SLIST_REMOVE(&xbdback_instances, xbdi, xbdback_instance, next); 468 mutex_exit(&xbdback_lock); 469 kmem_free(xbdi, sizeof(*xbdi)); 470 return error; 471 } 472 473 static int 474 xbdback_xenbus_destroy(void *arg) 475 { 476 struct xbdback_instance *xbdi = arg; 477 478 XENPRINTF(("xbdback_xenbus_destroy state %d\n", xbdi->xbdi_status)); 479 480 xbdback_disconnect(xbdi); 481 482 /* unregister watch */ 483 if (xbdi->xbdi_watch.node) 484 xenbus_unwatch_path(&xbdi->xbdi_watch); 485 /* unmap ring */ 486 if (xbdi->xbdi_ring_handle) { 487 xen_shm_unmap(xbdi->xbdi_ring_va, 1, &xbdi->xbdi_ring_handle); 488 } 489 490 if (xbdi->xbdi_ring_va != 0) { 491 uvm_km_free(kernel_map, xbdi->xbdi_ring_va, 492 PAGE_SIZE, UVM_KMF_VAONLY); 493 } 494 495 /* close device */ 496 if (xbdi->xbdi_size) { 497 const char *name; 498 struct dkwedge_info wi; 499 if (getdiskinfo(xbdi->xbdi_vp, &wi) == 0) 500 name = wi.dkw_devname; 501 else 502 name = "*unknown*"; 503 printf("xbd backend: detach device %s for domain %d\n", 504 name, xbdi->xbdi_domid); 505 vn_close(xbdi->xbdi_vp, FREAD, NOCRED); 506 } 507 mutex_enter(&xbdback_lock); 508 SLIST_REMOVE(&xbdback_instances, xbdi, xbdback_instance, next); 509 mutex_exit(&xbdback_lock); 510 511 for (int i = 0; i < BLKIF_RING_SIZE; i++) { 512 struct xbdback_io *xbd_io = &xbdi->xbdi_io[i]; 513 bus_dmamap_unload(xbdi->xbdi_xbusd->xbusd_dmat, 514 xbd_io->xio_seg_dmamap); 515 bus_dmamap_destroy(xbdi->xbdi_xbusd->xbusd_dmat, 516 xbd_io->xio_seg_dmamap); 517 if (xbdi->xbdi_va[i].xv_vaddr != 0) { 518 uvm_km_free(kernel_map, xbdi->xbdi_va[i].xv_vaddr, 519 VBD_VA_SIZE, UVM_KMF_VAONLY); 520 xbdi->xbdi_va[i].xv_vaddr = 0; 521 } 522 } 523 524 525 mutex_destroy(&xbdi->xbdi_lock); 526 cv_destroy(&xbdi->xbdi_cv); 527 kmem_free(xbdi, sizeof(*xbdi)); 528 return 0; 529 } 530 531 static int 532 xbdback_connect(struct xbdback_instance *xbdi) 533 { 534 int err; 535 evtchn_op_t evop; 536 grant_ref_t gring_ref; 537 u_long ring_ref, revtchn; 538 char xsproto[32]; 539 const char *proto; 540 struct xenbus_device *xbusd = xbdi->xbdi_xbusd; 541 542 XENPRINTF(("xbdback %s: connect\n", xbusd->xbusd_path)); 543 /* read communication informations */ 544 err = xenbus_read_ul(NULL, xbusd->xbusd_otherend, 545 "ring-ref", &ring_ref, 10); 546 if (err) { 547 xenbus_dev_fatal(xbusd, err, "reading %s/ring-ref", 548 xbusd->xbusd_otherend); 549 return -1; 550 } 551 XENPRINTF(("xbdback %s: connect ring-ref %lu\n", xbusd->xbusd_path, ring_ref)); 552 err = xenbus_read_ul(NULL, xbusd->xbusd_otherend, 553 "event-channel", &revtchn, 10); 554 if (err) { 555 xenbus_dev_fatal(xbusd, err, "reading %s/event-channel", 556 xbusd->xbusd_otherend); 557 return -1; 558 } 559 XENPRINTF(("xbdback %s: connect revtchn %lu\n", xbusd->xbusd_path, revtchn)); 560 err = xenbus_read(NULL, xbusd->xbusd_otherend, "protocol", 561 xsproto, sizeof(xsproto)); 562 if (err) { 563 xbdi->xbdi_proto = XBDIP_NATIVE; 564 proto = "unspecified"; 565 XENPRINTF(("xbdback %s: connect no xsproto\n", xbusd->xbusd_path)); 566 } else { 567 XENPRINTF(("xbdback %s: connect xsproto %s\n", xbusd->xbusd_path, xsproto)); 568 if (strcmp(xsproto, XEN_IO_PROTO_ABI_NATIVE) == 0) { 569 xbdi->xbdi_proto = XBDIP_NATIVE; 570 proto = XEN_IO_PROTO_ABI_NATIVE; 571 } else if (strcmp(xsproto, XEN_IO_PROTO_ABI_X86_32) == 0) { 572 xbdi->xbdi_proto = XBDIP_32; 573 proto = XEN_IO_PROTO_ABI_X86_32; 574 } else if (strcmp(xsproto, XEN_IO_PROTO_ABI_X86_64) == 0) { 575 xbdi->xbdi_proto = XBDIP_64; 576 proto = XEN_IO_PROTO_ABI_X86_64; 577 } else { 578 printf("xbd domain %d: unknown proto %s\n", 579 xbdi->xbdi_domid, xsproto); 580 return -1; 581 } 582 } 583 584 /* allocate VA space and map rings */ 585 xbdi->xbdi_ring_va = uvm_km_alloc(kernel_map, PAGE_SIZE, 0, 586 UVM_KMF_VAONLY); 587 if (xbdi->xbdi_ring_va == 0) { 588 xenbus_dev_fatal(xbusd, ENOMEM, 589 "can't get VA for ring", xbusd->xbusd_otherend); 590 return -1; 591 } 592 XENPRINTF(("xbdback %s: connect va 0x%" PRIxVADDR "\n", xbusd->xbusd_path, xbdi->xbdi_ring_va)); 593 594 gring_ref = ring_ref; 595 if (xen_shm_map(1, xbdi->xbdi_domid, &gring_ref, xbdi->xbdi_ring_va, 596 &xbdi->xbdi_ring_handle, 0) != 0) { 597 printf("xbdback %s: can't map grant ref\n", 598 xbusd->xbusd_path); 599 xenbus_dev_fatal(xbusd, EINVAL, 600 "can't map ring", xbusd->xbusd_otherend); 601 goto err1; 602 } 603 XENPRINTF(("xbdback %s: connect grhandle %d\n", xbusd->xbusd_path, xbdi->xbdi_ring_handle)); 604 605 switch(xbdi->xbdi_proto) { 606 case XBDIP_NATIVE: 607 { 608 blkif_sring_t *sring = (void *)xbdi->xbdi_ring_va; 609 BACK_RING_INIT(&xbdi->xbdi_ring.ring_n, sring, PAGE_SIZE); 610 break; 611 } 612 case XBDIP_32: 613 { 614 blkif_x86_32_sring_t *sring = (void *)xbdi->xbdi_ring_va; 615 BACK_RING_INIT(&xbdi->xbdi_ring.ring_32, sring, PAGE_SIZE); 616 break; 617 } 618 case XBDIP_64: 619 { 620 blkif_x86_64_sring_t *sring = (void *)xbdi->xbdi_ring_va; 621 BACK_RING_INIT(&xbdi->xbdi_ring.ring_64, sring, PAGE_SIZE); 622 break; 623 } 624 } 625 626 evop.cmd = EVTCHNOP_bind_interdomain; 627 evop.u.bind_interdomain.remote_dom = xbdi->xbdi_domid; 628 evop.u.bind_interdomain.remote_port = revtchn; 629 err = HYPERVISOR_event_channel_op(&evop); 630 if (err) { 631 printf("blkback %s: " 632 "can't get event channel: %d\n", 633 xbusd->xbusd_otherend, err); 634 xenbus_dev_fatal(xbusd, err, 635 "can't bind event channel", xbusd->xbusd_otherend); 636 goto err2; 637 } 638 xbdi->xbdi_evtchn = evop.u.bind_interdomain.local_port; 639 XENPRINTF(("xbdback %s: connect evchannel %d\n", xbusd->xbusd_path, xbdi->xbdi_evtchn)); 640 641 xbdi->xbdi_ih = xen_intr_establish_xname(-1, &xen_pic, 642 xbdi->xbdi_evtchn, IST_LEVEL, IPL_BIO, xbdback_evthandler, xbdi, 643 true, xbdi->xbdi_name); 644 KASSERT(xbdi->xbdi_ih != NULL); 645 printf("xbd backend domain %d handle %#x (%d) " 646 "using event channel %d, protocol %s\n", xbdi->xbdi_domid, 647 xbdi->xbdi_handle, xbdi->xbdi_handle, xbdi->xbdi_evtchn, proto); 648 649 /* enable the xbdback event handler machinery */ 650 xbdi->xbdi_status = WAITING; 651 hypervisor_unmask_event(xbdi->xbdi_evtchn); 652 hypervisor_notify_via_evtchn(xbdi->xbdi_evtchn); 653 654 if (kthread_create(PRI_NONE, KTHREAD_MPSAFE, NULL, 655 xbdback_thread, xbdi, NULL, "%s", xbdi->xbdi_name) == 0) 656 return 0; 657 658 err2: 659 /* unmap ring */ 660 xen_shm_unmap(xbdi->xbdi_ring_va, 1, &xbdi->xbdi_ring_handle); 661 err1: 662 /* free ring VA space */ 663 uvm_km_free(kernel_map, xbdi->xbdi_ring_va, PAGE_SIZE, UVM_KMF_VAONLY); 664 return -1; 665 } 666 667 /* 668 * Signal a xbdback thread to disconnect. Done in 'xenwatch' thread context. 669 */ 670 static void 671 xbdback_disconnect(struct xbdback_instance *xbdi) 672 { 673 674 mutex_enter(&xbdi->xbdi_lock); 675 if (xbdi->xbdi_status == DISCONNECTED) { 676 mutex_exit(&xbdi->xbdi_lock); 677 return; 678 } 679 hypervisor_mask_event(xbdi->xbdi_evtchn); 680 681 /* signal thread that we want to disconnect, then wait for it */ 682 xbdi->xbdi_status = DISCONNECTING; 683 cv_signal(&xbdi->xbdi_cv); 684 685 while (xbdi->xbdi_status != DISCONNECTED) 686 cv_wait(&xbdi->xbdi_cv, &xbdi->xbdi_lock); 687 688 mutex_exit(&xbdi->xbdi_lock); 689 xen_intr_disestablish(xbdi->xbdi_ih); 690 691 xenbus_switch_state(xbdi->xbdi_xbusd, NULL, XenbusStateClosing); 692 } 693 694 static void 695 xbdback_frontend_changed(void *arg, XenbusState new_state) 696 { 697 struct xbdback_instance *xbdi = arg; 698 struct xenbus_device *xbusd = xbdi->xbdi_xbusd; 699 700 XENPRINTF(("xbdback %s: new state %d\n", xbusd->xbusd_path, new_state)); 701 switch(new_state) { 702 case XenbusStateInitialising: 703 break; 704 case XenbusStateInitialised: 705 case XenbusStateConnected: 706 if (xbdi->xbdi_status == WAITING || xbdi->xbdi_status == RUN) 707 break; 708 xbdback_connect(xbdi); 709 break; 710 case XenbusStateClosing: 711 xbdback_disconnect(xbdi); 712 break; 713 case XenbusStateClosed: 714 /* otherend_changed() should handle it for us */ 715 panic("xbdback_frontend_changed: closed\n"); 716 case XenbusStateUnknown: 717 case XenbusStateInitWait: 718 default: 719 printf("xbdback %s: invalid frontend state %d\n", 720 xbusd->xbusd_path, new_state); 721 } 722 return; 723 } 724 725 static void 726 xbdback_backend_changed(struct xenbus_watch *watch, 727 const char **vec, unsigned int len) 728 { 729 struct xenbus_device *xbusd = watch->xbw_dev; 730 struct xbdback_instance *xbdi = xbusd->xbusd_u.b.b_cookie; 731 int err; 732 long dev; 733 char mode[32]; 734 struct xenbus_transaction *xbt; 735 const char *devname; 736 int major; 737 738 err = xenbus_read_ul(NULL, xbusd->xbusd_path, "physical-device", 739 &dev, 10); 740 /* 741 * An error can occur as the watch can fire up just after being 742 * registered. So we have to ignore error :( 743 */ 744 if (err) 745 return; 746 /* 747 * we can also fire up after having opened the device, don't try 748 * to do it twice. 749 */ 750 if (xbdi->xbdi_vp != NULL) { 751 if (xbdi->xbdi_status == WAITING || xbdi->xbdi_status == RUN) { 752 if (xbdi->xbdi_dev != dev) { 753 printf("xbdback %s: changing physical device " 754 "from %#"PRIx64" to %#lx not supported\n", 755 xbusd->xbusd_path, xbdi->xbdi_dev, dev); 756 } 757 } 758 return; 759 } 760 xbdi->xbdi_dev = dev; 761 err = xenbus_read(NULL, xbusd->xbusd_path, "mode", mode, sizeof(mode)); 762 if (err) { 763 printf("xbdback: failed to read %s/mode: %d\n", 764 xbusd->xbusd_path, err); 765 return; 766 } 767 if (mode[0] == 'w') 768 xbdi->xbdi_ro = false; 769 else 770 xbdi->xbdi_ro = true; 771 major = major(xbdi->xbdi_dev); 772 devname = devsw_blk2name(major); 773 if (devname == NULL) { 774 printf("xbdback %s: unknown device 0x%"PRIx64"\n", 775 xbusd->xbusd_path, xbdi->xbdi_dev); 776 return; 777 } 778 xbdi->xbdi_bdevsw = bdevsw_lookup(xbdi->xbdi_dev); 779 if (xbdi->xbdi_bdevsw == NULL) { 780 printf("xbdback %s: no bdevsw for device 0x%"PRIx64"\n", 781 xbusd->xbusd_path, xbdi->xbdi_dev); 782 return; 783 } 784 err = bdevvp(xbdi->xbdi_dev, &xbdi->xbdi_vp); 785 if (err) { 786 printf("xbdback %s: can't open device 0x%"PRIx64": %d\n", 787 xbusd->xbusd_path, xbdi->xbdi_dev, err); 788 return; 789 } 790 err = vn_lock(xbdi->xbdi_vp, LK_EXCLUSIVE | LK_RETRY); 791 if (err) { 792 printf("xbdback %s: can't vn_lock device 0x%"PRIx64": %d\n", 793 xbusd->xbusd_path, xbdi->xbdi_dev, err); 794 vrele(xbdi->xbdi_vp); 795 return; 796 } 797 err = VOP_OPEN(xbdi->xbdi_vp, FREAD, NOCRED); 798 if (err) { 799 printf("xbdback %s: can't VOP_OPEN device 0x%"PRIx64": %d\n", 800 xbusd->xbusd_path, xbdi->xbdi_dev, err); 801 vput(xbdi->xbdi_vp); 802 return; 803 } 804 VOP_UNLOCK(xbdi->xbdi_vp); 805 806 struct dkwedge_info wi; 807 if (getdiskinfo(xbdi->xbdi_vp, &wi) == 0) 808 devname = wi.dkw_devname; 809 810 err = getdisksize(xbdi->xbdi_vp, &xbdi->xbdi_size, &xbdi->xbdi_secsize); 811 if (err == 0) { 812 printf("xbd backend: attach device %s" 813 " (size %" PRIu64 " x %u) for domain %d\n", 814 devname, xbdi->xbdi_size, 815 xbdi->xbdi_secsize, xbdi->xbdi_domid); 816 } else { 817 /* If Ioctls failed set device size to 0 and return */ 818 printf("xbdback %s: can't get size of device " 819 "0x%"PRIx64": %d\n", xbusd->xbusd_path, 820 xbdi->xbdi_dev, err); 821 xbdi->xbdi_size = xbdi->xbdi_dev = 0; 822 vn_close(xbdi->xbdi_vp, FREAD, NOCRED); 823 xbdi->xbdi_vp = NULL; 824 return; 825 } 826 again: 827 xbt = xenbus_transaction_start(); 828 if (xbt == NULL) { 829 printf("xbdback %s: can't start transaction\n", 830 xbusd->xbusd_path); 831 return; 832 } 833 err = xenbus_printf(xbt, xbusd->xbusd_path, "sectors", "%" PRIu64 , 834 xbdi->xbdi_size * xbdi->xbdi_secsize / VBD_BSIZE); 835 if (err) { 836 printf("xbdback: failed to write %s/sectors: %d\n", 837 xbusd->xbusd_path, err); 838 goto abort; 839 } 840 err = xenbus_printf(xbt, xbusd->xbusd_path, "info", "%u", 841 xbdi->xbdi_ro ? VDISK_READONLY : 0); 842 if (err) { 843 printf("xbdback: failed to write %s/info: %d\n", 844 xbusd->xbusd_path, err); 845 goto abort; 846 } 847 848 /* 849 * don't pass true sector size as defined by spec but 850 * constant 512 bytes. 851 * 1. doesn't confuse guests that cannot handle large sectors 852 * 2. matches the disk-size above 853 * 3. what Linux does 854 * 4. guest needs to take care about alignment on its own 855 */ 856 err = xenbus_printf(xbt, xbusd->xbusd_path, "sector-size", "%lu", 857 (u_long)VBD_BSIZE); 858 if (err) { 859 printf("xbdback: failed to write %s/sector-size: %d\n", 860 xbusd->xbusd_path, err); 861 goto abort; 862 } 863 err = xenbus_printf(xbt, xbusd->xbusd_path, "feature-flush-cache", 864 "%u", 1); 865 if (err) { 866 printf("xbdback: failed to write %s/feature-flush-cache: %d\n", 867 xbusd->xbusd_path, err); 868 goto abort; 869 } 870 err = xenbus_printf(xbt, xbusd->xbusd_path, 871 "feature-max-indirect-segments", "%u", VBD_MAX_INDIRECT_SEGMENTS); 872 if (err) { 873 printf("xbdback: failed to write %s/feature-indirect: %d\n", 874 xbusd->xbusd_path, err); 875 goto abort; 876 } 877 err = xenbus_transaction_end(xbt, 0); 878 if (err == EAGAIN) 879 goto again; 880 if (err) { 881 printf("xbdback %s: can't end transaction: %d\n", 882 xbusd->xbusd_path, err); 883 } 884 err = xenbus_switch_state(xbusd, NULL, XenbusStateConnected); 885 if (err) { 886 printf("xbdback %s: can't switch state: %d\n", 887 xbusd->xbusd_path, err); 888 } 889 return; 890 abort: 891 xenbus_transaction_end(xbt, 1); 892 } 893 894 /* 895 * Used by a xbdi thread to signal that it is now disconnected. 896 */ 897 static void 898 xbdback_finish_disconnect(struct xbdback_instance *xbdi) 899 { 900 KASSERT(mutex_owned(&xbdi->xbdi_lock)); 901 KASSERT(xbdi->xbdi_status == DISCONNECTING); 902 903 xbdi->xbdi_status = DISCONNECTED; 904 905 cv_broadcast(&xbdi->xbdi_cv); 906 } 907 908 static bool 909 xbdif_lookup(domid_t dom , uint32_t handle) 910 { 911 struct xbdback_instance *xbdi; 912 bool found = false; 913 914 KASSERT(mutex_owned(&xbdback_lock)); 915 916 SLIST_FOREACH(xbdi, &xbdback_instances, next) { 917 if (xbdi->xbdi_domid == dom && xbdi->xbdi_handle == handle) { 918 found = true; 919 break; 920 } 921 } 922 923 return found; 924 } 925 926 static int 927 xbdback_evthandler(void *arg) 928 { 929 struct xbdback_instance *xbdi = arg; 930 931 XENPRINTF(("xbdback_evthandler domain %d: cont %p\n", 932 xbdi->xbdi_domid, xbdi->xbdi_cont)); 933 934 mutex_enter(&xbdi->xbdi_lock); 935 xbdback_wakeup_thread(xbdi); 936 mutex_exit(&xbdi->xbdi_lock); 937 938 return 1; 939 } 940 941 /* 942 * Main thread routine for one xbdback instance. Woken up by 943 * xbdback_evthandler when a domain has I/O work scheduled in a I/O ring. 944 */ 945 static void 946 xbdback_thread(void *arg) 947 { 948 struct xbdback_instance *xbdi = arg; 949 void *obj; 950 951 mutex_enter(&xbdi->xbdi_lock); 952 for (;;) { 953 switch (xbdi->xbdi_status) { 954 case WAITING: 955 cv_wait(&xbdi->xbdi_cv, &xbdi->xbdi_lock); 956 break; 957 case RUN: 958 xbdi->xbdi_status = WAITING; /* reset state */ 959 obj = xbdi; 960 if (xbdi->xbdi_cont_restart != NULL) { 961 KASSERT(xbdi->xbdi_cont == NULL); 962 xbdi->xbdi_cont = xbdi->xbdi_cont_restart; 963 obj = xbdi->xbdi_cont_restart_obj; 964 xbdi->xbdi_cont_restart = NULL; 965 xbdi->xbdi_cont_restart_obj = NULL; 966 } 967 if (xbdi->xbdi_cont == NULL) { 968 xbdi->xbdi_cont = xbdback_co_main; 969 } 970 971 xbdback_trampoline(xbdi, obj); 972 break; 973 case DISCONNECTING: 974 if (xbdi->xbdi_pendingreqs > 0) { 975 /* there are pending I/Os. Wait for them. */ 976 cv_wait(&xbdi->xbdi_cv, &xbdi->xbdi_lock); 977 continue; 978 } 979 980 /* All I/Os should have been processed by now, 981 * xbdi_refcnt should drop to 0 */ 982 xbdi_put(xbdi); 983 KASSERT(xbdi->xbdi_refcnt == 0); 984 goto out; 985 /* NOTREACHED */ 986 default: 987 panic("%s: invalid state %d", 988 xbdi->xbdi_name, xbdi->xbdi_status); 989 } 990 } 991 out: 992 mutex_exit(&xbdi->xbdi_lock); 993 994 kthread_exit(0); 995 } 996 997 static void * 998 xbdback_co_main(struct xbdback_instance *xbdi, void *obj) 999 { 1000 (void)obj; 1001 1002 xbdi->xbdi_req_prod = xbdi->xbdi_ring.ring_n.sring->req_prod; 1003 xen_rmb(); /* ensure we see all requests up to req_prod */ 1004 /* 1005 * note that we'll eventually get a full ring of request. 1006 * in this case, MASK_BLKIF_IDX(req_cons) == MASK_BLKIF_IDX(req_prod) 1007 */ 1008 xbdi->xbdi_cont = xbdback_co_main_loop; 1009 return xbdi; 1010 } 1011 1012 /* 1013 * Fetch a blkif request from the ring, and pass control to the appropriate 1014 * continuation. 1015 * If someone asked for disconnection, do not fetch any more request from 1016 * the ring. 1017 */ 1018 static void * 1019 xbdback_co_main_loop(struct xbdback_instance *xbdi, void *obj __unused) 1020 { 1021 blkif_request_t *req, *reqn; 1022 blkif_x86_32_request_t *req32; 1023 blkif_x86_64_request_t *req64; 1024 blkif_request_indirect_t *rinn; 1025 blkif_x86_32_request_indirect_t *rin32; 1026 blkif_x86_64_request_indirect_t *rin64; 1027 1028 if (xbdi->xbdi_ring.ring_n.req_cons != xbdi->xbdi_req_prod) { 1029 struct xbdback_io *xbd_io = xbdback_io_get(xbdi); 1030 uint8_t real_op = 0xff; 1031 1032 if (xbd_io == NULL) { 1033 /* retry after iodone */ 1034 xbdi->xbdi_cont = NULL; 1035 return NULL; 1036 } 1037 memset(&xbd_io->u, 0, sizeof(xbd_io->u)); 1038 1039 buf_init(&xbd_io->xio_buf); 1040 xbd_io->xio_xbdi = xbdi; 1041 1042 req = &xbd_io->xio_xen_req; 1043 memset(req, 0, sizeof(*req)); 1044 1045 switch(xbdi->xbdi_proto) { 1046 case XBDIP_NATIVE: 1047 reqn = RING_GET_REQUEST(&xbdi->xbdi_ring.ring_n, 1048 xbdi->xbdi_ring.ring_n.req_cons); 1049 real_op = req->operation = reqn->operation; 1050 if (real_op == BLKIF_OP_INDIRECT) { 1051 rinn = (blkif_request_indirect_t *)reqn; 1052 real_op = rinn->indirect_op; 1053 } 1054 req->id = reqn->id; 1055 break; 1056 case XBDIP_32: 1057 req32 = RING_GET_REQUEST(&xbdi->xbdi_ring.ring_32, 1058 xbdi->xbdi_ring.ring_n.req_cons); 1059 real_op = req->operation = req32->operation; 1060 if (real_op == BLKIF_OP_INDIRECT) { 1061 rin32 = (blkif_x86_32_request_indirect_t*)req32; 1062 real_op = rin32->indirect_op; 1063 } 1064 req->id = req32->id; 1065 break; 1066 case XBDIP_64: 1067 req64 = RING_GET_REQUEST(&xbdi->xbdi_ring.ring_64, 1068 xbdi->xbdi_ring.ring_n.req_cons); 1069 real_op = req->operation = req64->operation; 1070 if (real_op == BLKIF_OP_INDIRECT) { 1071 rin64 = (blkif_x86_64_request_indirect_t*)req64; 1072 real_op = rin64->indirect_op; 1073 } 1074 req->id = req64->id; 1075 break; 1076 } 1077 __insn_barrier(); 1078 XENPRINTF(("xbdback op %d req_cons 0x%x req_prod 0x%x " 1079 "resp_prod 0x%x id %" PRIu64 "\n", req->operation, 1080 xbdi->xbdi_ring.ring_n.req_cons, 1081 xbdi->xbdi_req_prod, 1082 xbdi->xbdi_ring.ring_n.rsp_prod_pvt, 1083 req->id)); 1084 switch (req->operation) { 1085 case BLKIF_OP_INDIRECT: 1086 /* just check indirect_op, rest is handled later */ 1087 if (real_op != BLKIF_OP_READ && 1088 real_op != BLKIF_OP_WRITE) { 1089 if (ratecheck(&xbdi->xbdi_lasterr_time, 1090 &xbdback_err_intvl)) { 1091 printf("%s: unknown ind operation %d\n", 1092 xbdi->xbdi_name, 1093 real_op); 1094 } 1095 goto fail; 1096 } 1097 /* FALLTHROUGH */ 1098 case BLKIF_OP_READ: 1099 case BLKIF_OP_WRITE: 1100 xbdi->xbdi_cont = xbdback_co_io; 1101 return xbd_io; 1102 case BLKIF_OP_FLUSH_DISKCACHE: 1103 xbdi->xbdi_cont = xbdback_co_cache_flush; 1104 return xbd_io; 1105 default: 1106 if (ratecheck(&xbdi->xbdi_lasterr_time, 1107 &xbdback_err_intvl)) { 1108 printf("%s: unknown operation %d\n", 1109 xbdi->xbdi_name, req->operation); 1110 } 1111 fail: 1112 xbdback_send_reply(xbdi, req->id, real_op, 1113 BLKIF_RSP_ERROR); 1114 xbdi->xbdi_cont = xbdback_co_main_incr; 1115 return xbdi; 1116 } 1117 } else { 1118 xbdi->xbdi_cont = xbdback_co_main_done2; 1119 return xbdi; 1120 } 1121 } 1122 1123 /* 1124 * Increment consumer index and move on to the next request. In case 1125 * we want to disconnect, leave continuation now. 1126 */ 1127 static void * 1128 xbdback_co_main_incr(struct xbdback_instance *xbdi, void *obj __unused) 1129 { 1130 KASSERT(mutex_owned(&xbdi->xbdi_lock)); 1131 1132 blkif_back_ring_t *ring = &xbdi->xbdi_ring.ring_n; 1133 1134 ring->req_cons++; 1135 1136 if (xbdi->xbdi_status == DISCONNECTING) 1137 xbdi->xbdi_cont = NULL; 1138 else 1139 xbdi->xbdi_cont = xbdback_co_main_loop; 1140 1141 return xbdi; 1142 } 1143 1144 /* 1145 * Check for requests in the instance's ring. In case there are, start again 1146 * from the beginning. If not, stall. 1147 */ 1148 static void * 1149 xbdback_co_main_done2(struct xbdback_instance *xbdi, void *obj) 1150 { 1151 int work_to_do; 1152 1153 xen_wmb(); 1154 RING_FINAL_CHECK_FOR_REQUESTS(&xbdi->xbdi_ring.ring_n, work_to_do); 1155 if (work_to_do) 1156 xbdi->xbdi_cont = xbdback_co_main; 1157 else 1158 xbdi->xbdi_cont = NULL; 1159 1160 return xbdi; 1161 } 1162 1163 /* 1164 * Frontend requested a cache flush operation. 1165 */ 1166 static void * 1167 xbdback_co_cache_flush(struct xbdback_instance *xbdi, void *obj) 1168 { 1169 struct xbdback_io *xbd_io = obj; 1170 KASSERT(xbd_io->xio_xen_req.operation == BLKIF_OP_FLUSH_DISKCACHE); 1171 if (xbdi->xbdi_pendingreqs > 0) { 1172 /* 1173 * There are pending requests. 1174 * Event or iodone() will restart processing 1175 */ 1176 xbdi->xbdi_cont_restart = xbdback_co_cache_flush; 1177 xbdi->xbdi_cont_restart_obj = xbd_io; 1178 xbdi->xbdi_cont = NULL; 1179 return NULL; 1180 } 1181 xbdi_get(xbdi); 1182 xbdi->xbdi_cont = xbdback_co_do_io; 1183 return xbd_io; 1184 } 1185 1186 /* 1187 * A read or write I/O request must be processed. Do some checks first, 1188 * then get the segment information directly from the ring request. 1189 */ 1190 static void * 1191 xbdback_co_io(struct xbdback_instance *xbdi, void *obj) 1192 { 1193 int i, error; 1194 blkif_request_t *req, *reqn; 1195 blkif_x86_32_request_t *req32; 1196 blkif_x86_64_request_t *req64; 1197 blkif_request_indirect_t *rinn; 1198 blkif_x86_32_request_indirect_t *rin32; 1199 blkif_x86_64_request_indirect_t *rin64; 1200 const char *errstr; 1201 struct xbdback_io *xbd_io = obj; 1202 grant_ref_t in_gntref = 0; 1203 1204 req = &xbd_io->xio_xen_req; 1205 1206 /* some sanity checks */ 1207 KASSERT(req->operation == BLKIF_OP_READ || 1208 req->operation == BLKIF_OP_WRITE || 1209 req->operation == BLKIF_OP_INDIRECT); 1210 1211 /* copy request segments */ 1212 switch (xbdi->xbdi_proto) { 1213 case XBDIP_NATIVE: 1214 reqn = RING_GET_REQUEST(&xbdi->xbdi_ring.ring_n, 1215 xbdi->xbdi_ring.ring_n.req_cons); 1216 req->handle = reqn->handle; 1217 req->sector_number = reqn->sector_number; 1218 if (reqn->operation == BLKIF_OP_INDIRECT) { 1219 rinn = (blkif_request_indirect_t *)reqn; 1220 req->operation = rinn->indirect_op; 1221 req->nr_segments = (uint8_t)rinn->nr_segments; 1222 if (req->nr_segments > VBD_MAX_INDIRECT_SEGMENTS) { 1223 errstr = "too many indirect segments"; 1224 goto bad_segments; 1225 } 1226 in_gntref = rinn->indirect_grefs[0]; 1227 /* first_sect and segment grefs fetched later */ 1228 } else { 1229 req->nr_segments = reqn->nr_segments; 1230 if (req->nr_segments > BLKIF_MAX_SEGMENTS_PER_REQUEST) { 1231 errstr = "too many segments"; 1232 goto bad_segments; 1233 } 1234 for (i = 0; i < req->nr_segments; i++) 1235 xbd_io->xio_seg[i] = reqn->seg[i]; 1236 } 1237 break; 1238 case XBDIP_32: 1239 req32 = RING_GET_REQUEST(&xbdi->xbdi_ring.ring_32, 1240 xbdi->xbdi_ring.ring_n.req_cons); 1241 req->handle = req32->handle; 1242 req->sector_number = req32->sector_number; 1243 if (req32->operation == BLKIF_OP_INDIRECT) { 1244 rin32 = (blkif_x86_32_request_indirect_t *)req32; 1245 req->operation = rin32->indirect_op; 1246 req->nr_segments = (uint8_t)rin32->nr_segments; 1247 if (req->nr_segments > VBD_MAX_INDIRECT_SEGMENTS) { 1248 errstr = "too many indirect segments"; 1249 goto bad_segments; 1250 } 1251 in_gntref = rin32->indirect_grefs[0]; 1252 /* first_sect and segment grefs fetched later */ 1253 } else { 1254 req->nr_segments = req32->nr_segments; 1255 if (req->nr_segments > BLKIF_MAX_SEGMENTS_PER_REQUEST) { 1256 errstr = "too many segments"; 1257 goto bad_segments; 1258 } 1259 for (i = 0; i < req->nr_segments; i++) 1260 xbd_io->xio_seg[i] = req32->seg[i]; 1261 } 1262 break; 1263 case XBDIP_64: 1264 req64 = RING_GET_REQUEST(&xbdi->xbdi_ring.ring_64, 1265 xbdi->xbdi_ring.ring_n.req_cons); 1266 req->handle = req64->handle; 1267 req->sector_number = req64->sector_number; 1268 if (req64->operation == BLKIF_OP_INDIRECT) { 1269 rin64 = (blkif_x86_64_request_indirect_t *)req64; 1270 req->nr_segments = (uint8_t)rin64->nr_segments; 1271 if (req->nr_segments > VBD_MAX_INDIRECT_SEGMENTS) { 1272 errstr = "too many indirect segments"; 1273 goto bad_segments; 1274 } 1275 in_gntref = rin64->indirect_grefs[0]; 1276 /* first_sect and segment grefs fetched later */ 1277 } else { 1278 req->nr_segments = req64->nr_segments; 1279 if (req->nr_segments > BLKIF_MAX_SEGMENTS_PER_REQUEST) { 1280 errstr = "too many segments"; 1281 goto bad_segments; 1282 } 1283 for (i = 0; i < req->nr_segments; i++) 1284 xbd_io->xio_seg[i] = req64->seg[i]; 1285 } 1286 break; 1287 } 1288 1289 if (req->operation == BLKIF_OP_WRITE) { 1290 if (xbdi->xbdi_ro) { 1291 error = EROFS; 1292 goto end; 1293 } 1294 } 1295 1296 /* Max value checked already earlier */ 1297 if (req->nr_segments < 1) { 1298 errstr = "invalid number of segments"; 1299 goto bad_segments; 1300 } 1301 1302 /* If segments are on an indirect page, copy them now */ 1303 if (in_gntref) { 1304 gnttab_copy_t gop; 1305 paddr_t ma; 1306 1307 gop.flags = GNTCOPY_source_gref; 1308 gop.len = req->nr_segments 1309 * sizeof(struct blkif_request_segment); 1310 1311 gop.source.u.ref = in_gntref; 1312 gop.source.offset = 0; 1313 gop.source.domid = xbdi->xbdi_domid; 1314 1315 ma = xbd_io->xio_seg_dmamap->dm_segs[0].ds_addr; 1316 gop.dest.offset = ma & PAGE_MASK; 1317 gop.dest.domid = DOMID_SELF; 1318 gop.dest.u.gmfn = ma >> PAGE_SHIFT; 1319 1320 if (HYPERVISOR_grant_table_op(GNTTABOP_copy, &gop, 1) != 0) { 1321 errstr = "GNTTABOP_copy failed"; 1322 goto bad_segments; 1323 } 1324 } 1325 1326 xbdi_get(xbdi); 1327 xbdi->xbdi_cont = xbdback_co_io_gotio; 1328 return xbd_io; 1329 1330 bad_segments: 1331 if (ratecheck(&xbdi->xbdi_lasterr_time, &xbdback_err_intvl)) { 1332 printf("%s: %s\n", xbdi->xbdi_name, errstr); 1333 } 1334 error = EINVAL; 1335 /* FALLTHROUGH */ 1336 1337 end: 1338 xbdback_send_reply(xbdi, req->id, req->operation, 1339 (error == EROFS) ? BLKIF_RSP_EOPNOTSUPP : BLKIF_RSP_ERROR); 1340 xbdi->xbdi_cont = xbdback_co_main_incr; 1341 return xbdi; 1342 } 1343 1344 /* Prepare an I/O buffer for a xbdback instance */ 1345 static void * 1346 xbdback_co_io_gotio(struct xbdback_instance *xbdi, void *obj) 1347 { 1348 struct xbdback_io *xbd_io = obj; 1349 int buf_flags; 1350 size_t bcount; 1351 blkif_request_t *req = &xbd_io->xio_xen_req; 1352 uint8_t last_sect; 1353 int error; 1354 1355 KASSERT(mutex_owned(&xbdi->xbdi_lock)); 1356 KASSERT(xbdi->xbdi_refcnt > 0); 1357 1358 /* Process segments */ 1359 bcount = 0; 1360 for (int i = 0; i < req->nr_segments; i++) { 1361 struct blkif_request_segment *seg = &xbd_io->xio_seg[i]; 1362 if (seg->last_sect > VBD_MAXSECT || 1363 seg->first_sect > VBD_MAXSECT) { 1364 if (ratecheck(&xbdi->xbdi_lasterr_time, 1365 &xbdback_err_intvl)) { 1366 printf("%s: invalid segment sectors %d %d\n", 1367 xbdi->xbdi_name, 1368 seg->first_sect, seg->last_sect); 1369 } 1370 xbdi->xbdi_pendingreqs++; /* xbdback_io_error will -- */ 1371 xbd_io->xio_need_bounce = 0; /* bouncebuf not setup yet */ 1372 xbdback_io_error(xbd_io, EINVAL); 1373 /* do not retry */ 1374 xbdi->xbdi_cont = xbdback_co_main_incr; 1375 return xbdi; 1376 } 1377 1378 if (i > 0) { 1379 if (last_sect != VBD_MAXSECT || 1380 seg->first_sect != 0) { 1381 xbd_io->xio_need_bounce = 1; 1382 } 1383 } 1384 last_sect = seg->last_sect; 1385 xbd_io->xio_gref[i] = seg->gref; 1386 bcount += (seg->last_sect - seg->first_sect + 1) 1387 * VBD_BSIZE; 1388 } 1389 xbd_io->xio_start_offset = xbd_io->xio_seg[0].first_sect * VBD_BSIZE; 1390 1391 KASSERT(bcount <= MAXPHYS); 1392 KASSERT(xbd_io->xio_start_offset < PAGE_SIZE); 1393 KASSERT(bcount + xbd_io->xio_start_offset <= VBD_VA_SIZE); 1394 1395 /* Fill-in the buf */ 1396 if (req->operation == BLKIF_OP_WRITE) { 1397 buf_flags = B_WRITE; 1398 } else { 1399 buf_flags = B_READ; 1400 } 1401 1402 xbd_io->xio_buf.b_flags = buf_flags; 1403 xbd_io->xio_buf.b_cflags = 0; 1404 xbd_io->xio_buf.b_oflags = 0; 1405 xbd_io->xio_buf.b_iodone = xbdback_iodone; 1406 xbd_io->xio_buf.b_proc = NULL; 1407 xbd_io->xio_buf.b_vp = xbdi->xbdi_vp; 1408 xbd_io->xio_buf.b_objlock = xbdi->xbdi_vp->v_interlock; 1409 xbd_io->xio_buf.b_dev = xbdi->xbdi_dev; 1410 xbd_io->xio_buf.b_blkno = req->sector_number; 1411 xbd_io->xio_buf.b_bcount = bcount; 1412 if (__predict_false(xbd_io->xio_need_bounce)) { 1413 if (__predict_false(xbdi->xbdi_bouncebuf_use)) { 1414 KASSERT(xbdi->xbdi_pendingreqs > 0); 1415 /* retry later */ 1416 xbdi->xbdi_cont_restart = xbdback_co_io_gotio; 1417 xbdi->xbdi_cont_restart_obj = xbd_io; 1418 xbdi->xbdi_cont = NULL; 1419 return NULL; 1420 } 1421 xbdi->xbdi_bouncebuf_use++; 1422 KASSERT(xbdi->xbdi_bouncebuf_use == 1); 1423 xbd_io->xio_buf.b_data = (void *)xbdi->xbdi_bouncebuf; 1424 } 1425 xbdi->xbdi_pendingreqs++; 1426 if ((error = xbdback_map_shm(xbd_io)) != 0) { 1427 xbdback_io_error(xbd_io, error); 1428 /* do not retry */ 1429 xbdi->xbdi_cont = xbdback_co_main_incr; 1430 return xbdi; 1431 } 1432 if (__predict_true(xbd_io->xio_need_bounce == 0)) { 1433 xbd_io->xio_buf.b_data = (void *) 1434 (xbd_io->xio_vaddr + xbd_io->xio_start_offset); 1435 } 1436 1437 1438 xbd_io->xio_buf.b_private = xbd_io; 1439 1440 xbdi->xbdi_cont = xbdback_co_do_io; 1441 return xbd_io; 1442 } 1443 1444 static void 1445 xbdback_io_error(struct xbdback_io *xbd_io, int error) 1446 { 1447 KASSERT(mutex_owned(&xbd_io->xio_xbdi->xbdi_lock)); 1448 1449 struct buf *bp = &xbd_io->xio_buf; 1450 1451 bp->b_error = error; 1452 xbdback_iodone_locked(xbd_io->xio_xbdi, xbd_io, bp); 1453 } 1454 1455 /* 1456 * Main xbdback I/O routine. It can either perform a flush operation or 1457 * schedule a read/write operation. 1458 */ 1459 static void * 1460 xbdback_co_do_io(struct xbdback_instance *xbdi, void *obj) 1461 { 1462 struct xbdback_io *xbd_io = obj; 1463 blkif_request_t *req = &xbd_io->xio_xen_req; 1464 1465 KASSERT(xbdi->xbdi_refcnt > 0); 1466 1467 switch (req->operation) { 1468 case BLKIF_OP_FLUSH_DISKCACHE: 1469 { 1470 int error; 1471 int force = 1; 1472 1473 KASSERT(mutex_owned(&xbdi->xbdi_lock)); 1474 mutex_exit(&xbdi->xbdi_lock); 1475 error = VOP_IOCTL(xbdi->xbdi_vp, DIOCCACHESYNC, &force, FWRITE, 1476 kauth_cred_get()); 1477 mutex_enter(&xbdi->xbdi_lock); 1478 if (error) { 1479 printf("xbdback %s: DIOCCACHESYNC returned %d\n", 1480 xbdi->xbdi_xbusd->xbusd_path, error); 1481 if (error == EOPNOTSUPP || error == ENOTTY) 1482 error = BLKIF_RSP_EOPNOTSUPP; 1483 else 1484 error = BLKIF_RSP_ERROR; 1485 } else 1486 error = BLKIF_RSP_OKAY; 1487 xbdback_send_reply(xbdi, req->id, req->operation, error); 1488 xbdback_io_put(xbdi, xbd_io); 1489 xbdi_put(xbdi); 1490 xbdi->xbdi_cont = xbdback_co_main_incr; 1491 return xbdi; 1492 } 1493 case BLKIF_OP_READ: 1494 case BLKIF_OP_WRITE: 1495 if (__predict_false(xbd_io->xio_need_bounce) && 1496 req->operation == BLKIF_OP_WRITE) { 1497 vaddr_t boffset = 0; 1498 for (int i = 0; i < req->nr_segments; i++) { 1499 struct blkif_request_segment *seg = 1500 &xbd_io->xio_seg[i]; 1501 vaddr_t segoffset = seg->first_sect * VBD_BSIZE; 1502 size_t segbcount = 1503 (seg->last_sect - seg->first_sect + 1) * 1504 VBD_BSIZE; 1505 KASSERT(segoffset + segbcount <= PAGE_SIZE); 1506 KASSERT(boffset + segbcount < MAXPHYS); 1507 segoffset += PAGE_SIZE * i; 1508 memcpy( 1509 (void *)(xbdi->xbdi_bouncebuf + boffset), 1510 (void *)(xbd_io->xio_vaddr + segoffset), 1511 segbcount); 1512 boffset += segbcount; 1513 } 1514 } 1515 KASSERT(mutex_owned(&xbdi->xbdi_lock)); 1516 mutex_exit(&xbdi->xbdi_lock); 1517 if ((xbd_io->xio_buf.b_flags & B_READ) == 0) { 1518 mutex_enter(xbd_io->xio_buf.b_vp->v_interlock); 1519 xbd_io->xio_buf.b_vp->v_numoutput++; 1520 mutex_exit(xbd_io->xio_buf.b_vp->v_interlock); 1521 } 1522 /* will call xbdback_iodone() asynchronously when done */ 1523 bdev_strategy(&xbd_io->xio_buf); 1524 mutex_enter(&xbdi->xbdi_lock); 1525 xbdi->xbdi_cont = xbdback_co_main_incr; 1526 return xbdi; 1527 default: 1528 /* Should never happen */ 1529 panic("xbdback_co_do_io: unsupported operation %d", 1530 req->operation); 1531 } 1532 } 1533 1534 /* 1535 * Called from softint(9) context when an I/O is done: for each request, send 1536 * back the associated reply to the domain. 1537 */ 1538 static void 1539 xbdback_iodone(struct buf *bp) 1540 { 1541 struct xbdback_io *xbd_io; 1542 struct xbdback_instance *xbdi; 1543 1544 xbd_io = bp->b_private; 1545 KASSERT(bp == &xbd_io->xio_buf); 1546 xbdi = xbd_io->xio_xbdi; 1547 1548 mutex_enter(&xbdi->xbdi_lock); 1549 xbdback_iodone_locked(xbdi, xbd_io, bp); 1550 mutex_exit(&xbdi->xbdi_lock); 1551 } 1552 1553 /* 1554 * This gets reused by xbdback_io_error to report errors from other sources. 1555 */ 1556 static void 1557 xbdback_iodone_locked(struct xbdback_instance *xbdi, struct xbdback_io *xbd_io, 1558 struct buf *bp) 1559 { 1560 int status; 1561 blkif_request_t *req = &xbd_io->xio_xen_req; 1562 1563 XENPRINTF(("xbdback_io domain %d: iodone ptr 0x%lx\n", 1564 xbdi->xbdi_domid, (long)xbd_io)); 1565 1566 KASSERT(mutex_owned(&xbdi->xbdi_lock)); 1567 1568 KASSERT(bp->b_error != 0 || xbd_io->xio_xv != NULL); 1569 if (__predict_false(xbd_io->xio_need_bounce)) { 1570 KASSERT(xbd_io->xio_buf.b_data == (void *)xbdi->xbdi_bouncebuf); 1571 1572 KASSERT(req->operation == BLKIF_OP_WRITE || 1573 req->operation == BLKIF_OP_READ); 1574 1575 if (req->operation == BLKIF_OP_READ && bp->b_error == 0) { 1576 vaddr_t boffset = 0; 1577 for (int i = 0; i < req->nr_segments; i++) { 1578 struct blkif_request_segment *seg = 1579 &xbd_io->xio_seg[i]; 1580 vaddr_t segoffset = seg->first_sect * VBD_BSIZE; 1581 size_t segbcount = 1582 (seg->last_sect - seg->first_sect + 1) * 1583 VBD_BSIZE; 1584 KASSERT(segoffset + segbcount <= PAGE_SIZE); 1585 KASSERT(boffset + segbcount < MAXPHYS); 1586 segoffset += PAGE_SIZE * i; 1587 memcpy( 1588 (void *)(xbd_io->xio_vaddr + segoffset), 1589 (void *)(xbdi->xbdi_bouncebuf + boffset), 1590 segbcount); 1591 boffset += segbcount; 1592 } 1593 } 1594 KASSERT(xbdi->xbdi_bouncebuf_use == 1); 1595 xbdi->xbdi_bouncebuf_use--; 1596 } 1597 if (xbd_io->xio_xv != NULL) 1598 xbdback_unmap_shm(xbd_io); 1599 1600 if (bp->b_error != 0) { 1601 printf("xbd IO domain %d: error %d\n", 1602 xbdi->xbdi_domid, bp->b_error); 1603 status = BLKIF_RSP_ERROR; 1604 } else 1605 status = BLKIF_RSP_OKAY; 1606 1607 xbdback_send_reply(xbdi, req->id, req->operation, status); 1608 1609 xbdi_put(xbdi); 1610 KASSERT(xbdi->xbdi_pendingreqs > 0); 1611 xbdi->xbdi_pendingreqs--; 1612 buf_destroy(&xbd_io->xio_buf); 1613 xbdback_io_put(xbdi, xbd_io); 1614 1615 xbdback_wakeup_thread(xbdi); 1616 } 1617 1618 /* 1619 * Wake up the per xbdback instance thread. 1620 */ 1621 static void 1622 xbdback_wakeup_thread(struct xbdback_instance *xbdi) 1623 { 1624 KASSERT(mutex_owned(&xbdi->xbdi_lock)); 1625 1626 /* only set RUN state when we are WAITING for work */ 1627 if (xbdi->xbdi_status == WAITING) 1628 xbdi->xbdi_status = RUN; 1629 cv_signal(&xbdi->xbdi_cv); 1630 } 1631 1632 /* 1633 * called once a request has completed. Place the reply in the ring and 1634 * notify the guest OS. 1635 */ 1636 static void 1637 xbdback_send_reply(struct xbdback_instance *xbdi, uint64_t id, 1638 int op, int status) 1639 { 1640 blkif_response_t *resp_n; 1641 blkif_x86_32_response_t *resp32; 1642 blkif_x86_64_response_t *resp64; 1643 int notify; 1644 1645 KASSERT(mutex_owned(&xbdi->xbdi_lock)); 1646 1647 /* 1648 * The ring can be accessed by the xbdback thread, xbdback_iodone() 1649 * handler, or any handler that triggered the shm callback. So 1650 * protect ring access via the xbdi_lock mutex. 1651 */ 1652 switch (xbdi->xbdi_proto) { 1653 case XBDIP_NATIVE: 1654 resp_n = RING_GET_RESPONSE(&xbdi->xbdi_ring.ring_n, 1655 xbdi->xbdi_ring.ring_n.rsp_prod_pvt); 1656 resp_n->id = id; 1657 resp_n->operation = op; 1658 resp_n->status = status; 1659 break; 1660 case XBDIP_32: 1661 resp32 = RING_GET_RESPONSE(&xbdi->xbdi_ring.ring_32, 1662 xbdi->xbdi_ring.ring_n.rsp_prod_pvt); 1663 resp32->id = id; 1664 resp32->operation = op; 1665 resp32->status = status; 1666 break; 1667 case XBDIP_64: 1668 resp64 = RING_GET_RESPONSE(&xbdi->xbdi_ring.ring_64, 1669 xbdi->xbdi_ring.ring_n.rsp_prod_pvt); 1670 resp64->id = id; 1671 resp64->operation = op; 1672 resp64->status = status; 1673 break; 1674 } 1675 xbdi->xbdi_ring.ring_n.rsp_prod_pvt++; 1676 RING_PUSH_RESPONSES_AND_CHECK_NOTIFY(&xbdi->xbdi_ring.ring_n, notify); 1677 1678 if (notify) { 1679 XENPRINTF(("xbdback_send_reply notify %d\n", xbdi->xbdi_domid)); 1680 hypervisor_notify_via_evtchn(xbdi->xbdi_evtchn); 1681 } 1682 } 1683 1684 /* 1685 * Map multiple entries of an I/O request into backend's VA space. 1686 * The xbd_io->xio_gref array has to be filled out by the caller. 1687 */ 1688 static int 1689 xbdback_map_shm(struct xbdback_io *xbd_io) 1690 { 1691 struct xbdback_instance *xbdi = xbd_io->xio_xbdi; 1692 blkif_request_t *req = &xbd_io->xio_xen_req; 1693 int error; 1694 1695 #ifdef XENDEBUG_VBD 1696 int i; 1697 printf("xbdback_map_shm map grant "); 1698 for (i = 0; i < req->nr_segments; i++) { 1699 printf("%u ", (u_int)xbd_io->xio_gref[i]); 1700 } 1701 #endif 1702 1703 KASSERT(mutex_owned(&xbdi->xbdi_lock)); 1704 KASSERT(xbd_io->xio_xv == NULL); 1705 1706 xbd_io->xio_xv = SLIST_FIRST(&xbdi->xbdi_va_free); 1707 KASSERT(xbd_io->xio_xv != NULL); 1708 SLIST_REMOVE_HEAD(&xbdi->xbdi_va_free, xv_next); 1709 xbd_io->xio_vaddr = xbd_io->xio_xv->xv_vaddr; 1710 1711 error = xen_shm_map(req->nr_segments, xbdi->xbdi_domid, 1712 xbd_io->xio_gref, xbd_io->xio_vaddr, xbd_io->xio_gh, 1713 (req->operation == BLKIF_OP_WRITE) ? XSHM_RO : 0); 1714 1715 switch(error) { 1716 case 0: 1717 #ifdef XENDEBUG_VBD 1718 printf("handle"); 1719 for (i = 0; i < req->nr_segments; i++) { 1720 printf(" %u ", (u_int)xbd_io->xio_gh[i]); 1721 } 1722 printf("\n"); 1723 #endif 1724 return 0; 1725 default: 1726 /* reset xio_xv so error handling won't try to unmap it */ 1727 SLIST_INSERT_HEAD(&xbdi->xbdi_va_free, xbd_io->xio_xv, xv_next); 1728 xbd_io->xio_xv = NULL; 1729 return error; 1730 } 1731 } 1732 1733 /* unmap a request from our virtual address space (request is done) */ 1734 static void 1735 xbdback_unmap_shm(struct xbdback_io *xbd_io) 1736 { 1737 struct xbdback_instance *xbdi = xbd_io->xio_xbdi; 1738 blkif_request_t *req = &xbd_io->xio_xen_req; 1739 1740 #ifdef XENDEBUG_VBD 1741 int i; 1742 printf("xbdback_unmap_shm handle "); 1743 for (i = 0; i < req->nr_segments; i++) { 1744 printf("%u ", (u_int)xbd_io->xio_gh[i]); 1745 } 1746 printf("\n"); 1747 #endif 1748 1749 KASSERT(xbd_io->xio_xv != NULL); 1750 xen_shm_unmap(xbd_io->xio_vaddr, req->nr_segments, 1751 xbd_io->xio_gh); 1752 SLIST_INSERT_HEAD(&xbdi->xbdi_va_free, xbd_io->xio_xv, xv_next); 1753 xbd_io->xio_xv = NULL; 1754 xbd_io->xio_vaddr = -1; 1755 } 1756 1757 /* Obtain memory from a pool */ 1758 static struct xbdback_io * 1759 xbdback_io_get(struct xbdback_instance *xbdi) 1760 { 1761 struct xbdback_io *xbd_io = SLIST_FIRST(&xbdi->xbdi_io_free); 1762 SLIST_REMOVE_HEAD(&xbdi->xbdi_io_free, xio_next); 1763 return xbd_io; 1764 } 1765 1766 /* Restore memory to a pool */ 1767 static void 1768 xbdback_io_put(struct xbdback_instance *xbdi, struct xbdback_io *xbd_io) 1769 { 1770 KASSERT(xbd_io->xio_xv == NULL); 1771 KASSERT(xbd_io != NULL); 1772 SLIST_INSERT_HEAD(&xbdi->xbdi_io_free, xbd_io, xio_next); 1773 } 1774 1775 /* 1776 * Trampoline routine. Calls continuations in a loop and only exits when 1777 * either the returned object or the next callback is NULL. 1778 */ 1779 static void 1780 xbdback_trampoline(struct xbdback_instance *xbdi, void *obj) 1781 { 1782 xbdback_cont_t cont; 1783 1784 while(obj != NULL && xbdi->xbdi_cont != NULL) { 1785 KASSERT(xbdi->xbdi_cont_restart == NULL); 1786 KASSERT(xbdi->xbdi_cont_restart_obj == NULL); 1787 cont = xbdi->xbdi_cont; 1788 #ifdef DIAGNOSTIC 1789 xbdi->xbdi_cont = (xbdback_cont_t)0xDEADBEEF; 1790 #endif 1791 obj = (*cont)(xbdi, obj); 1792 #ifdef DIAGNOSTIC 1793 if (xbdi->xbdi_cont == (xbdback_cont_t)0xDEADBEEF) { 1794 printf("xbdback_trampoline: 0x%lx didn't set " 1795 "xbdi->xbdi_cont!\n", (long)cont); 1796 panic("xbdback_trampoline: bad continuation"); 1797 } 1798 if (xbdi->xbdi_cont_restart != NULL || 1799 xbdi->xbdi_cont_restart_obj != NULL) { 1800 KASSERT(xbdi->xbdi_cont_restart != NULL); 1801 KASSERT(xbdi->xbdi_cont_restart_obj != NULL); 1802 KASSERT(xbdi->xbdi_cont == NULL); 1803 KASSERT(obj == NULL); 1804 } 1805 #endif 1806 } 1807 } 1808
Indexes created Sun Oct 25 00:26:45 UTC 2026