xhci.c revision 1.109
1 /* $NetBSD: xhci.c,v 1.109 2019/08/21 06:19:43 mrg Exp $ */ 2 3 /* 4 * Copyright (c) 2013 Jonathan A. Kollasch 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 17 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 18 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 19 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR 20 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 21 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 22 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 23 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 24 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 25 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 26 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 */ 28 29 /* 30 * USB rev 2.0 and rev 3.1 specification 31 * http://www.usb.org/developers/docs/ 32 * xHCI rev 1.1 specification 33 * http://www.intel.com/technology/usb/spec.htm 34 */ 35 36 #include <sys/cdefs.h> 37 __KERNEL_RCSID(0, "$NetBSD: xhci.c,v 1.109 2019/08/21 06:19:43 mrg Exp $"); 38 39 #ifdef _KERNEL_OPT 40 #include "opt_usb.h" 41 #endif 42 43 #include <sys/param.h> 44 #include <sys/systm.h> 45 #include <sys/kernel.h> 46 #include <sys/kmem.h> 47 #include <sys/device.h> 48 #include <sys/select.h> 49 #include <sys/proc.h> 50 #include <sys/queue.h> 51 #include <sys/mutex.h> 52 #include <sys/condvar.h> 53 #include <sys/bus.h> 54 #include <sys/cpu.h> 55 #include <sys/sysctl.h> 56 57 #include <machine/endian.h> 58 59 #include <dev/usb/usb.h> 60 #include <dev/usb/usbdi.h> 61 #include <dev/usb/usbdivar.h> 62 #include <dev/usb/usbdi_util.h> 63 #include <dev/usb/usbhist.h> 64 #include <dev/usb/usb_mem.h> 65 #include <dev/usb/usb_quirks.h> 66 67 #include <dev/usb/xhcireg.h> 68 #include <dev/usb/xhcivar.h> 69 #include <dev/usb/usbroothub.h> 70 71 72 #ifdef USB_DEBUG 73 #ifndef XHCI_DEBUG 74 #define xhcidebug 0 75 #else /* !XHCI_DEBUG */ 76 #define HEXDUMP(a, b, c) \ 77 do { \ 78 if (xhcidebug > 0) \ 79 hexdump(printf, a, b, c); \ 80 } while (/*CONSTCOND*/0) 81 static int xhcidebug = 0; 82 83 SYSCTL_SETUP(sysctl_hw_xhci_setup, "sysctl hw.xhci setup") 84 { 85 int err; 86 const struct sysctlnode *rnode; 87 const struct sysctlnode *cnode; 88 89 err = sysctl_createv(clog, 0, NULL, &rnode, 90 CTLFLAG_PERMANENT, CTLTYPE_NODE, "xhci", 91 SYSCTL_DESCR("xhci global controls"), 92 NULL, 0, NULL, 0, CTL_HW, CTL_CREATE, CTL_EOL); 93 94 if (err) 95 goto fail; 96 97 /* control debugging printfs */ 98 err = sysctl_createv(clog, 0, &rnode, &cnode, 99 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, CTLTYPE_INT, 100 "debug", SYSCTL_DESCR("Enable debugging output"), 101 NULL, 0, &xhcidebug, sizeof(xhcidebug), CTL_CREATE, CTL_EOL); 102 if (err) 103 goto fail; 104 105 return; 106 fail: 107 aprint_error("%s: sysctl_createv failed (err = %d)\n", __func__, err); 108 } 109 110 #endif /* !XHCI_DEBUG */ 111 #endif /* USB_DEBUG */ 112 113 #ifndef HEXDUMP 114 #define HEXDUMP(a, b, c) 115 #endif 116 117 #define DPRINTFN(N,FMT,A,B,C,D) USBHIST_LOGN(xhcidebug,N,FMT,A,B,C,D) 118 #define XHCIHIST_FUNC() USBHIST_FUNC() 119 #define XHCIHIST_CALLED(name) USBHIST_CALLED(xhcidebug) 120 121 #define XHCI_DCI_SLOT 0 122 #define XHCI_DCI_EP_CONTROL 1 123 124 #define XHCI_ICI_INPUT_CONTROL 0 125 126 struct xhci_pipe { 127 struct usbd_pipe xp_pipe; 128 struct usb_task xp_async_task; 129 }; 130 131 #define XHCI_COMMAND_RING_TRBS 256 132 #define XHCI_EVENT_RING_TRBS 256 133 #define XHCI_EVENT_RING_SEGMENTS 1 134 #define XHCI_TRB_3_ED_BIT XHCI_TRB_3_ISP_BIT 135 136 static usbd_status xhci_open(struct usbd_pipe *); 137 static void xhci_close_pipe(struct usbd_pipe *); 138 static int xhci_intr1(struct xhci_softc * const); 139 static void xhci_softintr(void *); 140 static void xhci_poll(struct usbd_bus *); 141 static struct usbd_xfer *xhci_allocx(struct usbd_bus *, unsigned int); 142 static void xhci_freex(struct usbd_bus *, struct usbd_xfer *); 143 static void xhci_get_lock(struct usbd_bus *, kmutex_t **); 144 static usbd_status xhci_new_device(device_t, struct usbd_bus *, int, int, int, 145 struct usbd_port *); 146 static int xhci_roothub_ctrl(struct usbd_bus *, usb_device_request_t *, 147 void *, int); 148 149 static usbd_status xhci_configure_endpoint(struct usbd_pipe *); 150 //static usbd_status xhci_unconfigure_endpoint(struct usbd_pipe *); 151 static usbd_status xhci_reset_endpoint(struct usbd_pipe *); 152 static usbd_status xhci_stop_endpoint(struct usbd_pipe *); 153 154 static void xhci_host_dequeue(struct xhci_ring * const); 155 static usbd_status xhci_set_dequeue(struct usbd_pipe *); 156 157 static usbd_status xhci_do_command(struct xhci_softc * const, 158 struct xhci_soft_trb * const, int); 159 static usbd_status xhci_do_command_locked(struct xhci_softc * const, 160 struct xhci_soft_trb * const, int); 161 static usbd_status xhci_init_slot(struct usbd_device *, uint32_t); 162 static void xhci_free_slot(struct xhci_softc *, struct xhci_slot *, int, int); 163 static usbd_status xhci_set_address(struct usbd_device *, uint32_t, bool); 164 static usbd_status xhci_enable_slot(struct xhci_softc * const, 165 uint8_t * const); 166 static usbd_status xhci_disable_slot(struct xhci_softc * const, uint8_t); 167 static usbd_status xhci_address_device(struct xhci_softc * const, 168 uint64_t, uint8_t, bool); 169 static void xhci_set_dcba(struct xhci_softc * const, uint64_t, int); 170 static usbd_status xhci_update_ep0_mps(struct xhci_softc * const, 171 struct xhci_slot * const, u_int); 172 static usbd_status xhci_ring_init(struct xhci_softc * const, 173 struct xhci_ring * const, size_t, size_t); 174 static void xhci_ring_free(struct xhci_softc * const, struct xhci_ring * const); 175 176 static void xhci_setup_ctx(struct usbd_pipe *); 177 static void xhci_setup_route(struct usbd_pipe *, uint32_t *); 178 static void xhci_setup_tthub(struct usbd_pipe *, uint32_t *); 179 static void xhci_setup_maxburst(struct usbd_pipe *, uint32_t *); 180 static uint32_t xhci_bival2ival(uint32_t, uint32_t); 181 182 static void xhci_noop(struct usbd_pipe *); 183 184 static usbd_status xhci_root_intr_transfer(struct usbd_xfer *); 185 static usbd_status xhci_root_intr_start(struct usbd_xfer *); 186 static void xhci_root_intr_abort(struct usbd_xfer *); 187 static void xhci_root_intr_close(struct usbd_pipe *); 188 static void xhci_root_intr_done(struct usbd_xfer *); 189 190 static usbd_status xhci_device_ctrl_transfer(struct usbd_xfer *); 191 static usbd_status xhci_device_ctrl_start(struct usbd_xfer *); 192 static void xhci_device_ctrl_abort(struct usbd_xfer *); 193 static void xhci_device_ctrl_close(struct usbd_pipe *); 194 static void xhci_device_ctrl_done(struct usbd_xfer *); 195 196 static usbd_status xhci_device_intr_transfer(struct usbd_xfer *); 197 static usbd_status xhci_device_intr_start(struct usbd_xfer *); 198 static void xhci_device_intr_abort(struct usbd_xfer *); 199 static void xhci_device_intr_close(struct usbd_pipe *); 200 static void xhci_device_intr_done(struct usbd_xfer *); 201 202 static usbd_status xhci_device_bulk_transfer(struct usbd_xfer *); 203 static usbd_status xhci_device_bulk_start(struct usbd_xfer *); 204 static void xhci_device_bulk_abort(struct usbd_xfer *); 205 static void xhci_device_bulk_close(struct usbd_pipe *); 206 static void xhci_device_bulk_done(struct usbd_xfer *); 207 208 static void xhci_timeout(void *); 209 static void xhci_timeout_task(void *); 210 211 static const struct usbd_bus_methods xhci_bus_methods = { 212 .ubm_open = xhci_open, 213 .ubm_softint = xhci_softintr, 214 .ubm_dopoll = xhci_poll, 215 .ubm_allocx = xhci_allocx, 216 .ubm_freex = xhci_freex, 217 .ubm_getlock = xhci_get_lock, 218 .ubm_newdev = xhci_new_device, 219 .ubm_rhctrl = xhci_roothub_ctrl, 220 }; 221 222 static const struct usbd_pipe_methods xhci_root_intr_methods = { 223 .upm_transfer = xhci_root_intr_transfer, 224 .upm_start = xhci_root_intr_start, 225 .upm_abort = xhci_root_intr_abort, 226 .upm_close = xhci_root_intr_close, 227 .upm_cleartoggle = xhci_noop, 228 .upm_done = xhci_root_intr_done, 229 }; 230 231 232 static const struct usbd_pipe_methods xhci_device_ctrl_methods = { 233 .upm_transfer = xhci_device_ctrl_transfer, 234 .upm_start = xhci_device_ctrl_start, 235 .upm_abort = xhci_device_ctrl_abort, 236 .upm_close = xhci_device_ctrl_close, 237 .upm_cleartoggle = xhci_noop, 238 .upm_done = xhci_device_ctrl_done, 239 }; 240 241 static const struct usbd_pipe_methods xhci_device_isoc_methods = { 242 .upm_cleartoggle = xhci_noop, 243 }; 244 245 static const struct usbd_pipe_methods xhci_device_bulk_methods = { 246 .upm_transfer = xhci_device_bulk_transfer, 247 .upm_start = xhci_device_bulk_start, 248 .upm_abort = xhci_device_bulk_abort, 249 .upm_close = xhci_device_bulk_close, 250 .upm_cleartoggle = xhci_noop, 251 .upm_done = xhci_device_bulk_done, 252 }; 253 254 static const struct usbd_pipe_methods xhci_device_intr_methods = { 255 .upm_transfer = xhci_device_intr_transfer, 256 .upm_start = xhci_device_intr_start, 257 .upm_abort = xhci_device_intr_abort, 258 .upm_close = xhci_device_intr_close, 259 .upm_cleartoggle = xhci_noop, 260 .upm_done = xhci_device_intr_done, 261 }; 262 263 static inline uint32_t 264 xhci_read_1(const struct xhci_softc * const sc, bus_size_t offset) 265 { 266 return bus_space_read_1(sc->sc_iot, sc->sc_ioh, offset); 267 } 268 269 static inline uint32_t 270 xhci_read_4(const struct xhci_softc * const sc, bus_size_t offset) 271 { 272 return bus_space_read_4(sc->sc_iot, sc->sc_ioh, offset); 273 } 274 275 static inline void 276 xhci_write_1(const struct xhci_softc * const sc, bus_size_t offset, 277 uint32_t value) 278 { 279 bus_space_write_1(sc->sc_iot, sc->sc_ioh, offset, value); 280 } 281 282 #if 0 /* unused */ 283 static inline void 284 xhci_write_4(const struct xhci_softc * const sc, bus_size_t offset, 285 uint32_t value) 286 { 287 bus_space_write_4(sc->sc_iot, sc->sc_ioh, offset, value); 288 } 289 #endif /* unused */ 290 291 static inline uint32_t 292 xhci_cap_read_4(const struct xhci_softc * const sc, bus_size_t offset) 293 { 294 return bus_space_read_4(sc->sc_iot, sc->sc_cbh, offset); 295 } 296 297 static inline uint32_t 298 xhci_op_read_4(const struct xhci_softc * const sc, bus_size_t offset) 299 { 300 return bus_space_read_4(sc->sc_iot, sc->sc_obh, offset); 301 } 302 303 static inline void 304 xhci_op_write_4(const struct xhci_softc * const sc, bus_size_t offset, 305 uint32_t value) 306 { 307 bus_space_write_4(sc->sc_iot, sc->sc_obh, offset, value); 308 } 309 310 static inline uint64_t 311 xhci_op_read_8(const struct xhci_softc * const sc, bus_size_t offset) 312 { 313 uint64_t value; 314 315 if (sc->sc_ac64) { 316 #ifdef XHCI_USE_BUS_SPACE_8 317 value = bus_space_read_8(sc->sc_iot, sc->sc_obh, offset); 318 #else 319 value = bus_space_read_4(sc->sc_iot, sc->sc_obh, offset); 320 value |= (uint64_t)bus_space_read_4(sc->sc_iot, sc->sc_obh, 321 offset + 4) << 32; 322 #endif 323 } else { 324 value = bus_space_read_4(sc->sc_iot, sc->sc_obh, offset); 325 } 326 327 return value; 328 } 329 330 static inline void 331 xhci_op_write_8(const struct xhci_softc * const sc, bus_size_t offset, 332 uint64_t value) 333 { 334 if (sc->sc_ac64) { 335 #ifdef XHCI_USE_BUS_SPACE_8 336 bus_space_write_8(sc->sc_iot, sc->sc_obh, offset, value); 337 #else 338 bus_space_write_4(sc->sc_iot, sc->sc_obh, offset + 0, 339 (value >> 0) & 0xffffffff); 340 bus_space_write_4(sc->sc_iot, sc->sc_obh, offset + 4, 341 (value >> 32) & 0xffffffff); 342 #endif 343 } else { 344 bus_space_write_4(sc->sc_iot, sc->sc_obh, offset, value); 345 } 346 } 347 348 static inline void 349 xhci_op_barrier(const struct xhci_softc * const sc, bus_size_t offset, 350 bus_size_t len, int flags) 351 { 352 bus_space_barrier(sc->sc_iot, sc->sc_obh, offset, len, flags); 353 } 354 355 static inline uint32_t 356 xhci_rt_read_4(const struct xhci_softc * const sc, bus_size_t offset) 357 { 358 return bus_space_read_4(sc->sc_iot, sc->sc_rbh, offset); 359 } 360 361 static inline void 362 xhci_rt_write_4(const struct xhci_softc * const sc, bus_size_t offset, 363 uint32_t value) 364 { 365 bus_space_write_4(sc->sc_iot, sc->sc_rbh, offset, value); 366 } 367 368 #if 0 /* unused */ 369 static inline uint64_t 370 xhci_rt_read_8(const struct xhci_softc * const sc, bus_size_t offset) 371 { 372 uint64_t value; 373 374 if (sc->sc_ac64) { 375 #ifdef XHCI_USE_BUS_SPACE_8 376 value = bus_space_read_8(sc->sc_iot, sc->sc_rbh, offset); 377 #else 378 value = bus_space_read_4(sc->sc_iot, sc->sc_rbh, offset); 379 value |= (uint64_t)bus_space_read_4(sc->sc_iot, sc->sc_rbh, 380 offset + 4) << 32; 381 #endif 382 } else { 383 value = bus_space_read_4(sc->sc_iot, sc->sc_rbh, offset); 384 } 385 386 return value; 387 } 388 #endif /* unused */ 389 390 static inline void 391 xhci_rt_write_8(const struct xhci_softc * const sc, bus_size_t offset, 392 uint64_t value) 393 { 394 if (sc->sc_ac64) { 395 #ifdef XHCI_USE_BUS_SPACE_8 396 bus_space_write_8(sc->sc_iot, sc->sc_rbh, offset, value); 397 #else 398 bus_space_write_4(sc->sc_iot, sc->sc_rbh, offset + 0, 399 (value >> 0) & 0xffffffff); 400 bus_space_write_4(sc->sc_iot, sc->sc_rbh, offset + 4, 401 (value >> 32) & 0xffffffff); 402 #endif 403 } else { 404 bus_space_write_4(sc->sc_iot, sc->sc_rbh, offset, value); 405 } 406 } 407 408 #if 0 /* unused */ 409 static inline uint32_t 410 xhci_db_read_4(const struct xhci_softc * const sc, bus_size_t offset) 411 { 412 return bus_space_read_4(sc->sc_iot, sc->sc_dbh, offset); 413 } 414 #endif /* unused */ 415 416 static inline void 417 xhci_db_write_4(const struct xhci_softc * const sc, bus_size_t offset, 418 uint32_t value) 419 { 420 bus_space_write_4(sc->sc_iot, sc->sc_dbh, offset, value); 421 } 422 423 /* --- */ 424 425 static inline uint8_t 426 xhci_ep_get_type(usb_endpoint_descriptor_t * const ed) 427 { 428 u_int eptype = 0; 429 430 switch (UE_GET_XFERTYPE(ed->bmAttributes)) { 431 case UE_CONTROL: 432 eptype = 0x0; 433 break; 434 case UE_ISOCHRONOUS: 435 eptype = 0x1; 436 break; 437 case UE_BULK: 438 eptype = 0x2; 439 break; 440 case UE_INTERRUPT: 441 eptype = 0x3; 442 break; 443 } 444 445 if ((UE_GET_XFERTYPE(ed->bmAttributes) == UE_CONTROL) || 446 (UE_GET_DIR(ed->bEndpointAddress) == UE_DIR_IN)) 447 return eptype | 0x4; 448 else 449 return eptype; 450 } 451 452 static u_int 453 xhci_ep_get_dci(usb_endpoint_descriptor_t * const ed) 454 { 455 /* xHCI 1.0 section 4.5.1 */ 456 u_int epaddr = UE_GET_ADDR(ed->bEndpointAddress); 457 u_int in = 0; 458 459 if ((UE_GET_XFERTYPE(ed->bmAttributes) == UE_CONTROL) || 460 (UE_GET_DIR(ed->bEndpointAddress) == UE_DIR_IN)) 461 in = 1; 462 463 return epaddr * 2 + in; 464 } 465 466 static inline u_int 467 xhci_dci_to_ici(const u_int i) 468 { 469 return i + 1; 470 } 471 472 static inline void * 473 xhci_slot_get_dcv(struct xhci_softc * const sc, struct xhci_slot * const xs, 474 const u_int dci) 475 { 476 return KERNADDR(&xs->xs_dc_dma, sc->sc_ctxsz * dci); 477 } 478 479 #if 0 /* unused */ 480 static inline bus_addr_t 481 xhci_slot_get_dcp(struct xhci_softc * const sc, struct xhci_slot * const xs, 482 const u_int dci) 483 { 484 return DMAADDR(&xs->xs_dc_dma, sc->sc_ctxsz * dci); 485 } 486 #endif /* unused */ 487 488 static inline void * 489 xhci_slot_get_icv(struct xhci_softc * const sc, struct xhci_slot * const xs, 490 const u_int ici) 491 { 492 return KERNADDR(&xs->xs_ic_dma, sc->sc_ctxsz * ici); 493 } 494 495 static inline bus_addr_t 496 xhci_slot_get_icp(struct xhci_softc * const sc, struct xhci_slot * const xs, 497 const u_int ici) 498 { 499 return DMAADDR(&xs->xs_ic_dma, sc->sc_ctxsz * ici); 500 } 501 502 static inline struct xhci_trb * 503 xhci_ring_trbv(struct xhci_ring * const xr, u_int idx) 504 { 505 return KERNADDR(&xr->xr_dma, XHCI_TRB_SIZE * idx); 506 } 507 508 static inline bus_addr_t 509 xhci_ring_trbp(struct xhci_ring * const xr, u_int idx) 510 { 511 return DMAADDR(&xr->xr_dma, XHCI_TRB_SIZE * idx); 512 } 513 514 static inline void 515 xhci_soft_trb_put(struct xhci_soft_trb * const trb, 516 uint64_t parameter, uint32_t status, uint32_t control) 517 { 518 trb->trb_0 = parameter; 519 trb->trb_2 = status; 520 trb->trb_3 = control; 521 } 522 523 static inline void 524 xhci_trb_put(struct xhci_trb * const trb, uint64_t parameter, uint32_t status, 525 uint32_t control) 526 { 527 trb->trb_0 = htole64(parameter); 528 trb->trb_2 = htole32(status); 529 trb->trb_3 = htole32(control); 530 } 531 532 static int 533 xhci_trb_get_idx(struct xhci_ring *xr, uint64_t trb_0, int *idx) 534 { 535 /* base address of TRBs */ 536 bus_addr_t trbp = xhci_ring_trbp(xr, 0); 537 538 /* trb_0 range sanity check */ 539 if (trb_0 == 0 || trb_0 < trbp || 540 (trb_0 - trbp) % sizeof(struct xhci_trb) != 0 || 541 (trb_0 - trbp) / sizeof(struct xhci_trb) >= xr->xr_ntrb) { 542 return 1; 543 } 544 *idx = (trb_0 - trbp) / sizeof(struct xhci_trb); 545 return 0; 546 } 547 548 static unsigned int 549 xhci_get_epstate(struct xhci_softc * const sc, struct xhci_slot * const xs, 550 u_int dci) 551 { 552 uint32_t *cp; 553 554 usb_syncmem(&xs->xs_dc_dma, 0, sc->sc_pgsz, BUS_DMASYNC_POSTREAD); 555 cp = xhci_slot_get_dcv(sc, xs, dci); 556 return XHCI_EPCTX_0_EPSTATE_GET(le32toh(cp[0])); 557 } 558 559 static inline unsigned int 560 xhci_ctlrport2bus(struct xhci_softc * const sc, unsigned int ctlrport) 561 { 562 const unsigned int port = ctlrport - 1; 563 const uint8_t bit = __BIT(port % NBBY); 564 565 return __SHIFTOUT(sc->sc_ctlrportbus[port / NBBY], bit); 566 } 567 568 /* 569 * Return the roothub port for a controller port. Both are 1..n. 570 */ 571 static inline unsigned int 572 xhci_ctlrport2rhport(struct xhci_softc * const sc, unsigned int ctrlport) 573 { 574 575 return sc->sc_ctlrportmap[ctrlport - 1]; 576 } 577 578 /* 579 * Return the controller port for a bus roothub port. Both are 1..n. 580 */ 581 static inline unsigned int 582 xhci_rhport2ctlrport(struct xhci_softc * const sc, unsigned int bn, 583 unsigned int rhport) 584 { 585 586 return sc->sc_rhportmap[bn][rhport - 1]; 587 } 588 589 /* --- */ 590 591 void 592 xhci_childdet(device_t self, device_t child) 593 { 594 struct xhci_softc * const sc = device_private(self); 595 596 KASSERT((sc->sc_child == child) || (sc->sc_child2 == child)); 597 if (child == sc->sc_child2) 598 sc->sc_child2 = NULL; 599 else if (child == sc->sc_child) 600 sc->sc_child = NULL; 601 } 602 603 int 604 xhci_detach(struct xhci_softc *sc, int flags) 605 { 606 int rv = 0; 607 608 if (sc->sc_child2 != NULL) { 609 rv = config_detach(sc->sc_child2, flags); 610 if (rv != 0) 611 return rv; 612 KASSERT(sc->sc_child2 == NULL); 613 } 614 615 if (sc->sc_child != NULL) { 616 rv = config_detach(sc->sc_child, flags); 617 if (rv != 0) 618 return rv; 619 KASSERT(sc->sc_child == NULL); 620 } 621 622 /* XXX unconfigure/free slots */ 623 624 /* verify: */ 625 xhci_rt_write_4(sc, XHCI_IMAN(0), 0); 626 xhci_op_write_4(sc, XHCI_USBCMD, 0); 627 /* do we need to wait for stop? */ 628 629 xhci_op_write_8(sc, XHCI_CRCR, 0); 630 xhci_ring_free(sc, &sc->sc_cr); 631 cv_destroy(&sc->sc_command_cv); 632 cv_destroy(&sc->sc_cmdbusy_cv); 633 634 xhci_rt_write_4(sc, XHCI_ERSTSZ(0), 0); 635 xhci_rt_write_8(sc, XHCI_ERSTBA(0), 0); 636 xhci_rt_write_8(sc, XHCI_ERDP(0), 0|XHCI_ERDP_LO_BUSY); 637 xhci_ring_free(sc, &sc->sc_er); 638 639 usb_freemem(&sc->sc_bus, &sc->sc_eventst_dma); 640 641 xhci_op_write_8(sc, XHCI_DCBAAP, 0); 642 usb_freemem(&sc->sc_bus, &sc->sc_dcbaa_dma); 643 644 kmem_free(sc->sc_slots, sizeof(*sc->sc_slots) * sc->sc_maxslots); 645 646 kmem_free(sc->sc_ctlrportbus, 647 howmany(sc->sc_maxports * sizeof(uint8_t), NBBY)); 648 kmem_free(sc->sc_ctlrportmap, sc->sc_maxports * sizeof(int)); 649 650 for (size_t j = 0; j < __arraycount(sc->sc_rhportmap); j++) { 651 kmem_free(sc->sc_rhportmap[j], sc->sc_maxports * sizeof(int)); 652 } 653 654 mutex_destroy(&sc->sc_lock); 655 mutex_destroy(&sc->sc_intr_lock); 656 657 pool_cache_destroy(sc->sc_xferpool); 658 659 return rv; 660 } 661 662 int 663 xhci_activate(device_t self, enum devact act) 664 { 665 struct xhci_softc * const sc = device_private(self); 666 667 switch (act) { 668 case DVACT_DEACTIVATE: 669 sc->sc_dying = true; 670 return 0; 671 default: 672 return EOPNOTSUPP; 673 } 674 } 675 676 bool 677 xhci_suspend(device_t dv, const pmf_qual_t *qual) 678 { 679 return false; 680 } 681 682 bool 683 xhci_resume(device_t dv, const pmf_qual_t *qual) 684 { 685 return false; 686 } 687 688 bool 689 xhci_shutdown(device_t self, int flags) 690 { 691 return false; 692 } 693 694 static int 695 xhci_hc_reset(struct xhci_softc * const sc) 696 { 697 uint32_t usbcmd, usbsts; 698 int i; 699 700 /* Check controller not ready */ 701 for (i = 0; i < XHCI_WAIT_CNR; i++) { 702 usbsts = xhci_op_read_4(sc, XHCI_USBSTS); 703 if ((usbsts & XHCI_STS_CNR) == 0) 704 break; 705 usb_delay_ms(&sc->sc_bus, 1); 706 } 707 if (i >= XHCI_WAIT_CNR) { 708 aprint_error_dev(sc->sc_dev, "controller not ready timeout\n"); 709 return EIO; 710 } 711 712 /* Halt controller */ 713 usbcmd = 0; 714 xhci_op_write_4(sc, XHCI_USBCMD, usbcmd); 715 usb_delay_ms(&sc->sc_bus, 1); 716 717 /* Reset controller */ 718 usbcmd = XHCI_CMD_HCRST; 719 xhci_op_write_4(sc, XHCI_USBCMD, usbcmd); 720 for (i = 0; i < XHCI_WAIT_HCRST; i++) { 721 /* 722 * Wait 1ms first. Existing Intel xHCI requies 1ms delay to 723 * prevent system hang (Errata). 724 */ 725 usb_delay_ms(&sc->sc_bus, 1); 726 usbcmd = xhci_op_read_4(sc, XHCI_USBCMD); 727 if ((usbcmd & XHCI_CMD_HCRST) == 0) 728 break; 729 } 730 if (i >= XHCI_WAIT_HCRST) { 731 aprint_error_dev(sc->sc_dev, "host controller reset timeout\n"); 732 return EIO; 733 } 734 735 /* Check controller not ready */ 736 for (i = 0; i < XHCI_WAIT_CNR; i++) { 737 usbsts = xhci_op_read_4(sc, XHCI_USBSTS); 738 if ((usbsts & XHCI_STS_CNR) == 0) 739 break; 740 usb_delay_ms(&sc->sc_bus, 1); 741 } 742 if (i >= XHCI_WAIT_CNR) { 743 aprint_error_dev(sc->sc_dev, 744 "controller not ready timeout after reset\n"); 745 return EIO; 746 } 747 748 return 0; 749 } 750 751 752 /* 7.2 xHCI Support Protocol Capability */ 753 static void 754 xhci_id_protocols(struct xhci_softc *sc, bus_size_t ecp) 755 { 756 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 757 758 /* XXX Cache this lot */ 759 760 const uint32_t w0 = xhci_read_4(sc, ecp); 761 const uint32_t w4 = xhci_read_4(sc, ecp + 4); 762 const uint32_t w8 = xhci_read_4(sc, ecp + 8); 763 const uint32_t wc = xhci_read_4(sc, ecp + 0xc); 764 765 aprint_debug_dev(sc->sc_dev, 766 " SP: %08x %08x %08x %08x\n", w0, w4, w8, wc); 767 768 if (w4 != XHCI_XECP_USBID) 769 return; 770 771 const int major = XHCI_XECP_SP_W0_MAJOR(w0); 772 const int minor = XHCI_XECP_SP_W0_MINOR(w0); 773 const uint8_t cpo = XHCI_XECP_SP_W8_CPO(w8); 774 const uint8_t cpc = XHCI_XECP_SP_W8_CPC(w8); 775 776 const uint16_t mm = __SHIFTOUT(w0, __BITS(31, 16)); 777 switch (mm) { 778 case 0x0200: 779 case 0x0300: 780 case 0x0301: 781 case 0x0310: 782 aprint_debug_dev(sc->sc_dev, " %s ports %d - %d\n", 783 major == 3 ? "ss" : "hs", cpo, cpo + cpc -1); 784 break; 785 default: 786 aprint_debug_dev(sc->sc_dev, " unknown major/minor (%d/%d)\n", 787 major, minor); 788 return; 789 } 790 791 const size_t bus = (major == 3) ? 0 : 1; 792 793 /* Index arrays with 0..n-1 where ports are numbered 1..n */ 794 for (size_t cp = cpo - 1; cp < cpo + cpc - 1; cp++) { 795 if (sc->sc_ctlrportmap[cp] != 0) { 796 aprint_error_dev(sc->sc_dev, "controller port %zu " 797 "already assigned", cp); 798 continue; 799 } 800 801 sc->sc_ctlrportbus[cp / NBBY] |= 802 bus == 0 ? 0 : __BIT(cp % NBBY); 803 804 const size_t rhp = sc->sc_rhportcount[bus]++; 805 806 KASSERTMSG(sc->sc_rhportmap[bus][rhp] == 0, 807 "bus %zu rhp %zu is %d", bus, rhp, 808 sc->sc_rhportmap[bus][rhp]); 809 810 sc->sc_rhportmap[bus][rhp] = cp + 1; 811 sc->sc_ctlrportmap[cp] = rhp + 1; 812 } 813 } 814 815 /* Process extended capabilities */ 816 static void 817 xhci_ecp(struct xhci_softc *sc, uint32_t hcc) 818 { 819 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 820 821 bus_size_t ecp = XHCI_HCC_XECP(hcc) * 4; 822 while (ecp != 0) { 823 uint32_t ecr = xhci_read_4(sc, ecp); 824 aprint_debug_dev(sc->sc_dev, "ECR: 0x%08x\n", ecr); 825 switch (XHCI_XECP_ID(ecr)) { 826 case XHCI_ID_PROTOCOLS: { 827 xhci_id_protocols(sc, ecp); 828 break; 829 } 830 case XHCI_ID_USB_LEGACY: { 831 uint8_t bios_sem; 832 833 /* Take host controller ownership from BIOS */ 834 bios_sem = xhci_read_1(sc, ecp + XHCI_XECP_BIOS_SEM); 835 if (bios_sem) { 836 /* sets xHCI to be owned by OS */ 837 xhci_write_1(sc, ecp + XHCI_XECP_OS_SEM, 1); 838 aprint_debug_dev(sc->sc_dev, 839 "waiting for BIOS to give up control\n"); 840 for (int i = 0; i < 5000; i++) { 841 bios_sem = xhci_read_1(sc, ecp + 842 XHCI_XECP_BIOS_SEM); 843 if (bios_sem == 0) 844 break; 845 DELAY(1000); 846 } 847 if (bios_sem) { 848 aprint_error_dev(sc->sc_dev, 849 "timed out waiting for BIOS\n"); 850 } 851 } 852 break; 853 } 854 default: 855 break; 856 } 857 ecr = xhci_read_4(sc, ecp); 858 if (XHCI_XECP_NEXT(ecr) == 0) { 859 ecp = 0; 860 } else { 861 ecp += XHCI_XECP_NEXT(ecr) * 4; 862 } 863 } 864 } 865 866 #define XHCI_HCCPREV1_BITS \ 867 "\177\020" /* New bitmask */ \ 868 "f\020\020XECP\0" \ 869 "f\014\4MAXPSA\0" \ 870 "b\013CFC\0" \ 871 "b\012SEC\0" \ 872 "b\011SBD\0" \ 873 "b\010FSE\0" \ 874 "b\7NSS\0" \ 875 "b\6LTC\0" \ 876 "b\5LHRC\0" \ 877 "b\4PIND\0" \ 878 "b\3PPC\0" \ 879 "b\2CZC\0" \ 880 "b\1BNC\0" \ 881 "b\0AC64\0" \ 882 "\0" 883 #define XHCI_HCCV1_x_BITS \ 884 "\177\020" /* New bitmask */ \ 885 "f\020\020XECP\0" \ 886 "f\014\4MAXPSA\0" \ 887 "b\013CFC\0" \ 888 "b\012SEC\0" \ 889 "b\011SPC\0" \ 890 "b\010PAE\0" \ 891 "b\7NSS\0" \ 892 "b\6LTC\0" \ 893 "b\5LHRC\0" \ 894 "b\4PIND\0" \ 895 "b\3PPC\0" \ 896 "b\2CSZ\0" \ 897 "b\1BNC\0" \ 898 "b\0AC64\0" \ 899 "\0" 900 901 #define XHCI_HCC2_BITS \ 902 "\177\020" /* New bitmask */ \ 903 "b\7ETC_TSC\0" \ 904 "b\6ETC\0" \ 905 "b\5CIC\0" \ 906 "b\4LEC\0" \ 907 "b\3CTC\0" \ 908 "b\2FSC\0" \ 909 "b\1CMC\0" \ 910 "b\0U3C\0" \ 911 "\0" 912 913 void 914 xhci_start(struct xhci_softc *sc) 915 { 916 xhci_rt_write_4(sc, XHCI_IMAN(0), XHCI_IMAN_INTR_ENA); 917 if ((sc->sc_quirks & XHCI_QUIRK_INTEL) != 0) 918 /* Intel xhci needs interrupt rate moderated. */ 919 xhci_rt_write_4(sc, XHCI_IMOD(0), XHCI_IMOD_DEFAULT_LP); 920 else 921 xhci_rt_write_4(sc, XHCI_IMOD(0), 0); 922 aprint_debug_dev(sc->sc_dev, "current IMOD %u\n", 923 xhci_rt_read_4(sc, XHCI_IMOD(0))); 924 925 /* Go! */ 926 xhci_op_write_4(sc, XHCI_USBCMD, XHCI_CMD_INTE|XHCI_CMD_RS); 927 aprint_debug_dev(sc->sc_dev, "USBCMD %08"PRIx32"\n", 928 xhci_op_read_4(sc, XHCI_USBCMD)); 929 } 930 931 int 932 xhci_init(struct xhci_softc *sc) 933 { 934 bus_size_t bsz; 935 uint32_t cap, hcs1, hcs2, hcs3, hcc, dboff, rtsoff, hcc2; 936 uint32_t pagesize, config; 937 int i = 0; 938 uint16_t hciversion; 939 uint8_t caplength; 940 941 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 942 943 /* Set up the bus struct for the usb 3 and usb 2 buses */ 944 sc->sc_bus.ub_methods = &xhci_bus_methods; 945 sc->sc_bus.ub_pipesize = sizeof(struct xhci_pipe); 946 sc->sc_bus.ub_usedma = true; 947 sc->sc_bus.ub_hcpriv = sc; 948 949 sc->sc_bus2.ub_methods = &xhci_bus_methods; 950 sc->sc_bus2.ub_pipesize = sizeof(struct xhci_pipe); 951 sc->sc_bus2.ub_revision = USBREV_2_0; 952 sc->sc_bus2.ub_usedma = true; 953 sc->sc_bus2.ub_hcpriv = sc; 954 sc->sc_bus2.ub_dmatag = sc->sc_bus.ub_dmatag; 955 956 cap = xhci_read_4(sc, XHCI_CAPLENGTH); 957 caplength = XHCI_CAP_CAPLENGTH(cap); 958 hciversion = XHCI_CAP_HCIVERSION(cap); 959 960 if (hciversion < XHCI_HCIVERSION_0_96 || 961 hciversion >= 0x0200) { 962 aprint_normal_dev(sc->sc_dev, 963 "xHCI version %x.%x not known to be supported\n", 964 (hciversion >> 8) & 0xff, (hciversion >> 0) & 0xff); 965 } else { 966 aprint_verbose_dev(sc->sc_dev, "xHCI version %x.%x\n", 967 (hciversion >> 8) & 0xff, (hciversion >> 0) & 0xff); 968 } 969 970 if (bus_space_subregion(sc->sc_iot, sc->sc_ioh, 0, caplength, 971 &sc->sc_cbh) != 0) { 972 aprint_error_dev(sc->sc_dev, "capability subregion failure\n"); 973 return ENOMEM; 974 } 975 976 hcs1 = xhci_cap_read_4(sc, XHCI_HCSPARAMS1); 977 sc->sc_maxslots = XHCI_HCS1_MAXSLOTS(hcs1); 978 sc->sc_maxintrs = XHCI_HCS1_MAXINTRS(hcs1); 979 sc->sc_maxports = XHCI_HCS1_MAXPORTS(hcs1); 980 hcs2 = xhci_cap_read_4(sc, XHCI_HCSPARAMS2); 981 hcs3 = xhci_cap_read_4(sc, XHCI_HCSPARAMS3); 982 aprint_debug_dev(sc->sc_dev, 983 "hcs1=%"PRIx32" hcs2=%"PRIx32" hcs3=%"PRIx32"\n", hcs1, hcs2, hcs3); 984 985 hcc = xhci_cap_read_4(sc, XHCI_HCCPARAMS); 986 sc->sc_ac64 = XHCI_HCC_AC64(hcc); 987 sc->sc_ctxsz = XHCI_HCC_CSZ(hcc) ? 64 : 32; 988 989 char sbuf[128]; 990 if (hciversion < XHCI_HCIVERSION_1_0) 991 snprintb(sbuf, sizeof(sbuf), XHCI_HCCPREV1_BITS, hcc); 992 else 993 snprintb(sbuf, sizeof(sbuf), XHCI_HCCV1_x_BITS, hcc); 994 aprint_debug_dev(sc->sc_dev, "hcc=%s\n", sbuf); 995 aprint_debug_dev(sc->sc_dev, "xECP %x\n", XHCI_HCC_XECP(hcc) * 4); 996 if (hciversion >= XHCI_HCIVERSION_1_1) { 997 hcc2 = xhci_cap_read_4(sc, XHCI_HCCPARAMS2); 998 snprintb(sbuf, sizeof(sbuf), XHCI_HCC2_BITS, hcc2); 999 aprint_debug_dev(sc->sc_dev, "hcc2=%s\n", sbuf); 1000 } 1001 1002 /* default all ports to bus 0, i.e. usb 3 */ 1003 sc->sc_ctlrportbus = kmem_zalloc( 1004 howmany(sc->sc_maxports * sizeof(uint8_t), NBBY), KM_SLEEP); 1005 sc->sc_ctlrportmap = kmem_zalloc(sc->sc_maxports * sizeof(int), KM_SLEEP); 1006 1007 /* controller port to bus roothub port map */ 1008 for (size_t j = 0; j < __arraycount(sc->sc_rhportmap); j++) { 1009 sc->sc_rhportmap[j] = kmem_zalloc(sc->sc_maxports * sizeof(int), KM_SLEEP); 1010 } 1011 1012 /* 1013 * Process all Extended Capabilities 1014 */ 1015 xhci_ecp(sc, hcc); 1016 1017 bsz = XHCI_PORTSC(sc->sc_maxports); 1018 if (bus_space_subregion(sc->sc_iot, sc->sc_ioh, caplength, bsz, 1019 &sc->sc_obh) != 0) { 1020 aprint_error_dev(sc->sc_dev, "operational subregion failure\n"); 1021 return ENOMEM; 1022 } 1023 1024 dboff = xhci_cap_read_4(sc, XHCI_DBOFF); 1025 if (bus_space_subregion(sc->sc_iot, sc->sc_ioh, dboff, 1026 sc->sc_maxslots * 4, &sc->sc_dbh) != 0) { 1027 aprint_error_dev(sc->sc_dev, "doorbell subregion failure\n"); 1028 return ENOMEM; 1029 } 1030 1031 rtsoff = xhci_cap_read_4(sc, XHCI_RTSOFF); 1032 if (bus_space_subregion(sc->sc_iot, sc->sc_ioh, rtsoff, 1033 sc->sc_maxintrs * 0x20, &sc->sc_rbh) != 0) { 1034 aprint_error_dev(sc->sc_dev, "runtime subregion failure\n"); 1035 return ENOMEM; 1036 } 1037 1038 int rv; 1039 rv = xhci_hc_reset(sc); 1040 if (rv != 0) { 1041 return rv; 1042 } 1043 1044 if (sc->sc_vendor_init) 1045 sc->sc_vendor_init(sc); 1046 1047 pagesize = xhci_op_read_4(sc, XHCI_PAGESIZE); 1048 aprint_debug_dev(sc->sc_dev, "PAGESIZE 0x%08x\n", pagesize); 1049 pagesize = ffs(pagesize); 1050 if (pagesize == 0) { 1051 aprint_error_dev(sc->sc_dev, "pagesize is 0\n"); 1052 return EIO; 1053 } 1054 sc->sc_pgsz = 1 << (12 + (pagesize - 1)); 1055 aprint_debug_dev(sc->sc_dev, "sc_pgsz 0x%08x\n", (uint32_t)sc->sc_pgsz); 1056 aprint_debug_dev(sc->sc_dev, "sc_maxslots 0x%08x\n", 1057 (uint32_t)sc->sc_maxslots); 1058 aprint_debug_dev(sc->sc_dev, "sc_maxports %d\n", sc->sc_maxports); 1059 1060 usbd_status err; 1061 1062 sc->sc_maxspbuf = XHCI_HCS2_MAXSPBUF(hcs2); 1063 aprint_debug_dev(sc->sc_dev, "sc_maxspbuf %d\n", sc->sc_maxspbuf); 1064 if (sc->sc_maxspbuf != 0) { 1065 err = usb_allocmem(&sc->sc_bus, 1066 sizeof(uint64_t) * sc->sc_maxspbuf, sizeof(uint64_t), 1067 &sc->sc_spbufarray_dma); 1068 if (err) { 1069 aprint_error_dev(sc->sc_dev, 1070 "spbufarray init fail, err %d\n", err); 1071 return ENOMEM; 1072 } 1073 1074 sc->sc_spbuf_dma = kmem_zalloc(sizeof(*sc->sc_spbuf_dma) * 1075 sc->sc_maxspbuf, KM_SLEEP); 1076 uint64_t *spbufarray = KERNADDR(&sc->sc_spbufarray_dma, 0); 1077 for (i = 0; i < sc->sc_maxspbuf; i++) { 1078 usb_dma_t * const dma = &sc->sc_spbuf_dma[i]; 1079 /* allocate contexts */ 1080 err = usb_allocmem(&sc->sc_bus, sc->sc_pgsz, 1081 sc->sc_pgsz, dma); 1082 if (err) { 1083 aprint_error_dev(sc->sc_dev, 1084 "spbufarray_dma init fail, err %d\n", err); 1085 rv = ENOMEM; 1086 goto bad1; 1087 } 1088 spbufarray[i] = htole64(DMAADDR(dma, 0)); 1089 usb_syncmem(dma, 0, sc->sc_pgsz, 1090 BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE); 1091 } 1092 1093 usb_syncmem(&sc->sc_spbufarray_dma, 0, 1094 sizeof(uint64_t) * sc->sc_maxspbuf, BUS_DMASYNC_PREWRITE); 1095 } 1096 1097 config = xhci_op_read_4(sc, XHCI_CONFIG); 1098 config &= ~0xFF; 1099 config |= sc->sc_maxslots & 0xFF; 1100 xhci_op_write_4(sc, XHCI_CONFIG, config); 1101 1102 err = xhci_ring_init(sc, &sc->sc_cr, XHCI_COMMAND_RING_TRBS, 1103 XHCI_COMMAND_RING_SEGMENTS_ALIGN); 1104 if (err) { 1105 aprint_error_dev(sc->sc_dev, "command ring init fail, err %d\n", 1106 err); 1107 rv = ENOMEM; 1108 goto bad1; 1109 } 1110 1111 err = xhci_ring_init(sc, &sc->sc_er, XHCI_EVENT_RING_TRBS, 1112 XHCI_EVENT_RING_SEGMENTS_ALIGN); 1113 if (err) { 1114 aprint_error_dev(sc->sc_dev, "event ring init fail, err %d\n", 1115 err); 1116 rv = ENOMEM; 1117 goto bad2; 1118 } 1119 1120 usb_dma_t *dma; 1121 size_t size; 1122 size_t align; 1123 1124 dma = &sc->sc_eventst_dma; 1125 size = roundup2(XHCI_EVENT_RING_SEGMENTS * XHCI_ERSTE_SIZE, 1126 XHCI_EVENT_RING_SEGMENT_TABLE_ALIGN); 1127 KASSERTMSG(size <= (512 * 1024), "eventst size %zu too large", size); 1128 align = XHCI_EVENT_RING_SEGMENT_TABLE_ALIGN; 1129 err = usb_allocmem(&sc->sc_bus, size, align, dma); 1130 if (err) { 1131 aprint_error_dev(sc->sc_dev, "eventst init fail, err %d\n", 1132 err); 1133 rv = ENOMEM; 1134 goto bad3; 1135 } 1136 1137 memset(KERNADDR(dma, 0), 0, size); 1138 usb_syncmem(dma, 0, size, BUS_DMASYNC_PREWRITE); 1139 aprint_debug_dev(sc->sc_dev, "eventst: %016jx %p %zx\n", 1140 (uintmax_t)DMAADDR(&sc->sc_eventst_dma, 0), 1141 KERNADDR(&sc->sc_eventst_dma, 0), 1142 sc->sc_eventst_dma.udma_block->size); 1143 1144 dma = &sc->sc_dcbaa_dma; 1145 size = (1 + sc->sc_maxslots) * sizeof(uint64_t); 1146 KASSERTMSG(size <= 2048, "dcbaa size %zu too large", size); 1147 align = XHCI_DEVICE_CONTEXT_BASE_ADDRESS_ARRAY_ALIGN; 1148 err = usb_allocmem(&sc->sc_bus, size, align, dma); 1149 if (err) { 1150 aprint_error_dev(sc->sc_dev, "dcbaa init fail, err %d\n", err); 1151 rv = ENOMEM; 1152 goto bad4; 1153 } 1154 aprint_debug_dev(sc->sc_dev, "dcbaa: %016jx %p %zx\n", 1155 (uintmax_t)DMAADDR(&sc->sc_dcbaa_dma, 0), 1156 KERNADDR(&sc->sc_dcbaa_dma, 0), 1157 sc->sc_dcbaa_dma.udma_block->size); 1158 1159 memset(KERNADDR(dma, 0), 0, size); 1160 if (sc->sc_maxspbuf != 0) { 1161 /* 1162 * DCBA entry 0 hold the scratchbuf array pointer. 1163 */ 1164 *(uint64_t *)KERNADDR(dma, 0) = 1165 htole64(DMAADDR(&sc->sc_spbufarray_dma, 0)); 1166 } 1167 usb_syncmem(dma, 0, size, BUS_DMASYNC_PREWRITE); 1168 1169 sc->sc_slots = kmem_zalloc(sizeof(*sc->sc_slots) * sc->sc_maxslots, 1170 KM_SLEEP); 1171 if (sc->sc_slots == NULL) { 1172 aprint_error_dev(sc->sc_dev, "slots init fail, err %d\n", err); 1173 rv = ENOMEM; 1174 goto bad; 1175 } 1176 1177 sc->sc_xferpool = pool_cache_init(sizeof(struct xhci_xfer), 0, 0, 0, 1178 "xhcixfer", NULL, IPL_USB, NULL, NULL, NULL); 1179 if (sc->sc_xferpool == NULL) { 1180 aprint_error_dev(sc->sc_dev, "pool_cache init fail, err %d\n", 1181 err); 1182 rv = ENOMEM; 1183 goto bad; 1184 } 1185 1186 cv_init(&sc->sc_command_cv, "xhcicmd"); 1187 cv_init(&sc->sc_cmdbusy_cv, "xhcicmdq"); 1188 mutex_init(&sc->sc_lock, MUTEX_DEFAULT, IPL_SOFTUSB); 1189 mutex_init(&sc->sc_intr_lock, MUTEX_DEFAULT, IPL_USB); 1190 1191 struct xhci_erste *erst; 1192 erst = KERNADDR(&sc->sc_eventst_dma, 0); 1193 erst[0].erste_0 = htole64(xhci_ring_trbp(&sc->sc_er, 0)); 1194 erst[0].erste_2 = htole32(sc->sc_er.xr_ntrb); 1195 erst[0].erste_3 = htole32(0); 1196 usb_syncmem(&sc->sc_eventst_dma, 0, 1197 XHCI_ERSTE_SIZE * XHCI_EVENT_RING_SEGMENTS, BUS_DMASYNC_PREWRITE); 1198 1199 xhci_rt_write_4(sc, XHCI_ERSTSZ(0), XHCI_EVENT_RING_SEGMENTS); 1200 xhci_rt_write_8(sc, XHCI_ERSTBA(0), DMAADDR(&sc->sc_eventst_dma, 0)); 1201 xhci_rt_write_8(sc, XHCI_ERDP(0), xhci_ring_trbp(&sc->sc_er, 0) | 1202 XHCI_ERDP_LO_BUSY); 1203 1204 xhci_op_write_8(sc, XHCI_DCBAAP, DMAADDR(&sc->sc_dcbaa_dma, 0)); 1205 xhci_op_write_8(sc, XHCI_CRCR, xhci_ring_trbp(&sc->sc_cr, 0) | 1206 sc->sc_cr.xr_cs); 1207 1208 xhci_op_barrier(sc, 0, 4, BUS_SPACE_BARRIER_WRITE); 1209 1210 HEXDUMP("eventst", KERNADDR(&sc->sc_eventst_dma, 0), 1211 XHCI_ERSTE_SIZE * XHCI_EVENT_RING_SEGMENTS); 1212 1213 if ((sc->sc_quirks & XHCI_DEFERRED_START) == 0) 1214 xhci_start(sc); 1215 1216 return 0; 1217 1218 bad: 1219 if (sc->sc_xferpool) { 1220 pool_cache_destroy(sc->sc_xferpool); 1221 sc->sc_xferpool = NULL; 1222 } 1223 1224 if (sc->sc_slots) { 1225 kmem_free(sc->sc_slots, sizeof(*sc->sc_slots) * 1226 sc->sc_maxslots); 1227 sc->sc_slots = NULL; 1228 } 1229 1230 usb_freemem(&sc->sc_bus, &sc->sc_dcbaa_dma); 1231 bad4: 1232 usb_freemem(&sc->sc_bus, &sc->sc_eventst_dma); 1233 bad3: 1234 xhci_ring_free(sc, &sc->sc_er); 1235 bad2: 1236 xhci_ring_free(sc, &sc->sc_cr); 1237 i = sc->sc_maxspbuf; 1238 bad1: 1239 for (int j = 0; j < i; j++) 1240 usb_freemem(&sc->sc_bus, &sc->sc_spbuf_dma[j]); 1241 usb_freemem(&sc->sc_bus, &sc->sc_spbufarray_dma); 1242 1243 return rv; 1244 } 1245 1246 static inline bool 1247 xhci_polling_p(struct xhci_softc * const sc) 1248 { 1249 return sc->sc_bus.ub_usepolling || sc->sc_bus2.ub_usepolling; 1250 } 1251 1252 int 1253 xhci_intr(void *v) 1254 { 1255 struct xhci_softc * const sc = v; 1256 int ret = 0; 1257 1258 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 1259 1260 if (sc == NULL) 1261 return 0; 1262 1263 mutex_spin_enter(&sc->sc_intr_lock); 1264 1265 if (sc->sc_dying || !device_has_power(sc->sc_dev)) 1266 goto done; 1267 1268 /* If we get an interrupt while polling, then just ignore it. */ 1269 if (xhci_polling_p(sc)) { 1270 #ifdef DIAGNOSTIC 1271 DPRINTFN(16, "ignored interrupt while polling", 0, 0, 0, 0); 1272 #endif 1273 goto done; 1274 } 1275 1276 ret = xhci_intr1(sc); 1277 if (ret) { 1278 KASSERT(sc->sc_child || sc->sc_child2); 1279 1280 /* 1281 * One of child busses could be already detached. It doesn't 1282 * matter on which of the two the softintr is scheduled. 1283 */ 1284 if (sc->sc_child) 1285 usb_schedsoftintr(&sc->sc_bus); 1286 else 1287 usb_schedsoftintr(&sc->sc_bus2); 1288 } 1289 done: 1290 mutex_spin_exit(&sc->sc_intr_lock); 1291 return ret; 1292 } 1293 1294 int 1295 xhci_intr1(struct xhci_softc * const sc) 1296 { 1297 uint32_t usbsts; 1298 uint32_t iman; 1299 1300 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 1301 1302 usbsts = xhci_op_read_4(sc, XHCI_USBSTS); 1303 DPRINTFN(16, "USBSTS %08jx", usbsts, 0, 0, 0); 1304 if ((usbsts & (XHCI_STS_HSE | XHCI_STS_EINT | XHCI_STS_PCD | 1305 XHCI_STS_HCE)) == 0) { 1306 DPRINTFN(16, "ignored intr not for %s", 1307 (uintptr_t)device_xname(sc->sc_dev), 0, 0, 0); 1308 return 0; 1309 } 1310 1311 /* 1312 * Clear EINT and other transient flags, to not misenterpret 1313 * next shared interrupt. Also, to avoid race, EINT must be cleared 1314 * before XHCI_IMAN_INTR_PEND is cleared. 1315 */ 1316 xhci_op_write_4(sc, XHCI_USBSTS, usbsts & XHCI_STS_RSVDP0); 1317 1318 #ifdef XHCI_DEBUG 1319 usbsts = xhci_op_read_4(sc, XHCI_USBSTS); 1320 DPRINTFN(16, "USBSTS %08jx", usbsts, 0, 0, 0); 1321 #endif 1322 1323 iman = xhci_rt_read_4(sc, XHCI_IMAN(0)); 1324 DPRINTFN(16, "IMAN0 %08jx", iman, 0, 0, 0); 1325 iman |= XHCI_IMAN_INTR_PEND; 1326 xhci_rt_write_4(sc, XHCI_IMAN(0), iman); 1327 1328 #ifdef XHCI_DEBUG 1329 iman = xhci_rt_read_4(sc, XHCI_IMAN(0)); 1330 DPRINTFN(16, "IMAN0 %08jx", iman, 0, 0, 0); 1331 usbsts = xhci_op_read_4(sc, XHCI_USBSTS); 1332 DPRINTFN(16, "USBSTS %08jx", usbsts, 0, 0, 0); 1333 #endif 1334 1335 return 1; 1336 } 1337 1338 /* 1339 * 3 port speed types used in USB stack 1340 * 1341 * usbdi speed 1342 * definition: USB_SPEED_* in usb.h 1343 * They are used in struct usbd_device in USB stack. 1344 * ioctl interface uses these values too. 1345 * port_status speed 1346 * definition: UPS_*_SPEED in usb.h 1347 * They are used in usb_port_status_t and valid only for USB 2.0. 1348 * Speed value is always 0 for Super Speed or more, and dwExtPortStatus 1349 * of usb_port_status_ext_t indicates port speed. 1350 * Note that some 3.0 values overlap with 2.0 values. 1351 * (e.g. 0x200 means UPS_POER_POWER_SS in SS and 1352 * means UPS_LOW_SPEED in HS.) 1353 * port status returned from hub also uses these values. 1354 * On NetBSD UPS_OTHER_SPEED indicates port speed is super speed 1355 * or more. 1356 * xspeed: 1357 * definition: Protocol Speed ID (PSI) (xHCI 1.1 7.2.1) 1358 * They are used in only slot context and PORTSC reg of xhci. 1359 * The difference between usbdi speed and xspeed is 1360 * that FS and LS values are swapped. 1361 */ 1362 1363 /* convert usbdi speed to xspeed */ 1364 static int 1365 xhci_speed2xspeed(int speed) 1366 { 1367 switch (speed) { 1368 case USB_SPEED_LOW: return 2; 1369 case USB_SPEED_FULL: return 1; 1370 default: return speed; 1371 } 1372 } 1373 1374 #if 0 1375 /* convert xspeed to usbdi speed */ 1376 static int 1377 xhci_xspeed2speed(int xspeed) 1378 { 1379 switch (xspeed) { 1380 case 1: return USB_SPEED_FULL; 1381 case 2: return USB_SPEED_LOW; 1382 default: return xspeed; 1383 } 1384 } 1385 #endif 1386 1387 /* convert xspeed to port status speed */ 1388 static int 1389 xhci_xspeed2psspeed(int xspeed) 1390 { 1391 switch (xspeed) { 1392 case 0: return 0; 1393 case 1: return UPS_FULL_SPEED; 1394 case 2: return UPS_LOW_SPEED; 1395 case 3: return UPS_HIGH_SPEED; 1396 default: return UPS_OTHER_SPEED; 1397 } 1398 } 1399 1400 /* 1401 * Construct input contexts and issue TRB to open pipe. 1402 */ 1403 static usbd_status 1404 xhci_configure_endpoint(struct usbd_pipe *pipe) 1405 { 1406 struct xhci_softc * const sc = XHCI_PIPE2SC(pipe); 1407 struct xhci_slot * const xs = pipe->up_dev->ud_hcpriv; 1408 #ifdef USB_DEBUG 1409 const u_int dci = xhci_ep_get_dci(pipe->up_endpoint->ue_edesc); 1410 #endif 1411 struct xhci_soft_trb trb; 1412 usbd_status err; 1413 1414 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 1415 DPRINTFN(4, "slot %ju dci %ju epaddr 0x%02jx attr 0x%02jx", 1416 xs->xs_idx, dci, pipe->up_endpoint->ue_edesc->bEndpointAddress, 1417 pipe->up_endpoint->ue_edesc->bmAttributes); 1418 1419 /* XXX ensure input context is available? */ 1420 1421 memset(xhci_slot_get_icv(sc, xs, 0), 0, sc->sc_pgsz); 1422 1423 /* set up context */ 1424 xhci_setup_ctx(pipe); 1425 1426 HEXDUMP("input control context", xhci_slot_get_icv(sc, xs, 0), 1427 sc->sc_ctxsz * 1); 1428 HEXDUMP("input endpoint context", xhci_slot_get_icv(sc, xs, 1429 xhci_dci_to_ici(dci)), sc->sc_ctxsz * 1); 1430 1431 trb.trb_0 = xhci_slot_get_icp(sc, xs, 0); 1432 trb.trb_2 = 0; 1433 trb.trb_3 = XHCI_TRB_3_SLOT_SET(xs->xs_idx) | 1434 XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_CONFIGURE_EP); 1435 1436 err = xhci_do_command(sc, &trb, USBD_DEFAULT_TIMEOUT); 1437 1438 usb_syncmem(&xs->xs_dc_dma, 0, sc->sc_pgsz, BUS_DMASYNC_POSTREAD); 1439 HEXDUMP("output context", xhci_slot_get_dcv(sc, xs, dci), 1440 sc->sc_ctxsz * 1); 1441 1442 return err; 1443 } 1444 1445 #if 0 1446 static usbd_status 1447 xhci_unconfigure_endpoint(struct usbd_pipe *pipe) 1448 { 1449 #ifdef USB_DEBUG 1450 struct xhci_slot * const xs = pipe->up_dev->ud_hcpriv; 1451 #endif 1452 1453 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 1454 DPRINTFN(4, "slot %ju", xs->xs_idx, 0, 0, 0); 1455 1456 return USBD_NORMAL_COMPLETION; 1457 } 1458 #endif 1459 1460 /* 4.6.8, 6.4.3.7 */ 1461 static usbd_status 1462 xhci_reset_endpoint_locked(struct usbd_pipe *pipe) 1463 { 1464 struct xhci_softc * const sc = XHCI_PIPE2SC(pipe); 1465 struct xhci_slot * const xs = pipe->up_dev->ud_hcpriv; 1466 const u_int dci = xhci_ep_get_dci(pipe->up_endpoint->ue_edesc); 1467 struct xhci_soft_trb trb; 1468 usbd_status err; 1469 1470 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 1471 DPRINTFN(4, "slot %ju dci %ju", xs->xs_idx, dci, 0, 0); 1472 1473 KASSERT(mutex_owned(&sc->sc_lock)); 1474 1475 trb.trb_0 = 0; 1476 trb.trb_2 = 0; 1477 trb.trb_3 = XHCI_TRB_3_SLOT_SET(xs->xs_idx) | 1478 XHCI_TRB_3_EP_SET(dci) | 1479 XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_RESET_EP); 1480 1481 err = xhci_do_command_locked(sc, &trb, USBD_DEFAULT_TIMEOUT); 1482 1483 return err; 1484 } 1485 1486 static usbd_status 1487 xhci_reset_endpoint(struct usbd_pipe *pipe) 1488 { 1489 struct xhci_softc * const sc = XHCI_PIPE2SC(pipe); 1490 1491 mutex_enter(&sc->sc_lock); 1492 usbd_status ret = xhci_reset_endpoint_locked(pipe); 1493 mutex_exit(&sc->sc_lock); 1494 1495 return ret; 1496 } 1497 1498 /* 1499 * 4.6.9, 6.4.3.8 1500 * Stop execution of TDs on xfer ring. 1501 * Should be called with sc_lock held. 1502 */ 1503 static usbd_status 1504 xhci_stop_endpoint(struct usbd_pipe *pipe) 1505 { 1506 struct xhci_softc * const sc = XHCI_PIPE2SC(pipe); 1507 struct xhci_slot * const xs = pipe->up_dev->ud_hcpriv; 1508 struct xhci_soft_trb trb; 1509 usbd_status err; 1510 const u_int dci = xhci_ep_get_dci(pipe->up_endpoint->ue_edesc); 1511 1512 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 1513 DPRINTFN(4, "slot %ju dci %ju", xs->xs_idx, dci, 0, 0); 1514 1515 KASSERT(mutex_owned(&sc->sc_lock)); 1516 1517 trb.trb_0 = 0; 1518 trb.trb_2 = 0; 1519 trb.trb_3 = XHCI_TRB_3_SLOT_SET(xs->xs_idx) | 1520 XHCI_TRB_3_EP_SET(dci) | 1521 XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_STOP_EP); 1522 1523 err = xhci_do_command_locked(sc, &trb, USBD_DEFAULT_TIMEOUT); 1524 1525 return err; 1526 } 1527 1528 /* 1529 * Set TR Dequeue Pointer. 1530 * xHCI 1.1 4.6.10 6.4.3.9 1531 * Purge all of the TRBs on ring and reinitialize ring. 1532 * Set TR dequeue Pointr to 0 and Cycle State to 1. 1533 * EPSTATE of endpoint must be ERROR or STOPPED, otherwise CONTEXT_STATE 1534 * error will be generated. 1535 */ 1536 static usbd_status 1537 xhci_set_dequeue_locked(struct usbd_pipe *pipe) 1538 { 1539 struct xhci_softc * const sc = XHCI_PIPE2SC(pipe); 1540 struct xhci_slot * const xs = pipe->up_dev->ud_hcpriv; 1541 const u_int dci = xhci_ep_get_dci(pipe->up_endpoint->ue_edesc); 1542 struct xhci_ring * const xr = &xs->xs_ep[dci].xe_tr; 1543 struct xhci_soft_trb trb; 1544 usbd_status err; 1545 1546 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 1547 DPRINTFN(4, "slot %ju dci %ju", xs->xs_idx, dci, 0, 0); 1548 1549 KASSERT(mutex_owned(&sc->sc_lock)); 1550 1551 xhci_host_dequeue(xr); 1552 1553 /* set DCS */ 1554 trb.trb_0 = xhci_ring_trbp(xr, 0) | 1; /* XXX */ 1555 trb.trb_2 = 0; 1556 trb.trb_3 = XHCI_TRB_3_SLOT_SET(xs->xs_idx) | 1557 XHCI_TRB_3_EP_SET(dci) | 1558 XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_SET_TR_DEQUEUE); 1559 1560 err = xhci_do_command_locked(sc, &trb, USBD_DEFAULT_TIMEOUT); 1561 1562 return err; 1563 } 1564 1565 static usbd_status 1566 xhci_set_dequeue(struct usbd_pipe *pipe) 1567 { 1568 struct xhci_softc * const sc = XHCI_PIPE2SC(pipe); 1569 1570 mutex_enter(&sc->sc_lock); 1571 usbd_status ret = xhci_set_dequeue_locked(pipe); 1572 mutex_exit(&sc->sc_lock); 1573 1574 return ret; 1575 } 1576 1577 /* 1578 * Open new pipe: called from usbd_setup_pipe_flags. 1579 * Fills methods of pipe. 1580 * If pipe is not for ep0, calls configure_endpoint. 1581 */ 1582 static usbd_status 1583 xhci_open(struct usbd_pipe *pipe) 1584 { 1585 struct usbd_device * const dev = pipe->up_dev; 1586 struct xhci_softc * const sc = XHCI_BUS2SC(dev->ud_bus); 1587 usb_endpoint_descriptor_t * const ed = pipe->up_endpoint->ue_edesc; 1588 const uint8_t xfertype = UE_GET_XFERTYPE(ed->bmAttributes); 1589 1590 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 1591 DPRINTFN(1, "addr %jd depth %jd port %jd speed %jd", dev->ud_addr, 1592 dev->ud_depth, dev->ud_powersrc->up_portno, dev->ud_speed); 1593 DPRINTFN(1, " dci %ju type 0x%02jx epaddr 0x%02jx attr 0x%02jx", 1594 xhci_ep_get_dci(ed), ed->bDescriptorType, ed->bEndpointAddress, 1595 ed->bmAttributes); 1596 DPRINTFN(1, " mps %ju ival %ju", UGETW(ed->wMaxPacketSize), 1597 ed->bInterval, 0, 0); 1598 1599 if (sc->sc_dying) 1600 return USBD_IOERROR; 1601 1602 /* Root Hub */ 1603 if (dev->ud_depth == 0 && dev->ud_powersrc->up_portno == 0) { 1604 switch (ed->bEndpointAddress) { 1605 case USB_CONTROL_ENDPOINT: 1606 pipe->up_methods = &roothub_ctrl_methods; 1607 break; 1608 case UE_DIR_IN | USBROOTHUB_INTR_ENDPT: 1609 pipe->up_methods = &xhci_root_intr_methods; 1610 break; 1611 default: 1612 pipe->up_methods = NULL; 1613 DPRINTFN(0, "bad bEndpointAddress 0x%02jx", 1614 ed->bEndpointAddress, 0, 0, 0); 1615 return USBD_INVAL; 1616 } 1617 return USBD_NORMAL_COMPLETION; 1618 } 1619 1620 switch (xfertype) { 1621 case UE_CONTROL: 1622 pipe->up_methods = &xhci_device_ctrl_methods; 1623 break; 1624 case UE_ISOCHRONOUS: 1625 pipe->up_methods = &xhci_device_isoc_methods; 1626 return USBD_INVAL; 1627 break; 1628 case UE_BULK: 1629 pipe->up_methods = &xhci_device_bulk_methods; 1630 break; 1631 case UE_INTERRUPT: 1632 pipe->up_methods = &xhci_device_intr_methods; 1633 break; 1634 default: 1635 return USBD_IOERROR; 1636 break; 1637 } 1638 1639 if (ed->bEndpointAddress != USB_CONTROL_ENDPOINT) 1640 return xhci_configure_endpoint(pipe); 1641 1642 return USBD_NORMAL_COMPLETION; 1643 } 1644 1645 /* 1646 * Closes pipe, called from usbd_kill_pipe via close methods. 1647 * If the endpoint to be closed is ep0, disable_slot. 1648 * Should be called with sc_lock held. 1649 */ 1650 static void 1651 xhci_close_pipe(struct usbd_pipe *pipe) 1652 { 1653 struct xhci_softc * const sc = XHCI_PIPE2SC(pipe); 1654 struct xhci_slot * const xs = pipe->up_dev->ud_hcpriv; 1655 usb_endpoint_descriptor_t * const ed = pipe->up_endpoint->ue_edesc; 1656 const u_int dci = xhci_ep_get_dci(ed); 1657 struct xhci_soft_trb trb; 1658 uint32_t *cp; 1659 1660 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 1661 1662 if (sc->sc_dying) 1663 return; 1664 1665 /* xs is uninitialized before xhci_init_slot */ 1666 if (xs == NULL || xs->xs_idx == 0) 1667 return; 1668 1669 DPRINTFN(4, "pipe %#jx slot %ju dci %ju", (uintptr_t)pipe, xs->xs_idx, 1670 dci, 0); 1671 1672 KASSERTMSG(!cpu_intr_p() && !cpu_softintr_p(), "called from intr ctx"); 1673 KASSERT(mutex_owned(&sc->sc_lock)); 1674 1675 if (pipe->up_dev->ud_depth == 0) 1676 return; 1677 1678 if (dci == XHCI_DCI_EP_CONTROL) { 1679 DPRINTFN(4, "closing ep0", 0, 0, 0, 0); 1680 xhci_disable_slot(sc, xs->xs_idx); 1681 return; 1682 } 1683 1684 if (xhci_get_epstate(sc, xs, dci) != XHCI_EPSTATE_STOPPED) 1685 (void)xhci_stop_endpoint(pipe); 1686 1687 /* 1688 * set appropriate bit to be dropped. 1689 * don't set DC bit to 1, otherwise all endpoints 1690 * would be deconfigured. 1691 */ 1692 cp = xhci_slot_get_icv(sc, xs, XHCI_ICI_INPUT_CONTROL); 1693 cp[0] = htole32(XHCI_INCTX_0_DROP_MASK(dci)); 1694 cp[1] = htole32(0); 1695 1696 /* XXX should be most significant one, not dci? */ 1697 cp = xhci_slot_get_icv(sc, xs, xhci_dci_to_ici(XHCI_DCI_SLOT)); 1698 cp[0] = htole32(XHCI_SCTX_0_CTX_NUM_SET(dci)); 1699 1700 /* configure ep context performs an implicit dequeue */ 1701 xhci_host_dequeue(&xs->xs_ep[dci].xe_tr); 1702 1703 /* sync input contexts before they are read from memory */ 1704 usb_syncmem(&xs->xs_ic_dma, 0, sc->sc_pgsz, BUS_DMASYNC_PREWRITE); 1705 1706 trb.trb_0 = xhci_slot_get_icp(sc, xs, 0); 1707 trb.trb_2 = 0; 1708 trb.trb_3 = XHCI_TRB_3_SLOT_SET(xs->xs_idx) | 1709 XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_CONFIGURE_EP); 1710 1711 (void)xhci_do_command_locked(sc, &trb, USBD_DEFAULT_TIMEOUT); 1712 usb_syncmem(&xs->xs_dc_dma, 0, sc->sc_pgsz, BUS_DMASYNC_POSTREAD); 1713 } 1714 1715 /* 1716 * Abort transfer. 1717 * Should be called with sc_lock held. 1718 */ 1719 static void 1720 xhci_abort_xfer(struct usbd_xfer *xfer, usbd_status status) 1721 { 1722 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 1723 struct xhci_softc * const sc = XHCI_XFER2SC(xfer); 1724 struct xhci_slot * const xs = xfer->ux_pipe->up_dev->ud_hcpriv; 1725 const u_int dci = xhci_ep_get_dci(xfer->ux_pipe->up_endpoint->ue_edesc); 1726 1727 KASSERTMSG((status == USBD_CANCELLED || status == USBD_TIMEOUT), 1728 "invalid status for abort: %d", (int)status); 1729 1730 DPRINTFN(4, "xfer %#jx pipe %#jx status %jd", 1731 (uintptr_t)xfer, (uintptr_t)xfer->ux_pipe, status, 0); 1732 1733 KASSERT(mutex_owned(&sc->sc_lock)); 1734 ASSERT_SLEEPABLE(); 1735 1736 if (status == USBD_CANCELLED) { 1737 /* 1738 * We are synchronously aborting. Try to stop the 1739 * callout and task, but if we can't, wait for them to 1740 * complete. 1741 */ 1742 callout_halt(&xfer->ux_callout, &sc->sc_lock); 1743 usb_rem_task_wait(xfer->ux_pipe->up_dev, &xfer->ux_aborttask, 1744 USB_TASKQ_HC, &sc->sc_lock); 1745 } else { 1746 /* Otherwise, we are timing out. */ 1747 KASSERT(status == USBD_TIMEOUT); 1748 } 1749 1750 /* 1751 * The xfer cannot have been cancelled already. It is the 1752 * responsibility of the caller of usbd_abort_pipe not to try 1753 * to abort a pipe multiple times, whether concurrently or 1754 * sequentially. 1755 */ 1756 KASSERT(xfer->ux_status != USBD_CANCELLED); 1757 1758 /* Only the timeout, which runs only once, can time it out. */ 1759 KASSERT(xfer->ux_status != USBD_TIMEOUT); 1760 1761 /* If anyone else beat us, we're done. */ 1762 if (xfer->ux_status != USBD_IN_PROGRESS) 1763 return; 1764 1765 /* We beat everyone else. Claim the status. */ 1766 xfer->ux_status = status; 1767 1768 /* 1769 * If we're dying, skip the hardware action and just notify the 1770 * software that we're done. 1771 */ 1772 if (sc->sc_dying) { 1773 DPRINTFN(4, "xfer %#jx dying %ju", (uintptr_t)xfer, 1774 xfer->ux_status, 0, 0); 1775 goto dying; 1776 } 1777 1778 /* 1779 * HC Step 1: Stop execution of TD on the ring. 1780 */ 1781 switch (xhci_get_epstate(sc, xs, dci)) { 1782 case XHCI_EPSTATE_HALTED: 1783 (void)xhci_reset_endpoint_locked(xfer->ux_pipe); 1784 break; 1785 case XHCI_EPSTATE_STOPPED: 1786 break; 1787 default: 1788 (void)xhci_stop_endpoint(xfer->ux_pipe); 1789 break; 1790 } 1791 #ifdef DIAGNOSTIC 1792 uint32_t epst = xhci_get_epstate(sc, xs, dci); 1793 if (epst != XHCI_EPSTATE_STOPPED) 1794 DPRINTFN(4, "dci %ju not stopped %ju", dci, epst, 0, 0); 1795 #endif 1796 1797 /* 1798 * HC Step 2: Remove any vestiges of the xfer from the ring. 1799 */ 1800 xhci_set_dequeue_locked(xfer->ux_pipe); 1801 1802 /* 1803 * Final Step: Notify completion to waiting xfers. 1804 */ 1805 dying: 1806 usb_transfer_complete(xfer); 1807 DPRINTFN(14, "end", 0, 0, 0, 0); 1808 1809 KASSERT(mutex_owned(&sc->sc_lock)); 1810 } 1811 1812 static void 1813 xhci_host_dequeue(struct xhci_ring * const xr) 1814 { 1815 /* When dequeueing the controller, update our struct copy too */ 1816 memset(xr->xr_trb, 0, xr->xr_ntrb * XHCI_TRB_SIZE); 1817 usb_syncmem(&xr->xr_dma, 0, xr->xr_ntrb * XHCI_TRB_SIZE, 1818 BUS_DMASYNC_PREWRITE); 1819 memset(xr->xr_cookies, 0, xr->xr_ntrb * sizeof(*xr->xr_cookies)); 1820 1821 xr->xr_ep = 0; 1822 xr->xr_cs = 1; 1823 } 1824 1825 /* 1826 * Recover STALLed endpoint. 1827 * xHCI 1.1 sect 4.10.2.1 1828 * Issue RESET_EP to recover halt condition and SET_TR_DEQUEUE to remove 1829 * all transfers on transfer ring. 1830 * These are done in thread context asynchronously. 1831 */ 1832 static void 1833 xhci_clear_endpoint_stall_async_task(void *cookie) 1834 { 1835 struct usbd_xfer * const xfer = cookie; 1836 struct xhci_softc * const sc = XHCI_XFER2SC(xfer); 1837 struct xhci_slot * const xs = xfer->ux_pipe->up_dev->ud_hcpriv; 1838 const u_int dci = xhci_ep_get_dci(xfer->ux_pipe->up_endpoint->ue_edesc); 1839 struct xhci_ring * const tr = &xs->xs_ep[dci].xe_tr; 1840 1841 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 1842 DPRINTFN(4, "xfer %#jx slot %ju dci %ju", (uintptr_t)xfer, xs->xs_idx, 1843 dci, 0); 1844 1845 /* 1846 * XXXMRG: Stall task can run after slot is disabled when yanked. 1847 * This hack notices that the xs has been memset() in 1848 * xhci_disable_slot() and returns. Both xhci_reset_endpoint() 1849 * and xhci_set_dequeue() rely upon a valid ring setup for correct 1850 * operation, and the latter will fault, as would 1851 * usb_transfer_complete() if it got that far. 1852 */ 1853 if (xs->xs_idx == 0) { 1854 DPRINTFN(4, "ends xs_idx is 0", 0, 0, 0, 0); 1855 return; 1856 } 1857 1858 xhci_reset_endpoint(xfer->ux_pipe); 1859 xhci_set_dequeue(xfer->ux_pipe); 1860 1861 mutex_enter(&sc->sc_lock); 1862 tr->is_halted = false; 1863 usb_transfer_complete(xfer); 1864 mutex_exit(&sc->sc_lock); 1865 DPRINTFN(4, "ends", 0, 0, 0, 0); 1866 } 1867 1868 static usbd_status 1869 xhci_clear_endpoint_stall_async(struct usbd_xfer *xfer) 1870 { 1871 struct xhci_softc * const sc = XHCI_XFER2SC(xfer); 1872 struct xhci_pipe * const xp = (struct xhci_pipe *)xfer->ux_pipe; 1873 1874 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 1875 DPRINTFN(4, "xfer %#jx", (uintptr_t)xfer, 0, 0, 0); 1876 1877 if (sc->sc_dying) { 1878 return USBD_IOERROR; 1879 } 1880 1881 usb_init_task(&xp->xp_async_task, 1882 xhci_clear_endpoint_stall_async_task, xfer, USB_TASKQ_MPSAFE); 1883 usb_add_task(xfer->ux_pipe->up_dev, &xp->xp_async_task, USB_TASKQ_HC); 1884 DPRINTFN(4, "ends", 0, 0, 0, 0); 1885 1886 return USBD_NORMAL_COMPLETION; 1887 } 1888 1889 /* Process roothub port status/change events and notify to uhub_intr. */ 1890 static void 1891 xhci_rhpsc(struct xhci_softc * const sc, u_int ctlrport) 1892 { 1893 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 1894 DPRINTFN(4, "xhci%jd: port %ju status change", device_unit(sc->sc_dev), 1895 ctlrport, 0, 0); 1896 1897 if (ctlrport > sc->sc_maxports) 1898 return; 1899 1900 const size_t bn = xhci_ctlrport2bus(sc, ctlrport); 1901 const size_t rhp = xhci_ctlrport2rhport(sc, ctlrport); 1902 struct usbd_xfer * const xfer = sc->sc_intrxfer[bn]; 1903 1904 DPRINTFN(4, "xhci%jd: bus %jd bp %ju xfer %#jx status change", 1905 device_unit(sc->sc_dev), bn, rhp, (uintptr_t)xfer); 1906 1907 if (xfer == NULL) 1908 return; 1909 1910 uint8_t *p = xfer->ux_buf; 1911 memset(p, 0, xfer->ux_length); 1912 p[rhp / NBBY] |= 1 << (rhp % NBBY); 1913 xfer->ux_actlen = xfer->ux_length; 1914 xfer->ux_status = USBD_NORMAL_COMPLETION; 1915 usb_transfer_complete(xfer); 1916 } 1917 1918 /* Process Transfer Events */ 1919 static void 1920 xhci_event_transfer(struct xhci_softc * const sc, 1921 const struct xhci_trb * const trb) 1922 { 1923 uint64_t trb_0; 1924 uint32_t trb_2, trb_3; 1925 uint8_t trbcode; 1926 u_int slot, dci; 1927 struct xhci_slot *xs; 1928 struct xhci_ring *xr; 1929 struct xhci_xfer *xx; 1930 struct usbd_xfer *xfer; 1931 usbd_status err; 1932 1933 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 1934 1935 trb_0 = le64toh(trb->trb_0); 1936 trb_2 = le32toh(trb->trb_2); 1937 trb_3 = le32toh(trb->trb_3); 1938 trbcode = XHCI_TRB_2_ERROR_GET(trb_2); 1939 slot = XHCI_TRB_3_SLOT_GET(trb_3); 1940 dci = XHCI_TRB_3_EP_GET(trb_3); 1941 xs = &sc->sc_slots[slot]; 1942 xr = &xs->xs_ep[dci].xe_tr; 1943 1944 /* sanity check */ 1945 KASSERTMSG(xs->xs_idx != 0 && xs->xs_idx <= sc->sc_maxslots, 1946 "invalid xs_idx %u slot %u", xs->xs_idx, slot); 1947 1948 int idx = 0; 1949 if ((trb_3 & XHCI_TRB_3_ED_BIT) == 0) { 1950 if (xhci_trb_get_idx(xr, trb_0, &idx)) { 1951 DPRINTFN(0, "invalid trb_0 0x%jx", trb_0, 0, 0, 0); 1952 return; 1953 } 1954 xx = xr->xr_cookies[idx]; 1955 1956 /* clear cookie of consumed TRB */ 1957 xr->xr_cookies[idx] = NULL; 1958 1959 /* 1960 * xx is NULL if pipe is opened but xfer is not started. 1961 * It happens when stopping idle pipe. 1962 */ 1963 if (xx == NULL || trbcode == XHCI_TRB_ERROR_LENGTH) { 1964 DPRINTFN(1, "Ignore #%ju: cookie %#jx cc %ju dci %ju", 1965 idx, (uintptr_t)xx, trbcode, dci); 1966 DPRINTFN(1, " orig TRB %jx type %ju", trb_0, 1967 XHCI_TRB_3_TYPE_GET(le32toh(xr->xr_trb[idx].trb_3)), 1968 0, 0); 1969 return; 1970 } 1971 } else { 1972 /* When ED != 0, trb_0 is virtual addr of struct xhci_xfer. */ 1973 xx = (void *)(uintptr_t)(trb_0 & ~0x3); 1974 } 1975 /* XXX this may not happen */ 1976 if (xx == NULL) { 1977 DPRINTFN(1, "xfer done: xx is NULL", 0, 0, 0, 0); 1978 return; 1979 } 1980 xfer = &xx->xx_xfer; 1981 /* XXX this may happen when detaching */ 1982 if (xfer == NULL) { 1983 DPRINTFN(1, "xx(%#jx)->xx_xfer is NULL trb_0 %#jx", 1984 (uintptr_t)xx, trb_0, 0, 0); 1985 return; 1986 } 1987 DPRINTFN(14, "xfer %#jx", (uintptr_t)xfer, 0, 0, 0); 1988 /* XXX I dunno why this happens */ 1989 KASSERTMSG(xfer->ux_pipe != NULL, "xfer(%p)->ux_pipe is NULL", xfer); 1990 1991 if (!xfer->ux_pipe->up_repeat && 1992 SIMPLEQ_EMPTY(&xfer->ux_pipe->up_queue)) { 1993 DPRINTFN(1, "xfer(%#jx)->pipe not queued", (uintptr_t)xfer, 1994 0, 0, 0); 1995 return; 1996 } 1997 1998 /* 4.11.5.2 Event Data TRB */ 1999 if ((trb_3 & XHCI_TRB_3_ED_BIT) != 0) { 2000 DPRINTFN(14, "transfer Event Data: 0x%016jx 0x%08jx" 2001 " %02jx", trb_0, XHCI_TRB_2_REM_GET(trb_2), trbcode, 0); 2002 if ((trb_0 & 0x3) == 0x3) { 2003 xfer->ux_actlen = XHCI_TRB_2_REM_GET(trb_2); 2004 } 2005 } 2006 2007 switch (trbcode) { 2008 case XHCI_TRB_ERROR_SHORT_PKT: 2009 case XHCI_TRB_ERROR_SUCCESS: 2010 /* 2011 * A ctrl transfer can generate two events if it has a Data 2012 * stage. A short data stage can be OK and should not 2013 * complete the transfer as the status stage needs to be 2014 * performed. 2015 * 2016 * Note: Data and Status stage events point at same xfer. 2017 * ux_actlen and ux_dmabuf will be passed to 2018 * usb_transfer_complete after the Status stage event. 2019 * 2020 * It can be distingished which stage generates the event: 2021 * + by checking least 3 bits of trb_0 if ED==1. 2022 * (see xhci_device_ctrl_start). 2023 * + by checking the type of original TRB if ED==0. 2024 * 2025 * In addition, intr, bulk, and isoc transfer currently 2026 * consists of single TD, so the "skip" is not needed. 2027 * ctrl xfer uses EVENT_DATA, and others do not. 2028 * Thus driver can switch the flow by checking ED bit. 2029 */ 2030 if ((trb_3 & XHCI_TRB_3_ED_BIT) == 0) { 2031 if (xfer->ux_actlen == 0) 2032 xfer->ux_actlen = xfer->ux_length - 2033 XHCI_TRB_2_REM_GET(trb_2); 2034 if (XHCI_TRB_3_TYPE_GET(le32toh(xr->xr_trb[idx].trb_3)) 2035 == XHCI_TRB_TYPE_DATA_STAGE) { 2036 return; 2037 } 2038 } else if ((trb_0 & 0x3) == 0x3) { 2039 return; 2040 } 2041 err = USBD_NORMAL_COMPLETION; 2042 break; 2043 case XHCI_TRB_ERROR_STOPPED: 2044 case XHCI_TRB_ERROR_LENGTH: 2045 case XHCI_TRB_ERROR_STOPPED_SHORT: 2046 /* 2047 * don't complete the transfer being aborted 2048 * as abort_xfer does instead. 2049 */ 2050 if (xfer->ux_status == USBD_CANCELLED || 2051 xfer->ux_status == USBD_TIMEOUT) { 2052 DPRINTFN(14, "ignore aborting xfer %#jx", 2053 (uintptr_t)xfer, 0, 0, 0); 2054 return; 2055 } 2056 err = USBD_CANCELLED; 2057 break; 2058 case XHCI_TRB_ERROR_STALL: 2059 case XHCI_TRB_ERROR_BABBLE: 2060 DPRINTFN(1, "ERR %ju slot %ju dci %ju", trbcode, slot, dci, 0); 2061 xr->is_halted = true; 2062 /* 2063 * Stalled endpoints can be recoverd by issuing 2064 * command TRB TYPE_RESET_EP on xHCI instead of 2065 * issuing request CLEAR_FEATURE UF_ENDPOINT_HALT 2066 * on the endpoint. However, this function may be 2067 * called from softint context (e.g. from umass), 2068 * in that case driver gets KASSERT in cv_timedwait 2069 * in xhci_do_command. 2070 * To avoid this, this runs reset_endpoint and 2071 * usb_transfer_complete in usb task thread 2072 * asynchronously (and then umass issues clear 2073 * UF_ENDPOINT_HALT). 2074 */ 2075 2076 /* Override the status. */ 2077 xfer->ux_status = USBD_STALLED; 2078 2079 /* 2080 * Cancel the timeout and the task, which have not yet 2081 * run. If they have already fired, at worst they are 2082 * waiting for the lock. They will see that the xfer 2083 * is no longer in progress and give up. 2084 */ 2085 callout_stop(&xfer->ux_callout); 2086 usb_rem_task(xfer->ux_pipe->up_dev, &xfer->ux_aborttask); 2087 2088 xhci_clear_endpoint_stall_async(xfer); 2089 return; 2090 default: 2091 DPRINTFN(1, "ERR %ju slot %ju dci %ju", trbcode, slot, dci, 0); 2092 err = USBD_IOERROR; 2093 break; 2094 } 2095 2096 /* 2097 * If software has completed it, either by cancellation 2098 * or timeout, drop it on the floor. 2099 */ 2100 if (xfer->ux_status != USBD_IN_PROGRESS) { 2101 KASSERTMSG((xfer->ux_status == USBD_CANCELLED || 2102 xfer->ux_status == USBD_TIMEOUT), 2103 "xfer %p status %x", xfer, xfer->ux_status); 2104 return; 2105 } 2106 2107 /* Otherwise, set the status. */ 2108 xfer->ux_status = err; 2109 2110 /* 2111 * Cancel the timeout and the task, which have not yet 2112 * run. If they have already fired, at worst they are 2113 * waiting for the lock. They will see that the xfer 2114 * is no longer in progress and give up. 2115 */ 2116 callout_stop(&xfer->ux_callout); 2117 usb_rem_task(xfer->ux_pipe->up_dev, &xfer->ux_aborttask); 2118 2119 if ((trb_3 & XHCI_TRB_3_ED_BIT) == 0 || 2120 (trb_0 & 0x3) == 0x0) { 2121 usb_transfer_complete(xfer); 2122 } 2123 } 2124 2125 /* Process Command complete events */ 2126 static void 2127 xhci_event_cmd(struct xhci_softc * const sc, const struct xhci_trb * const trb) 2128 { 2129 uint64_t trb_0; 2130 uint32_t trb_2, trb_3; 2131 2132 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 2133 2134 KASSERT(mutex_owned(&sc->sc_lock)); 2135 2136 trb_0 = le64toh(trb->trb_0); 2137 trb_2 = le32toh(trb->trb_2); 2138 trb_3 = le32toh(trb->trb_3); 2139 2140 if (trb_0 == sc->sc_command_addr) { 2141 sc->sc_resultpending = false; 2142 2143 sc->sc_result_trb.trb_0 = trb_0; 2144 sc->sc_result_trb.trb_2 = trb_2; 2145 sc->sc_result_trb.trb_3 = trb_3; 2146 if (XHCI_TRB_2_ERROR_GET(trb_2) != 2147 XHCI_TRB_ERROR_SUCCESS) { 2148 DPRINTFN(1, "command completion " 2149 "failure: 0x%016jx 0x%08jx 0x%08jx", 2150 trb_0, trb_2, trb_3, 0); 2151 } 2152 cv_signal(&sc->sc_command_cv); 2153 } else { 2154 DPRINTFN(1, "spurious event: %#jx 0x%016jx " 2155 "0x%08jx 0x%08jx", (uintptr_t)trb, trb_0, trb_2, trb_3); 2156 } 2157 } 2158 2159 /* 2160 * Process events. 2161 * called from xhci_softintr 2162 */ 2163 static void 2164 xhci_handle_event(struct xhci_softc * const sc, 2165 const struct xhci_trb * const trb) 2166 { 2167 uint64_t trb_0; 2168 uint32_t trb_2, trb_3; 2169 2170 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 2171 2172 trb_0 = le64toh(trb->trb_0); 2173 trb_2 = le32toh(trb->trb_2); 2174 trb_3 = le32toh(trb->trb_3); 2175 2176 DPRINTFN(14, "event: %#jx 0x%016jx 0x%08jx 0x%08jx", 2177 (uintptr_t)trb, trb_0, trb_2, trb_3); 2178 2179 /* 2180 * 4.11.3.1, 6.4.2.1 2181 * TRB Pointer is invalid for these completion codes. 2182 */ 2183 switch (XHCI_TRB_2_ERROR_GET(trb_2)) { 2184 case XHCI_TRB_ERROR_RING_UNDERRUN: 2185 case XHCI_TRB_ERROR_RING_OVERRUN: 2186 case XHCI_TRB_ERROR_VF_RING_FULL: 2187 return; 2188 default: 2189 if (trb_0 == 0) { 2190 return; 2191 } 2192 break; 2193 } 2194 2195 switch (XHCI_TRB_3_TYPE_GET(trb_3)) { 2196 case XHCI_TRB_EVENT_TRANSFER: 2197 xhci_event_transfer(sc, trb); 2198 break; 2199 case XHCI_TRB_EVENT_CMD_COMPLETE: 2200 xhci_event_cmd(sc, trb); 2201 break; 2202 case XHCI_TRB_EVENT_PORT_STS_CHANGE: 2203 xhci_rhpsc(sc, (uint32_t)((trb_0 >> 24) & 0xff)); 2204 break; 2205 default: 2206 break; 2207 } 2208 } 2209 2210 static void 2211 xhci_softintr(void *v) 2212 { 2213 struct usbd_bus * const bus = v; 2214 struct xhci_softc * const sc = XHCI_BUS2SC(bus); 2215 struct xhci_ring * const er = &sc->sc_er; 2216 struct xhci_trb *trb; 2217 int i, j, k; 2218 2219 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 2220 2221 KASSERT(xhci_polling_p(sc) || mutex_owned(&sc->sc_lock)); 2222 2223 i = er->xr_ep; 2224 j = er->xr_cs; 2225 2226 DPRINTFN(16, "er: xr_ep %jd xr_cs %jd", i, j, 0, 0); 2227 2228 while (1) { 2229 usb_syncmem(&er->xr_dma, XHCI_TRB_SIZE * i, XHCI_TRB_SIZE, 2230 BUS_DMASYNC_POSTREAD); 2231 trb = &er->xr_trb[i]; 2232 k = (le32toh(trb->trb_3) & XHCI_TRB_3_CYCLE_BIT) ? 1 : 0; 2233 2234 if (j != k) 2235 break; 2236 2237 xhci_handle_event(sc, trb); 2238 2239 i++; 2240 if (i == er->xr_ntrb) { 2241 i = 0; 2242 j ^= 1; 2243 } 2244 } 2245 2246 er->xr_ep = i; 2247 er->xr_cs = j; 2248 2249 xhci_rt_write_8(sc, XHCI_ERDP(0), xhci_ring_trbp(er, er->xr_ep) | 2250 XHCI_ERDP_LO_BUSY); 2251 2252 DPRINTFN(16, "ends", 0, 0, 0, 0); 2253 2254 return; 2255 } 2256 2257 static void 2258 xhci_poll(struct usbd_bus *bus) 2259 { 2260 struct xhci_softc * const sc = XHCI_BUS2SC(bus); 2261 2262 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 2263 2264 mutex_enter(&sc->sc_intr_lock); 2265 int ret = xhci_intr1(sc); 2266 if (ret) { 2267 xhci_softintr(bus); 2268 } 2269 mutex_exit(&sc->sc_intr_lock); 2270 2271 return; 2272 } 2273 2274 static struct usbd_xfer * 2275 xhci_allocx(struct usbd_bus *bus, unsigned int nframes) 2276 { 2277 struct xhci_softc * const sc = XHCI_BUS2SC(bus); 2278 struct usbd_xfer *xfer; 2279 2280 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 2281 2282 xfer = pool_cache_get(sc->sc_xferpool, PR_WAITOK); 2283 if (xfer != NULL) { 2284 memset(xfer, 0, sizeof(struct xhci_xfer)); 2285 usb_init_task(&xfer->ux_aborttask, xhci_timeout_task, xfer, 2286 USB_TASKQ_MPSAFE); 2287 #ifdef DIAGNOSTIC 2288 xfer->ux_state = XFER_BUSY; 2289 #endif 2290 } 2291 2292 return xfer; 2293 } 2294 2295 static void 2296 xhci_freex(struct usbd_bus *bus, struct usbd_xfer *xfer) 2297 { 2298 struct xhci_softc * const sc = XHCI_BUS2SC(bus); 2299 2300 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 2301 2302 #ifdef DIAGNOSTIC 2303 if (xfer->ux_state != XFER_BUSY && 2304 xfer->ux_status != USBD_NOT_STARTED) { 2305 DPRINTFN(0, "xfer=%#jx not busy, 0x%08jx", 2306 (uintptr_t)xfer, xfer->ux_state, 0, 0); 2307 } 2308 xfer->ux_state = XFER_FREE; 2309 #endif 2310 pool_cache_put(sc->sc_xferpool, xfer); 2311 } 2312 2313 static void 2314 xhci_get_lock(struct usbd_bus *bus, kmutex_t **lock) 2315 { 2316 struct xhci_softc * const sc = XHCI_BUS2SC(bus); 2317 2318 *lock = &sc->sc_lock; 2319 } 2320 2321 extern uint32_t usb_cookie_no; 2322 2323 /* 2324 * xHCI 4.3 2325 * Called when uhub_explore finds a new device (via usbd_new_device). 2326 * Port initialization and speed detection (4.3.1) are already done in uhub.c. 2327 * This function does: 2328 * Allocate and construct dev structure of default endpoint (ep0). 2329 * Allocate and open pipe of ep0. 2330 * Enable slot and initialize slot context. 2331 * Set Address. 2332 * Read initial device descriptor. 2333 * Determine initial MaxPacketSize (mps) by speed. 2334 * Read full device descriptor. 2335 * Register this device. 2336 * Finally state of device transitions ADDRESSED. 2337 */ 2338 static usbd_status 2339 xhci_new_device(device_t parent, struct usbd_bus *bus, int depth, 2340 int speed, int port, struct usbd_port *up) 2341 { 2342 struct xhci_softc * const sc = XHCI_BUS2SC(bus); 2343 struct usbd_device *dev; 2344 usbd_status err; 2345 usb_device_descriptor_t *dd; 2346 struct xhci_slot *xs; 2347 uint32_t *cp; 2348 2349 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 2350 DPRINTFN(4, "port %ju depth %ju speed %ju up %#jx", 2351 port, depth, speed, (uintptr_t)up); 2352 2353 dev = kmem_zalloc(sizeof(*dev), KM_SLEEP); 2354 dev->ud_bus = bus; 2355 dev->ud_quirks = &usbd_no_quirk; 2356 dev->ud_addr = 0; 2357 dev->ud_ddesc.bMaxPacketSize = 0; 2358 dev->ud_depth = depth; 2359 dev->ud_powersrc = up; 2360 dev->ud_myhub = up->up_parent; 2361 dev->ud_speed = speed; 2362 dev->ud_langid = USBD_NOLANG; 2363 dev->ud_cookie.cookie = ++usb_cookie_no; 2364 2365 /* Set up default endpoint handle. */ 2366 dev->ud_ep0.ue_edesc = &dev->ud_ep0desc; 2367 /* doesn't matter, just don't let it uninitialized */ 2368 dev->ud_ep0.ue_toggle = 0; 2369 2370 /* Set up default endpoint descriptor. */ 2371 dev->ud_ep0desc.bLength = USB_ENDPOINT_DESCRIPTOR_SIZE; 2372 dev->ud_ep0desc.bDescriptorType = UDESC_ENDPOINT; 2373 dev->ud_ep0desc.bEndpointAddress = USB_CONTROL_ENDPOINT; 2374 dev->ud_ep0desc.bmAttributes = UE_CONTROL; 2375 dev->ud_ep0desc.bInterval = 0; 2376 2377 /* 4.3, 4.8.2.1 */ 2378 switch (speed) { 2379 case USB_SPEED_SUPER: 2380 case USB_SPEED_SUPER_PLUS: 2381 USETW(dev->ud_ep0desc.wMaxPacketSize, USB_3_MAX_CTRL_PACKET); 2382 break; 2383 case USB_SPEED_FULL: 2384 /* XXX using 64 as initial mps of ep0 in FS */ 2385 case USB_SPEED_HIGH: 2386 USETW(dev->ud_ep0desc.wMaxPacketSize, USB_2_MAX_CTRL_PACKET); 2387 break; 2388 case USB_SPEED_LOW: 2389 default: 2390 USETW(dev->ud_ep0desc.wMaxPacketSize, USB_MAX_IPACKET); 2391 break; 2392 } 2393 2394 up->up_dev = dev; 2395 2396 /* Establish the default pipe. */ 2397 err = usbd_setup_pipe(dev, 0, &dev->ud_ep0, USBD_DEFAULT_INTERVAL, 2398 &dev->ud_pipe0); 2399 if (err) { 2400 goto bad; 2401 } 2402 2403 dd = &dev->ud_ddesc; 2404 2405 if (depth == 0 && port == 0) { 2406 KASSERT(bus->ub_devices[USB_ROOTHUB_INDEX] == NULL); 2407 bus->ub_devices[USB_ROOTHUB_INDEX] = dev; 2408 err = usbd_get_initial_ddesc(dev, dd); 2409 if (err) { 2410 DPRINTFN(1, "get_initial_ddesc %ju", err, 0, 0, 0); 2411 goto bad; 2412 } 2413 2414 err = usbd_reload_device_desc(dev); 2415 if (err) { 2416 DPRINTFN(1, "reload desc %ju", err, 0, 0, 0); 2417 goto bad; 2418 } 2419 } else { 2420 uint8_t slot = 0; 2421 2422 /* 4.3.2 */ 2423 err = xhci_enable_slot(sc, &slot); 2424 if (err) { 2425 DPRINTFN(1, "enable slot %ju", err, 0, 0, 0); 2426 goto bad; 2427 } 2428 2429 xs = &sc->sc_slots[slot]; 2430 dev->ud_hcpriv = xs; 2431 2432 /* 4.3.3 initialize slot structure */ 2433 err = xhci_init_slot(dev, slot); 2434 if (err) { 2435 DPRINTFN(1, "init slot %ju", err, 0, 0, 0); 2436 dev->ud_hcpriv = NULL; 2437 /* 2438 * We have to disable_slot here because 2439 * xs->xs_idx == 0 when xhci_init_slot fails, 2440 * in that case usbd_remove_dev won't work. 2441 */ 2442 mutex_enter(&sc->sc_lock); 2443 xhci_disable_slot(sc, slot); 2444 mutex_exit(&sc->sc_lock); 2445 goto bad; 2446 } 2447 2448 /* 4.3.4 Address Assignment */ 2449 err = xhci_set_address(dev, slot, false); 2450 if (err) { 2451 DPRINTFN(1, "set address w/o bsr %ju", err, 0, 0, 0); 2452 goto bad; 2453 } 2454 2455 /* Allow device time to set new address */ 2456 usbd_delay_ms(dev, USB_SET_ADDRESS_SETTLE); 2457 2458 usb_syncmem(&xs->xs_dc_dma, 0, sc->sc_pgsz, BUS_DMASYNC_POSTREAD); 2459 cp = xhci_slot_get_dcv(sc, xs, XHCI_DCI_SLOT); 2460 HEXDUMP("slot context", cp, sc->sc_ctxsz); 2461 uint8_t addr = XHCI_SCTX_3_DEV_ADDR_GET(le32toh(cp[3])); 2462 DPRINTFN(4, "device address %ju", addr, 0, 0, 0); 2463 /* 2464 * XXX ensure we know when the hardware does something 2465 * we can't yet cope with 2466 */ 2467 KASSERTMSG(addr >= 1 && addr <= 127, "addr %d", addr); 2468 dev->ud_addr = addr; 2469 2470 KASSERTMSG(bus->ub_devices[usb_addr2dindex(dev->ud_addr)] == NULL, 2471 "addr %d already allocated", dev->ud_addr); 2472 /* 2473 * The root hub is given its own slot 2474 */ 2475 bus->ub_devices[usb_addr2dindex(dev->ud_addr)] = dev; 2476 2477 err = usbd_get_initial_ddesc(dev, dd); 2478 if (err) { 2479 DPRINTFN(1, "get_initial_ddesc %ju", err, 0, 0, 0); 2480 goto bad; 2481 } 2482 2483 /* 4.8.2.1 */ 2484 if (USB_IS_SS(speed)) { 2485 if (dd->bMaxPacketSize != 9) { 2486 printf("%s: invalid mps 2^%u for SS ep0," 2487 " using 512\n", 2488 device_xname(sc->sc_dev), 2489 dd->bMaxPacketSize); 2490 dd->bMaxPacketSize = 9; 2491 } 2492 USETW(dev->ud_ep0desc.wMaxPacketSize, 2493 (1 << dd->bMaxPacketSize)); 2494 } else 2495 USETW(dev->ud_ep0desc.wMaxPacketSize, 2496 dd->bMaxPacketSize); 2497 DPRINTFN(4, "bMaxPacketSize %ju", dd->bMaxPacketSize, 0, 0, 0); 2498 err = xhci_update_ep0_mps(sc, xs, 2499 UGETW(dev->ud_ep0desc.wMaxPacketSize)); 2500 if (err) { 2501 DPRINTFN(1, "update mps of ep0 %ju", err, 0, 0, 0); 2502 goto bad; 2503 } 2504 2505 err = usbd_reload_device_desc(dev); 2506 if (err) { 2507 DPRINTFN(1, "reload desc %ju", err, 0, 0, 0); 2508 goto bad; 2509 } 2510 } 2511 2512 DPRINTFN(1, "adding unit addr=%jd, rev=%02jx,", 2513 dev->ud_addr, UGETW(dd->bcdUSB), 0, 0); 2514 DPRINTFN(1, " class=%jd, subclass=%jd, protocol=%jd,", 2515 dd->bDeviceClass, dd->bDeviceSubClass, 2516 dd->bDeviceProtocol, 0); 2517 DPRINTFN(1, " mps=%jd, len=%jd, noconf=%jd, speed=%jd", 2518 dd->bMaxPacketSize, dd->bLength, dd->bNumConfigurations, 2519 dev->ud_speed); 2520 2521 usbd_get_device_strings(dev); 2522 2523 usbd_add_dev_event(USB_EVENT_DEVICE_ATTACH, dev); 2524 2525 if (depth == 0 && port == 0) { 2526 usbd_attach_roothub(parent, dev); 2527 DPRINTFN(1, "root hub %#jx", (uintptr_t)dev, 0, 0, 0); 2528 return USBD_NORMAL_COMPLETION; 2529 } 2530 2531 err = usbd_probe_and_attach(parent, dev, port, dev->ud_addr); 2532 bad: 2533 if (err != USBD_NORMAL_COMPLETION) { 2534 usbd_remove_device(dev, up); 2535 } 2536 2537 return err; 2538 } 2539 2540 static usbd_status 2541 xhci_ring_init(struct xhci_softc * const sc, struct xhci_ring * const xr, 2542 size_t ntrb, size_t align) 2543 { 2544 usbd_status err; 2545 size_t size = ntrb * XHCI_TRB_SIZE; 2546 2547 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 2548 2549 err = usb_allocmem(&sc->sc_bus, size, align, &xr->xr_dma); 2550 if (err) 2551 return err; 2552 mutex_init(&xr->xr_lock, MUTEX_DEFAULT, IPL_SOFTUSB); 2553 xr->xr_cookies = kmem_zalloc(sizeof(*xr->xr_cookies) * ntrb, KM_SLEEP); 2554 xr->xr_trb = xhci_ring_trbv(xr, 0); 2555 xr->xr_ntrb = ntrb; 2556 xr->is_halted = false; 2557 xhci_host_dequeue(xr); 2558 2559 return USBD_NORMAL_COMPLETION; 2560 } 2561 2562 static void 2563 xhci_ring_free(struct xhci_softc * const sc, struct xhci_ring * const xr) 2564 { 2565 usb_freemem(&sc->sc_bus, &xr->xr_dma); 2566 mutex_destroy(&xr->xr_lock); 2567 kmem_free(xr->xr_cookies, sizeof(*xr->xr_cookies) * xr->xr_ntrb); 2568 } 2569 2570 static void 2571 xhci_ring_put(struct xhci_softc * const sc, struct xhci_ring * const xr, 2572 void *cookie, struct xhci_soft_trb * const trbs, size_t ntrbs) 2573 { 2574 size_t i; 2575 u_int ri; 2576 u_int cs; 2577 uint64_t parameter; 2578 uint32_t status; 2579 uint32_t control; 2580 2581 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 2582 2583 KASSERTMSG(ntrbs <= XHCI_XFER_NTRB, "ntrbs %zu", ntrbs); 2584 for (i = 0; i < ntrbs; i++) { 2585 DPRINTFN(12, "xr %#jx trbs %#jx num %ju", (uintptr_t)xr, 2586 (uintptr_t)trbs, i, 0); 2587 DPRINTFN(12, " %016jx %08jx %08jx", 2588 trbs[i].trb_0, trbs[i].trb_2, trbs[i].trb_3, 0); 2589 KASSERTMSG(XHCI_TRB_3_TYPE_GET(trbs[i].trb_3) != 2590 XHCI_TRB_TYPE_LINK, "trbs[%zu].trb3 %#x", i, trbs[i].trb_3); 2591 } 2592 2593 DPRINTFN(12, "%#jx xr_ep 0x%jx xr_cs %ju", (uintptr_t)xr, xr->xr_ep, 2594 xr->xr_cs, 0); 2595 2596 ri = xr->xr_ep; 2597 cs = xr->xr_cs; 2598 2599 /* 2600 * Although the xhci hardware can do scatter/gather dma from 2601 * arbitrary sized buffers, there is a non-obvious restriction 2602 * that a LINK trb is only allowed at the end of a burst of 2603 * transfers - which might be 16kB. 2604 * Arbitrary aligned LINK trb definitely fail on Ivy bridge. 2605 * The simple solution is not to allow a LINK trb in the middle 2606 * of anything - as here. 2607 * XXX: (dsl) There are xhci controllers out there (eg some made by 2608 * ASMedia) that seem to lock up if they process a LINK trb but 2609 * cannot process the linked-to trb yet. 2610 * The code should write the 'cycle' bit on the link trb AFTER 2611 * adding the other trb. 2612 */ 2613 u_int firstep = xr->xr_ep; 2614 u_int firstcs = xr->xr_cs; 2615 2616 for (i = 0; i < ntrbs; ) { 2617 u_int oldri = ri; 2618 u_int oldcs = cs; 2619 2620 if (ri >= (xr->xr_ntrb - 1)) { 2621 /* Put Link TD at the end of ring */ 2622 parameter = xhci_ring_trbp(xr, 0); 2623 status = 0; 2624 control = XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_LINK) | 2625 XHCI_TRB_3_TC_BIT; 2626 xr->xr_cookies[ri] = NULL; 2627 xr->xr_ep = 0; 2628 xr->xr_cs ^= 1; 2629 ri = xr->xr_ep; 2630 cs = xr->xr_cs; 2631 } else { 2632 parameter = trbs[i].trb_0; 2633 status = trbs[i].trb_2; 2634 control = trbs[i].trb_3; 2635 2636 xr->xr_cookies[ri] = cookie; 2637 ri++; 2638 i++; 2639 } 2640 /* 2641 * If this is a first TRB, mark it invalid to prevent 2642 * xHC from running it immediately. 2643 */ 2644 if (oldri == firstep) { 2645 if (oldcs) { 2646 control &= ~XHCI_TRB_3_CYCLE_BIT; 2647 } else { 2648 control |= XHCI_TRB_3_CYCLE_BIT; 2649 } 2650 } else { 2651 if (oldcs) { 2652 control |= XHCI_TRB_3_CYCLE_BIT; 2653 } else { 2654 control &= ~XHCI_TRB_3_CYCLE_BIT; 2655 } 2656 } 2657 xhci_trb_put(&xr->xr_trb[oldri], parameter, status, control); 2658 usb_syncmem(&xr->xr_dma, XHCI_TRB_SIZE * oldri, 2659 XHCI_TRB_SIZE * 1, BUS_DMASYNC_PREWRITE); 2660 } 2661 2662 /* Now invert cycle bit of first TRB */ 2663 if (firstcs) { 2664 xr->xr_trb[firstep].trb_3 |= htole32(XHCI_TRB_3_CYCLE_BIT); 2665 } else { 2666 xr->xr_trb[firstep].trb_3 &= ~htole32(XHCI_TRB_3_CYCLE_BIT); 2667 } 2668 usb_syncmem(&xr->xr_dma, XHCI_TRB_SIZE * firstep, 2669 XHCI_TRB_SIZE * 1, BUS_DMASYNC_PREWRITE); 2670 2671 xr->xr_ep = ri; 2672 xr->xr_cs = cs; 2673 2674 DPRINTFN(12, "%#jx xr_ep 0x%jx xr_cs %ju", (uintptr_t)xr, xr->xr_ep, 2675 xr->xr_cs, 0); 2676 } 2677 2678 /* 2679 * Stop execution commands, purge all commands on command ring, and 2680 * rewind dequeue pointer. 2681 */ 2682 static void 2683 xhci_abort_command(struct xhci_softc *sc) 2684 { 2685 struct xhci_ring * const cr = &sc->sc_cr; 2686 uint64_t crcr; 2687 int i; 2688 2689 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 2690 DPRINTFN(14, "command %#jx timeout, aborting", 2691 sc->sc_command_addr, 0, 0, 0); 2692 2693 mutex_enter(&cr->xr_lock); 2694 2695 /* 4.6.1.2 Aborting a Command */ 2696 crcr = xhci_op_read_8(sc, XHCI_CRCR); 2697 xhci_op_write_8(sc, XHCI_CRCR, crcr | XHCI_CRCR_LO_CA); 2698 2699 for (i = 0; i < 500; i++) { 2700 crcr = xhci_op_read_8(sc, XHCI_CRCR); 2701 if ((crcr & XHCI_CRCR_LO_CRR) == 0) 2702 break; 2703 usb_delay_ms(&sc->sc_bus, 1); 2704 } 2705 if ((crcr & XHCI_CRCR_LO_CRR) != 0) { 2706 DPRINTFN(1, "Command Abort timeout", 0, 0, 0, 0); 2707 /* reset HC here? */ 2708 } 2709 2710 /* reset command ring dequeue pointer */ 2711 cr->xr_ep = 0; 2712 cr->xr_cs = 1; 2713 xhci_op_write_8(sc, XHCI_CRCR, xhci_ring_trbp(cr, 0) | cr->xr_cs); 2714 2715 mutex_exit(&cr->xr_lock); 2716 } 2717 2718 /* 2719 * Put a command on command ring, ring bell, set timer, and cv_timedwait. 2720 * Command completion is notified by cv_signal from xhci_event_cmd() 2721 * (called from xhci_softint), or timed-out. 2722 * The completion code is copied to sc->sc_result_trb in xhci_event_cmd(), 2723 * then do_command examines it. 2724 */ 2725 static usbd_status 2726 xhci_do_command_locked(struct xhci_softc * const sc, 2727 struct xhci_soft_trb * const trb, int timeout) 2728 { 2729 struct xhci_ring * const cr = &sc->sc_cr; 2730 usbd_status err; 2731 2732 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 2733 DPRINTFN(12, "input: 0x%016jx 0x%08jx 0x%08jx", 2734 trb->trb_0, trb->trb_2, trb->trb_3, 0); 2735 2736 KASSERTMSG(!cpu_intr_p() && !cpu_softintr_p(), "called from intr ctx"); 2737 KASSERT(mutex_owned(&sc->sc_lock)); 2738 2739 while (sc->sc_command_addr != 0) 2740 cv_wait(&sc->sc_cmdbusy_cv, &sc->sc_lock); 2741 2742 /* 2743 * If enqueue pointer points at last of ring, it's Link TRB, 2744 * command TRB will be stored in 0th TRB. 2745 */ 2746 if (cr->xr_ep == cr->xr_ntrb - 1) 2747 sc->sc_command_addr = xhci_ring_trbp(cr, 0); 2748 else 2749 sc->sc_command_addr = xhci_ring_trbp(cr, cr->xr_ep); 2750 2751 sc->sc_resultpending = true; 2752 2753 mutex_enter(&cr->xr_lock); 2754 xhci_ring_put(sc, cr, NULL, trb, 1); 2755 mutex_exit(&cr->xr_lock); 2756 2757 xhci_db_write_4(sc, XHCI_DOORBELL(0), 0); 2758 2759 while (sc->sc_resultpending) { 2760 if (cv_timedwait(&sc->sc_command_cv, &sc->sc_lock, 2761 MAX(1, mstohz(timeout))) == EWOULDBLOCK) { 2762 xhci_abort_command(sc); 2763 err = USBD_TIMEOUT; 2764 goto timedout; 2765 } 2766 } 2767 2768 trb->trb_0 = sc->sc_result_trb.trb_0; 2769 trb->trb_2 = sc->sc_result_trb.trb_2; 2770 trb->trb_3 = sc->sc_result_trb.trb_3; 2771 2772 DPRINTFN(12, "output: 0x%016jx 0x%08jx 0x%08jx", 2773 trb->trb_0, trb->trb_2, trb->trb_3, 0); 2774 2775 switch (XHCI_TRB_2_ERROR_GET(trb->trb_2)) { 2776 case XHCI_TRB_ERROR_SUCCESS: 2777 err = USBD_NORMAL_COMPLETION; 2778 break; 2779 default: 2780 case 192 ... 223: 2781 err = USBD_IOERROR; 2782 break; 2783 case 224 ... 255: 2784 err = USBD_NORMAL_COMPLETION; 2785 break; 2786 } 2787 2788 timedout: 2789 sc->sc_resultpending = false; 2790 sc->sc_command_addr = 0; 2791 cv_broadcast(&sc->sc_cmdbusy_cv); 2792 2793 return err; 2794 } 2795 2796 static usbd_status 2797 xhci_do_command(struct xhci_softc * const sc, struct xhci_soft_trb * const trb, 2798 int timeout) 2799 { 2800 2801 mutex_enter(&sc->sc_lock); 2802 usbd_status ret = xhci_do_command_locked(sc, trb, timeout); 2803 mutex_exit(&sc->sc_lock); 2804 2805 return ret; 2806 } 2807 2808 static usbd_status 2809 xhci_enable_slot(struct xhci_softc * const sc, uint8_t * const slotp) 2810 { 2811 struct xhci_soft_trb trb; 2812 usbd_status err; 2813 2814 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 2815 2816 trb.trb_0 = 0; 2817 trb.trb_2 = 0; 2818 trb.trb_3 = XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_ENABLE_SLOT); 2819 2820 err = xhci_do_command(sc, &trb, USBD_DEFAULT_TIMEOUT); 2821 if (err != USBD_NORMAL_COMPLETION) { 2822 return err; 2823 } 2824 2825 *slotp = XHCI_TRB_3_SLOT_GET(trb.trb_3); 2826 2827 return err; 2828 } 2829 2830 /* 2831 * xHCI 4.6.4 2832 * Deallocate ring and device/input context DMA buffers, and disable_slot. 2833 * All endpoints in the slot should be stopped. 2834 * Should be called with sc_lock held. 2835 */ 2836 static usbd_status 2837 xhci_disable_slot(struct xhci_softc * const sc, uint8_t slot) 2838 { 2839 struct xhci_soft_trb trb; 2840 struct xhci_slot *xs; 2841 usbd_status err; 2842 2843 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 2844 2845 if (sc->sc_dying) 2846 return USBD_IOERROR; 2847 2848 trb.trb_0 = 0; 2849 trb.trb_2 = 0; 2850 trb.trb_3 = XHCI_TRB_3_SLOT_SET(slot) | 2851 XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_DISABLE_SLOT); 2852 2853 err = xhci_do_command_locked(sc, &trb, USBD_DEFAULT_TIMEOUT); 2854 2855 if (!err) { 2856 xs = &sc->sc_slots[slot]; 2857 if (xs->xs_idx != 0) { 2858 xhci_free_slot(sc, xs, XHCI_DCI_SLOT + 1, 32); 2859 xhci_set_dcba(sc, 0, slot); 2860 memset(xs, 0, sizeof(*xs)); 2861 } 2862 } 2863 2864 return err; 2865 } 2866 2867 /* 2868 * Set address of device and transition slot state from ENABLED to ADDRESSED 2869 * if Block Setaddress Request (BSR) is false. 2870 * If BSR==true, transition slot state from ENABLED to DEFAULT. 2871 * see xHCI 1.1 4.5.3, 3.3.4 2872 * Should be called without sc_lock held. 2873 */ 2874 static usbd_status 2875 xhci_address_device(struct xhci_softc * const sc, 2876 uint64_t icp, uint8_t slot_id, bool bsr) 2877 { 2878 struct xhci_soft_trb trb; 2879 usbd_status err; 2880 2881 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 2882 2883 trb.trb_0 = icp; 2884 trb.trb_2 = 0; 2885 trb.trb_3 = XHCI_TRB_3_SLOT_SET(slot_id) | 2886 XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_ADDRESS_DEVICE) | 2887 (bsr ? XHCI_TRB_3_BSR_BIT : 0); 2888 2889 err = xhci_do_command(sc, &trb, USBD_DEFAULT_TIMEOUT); 2890 2891 if (XHCI_TRB_2_ERROR_GET(trb.trb_2) == XHCI_TRB_ERROR_NO_SLOTS) 2892 err = USBD_NO_ADDR; 2893 2894 return err; 2895 } 2896 2897 static usbd_status 2898 xhci_update_ep0_mps(struct xhci_softc * const sc, 2899 struct xhci_slot * const xs, u_int mps) 2900 { 2901 struct xhci_soft_trb trb; 2902 usbd_status err; 2903 uint32_t * cp; 2904 2905 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 2906 DPRINTFN(4, "slot %ju mps %ju", xs->xs_idx, mps, 0, 0); 2907 2908 cp = xhci_slot_get_icv(sc, xs, XHCI_ICI_INPUT_CONTROL); 2909 cp[0] = htole32(0); 2910 cp[1] = htole32(XHCI_INCTX_1_ADD_MASK(XHCI_DCI_EP_CONTROL)); 2911 2912 cp = xhci_slot_get_icv(sc, xs, xhci_dci_to_ici(XHCI_DCI_EP_CONTROL)); 2913 cp[1] = htole32(XHCI_EPCTX_1_MAXP_SIZE_SET(mps)); 2914 2915 /* sync input contexts before they are read from memory */ 2916 usb_syncmem(&xs->xs_ic_dma, 0, sc->sc_pgsz, BUS_DMASYNC_PREWRITE); 2917 HEXDUMP("input context", xhci_slot_get_icv(sc, xs, 0), 2918 sc->sc_ctxsz * 4); 2919 2920 trb.trb_0 = xhci_slot_get_icp(sc, xs, 0); 2921 trb.trb_2 = 0; 2922 trb.trb_3 = XHCI_TRB_3_SLOT_SET(xs->xs_idx) | 2923 XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_EVALUATE_CTX); 2924 2925 err = xhci_do_command(sc, &trb, USBD_DEFAULT_TIMEOUT); 2926 return err; 2927 } 2928 2929 static void 2930 xhci_set_dcba(struct xhci_softc * const sc, uint64_t dcba, int si) 2931 { 2932 uint64_t * const dcbaa = KERNADDR(&sc->sc_dcbaa_dma, 0); 2933 2934 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 2935 DPRINTFN(4, "dcbaa %#jx dc %016jx slot %jd", 2936 (uintptr_t)&dcbaa[si], dcba, si, 0); 2937 2938 dcbaa[si] = htole64(dcba); 2939 usb_syncmem(&sc->sc_dcbaa_dma, si * sizeof(uint64_t), sizeof(uint64_t), 2940 BUS_DMASYNC_PREWRITE); 2941 } 2942 2943 /* 2944 * Allocate device and input context DMA buffer, and 2945 * TRB DMA buffer for each endpoint. 2946 */ 2947 static usbd_status 2948 xhci_init_slot(struct usbd_device *dev, uint32_t slot) 2949 { 2950 struct xhci_softc * const sc = XHCI_BUS2SC(dev->ud_bus); 2951 struct xhci_slot *xs; 2952 usbd_status err; 2953 u_int dci; 2954 2955 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 2956 DPRINTFN(4, "slot %ju", slot, 0, 0, 0); 2957 2958 xs = &sc->sc_slots[slot]; 2959 2960 /* allocate contexts */ 2961 err = usb_allocmem(&sc->sc_bus, sc->sc_pgsz, sc->sc_pgsz, 2962 &xs->xs_dc_dma); 2963 if (err) 2964 return err; 2965 memset(KERNADDR(&xs->xs_dc_dma, 0), 0, sc->sc_pgsz); 2966 2967 err = usb_allocmem(&sc->sc_bus, sc->sc_pgsz, sc->sc_pgsz, 2968 &xs->xs_ic_dma); 2969 if (err) 2970 goto bad1; 2971 memset(KERNADDR(&xs->xs_ic_dma, 0), 0, sc->sc_pgsz); 2972 2973 for (dci = 0; dci < 32; dci++) { 2974 //CTASSERT(sizeof(xs->xs_ep[dci]) == sizeof(struct xhci_endpoint)); 2975 memset(&xs->xs_ep[dci], 0, sizeof(xs->xs_ep[dci])); 2976 if (dci == XHCI_DCI_SLOT) 2977 continue; 2978 err = xhci_ring_init(sc, &xs->xs_ep[dci].xe_tr, 2979 XHCI_TRANSFER_RING_TRBS, XHCI_TRB_ALIGN); 2980 if (err) { 2981 DPRINTFN(0, "ring init failure", 0, 0, 0, 0); 2982 goto bad2; 2983 } 2984 } 2985 2986 bad2: 2987 if (err == USBD_NORMAL_COMPLETION) { 2988 xs->xs_idx = slot; 2989 } else { 2990 xhci_free_slot(sc, xs, XHCI_DCI_SLOT + 1, dci); 2991 } 2992 2993 return err; 2994 2995 bad1: 2996 usb_freemem(&sc->sc_bus, &xs->xs_dc_dma); 2997 xs->xs_idx = 0; 2998 return err; 2999 } 3000 3001 static void 3002 xhci_free_slot(struct xhci_softc *sc, struct xhci_slot *xs, int start_dci, 3003 int end_dci) 3004 { 3005 u_int dci; 3006 3007 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 3008 DPRINTFN(4, "slot %ju start %ju end %ju", xs->xs_idx, start_dci, 3009 end_dci, 0); 3010 3011 for (dci = start_dci; dci < end_dci; dci++) { 3012 xhci_ring_free(sc, &xs->xs_ep[dci].xe_tr); 3013 memset(&xs->xs_ep[dci], 0, sizeof(xs->xs_ep[dci])); 3014 } 3015 usb_freemem(&sc->sc_bus, &xs->xs_ic_dma); 3016 usb_freemem(&sc->sc_bus, &xs->xs_dc_dma); 3017 xs->xs_idx = 0; 3018 } 3019 3020 /* 3021 * Setup slot context, set Device Context Base Address, and issue 3022 * Set Address Device command. 3023 */ 3024 static usbd_status 3025 xhci_set_address(struct usbd_device *dev, uint32_t slot, bool bsr) 3026 { 3027 struct xhci_softc * const sc = XHCI_BUS2SC(dev->ud_bus); 3028 struct xhci_slot *xs; 3029 usbd_status err; 3030 3031 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 3032 DPRINTFN(4, "slot %ju bsr %ju", slot, bsr, 0, 0); 3033 3034 xs = &sc->sc_slots[slot]; 3035 3036 xhci_setup_ctx(dev->ud_pipe0); 3037 3038 HEXDUMP("input context", xhci_slot_get_icv(sc, xs, 0), 3039 sc->sc_ctxsz * 3); 3040 3041 xhci_set_dcba(sc, DMAADDR(&xs->xs_dc_dma, 0), slot); 3042 3043 err = xhci_address_device(sc, xhci_slot_get_icp(sc, xs, 0), slot, bsr); 3044 3045 usb_syncmem(&xs->xs_dc_dma, 0, sc->sc_pgsz, BUS_DMASYNC_POSTREAD); 3046 HEXDUMP("output context", xhci_slot_get_dcv(sc, xs, 0), 3047 sc->sc_ctxsz * 2); 3048 3049 return err; 3050 } 3051 3052 /* 3053 * 4.8.2, 6.2.3.2 3054 * construct slot/endpoint context parameters and do syncmem 3055 */ 3056 static void 3057 xhci_setup_ctx(struct usbd_pipe *pipe) 3058 { 3059 struct xhci_softc * const sc = XHCI_PIPE2SC(pipe); 3060 struct usbd_device *dev = pipe->up_dev; 3061 struct xhci_slot * const xs = dev->ud_hcpriv; 3062 usb_endpoint_descriptor_t * const ed = pipe->up_endpoint->ue_edesc; 3063 const u_int dci = xhci_ep_get_dci(ed); 3064 const uint8_t xfertype = UE_GET_XFERTYPE(ed->bmAttributes); 3065 uint32_t *cp; 3066 uint16_t mps = UGETW(ed->wMaxPacketSize); 3067 uint8_t speed = dev->ud_speed; 3068 uint8_t ival = ed->bInterval; 3069 3070 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 3071 DPRINTFN(4, "pipe %#jx: slot %ju dci %ju speed %ju", 3072 (uintptr_t)pipe, xs->xs_idx, dci, speed); 3073 3074 /* set up initial input control context */ 3075 cp = xhci_slot_get_icv(sc, xs, XHCI_ICI_INPUT_CONTROL); 3076 cp[0] = htole32(0); 3077 cp[1] = htole32(XHCI_INCTX_1_ADD_MASK(dci)); 3078 cp[1] |= htole32(XHCI_INCTX_1_ADD_MASK(XHCI_DCI_SLOT)); 3079 cp[7] = htole32(0); 3080 3081 /* set up input slot context */ 3082 cp = xhci_slot_get_icv(sc, xs, xhci_dci_to_ici(XHCI_DCI_SLOT)); 3083 cp[0] = 3084 XHCI_SCTX_0_CTX_NUM_SET(dci) | 3085 XHCI_SCTX_0_SPEED_SET(xhci_speed2xspeed(speed)); 3086 cp[1] = 0; 3087 cp[2] = XHCI_SCTX_2_IRQ_TARGET_SET(0); 3088 cp[3] = 0; 3089 xhci_setup_route(pipe, cp); 3090 xhci_setup_tthub(pipe, cp); 3091 3092 cp[0] = htole32(cp[0]); 3093 cp[1] = htole32(cp[1]); 3094 cp[2] = htole32(cp[2]); 3095 cp[3] = htole32(cp[3]); 3096 3097 /* set up input endpoint context */ 3098 cp = xhci_slot_get_icv(sc, xs, xhci_dci_to_ici(dci)); 3099 cp[0] = 3100 XHCI_EPCTX_0_EPSTATE_SET(0) | 3101 XHCI_EPCTX_0_MULT_SET(0) | 3102 XHCI_EPCTX_0_MAXP_STREAMS_SET(0) | 3103 XHCI_EPCTX_0_LSA_SET(0) | 3104 XHCI_EPCTX_0_MAX_ESIT_PAYLOAD_HI_SET(0); 3105 cp[1] = 3106 XHCI_EPCTX_1_EPTYPE_SET(xhci_ep_get_type(ed)) | 3107 XHCI_EPCTX_1_HID_SET(0) | 3108 XHCI_EPCTX_1_MAXB_SET(0); 3109 3110 if (xfertype != UE_ISOCHRONOUS) 3111 cp[1] |= XHCI_EPCTX_1_CERR_SET(3); 3112 3113 if (xfertype == UE_CONTROL) 3114 cp[4] = XHCI_EPCTX_4_AVG_TRB_LEN_SET(8); /* 6.2.3 */ 3115 else if (USB_IS_SS(speed)) 3116 cp[4] = XHCI_EPCTX_4_AVG_TRB_LEN_SET(mps); 3117 else 3118 cp[4] = XHCI_EPCTX_4_AVG_TRB_LEN_SET(UE_GET_SIZE(mps)); 3119 3120 xhci_setup_maxburst(pipe, cp); 3121 3122 switch (xfertype) { 3123 case UE_CONTROL: 3124 break; 3125 case UE_BULK: 3126 /* XXX Set MaxPStreams, HID, and LSA if streams enabled */ 3127 break; 3128 case UE_INTERRUPT: 3129 if (pipe->up_interval != USBD_DEFAULT_INTERVAL) 3130 ival = pipe->up_interval; 3131 3132 ival = xhci_bival2ival(ival, speed); 3133 cp[0] |= XHCI_EPCTX_0_IVAL_SET(ival); 3134 break; 3135 case UE_ISOCHRONOUS: 3136 if (pipe->up_interval != USBD_DEFAULT_INTERVAL) 3137 ival = pipe->up_interval; 3138 3139 /* xHCI 6.2.3.6 Table 65, USB 2.0 9.6.6 */ 3140 if (speed == USB_SPEED_FULL) 3141 ival += 3; /* 1ms -> 125us */ 3142 ival--; 3143 cp[0] |= XHCI_EPCTX_0_IVAL_SET(ival); 3144 break; 3145 default: 3146 break; 3147 } 3148 DPRINTFN(4, "setting ival %ju MaxBurst %#jx", 3149 XHCI_EPCTX_0_IVAL_GET(cp[0]), XHCI_EPCTX_1_MAXB_GET(cp[1]), 0, 0); 3150 3151 /* rewind TR dequeue pointer in xHC */ 3152 /* can't use xhci_ep_get_dci() yet? */ 3153 *(uint64_t *)(&cp[2]) = htole64( 3154 xhci_ring_trbp(&xs->xs_ep[dci].xe_tr, 0) | 3155 XHCI_EPCTX_2_DCS_SET(1)); 3156 3157 cp[0] = htole32(cp[0]); 3158 cp[1] = htole32(cp[1]); 3159 cp[4] = htole32(cp[4]); 3160 3161 /* rewind TR dequeue pointer in driver */ 3162 struct xhci_ring *xr = &xs->xs_ep[dci].xe_tr; 3163 mutex_enter(&xr->xr_lock); 3164 xhci_host_dequeue(xr); 3165 mutex_exit(&xr->xr_lock); 3166 3167 /* sync input contexts before they are read from memory */ 3168 usb_syncmem(&xs->xs_ic_dma, 0, sc->sc_pgsz, BUS_DMASYNC_PREWRITE); 3169 } 3170 3171 /* 3172 * Setup route string and roothub port of given device for slot context 3173 */ 3174 static void 3175 xhci_setup_route(struct usbd_pipe *pipe, uint32_t *cp) 3176 { 3177 struct xhci_softc * const sc = XHCI_PIPE2SC(pipe); 3178 struct usbd_device *dev = pipe->up_dev; 3179 struct usbd_port *up = dev->ud_powersrc; 3180 struct usbd_device *hub; 3181 struct usbd_device *adev; 3182 uint8_t rhport = 0; 3183 uint32_t route = 0; 3184 3185 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 3186 3187 /* Locate root hub port and Determine route string */ 3188 /* 4.3.3 route string does not include roothub port */ 3189 for (hub = dev; hub != NULL; hub = hub->ud_myhub) { 3190 uint32_t dep; 3191 3192 DPRINTFN(4, "hub %#jx depth %jd upport %jp upportno %jd", 3193 (uintptr_t)hub, hub->ud_depth, (uintptr_t)hub->ud_powersrc, 3194 hub->ud_powersrc ? (uintptr_t)hub->ud_powersrc->up_portno : 3195 -1); 3196 3197 if (hub->ud_powersrc == NULL) 3198 break; 3199 dep = hub->ud_depth; 3200 if (dep == 0) 3201 break; 3202 rhport = hub->ud_powersrc->up_portno; 3203 if (dep > USB_HUB_MAX_DEPTH) 3204 continue; 3205 3206 route |= 3207 (rhport > UHD_SS_NPORTS_MAX ? UHD_SS_NPORTS_MAX : rhport) 3208 << ((dep - 1) * 4); 3209 } 3210 route = route >> 4; 3211 size_t bn = hub == sc->sc_bus.ub_roothub ? 0 : 1; 3212 3213 /* Locate port on upstream high speed hub */ 3214 for (adev = dev, hub = up->up_parent; 3215 hub != NULL && hub->ud_speed != USB_SPEED_HIGH; 3216 adev = hub, hub = hub->ud_myhub) 3217 ; 3218 if (hub) { 3219 int p; 3220 for (p = 0; p < hub->ud_hub->uh_hubdesc.bNbrPorts; p++) { 3221 if (hub->ud_hub->uh_ports[p].up_dev == adev) { 3222 dev->ud_myhsport = &hub->ud_hub->uh_ports[p]; 3223 goto found; 3224 } 3225 } 3226 panic("%s: cannot find HS port", __func__); 3227 found: 3228 DPRINTFN(4, "high speed port %jd", p, 0, 0, 0); 3229 } else { 3230 dev->ud_myhsport = NULL; 3231 } 3232 3233 const size_t ctlrport = xhci_rhport2ctlrport(sc, bn, rhport); 3234 3235 DPRINTFN(4, "rhport %ju ctlrport %ju Route %05jx hub %#jx", rhport, 3236 ctlrport, route, (uintptr_t)hub); 3237 3238 cp[0] |= XHCI_SCTX_0_ROUTE_SET(route); 3239 cp[1] |= XHCI_SCTX_1_RH_PORT_SET(ctlrport); 3240 } 3241 3242 /* 3243 * Setup whether device is hub, whether device uses MTT, and 3244 * TT informations if it uses MTT. 3245 */ 3246 static void 3247 xhci_setup_tthub(struct usbd_pipe *pipe, uint32_t *cp) 3248 { 3249 struct usbd_device *dev = pipe->up_dev; 3250 struct usbd_port *myhsport = dev->ud_myhsport; 3251 usb_device_descriptor_t * const dd = &dev->ud_ddesc; 3252 uint32_t speed = dev->ud_speed; 3253 uint8_t rhaddr = dev->ud_bus->ub_rhaddr; 3254 uint8_t tthubslot, ttportnum; 3255 bool ishub; 3256 bool usemtt; 3257 3258 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 3259 3260 /* 3261 * 6.2.2, Table 57-60, 6.2.2.1, 6.2.2.2 3262 * tthubslot: 3263 * This is the slot ID of parent HS hub 3264 * if LS/FS device is connected && connected through HS hub. 3265 * This is 0 if device is not LS/FS device || 3266 * parent hub is not HS hub || 3267 * attached to root hub. 3268 * ttportnum: 3269 * This is the downstream facing port of parent HS hub 3270 * if LS/FS device is connected. 3271 * This is 0 if device is not LS/FS device || 3272 * parent hub is not HS hub || 3273 * attached to root hub. 3274 */ 3275 if (myhsport && 3276 myhsport->up_parent->ud_addr != rhaddr && 3277 (speed == USB_SPEED_LOW || speed == USB_SPEED_FULL)) { 3278 ttportnum = myhsport->up_portno; 3279 tthubslot = myhsport->up_parent->ud_addr; 3280 } else { 3281 ttportnum = 0; 3282 tthubslot = 0; 3283 } 3284 DPRINTFN(4, "myhsport %#jx ttportnum=%jd tthubslot=%jd", 3285 (uintptr_t)myhsport, ttportnum, tthubslot, 0); 3286 3287 /* ishub is valid after reading UDESC_DEVICE */ 3288 ishub = (dd->bDeviceClass == UDCLASS_HUB); 3289 3290 /* dev->ud_hub is valid after reading UDESC_HUB */ 3291 if (ishub && dev->ud_hub) { 3292 usb_hub_descriptor_t *hd = &dev->ud_hub->uh_hubdesc; 3293 uint8_t ttt = 3294 __SHIFTOUT(UGETW(hd->wHubCharacteristics), UHD_TT_THINK); 3295 3296 cp[1] |= XHCI_SCTX_1_NUM_PORTS_SET(hd->bNbrPorts); 3297 cp[2] |= XHCI_SCTX_2_TT_THINK_TIME_SET(ttt); 3298 DPRINTFN(4, "nports=%jd ttt=%jd", hd->bNbrPorts, ttt, 0, 0); 3299 } 3300 3301 #define IS_MTTHUB(dd) \ 3302 ((dd)->bDeviceProtocol == UDPROTO_HSHUBMTT) 3303 3304 /* 3305 * MTT flag is set if 3306 * 1. this is HS hub && MTTs are supported and enabled; or 3307 * 2. this is LS or FS device && there is a parent HS hub where MTTs 3308 * are supported and enabled. 3309 * 3310 * XXX enabled is not tested yet 3311 */ 3312 if (ishub && speed == USB_SPEED_HIGH && IS_MTTHUB(dd)) 3313 usemtt = true; 3314 else if ((speed == USB_SPEED_LOW || speed == USB_SPEED_FULL) && 3315 myhsport && 3316 myhsport->up_parent->ud_addr != rhaddr && 3317 IS_MTTHUB(&myhsport->up_parent->ud_ddesc)) 3318 usemtt = true; 3319 else 3320 usemtt = false; 3321 DPRINTFN(4, "class %ju proto %ju ishub %jd usemtt %jd", 3322 dd->bDeviceClass, dd->bDeviceProtocol, ishub, usemtt); 3323 3324 #undef IS_MTTHUB 3325 3326 cp[0] |= 3327 XHCI_SCTX_0_HUB_SET(ishub ? 1 : 0) | 3328 XHCI_SCTX_0_MTT_SET(usemtt ? 1 : 0); 3329 cp[2] |= 3330 XHCI_SCTX_2_TT_HUB_SID_SET(tthubslot) | 3331 XHCI_SCTX_2_TT_PORT_NUM_SET(ttportnum); 3332 } 3333 3334 /* set up params for periodic endpoint */ 3335 static void 3336 xhci_setup_maxburst(struct usbd_pipe *pipe, uint32_t *cp) 3337 { 3338 struct usbd_device *dev = pipe->up_dev; 3339 usb_endpoint_descriptor_t * const ed = pipe->up_endpoint->ue_edesc; 3340 const uint8_t xfertype = UE_GET_XFERTYPE(ed->bmAttributes); 3341 usbd_desc_iter_t iter; 3342 const usb_cdc_descriptor_t *cdcd; 3343 uint32_t maxb = 0; 3344 uint16_t mps = UGETW(ed->wMaxPacketSize); 3345 uint8_t speed = dev->ud_speed; 3346 uint8_t ep; 3347 3348 /* config desc is NULL when opening ep0 */ 3349 if (dev == NULL || dev->ud_cdesc == NULL) 3350 goto no_cdcd; 3351 cdcd = (const usb_cdc_descriptor_t *)usb_find_desc(dev, 3352 UDESC_INTERFACE, USBD_CDCSUBTYPE_ANY); 3353 if (cdcd == NULL) 3354 goto no_cdcd; 3355 usb_desc_iter_init(dev, &iter); 3356 iter.cur = (const void *)cdcd; 3357 3358 /* find endpoint_ss_comp desc for ep of this pipe */ 3359 for (ep = 0;;) { 3360 cdcd = (const usb_cdc_descriptor_t *)usb_desc_iter_next(&iter); 3361 if (cdcd == NULL) 3362 break; 3363 if (ep == 0 && cdcd->bDescriptorType == UDESC_ENDPOINT) { 3364 ep = ((const usb_endpoint_descriptor_t *)cdcd)-> 3365 bEndpointAddress; 3366 if (UE_GET_ADDR(ep) == 3367 UE_GET_ADDR(ed->bEndpointAddress)) { 3368 cdcd = (const usb_cdc_descriptor_t *) 3369 usb_desc_iter_next(&iter); 3370 break; 3371 } 3372 ep = 0; 3373 } 3374 } 3375 if (cdcd != NULL && cdcd->bDescriptorType == UDESC_ENDPOINT_SS_COMP) { 3376 const usb_endpoint_ss_comp_descriptor_t * esscd = 3377 (const usb_endpoint_ss_comp_descriptor_t *)cdcd; 3378 maxb = esscd->bMaxBurst; 3379 } 3380 3381 no_cdcd: 3382 /* 6.2.3.4, 4.8.2.4 */ 3383 if (USB_IS_SS(speed)) { 3384 /* USB 3.1 9.6.6 */ 3385 cp[1] |= XHCI_EPCTX_1_MAXP_SIZE_SET(mps); 3386 /* USB 3.1 9.6.7 */ 3387 cp[1] |= XHCI_EPCTX_1_MAXB_SET(maxb); 3388 #ifdef notyet 3389 if (xfertype == UE_ISOCHRONOUS) { 3390 } 3391 if (XHCI_HCC2_LEC(sc->sc_hcc2) != 0) { 3392 /* use ESIT */ 3393 cp[4] |= XHCI_EPCTX_4_MAX_ESIT_PAYLOAD_SET(x); 3394 cp[0] |= XHCI_EPCTX_0_MAX_ESIT_PAYLOAD_HI_SET(x); 3395 3396 /* XXX if LEC = 1, set ESIT instead */ 3397 cp[0] |= XHCI_EPCTX_0_MULT_SET(0); 3398 } else { 3399 /* use ival */ 3400 } 3401 #endif 3402 } else { 3403 /* USB 2.0 9.6.6 */ 3404 cp[1] |= XHCI_EPCTX_1_MAXP_SIZE_SET(UE_GET_SIZE(mps)); 3405 3406 /* 6.2.3.4 */ 3407 if (speed == USB_SPEED_HIGH && 3408 (xfertype == UE_ISOCHRONOUS || xfertype == UE_INTERRUPT)) { 3409 maxb = UE_GET_TRANS(mps); 3410 } else { 3411 /* LS/FS or HS CTRL or HS BULK */ 3412 maxb = 0; 3413 } 3414 cp[1] |= XHCI_EPCTX_1_MAXB_SET(maxb); 3415 } 3416 } 3417 3418 /* 3419 * Convert endpoint bInterval value to endpoint context interval value 3420 * for Interrupt pipe. 3421 * xHCI 6.2.3.6 Table 65, USB 2.0 9.6.6 3422 */ 3423 static uint32_t 3424 xhci_bival2ival(uint32_t ival, uint32_t speed) 3425 { 3426 if (speed == USB_SPEED_LOW || speed == USB_SPEED_FULL) { 3427 int i; 3428 3429 /* 3430 * round ival down to "the nearest base 2 multiple of 3431 * bInterval * 8". 3432 * bInterval is at most 255 as its type is uByte. 3433 * 255(ms) = 2040(x 125us) < 2^11, so start with 10. 3434 */ 3435 for (i = 10; i > 0; i--) { 3436 if ((ival * 8) >= (1 << i)) 3437 break; 3438 } 3439 ival = i; 3440 } else { 3441 /* Interval = bInterval-1 for SS/HS */ 3442 ival--; 3443 } 3444 3445 return ival; 3446 } 3447 3448 /* ----- */ 3449 3450 static void 3451 xhci_noop(struct usbd_pipe *pipe) 3452 { 3453 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 3454 } 3455 3456 /* 3457 * Process root hub request. 3458 */ 3459 static int 3460 xhci_roothub_ctrl(struct usbd_bus *bus, usb_device_request_t *req, 3461 void *buf, int buflen) 3462 { 3463 struct xhci_softc * const sc = XHCI_BUS2SC(bus); 3464 usb_port_status_t ps; 3465 int l, totlen = 0; 3466 uint16_t len, value, index; 3467 int port, i; 3468 uint32_t v; 3469 3470 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 3471 3472 if (sc->sc_dying) 3473 return -1; 3474 3475 size_t bn = bus == &sc->sc_bus ? 0 : 1; 3476 3477 len = UGETW(req->wLength); 3478 value = UGETW(req->wValue); 3479 index = UGETW(req->wIndex); 3480 3481 DPRINTFN(12, "rhreq: %04jx %04jx %04jx %04jx", 3482 req->bmRequestType | (req->bRequest << 8), value, index, len); 3483 3484 #define C(x,y) ((x) | ((y) << 8)) 3485 switch (C(req->bRequest, req->bmRequestType)) { 3486 case C(UR_GET_DESCRIPTOR, UT_READ_DEVICE): 3487 DPRINTFN(8, "getdesc: wValue=0x%04jx", value, 0, 0, 0); 3488 if (len == 0) 3489 break; 3490 switch (value) { 3491 #define sd ((usb_string_descriptor_t *)buf) 3492 case C(2, UDESC_STRING): 3493 /* Product */ 3494 totlen = usb_makestrdesc(sd, len, "xHCI root hub"); 3495 break; 3496 #undef sd 3497 default: 3498 /* default from usbroothub */ 3499 return buflen; 3500 } 3501 break; 3502 3503 /* Hub requests */ 3504 case C(UR_CLEAR_FEATURE, UT_WRITE_CLASS_DEVICE): 3505 break; 3506 /* Clear Port Feature request */ 3507 case C(UR_CLEAR_FEATURE, UT_WRITE_CLASS_OTHER): { 3508 const size_t cp = xhci_rhport2ctlrport(sc, bn, index); 3509 3510 DPRINTFN(4, "UR_CLEAR_PORT_FEAT bp=%jd feat=%jd bus=%jd cp=%jd", 3511 index, value, bn, cp); 3512 if (index < 1 || index > sc->sc_rhportcount[bn]) { 3513 return -1; 3514 } 3515 port = XHCI_PORTSC(cp); 3516 v = xhci_op_read_4(sc, port); 3517 DPRINTFN(4, "portsc=0x%08jx", v, 0, 0, 0); 3518 v &= ~XHCI_PS_CLEAR; 3519 switch (value) { 3520 case UHF_PORT_ENABLE: 3521 xhci_op_write_4(sc, port, v & ~XHCI_PS_PED); 3522 break; 3523 case UHF_PORT_SUSPEND: 3524 return -1; 3525 case UHF_PORT_POWER: 3526 break; 3527 case UHF_PORT_TEST: 3528 case UHF_PORT_INDICATOR: 3529 return -1; 3530 case UHF_C_PORT_CONNECTION: 3531 xhci_op_write_4(sc, port, v | XHCI_PS_CSC); 3532 break; 3533 case UHF_C_PORT_ENABLE: 3534 case UHF_C_PORT_SUSPEND: 3535 case UHF_C_PORT_OVER_CURRENT: 3536 return -1; 3537 case UHF_C_BH_PORT_RESET: 3538 xhci_op_write_4(sc, port, v | XHCI_PS_WRC); 3539 break; 3540 case UHF_C_PORT_RESET: 3541 xhci_op_write_4(sc, port, v | XHCI_PS_PRC); 3542 break; 3543 case UHF_C_PORT_LINK_STATE: 3544 xhci_op_write_4(sc, port, v | XHCI_PS_PLC); 3545 break; 3546 case UHF_C_PORT_CONFIG_ERROR: 3547 xhci_op_write_4(sc, port, v | XHCI_PS_CEC); 3548 break; 3549 default: 3550 return -1; 3551 } 3552 break; 3553 } 3554 case C(UR_GET_DESCRIPTOR, UT_READ_CLASS_DEVICE): 3555 if (len == 0) 3556 break; 3557 if ((value & 0xff) != 0) { 3558 return -1; 3559 } 3560 usb_hub_descriptor_t hubd; 3561 3562 totlen = uimin(buflen, sizeof(hubd)); 3563 memcpy(&hubd, buf, totlen); 3564 hubd.bNbrPorts = sc->sc_rhportcount[bn]; 3565 USETW(hubd.wHubCharacteristics, UHD_PWR_NO_SWITCH); 3566 hubd.bPwrOn2PwrGood = 200; 3567 for (i = 0, l = sc->sc_rhportcount[bn]; l > 0; i++, l -= 8) { 3568 /* XXX can't find out? */ 3569 hubd.DeviceRemovable[i++] = 0; 3570 } 3571 hubd.bDescLength = USB_HUB_DESCRIPTOR_SIZE + i; 3572 totlen = uimin(totlen, hubd.bDescLength); 3573 memcpy(buf, &hubd, totlen); 3574 break; 3575 case C(UR_GET_STATUS, UT_READ_CLASS_DEVICE): 3576 if (len != 4) { 3577 return -1; 3578 } 3579 memset(buf, 0, len); /* ? XXX */ 3580 totlen = len; 3581 break; 3582 /* Get Port Status request */ 3583 case C(UR_GET_STATUS, UT_READ_CLASS_OTHER): { 3584 const size_t cp = xhci_rhport2ctlrport(sc, bn, index); 3585 3586 DPRINTFN(8, "get port status bn=%jd i=%jd cp=%ju", 3587 bn, index, cp, 0); 3588 if (index < 1 || index > sc->sc_rhportcount[bn]) { 3589 return -1; 3590 } 3591 if (len != 4) { 3592 return -1; 3593 } 3594 v = xhci_op_read_4(sc, XHCI_PORTSC(cp)); 3595 DPRINTFN(4, "getrhportsc %jd %08jx", cp, v, 0, 0); 3596 i = xhci_xspeed2psspeed(XHCI_PS_SPEED_GET(v)); 3597 if (v & XHCI_PS_CCS) i |= UPS_CURRENT_CONNECT_STATUS; 3598 if (v & XHCI_PS_PED) i |= UPS_PORT_ENABLED; 3599 if (v & XHCI_PS_OCA) i |= UPS_OVERCURRENT_INDICATOR; 3600 //if (v & XHCI_PS_SUSP) i |= UPS_SUSPEND; 3601 if (v & XHCI_PS_PR) i |= UPS_RESET; 3602 if (v & XHCI_PS_PP) { 3603 if (i & UPS_OTHER_SPEED) 3604 i |= UPS_PORT_POWER_SS; 3605 else 3606 i |= UPS_PORT_POWER; 3607 } 3608 if (i & UPS_OTHER_SPEED) 3609 i |= UPS_PORT_LS_SET(XHCI_PS_PLS_GET(v)); 3610 if (sc->sc_vendor_port_status) 3611 i = sc->sc_vendor_port_status(sc, v, i); 3612 USETW(ps.wPortStatus, i); 3613 i = 0; 3614 if (v & XHCI_PS_CSC) i |= UPS_C_CONNECT_STATUS; 3615 if (v & XHCI_PS_PEC) i |= UPS_C_PORT_ENABLED; 3616 if (v & XHCI_PS_OCC) i |= UPS_C_OVERCURRENT_INDICATOR; 3617 if (v & XHCI_PS_PRC) i |= UPS_C_PORT_RESET; 3618 if (v & XHCI_PS_WRC) i |= UPS_C_BH_PORT_RESET; 3619 if (v & XHCI_PS_PLC) i |= UPS_C_PORT_LINK_STATE; 3620 if (v & XHCI_PS_CEC) i |= UPS_C_PORT_CONFIG_ERROR; 3621 USETW(ps.wPortChange, i); 3622 totlen = uimin(len, sizeof(ps)); 3623 memcpy(buf, &ps, totlen); 3624 break; 3625 } 3626 case C(UR_SET_DESCRIPTOR, UT_WRITE_CLASS_DEVICE): 3627 return -1; 3628 case C(UR_SET_HUB_DEPTH, UT_WRITE_CLASS_DEVICE): 3629 break; 3630 case C(UR_SET_FEATURE, UT_WRITE_CLASS_DEVICE): 3631 break; 3632 /* Set Port Feature request */ 3633 case C(UR_SET_FEATURE, UT_WRITE_CLASS_OTHER): { 3634 int optval = (index >> 8) & 0xff; 3635 index &= 0xff; 3636 if (index < 1 || index > sc->sc_rhportcount[bn]) { 3637 return -1; 3638 } 3639 3640 const size_t cp = xhci_rhport2ctlrport(sc, bn, index); 3641 3642 port = XHCI_PORTSC(cp); 3643 v = xhci_op_read_4(sc, port); 3644 DPRINTFN(4, "index %jd cp %jd portsc=0x%08jx", index, cp, v, 0); 3645 v &= ~XHCI_PS_CLEAR; 3646 switch (value) { 3647 case UHF_PORT_ENABLE: 3648 xhci_op_write_4(sc, port, v | XHCI_PS_PED); 3649 break; 3650 case UHF_PORT_SUSPEND: 3651 /* XXX suspend */ 3652 break; 3653 case UHF_PORT_RESET: 3654 v &= ~(XHCI_PS_PED | XHCI_PS_PR); 3655 xhci_op_write_4(sc, port, v | XHCI_PS_PR); 3656 /* Wait for reset to complete. */ 3657 usb_delay_ms(&sc->sc_bus, USB_PORT_ROOT_RESET_DELAY); 3658 if (sc->sc_dying) { 3659 return -1; 3660 } 3661 v = xhci_op_read_4(sc, port); 3662 if (v & XHCI_PS_PR) { 3663 xhci_op_write_4(sc, port, v & ~XHCI_PS_PR); 3664 usb_delay_ms(&sc->sc_bus, 10); 3665 /* XXX */ 3666 } 3667 break; 3668 case UHF_PORT_POWER: 3669 /* XXX power control */ 3670 break; 3671 /* XXX more */ 3672 case UHF_C_PORT_RESET: 3673 xhci_op_write_4(sc, port, v | XHCI_PS_PRC); 3674 break; 3675 case UHF_PORT_U1_TIMEOUT: 3676 if (XHCI_PS_SPEED_GET(v) < XHCI_PS_SPEED_SS) { 3677 return -1; 3678 } 3679 port = XHCI_PORTPMSC(cp); 3680 v = xhci_op_read_4(sc, port); 3681 DPRINTFN(4, "index %jd cp %jd portpmsc=0x%08jx", 3682 index, cp, v, 0); 3683 v &= ~XHCI_PM3_U1TO_SET(0xff); 3684 v |= XHCI_PM3_U1TO_SET(optval); 3685 xhci_op_write_4(sc, port, v); 3686 break; 3687 case UHF_PORT_U2_TIMEOUT: 3688 if (XHCI_PS_SPEED_GET(v) < XHCI_PS_SPEED_SS) { 3689 return -1; 3690 } 3691 port = XHCI_PORTPMSC(cp); 3692 v = xhci_op_read_4(sc, port); 3693 DPRINTFN(4, "index %jd cp %jd portpmsc=0x%08jx", 3694 index, cp, v, 0); 3695 v &= ~XHCI_PM3_U2TO_SET(0xff); 3696 v |= XHCI_PM3_U2TO_SET(optval); 3697 xhci_op_write_4(sc, port, v); 3698 break; 3699 default: 3700 return -1; 3701 } 3702 } 3703 break; 3704 case C(UR_CLEAR_TT_BUFFER, UT_WRITE_CLASS_OTHER): 3705 case C(UR_RESET_TT, UT_WRITE_CLASS_OTHER): 3706 case C(UR_GET_TT_STATE, UT_READ_CLASS_OTHER): 3707 case C(UR_STOP_TT, UT_WRITE_CLASS_OTHER): 3708 break; 3709 default: 3710 /* default from usbroothub */ 3711 return buflen; 3712 } 3713 3714 return totlen; 3715 } 3716 3717 /* root hub interrupt */ 3718 3719 static usbd_status 3720 xhci_root_intr_transfer(struct usbd_xfer *xfer) 3721 { 3722 struct xhci_softc * const sc = XHCI_XFER2SC(xfer); 3723 usbd_status err; 3724 3725 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 3726 3727 /* Insert last in queue. */ 3728 mutex_enter(&sc->sc_lock); 3729 err = usb_insert_transfer(xfer); 3730 mutex_exit(&sc->sc_lock); 3731 if (err) 3732 return err; 3733 3734 /* Pipe isn't running, start first */ 3735 return xhci_root_intr_start(SIMPLEQ_FIRST(&xfer->ux_pipe->up_queue)); 3736 } 3737 3738 /* Wait for roothub port status/change */ 3739 static usbd_status 3740 xhci_root_intr_start(struct usbd_xfer *xfer) 3741 { 3742 struct xhci_softc * const sc = XHCI_XFER2SC(xfer); 3743 const size_t bn = XHCI_XFER2BUS(xfer) == &sc->sc_bus ? 0 : 1; 3744 const bool polling = xhci_polling_p(sc); 3745 3746 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 3747 3748 if (sc->sc_dying) 3749 return USBD_IOERROR; 3750 3751 if (!polling) 3752 mutex_enter(&sc->sc_lock); 3753 sc->sc_intrxfer[bn] = xfer; 3754 if (!polling) 3755 mutex_exit(&sc->sc_lock); 3756 3757 return USBD_IN_PROGRESS; 3758 } 3759 3760 static void 3761 xhci_root_intr_abort(struct usbd_xfer *xfer) 3762 { 3763 struct xhci_softc * const sc __diagused = XHCI_XFER2SC(xfer); 3764 3765 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 3766 3767 KASSERT(mutex_owned(&sc->sc_lock)); 3768 KASSERT(xfer->ux_pipe->up_intrxfer == xfer); 3769 3770 xfer->ux_status = USBD_CANCELLED; 3771 usb_transfer_complete(xfer); 3772 } 3773 3774 static void 3775 xhci_root_intr_close(struct usbd_pipe *pipe) 3776 { 3777 struct xhci_softc * const sc = XHCI_PIPE2SC(pipe); 3778 const struct usbd_xfer *xfer = pipe->up_intrxfer; 3779 const size_t bn = XHCI_XFER2BUS(xfer) == &sc->sc_bus ? 0 : 1; 3780 3781 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 3782 3783 KASSERT(mutex_owned(&sc->sc_lock)); 3784 3785 sc->sc_intrxfer[bn] = NULL; 3786 } 3787 3788 static void 3789 xhci_root_intr_done(struct usbd_xfer *xfer) 3790 { 3791 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 3792 3793 } 3794 3795 /* -------------- */ 3796 /* device control */ 3797 3798 static usbd_status 3799 xhci_device_ctrl_transfer(struct usbd_xfer *xfer) 3800 { 3801 struct xhci_softc * const sc = XHCI_XFER2SC(xfer); 3802 usbd_status err; 3803 3804 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 3805 3806 /* Insert last in queue. */ 3807 mutex_enter(&sc->sc_lock); 3808 err = usb_insert_transfer(xfer); 3809 mutex_exit(&sc->sc_lock); 3810 if (err) 3811 return err; 3812 3813 /* Pipe isn't running, start first */ 3814 return xhci_device_ctrl_start(SIMPLEQ_FIRST(&xfer->ux_pipe->up_queue)); 3815 } 3816 3817 static usbd_status 3818 xhci_device_ctrl_start(struct usbd_xfer *xfer) 3819 { 3820 struct xhci_softc * const sc = XHCI_XFER2SC(xfer); 3821 struct xhci_slot * const xs = xfer->ux_pipe->up_dev->ud_hcpriv; 3822 const u_int dci = xhci_ep_get_dci(xfer->ux_pipe->up_endpoint->ue_edesc); 3823 struct xhci_ring * const tr = &xs->xs_ep[dci].xe_tr; 3824 struct xhci_xfer * const xx = XHCI_XFER2XXFER(xfer); 3825 usb_device_request_t * const req = &xfer->ux_request; 3826 const int isread = usbd_xfer_isread(xfer); 3827 const uint32_t len = UGETW(req->wLength); 3828 usb_dma_t * const dma = &xfer->ux_dmabuf; 3829 uint64_t parameter; 3830 uint32_t status; 3831 uint32_t control; 3832 u_int i; 3833 const bool polling = xhci_polling_p(sc); 3834 3835 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 3836 DPRINTFN(12, "req: %04jx %04jx %04jx %04jx", 3837 req->bmRequestType | (req->bRequest << 8), UGETW(req->wValue), 3838 UGETW(req->wIndex), UGETW(req->wLength)); 3839 3840 /* we rely on the bottom bits for extra info */ 3841 KASSERTMSG(((uintptr_t)xfer & 0x3) == 0x0, "xfer %zx", 3842 (uintptr_t) xfer); 3843 3844 KASSERT((xfer->ux_rqflags & URQ_REQUEST) != 0); 3845 3846 i = 0; 3847 3848 /* setup phase */ 3849 memcpy(¶meter, req, sizeof(parameter)); 3850 status = XHCI_TRB_2_IRQ_SET(0) | XHCI_TRB_2_BYTES_SET(sizeof(*req)); 3851 control = ((len == 0) ? XHCI_TRB_3_TRT_NONE : 3852 (isread ? XHCI_TRB_3_TRT_IN : XHCI_TRB_3_TRT_OUT)) | 3853 XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_SETUP_STAGE) | 3854 XHCI_TRB_3_IDT_BIT; 3855 /* we need parameter un-swapped on big endian, so pre-swap it here */ 3856 xhci_soft_trb_put(&xx->xx_trb[i++], htole64(parameter), status, control); 3857 3858 if (len != 0) { 3859 /* data phase */ 3860 parameter = DMAADDR(dma, 0); 3861 KASSERTMSG(len <= 0x10000, "len %d", len); 3862 status = XHCI_TRB_2_IRQ_SET(0) | 3863 XHCI_TRB_2_TDSZ_SET(1) | 3864 XHCI_TRB_2_BYTES_SET(len); 3865 control = (isread ? XHCI_TRB_3_DIR_IN : 0) | 3866 XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_DATA_STAGE) | 3867 (usbd_xfer_isread(xfer) ? XHCI_TRB_3_ISP_BIT : 0) | 3868 XHCI_TRB_3_IOC_BIT; 3869 xhci_soft_trb_put(&xx->xx_trb[i++], parameter, status, control); 3870 } 3871 3872 parameter = 0; 3873 status = XHCI_TRB_2_IRQ_SET(0); 3874 /* the status stage has inverted direction */ 3875 control = ((isread && (len > 0)) ? 0 : XHCI_TRB_3_DIR_IN) | 3876 XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_STATUS_STAGE) | 3877 XHCI_TRB_3_IOC_BIT; 3878 xhci_soft_trb_put(&xx->xx_trb[i++], parameter, status, control); 3879 xfer->ux_status = USBD_IN_PROGRESS; 3880 3881 if (!polling) 3882 mutex_enter(&tr->xr_lock); 3883 xhci_ring_put(sc, tr, xfer, xx->xx_trb, i); 3884 if (!polling) 3885 mutex_exit(&tr->xr_lock); 3886 3887 xhci_db_write_4(sc, XHCI_DOORBELL(xs->xs_idx), dci); 3888 3889 if (xfer->ux_timeout && !xhci_polling_p(sc)) { 3890 callout_reset(&xfer->ux_callout, mstohz(xfer->ux_timeout), 3891 xhci_timeout, xfer); 3892 } 3893 3894 return USBD_IN_PROGRESS; 3895 } 3896 3897 static void 3898 xhci_device_ctrl_done(struct usbd_xfer *xfer) 3899 { 3900 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 3901 usb_device_request_t *req = &xfer->ux_request; 3902 int len = UGETW(req->wLength); 3903 int rd = req->bmRequestType & UT_READ; 3904 3905 if (len) 3906 usb_syncmem(&xfer->ux_dmabuf, 0, len, 3907 rd ? BUS_DMASYNC_POSTREAD : BUS_DMASYNC_POSTWRITE); 3908 } 3909 3910 static void 3911 xhci_device_ctrl_abort(struct usbd_xfer *xfer) 3912 { 3913 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 3914 3915 xhci_abort_xfer(xfer, USBD_CANCELLED); 3916 } 3917 3918 static void 3919 xhci_device_ctrl_close(struct usbd_pipe *pipe) 3920 { 3921 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 3922 3923 xhci_close_pipe(pipe); 3924 } 3925 3926 /* ------------------ */ 3927 /* device isochronous */ 3928 3929 /* ----------- */ 3930 /* device bulk */ 3931 3932 static usbd_status 3933 xhci_device_bulk_transfer(struct usbd_xfer *xfer) 3934 { 3935 struct xhci_softc * const sc = XHCI_XFER2SC(xfer); 3936 usbd_status err; 3937 3938 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 3939 3940 /* Insert last in queue. */ 3941 mutex_enter(&sc->sc_lock); 3942 err = usb_insert_transfer(xfer); 3943 mutex_exit(&sc->sc_lock); 3944 if (err) 3945 return err; 3946 3947 /* 3948 * Pipe isn't running (otherwise err would be USBD_INPROG), 3949 * so start it first. 3950 */ 3951 return xhci_device_bulk_start(SIMPLEQ_FIRST(&xfer->ux_pipe->up_queue)); 3952 } 3953 3954 static usbd_status 3955 xhci_device_bulk_start(struct usbd_xfer *xfer) 3956 { 3957 struct xhci_softc * const sc = XHCI_XFER2SC(xfer); 3958 struct xhci_slot * const xs = xfer->ux_pipe->up_dev->ud_hcpriv; 3959 const u_int dci = xhci_ep_get_dci(xfer->ux_pipe->up_endpoint->ue_edesc); 3960 struct xhci_ring * const tr = &xs->xs_ep[dci].xe_tr; 3961 struct xhci_xfer * const xx = XHCI_XFER2XXFER(xfer); 3962 const uint32_t len = xfer->ux_length; 3963 usb_dma_t * const dma = &xfer->ux_dmabuf; 3964 uint64_t parameter; 3965 uint32_t status; 3966 uint32_t control; 3967 u_int i = 0; 3968 const bool polling = xhci_polling_p(sc); 3969 3970 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 3971 3972 DPRINTFN(15, "%#jx slot %ju dci %ju", (uintptr_t)xfer, xs->xs_idx, dci, 3973 0); 3974 3975 if (sc->sc_dying) 3976 return USBD_IOERROR; 3977 3978 KASSERT((xfer->ux_rqflags & URQ_REQUEST) == 0); 3979 3980 parameter = DMAADDR(dma, 0); 3981 /* 3982 * XXX: (dsl) The physical buffer must not cross a 64k boundary. 3983 * If the user supplied buffer crosses such a boundary then 2 3984 * (or more) TRB should be used. 3985 * If multiple TRB are used the td_size field must be set correctly. 3986 * For v1.0 devices (like ivy bridge) this is the number of usb data 3987 * blocks needed to complete the transfer. 3988 * Setting it to 1 in the last TRB causes an extra zero-length 3989 * data block be sent. 3990 * The earlier documentation differs, I don't know how it behaves. 3991 */ 3992 KASSERTMSG(len <= 0x10000, "len %d", len); 3993 status = XHCI_TRB_2_IRQ_SET(0) | 3994 XHCI_TRB_2_TDSZ_SET(1) | 3995 XHCI_TRB_2_BYTES_SET(len); 3996 control = XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_NORMAL) | 3997 (usbd_xfer_isread(xfer) ? XHCI_TRB_3_ISP_BIT : 0) | 3998 XHCI_TRB_3_IOC_BIT; 3999 xhci_soft_trb_put(&xx->xx_trb[i++], parameter, status, control); 4000 xfer->ux_status = USBD_IN_PROGRESS; 4001 4002 if (!polling) 4003 mutex_enter(&tr->xr_lock); 4004 xhci_ring_put(sc, tr, xfer, xx->xx_trb, i); 4005 if (!polling) 4006 mutex_exit(&tr->xr_lock); 4007 4008 xhci_db_write_4(sc, XHCI_DOORBELL(xs->xs_idx), dci); 4009 4010 if (xfer->ux_timeout && !xhci_polling_p(sc)) { 4011 callout_reset(&xfer->ux_callout, mstohz(xfer->ux_timeout), 4012 xhci_timeout, xfer); 4013 } 4014 4015 return USBD_IN_PROGRESS; 4016 } 4017 4018 static void 4019 xhci_device_bulk_done(struct usbd_xfer *xfer) 4020 { 4021 #ifdef USB_DEBUG 4022 struct xhci_slot * const xs = xfer->ux_pipe->up_dev->ud_hcpriv; 4023 const u_int dci = xhci_ep_get_dci(xfer->ux_pipe->up_endpoint->ue_edesc); 4024 #endif 4025 const int isread = usbd_xfer_isread(xfer); 4026 4027 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 4028 4029 DPRINTFN(15, "%#jx slot %ju dci %ju", (uintptr_t)xfer, xs->xs_idx, dci, 4030 0); 4031 4032 usb_syncmem(&xfer->ux_dmabuf, 0, xfer->ux_length, 4033 isread ? BUS_DMASYNC_POSTREAD : BUS_DMASYNC_POSTWRITE); 4034 } 4035 4036 static void 4037 xhci_device_bulk_abort(struct usbd_xfer *xfer) 4038 { 4039 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 4040 4041 xhci_abort_xfer(xfer, USBD_CANCELLED); 4042 } 4043 4044 static void 4045 xhci_device_bulk_close(struct usbd_pipe *pipe) 4046 { 4047 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 4048 4049 xhci_close_pipe(pipe); 4050 } 4051 4052 /* ---------------- */ 4053 /* device interrupt */ 4054 4055 static usbd_status 4056 xhci_device_intr_transfer(struct usbd_xfer *xfer) 4057 { 4058 struct xhci_softc * const sc = XHCI_XFER2SC(xfer); 4059 usbd_status err; 4060 4061 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 4062 4063 /* Insert last in queue. */ 4064 mutex_enter(&sc->sc_lock); 4065 err = usb_insert_transfer(xfer); 4066 mutex_exit(&sc->sc_lock); 4067 if (err) 4068 return err; 4069 4070 /* 4071 * Pipe isn't running (otherwise err would be USBD_INPROG), 4072 * so start it first. 4073 */ 4074 return xhci_device_intr_start(SIMPLEQ_FIRST(&xfer->ux_pipe->up_queue)); 4075 } 4076 4077 static usbd_status 4078 xhci_device_intr_start(struct usbd_xfer *xfer) 4079 { 4080 struct xhci_softc * const sc = XHCI_XFER2SC(xfer); 4081 struct xhci_slot * const xs = xfer->ux_pipe->up_dev->ud_hcpriv; 4082 const u_int dci = xhci_ep_get_dci(xfer->ux_pipe->up_endpoint->ue_edesc); 4083 struct xhci_ring * const tr = &xs->xs_ep[dci].xe_tr; 4084 struct xhci_xfer * const xx = XHCI_XFER2XXFER(xfer); 4085 const uint32_t len = xfer->ux_length; 4086 const bool polling = xhci_polling_p(sc); 4087 usb_dma_t * const dma = &xfer->ux_dmabuf; 4088 uint64_t parameter; 4089 uint32_t status; 4090 uint32_t control; 4091 u_int i = 0; 4092 4093 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 4094 4095 DPRINTFN(15, "%#jx slot %ju dci %ju", (uintptr_t)xfer, xs->xs_idx, dci, 4096 0); 4097 4098 if (sc->sc_dying) 4099 return USBD_IOERROR; 4100 4101 KASSERT((xfer->ux_rqflags & URQ_REQUEST) == 0); 4102 4103 parameter = DMAADDR(dma, 0); 4104 KASSERTMSG(len <= 0x10000, "len %d", len); 4105 status = XHCI_TRB_2_IRQ_SET(0) | 4106 XHCI_TRB_2_TDSZ_SET(1) | 4107 XHCI_TRB_2_BYTES_SET(len); 4108 control = XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_NORMAL) | 4109 (usbd_xfer_isread(xfer) ? XHCI_TRB_3_ISP_BIT : 0) | 4110 XHCI_TRB_3_IOC_BIT; 4111 xhci_soft_trb_put(&xx->xx_trb[i++], parameter, status, control); 4112 xfer->ux_status = USBD_IN_PROGRESS; 4113 4114 if (!polling) 4115 mutex_enter(&tr->xr_lock); 4116 xhci_ring_put(sc, tr, xfer, xx->xx_trb, i); 4117 if (!polling) 4118 mutex_exit(&tr->xr_lock); 4119 4120 xhci_db_write_4(sc, XHCI_DOORBELL(xs->xs_idx), dci); 4121 4122 if (xfer->ux_timeout && !polling) { 4123 callout_reset(&xfer->ux_callout, mstohz(xfer->ux_timeout), 4124 xhci_timeout, xfer); 4125 } 4126 4127 return USBD_IN_PROGRESS; 4128 } 4129 4130 static void 4131 xhci_device_intr_done(struct usbd_xfer *xfer) 4132 { 4133 struct xhci_softc * const sc __diagused = XHCI_XFER2SC(xfer); 4134 #ifdef USB_DEBUG 4135 struct xhci_slot * const xs = xfer->ux_pipe->up_dev->ud_hcpriv; 4136 const u_int dci = xhci_ep_get_dci(xfer->ux_pipe->up_endpoint->ue_edesc); 4137 #endif 4138 const int isread = usbd_xfer_isread(xfer); 4139 4140 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 4141 4142 DPRINTFN(15, "%#jx slot %ju dci %ju", (uintptr_t)xfer, xs->xs_idx, dci, 4143 0); 4144 4145 KASSERT(xhci_polling_p(sc) || mutex_owned(&sc->sc_lock)); 4146 4147 usb_syncmem(&xfer->ux_dmabuf, 0, xfer->ux_length, 4148 isread ? BUS_DMASYNC_POSTREAD : BUS_DMASYNC_POSTWRITE); 4149 } 4150 4151 static void 4152 xhci_device_intr_abort(struct usbd_xfer *xfer) 4153 { 4154 struct xhci_softc * const sc __diagused = XHCI_XFER2SC(xfer); 4155 4156 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 4157 4158 KASSERT(mutex_owned(&sc->sc_lock)); 4159 DPRINTFN(15, "%#jx", (uintptr_t)xfer, 0, 0, 0); 4160 KASSERT(xfer->ux_pipe->up_intrxfer == xfer); 4161 xhci_abort_xfer(xfer, USBD_CANCELLED); 4162 } 4163 4164 static void 4165 xhci_device_intr_close(struct usbd_pipe *pipe) 4166 { 4167 //struct xhci_softc * const sc = XHCI_PIPE2SC(pipe); 4168 4169 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 4170 DPRINTFN(15, "%#jx", (uintptr_t)pipe, 0, 0, 0); 4171 4172 xhci_close_pipe(pipe); 4173 } 4174 4175 /* ------------ */ 4176 4177 static void 4178 xhci_timeout(void *addr) 4179 { 4180 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 4181 struct xhci_xfer * const xx = addr; 4182 struct usbd_xfer * const xfer = &xx->xx_xfer; 4183 struct xhci_softc * const sc = XHCI_XFER2SC(xfer); 4184 struct usbd_device *dev = xfer->ux_pipe->up_dev; 4185 4186 mutex_enter(&sc->sc_lock); 4187 if (!sc->sc_dying && xfer->ux_status == USBD_IN_PROGRESS) 4188 usb_add_task(dev, &xfer->ux_aborttask, USB_TASKQ_HC); 4189 mutex_exit(&sc->sc_lock); 4190 } 4191 4192 static void 4193 xhci_timeout_task(void *addr) 4194 { 4195 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 4196 struct usbd_xfer * const xfer = addr; 4197 struct xhci_softc * const sc = XHCI_XFER2SC(xfer); 4198 4199 mutex_enter(&sc->sc_lock); 4200 xhci_abort_xfer(xfer, USBD_TIMEOUT); 4201 mutex_exit(&sc->sc_lock); 4202 } 4203
Indexes created Sat Sep 27 14:09:57 GMT 2025