xhci.c revision 1.107.2.10
1 /* $NetBSD: xhci.c,v 1.107.2.10 2022/09/16 18:32:49 martin Exp $ */ 2 3 /* 4 * Copyright (c) 2013 Jonathan A. Kollasch 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 17 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 18 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 19 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR 20 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 21 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 22 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 23 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 24 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 25 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 26 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 */ 28 29 /* 30 * USB rev 2.0 and rev 3.1 specification 31 * http://www.usb.org/developers/docs/ 32 * xHCI rev 1.1 specification 33 * http://www.intel.com/technology/usb/spec.htm 34 */ 35 36 #include <sys/cdefs.h> 37 __KERNEL_RCSID(0, "$NetBSD: xhci.c,v 1.107.2.10 2022/09/16 18:32:49 martin Exp $"); 38 39 #ifdef _KERNEL_OPT 40 #include "opt_usb.h" 41 #endif 42 43 #include <sys/param.h> 44 #include <sys/systm.h> 45 #include <sys/kernel.h> 46 #include <sys/kmem.h> 47 #include <sys/device.h> 48 #include <sys/select.h> 49 #include <sys/proc.h> 50 #include <sys/queue.h> 51 #include <sys/mutex.h> 52 #include <sys/condvar.h> 53 #include <sys/bus.h> 54 #include <sys/cpu.h> 55 #include <sys/sysctl.h> 56 57 #include <machine/endian.h> 58 59 #include <dev/usb/usb.h> 60 #include <dev/usb/usbdi.h> 61 #include <dev/usb/usbdivar.h> 62 #include <dev/usb/usbdi_util.h> 63 #include <dev/usb/usbhist.h> 64 #include <dev/usb/usb_mem.h> 65 #include <dev/usb/usb_quirks.h> 66 67 #include <dev/usb/xhcireg.h> 68 #include <dev/usb/xhcivar.h> 69 #include <dev/usb/usbroothub.h> 70 71 72 #ifdef USB_DEBUG 73 #ifndef XHCI_DEBUG 74 #define xhcidebug 0 75 #else /* !XHCI_DEBUG */ 76 #define HEXDUMP(a, b, c) \ 77 do { \ 78 if (xhcidebug > 0) \ 79 hexdump(printf, a, b, c); \ 80 } while (/*CONSTCOND*/0) 81 static int xhcidebug = 0; 82 83 SYSCTL_SETUP(sysctl_hw_xhci_setup, "sysctl hw.xhci setup") 84 { 85 int err; 86 const struct sysctlnode *rnode; 87 const struct sysctlnode *cnode; 88 89 err = sysctl_createv(clog, 0, NULL, &rnode, 90 CTLFLAG_PERMANENT, CTLTYPE_NODE, "xhci", 91 SYSCTL_DESCR("xhci global controls"), 92 NULL, 0, NULL, 0, CTL_HW, CTL_CREATE, CTL_EOL); 93 94 if (err) 95 goto fail; 96 97 /* control debugging printfs */ 98 err = sysctl_createv(clog, 0, &rnode, &cnode, 99 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, CTLTYPE_INT, 100 "debug", SYSCTL_DESCR("Enable debugging output"), 101 NULL, 0, &xhcidebug, sizeof(xhcidebug), CTL_CREATE, CTL_EOL); 102 if (err) 103 goto fail; 104 105 return; 106 fail: 107 aprint_error("%s: sysctl_createv failed (err = %d)\n", __func__, err); 108 } 109 110 #endif /* !XHCI_DEBUG */ 111 #endif /* USB_DEBUG */ 112 113 #ifndef HEXDUMP 114 #define HEXDUMP(a, b, c) 115 #endif 116 117 #define DPRINTF(FMT,A,B,C,D) USBHIST_LOG(xhcidebug,FMT,A,B,C,D) 118 #define DPRINTFN(N,FMT,A,B,C,D) USBHIST_LOGN(xhcidebug,N,FMT,A,B,C,D) 119 #define XHCIHIST_FUNC() USBHIST_FUNC() 120 #define XHCIHIST_CALLED(name) USBHIST_CALLED(xhcidebug) 121 #define XHCIHIST_CALLARGS(FMT,A,B,C,D) \ 122 USBHIST_CALLARGS(xhcidebug,FMT,A,B,C,D) 123 124 #define XHCI_DCI_SLOT 0 125 #define XHCI_DCI_EP_CONTROL 1 126 127 #define XHCI_ICI_INPUT_CONTROL 0 128 129 struct xhci_pipe { 130 struct usbd_pipe xp_pipe; 131 struct usb_task xp_async_task; 132 }; 133 134 #define XHCI_COMMAND_RING_TRBS 256 135 #define XHCI_EVENT_RING_TRBS 256 136 #define XHCI_EVENT_RING_SEGMENTS 1 137 #define XHCI_TRB_3_ED_BIT XHCI_TRB_3_ISP_BIT 138 139 static usbd_status xhci_open(struct usbd_pipe *); 140 static void xhci_close_pipe(struct usbd_pipe *); 141 static int xhci_intr1(struct xhci_softc * const); 142 static void xhci_softintr(void *); 143 static void xhci_poll(struct usbd_bus *); 144 static struct usbd_xfer *xhci_allocx(struct usbd_bus *, unsigned int); 145 static void xhci_freex(struct usbd_bus *, struct usbd_xfer *); 146 static void xhci_abortx(struct usbd_xfer *); 147 static bool xhci_dying(struct usbd_bus *); 148 static void xhci_get_lock(struct usbd_bus *, kmutex_t **); 149 static usbd_status xhci_new_device(device_t, struct usbd_bus *, int, int, int, 150 struct usbd_port *); 151 static int xhci_roothub_ctrl(struct usbd_bus *, usb_device_request_t *, 152 void *, int); 153 154 static usbd_status xhci_configure_endpoint(struct usbd_pipe *); 155 //static usbd_status xhci_unconfigure_endpoint(struct usbd_pipe *); 156 static usbd_status xhci_reset_endpoint(struct usbd_pipe *); 157 static usbd_status xhci_stop_endpoint_cmd(struct xhci_softc *, 158 struct xhci_slot *, u_int, uint32_t); 159 static usbd_status xhci_stop_endpoint(struct usbd_pipe *); 160 161 static void xhci_host_dequeue(struct xhci_ring * const); 162 static usbd_status xhci_set_dequeue(struct usbd_pipe *); 163 164 static usbd_status xhci_do_command(struct xhci_softc * const, 165 struct xhci_soft_trb * const, int); 166 static usbd_status xhci_do_command_locked(struct xhci_softc * const, 167 struct xhci_soft_trb * const, int); 168 static usbd_status xhci_init_slot(struct usbd_device *, uint32_t); 169 static void xhci_free_slot(struct xhci_softc *, struct xhci_slot *, int, int); 170 static usbd_status xhci_set_address(struct usbd_device *, uint32_t, bool); 171 static usbd_status xhci_enable_slot(struct xhci_softc * const, 172 uint8_t * const); 173 static usbd_status xhci_disable_slot(struct xhci_softc * const, uint8_t); 174 static usbd_status xhci_address_device(struct xhci_softc * const, 175 uint64_t, uint8_t, bool); 176 static void xhci_set_dcba(struct xhci_softc * const, uint64_t, int); 177 static usbd_status xhci_update_ep0_mps(struct xhci_softc * const, 178 struct xhci_slot * const, u_int); 179 static usbd_status xhci_ring_init(struct xhci_softc * const, 180 struct xhci_ring * const, size_t, size_t); 181 static void xhci_ring_free(struct xhci_softc * const, struct xhci_ring * const); 182 183 static void xhci_setup_ctx(struct usbd_pipe *); 184 static void xhci_setup_route(struct usbd_pipe *, uint32_t *); 185 static void xhci_setup_tthub(struct usbd_pipe *, uint32_t *); 186 static void xhci_setup_maxburst(struct usbd_pipe *, uint32_t *); 187 static uint32_t xhci_bival2ival(uint32_t, uint32_t); 188 189 static void xhci_noop(struct usbd_pipe *); 190 191 static usbd_status xhci_root_intr_transfer(struct usbd_xfer *); 192 static usbd_status xhci_root_intr_start(struct usbd_xfer *); 193 static void xhci_root_intr_abort(struct usbd_xfer *); 194 static void xhci_root_intr_close(struct usbd_pipe *); 195 static void xhci_root_intr_done(struct usbd_xfer *); 196 197 static usbd_status xhci_device_ctrl_transfer(struct usbd_xfer *); 198 static usbd_status xhci_device_ctrl_start(struct usbd_xfer *); 199 static void xhci_device_ctrl_abort(struct usbd_xfer *); 200 static void xhci_device_ctrl_close(struct usbd_pipe *); 201 static void xhci_device_ctrl_done(struct usbd_xfer *); 202 203 static usbd_status xhci_device_intr_transfer(struct usbd_xfer *); 204 static usbd_status xhci_device_intr_start(struct usbd_xfer *); 205 static void xhci_device_intr_abort(struct usbd_xfer *); 206 static void xhci_device_intr_close(struct usbd_pipe *); 207 static void xhci_device_intr_done(struct usbd_xfer *); 208 209 static usbd_status xhci_device_bulk_transfer(struct usbd_xfer *); 210 static usbd_status xhci_device_bulk_start(struct usbd_xfer *); 211 static void xhci_device_bulk_abort(struct usbd_xfer *); 212 static void xhci_device_bulk_close(struct usbd_pipe *); 213 static void xhci_device_bulk_done(struct usbd_xfer *); 214 215 static const struct usbd_bus_methods xhci_bus_methods = { 216 .ubm_open = xhci_open, 217 .ubm_softint = xhci_softintr, 218 .ubm_dopoll = xhci_poll, 219 .ubm_allocx = xhci_allocx, 220 .ubm_freex = xhci_freex, 221 .ubm_abortx = xhci_abortx, 222 .ubm_dying = xhci_dying, 223 .ubm_getlock = xhci_get_lock, 224 .ubm_newdev = xhci_new_device, 225 .ubm_rhctrl = xhci_roothub_ctrl, 226 }; 227 228 static const struct usbd_pipe_methods xhci_root_intr_methods = { 229 .upm_transfer = xhci_root_intr_transfer, 230 .upm_start = xhci_root_intr_start, 231 .upm_abort = xhci_root_intr_abort, 232 .upm_close = xhci_root_intr_close, 233 .upm_cleartoggle = xhci_noop, 234 .upm_done = xhci_root_intr_done, 235 }; 236 237 238 static const struct usbd_pipe_methods xhci_device_ctrl_methods = { 239 .upm_transfer = xhci_device_ctrl_transfer, 240 .upm_start = xhci_device_ctrl_start, 241 .upm_abort = xhci_device_ctrl_abort, 242 .upm_close = xhci_device_ctrl_close, 243 .upm_cleartoggle = xhci_noop, 244 .upm_done = xhci_device_ctrl_done, 245 }; 246 247 static const struct usbd_pipe_methods xhci_device_isoc_methods = { 248 .upm_cleartoggle = xhci_noop, 249 }; 250 251 static const struct usbd_pipe_methods xhci_device_bulk_methods = { 252 .upm_transfer = xhci_device_bulk_transfer, 253 .upm_start = xhci_device_bulk_start, 254 .upm_abort = xhci_device_bulk_abort, 255 .upm_close = xhci_device_bulk_close, 256 .upm_cleartoggle = xhci_noop, 257 .upm_done = xhci_device_bulk_done, 258 }; 259 260 static const struct usbd_pipe_methods xhci_device_intr_methods = { 261 .upm_transfer = xhci_device_intr_transfer, 262 .upm_start = xhci_device_intr_start, 263 .upm_abort = xhci_device_intr_abort, 264 .upm_close = xhci_device_intr_close, 265 .upm_cleartoggle = xhci_noop, 266 .upm_done = xhci_device_intr_done, 267 }; 268 269 static inline uint32_t 270 xhci_read_1(const struct xhci_softc * const sc, bus_size_t offset) 271 { 272 return bus_space_read_1(sc->sc_iot, sc->sc_ioh, offset); 273 } 274 275 static inline uint32_t 276 xhci_read_4(const struct xhci_softc * const sc, bus_size_t offset) 277 { 278 return bus_space_read_4(sc->sc_iot, sc->sc_ioh, offset); 279 } 280 281 static inline void 282 xhci_write_1(const struct xhci_softc * const sc, bus_size_t offset, 283 uint32_t value) 284 { 285 bus_space_write_1(sc->sc_iot, sc->sc_ioh, offset, value); 286 } 287 288 #if 0 /* unused */ 289 static inline void 290 xhci_write_4(const struct xhci_softc * const sc, bus_size_t offset, 291 uint32_t value) 292 { 293 bus_space_write_4(sc->sc_iot, sc->sc_ioh, offset, value); 294 } 295 #endif /* unused */ 296 297 static inline uint32_t 298 xhci_cap_read_4(const struct xhci_softc * const sc, bus_size_t offset) 299 { 300 return bus_space_read_4(sc->sc_iot, sc->sc_cbh, offset); 301 } 302 303 static inline uint32_t 304 xhci_op_read_4(const struct xhci_softc * const sc, bus_size_t offset) 305 { 306 return bus_space_read_4(sc->sc_iot, sc->sc_obh, offset); 307 } 308 309 static inline void 310 xhci_op_write_4(const struct xhci_softc * const sc, bus_size_t offset, 311 uint32_t value) 312 { 313 bus_space_write_4(sc->sc_iot, sc->sc_obh, offset, value); 314 } 315 316 static inline uint64_t 317 xhci_op_read_8(const struct xhci_softc * const sc, bus_size_t offset) 318 { 319 uint64_t value; 320 321 if (sc->sc_ac64) { 322 #ifdef XHCI_USE_BUS_SPACE_8 323 value = bus_space_read_8(sc->sc_iot, sc->sc_obh, offset); 324 #else 325 value = bus_space_read_4(sc->sc_iot, sc->sc_obh, offset); 326 value |= (uint64_t)bus_space_read_4(sc->sc_iot, sc->sc_obh, 327 offset + 4) << 32; 328 #endif 329 } else { 330 value = bus_space_read_4(sc->sc_iot, sc->sc_obh, offset); 331 } 332 333 return value; 334 } 335 336 static inline void 337 xhci_op_write_8(const struct xhci_softc * const sc, bus_size_t offset, 338 uint64_t value) 339 { 340 if (sc->sc_ac64) { 341 #ifdef XHCI_USE_BUS_SPACE_8 342 bus_space_write_8(sc->sc_iot, sc->sc_obh, offset, value); 343 #else 344 bus_space_write_4(sc->sc_iot, sc->sc_obh, offset + 0, 345 (value >> 0) & 0xffffffff); 346 bus_space_write_4(sc->sc_iot, sc->sc_obh, offset + 4, 347 (value >> 32) & 0xffffffff); 348 #endif 349 } else { 350 bus_space_write_4(sc->sc_iot, sc->sc_obh, offset, value); 351 } 352 } 353 354 static inline void 355 xhci_op_barrier(const struct xhci_softc * const sc, bus_size_t offset, 356 bus_size_t len, int flags) 357 { 358 bus_space_barrier(sc->sc_iot, sc->sc_obh, offset, len, flags); 359 } 360 361 static inline uint32_t 362 xhci_rt_read_4(const struct xhci_softc * const sc, bus_size_t offset) 363 { 364 return bus_space_read_4(sc->sc_iot, sc->sc_rbh, offset); 365 } 366 367 static inline void 368 xhci_rt_write_4(const struct xhci_softc * const sc, bus_size_t offset, 369 uint32_t value) 370 { 371 bus_space_write_4(sc->sc_iot, sc->sc_rbh, offset, value); 372 } 373 374 static inline uint64_t 375 xhci_rt_read_8(const struct xhci_softc * const sc, bus_size_t offset) 376 { 377 uint64_t value; 378 379 if (sc->sc_ac64) { 380 #ifdef XHCI_USE_BUS_SPACE_8 381 value = bus_space_read_8(sc->sc_iot, sc->sc_rbh, offset); 382 #else 383 value = bus_space_read_4(sc->sc_iot, sc->sc_rbh, offset); 384 value |= (uint64_t)bus_space_read_4(sc->sc_iot, sc->sc_rbh, 385 offset + 4) << 32; 386 #endif 387 } else { 388 value = bus_space_read_4(sc->sc_iot, sc->sc_rbh, offset); 389 } 390 391 return value; 392 } 393 394 static inline void 395 xhci_rt_write_8(const struct xhci_softc * const sc, bus_size_t offset, 396 uint64_t value) 397 { 398 if (sc->sc_ac64) { 399 #ifdef XHCI_USE_BUS_SPACE_8 400 bus_space_write_8(sc->sc_iot, sc->sc_rbh, offset, value); 401 #else 402 bus_space_write_4(sc->sc_iot, sc->sc_rbh, offset + 0, 403 (value >> 0) & 0xffffffff); 404 bus_space_write_4(sc->sc_iot, sc->sc_rbh, offset + 4, 405 (value >> 32) & 0xffffffff); 406 #endif 407 } else { 408 bus_space_write_4(sc->sc_iot, sc->sc_rbh, offset, value); 409 } 410 } 411 412 #if 0 /* unused */ 413 static inline uint32_t 414 xhci_db_read_4(const struct xhci_softc * const sc, bus_size_t offset) 415 { 416 return bus_space_read_4(sc->sc_iot, sc->sc_dbh, offset); 417 } 418 #endif /* unused */ 419 420 static inline void 421 xhci_db_write_4(const struct xhci_softc * const sc, bus_size_t offset, 422 uint32_t value) 423 { 424 bus_space_write_4(sc->sc_iot, sc->sc_dbh, offset, value); 425 } 426 427 /* --- */ 428 429 static inline uint8_t 430 xhci_ep_get_type(usb_endpoint_descriptor_t * const ed) 431 { 432 u_int eptype = 0; 433 434 switch (UE_GET_XFERTYPE(ed->bmAttributes)) { 435 case UE_CONTROL: 436 eptype = 0x0; 437 break; 438 case UE_ISOCHRONOUS: 439 eptype = 0x1; 440 break; 441 case UE_BULK: 442 eptype = 0x2; 443 break; 444 case UE_INTERRUPT: 445 eptype = 0x3; 446 break; 447 } 448 449 if ((UE_GET_XFERTYPE(ed->bmAttributes) == UE_CONTROL) || 450 (UE_GET_DIR(ed->bEndpointAddress) == UE_DIR_IN)) 451 return eptype | 0x4; 452 else 453 return eptype; 454 } 455 456 static u_int 457 xhci_ep_get_dci(usb_endpoint_descriptor_t * const ed) 458 { 459 /* xHCI 1.0 section 4.5.1 */ 460 u_int epaddr = UE_GET_ADDR(ed->bEndpointAddress); 461 u_int in = 0; 462 463 if ((UE_GET_XFERTYPE(ed->bmAttributes) == UE_CONTROL) || 464 (UE_GET_DIR(ed->bEndpointAddress) == UE_DIR_IN)) 465 in = 1; 466 467 return epaddr * 2 + in; 468 } 469 470 static inline u_int 471 xhci_dci_to_ici(const u_int i) 472 { 473 return i + 1; 474 } 475 476 static inline void * 477 xhci_slot_get_dcv(struct xhci_softc * const sc, struct xhci_slot * const xs, 478 const u_int dci) 479 { 480 return KERNADDR(&xs->xs_dc_dma, sc->sc_ctxsz * dci); 481 } 482 483 #if 0 /* unused */ 484 static inline bus_addr_t 485 xhci_slot_get_dcp(struct xhci_softc * const sc, struct xhci_slot * const xs, 486 const u_int dci) 487 { 488 return DMAADDR(&xs->xs_dc_dma, sc->sc_ctxsz * dci); 489 } 490 #endif /* unused */ 491 492 static inline void * 493 xhci_slot_get_icv(struct xhci_softc * const sc, struct xhci_slot * const xs, 494 const u_int ici) 495 { 496 return KERNADDR(&xs->xs_ic_dma, sc->sc_ctxsz * ici); 497 } 498 499 static inline bus_addr_t 500 xhci_slot_get_icp(struct xhci_softc * const sc, struct xhci_slot * const xs, 501 const u_int ici) 502 { 503 return DMAADDR(&xs->xs_ic_dma, sc->sc_ctxsz * ici); 504 } 505 506 static inline struct xhci_trb * 507 xhci_ring_trbv(struct xhci_ring * const xr, u_int idx) 508 { 509 return KERNADDR(&xr->xr_dma, XHCI_TRB_SIZE * idx); 510 } 511 512 static inline bus_addr_t 513 xhci_ring_trbp(struct xhci_ring * const xr, u_int idx) 514 { 515 return DMAADDR(&xr->xr_dma, XHCI_TRB_SIZE * idx); 516 } 517 518 static inline void 519 xhci_soft_trb_put(struct xhci_soft_trb * const trb, 520 uint64_t parameter, uint32_t status, uint32_t control) 521 { 522 trb->trb_0 = parameter; 523 trb->trb_2 = status; 524 trb->trb_3 = control; 525 } 526 527 static inline void 528 xhci_trb_put(struct xhci_trb * const trb, uint64_t parameter, uint32_t status, 529 uint32_t control) 530 { 531 trb->trb_0 = htole64(parameter); 532 trb->trb_2 = htole32(status); 533 trb->trb_3 = htole32(control); 534 } 535 536 static int 537 xhci_trb_get_idx(struct xhci_ring *xr, uint64_t trb_0, int *idx) 538 { 539 /* base address of TRBs */ 540 bus_addr_t trbp = xhci_ring_trbp(xr, 0); 541 542 /* trb_0 range sanity check */ 543 if (trb_0 == 0 || trb_0 < trbp || 544 (trb_0 - trbp) % sizeof(struct xhci_trb) != 0 || 545 (trb_0 - trbp) / sizeof(struct xhci_trb) >= xr->xr_ntrb) { 546 return 1; 547 } 548 *idx = (trb_0 - trbp) / sizeof(struct xhci_trb); 549 return 0; 550 } 551 552 static unsigned int 553 xhci_get_epstate(struct xhci_softc * const sc, struct xhci_slot * const xs, 554 u_int dci) 555 { 556 uint32_t *cp; 557 558 usb_syncmem(&xs->xs_dc_dma, 0, sc->sc_pgsz, BUS_DMASYNC_POSTREAD); 559 cp = xhci_slot_get_dcv(sc, xs, dci); 560 return XHCI_EPCTX_0_EPSTATE_GET(le32toh(cp[0])); 561 } 562 563 static inline unsigned int 564 xhci_ctlrport2bus(struct xhci_softc * const sc, unsigned int ctlrport) 565 { 566 const unsigned int port = ctlrport - 1; 567 const uint8_t bit = __BIT(port % NBBY); 568 569 return __SHIFTOUT(sc->sc_ctlrportbus[port / NBBY], bit); 570 } 571 572 /* 573 * Return the roothub port for a controller port. Both are 1..n. 574 */ 575 static inline unsigned int 576 xhci_ctlrport2rhport(struct xhci_softc * const sc, unsigned int ctrlport) 577 { 578 579 return sc->sc_ctlrportmap[ctrlport - 1]; 580 } 581 582 /* 583 * Return the controller port for a bus roothub port. Both are 1..n. 584 */ 585 static inline unsigned int 586 xhci_rhport2ctlrport(struct xhci_softc * const sc, unsigned int bn, 587 unsigned int rhport) 588 { 589 590 return sc->sc_rhportmap[bn][rhport - 1]; 591 } 592 593 /* --- */ 594 595 void 596 xhci_childdet(device_t self, device_t child) 597 { 598 struct xhci_softc * const sc = device_private(self); 599 600 KASSERT((sc->sc_child == child) || (sc->sc_child2 == child)); 601 if (child == sc->sc_child2) 602 sc->sc_child2 = NULL; 603 else if (child == sc->sc_child) 604 sc->sc_child = NULL; 605 } 606 607 int 608 xhci_detach(struct xhci_softc *sc, int flags) 609 { 610 int rv = 0; 611 612 if (sc->sc_child2 != NULL) { 613 rv = config_detach(sc->sc_child2, flags); 614 if (rv != 0) 615 return rv; 616 KASSERT(sc->sc_child2 == NULL); 617 } 618 619 if (sc->sc_child != NULL) { 620 rv = config_detach(sc->sc_child, flags); 621 if (rv != 0) 622 return rv; 623 KASSERT(sc->sc_child == NULL); 624 } 625 626 /* XXX unconfigure/free slots */ 627 628 /* verify: */ 629 xhci_rt_write_4(sc, XHCI_IMAN(0), 0); 630 xhci_op_write_4(sc, XHCI_USBCMD, 0); 631 /* do we need to wait for stop? */ 632 633 xhci_op_write_8(sc, XHCI_CRCR, 0); 634 xhci_ring_free(sc, &sc->sc_cr); 635 cv_destroy(&sc->sc_command_cv); 636 cv_destroy(&sc->sc_cmdbusy_cv); 637 638 xhci_rt_write_4(sc, XHCI_ERSTSZ(0), 0); 639 xhci_rt_write_8(sc, XHCI_ERSTBA(0), 0); 640 xhci_rt_write_8(sc, XHCI_ERDP(0), 0|XHCI_ERDP_LO_BUSY); 641 xhci_ring_free(sc, &sc->sc_er); 642 643 usb_freemem(&sc->sc_bus, &sc->sc_eventst_dma); 644 645 xhci_op_write_8(sc, XHCI_DCBAAP, 0); 646 usb_freemem(&sc->sc_bus, &sc->sc_dcbaa_dma); 647 648 kmem_free(sc->sc_slots, sizeof(*sc->sc_slots) * sc->sc_maxslots); 649 650 kmem_free(sc->sc_ctlrportbus, 651 howmany(sc->sc_maxports * sizeof(uint8_t), NBBY)); 652 kmem_free(sc->sc_ctlrportmap, sc->sc_maxports * sizeof(int)); 653 654 for (size_t j = 0; j < __arraycount(sc->sc_rhportmap); j++) { 655 kmem_free(sc->sc_rhportmap[j], sc->sc_maxports * sizeof(int)); 656 } 657 658 mutex_destroy(&sc->sc_lock); 659 mutex_destroy(&sc->sc_intr_lock); 660 661 pool_cache_destroy(sc->sc_xferpool); 662 663 return rv; 664 } 665 666 int 667 xhci_activate(device_t self, enum devact act) 668 { 669 struct xhci_softc * const sc = device_private(self); 670 671 switch (act) { 672 case DVACT_DEACTIVATE: 673 sc->sc_dying = true; 674 return 0; 675 default: 676 return EOPNOTSUPP; 677 } 678 } 679 680 bool 681 xhci_suspend(device_t self, const pmf_qual_t *qual) 682 { 683 struct xhci_softc * const sc = device_private(self); 684 size_t i, j, bn, dci; 685 int port; 686 uint32_t v; 687 usbd_status err; 688 bool ok = false; 689 690 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 691 692 mutex_enter(&sc->sc_lock); 693 694 /* 695 * Block issuance of new commands, and wait for all pending 696 * commands to complete. 697 */ 698 KASSERT(sc->sc_suspender == NULL); 699 sc->sc_suspender = curlwp; 700 while (sc->sc_command_addr != 0) 701 cv_wait(&sc->sc_cmdbusy_cv, &sc->sc_lock); 702 703 /* 704 * xHCI Requirements Specification 1.2, May 2019, Sec. 4.23.2: 705 * xHCI Power Management, p. 342 706 * https://www.intel.com/content/dam/www/public/us/en/documents/technical-specifications/extensible-host-controler-interface-usb-xhci.pdf#page=342 707 */ 708 709 /* 710 * `1. Stop all USB activity by issuing Stop Endpoint Commands 711 * for Busy endpoints in the Running state. If the Force 712 * Save Context Capability (FSC = ``0'') is not supported, 713 * then Stop Endpoint Commands shall be issued for all idle 714 * endpoints in the Running state as well. The Stop 715 * Endpoint Command causes the xHC to update the respective 716 * Endpoint or Stream Contexts in system memory, e.g. the 717 * TR Dequeue Pointer, DCS, etc. fields. Refer to 718 * Implementation Note "0".' 719 */ 720 for (i = 0; i < sc->sc_maxslots; i++) { 721 struct xhci_slot *xs = &sc->sc_slots[i]; 722 723 /* Skip if the slot is not in use. */ 724 if (xs->xs_idx == 0) 725 continue; 726 727 for (dci = 0; dci < 32; dci++) { 728 /* Skip if the endpoint is not Running. */ 729 /* XXX What about Busy? */ 730 if (xhci_get_epstate(sc, xs, dci) != 731 XHCI_EPSTATE_RUNNING) 732 continue; 733 734 /* Stop endpoint. */ 735 err = xhci_stop_endpoint_cmd(sc, xs, dci, 736 XHCI_TRB_3_SUSP_EP_BIT); 737 if (err) { 738 device_printf(self, "failed to stop endpoint" 739 " slot %zu dci %zu err %d\n", 740 i, dci, err); 741 goto out; 742 } 743 } 744 } 745 746 /* 747 * Next, suspend all the ports: 748 * 749 * xHCI Requirements Specification 1.2, May 2019, Sec. 4.15: 750 * Suspend-Resume, pp. 276-283 751 * https://www.intel.com/content/dam/www/public/us/en/documents/technical-specifications/extensible-host-controler-interface-usb-xhci.pdf#page=276 752 */ 753 for (bn = 0; bn < 2; bn++) { 754 for (i = 1; i <= sc->sc_rhportcount[bn]; i++) { 755 /* 4.15.1: Port Suspend. */ 756 port = XHCI_PORTSC(xhci_rhport2ctlrport(sc, bn, i)); 757 758 /* 759 * `System software places individual ports 760 * into suspend mode by writing a ``3'' into 761 * the appropriate PORTSC register Port Link 762 * State (PLS) field (refer to Section 5.4.8). 763 * Software should only set the PLS field to 764 * ``3'' when the port is in the Enabled 765 * state.' 766 * 767 * `Software should not attempt to suspend a 768 * port unless the port reports that it is in 769 * the enabled (PED = ``1''; PLS < ``3'') 770 * state (refer to Section 5.4.8 for more 771 * information about PED and PLS).' 772 */ 773 v = xhci_op_read_4(sc, port); 774 if (((v & XHCI_PS_PED) == 0) || 775 XHCI_PS_PLS_GET(v) >= XHCI_PS_PLS_U3) 776 continue; 777 v &= ~(XHCI_PS_PLS_MASK | XHCI_PS_CLEAR); 778 v |= XHCI_PS_LWS | XHCI_PS_PLS_SET(XHCI_PS_PLS_SETU3); 779 xhci_op_write_4(sc, port, v); 780 781 /* 782 * `When the PLS field is written with U3 783 * (``3''), the status of the PLS bit will not 784 * change to the target U state U3 until the 785 * suspend signaling has completed to the 786 * attached device (which may be as long as 787 * 10ms.).' 788 * 789 * `Software is required to wait for U3 790 * transitions to complete before it puts the 791 * xHC into a low power state, and before 792 * resuming the port.' 793 * 794 * XXX Take advantage of the technique to 795 * reduce polling on host controllers that 796 * support the U3C capability. 797 */ 798 for (j = 0; j < XHCI_WAIT_PLS_U3; j++) { 799 v = xhci_op_read_4(sc, port); 800 if (XHCI_PS_PLS_GET(v) == XHCI_PS_PLS_U3) 801 break; 802 usb_delay_ms(&sc->sc_bus, 1); 803 } 804 if (j == XHCI_WAIT_PLS_U3) { 805 device_printf(self, 806 "suspend timeout on bus %zu port %zu\n", 807 bn, i); 808 goto out; 809 } 810 } 811 } 812 813 /* 814 * `2. Ensure that the Command Ring is in the Stopped state 815 * (CRR = ``0'') or Idle (i.e. the Command Transfer Ring is 816 * empty), and all Command Completion Events associated 817 * with them have been received.' 818 * 819 * XXX 820 */ 821 822 /* `3. Stop the controller by setting Run/Stop (R/S) = ``0''.' */ 823 xhci_op_write_4(sc, XHCI_USBCMD, 824 xhci_op_read_4(sc, XHCI_USBCMD) & ~XHCI_CMD_RS); 825 826 /* 827 * `4. Read the Operational Runtime, and VTIO registers in the 828 * following order: USBCMD, DNCTRL, DCBAAP, CONFIG, ERSTSZ, 829 * ERSTBA, ERDP, IMAN, IMOD, and VTIO and save their 830 * state.' 831 * 832 * (We don't use VTIO here (XXX for now?).) 833 */ 834 sc->sc_regs.usbcmd = xhci_op_read_4(sc, XHCI_USBCMD); 835 sc->sc_regs.dnctrl = xhci_op_read_4(sc, XHCI_DNCTRL); 836 sc->sc_regs.dcbaap = xhci_op_read_8(sc, XHCI_DCBAAP); 837 sc->sc_regs.config = xhci_op_read_4(sc, XHCI_CONFIG); 838 sc->sc_regs.erstsz0 = xhci_rt_read_4(sc, XHCI_ERSTSZ(0)); 839 sc->sc_regs.erstba0 = xhci_rt_read_8(sc, XHCI_ERSTBA(0)); 840 sc->sc_regs.erdp0 = xhci_rt_read_8(sc, XHCI_ERDP(0)); 841 sc->sc_regs.iman0 = xhci_rt_read_4(sc, XHCI_IMAN(0)); 842 sc->sc_regs.imod0 = xhci_rt_read_4(sc, XHCI_IMOD(0)); 843 844 /* 845 * `5. Set the Controller Save State (CSS) flag in the USBCMD 846 * register (5.4.1)...' 847 */ 848 xhci_op_write_4(sc, XHCI_USBCMD, 849 xhci_op_read_4(sc, XHCI_USBCMD) | XHCI_CMD_CSS); 850 851 /* 852 * `...and wait for the Save State Status (SSS) flag in the 853 * USBSTS register (5.4.2) to transition to ``0''.' 854 */ 855 for (i = 0; i < XHCI_WAIT_SSS; i++) { 856 if ((xhci_op_read_4(sc, XHCI_USBSTS) & XHCI_STS_SSS) == 0) 857 break; 858 usb_delay_ms(&sc->sc_bus, 1); 859 } 860 if (i >= XHCI_WAIT_SSS) { 861 device_printf(self, "suspend timeout, USBSTS.SSS\n"); 862 /* 863 * Just optimistically go on and check SRE anyway -- 864 * what's the worst that could happen? 865 */ 866 } 867 868 /* 869 * `Note: After a Save or Restore operation completes, the 870 * Save/Restore Error (SRE) flag in the USBSTS register should 871 * be checked to ensure that the operation completed 872 * successfully.' 873 */ 874 if (xhci_op_read_4(sc, XHCI_USBSTS) & XHCI_STS_SRE) { 875 device_printf(self, "suspend error, USBSTS.SRE\n"); 876 goto out; 877 } 878 879 /* Success! */ 880 ok = true; 881 882 out: mutex_exit(&sc->sc_lock); 883 return ok; 884 } 885 886 bool 887 xhci_resume(device_t self, const pmf_qual_t *qual) 888 { 889 struct xhci_softc * const sc = device_private(self); 890 size_t i, j, bn, dci; 891 int port; 892 uint32_t v; 893 bool ok = false; 894 895 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 896 897 mutex_enter(&sc->sc_lock); 898 KASSERT(sc->sc_suspender); 899 900 /* 901 * xHCI Requirements Specification 1.2, May 2019, Sec. 4.23.2: 902 * xHCI Power Management, p. 343 903 * https://www.intel.com/content/dam/www/public/us/en/documents/technical-specifications/extensible-host-controler-interface-usb-xhci.pdf#page=343 904 */ 905 906 /* 907 * `4. Restore the Operational Runtime, and VTIO registers with 908 * their previously saved state in the following order: 909 * DNCTRL, DCBAAP, CONFIG, ERSTSZ, ERSTBA, ERDP, IMAN, 910 * IMOD, and VTIO.' 911 * 912 * (We don't use VTIO here (for now?).) 913 */ 914 xhci_op_write_4(sc, XHCI_USBCMD, sc->sc_regs.usbcmd); 915 xhci_op_write_4(sc, XHCI_DNCTRL, sc->sc_regs.dnctrl); 916 xhci_op_write_8(sc, XHCI_DCBAAP, sc->sc_regs.dcbaap); 917 xhci_op_write_4(sc, XHCI_CONFIG, sc->sc_regs.config); 918 xhci_rt_write_4(sc, XHCI_ERSTSZ(0), sc->sc_regs.erstsz0); 919 xhci_rt_write_8(sc, XHCI_ERSTBA(0), sc->sc_regs.erstba0); 920 xhci_rt_write_8(sc, XHCI_ERDP(0), sc->sc_regs.erdp0); 921 xhci_rt_write_4(sc, XHCI_IMAN(0), sc->sc_regs.iman0); 922 xhci_rt_write_4(sc, XHCI_IMOD(0), sc->sc_regs.imod0); 923 924 memset(&sc->sc_regs, 0, sizeof(sc->sc_regs)); /* paranoia */ 925 926 /* 927 * `5. Set the Controller Restore State (CRS) flag in the 928 * USBCMD register (5.4.1) to ``1''...' 929 */ 930 xhci_op_write_4(sc, XHCI_USBCMD, 931 xhci_op_read_4(sc, XHCI_USBCMD) | XHCI_CMD_CRS); 932 933 /* 934 * `...and wait for the Restore State Status (RSS) in the 935 * USBSTS register (5.4.2) to transition to ``0''.' 936 */ 937 for (i = 0; i < XHCI_WAIT_RSS; i++) { 938 if ((xhci_op_read_4(sc, XHCI_USBSTS) & XHCI_STS_RSS) == 0) 939 break; 940 usb_delay_ms(&sc->sc_bus, 1); 941 } 942 if (i >= XHCI_WAIT_RSS) { 943 device_printf(self, "suspend timeout, USBSTS.RSS\n"); 944 goto out; 945 } 946 947 /* 948 * `6. Reinitialize the Command Ring, i.e. so its Cycle bits 949 * are consistent with the RCS values to be written to the 950 * CRCR.' 951 * 952 * XXX Hope just zeroing it is good enough! 953 */ 954 xhci_host_dequeue(&sc->sc_cr); 955 956 /* 957 * `7. Write the CRCR with the address and RCS value of the 958 * reinitialized Command Ring. Note that this write will 959 * cause the Command Ring to restart at the address 960 * specified by the CRCR.' 961 */ 962 xhci_op_write_8(sc, XHCI_CRCR, xhci_ring_trbp(&sc->sc_cr, 0) | 963 sc->sc_cr.xr_cs); 964 965 /* 966 * `8. Enable the controller by setting Run/Stop (R/S) = 967 * ``1''.' 968 */ 969 xhci_op_write_4(sc, XHCI_USBCMD, 970 xhci_op_read_4(sc, XHCI_USBCMD) | XHCI_CMD_RS); 971 972 /* 973 * `9. Software shall walk the USB topology and initialize each 974 * of the xHC PORTSC, PORTPMSC, and PORTLI registers, and 975 * external hub ports attached to USB devices.' 976 * 977 * This follows the procedure in 4.15 `Suspend-Resume', 4.15.2 978 * `Port Resume', 4.15.2.1 `Host Initiated'. 979 * 980 * XXX We should maybe batch up initiating the state 981 * transitions, and then wait for them to complete all at once. 982 */ 983 for (bn = 0; bn < 2; bn++) { 984 for (i = 1; i <= sc->sc_rhportcount[bn]; i++) { 985 port = XHCI_PORTSC(xhci_rhport2ctlrport(sc, bn, i)); 986 987 /* `When a port is in the U3 state: ...' */ 988 v = xhci_op_read_4(sc, port); 989 if (XHCI_PS_PLS_GET(v) != XHCI_PS_PLS_U3) 990 continue; 991 992 /* 993 * `For a USB2 protocol port, software shall 994 * write a ``15'' (Resume) to the PLS field to 995 * initiate resume signaling. The port shall 996 * transition to the Resume substate and the 997 * xHC shall transmit the resume signaling 998 * within 1ms (T_URSM). Software shall ensure 999 * that resume is signaled for at least 20ms 1000 * (T_DRSMDN). Software shall start timing 1001 * T_DRSMDN from the write of ``15'' (Resume) 1002 * to PLS.' 1003 */ 1004 if (bn == 1) { 1005 KASSERT(sc->sc_bus2.ub_revision == USBREV_2_0); 1006 v &= ~(XHCI_PS_PLS_MASK | XHCI_PS_CLEAR); 1007 v |= XHCI_PS_LWS; 1008 v |= XHCI_PS_PLS_SET(XHCI_PS_PLS_SETRESUME); 1009 xhci_op_write_4(sc, port, v); 1010 usb_delay_ms(&sc->sc_bus, USB_RESUME_WAIT); 1011 } else { 1012 KASSERT(sc->sc_bus.ub_revision > USBREV_2_0); 1013 } 1014 1015 /* 1016 * `For a USB3 protocol port [and a USB2 1017 * protocol port after transitioning to 1018 * Resume], software shall write a ``0'' (U0) 1019 * to the PLS field...' 1020 */ 1021 v = xhci_op_read_4(sc, port); 1022 v &= ~(XHCI_PS_PLS_MASK | XHCI_PS_CLEAR); 1023 v |= XHCI_PS_LWS | XHCI_PS_PLS_SET(XHCI_PS_PLS_SETU0); 1024 xhci_op_write_4(sc, port, v); 1025 1026 for (j = 0; j < XHCI_WAIT_PLS_U0; j++) { 1027 v = xhci_op_read_4(sc, port); 1028 if (XHCI_PS_PLS_GET(v) == XHCI_PS_PLS_U0) 1029 break; 1030 usb_delay_ms(&sc->sc_bus, 1); 1031 } 1032 if (j == XHCI_WAIT_PLS_U0) { 1033 device_printf(self, 1034 "resume timeout on bus %zu port %zu\n", 1035 bn, i); 1036 goto out; 1037 } 1038 } 1039 } 1040 1041 /* 1042 * `10. Restart each of the previously Running endpoints by 1043 * ringing their doorbells.' 1044 */ 1045 for (i = 0; i < sc->sc_maxslots; i++) { 1046 struct xhci_slot *xs = &sc->sc_slots[i]; 1047 1048 /* Skip if the slot is not in use. */ 1049 if (xs->xs_idx == 0) 1050 continue; 1051 1052 for (dci = 0; dci < 32; dci++) { 1053 /* Skip if the endpoint is not Running. */ 1054 if (xhci_get_epstate(sc, xs, dci) != 1055 XHCI_EPSTATE_RUNNING) 1056 continue; 1057 1058 /* Ring the doorbell. */ 1059 xhci_db_write_4(sc, XHCI_DOORBELL(xs->xs_idx), dci); 1060 } 1061 } 1062 1063 /* 1064 * `Note: After a Save or Restore operation completes, the 1065 * Save/Restore Error (SRE) flag in the USBSTS register should 1066 * be checked to ensure that the operation completed 1067 * successfully.' 1068 */ 1069 if (xhci_op_read_4(sc, XHCI_USBSTS) & XHCI_STS_SRE) { 1070 device_printf(self, "resume error, USBSTS.SRE\n"); 1071 goto out; 1072 } 1073 1074 /* Resume command issuance. */ 1075 sc->sc_suspender = NULL; 1076 cv_broadcast(&sc->sc_cmdbusy_cv); 1077 1078 /* Success! */ 1079 ok = true; 1080 1081 out: mutex_exit(&sc->sc_lock); 1082 return ok; 1083 } 1084 1085 bool 1086 xhci_shutdown(device_t self, int flags) 1087 { 1088 return false; 1089 } 1090 1091 static int 1092 xhci_hc_reset(struct xhci_softc * const sc) 1093 { 1094 uint32_t usbcmd, usbsts; 1095 int i; 1096 1097 /* Check controller not ready */ 1098 for (i = 0; i < XHCI_WAIT_CNR; i++) { 1099 usbsts = xhci_op_read_4(sc, XHCI_USBSTS); 1100 if ((usbsts & XHCI_STS_CNR) == 0) 1101 break; 1102 usb_delay_ms(&sc->sc_bus, 1); 1103 } 1104 if (i >= XHCI_WAIT_CNR) { 1105 aprint_error_dev(sc->sc_dev, "controller not ready timeout\n"); 1106 return EIO; 1107 } 1108 1109 /* Halt controller */ 1110 usbcmd = 0; 1111 xhci_op_write_4(sc, XHCI_USBCMD, usbcmd); 1112 usb_delay_ms(&sc->sc_bus, 1); 1113 1114 /* Reset controller */ 1115 usbcmd = XHCI_CMD_HCRST; 1116 xhci_op_write_4(sc, XHCI_USBCMD, usbcmd); 1117 for (i = 0; i < XHCI_WAIT_HCRST; i++) { 1118 /* 1119 * Wait 1ms first. Existing Intel xHCI requies 1ms delay to 1120 * prevent system hang (Errata). 1121 */ 1122 usb_delay_ms(&sc->sc_bus, 1); 1123 usbcmd = xhci_op_read_4(sc, XHCI_USBCMD); 1124 if ((usbcmd & XHCI_CMD_HCRST) == 0) 1125 break; 1126 } 1127 if (i >= XHCI_WAIT_HCRST) { 1128 aprint_error_dev(sc->sc_dev, "host controller reset timeout\n"); 1129 return EIO; 1130 } 1131 1132 /* Check controller not ready */ 1133 for (i = 0; i < XHCI_WAIT_CNR; i++) { 1134 usbsts = xhci_op_read_4(sc, XHCI_USBSTS); 1135 if ((usbsts & XHCI_STS_CNR) == 0) 1136 break; 1137 usb_delay_ms(&sc->sc_bus, 1); 1138 } 1139 if (i >= XHCI_WAIT_CNR) { 1140 aprint_error_dev(sc->sc_dev, 1141 "controller not ready timeout after reset\n"); 1142 return EIO; 1143 } 1144 1145 return 0; 1146 } 1147 1148 /* 7.2 xHCI Support Protocol Capability */ 1149 static void 1150 xhci_id_protocols(struct xhci_softc *sc, bus_size_t ecp) 1151 { 1152 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 1153 1154 /* XXX Cache this lot */ 1155 1156 const uint32_t w0 = xhci_read_4(sc, ecp); 1157 const uint32_t w4 = xhci_read_4(sc, ecp + 4); 1158 const uint32_t w8 = xhci_read_4(sc, ecp + 8); 1159 const uint32_t wc = xhci_read_4(sc, ecp + 0xc); 1160 1161 aprint_debug_dev(sc->sc_dev, 1162 " SP: %08x %08x %08x %08x\n", w0, w4, w8, wc); 1163 1164 if (w4 != XHCI_XECP_USBID) 1165 return; 1166 1167 const int major = XHCI_XECP_SP_W0_MAJOR(w0); 1168 const int minor = XHCI_XECP_SP_W0_MINOR(w0); 1169 const uint8_t cpo = XHCI_XECP_SP_W8_CPO(w8); 1170 const uint8_t cpc = XHCI_XECP_SP_W8_CPC(w8); 1171 1172 const uint16_t mm = __SHIFTOUT(w0, __BITS(31, 16)); 1173 switch (mm) { 1174 case 0x0200: 1175 case 0x0300: 1176 case 0x0301: 1177 case 0x0310: 1178 case 0x0320: 1179 aprint_debug_dev(sc->sc_dev, " %s ports %d - %d\n", 1180 major == 3 ? "ss" : "hs", cpo, cpo + cpc -1); 1181 break; 1182 default: 1183 aprint_error_dev(sc->sc_dev, " unknown major/minor (%d/%d)\n", 1184 major, minor); 1185 return; 1186 } 1187 1188 const size_t bus = (major == 3) ? 0 : 1; 1189 1190 /* Index arrays with 0..n-1 where ports are numbered 1..n */ 1191 for (size_t cp = cpo - 1; cp < cpo + cpc - 1; cp++) { 1192 if (sc->sc_ctlrportmap[cp] != 0) { 1193 aprint_error_dev(sc->sc_dev, "controller port %zu " 1194 "already assigned", cp); 1195 continue; 1196 } 1197 1198 sc->sc_ctlrportbus[cp / NBBY] |= 1199 bus == 0 ? 0 : __BIT(cp % NBBY); 1200 1201 const size_t rhp = sc->sc_rhportcount[bus]++; 1202 1203 KASSERTMSG(sc->sc_rhportmap[bus][rhp] == 0, 1204 "bus %zu rhp %zu is %d", bus, rhp, 1205 sc->sc_rhportmap[bus][rhp]); 1206 1207 sc->sc_rhportmap[bus][rhp] = cp + 1; 1208 sc->sc_ctlrportmap[cp] = rhp + 1; 1209 } 1210 } 1211 1212 /* Process extended capabilities */ 1213 static void 1214 xhci_ecp(struct xhci_softc *sc, uint32_t hcc) 1215 { 1216 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 1217 1218 bus_size_t ecp = XHCI_HCC_XECP(hcc) * 4; 1219 while (ecp != 0) { 1220 uint32_t ecr = xhci_read_4(sc, ecp); 1221 aprint_debug_dev(sc->sc_dev, "ECR: 0x%08x\n", ecr); 1222 switch (XHCI_XECP_ID(ecr)) { 1223 case XHCI_ID_PROTOCOLS: { 1224 xhci_id_protocols(sc, ecp); 1225 break; 1226 } 1227 case XHCI_ID_USB_LEGACY: { 1228 uint8_t bios_sem; 1229 1230 /* Take host controller ownership from BIOS */ 1231 bios_sem = xhci_read_1(sc, ecp + XHCI_XECP_BIOS_SEM); 1232 if (bios_sem) { 1233 /* sets xHCI to be owned by OS */ 1234 xhci_write_1(sc, ecp + XHCI_XECP_OS_SEM, 1); 1235 aprint_debug_dev(sc->sc_dev, 1236 "waiting for BIOS to give up control\n"); 1237 for (int i = 0; i < 5000; i++) { 1238 bios_sem = xhci_read_1(sc, ecp + 1239 XHCI_XECP_BIOS_SEM); 1240 if (bios_sem == 0) 1241 break; 1242 DELAY(1000); 1243 } 1244 if (bios_sem) { 1245 aprint_error_dev(sc->sc_dev, 1246 "timed out waiting for BIOS\n"); 1247 } 1248 } 1249 break; 1250 } 1251 default: 1252 break; 1253 } 1254 ecr = xhci_read_4(sc, ecp); 1255 if (XHCI_XECP_NEXT(ecr) == 0) { 1256 ecp = 0; 1257 } else { 1258 ecp += XHCI_XECP_NEXT(ecr) * 4; 1259 } 1260 } 1261 } 1262 1263 #define XHCI_HCCPREV1_BITS \ 1264 "\177\020" /* New bitmask */ \ 1265 "f\020\020XECP\0" \ 1266 "f\014\4MAXPSA\0" \ 1267 "b\013CFC\0" \ 1268 "b\012SEC\0" \ 1269 "b\011SBD\0" \ 1270 "b\010FSE\0" \ 1271 "b\7NSS\0" \ 1272 "b\6LTC\0" \ 1273 "b\5LHRC\0" \ 1274 "b\4PIND\0" \ 1275 "b\3PPC\0" \ 1276 "b\2CZC\0" \ 1277 "b\1BNC\0" \ 1278 "b\0AC64\0" \ 1279 "\0" 1280 #define XHCI_HCCV1_x_BITS \ 1281 "\177\020" /* New bitmask */ \ 1282 "f\020\020XECP\0" \ 1283 "f\014\4MAXPSA\0" \ 1284 "b\013CFC\0" \ 1285 "b\012SEC\0" \ 1286 "b\011SPC\0" \ 1287 "b\010PAE\0" \ 1288 "b\7NSS\0" \ 1289 "b\6LTC\0" \ 1290 "b\5LHRC\0" \ 1291 "b\4PIND\0" \ 1292 "b\3PPC\0" \ 1293 "b\2CSZ\0" \ 1294 "b\1BNC\0" \ 1295 "b\0AC64\0" \ 1296 "\0" 1297 1298 #define XHCI_HCC2_BITS \ 1299 "\177\020" /* New bitmask */ \ 1300 "b\7ETC_TSC\0" \ 1301 "b\6ETC\0" \ 1302 "b\5CIC\0" \ 1303 "b\4LEC\0" \ 1304 "b\3CTC\0" \ 1305 "b\2FSC\0" \ 1306 "b\1CMC\0" \ 1307 "b\0U3C\0" \ 1308 "\0" 1309 1310 void 1311 xhci_start(struct xhci_softc *sc) 1312 { 1313 xhci_rt_write_4(sc, XHCI_IMAN(0), XHCI_IMAN_INTR_ENA); 1314 if ((sc->sc_quirks & XHCI_QUIRK_INTEL) != 0) 1315 /* Intel xhci needs interrupt rate moderated. */ 1316 xhci_rt_write_4(sc, XHCI_IMOD(0), XHCI_IMOD_DEFAULT_LP); 1317 else 1318 xhci_rt_write_4(sc, XHCI_IMOD(0), 0); 1319 aprint_debug_dev(sc->sc_dev, "current IMOD %u\n", 1320 xhci_rt_read_4(sc, XHCI_IMOD(0))); 1321 1322 /* Go! */ 1323 xhci_op_write_4(sc, XHCI_USBCMD, XHCI_CMD_INTE|XHCI_CMD_RS); 1324 aprint_debug_dev(sc->sc_dev, "USBCMD %08"PRIx32"\n", 1325 xhci_op_read_4(sc, XHCI_USBCMD)); 1326 } 1327 1328 int 1329 xhci_init(struct xhci_softc *sc) 1330 { 1331 bus_size_t bsz; 1332 uint32_t cap, hcs1, hcs2, hcs3, hcc, dboff, rtsoff, hcc2; 1333 uint32_t pagesize, config; 1334 int i = 0; 1335 uint16_t hciversion; 1336 uint8_t caplength; 1337 1338 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 1339 1340 /* Set up the bus struct for the usb 3 and usb 2 buses */ 1341 sc->sc_bus.ub_methods = &xhci_bus_methods; 1342 sc->sc_bus.ub_pipesize = sizeof(struct xhci_pipe); 1343 sc->sc_bus.ub_usedma = true; 1344 sc->sc_bus.ub_hcpriv = sc; 1345 1346 sc->sc_bus2.ub_methods = &xhci_bus_methods; 1347 sc->sc_bus2.ub_pipesize = sizeof(struct xhci_pipe); 1348 sc->sc_bus2.ub_revision = USBREV_2_0; 1349 sc->sc_bus2.ub_usedma = true; 1350 sc->sc_bus2.ub_hcpriv = sc; 1351 sc->sc_bus2.ub_dmatag = sc->sc_bus.ub_dmatag; 1352 1353 cap = xhci_read_4(sc, XHCI_CAPLENGTH); 1354 caplength = XHCI_CAP_CAPLENGTH(cap); 1355 hciversion = XHCI_CAP_HCIVERSION(cap); 1356 1357 if (hciversion < XHCI_HCIVERSION_0_96 || 1358 hciversion >= 0x0200) { 1359 aprint_normal_dev(sc->sc_dev, 1360 "xHCI version %x.%x not known to be supported\n", 1361 (hciversion >> 8) & 0xff, (hciversion >> 0) & 0xff); 1362 } else { 1363 aprint_verbose_dev(sc->sc_dev, "xHCI version %x.%x\n", 1364 (hciversion >> 8) & 0xff, (hciversion >> 0) & 0xff); 1365 } 1366 1367 if (bus_space_subregion(sc->sc_iot, sc->sc_ioh, 0, caplength, 1368 &sc->sc_cbh) != 0) { 1369 aprint_error_dev(sc->sc_dev, "capability subregion failure\n"); 1370 return ENOMEM; 1371 } 1372 1373 hcs1 = xhci_cap_read_4(sc, XHCI_HCSPARAMS1); 1374 sc->sc_maxslots = XHCI_HCS1_MAXSLOTS(hcs1); 1375 sc->sc_maxintrs = XHCI_HCS1_MAXINTRS(hcs1); 1376 sc->sc_maxports = XHCI_HCS1_MAXPORTS(hcs1); 1377 hcs2 = xhci_cap_read_4(sc, XHCI_HCSPARAMS2); 1378 hcs3 = xhci_cap_read_4(sc, XHCI_HCSPARAMS3); 1379 aprint_debug_dev(sc->sc_dev, 1380 "hcs1=%"PRIx32" hcs2=%"PRIx32" hcs3=%"PRIx32"\n", hcs1, hcs2, hcs3); 1381 1382 hcc = xhci_cap_read_4(sc, XHCI_HCCPARAMS); 1383 sc->sc_ac64 = XHCI_HCC_AC64(hcc); 1384 sc->sc_ctxsz = XHCI_HCC_CSZ(hcc) ? 64 : 32; 1385 1386 char sbuf[128]; 1387 if (hciversion < XHCI_HCIVERSION_1_0) 1388 snprintb(sbuf, sizeof(sbuf), XHCI_HCCPREV1_BITS, hcc); 1389 else 1390 snprintb(sbuf, sizeof(sbuf), XHCI_HCCV1_x_BITS, hcc); 1391 aprint_debug_dev(sc->sc_dev, "hcc=%s\n", sbuf); 1392 aprint_debug_dev(sc->sc_dev, "xECP %x\n", XHCI_HCC_XECP(hcc) * 4); 1393 if (hciversion >= XHCI_HCIVERSION_1_1) { 1394 hcc2 = xhci_cap_read_4(sc, XHCI_HCCPARAMS2); 1395 snprintb(sbuf, sizeof(sbuf), XHCI_HCC2_BITS, hcc2); 1396 aprint_debug_dev(sc->sc_dev, "hcc2=%s\n", sbuf); 1397 } 1398 1399 /* default all ports to bus 0, i.e. usb 3 */ 1400 sc->sc_ctlrportbus = kmem_zalloc( 1401 howmany(sc->sc_maxports * sizeof(uint8_t), NBBY), KM_SLEEP); 1402 sc->sc_ctlrportmap = kmem_zalloc(sc->sc_maxports * sizeof(int), KM_SLEEP); 1403 1404 /* controller port to bus roothub port map */ 1405 for (size_t j = 0; j < __arraycount(sc->sc_rhportmap); j++) { 1406 sc->sc_rhportmap[j] = kmem_zalloc(sc->sc_maxports * sizeof(int), KM_SLEEP); 1407 } 1408 1409 /* 1410 * Process all Extended Capabilities 1411 */ 1412 xhci_ecp(sc, hcc); 1413 1414 bsz = XHCI_PORTSC(sc->sc_maxports); 1415 if (bus_space_subregion(sc->sc_iot, sc->sc_ioh, caplength, bsz, 1416 &sc->sc_obh) != 0) { 1417 aprint_error_dev(sc->sc_dev, "operational subregion failure\n"); 1418 return ENOMEM; 1419 } 1420 1421 dboff = xhci_cap_read_4(sc, XHCI_DBOFF); 1422 if (bus_space_subregion(sc->sc_iot, sc->sc_ioh, dboff, 1423 sc->sc_maxslots * 4, &sc->sc_dbh) != 0) { 1424 aprint_error_dev(sc->sc_dev, "doorbell subregion failure\n"); 1425 return ENOMEM; 1426 } 1427 1428 rtsoff = xhci_cap_read_4(sc, XHCI_RTSOFF); 1429 if (bus_space_subregion(sc->sc_iot, sc->sc_ioh, rtsoff, 1430 sc->sc_maxintrs * 0x20, &sc->sc_rbh) != 0) { 1431 aprint_error_dev(sc->sc_dev, "runtime subregion failure\n"); 1432 return ENOMEM; 1433 } 1434 1435 int rv; 1436 rv = xhci_hc_reset(sc); 1437 if (rv != 0) { 1438 return rv; 1439 } 1440 1441 if (sc->sc_vendor_init) 1442 sc->sc_vendor_init(sc); 1443 1444 pagesize = xhci_op_read_4(sc, XHCI_PAGESIZE); 1445 aprint_debug_dev(sc->sc_dev, "PAGESIZE 0x%08x\n", pagesize); 1446 pagesize = ffs(pagesize); 1447 if (pagesize == 0) { 1448 aprint_error_dev(sc->sc_dev, "pagesize is 0\n"); 1449 return EIO; 1450 } 1451 sc->sc_pgsz = 1 << (12 + (pagesize - 1)); 1452 aprint_debug_dev(sc->sc_dev, "sc_pgsz 0x%08x\n", (uint32_t)sc->sc_pgsz); 1453 aprint_debug_dev(sc->sc_dev, "sc_maxslots 0x%08x\n", 1454 (uint32_t)sc->sc_maxslots); 1455 aprint_debug_dev(sc->sc_dev, "sc_maxports %d\n", sc->sc_maxports); 1456 1457 usbd_status err; 1458 1459 sc->sc_maxspbuf = XHCI_HCS2_MAXSPBUF(hcs2); 1460 aprint_debug_dev(sc->sc_dev, "sc_maxspbuf %d\n", sc->sc_maxspbuf); 1461 if (sc->sc_maxspbuf != 0) { 1462 err = usb_allocmem(&sc->sc_bus, 1463 sizeof(uint64_t) * sc->sc_maxspbuf, sizeof(uint64_t), 1464 &sc->sc_spbufarray_dma); 1465 if (err) { 1466 aprint_error_dev(sc->sc_dev, 1467 "spbufarray init fail, err %d\n", err); 1468 return ENOMEM; 1469 } 1470 1471 sc->sc_spbuf_dma = kmem_zalloc(sizeof(*sc->sc_spbuf_dma) * 1472 sc->sc_maxspbuf, KM_SLEEP); 1473 uint64_t *spbufarray = KERNADDR(&sc->sc_spbufarray_dma, 0); 1474 for (i = 0; i < sc->sc_maxspbuf; i++) { 1475 usb_dma_t * const dma = &sc->sc_spbuf_dma[i]; 1476 /* allocate contexts */ 1477 err = usb_allocmem(&sc->sc_bus, sc->sc_pgsz, 1478 sc->sc_pgsz, dma); 1479 if (err) { 1480 aprint_error_dev(sc->sc_dev, 1481 "spbufarray_dma init fail, err %d\n", err); 1482 rv = ENOMEM; 1483 goto bad1; 1484 } 1485 spbufarray[i] = htole64(DMAADDR(dma, 0)); 1486 usb_syncmem(dma, 0, sc->sc_pgsz, 1487 BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE); 1488 } 1489 1490 usb_syncmem(&sc->sc_spbufarray_dma, 0, 1491 sizeof(uint64_t) * sc->sc_maxspbuf, BUS_DMASYNC_PREWRITE); 1492 } 1493 1494 config = xhci_op_read_4(sc, XHCI_CONFIG); 1495 config &= ~0xFF; 1496 config |= sc->sc_maxslots & 0xFF; 1497 xhci_op_write_4(sc, XHCI_CONFIG, config); 1498 1499 err = xhci_ring_init(sc, &sc->sc_cr, XHCI_COMMAND_RING_TRBS, 1500 XHCI_COMMAND_RING_SEGMENTS_ALIGN); 1501 if (err) { 1502 aprint_error_dev(sc->sc_dev, "command ring init fail, err %d\n", 1503 err); 1504 rv = ENOMEM; 1505 goto bad1; 1506 } 1507 1508 err = xhci_ring_init(sc, &sc->sc_er, XHCI_EVENT_RING_TRBS, 1509 XHCI_EVENT_RING_SEGMENTS_ALIGN); 1510 if (err) { 1511 aprint_error_dev(sc->sc_dev, "event ring init fail, err %d\n", 1512 err); 1513 rv = ENOMEM; 1514 goto bad2; 1515 } 1516 1517 usb_dma_t *dma; 1518 size_t size; 1519 size_t align; 1520 1521 dma = &sc->sc_eventst_dma; 1522 size = roundup2(XHCI_EVENT_RING_SEGMENTS * XHCI_ERSTE_SIZE, 1523 XHCI_EVENT_RING_SEGMENT_TABLE_ALIGN); 1524 KASSERTMSG(size <= (512 * 1024), "eventst size %zu too large", size); 1525 align = XHCI_EVENT_RING_SEGMENT_TABLE_ALIGN; 1526 err = usb_allocmem(&sc->sc_bus, size, align, dma); 1527 if (err) { 1528 aprint_error_dev(sc->sc_dev, "eventst init fail, err %d\n", 1529 err); 1530 rv = ENOMEM; 1531 goto bad3; 1532 } 1533 1534 memset(KERNADDR(dma, 0), 0, size); 1535 usb_syncmem(dma, 0, size, BUS_DMASYNC_PREWRITE); 1536 aprint_debug_dev(sc->sc_dev, "eventst: %016jx %p %zx\n", 1537 (uintmax_t)DMAADDR(&sc->sc_eventst_dma, 0), 1538 KERNADDR(&sc->sc_eventst_dma, 0), 1539 sc->sc_eventst_dma.udma_block->size); 1540 1541 dma = &sc->sc_dcbaa_dma; 1542 size = (1 + sc->sc_maxslots) * sizeof(uint64_t); 1543 KASSERTMSG(size <= 2048, "dcbaa size %zu too large", size); 1544 align = XHCI_DEVICE_CONTEXT_BASE_ADDRESS_ARRAY_ALIGN; 1545 err = usb_allocmem(&sc->sc_bus, size, align, dma); 1546 if (err) { 1547 aprint_error_dev(sc->sc_dev, "dcbaa init fail, err %d\n", err); 1548 rv = ENOMEM; 1549 goto bad4; 1550 } 1551 aprint_debug_dev(sc->sc_dev, "dcbaa: %016jx %p %zx\n", 1552 (uintmax_t)DMAADDR(&sc->sc_dcbaa_dma, 0), 1553 KERNADDR(&sc->sc_dcbaa_dma, 0), 1554 sc->sc_dcbaa_dma.udma_block->size); 1555 1556 memset(KERNADDR(dma, 0), 0, size); 1557 if (sc->sc_maxspbuf != 0) { 1558 /* 1559 * DCBA entry 0 hold the scratchbuf array pointer. 1560 */ 1561 *(uint64_t *)KERNADDR(dma, 0) = 1562 htole64(DMAADDR(&sc->sc_spbufarray_dma, 0)); 1563 } 1564 usb_syncmem(dma, 0, size, BUS_DMASYNC_PREWRITE); 1565 1566 sc->sc_slots = kmem_zalloc(sizeof(*sc->sc_slots) * sc->sc_maxslots, 1567 KM_SLEEP); 1568 if (sc->sc_slots == NULL) { 1569 aprint_error_dev(sc->sc_dev, "slots init fail, err %d\n", err); 1570 rv = ENOMEM; 1571 goto bad; 1572 } 1573 1574 sc->sc_xferpool = pool_cache_init(sizeof(struct xhci_xfer), 0, 0, 0, 1575 "xhcixfer", NULL, IPL_USB, NULL, NULL, NULL); 1576 if (sc->sc_xferpool == NULL) { 1577 aprint_error_dev(sc->sc_dev, "pool_cache init fail, err %d\n", 1578 err); 1579 rv = ENOMEM; 1580 goto bad; 1581 } 1582 1583 cv_init(&sc->sc_command_cv, "xhcicmd"); 1584 cv_init(&sc->sc_cmdbusy_cv, "xhcicmdq"); 1585 mutex_init(&sc->sc_lock, MUTEX_DEFAULT, IPL_SOFTUSB); 1586 mutex_init(&sc->sc_intr_lock, MUTEX_DEFAULT, IPL_USB); 1587 1588 struct xhci_erste *erst; 1589 erst = KERNADDR(&sc->sc_eventst_dma, 0); 1590 erst[0].erste_0 = htole64(xhci_ring_trbp(&sc->sc_er, 0)); 1591 erst[0].erste_2 = htole32(sc->sc_er.xr_ntrb); 1592 erst[0].erste_3 = htole32(0); 1593 usb_syncmem(&sc->sc_eventst_dma, 0, 1594 XHCI_ERSTE_SIZE * XHCI_EVENT_RING_SEGMENTS, BUS_DMASYNC_PREWRITE); 1595 1596 xhci_rt_write_4(sc, XHCI_ERSTSZ(0), XHCI_EVENT_RING_SEGMENTS); 1597 xhci_rt_write_8(sc, XHCI_ERSTBA(0), DMAADDR(&sc->sc_eventst_dma, 0)); 1598 xhci_rt_write_8(sc, XHCI_ERDP(0), xhci_ring_trbp(&sc->sc_er, 0) | 1599 XHCI_ERDP_LO_BUSY); 1600 1601 xhci_op_write_8(sc, XHCI_DCBAAP, DMAADDR(&sc->sc_dcbaa_dma, 0)); 1602 xhci_op_write_8(sc, XHCI_CRCR, xhci_ring_trbp(&sc->sc_cr, 0) | 1603 sc->sc_cr.xr_cs); 1604 1605 xhci_op_barrier(sc, 0, 4, BUS_SPACE_BARRIER_WRITE); 1606 1607 HEXDUMP("eventst", KERNADDR(&sc->sc_eventst_dma, 0), 1608 XHCI_ERSTE_SIZE * XHCI_EVENT_RING_SEGMENTS); 1609 1610 if ((sc->sc_quirks & XHCI_DEFERRED_START) == 0) 1611 xhci_start(sc); 1612 1613 return 0; 1614 1615 bad: 1616 if (sc->sc_xferpool) { 1617 pool_cache_destroy(sc->sc_xferpool); 1618 sc->sc_xferpool = NULL; 1619 } 1620 1621 if (sc->sc_slots) { 1622 kmem_free(sc->sc_slots, sizeof(*sc->sc_slots) * 1623 sc->sc_maxslots); 1624 sc->sc_slots = NULL; 1625 } 1626 1627 usb_freemem(&sc->sc_bus, &sc->sc_dcbaa_dma); 1628 bad4: 1629 usb_freemem(&sc->sc_bus, &sc->sc_eventst_dma); 1630 bad3: 1631 xhci_ring_free(sc, &sc->sc_er); 1632 bad2: 1633 xhci_ring_free(sc, &sc->sc_cr); 1634 i = sc->sc_maxspbuf; 1635 bad1: 1636 for (int j = 0; j < i; j++) 1637 usb_freemem(&sc->sc_bus, &sc->sc_spbuf_dma[j]); 1638 usb_freemem(&sc->sc_bus, &sc->sc_spbufarray_dma); 1639 1640 return rv; 1641 } 1642 1643 static inline bool 1644 xhci_polling_p(struct xhci_softc * const sc) 1645 { 1646 return sc->sc_bus.ub_usepolling || sc->sc_bus2.ub_usepolling; 1647 } 1648 1649 int 1650 xhci_intr(void *v) 1651 { 1652 struct xhci_softc * const sc = v; 1653 int ret = 0; 1654 1655 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 1656 1657 if (sc == NULL) 1658 return 0; 1659 1660 mutex_spin_enter(&sc->sc_intr_lock); 1661 1662 if (sc->sc_dying || !device_has_power(sc->sc_dev)) 1663 goto done; 1664 1665 /* If we get an interrupt while polling, then just ignore it. */ 1666 if (xhci_polling_p(sc)) { 1667 #ifdef DIAGNOSTIC 1668 DPRINTFN(16, "ignored interrupt while polling", 0, 0, 0, 0); 1669 #endif 1670 goto done; 1671 } 1672 1673 ret = xhci_intr1(sc); 1674 if (ret) { 1675 KASSERT(sc->sc_child || sc->sc_child2); 1676 1677 /* 1678 * One of child busses could be already detached. It doesn't 1679 * matter on which of the two the softintr is scheduled. 1680 */ 1681 if (sc->sc_child) 1682 usb_schedsoftintr(&sc->sc_bus); 1683 else 1684 usb_schedsoftintr(&sc->sc_bus2); 1685 } 1686 done: 1687 mutex_spin_exit(&sc->sc_intr_lock); 1688 return ret; 1689 } 1690 1691 int 1692 xhci_intr1(struct xhci_softc * const sc) 1693 { 1694 uint32_t usbsts; 1695 uint32_t iman; 1696 1697 XHCIHIST_FUNC(); 1698 1699 usbsts = xhci_op_read_4(sc, XHCI_USBSTS); 1700 XHCIHIST_CALLARGS("USBSTS %08jx", usbsts, 0, 0, 0); 1701 if ((usbsts & (XHCI_STS_HSE | XHCI_STS_EINT | XHCI_STS_PCD | 1702 XHCI_STS_HCE)) == 0) { 1703 DPRINTFN(16, "ignored intr not for %s", 1704 (uintptr_t)device_xname(sc->sc_dev), 0, 0, 0); 1705 return 0; 1706 } 1707 1708 /* 1709 * Clear EINT and other transient flags, to not misenterpret 1710 * next shared interrupt. Also, to avoid race, EINT must be cleared 1711 * before XHCI_IMAN_INTR_PEND is cleared. 1712 */ 1713 xhci_op_write_4(sc, XHCI_USBSTS, usbsts & XHCI_STS_RSVDP0); 1714 1715 #ifdef XHCI_DEBUG 1716 usbsts = xhci_op_read_4(sc, XHCI_USBSTS); 1717 DPRINTFN(16, "USBSTS %08jx", usbsts, 0, 0, 0); 1718 #endif 1719 1720 iman = xhci_rt_read_4(sc, XHCI_IMAN(0)); 1721 DPRINTFN(16, "IMAN0 %08jx", iman, 0, 0, 0); 1722 iman |= XHCI_IMAN_INTR_PEND; 1723 xhci_rt_write_4(sc, XHCI_IMAN(0), iman); 1724 1725 #ifdef XHCI_DEBUG 1726 iman = xhci_rt_read_4(sc, XHCI_IMAN(0)); 1727 DPRINTFN(16, "IMAN0 %08jx", iman, 0, 0, 0); 1728 usbsts = xhci_op_read_4(sc, XHCI_USBSTS); 1729 DPRINTFN(16, "USBSTS %08jx", usbsts, 0, 0, 0); 1730 #endif 1731 1732 return 1; 1733 } 1734 1735 /* 1736 * 3 port speed types used in USB stack 1737 * 1738 * usbdi speed 1739 * definition: USB_SPEED_* in usb.h 1740 * They are used in struct usbd_device in USB stack. 1741 * ioctl interface uses these values too. 1742 * port_status speed 1743 * definition: UPS_*_SPEED in usb.h 1744 * They are used in usb_port_status_t and valid only for USB 2.0. 1745 * Speed value is always 0 for Super Speed or more, and dwExtPortStatus 1746 * of usb_port_status_ext_t indicates port speed. 1747 * Note that some 3.0 values overlap with 2.0 values. 1748 * (e.g. 0x200 means UPS_POER_POWER_SS in SS and 1749 * means UPS_LOW_SPEED in HS.) 1750 * port status returned from hub also uses these values. 1751 * On NetBSD UPS_OTHER_SPEED indicates port speed is super speed 1752 * or more. 1753 * xspeed: 1754 * definition: Protocol Speed ID (PSI) (xHCI 1.1 7.2.1) 1755 * They are used in only slot context and PORTSC reg of xhci. 1756 * The difference between usbdi speed and xspeed is 1757 * that FS and LS values are swapped. 1758 */ 1759 1760 /* convert usbdi speed to xspeed */ 1761 static int 1762 xhci_speed2xspeed(int speed) 1763 { 1764 switch (speed) { 1765 case USB_SPEED_LOW: return 2; 1766 case USB_SPEED_FULL: return 1; 1767 default: return speed; 1768 } 1769 } 1770 1771 #if 0 1772 /* convert xspeed to usbdi speed */ 1773 static int 1774 xhci_xspeed2speed(int xspeed) 1775 { 1776 switch (xspeed) { 1777 case 1: return USB_SPEED_FULL; 1778 case 2: return USB_SPEED_LOW; 1779 default: return xspeed; 1780 } 1781 } 1782 #endif 1783 1784 /* convert xspeed to port status speed */ 1785 static int 1786 xhci_xspeed2psspeed(int xspeed) 1787 { 1788 switch (xspeed) { 1789 case 0: return 0; 1790 case 1: return UPS_FULL_SPEED; 1791 case 2: return UPS_LOW_SPEED; 1792 case 3: return UPS_HIGH_SPEED; 1793 default: return UPS_OTHER_SPEED; 1794 } 1795 } 1796 1797 /* 1798 * Construct input contexts and issue TRB to open pipe. 1799 */ 1800 static usbd_status 1801 xhci_configure_endpoint(struct usbd_pipe *pipe) 1802 { 1803 struct xhci_softc * const sc = XHCI_PIPE2SC(pipe); 1804 struct xhci_slot * const xs = pipe->up_dev->ud_hcpriv; 1805 #ifdef USB_DEBUG 1806 const u_int dci = xhci_ep_get_dci(pipe->up_endpoint->ue_edesc); 1807 #endif 1808 struct xhci_soft_trb trb; 1809 usbd_status err; 1810 1811 XHCIHIST_FUNC(); 1812 XHCIHIST_CALLARGS("slot %ju dci %ju epaddr 0x%02jx attr 0x%02jx", 1813 xs->xs_idx, dci, pipe->up_endpoint->ue_edesc->bEndpointAddress, 1814 pipe->up_endpoint->ue_edesc->bmAttributes); 1815 1816 /* XXX ensure input context is available? */ 1817 1818 memset(xhci_slot_get_icv(sc, xs, 0), 0, sc->sc_pgsz); 1819 1820 /* set up context */ 1821 xhci_setup_ctx(pipe); 1822 1823 HEXDUMP("input control context", xhci_slot_get_icv(sc, xs, 0), 1824 sc->sc_ctxsz * 1); 1825 HEXDUMP("input endpoint context", xhci_slot_get_icv(sc, xs, 1826 xhci_dci_to_ici(dci)), sc->sc_ctxsz * 1); 1827 1828 trb.trb_0 = xhci_slot_get_icp(sc, xs, 0); 1829 trb.trb_2 = 0; 1830 trb.trb_3 = XHCI_TRB_3_SLOT_SET(xs->xs_idx) | 1831 XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_CONFIGURE_EP); 1832 1833 err = xhci_do_command(sc, &trb, USBD_DEFAULT_TIMEOUT); 1834 1835 usb_syncmem(&xs->xs_dc_dma, 0, sc->sc_pgsz, BUS_DMASYNC_POSTREAD); 1836 HEXDUMP("output context", xhci_slot_get_dcv(sc, xs, dci), 1837 sc->sc_ctxsz * 1); 1838 1839 return err; 1840 } 1841 1842 #if 0 1843 static usbd_status 1844 xhci_unconfigure_endpoint(struct usbd_pipe *pipe) 1845 { 1846 #ifdef USB_DEBUG 1847 struct xhci_slot * const xs = pipe->up_dev->ud_hcpriv; 1848 #endif 1849 1850 XHCIHIST_FUNC(); 1851 XHCIHIST_CALLARGS("slot %ju", xs->xs_idx, 0, 0, 0); 1852 1853 return USBD_NORMAL_COMPLETION; 1854 } 1855 #endif 1856 1857 /* 4.6.8, 6.4.3.7 */ 1858 static usbd_status 1859 xhci_reset_endpoint_locked(struct usbd_pipe *pipe) 1860 { 1861 struct xhci_softc * const sc = XHCI_PIPE2SC(pipe); 1862 struct xhci_slot * const xs = pipe->up_dev->ud_hcpriv; 1863 const u_int dci = xhci_ep_get_dci(pipe->up_endpoint->ue_edesc); 1864 struct xhci_soft_trb trb; 1865 usbd_status err; 1866 1867 XHCIHIST_FUNC(); 1868 XHCIHIST_CALLARGS("slot %ju dci %ju", xs->xs_idx, dci, 0, 0); 1869 1870 KASSERT(mutex_owned(&sc->sc_lock)); 1871 1872 trb.trb_0 = 0; 1873 trb.trb_2 = 0; 1874 trb.trb_3 = XHCI_TRB_3_SLOT_SET(xs->xs_idx) | 1875 XHCI_TRB_3_EP_SET(dci) | 1876 XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_RESET_EP); 1877 1878 err = xhci_do_command_locked(sc, &trb, USBD_DEFAULT_TIMEOUT); 1879 1880 return err; 1881 } 1882 1883 static usbd_status 1884 xhci_reset_endpoint(struct usbd_pipe *pipe) 1885 { 1886 struct xhci_softc * const sc = XHCI_PIPE2SC(pipe); 1887 1888 mutex_enter(&sc->sc_lock); 1889 usbd_status ret = xhci_reset_endpoint_locked(pipe); 1890 mutex_exit(&sc->sc_lock); 1891 1892 return ret; 1893 } 1894 1895 /* 1896 * 4.6.9, 6.4.3.8 1897 * Stop execution of TDs on xfer ring. 1898 * Should be called with sc_lock held. 1899 */ 1900 static usbd_status 1901 xhci_stop_endpoint_cmd(struct xhci_softc *sc, struct xhci_slot *xs, u_int dci, 1902 uint32_t trb3flags) 1903 { 1904 struct xhci_soft_trb trb; 1905 usbd_status err; 1906 1907 XHCIHIST_FUNC(); 1908 XHCIHIST_CALLARGS("slot %ju dci %ju", xs->xs_idx, dci, 0, 0); 1909 1910 KASSERT(mutex_owned(&sc->sc_lock)); 1911 1912 trb.trb_0 = 0; 1913 trb.trb_2 = 0; 1914 trb.trb_3 = XHCI_TRB_3_SLOT_SET(xs->xs_idx) | 1915 XHCI_TRB_3_EP_SET(dci) | 1916 XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_STOP_EP) | 1917 trb3flags; 1918 1919 err = xhci_do_command_locked(sc, &trb, USBD_DEFAULT_TIMEOUT); 1920 1921 return err; 1922 } 1923 1924 static usbd_status 1925 xhci_stop_endpoint(struct usbd_pipe *pipe) 1926 { 1927 struct xhci_softc * const sc = XHCI_PIPE2SC(pipe); 1928 struct xhci_slot * const xs = pipe->up_dev->ud_hcpriv; 1929 const u_int dci = xhci_ep_get_dci(pipe->up_endpoint->ue_edesc); 1930 1931 XHCIHIST_FUNC(); 1932 XHCIHIST_CALLARGS("slot %ju dci %ju", xs->xs_idx, dci, 0, 0); 1933 1934 KASSERT(mutex_owned(&sc->sc_lock)); 1935 1936 return xhci_stop_endpoint_cmd(sc, xs, dci, 0); 1937 } 1938 1939 /* 1940 * Set TR Dequeue Pointer. 1941 * xHCI 1.1 4.6.10 6.4.3.9 1942 * Purge all of the TRBs on ring and reinitialize ring. 1943 * Set TR dequeue Pointr to 0 and Cycle State to 1. 1944 * EPSTATE of endpoint must be ERROR or STOPPED, otherwise CONTEXT_STATE 1945 * error will be generated. 1946 */ 1947 static usbd_status 1948 xhci_set_dequeue_locked(struct usbd_pipe *pipe) 1949 { 1950 struct xhci_softc * const sc = XHCI_PIPE2SC(pipe); 1951 struct xhci_slot * const xs = pipe->up_dev->ud_hcpriv; 1952 const u_int dci = xhci_ep_get_dci(pipe->up_endpoint->ue_edesc); 1953 struct xhci_ring * const xr = &xs->xs_ep[dci].xe_tr; 1954 struct xhci_soft_trb trb; 1955 usbd_status err; 1956 1957 XHCIHIST_FUNC(); 1958 XHCIHIST_CALLARGS("slot %ju dci %ju", xs->xs_idx, dci, 0, 0); 1959 1960 KASSERT(mutex_owned(&sc->sc_lock)); 1961 1962 xhci_host_dequeue(xr); 1963 1964 /* set DCS */ 1965 trb.trb_0 = xhci_ring_trbp(xr, 0) | 1; /* XXX */ 1966 trb.trb_2 = 0; 1967 trb.trb_3 = XHCI_TRB_3_SLOT_SET(xs->xs_idx) | 1968 XHCI_TRB_3_EP_SET(dci) | 1969 XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_SET_TR_DEQUEUE); 1970 1971 err = xhci_do_command_locked(sc, &trb, USBD_DEFAULT_TIMEOUT); 1972 1973 return err; 1974 } 1975 1976 static usbd_status 1977 xhci_set_dequeue(struct usbd_pipe *pipe) 1978 { 1979 struct xhci_softc * const sc = XHCI_PIPE2SC(pipe); 1980 1981 mutex_enter(&sc->sc_lock); 1982 usbd_status ret = xhci_set_dequeue_locked(pipe); 1983 mutex_exit(&sc->sc_lock); 1984 1985 return ret; 1986 } 1987 1988 /* 1989 * Open new pipe: called from usbd_setup_pipe_flags. 1990 * Fills methods of pipe. 1991 * If pipe is not for ep0, calls configure_endpoint. 1992 */ 1993 static usbd_status 1994 xhci_open(struct usbd_pipe *pipe) 1995 { 1996 struct usbd_device * const dev = pipe->up_dev; 1997 struct xhci_softc * const sc = XHCI_BUS2SC(dev->ud_bus); 1998 usb_endpoint_descriptor_t * const ed = pipe->up_endpoint->ue_edesc; 1999 const uint8_t xfertype = UE_GET_XFERTYPE(ed->bmAttributes); 2000 2001 XHCIHIST_FUNC(); 2002 XHCIHIST_CALLARGS("addr %jd depth %jd port %jd speed %jd", dev->ud_addr, 2003 dev->ud_depth, dev->ud_powersrc->up_portno, dev->ud_speed); 2004 DPRINTFN(1, " dci %ju type 0x%02jx epaddr 0x%02jx attr 0x%02jx", 2005 xhci_ep_get_dci(ed), ed->bDescriptorType, ed->bEndpointAddress, 2006 ed->bmAttributes); 2007 DPRINTFN(1, " mps %ju ival %ju", UGETW(ed->wMaxPacketSize), 2008 ed->bInterval, 0, 0); 2009 2010 if (sc->sc_dying) 2011 return USBD_IOERROR; 2012 2013 /* Root Hub */ 2014 if (dev->ud_depth == 0 && dev->ud_powersrc->up_portno == 0) { 2015 switch (ed->bEndpointAddress) { 2016 case USB_CONTROL_ENDPOINT: 2017 pipe->up_methods = &roothub_ctrl_methods; 2018 break; 2019 case UE_DIR_IN | USBROOTHUB_INTR_ENDPT: 2020 pipe->up_methods = &xhci_root_intr_methods; 2021 break; 2022 default: 2023 pipe->up_methods = NULL; 2024 DPRINTFN(0, "bad bEndpointAddress 0x%02jx", 2025 ed->bEndpointAddress, 0, 0, 0); 2026 return USBD_INVAL; 2027 } 2028 return USBD_NORMAL_COMPLETION; 2029 } 2030 2031 switch (xfertype) { 2032 case UE_CONTROL: 2033 pipe->up_methods = &xhci_device_ctrl_methods; 2034 break; 2035 case UE_ISOCHRONOUS: 2036 pipe->up_methods = &xhci_device_isoc_methods; 2037 return USBD_INVAL; 2038 break; 2039 case UE_BULK: 2040 pipe->up_methods = &xhci_device_bulk_methods; 2041 break; 2042 case UE_INTERRUPT: 2043 pipe->up_methods = &xhci_device_intr_methods; 2044 break; 2045 default: 2046 return USBD_IOERROR; 2047 break; 2048 } 2049 2050 if (ed->bEndpointAddress != USB_CONTROL_ENDPOINT) 2051 return xhci_configure_endpoint(pipe); 2052 2053 return USBD_NORMAL_COMPLETION; 2054 } 2055 2056 /* 2057 * Closes pipe, called from usbd_kill_pipe via close methods. 2058 * If the endpoint to be closed is ep0, disable_slot. 2059 * Should be called with sc_lock held. 2060 */ 2061 static void 2062 xhci_close_pipe(struct usbd_pipe *pipe) 2063 { 2064 struct xhci_softc * const sc = XHCI_PIPE2SC(pipe); 2065 struct xhci_slot * const xs = pipe->up_dev->ud_hcpriv; 2066 usb_endpoint_descriptor_t * const ed = pipe->up_endpoint->ue_edesc; 2067 const u_int dci = xhci_ep_get_dci(ed); 2068 struct xhci_soft_trb trb; 2069 uint32_t *cp; 2070 2071 XHCIHIST_FUNC(); 2072 2073 if (sc->sc_dying) 2074 return; 2075 2076 /* xs is uninitialized before xhci_init_slot */ 2077 if (xs == NULL || xs->xs_idx == 0) 2078 return; 2079 2080 XHCIHIST_CALLARGS("pipe %#jx slot %ju dci %ju", 2081 (uintptr_t)pipe, xs->xs_idx, dci, 0); 2082 2083 KASSERTMSG(!cpu_intr_p() && !cpu_softintr_p(), "called from intr ctx"); 2084 KASSERT(mutex_owned(&sc->sc_lock)); 2085 2086 if (pipe->up_dev->ud_depth == 0) 2087 return; 2088 2089 if (dci == XHCI_DCI_EP_CONTROL) { 2090 DPRINTFN(4, "closing ep0", 0, 0, 0, 0); 2091 xhci_disable_slot(sc, xs->xs_idx); 2092 return; 2093 } 2094 2095 if (xhci_get_epstate(sc, xs, dci) != XHCI_EPSTATE_STOPPED) 2096 (void)xhci_stop_endpoint(pipe); 2097 2098 /* 2099 * set appropriate bit to be dropped. 2100 * don't set DC bit to 1, otherwise all endpoints 2101 * would be deconfigured. 2102 */ 2103 cp = xhci_slot_get_icv(sc, xs, XHCI_ICI_INPUT_CONTROL); 2104 cp[0] = htole32(XHCI_INCTX_0_DROP_MASK(dci)); 2105 cp[1] = htole32(0); 2106 2107 /* XXX should be most significant one, not dci? */ 2108 cp = xhci_slot_get_icv(sc, xs, xhci_dci_to_ici(XHCI_DCI_SLOT)); 2109 cp[0] = htole32(XHCI_SCTX_0_CTX_NUM_SET(dci)); 2110 2111 /* configure ep context performs an implicit dequeue */ 2112 xhci_host_dequeue(&xs->xs_ep[dci].xe_tr); 2113 2114 /* sync input contexts before they are read from memory */ 2115 usb_syncmem(&xs->xs_ic_dma, 0, sc->sc_pgsz, BUS_DMASYNC_PREWRITE); 2116 2117 trb.trb_0 = xhci_slot_get_icp(sc, xs, 0); 2118 trb.trb_2 = 0; 2119 trb.trb_3 = XHCI_TRB_3_SLOT_SET(xs->xs_idx) | 2120 XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_CONFIGURE_EP); 2121 2122 (void)xhci_do_command_locked(sc, &trb, USBD_DEFAULT_TIMEOUT); 2123 usb_syncmem(&xs->xs_dc_dma, 0, sc->sc_pgsz, BUS_DMASYNC_POSTREAD); 2124 } 2125 2126 /* 2127 * Abort transfer. 2128 * Should be called with sc_lock held. 2129 */ 2130 static void 2131 xhci_abortx(struct usbd_xfer *xfer) 2132 { 2133 XHCIHIST_FUNC(); 2134 struct xhci_softc * const sc = XHCI_XFER2SC(xfer); 2135 struct xhci_slot * const xs = xfer->ux_pipe->up_dev->ud_hcpriv; 2136 const u_int dci = xhci_ep_get_dci(xfer->ux_pipe->up_endpoint->ue_edesc); 2137 2138 XHCIHIST_CALLARGS("xfer %#jx pipe %#jx", 2139 (uintptr_t)xfer, (uintptr_t)xfer->ux_pipe, 0, 0); 2140 2141 KASSERT(mutex_owned(&sc->sc_lock)); 2142 ASSERT_SLEEPABLE(); 2143 2144 KASSERTMSG((xfer->ux_status == USBD_CANCELLED || 2145 xfer->ux_status == USBD_TIMEOUT), 2146 "bad abort status: %d", xfer->ux_status); 2147 2148 /* 2149 * If we're dying, skip the hardware action and just notify the 2150 * software that we're done. 2151 */ 2152 if (sc->sc_dying) { 2153 DPRINTFN(4, "xfer %#jx dying %ju", (uintptr_t)xfer, 2154 xfer->ux_status, 0, 0); 2155 goto dying; 2156 } 2157 2158 /* 2159 * HC Step 1: Stop execution of TD on the ring. 2160 */ 2161 switch (xhci_get_epstate(sc, xs, dci)) { 2162 case XHCI_EPSTATE_HALTED: 2163 (void)xhci_reset_endpoint_locked(xfer->ux_pipe); 2164 break; 2165 case XHCI_EPSTATE_STOPPED: 2166 break; 2167 default: 2168 (void)xhci_stop_endpoint(xfer->ux_pipe); 2169 break; 2170 } 2171 #ifdef DIAGNOSTIC 2172 uint32_t epst = xhci_get_epstate(sc, xs, dci); 2173 if (epst != XHCI_EPSTATE_STOPPED) 2174 DPRINTFN(4, "dci %ju not stopped %ju", dci, epst, 0, 0); 2175 #endif 2176 2177 /* 2178 * HC Step 2: Remove any vestiges of the xfer from the ring. 2179 */ 2180 xhci_set_dequeue_locked(xfer->ux_pipe); 2181 2182 /* 2183 * Final Step: Notify completion to waiting xfers. 2184 */ 2185 dying: 2186 usb_transfer_complete(xfer); 2187 DPRINTFN(14, "end", 0, 0, 0, 0); 2188 2189 KASSERT(mutex_owned(&sc->sc_lock)); 2190 } 2191 2192 static void 2193 xhci_host_dequeue(struct xhci_ring * const xr) 2194 { 2195 /* When dequeueing the controller, update our struct copy too */ 2196 memset(xr->xr_trb, 0, xr->xr_ntrb * XHCI_TRB_SIZE); 2197 usb_syncmem(&xr->xr_dma, 0, xr->xr_ntrb * XHCI_TRB_SIZE, 2198 BUS_DMASYNC_PREWRITE); 2199 memset(xr->xr_cookies, 0, xr->xr_ntrb * sizeof(*xr->xr_cookies)); 2200 2201 xr->xr_ep = 0; 2202 xr->xr_cs = 1; 2203 } 2204 2205 /* 2206 * Recover STALLed endpoint. 2207 * xHCI 1.1 sect 4.10.2.1 2208 * Issue RESET_EP to recover halt condition and SET_TR_DEQUEUE to remove 2209 * all transfers on transfer ring. 2210 * These are done in thread context asynchronously. 2211 */ 2212 static void 2213 xhci_clear_endpoint_stall_async_task(void *cookie) 2214 { 2215 struct usbd_xfer * const xfer = cookie; 2216 struct xhci_softc * const sc = XHCI_XFER2SC(xfer); 2217 struct xhci_slot * const xs = xfer->ux_pipe->up_dev->ud_hcpriv; 2218 const u_int dci = xhci_ep_get_dci(xfer->ux_pipe->up_endpoint->ue_edesc); 2219 struct xhci_ring * const tr = &xs->xs_ep[dci].xe_tr; 2220 2221 XHCIHIST_FUNC(); 2222 XHCIHIST_CALLARGS("xfer %#jx slot %ju dci %ju", (uintptr_t)xfer, xs->xs_idx, 2223 dci, 0); 2224 2225 /* 2226 * XXXMRG: Stall task can run after slot is disabled when yanked. 2227 * This hack notices that the xs has been memset() in 2228 * xhci_disable_slot() and returns. Both xhci_reset_endpoint() 2229 * and xhci_set_dequeue() rely upon a valid ring setup for correct 2230 * operation, and the latter will fault, as would 2231 * usb_transfer_complete() if it got that far. 2232 */ 2233 if (xs->xs_idx == 0) { 2234 DPRINTFN(4, "ends xs_idx is 0", 0, 0, 0, 0); 2235 return; 2236 } 2237 2238 xhci_reset_endpoint(xfer->ux_pipe); 2239 xhci_set_dequeue(xfer->ux_pipe); 2240 2241 mutex_enter(&sc->sc_lock); 2242 tr->is_halted = false; 2243 usb_transfer_complete(xfer); 2244 mutex_exit(&sc->sc_lock); 2245 DPRINTFN(4, "ends", 0, 0, 0, 0); 2246 } 2247 2248 static usbd_status 2249 xhci_clear_endpoint_stall_async(struct usbd_xfer *xfer) 2250 { 2251 struct xhci_softc * const sc = XHCI_XFER2SC(xfer); 2252 struct xhci_pipe * const xp = (struct xhci_pipe *)xfer->ux_pipe; 2253 2254 XHCIHIST_FUNC(); 2255 XHCIHIST_CALLARGS("xfer %#jx", (uintptr_t)xfer, 0, 0, 0); 2256 2257 if (sc->sc_dying) { 2258 return USBD_IOERROR; 2259 } 2260 2261 usb_init_task(&xp->xp_async_task, 2262 xhci_clear_endpoint_stall_async_task, xfer, USB_TASKQ_MPSAFE); 2263 usb_add_task(xfer->ux_pipe->up_dev, &xp->xp_async_task, USB_TASKQ_HC); 2264 DPRINTFN(4, "ends", 0, 0, 0, 0); 2265 2266 return USBD_NORMAL_COMPLETION; 2267 } 2268 2269 /* Process roothub port status/change events and notify to uhub_intr. */ 2270 static void 2271 xhci_rhpsc(struct xhci_softc * const sc, u_int ctlrport) 2272 { 2273 XHCIHIST_FUNC(); 2274 XHCIHIST_CALLARGS("xhci%jd: port %ju status change", 2275 device_unit(sc->sc_dev), ctlrport, 0, 0); 2276 2277 if (ctlrport > sc->sc_maxports) 2278 return; 2279 2280 const size_t bn = xhci_ctlrport2bus(sc, ctlrport); 2281 const size_t rhp = xhci_ctlrport2rhport(sc, ctlrport); 2282 struct usbd_xfer * const xfer = sc->sc_intrxfer[bn]; 2283 2284 DPRINTFN(4, "xhci%jd: bus %jd bp %ju xfer %#jx status change", 2285 device_unit(sc->sc_dev), bn, rhp, (uintptr_t)xfer); 2286 2287 if (xfer == NULL) 2288 return; 2289 KASSERT(xfer->ux_status == USBD_IN_PROGRESS); 2290 2291 uint8_t *p = xfer->ux_buf; 2292 memset(p, 0, xfer->ux_length); 2293 p[rhp / NBBY] |= 1 << (rhp % NBBY); 2294 xfer->ux_actlen = xfer->ux_length; 2295 xfer->ux_status = USBD_NORMAL_COMPLETION; 2296 usb_transfer_complete(xfer); 2297 } 2298 2299 /* Process Transfer Events */ 2300 static void 2301 xhci_event_transfer(struct xhci_softc * const sc, 2302 const struct xhci_trb * const trb) 2303 { 2304 uint64_t trb_0; 2305 uint32_t trb_2, trb_3; 2306 uint8_t trbcode; 2307 u_int slot, dci; 2308 struct xhci_slot *xs; 2309 struct xhci_ring *xr; 2310 struct xhci_xfer *xx; 2311 struct usbd_xfer *xfer; 2312 usbd_status err; 2313 2314 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 2315 2316 trb_0 = le64toh(trb->trb_0); 2317 trb_2 = le32toh(trb->trb_2); 2318 trb_3 = le32toh(trb->trb_3); 2319 trbcode = XHCI_TRB_2_ERROR_GET(trb_2); 2320 slot = XHCI_TRB_3_SLOT_GET(trb_3); 2321 dci = XHCI_TRB_3_EP_GET(trb_3); 2322 xs = &sc->sc_slots[slot]; 2323 xr = &xs->xs_ep[dci].xe_tr; 2324 2325 /* sanity check */ 2326 KASSERTMSG(xs->xs_idx != 0 && xs->xs_idx <= sc->sc_maxslots, 2327 "invalid xs_idx %u slot %u", xs->xs_idx, slot); 2328 2329 int idx = 0; 2330 if ((trb_3 & XHCI_TRB_3_ED_BIT) == 0) { 2331 if (xhci_trb_get_idx(xr, trb_0, &idx)) { 2332 DPRINTFN(0, "invalid trb_0 0x%jx", trb_0, 0, 0, 0); 2333 return; 2334 } 2335 xx = xr->xr_cookies[idx]; 2336 2337 /* clear cookie of consumed TRB */ 2338 xr->xr_cookies[idx] = NULL; 2339 2340 /* 2341 * xx is NULL if pipe is opened but xfer is not started. 2342 * It happens when stopping idle pipe. 2343 */ 2344 if (xx == NULL || trbcode == XHCI_TRB_ERROR_LENGTH) { 2345 DPRINTFN(1, "Ignore #%ju: cookie %#jx cc %ju dci %ju", 2346 idx, (uintptr_t)xx, trbcode, dci); 2347 DPRINTFN(1, " orig TRB %jx type %ju", trb_0, 2348 XHCI_TRB_3_TYPE_GET(le32toh(xr->xr_trb[idx].trb_3)), 2349 0, 0); 2350 return; 2351 } 2352 } else { 2353 /* When ED != 0, trb_0 is virtual addr of struct xhci_xfer. */ 2354 xx = (void *)(uintptr_t)(trb_0 & ~0x3); 2355 } 2356 /* XXX this may not happen */ 2357 if (xx == NULL) { 2358 DPRINTFN(1, "xfer done: xx is NULL", 0, 0, 0, 0); 2359 return; 2360 } 2361 xfer = &xx->xx_xfer; 2362 /* XXX this may happen when detaching */ 2363 if (xfer == NULL) { 2364 DPRINTFN(1, "xx(%#jx)->xx_xfer is NULL trb_0 %#jx", 2365 (uintptr_t)xx, trb_0, 0, 0); 2366 return; 2367 } 2368 DPRINTFN(14, "xfer %#jx", (uintptr_t)xfer, 0, 0, 0); 2369 /* XXX I dunno why this happens */ 2370 KASSERTMSG(xfer->ux_pipe != NULL, "xfer(%p)->ux_pipe is NULL", xfer); 2371 2372 if (!xfer->ux_pipe->up_repeat && 2373 SIMPLEQ_EMPTY(&xfer->ux_pipe->up_queue)) { 2374 DPRINTFN(1, "xfer(%#jx)->pipe not queued", (uintptr_t)xfer, 2375 0, 0, 0); 2376 return; 2377 } 2378 2379 /* 4.11.5.2 Event Data TRB */ 2380 if ((trb_3 & XHCI_TRB_3_ED_BIT) != 0) { 2381 DPRINTFN(14, "transfer Event Data: 0x%016jx 0x%08jx" 2382 " %02jx", trb_0, XHCI_TRB_2_REM_GET(trb_2), trbcode, 0); 2383 if ((trb_0 & 0x3) == 0x3) { 2384 xfer->ux_actlen = XHCI_TRB_2_REM_GET(trb_2); 2385 } 2386 } 2387 2388 switch (trbcode) { 2389 case XHCI_TRB_ERROR_SHORT_PKT: 2390 case XHCI_TRB_ERROR_SUCCESS: 2391 /* 2392 * A ctrl transfer can generate two events if it has a Data 2393 * stage. A short data stage can be OK and should not 2394 * complete the transfer as the status stage needs to be 2395 * performed. 2396 * 2397 * Note: Data and Status stage events point at same xfer. 2398 * ux_actlen and ux_dmabuf will be passed to 2399 * usb_transfer_complete after the Status stage event. 2400 * 2401 * It can be distingished which stage generates the event: 2402 * + by checking least 3 bits of trb_0 if ED==1. 2403 * (see xhci_device_ctrl_start). 2404 * + by checking the type of original TRB if ED==0. 2405 * 2406 * In addition, intr, bulk, and isoc transfer currently 2407 * consists of single TD, so the "skip" is not needed. 2408 * ctrl xfer uses EVENT_DATA, and others do not. 2409 * Thus driver can switch the flow by checking ED bit. 2410 */ 2411 if ((trb_3 & XHCI_TRB_3_ED_BIT) == 0) { 2412 if (xfer->ux_actlen == 0) 2413 xfer->ux_actlen = xfer->ux_length - 2414 XHCI_TRB_2_REM_GET(trb_2); 2415 if (XHCI_TRB_3_TYPE_GET(le32toh(xr->xr_trb[idx].trb_3)) 2416 == XHCI_TRB_TYPE_DATA_STAGE) { 2417 return; 2418 } 2419 } else if ((trb_0 & 0x3) == 0x3) { 2420 return; 2421 } 2422 err = USBD_NORMAL_COMPLETION; 2423 break; 2424 case XHCI_TRB_ERROR_STOPPED: 2425 case XHCI_TRB_ERROR_LENGTH: 2426 case XHCI_TRB_ERROR_STOPPED_SHORT: 2427 err = USBD_IOERROR; 2428 break; 2429 case XHCI_TRB_ERROR_STALL: 2430 case XHCI_TRB_ERROR_BABBLE: 2431 DPRINTFN(1, "ERR %ju slot %ju dci %ju", trbcode, slot, dci, 0); 2432 xr->is_halted = true; 2433 /* 2434 * Try to claim this xfer for completion. If it has already 2435 * completed or aborted, drop it on the floor. 2436 */ 2437 if (!usbd_xfer_trycomplete(xfer)) 2438 return; 2439 2440 /* 2441 * Stalled endpoints can be recoverd by issuing 2442 * command TRB TYPE_RESET_EP on xHCI instead of 2443 * issuing request CLEAR_FEATURE UF_ENDPOINT_HALT 2444 * on the endpoint. However, this function may be 2445 * called from softint context (e.g. from umass), 2446 * in that case driver gets KASSERT in cv_timedwait 2447 * in xhci_do_command. 2448 * To avoid this, this runs reset_endpoint and 2449 * usb_transfer_complete in usb task thread 2450 * asynchronously (and then umass issues clear 2451 * UF_ENDPOINT_HALT). 2452 */ 2453 2454 /* Override the status. */ 2455 xfer->ux_status = USBD_STALLED; 2456 2457 xhci_clear_endpoint_stall_async(xfer); 2458 return; 2459 default: 2460 DPRINTFN(1, "ERR %ju slot %ju dci %ju", trbcode, slot, dci, 0); 2461 err = USBD_IOERROR; 2462 break; 2463 } 2464 2465 /* 2466 * Try to claim this xfer for completion. If it has already 2467 * completed or aborted, drop it on the floor. 2468 */ 2469 if (!usbd_xfer_trycomplete(xfer)) 2470 return; 2471 2472 /* Set the status. */ 2473 xfer->ux_status = err; 2474 2475 if ((trb_3 & XHCI_TRB_3_ED_BIT) == 0 || 2476 (trb_0 & 0x3) == 0x0) { 2477 usb_transfer_complete(xfer); 2478 } 2479 } 2480 2481 /* Process Command complete events */ 2482 static void 2483 xhci_event_cmd(struct xhci_softc * const sc, const struct xhci_trb * const trb) 2484 { 2485 uint64_t trb_0; 2486 uint32_t trb_2, trb_3; 2487 2488 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 2489 2490 KASSERT(mutex_owned(&sc->sc_lock)); 2491 2492 trb_0 = le64toh(trb->trb_0); 2493 trb_2 = le32toh(trb->trb_2); 2494 trb_3 = le32toh(trb->trb_3); 2495 2496 if (trb_0 == sc->sc_command_addr) { 2497 sc->sc_resultpending = false; 2498 2499 sc->sc_result_trb.trb_0 = trb_0; 2500 sc->sc_result_trb.trb_2 = trb_2; 2501 sc->sc_result_trb.trb_3 = trb_3; 2502 if (XHCI_TRB_2_ERROR_GET(trb_2) != 2503 XHCI_TRB_ERROR_SUCCESS) { 2504 DPRINTFN(1, "command completion " 2505 "failure: 0x%016jx 0x%08jx 0x%08jx", 2506 trb_0, trb_2, trb_3, 0); 2507 } 2508 cv_signal(&sc->sc_command_cv); 2509 } else { 2510 DPRINTFN(1, "spurious event: %#jx 0x%016jx " 2511 "0x%08jx 0x%08jx", (uintptr_t)trb, trb_0, trb_2, trb_3); 2512 } 2513 } 2514 2515 /* 2516 * Process events. 2517 * called from xhci_softintr 2518 */ 2519 static void 2520 xhci_handle_event(struct xhci_softc * const sc, 2521 const struct xhci_trb * const trb) 2522 { 2523 uint64_t trb_0; 2524 uint32_t trb_2, trb_3; 2525 2526 XHCIHIST_FUNC(); 2527 2528 trb_0 = le64toh(trb->trb_0); 2529 trb_2 = le32toh(trb->trb_2); 2530 trb_3 = le32toh(trb->trb_3); 2531 2532 XHCIHIST_CALLARGS("event: %#jx 0x%016jx 0x%08jx 0x%08jx", 2533 (uintptr_t)trb, trb_0, trb_2, trb_3); 2534 2535 /* 2536 * 4.11.3.1, 6.4.2.1 2537 * TRB Pointer is invalid for these completion codes. 2538 */ 2539 switch (XHCI_TRB_2_ERROR_GET(trb_2)) { 2540 case XHCI_TRB_ERROR_RING_UNDERRUN: 2541 case XHCI_TRB_ERROR_RING_OVERRUN: 2542 case XHCI_TRB_ERROR_VF_RING_FULL: 2543 return; 2544 default: 2545 if (trb_0 == 0) { 2546 return; 2547 } 2548 break; 2549 } 2550 2551 switch (XHCI_TRB_3_TYPE_GET(trb_3)) { 2552 case XHCI_TRB_EVENT_TRANSFER: 2553 xhci_event_transfer(sc, trb); 2554 break; 2555 case XHCI_TRB_EVENT_CMD_COMPLETE: 2556 xhci_event_cmd(sc, trb); 2557 break; 2558 case XHCI_TRB_EVENT_PORT_STS_CHANGE: 2559 xhci_rhpsc(sc, (uint32_t)((trb_0 >> 24) & 0xff)); 2560 break; 2561 default: 2562 break; 2563 } 2564 } 2565 2566 static void 2567 xhci_softintr(void *v) 2568 { 2569 struct usbd_bus * const bus = v; 2570 struct xhci_softc * const sc = XHCI_BUS2SC(bus); 2571 struct xhci_ring * const er = &sc->sc_er; 2572 struct xhci_trb *trb; 2573 int i, j, k; 2574 2575 XHCIHIST_FUNC(); 2576 2577 KASSERT(xhci_polling_p(sc) || mutex_owned(&sc->sc_lock)); 2578 2579 i = er->xr_ep; 2580 j = er->xr_cs; 2581 2582 XHCIHIST_CALLARGS("er: xr_ep %jd xr_cs %jd", i, j, 0, 0); 2583 2584 while (1) { 2585 usb_syncmem(&er->xr_dma, XHCI_TRB_SIZE * i, XHCI_TRB_SIZE, 2586 BUS_DMASYNC_POSTREAD); 2587 trb = &er->xr_trb[i]; 2588 k = (le32toh(trb->trb_3) & XHCI_TRB_3_CYCLE_BIT) ? 1 : 0; 2589 2590 if (j != k) 2591 break; 2592 2593 xhci_handle_event(sc, trb); 2594 2595 i++; 2596 if (i == er->xr_ntrb) { 2597 i = 0; 2598 j ^= 1; 2599 } 2600 } 2601 2602 er->xr_ep = i; 2603 er->xr_cs = j; 2604 2605 xhci_rt_write_8(sc, XHCI_ERDP(0), xhci_ring_trbp(er, er->xr_ep) | 2606 XHCI_ERDP_LO_BUSY); 2607 2608 DPRINTFN(16, "ends", 0, 0, 0, 0); 2609 2610 return; 2611 } 2612 2613 static void 2614 xhci_poll(struct usbd_bus *bus) 2615 { 2616 struct xhci_softc * const sc = XHCI_BUS2SC(bus); 2617 2618 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 2619 2620 mutex_enter(&sc->sc_intr_lock); 2621 int ret = xhci_intr1(sc); 2622 if (ret) { 2623 xhci_softintr(bus); 2624 } 2625 mutex_exit(&sc->sc_intr_lock); 2626 2627 return; 2628 } 2629 2630 static struct usbd_xfer * 2631 xhci_allocx(struct usbd_bus *bus, unsigned int nframes) 2632 { 2633 struct xhci_softc * const sc = XHCI_BUS2SC(bus); 2634 struct usbd_xfer *xfer; 2635 2636 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 2637 2638 xfer = pool_cache_get(sc->sc_xferpool, PR_WAITOK); 2639 if (xfer != NULL) { 2640 memset(xfer, 0, sizeof(struct xhci_xfer)); 2641 #ifdef DIAGNOSTIC 2642 xfer->ux_state = XFER_BUSY; 2643 #endif 2644 } 2645 2646 return xfer; 2647 } 2648 2649 static void 2650 xhci_freex(struct usbd_bus *bus, struct usbd_xfer *xfer) 2651 { 2652 struct xhci_softc * const sc = XHCI_BUS2SC(bus); 2653 2654 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 2655 2656 #ifdef DIAGNOSTIC 2657 if (xfer->ux_state != XFER_BUSY && 2658 xfer->ux_status != USBD_NOT_STARTED) { 2659 DPRINTFN(0, "xfer=%#jx not busy, 0x%08jx", 2660 (uintptr_t)xfer, xfer->ux_state, 0, 0); 2661 } 2662 xfer->ux_state = XFER_FREE; 2663 #endif 2664 pool_cache_put(sc->sc_xferpool, xfer); 2665 } 2666 2667 static bool 2668 xhci_dying(struct usbd_bus *bus) 2669 { 2670 struct xhci_softc * const sc = XHCI_BUS2SC(bus); 2671 2672 return sc->sc_dying; 2673 } 2674 2675 static void 2676 xhci_get_lock(struct usbd_bus *bus, kmutex_t **lock) 2677 { 2678 struct xhci_softc * const sc = XHCI_BUS2SC(bus); 2679 2680 *lock = &sc->sc_lock; 2681 } 2682 2683 extern uint32_t usb_cookie_no; 2684 2685 /* 2686 * xHCI 4.3 2687 * Called when uhub_explore finds a new device (via usbd_new_device). 2688 * Port initialization and speed detection (4.3.1) are already done in uhub.c. 2689 * This function does: 2690 * Allocate and construct dev structure of default endpoint (ep0). 2691 * Allocate and open pipe of ep0. 2692 * Enable slot and initialize slot context. 2693 * Set Address. 2694 * Read initial device descriptor. 2695 * Determine initial MaxPacketSize (mps) by speed. 2696 * Read full device descriptor. 2697 * Register this device. 2698 * Finally state of device transitions ADDRESSED. 2699 */ 2700 static usbd_status 2701 xhci_new_device(device_t parent, struct usbd_bus *bus, int depth, 2702 int speed, int port, struct usbd_port *up) 2703 { 2704 struct xhci_softc * const sc = XHCI_BUS2SC(bus); 2705 struct usbd_device *dev; 2706 usbd_status err; 2707 usb_device_descriptor_t *dd; 2708 struct xhci_slot *xs; 2709 uint32_t *cp; 2710 2711 XHCIHIST_FUNC(); 2712 XHCIHIST_CALLARGS("port %ju depth %ju speed %ju up %#jx", 2713 port, depth, speed, (uintptr_t)up); 2714 2715 dev = kmem_zalloc(sizeof(*dev), KM_SLEEP); 2716 dev->ud_bus = bus; 2717 dev->ud_quirks = &usbd_no_quirk; 2718 dev->ud_addr = 0; 2719 dev->ud_ddesc.bMaxPacketSize = 0; 2720 dev->ud_depth = depth; 2721 dev->ud_powersrc = up; 2722 dev->ud_myhub = up->up_parent; 2723 dev->ud_speed = speed; 2724 dev->ud_langid = USBD_NOLANG; 2725 dev->ud_cookie.cookie = ++usb_cookie_no; 2726 2727 /* Set up default endpoint handle. */ 2728 dev->ud_ep0.ue_edesc = &dev->ud_ep0desc; 2729 /* doesn't matter, just don't let it uninitialized */ 2730 dev->ud_ep0.ue_toggle = 0; 2731 2732 /* Set up default endpoint descriptor. */ 2733 dev->ud_ep0desc.bLength = USB_ENDPOINT_DESCRIPTOR_SIZE; 2734 dev->ud_ep0desc.bDescriptorType = UDESC_ENDPOINT; 2735 dev->ud_ep0desc.bEndpointAddress = USB_CONTROL_ENDPOINT; 2736 dev->ud_ep0desc.bmAttributes = UE_CONTROL; 2737 dev->ud_ep0desc.bInterval = 0; 2738 2739 /* 4.3, 4.8.2.1 */ 2740 switch (speed) { 2741 case USB_SPEED_SUPER: 2742 case USB_SPEED_SUPER_PLUS: 2743 USETW(dev->ud_ep0desc.wMaxPacketSize, USB_3_MAX_CTRL_PACKET); 2744 break; 2745 case USB_SPEED_FULL: 2746 /* XXX using 64 as initial mps of ep0 in FS */ 2747 case USB_SPEED_HIGH: 2748 USETW(dev->ud_ep0desc.wMaxPacketSize, USB_2_MAX_CTRL_PACKET); 2749 break; 2750 case USB_SPEED_LOW: 2751 default: 2752 USETW(dev->ud_ep0desc.wMaxPacketSize, USB_MAX_IPACKET); 2753 break; 2754 } 2755 2756 up->up_dev = dev; 2757 2758 /* Establish the default pipe. */ 2759 err = usbd_setup_pipe(dev, 0, &dev->ud_ep0, USBD_DEFAULT_INTERVAL, 2760 &dev->ud_pipe0); 2761 if (err) { 2762 goto bad; 2763 } 2764 2765 dd = &dev->ud_ddesc; 2766 2767 if (depth == 0 && port == 0) { 2768 KASSERT(bus->ub_devices[USB_ROOTHUB_INDEX] == NULL); 2769 bus->ub_devices[USB_ROOTHUB_INDEX] = dev; 2770 err = usbd_get_initial_ddesc(dev, dd); 2771 if (err) { 2772 DPRINTFN(1, "get_initial_ddesc %ju", err, 0, 0, 0); 2773 goto bad; 2774 } 2775 2776 err = usbd_reload_device_desc(dev); 2777 if (err) { 2778 DPRINTFN(1, "reload desc %ju", err, 0, 0, 0); 2779 goto bad; 2780 } 2781 } else { 2782 uint8_t slot = 0; 2783 2784 /* 4.3.2 */ 2785 err = xhci_enable_slot(sc, &slot); 2786 if (err) { 2787 DPRINTFN(1, "enable slot %ju", err, 0, 0, 0); 2788 goto bad; 2789 } 2790 2791 xs = &sc->sc_slots[slot]; 2792 dev->ud_hcpriv = xs; 2793 2794 /* 4.3.3 initialize slot structure */ 2795 err = xhci_init_slot(dev, slot); 2796 if (err) { 2797 DPRINTFN(1, "init slot %ju", err, 0, 0, 0); 2798 dev->ud_hcpriv = NULL; 2799 /* 2800 * We have to disable_slot here because 2801 * xs->xs_idx == 0 when xhci_init_slot fails, 2802 * in that case usbd_remove_dev won't work. 2803 */ 2804 mutex_enter(&sc->sc_lock); 2805 xhci_disable_slot(sc, slot); 2806 mutex_exit(&sc->sc_lock); 2807 goto bad; 2808 } 2809 2810 /* 4.3.4 Address Assignment */ 2811 err = xhci_set_address(dev, slot, false); 2812 if (err) { 2813 DPRINTFN(1, "failed! to set address: %ju", err, 0, 0, 0); 2814 goto bad; 2815 } 2816 2817 /* Allow device time to set new address */ 2818 usbd_delay_ms(dev, USB_SET_ADDRESS_SETTLE); 2819 2820 usb_syncmem(&xs->xs_dc_dma, 0, sc->sc_pgsz, BUS_DMASYNC_POSTREAD); 2821 cp = xhci_slot_get_dcv(sc, xs, XHCI_DCI_SLOT); 2822 HEXDUMP("slot context", cp, sc->sc_ctxsz); 2823 uint8_t addr = XHCI_SCTX_3_DEV_ADDR_GET(le32toh(cp[3])); 2824 DPRINTFN(4, "device address %ju", addr, 0, 0, 0); 2825 /* 2826 * XXX ensure we know when the hardware does something 2827 * we can't yet cope with 2828 */ 2829 KASSERTMSG(addr >= 1 && addr <= 127, "addr %d", addr); 2830 dev->ud_addr = addr; 2831 2832 KASSERTMSG(bus->ub_devices[usb_addr2dindex(dev->ud_addr)] == NULL, 2833 "addr %d already allocated", dev->ud_addr); 2834 /* 2835 * The root hub is given its own slot 2836 */ 2837 bus->ub_devices[usb_addr2dindex(dev->ud_addr)] = dev; 2838 2839 err = usbd_get_initial_ddesc(dev, dd); 2840 if (err) { 2841 DPRINTFN(1, "get_initial_ddesc %ju", err, 0, 0, 0); 2842 goto bad; 2843 } 2844 2845 /* 4.8.2.1 */ 2846 if (USB_IS_SS(speed)) { 2847 if (dd->bMaxPacketSize != 9) { 2848 printf("%s: invalid mps 2^%u for SS ep0," 2849 " using 512\n", 2850 device_xname(sc->sc_dev), 2851 dd->bMaxPacketSize); 2852 dd->bMaxPacketSize = 9; 2853 } 2854 USETW(dev->ud_ep0desc.wMaxPacketSize, 2855 (1 << dd->bMaxPacketSize)); 2856 } else 2857 USETW(dev->ud_ep0desc.wMaxPacketSize, 2858 dd->bMaxPacketSize); 2859 DPRINTFN(4, "bMaxPacketSize %ju", dd->bMaxPacketSize, 0, 0, 0); 2860 err = xhci_update_ep0_mps(sc, xs, 2861 UGETW(dev->ud_ep0desc.wMaxPacketSize)); 2862 if (err) { 2863 DPRINTFN(1, "update mps of ep0 %ju", err, 0, 0, 0); 2864 goto bad; 2865 } 2866 2867 err = usbd_reload_device_desc(dev); 2868 if (err) { 2869 DPRINTFN(1, "reload desc %ju", err, 0, 0, 0); 2870 goto bad; 2871 } 2872 } 2873 2874 DPRINTFN(1, "adding unit addr=%jd, rev=%02jx,", 2875 dev->ud_addr, UGETW(dd->bcdUSB), 0, 0); 2876 DPRINTFN(1, " class=%jd, subclass=%jd, protocol=%jd,", 2877 dd->bDeviceClass, dd->bDeviceSubClass, 2878 dd->bDeviceProtocol, 0); 2879 DPRINTFN(1, " mps=%jd, len=%jd, noconf=%jd, speed=%jd", 2880 dd->bMaxPacketSize, dd->bLength, dd->bNumConfigurations, 2881 dev->ud_speed); 2882 2883 usbd_get_device_strings(dev); 2884 2885 usbd_add_dev_event(USB_EVENT_DEVICE_ATTACH, dev); 2886 2887 if (depth == 0 && port == 0) { 2888 usbd_attach_roothub(parent, dev); 2889 DPRINTFN(1, "root hub %#jx", (uintptr_t)dev, 0, 0, 0); 2890 return USBD_NORMAL_COMPLETION; 2891 } 2892 2893 err = usbd_probe_and_attach(parent, dev, port, dev->ud_addr); 2894 bad: 2895 if (err != USBD_NORMAL_COMPLETION) { 2896 usbd_remove_device(dev, up); 2897 } 2898 2899 return err; 2900 } 2901 2902 static usbd_status 2903 xhci_ring_init(struct xhci_softc * const sc, struct xhci_ring * const xr, 2904 size_t ntrb, size_t align) 2905 { 2906 usbd_status err; 2907 size_t size = ntrb * XHCI_TRB_SIZE; 2908 2909 XHCIHIST_FUNC(); 2910 XHCIHIST_CALLARGS("xr %#jx ntrb %#jx align %#jx", 2911 (uintptr_t)xr, ntrb, align, 0); 2912 2913 err = usb_allocmem(&sc->sc_bus, size, align, &xr->xr_dma); 2914 if (err) 2915 return err; 2916 mutex_init(&xr->xr_lock, MUTEX_DEFAULT, IPL_SOFTUSB); 2917 xr->xr_cookies = kmem_zalloc(sizeof(*xr->xr_cookies) * ntrb, KM_SLEEP); 2918 xr->xr_trb = xhci_ring_trbv(xr, 0); 2919 xr->xr_ntrb = ntrb; 2920 xr->is_halted = false; 2921 xhci_host_dequeue(xr); 2922 2923 return USBD_NORMAL_COMPLETION; 2924 } 2925 2926 static void 2927 xhci_ring_free(struct xhci_softc * const sc, struct xhci_ring * const xr) 2928 { 2929 usb_freemem(&sc->sc_bus, &xr->xr_dma); 2930 mutex_destroy(&xr->xr_lock); 2931 kmem_free(xr->xr_cookies, sizeof(*xr->xr_cookies) * xr->xr_ntrb); 2932 } 2933 2934 static void 2935 xhci_ring_put(struct xhci_softc * const sc, struct xhci_ring * const xr, 2936 void *cookie, struct xhci_soft_trb * const trbs, size_t ntrbs) 2937 { 2938 size_t i; 2939 u_int ri; 2940 u_int cs; 2941 uint64_t parameter; 2942 uint32_t status; 2943 uint32_t control; 2944 2945 XHCIHIST_FUNC(); 2946 XHCIHIST_CALLARGS("%#jx xr_ep 0x%jx xr_cs %ju", 2947 (uintptr_t)xr, xr->xr_ep, xr->xr_cs, 0); 2948 2949 KASSERTMSG(ntrbs <= XHCI_XFER_NTRB, "ntrbs %zu", ntrbs); 2950 for (i = 0; i < ntrbs; i++) { 2951 DPRINTFN(12, "xr %#jx trbs %#jx num %ju", (uintptr_t)xr, 2952 (uintptr_t)trbs, i, 0); 2953 DPRINTFN(12, " %016jx %08jx %08jx", 2954 trbs[i].trb_0, trbs[i].trb_2, trbs[i].trb_3, 0); 2955 KASSERTMSG(XHCI_TRB_3_TYPE_GET(trbs[i].trb_3) != 2956 XHCI_TRB_TYPE_LINK, "trbs[%zu].trb3 %#x", i, trbs[i].trb_3); 2957 } 2958 2959 ri = xr->xr_ep; 2960 cs = xr->xr_cs; 2961 2962 /* 2963 * Although the xhci hardware can do scatter/gather dma from 2964 * arbitrary sized buffers, there is a non-obvious restriction 2965 * that a LINK trb is only allowed at the end of a burst of 2966 * transfers - which might be 16kB. 2967 * Arbitrary aligned LINK trb definitely fail on Ivy bridge. 2968 * The simple solution is not to allow a LINK trb in the middle 2969 * of anything - as here. 2970 * XXX: (dsl) There are xhci controllers out there (eg some made by 2971 * ASMedia) that seem to lock up if they process a LINK trb but 2972 * cannot process the linked-to trb yet. 2973 * The code should write the 'cycle' bit on the link trb AFTER 2974 * adding the other trb. 2975 */ 2976 u_int firstep = xr->xr_ep; 2977 u_int firstcs = xr->xr_cs; 2978 2979 for (i = 0; i < ntrbs; ) { 2980 u_int oldri = ri; 2981 u_int oldcs = cs; 2982 2983 if (ri >= (xr->xr_ntrb - 1)) { 2984 /* Put Link TD at the end of ring */ 2985 parameter = xhci_ring_trbp(xr, 0); 2986 status = 0; 2987 control = XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_LINK) | 2988 XHCI_TRB_3_TC_BIT; 2989 xr->xr_cookies[ri] = NULL; 2990 xr->xr_ep = 0; 2991 xr->xr_cs ^= 1; 2992 ri = xr->xr_ep; 2993 cs = xr->xr_cs; 2994 } else { 2995 parameter = trbs[i].trb_0; 2996 status = trbs[i].trb_2; 2997 control = trbs[i].trb_3; 2998 2999 xr->xr_cookies[ri] = cookie; 3000 ri++; 3001 i++; 3002 } 3003 /* 3004 * If this is a first TRB, mark it invalid to prevent 3005 * xHC from running it immediately. 3006 */ 3007 if (oldri == firstep) { 3008 if (oldcs) { 3009 control &= ~XHCI_TRB_3_CYCLE_BIT; 3010 } else { 3011 control |= XHCI_TRB_3_CYCLE_BIT; 3012 } 3013 } else { 3014 if (oldcs) { 3015 control |= XHCI_TRB_3_CYCLE_BIT; 3016 } else { 3017 control &= ~XHCI_TRB_3_CYCLE_BIT; 3018 } 3019 } 3020 xhci_trb_put(&xr->xr_trb[oldri], parameter, status, control); 3021 usb_syncmem(&xr->xr_dma, XHCI_TRB_SIZE * oldri, 3022 XHCI_TRB_SIZE * 1, BUS_DMASYNC_PREWRITE); 3023 } 3024 3025 /* Now invert cycle bit of first TRB */ 3026 if (firstcs) { 3027 xr->xr_trb[firstep].trb_3 |= htole32(XHCI_TRB_3_CYCLE_BIT); 3028 } else { 3029 xr->xr_trb[firstep].trb_3 &= ~htole32(XHCI_TRB_3_CYCLE_BIT); 3030 } 3031 usb_syncmem(&xr->xr_dma, XHCI_TRB_SIZE * firstep, 3032 XHCI_TRB_SIZE * 1, BUS_DMASYNC_PREWRITE); 3033 3034 xr->xr_ep = ri; 3035 xr->xr_cs = cs; 3036 3037 DPRINTFN(12, "%#jx xr_ep 0x%jx xr_cs %ju", (uintptr_t)xr, xr->xr_ep, 3038 xr->xr_cs, 0); 3039 } 3040 3041 /* 3042 * Stop execution commands, purge all commands on command ring, and 3043 * rewind dequeue pointer. 3044 */ 3045 static void 3046 xhci_abort_command(struct xhci_softc *sc) 3047 { 3048 struct xhci_ring * const cr = &sc->sc_cr; 3049 uint64_t crcr; 3050 int i; 3051 3052 XHCIHIST_FUNC(); 3053 XHCIHIST_CALLARGS("command %#jx timeout, aborting", 3054 sc->sc_command_addr, 0, 0, 0); 3055 3056 mutex_enter(&cr->xr_lock); 3057 3058 /* 4.6.1.2 Aborting a Command */ 3059 crcr = xhci_op_read_8(sc, XHCI_CRCR); 3060 xhci_op_write_8(sc, XHCI_CRCR, crcr | XHCI_CRCR_LO_CA); 3061 3062 for (i = 0; i < 500; i++) { 3063 crcr = xhci_op_read_8(sc, XHCI_CRCR); 3064 if ((crcr & XHCI_CRCR_LO_CRR) == 0) 3065 break; 3066 usb_delay_ms(&sc->sc_bus, 1); 3067 } 3068 if ((crcr & XHCI_CRCR_LO_CRR) != 0) { 3069 DPRINTFN(1, "Command Abort timeout", 0, 0, 0, 0); 3070 /* reset HC here? */ 3071 } 3072 3073 /* reset command ring dequeue pointer */ 3074 cr->xr_ep = 0; 3075 cr->xr_cs = 1; 3076 xhci_op_write_8(sc, XHCI_CRCR, xhci_ring_trbp(cr, 0) | cr->xr_cs); 3077 3078 mutex_exit(&cr->xr_lock); 3079 } 3080 3081 /* 3082 * Put a command on command ring, ring bell, set timer, and cv_timedwait. 3083 * Command completion is notified by cv_signal from xhci_event_cmd() 3084 * (called from xhci_softint), or timed-out. 3085 * The completion code is copied to sc->sc_result_trb in xhci_event_cmd(), 3086 * then do_command examines it. 3087 */ 3088 static usbd_status 3089 xhci_do_command_locked(struct xhci_softc * const sc, 3090 struct xhci_soft_trb * const trb, int timeout) 3091 { 3092 struct xhci_ring * const cr = &sc->sc_cr; 3093 usbd_status err; 3094 3095 XHCIHIST_FUNC(); 3096 XHCIHIST_CALLARGS("input: 0x%016jx 0x%08jx 0x%08jx", 3097 trb->trb_0, trb->trb_2, trb->trb_3, 0); 3098 3099 KASSERTMSG(!cpu_intr_p() && !cpu_softintr_p(), "called from intr ctx"); 3100 KASSERT(mutex_owned(&sc->sc_lock)); 3101 3102 while (sc->sc_command_addr != 0 || 3103 (sc->sc_suspender != NULL && sc->sc_suspender != curlwp)) 3104 cv_wait(&sc->sc_cmdbusy_cv, &sc->sc_lock); 3105 3106 /* 3107 * If enqueue pointer points at last of ring, it's Link TRB, 3108 * command TRB will be stored in 0th TRB. 3109 */ 3110 if (cr->xr_ep == cr->xr_ntrb - 1) 3111 sc->sc_command_addr = xhci_ring_trbp(cr, 0); 3112 else 3113 sc->sc_command_addr = xhci_ring_trbp(cr, cr->xr_ep); 3114 3115 sc->sc_resultpending = true; 3116 3117 mutex_enter(&cr->xr_lock); 3118 xhci_ring_put(sc, cr, NULL, trb, 1); 3119 mutex_exit(&cr->xr_lock); 3120 3121 xhci_db_write_4(sc, XHCI_DOORBELL(0), 0); 3122 3123 while (sc->sc_resultpending) { 3124 if (cv_timedwait(&sc->sc_command_cv, &sc->sc_lock, 3125 MAX(1, mstohz(timeout))) == EWOULDBLOCK) { 3126 xhci_abort_command(sc); 3127 err = USBD_TIMEOUT; 3128 goto timedout; 3129 } 3130 } 3131 3132 trb->trb_0 = sc->sc_result_trb.trb_0; 3133 trb->trb_2 = sc->sc_result_trb.trb_2; 3134 trb->trb_3 = sc->sc_result_trb.trb_3; 3135 3136 DPRINTFN(12, "output: 0x%016jx 0x%08jx 0x%08jx", 3137 trb->trb_0, trb->trb_2, trb->trb_3, 0); 3138 3139 switch (XHCI_TRB_2_ERROR_GET(trb->trb_2)) { 3140 case XHCI_TRB_ERROR_SUCCESS: 3141 err = USBD_NORMAL_COMPLETION; 3142 break; 3143 default: 3144 case 192 ... 223: 3145 DPRINTFN(5, "error %x", 3146 XHCI_TRB_2_ERROR_GET(trb->trb_2), 0, 0, 0); 3147 err = USBD_IOERROR; 3148 break; 3149 case 224 ... 255: 3150 err = USBD_NORMAL_COMPLETION; 3151 break; 3152 } 3153 3154 timedout: 3155 sc->sc_resultpending = false; 3156 sc->sc_command_addr = 0; 3157 cv_broadcast(&sc->sc_cmdbusy_cv); 3158 3159 return err; 3160 } 3161 3162 static usbd_status 3163 xhci_do_command(struct xhci_softc * const sc, struct xhci_soft_trb * const trb, 3164 int timeout) 3165 { 3166 3167 mutex_enter(&sc->sc_lock); 3168 usbd_status ret = xhci_do_command_locked(sc, trb, timeout); 3169 mutex_exit(&sc->sc_lock); 3170 3171 return ret; 3172 } 3173 3174 static usbd_status 3175 xhci_enable_slot(struct xhci_softc * const sc, uint8_t * const slotp) 3176 { 3177 struct xhci_soft_trb trb; 3178 usbd_status err; 3179 3180 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 3181 3182 trb.trb_0 = 0; 3183 trb.trb_2 = 0; 3184 trb.trb_3 = XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_ENABLE_SLOT); 3185 3186 err = xhci_do_command(sc, &trb, USBD_DEFAULT_TIMEOUT); 3187 if (err != USBD_NORMAL_COMPLETION) { 3188 return err; 3189 } 3190 3191 *slotp = XHCI_TRB_3_SLOT_GET(trb.trb_3); 3192 3193 return err; 3194 } 3195 3196 /* 3197 * xHCI 4.6.4 3198 * Deallocate ring and device/input context DMA buffers, and disable_slot. 3199 * All endpoints in the slot should be stopped. 3200 * Should be called with sc_lock held. 3201 */ 3202 static usbd_status 3203 xhci_disable_slot(struct xhci_softc * const sc, uint8_t slot) 3204 { 3205 struct xhci_soft_trb trb; 3206 struct xhci_slot *xs; 3207 usbd_status err; 3208 3209 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 3210 3211 if (sc->sc_dying) 3212 return USBD_IOERROR; 3213 3214 trb.trb_0 = 0; 3215 trb.trb_2 = 0; 3216 trb.trb_3 = XHCI_TRB_3_SLOT_SET(slot) | 3217 XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_DISABLE_SLOT); 3218 3219 err = xhci_do_command_locked(sc, &trb, USBD_DEFAULT_TIMEOUT); 3220 3221 if (!err) { 3222 xs = &sc->sc_slots[slot]; 3223 if (xs->xs_idx != 0) { 3224 xhci_free_slot(sc, xs, XHCI_DCI_SLOT + 1, 32); 3225 xhci_set_dcba(sc, 0, slot); 3226 memset(xs, 0, sizeof(*xs)); 3227 } 3228 } 3229 3230 return err; 3231 } 3232 3233 /* 3234 * Set address of device and transition slot state from ENABLED to ADDRESSED 3235 * if Block Setaddress Request (BSR) is false. 3236 * If BSR==true, transition slot state from ENABLED to DEFAULT. 3237 * see xHCI 1.1 4.5.3, 3.3.4 3238 * Should be called without sc_lock held. 3239 */ 3240 static usbd_status 3241 xhci_address_device(struct xhci_softc * const sc, 3242 uint64_t icp, uint8_t slot_id, bool bsr) 3243 { 3244 struct xhci_soft_trb trb; 3245 usbd_status err; 3246 3247 XHCIHIST_FUNC(); 3248 if (bsr) { 3249 XHCIHIST_CALLARGS("icp %jx slot %jx with bsr", 3250 icp, slot_id, 0, 0); 3251 } else { 3252 XHCIHIST_CALLARGS("icp %jx slot %jx nobsr", 3253 icp, slot_id, 0, 0); 3254 } 3255 3256 trb.trb_0 = icp; 3257 trb.trb_2 = 0; 3258 trb.trb_3 = XHCI_TRB_3_SLOT_SET(slot_id) | 3259 XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_ADDRESS_DEVICE) | 3260 (bsr ? XHCI_TRB_3_BSR_BIT : 0); 3261 3262 err = xhci_do_command(sc, &trb, USBD_DEFAULT_TIMEOUT); 3263 3264 if (XHCI_TRB_2_ERROR_GET(trb.trb_2) == XHCI_TRB_ERROR_NO_SLOTS) 3265 err = USBD_NO_ADDR; 3266 3267 return err; 3268 } 3269 3270 static usbd_status 3271 xhci_update_ep0_mps(struct xhci_softc * const sc, 3272 struct xhci_slot * const xs, u_int mps) 3273 { 3274 struct xhci_soft_trb trb; 3275 usbd_status err; 3276 uint32_t * cp; 3277 3278 XHCIHIST_FUNC(); 3279 XHCIHIST_CALLARGS("slot %ju mps %ju", xs->xs_idx, mps, 0, 0); 3280 3281 cp = xhci_slot_get_icv(sc, xs, XHCI_ICI_INPUT_CONTROL); 3282 cp[0] = htole32(0); 3283 cp[1] = htole32(XHCI_INCTX_1_ADD_MASK(XHCI_DCI_EP_CONTROL)); 3284 3285 cp = xhci_slot_get_icv(sc, xs, xhci_dci_to_ici(XHCI_DCI_EP_CONTROL)); 3286 cp[1] = htole32(XHCI_EPCTX_1_MAXP_SIZE_SET(mps)); 3287 3288 /* sync input contexts before they are read from memory */ 3289 usb_syncmem(&xs->xs_ic_dma, 0, sc->sc_pgsz, BUS_DMASYNC_PREWRITE); 3290 HEXDUMP("input context", xhci_slot_get_icv(sc, xs, 0), 3291 sc->sc_ctxsz * 4); 3292 3293 trb.trb_0 = xhci_slot_get_icp(sc, xs, 0); 3294 trb.trb_2 = 0; 3295 trb.trb_3 = XHCI_TRB_3_SLOT_SET(xs->xs_idx) | 3296 XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_EVALUATE_CTX); 3297 3298 err = xhci_do_command(sc, &trb, USBD_DEFAULT_TIMEOUT); 3299 return err; 3300 } 3301 3302 static void 3303 xhci_set_dcba(struct xhci_softc * const sc, uint64_t dcba, int si) 3304 { 3305 uint64_t * const dcbaa = KERNADDR(&sc->sc_dcbaa_dma, 0); 3306 3307 XHCIHIST_FUNC(); 3308 XHCIHIST_CALLARGS("dcbaa %#jx dc %016jx slot %jd", 3309 (uintptr_t)&dcbaa[si], dcba, si, 0); 3310 3311 dcbaa[si] = htole64(dcba); 3312 usb_syncmem(&sc->sc_dcbaa_dma, si * sizeof(uint64_t), sizeof(uint64_t), 3313 BUS_DMASYNC_PREWRITE); 3314 } 3315 3316 /* 3317 * Allocate device and input context DMA buffer, and 3318 * TRB DMA buffer for each endpoint. 3319 */ 3320 static usbd_status 3321 xhci_init_slot(struct usbd_device *dev, uint32_t slot) 3322 { 3323 struct xhci_softc * const sc = XHCI_BUS2SC(dev->ud_bus); 3324 struct xhci_slot *xs; 3325 usbd_status err; 3326 u_int dci; 3327 3328 XHCIHIST_FUNC(); 3329 XHCIHIST_CALLARGS("slot %ju", slot, 0, 0, 0); 3330 3331 xs = &sc->sc_slots[slot]; 3332 3333 /* allocate contexts */ 3334 err = usb_allocmem(&sc->sc_bus, sc->sc_pgsz, sc->sc_pgsz, 3335 &xs->xs_dc_dma); 3336 if (err) 3337 return err; 3338 memset(KERNADDR(&xs->xs_dc_dma, 0), 0, sc->sc_pgsz); 3339 3340 err = usb_allocmem(&sc->sc_bus, sc->sc_pgsz, sc->sc_pgsz, 3341 &xs->xs_ic_dma); 3342 if (err) 3343 goto bad1; 3344 memset(KERNADDR(&xs->xs_ic_dma, 0), 0, sc->sc_pgsz); 3345 3346 for (dci = 0; dci < 32; dci++) { 3347 //CTASSERT(sizeof(xs->xs_ep[dci]) == sizeof(struct xhci_endpoint)); 3348 memset(&xs->xs_ep[dci], 0, sizeof(xs->xs_ep[dci])); 3349 if (dci == XHCI_DCI_SLOT) 3350 continue; 3351 err = xhci_ring_init(sc, &xs->xs_ep[dci].xe_tr, 3352 XHCI_TRANSFER_RING_TRBS, XHCI_TRB_ALIGN); 3353 if (err) { 3354 DPRINTFN(0, "ring init failure", 0, 0, 0, 0); 3355 goto bad2; 3356 } 3357 } 3358 3359 bad2: 3360 if (err == USBD_NORMAL_COMPLETION) { 3361 xs->xs_idx = slot; 3362 } else { 3363 xhci_free_slot(sc, xs, XHCI_DCI_SLOT + 1, dci); 3364 } 3365 3366 return err; 3367 3368 bad1: 3369 usb_freemem(&sc->sc_bus, &xs->xs_dc_dma); 3370 xs->xs_idx = 0; 3371 return err; 3372 } 3373 3374 static void 3375 xhci_free_slot(struct xhci_softc *sc, struct xhci_slot *xs, int start_dci, 3376 int end_dci) 3377 { 3378 u_int dci; 3379 3380 XHCIHIST_FUNC(); 3381 XHCIHIST_CALLARGS("slot %ju start %ju end %ju", 3382 xs->xs_idx, start_dci, end_dci, 0); 3383 3384 for (dci = start_dci; dci < end_dci; dci++) { 3385 xhci_ring_free(sc, &xs->xs_ep[dci].xe_tr); 3386 memset(&xs->xs_ep[dci], 0, sizeof(xs->xs_ep[dci])); 3387 } 3388 usb_freemem(&sc->sc_bus, &xs->xs_ic_dma); 3389 usb_freemem(&sc->sc_bus, &xs->xs_dc_dma); 3390 xs->xs_idx = 0; 3391 } 3392 3393 /* 3394 * Setup slot context, set Device Context Base Address, and issue 3395 * Set Address Device command. 3396 */ 3397 static usbd_status 3398 xhci_set_address(struct usbd_device *dev, uint32_t slot, bool bsr) 3399 { 3400 struct xhci_softc * const sc = XHCI_BUS2SC(dev->ud_bus); 3401 struct xhci_slot *xs; 3402 usbd_status err; 3403 3404 XHCIHIST_FUNC(); 3405 XHCIHIST_CALLARGS("slot %ju bsr %ju", slot, bsr, 0, 0); 3406 3407 xs = &sc->sc_slots[slot]; 3408 3409 xhci_setup_ctx(dev->ud_pipe0); 3410 3411 HEXDUMP("input context", xhci_slot_get_icv(sc, xs, 0), 3412 sc->sc_ctxsz * 3); 3413 3414 xhci_set_dcba(sc, DMAADDR(&xs->xs_dc_dma, 0), slot); 3415 3416 err = xhci_address_device(sc, xhci_slot_get_icp(sc, xs, 0), slot, bsr); 3417 3418 usb_syncmem(&xs->xs_dc_dma, 0, sc->sc_pgsz, BUS_DMASYNC_POSTREAD); 3419 HEXDUMP("output context", xhci_slot_get_dcv(sc, xs, 0), 3420 sc->sc_ctxsz * 2); 3421 3422 return err; 3423 } 3424 3425 /* 3426 * 4.8.2, 6.2.3.2 3427 * construct slot/endpoint context parameters and do syncmem 3428 */ 3429 static void 3430 xhci_setup_ctx(struct usbd_pipe *pipe) 3431 { 3432 struct xhci_softc * const sc = XHCI_PIPE2SC(pipe); 3433 struct usbd_device *dev = pipe->up_dev; 3434 struct xhci_slot * const xs = dev->ud_hcpriv; 3435 usb_endpoint_descriptor_t * const ed = pipe->up_endpoint->ue_edesc; 3436 const u_int dci = xhci_ep_get_dci(ed); 3437 const uint8_t xfertype = UE_GET_XFERTYPE(ed->bmAttributes); 3438 uint32_t *cp; 3439 uint16_t mps = UGETW(ed->wMaxPacketSize); 3440 uint8_t speed = dev->ud_speed; 3441 uint8_t ival = ed->bInterval; 3442 3443 XHCIHIST_FUNC(); 3444 XHCIHIST_CALLARGS("pipe %#jx: slot %ju dci %ju speed %ju", 3445 (uintptr_t)pipe, xs->xs_idx, dci, speed); 3446 3447 /* set up initial input control context */ 3448 cp = xhci_slot_get_icv(sc, xs, XHCI_ICI_INPUT_CONTROL); 3449 cp[0] = htole32(0); 3450 cp[1] = htole32(XHCI_INCTX_1_ADD_MASK(dci)); 3451 cp[1] |= htole32(XHCI_INCTX_1_ADD_MASK(XHCI_DCI_SLOT)); 3452 cp[7] = htole32(0); 3453 3454 /* set up input slot context */ 3455 cp = xhci_slot_get_icv(sc, xs, xhci_dci_to_ici(XHCI_DCI_SLOT)); 3456 cp[0] = 3457 XHCI_SCTX_0_CTX_NUM_SET(dci) | 3458 XHCI_SCTX_0_SPEED_SET(xhci_speed2xspeed(speed)); 3459 cp[1] = 0; 3460 cp[2] = XHCI_SCTX_2_IRQ_TARGET_SET(0); 3461 cp[3] = 0; 3462 xhci_setup_route(pipe, cp); 3463 xhci_setup_tthub(pipe, cp); 3464 3465 cp[0] = htole32(cp[0]); 3466 cp[1] = htole32(cp[1]); 3467 cp[2] = htole32(cp[2]); 3468 cp[3] = htole32(cp[3]); 3469 3470 /* set up input endpoint context */ 3471 cp = xhci_slot_get_icv(sc, xs, xhci_dci_to_ici(dci)); 3472 cp[0] = 3473 XHCI_EPCTX_0_EPSTATE_SET(0) | 3474 XHCI_EPCTX_0_MULT_SET(0) | 3475 XHCI_EPCTX_0_MAXP_STREAMS_SET(0) | 3476 XHCI_EPCTX_0_LSA_SET(0) | 3477 XHCI_EPCTX_0_MAX_ESIT_PAYLOAD_HI_SET(0); 3478 cp[1] = 3479 XHCI_EPCTX_1_EPTYPE_SET(xhci_ep_get_type(ed)) | 3480 XHCI_EPCTX_1_HID_SET(0) | 3481 XHCI_EPCTX_1_MAXB_SET(0); 3482 3483 if (xfertype != UE_ISOCHRONOUS) 3484 cp[1] |= XHCI_EPCTX_1_CERR_SET(3); 3485 3486 if (xfertype == UE_CONTROL) 3487 cp[4] = XHCI_EPCTX_4_AVG_TRB_LEN_SET(8); /* 6.2.3 */ 3488 else if (USB_IS_SS(speed)) 3489 cp[4] = XHCI_EPCTX_4_AVG_TRB_LEN_SET(mps); 3490 else 3491 cp[4] = XHCI_EPCTX_4_AVG_TRB_LEN_SET(UE_GET_SIZE(mps)); 3492 3493 xhci_setup_maxburst(pipe, cp); 3494 3495 switch (xfertype) { 3496 case UE_CONTROL: 3497 break; 3498 case UE_BULK: 3499 /* XXX Set MaxPStreams, HID, and LSA if streams enabled */ 3500 break; 3501 case UE_INTERRUPT: 3502 if (pipe->up_interval != USBD_DEFAULT_INTERVAL) 3503 ival = pipe->up_interval; 3504 3505 ival = xhci_bival2ival(ival, speed); 3506 cp[0] |= XHCI_EPCTX_0_IVAL_SET(ival); 3507 break; 3508 case UE_ISOCHRONOUS: 3509 if (pipe->up_interval != USBD_DEFAULT_INTERVAL) 3510 ival = pipe->up_interval; 3511 3512 /* xHCI 6.2.3.6 Table 65, USB 2.0 9.6.6 */ 3513 if (speed == USB_SPEED_FULL) 3514 ival += 3; /* 1ms -> 125us */ 3515 ival--; 3516 cp[0] |= XHCI_EPCTX_0_IVAL_SET(ival); 3517 break; 3518 default: 3519 break; 3520 } 3521 DPRINTFN(4, "setting ival %ju MaxBurst %#jx", 3522 XHCI_EPCTX_0_IVAL_GET(cp[0]), XHCI_EPCTX_1_MAXB_GET(cp[1]), 0, 0); 3523 3524 /* rewind TR dequeue pointer in xHC */ 3525 /* can't use xhci_ep_get_dci() yet? */ 3526 *(uint64_t *)(&cp[2]) = htole64( 3527 xhci_ring_trbp(&xs->xs_ep[dci].xe_tr, 0) | 3528 XHCI_EPCTX_2_DCS_SET(1)); 3529 3530 cp[0] = htole32(cp[0]); 3531 cp[1] = htole32(cp[1]); 3532 cp[4] = htole32(cp[4]); 3533 3534 /* rewind TR dequeue pointer in driver */ 3535 struct xhci_ring *xr = &xs->xs_ep[dci].xe_tr; 3536 mutex_enter(&xr->xr_lock); 3537 xhci_host_dequeue(xr); 3538 mutex_exit(&xr->xr_lock); 3539 3540 /* sync input contexts before they are read from memory */ 3541 usb_syncmem(&xs->xs_ic_dma, 0, sc->sc_pgsz, BUS_DMASYNC_PREWRITE); 3542 } 3543 3544 /* 3545 * Setup route string and roothub port of given device for slot context 3546 */ 3547 static void 3548 xhci_setup_route(struct usbd_pipe *pipe, uint32_t *cp) 3549 { 3550 struct xhci_softc * const sc = XHCI_PIPE2SC(pipe); 3551 struct usbd_device *dev = pipe->up_dev; 3552 struct usbd_port *up = dev->ud_powersrc; 3553 struct usbd_device *hub; 3554 struct usbd_device *adev; 3555 uint8_t rhport = 0; 3556 uint32_t route = 0; 3557 3558 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 3559 3560 /* Locate root hub port and Determine route string */ 3561 /* 4.3.3 route string does not include roothub port */ 3562 for (hub = dev; hub != NULL; hub = hub->ud_myhub) { 3563 uint32_t dep; 3564 3565 DPRINTFN(4, "hub %#jx depth %jd upport %jp upportno %jd", 3566 (uintptr_t)hub, hub->ud_depth, (uintptr_t)hub->ud_powersrc, 3567 hub->ud_powersrc ? (uintptr_t)hub->ud_powersrc->up_portno : 3568 -1); 3569 3570 if (hub->ud_powersrc == NULL) 3571 break; 3572 dep = hub->ud_depth; 3573 if (dep == 0) 3574 break; 3575 rhport = hub->ud_powersrc->up_portno; 3576 if (dep > USB_HUB_MAX_DEPTH) 3577 continue; 3578 3579 route |= 3580 (rhport > UHD_SS_NPORTS_MAX ? UHD_SS_NPORTS_MAX : rhport) 3581 << ((dep - 1) * 4); 3582 } 3583 route = route >> 4; 3584 size_t bn = hub == sc->sc_bus.ub_roothub ? 0 : 1; 3585 3586 /* Locate port on upstream high speed hub */ 3587 for (adev = dev, hub = up->up_parent; 3588 hub != NULL && hub->ud_speed != USB_SPEED_HIGH; 3589 adev = hub, hub = hub->ud_myhub) 3590 ; 3591 if (hub) { 3592 int p; 3593 for (p = 0; p < hub->ud_hub->uh_hubdesc.bNbrPorts; p++) { 3594 if (hub->ud_hub->uh_ports[p].up_dev == adev) { 3595 dev->ud_myhsport = &hub->ud_hub->uh_ports[p]; 3596 goto found; 3597 } 3598 } 3599 panic("%s: cannot find HS port", __func__); 3600 found: 3601 DPRINTFN(4, "high speed port %jd", p, 0, 0, 0); 3602 } else { 3603 dev->ud_myhsport = NULL; 3604 } 3605 3606 const size_t ctlrport = xhci_rhport2ctlrport(sc, bn, rhport); 3607 3608 DPRINTFN(4, "rhport %ju ctlrport %ju Route %05jx hub %#jx", rhport, 3609 ctlrport, route, (uintptr_t)hub); 3610 3611 cp[0] |= XHCI_SCTX_0_ROUTE_SET(route); 3612 cp[1] |= XHCI_SCTX_1_RH_PORT_SET(ctlrport); 3613 } 3614 3615 /* 3616 * Setup whether device is hub, whether device uses MTT, and 3617 * TT informations if it uses MTT. 3618 */ 3619 static void 3620 xhci_setup_tthub(struct usbd_pipe *pipe, uint32_t *cp) 3621 { 3622 struct usbd_device *dev = pipe->up_dev; 3623 struct usbd_port *myhsport = dev->ud_myhsport; 3624 usb_device_descriptor_t * const dd = &dev->ud_ddesc; 3625 uint32_t speed = dev->ud_speed; 3626 uint8_t rhaddr = dev->ud_bus->ub_rhaddr; 3627 uint8_t tthubslot, ttportnum; 3628 bool ishub; 3629 bool usemtt; 3630 3631 XHCIHIST_FUNC(); 3632 3633 /* 3634 * 6.2.2, Table 57-60, 6.2.2.1, 6.2.2.2 3635 * tthubslot: 3636 * This is the slot ID of parent HS hub 3637 * if LS/FS device is connected && connected through HS hub. 3638 * This is 0 if device is not LS/FS device || 3639 * parent hub is not HS hub || 3640 * attached to root hub. 3641 * ttportnum: 3642 * This is the downstream facing port of parent HS hub 3643 * if LS/FS device is connected. 3644 * This is 0 if device is not LS/FS device || 3645 * parent hub is not HS hub || 3646 * attached to root hub. 3647 */ 3648 if (myhsport && 3649 myhsport->up_parent->ud_addr != rhaddr && 3650 (speed == USB_SPEED_LOW || speed == USB_SPEED_FULL)) { 3651 ttportnum = myhsport->up_portno; 3652 tthubslot = myhsport->up_parent->ud_addr; 3653 } else { 3654 ttportnum = 0; 3655 tthubslot = 0; 3656 } 3657 XHCIHIST_CALLARGS("myhsport %#jx ttportnum=%jd tthubslot=%jd", 3658 (uintptr_t)myhsport, ttportnum, tthubslot, 0); 3659 3660 /* ishub is valid after reading UDESC_DEVICE */ 3661 ishub = (dd->bDeviceClass == UDCLASS_HUB); 3662 3663 /* dev->ud_hub is valid after reading UDESC_HUB */ 3664 if (ishub && dev->ud_hub) { 3665 usb_hub_descriptor_t *hd = &dev->ud_hub->uh_hubdesc; 3666 uint8_t ttt = 3667 __SHIFTOUT(UGETW(hd->wHubCharacteristics), UHD_TT_THINK); 3668 3669 cp[1] |= XHCI_SCTX_1_NUM_PORTS_SET(hd->bNbrPorts); 3670 cp[2] |= XHCI_SCTX_2_TT_THINK_TIME_SET(ttt); 3671 DPRINTFN(4, "nports=%jd ttt=%jd", hd->bNbrPorts, ttt, 0, 0); 3672 } 3673 3674 #define IS_MTTHUB(dd) \ 3675 ((dd)->bDeviceProtocol == UDPROTO_HSHUBMTT) 3676 3677 /* 3678 * MTT flag is set if 3679 * 1. this is HS hub && MTTs are supported and enabled; or 3680 * 2. this is LS or FS device && there is a parent HS hub where MTTs 3681 * are supported and enabled. 3682 * 3683 * XXX enabled is not tested yet 3684 */ 3685 if (ishub && speed == USB_SPEED_HIGH && IS_MTTHUB(dd)) 3686 usemtt = true; 3687 else if ((speed == USB_SPEED_LOW || speed == USB_SPEED_FULL) && 3688 myhsport && 3689 myhsport->up_parent->ud_addr != rhaddr && 3690 IS_MTTHUB(&myhsport->up_parent->ud_ddesc)) 3691 usemtt = true; 3692 else 3693 usemtt = false; 3694 DPRINTFN(4, "class %ju proto %ju ishub %jd usemtt %jd", 3695 dd->bDeviceClass, dd->bDeviceProtocol, ishub, usemtt); 3696 3697 #undef IS_MTTHUB 3698 3699 cp[0] |= 3700 XHCI_SCTX_0_HUB_SET(ishub ? 1 : 0) | 3701 XHCI_SCTX_0_MTT_SET(usemtt ? 1 : 0); 3702 cp[2] |= 3703 XHCI_SCTX_2_TT_HUB_SID_SET(tthubslot) | 3704 XHCI_SCTX_2_TT_PORT_NUM_SET(ttportnum); 3705 } 3706 3707 /* set up params for periodic endpoint */ 3708 static void 3709 xhci_setup_maxburst(struct usbd_pipe *pipe, uint32_t *cp) 3710 { 3711 struct usbd_device *dev = pipe->up_dev; 3712 usb_endpoint_descriptor_t * const ed = pipe->up_endpoint->ue_edesc; 3713 const uint8_t xfertype = UE_GET_XFERTYPE(ed->bmAttributes); 3714 usbd_desc_iter_t iter; 3715 const usb_cdc_descriptor_t *cdcd; 3716 uint32_t maxb = 0; 3717 uint16_t mps = UGETW(ed->wMaxPacketSize); 3718 uint8_t speed = dev->ud_speed; 3719 uint8_t ep; 3720 3721 /* config desc is NULL when opening ep0 */ 3722 if (dev == NULL || dev->ud_cdesc == NULL) 3723 goto no_cdcd; 3724 cdcd = (const usb_cdc_descriptor_t *)usb_find_desc(dev, 3725 UDESC_INTERFACE, USBD_CDCSUBTYPE_ANY); 3726 if (cdcd == NULL) 3727 goto no_cdcd; 3728 usb_desc_iter_init(dev, &iter); 3729 iter.cur = (const void *)cdcd; 3730 3731 /* find endpoint_ss_comp desc for ep of this pipe */ 3732 for (ep = 0;;) { 3733 cdcd = (const usb_cdc_descriptor_t *)usb_desc_iter_next(&iter); 3734 if (cdcd == NULL) 3735 break; 3736 if (ep == 0 && cdcd->bDescriptorType == UDESC_ENDPOINT) { 3737 ep = ((const usb_endpoint_descriptor_t *)cdcd)-> 3738 bEndpointAddress; 3739 if (UE_GET_ADDR(ep) == 3740 UE_GET_ADDR(ed->bEndpointAddress)) { 3741 cdcd = (const usb_cdc_descriptor_t *) 3742 usb_desc_iter_next(&iter); 3743 break; 3744 } 3745 ep = 0; 3746 } 3747 } 3748 if (cdcd != NULL && cdcd->bDescriptorType == UDESC_ENDPOINT_SS_COMP) { 3749 const usb_endpoint_ss_comp_descriptor_t * esscd = 3750 (const usb_endpoint_ss_comp_descriptor_t *)cdcd; 3751 maxb = esscd->bMaxBurst; 3752 } 3753 3754 no_cdcd: 3755 /* 6.2.3.4, 4.8.2.4 */ 3756 if (USB_IS_SS(speed)) { 3757 /* USB 3.1 9.6.6 */ 3758 cp[1] |= XHCI_EPCTX_1_MAXP_SIZE_SET(mps); 3759 /* USB 3.1 9.6.7 */ 3760 cp[1] |= XHCI_EPCTX_1_MAXB_SET(maxb); 3761 #ifdef notyet 3762 if (xfertype == UE_ISOCHRONOUS) { 3763 } 3764 if (XHCI_HCC2_LEC(sc->sc_hcc2) != 0) { 3765 /* use ESIT */ 3766 cp[4] |= XHCI_EPCTX_4_MAX_ESIT_PAYLOAD_SET(x); 3767 cp[0] |= XHCI_EPCTX_0_MAX_ESIT_PAYLOAD_HI_SET(x); 3768 3769 /* XXX if LEC = 1, set ESIT instead */ 3770 cp[0] |= XHCI_EPCTX_0_MULT_SET(0); 3771 } else { 3772 /* use ival */ 3773 } 3774 #endif 3775 } else { 3776 /* USB 2.0 9.6.6 */ 3777 cp[1] |= XHCI_EPCTX_1_MAXP_SIZE_SET(UE_GET_SIZE(mps)); 3778 3779 /* 6.2.3.4 */ 3780 if (speed == USB_SPEED_HIGH && 3781 (xfertype == UE_ISOCHRONOUS || xfertype == UE_INTERRUPT)) { 3782 maxb = UE_GET_TRANS(mps); 3783 } else { 3784 /* LS/FS or HS CTRL or HS BULK */ 3785 maxb = 0; 3786 } 3787 cp[1] |= XHCI_EPCTX_1_MAXB_SET(maxb); 3788 } 3789 } 3790 3791 /* 3792 * Convert endpoint bInterval value to endpoint context interval value 3793 * for Interrupt pipe. 3794 * xHCI 6.2.3.6 Table 65, USB 2.0 9.6.6 3795 */ 3796 static uint32_t 3797 xhci_bival2ival(uint32_t ival, uint32_t speed) 3798 { 3799 if (speed == USB_SPEED_LOW || speed == USB_SPEED_FULL) { 3800 int i; 3801 3802 /* 3803 * round ival down to "the nearest base 2 multiple of 3804 * bInterval * 8". 3805 * bInterval is at most 255 as its type is uByte. 3806 * 255(ms) = 2040(x 125us) < 2^11, so start with 10. 3807 */ 3808 for (i = 10; i > 0; i--) { 3809 if ((ival * 8) >= (1 << i)) 3810 break; 3811 } 3812 ival = i; 3813 } else { 3814 /* Interval = bInterval-1 for SS/HS */ 3815 ival--; 3816 } 3817 3818 return ival; 3819 } 3820 3821 /* ----- */ 3822 3823 static void 3824 xhci_noop(struct usbd_pipe *pipe) 3825 { 3826 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 3827 } 3828 3829 /* 3830 * Process root hub request. 3831 */ 3832 static int 3833 xhci_roothub_ctrl(struct usbd_bus *bus, usb_device_request_t *req, 3834 void *buf, int buflen) 3835 { 3836 struct xhci_softc * const sc = XHCI_BUS2SC(bus); 3837 usb_port_status_t ps; 3838 int l, totlen = 0; 3839 uint16_t len, value, index; 3840 int port, i; 3841 uint32_t v; 3842 3843 XHCIHIST_FUNC(); 3844 3845 if (sc->sc_dying) 3846 return -1; 3847 3848 size_t bn = bus == &sc->sc_bus ? 0 : 1; 3849 3850 len = UGETW(req->wLength); 3851 value = UGETW(req->wValue); 3852 index = UGETW(req->wIndex); 3853 3854 XHCIHIST_CALLARGS("rhreq: %04jx %04jx %04jx %04jx", 3855 req->bmRequestType | (req->bRequest << 8), value, index, len); 3856 3857 #define C(x,y) ((x) | ((y) << 8)) 3858 switch (C(req->bRequest, req->bmRequestType)) { 3859 case C(UR_GET_DESCRIPTOR, UT_READ_DEVICE): 3860 DPRINTFN(8, "getdesc: wValue=0x%04jx", value, 0, 0, 0); 3861 if (len == 0) 3862 break; 3863 switch (value) { 3864 #define sd ((usb_string_descriptor_t *)buf) 3865 case C(2, UDESC_STRING): 3866 /* Product */ 3867 totlen = usb_makestrdesc(sd, len, "xHCI root hub"); 3868 break; 3869 #undef sd 3870 default: 3871 /* default from usbroothub */ 3872 return buflen; 3873 } 3874 break; 3875 3876 /* Hub requests */ 3877 case C(UR_CLEAR_FEATURE, UT_WRITE_CLASS_DEVICE): 3878 break; 3879 /* Clear Port Feature request */ 3880 case C(UR_CLEAR_FEATURE, UT_WRITE_CLASS_OTHER): { 3881 const size_t cp = xhci_rhport2ctlrport(sc, bn, index); 3882 3883 DPRINTFN(4, "UR_CLEAR_PORT_FEAT bp=%jd feat=%jd bus=%jd cp=%jd", 3884 index, value, bn, cp); 3885 if (index < 1 || index > sc->sc_rhportcount[bn]) { 3886 return -1; 3887 } 3888 port = XHCI_PORTSC(cp); 3889 v = xhci_op_read_4(sc, port); 3890 DPRINTFN(4, "portsc=0x%08jx", v, 0, 0, 0); 3891 v &= ~XHCI_PS_CLEAR; 3892 switch (value) { 3893 case UHF_PORT_ENABLE: 3894 xhci_op_write_4(sc, port, v & ~XHCI_PS_PED); 3895 break; 3896 case UHF_PORT_SUSPEND: 3897 return -1; 3898 case UHF_PORT_POWER: 3899 break; 3900 case UHF_PORT_TEST: 3901 case UHF_PORT_INDICATOR: 3902 return -1; 3903 case UHF_C_PORT_CONNECTION: 3904 xhci_op_write_4(sc, port, v | XHCI_PS_CSC); 3905 break; 3906 case UHF_C_PORT_ENABLE: 3907 case UHF_C_PORT_SUSPEND: 3908 case UHF_C_PORT_OVER_CURRENT: 3909 return -1; 3910 case UHF_C_BH_PORT_RESET: 3911 xhci_op_write_4(sc, port, v | XHCI_PS_WRC); 3912 break; 3913 case UHF_C_PORT_RESET: 3914 xhci_op_write_4(sc, port, v | XHCI_PS_PRC); 3915 break; 3916 case UHF_C_PORT_LINK_STATE: 3917 xhci_op_write_4(sc, port, v | XHCI_PS_PLC); 3918 break; 3919 case UHF_C_PORT_CONFIG_ERROR: 3920 xhci_op_write_4(sc, port, v | XHCI_PS_CEC); 3921 break; 3922 default: 3923 return -1; 3924 } 3925 break; 3926 } 3927 case C(UR_GET_DESCRIPTOR, UT_READ_CLASS_DEVICE): 3928 if (len == 0) 3929 break; 3930 if ((value & 0xff) != 0) { 3931 return -1; 3932 } 3933 usb_hub_descriptor_t hubd; 3934 3935 totlen = uimin(buflen, sizeof(hubd)); 3936 memcpy(&hubd, buf, totlen); 3937 hubd.bNbrPorts = sc->sc_rhportcount[bn]; 3938 USETW(hubd.wHubCharacteristics, UHD_PWR_NO_SWITCH); 3939 hubd.bPwrOn2PwrGood = 200; 3940 for (i = 0, l = sc->sc_rhportcount[bn]; l > 0; i++, l -= 8) { 3941 /* XXX can't find out? */ 3942 hubd.DeviceRemovable[i++] = 0; 3943 } 3944 hubd.bDescLength = USB_HUB_DESCRIPTOR_SIZE + i; 3945 totlen = uimin(totlen, hubd.bDescLength); 3946 memcpy(buf, &hubd, totlen); 3947 break; 3948 case C(UR_GET_STATUS, UT_READ_CLASS_DEVICE): 3949 if (len != 4) { 3950 return -1; 3951 } 3952 memset(buf, 0, len); /* ? XXX */ 3953 totlen = len; 3954 break; 3955 /* Get Port Status request */ 3956 case C(UR_GET_STATUS, UT_READ_CLASS_OTHER): { 3957 const size_t cp = xhci_rhport2ctlrport(sc, bn, index); 3958 3959 DPRINTFN(8, "get port status bn=%jd i=%jd cp=%ju", 3960 bn, index, cp, 0); 3961 if (index < 1 || index > sc->sc_rhportcount[bn]) { 3962 DPRINTFN(5, "bad get port status: index=%jd bn=%jd " 3963 "portcount=%jd", 3964 index, bn, sc->sc_rhportcount[bn], 0); 3965 return -1; 3966 } 3967 if (len != 4) { 3968 DPRINTFN(5, "bad get port status: len %d != 4", 3969 len, 0, 0, 0); 3970 return -1; 3971 } 3972 v = xhci_op_read_4(sc, XHCI_PORTSC(cp)); 3973 DPRINTFN(4, "getrhportsc %jd %08jx", cp, v, 0, 0); 3974 i = xhci_xspeed2psspeed(XHCI_PS_SPEED_GET(v)); 3975 if (v & XHCI_PS_CCS) i |= UPS_CURRENT_CONNECT_STATUS; 3976 if (v & XHCI_PS_PED) i |= UPS_PORT_ENABLED; 3977 if (v & XHCI_PS_OCA) i |= UPS_OVERCURRENT_INDICATOR; 3978 //if (v & XHCI_PS_SUSP) i |= UPS_SUSPEND; 3979 if (v & XHCI_PS_PR) i |= UPS_RESET; 3980 if (v & XHCI_PS_PP) { 3981 if (i & UPS_OTHER_SPEED) 3982 i |= UPS_PORT_POWER_SS; 3983 else 3984 i |= UPS_PORT_POWER; 3985 } 3986 if (i & UPS_OTHER_SPEED) 3987 i |= UPS_PORT_LS_SET(XHCI_PS_PLS_GET(v)); 3988 if (sc->sc_vendor_port_status) 3989 i = sc->sc_vendor_port_status(sc, v, i); 3990 USETW(ps.wPortStatus, i); 3991 i = 0; 3992 if (v & XHCI_PS_CSC) i |= UPS_C_CONNECT_STATUS; 3993 if (v & XHCI_PS_PEC) i |= UPS_C_PORT_ENABLED; 3994 if (v & XHCI_PS_OCC) i |= UPS_C_OVERCURRENT_INDICATOR; 3995 if (v & XHCI_PS_PRC) i |= UPS_C_PORT_RESET; 3996 if (v & XHCI_PS_WRC) i |= UPS_C_BH_PORT_RESET; 3997 if (v & XHCI_PS_PLC) i |= UPS_C_PORT_LINK_STATE; 3998 if (v & XHCI_PS_CEC) i |= UPS_C_PORT_CONFIG_ERROR; 3999 USETW(ps.wPortChange, i); 4000 totlen = uimin(len, sizeof(ps)); 4001 memcpy(buf, &ps, totlen); 4002 DPRINTFN(5, "get port status: wPortStatus %x wPortChange %x " 4003 "totlen %d", 4004 UGETW(ps.wPortStatus), UGETW(ps.wPortChange), totlen, 0); 4005 break; 4006 } 4007 case C(UR_SET_DESCRIPTOR, UT_WRITE_CLASS_DEVICE): 4008 return -1; 4009 case C(UR_SET_HUB_DEPTH, UT_WRITE_CLASS_DEVICE): 4010 break; 4011 case C(UR_SET_FEATURE, UT_WRITE_CLASS_DEVICE): 4012 break; 4013 /* Set Port Feature request */ 4014 case C(UR_SET_FEATURE, UT_WRITE_CLASS_OTHER): { 4015 int optval = (index >> 8) & 0xff; 4016 index &= 0xff; 4017 if (index < 1 || index > sc->sc_rhportcount[bn]) { 4018 return -1; 4019 } 4020 4021 const size_t cp = xhci_rhport2ctlrport(sc, bn, index); 4022 4023 port = XHCI_PORTSC(cp); 4024 v = xhci_op_read_4(sc, port); 4025 DPRINTFN(4, "index %jd cp %jd portsc=0x%08jx", index, cp, v, 0); 4026 v &= ~XHCI_PS_CLEAR; 4027 switch (value) { 4028 case UHF_PORT_ENABLE: 4029 xhci_op_write_4(sc, port, v | XHCI_PS_PED); 4030 break; 4031 case UHF_PORT_SUSPEND: 4032 /* XXX suspend */ 4033 break; 4034 case UHF_PORT_RESET: 4035 v &= ~(XHCI_PS_PED | XHCI_PS_PR); 4036 xhci_op_write_4(sc, port, v | XHCI_PS_PR); 4037 /* Wait for reset to complete. */ 4038 usb_delay_ms(&sc->sc_bus, USB_PORT_ROOT_RESET_DELAY); 4039 if (sc->sc_dying) { 4040 return -1; 4041 } 4042 v = xhci_op_read_4(sc, port); 4043 if (v & XHCI_PS_PR) { 4044 xhci_op_write_4(sc, port, v & ~XHCI_PS_PR); 4045 usb_delay_ms(&sc->sc_bus, 10); 4046 /* XXX */ 4047 } 4048 break; 4049 case UHF_PORT_POWER: 4050 /* XXX power control */ 4051 break; 4052 /* XXX more */ 4053 case UHF_C_PORT_RESET: 4054 xhci_op_write_4(sc, port, v | XHCI_PS_PRC); 4055 break; 4056 case UHF_PORT_U1_TIMEOUT: 4057 if (XHCI_PS_SPEED_GET(v) < XHCI_PS_SPEED_SS) { 4058 return -1; 4059 } 4060 port = XHCI_PORTPMSC(cp); 4061 v = xhci_op_read_4(sc, port); 4062 DPRINTFN(4, "index %jd cp %jd portpmsc=0x%08jx", 4063 index, cp, v, 0); 4064 v &= ~XHCI_PM3_U1TO_SET(0xff); 4065 v |= XHCI_PM3_U1TO_SET(optval); 4066 xhci_op_write_4(sc, port, v); 4067 break; 4068 case UHF_PORT_U2_TIMEOUT: 4069 if (XHCI_PS_SPEED_GET(v) < XHCI_PS_SPEED_SS) { 4070 return -1; 4071 } 4072 port = XHCI_PORTPMSC(cp); 4073 v = xhci_op_read_4(sc, port); 4074 DPRINTFN(4, "index %jd cp %jd portpmsc=0x%08jx", 4075 index, cp, v, 0); 4076 v &= ~XHCI_PM3_U2TO_SET(0xff); 4077 v |= XHCI_PM3_U2TO_SET(optval); 4078 xhci_op_write_4(sc, port, v); 4079 break; 4080 default: 4081 return -1; 4082 } 4083 } 4084 break; 4085 case C(UR_CLEAR_TT_BUFFER, UT_WRITE_CLASS_OTHER): 4086 case C(UR_RESET_TT, UT_WRITE_CLASS_OTHER): 4087 case C(UR_GET_TT_STATE, UT_READ_CLASS_OTHER): 4088 case C(UR_STOP_TT, UT_WRITE_CLASS_OTHER): 4089 break; 4090 default: 4091 /* default from usbroothub */ 4092 return buflen; 4093 } 4094 4095 return totlen; 4096 } 4097 4098 /* root hub interrupt */ 4099 4100 static usbd_status 4101 xhci_root_intr_transfer(struct usbd_xfer *xfer) 4102 { 4103 struct xhci_softc * const sc = XHCI_XFER2SC(xfer); 4104 usbd_status err; 4105 4106 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 4107 4108 /* Insert last in queue. */ 4109 mutex_enter(&sc->sc_lock); 4110 err = usb_insert_transfer(xfer); 4111 mutex_exit(&sc->sc_lock); 4112 if (err) 4113 return err; 4114 4115 /* Pipe isn't running, start first */ 4116 return xhci_root_intr_start(SIMPLEQ_FIRST(&xfer->ux_pipe->up_queue)); 4117 } 4118 4119 /* Wait for roothub port status/change */ 4120 static usbd_status 4121 xhci_root_intr_start(struct usbd_xfer *xfer) 4122 { 4123 struct xhci_softc * const sc = XHCI_XFER2SC(xfer); 4124 const size_t bn = XHCI_XFER2BUS(xfer) == &sc->sc_bus ? 0 : 1; 4125 const bool polling = xhci_polling_p(sc); 4126 4127 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 4128 4129 if (sc->sc_dying) 4130 return USBD_IOERROR; 4131 4132 if (!polling) 4133 mutex_enter(&sc->sc_lock); 4134 KASSERT(sc->sc_intrxfer[bn] == NULL); 4135 sc->sc_intrxfer[bn] = xfer; 4136 xfer->ux_status = USBD_IN_PROGRESS; 4137 if (!polling) 4138 mutex_exit(&sc->sc_lock); 4139 4140 return USBD_IN_PROGRESS; 4141 } 4142 4143 static void 4144 xhci_root_intr_abort(struct usbd_xfer *xfer) 4145 { 4146 struct xhci_softc * const sc = XHCI_XFER2SC(xfer); 4147 const size_t bn = XHCI_XFER2BUS(xfer) == &sc->sc_bus ? 0 : 1; 4148 4149 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 4150 4151 KASSERT(mutex_owned(&sc->sc_lock)); 4152 KASSERT(xfer->ux_pipe->up_intrxfer == xfer); 4153 4154 /* If xfer has already completed, nothing to do here. */ 4155 if (sc->sc_intrxfer[bn] == NULL) 4156 return; 4157 4158 /* 4159 * Otherwise, sc->sc_intrxfer[bn] had better be this transfer. 4160 * Cancel it. 4161 */ 4162 KASSERT(sc->sc_intrxfer[bn] == xfer); 4163 xfer->ux_status = USBD_CANCELLED; 4164 usb_transfer_complete(xfer); 4165 } 4166 4167 static void 4168 xhci_root_intr_close(struct usbd_pipe *pipe) 4169 { 4170 struct xhci_softc * const sc __diagused = XHCI_PIPE2SC(pipe); 4171 const struct usbd_xfer *xfer __diagused = pipe->up_intrxfer; 4172 const size_t bn __diagused = XHCI_XFER2BUS(xfer) == &sc->sc_bus ? 0 : 1; 4173 4174 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 4175 4176 KASSERT(mutex_owned(&sc->sc_lock)); 4177 4178 /* 4179 * Caller must guarantee the xfer has completed first, by 4180 * closing the pipe only after normal completion or an abort. 4181 */ 4182 KASSERT(sc->sc_intrxfer[bn] == NULL); 4183 } 4184 4185 static void 4186 xhci_root_intr_done(struct usbd_xfer *xfer) 4187 { 4188 struct xhci_softc * const sc = XHCI_XFER2SC(xfer); 4189 const size_t bn = XHCI_XFER2BUS(xfer) == &sc->sc_bus ? 0 : 1; 4190 4191 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 4192 4193 KASSERT(mutex_owned(&sc->sc_lock)); 4194 4195 /* Claim the xfer so it doesn't get completed again. */ 4196 KASSERT(sc->sc_intrxfer[bn] == xfer); 4197 KASSERT(xfer->ux_status != USBD_IN_PROGRESS); 4198 sc->sc_intrxfer[bn] = NULL; 4199 } 4200 4201 /* -------------- */ 4202 /* device control */ 4203 4204 static usbd_status 4205 xhci_device_ctrl_transfer(struct usbd_xfer *xfer) 4206 { 4207 struct xhci_softc * const sc = XHCI_XFER2SC(xfer); 4208 usbd_status err; 4209 4210 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 4211 4212 /* Insert last in queue. */ 4213 mutex_enter(&sc->sc_lock); 4214 err = usb_insert_transfer(xfer); 4215 mutex_exit(&sc->sc_lock); 4216 if (err) 4217 return err; 4218 4219 /* Pipe isn't running, start first */ 4220 return xhci_device_ctrl_start(SIMPLEQ_FIRST(&xfer->ux_pipe->up_queue)); 4221 } 4222 4223 static usbd_status 4224 xhci_device_ctrl_start(struct usbd_xfer *xfer) 4225 { 4226 struct xhci_softc * const sc = XHCI_XFER2SC(xfer); 4227 struct xhci_slot * const xs = xfer->ux_pipe->up_dev->ud_hcpriv; 4228 const u_int dci = xhci_ep_get_dci(xfer->ux_pipe->up_endpoint->ue_edesc); 4229 struct xhci_ring * const tr = &xs->xs_ep[dci].xe_tr; 4230 struct xhci_xfer * const xx = XHCI_XFER2XXFER(xfer); 4231 usb_device_request_t * const req = &xfer->ux_request; 4232 const int isread = usbd_xfer_isread(xfer); 4233 const uint32_t len = UGETW(req->wLength); 4234 usb_dma_t * const dma = &xfer->ux_dmabuf; 4235 uint64_t parameter; 4236 uint32_t status; 4237 uint32_t control; 4238 u_int i; 4239 const bool polling = xhci_polling_p(sc); 4240 4241 XHCIHIST_FUNC(); 4242 XHCIHIST_CALLARGS("req: %04jx %04jx %04jx %04jx", 4243 req->bmRequestType | (req->bRequest << 8), UGETW(req->wValue), 4244 UGETW(req->wIndex), UGETW(req->wLength)); 4245 4246 /* we rely on the bottom bits for extra info */ 4247 KASSERTMSG(((uintptr_t)xfer & 0x3) == 0x0, "xfer %zx", 4248 (uintptr_t) xfer); 4249 4250 KASSERT((xfer->ux_rqflags & URQ_REQUEST) != 0); 4251 4252 i = 0; 4253 4254 /* setup phase */ 4255 memcpy(¶meter, req, sizeof(parameter)); 4256 status = XHCI_TRB_2_IRQ_SET(0) | XHCI_TRB_2_BYTES_SET(sizeof(*req)); 4257 control = ((len == 0) ? XHCI_TRB_3_TRT_NONE : 4258 (isread ? XHCI_TRB_3_TRT_IN : XHCI_TRB_3_TRT_OUT)) | 4259 XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_SETUP_STAGE) | 4260 XHCI_TRB_3_IDT_BIT; 4261 /* we need parameter un-swapped on big endian, so pre-swap it here */ 4262 xhci_soft_trb_put(&xx->xx_trb[i++], htole64(parameter), status, control); 4263 4264 if (len != 0) { 4265 /* data phase */ 4266 parameter = DMAADDR(dma, 0); 4267 KASSERTMSG(len <= 0x10000, "len %d", len); 4268 status = XHCI_TRB_2_IRQ_SET(0) | 4269 XHCI_TRB_2_TDSZ_SET(0) | 4270 XHCI_TRB_2_BYTES_SET(len); 4271 control = (isread ? XHCI_TRB_3_DIR_IN : 0) | 4272 XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_DATA_STAGE) | 4273 (usbd_xfer_isread(xfer) ? XHCI_TRB_3_ISP_BIT : 0) | 4274 XHCI_TRB_3_IOC_BIT; 4275 xhci_soft_trb_put(&xx->xx_trb[i++], parameter, status, control); 4276 } 4277 4278 parameter = 0; 4279 status = XHCI_TRB_2_IRQ_SET(0); 4280 /* the status stage has inverted direction */ 4281 control = ((isread && (len > 0)) ? 0 : XHCI_TRB_3_DIR_IN) | 4282 XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_STATUS_STAGE) | 4283 XHCI_TRB_3_IOC_BIT; 4284 xhci_soft_trb_put(&xx->xx_trb[i++], parameter, status, control); 4285 4286 if (!polling) 4287 mutex_enter(&tr->xr_lock); 4288 xhci_ring_put(sc, tr, xfer, xx->xx_trb, i); 4289 if (!polling) 4290 mutex_exit(&tr->xr_lock); 4291 4292 if (!polling) 4293 mutex_enter(&sc->sc_lock); 4294 xfer->ux_status = USBD_IN_PROGRESS; 4295 xhci_db_write_4(sc, XHCI_DOORBELL(xs->xs_idx), dci); 4296 usbd_xfer_schedule_timeout(xfer); 4297 if (!polling) 4298 mutex_exit(&sc->sc_lock); 4299 4300 return USBD_IN_PROGRESS; 4301 } 4302 4303 static void 4304 xhci_device_ctrl_done(struct usbd_xfer *xfer) 4305 { 4306 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 4307 usb_device_request_t *req = &xfer->ux_request; 4308 int len = UGETW(req->wLength); 4309 int rd = req->bmRequestType & UT_READ; 4310 4311 if (len) 4312 usb_syncmem(&xfer->ux_dmabuf, 0, len, 4313 rd ? BUS_DMASYNC_POSTREAD : BUS_DMASYNC_POSTWRITE); 4314 } 4315 4316 static void 4317 xhci_device_ctrl_abort(struct usbd_xfer *xfer) 4318 { 4319 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 4320 4321 usbd_xfer_abort(xfer); 4322 } 4323 4324 static void 4325 xhci_device_ctrl_close(struct usbd_pipe *pipe) 4326 { 4327 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 4328 4329 xhci_close_pipe(pipe); 4330 } 4331 4332 /* ------------------ */ 4333 /* device isochronous */ 4334 4335 /* ----------- */ 4336 /* device bulk */ 4337 4338 static usbd_status 4339 xhci_device_bulk_transfer(struct usbd_xfer *xfer) 4340 { 4341 struct xhci_softc * const sc = XHCI_XFER2SC(xfer); 4342 usbd_status err; 4343 4344 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 4345 4346 /* Insert last in queue. */ 4347 mutex_enter(&sc->sc_lock); 4348 err = usb_insert_transfer(xfer); 4349 mutex_exit(&sc->sc_lock); 4350 if (err) 4351 return err; 4352 4353 /* 4354 * Pipe isn't running (otherwise err would be USBD_INPROG), 4355 * so start it first. 4356 */ 4357 return xhci_device_bulk_start(SIMPLEQ_FIRST(&xfer->ux_pipe->up_queue)); 4358 } 4359 4360 static usbd_status 4361 xhci_device_bulk_start(struct usbd_xfer *xfer) 4362 { 4363 struct xhci_softc * const sc = XHCI_XFER2SC(xfer); 4364 struct xhci_slot * const xs = xfer->ux_pipe->up_dev->ud_hcpriv; 4365 const u_int dci = xhci_ep_get_dci(xfer->ux_pipe->up_endpoint->ue_edesc); 4366 struct xhci_ring * const tr = &xs->xs_ep[dci].xe_tr; 4367 struct xhci_xfer * const xx = XHCI_XFER2XXFER(xfer); 4368 const uint32_t len = xfer->ux_length; 4369 usb_dma_t * const dma = &xfer->ux_dmabuf; 4370 uint64_t parameter; 4371 uint32_t status; 4372 uint32_t control; 4373 u_int i = 0; 4374 const bool polling = xhci_polling_p(sc); 4375 4376 XHCIHIST_FUNC(); 4377 XHCIHIST_CALLARGS("%#jx slot %ju dci %ju", 4378 (uintptr_t)xfer, xs->xs_idx, dci, 0); 4379 4380 if (sc->sc_dying) 4381 return USBD_IOERROR; 4382 4383 KASSERT((xfer->ux_rqflags & URQ_REQUEST) == 0); 4384 4385 parameter = DMAADDR(dma, 0); 4386 /* 4387 * XXX: (dsl) The physical buffer must not cross a 64k boundary. 4388 * If the user supplied buffer crosses such a boundary then 2 4389 * (or more) TRB should be used. 4390 * If multiple TRB are used the td_size field must be set correctly. 4391 * For v1.0 devices (like ivy bridge) this is the number of usb data 4392 * blocks needed to complete the transfer. 4393 * Setting it to 1 in the last TRB causes an extra zero-length 4394 * data block be sent. 4395 * The earlier documentation differs, I don't know how it behaves. 4396 */ 4397 KASSERTMSG(len <= 0x10000, "len %d", len); 4398 status = XHCI_TRB_2_IRQ_SET(0) | 4399 XHCI_TRB_2_TDSZ_SET(0) | 4400 XHCI_TRB_2_BYTES_SET(len); 4401 control = XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_NORMAL) | 4402 (usbd_xfer_isread(xfer) ? XHCI_TRB_3_ISP_BIT : 0) | 4403 XHCI_TRB_3_IOC_BIT; 4404 xhci_soft_trb_put(&xx->xx_trb[i++], parameter, status, control); 4405 4406 if (!polling) 4407 mutex_enter(&tr->xr_lock); 4408 xhci_ring_put(sc, tr, xfer, xx->xx_trb, i); 4409 if (!polling) 4410 mutex_exit(&tr->xr_lock); 4411 4412 if (!polling) 4413 mutex_enter(&sc->sc_lock); 4414 xfer->ux_status = USBD_IN_PROGRESS; 4415 xhci_db_write_4(sc, XHCI_DOORBELL(xs->xs_idx), dci); 4416 usbd_xfer_schedule_timeout(xfer); 4417 if (!polling) 4418 mutex_exit(&sc->sc_lock); 4419 4420 return USBD_IN_PROGRESS; 4421 } 4422 4423 static void 4424 xhci_device_bulk_done(struct usbd_xfer *xfer) 4425 { 4426 #ifdef USB_DEBUG 4427 struct xhci_slot * const xs = xfer->ux_pipe->up_dev->ud_hcpriv; 4428 const u_int dci = xhci_ep_get_dci(xfer->ux_pipe->up_endpoint->ue_edesc); 4429 #endif 4430 const int isread = usbd_xfer_isread(xfer); 4431 4432 XHCIHIST_FUNC(); 4433 XHCIHIST_CALLARGS("%#jx slot %ju dci %ju", 4434 (uintptr_t)xfer, xs->xs_idx, dci, 0); 4435 4436 usb_syncmem(&xfer->ux_dmabuf, 0, xfer->ux_length, 4437 isread ? BUS_DMASYNC_POSTREAD : BUS_DMASYNC_POSTWRITE); 4438 } 4439 4440 static void 4441 xhci_device_bulk_abort(struct usbd_xfer *xfer) 4442 { 4443 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 4444 4445 usbd_xfer_abort(xfer); 4446 } 4447 4448 static void 4449 xhci_device_bulk_close(struct usbd_pipe *pipe) 4450 { 4451 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 4452 4453 xhci_close_pipe(pipe); 4454 } 4455 4456 /* ---------------- */ 4457 /* device interrupt */ 4458 4459 static usbd_status 4460 xhci_device_intr_transfer(struct usbd_xfer *xfer) 4461 { 4462 struct xhci_softc * const sc = XHCI_XFER2SC(xfer); 4463 usbd_status err; 4464 4465 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 4466 4467 /* Insert last in queue. */ 4468 mutex_enter(&sc->sc_lock); 4469 err = usb_insert_transfer(xfer); 4470 mutex_exit(&sc->sc_lock); 4471 if (err) 4472 return err; 4473 4474 /* 4475 * Pipe isn't running (otherwise err would be USBD_INPROG), 4476 * so start it first. 4477 */ 4478 return xhci_device_intr_start(SIMPLEQ_FIRST(&xfer->ux_pipe->up_queue)); 4479 } 4480 4481 static usbd_status 4482 xhci_device_intr_start(struct usbd_xfer *xfer) 4483 { 4484 struct xhci_softc * const sc = XHCI_XFER2SC(xfer); 4485 struct xhci_slot * const xs = xfer->ux_pipe->up_dev->ud_hcpriv; 4486 const u_int dci = xhci_ep_get_dci(xfer->ux_pipe->up_endpoint->ue_edesc); 4487 struct xhci_ring * const tr = &xs->xs_ep[dci].xe_tr; 4488 struct xhci_xfer * const xx = XHCI_XFER2XXFER(xfer); 4489 const uint32_t len = xfer->ux_length; 4490 const bool polling = xhci_polling_p(sc); 4491 usb_dma_t * const dma = &xfer->ux_dmabuf; 4492 uint64_t parameter; 4493 uint32_t status; 4494 uint32_t control; 4495 u_int i = 0; 4496 4497 XHCIHIST_FUNC(); 4498 XHCIHIST_CALLARGS("%#jx slot %ju dci %ju", 4499 (uintptr_t)xfer, xs->xs_idx, dci, 0); 4500 4501 if (sc->sc_dying) 4502 return USBD_IOERROR; 4503 4504 KASSERT((xfer->ux_rqflags & URQ_REQUEST) == 0); 4505 4506 parameter = DMAADDR(dma, 0); 4507 KASSERTMSG(len <= 0x10000, "len %d", len); 4508 status = XHCI_TRB_2_IRQ_SET(0) | 4509 XHCI_TRB_2_TDSZ_SET(0) | 4510 XHCI_TRB_2_BYTES_SET(len); 4511 control = XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_NORMAL) | 4512 (usbd_xfer_isread(xfer) ? XHCI_TRB_3_ISP_BIT : 0) | 4513 XHCI_TRB_3_IOC_BIT; 4514 xhci_soft_trb_put(&xx->xx_trb[i++], parameter, status, control); 4515 4516 if (!polling) 4517 mutex_enter(&tr->xr_lock); 4518 xhci_ring_put(sc, tr, xfer, xx->xx_trb, i); 4519 if (!polling) 4520 mutex_exit(&tr->xr_lock); 4521 4522 if (!polling) 4523 mutex_enter(&sc->sc_lock); 4524 xfer->ux_status = USBD_IN_PROGRESS; 4525 xhci_db_write_4(sc, XHCI_DOORBELL(xs->xs_idx), dci); 4526 usbd_xfer_schedule_timeout(xfer); 4527 if (!polling) 4528 mutex_exit(&sc->sc_lock); 4529 4530 return USBD_IN_PROGRESS; 4531 } 4532 4533 static void 4534 xhci_device_intr_done(struct usbd_xfer *xfer) 4535 { 4536 struct xhci_softc * const sc __diagused = XHCI_XFER2SC(xfer); 4537 #ifdef USB_DEBUG 4538 struct xhci_slot * const xs = xfer->ux_pipe->up_dev->ud_hcpriv; 4539 const u_int dci = xhci_ep_get_dci(xfer->ux_pipe->up_endpoint->ue_edesc); 4540 #endif 4541 const int isread = usbd_xfer_isread(xfer); 4542 4543 XHCIHIST_FUNC(); 4544 XHCIHIST_CALLARGS("%#jx slot %ju dci %ju", 4545 (uintptr_t)xfer, xs->xs_idx, dci, 0); 4546 4547 KASSERT(xhci_polling_p(sc) || mutex_owned(&sc->sc_lock)); 4548 4549 usb_syncmem(&xfer->ux_dmabuf, 0, xfer->ux_length, 4550 isread ? BUS_DMASYNC_POSTREAD : BUS_DMASYNC_POSTWRITE); 4551 } 4552 4553 static void 4554 xhci_device_intr_abort(struct usbd_xfer *xfer) 4555 { 4556 struct xhci_softc * const sc __diagused = XHCI_XFER2SC(xfer); 4557 4558 XHCIHIST_FUNC(); 4559 XHCIHIST_CALLARGS("%#jx", (uintptr_t)xfer, 0, 0, 0); 4560 4561 KASSERT(mutex_owned(&sc->sc_lock)); 4562 usbd_xfer_abort(xfer); 4563 } 4564 4565 static void 4566 xhci_device_intr_close(struct usbd_pipe *pipe) 4567 { 4568 //struct xhci_softc * const sc = XHCI_PIPE2SC(pipe); 4569 4570 XHCIHIST_FUNC(); 4571 XHCIHIST_CALLARGS("%#jx", (uintptr_t)pipe, 0, 0, 0); 4572 4573 xhci_close_pipe(pipe); 4574 } 4575
Indexes created Wed Oct 01 18:09:54 GMT 2025