xhci.c revision 1.48
1 /* $NetBSD: xhci.c,v 1.48 2016/06/05 07:52:16 skrll Exp $ */ 2 3 /* 4 * Copyright (c) 2013 Jonathan A. Kollasch 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 17 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 18 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 19 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR 20 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 21 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 22 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 23 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 24 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 25 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 26 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 */ 28 29 /* 30 * USB rev 2.0 and rev 3.1 specification 31 * http://www.usb.org/developers/docs/ 32 * xHCI rev 1.1 specification 33 * http://www.intel.com/technology/usb/spec.htm 34 */ 35 36 #include <sys/cdefs.h> 37 __KERNEL_RCSID(0, "$NetBSD: xhci.c,v 1.48 2016/06/05 07:52:16 skrll Exp $"); 38 39 #ifdef _KERNEL_OPT 40 #include "opt_usb.h" 41 #endif 42 43 #include <sys/param.h> 44 #include <sys/systm.h> 45 #include <sys/kernel.h> 46 #include <sys/kmem.h> 47 #include <sys/device.h> 48 #include <sys/select.h> 49 #include <sys/proc.h> 50 #include <sys/queue.h> 51 #include <sys/mutex.h> 52 #include <sys/condvar.h> 53 #include <sys/bus.h> 54 #include <sys/cpu.h> 55 #include <sys/sysctl.h> 56 57 #include <machine/endian.h> 58 59 #include <dev/usb/usb.h> 60 #include <dev/usb/usbdi.h> 61 #include <dev/usb/usbdivar.h> 62 #include <dev/usb/usbdi_util.h> 63 #include <dev/usb/usbhist.h> 64 #include <dev/usb/usb_mem.h> 65 #include <dev/usb/usb_quirks.h> 66 67 #include <dev/usb/xhcireg.h> 68 #include <dev/usb/xhcivar.h> 69 #include <dev/usb/usbroothub.h> 70 71 72 #ifdef USB_DEBUG 73 #ifndef XHCI_DEBUG 74 #define xhcidebug 0 75 #else /* !XHCI_DEBUG */ 76 static int xhcidebug = 0; 77 78 SYSCTL_SETUP(sysctl_hw_xhci_setup, "sysctl hw.xhci setup") 79 { 80 int err; 81 const struct sysctlnode *rnode; 82 const struct sysctlnode *cnode; 83 84 err = sysctl_createv(clog, 0, NULL, &rnode, 85 CTLFLAG_PERMANENT, CTLTYPE_NODE, "xhci", 86 SYSCTL_DESCR("xhci global controls"), 87 NULL, 0, NULL, 0, CTL_HW, CTL_CREATE, CTL_EOL); 88 89 if (err) 90 goto fail; 91 92 /* control debugging printfs */ 93 err = sysctl_createv(clog, 0, &rnode, &cnode, 94 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, CTLTYPE_INT, 95 "debug", SYSCTL_DESCR("Enable debugging output"), 96 NULL, 0, &xhcidebug, sizeof(xhcidebug), CTL_CREATE, CTL_EOL); 97 if (err) 98 goto fail; 99 100 return; 101 fail: 102 aprint_error("%s: sysctl_createv failed (err = %d)\n", __func__, err); 103 } 104 105 #endif /* !XHCI_DEBUG */ 106 #endif /* USB_DEBUG */ 107 108 #define DPRINTFN(N,FMT,A,B,C,D) USBHIST_LOGN(xhcidebug,N,FMT,A,B,C,D) 109 #define XHCIHIST_FUNC() USBHIST_FUNC() 110 #define XHCIHIST_CALLED(name) USBHIST_CALLED(xhcidebug) 111 112 #define XHCI_DCI_SLOT 0 113 #define XHCI_DCI_EP_CONTROL 1 114 115 #define XHCI_ICI_INPUT_CONTROL 0 116 117 struct xhci_pipe { 118 struct usbd_pipe xp_pipe; 119 struct usb_task xp_async_task; 120 }; 121 122 #define XHCI_COMMAND_RING_TRBS 256 123 #define XHCI_EVENT_RING_TRBS 256 124 #define XHCI_EVENT_RING_SEGMENTS 1 125 #define XHCI_TRB_3_ED_BIT XHCI_TRB_3_ISP_BIT 126 127 static usbd_status xhci_open(struct usbd_pipe *); 128 static void xhci_close_pipe(struct usbd_pipe *); 129 static int xhci_intr1(struct xhci_softc * const); 130 static void xhci_softintr(void *); 131 static void xhci_poll(struct usbd_bus *); 132 static struct usbd_xfer *xhci_allocx(struct usbd_bus *, unsigned int); 133 static void xhci_freex(struct usbd_bus *, struct usbd_xfer *); 134 static void xhci_get_lock(struct usbd_bus *, kmutex_t **); 135 static usbd_status xhci_new_device(device_t, struct usbd_bus *, int, int, int, 136 struct usbd_port *); 137 static int xhci_roothub_ctrl(struct usbd_bus *, usb_device_request_t *, 138 void *, int); 139 140 static usbd_status xhci_configure_endpoint(struct usbd_pipe *); 141 //static usbd_status xhci_unconfigure_endpoint(struct usbd_pipe *); 142 static usbd_status xhci_reset_endpoint(struct usbd_pipe *); 143 static usbd_status xhci_stop_endpoint(struct usbd_pipe *); 144 145 static usbd_status xhci_set_dequeue(struct usbd_pipe *); 146 147 static usbd_status xhci_do_command(struct xhci_softc * const, 148 struct xhci_trb * const, int); 149 static usbd_status xhci_do_command_locked(struct xhci_softc * const, 150 struct xhci_trb * const, int); 151 static usbd_status xhci_init_slot(struct usbd_device *, uint32_t); 152 static void xhci_free_slot(struct xhci_softc *, struct xhci_slot *, int, int); 153 static usbd_status xhci_set_address(struct usbd_device *, uint32_t, uint32_t, int, bool); 154 static usbd_status xhci_enable_slot(struct xhci_softc * const, 155 uint8_t * const); 156 static usbd_status xhci_disable_slot(struct xhci_softc * const, uint8_t); 157 static usbd_status xhci_address_device(struct xhci_softc * const, 158 uint64_t, uint8_t, bool); 159 static void xhci_set_dcba(struct xhci_softc * const, uint64_t, int); 160 static usbd_status xhci_update_ep0_mps(struct xhci_softc * const, 161 struct xhci_slot * const, u_int); 162 static usbd_status xhci_ring_init(struct xhci_softc * const, 163 struct xhci_ring * const, size_t, size_t); 164 static void xhci_ring_free(struct xhci_softc * const, struct xhci_ring * const); 165 166 static void xhci_noop(struct usbd_pipe *); 167 168 static usbd_status xhci_root_intr_transfer(struct usbd_xfer *); 169 static usbd_status xhci_root_intr_start(struct usbd_xfer *); 170 static void xhci_root_intr_abort(struct usbd_xfer *); 171 static void xhci_root_intr_close(struct usbd_pipe *); 172 static void xhci_root_intr_done(struct usbd_xfer *); 173 174 static usbd_status xhci_device_ctrl_transfer(struct usbd_xfer *); 175 static usbd_status xhci_device_ctrl_start(struct usbd_xfer *); 176 static void xhci_device_ctrl_abort(struct usbd_xfer *); 177 static void xhci_device_ctrl_close(struct usbd_pipe *); 178 static void xhci_device_ctrl_done(struct usbd_xfer *); 179 180 static usbd_status xhci_device_intr_transfer(struct usbd_xfer *); 181 static usbd_status xhci_device_intr_start(struct usbd_xfer *); 182 static void xhci_device_intr_abort(struct usbd_xfer *); 183 static void xhci_device_intr_close(struct usbd_pipe *); 184 static void xhci_device_intr_done(struct usbd_xfer *); 185 186 static usbd_status xhci_device_bulk_transfer(struct usbd_xfer *); 187 static usbd_status xhci_device_bulk_start(struct usbd_xfer *); 188 static void xhci_device_bulk_abort(struct usbd_xfer *); 189 static void xhci_device_bulk_close(struct usbd_pipe *); 190 static void xhci_device_bulk_done(struct usbd_xfer *); 191 192 static void xhci_timeout(void *); 193 static void xhci_timeout_task(void *); 194 195 static const struct usbd_bus_methods xhci_bus_methods = { 196 .ubm_open = xhci_open, 197 .ubm_softint = xhci_softintr, 198 .ubm_dopoll = xhci_poll, 199 .ubm_allocx = xhci_allocx, 200 .ubm_freex = xhci_freex, 201 .ubm_getlock = xhci_get_lock, 202 .ubm_newdev = xhci_new_device, 203 .ubm_rhctrl = xhci_roothub_ctrl, 204 }; 205 206 static const struct usbd_pipe_methods xhci_root_intr_methods = { 207 .upm_transfer = xhci_root_intr_transfer, 208 .upm_start = xhci_root_intr_start, 209 .upm_abort = xhci_root_intr_abort, 210 .upm_close = xhci_root_intr_close, 211 .upm_cleartoggle = xhci_noop, 212 .upm_done = xhci_root_intr_done, 213 }; 214 215 216 static const struct usbd_pipe_methods xhci_device_ctrl_methods = { 217 .upm_transfer = xhci_device_ctrl_transfer, 218 .upm_start = xhci_device_ctrl_start, 219 .upm_abort = xhci_device_ctrl_abort, 220 .upm_close = xhci_device_ctrl_close, 221 .upm_cleartoggle = xhci_noop, 222 .upm_done = xhci_device_ctrl_done, 223 }; 224 225 static const struct usbd_pipe_methods xhci_device_isoc_methods = { 226 .upm_cleartoggle = xhci_noop, 227 }; 228 229 static const struct usbd_pipe_methods xhci_device_bulk_methods = { 230 .upm_transfer = xhci_device_bulk_transfer, 231 .upm_start = xhci_device_bulk_start, 232 .upm_abort = xhci_device_bulk_abort, 233 .upm_close = xhci_device_bulk_close, 234 .upm_cleartoggle = xhci_noop, 235 .upm_done = xhci_device_bulk_done, 236 }; 237 238 static const struct usbd_pipe_methods xhci_device_intr_methods = { 239 .upm_transfer = xhci_device_intr_transfer, 240 .upm_start = xhci_device_intr_start, 241 .upm_abort = xhci_device_intr_abort, 242 .upm_close = xhci_device_intr_close, 243 .upm_cleartoggle = xhci_noop, 244 .upm_done = xhci_device_intr_done, 245 }; 246 247 static inline uint32_t 248 xhci_read_1(const struct xhci_softc * const sc, bus_size_t offset) 249 { 250 return bus_space_read_1(sc->sc_iot, sc->sc_ioh, offset); 251 } 252 253 static inline uint32_t 254 xhci_read_4(const struct xhci_softc * const sc, bus_size_t offset) 255 { 256 return bus_space_read_4(sc->sc_iot, sc->sc_ioh, offset); 257 } 258 259 static inline void 260 xhci_write_1(const struct xhci_softc * const sc, bus_size_t offset, 261 uint32_t value) 262 { 263 bus_space_write_1(sc->sc_iot, sc->sc_ioh, offset, value); 264 } 265 266 #if 0 /* unused */ 267 static inline void 268 xhci_write_4(const struct xhci_softc * const sc, bus_size_t offset, 269 uint32_t value) 270 { 271 bus_space_write_4(sc->sc_iot, sc->sc_ioh, offset, value); 272 } 273 #endif /* unused */ 274 275 static inline uint32_t 276 xhci_cap_read_4(const struct xhci_softc * const sc, bus_size_t offset) 277 { 278 return bus_space_read_4(sc->sc_iot, sc->sc_cbh, offset); 279 } 280 281 static inline uint32_t 282 xhci_op_read_4(const struct xhci_softc * const sc, bus_size_t offset) 283 { 284 return bus_space_read_4(sc->sc_iot, sc->sc_obh, offset); 285 } 286 287 static inline void 288 xhci_op_write_4(const struct xhci_softc * const sc, bus_size_t offset, 289 uint32_t value) 290 { 291 bus_space_write_4(sc->sc_iot, sc->sc_obh, offset, value); 292 } 293 294 static inline uint64_t 295 xhci_op_read_8(const struct xhci_softc * const sc, bus_size_t offset) 296 { 297 uint64_t value; 298 299 if (sc->sc_ac64) { 300 #ifdef XHCI_USE_BUS_SPACE_8 301 value = bus_space_read_8(sc->sc_iot, sc->sc_obh, offset); 302 #else 303 value = bus_space_read_4(sc->sc_iot, sc->sc_obh, offset); 304 value |= (uint64_t)bus_space_read_4(sc->sc_iot, sc->sc_obh, 305 offset + 4) << 32; 306 #endif 307 } else { 308 value = bus_space_read_4(sc->sc_iot, sc->sc_obh, offset); 309 } 310 311 return value; 312 } 313 314 static inline void 315 xhci_op_write_8(const struct xhci_softc * const sc, bus_size_t offset, 316 uint64_t value) 317 { 318 if (sc->sc_ac64) { 319 #ifdef XHCI_USE_BUS_SPACE_8 320 bus_space_write_8(sc->sc_iot, sc->sc_obh, offset, value); 321 #else 322 bus_space_write_4(sc->sc_iot, sc->sc_obh, offset + 0, 323 (value >> 0) & 0xffffffff); 324 bus_space_write_4(sc->sc_iot, sc->sc_obh, offset + 4, 325 (value >> 32) & 0xffffffff); 326 #endif 327 } else { 328 bus_space_write_4(sc->sc_iot, sc->sc_obh, offset, value); 329 } 330 } 331 332 static inline uint32_t 333 xhci_rt_read_4(const struct xhci_softc * const sc, bus_size_t offset) 334 { 335 return bus_space_read_4(sc->sc_iot, sc->sc_rbh, offset); 336 } 337 338 static inline void 339 xhci_rt_write_4(const struct xhci_softc * const sc, bus_size_t offset, 340 uint32_t value) 341 { 342 bus_space_write_4(sc->sc_iot, sc->sc_rbh, offset, value); 343 } 344 345 #if 0 /* unused */ 346 static inline uint64_t 347 xhci_rt_read_8(const struct xhci_softc * const sc, bus_size_t offset) 348 { 349 uint64_t value; 350 351 if (sc->sc_ac64) { 352 #ifdef XHCI_USE_BUS_SPACE_8 353 value = bus_space_read_8(sc->sc_iot, sc->sc_rbh, offset); 354 #else 355 value = bus_space_read_4(sc->sc_iot, sc->sc_rbh, offset); 356 value |= (uint64_t)bus_space_read_4(sc->sc_iot, sc->sc_rbh, 357 offset + 4) << 32; 358 #endif 359 } else { 360 value = bus_space_read_4(sc->sc_iot, sc->sc_rbh, offset); 361 } 362 363 return value; 364 } 365 #endif /* unused */ 366 367 static inline void 368 xhci_rt_write_8(const struct xhci_softc * const sc, bus_size_t offset, 369 uint64_t value) 370 { 371 if (sc->sc_ac64) { 372 #ifdef XHCI_USE_BUS_SPACE_8 373 bus_space_write_8(sc->sc_iot, sc->sc_rbh, offset, value); 374 #else 375 bus_space_write_4(sc->sc_iot, sc->sc_rbh, offset + 0, 376 (value >> 0) & 0xffffffff); 377 bus_space_write_4(sc->sc_iot, sc->sc_rbh, offset + 4, 378 (value >> 32) & 0xffffffff); 379 #endif 380 } else { 381 bus_space_write_4(sc->sc_iot, sc->sc_rbh, offset, value); 382 } 383 } 384 385 #if 0 /* unused */ 386 static inline uint32_t 387 xhci_db_read_4(const struct xhci_softc * const sc, bus_size_t offset) 388 { 389 return bus_space_read_4(sc->sc_iot, sc->sc_dbh, offset); 390 } 391 #endif /* unused */ 392 393 static inline void 394 xhci_db_write_4(const struct xhci_softc * const sc, bus_size_t offset, 395 uint32_t value) 396 { 397 bus_space_write_4(sc->sc_iot, sc->sc_dbh, offset, value); 398 } 399 400 /* --- */ 401 402 static inline uint8_t 403 xhci_ep_get_type(usb_endpoint_descriptor_t * const ed) 404 { 405 u_int eptype = 0; 406 407 switch (UE_GET_XFERTYPE(ed->bmAttributes)) { 408 case UE_CONTROL: 409 eptype = 0x0; 410 break; 411 case UE_ISOCHRONOUS: 412 eptype = 0x1; 413 break; 414 case UE_BULK: 415 eptype = 0x2; 416 break; 417 case UE_INTERRUPT: 418 eptype = 0x3; 419 break; 420 } 421 422 if ((UE_GET_XFERTYPE(ed->bmAttributes) == UE_CONTROL) || 423 (UE_GET_DIR(ed->bEndpointAddress) == UE_DIR_IN)) 424 return eptype | 0x4; 425 else 426 return eptype; 427 } 428 429 static u_int 430 xhci_ep_get_dci(usb_endpoint_descriptor_t * const ed) 431 { 432 /* xHCI 1.0 section 4.5.1 */ 433 u_int epaddr = UE_GET_ADDR(ed->bEndpointAddress); 434 u_int in = 0; 435 436 if ((UE_GET_XFERTYPE(ed->bmAttributes) == UE_CONTROL) || 437 (UE_GET_DIR(ed->bEndpointAddress) == UE_DIR_IN)) 438 in = 1; 439 440 return epaddr * 2 + in; 441 } 442 443 static inline u_int 444 xhci_dci_to_ici(const u_int i) 445 { 446 return i + 1; 447 } 448 449 static inline void * 450 xhci_slot_get_dcv(struct xhci_softc * const sc, struct xhci_slot * const xs, 451 const u_int dci) 452 { 453 return KERNADDR(&xs->xs_dc_dma, sc->sc_ctxsz * dci); 454 } 455 456 #if 0 /* unused */ 457 static inline bus_addr_t 458 xhci_slot_get_dcp(struct xhci_softc * const sc, struct xhci_slot * const xs, 459 const u_int dci) 460 { 461 return DMAADDR(&xs->xs_dc_dma, sc->sc_ctxsz * dci); 462 } 463 #endif /* unused */ 464 465 static inline void * 466 xhci_slot_get_icv(struct xhci_softc * const sc, struct xhci_slot * const xs, 467 const u_int ici) 468 { 469 return KERNADDR(&xs->xs_ic_dma, sc->sc_ctxsz * ici); 470 } 471 472 static inline bus_addr_t 473 xhci_slot_get_icp(struct xhci_softc * const sc, struct xhci_slot * const xs, 474 const u_int ici) 475 { 476 return DMAADDR(&xs->xs_ic_dma, sc->sc_ctxsz * ici); 477 } 478 479 static inline struct xhci_trb * 480 xhci_ring_trbv(struct xhci_ring * const xr, u_int idx) 481 { 482 return KERNADDR(&xr->xr_dma, XHCI_TRB_SIZE * idx); 483 } 484 485 static inline bus_addr_t 486 xhci_ring_trbp(struct xhci_ring * const xr, u_int idx) 487 { 488 return DMAADDR(&xr->xr_dma, XHCI_TRB_SIZE * idx); 489 } 490 491 static inline void 492 xhci_trb_put(struct xhci_trb * const trb, uint64_t parameter, uint32_t status, 493 uint32_t control) 494 { 495 trb->trb_0 = htole64(parameter); 496 trb->trb_2 = htole32(status); 497 trb->trb_3 = htole32(control); 498 } 499 500 static int 501 xhci_trb_get_idx(struct xhci_ring *xr, uint64_t trb_0, int *idx) 502 { 503 /* base address of TRBs */ 504 bus_addr_t trbp = xhci_ring_trbp(xr, 0); 505 506 /* trb_0 range sanity check */ 507 if (trb_0 == 0 || trb_0 < trbp || 508 (trb_0 - trbp) % sizeof(struct xhci_trb) != 0 || 509 (trb_0 - trbp) / sizeof(struct xhci_trb) >= xr->xr_ntrb) { 510 return 1; 511 } 512 *idx = (trb_0 - trbp) / sizeof(struct xhci_trb); 513 return 0; 514 } 515 516 /* --- */ 517 518 void 519 xhci_childdet(device_t self, device_t child) 520 { 521 struct xhci_softc * const sc = device_private(self); 522 523 KASSERT(sc->sc_child == child); 524 if (child == sc->sc_child) 525 sc->sc_child = NULL; 526 } 527 528 int 529 xhci_detach(struct xhci_softc *sc, int flags) 530 { 531 int rv = 0; 532 533 if (sc->sc_child != NULL) 534 rv = config_detach(sc->sc_child, flags); 535 536 if (rv != 0) 537 return rv; 538 539 /* XXX unconfigure/free slots */ 540 541 /* verify: */ 542 xhci_rt_write_4(sc, XHCI_IMAN(0), 0); 543 xhci_op_write_4(sc, XHCI_USBCMD, 0); 544 /* do we need to wait for stop? */ 545 546 xhci_op_write_8(sc, XHCI_CRCR, 0); 547 xhci_ring_free(sc, &sc->sc_cr); 548 cv_destroy(&sc->sc_command_cv); 549 550 xhci_rt_write_4(sc, XHCI_ERSTSZ(0), 0); 551 xhci_rt_write_8(sc, XHCI_ERSTBA(0), 0); 552 xhci_rt_write_8(sc, XHCI_ERDP(0), 0|XHCI_ERDP_LO_BUSY); 553 xhci_ring_free(sc, &sc->sc_er); 554 555 usb_freemem(&sc->sc_bus, &sc->sc_eventst_dma); 556 557 xhci_op_write_8(sc, XHCI_DCBAAP, 0); 558 usb_freemem(&sc->sc_bus, &sc->sc_dcbaa_dma); 559 560 kmem_free(sc->sc_slots, sizeof(*sc->sc_slots) * sc->sc_maxslots); 561 562 mutex_destroy(&sc->sc_lock); 563 mutex_destroy(&sc->sc_intr_lock); 564 cv_destroy(&sc->sc_softwake_cv); 565 566 pool_cache_destroy(sc->sc_xferpool); 567 568 return rv; 569 } 570 571 int 572 xhci_activate(device_t self, enum devact act) 573 { 574 struct xhci_softc * const sc = device_private(self); 575 576 switch (act) { 577 case DVACT_DEACTIVATE: 578 sc->sc_dying = true; 579 return 0; 580 default: 581 return EOPNOTSUPP; 582 } 583 } 584 585 bool 586 xhci_suspend(device_t dv, const pmf_qual_t *qual) 587 { 588 return false; 589 } 590 591 bool 592 xhci_resume(device_t dv, const pmf_qual_t *qual) 593 { 594 return false; 595 } 596 597 bool 598 xhci_shutdown(device_t self, int flags) 599 { 600 return false; 601 } 602 603 static int 604 xhci_hc_reset(struct xhci_softc * const sc) 605 { 606 uint32_t usbcmd, usbsts; 607 int i; 608 609 /* Check controller not ready */ 610 for (i = 0; i < XHCI_WAIT_CNR; i++) { 611 usbsts = xhci_op_read_4(sc, XHCI_USBSTS); 612 if ((usbsts & XHCI_STS_CNR) == 0) 613 break; 614 usb_delay_ms(&sc->sc_bus, 1); 615 } 616 if (i >= XHCI_WAIT_CNR) { 617 aprint_error_dev(sc->sc_dev, "controller not ready timeout\n"); 618 return EIO; 619 } 620 621 /* Halt controller */ 622 usbcmd = 0; 623 xhci_op_write_4(sc, XHCI_USBCMD, usbcmd); 624 usb_delay_ms(&sc->sc_bus, 1); 625 626 /* Reset controller */ 627 usbcmd = XHCI_CMD_HCRST; 628 xhci_op_write_4(sc, XHCI_USBCMD, usbcmd); 629 for (i = 0; i < XHCI_WAIT_HCRST; i++) { 630 usbcmd = xhci_op_read_4(sc, XHCI_USBCMD); 631 if ((usbcmd & XHCI_CMD_HCRST) == 0) 632 break; 633 usb_delay_ms(&sc->sc_bus, 1); 634 } 635 if (i >= XHCI_WAIT_HCRST) { 636 aprint_error_dev(sc->sc_dev, "host controller reset timeout\n"); 637 return EIO; 638 } 639 640 /* Check controller not ready */ 641 for (i = 0; i < XHCI_WAIT_CNR; i++) { 642 usbsts = xhci_op_read_4(sc, XHCI_USBSTS); 643 if ((usbsts & XHCI_STS_CNR) == 0) 644 break; 645 usb_delay_ms(&sc->sc_bus, 1); 646 } 647 if (i >= XHCI_WAIT_CNR) { 648 aprint_error_dev(sc->sc_dev, 649 "controller not ready timeout after reset\n"); 650 return EIO; 651 } 652 653 return 0; 654 } 655 656 657 static void 658 hexdump(const char *msg, const void *base, size_t len) 659 { 660 #if 0 661 size_t cnt; 662 const uint32_t *p; 663 extern paddr_t vtophys(vaddr_t); 664 665 p = base; 666 cnt = 0; 667 668 printf("*** %s (%zu bytes @ %p %p)\n", msg, len, base, 669 (void *)vtophys((vaddr_t)base)); 670 671 while (cnt < len) { 672 if (cnt % 16 == 0) 673 printf("%p: ", p); 674 else if (cnt % 8 == 0) 675 printf(" |"); 676 printf(" %08x", *p++); 677 cnt += 4; 678 if (cnt % 16 == 0) 679 printf("\n"); 680 } 681 if (cnt % 16 != 0) 682 printf("\n"); 683 #endif 684 } 685 686 /* Process extended capabilities */ 687 static void 688 xhci_ecp(struct xhci_softc *sc, uint32_t hcc) 689 { 690 uint32_t ecp, ecr; 691 692 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 693 694 ecp = XHCI_HCC_XECP(hcc) * 4; 695 while (ecp != 0) { 696 ecr = xhci_read_4(sc, ecp); 697 aprint_debug_dev(sc->sc_dev, "ECR %x: %08x\n", ecp, ecr); 698 switch (XHCI_XECP_ID(ecr)) { 699 case XHCI_ID_PROTOCOLS: { 700 uint32_t w4, w8, wc; 701 uint16_t w2; 702 w2 = (ecr >> 16) & 0xffff; 703 w4 = xhci_read_4(sc, ecp + 4); 704 w8 = xhci_read_4(sc, ecp + 8); 705 wc = xhci_read_4(sc, ecp + 0xc); 706 aprint_debug_dev(sc->sc_dev, 707 " SP: %08x %08x %08x %08x\n", ecr, w4, w8, wc); 708 /* unused */ 709 if (w4 == 0x20425355 && (w2 & 0xff00) == 0x0300) { 710 sc->sc_ss_port_start = (w8 >> 0) & 0xff;; 711 sc->sc_ss_port_count = (w8 >> 8) & 0xff;; 712 } 713 if (w4 == 0x20425355 && (w2 & 0xff00) == 0x0200) { 714 sc->sc_hs_port_start = (w8 >> 0) & 0xff; 715 sc->sc_hs_port_count = (w8 >> 8) & 0xff; 716 } 717 break; 718 } 719 case XHCI_ID_USB_LEGACY: { 720 uint8_t bios_sem; 721 722 /* Take host controller ownership from BIOS */ 723 bios_sem = xhci_read_1(sc, ecp + XHCI_XECP_BIOS_SEM); 724 if (bios_sem) { 725 /* sets xHCI to be owned by OS */ 726 xhci_write_1(sc, ecp + XHCI_XECP_OS_SEM, 1); 727 aprint_debug_dev(sc->sc_dev, 728 "waiting for BIOS to give up control\n"); 729 for (int i = 0; i < 5000; i++) { 730 bios_sem = xhci_read_1(sc, ecp + 731 XHCI_XECP_BIOS_SEM); 732 if (bios_sem == 0) 733 break; 734 DELAY(1000); 735 } 736 if (bios_sem) { 737 aprint_error_dev(sc->sc_dev, 738 "timed out waiting for BIOS\n"); 739 } 740 } 741 break; 742 } 743 default: 744 break; 745 } 746 ecr = xhci_read_4(sc, ecp); 747 if (XHCI_XECP_NEXT(ecr) == 0) { 748 ecp = 0; 749 } else { 750 ecp += XHCI_XECP_NEXT(ecr) * 4; 751 } 752 } 753 } 754 755 #define XHCI_HCCPREV1_BITS \ 756 "\177\020" /* New bitmask */ \ 757 "f\020\020XECP\0" \ 758 "f\014\4MAXPSA\0" \ 759 "b\013CFC\0" \ 760 "b\012SEC\0" \ 761 "b\011SBD\0" \ 762 "b\010FSE\0" \ 763 "b\7NSS\0" \ 764 "b\6LTC\0" \ 765 "b\5LHRC\0" \ 766 "b\4PIND\0" \ 767 "b\3PPC\0" \ 768 "b\2CZC\0" \ 769 "b\1BNC\0" \ 770 "b\0AC64\0" \ 771 "\0" 772 #define XHCI_HCCV1_x_BITS \ 773 "\177\020" /* New bitmask */ \ 774 "f\020\020XECP\0" \ 775 "f\014\4MAXPSA\0" \ 776 "b\013CFC\0" \ 777 "b\012SEC\0" \ 778 "b\011SPC\0" \ 779 "b\010PAE\0" \ 780 "b\7NSS\0" \ 781 "b\6LTC\0" \ 782 "b\5LHRC\0" \ 783 "b\4PIND\0" \ 784 "b\3PPC\0" \ 785 "b\2CSZ\0" \ 786 "b\1BNC\0" \ 787 "b\0AC64\0" \ 788 "\0" 789 790 int 791 xhci_init(struct xhci_softc *sc) 792 { 793 bus_size_t bsz; 794 uint32_t cap, hcs1, hcs2, hcs3, hcc, dboff, rtsoff; 795 uint32_t pagesize, config; 796 int i = 0; 797 uint16_t hciversion; 798 uint8_t caplength; 799 800 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 801 802 sc->sc_bus.ub_revision = USBREV_3_0; 803 sc->sc_bus.ub_usedma = true; 804 805 cap = xhci_read_4(sc, XHCI_CAPLENGTH); 806 caplength = XHCI_CAP_CAPLENGTH(cap); 807 hciversion = XHCI_CAP_HCIVERSION(cap); 808 809 if (hciversion < XHCI_HCIVERSION_0_96 || 810 hciversion > XHCI_HCIVERSION_1_0) { 811 aprint_normal_dev(sc->sc_dev, 812 "xHCI version %x.%x not known to be supported\n", 813 (hciversion >> 8) & 0xff, (hciversion >> 0) & 0xff); 814 } else { 815 aprint_verbose_dev(sc->sc_dev, "xHCI version %x.%x\n", 816 (hciversion >> 8) & 0xff, (hciversion >> 0) & 0xff); 817 } 818 819 if (bus_space_subregion(sc->sc_iot, sc->sc_ioh, 0, caplength, 820 &sc->sc_cbh) != 0) { 821 aprint_error_dev(sc->sc_dev, "capability subregion failure\n"); 822 return ENOMEM; 823 } 824 825 hcs1 = xhci_cap_read_4(sc, XHCI_HCSPARAMS1); 826 sc->sc_maxslots = XHCI_HCS1_MAXSLOTS(hcs1); 827 sc->sc_maxintrs = XHCI_HCS1_MAXINTRS(hcs1); 828 sc->sc_maxports = XHCI_HCS1_MAXPORTS(hcs1); 829 hcs2 = xhci_cap_read_4(sc, XHCI_HCSPARAMS2); 830 hcs3 = xhci_cap_read_4(sc, XHCI_HCSPARAMS3); 831 aprint_debug_dev(sc->sc_dev, 832 "hcs1=%"PRIx32" hcs2=%"PRIx32" hcs3=%"PRIx32"\n", hcs1, hcs2, hcs3); 833 834 hcc = xhci_cap_read_4(sc, XHCI_HCCPARAMS); 835 sc->sc_ac64 = XHCI_HCC_AC64(hcc); 836 sc->sc_ctxsz = XHCI_HCC_CSZ(hcc) ? 64 : 32; 837 838 char sbuf[128]; 839 if (hciversion < XHCI_HCIVERSION_1_0) 840 snprintb(sbuf, sizeof(sbuf), XHCI_HCCPREV1_BITS, hcc); 841 else 842 snprintb(sbuf, sizeof(sbuf), XHCI_HCCV1_x_BITS, hcc); 843 aprint_debug_dev(sc->sc_dev, "hcc=%s\n", sbuf); 844 aprint_debug_dev(sc->sc_dev, "xECP %x\n", XHCI_HCC_XECP(hcc) * 4); 845 846 /* print PSI and take ownership from BIOS */ 847 xhci_ecp(sc, hcc); 848 849 bsz = XHCI_PORTSC(sc->sc_maxports + 1); 850 if (bus_space_subregion(sc->sc_iot, sc->sc_ioh, caplength, bsz, 851 &sc->sc_obh) != 0) { 852 aprint_error_dev(sc->sc_dev, "operational subregion failure\n"); 853 return ENOMEM; 854 } 855 856 dboff = xhci_cap_read_4(sc, XHCI_DBOFF); 857 if (bus_space_subregion(sc->sc_iot, sc->sc_ioh, dboff, 858 sc->sc_maxslots * 4, &sc->sc_dbh) != 0) { 859 aprint_error_dev(sc->sc_dev, "doorbell subregion failure\n"); 860 return ENOMEM; 861 } 862 863 rtsoff = xhci_cap_read_4(sc, XHCI_RTSOFF); 864 if (bus_space_subregion(sc->sc_iot, sc->sc_ioh, rtsoff, 865 sc->sc_maxintrs * 0x20, &sc->sc_rbh) != 0) { 866 aprint_error_dev(sc->sc_dev, "runtime subregion failure\n"); 867 return ENOMEM; 868 } 869 870 int rv; 871 rv = xhci_hc_reset(sc); 872 if (rv != 0) { 873 return rv; 874 } 875 876 if (sc->sc_vendor_init) 877 sc->sc_vendor_init(sc); 878 879 pagesize = xhci_op_read_4(sc, XHCI_PAGESIZE); 880 aprint_debug_dev(sc->sc_dev, "PAGESIZE 0x%08x\n", pagesize); 881 pagesize = ffs(pagesize); 882 if (pagesize == 0) { 883 aprint_error_dev(sc->sc_dev, "pagesize is 0\n"); 884 return EIO; 885 } 886 sc->sc_pgsz = 1 << (12 + (pagesize - 1)); 887 aprint_debug_dev(sc->sc_dev, "sc_pgsz 0x%08x\n", (uint32_t)sc->sc_pgsz); 888 aprint_debug_dev(sc->sc_dev, "sc_maxslots 0x%08x\n", 889 (uint32_t)sc->sc_maxslots); 890 aprint_debug_dev(sc->sc_dev, "sc_maxports %d\n", sc->sc_maxports); 891 892 usbd_status err; 893 894 sc->sc_maxspbuf = XHCI_HCS2_MAXSPBUF(hcs2); 895 aprint_debug_dev(sc->sc_dev, "sc_maxspbuf %d\n", sc->sc_maxspbuf); 896 if (sc->sc_maxspbuf != 0) { 897 err = usb_allocmem(&sc->sc_bus, 898 sizeof(uint64_t) * sc->sc_maxspbuf, sizeof(uint64_t), 899 &sc->sc_spbufarray_dma); 900 if (err) { 901 aprint_error_dev(sc->sc_dev, 902 "spbufarray init fail, err %d\n", err); 903 return ENOMEM; 904 } 905 906 sc->sc_spbuf_dma = kmem_zalloc(sizeof(*sc->sc_spbuf_dma) * 907 sc->sc_maxspbuf, KM_SLEEP); 908 uint64_t *spbufarray = KERNADDR(&sc->sc_spbufarray_dma, 0); 909 for (i = 0; i < sc->sc_maxspbuf; i++) { 910 usb_dma_t * const dma = &sc->sc_spbuf_dma[i]; 911 /* allocate contexts */ 912 err = usb_allocmem(&sc->sc_bus, sc->sc_pgsz, 913 sc->sc_pgsz, dma); 914 if (err) { 915 aprint_error_dev(sc->sc_dev, 916 "spbufarray_dma init fail, err %d\n", err); 917 rv = ENOMEM; 918 goto bad1; 919 } 920 spbufarray[i] = htole64(DMAADDR(dma, 0)); 921 usb_syncmem(dma, 0, sc->sc_pgsz, 922 BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE); 923 } 924 925 usb_syncmem(&sc->sc_spbufarray_dma, 0, 926 sizeof(uint64_t) * sc->sc_maxspbuf, BUS_DMASYNC_PREWRITE); 927 } 928 929 config = xhci_op_read_4(sc, XHCI_CONFIG); 930 config &= ~0xFF; 931 config |= sc->sc_maxslots & 0xFF; 932 xhci_op_write_4(sc, XHCI_CONFIG, config); 933 934 err = xhci_ring_init(sc, &sc->sc_cr, XHCI_COMMAND_RING_TRBS, 935 XHCI_COMMAND_RING_SEGMENTS_ALIGN); 936 if (err) { 937 aprint_error_dev(sc->sc_dev, "command ring init fail, err %d\n", 938 err); 939 rv = ENOMEM; 940 goto bad1; 941 } 942 943 err = xhci_ring_init(sc, &sc->sc_er, XHCI_EVENT_RING_TRBS, 944 XHCI_EVENT_RING_SEGMENTS_ALIGN); 945 if (err) { 946 aprint_error_dev(sc->sc_dev, "event ring init fail, err %d\n", 947 err); 948 rv = ENOMEM; 949 goto bad2; 950 } 951 952 usb_dma_t *dma; 953 size_t size; 954 size_t align; 955 956 dma = &sc->sc_eventst_dma; 957 size = roundup2(XHCI_EVENT_RING_SEGMENTS * XHCI_ERSTE_SIZE, 958 XHCI_EVENT_RING_SEGMENT_TABLE_ALIGN); 959 KASSERTMSG(size <= (512 * 1024), "eventst size %zu too large", size); 960 align = XHCI_EVENT_RING_SEGMENT_TABLE_ALIGN; 961 err = usb_allocmem(&sc->sc_bus, size, align, dma); 962 if (err) { 963 aprint_error_dev(sc->sc_dev, "eventst init fail, err %d\n", 964 err); 965 rv = ENOMEM; 966 goto bad3; 967 } 968 969 memset(KERNADDR(dma, 0), 0, size); 970 usb_syncmem(dma, 0, size, BUS_DMASYNC_PREWRITE); 971 aprint_debug_dev(sc->sc_dev, "eventst: %016jx %p %zx\n", 972 (uintmax_t)DMAADDR(&sc->sc_eventst_dma, 0), 973 KERNADDR(&sc->sc_eventst_dma, 0), 974 sc->sc_eventst_dma.udma_block->size); 975 976 dma = &sc->sc_dcbaa_dma; 977 size = (1 + sc->sc_maxslots) * sizeof(uint64_t); 978 KASSERTMSG(size <= 2048, "dcbaa size %zu too large", size); 979 align = XHCI_DEVICE_CONTEXT_BASE_ADDRESS_ARRAY_ALIGN; 980 err = usb_allocmem(&sc->sc_bus, size, align, dma); 981 if (err) { 982 aprint_error_dev(sc->sc_dev, "dcbaa init fail, err %d\n", err); 983 rv = ENOMEM; 984 goto bad4; 985 } 986 aprint_debug_dev(sc->sc_dev, "dcbaa: %016jx %p %zx\n", 987 (uintmax_t)DMAADDR(&sc->sc_dcbaa_dma, 0), 988 KERNADDR(&sc->sc_dcbaa_dma, 0), 989 sc->sc_dcbaa_dma.udma_block->size); 990 991 memset(KERNADDR(dma, 0), 0, size); 992 if (sc->sc_maxspbuf != 0) { 993 /* 994 * DCBA entry 0 hold the scratchbuf array pointer. 995 */ 996 *(uint64_t *)KERNADDR(dma, 0) = 997 htole64(DMAADDR(&sc->sc_spbufarray_dma, 0)); 998 } 999 usb_syncmem(dma, 0, size, BUS_DMASYNC_PREWRITE); 1000 1001 sc->sc_slots = kmem_zalloc(sizeof(*sc->sc_slots) * sc->sc_maxslots, 1002 KM_SLEEP); 1003 if (sc->sc_slots == NULL) { 1004 aprint_error_dev(sc->sc_dev, "slots init fail, err %d\n", err); 1005 rv = ENOMEM; 1006 goto bad; 1007 } 1008 1009 sc->sc_xferpool = pool_cache_init(sizeof(struct xhci_xfer), 0, 0, 0, 1010 "xhcixfer", NULL, IPL_USB, NULL, NULL, NULL); 1011 if (sc->sc_xferpool == NULL) { 1012 aprint_error_dev(sc->sc_dev, "pool_cache init fail, err %d\n", 1013 err); 1014 rv = ENOMEM; 1015 goto bad; 1016 } 1017 1018 cv_init(&sc->sc_command_cv, "xhcicmd"); 1019 mutex_init(&sc->sc_lock, MUTEX_DEFAULT, IPL_SOFTUSB); 1020 mutex_init(&sc->sc_intr_lock, MUTEX_DEFAULT, IPL_USB); 1021 cv_init(&sc->sc_softwake_cv, "xhciab"); 1022 1023 /* Set up the bus struct. */ 1024 sc->sc_bus.ub_methods = &xhci_bus_methods; 1025 sc->sc_bus.ub_pipesize = sizeof(struct xhci_pipe); 1026 1027 struct xhci_erste *erst; 1028 erst = KERNADDR(&sc->sc_eventst_dma, 0); 1029 erst[0].erste_0 = htole64(xhci_ring_trbp(&sc->sc_er, 0)); 1030 erst[0].erste_2 = htole32(XHCI_EVENT_RING_TRBS); 1031 erst[0].erste_3 = htole32(0); 1032 usb_syncmem(&sc->sc_eventst_dma, 0, 1033 XHCI_ERSTE_SIZE * XHCI_EVENT_RING_SEGMENTS, BUS_DMASYNC_PREWRITE); 1034 1035 xhci_rt_write_4(sc, XHCI_ERSTSZ(0), XHCI_EVENT_RING_SEGMENTS); 1036 xhci_rt_write_8(sc, XHCI_ERSTBA(0), DMAADDR(&sc->sc_eventst_dma, 0)); 1037 xhci_rt_write_8(sc, XHCI_ERDP(0), xhci_ring_trbp(&sc->sc_er, 0) | 1038 XHCI_ERDP_LO_BUSY); 1039 xhci_op_write_8(sc, XHCI_DCBAAP, DMAADDR(&sc->sc_dcbaa_dma, 0)); 1040 xhci_op_write_8(sc, XHCI_CRCR, xhci_ring_trbp(&sc->sc_cr, 0) | 1041 sc->sc_cr.xr_cs); 1042 1043 #if 0 1044 hexdump("eventst", KERNADDR(&sc->sc_eventst_dma, 0), 1045 XHCI_ERSTE_SIZE * XHCI_EVENT_RING_SEGMENTS); 1046 #endif 1047 1048 xhci_rt_write_4(sc, XHCI_IMAN(0), XHCI_IMAN_INTR_ENA); 1049 if ((sc->sc_quirks & XHCI_QUIRK_INTEL) != 0) 1050 /* Intel xhci needs interrupt rate moderated. */ 1051 xhci_rt_write_4(sc, XHCI_IMOD(0), XHCI_IMOD_DEFAULT_LP); 1052 else 1053 xhci_rt_write_4(sc, XHCI_IMOD(0), 0); 1054 aprint_debug_dev(sc->sc_dev, "setting IMOD %u\n", 1055 xhci_rt_read_4(sc, XHCI_IMOD(0))); 1056 1057 xhci_op_write_4(sc, XHCI_USBCMD, XHCI_CMD_INTE|XHCI_CMD_RS); /* Go! */ 1058 aprint_debug_dev(sc->sc_dev, "USBCMD %08"PRIx32"\n", 1059 xhci_op_read_4(sc, XHCI_USBCMD)); 1060 1061 return 0; 1062 1063 bad: 1064 if (sc->sc_xferpool) { 1065 pool_cache_destroy(sc->sc_xferpool); 1066 sc->sc_xferpool = NULL; 1067 } 1068 1069 if (sc->sc_slots) { 1070 kmem_free(sc->sc_slots, sizeof(*sc->sc_slots) * 1071 sc->sc_maxslots); 1072 sc->sc_slots = NULL; 1073 } 1074 1075 usb_freemem(&sc->sc_bus, &sc->sc_dcbaa_dma); 1076 bad4: 1077 usb_freemem(&sc->sc_bus, &sc->sc_eventst_dma); 1078 bad3: 1079 xhci_ring_free(sc, &sc->sc_er); 1080 bad2: 1081 xhci_ring_free(sc, &sc->sc_cr); 1082 i = sc->sc_maxspbuf; 1083 bad1: 1084 for (int j = 0; j < i; j++) 1085 usb_freemem(&sc->sc_bus, &sc->sc_spbuf_dma[j]); 1086 usb_freemem(&sc->sc_bus, &sc->sc_spbufarray_dma); 1087 1088 return rv; 1089 } 1090 1091 int 1092 xhci_intr(void *v) 1093 { 1094 struct xhci_softc * const sc = v; 1095 int ret = 0; 1096 1097 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 1098 1099 if (sc == NULL) 1100 return 0; 1101 1102 mutex_spin_enter(&sc->sc_intr_lock); 1103 1104 if (sc->sc_dying || !device_has_power(sc->sc_dev)) 1105 goto done; 1106 1107 /* If we get an interrupt while polling, then just ignore it. */ 1108 if (sc->sc_bus.ub_usepolling) { 1109 #ifdef DIAGNOSTIC 1110 DPRINTFN(16, "ignored interrupt while polling", 0, 0, 0, 0); 1111 #endif 1112 goto done; 1113 } 1114 1115 ret = xhci_intr1(sc); 1116 done: 1117 mutex_spin_exit(&sc->sc_intr_lock); 1118 return ret; 1119 } 1120 1121 int 1122 xhci_intr1(struct xhci_softc * const sc) 1123 { 1124 uint32_t usbsts; 1125 uint32_t iman; 1126 1127 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 1128 1129 usbsts = xhci_op_read_4(sc, XHCI_USBSTS); 1130 DPRINTFN(16, "USBSTS %08x", usbsts, 0, 0, 0); 1131 #if 0 1132 if ((usbsts & (XHCI_STS_EINT|XHCI_STS_PCD)) == 0) { 1133 return 0; 1134 } 1135 #endif 1136 xhci_op_write_4(sc, XHCI_USBSTS, 1137 usbsts & (2|XHCI_STS_EINT|XHCI_STS_PCD)); /* XXX */ 1138 usbsts = xhci_op_read_4(sc, XHCI_USBSTS); 1139 DPRINTFN(16, "USBSTS %08x", usbsts, 0, 0, 0); 1140 1141 iman = xhci_rt_read_4(sc, XHCI_IMAN(0)); 1142 DPRINTFN(16, "IMAN0 %08x", iman, 0, 0, 0); 1143 iman |= XHCI_IMAN_INTR_PEND; 1144 xhci_rt_write_4(sc, XHCI_IMAN(0), iman); 1145 iman = xhci_rt_read_4(sc, XHCI_IMAN(0)); 1146 DPRINTFN(16, "IMAN0 %08x", iman, 0, 0, 0); 1147 usbsts = xhci_op_read_4(sc, XHCI_USBSTS); 1148 DPRINTFN(16, "USBSTS %08x", usbsts, 0, 0, 0); 1149 1150 usb_schedsoftintr(&sc->sc_bus); 1151 1152 return 1; 1153 } 1154 1155 /* 1156 * 3 port speed types used in USB stack 1157 * 1158 * usbdi speed 1159 * definition: USB_SPEED_* in usb.h 1160 * They are used in struct usbd_device in USB stack. 1161 * ioctl interface uses these values too. 1162 * port_status speed 1163 * definition: UPS_*_SPEED in usb.h 1164 * They are used in usb_port_status_t and valid only for USB 2.0. 1165 * Speed value is always 0 for Super Speed or more, and dwExtPortStatus 1166 * of usb_port_status_ext_t indicates port speed. 1167 * Note that some 3.0 values overlap with 2.0 values. 1168 * (e.g. 0x200 means UPS_POER_POWER_SS in SS and 1169 * means UPS_LOW_SPEED in HS.) 1170 * port status returned from hub also uses these values. 1171 * On NetBSD UPS_OTHER_SPEED indicates port speed is super speed 1172 * or more. 1173 * xspeed: 1174 * definition: Protocol Speed ID (PSI) (xHCI 1.1 7.2.1) 1175 * They are used in only slot context and PORTSC reg of xhci. 1176 * The difference between usbdi speed and xspeed is 1177 * that FS and LS values are swapped. 1178 */ 1179 1180 /* convert usbdi speed to xspeed */ 1181 static int 1182 xhci_speed2xspeed(int speed) 1183 { 1184 switch (speed) { 1185 case USB_SPEED_LOW: return 2; 1186 case USB_SPEED_FULL: return 1; 1187 default: return speed; 1188 } 1189 } 1190 1191 #if 0 1192 /* convert xspeed to usbdi speed */ 1193 static int 1194 xhci_xspeed2speed(int xspeed) 1195 { 1196 switch (xspeed) { 1197 case 1: return USB_SPEED_FULL; 1198 case 2: return USB_SPEED_LOW; 1199 default: return xspeed; 1200 } 1201 } 1202 #endif 1203 1204 /* convert xspeed to port status speed */ 1205 static int 1206 xhci_xspeed2psspeed(int xspeed) 1207 { 1208 switch (xspeed) { 1209 case 0: return 0; 1210 case 1: return UPS_FULL_SPEED; 1211 case 2: return UPS_LOW_SPEED; 1212 case 3: return UPS_HIGH_SPEED; 1213 default: return UPS_OTHER_SPEED; 1214 } 1215 } 1216 1217 /* construct slot context */ 1218 static void 1219 xhci_setup_sctx(struct usbd_device *dev, uint32_t *cp) 1220 { 1221 usb_device_descriptor_t * const dd = &dev->ud_ddesc; 1222 int speed = dev->ud_speed; 1223 int tthubslot, ttportnum; 1224 bool ishub; 1225 bool usemtt; 1226 1227 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 1228 1229 /* 1230 * 6.2.2, Table 57-60, 6.2.2.1, 6.2.2.2 1231 * tthubslot: 1232 * This is the slot ID of parent HS hub 1233 * if LS/FS device is connected && connected through HS hub. 1234 * This is 0 if device is not LS/FS device || 1235 * parent hub is not HS hub || 1236 * attached to root hub. 1237 * ttportnum: 1238 * This is the downstream facing port of parent HS hub 1239 * if LS/FS device is connected. 1240 * This is 0 if device is not LS/FS device || 1241 * parent hub is not HS hub || 1242 * attached to root hub. 1243 */ 1244 if (dev->ud_myhsport != NULL && 1245 dev->ud_myhub != NULL && dev->ud_myhub->ud_depth != 0 && 1246 (dev->ud_myhub != NULL && 1247 dev->ud_myhub->ud_speed == USB_SPEED_HIGH) && 1248 (speed == USB_SPEED_LOW || speed == USB_SPEED_FULL)) { 1249 ttportnum = dev->ud_myhsport->up_portno; 1250 tthubslot = dev->ud_myhsport->up_parent->ud_addr; 1251 } else { 1252 ttportnum = 0; 1253 tthubslot = 0; 1254 } 1255 DPRINTFN(4, "myhsport %p ttportnum=%d tthubslot=%d", 1256 dev->ud_myhsport, ttportnum, tthubslot, 0); 1257 1258 /* ishub is valid after reading UDESC_DEVICE */ 1259 ishub = (dd->bDeviceClass == UDCLASS_HUB); 1260 1261 /* dev->ud_hub is valid after reading UDESC_HUB */ 1262 if (ishub && dev->ud_hub) { 1263 usb_hub_descriptor_t *hd = &dev->ud_hub->uh_hubdesc; 1264 1265 cp[1] |= htole32(XHCI_SCTX_1_NUM_PORTS_SET(hd->bNbrPorts)); 1266 cp[2] |= htole32(XHCI_SCTX_2_TT_THINK_TIME_SET( 1267 __SHIFTOUT(UGETW(hd->wHubCharacteristics), UHD_TT_THINK))); 1268 DPRINTFN(4, "nports=%d ttt=%d", 1269 hd->bNbrPorts, XHCI_SCTX_2_TT_THINK_TIME_GET(cp[2]), 0, 0); 1270 } 1271 1272 #define IS_TTHUB(dd) \ 1273 ((dd)->bDeviceProtocol == UDPROTO_HSHUBSTT || \ 1274 (dd)->bDeviceProtocol == UDPROTO_HSHUBMTT) 1275 1276 /* 1277 * MTT flag is set if 1278 * 1. this is HS hub && MTT is enabled 1279 * or 1280 * 2. this is not hub && this is LS or FS device && 1281 * MTT of parent HS hub (and its parent, too) is enabled 1282 */ 1283 if (ishub && speed == USB_SPEED_HIGH && IS_TTHUB(dd)) 1284 usemtt = true; 1285 else if (!ishub && 1286 (speed == USB_SPEED_LOW || speed == USB_SPEED_FULL) && 1287 dev->ud_myhub != NULL && dev->ud_myhub->ud_depth != 0 && 1288 (dev->ud_myhub != NULL && 1289 dev->ud_myhub->ud_speed == USB_SPEED_HIGH) && 1290 dev->ud_myhsport != NULL && 1291 IS_TTHUB(&dev->ud_myhsport->up_parent->ud_ddesc)) 1292 usemtt = true; 1293 else 1294 usemtt = false; 1295 DPRINTFN(4, "class %u proto %u ishub %d usemtt %d", 1296 dd->bDeviceClass, dd->bDeviceProtocol, ishub, usemtt); 1297 1298 cp[0] |= htole32( 1299 XHCI_SCTX_0_SPEED_SET(xhci_speed2xspeed(speed)) | 1300 XHCI_SCTX_0_HUB_SET(ishub ? 1 : 0) | 1301 XHCI_SCTX_0_MTT_SET(usemtt ? 1 : 0) 1302 ); 1303 cp[1] |= htole32(0); 1304 cp[2] |= htole32( 1305 XHCI_SCTX_2_IRQ_TARGET_SET(0) | 1306 XHCI_SCTX_2_TT_HUB_SID_SET(tthubslot) | 1307 XHCI_SCTX_2_TT_PORT_NUM_SET(ttportnum) 1308 ); 1309 cp[3] |= htole32(0); 1310 } 1311 1312 static uint32_t 1313 xhci_get_maxburst(struct usbd_pipe *pipe) 1314 { 1315 usb_endpoint_descriptor_t * const ed = pipe->up_endpoint->ue_edesc; 1316 usbd_desc_iter_t iter; 1317 const usb_cdc_descriptor_t *cdcd; 1318 const usb_endpoint_ss_comp_descriptor_t * esscd = NULL; 1319 uint32_t maxb = 0; 1320 uint8_t ep; 1321 1322 cdcd = (const usb_cdc_descriptor_t *)usb_find_desc( 1323 pipe->up_dev, UDESC_INTERFACE, USBD_CDCSUBTYPE_ANY); 1324 usb_desc_iter_init(pipe->up_dev, &iter); 1325 iter.cur = (const void *)cdcd; 1326 1327 /* find endpoint_ss_comp desc for ep of this pipe */ 1328 for (ep = 0;;) { 1329 cdcd = (const usb_cdc_descriptor_t *)usb_desc_iter_next(&iter); 1330 if (cdcd == NULL) 1331 break; 1332 if (ep == 0 && cdcd->bDescriptorType == UDESC_ENDPOINT) { 1333 ep = ((const usb_endpoint_descriptor_t *)cdcd)-> 1334 bEndpointAddress; 1335 if (UE_GET_ADDR(ep) == 1336 UE_GET_ADDR(ed->bEndpointAddress)) { 1337 cdcd = (const usb_cdc_descriptor_t *) 1338 usb_desc_iter_next(&iter); 1339 break; 1340 } 1341 ep = 0; 1342 } 1343 } 1344 if (cdcd != NULL && cdcd->bDescriptorType == UDESC_ENDPOINT_SS_COMP) { 1345 esscd = (const usb_endpoint_ss_comp_descriptor_t *)cdcd; 1346 maxb = esscd->bMaxBurst; 1347 } 1348 1349 return maxb; 1350 } 1351 1352 /* 1353 * Convert endpoint bInterval value to endpoint context interval value 1354 * for Interrupt pipe. 1355 * xHCI 6.2.3.6 Table 65, USB 2.0 9.6.6 1356 */ 1357 static uint32_t 1358 xhci_bival2ival(uint32_t ival, int speed) 1359 { 1360 if (speed == USB_SPEED_LOW || speed == USB_SPEED_FULL) { 1361 int i; 1362 1363 /* 1364 * round ival down to "the nearest base 2 multiple of 1365 * bInterval * 8". 1366 * bInterval is at most 255 as its type is uByte. 1367 * 255(ms) = 2040(x 125us) < 2^11, so start with 10. 1368 */ 1369 for (i = 10; i > 0; i--) { 1370 if ((ival * 8) >= (1 << i)) 1371 break; 1372 } 1373 ival = i; 1374 } else { 1375 /* Interval = bInterval-1 for SS/HS */ 1376 ival--; 1377 } 1378 1379 return ival; 1380 } 1381 1382 /* 1383 * 4.8.2, 6.2.3.2 1384 * construct common endpoint parameters 1385 */ 1386 static void 1387 xhci_setup_endp_ctx(struct usbd_pipe *pipe, uint32_t *cp) 1388 { 1389 struct xhci_slot * const xs = pipe->up_dev->ud_hcpriv; 1390 usb_endpoint_descriptor_t * const ed = pipe->up_endpoint->ue_edesc; 1391 const u_int dci = xhci_ep_get_dci(ed); 1392 const uint8_t xfertype = UE_GET_XFERTYPE(ed->bmAttributes); 1393 uint32_t mps = UGETW(ed->wMaxPacketSize); 1394 uint32_t maxb = 0; 1395 int speed = pipe->up_dev->ud_speed; 1396 uint32_t ival = ed->bInterval; 1397 1398 cp[0] = htole32( 1399 XHCI_EPCTX_0_EPSTATE_SET(0) | 1400 XHCI_EPCTX_0_MULT_SET(0) | /* always 0 except SS iscoh */ 1401 XHCI_EPCTX_0_MAXP_STREAMS_SET(0) | 1402 XHCI_EPCTX_0_LSA_SET(0) | 1403 XHCI_EPCTX_0_MAX_ESIT_PAYLOAD_HI_SET(0) 1404 ); 1405 cp[1] = htole32( 1406 XHCI_EPCTX_1_EPTYPE_SET(xhci_ep_get_type(ed)) | 1407 XHCI_EPCTX_1_HID_SET(0) | 1408 XHCI_EPCTX_1_MAXB_SET(0) 1409 ); 1410 if (xfertype != UE_ISOCHRONOUS) 1411 cp[1] |= htole32(XHCI_EPCTX_1_CERR_SET(3)); 1412 1413 /* 6.2.3.4, 4.8.2.4 */ 1414 if (USB_IS_SS(speed)) { 1415 /* UBS 3.1 9.6.6 */ 1416 cp[1] |= htole32(XHCI_EPCTX_1_MAXP_SIZE_SET(mps)); 1417 /* UBS 3.1 9.6.7 */ 1418 maxb = xhci_get_maxburst(pipe); 1419 cp[1] |= htole32(XHCI_EPCTX_1_MAXB_SET(maxb)); 1420 } else { 1421 /* UBS 2.0 9.6.6 */ 1422 cp[1] |= htole32(XHCI_EPCTX_1_MAXP_SIZE_SET(UE_GET_SIZE(mps))); 1423 1424 /* 6.2.3.4 */ 1425 if (speed == USB_SPEED_HIGH && 1426 (xfertype == UE_ISOCHRONOUS || xfertype == UE_INTERRUPT)) { 1427 maxb = UE_GET_TRANS(mps); 1428 } else { 1429 /* LS/FS or HS CTRL or HS BULK */ 1430 maxb = 0; 1431 } 1432 cp[1] |= htole32(XHCI_EPCTX_1_MAXB_SET(maxb)); 1433 } 1434 1435 if (xfertype == UE_CONTROL) 1436 cp[4] = htole32(XHCI_EPCTX_4_AVG_TRB_LEN_SET(8)); /* 6.2.3 */ 1437 else if (USB_IS_SS(speed)) 1438 cp[4] = htole32(XHCI_EPCTX_4_AVG_TRB_LEN_SET(mps)); 1439 else 1440 cp[4] = htole32(XHCI_EPCTX_4_AVG_TRB_LEN_SET(UE_GET_SIZE(mps))); 1441 1442 switch (xfertype) { 1443 case UE_CONTROL: 1444 break; 1445 case UE_BULK: 1446 /* XXX Set MaxPStreams, HID, and LSA if streams enabled */ 1447 break; 1448 case UE_INTERRUPT: 1449 if (pipe->up_interval != USBD_DEFAULT_INTERVAL) 1450 ival = pipe->up_interval; 1451 1452 ival = xhci_bival2ival(ival, speed); 1453 cp[0] |= htole32(XHCI_EPCTX_0_IVAL_SET(ival)); 1454 break; 1455 case UE_ISOCHRONOUS: 1456 if (pipe->up_interval != USBD_DEFAULT_INTERVAL) 1457 ival = pipe->up_interval; 1458 1459 /* xHCI 6.2.3.6 Table 65, USB 2.0 9.6.6 */ 1460 if (speed == USB_SPEED_FULL) 1461 ival += 3; /* 1ms -> 125us */ 1462 ival--; 1463 cp[0] |= htole32(XHCI_EPCTX_0_IVAL_SET(ival)); 1464 1465 if (USB_IS_SS(speed)) { 1466 /* XXX if LEC = 1, set ESIT instead */ 1467 cp[0] |= htole32(XHCI_EPCTX_0_MULT_SET(0)); 1468 } 1469 break; 1470 default: 1471 break; 1472 } 1473 *(uint64_t *)(&cp[2]) = htole64( 1474 xhci_ring_trbp(&xs->xs_ep[dci].xe_tr, 0) | 1475 XHCI_EPCTX_2_DCS_SET(1)); 1476 } 1477 1478 /* 1479 * Construct input contexts and issue TRB 1480 */ 1481 static usbd_status 1482 xhci_configure_endpoint(struct usbd_pipe *pipe) 1483 { 1484 struct xhci_softc * const sc = XHCI_PIPE2SC(pipe); 1485 struct xhci_slot * const xs = pipe->up_dev->ud_hcpriv; 1486 const u_int dci = xhci_ep_get_dci(pipe->up_endpoint->ue_edesc); 1487 struct xhci_trb trb; 1488 usbd_status err; 1489 uint32_t *cp; 1490 1491 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 1492 DPRINTFN(4, "slot %u dci %u epaddr 0x%02x attr 0x%02x", 1493 xs->xs_idx, dci, pipe->up_endpoint->ue_edesc->bEndpointAddress, 1494 pipe->up_endpoint->ue_edesc->bmAttributes); 1495 1496 /* XXX ensure input context is available? */ 1497 1498 memset(xhci_slot_get_icv(sc, xs, 0), 0, sc->sc_pgsz); 1499 1500 cp = xhci_slot_get_icv(sc, xs, XHCI_ICI_INPUT_CONTROL); 1501 cp[0] = htole32(0); 1502 cp[1] = htole32(XHCI_INCTX_1_ADD_MASK(dci)); 1503 1504 /* set up input slot context */ 1505 cp = xhci_slot_get_icv(sc, xs, xhci_dci_to_ici(XHCI_DCI_SLOT)); 1506 xhci_setup_sctx(pipe->up_dev, cp); 1507 cp[0] |= htole32(XHCI_SCTX_0_CTX_NUM_SET(dci)); 1508 1509 /* set up input endpoint context */ 1510 cp = xhci_slot_get_icv(sc, xs, xhci_dci_to_ici(dci)); 1511 xhci_setup_endp_ctx(pipe, cp); 1512 1513 /* sync input contexts before they are read from memory */ 1514 usb_syncmem(&xs->xs_ic_dma, 0, sc->sc_pgsz, BUS_DMASYNC_PREWRITE); 1515 hexdump("input control context", xhci_slot_get_icv(sc, xs, 0), 1516 sc->sc_ctxsz * 1); 1517 hexdump("input endpoint context", xhci_slot_get_icv(sc, xs, 1518 xhci_dci_to_ici(dci)), sc->sc_ctxsz * 1); 1519 1520 trb.trb_0 = xhci_slot_get_icp(sc, xs, 0); 1521 trb.trb_2 = 0; 1522 trb.trb_3 = XHCI_TRB_3_SLOT_SET(xs->xs_idx) | 1523 XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_CONFIGURE_EP); 1524 1525 err = xhci_do_command(sc, &trb, USBD_DEFAULT_TIMEOUT); 1526 1527 usb_syncmem(&xs->xs_dc_dma, 0, sc->sc_pgsz, BUS_DMASYNC_POSTREAD); 1528 hexdump("output context", xhci_slot_get_dcv(sc, xs, dci), 1529 sc->sc_ctxsz * 1); 1530 1531 return err; 1532 } 1533 1534 #if 0 1535 static usbd_status 1536 xhci_unconfigure_endpoint(struct usbd_pipe *pipe) 1537 { 1538 #ifdef USB_DEBUG 1539 struct xhci_slot * const xs = pipe->up_dev->ud_hcpriv; 1540 #endif 1541 1542 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 1543 DPRINTFN(4, "slot %u", xs->xs_idx, 0, 0, 0); 1544 1545 return USBD_NORMAL_COMPLETION; 1546 } 1547 #endif 1548 1549 /* 4.6.8, 6.4.3.7 */ 1550 static usbd_status 1551 xhci_reset_endpoint(struct usbd_pipe *pipe) 1552 { 1553 struct xhci_softc * const sc = XHCI_PIPE2SC(pipe); 1554 struct xhci_slot * const xs = pipe->up_dev->ud_hcpriv; 1555 const u_int dci = xhci_ep_get_dci(pipe->up_endpoint->ue_edesc); 1556 struct xhci_trb trb; 1557 usbd_status err; 1558 1559 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 1560 DPRINTFN(4, "slot %u dci %u", xs->xs_idx, dci, 0, 0); 1561 1562 KASSERT(!mutex_owned(&sc->sc_lock)); 1563 1564 trb.trb_0 = 0; 1565 trb.trb_2 = 0; 1566 trb.trb_3 = XHCI_TRB_3_SLOT_SET(xs->xs_idx) | 1567 XHCI_TRB_3_EP_SET(dci) | 1568 XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_RESET_EP); 1569 1570 err = xhci_do_command(sc, &trb, USBD_DEFAULT_TIMEOUT); 1571 1572 return err; 1573 } 1574 1575 /* 1576 * 4.6.9, 6.4.3.8 1577 * Stop execution of TDs on xfer ring. 1578 * Should be called with sc_lock held. 1579 */ 1580 static usbd_status 1581 xhci_stop_endpoint(struct usbd_pipe *pipe) 1582 { 1583 struct xhci_softc * const sc = XHCI_PIPE2SC(pipe); 1584 struct xhci_slot * const xs = pipe->up_dev->ud_hcpriv; 1585 struct xhci_trb trb; 1586 usbd_status err; 1587 const u_int dci = xhci_ep_get_dci(pipe->up_endpoint->ue_edesc); 1588 1589 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 1590 DPRINTFN(4, "slot %u dci %u", xs->xs_idx, dci, 0, 0); 1591 1592 KASSERT(mutex_owned(&sc->sc_lock)); 1593 1594 trb.trb_0 = 0; 1595 trb.trb_2 = 0; 1596 trb.trb_3 = XHCI_TRB_3_SLOT_SET(xs->xs_idx) | 1597 XHCI_TRB_3_EP_SET(dci) | 1598 XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_STOP_EP); 1599 1600 err = xhci_do_command_locked(sc, &trb, USBD_DEFAULT_TIMEOUT); 1601 1602 return err; 1603 } 1604 1605 /* 1606 * Set TR Dequeue Pointer. 1607 * xCHI 1.1 4.6.10 6.4.3.9 1608 * Purge all of the transfer requests on ring. 1609 * EPSTATE of endpoint must be ERROR or STOPPED, or CONTEXT_STATE error. 1610 */ 1611 static usbd_status 1612 xhci_set_dequeue(struct usbd_pipe *pipe) 1613 { 1614 struct xhci_softc * const sc = XHCI_PIPE2SC(pipe); 1615 struct xhci_slot * const xs = pipe->up_dev->ud_hcpriv; 1616 const u_int dci = xhci_ep_get_dci(pipe->up_endpoint->ue_edesc); 1617 struct xhci_ring * const xr = &xs->xs_ep[dci].xe_tr; 1618 struct xhci_trb trb; 1619 usbd_status err; 1620 1621 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 1622 DPRINTFN(4, "slot %u dci %u", xs->xs_idx, dci, 0, 0); 1623 1624 memset(xr->xr_trb, 0, xr->xr_ntrb * XHCI_TRB_SIZE); 1625 usb_syncmem(&xr->xr_dma, 0, xr->xr_ntrb * XHCI_TRB_SIZE, 1626 BUS_DMASYNC_PREWRITE); 1627 memset(xr->xr_cookies, 0, xr->xr_ntrb * sizeof(*xr->xr_cookies)); 1628 1629 xr->xr_ep = 0; 1630 xr->xr_cs = 1; 1631 1632 /* set DCS */ 1633 trb.trb_0 = xhci_ring_trbp(xr, 0) | 1; /* XXX */ 1634 trb.trb_2 = 0; 1635 trb.trb_3 = XHCI_TRB_3_SLOT_SET(xs->xs_idx) | 1636 XHCI_TRB_3_EP_SET(dci) | 1637 XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_SET_TR_DEQUEUE); 1638 1639 err = xhci_do_command(sc, &trb, USBD_DEFAULT_TIMEOUT); 1640 1641 return err; 1642 } 1643 1644 /* 1645 * Open new pipe: called from usbd_setup_pipe_flags. 1646 * Fills methods of pipe. 1647 * If pipe is not for ep0, calls configure_endpoint. 1648 */ 1649 static usbd_status 1650 xhci_open(struct usbd_pipe *pipe) 1651 { 1652 struct usbd_device * const dev = pipe->up_dev; 1653 struct xhci_softc * const sc = XHCI_BUS2SC(dev->ud_bus); 1654 usb_endpoint_descriptor_t * const ed = pipe->up_endpoint->ue_edesc; 1655 const uint8_t xfertype = UE_GET_XFERTYPE(ed->bmAttributes); 1656 1657 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 1658 DPRINTFN(1, "addr %d depth %d port %d speed %d", 1659 dev->ud_addr, dev->ud_depth, dev->ud_powersrc->up_portno, 1660 dev->ud_speed); 1661 1662 if (sc->sc_dying) 1663 return USBD_IOERROR; 1664 1665 /* Root Hub */ 1666 if (dev->ud_depth == 0 && dev->ud_powersrc->up_portno == 0) { 1667 switch (ed->bEndpointAddress) { 1668 case USB_CONTROL_ENDPOINT: 1669 pipe->up_methods = &roothub_ctrl_methods; 1670 break; 1671 case UE_DIR_IN | USBROOTHUB_INTR_ENDPT: 1672 pipe->up_methods = &xhci_root_intr_methods; 1673 break; 1674 default: 1675 pipe->up_methods = NULL; 1676 DPRINTFN(0, "bad bEndpointAddress 0x%02x", 1677 ed->bEndpointAddress, 0, 0, 0); 1678 return USBD_INVAL; 1679 } 1680 return USBD_NORMAL_COMPLETION; 1681 } 1682 1683 switch (xfertype) { 1684 case UE_CONTROL: 1685 pipe->up_methods = &xhci_device_ctrl_methods; 1686 break; 1687 case UE_ISOCHRONOUS: 1688 pipe->up_methods = &xhci_device_isoc_methods; 1689 return USBD_INVAL; 1690 break; 1691 case UE_BULK: 1692 pipe->up_methods = &xhci_device_bulk_methods; 1693 break; 1694 case UE_INTERRUPT: 1695 pipe->up_methods = &xhci_device_intr_methods; 1696 break; 1697 default: 1698 return USBD_IOERROR; 1699 break; 1700 } 1701 1702 if (ed->bEndpointAddress != USB_CONTROL_ENDPOINT) 1703 return xhci_configure_endpoint(pipe); 1704 1705 return USBD_NORMAL_COMPLETION; 1706 } 1707 1708 /* 1709 * Closes pipe, called from usbd_kill_pipe via close methods. 1710 * If the endpoint to be closed is ep0, disable_slot. 1711 * Should be called with sc_lock held. 1712 */ 1713 static void 1714 xhci_close_pipe(struct usbd_pipe *pipe) 1715 { 1716 struct xhci_softc * const sc = XHCI_PIPE2SC(pipe); 1717 struct xhci_slot * const xs = pipe->up_dev->ud_hcpriv; 1718 usb_endpoint_descriptor_t * const ed = pipe->up_endpoint->ue_edesc; 1719 const u_int dci = xhci_ep_get_dci(ed); 1720 struct xhci_trb trb; 1721 uint32_t *cp; 1722 1723 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 1724 1725 if (sc->sc_dying) 1726 return; 1727 1728 /* xs is uninitialized before xhci_init_slot */ 1729 if (xs == NULL || xs->xs_idx == 0) 1730 return; 1731 1732 DPRINTFN(4, "pipe %p slot %u dci %u", pipe, xs->xs_idx, dci, 0); 1733 1734 KASSERTMSG(!cpu_intr_p() && !cpu_softintr_p(), "called from intr ctx"); 1735 KASSERT(mutex_owned(&sc->sc_lock)); 1736 1737 if (pipe->up_dev->ud_depth == 0) 1738 return; 1739 1740 if (dci == XHCI_DCI_EP_CONTROL) { 1741 DPRINTFN(4, "closing ep0", 0, 0, 0, 0); 1742 xhci_disable_slot(sc, xs->xs_idx); 1743 return; 1744 } 1745 1746 /* 1747 * This may fail in the case that xhci_close_pipe is called after 1748 * xhci_abort_xfer e.g. usbd_kill_pipe. 1749 */ 1750 (void)xhci_stop_endpoint(pipe); 1751 1752 /* 1753 * set appropriate bit to be dropped. 1754 * don't set DC bit to 1, otherwise all endpoints 1755 * would be deconfigured. 1756 */ 1757 cp = xhci_slot_get_icv(sc, xs, XHCI_ICI_INPUT_CONTROL); 1758 cp[0] = htole32(XHCI_INCTX_0_DROP_MASK(dci)); 1759 cp[1] = htole32(0); 1760 1761 /* XXX should be most significant one, not dci? */ 1762 cp = xhci_slot_get_icv(sc, xs, xhci_dci_to_ici(XHCI_DCI_SLOT)); 1763 cp[0] = htole32(XHCI_SCTX_0_CTX_NUM_SET(dci)); 1764 1765 /* sync input contexts before they are read from memory */ 1766 usb_syncmem(&xs->xs_ic_dma, 0, sc->sc_pgsz, BUS_DMASYNC_PREWRITE); 1767 1768 trb.trb_0 = xhci_slot_get_icp(sc, xs, 0); 1769 trb.trb_2 = 0; 1770 trb.trb_3 = XHCI_TRB_3_SLOT_SET(xs->xs_idx) | 1771 XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_CONFIGURE_EP); 1772 1773 (void)xhci_do_command_locked(sc, &trb, USBD_DEFAULT_TIMEOUT); 1774 usb_syncmem(&xs->xs_dc_dma, 0, sc->sc_pgsz, BUS_DMASYNC_POSTREAD); 1775 } 1776 1777 /* 1778 * Abort transfer. 1779 * Called with sc_lock held. 1780 * May be called from softintr context. 1781 */ 1782 static void 1783 xhci_abort_xfer(struct usbd_xfer *xfer, usbd_status status) 1784 { 1785 struct xhci_softc * const sc = XHCI_XFER2SC(xfer); 1786 1787 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 1788 DPRINTFN(4, "xfer %p pipe %p status %d", 1789 xfer, xfer->ux_pipe, status, 0); 1790 1791 KASSERT(mutex_owned(&sc->sc_lock)); 1792 1793 if (sc->sc_dying) { 1794 /* If we're dying, just do the software part. */ 1795 DPRINTFN(4, "dying", 0, 0, 0, 0); 1796 xfer->ux_status = status; /* make software ignore it */ 1797 callout_stop(&xfer->ux_callout); 1798 usb_transfer_complete(xfer); 1799 return; 1800 } 1801 1802 /* XXX need more stuff */ 1803 xfer->ux_status = status; 1804 callout_stop(&xfer->ux_callout); 1805 usb_transfer_complete(xfer); 1806 DPRINTFN(14, "end", 0, 0, 0, 0); 1807 1808 KASSERT(mutex_owned(&sc->sc_lock)); 1809 } 1810 1811 /* 1812 * Recover STALLed endpoint. 1813 * xHCI 1.1 sect 4.10.2.1 1814 * Issue RESET_EP to recover halt condition and SET_TR_DEQUEUE to remove 1815 * all transfers on transfer ring. 1816 * These are done in thread context asynchronously. 1817 */ 1818 static void 1819 xhci_clear_endpoint_stall_async_task(void *cookie) 1820 { 1821 struct usbd_xfer * const xfer = cookie; 1822 struct xhci_softc * const sc = XHCI_XFER2SC(xfer); 1823 struct xhci_slot * const xs = xfer->ux_pipe->up_dev->ud_hcpriv; 1824 const u_int dci = xhci_ep_get_dci(xfer->ux_pipe->up_endpoint->ue_edesc); 1825 struct xhci_ring * const tr = &xs->xs_ep[dci].xe_tr; 1826 1827 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 1828 DPRINTFN(4, "xfer %p slot %u dci %u", xfer, xs->xs_idx, dci, 0); 1829 1830 xhci_reset_endpoint(xfer->ux_pipe); 1831 xhci_set_dequeue(xfer->ux_pipe); 1832 1833 mutex_enter(&sc->sc_lock); 1834 tr->is_halted = false; 1835 usb_transfer_complete(xfer); 1836 mutex_exit(&sc->sc_lock); 1837 DPRINTFN(4, "ends", 0, 0, 0, 0); 1838 } 1839 1840 static usbd_status 1841 xhci_clear_endpoint_stall_async(struct usbd_xfer *xfer) 1842 { 1843 struct xhci_softc * const sc = XHCI_XFER2SC(xfer); 1844 struct xhci_pipe * const xp = (struct xhci_pipe *)xfer->ux_pipe; 1845 1846 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 1847 DPRINTFN(4, "xfer %p", xfer, 0, 0, 0); 1848 1849 if (sc->sc_dying) { 1850 return USBD_IOERROR; 1851 } 1852 1853 usb_init_task(&xp->xp_async_task, 1854 xhci_clear_endpoint_stall_async_task, xfer, USB_TASKQ_MPSAFE); 1855 usb_add_task(xfer->ux_pipe->up_dev, &xp->xp_async_task, USB_TASKQ_HC); 1856 DPRINTFN(4, "ends", 0, 0, 0, 0); 1857 1858 return USBD_NORMAL_COMPLETION; 1859 } 1860 1861 /* Process roothub port status/change events and notify to uhub_intr. */ 1862 static void 1863 xhci_rhpsc(struct xhci_softc * const sc, u_int port) 1864 { 1865 struct usbd_xfer * const xfer = sc->sc_intrxfer; 1866 uint8_t *p; 1867 1868 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 1869 DPRINTFN(4, "xhci%d: port %u status change", device_unit(sc->sc_dev), 1870 port, 0, 0); 1871 1872 if (xfer == NULL) 1873 return; 1874 1875 if (port > sc->sc_maxports) 1876 return; 1877 1878 p = xfer->ux_buf; 1879 memset(p, 0, xfer->ux_length); 1880 p[port/NBBY] |= 1 << (port%NBBY); 1881 xfer->ux_actlen = xfer->ux_length; 1882 xfer->ux_status = USBD_NORMAL_COMPLETION; 1883 usb_transfer_complete(xfer); 1884 } 1885 1886 /* Process Transfer Events */ 1887 static void 1888 xhci_event_transfer(struct xhci_softc * const sc, 1889 const struct xhci_trb * const trb) 1890 { 1891 uint64_t trb_0; 1892 uint32_t trb_2, trb_3; 1893 uint8_t trbcode; 1894 u_int slot, dci; 1895 struct xhci_slot *xs; 1896 struct xhci_ring *xr; 1897 struct xhci_xfer *xx; 1898 struct usbd_xfer *xfer; 1899 usbd_status err; 1900 1901 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 1902 1903 trb_0 = le64toh(trb->trb_0); 1904 trb_2 = le32toh(trb->trb_2); 1905 trb_3 = le32toh(trb->trb_3); 1906 trbcode = XHCI_TRB_2_ERROR_GET(trb_2); 1907 slot = XHCI_TRB_3_SLOT_GET(trb_3); 1908 dci = XHCI_TRB_3_EP_GET(trb_3); 1909 xs = &sc->sc_slots[slot]; 1910 xr = &xs->xs_ep[dci].xe_tr; 1911 1912 /* sanity check */ 1913 KASSERTMSG(xs->xs_idx != 0 && xs->xs_idx <= sc->sc_maxslots, 1914 "invalid xs_idx %u slot %u", xs->xs_idx, slot); 1915 1916 int idx = 0; 1917 if ((trb_3 & XHCI_TRB_3_ED_BIT) == 0) { 1918 if (xhci_trb_get_idx(xr, trb_0, &idx)) { 1919 DPRINTFN(0, "invalid trb_0 0x%"PRIx64, trb_0, 0, 0, 0); 1920 return; 1921 } 1922 xx = xr->xr_cookies[idx]; 1923 1924 /* 1925 * If endpoint is stopped between TDs, TRB pointer points at 1926 * next TRB, however, it is not put yet or is a garbage TRB. 1927 * That's why xr_cookies may be NULL or look like broken. 1928 * Note: this ev happens only when hciversion >= 1.0 or 1929 * hciversion == 0.96 and FSE of hcc1 is set. 1930 */ 1931 if (xx == NULL || trbcode == XHCI_TRB_ERROR_LENGTH) { 1932 DPRINTFN(1, "xx NULL: #%u: cookie %p: code %u trb_0 %" 1933 PRIx64, idx, xx, trbcode, trb_0); 1934 } 1935 } else { 1936 /* When ED != 0, trb_0 is kaddr of struct xhci_xfer. */ 1937 xx = (void *)(uintptr_t)(trb_0 & ~0x3); 1938 } 1939 /* XXX this may not happen */ 1940 if (xx == NULL) { 1941 DPRINTFN(1, "xfer done: xx is NULL", 0, 0, 0, 0); 1942 return; 1943 } 1944 xfer = &xx->xx_xfer; 1945 /* XXX this may happen when detaching */ 1946 if (xfer == NULL) { 1947 DPRINTFN(1, "xx(%p)->xx_xfer is NULL trb_0 %#"PRIx64, 1948 xx, trb_0, 0, 0); 1949 return; 1950 } 1951 DPRINTFN(14, "xfer %p", xfer, 0, 0, 0); 1952 /* XXX I dunno why this happens */ 1953 KASSERTMSG(xfer->ux_pipe != NULL, "xfer(%p)->ux_pipe is NULL", xfer); 1954 1955 if (!xfer->ux_pipe->up_repeat && 1956 SIMPLEQ_EMPTY(&xfer->ux_pipe->up_queue)) { 1957 DPRINTFN(1, "xfer(%p)->pipe not queued", xfer, 0, 0, 0); 1958 return; 1959 } 1960 1961 /* 4.11.5.2 Event Data TRB */ 1962 if ((trb_3 & XHCI_TRB_3_ED_BIT) != 0) { 1963 DPRINTFN(14, "transfer Event Data: 0x%016"PRIx64" 0x%08"PRIx32 1964 " %02x", trb_0, XHCI_TRB_2_REM_GET(trb_2), trbcode, 0); 1965 if ((trb_0 & 0x3) == 0x3) { 1966 xfer->ux_actlen = XHCI_TRB_2_REM_GET(trb_2); 1967 } 1968 } 1969 1970 switch (trbcode) { 1971 case XHCI_TRB_ERROR_SHORT_PKT: 1972 case XHCI_TRB_ERROR_SUCCESS: 1973 xfer->ux_actlen = 1974 xfer->ux_length - XHCI_TRB_2_REM_GET(trb_2); 1975 err = USBD_NORMAL_COMPLETION; 1976 break; 1977 case XHCI_TRB_ERROR_STALL: 1978 case XHCI_TRB_ERROR_BABBLE: 1979 DPRINTFN(1, "ERR %u slot %u dci %u", trbcode, slot, dci, 0); 1980 xr->is_halted = true; 1981 err = USBD_STALLED; 1982 /* 1983 * Stalled endpoints can be recoverd by issuing 1984 * command TRB TYPE_RESET_EP on xHCI instead of 1985 * issuing request CLEAR_FEATURE UF_ENDPOINT_HALT 1986 * on the endpoint. However, this function may be 1987 * called from softint context (e.g. from umass), 1988 * in that case driver gets KASSERT in cv_timedwait 1989 * in xhci_do_command. 1990 * To avoid this, this runs reset_endpoint and 1991 * usb_transfer_complete in usb task thread 1992 * asynchronously (and then umass issues clear 1993 * UF_ENDPOINT_HALT). 1994 */ 1995 xfer->ux_status = err; 1996 xhci_clear_endpoint_stall_async(xfer); 1997 return; 1998 default: 1999 DPRINTFN(1, "ERR %u slot %u dci %u", trbcode, slot, dci, 0); 2000 err = USBD_IOERROR; 2001 break; 2002 } 2003 xfer->ux_status = err; 2004 2005 if ((trb_3 & XHCI_TRB_3_ED_BIT) != 0) { 2006 if ((trb_0 & 0x3) == 0x0) { 2007 callout_stop(&xfer->ux_callout); 2008 usb_transfer_complete(xfer); 2009 } 2010 } else { 2011 callout_stop(&xfer->ux_callout); 2012 usb_transfer_complete(xfer); 2013 } 2014 } 2015 2016 /* Process Command complete events */ 2017 static void 2018 xhci_event_cmd(struct xhci_softc * const sc, 2019 const struct xhci_trb * const trb) 2020 { 2021 uint64_t trb_0; 2022 uint32_t trb_2, trb_3; 2023 2024 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 2025 2026 trb_0 = le64toh(trb->trb_0); 2027 trb_2 = le32toh(trb->trb_2); 2028 trb_3 = le32toh(trb->trb_3); 2029 2030 if (trb_0 == sc->sc_command_addr) { 2031 sc->sc_result_trb.trb_0 = trb_0; 2032 sc->sc_result_trb.trb_2 = trb_2; 2033 sc->sc_result_trb.trb_3 = trb_3; 2034 if (XHCI_TRB_2_ERROR_GET(trb_2) != 2035 XHCI_TRB_ERROR_SUCCESS) { 2036 DPRINTFN(1, "command completion " 2037 "failure: 0x%016"PRIx64" 0x%08"PRIx32" " 2038 "0x%08"PRIx32, trb_0, trb_2, trb_3, 0); 2039 } 2040 cv_signal(&sc->sc_command_cv); 2041 } else { 2042 DPRINTFN(1, "spurious event: %p 0x%016"PRIx64" " 2043 "0x%08"PRIx32" 0x%08"PRIx32, trb, trb_0, 2044 trb_2, trb_3); 2045 } 2046 } 2047 2048 /* 2049 * Process events. 2050 * called from xhci_softintr 2051 */ 2052 static void 2053 xhci_handle_event(struct xhci_softc * const sc, 2054 const struct xhci_trb * const trb) 2055 { 2056 uint64_t trb_0; 2057 uint32_t trb_2, trb_3; 2058 2059 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 2060 2061 trb_0 = le64toh(trb->trb_0); 2062 trb_2 = le32toh(trb->trb_2); 2063 trb_3 = le32toh(trb->trb_3); 2064 2065 DPRINTFN(14, "event: %p 0x%016"PRIx64" 0x%08"PRIx32" 0x%08"PRIx32, 2066 trb, trb_0, trb_2, trb_3); 2067 2068 /* 2069 * 4.11.3.1, 6.4.2.1 2070 * TRB Pointer is invalid for these completion codes. 2071 */ 2072 switch (XHCI_TRB_2_ERROR_GET(trb_2)) { 2073 case XHCI_TRB_ERROR_RING_UNDERRUN: 2074 case XHCI_TRB_ERROR_RING_OVERRUN: 2075 case XHCI_TRB_ERROR_VF_RING_FULL: 2076 return; 2077 default: 2078 if (trb_0 == 0) { 2079 return; 2080 } 2081 break; 2082 } 2083 2084 switch (XHCI_TRB_3_TYPE_GET(trb_3)) { 2085 case XHCI_TRB_EVENT_TRANSFER: 2086 xhci_event_transfer(sc, trb); 2087 break; 2088 case XHCI_TRB_EVENT_CMD_COMPLETE: 2089 xhci_event_cmd(sc, trb); 2090 break; 2091 case XHCI_TRB_EVENT_PORT_STS_CHANGE: 2092 xhci_rhpsc(sc, (uint32_t)((trb_0 >> 24) & 0xff)); 2093 break; 2094 default: 2095 break; 2096 } 2097 } 2098 2099 static void 2100 xhci_softintr(void *v) 2101 { 2102 struct usbd_bus * const bus = v; 2103 struct xhci_softc * const sc = XHCI_BUS2SC(bus); 2104 struct xhci_ring * const er = &sc->sc_er; 2105 struct xhci_trb *trb; 2106 int i, j, k; 2107 2108 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 2109 2110 KASSERT(sc->sc_bus.ub_usepolling || mutex_owned(&sc->sc_lock)); 2111 2112 i = er->xr_ep; 2113 j = er->xr_cs; 2114 2115 DPRINTFN(16, "xr_ep %d xr_cs %d", i, j, 0, 0); 2116 2117 while (1) { 2118 usb_syncmem(&er->xr_dma, XHCI_TRB_SIZE * i, XHCI_TRB_SIZE, 2119 BUS_DMASYNC_POSTREAD); 2120 trb = &er->xr_trb[i]; 2121 k = (le32toh(trb->trb_3) & XHCI_TRB_3_CYCLE_BIT) ? 1 : 0; 2122 2123 if (j != k) 2124 break; 2125 2126 xhci_handle_event(sc, trb); 2127 2128 i++; 2129 if (i == XHCI_EVENT_RING_TRBS) { 2130 i = 0; 2131 j ^= 1; 2132 } 2133 } 2134 2135 er->xr_ep = i; 2136 er->xr_cs = j; 2137 2138 xhci_rt_write_8(sc, XHCI_ERDP(0), xhci_ring_trbp(er, er->xr_ep) | 2139 XHCI_ERDP_LO_BUSY); 2140 2141 DPRINTFN(16, "ends", 0, 0, 0, 0); 2142 2143 return; 2144 } 2145 2146 static void 2147 xhci_poll(struct usbd_bus *bus) 2148 { 2149 struct xhci_softc * const sc = XHCI_BUS2SC(bus); 2150 2151 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 2152 2153 mutex_spin_enter(&sc->sc_intr_lock); 2154 xhci_intr1(sc); 2155 mutex_spin_exit(&sc->sc_intr_lock); 2156 2157 return; 2158 } 2159 2160 static struct usbd_xfer * 2161 xhci_allocx(struct usbd_bus *bus, unsigned int nframes) 2162 { 2163 struct xhci_softc * const sc = XHCI_BUS2SC(bus); 2164 struct usbd_xfer *xfer; 2165 2166 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 2167 2168 xfer = pool_cache_get(sc->sc_xferpool, PR_NOWAIT); 2169 if (xfer != NULL) { 2170 memset(xfer, 0, sizeof(struct xhci_xfer)); 2171 #ifdef DIAGNOSTIC 2172 xfer->ux_state = XFER_BUSY; 2173 #endif 2174 } 2175 2176 return xfer; 2177 } 2178 2179 static void 2180 xhci_freex(struct usbd_bus *bus, struct usbd_xfer *xfer) 2181 { 2182 struct xhci_softc * const sc = XHCI_BUS2SC(bus); 2183 2184 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 2185 2186 #ifdef DIAGNOSTIC 2187 if (xfer->ux_state != XFER_BUSY) { 2188 DPRINTFN(0, "xfer=%p not busy, 0x%08x", 2189 xfer, xfer->ux_state, 0, 0); 2190 } 2191 xfer->ux_state = XFER_FREE; 2192 #endif 2193 pool_cache_put(sc->sc_xferpool, xfer); 2194 } 2195 2196 static void 2197 xhci_get_lock(struct usbd_bus *bus, kmutex_t **lock) 2198 { 2199 struct xhci_softc * const sc = XHCI_BUS2SC(bus); 2200 2201 *lock = &sc->sc_lock; 2202 } 2203 2204 extern uint32_t usb_cookie_no; 2205 2206 /* 2207 * xHCI 4.3 2208 * Called when uhub_explore finds a new device (via usbd_new_device). 2209 * Port initialization and speed detection (4.3.1) are already done in uhub.c. 2210 * This function does: 2211 * Allocate and construct dev structure of default endpoint (ep0). 2212 * Allocate and open pipe of ep0. 2213 * Enable slot and initialize slot context. 2214 * Set Address. 2215 * Read initial device descriptor. 2216 * Determine initial MaxPacketSize (mps) by speed. 2217 * Read full device descriptor. 2218 * Register this device. 2219 */ 2220 static usbd_status 2221 xhci_new_device(device_t parent, struct usbd_bus *bus, int depth, 2222 int speed, int port, struct usbd_port *up) 2223 { 2224 struct xhci_softc * const sc = XHCI_BUS2SC(bus); 2225 struct usbd_device *dev; 2226 usbd_status err; 2227 usb_device_descriptor_t *dd; 2228 struct usbd_device *hub; 2229 struct usbd_device *adev; 2230 int rhport = 0; 2231 struct xhci_slot *xs; 2232 uint32_t *cp; 2233 uint32_t route = 0; 2234 uint8_t slot = 0; 2235 uint8_t addr; 2236 2237 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 2238 DPRINTFN(4, "port=%d depth=%d speed=%d upport %d", 2239 port, depth, speed, up->up_portno); 2240 2241 dev = kmem_zalloc(sizeof(*dev), KM_SLEEP); 2242 if (dev == NULL) 2243 return USBD_NOMEM; 2244 2245 dev->ud_bus = bus; 2246 2247 /* Set up default endpoint handle. */ 2248 dev->ud_ep0.ue_edesc = &dev->ud_ep0desc; 2249 2250 /* Set up default endpoint descriptor. */ 2251 dev->ud_ep0desc.bLength = USB_ENDPOINT_DESCRIPTOR_SIZE; 2252 dev->ud_ep0desc.bDescriptorType = UDESC_ENDPOINT; 2253 dev->ud_ep0desc.bEndpointAddress = USB_CONTROL_ENDPOINT; 2254 dev->ud_ep0desc.bmAttributes = UE_CONTROL; 2255 /* 4.3, 4.8.2.1 */ 2256 switch (speed) { 2257 case USB_SPEED_SUPER: 2258 case USB_SPEED_SUPER_PLUS: 2259 USETW(dev->ud_ep0desc.wMaxPacketSize, USB_3_MAX_CTRL_PACKET); 2260 break; 2261 case USB_SPEED_FULL: 2262 /* XXX using 64 as initial mps of ep0 in FS */ 2263 case USB_SPEED_HIGH: 2264 USETW(dev->ud_ep0desc.wMaxPacketSize, USB_2_MAX_CTRL_PACKET); 2265 break; 2266 case USB_SPEED_LOW: 2267 default: 2268 USETW(dev->ud_ep0desc.wMaxPacketSize, USB_MAX_IPACKET); 2269 break; 2270 } 2271 dev->ud_ep0desc.bInterval = 0; 2272 2273 /* doesn't matter, just don't let it uninitialized */ 2274 dev->ud_ep0.ue_toggle = 0; 2275 2276 DPRINTFN(4, "up %p portno %d", up, up->up_portno, 0, 0); 2277 2278 dev->ud_quirks = &usbd_no_quirk; 2279 dev->ud_addr = 0; 2280 dev->ud_ddesc.bMaxPacketSize = 0; 2281 dev->ud_depth = depth; 2282 dev->ud_powersrc = up; 2283 dev->ud_myhub = up->up_parent; 2284 2285 up->up_dev = dev; 2286 2287 /* Locate root hub port */ 2288 for (hub = dev; hub != NULL; hub = hub->ud_myhub) { 2289 uint32_t dep; 2290 2291 DPRINTFN(4, "hub %p depth %d upport %p upportno %d", 2292 hub, hub->ud_depth, hub->ud_powersrc, 2293 hub->ud_powersrc ? hub->ud_powersrc->up_portno : -1); 2294 2295 if (hub->ud_powersrc == NULL) 2296 break; 2297 dep = hub->ud_depth; 2298 if (dep == 0) 2299 break; 2300 rhport = hub->ud_powersrc->up_portno; 2301 if (dep > USB_HUB_MAX_DEPTH) 2302 continue; 2303 2304 route |= 2305 (rhport > UHD_SS_NPORTS_MAX ? UHD_SS_NPORTS_MAX : rhport) 2306 << ((dep - 1) * 4); 2307 } 2308 route = route >> 4; 2309 DPRINTFN(4, "rhport %d Route %05x hub %p", rhport, route, hub, 0); 2310 2311 /* Locate port on upstream high speed hub */ 2312 for (adev = dev, hub = up->up_parent; 2313 hub != NULL && hub->ud_speed != USB_SPEED_HIGH; 2314 adev = hub, hub = hub->ud_myhub) 2315 ; 2316 if (hub) { 2317 int p; 2318 for (p = 0; p < hub->ud_hub->uh_hubdesc.bNbrPorts; p++) { 2319 if (hub->ud_hub->uh_ports[p].up_dev == adev) { 2320 dev->ud_myhsport = &hub->ud_hub->uh_ports[p]; 2321 goto found; 2322 } 2323 } 2324 panic("xhci_new_device: cannot find HS port"); 2325 found: 2326 DPRINTFN(4, "high speed port %d", p, 0, 0, 0); 2327 } else { 2328 dev->ud_myhsport = NULL; 2329 } 2330 2331 dev->ud_speed = speed; 2332 dev->ud_langid = USBD_NOLANG; 2333 dev->ud_cookie.cookie = ++usb_cookie_no; 2334 2335 /* Establish the default pipe. */ 2336 err = usbd_setup_pipe(dev, 0, &dev->ud_ep0, USBD_DEFAULT_INTERVAL, 2337 &dev->ud_pipe0); 2338 if (err) { 2339 goto bad; 2340 } 2341 2342 dd = &dev->ud_ddesc; 2343 2344 if ((depth == 0) && (port == 0)) { 2345 KASSERT(bus->ub_devices[dev->ud_addr] == NULL); 2346 bus->ub_devices[dev->ud_addr] = dev; 2347 err = usbd_get_initial_ddesc(dev, dd); 2348 if (err) 2349 goto bad; 2350 err = usbd_reload_device_desc(dev); 2351 if (err) 2352 goto bad; 2353 } else { 2354 /* 4.3.2 */ 2355 err = xhci_enable_slot(sc, &slot); 2356 if (err) 2357 goto bad; 2358 xs = &sc->sc_slots[slot]; 2359 dev->ud_hcpriv = xs; 2360 /* 4.3.3 initialize slot structure */ 2361 err = xhci_init_slot(dev, slot); 2362 if (err) { 2363 dev->ud_hcpriv = NULL; 2364 /* 2365 * We have to disable_slot here because 2366 * xs->xs_idx == 0 when xhci_init_slot fails, 2367 * in that case usbd_remove_dev won't work. 2368 */ 2369 mutex_enter(&sc->sc_lock); 2370 xhci_disable_slot(sc, slot); 2371 mutex_exit(&sc->sc_lock); 2372 goto bad; 2373 } 2374 2375 /* 4.3.4 Address Assignment */ 2376 err = xhci_set_address(dev, slot, route, rhport, false); 2377 if (err) 2378 goto bad; 2379 2380 /* Allow device time to set new address */ 2381 usbd_delay_ms(dev, USB_SET_ADDRESS_SETTLE); 2382 cp = xhci_slot_get_dcv(sc, xs, XHCI_DCI_SLOT); 2383 //hexdump("slot context", cp, sc->sc_ctxsz); 2384 addr = XHCI_SCTX_3_DEV_ADDR_GET(cp[3]); 2385 DPRINTFN(4, "device address %u", addr, 0, 0, 0); 2386 /* XXX ensure we know when the hardware does something 2387 we can't yet cope with */ 2388 KASSERT(addr >= 1 && addr <= 127); 2389 dev->ud_addr = addr; 2390 /* XXX dev->ud_addr not necessarily unique on bus */ 2391 KASSERT(bus->ub_devices[dev->ud_addr] == NULL); 2392 bus->ub_devices[dev->ud_addr] = dev; 2393 2394 err = usbd_get_initial_ddesc(dev, dd); 2395 if (err) 2396 goto bad; 2397 /* 4.8.2.1 */ 2398 if (USB_IS_SS(speed)) { 2399 if (dd->bMaxPacketSize != 9) { 2400 printf("%s: invalid mps 2^%u for SS ep0," 2401 " using 512\n", 2402 device_xname(sc->sc_dev), 2403 dd->bMaxPacketSize); 2404 dd->bMaxPacketSize = 9; 2405 } 2406 USETW(dev->ud_ep0desc.wMaxPacketSize, 2407 (1 << dd->bMaxPacketSize)); 2408 } else 2409 USETW(dev->ud_ep0desc.wMaxPacketSize, 2410 dd->bMaxPacketSize); 2411 DPRINTFN(4, "bMaxPacketSize %u", dd->bMaxPacketSize, 0, 0, 0); 2412 xhci_update_ep0_mps(sc, xs, 2413 UGETW(dev->ud_ep0desc.wMaxPacketSize)); 2414 err = usbd_reload_device_desc(dev); 2415 if (err) 2416 goto bad; 2417 } 2418 2419 DPRINTFN(1, "adding unit addr=%d, rev=%02x,", 2420 dev->ud_addr, UGETW(dd->bcdUSB), 0, 0); 2421 DPRINTFN(1, " class=%d, subclass=%d, protocol=%d,", 2422 dd->bDeviceClass, dd->bDeviceSubClass, 2423 dd->bDeviceProtocol, 0); 2424 DPRINTFN(1, " mps=%d, len=%d, noconf=%d, speed=%d", 2425 dd->bMaxPacketSize, dd->bLength, dd->bNumConfigurations, 2426 dev->ud_speed); 2427 2428 usbd_get_device_strings(dev); 2429 2430 usbd_add_dev_event(USB_EVENT_DEVICE_ATTACH, dev); 2431 2432 if ((depth == 0) && (port == 0)) { 2433 usbd_attach_roothub(parent, dev); 2434 DPRINTFN(1, "root_hub %p", bus->ub_roothub, 0, 0, 0); 2435 return USBD_NORMAL_COMPLETION; 2436 } 2437 2438 2439 err = usbd_probe_and_attach(parent, dev, port, dev->ud_addr); 2440 bad: 2441 if (err != USBD_NORMAL_COMPLETION) { 2442 usbd_remove_device(dev, up); 2443 } 2444 2445 return err; 2446 } 2447 2448 static usbd_status 2449 xhci_ring_init(struct xhci_softc * const sc, struct xhci_ring * const xr, 2450 size_t ntrb, size_t align) 2451 { 2452 usbd_status err; 2453 size_t size = ntrb * XHCI_TRB_SIZE; 2454 2455 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 2456 2457 err = usb_allocmem(&sc->sc_bus, size, align, &xr->xr_dma); 2458 if (err) 2459 return err; 2460 mutex_init(&xr->xr_lock, MUTEX_DEFAULT, IPL_SOFTUSB); 2461 xr->xr_cookies = kmem_zalloc(sizeof(*xr->xr_cookies) * ntrb, KM_SLEEP); 2462 xr->xr_trb = xhci_ring_trbv(xr, 0); 2463 xr->xr_ntrb = ntrb; 2464 xr->xr_ep = 0; 2465 xr->xr_cs = 1; 2466 memset(xr->xr_trb, 0, size); 2467 usb_syncmem(&xr->xr_dma, 0, size, BUS_DMASYNC_PREWRITE); 2468 xr->is_halted = false; 2469 2470 return USBD_NORMAL_COMPLETION; 2471 } 2472 2473 static void 2474 xhci_ring_free(struct xhci_softc * const sc, struct xhci_ring * const xr) 2475 { 2476 usb_freemem(&sc->sc_bus, &xr->xr_dma); 2477 mutex_destroy(&xr->xr_lock); 2478 kmem_free(xr->xr_cookies, sizeof(*xr->xr_cookies) * xr->xr_ntrb); 2479 } 2480 2481 static void 2482 xhci_ring_put(struct xhci_softc * const sc, struct xhci_ring * const xr, 2483 void *cookie, struct xhci_trb * const trbs, size_t ntrbs) 2484 { 2485 size_t i; 2486 u_int ri; 2487 u_int cs; 2488 uint64_t parameter; 2489 uint32_t status; 2490 uint32_t control; 2491 2492 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 2493 2494 KASSERT(ntrbs <= XHCI_XFER_NTRB); 2495 for (i = 0; i < ntrbs; i++) { 2496 DPRINTFN(12, "xr %p trbs %p num %zu", xr, trbs, i, 0); 2497 DPRINTFN(12, " %016"PRIx64" %08"PRIx32" %08"PRIx32, 2498 trbs[i].trb_0, trbs[i].trb_2, trbs[i].trb_3, 0); 2499 KASSERT(XHCI_TRB_3_TYPE_GET(trbs[i].trb_3) != 2500 XHCI_TRB_TYPE_LINK); 2501 } 2502 2503 DPRINTFN(12, "%p xr_ep 0x%x xr_cs %u", xr, xr->xr_ep, xr->xr_cs, 0); 2504 2505 ri = xr->xr_ep; 2506 cs = xr->xr_cs; 2507 2508 /* 2509 * Although the xhci hardware can do scatter/gather dma from 2510 * arbitrary sized buffers, there is a non-obvious restriction 2511 * that a LINK trb is only allowed at the end of a burst of 2512 * transfers - which might be 16kB. 2513 * Arbitrary aligned LINK trb definitely fail on Ivy bridge. 2514 * The simple solution is not to allow a LINK trb in the middle 2515 * of anything - as here. 2516 * XXX: (dsl) There are xhci controllers out there (eg some made by 2517 * ASMedia) that seem to lock up if they process a LINK trb but 2518 * cannot process the linked-to trb yet. 2519 * The code should write the 'cycle' bit on the link trb AFTER 2520 * adding the other trb. 2521 */ 2522 if (ri + ntrbs >= (xr->xr_ntrb - 1)) { 2523 parameter = xhci_ring_trbp(xr, 0); 2524 status = 0; 2525 control = XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_LINK) | 2526 XHCI_TRB_3_TC_BIT | (cs ? XHCI_TRB_3_CYCLE_BIT : 0); 2527 xhci_trb_put(&xr->xr_trb[ri], parameter, status, control); 2528 usb_syncmem(&xr->xr_dma, XHCI_TRB_SIZE * ri, XHCI_TRB_SIZE * 1, 2529 BUS_DMASYNC_PREWRITE); 2530 xr->xr_cookies[ri] = NULL; 2531 xr->xr_ep = 0; 2532 xr->xr_cs ^= 1; 2533 ri = xr->xr_ep; 2534 cs = xr->xr_cs; 2535 } 2536 2537 ri++; 2538 2539 /* Write any subsequent TRB first */ 2540 for (i = 1; i < ntrbs; i++) { 2541 parameter = trbs[i].trb_0; 2542 status = trbs[i].trb_2; 2543 control = trbs[i].trb_3; 2544 2545 if (cs) { 2546 control |= XHCI_TRB_3_CYCLE_BIT; 2547 } else { 2548 control &= ~XHCI_TRB_3_CYCLE_BIT; 2549 } 2550 2551 xhci_trb_put(&xr->xr_trb[ri], parameter, status, control); 2552 usb_syncmem(&xr->xr_dma, XHCI_TRB_SIZE * ri, XHCI_TRB_SIZE * 1, 2553 BUS_DMASYNC_PREWRITE); 2554 xr->xr_cookies[ri] = cookie; 2555 ri++; 2556 } 2557 2558 /* Write the first TRB last */ 2559 i = 0; 2560 parameter = trbs[i].trb_0; 2561 status = trbs[i].trb_2; 2562 control = trbs[i].trb_3; 2563 2564 if (xr->xr_cs) { 2565 control |= XHCI_TRB_3_CYCLE_BIT; 2566 } else { 2567 control &= ~XHCI_TRB_3_CYCLE_BIT; 2568 } 2569 2570 xhci_trb_put(&xr->xr_trb[xr->xr_ep], parameter, status, control); 2571 usb_syncmem(&xr->xr_dma, XHCI_TRB_SIZE * xr->xr_ep, XHCI_TRB_SIZE * 1, 2572 BUS_DMASYNC_PREWRITE); 2573 xr->xr_cookies[xr->xr_ep] = cookie; 2574 2575 xr->xr_ep = ri; 2576 xr->xr_cs = cs; 2577 2578 DPRINTFN(12, "%p xr_ep 0x%x xr_cs %u", xr, xr->xr_ep, xr->xr_cs, 0); 2579 } 2580 2581 /* 2582 * Stop execution commands, purge all commands on command ring, and 2583 * rewind enqueue pointer. 2584 */ 2585 static void 2586 xhci_abort_command(struct xhci_softc *sc) 2587 { 2588 struct xhci_ring * const cr = &sc->sc_cr; 2589 uint64_t crcr; 2590 int i; 2591 2592 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 2593 DPRINTFN(14, "command %#"PRIx64" timeout, aborting", 2594 sc->sc_command_addr, 0, 0, 0); 2595 2596 mutex_enter(&cr->xr_lock); 2597 2598 /* 4.6.1.2 Aborting a Command */ 2599 crcr = xhci_op_read_8(sc, XHCI_CRCR); 2600 xhci_op_write_8(sc, XHCI_CRCR, crcr | XHCI_CRCR_LO_CA); 2601 2602 for (i = 0; i < 500; i++) { 2603 crcr = xhci_op_read_8(sc, XHCI_CRCR); 2604 if ((crcr & XHCI_CRCR_LO_CRR) == 0) 2605 break; 2606 usb_delay_ms(&sc->sc_bus, 1); 2607 } 2608 if ((crcr & XHCI_CRCR_LO_CRR) != 0) { 2609 DPRINTFN(1, "Command Abort timeout", 0, 0, 0, 0); 2610 /* reset HC here? */ 2611 } 2612 2613 /* reset command ring dequeue pointer */ 2614 cr->xr_ep = 0; 2615 cr->xr_cs = 1; 2616 xhci_op_write_8(sc, XHCI_CRCR, xhci_ring_trbp(cr, 0) | cr->xr_cs); 2617 2618 mutex_exit(&cr->xr_lock); 2619 } 2620 2621 /* 2622 * Put a command on command ring, ring bell, set timer, and cv_timedwait. 2623 * Command completion is notified by cv_signal from xhci_handle_event 2624 * (called from interrupt from xHCI), or timed-out. 2625 * Command validation is performed in xhci_handle_event by checking if 2626 * trb_0 in CMD_COMPLETE TRB and sc->sc_command_addr are identical. 2627 */ 2628 static usbd_status 2629 xhci_do_command_locked(struct xhci_softc * const sc, struct xhci_trb * const trb, 2630 int timeout) 2631 { 2632 struct xhci_ring * const cr = &sc->sc_cr; 2633 usbd_status err; 2634 2635 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 2636 DPRINTFN(12, "input: 0x%016"PRIx64" 0x%08"PRIx32" 0x%08"PRIx32, 2637 trb->trb_0, trb->trb_2, trb->trb_3, 0); 2638 2639 KASSERTMSG(!cpu_intr_p() && !cpu_softintr_p(), "called from intr ctx"); 2640 KASSERT(mutex_owned(&sc->sc_lock)); 2641 2642 /* XXX KASSERT may fire when cv_timedwait unlocks sc_lock */ 2643 KASSERT(sc->sc_command_addr == 0); 2644 sc->sc_command_addr = xhci_ring_trbp(cr, cr->xr_ep); 2645 2646 mutex_enter(&cr->xr_lock); 2647 xhci_ring_put(sc, cr, NULL, trb, 1); 2648 mutex_exit(&cr->xr_lock); 2649 2650 xhci_db_write_4(sc, XHCI_DOORBELL(0), 0); 2651 2652 if (cv_timedwait(&sc->sc_command_cv, &sc->sc_lock, 2653 MAX(1, mstohz(timeout))) == EWOULDBLOCK) { 2654 xhci_abort_command(sc); 2655 err = USBD_TIMEOUT; 2656 goto timedout; 2657 } 2658 2659 trb->trb_0 = sc->sc_result_trb.trb_0; 2660 trb->trb_2 = sc->sc_result_trb.trb_2; 2661 trb->trb_3 = sc->sc_result_trb.trb_3; 2662 2663 DPRINTFN(12, "output: 0x%016"PRIx64" 0x%08"PRIx32" 0x%08"PRIx32"", 2664 trb->trb_0, trb->trb_2, trb->trb_3, 0); 2665 2666 switch (XHCI_TRB_2_ERROR_GET(trb->trb_2)) { 2667 case XHCI_TRB_ERROR_SUCCESS: 2668 err = USBD_NORMAL_COMPLETION; 2669 break; 2670 default: 2671 case 192 ... 223: 2672 err = USBD_IOERROR; 2673 break; 2674 case 224 ... 255: 2675 err = USBD_NORMAL_COMPLETION; 2676 break; 2677 } 2678 2679 timedout: 2680 sc->sc_command_addr = 0; 2681 return err; 2682 } 2683 2684 static usbd_status 2685 xhci_do_command(struct xhci_softc * const sc, struct xhci_trb * const trb, 2686 int timeout) 2687 { 2688 2689 mutex_enter(&sc->sc_lock); 2690 usbd_status ret = xhci_do_command_locked(sc, trb, timeout); 2691 mutex_exit(&sc->sc_lock); 2692 2693 return ret; 2694 } 2695 2696 static usbd_status 2697 xhci_enable_slot(struct xhci_softc * const sc, uint8_t * const slotp) 2698 { 2699 struct xhci_trb trb; 2700 usbd_status err; 2701 2702 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 2703 2704 trb.trb_0 = 0; 2705 trb.trb_2 = 0; 2706 trb.trb_3 = XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_ENABLE_SLOT); 2707 2708 err = xhci_do_command(sc, &trb, USBD_DEFAULT_TIMEOUT); 2709 if (err != USBD_NORMAL_COMPLETION) { 2710 return err; 2711 } 2712 2713 *slotp = XHCI_TRB_3_SLOT_GET(trb.trb_3); 2714 2715 return err; 2716 } 2717 2718 /* 2719 * xHCI 4.6.4 2720 * Deallocate ring and device/input context DMA buffers, and disable_slot. 2721 * All endpoints in the slot should be stopped. 2722 * Should be called with sc_lock held. 2723 */ 2724 static usbd_status 2725 xhci_disable_slot(struct xhci_softc * const sc, uint8_t slot) 2726 { 2727 struct xhci_trb trb; 2728 struct xhci_slot *xs; 2729 usbd_status err; 2730 2731 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 2732 2733 if (sc->sc_dying) 2734 return USBD_IOERROR; 2735 2736 trb.trb_0 = 0; 2737 trb.trb_2 = 0; 2738 trb.trb_3 = htole32( 2739 XHCI_TRB_3_SLOT_SET(slot) | 2740 XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_DISABLE_SLOT)); 2741 2742 err = xhci_do_command_locked(sc, &trb, USBD_DEFAULT_TIMEOUT); 2743 2744 if (!err) { 2745 xs = &sc->sc_slots[slot]; 2746 if (xs->xs_idx != 0) { 2747 xhci_free_slot(sc, xs, XHCI_DCI_SLOT + 1, 32); 2748 xhci_set_dcba(sc, 0, slot); 2749 memset(xs, 0, sizeof(*xs)); 2750 } 2751 } 2752 2753 return err; 2754 } 2755 2756 /* 2757 * Set address of device and transition slot state from ENABLED to ADDRESSED 2758 * if Block Setaddress Request (BSR) is false. 2759 * If BSR==true, transition slot state from ENABLED to DEFAULT. 2760 * see xHCI 1.1 4.5.3, 3.3.4 2761 * Should be called without sc_lock held. 2762 */ 2763 static usbd_status 2764 xhci_address_device(struct xhci_softc * const sc, 2765 uint64_t icp, uint8_t slot_id, bool bsr) 2766 { 2767 struct xhci_trb trb; 2768 usbd_status err; 2769 2770 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 2771 2772 trb.trb_0 = icp; 2773 trb.trb_2 = 0; 2774 trb.trb_3 = XHCI_TRB_3_SLOT_SET(slot_id) | 2775 XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_ADDRESS_DEVICE) | 2776 (bsr ? XHCI_TRB_3_BSR_BIT : 0); 2777 2778 err = xhci_do_command(sc, &trb, USBD_DEFAULT_TIMEOUT); 2779 2780 if (XHCI_TRB_2_ERROR_GET(trb.trb_2) == XHCI_TRB_ERROR_NO_SLOTS) 2781 err = USBD_NO_ADDR; 2782 2783 return err; 2784 } 2785 2786 static usbd_status 2787 xhci_update_ep0_mps(struct xhci_softc * const sc, 2788 struct xhci_slot * const xs, u_int mps) 2789 { 2790 struct xhci_trb trb; 2791 usbd_status err; 2792 uint32_t * cp; 2793 2794 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 2795 DPRINTFN(4, "slot %u mps %u", xs->xs_idx, mps, 0, 0); 2796 2797 cp = xhci_slot_get_icv(sc, xs, XHCI_ICI_INPUT_CONTROL); 2798 cp[0] = htole32(0); 2799 cp[1] = htole32(XHCI_INCTX_1_ADD_MASK(XHCI_DCI_EP_CONTROL)); 2800 2801 cp = xhci_slot_get_icv(sc, xs, xhci_dci_to_ici(XHCI_DCI_EP_CONTROL)); 2802 cp[1] = htole32(XHCI_EPCTX_1_MAXP_SIZE_SET(mps)); 2803 2804 /* sync input contexts before they are read from memory */ 2805 usb_syncmem(&xs->xs_ic_dma, 0, sc->sc_pgsz, BUS_DMASYNC_PREWRITE); 2806 hexdump("input context", xhci_slot_get_icv(sc, xs, 0), 2807 sc->sc_ctxsz * 4); 2808 2809 trb.trb_0 = xhci_slot_get_icp(sc, xs, 0); 2810 trb.trb_2 = 0; 2811 trb.trb_3 = XHCI_TRB_3_SLOT_SET(xs->xs_idx) | 2812 XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_EVALUATE_CTX); 2813 2814 err = xhci_do_command(sc, &trb, USBD_DEFAULT_TIMEOUT); 2815 KASSERT(err == USBD_NORMAL_COMPLETION); /* XXX */ 2816 return err; 2817 } 2818 2819 static void 2820 xhci_set_dcba(struct xhci_softc * const sc, uint64_t dcba, int si) 2821 { 2822 uint64_t * const dcbaa = KERNADDR(&sc->sc_dcbaa_dma, 0); 2823 2824 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 2825 DPRINTFN(4, "dcbaa %p dc %016"PRIx64" slot %d", 2826 &dcbaa[si], dcba, si, 0); 2827 2828 dcbaa[si] = htole64(dcba); 2829 usb_syncmem(&sc->sc_dcbaa_dma, si * sizeof(uint64_t), sizeof(uint64_t), 2830 BUS_DMASYNC_PREWRITE); 2831 } 2832 2833 /* 2834 * Allocate device and input context DMA buffer, and 2835 * TRB DMA buffer for each endpoint. 2836 */ 2837 static usbd_status 2838 xhci_init_slot(struct usbd_device *dev, uint32_t slot) 2839 { 2840 struct xhci_softc * const sc = XHCI_BUS2SC(dev->ud_bus); 2841 struct xhci_slot *xs; 2842 usbd_status err; 2843 u_int dci; 2844 2845 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 2846 DPRINTFN(4, "slot %u", slot, 0, 0, 0); 2847 2848 xs = &sc->sc_slots[slot]; 2849 2850 /* allocate contexts */ 2851 err = usb_allocmem(&sc->sc_bus, sc->sc_pgsz, sc->sc_pgsz, 2852 &xs->xs_dc_dma); 2853 if (err) 2854 return err; 2855 memset(KERNADDR(&xs->xs_dc_dma, 0), 0, sc->sc_pgsz); 2856 2857 err = usb_allocmem(&sc->sc_bus, sc->sc_pgsz, sc->sc_pgsz, 2858 &xs->xs_ic_dma); 2859 if (err) 2860 goto bad1; 2861 memset(KERNADDR(&xs->xs_ic_dma, 0), 0, sc->sc_pgsz); 2862 2863 for (dci = 0; dci < 32; dci++) { 2864 //CTASSERT(sizeof(xs->xs_ep[dci]) == sizeof(struct xhci_endpoint)); 2865 memset(&xs->xs_ep[dci], 0, sizeof(xs->xs_ep[dci])); 2866 if (dci == XHCI_DCI_SLOT) 2867 continue; 2868 err = xhci_ring_init(sc, &xs->xs_ep[dci].xe_tr, 2869 XHCI_TRANSFER_RING_TRBS, XHCI_TRB_ALIGN); 2870 if (err) { 2871 DPRINTFN(0, "ring init failure", 0, 0, 0, 0); 2872 goto bad2; 2873 } 2874 } 2875 2876 bad2: 2877 if (err == USBD_NORMAL_COMPLETION) { 2878 xs->xs_idx = slot; 2879 } else { 2880 xhci_free_slot(sc, xs, XHCI_DCI_SLOT + 1, dci); 2881 } 2882 2883 return err; 2884 2885 bad1: 2886 usb_freemem(&sc->sc_bus, &xs->xs_dc_dma); 2887 xs->xs_idx = 0; 2888 return err; 2889 } 2890 2891 static void 2892 xhci_free_slot(struct xhci_softc *sc, struct xhci_slot *xs, int start_dci, 2893 int end_dci) 2894 { 2895 u_int dci; 2896 2897 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 2898 DPRINTFN(4, "slot %u start %u end %u", xs->xs_idx, start_dci, end_dci, 2899 0); 2900 2901 for (dci = start_dci; dci < end_dci; dci++) { 2902 xhci_ring_free(sc, &xs->xs_ep[dci].xe_tr); 2903 memset(&xs->xs_ep[dci], 0, sizeof(xs->xs_ep[dci])); 2904 } 2905 usb_freemem(&sc->sc_bus, &xs->xs_ic_dma); 2906 usb_freemem(&sc->sc_bus, &xs->xs_dc_dma); 2907 xs->xs_idx = 0; 2908 } 2909 2910 /* 2911 * Setup slot context, set Device Context Base Address, and issue 2912 * Set Address Device command. 2913 */ 2914 static usbd_status 2915 xhci_set_address(struct usbd_device *dev, uint32_t slot, uint32_t route, 2916 int rhport, bool bsr) 2917 { 2918 struct xhci_softc * const sc = XHCI_BUS2SC(dev->ud_bus); 2919 struct xhci_slot *xs; 2920 usbd_status err; 2921 uint32_t *cp; 2922 uint32_t mps = UGETW(dev->ud_ep0desc.wMaxPacketSize); 2923 2924 xs = &sc->sc_slots[slot]; 2925 2926 /* set up initial input control context */ 2927 cp = xhci_slot_get_icv(sc, xs, XHCI_ICI_INPUT_CONTROL); 2928 cp[0] = htole32(0); 2929 cp[1] = htole32(XHCI_INCTX_1_ADD_MASK(XHCI_DCI_EP_CONTROL)| 2930 XHCI_INCTX_1_ADD_MASK(XHCI_DCI_SLOT)); 2931 2932 /* set up input slot context */ 2933 cp = xhci_slot_get_icv(sc, xs, xhci_dci_to_ici(XHCI_DCI_SLOT)); 2934 xhci_setup_sctx(dev, cp); 2935 cp[0] |= htole32(XHCI_SCTX_0_CTX_NUM_SET(1)); 2936 cp[0] |= htole32(XHCI_SCTX_0_ROUTE_SET(route)); 2937 cp[1] |= htole32(XHCI_SCTX_1_RH_PORT_SET(rhport)); 2938 2939 /* set up input EP0 context */ 2940 cp = xhci_slot_get_icv(sc, xs, xhci_dci_to_ici(XHCI_DCI_EP_CONTROL)); 2941 cp[0] = htole32(0); 2942 cp[1] = htole32( 2943 XHCI_EPCTX_1_MAXP_SIZE_SET(mps) | 2944 XHCI_EPCTX_1_EPTYPE_SET(4) | 2945 XHCI_EPCTX_1_CERR_SET(3) 2946 ); 2947 /* can't use xhci_ep_get_dci() yet? */ 2948 *(uint64_t *)(&cp[2]) = htole64( 2949 xhci_ring_trbp(&xs->xs_ep[XHCI_DCI_EP_CONTROL].xe_tr, 0) | 2950 XHCI_EPCTX_2_DCS_SET(1)); 2951 cp[4] = htole32( 2952 XHCI_EPCTX_4_AVG_TRB_LEN_SET(8) 2953 ); 2954 2955 /* sync input contexts before they are read from memory */ 2956 usb_syncmem(&xs->xs_ic_dma, 0, sc->sc_pgsz, BUS_DMASYNC_PREWRITE); 2957 hexdump("input context", xhci_slot_get_icv(sc, xs, 0), 2958 sc->sc_ctxsz * 3); 2959 2960 xhci_set_dcba(sc, DMAADDR(&xs->xs_dc_dma, 0), slot); 2961 2962 err = xhci_address_device(sc, xhci_slot_get_icp(sc, xs, 0), slot, bsr); 2963 2964 usb_syncmem(&xs->xs_dc_dma, 0, sc->sc_pgsz, BUS_DMASYNC_POSTREAD); 2965 hexdump("output context", xhci_slot_get_dcv(sc, xs, 0), 2966 sc->sc_ctxsz * 2); 2967 2968 return err; 2969 } 2970 2971 /* ----- */ 2972 2973 static void 2974 xhci_noop(struct usbd_pipe *pipe) 2975 { 2976 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 2977 } 2978 2979 /* 2980 * Process root hub request. 2981 */ 2982 static int 2983 xhci_roothub_ctrl(struct usbd_bus *bus, usb_device_request_t *req, 2984 void *buf, int buflen) 2985 { 2986 struct xhci_softc * const sc = XHCI_BUS2SC(bus); 2987 usb_port_status_t ps; 2988 int l, totlen = 0; 2989 uint16_t len, value, index; 2990 int port, i; 2991 uint32_t v; 2992 2993 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 2994 2995 if (sc->sc_dying) 2996 return -1; 2997 2998 len = UGETW(req->wLength); 2999 value = UGETW(req->wValue); 3000 index = UGETW(req->wIndex); 3001 3002 DPRINTFN(12, "rhreq: %04x %04x %04x %04x", 3003 req->bmRequestType | (req->bRequest << 8), value, index, len); 3004 3005 #define C(x,y) ((x) | ((y) << 8)) 3006 switch (C(req->bRequest, req->bmRequestType)) { 3007 case C(UR_GET_DESCRIPTOR, UT_READ_DEVICE): 3008 DPRINTFN(8, "getdesc: wValue=0x%04x", value, 0, 0, 0); 3009 if (len == 0) 3010 break; 3011 switch (value) { 3012 case C(0, UDESC_DEVICE): { 3013 usb_device_descriptor_t devd; 3014 totlen = min(buflen, sizeof(devd)); 3015 memcpy(&devd, buf, totlen); 3016 USETW(devd.idVendor, sc->sc_id_vendor); 3017 memcpy(buf, &devd, totlen); 3018 break; 3019 } 3020 #define sd ((usb_string_descriptor_t *)buf) 3021 case C(1, UDESC_STRING): 3022 /* Vendor */ 3023 totlen = usb_makestrdesc(sd, len, sc->sc_vendor); 3024 break; 3025 case C(2, UDESC_STRING): 3026 /* Product */ 3027 totlen = usb_makestrdesc(sd, len, "xHCI Root Hub"); 3028 break; 3029 #undef sd 3030 default: 3031 /* default from usbroothub */ 3032 return buflen; 3033 } 3034 break; 3035 3036 /* Hub requests */ 3037 case C(UR_CLEAR_FEATURE, UT_WRITE_CLASS_DEVICE): 3038 break; 3039 /* Clear Port Feature request */ 3040 case C(UR_CLEAR_FEATURE, UT_WRITE_CLASS_OTHER): 3041 DPRINTFN(4, "UR_CLEAR_PORT_FEATURE port=%d feature=%d", 3042 index, value, 0, 0); 3043 if (index < 1 || index > sc->sc_maxports) { 3044 return -1; 3045 } 3046 port = XHCI_PORTSC(index); 3047 v = xhci_op_read_4(sc, port); 3048 DPRINTFN(4, "portsc=0x%08x", v, 0, 0, 0); 3049 v &= ~XHCI_PS_CLEAR; 3050 switch (value) { 3051 case UHF_PORT_ENABLE: 3052 xhci_op_write_4(sc, port, v & ~XHCI_PS_PED); 3053 break; 3054 case UHF_PORT_SUSPEND: 3055 return -1; 3056 case UHF_PORT_POWER: 3057 break; 3058 case UHF_PORT_TEST: 3059 case UHF_PORT_INDICATOR: 3060 return -1; 3061 case UHF_C_PORT_CONNECTION: 3062 xhci_op_write_4(sc, port, v | XHCI_PS_CSC); 3063 break; 3064 case UHF_C_PORT_ENABLE: 3065 case UHF_C_PORT_SUSPEND: 3066 case UHF_C_PORT_OVER_CURRENT: 3067 return -1; 3068 case UHF_C_BH_PORT_RESET: 3069 xhci_op_write_4(sc, port, v | XHCI_PS_WRC); 3070 break; 3071 case UHF_C_PORT_RESET: 3072 xhci_op_write_4(sc, port, v | XHCI_PS_PRC); 3073 break; 3074 case UHF_C_PORT_LINK_STATE: 3075 xhci_op_write_4(sc, port, v | XHCI_PS_PLC); 3076 break; 3077 case UHF_C_PORT_CONFIG_ERROR: 3078 xhci_op_write_4(sc, port, v | XHCI_PS_CEC); 3079 break; 3080 default: 3081 return -1; 3082 } 3083 break; 3084 case C(UR_GET_DESCRIPTOR, UT_READ_CLASS_DEVICE): 3085 if (len == 0) 3086 break; 3087 if ((value & 0xff) != 0) { 3088 return -1; 3089 } 3090 usb_hub_descriptor_t hubd; 3091 3092 totlen = min(buflen, sizeof(hubd)); 3093 memcpy(&hubd, buf, totlen); 3094 hubd.bNbrPorts = sc->sc_maxports; 3095 USETW(hubd.wHubCharacteristics, UHD_PWR_NO_SWITCH); 3096 hubd.bPwrOn2PwrGood = 200; 3097 for (i = 0, l = sc->sc_maxports; l > 0; i++, l -= 8) 3098 hubd.DeviceRemovable[i++] = 0; /* XXX can't find out? */ 3099 hubd.bDescLength = USB_HUB_DESCRIPTOR_SIZE + i; 3100 totlen = min(totlen, hubd.bDescLength); 3101 memcpy(buf, &hubd, totlen); 3102 break; 3103 case C(UR_GET_STATUS, UT_READ_CLASS_DEVICE): 3104 if (len != 4) { 3105 return -1; 3106 } 3107 memset(buf, 0, len); /* ? XXX */ 3108 totlen = len; 3109 break; 3110 /* Get Port Status request */ 3111 case C(UR_GET_STATUS, UT_READ_CLASS_OTHER): 3112 DPRINTFN(8, "get port status i=%d", index, 0, 0, 0); 3113 if (index < 1 || index > sc->sc_maxports) { 3114 return -1; 3115 } 3116 if (len != 4) { 3117 return -1; 3118 } 3119 v = xhci_op_read_4(sc, XHCI_PORTSC(index)); 3120 DPRINTFN(4, "getrhportsc %d %08x", index, v, 0, 0); 3121 i = xhci_xspeed2psspeed(XHCI_PS_SPEED_GET(v)); 3122 if (v & XHCI_PS_CCS) i |= UPS_CURRENT_CONNECT_STATUS; 3123 if (v & XHCI_PS_PED) i |= UPS_PORT_ENABLED; 3124 if (v & XHCI_PS_OCA) i |= UPS_OVERCURRENT_INDICATOR; 3125 //if (v & XHCI_PS_SUSP) i |= UPS_SUSPEND; 3126 if (v & XHCI_PS_PR) i |= UPS_RESET; 3127 if (v & XHCI_PS_PP) { 3128 if (i & UPS_OTHER_SPEED) 3129 i |= UPS_PORT_POWER_SS; 3130 else 3131 i |= UPS_PORT_POWER; 3132 } 3133 if (i & UPS_OTHER_SPEED) 3134 i |= UPS_PORT_LS_SET(XHCI_PS_PLS_GET(v)); 3135 if (sc->sc_vendor_port_status) 3136 i = sc->sc_vendor_port_status(sc, v, i); 3137 USETW(ps.wPortStatus, i); 3138 i = 0; 3139 if (v & XHCI_PS_CSC) i |= UPS_C_CONNECT_STATUS; 3140 if (v & XHCI_PS_PEC) i |= UPS_C_PORT_ENABLED; 3141 if (v & XHCI_PS_OCC) i |= UPS_C_OVERCURRENT_INDICATOR; 3142 if (v & XHCI_PS_PRC) i |= UPS_C_PORT_RESET; 3143 if (v & XHCI_PS_WRC) i |= UPS_C_BH_PORT_RESET; 3144 if (v & XHCI_PS_PLC) i |= UPS_C_PORT_LINK_STATE; 3145 if (v & XHCI_PS_CEC) i |= UPS_C_PORT_CONFIG_ERROR; 3146 USETW(ps.wPortChange, i); 3147 totlen = min(len, sizeof(ps)); 3148 memcpy(buf, &ps, totlen); 3149 break; 3150 case C(UR_SET_DESCRIPTOR, UT_WRITE_CLASS_DEVICE): 3151 return -1; 3152 case C(UR_SET_HUB_DEPTH, UT_WRITE_CLASS_DEVICE): 3153 break; 3154 case C(UR_SET_FEATURE, UT_WRITE_CLASS_DEVICE): 3155 break; 3156 /* Set Port Feature request */ 3157 case C(UR_SET_FEATURE, UT_WRITE_CLASS_OTHER): { 3158 int optval = (index >> 8) & 0xff; 3159 index &= 0xff; 3160 if (index < 1 || index > sc->sc_maxports) { 3161 return -1; 3162 } 3163 port = XHCI_PORTSC(index); 3164 v = xhci_op_read_4(sc, port); 3165 DPRINTFN(4, "portsc=0x%08x", v, 0, 0, 0); 3166 v &= ~XHCI_PS_CLEAR; 3167 switch (value) { 3168 case UHF_PORT_ENABLE: 3169 xhci_op_write_4(sc, port, v | XHCI_PS_PED); 3170 break; 3171 case UHF_PORT_SUSPEND: 3172 /* XXX suspend */ 3173 break; 3174 case UHF_PORT_RESET: 3175 v &= ~(XHCI_PS_PED | XHCI_PS_PR); 3176 xhci_op_write_4(sc, port, v | XHCI_PS_PR); 3177 /* Wait for reset to complete. */ 3178 usb_delay_ms(&sc->sc_bus, USB_PORT_ROOT_RESET_DELAY); 3179 if (sc->sc_dying) { 3180 return -1; 3181 } 3182 v = xhci_op_read_4(sc, port); 3183 if (v & XHCI_PS_PR) { 3184 xhci_op_write_4(sc, port, v & ~XHCI_PS_PR); 3185 usb_delay_ms(&sc->sc_bus, 10); 3186 /* XXX */ 3187 } 3188 break; 3189 case UHF_PORT_POWER: 3190 /* XXX power control */ 3191 break; 3192 /* XXX more */ 3193 case UHF_C_PORT_RESET: 3194 xhci_op_write_4(sc, port, v | XHCI_PS_PRC); 3195 break; 3196 case UHF_PORT_U1_TIMEOUT: 3197 if (XHCI_PS_SPEED_GET(v) < XHCI_PS_SPEED_SS) { 3198 return -1; 3199 } 3200 port = XHCI_PORTPMSC(index); 3201 v = xhci_op_read_4(sc, port); 3202 v &= ~XHCI_PM3_U1TO_SET(0xff); 3203 v |= XHCI_PM3_U1TO_SET(optval); 3204 xhci_op_write_4(sc, port, v); 3205 break; 3206 case UHF_PORT_U2_TIMEOUT: 3207 if (XHCI_PS_SPEED_GET(v) < XHCI_PS_SPEED_SS) { 3208 return -1; 3209 } 3210 port = XHCI_PORTPMSC(index); 3211 v = xhci_op_read_4(sc, port); 3212 v &= ~XHCI_PM3_U2TO_SET(0xff); 3213 v |= XHCI_PM3_U2TO_SET(optval); 3214 xhci_op_write_4(sc, port, v); 3215 break; 3216 default: 3217 return -1; 3218 } 3219 } 3220 break; 3221 case C(UR_CLEAR_TT_BUFFER, UT_WRITE_CLASS_OTHER): 3222 case C(UR_RESET_TT, UT_WRITE_CLASS_OTHER): 3223 case C(UR_GET_TT_STATE, UT_READ_CLASS_OTHER): 3224 case C(UR_STOP_TT, UT_WRITE_CLASS_OTHER): 3225 break; 3226 default: 3227 /* default from usbroothub */ 3228 return buflen; 3229 } 3230 3231 return totlen; 3232 } 3233 3234 /* root hub interrupt */ 3235 3236 static usbd_status 3237 xhci_root_intr_transfer(struct usbd_xfer *xfer) 3238 { 3239 struct xhci_softc * const sc = XHCI_XFER2SC(xfer); 3240 usbd_status err; 3241 3242 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 3243 3244 /* Insert last in queue. */ 3245 mutex_enter(&sc->sc_lock); 3246 err = usb_insert_transfer(xfer); 3247 mutex_exit(&sc->sc_lock); 3248 if (err) 3249 return err; 3250 3251 /* Pipe isn't running, start first */ 3252 return xhci_root_intr_start(SIMPLEQ_FIRST(&xfer->ux_pipe->up_queue)); 3253 } 3254 3255 /* Wait for roothub port status/change */ 3256 static usbd_status 3257 xhci_root_intr_start(struct usbd_xfer *xfer) 3258 { 3259 struct xhci_softc * const sc = XHCI_XFER2SC(xfer); 3260 3261 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 3262 3263 if (sc->sc_dying) 3264 return USBD_IOERROR; 3265 3266 mutex_enter(&sc->sc_lock); 3267 sc->sc_intrxfer = xfer; 3268 mutex_exit(&sc->sc_lock); 3269 3270 return USBD_IN_PROGRESS; 3271 } 3272 3273 static void 3274 xhci_root_intr_abort(struct usbd_xfer *xfer) 3275 { 3276 struct xhci_softc * const sc = XHCI_XFER2SC(xfer); 3277 3278 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 3279 3280 KASSERT(mutex_owned(&sc->sc_lock)); 3281 KASSERT(xfer->ux_pipe->up_intrxfer == xfer); 3282 3283 sc->sc_intrxfer = NULL; 3284 3285 xfer->ux_status = USBD_CANCELLED; 3286 usb_transfer_complete(xfer); 3287 } 3288 3289 static void 3290 xhci_root_intr_close(struct usbd_pipe *pipe) 3291 { 3292 struct xhci_softc * const sc = XHCI_PIPE2SC(pipe); 3293 3294 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 3295 3296 KASSERT(mutex_owned(&sc->sc_lock)); 3297 3298 sc->sc_intrxfer = NULL; 3299 } 3300 3301 static void 3302 xhci_root_intr_done(struct usbd_xfer *xfer) 3303 { 3304 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 3305 3306 } 3307 3308 /* -------------- */ 3309 /* device control */ 3310 3311 static usbd_status 3312 xhci_device_ctrl_transfer(struct usbd_xfer *xfer) 3313 { 3314 struct xhci_softc * const sc = XHCI_XFER2SC(xfer); 3315 usbd_status err; 3316 3317 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 3318 3319 /* Insert last in queue. */ 3320 mutex_enter(&sc->sc_lock); 3321 err = usb_insert_transfer(xfer); 3322 mutex_exit(&sc->sc_lock); 3323 if (err) 3324 return err; 3325 3326 /* Pipe isn't running, start first */ 3327 return xhci_device_ctrl_start(SIMPLEQ_FIRST(&xfer->ux_pipe->up_queue)); 3328 } 3329 3330 static usbd_status 3331 xhci_device_ctrl_start(struct usbd_xfer *xfer) 3332 { 3333 struct xhci_softc * const sc = XHCI_XFER2SC(xfer); 3334 struct xhci_slot * const xs = xfer->ux_pipe->up_dev->ud_hcpriv; 3335 const u_int dci = xhci_ep_get_dci(xfer->ux_pipe->up_endpoint->ue_edesc); 3336 struct xhci_ring * const tr = &xs->xs_ep[dci].xe_tr; 3337 struct xhci_xfer * const xx = XHCI_XFER2XXFER(xfer); 3338 usb_device_request_t * const req = &xfer->ux_request; 3339 const int isread = usbd_xfer_isread(xfer); 3340 const uint32_t len = UGETW(req->wLength); 3341 usb_dma_t * const dma = &xfer->ux_dmabuf; 3342 uint64_t parameter; 3343 uint32_t status; 3344 uint32_t control; 3345 u_int i; 3346 3347 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 3348 DPRINTFN(12, "req: %04x %04x %04x %04x", 3349 req->bmRequestType | (req->bRequest << 8), UGETW(req->wValue), 3350 UGETW(req->wIndex), UGETW(req->wLength)); 3351 3352 /* we rely on the bottom bits for extra info */ 3353 KASSERT(((uintptr_t)xfer & 0x3) == 0x0); 3354 3355 KASSERT((xfer->ux_rqflags & URQ_REQUEST) != 0); 3356 3357 i = 0; 3358 3359 /* setup phase */ 3360 memcpy(¶meter, req, sizeof(*req)); 3361 status = XHCI_TRB_2_IRQ_SET(0) | XHCI_TRB_2_BYTES_SET(sizeof(*req)); 3362 control = ((len == 0) ? XHCI_TRB_3_TRT_NONE : 3363 (isread ? XHCI_TRB_3_TRT_IN : XHCI_TRB_3_TRT_OUT)) | 3364 XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_SETUP_STAGE) | 3365 XHCI_TRB_3_IDT_BIT; 3366 xhci_trb_put(&xx->xx_trb[i++], parameter, status, control); 3367 3368 if (len != 0) { 3369 /* data phase */ 3370 parameter = DMAADDR(dma, 0); 3371 KASSERT(len <= 0x10000); 3372 status = XHCI_TRB_2_IRQ_SET(0) | 3373 XHCI_TRB_2_TDSZ_SET(1) | 3374 XHCI_TRB_2_BYTES_SET(len); 3375 control = (isread ? XHCI_TRB_3_DIR_IN : 0) | 3376 XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_DATA_STAGE) | 3377 XHCI_TRB_3_CHAIN_BIT | XHCI_TRB_3_ENT_BIT; 3378 xhci_trb_put(&xx->xx_trb[i++], parameter, status, control); 3379 3380 parameter = (uintptr_t)xfer | 0x3; 3381 status = XHCI_TRB_2_IRQ_SET(0); 3382 control = XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_EVENT_DATA) | 3383 XHCI_TRB_3_IOC_BIT; 3384 xhci_trb_put(&xx->xx_trb[i++], parameter, status, control); 3385 } 3386 3387 parameter = 0; 3388 status = XHCI_TRB_2_IRQ_SET(0); 3389 /* the status stage has inverted direction */ 3390 control = ((isread && (len > 0)) ? 0 : XHCI_TRB_3_DIR_IN) | 3391 XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_STATUS_STAGE) | 3392 XHCI_TRB_3_CHAIN_BIT | XHCI_TRB_3_ENT_BIT; 3393 xhci_trb_put(&xx->xx_trb[i++], parameter, status, control); 3394 3395 parameter = (uintptr_t)xfer | 0x0; 3396 status = XHCI_TRB_2_IRQ_SET(0); 3397 control = XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_EVENT_DATA) | 3398 XHCI_TRB_3_IOC_BIT; 3399 xhci_trb_put(&xx->xx_trb[i++], parameter, status, control); 3400 3401 mutex_enter(&tr->xr_lock); 3402 xhci_ring_put(sc, tr, xfer, xx->xx_trb, i); 3403 mutex_exit(&tr->xr_lock); 3404 3405 xhci_db_write_4(sc, XHCI_DOORBELL(xs->xs_idx), dci); 3406 3407 if (xfer->ux_timeout && !sc->sc_bus.ub_usepolling) { 3408 callout_reset(&xfer->ux_callout, mstohz(xfer->ux_timeout), 3409 xhci_timeout, xfer); 3410 } 3411 3412 return USBD_IN_PROGRESS; 3413 } 3414 3415 static void 3416 xhci_device_ctrl_done(struct usbd_xfer *xfer) 3417 { 3418 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 3419 usb_device_request_t *req = &xfer->ux_request; 3420 int len = UGETW(req->wLength); 3421 int rd = req->bmRequestType & UT_READ; 3422 3423 if (len) 3424 usb_syncmem(&xfer->ux_dmabuf, 0, len, 3425 rd ? BUS_DMASYNC_POSTREAD : BUS_DMASYNC_POSTWRITE); 3426 } 3427 3428 static void 3429 xhci_device_ctrl_abort(struct usbd_xfer *xfer) 3430 { 3431 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 3432 3433 xhci_abort_xfer(xfer, USBD_CANCELLED); 3434 } 3435 3436 static void 3437 xhci_device_ctrl_close(struct usbd_pipe *pipe) 3438 { 3439 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 3440 3441 xhci_close_pipe(pipe); 3442 } 3443 3444 /* ------------------ */ 3445 /* device isochronous */ 3446 3447 /* ----------- */ 3448 /* device bulk */ 3449 3450 static usbd_status 3451 xhci_device_bulk_transfer(struct usbd_xfer *xfer) 3452 { 3453 struct xhci_softc * const sc = XHCI_XFER2SC(xfer); 3454 usbd_status err; 3455 3456 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 3457 3458 /* Insert last in queue. */ 3459 mutex_enter(&sc->sc_lock); 3460 err = usb_insert_transfer(xfer); 3461 mutex_exit(&sc->sc_lock); 3462 if (err) 3463 return err; 3464 3465 /* 3466 * Pipe isn't running (otherwise err would be USBD_INPROG), 3467 * so start it first. 3468 */ 3469 return xhci_device_bulk_start(SIMPLEQ_FIRST(&xfer->ux_pipe->up_queue)); 3470 } 3471 3472 static usbd_status 3473 xhci_device_bulk_start(struct usbd_xfer *xfer) 3474 { 3475 struct xhci_softc * const sc = XHCI_XFER2SC(xfer); 3476 struct xhci_slot * const xs = xfer->ux_pipe->up_dev->ud_hcpriv; 3477 const u_int dci = xhci_ep_get_dci(xfer->ux_pipe->up_endpoint->ue_edesc); 3478 struct xhci_ring * const tr = &xs->xs_ep[dci].xe_tr; 3479 struct xhci_xfer * const xx = XHCI_XFER2XXFER(xfer); 3480 const uint32_t len = xfer->ux_length; 3481 usb_dma_t * const dma = &xfer->ux_dmabuf; 3482 uint64_t parameter; 3483 uint32_t status; 3484 uint32_t control; 3485 u_int i = 0; 3486 3487 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 3488 3489 DPRINTFN(15, "%p slot %u dci %u", xfer, xs->xs_idx, dci, 0); 3490 3491 if (sc->sc_dying) 3492 return USBD_IOERROR; 3493 3494 KASSERT((xfer->ux_rqflags & URQ_REQUEST) == 0); 3495 3496 parameter = DMAADDR(dma, 0); 3497 /* 3498 * XXX: (dsl) The physical buffer must not cross a 64k boundary. 3499 * If the user supplied buffer crosses such a boundary then 2 3500 * (or more) TRB should be used. 3501 * If multiple TRB are used the td_size field must be set correctly. 3502 * For v1.0 devices (like ivy bridge) this is the number of usb data 3503 * blocks needed to complete the transfer. 3504 * Setting it to 1 in the last TRB causes an extra zero-length 3505 * data block be sent. 3506 * The earlier documentation differs, I don't know how it behaves. 3507 */ 3508 KASSERT(len <= 0x10000); 3509 status = XHCI_TRB_2_IRQ_SET(0) | 3510 XHCI_TRB_2_TDSZ_SET(1) | 3511 XHCI_TRB_2_BYTES_SET(len); 3512 control = XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_NORMAL) | 3513 XHCI_TRB_3_ISP_BIT | XHCI_TRB_3_IOC_BIT; 3514 xhci_trb_put(&xx->xx_trb[i++], parameter, status, control); 3515 3516 mutex_enter(&tr->xr_lock); 3517 xhci_ring_put(sc, tr, xfer, xx->xx_trb, i); 3518 mutex_exit(&tr->xr_lock); 3519 3520 xhci_db_write_4(sc, XHCI_DOORBELL(xs->xs_idx), dci); 3521 3522 if (xfer->ux_timeout && !sc->sc_bus.ub_usepolling) { 3523 callout_reset(&xfer->ux_callout, mstohz(xfer->ux_timeout), 3524 xhci_timeout, xfer); 3525 } 3526 3527 return USBD_IN_PROGRESS; 3528 } 3529 3530 static void 3531 xhci_device_bulk_done(struct usbd_xfer *xfer) 3532 { 3533 #ifdef USB_DEBUG 3534 struct xhci_slot * const xs = xfer->ux_pipe->up_dev->ud_hcpriv; 3535 const u_int dci = xhci_ep_get_dci(xfer->ux_pipe->up_endpoint->ue_edesc); 3536 #endif 3537 const int isread = usbd_xfer_isread(xfer); 3538 3539 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 3540 3541 DPRINTFN(15, "%p slot %u dci %u", xfer, xs->xs_idx, dci, 0); 3542 3543 usb_syncmem(&xfer->ux_dmabuf, 0, xfer->ux_length, 3544 isread ? BUS_DMASYNC_POSTREAD : BUS_DMASYNC_POSTWRITE); 3545 } 3546 3547 static void 3548 xhci_device_bulk_abort(struct usbd_xfer *xfer) 3549 { 3550 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 3551 3552 xhci_abort_xfer(xfer, USBD_CANCELLED); 3553 } 3554 3555 static void 3556 xhci_device_bulk_close(struct usbd_pipe *pipe) 3557 { 3558 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 3559 3560 xhci_close_pipe(pipe); 3561 } 3562 3563 /* ---------------- */ 3564 /* device interrupt */ 3565 3566 static usbd_status 3567 xhci_device_intr_transfer(struct usbd_xfer *xfer) 3568 { 3569 struct xhci_softc * const sc = XHCI_XFER2SC(xfer); 3570 usbd_status err; 3571 3572 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 3573 3574 /* Insert last in queue. */ 3575 mutex_enter(&sc->sc_lock); 3576 err = usb_insert_transfer(xfer); 3577 mutex_exit(&sc->sc_lock); 3578 if (err) 3579 return err; 3580 3581 /* 3582 * Pipe isn't running (otherwise err would be USBD_INPROG), 3583 * so start it first. 3584 */ 3585 return xhci_device_intr_start(SIMPLEQ_FIRST(&xfer->ux_pipe->up_queue)); 3586 } 3587 3588 static usbd_status 3589 xhci_device_intr_start(struct usbd_xfer *xfer) 3590 { 3591 struct xhci_softc * const sc = XHCI_XFER2SC(xfer); 3592 struct xhci_slot * const xs = xfer->ux_pipe->up_dev->ud_hcpriv; 3593 const u_int dci = xhci_ep_get_dci(xfer->ux_pipe->up_endpoint->ue_edesc); 3594 struct xhci_ring * const tr = &xs->xs_ep[dci].xe_tr; 3595 struct xhci_xfer * const xx = XHCI_XFER2XXFER(xfer); 3596 const uint32_t len = xfer->ux_length; 3597 usb_dma_t * const dma = &xfer->ux_dmabuf; 3598 uint64_t parameter; 3599 uint32_t status; 3600 uint32_t control; 3601 u_int i = 0; 3602 3603 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 3604 3605 DPRINTFN(15, "%p slot %u dci %u", xfer, xs->xs_idx, dci, 0); 3606 3607 if (sc->sc_dying) 3608 return USBD_IOERROR; 3609 3610 KASSERT((xfer->ux_rqflags & URQ_REQUEST) == 0); 3611 3612 parameter = DMAADDR(dma, 0); 3613 KASSERT(len <= 0x10000); 3614 status = XHCI_TRB_2_IRQ_SET(0) | 3615 XHCI_TRB_2_TDSZ_SET(1) | 3616 XHCI_TRB_2_BYTES_SET(len); 3617 control = XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_NORMAL) | 3618 XHCI_TRB_3_ISP_BIT | XHCI_TRB_3_IOC_BIT; 3619 xhci_trb_put(&xx->xx_trb[i++], parameter, status, control); 3620 3621 mutex_enter(&tr->xr_lock); 3622 xhci_ring_put(sc, tr, xfer, xx->xx_trb, i); 3623 mutex_exit(&tr->xr_lock); 3624 3625 xhci_db_write_4(sc, XHCI_DOORBELL(xs->xs_idx), dci); 3626 3627 if (xfer->ux_timeout && !sc->sc_bus.ub_usepolling) { 3628 callout_reset(&xfer->ux_callout, mstohz(xfer->ux_timeout), 3629 xhci_timeout, xfer); 3630 } 3631 3632 return USBD_IN_PROGRESS; 3633 } 3634 3635 static void 3636 xhci_device_intr_done(struct usbd_xfer *xfer) 3637 { 3638 struct xhci_softc * const sc __diagused = XHCI_XFER2SC(xfer); 3639 #ifdef USB_DEBUG 3640 struct xhci_slot * const xs = xfer->ux_pipe->up_dev->ud_hcpriv; 3641 const u_int dci = xhci_ep_get_dci(xfer->ux_pipe->up_endpoint->ue_edesc); 3642 #endif 3643 const int isread = usbd_xfer_isread(xfer); 3644 3645 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 3646 3647 DPRINTFN(15, "%p slot %u dci %u", xfer, xs->xs_idx, dci, 0); 3648 3649 KASSERT(sc->sc_bus.ub_usepolling || mutex_owned(&sc->sc_lock)); 3650 3651 usb_syncmem(&xfer->ux_dmabuf, 0, xfer->ux_length, 3652 isread ? BUS_DMASYNC_POSTREAD : BUS_DMASYNC_POSTWRITE); 3653 } 3654 3655 static void 3656 xhci_device_intr_abort(struct usbd_xfer *xfer) 3657 { 3658 struct xhci_softc * const sc __diagused = XHCI_XFER2SC(xfer); 3659 3660 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 3661 3662 KASSERT(mutex_owned(&sc->sc_lock)); 3663 DPRINTFN(15, "%p", xfer, 0, 0, 0); 3664 KASSERT(xfer->ux_pipe->up_intrxfer == xfer); 3665 xhci_abort_xfer(xfer, USBD_CANCELLED); 3666 } 3667 3668 static void 3669 xhci_device_intr_close(struct usbd_pipe *pipe) 3670 { 3671 //struct xhci_softc * const sc = XHCI_PIPE2SC(pipe); 3672 3673 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 3674 DPRINTFN(15, "%p", pipe, 0, 0, 0); 3675 3676 xhci_close_pipe(pipe); 3677 } 3678 3679 /* ------------ */ 3680 3681 static void 3682 xhci_timeout(void *addr) 3683 { 3684 struct xhci_xfer * const xx = addr; 3685 struct usbd_xfer * const xfer = &xx->xx_xfer; 3686 struct xhci_softc * const sc = XHCI_XFER2SC(xfer); 3687 3688 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 3689 3690 if (sc->sc_dying) { 3691 return; 3692 } 3693 3694 usb_init_task(&xx->xx_abort_task, xhci_timeout_task, addr, 3695 USB_TASKQ_MPSAFE); 3696 usb_add_task(xx->xx_xfer.ux_pipe->up_dev, &xx->xx_abort_task, 3697 USB_TASKQ_HC); 3698 } 3699 3700 static void 3701 xhci_timeout_task(void *addr) 3702 { 3703 struct usbd_xfer * const xfer = addr; 3704 struct xhci_softc * const sc = XHCI_XFER2SC(xfer); 3705 3706 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 3707 3708 mutex_enter(&sc->sc_lock); 3709 #if 0 3710 xhci_abort_xfer(xfer, USBD_TIMEOUT); 3711 #else 3712 xfer->ux_status = USBD_TIMEOUT; 3713 usb_transfer_complete(xfer); 3714 #endif 3715 mutex_exit(&sc->sc_lock); 3716 } 3717
Indexes created Thu Sep 25 16:09:42 GMT 2025