xhci.c revision 1.28.2.69
1 /* $NetBSD: xhci.c,v 1.28.2.69 2016/05/29 08:44:31 skrll Exp $ */ 2 3 /* 4 * Copyright (c) 2013 Jonathan A. Kollasch 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 17 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 18 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 19 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR 20 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 21 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 22 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 23 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 24 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 25 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 26 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 */ 28 29 /* 30 * USB rev 2.0 and rev 3.1 specification 31 * http://www.usb.org/developers/docs/ 32 * xHCI rev 1.1 specification 33 * http://www.intel.com/technology/usb/spec.htm 34 */ 35 36 #include <sys/cdefs.h> 37 __KERNEL_RCSID(0, "$NetBSD: xhci.c,v 1.28.2.69 2016/05/29 08:44:31 skrll Exp $"); 38 39 #ifdef _KERNEL_OPT 40 #include "opt_usb.h" 41 #endif 42 43 #include <sys/param.h> 44 #include <sys/systm.h> 45 #include <sys/kernel.h> 46 #include <sys/kmem.h> 47 #include <sys/device.h> 48 #include <sys/select.h> 49 #include <sys/proc.h> 50 #include <sys/queue.h> 51 #include <sys/mutex.h> 52 #include <sys/condvar.h> 53 #include <sys/bus.h> 54 #include <sys/cpu.h> 55 #include <sys/sysctl.h> 56 57 #include <machine/endian.h> 58 59 #include <dev/usb/usb.h> 60 #include <dev/usb/usbdi.h> 61 #include <dev/usb/usbdivar.h> 62 #include <dev/usb/usbdi_util.h> 63 #include <dev/usb/usbhist.h> 64 #include <dev/usb/usb_mem.h> 65 #include <dev/usb/usb_quirks.h> 66 67 #include <dev/usb/xhcireg.h> 68 #include <dev/usb/xhcivar.h> 69 #include <dev/usb/usbroothub.h> 70 71 72 #ifdef USB_DEBUG 73 #ifndef XHCI_DEBUG 74 #define xhcidebug 0 75 #else /* !XHCI_DEBUG */ 76 static int xhcidebug = 0; 77 78 SYSCTL_SETUP(sysctl_hw_xhci_setup, "sysctl hw.xhci setup") 79 { 80 int err; 81 const struct sysctlnode *rnode; 82 const struct sysctlnode *cnode; 83 84 err = sysctl_createv(clog, 0, NULL, &rnode, 85 CTLFLAG_PERMANENT, CTLTYPE_NODE, "xhci", 86 SYSCTL_DESCR("xhci global controls"), 87 NULL, 0, NULL, 0, CTL_HW, CTL_CREATE, CTL_EOL); 88 89 if (err) 90 goto fail; 91 92 /* control debugging printfs */ 93 err = sysctl_createv(clog, 0, &rnode, &cnode, 94 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, CTLTYPE_INT, 95 "debug", SYSCTL_DESCR("Enable debugging output"), 96 NULL, 0, &xhcidebug, sizeof(xhcidebug), CTL_CREATE, CTL_EOL); 97 if (err) 98 goto fail; 99 100 return; 101 fail: 102 aprint_error("%s: sysctl_createv failed (err = %d)\n", __func__, err); 103 } 104 105 #endif /* !XHCI_DEBUG */ 106 #endif /* USB_DEBUG */ 107 108 #define DPRINTFN(N,FMT,A,B,C,D) USBHIST_LOGN(xhcidebug,N,FMT,A,B,C,D) 109 #define XHCIHIST_FUNC() USBHIST_FUNC() 110 #define XHCIHIST_CALLED(name) USBHIST_CALLED(xhcidebug) 111 112 #define XHCI_DCI_SLOT 0 113 #define XHCI_DCI_EP_CONTROL 1 114 115 #define XHCI_ICI_INPUT_CONTROL 0 116 117 struct xhci_pipe { 118 struct usbd_pipe xp_pipe; 119 struct usb_task xp_async_task; 120 }; 121 122 #define XHCI_COMMAND_RING_TRBS 256 123 #define XHCI_EVENT_RING_TRBS 256 124 #define XHCI_EVENT_RING_SEGMENTS 1 125 #define XHCI_TRB_3_ED_BIT XHCI_TRB_3_ISP_BIT 126 127 static usbd_status xhci_open(struct usbd_pipe *); 128 static void xhci_close_pipe(struct usbd_pipe *); 129 static int xhci_intr1(struct xhci_softc * const); 130 static void xhci_softintr(void *); 131 static void xhci_poll(struct usbd_bus *); 132 static struct usbd_xfer *xhci_allocx(struct usbd_bus *, unsigned int); 133 static void xhci_freex(struct usbd_bus *, struct usbd_xfer *); 134 static void xhci_get_lock(struct usbd_bus *, kmutex_t **); 135 static usbd_status xhci_new_device(device_t, struct usbd_bus *, int, int, int, 136 struct usbd_port *); 137 static int xhci_roothub_ctrl(struct usbd_bus *, usb_device_request_t *, 138 void *, int); 139 140 static usbd_status xhci_configure_endpoint(struct usbd_pipe *); 141 //static usbd_status xhci_unconfigure_endpoint(struct usbd_pipe *); 142 static usbd_status xhci_reset_endpoint(struct usbd_pipe *); 143 static usbd_status xhci_stop_endpoint(struct usbd_pipe *); 144 145 static usbd_status xhci_set_dequeue(struct usbd_pipe *); 146 147 static usbd_status xhci_do_command(struct xhci_softc * const, 148 struct xhci_trb * const, int); 149 static usbd_status xhci_do_command_locked(struct xhci_softc * const, 150 struct xhci_trb * const, int); 151 static usbd_status xhci_init_slot(struct usbd_device *, uint32_t, uint32_t, int); 152 static usbd_status xhci_enable_slot(struct xhci_softc * const, 153 uint8_t * const); 154 static usbd_status xhci_disable_slot(struct xhci_softc * const, uint8_t); 155 static usbd_status xhci_address_device(struct xhci_softc * const, 156 uint64_t, uint8_t, bool); 157 static void xhci_set_dcba(struct xhci_softc * const, uint64_t, int); 158 static usbd_status xhci_update_ep0_mps(struct xhci_softc * const, 159 struct xhci_slot * const, u_int); 160 static usbd_status xhci_ring_init(struct xhci_softc * const, 161 struct xhci_ring * const, size_t, size_t); 162 static void xhci_ring_free(struct xhci_softc * const, struct xhci_ring * const); 163 164 static void xhci_noop(struct usbd_pipe *); 165 166 static usbd_status xhci_root_intr_transfer(struct usbd_xfer *); 167 static usbd_status xhci_root_intr_start(struct usbd_xfer *); 168 static void xhci_root_intr_abort(struct usbd_xfer *); 169 static void xhci_root_intr_close(struct usbd_pipe *); 170 static void xhci_root_intr_done(struct usbd_xfer *); 171 172 static usbd_status xhci_device_ctrl_transfer(struct usbd_xfer *); 173 static usbd_status xhci_device_ctrl_start(struct usbd_xfer *); 174 static void xhci_device_ctrl_abort(struct usbd_xfer *); 175 static void xhci_device_ctrl_close(struct usbd_pipe *); 176 static void xhci_device_ctrl_done(struct usbd_xfer *); 177 178 static usbd_status xhci_device_intr_transfer(struct usbd_xfer *); 179 static usbd_status xhci_device_intr_start(struct usbd_xfer *); 180 static void xhci_device_intr_abort(struct usbd_xfer *); 181 static void xhci_device_intr_close(struct usbd_pipe *); 182 static void xhci_device_intr_done(struct usbd_xfer *); 183 184 static usbd_status xhci_device_bulk_transfer(struct usbd_xfer *); 185 static usbd_status xhci_device_bulk_start(struct usbd_xfer *); 186 static void xhci_device_bulk_abort(struct usbd_xfer *); 187 static void xhci_device_bulk_close(struct usbd_pipe *); 188 static void xhci_device_bulk_done(struct usbd_xfer *); 189 190 static void xhci_timeout(void *); 191 static void xhci_timeout_task(void *); 192 193 static const struct usbd_bus_methods xhci_bus_methods = { 194 .ubm_open = xhci_open, 195 .ubm_softint = xhci_softintr, 196 .ubm_dopoll = xhci_poll, 197 .ubm_allocx = xhci_allocx, 198 .ubm_freex = xhci_freex, 199 .ubm_getlock = xhci_get_lock, 200 .ubm_newdev = xhci_new_device, 201 .ubm_rhctrl = xhci_roothub_ctrl, 202 }; 203 204 static const struct usbd_pipe_methods xhci_root_intr_methods = { 205 .upm_transfer = xhci_root_intr_transfer, 206 .upm_start = xhci_root_intr_start, 207 .upm_abort = xhci_root_intr_abort, 208 .upm_close = xhci_root_intr_close, 209 .upm_cleartoggle = xhci_noop, 210 .upm_done = xhci_root_intr_done, 211 }; 212 213 214 static const struct usbd_pipe_methods xhci_device_ctrl_methods = { 215 .upm_transfer = xhci_device_ctrl_transfer, 216 .upm_start = xhci_device_ctrl_start, 217 .upm_abort = xhci_device_ctrl_abort, 218 .upm_close = xhci_device_ctrl_close, 219 .upm_cleartoggle = xhci_noop, 220 .upm_done = xhci_device_ctrl_done, 221 }; 222 223 static const struct usbd_pipe_methods xhci_device_isoc_methods = { 224 .upm_cleartoggle = xhci_noop, 225 }; 226 227 static const struct usbd_pipe_methods xhci_device_bulk_methods = { 228 .upm_transfer = xhci_device_bulk_transfer, 229 .upm_start = xhci_device_bulk_start, 230 .upm_abort = xhci_device_bulk_abort, 231 .upm_close = xhci_device_bulk_close, 232 .upm_cleartoggle = xhci_noop, 233 .upm_done = xhci_device_bulk_done, 234 }; 235 236 static const struct usbd_pipe_methods xhci_device_intr_methods = { 237 .upm_transfer = xhci_device_intr_transfer, 238 .upm_start = xhci_device_intr_start, 239 .upm_abort = xhci_device_intr_abort, 240 .upm_close = xhci_device_intr_close, 241 .upm_cleartoggle = xhci_noop, 242 .upm_done = xhci_device_intr_done, 243 }; 244 245 static inline uint32_t 246 xhci_read_1(const struct xhci_softc * const sc, bus_size_t offset) 247 { 248 return bus_space_read_1(sc->sc_iot, sc->sc_ioh, offset); 249 } 250 251 static inline uint32_t 252 xhci_read_4(const struct xhci_softc * const sc, bus_size_t offset) 253 { 254 return bus_space_read_4(sc->sc_iot, sc->sc_ioh, offset); 255 } 256 257 static inline void 258 xhci_write_1(const struct xhci_softc * const sc, bus_size_t offset, 259 uint32_t value) 260 { 261 bus_space_write_1(sc->sc_iot, sc->sc_ioh, offset, value); 262 } 263 264 #if 0 /* unused */ 265 static inline void 266 xhci_write_4(const struct xhci_softc * const sc, bus_size_t offset, 267 uint32_t value) 268 { 269 bus_space_write_4(sc->sc_iot, sc->sc_ioh, offset, value); 270 } 271 #endif /* unused */ 272 273 static inline uint32_t 274 xhci_cap_read_4(const struct xhci_softc * const sc, bus_size_t offset) 275 { 276 return bus_space_read_4(sc->sc_iot, sc->sc_cbh, offset); 277 } 278 279 static inline uint32_t 280 xhci_op_read_4(const struct xhci_softc * const sc, bus_size_t offset) 281 { 282 return bus_space_read_4(sc->sc_iot, sc->sc_obh, offset); 283 } 284 285 static inline void 286 xhci_op_write_4(const struct xhci_softc * const sc, bus_size_t offset, 287 uint32_t value) 288 { 289 bus_space_write_4(sc->sc_iot, sc->sc_obh, offset, value); 290 } 291 292 static inline uint64_t 293 xhci_op_read_8(const struct xhci_softc * const sc, bus_size_t offset) 294 { 295 uint64_t value; 296 297 if (sc->sc_ac64) { 298 #ifdef XHCI_USE_BUS_SPACE_8 299 value = bus_space_read_8(sc->sc_iot, sc->sc_obh, offset); 300 #else 301 value = bus_space_read_4(sc->sc_iot, sc->sc_obh, offset); 302 value |= (uint64_t)bus_space_read_4(sc->sc_iot, sc->sc_obh, 303 offset + 4) << 32; 304 #endif 305 } else { 306 value = bus_space_read_4(sc->sc_iot, sc->sc_obh, offset); 307 } 308 309 return value; 310 } 311 312 static inline void 313 xhci_op_write_8(const struct xhci_softc * const sc, bus_size_t offset, 314 uint64_t value) 315 { 316 if (sc->sc_ac64) { 317 #ifdef XHCI_USE_BUS_SPACE_8 318 bus_space_write_8(sc->sc_iot, sc->sc_obh, offset, value); 319 #else 320 bus_space_write_4(sc->sc_iot, sc->sc_obh, offset + 0, 321 (value >> 0) & 0xffffffff); 322 bus_space_write_4(sc->sc_iot, sc->sc_obh, offset + 4, 323 (value >> 32) & 0xffffffff); 324 #endif 325 } else { 326 bus_space_write_4(sc->sc_iot, sc->sc_obh, offset, value); 327 } 328 } 329 330 static inline uint32_t 331 xhci_rt_read_4(const struct xhci_softc * const sc, bus_size_t offset) 332 { 333 return bus_space_read_4(sc->sc_iot, sc->sc_rbh, offset); 334 } 335 336 static inline void 337 xhci_rt_write_4(const struct xhci_softc * const sc, bus_size_t offset, 338 uint32_t value) 339 { 340 bus_space_write_4(sc->sc_iot, sc->sc_rbh, offset, value); 341 } 342 343 #if 0 /* unused */ 344 static inline uint64_t 345 xhci_rt_read_8(const struct xhci_softc * const sc, bus_size_t offset) 346 { 347 uint64_t value; 348 349 if (sc->sc_ac64) { 350 #ifdef XHCI_USE_BUS_SPACE_8 351 value = bus_space_read_8(sc->sc_iot, sc->sc_rbh, offset); 352 #else 353 value = bus_space_read_4(sc->sc_iot, sc->sc_rbh, offset); 354 value |= (uint64_t)bus_space_read_4(sc->sc_iot, sc->sc_rbh, 355 offset + 4) << 32; 356 #endif 357 } else { 358 value = bus_space_read_4(sc->sc_iot, sc->sc_rbh, offset); 359 } 360 361 return value; 362 } 363 #endif /* unused */ 364 365 static inline void 366 xhci_rt_write_8(const struct xhci_softc * const sc, bus_size_t offset, 367 uint64_t value) 368 { 369 if (sc->sc_ac64) { 370 #ifdef XHCI_USE_BUS_SPACE_8 371 bus_space_write_8(sc->sc_iot, sc->sc_rbh, offset, value); 372 #else 373 bus_space_write_4(sc->sc_iot, sc->sc_rbh, offset + 0, 374 (value >> 0) & 0xffffffff); 375 bus_space_write_4(sc->sc_iot, sc->sc_rbh, offset + 4, 376 (value >> 32) & 0xffffffff); 377 #endif 378 } else { 379 bus_space_write_4(sc->sc_iot, sc->sc_rbh, offset, value); 380 } 381 } 382 383 #if 0 /* unused */ 384 static inline uint32_t 385 xhci_db_read_4(const struct xhci_softc * const sc, bus_size_t offset) 386 { 387 return bus_space_read_4(sc->sc_iot, sc->sc_dbh, offset); 388 } 389 #endif /* unused */ 390 391 static inline void 392 xhci_db_write_4(const struct xhci_softc * const sc, bus_size_t offset, 393 uint32_t value) 394 { 395 bus_space_write_4(sc->sc_iot, sc->sc_dbh, offset, value); 396 } 397 398 /* --- */ 399 400 static inline uint8_t 401 xhci_ep_get_type(usb_endpoint_descriptor_t * const ed) 402 { 403 u_int eptype = 0; 404 405 switch (UE_GET_XFERTYPE(ed->bmAttributes)) { 406 case UE_CONTROL: 407 eptype = 0x0; 408 break; 409 case UE_ISOCHRONOUS: 410 eptype = 0x1; 411 break; 412 case UE_BULK: 413 eptype = 0x2; 414 break; 415 case UE_INTERRUPT: 416 eptype = 0x3; 417 break; 418 } 419 420 if ((UE_GET_XFERTYPE(ed->bmAttributes) == UE_CONTROL) || 421 (UE_GET_DIR(ed->bEndpointAddress) == UE_DIR_IN)) 422 return eptype | 0x4; 423 else 424 return eptype; 425 } 426 427 static u_int 428 xhci_ep_get_dci(usb_endpoint_descriptor_t * const ed) 429 { 430 /* xHCI 1.0 section 4.5.1 */ 431 u_int epaddr = UE_GET_ADDR(ed->bEndpointAddress); 432 u_int in = 0; 433 434 if ((UE_GET_XFERTYPE(ed->bmAttributes) == UE_CONTROL) || 435 (UE_GET_DIR(ed->bEndpointAddress) == UE_DIR_IN)) 436 in = 1; 437 438 return epaddr * 2 + in; 439 } 440 441 static inline u_int 442 xhci_dci_to_ici(const u_int i) 443 { 444 return i + 1; 445 } 446 447 static inline void * 448 xhci_slot_get_dcv(struct xhci_softc * const sc, struct xhci_slot * const xs, 449 const u_int dci) 450 { 451 return KERNADDR(&xs->xs_dc_dma, sc->sc_ctxsz * dci); 452 } 453 454 #if 0 /* unused */ 455 static inline bus_addr_t 456 xhci_slot_get_dcp(struct xhci_softc * const sc, struct xhci_slot * const xs, 457 const u_int dci) 458 { 459 return DMAADDR(&xs->xs_dc_dma, sc->sc_ctxsz * dci); 460 } 461 #endif /* unused */ 462 463 static inline void * 464 xhci_slot_get_icv(struct xhci_softc * const sc, struct xhci_slot * const xs, 465 const u_int ici) 466 { 467 return KERNADDR(&xs->xs_ic_dma, sc->sc_ctxsz * ici); 468 } 469 470 static inline bus_addr_t 471 xhci_slot_get_icp(struct xhci_softc * const sc, struct xhci_slot * const xs, 472 const u_int ici) 473 { 474 return DMAADDR(&xs->xs_ic_dma, sc->sc_ctxsz * ici); 475 } 476 477 static inline struct xhci_trb * 478 xhci_ring_trbv(struct xhci_ring * const xr, u_int idx) 479 { 480 return KERNADDR(&xr->xr_dma, XHCI_TRB_SIZE * idx); 481 } 482 483 static inline bus_addr_t 484 xhci_ring_trbp(struct xhci_ring * const xr, u_int idx) 485 { 486 return DMAADDR(&xr->xr_dma, XHCI_TRB_SIZE * idx); 487 } 488 489 static inline void 490 xhci_trb_put(struct xhci_trb * const trb, uint64_t parameter, uint32_t status, 491 uint32_t control) 492 { 493 trb->trb_0 = htole64(parameter); 494 trb->trb_2 = htole32(status); 495 trb->trb_3 = htole32(control); 496 } 497 498 static int 499 xhci_trb_get_idx(struct xhci_ring *xr, uint64_t trb_0, int *idx) 500 { 501 /* base address of TRBs */ 502 bus_addr_t trbp = xhci_ring_trbp(xr, 0); 503 504 /* trb_0 range sanity check */ 505 if (trb_0 == 0 || trb_0 < trbp || 506 (trb_0 - trbp) % sizeof(struct xhci_trb) != 0 || 507 (trb_0 - trbp) / sizeof(struct xhci_trb) >= xr->xr_ntrb) { 508 return 1; 509 } 510 *idx = (trb_0 - trbp) / sizeof(struct xhci_trb); 511 return 0; 512 } 513 514 /* --- */ 515 516 void 517 xhci_childdet(device_t self, device_t child) 518 { 519 struct xhci_softc * const sc = device_private(self); 520 521 KASSERT(sc->sc_child == child); 522 if (child == sc->sc_child) 523 sc->sc_child = NULL; 524 } 525 526 int 527 xhci_detach(struct xhci_softc *sc, int flags) 528 { 529 int rv = 0; 530 531 if (sc->sc_child != NULL) 532 rv = config_detach(sc->sc_child, flags); 533 534 if (rv != 0) 535 return rv; 536 537 /* XXX unconfigure/free slots */ 538 539 /* verify: */ 540 xhci_rt_write_4(sc, XHCI_IMAN(0), 0); 541 xhci_op_write_4(sc, XHCI_USBCMD, 0); 542 /* do we need to wait for stop? */ 543 544 xhci_op_write_8(sc, XHCI_CRCR, 0); 545 xhci_ring_free(sc, &sc->sc_cr); 546 cv_destroy(&sc->sc_command_cv); 547 548 xhci_rt_write_4(sc, XHCI_ERSTSZ(0), 0); 549 xhci_rt_write_8(sc, XHCI_ERSTBA(0), 0); 550 xhci_rt_write_8(sc, XHCI_ERDP(0), 0|XHCI_ERDP_LO_BUSY); 551 xhci_ring_free(sc, &sc->sc_er); 552 553 usb_freemem(&sc->sc_bus, &sc->sc_eventst_dma); 554 555 xhci_op_write_8(sc, XHCI_DCBAAP, 0); 556 usb_freemem(&sc->sc_bus, &sc->sc_dcbaa_dma); 557 558 kmem_free(sc->sc_slots, sizeof(*sc->sc_slots) * sc->sc_maxslots); 559 560 mutex_destroy(&sc->sc_lock); 561 mutex_destroy(&sc->sc_intr_lock); 562 cv_destroy(&sc->sc_softwake_cv); 563 564 pool_cache_destroy(sc->sc_xferpool); 565 566 return rv; 567 } 568 569 int 570 xhci_activate(device_t self, enum devact act) 571 { 572 struct xhci_softc * const sc = device_private(self); 573 574 switch (act) { 575 case DVACT_DEACTIVATE: 576 sc->sc_dying = true; 577 return 0; 578 default: 579 return EOPNOTSUPP; 580 } 581 } 582 583 bool 584 xhci_suspend(device_t dv, const pmf_qual_t *qual) 585 { 586 return false; 587 } 588 589 bool 590 xhci_resume(device_t dv, const pmf_qual_t *qual) 591 { 592 return false; 593 } 594 595 bool 596 xhci_shutdown(device_t self, int flags) 597 { 598 return false; 599 } 600 601 static int 602 xhci_hc_reset(struct xhci_softc * const sc) 603 { 604 uint32_t usbcmd, usbsts; 605 int i; 606 607 /* Check controller not ready */ 608 for (i = 0; i < XHCI_WAIT_CNR; i++) { 609 usbsts = xhci_op_read_4(sc, XHCI_USBSTS); 610 if ((usbsts & XHCI_STS_CNR) == 0) 611 break; 612 usb_delay_ms(&sc->sc_bus, 1); 613 } 614 if (i >= XHCI_WAIT_CNR) { 615 aprint_error_dev(sc->sc_dev, "controller not ready timeout\n"); 616 return EIO; 617 } 618 619 /* Halt controller */ 620 usbcmd = 0; 621 xhci_op_write_4(sc, XHCI_USBCMD, usbcmd); 622 usb_delay_ms(&sc->sc_bus, 1); 623 624 /* Reset controller */ 625 usbcmd = XHCI_CMD_HCRST; 626 xhci_op_write_4(sc, XHCI_USBCMD, usbcmd); 627 for (i = 0; i < XHCI_WAIT_HCRST; i++) { 628 usbcmd = xhci_op_read_4(sc, XHCI_USBCMD); 629 if ((usbcmd & XHCI_CMD_HCRST) == 0) 630 break; 631 usb_delay_ms(&sc->sc_bus, 1); 632 } 633 if (i >= XHCI_WAIT_HCRST) { 634 aprint_error_dev(sc->sc_dev, "host controller reset timeout\n"); 635 return EIO; 636 } 637 638 /* Check controller not ready */ 639 for (i = 0; i < XHCI_WAIT_CNR; i++) { 640 usbsts = xhci_op_read_4(sc, XHCI_USBSTS); 641 if ((usbsts & XHCI_STS_CNR) == 0) 642 break; 643 usb_delay_ms(&sc->sc_bus, 1); 644 } 645 if (i >= XHCI_WAIT_CNR) { 646 aprint_error_dev(sc->sc_dev, 647 "controller not ready timeout after reset\n"); 648 return EIO; 649 } 650 651 return 0; 652 } 653 654 655 static void 656 hexdump(const char *msg, const void *base, size_t len) 657 { 658 #if 0 659 size_t cnt; 660 const uint32_t *p; 661 extern paddr_t vtophys(vaddr_t); 662 663 p = base; 664 cnt = 0; 665 666 printf("*** %s (%zu bytes @ %p %p)\n", msg, len, base, 667 (void *)vtophys((vaddr_t)base)); 668 669 while (cnt < len) { 670 if (cnt % 16 == 0) 671 printf("%p: ", p); 672 else if (cnt % 8 == 0) 673 printf(" |"); 674 printf(" %08x", *p++); 675 cnt += 4; 676 if (cnt % 16 == 0) 677 printf("\n"); 678 } 679 if (cnt % 16 != 0) 680 printf("\n"); 681 #endif 682 } 683 684 /* Process extended capabilities */ 685 static void 686 xhci_ecp(struct xhci_softc *sc, uint32_t hcc) 687 { 688 uint32_t ecp, ecr; 689 690 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 691 692 ecp = XHCI_HCC_XECP(hcc) * 4; 693 while (ecp != 0) { 694 ecr = xhci_read_4(sc, ecp); 695 aprint_debug_dev(sc->sc_dev, "ECR %x: %08x\n", ecp, ecr); 696 switch (XHCI_XECP_ID(ecr)) { 697 case XHCI_ID_PROTOCOLS: { 698 uint32_t w4, w8, wc; 699 uint16_t w2; 700 w2 = (ecr >> 16) & 0xffff; 701 w4 = xhci_read_4(sc, ecp + 4); 702 w8 = xhci_read_4(sc, ecp + 8); 703 wc = xhci_read_4(sc, ecp + 0xc); 704 aprint_debug_dev(sc->sc_dev, 705 " SP: %08x %08x %08x %08x\n", ecr, w4, w8, wc); 706 /* unused */ 707 if (w4 == 0x20425355 && (w2 & 0xff00) == 0x0300) { 708 sc->sc_ss_port_start = (w8 >> 0) & 0xff;; 709 sc->sc_ss_port_count = (w8 >> 8) & 0xff;; 710 } 711 if (w4 == 0x20425355 && (w2 & 0xff00) == 0x0200) { 712 sc->sc_hs_port_start = (w8 >> 0) & 0xff; 713 sc->sc_hs_port_count = (w8 >> 8) & 0xff; 714 } 715 break; 716 } 717 case XHCI_ID_USB_LEGACY: { 718 uint8_t bios_sem; 719 720 /* Take host controller ownership from BIOS */ 721 bios_sem = xhci_read_1(sc, ecp + XHCI_XECP_BIOS_SEM); 722 if (bios_sem) { 723 /* sets xHCI to be owned by OS */ 724 xhci_write_1(sc, ecp + XHCI_XECP_OS_SEM, 1); 725 aprint_debug_dev(sc->sc_dev, 726 "waiting for BIOS to give up control\n"); 727 for (int i = 0; i < 5000; i++) { 728 bios_sem = xhci_read_1(sc, ecp + 729 XHCI_XECP_BIOS_SEM); 730 if (bios_sem == 0) 731 break; 732 DELAY(1000); 733 } 734 if (bios_sem) { 735 aprint_error_dev(sc->sc_dev, 736 "timed out waiting for BIOS\n"); 737 } 738 } 739 break; 740 } 741 default: 742 break; 743 } 744 ecr = xhci_read_4(sc, ecp); 745 if (XHCI_XECP_NEXT(ecr) == 0) { 746 ecp = 0; 747 } else { 748 ecp += XHCI_XECP_NEXT(ecr) * 4; 749 } 750 } 751 } 752 753 #define XHCI_HCCPREV1_BITS \ 754 "\177\020" /* New bitmask */ \ 755 "f\020\020XECP\0" \ 756 "f\014\4MAXPSA\0" \ 757 "b\013CFC\0" \ 758 "b\012SEC\0" \ 759 "b\011SBD\0" \ 760 "b\010FSE\0" \ 761 "b\7NSS\0" \ 762 "b\6LTC\0" \ 763 "b\5LHRC\0" \ 764 "b\4PIND\0" \ 765 "b\3PPC\0" \ 766 "b\2CZC\0" \ 767 "b\1BNC\0" \ 768 "b\0AC64\0" \ 769 "\0" 770 #define XHCI_HCCV1_x_BITS \ 771 "\177\020" /* New bitmask */ \ 772 "f\020\020XECP\0" \ 773 "f\014\4MAXPSA\0" \ 774 "b\013CFC\0" \ 775 "b\012SEC\0" \ 776 "b\011SPC\0" \ 777 "b\010PAE\0" \ 778 "b\7NSS\0" \ 779 "b\6LTC\0" \ 780 "b\5LHRC\0" \ 781 "b\4PIND\0" \ 782 "b\3PPC\0" \ 783 "b\2CSZ\0" \ 784 "b\1BNC\0" \ 785 "b\0AC64\0" \ 786 "\0" 787 788 int 789 xhci_init(struct xhci_softc *sc) 790 { 791 bus_size_t bsz; 792 uint32_t cap, hcs1, hcs2, hcs3, hcc, dboff, rtsoff; 793 uint32_t pagesize, config; 794 int i = 0; 795 uint16_t hciversion; 796 uint8_t caplength; 797 798 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 799 800 sc->sc_bus.ub_revision = USBREV_3_0; 801 sc->sc_bus.ub_usedma = true; 802 803 cap = xhci_read_4(sc, XHCI_CAPLENGTH); 804 caplength = XHCI_CAP_CAPLENGTH(cap); 805 hciversion = XHCI_CAP_HCIVERSION(cap); 806 807 if (hciversion < XHCI_HCIVERSION_0_96 || 808 hciversion > XHCI_HCIVERSION_1_0) { 809 aprint_normal_dev(sc->sc_dev, 810 "xHCI version %x.%x not known to be supported\n", 811 (hciversion >> 8) & 0xff, (hciversion >> 0) & 0xff); 812 } else { 813 aprint_verbose_dev(sc->sc_dev, "xHCI version %x.%x\n", 814 (hciversion >> 8) & 0xff, (hciversion >> 0) & 0xff); 815 } 816 817 if (bus_space_subregion(sc->sc_iot, sc->sc_ioh, 0, caplength, 818 &sc->sc_cbh) != 0) { 819 aprint_error_dev(sc->sc_dev, "capability subregion failure\n"); 820 return ENOMEM; 821 } 822 823 hcs1 = xhci_cap_read_4(sc, XHCI_HCSPARAMS1); 824 sc->sc_maxslots = XHCI_HCS1_MAXSLOTS(hcs1); 825 sc->sc_maxintrs = XHCI_HCS1_MAXINTRS(hcs1); 826 sc->sc_maxports = XHCI_HCS1_MAXPORTS(hcs1); 827 hcs2 = xhci_cap_read_4(sc, XHCI_HCSPARAMS2); 828 hcs3 = xhci_cap_read_4(sc, XHCI_HCSPARAMS3); 829 aprint_debug_dev(sc->sc_dev, 830 "hcs1=%"PRIx32" hcs2=%"PRIx32" hcs3=%"PRIx32"\n", hcs1, hcs2, hcs3); 831 832 hcc = xhci_cap_read_4(sc, XHCI_HCCPARAMS); 833 sc->sc_ac64 = XHCI_HCC_AC64(hcc); 834 sc->sc_ctxsz = XHCI_HCC_CSZ(hcc) ? 64 : 32; 835 836 char sbuf[128]; 837 if (hciversion < XHCI_HCIVERSION_1_0) 838 snprintb(sbuf, sizeof(sbuf), XHCI_HCCPREV1_BITS, hcc); 839 else 840 snprintb(sbuf, sizeof(sbuf), XHCI_HCCV1_x_BITS, hcc); 841 aprint_debug_dev(sc->sc_dev, "hcc=%s\n", sbuf); 842 aprint_debug_dev(sc->sc_dev, "xECP %x\n", XHCI_HCC_XECP(hcc) * 4); 843 844 /* print PSI and take ownership from BIOS */ 845 xhci_ecp(sc, hcc); 846 847 bsz = XHCI_PORTSC(sc->sc_maxports + 1); 848 if (bus_space_subregion(sc->sc_iot, sc->sc_ioh, caplength, bsz, 849 &sc->sc_obh) != 0) { 850 aprint_error_dev(sc->sc_dev, "operational subregion failure\n"); 851 return ENOMEM; 852 } 853 854 dboff = xhci_cap_read_4(sc, XHCI_DBOFF); 855 if (bus_space_subregion(sc->sc_iot, sc->sc_ioh, dboff, 856 sc->sc_maxslots * 4, &sc->sc_dbh) != 0) { 857 aprint_error_dev(sc->sc_dev, "doorbell subregion failure\n"); 858 return ENOMEM; 859 } 860 861 rtsoff = xhci_cap_read_4(sc, XHCI_RTSOFF); 862 if (bus_space_subregion(sc->sc_iot, sc->sc_ioh, rtsoff, 863 sc->sc_maxintrs * 0x20, &sc->sc_rbh) != 0) { 864 aprint_error_dev(sc->sc_dev, "runtime subregion failure\n"); 865 return ENOMEM; 866 } 867 868 int rv; 869 rv = xhci_hc_reset(sc); 870 if (rv != 0) { 871 return rv; 872 } 873 874 if (sc->sc_vendor_init) 875 sc->sc_vendor_init(sc); 876 877 if (sc->sc_vendor_init) 878 sc->sc_vendor_init(sc); 879 880 pagesize = xhci_op_read_4(sc, XHCI_PAGESIZE); 881 aprint_debug_dev(sc->sc_dev, "PAGESIZE 0x%08x\n", pagesize); 882 pagesize = ffs(pagesize); 883 if (pagesize == 0) { 884 aprint_error_dev(sc->sc_dev, "pagesize is 0\n"); 885 return EIO; 886 } 887 sc->sc_pgsz = 1 << (12 + (pagesize - 1)); 888 aprint_debug_dev(sc->sc_dev, "sc_pgsz 0x%08x\n", (uint32_t)sc->sc_pgsz); 889 aprint_debug_dev(sc->sc_dev, "sc_maxslots 0x%08x\n", 890 (uint32_t)sc->sc_maxslots); 891 aprint_debug_dev(sc->sc_dev, "sc_maxports %d\n", sc->sc_maxports); 892 893 usbd_status err; 894 895 sc->sc_maxspbuf = XHCI_HCS2_MAXSPBUF(hcs2); 896 aprint_debug_dev(sc->sc_dev, "sc_maxspbuf %d\n", sc->sc_maxspbuf); 897 if (sc->sc_maxspbuf != 0) { 898 err = usb_allocmem(&sc->sc_bus, 899 sizeof(uint64_t) * sc->sc_maxspbuf, sizeof(uint64_t), 900 &sc->sc_spbufarray_dma); 901 if (err) { 902 aprint_error_dev(sc->sc_dev, 903 "spbufarray init fail, err %d\n", err); 904 return ENOMEM; 905 } 906 907 sc->sc_spbuf_dma = kmem_zalloc(sizeof(*sc->sc_spbuf_dma) * 908 sc->sc_maxspbuf, KM_SLEEP); 909 uint64_t *spbufarray = KERNADDR(&sc->sc_spbufarray_dma, 0); 910 for (i = 0; i < sc->sc_maxspbuf; i++) { 911 usb_dma_t * const dma = &sc->sc_spbuf_dma[i]; 912 /* allocate contexts */ 913 err = usb_allocmem(&sc->sc_bus, sc->sc_pgsz, 914 sc->sc_pgsz, dma); 915 if (err) { 916 aprint_error_dev(sc->sc_dev, 917 "spbufarray_dma init fail, err %d\n", err); 918 rv = ENOMEM; 919 goto bad1; 920 } 921 spbufarray[i] = htole64(DMAADDR(dma, 0)); 922 usb_syncmem(dma, 0, sc->sc_pgsz, 923 BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE); 924 } 925 926 usb_syncmem(&sc->sc_spbufarray_dma, 0, 927 sizeof(uint64_t) * sc->sc_maxspbuf, BUS_DMASYNC_PREWRITE); 928 } 929 930 config = xhci_op_read_4(sc, XHCI_CONFIG); 931 config &= ~0xFF; 932 config |= sc->sc_maxslots & 0xFF; 933 xhci_op_write_4(sc, XHCI_CONFIG, config); 934 935 err = xhci_ring_init(sc, &sc->sc_cr, XHCI_COMMAND_RING_TRBS, 936 XHCI_COMMAND_RING_SEGMENTS_ALIGN); 937 if (err) { 938 aprint_error_dev(sc->sc_dev, "command ring init fail, err %d\n", 939 err); 940 rv = ENOMEM; 941 goto bad1; 942 } 943 944 err = xhci_ring_init(sc, &sc->sc_er, XHCI_EVENT_RING_TRBS, 945 XHCI_EVENT_RING_SEGMENTS_ALIGN); 946 if (err) { 947 aprint_error_dev(sc->sc_dev, "event ring init fail, err %d\n", 948 err); 949 rv = ENOMEM; 950 goto bad2; 951 } 952 953 usb_dma_t *dma; 954 size_t size; 955 size_t align; 956 957 dma = &sc->sc_eventst_dma; 958 size = roundup2(XHCI_EVENT_RING_SEGMENTS * XHCI_ERSTE_SIZE, 959 XHCI_EVENT_RING_SEGMENT_TABLE_ALIGN); 960 KASSERTMSG(size <= (512 * 1024), "eventst size %zu too large", size); 961 align = XHCI_EVENT_RING_SEGMENT_TABLE_ALIGN; 962 err = usb_allocmem(&sc->sc_bus, size, align, dma); 963 if (err) { 964 aprint_error_dev(sc->sc_dev, "eventst init fail, err %d\n", 965 err); 966 rv = ENOMEM; 967 goto bad3; 968 } 969 970 memset(KERNADDR(dma, 0), 0, size); 971 usb_syncmem(dma, 0, size, BUS_DMASYNC_PREWRITE); 972 aprint_debug_dev(sc->sc_dev, "eventst: %016jx %p %zx\n", 973 (uintmax_t)DMAADDR(&sc->sc_eventst_dma, 0), 974 KERNADDR(&sc->sc_eventst_dma, 0), 975 sc->sc_eventst_dma.udma_block->size); 976 977 dma = &sc->sc_dcbaa_dma; 978 size = (1 + sc->sc_maxslots) * sizeof(uint64_t); 979 KASSERTMSG(size <= 2048, "dcbaa size %zu too large", size); 980 align = XHCI_DEVICE_CONTEXT_BASE_ADDRESS_ARRAY_ALIGN; 981 err = usb_allocmem(&sc->sc_bus, size, align, dma); 982 if (err) { 983 aprint_error_dev(sc->sc_dev, "dcbaa init fail, err %d\n", err); 984 rv = ENOMEM; 985 goto bad4; 986 } 987 aprint_debug_dev(sc->sc_dev, "dcbaa: %016jx %p %zx\n", 988 (uintmax_t)DMAADDR(&sc->sc_dcbaa_dma, 0), 989 KERNADDR(&sc->sc_dcbaa_dma, 0), 990 sc->sc_dcbaa_dma.udma_block->size); 991 992 memset(KERNADDR(dma, 0), 0, size); 993 if (sc->sc_maxspbuf != 0) { 994 /* 995 * DCBA entry 0 hold the scratchbuf array pointer. 996 */ 997 *(uint64_t *)KERNADDR(dma, 0) = 998 htole64(DMAADDR(&sc->sc_spbufarray_dma, 0)); 999 } 1000 usb_syncmem(dma, 0, size, BUS_DMASYNC_PREWRITE); 1001 1002 sc->sc_slots = kmem_zalloc(sizeof(*sc->sc_slots) * sc->sc_maxslots, 1003 KM_SLEEP); 1004 if (sc->sc_slots == NULL) { 1005 aprint_error_dev(sc->sc_dev, "slots init fail, err %d\n", err); 1006 rv = ENOMEM; 1007 goto bad; 1008 } 1009 1010 sc->sc_xferpool = pool_cache_init(sizeof(struct xhci_xfer), 0, 0, 0, 1011 "xhcixfer", NULL, IPL_USB, NULL, NULL, NULL); 1012 if (sc->sc_xferpool == NULL) { 1013 aprint_error_dev(sc->sc_dev, "pool_cache init fail, err %d\n", 1014 err); 1015 rv = ENOMEM; 1016 goto bad; 1017 } 1018 1019 cv_init(&sc->sc_command_cv, "xhcicmd"); 1020 mutex_init(&sc->sc_lock, MUTEX_DEFAULT, IPL_SOFTUSB); 1021 mutex_init(&sc->sc_intr_lock, MUTEX_DEFAULT, IPL_USB); 1022 cv_init(&sc->sc_softwake_cv, "xhciab"); 1023 1024 /* Set up the bus struct. */ 1025 sc->sc_bus.ub_methods = &xhci_bus_methods; 1026 sc->sc_bus.ub_pipesize = sizeof(struct xhci_pipe); 1027 1028 struct xhci_erste *erst; 1029 erst = KERNADDR(&sc->sc_eventst_dma, 0); 1030 erst[0].erste_0 = htole64(xhci_ring_trbp(&sc->sc_er, 0)); 1031 erst[0].erste_2 = htole32(XHCI_EVENT_RING_TRBS); 1032 erst[0].erste_3 = htole32(0); 1033 usb_syncmem(&sc->sc_eventst_dma, 0, 1034 XHCI_ERSTE_SIZE * XHCI_EVENT_RING_SEGMENTS, BUS_DMASYNC_PREWRITE); 1035 1036 xhci_rt_write_4(sc, XHCI_ERSTSZ(0), XHCI_EVENT_RING_SEGMENTS); 1037 xhci_rt_write_8(sc, XHCI_ERSTBA(0), DMAADDR(&sc->sc_eventst_dma, 0)); 1038 xhci_rt_write_8(sc, XHCI_ERDP(0), xhci_ring_trbp(&sc->sc_er, 0) | 1039 XHCI_ERDP_LO_BUSY); 1040 xhci_op_write_8(sc, XHCI_DCBAAP, DMAADDR(&sc->sc_dcbaa_dma, 0)); 1041 xhci_op_write_8(sc, XHCI_CRCR, xhci_ring_trbp(&sc->sc_cr, 0) | 1042 sc->sc_cr.xr_cs); 1043 1044 #if 0 1045 hexdump("eventst", KERNADDR(&sc->sc_eventst_dma, 0), 1046 XHCI_ERSTE_SIZE * XHCI_EVENT_RING_SEGMENTS); 1047 #endif 1048 1049 xhci_rt_write_4(sc, XHCI_IMAN(0), XHCI_IMAN_INTR_ENA); 1050 if ((sc->sc_quirks & XHCI_QUIRK_INTEL) != 0) 1051 /* Intel xhci needs interrupt rate moderated. */ 1052 xhci_rt_write_4(sc, XHCI_IMOD(0), XHCI_IMOD_DEFAULT_LP); 1053 else 1054 xhci_rt_write_4(sc, XHCI_IMOD(0), 0); 1055 aprint_debug_dev(sc->sc_dev, "setting IMOD %u\n", 1056 xhci_rt_read_4(sc, XHCI_IMOD(0))); 1057 1058 xhci_op_write_4(sc, XHCI_USBCMD, XHCI_CMD_INTE|XHCI_CMD_RS); /* Go! */ 1059 aprint_debug_dev(sc->sc_dev, "USBCMD %08"PRIx32"\n", 1060 xhci_op_read_4(sc, XHCI_USBCMD)); 1061 1062 return 0; 1063 1064 bad: 1065 if (sc->sc_xferpool) { 1066 pool_cache_destroy(sc->sc_xferpool); 1067 sc->sc_xferpool = NULL; 1068 } 1069 1070 if (sc->sc_slots) { 1071 kmem_free(sc->sc_slots, sizeof(*sc->sc_slots) * 1072 sc->sc_maxslots); 1073 sc->sc_slots = NULL; 1074 } 1075 1076 usb_freemem(&sc->sc_bus, &sc->sc_dcbaa_dma); 1077 bad4: 1078 usb_freemem(&sc->sc_bus, &sc->sc_eventst_dma); 1079 bad3: 1080 xhci_ring_free(sc, &sc->sc_er); 1081 bad2: 1082 xhci_ring_free(sc, &sc->sc_cr); 1083 i = sc->sc_maxspbuf; 1084 bad1: 1085 for (int j = 0; j < i; j++) 1086 usb_freemem(&sc->sc_bus, &sc->sc_spbuf_dma[j]); 1087 usb_freemem(&sc->sc_bus, &sc->sc_spbufarray_dma); 1088 1089 return rv; 1090 } 1091 1092 int 1093 xhci_intr(void *v) 1094 { 1095 struct xhci_softc * const sc = v; 1096 int ret = 0; 1097 1098 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 1099 1100 if (sc == NULL) 1101 return 0; 1102 1103 mutex_spin_enter(&sc->sc_intr_lock); 1104 1105 if (sc->sc_dying || !device_has_power(sc->sc_dev)) 1106 goto done; 1107 1108 /* If we get an interrupt while polling, then just ignore it. */ 1109 if (sc->sc_bus.ub_usepolling) { 1110 #ifdef DIAGNOSTIC 1111 DPRINTFN(16, "ignored interrupt while polling", 0, 0, 0, 0); 1112 #endif 1113 goto done; 1114 } 1115 1116 ret = xhci_intr1(sc); 1117 done: 1118 mutex_spin_exit(&sc->sc_intr_lock); 1119 return ret; 1120 } 1121 1122 int 1123 xhci_intr1(struct xhci_softc * const sc) 1124 { 1125 uint32_t usbsts; 1126 uint32_t iman; 1127 1128 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 1129 1130 usbsts = xhci_op_read_4(sc, XHCI_USBSTS); 1131 DPRINTFN(16, "USBSTS %08x", usbsts, 0, 0, 0); 1132 #if 0 1133 if ((usbsts & (XHCI_STS_EINT|XHCI_STS_PCD)) == 0) { 1134 return 0; 1135 } 1136 #endif 1137 xhci_op_write_4(sc, XHCI_USBSTS, 1138 usbsts & (2|XHCI_STS_EINT|XHCI_STS_PCD)); /* XXX */ 1139 usbsts = xhci_op_read_4(sc, XHCI_USBSTS); 1140 DPRINTFN(16, "USBSTS %08x", usbsts, 0, 0, 0); 1141 1142 iman = xhci_rt_read_4(sc, XHCI_IMAN(0)); 1143 DPRINTFN(16, "IMAN0 %08x", iman, 0, 0, 0); 1144 iman |= XHCI_IMAN_INTR_PEND; 1145 xhci_rt_write_4(sc, XHCI_IMAN(0), iman); 1146 iman = xhci_rt_read_4(sc, XHCI_IMAN(0)); 1147 DPRINTFN(16, "IMAN0 %08x", iman, 0, 0, 0); 1148 usbsts = xhci_op_read_4(sc, XHCI_USBSTS); 1149 DPRINTFN(16, "USBSTS %08x", usbsts, 0, 0, 0); 1150 1151 usb_schedsoftintr(&sc->sc_bus); 1152 1153 return 1; 1154 } 1155 1156 /* 1157 * 3 port speed types used in USB stack 1158 * 1159 * usbdi speed 1160 * definition: USB_SPEED_* in usb.h 1161 * They are used in struct usbd_device in USB stack. 1162 * ioctl interface uses these values too. 1163 * port_status speed 1164 * definition: UPS_*_SPEED in usb.h 1165 * They are used in usb_port_status_t and valid only for USB 2.0. 1166 * Speed value is always 0 for Super Speed or more, and dwExtPortStatus 1167 * of usb_port_status_ext_t indicates port speed. 1168 * Note that some 3.0 values overlap with 2.0 values. 1169 * (e.g. 0x200 means UPS_POER_POWER_SS in SS and 1170 * means UPS_LOW_SPEED in HS.) 1171 * port status returned from hub also uses these values. 1172 * On NetBSD UPS_OTHER_SPEED indicates port speed is super speed 1173 * or more. 1174 * xspeed: 1175 * definition: Protocol Speed ID (PSI) (xHCI 1.1 7.2.1) 1176 * They are used in only slot context and PORTSC reg of xhci. 1177 * The difference between usbdi speed and xspeed is 1178 * that FS and LS values are swapped. 1179 */ 1180 1181 /* convert usbdi speed to xspeed */ 1182 static int 1183 xhci_speed2xspeed(int speed) 1184 { 1185 switch (speed) { 1186 case USB_SPEED_LOW: return 2; 1187 case USB_SPEED_FULL: return 1; 1188 default: return speed; 1189 } 1190 } 1191 1192 #if 0 1193 /* convert xspeed to usbdi speed */ 1194 static int 1195 xhci_xspeed2speed(int xspeed) 1196 { 1197 switch (xspeed) { 1198 case 1: return USB_SPEED_FULL; 1199 case 2: return USB_SPEED_LOW; 1200 default: return xspeed; 1201 } 1202 } 1203 #endif 1204 1205 /* convert xspeed to port status speed */ 1206 static int 1207 xhci_xspeed2psspeed(int xspeed) 1208 { 1209 switch (xspeed) { 1210 case 0: return 0; 1211 case 1: return UPS_FULL_SPEED; 1212 case 2: return UPS_LOW_SPEED; 1213 case 3: return UPS_HIGH_SPEED; 1214 default: return UPS_OTHER_SPEED; 1215 } 1216 } 1217 1218 /* construct slot context */ 1219 static void 1220 xhci_setup_sctx(struct usbd_device *dev, uint32_t *cp) 1221 { 1222 usb_device_descriptor_t * const dd = &dev->ud_ddesc; 1223 int speed = dev->ud_speed; 1224 int tthubslot, ttportnum; 1225 bool ishub; 1226 bool usemtt; 1227 1228 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 1229 1230 /* 1231 * 6.2.2, Table 57-60, 6.2.2.1, 6.2.2.2 1232 * tthubslot: 1233 * This is the slot ID of parent HS hub 1234 * if LS/FS device is connected && connected through HS hub. 1235 * This is 0 if device is not LS/FS device || 1236 * parent hub is not HS hub || 1237 * attached to root hub. 1238 * ttportnum: 1239 * This is the downstream facing port of parent HS hub 1240 * if LS/FS device is connected. 1241 * This is 0 if device is not LS/FS device || 1242 * parent hub is not HS hub || 1243 * attached to root hub. 1244 */ 1245 if (dev->ud_myhsport != NULL && 1246 dev->ud_myhub != NULL && dev->ud_myhub->ud_depth != 0 && 1247 (dev->ud_myhub != NULL && 1248 dev->ud_myhub->ud_speed == USB_SPEED_HIGH) && 1249 (speed == USB_SPEED_LOW || speed == USB_SPEED_FULL)) { 1250 ttportnum = dev->ud_myhsport->up_portno; 1251 tthubslot = dev->ud_myhsport->up_parent->ud_addr; 1252 } else { 1253 ttportnum = 0; 1254 tthubslot = 0; 1255 } 1256 DPRINTFN(4, "myhsport %p ttportnum=%d tthubslot=%d", 1257 dev->ud_myhsport, ttportnum, tthubslot, 0); 1258 1259 /* ishub is valid after reading UDESC_DEVICE */ 1260 ishub = (dd->bDeviceClass == UDCLASS_HUB); 1261 1262 /* dev->ud_hub is valid after reading UDESC_HUB */ 1263 if (ishub && dev->ud_hub) { 1264 usb_hub_descriptor_t *hd = &dev->ud_hub->uh_hubdesc; 1265 1266 cp[1] |= htole32(XHCI_SCTX_1_NUM_PORTS_SET(hd->bNbrPorts)); 1267 cp[2] |= htole32(XHCI_SCTX_2_TT_THINK_TIME_SET( 1268 __SHIFTOUT(UGETW(hd->wHubCharacteristics), UHD_TT_THINK))); 1269 DPRINTFN(4, "nports=%d ttt=%d", 1270 hd->bNbrPorts, XHCI_SCTX_2_TT_THINK_TIME_GET(cp[2]), 0, 0); 1271 } 1272 1273 #define IS_TTHUB(dd) \ 1274 ((dd)->bDeviceProtocol == UDPROTO_HSHUBSTT || \ 1275 (dd)->bDeviceProtocol == UDPROTO_HSHUBMTT) 1276 1277 /* 1278 * MTT flag is set if 1279 * 1. this is HS hub && MTT is enabled 1280 * or 1281 * 2. this is not hub && this is LS or FS device && 1282 * MTT of parent HS hub (and its parent, too) is enabled 1283 */ 1284 if (ishub && speed == USB_SPEED_HIGH && IS_TTHUB(dd)) 1285 usemtt = true; 1286 else if (!ishub && 1287 (speed == USB_SPEED_LOW || speed == USB_SPEED_FULL) && 1288 dev->ud_myhub != NULL && dev->ud_myhub->ud_depth != 0 && 1289 (dev->ud_myhub != NULL && 1290 dev->ud_myhub->ud_speed == USB_SPEED_HIGH) && 1291 dev->ud_myhsport != NULL && 1292 IS_TTHUB(&dev->ud_myhsport->up_parent->ud_ddesc)) 1293 usemtt = true; 1294 else 1295 usemtt = false; 1296 DPRINTFN(4, "class %u proto %u ishub %d usemtt %d", 1297 dd->bDeviceClass, dd->bDeviceProtocol, ishub, usemtt); 1298 1299 cp[0] |= htole32( 1300 XHCI_SCTX_0_SPEED_SET(xhci_speed2xspeed(speed)) | 1301 XHCI_SCTX_0_HUB_SET(ishub ? 1 : 0) | 1302 XHCI_SCTX_0_MTT_SET(usemtt ? 1 : 0) 1303 ); 1304 cp[1] |= htole32(0); 1305 cp[2] |= htole32( 1306 XHCI_SCTX_2_IRQ_TARGET_SET(0) | 1307 XHCI_SCTX_2_TT_HUB_SID_SET(tthubslot) | 1308 XHCI_SCTX_2_TT_PORT_NUM_SET(ttportnum) 1309 ); 1310 cp[3] |= htole32(0); 1311 } 1312 1313 static uint32_t 1314 xhci_get_maxburst(struct usbd_pipe *pipe) 1315 { 1316 usb_endpoint_descriptor_t * const ed = pipe->up_endpoint->ue_edesc; 1317 usbd_desc_iter_t iter; 1318 const usb_cdc_descriptor_t *cdcd; 1319 const usb_endpoint_ss_comp_descriptor_t * esscd = NULL; 1320 uint32_t maxb = 0; 1321 uint8_t ep; 1322 1323 cdcd = (const usb_cdc_descriptor_t *)usb_find_desc( 1324 pipe->up_dev, UDESC_INTERFACE, USBD_CDCSUBTYPE_ANY); 1325 usb_desc_iter_init(pipe->up_dev, &iter); 1326 iter.cur = (const void *)cdcd; 1327 1328 /* find endpoint_ss_comp desc for ep of this pipe */ 1329 for (ep = 0;;) { 1330 cdcd = (const usb_cdc_descriptor_t *)usb_desc_iter_next(&iter); 1331 if (cdcd == NULL) 1332 break; 1333 if (ep == 0 && cdcd->bDescriptorType == UDESC_ENDPOINT) { 1334 ep = ((const usb_endpoint_descriptor_t *)cdcd)-> 1335 bEndpointAddress; 1336 if (UE_GET_ADDR(ep) == 1337 UE_GET_ADDR(ed->bEndpointAddress)) { 1338 cdcd = (const usb_cdc_descriptor_t *) 1339 usb_desc_iter_next(&iter); 1340 break; 1341 } 1342 ep = 0; 1343 } 1344 } 1345 if (cdcd != NULL && cdcd->bDescriptorType == UDESC_ENDPOINT_SS_COMP) { 1346 esscd = (const usb_endpoint_ss_comp_descriptor_t *)cdcd; 1347 maxb = esscd->bMaxBurst; 1348 } 1349 1350 return maxb; 1351 } 1352 1353 /* 1354 * Convert endpoint bInterval value to endpoint context interval value 1355 * for Interrupt pipe. 1356 * xHCI 6.2.3.6 Table 65, USB 2.0 9.6.6 1357 */ 1358 static uint32_t 1359 xhci_bival2ival(uint32_t ival, int speed) 1360 { 1361 if (speed == USB_SPEED_LOW || speed == USB_SPEED_FULL) { 1362 int i; 1363 1364 /* 1365 * round ival down to "the nearest base 2 multiple of 1366 * bInterval * 8". 1367 * bInterval is at most 255 as its type is uByte. 1368 * 255(ms) = 2040(x 125us) < 2^11, so start with 10. 1369 */ 1370 for (i = 10; i > 0; i--) { 1371 if ((ival * 8) >= (1 << i)) 1372 break; 1373 } 1374 ival = i; 1375 } else { 1376 /* Interval = bInterval-1 for SS/HS */ 1377 ival--; 1378 } 1379 1380 return ival; 1381 } 1382 1383 /* 1384 * 4.8.2, 6.2.3.2 1385 * construct common endpoint parameters 1386 */ 1387 static void 1388 xhci_setup_endp_ctx(struct usbd_pipe *pipe, uint32_t *cp) 1389 { 1390 struct xhci_slot * const xs = pipe->up_dev->ud_hcpriv; 1391 usb_endpoint_descriptor_t * const ed = pipe->up_endpoint->ue_edesc; 1392 const u_int dci = xhci_ep_get_dci(ed); 1393 const uint8_t xfertype = UE_GET_XFERTYPE(ed->bmAttributes); 1394 uint32_t mps = UGETW(ed->wMaxPacketSize); 1395 uint32_t maxb = 0; 1396 int speed = pipe->up_dev->ud_speed; 1397 uint32_t ival = ed->bInterval; 1398 1399 cp[0] = htole32( 1400 XHCI_EPCTX_0_EPSTATE_SET(0) | 1401 XHCI_EPCTX_0_MULT_SET(0) | /* always 0 except SS iscoh */ 1402 XHCI_EPCTX_0_MAXP_STREAMS_SET(0) | 1403 XHCI_EPCTX_0_LSA_SET(0) | 1404 XHCI_EPCTX_0_MAX_ESIT_PAYLOAD_HI_SET(0) 1405 ); 1406 cp[1] = htole32( 1407 XHCI_EPCTX_1_EPTYPE_SET(xhci_ep_get_type(ed)) | 1408 XHCI_EPCTX_1_HID_SET(0) | 1409 XHCI_EPCTX_1_MAXB_SET(0) 1410 ); 1411 if (xfertype != UE_ISOCHRONOUS) 1412 cp[1] |= htole32(XHCI_EPCTX_1_CERR_SET(3)); 1413 1414 /* 6.2.3.4, 4.8.2.4 */ 1415 if (USB_IS_SS(speed)) { 1416 /* UBS 3.1 9.6.6 */ 1417 cp[1] |= htole32(XHCI_EPCTX_1_MAXP_SIZE_SET(mps)); 1418 /* UBS 3.1 9.6.7 */ 1419 maxb = xhci_get_maxburst(pipe); 1420 cp[1] |= htole32(XHCI_EPCTX_1_MAXB_SET(maxb)); 1421 } else { 1422 /* UBS 2.0 9.6.6 */ 1423 cp[1] |= htole32(XHCI_EPCTX_1_MAXP_SIZE_SET(UE_GET_SIZE(mps))); 1424 1425 /* 6.2.3.4 */ 1426 if (speed == USB_SPEED_HIGH && 1427 (xfertype == UE_ISOCHRONOUS || xfertype == UE_INTERRUPT)) { 1428 maxb = UE_GET_TRANS(mps); 1429 } else { 1430 /* LS/FS or HS CTRL or HS BULK */ 1431 maxb = 0; 1432 } 1433 cp[1] |= htole32(XHCI_EPCTX_1_MAXB_SET(maxb)); 1434 } 1435 1436 if (xfertype == UE_CONTROL) 1437 cp[4] = htole32(XHCI_EPCTX_4_AVG_TRB_LEN_SET(8)); /* 6.2.3 */ 1438 else if (USB_IS_SS(speed)) 1439 cp[4] = htole32(XHCI_EPCTX_4_AVG_TRB_LEN_SET(mps)); 1440 else 1441 cp[4] = htole32(XHCI_EPCTX_4_AVG_TRB_LEN_SET(UE_GET_SIZE(mps))); 1442 1443 switch (xfertype) { 1444 case UE_CONTROL: 1445 break; 1446 case UE_BULK: 1447 /* XXX Set MaxPStreams, HID, and LSA if streams enabled */ 1448 break; 1449 case UE_INTERRUPT: 1450 if (pipe->up_interval != USBD_DEFAULT_INTERVAL) 1451 ival = pipe->up_interval; 1452 1453 ival = xhci_bival2ival(ival, speed); 1454 cp[0] |= htole32(XHCI_EPCTX_0_IVAL_SET(ival)); 1455 break; 1456 case UE_ISOCHRONOUS: 1457 if (pipe->up_interval != USBD_DEFAULT_INTERVAL) 1458 ival = pipe->up_interval; 1459 1460 /* xHCI 6.2.3.6 Table 65, USB 2.0 9.6.6 */ 1461 if (speed == USB_SPEED_FULL) 1462 ival += 3; /* 1ms -> 125us */ 1463 ival--; 1464 cp[0] |= htole32(XHCI_EPCTX_0_IVAL_SET(ival)); 1465 1466 if (USB_IS_SS(speed)) { 1467 /* XXX if LEC = 1, set ESIT instead */ 1468 cp[0] |= htole32(XHCI_EPCTX_0_MULT_SET(0)); 1469 } 1470 break; 1471 default: 1472 break; 1473 } 1474 *(uint64_t *)(&cp[2]) = htole64( 1475 xhci_ring_trbp(&xs->xs_ep[dci].xe_tr, 0) | 1476 XHCI_EPCTX_2_DCS_SET(1)); 1477 } 1478 1479 /* 1480 * Construct input contexts and issue TRB 1481 */ 1482 static usbd_status 1483 xhci_configure_endpoint(struct usbd_pipe *pipe) 1484 { 1485 struct xhci_softc * const sc = XHCI_PIPE2SC(pipe); 1486 struct xhci_slot * const xs = pipe->up_dev->ud_hcpriv; 1487 const u_int dci = xhci_ep_get_dci(pipe->up_endpoint->ue_edesc); 1488 struct xhci_trb trb; 1489 usbd_status err; 1490 uint32_t *cp; 1491 1492 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 1493 DPRINTFN(4, "slot %u dci %u epaddr 0x%02x attr 0x%02x", 1494 xs->xs_idx, dci, pipe->up_endpoint->ue_edesc->bEndpointAddress, 1495 pipe->up_endpoint->ue_edesc->bmAttributes); 1496 1497 /* XXX ensure input context is available? */ 1498 1499 memset(xhci_slot_get_icv(sc, xs, 0), 0, sc->sc_pgsz); 1500 1501 cp = xhci_slot_get_icv(sc, xs, XHCI_ICI_INPUT_CONTROL); 1502 cp[0] = htole32(0); 1503 cp[1] = htole32(XHCI_INCTX_1_ADD_MASK(dci)); 1504 1505 /* set up input slot context */ 1506 cp = xhci_slot_get_icv(sc, xs, xhci_dci_to_ici(XHCI_DCI_SLOT)); 1507 xhci_setup_sctx(pipe->up_dev, cp); 1508 cp[0] |= htole32(XHCI_SCTX_0_CTX_NUM_SET(dci)); 1509 1510 /* set up input endpoint context */ 1511 cp = xhci_slot_get_icv(sc, xs, xhci_dci_to_ici(dci)); 1512 xhci_setup_endp_ctx(pipe, cp); 1513 1514 /* sync input contexts before they are read from memory */ 1515 usb_syncmem(&xs->xs_ic_dma, 0, sc->sc_pgsz, BUS_DMASYNC_PREWRITE); 1516 hexdump("input control context", xhci_slot_get_icv(sc, xs, 0), 1517 sc->sc_ctxsz * 1); 1518 hexdump("input endpoint context", xhci_slot_get_icv(sc, xs, 1519 xhci_dci_to_ici(dci)), sc->sc_ctxsz * 1); 1520 1521 trb.trb_0 = xhci_slot_get_icp(sc, xs, 0); 1522 trb.trb_2 = 0; 1523 trb.trb_3 = XHCI_TRB_3_SLOT_SET(xs->xs_idx) | 1524 XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_CONFIGURE_EP); 1525 1526 err = xhci_do_command(sc, &trb, USBD_DEFAULT_TIMEOUT); 1527 1528 usb_syncmem(&xs->xs_dc_dma, 0, sc->sc_pgsz, BUS_DMASYNC_POSTREAD); 1529 hexdump("output context", xhci_slot_get_dcv(sc, xs, dci), 1530 sc->sc_ctxsz * 1); 1531 1532 return err; 1533 } 1534 1535 #if 0 1536 static usbd_status 1537 xhci_unconfigure_endpoint(struct usbd_pipe *pipe) 1538 { 1539 #ifdef USB_DEBUG 1540 struct xhci_slot * const xs = pipe->up_dev->ud_hcpriv; 1541 #endif 1542 1543 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 1544 DPRINTFN(4, "slot %u", xs->xs_idx, 0, 0, 0); 1545 1546 return USBD_NORMAL_COMPLETION; 1547 } 1548 #endif 1549 1550 /* 4.6.8, 6.4.3.7 */ 1551 static usbd_status 1552 xhci_reset_endpoint(struct usbd_pipe *pipe) 1553 { 1554 struct xhci_softc * const sc = XHCI_PIPE2SC(pipe); 1555 struct xhci_slot * const xs = pipe->up_dev->ud_hcpriv; 1556 const u_int dci = xhci_ep_get_dci(pipe->up_endpoint->ue_edesc); 1557 struct xhci_trb trb; 1558 usbd_status err; 1559 1560 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 1561 DPRINTFN(4, "slot %u dci %u", xs->xs_idx, dci, 0, 0); 1562 1563 KASSERT(!mutex_owned(&sc->sc_lock)); 1564 1565 trb.trb_0 = 0; 1566 trb.trb_2 = 0; 1567 trb.trb_3 = XHCI_TRB_3_SLOT_SET(xs->xs_idx) | 1568 XHCI_TRB_3_EP_SET(dci) | 1569 XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_RESET_EP); 1570 1571 err = xhci_do_command(sc, &trb, USBD_DEFAULT_TIMEOUT); 1572 1573 return err; 1574 } 1575 1576 /* 1577 * 4.6.9, 6.4.3.8 1578 * Stop execution of TDs on xfer ring. 1579 * Should be called with sc_lock held. 1580 */ 1581 static usbd_status 1582 xhci_stop_endpoint(struct usbd_pipe *pipe) 1583 { 1584 struct xhci_softc * const sc = XHCI_PIPE2SC(pipe); 1585 struct xhci_slot * const xs = pipe->up_dev->ud_hcpriv; 1586 struct xhci_trb trb; 1587 usbd_status err; 1588 const u_int dci = xhci_ep_get_dci(pipe->up_endpoint->ue_edesc); 1589 1590 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 1591 DPRINTFN(4, "slot %u dci %u", xs->xs_idx, dci, 0, 0); 1592 1593 KASSERT(mutex_owned(&sc->sc_lock)); 1594 1595 trb.trb_0 = 0; 1596 trb.trb_2 = 0; 1597 trb.trb_3 = XHCI_TRB_3_SLOT_SET(xs->xs_idx) | 1598 XHCI_TRB_3_EP_SET(dci) | 1599 XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_STOP_EP); 1600 1601 err = xhci_do_command_locked(sc, &trb, USBD_DEFAULT_TIMEOUT); 1602 1603 return err; 1604 } 1605 1606 /* 1607 * Set TR Dequeue Pointer. 1608 * xCHI 1.1 4.6.10 6.4.3.9 1609 * Purge all of the transfer requests on ring. 1610 * EPSTATE of endpoint must be ERROR or STOPPED, or CONTEXT_STATE error. 1611 */ 1612 static usbd_status 1613 xhci_set_dequeue(struct usbd_pipe *pipe) 1614 { 1615 struct xhci_softc * const sc = XHCI_PIPE2SC(pipe); 1616 struct xhci_slot * const xs = pipe->up_dev->ud_hcpriv; 1617 const u_int dci = xhci_ep_get_dci(pipe->up_endpoint->ue_edesc); 1618 struct xhci_ring * const xr = &xs->xs_ep[dci].xe_tr; 1619 struct xhci_trb trb; 1620 usbd_status err; 1621 1622 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 1623 DPRINTFN(4, "slot %u dci %u", xs->xs_idx, dci, 0, 0); 1624 1625 memset(xr->xr_trb, 0, xr->xr_ntrb * XHCI_TRB_SIZE); 1626 usb_syncmem(&xr->xr_dma, 0, xr->xr_ntrb * XHCI_TRB_SIZE, 1627 BUS_DMASYNC_PREWRITE); 1628 memset(xr->xr_cookies, 0, xr->xr_ntrb * sizeof(*xr->xr_cookies)); 1629 1630 xr->xr_ep = 0; 1631 xr->xr_cs = 1; 1632 1633 /* set DCS */ 1634 trb.trb_0 = xhci_ring_trbp(xr, 0) | 1; /* XXX */ 1635 trb.trb_2 = 0; 1636 trb.trb_3 = XHCI_TRB_3_SLOT_SET(xs->xs_idx) | 1637 XHCI_TRB_3_EP_SET(dci) | 1638 XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_SET_TR_DEQUEUE); 1639 1640 err = xhci_do_command(sc, &trb, USBD_DEFAULT_TIMEOUT); 1641 1642 return err; 1643 } 1644 1645 /* 1646 * Open new pipe: called from usbd_setup_pipe_flags. 1647 * Fills methods of pipe. 1648 * If pipe is not for ep0, calls configure_endpoint. 1649 */ 1650 static usbd_status 1651 xhci_open(struct usbd_pipe *pipe) 1652 { 1653 struct usbd_device * const dev = pipe->up_dev; 1654 struct xhci_softc * const sc = XHCI_BUS2SC(dev->ud_bus); 1655 usb_endpoint_descriptor_t * const ed = pipe->up_endpoint->ue_edesc; 1656 const uint8_t xfertype = UE_GET_XFERTYPE(ed->bmAttributes); 1657 1658 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 1659 DPRINTFN(1, "addr %d depth %d port %d speed %d", 1660 dev->ud_addr, dev->ud_depth, dev->ud_powersrc->up_portno, 1661 dev->ud_speed); 1662 1663 if (sc->sc_dying) 1664 return USBD_IOERROR; 1665 1666 /* Root Hub */ 1667 if (dev->ud_depth == 0 && dev->ud_powersrc->up_portno == 0) { 1668 switch (ed->bEndpointAddress) { 1669 case USB_CONTROL_ENDPOINT: 1670 pipe->up_methods = &roothub_ctrl_methods; 1671 break; 1672 case UE_DIR_IN | USBROOTHUB_INTR_ENDPT: 1673 pipe->up_methods = &xhci_root_intr_methods; 1674 break; 1675 default: 1676 pipe->up_methods = NULL; 1677 DPRINTFN(0, "bad bEndpointAddress 0x%02x", 1678 ed->bEndpointAddress, 0, 0, 0); 1679 return USBD_INVAL; 1680 } 1681 return USBD_NORMAL_COMPLETION; 1682 } 1683 1684 switch (xfertype) { 1685 case UE_CONTROL: 1686 pipe->up_methods = &xhci_device_ctrl_methods; 1687 break; 1688 case UE_ISOCHRONOUS: 1689 pipe->up_methods = &xhci_device_isoc_methods; 1690 return USBD_INVAL; 1691 break; 1692 case UE_BULK: 1693 pipe->up_methods = &xhci_device_bulk_methods; 1694 break; 1695 case UE_INTERRUPT: 1696 pipe->up_methods = &xhci_device_intr_methods; 1697 break; 1698 default: 1699 return USBD_IOERROR; 1700 break; 1701 } 1702 1703 if (ed->bEndpointAddress != USB_CONTROL_ENDPOINT) 1704 return xhci_configure_endpoint(pipe); 1705 1706 return USBD_NORMAL_COMPLETION; 1707 } 1708 1709 /* 1710 * Closes pipe, called from usbd_kill_pipe via close methods. 1711 * If the endpoint to be closed is ep0, disable_slot. 1712 * Should be called with sc_lock held. 1713 */ 1714 static void 1715 xhci_close_pipe(struct usbd_pipe *pipe) 1716 { 1717 struct xhci_softc * const sc = XHCI_PIPE2SC(pipe); 1718 struct xhci_slot * const xs = pipe->up_dev->ud_hcpriv; 1719 usb_endpoint_descriptor_t * const ed = pipe->up_endpoint->ue_edesc; 1720 const u_int dci = xhci_ep_get_dci(ed); 1721 struct xhci_trb trb; 1722 uint32_t *cp; 1723 1724 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 1725 1726 if (sc->sc_dying) 1727 return; 1728 1729 /* xs is uninitialized before xhci_init_slot */ 1730 if (xs == NULL || xs->xs_idx == 0) 1731 return; 1732 1733 DPRINTFN(4, "pipe %p slot %u dci %u", pipe, xs->xs_idx, dci, 0); 1734 1735 KASSERTMSG(!cpu_intr_p() && !cpu_softintr_p(), "called from intr ctx"); 1736 KASSERT(mutex_owned(&sc->sc_lock)); 1737 1738 if (pipe->up_dev->ud_depth == 0) 1739 return; 1740 1741 if (dci == XHCI_DCI_EP_CONTROL) { 1742 DPRINTFN(4, "closing ep0", 0, 0, 0, 0); 1743 xhci_disable_slot(sc, xs->xs_idx); 1744 return; 1745 } 1746 1747 /* 1748 * This may fail in the case that xhci_close_pipe is called after 1749 * xhci_abort_xfer e.g. usbd_kill_pipe. 1750 */ 1751 (void)xhci_stop_endpoint(pipe); 1752 1753 /* 1754 * set appropriate bit to be dropped. 1755 * don't set DC bit to 1, otherwise all endpoints 1756 * would be deconfigured. 1757 */ 1758 cp = xhci_slot_get_icv(sc, xs, XHCI_ICI_INPUT_CONTROL); 1759 cp[0] = htole32(XHCI_INCTX_0_DROP_MASK(dci)); 1760 cp[1] = htole32(0); 1761 1762 /* XXX should be most significant one, not dci? */ 1763 cp = xhci_slot_get_icv(sc, xs, xhci_dci_to_ici(XHCI_DCI_SLOT)); 1764 cp[0] = htole32(XHCI_SCTX_0_CTX_NUM_SET(dci)); 1765 1766 /* sync input contexts before they are read from memory */ 1767 usb_syncmem(&xs->xs_ic_dma, 0, sc->sc_pgsz, BUS_DMASYNC_PREWRITE); 1768 1769 trb.trb_0 = xhci_slot_get_icp(sc, xs, 0); 1770 trb.trb_2 = 0; 1771 trb.trb_3 = XHCI_TRB_3_SLOT_SET(xs->xs_idx) | 1772 XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_CONFIGURE_EP); 1773 1774 (void)xhci_do_command_locked(sc, &trb, USBD_DEFAULT_TIMEOUT); 1775 usb_syncmem(&xs->xs_dc_dma, 0, sc->sc_pgsz, BUS_DMASYNC_POSTREAD); 1776 } 1777 1778 /* 1779 * Abort transfer. 1780 * Called with sc_lock held. 1781 * May be called from softintr context. 1782 */ 1783 static void 1784 xhci_abort_xfer(struct usbd_xfer *xfer, usbd_status status) 1785 { 1786 struct xhci_softc * const sc = XHCI_XFER2SC(xfer); 1787 1788 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 1789 DPRINTFN(4, "xfer %p pipe %p status %d", 1790 xfer, xfer->ux_pipe, status, 0); 1791 1792 KASSERT(mutex_owned(&sc->sc_lock)); 1793 1794 if (sc->sc_dying) { 1795 /* If we're dying, just do the software part. */ 1796 DPRINTFN(4, "dying", 0, 0, 0, 0); 1797 xfer->ux_status = status; /* make software ignore it */ 1798 callout_stop(&xfer->ux_callout); 1799 usb_transfer_complete(xfer); 1800 return; 1801 } 1802 1803 /* XXX need more stuff */ 1804 xfer->ux_status = status; 1805 callout_stop(&xfer->ux_callout); 1806 usb_transfer_complete(xfer); 1807 DPRINTFN(14, "end", 0, 0, 0, 0); 1808 1809 KASSERT(mutex_owned(&sc->sc_lock)); 1810 } 1811 1812 /* 1813 * Recover STALLed endpoint. 1814 * xHCI 1.1 sect 4.10.2.1 1815 * Issue RESET_EP to recover halt condition and SET_TR_DEQUEUE to remove 1816 * all transfers on transfer ring. 1817 * These are done in thread context asynchronously. 1818 */ 1819 static void 1820 xhci_clear_endpoint_stall_async_task(void *cookie) 1821 { 1822 struct usbd_xfer * const xfer = cookie; 1823 struct xhci_softc * const sc = XHCI_XFER2SC(xfer); 1824 struct xhci_slot * const xs = xfer->ux_pipe->up_dev->ud_hcpriv; 1825 const u_int dci = xhci_ep_get_dci(xfer->ux_pipe->up_endpoint->ue_edesc); 1826 struct xhci_ring * const tr = &xs->xs_ep[dci].xe_tr; 1827 1828 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 1829 DPRINTFN(4, "xfer %p slot %u dci %u", xfer, xs->xs_idx, dci, 0); 1830 1831 xhci_reset_endpoint(xfer->ux_pipe); 1832 xhci_set_dequeue(xfer->ux_pipe); 1833 1834 mutex_enter(&sc->sc_lock); 1835 tr->is_halted = false; 1836 usb_transfer_complete(xfer); 1837 mutex_exit(&sc->sc_lock); 1838 DPRINTFN(4, "ends", 0, 0, 0, 0); 1839 } 1840 1841 static usbd_status 1842 xhci_clear_endpoint_stall_async(struct usbd_xfer *xfer) 1843 { 1844 struct xhci_softc * const sc = XHCI_XFER2SC(xfer); 1845 struct xhci_pipe * const xp = (struct xhci_pipe *)xfer->ux_pipe; 1846 1847 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 1848 DPRINTFN(4, "xfer %p", xfer, 0, 0, 0); 1849 1850 if (sc->sc_dying) { 1851 return USBD_IOERROR; 1852 } 1853 1854 usb_init_task(&xp->xp_async_task, 1855 xhci_clear_endpoint_stall_async_task, xfer, USB_TASKQ_MPSAFE); 1856 usb_add_task(xfer->ux_pipe->up_dev, &xp->xp_async_task, USB_TASKQ_HC); 1857 DPRINTFN(4, "ends", 0, 0, 0, 0); 1858 1859 return USBD_NORMAL_COMPLETION; 1860 } 1861 1862 /* Process roothub port status/change events and notify to uhub_intr. */ 1863 static void 1864 xhci_rhpsc(struct xhci_softc * const sc, u_int port) 1865 { 1866 struct usbd_xfer * const xfer = sc->sc_intrxfer; 1867 uint8_t *p; 1868 1869 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 1870 DPRINTFN(4, "xhci%d: port %u status change", device_unit(sc->sc_dev), 1871 port, 0, 0); 1872 1873 if (xfer == NULL) 1874 return; 1875 1876 if (port > sc->sc_maxports) 1877 return; 1878 1879 p = xfer->ux_buf; 1880 memset(p, 0, xfer->ux_length); 1881 p[port/NBBY] |= 1 << (port%NBBY); 1882 xfer->ux_actlen = xfer->ux_length; 1883 xfer->ux_status = USBD_NORMAL_COMPLETION; 1884 usb_transfer_complete(xfer); 1885 } 1886 1887 /* Process Transfer Events */ 1888 static void 1889 xhci_event_transfer(struct xhci_softc * const sc, 1890 const struct xhci_trb * const trb) 1891 { 1892 uint64_t trb_0; 1893 uint32_t trb_2, trb_3; 1894 uint8_t trbcode; 1895 u_int slot, dci; 1896 struct xhci_slot *xs; 1897 struct xhci_ring *xr; 1898 struct xhci_xfer *xx; 1899 struct usbd_xfer *xfer; 1900 usbd_status err; 1901 1902 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 1903 1904 trb_0 = le64toh(trb->trb_0); 1905 trb_2 = le32toh(trb->trb_2); 1906 trb_3 = le32toh(trb->trb_3); 1907 trbcode = XHCI_TRB_2_ERROR_GET(trb_2); 1908 slot = XHCI_TRB_3_SLOT_GET(trb_3); 1909 dci = XHCI_TRB_3_EP_GET(trb_3); 1910 xs = &sc->sc_slots[slot]; 1911 xr = &xs->xs_ep[dci].xe_tr; 1912 1913 /* sanity check */ 1914 KASSERTMSG(xs->xs_idx != 0 && xs->xs_idx <= sc->sc_maxslots, 1915 "invalid xs_idx %u slot %u", xs->xs_idx, slot); 1916 1917 int idx = 0; 1918 if ((trb_3 & XHCI_TRB_3_ED_BIT) == 0) { 1919 if (xhci_trb_get_idx(xr, trb_0, &idx)) { 1920 DPRINTFN(0, "invalid trb_0 0x%"PRIx64, trb_0, 0, 0, 0); 1921 return; 1922 } 1923 xx = xr->xr_cookies[idx]; 1924 1925 /* 1926 * If endpoint is stopped between TDs, TRB pointer points at 1927 * next TRB, however, it is not put yet or is a garbage TRB. 1928 * That's why xr_cookies may be NULL or look like broken. 1929 * Note: this ev happens only when hciversion >= 1.0 or 1930 * hciversion == 0.96 and FSE of hcc1 is set. 1931 */ 1932 if (xx == NULL || trbcode == XHCI_TRB_ERROR_LENGTH) { 1933 DPRINTFN(1, "xx NULL: #%u: cookie %p: code %u trb_0 %" 1934 PRIx64, idx, xx, trbcode, trb_0); 1935 } 1936 } else { 1937 /* When ED != 0, trb_0 is kaddr of struct xhci_xfer. */ 1938 xx = (void *)(uintptr_t)(trb_0 & ~0x3); 1939 } 1940 /* XXX this may not happen */ 1941 if (xx == NULL) { 1942 DPRINTFN(1, "xfer done: xx is NULL", 0, 0, 0, 0); 1943 return; 1944 } 1945 xfer = &xx->xx_xfer; 1946 /* XXX this may happen when detaching */ 1947 if (xfer == NULL) { 1948 DPRINTFN(1, "xx(%p)->xx_xfer is NULL trb_0 %#"PRIx64, 1949 xx, trb_0, 0, 0); 1950 return; 1951 } 1952 DPRINTFN(14, "xfer %p", xfer, 0, 0, 0); 1953 /* XXX I dunno why this happens */ 1954 KASSERTMSG(xfer->ux_pipe != NULL, "xfer(%p)->ux_pipe is NULL", xfer); 1955 1956 if (!xfer->ux_pipe->up_repeat && 1957 SIMPLEQ_EMPTY(&xfer->ux_pipe->up_queue)) { 1958 DPRINTFN(1, "xfer(%p)->pipe not queued", xfer, 0, 0, 0); 1959 return; 1960 } 1961 1962 /* 4.11.5.2 Event Data TRB */ 1963 if ((trb_3 & XHCI_TRB_3_ED_BIT) != 0) { 1964 DPRINTFN(14, "transfer Event Data: 0x%016"PRIx64" 0x%08"PRIx32 1965 " %02x", trb_0, XHCI_TRB_2_REM_GET(trb_2), trbcode, 0); 1966 if ((trb_0 & 0x3) == 0x3) { 1967 xfer->ux_actlen = XHCI_TRB_2_REM_GET(trb_2); 1968 } 1969 } 1970 1971 switch (trbcode) { 1972 case XHCI_TRB_ERROR_SHORT_PKT: 1973 case XHCI_TRB_ERROR_SUCCESS: 1974 xfer->ux_actlen = 1975 xfer->ux_length - XHCI_TRB_2_REM_GET(trb_2); 1976 err = USBD_NORMAL_COMPLETION; 1977 break; 1978 case XHCI_TRB_ERROR_STALL: 1979 case XHCI_TRB_ERROR_BABBLE: 1980 DPRINTFN(1, "ERR %u slot %u dci %u", trbcode, slot, dci, 0); 1981 xr->is_halted = true; 1982 err = USBD_STALLED; 1983 /* 1984 * Stalled endpoints can be recoverd by issuing 1985 * command TRB TYPE_RESET_EP on xHCI instead of 1986 * issuing request CLEAR_FEATURE UF_ENDPOINT_HALT 1987 * on the endpoint. However, this function may be 1988 * called from softint context (e.g. from umass), 1989 * in that case driver gets KASSERT in cv_timedwait 1990 * in xhci_do_command. 1991 * To avoid this, this runs reset_endpoint and 1992 * usb_transfer_complete in usb task thread 1993 * asynchronously (and then umass issues clear 1994 * UF_ENDPOINT_HALT). 1995 */ 1996 xfer->ux_status = err; 1997 xhci_clear_endpoint_stall_async(xfer); 1998 return; 1999 default: 2000 DPRINTFN(1, "ERR %u slot %u dci %u", trbcode, slot, dci, 0); 2001 err = USBD_IOERROR; 2002 break; 2003 } 2004 xfer->ux_status = err; 2005 2006 if ((trb_3 & XHCI_TRB_3_ED_BIT) != 0) { 2007 if ((trb_0 & 0x3) == 0x0) { 2008 callout_stop(&xfer->ux_callout); 2009 usb_transfer_complete(xfer); 2010 } 2011 } else { 2012 callout_stop(&xfer->ux_callout); 2013 usb_transfer_complete(xfer); 2014 } 2015 } 2016 2017 /* Process Command complete events */ 2018 static void 2019 xhci_event_cmd(struct xhci_softc * const sc, 2020 const struct xhci_trb * const trb) 2021 { 2022 uint64_t trb_0; 2023 uint32_t trb_2, trb_3; 2024 2025 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 2026 2027 trb_0 = le64toh(trb->trb_0); 2028 trb_2 = le32toh(trb->trb_2); 2029 trb_3 = le32toh(trb->trb_3); 2030 2031 if (trb_0 == sc->sc_command_addr) { 2032 sc->sc_result_trb.trb_0 = trb_0; 2033 sc->sc_result_trb.trb_2 = trb_2; 2034 sc->sc_result_trb.trb_3 = trb_3; 2035 if (XHCI_TRB_2_ERROR_GET(trb_2) != 2036 XHCI_TRB_ERROR_SUCCESS) { 2037 DPRINTFN(1, "command completion " 2038 "failure: 0x%016"PRIx64" 0x%08"PRIx32" " 2039 "0x%08"PRIx32, trb_0, trb_2, trb_3, 0); 2040 } 2041 cv_signal(&sc->sc_command_cv); 2042 } else { 2043 DPRINTFN(1, "spurious event: %p 0x%016"PRIx64" " 2044 "0x%08"PRIx32" 0x%08"PRIx32, trb, trb_0, 2045 trb_2, trb_3); 2046 } 2047 } 2048 2049 /* 2050 * Process events. 2051 * called from xhci_softintr 2052 */ 2053 static void 2054 xhci_handle_event(struct xhci_softc * const sc, 2055 const struct xhci_trb * const trb) 2056 { 2057 uint64_t trb_0; 2058 uint32_t trb_2, trb_3; 2059 2060 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 2061 2062 trb_0 = le64toh(trb->trb_0); 2063 trb_2 = le32toh(trb->trb_2); 2064 trb_3 = le32toh(trb->trb_3); 2065 2066 DPRINTFN(14, "event: %p 0x%016"PRIx64" 0x%08"PRIx32" 0x%08"PRIx32, 2067 trb, trb_0, trb_2, trb_3); 2068 2069 /* 2070 * 4.11.3.1, 6.4.2.1 2071 * TRB Pointer is invalid for these completion codes. 2072 */ 2073 switch (XHCI_TRB_2_ERROR_GET(trb_2)) { 2074 case XHCI_TRB_ERROR_RING_UNDERRUN: 2075 case XHCI_TRB_ERROR_RING_OVERRUN: 2076 case XHCI_TRB_ERROR_VF_RING_FULL: 2077 return; 2078 default: 2079 if (trb_0 == 0) { 2080 return; 2081 } 2082 break; 2083 } 2084 2085 switch (XHCI_TRB_3_TYPE_GET(trb_3)) { 2086 case XHCI_TRB_EVENT_TRANSFER: 2087 xhci_event_transfer(sc, trb); 2088 break; 2089 case XHCI_TRB_EVENT_CMD_COMPLETE: 2090 xhci_event_cmd(sc, trb); 2091 break; 2092 case XHCI_TRB_EVENT_PORT_STS_CHANGE: 2093 xhci_rhpsc(sc, (uint32_t)((trb_0 >> 24) & 0xff)); 2094 break; 2095 default: 2096 break; 2097 } 2098 } 2099 2100 static void 2101 xhci_softintr(void *v) 2102 { 2103 struct usbd_bus * const bus = v; 2104 struct xhci_softc * const sc = XHCI_BUS2SC(bus); 2105 struct xhci_ring * const er = &sc->sc_er; 2106 struct xhci_trb *trb; 2107 int i, j, k; 2108 2109 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 2110 2111 KASSERT(sc->sc_bus.ub_usepolling || mutex_owned(&sc->sc_lock)); 2112 2113 i = er->xr_ep; 2114 j = er->xr_cs; 2115 2116 DPRINTFN(16, "xr_ep %d xr_cs %d", i, j, 0, 0); 2117 2118 while (1) { 2119 usb_syncmem(&er->xr_dma, XHCI_TRB_SIZE * i, XHCI_TRB_SIZE, 2120 BUS_DMASYNC_POSTREAD); 2121 trb = &er->xr_trb[i]; 2122 k = (le32toh(trb->trb_3) & XHCI_TRB_3_CYCLE_BIT) ? 1 : 0; 2123 2124 if (j != k) 2125 break; 2126 2127 xhci_handle_event(sc, trb); 2128 2129 i++; 2130 if (i == XHCI_EVENT_RING_TRBS) { 2131 i = 0; 2132 j ^= 1; 2133 } 2134 } 2135 2136 er->xr_ep = i; 2137 er->xr_cs = j; 2138 2139 xhci_rt_write_8(sc, XHCI_ERDP(0), xhci_ring_trbp(er, er->xr_ep) | 2140 XHCI_ERDP_LO_BUSY); 2141 2142 DPRINTFN(16, "ends", 0, 0, 0, 0); 2143 2144 return; 2145 } 2146 2147 static void 2148 xhci_poll(struct usbd_bus *bus) 2149 { 2150 struct xhci_softc * const sc = XHCI_BUS2SC(bus); 2151 2152 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 2153 2154 mutex_spin_enter(&sc->sc_intr_lock); 2155 xhci_intr1(sc); 2156 mutex_spin_exit(&sc->sc_intr_lock); 2157 2158 return; 2159 } 2160 2161 static struct usbd_xfer * 2162 xhci_allocx(struct usbd_bus *bus, unsigned int nframes) 2163 { 2164 struct xhci_softc * const sc = XHCI_BUS2SC(bus); 2165 struct usbd_xfer *xfer; 2166 2167 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 2168 2169 xfer = pool_cache_get(sc->sc_xferpool, PR_NOWAIT); 2170 if (xfer != NULL) { 2171 memset(xfer, 0, sizeof(struct xhci_xfer)); 2172 #ifdef DIAGNOSTIC 2173 xfer->ux_state = XFER_BUSY; 2174 #endif 2175 } 2176 2177 return xfer; 2178 } 2179 2180 static void 2181 xhci_freex(struct usbd_bus *bus, struct usbd_xfer *xfer) 2182 { 2183 struct xhci_softc * const sc = XHCI_BUS2SC(bus); 2184 2185 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 2186 2187 #ifdef DIAGNOSTIC 2188 if (xfer->ux_state != XFER_BUSY) { 2189 DPRINTFN(0, "xfer=%p not busy, 0x%08x", 2190 xfer, xfer->ux_state, 0, 0); 2191 } 2192 xfer->ux_state = XFER_FREE; 2193 #endif 2194 pool_cache_put(sc->sc_xferpool, xfer); 2195 } 2196 2197 static void 2198 xhci_get_lock(struct usbd_bus *bus, kmutex_t **lock) 2199 { 2200 struct xhci_softc * const sc = XHCI_BUS2SC(bus); 2201 2202 *lock = &sc->sc_lock; 2203 } 2204 2205 extern uint32_t usb_cookie_no; 2206 2207 /* 2208 * xHCI 4.3 2209 * Called when uhub_explore finds a new device (via usbd_new_device). 2210 * Port initialization and speed detection (4.3.1) are already done in uhub.c. 2211 * This function does: 2212 * Allocate and construct dev structure of default endpoint (ep0). 2213 * Allocate and open pipe of ep0. 2214 * Enable slot and initialize slot context. 2215 * Set Address. 2216 * Read initial device descriptor. 2217 * Determine initial MaxPacketSize (mps) by speed. 2218 * Read full device descriptor. 2219 * Register this device. 2220 */ 2221 static usbd_status 2222 xhci_new_device(device_t parent, struct usbd_bus *bus, int depth, 2223 int speed, int port, struct usbd_port *up) 2224 { 2225 struct xhci_softc * const sc = XHCI_BUS2SC(bus); 2226 struct usbd_device *dev; 2227 usbd_status err; 2228 usb_device_descriptor_t *dd; 2229 struct usbd_device *hub; 2230 struct usbd_device *adev; 2231 int rhport = 0; 2232 struct xhci_slot *xs; 2233 uint32_t *cp; 2234 uint32_t route = 0; 2235 uint8_t slot = 0; 2236 uint8_t addr; 2237 2238 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 2239 DPRINTFN(4, "port=%d depth=%d speed=%d upport %d", 2240 port, depth, speed, up->up_portno); 2241 2242 dev = kmem_zalloc(sizeof(*dev), KM_SLEEP); 2243 if (dev == NULL) 2244 return USBD_NOMEM; 2245 2246 dev->ud_bus = bus; 2247 2248 /* Set up default endpoint handle. */ 2249 dev->ud_ep0.ue_edesc = &dev->ud_ep0desc; 2250 2251 /* Set up default endpoint descriptor. */ 2252 dev->ud_ep0desc.bLength = USB_ENDPOINT_DESCRIPTOR_SIZE; 2253 dev->ud_ep0desc.bDescriptorType = UDESC_ENDPOINT; 2254 dev->ud_ep0desc.bEndpointAddress = USB_CONTROL_ENDPOINT; 2255 dev->ud_ep0desc.bmAttributes = UE_CONTROL; 2256 /* 4.3, 4.8.2.1 */ 2257 switch (speed) { 2258 case USB_SPEED_SUPER: 2259 case USB_SPEED_SUPER_PLUS: 2260 USETW(dev->ud_ep0desc.wMaxPacketSize, USB_3_MAX_CTRL_PACKET); 2261 break; 2262 case USB_SPEED_FULL: 2263 /* XXX using 64 as initial mps of ep0 in FS */ 2264 case USB_SPEED_HIGH: 2265 USETW(dev->ud_ep0desc.wMaxPacketSize, USB_2_MAX_CTRL_PACKET); 2266 break; 2267 case USB_SPEED_LOW: 2268 default: 2269 USETW(dev->ud_ep0desc.wMaxPacketSize, USB_MAX_IPACKET); 2270 break; 2271 } 2272 dev->ud_ep0desc.bInterval = 0; 2273 2274 /* doesn't matter, just don't let it uninitialized */ 2275 dev->ud_ep0.ue_toggle = 0; 2276 2277 DPRINTFN(4, "up %p portno %d", up, up->up_portno, 0, 0); 2278 2279 dev->ud_quirks = &usbd_no_quirk; 2280 dev->ud_addr = 0; 2281 dev->ud_ddesc.bMaxPacketSize = 0; 2282 dev->ud_depth = depth; 2283 dev->ud_powersrc = up; 2284 dev->ud_myhub = up->up_parent; 2285 2286 up->up_dev = dev; 2287 2288 /* Locate root hub port */ 2289 for (hub = dev; hub != NULL; hub = hub->ud_myhub) { 2290 uint32_t dep; 2291 2292 DPRINTFN(4, "hub %p depth %d upport %p upportno %d", 2293 hub, hub->ud_depth, hub->ud_powersrc, 2294 hub->ud_powersrc ? hub->ud_powersrc->up_portno : -1); 2295 2296 if (hub->ud_powersrc == NULL) 2297 break; 2298 dep = hub->ud_depth; 2299 if (dep == 0) 2300 break; 2301 rhport = hub->ud_powersrc->up_portno; 2302 if (dep > USB_HUB_MAX_DEPTH) 2303 continue; 2304 2305 route |= 2306 (rhport > UHD_SS_NPORTS_MAX ? UHD_SS_NPORTS_MAX : rhport) 2307 << ((dep - 1) * 4); 2308 } 2309 route = route >> 4; 2310 DPRINTFN(4, "rhport %d Route %05x hub %p", rhport, route, hub, 0); 2311 2312 /* Locate port on upstream high speed hub */ 2313 for (adev = dev, hub = up->up_parent; 2314 hub != NULL && hub->ud_speed != USB_SPEED_HIGH; 2315 adev = hub, hub = hub->ud_myhub) 2316 ; 2317 if (hub) { 2318 int p; 2319 for (p = 0; p < hub->ud_hub->uh_hubdesc.bNbrPorts; p++) { 2320 if (hub->ud_hub->uh_ports[p].up_dev == adev) { 2321 dev->ud_myhsport = &hub->ud_hub->uh_ports[p]; 2322 goto found; 2323 } 2324 } 2325 panic("xhci_new_device: cannot find HS port"); 2326 found: 2327 DPRINTFN(4, "high speed port %d", p, 0, 0, 0); 2328 } else { 2329 dev->ud_myhsport = NULL; 2330 } 2331 2332 dev->ud_speed = speed; 2333 dev->ud_langid = USBD_NOLANG; 2334 dev->ud_cookie.cookie = ++usb_cookie_no; 2335 2336 /* Establish the default pipe. */ 2337 err = usbd_setup_pipe(dev, 0, &dev->ud_ep0, USBD_DEFAULT_INTERVAL, 2338 &dev->ud_pipe0); 2339 if (err) { 2340 goto bad; 2341 } 2342 2343 dd = &dev->ud_ddesc; 2344 2345 if ((depth == 0) && (port == 0)) { 2346 KASSERT(bus->ub_devices[dev->ud_addr] == NULL); 2347 bus->ub_devices[dev->ud_addr] = dev; 2348 err = usbd_get_initial_ddesc(dev, dd); 2349 if (err) 2350 goto bad; 2351 err = usbd_reload_device_desc(dev); 2352 if (err) 2353 goto bad; 2354 } else { 2355 err = xhci_enable_slot(sc, &slot); 2356 if (err) 2357 goto bad; 2358 xs = &sc->sc_slots[slot]; 2359 dev->ud_hcpriv = xs; 2360 err = xhci_init_slot(dev, slot, route, rhport); 2361 if (err) { 2362 dev->ud_hcpriv = NULL; 2363 /* 2364 * We have to disable_slot here because 2365 * xs->xs_idx == 0 when xhci_init_slot fails, 2366 * in that case usbd_remove_dev won't work. 2367 */ 2368 mutex_enter(&sc->sc_lock); 2369 xhci_disable_slot(sc, slot); 2370 mutex_exit(&sc->sc_lock); 2371 goto bad; 2372 } 2373 2374 /* Allow device time to set new address */ 2375 usbd_delay_ms(dev, USB_SET_ADDRESS_SETTLE); 2376 cp = xhci_slot_get_dcv(sc, xs, XHCI_DCI_SLOT); 2377 //hexdump("slot context", cp, sc->sc_ctxsz); 2378 addr = XHCI_SCTX_3_DEV_ADDR_GET(cp[3]); 2379 DPRINTFN(4, "device address %u", addr, 0, 0, 0); 2380 /* XXX ensure we know when the hardware does something 2381 we can't yet cope with */ 2382 KASSERT(addr >= 1 && addr <= 127); 2383 dev->ud_addr = addr; 2384 /* XXX dev->ud_addr not necessarily unique on bus */ 2385 KASSERT(bus->ub_devices[dev->ud_addr] == NULL); 2386 bus->ub_devices[dev->ud_addr] = dev; 2387 2388 err = usbd_get_initial_ddesc(dev, dd); 2389 if (err) 2390 goto bad; 2391 /* 4.8.2.1 */ 2392 if (USB_IS_SS(speed)) { 2393 if (dd->bMaxPacketSize != 9) { 2394 printf("%s: invalid mps 2^%u for SS ep0," 2395 " using 512\n", 2396 device_xname(sc->sc_dev), 2397 dd->bMaxPacketSize); 2398 dd->bMaxPacketSize = 9; 2399 } 2400 USETW(dev->ud_ep0desc.wMaxPacketSize, 2401 (1 << dd->bMaxPacketSize)); 2402 } else 2403 USETW(dev->ud_ep0desc.wMaxPacketSize, 2404 dd->bMaxPacketSize); 2405 DPRINTFN(4, "bMaxPacketSize %u", dd->bMaxPacketSize, 0, 0, 0); 2406 xhci_update_ep0_mps(sc, xs, 2407 UGETW(dev->ud_ep0desc.wMaxPacketSize)); 2408 err = usbd_reload_device_desc(dev); 2409 if (err) 2410 goto bad; 2411 } 2412 2413 DPRINTFN(1, "adding unit addr=%d, rev=%02x,", 2414 dev->ud_addr, UGETW(dd->bcdUSB), 0, 0); 2415 DPRINTFN(1, " class=%d, subclass=%d, protocol=%d,", 2416 dd->bDeviceClass, dd->bDeviceSubClass, 2417 dd->bDeviceProtocol, 0); 2418 DPRINTFN(1, " mps=%d, len=%d, noconf=%d, speed=%d", 2419 dd->bMaxPacketSize, dd->bLength, dd->bNumConfigurations, 2420 dev->ud_speed); 2421 2422 usbd_get_device_strings(dev); 2423 2424 usbd_add_dev_event(USB_EVENT_DEVICE_ATTACH, dev); 2425 2426 if ((depth == 0) && (port == 0)) { 2427 usbd_attach_roothub(parent, dev); 2428 DPRINTFN(1, "root_hub %p", bus->ub_roothub, 0, 0, 0); 2429 return USBD_NORMAL_COMPLETION; 2430 } 2431 2432 2433 err = usbd_probe_and_attach(parent, dev, port, dev->ud_addr); 2434 bad: 2435 if (err != USBD_NORMAL_COMPLETION) { 2436 usbd_remove_device(dev, up); 2437 } 2438 2439 return err; 2440 } 2441 2442 static usbd_status 2443 xhci_ring_init(struct xhci_softc * const sc, struct xhci_ring * const xr, 2444 size_t ntrb, size_t align) 2445 { 2446 usbd_status err; 2447 size_t size = ntrb * XHCI_TRB_SIZE; 2448 2449 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 2450 2451 err = usb_allocmem(&sc->sc_bus, size, align, &xr->xr_dma); 2452 if (err) 2453 return err; 2454 mutex_init(&xr->xr_lock, MUTEX_DEFAULT, IPL_SOFTUSB); 2455 xr->xr_cookies = kmem_zalloc(sizeof(*xr->xr_cookies) * ntrb, KM_SLEEP); 2456 xr->xr_trb = xhci_ring_trbv(xr, 0); 2457 xr->xr_ntrb = ntrb; 2458 xr->xr_ep = 0; 2459 xr->xr_cs = 1; 2460 memset(xr->xr_trb, 0, size); 2461 usb_syncmem(&xr->xr_dma, 0, size, BUS_DMASYNC_PREWRITE); 2462 xr->is_halted = false; 2463 2464 return USBD_NORMAL_COMPLETION; 2465 } 2466 2467 static void 2468 xhci_ring_free(struct xhci_softc * const sc, struct xhci_ring * const xr) 2469 { 2470 usb_freemem(&sc->sc_bus, &xr->xr_dma); 2471 mutex_destroy(&xr->xr_lock); 2472 kmem_free(xr->xr_cookies, sizeof(*xr->xr_cookies) * xr->xr_ntrb); 2473 } 2474 2475 static void 2476 xhci_ring_put(struct xhci_softc * const sc, struct xhci_ring * const xr, 2477 void *cookie, struct xhci_trb * const trbs, size_t ntrbs) 2478 { 2479 size_t i; 2480 u_int ri; 2481 u_int cs; 2482 uint64_t parameter; 2483 uint32_t status; 2484 uint32_t control; 2485 2486 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 2487 2488 KASSERT(ntrbs <= XHCI_XFER_NTRB); 2489 for (i = 0; i < ntrbs; i++) { 2490 DPRINTFN(12, "xr %p trbs %p num %zu", xr, trbs, i, 0); 2491 DPRINTFN(12, " %016"PRIx64" %08"PRIx32" %08"PRIx32, 2492 trbs[i].trb_0, trbs[i].trb_2, trbs[i].trb_3, 0); 2493 KASSERT(XHCI_TRB_3_TYPE_GET(trbs[i].trb_3) != 2494 XHCI_TRB_TYPE_LINK); 2495 } 2496 2497 DPRINTFN(12, "%p xr_ep 0x%x xr_cs %u", xr, xr->xr_ep, xr->xr_cs, 0); 2498 2499 ri = xr->xr_ep; 2500 cs = xr->xr_cs; 2501 2502 /* 2503 * Although the xhci hardware can do scatter/gather dma from 2504 * arbitrary sized buffers, there is a non-obvious restriction 2505 * that a LINK trb is only allowed at the end of a burst of 2506 * transfers - which might be 16kB. 2507 * Arbitrary aligned LINK trb definitely fail on Ivy bridge. 2508 * The simple solution is not to allow a LINK trb in the middle 2509 * of anything - as here. 2510 * XXX: (dsl) There are xhci controllers out there (eg some made by 2511 * ASMedia) that seem to lock up if they process a LINK trb but 2512 * cannot process the linked-to trb yet. 2513 * The code should write the 'cycle' bit on the link trb AFTER 2514 * adding the other trb. 2515 */ 2516 if (ri + ntrbs >= (xr->xr_ntrb - 1)) { 2517 parameter = xhci_ring_trbp(xr, 0); 2518 status = 0; 2519 control = XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_LINK) | 2520 XHCI_TRB_3_TC_BIT | (cs ? XHCI_TRB_3_CYCLE_BIT : 0); 2521 xhci_trb_put(&xr->xr_trb[ri], parameter, status, control); 2522 usb_syncmem(&xr->xr_dma, XHCI_TRB_SIZE * ri, XHCI_TRB_SIZE * 1, 2523 BUS_DMASYNC_PREWRITE); 2524 xr->xr_cookies[ri] = NULL; 2525 xr->xr_ep = 0; 2526 xr->xr_cs ^= 1; 2527 ri = xr->xr_ep; 2528 cs = xr->xr_cs; 2529 } 2530 2531 ri++; 2532 2533 /* Write any subsequent TRB first */ 2534 for (i = 1; i < ntrbs; i++) { 2535 parameter = trbs[i].trb_0; 2536 status = trbs[i].trb_2; 2537 control = trbs[i].trb_3; 2538 2539 if (cs) { 2540 control |= XHCI_TRB_3_CYCLE_BIT; 2541 } else { 2542 control &= ~XHCI_TRB_3_CYCLE_BIT; 2543 } 2544 2545 xhci_trb_put(&xr->xr_trb[ri], parameter, status, control); 2546 usb_syncmem(&xr->xr_dma, XHCI_TRB_SIZE * ri, XHCI_TRB_SIZE * 1, 2547 BUS_DMASYNC_PREWRITE); 2548 xr->xr_cookies[ri] = cookie; 2549 ri++; 2550 } 2551 2552 /* Write the first TRB last */ 2553 i = 0; 2554 parameter = trbs[i].trb_0; 2555 status = trbs[i].trb_2; 2556 control = trbs[i].trb_3; 2557 2558 if (xr->xr_cs) { 2559 control |= XHCI_TRB_3_CYCLE_BIT; 2560 } else { 2561 control &= ~XHCI_TRB_3_CYCLE_BIT; 2562 } 2563 2564 xhci_trb_put(&xr->xr_trb[xr->xr_ep], parameter, status, control); 2565 usb_syncmem(&xr->xr_dma, XHCI_TRB_SIZE * ri, XHCI_TRB_SIZE * 1, 2566 BUS_DMASYNC_PREWRITE); 2567 xr->xr_cookies[xr->xr_ep] = cookie; 2568 2569 xr->xr_ep = ri; 2570 xr->xr_cs = cs; 2571 2572 DPRINTFN(12, "%p xr_ep 0x%x xr_cs %u", xr, xr->xr_ep, xr->xr_cs, 0); 2573 } 2574 2575 /* 2576 * Stop execution commands, purge all commands on command ring, and 2577 * rewind enqueue pointer. 2578 */ 2579 static void 2580 xhci_abort_command(struct xhci_softc *sc) 2581 { 2582 struct xhci_ring * const cr = &sc->sc_cr; 2583 uint64_t crcr; 2584 int i; 2585 2586 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 2587 DPRINTFN(14, "command %#"PRIx64" timeout, aborting", 2588 sc->sc_command_addr, 0, 0, 0); 2589 2590 mutex_enter(&cr->xr_lock); 2591 2592 /* 4.6.1.2 Aborting a Command */ 2593 crcr = xhci_op_read_8(sc, XHCI_CRCR); 2594 xhci_op_write_8(sc, XHCI_CRCR, crcr | XHCI_CRCR_LO_CA); 2595 2596 for (i = 0; i < 500; i++) { 2597 crcr = xhci_op_read_8(sc, XHCI_CRCR); 2598 if ((crcr & XHCI_CRCR_LO_CRR) == 0) 2599 break; 2600 usb_delay_ms(&sc->sc_bus, 1); 2601 } 2602 if ((crcr & XHCI_CRCR_LO_CRR) != 0) { 2603 DPRINTFN(1, "Command Abort timeout", 0, 0, 0, 0); 2604 /* reset HC here? */ 2605 } 2606 2607 /* reset command ring dequeue pointer */ 2608 cr->xr_ep = 0; 2609 cr->xr_cs = 1; 2610 xhci_op_write_8(sc, XHCI_CRCR, xhci_ring_trbp(cr, 0) | cr->xr_cs); 2611 2612 mutex_exit(&cr->xr_lock); 2613 } 2614 2615 /* 2616 * Put a command on command ring, ring bell, set timer, and cv_timedwait. 2617 * Command completion is notified by cv_signal from xhci_handle_event 2618 * (called from interrupt from xHCI), or timed-out. 2619 * Command validation is performed in xhci_handle_event by checking if 2620 * trb_0 in CMD_COMPLETE TRB and sc->sc_command_addr are identical. 2621 */ 2622 static usbd_status 2623 xhci_do_command_locked(struct xhci_softc * const sc, struct xhci_trb * const trb, 2624 int timeout) 2625 { 2626 struct xhci_ring * const cr = &sc->sc_cr; 2627 usbd_status err; 2628 2629 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 2630 DPRINTFN(12, "input: 0x%016"PRIx64" 0x%08"PRIx32" 0x%08"PRIx32, 2631 trb->trb_0, trb->trb_2, trb->trb_3, 0); 2632 2633 KASSERTMSG(!cpu_intr_p() && !cpu_softintr_p(), "called from intr ctx"); 2634 KASSERT(mutex_owned(&sc->sc_lock)); 2635 2636 /* XXX KASSERT may fire when cv_timedwait unlocks sc_lock */ 2637 KASSERT(sc->sc_command_addr == 0); 2638 sc->sc_command_addr = xhci_ring_trbp(cr, cr->xr_ep); 2639 2640 mutex_enter(&cr->xr_lock); 2641 xhci_ring_put(sc, cr, NULL, trb, 1); 2642 mutex_exit(&cr->xr_lock); 2643 2644 xhci_db_write_4(sc, XHCI_DOORBELL(0), 0); 2645 2646 if (cv_timedwait(&sc->sc_command_cv, &sc->sc_lock, 2647 MAX(1, mstohz(timeout))) == EWOULDBLOCK) { 2648 xhci_abort_command(sc); 2649 err = USBD_TIMEOUT; 2650 goto timedout; 2651 } 2652 2653 trb->trb_0 = sc->sc_result_trb.trb_0; 2654 trb->trb_2 = sc->sc_result_trb.trb_2; 2655 trb->trb_3 = sc->sc_result_trb.trb_3; 2656 2657 DPRINTFN(12, "output: 0x%016"PRIx64" 0x%08"PRIx32" 0x%08"PRIx32"", 2658 trb->trb_0, trb->trb_2, trb->trb_3, 0); 2659 2660 switch (XHCI_TRB_2_ERROR_GET(trb->trb_2)) { 2661 case XHCI_TRB_ERROR_SUCCESS: 2662 err = USBD_NORMAL_COMPLETION; 2663 break; 2664 default: 2665 case 192 ... 223: 2666 err = USBD_IOERROR; 2667 break; 2668 case 224 ... 255: 2669 err = USBD_NORMAL_COMPLETION; 2670 break; 2671 } 2672 2673 timedout: 2674 sc->sc_command_addr = 0; 2675 return err; 2676 } 2677 2678 static usbd_status 2679 xhci_do_command(struct xhci_softc * const sc, struct xhci_trb * const trb, 2680 int timeout) 2681 { 2682 2683 mutex_enter(&sc->sc_lock); 2684 usbd_status ret = xhci_do_command_locked(sc, trb, timeout); 2685 mutex_exit(&sc->sc_lock); 2686 2687 return ret; 2688 } 2689 2690 static usbd_status 2691 xhci_enable_slot(struct xhci_softc * const sc, uint8_t * const slotp) 2692 { 2693 struct xhci_trb trb; 2694 usbd_status err; 2695 2696 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 2697 2698 trb.trb_0 = 0; 2699 trb.trb_2 = 0; 2700 trb.trb_3 = XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_ENABLE_SLOT); 2701 2702 err = xhci_do_command(sc, &trb, USBD_DEFAULT_TIMEOUT); 2703 if (err != USBD_NORMAL_COMPLETION) { 2704 return err; 2705 } 2706 2707 *slotp = XHCI_TRB_3_SLOT_GET(trb.trb_3); 2708 2709 return err; 2710 } 2711 2712 /* 2713 * xHCI 4.6.4 2714 * Deallocate ring and device/input context DMA buffers, and disable_slot. 2715 * All endpoints in the slot should be stopped. 2716 * Should be called with sc_lock held. 2717 */ 2718 static usbd_status 2719 xhci_disable_slot(struct xhci_softc * const sc, uint8_t slot) 2720 { 2721 struct xhci_trb trb; 2722 struct xhci_slot *xs; 2723 usbd_status err; 2724 2725 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 2726 2727 if (sc->sc_dying) 2728 return USBD_IOERROR; 2729 2730 trb.trb_0 = 0; 2731 trb.trb_2 = 0; 2732 trb.trb_3 = htole32( 2733 XHCI_TRB_3_SLOT_SET(slot) | 2734 XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_DISABLE_SLOT)); 2735 2736 err = xhci_do_command_locked(sc, &trb, USBD_DEFAULT_TIMEOUT); 2737 2738 if (!err) { 2739 xs = &sc->sc_slots[slot]; 2740 if (xs->xs_idx != 0) { 2741 for (int i = XHCI_DCI_SLOT + 1; i < 32; i++) { 2742 xhci_ring_free(sc, &xs->xs_ep[i].xe_tr); 2743 memset(&xs->xs_ep[i], 0, sizeof(xs->xs_ep[i])); 2744 } 2745 usb_freemem(&sc->sc_bus, &xs->xs_ic_dma); 2746 usb_freemem(&sc->sc_bus, &xs->xs_dc_dma); 2747 xhci_set_dcba(sc, 0, slot); 2748 memset(xs, 0, sizeof(*xs)); 2749 } 2750 } 2751 2752 return err; 2753 } 2754 2755 /* 2756 * Set address of device and transition slot state from ENABLED to ADDRESSED 2757 * if Block Setaddress Request (BSR) is false. 2758 * If BSR==true, transition slot state from ENABLED to DEFAULT. 2759 * see xHCI 1.1 4.5.3, 3.3.4 2760 * Should be called without sc_lock held. 2761 */ 2762 static usbd_status 2763 xhci_address_device(struct xhci_softc * const sc, 2764 uint64_t icp, uint8_t slot_id, bool bsr) 2765 { 2766 struct xhci_trb trb; 2767 usbd_status err; 2768 2769 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 2770 2771 trb.trb_0 = icp; 2772 trb.trb_2 = 0; 2773 trb.trb_3 = XHCI_TRB_3_SLOT_SET(slot_id) | 2774 XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_ADDRESS_DEVICE) | 2775 (bsr ? XHCI_TRB_3_BSR_BIT : 0); 2776 2777 err = xhci_do_command(sc, &trb, USBD_DEFAULT_TIMEOUT); 2778 2779 if (XHCI_TRB_2_ERROR_GET(trb.trb_2) == XHCI_TRB_ERROR_NO_SLOTS) 2780 err = USBD_NO_ADDR; 2781 2782 return err; 2783 } 2784 2785 static usbd_status 2786 xhci_update_ep0_mps(struct xhci_softc * const sc, 2787 struct xhci_slot * const xs, u_int mps) 2788 { 2789 struct xhci_trb trb; 2790 usbd_status err; 2791 uint32_t * cp; 2792 2793 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 2794 DPRINTFN(4, "slot %u mps %u", xs->xs_idx, mps, 0, 0); 2795 2796 cp = xhci_slot_get_icv(sc, xs, XHCI_ICI_INPUT_CONTROL); 2797 cp[0] = htole32(0); 2798 cp[1] = htole32(XHCI_INCTX_1_ADD_MASK(XHCI_DCI_EP_CONTROL)); 2799 2800 cp = xhci_slot_get_icv(sc, xs, xhci_dci_to_ici(XHCI_DCI_EP_CONTROL)); 2801 cp[1] = htole32(XHCI_EPCTX_1_MAXP_SIZE_SET(mps)); 2802 2803 /* sync input contexts before they are read from memory */ 2804 usb_syncmem(&xs->xs_ic_dma, 0, sc->sc_pgsz, BUS_DMASYNC_PREWRITE); 2805 hexdump("input context", xhci_slot_get_icv(sc, xs, 0), 2806 sc->sc_ctxsz * 4); 2807 2808 trb.trb_0 = xhci_slot_get_icp(sc, xs, 0); 2809 trb.trb_2 = 0; 2810 trb.trb_3 = XHCI_TRB_3_SLOT_SET(xs->xs_idx) | 2811 XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_EVALUATE_CTX); 2812 2813 err = xhci_do_command(sc, &trb, USBD_DEFAULT_TIMEOUT); 2814 KASSERT(err == USBD_NORMAL_COMPLETION); /* XXX */ 2815 return err; 2816 } 2817 2818 static void 2819 xhci_set_dcba(struct xhci_softc * const sc, uint64_t dcba, int si) 2820 { 2821 uint64_t * const dcbaa = KERNADDR(&sc->sc_dcbaa_dma, 0); 2822 2823 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 2824 DPRINTFN(4, "dcbaa %p dc %016"PRIx64" slot %d", 2825 &dcbaa[si], dcba, si, 0); 2826 2827 dcbaa[si] = htole64(dcba); 2828 usb_syncmem(&sc->sc_dcbaa_dma, si * sizeof(uint64_t), sizeof(uint64_t), 2829 BUS_DMASYNC_PREWRITE); 2830 } 2831 2832 /* 2833 * Allocate DMA buffer and ring buffer for specified slot 2834 * and set Device Context Base Address 2835 * and issue Set Address device command. 2836 */ 2837 static usbd_status 2838 xhci_init_slot(struct usbd_device *dev, uint32_t slot, uint32_t route, int rhport) 2839 { 2840 struct xhci_softc * const sc = XHCI_BUS2SC(dev->ud_bus); 2841 struct xhci_slot *xs; 2842 usbd_status err; 2843 u_int dci; 2844 uint32_t *cp; 2845 uint32_t mps = UGETW(dev->ud_ep0desc.wMaxPacketSize); 2846 2847 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 2848 DPRINTFN(4, "slot %u speed %d route %05x rhport %d", 2849 slot, dev->ud_speed, route, rhport); 2850 2851 xs = &sc->sc_slots[slot]; 2852 2853 /* allocate contexts */ 2854 err = usb_allocmem(&sc->sc_bus, sc->sc_pgsz, sc->sc_pgsz, 2855 &xs->xs_dc_dma); 2856 if (err) 2857 return err; 2858 memset(KERNADDR(&xs->xs_dc_dma, 0), 0, sc->sc_pgsz); 2859 2860 err = usb_allocmem(&sc->sc_bus, sc->sc_pgsz, sc->sc_pgsz, 2861 &xs->xs_ic_dma); 2862 if (err) 2863 goto bad1; 2864 memset(KERNADDR(&xs->xs_ic_dma, 0), 0, sc->sc_pgsz); 2865 2866 for (dci = 0; dci < 32; dci++) { 2867 //CTASSERT(sizeof(xs->xs_ep[dci]) == sizeof(struct xhci_endpoint)); 2868 memset(&xs->xs_ep[dci], 0, sizeof(xs->xs_ep[dci])); 2869 if (dci == XHCI_DCI_SLOT) 2870 continue; 2871 err = xhci_ring_init(sc, &xs->xs_ep[dci].xe_tr, 2872 XHCI_TRANSFER_RING_TRBS, XHCI_TRB_ALIGN); 2873 if (err) { 2874 DPRINTFN(0, "ring init failure", 0, 0, 0, 0); 2875 goto bad2; 2876 } 2877 } 2878 2879 /* set up initial input control context */ 2880 cp = xhci_slot_get_icv(sc, xs, XHCI_ICI_INPUT_CONTROL); 2881 cp[0] = htole32(0); 2882 cp[1] = htole32(XHCI_INCTX_1_ADD_MASK(XHCI_DCI_EP_CONTROL)| 2883 XHCI_INCTX_1_ADD_MASK(XHCI_DCI_SLOT)); 2884 2885 /* set up input slot context */ 2886 cp = xhci_slot_get_icv(sc, xs, xhci_dci_to_ici(XHCI_DCI_SLOT)); 2887 xhci_setup_sctx(dev, cp); 2888 cp[0] |= htole32(XHCI_SCTX_0_CTX_NUM_SET(1)); 2889 cp[0] |= htole32(XHCI_SCTX_0_ROUTE_SET(route)); 2890 cp[1] |= htole32(XHCI_SCTX_1_RH_PORT_SET(rhport)); 2891 2892 /* set up input EP0 context */ 2893 cp = xhci_slot_get_icv(sc, xs, xhci_dci_to_ici(XHCI_DCI_EP_CONTROL)); 2894 cp[0] = htole32(0); 2895 cp[1] = htole32( 2896 XHCI_EPCTX_1_MAXP_SIZE_SET(mps) | 2897 XHCI_EPCTX_1_EPTYPE_SET(4) | 2898 XHCI_EPCTX_1_CERR_SET(3) 2899 ); 2900 /* can't use xhci_ep_get_dci() yet? */ 2901 *(uint64_t *)(&cp[2]) = htole64( 2902 xhci_ring_trbp(&xs->xs_ep[XHCI_DCI_EP_CONTROL].xe_tr, 0) | 2903 XHCI_EPCTX_2_DCS_SET(1)); 2904 cp[4] = htole32( 2905 XHCI_EPCTX_4_AVG_TRB_LEN_SET(8) 2906 ); 2907 2908 /* sync input contexts before they are read from memory */ 2909 usb_syncmem(&xs->xs_ic_dma, 0, sc->sc_pgsz, BUS_DMASYNC_PREWRITE); 2910 hexdump("input context", xhci_slot_get_icv(sc, xs, 0), 2911 sc->sc_ctxsz * 3); 2912 2913 xhci_set_dcba(sc, DMAADDR(&xs->xs_dc_dma, 0), slot); 2914 2915 err = xhci_address_device(sc, xhci_slot_get_icp(sc, xs, 0), slot, 2916 false); 2917 2918 usb_syncmem(&xs->xs_dc_dma, 0, sc->sc_pgsz, BUS_DMASYNC_POSTREAD); 2919 hexdump("output context", xhci_slot_get_dcv(sc, xs, 0), 2920 sc->sc_ctxsz * 2); 2921 2922 bad2: 2923 if (err == USBD_NORMAL_COMPLETION) { 2924 xs->xs_idx = slot; 2925 } else { 2926 for (int i = 1; i < dci; i++) { 2927 xhci_ring_free(sc, &xs->xs_ep[i].xe_tr); 2928 memset(&xs->xs_ep[i], 0, sizeof(xs->xs_ep[i])); 2929 } 2930 usb_freemem(&sc->sc_bus, &xs->xs_ic_dma); 2931 bad1: 2932 usb_freemem(&sc->sc_bus, &xs->xs_dc_dma); 2933 xs->xs_idx = 0; 2934 } 2935 2936 return err; 2937 } 2938 2939 /* ----- */ 2940 2941 static void 2942 xhci_noop(struct usbd_pipe *pipe) 2943 { 2944 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 2945 } 2946 2947 /* 2948 * Process root hub request. 2949 */ 2950 static int 2951 xhci_roothub_ctrl(struct usbd_bus *bus, usb_device_request_t *req, 2952 void *buf, int buflen) 2953 { 2954 struct xhci_softc * const sc = XHCI_BUS2SC(bus); 2955 usb_port_status_t ps; 2956 int l, totlen = 0; 2957 uint16_t len, value, index; 2958 int port, i; 2959 uint32_t v; 2960 2961 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 2962 2963 if (sc->sc_dying) 2964 return -1; 2965 2966 len = UGETW(req->wLength); 2967 value = UGETW(req->wValue); 2968 index = UGETW(req->wIndex); 2969 2970 DPRINTFN(12, "rhreq: %04x %04x %04x %04x", 2971 req->bmRequestType | (req->bRequest << 8), value, index, len); 2972 2973 #define C(x,y) ((x) | ((y) << 8)) 2974 switch (C(req->bRequest, req->bmRequestType)) { 2975 case C(UR_GET_DESCRIPTOR, UT_READ_DEVICE): 2976 DPRINTFN(8, "getdesc: wValue=0x%04x", value, 0, 0, 0); 2977 if (len == 0) 2978 break; 2979 switch (value) { 2980 case C(0, UDESC_DEVICE): { 2981 usb_device_descriptor_t devd; 2982 totlen = min(buflen, sizeof(devd)); 2983 memcpy(&devd, buf, totlen); 2984 USETW(devd.idVendor, sc->sc_id_vendor); 2985 memcpy(buf, &devd, totlen); 2986 break; 2987 } 2988 #define sd ((usb_string_descriptor_t *)buf) 2989 case C(1, UDESC_STRING): 2990 /* Vendor */ 2991 totlen = usb_makestrdesc(sd, len, sc->sc_vendor); 2992 break; 2993 case C(2, UDESC_STRING): 2994 /* Product */ 2995 totlen = usb_makestrdesc(sd, len, "xHCI Root Hub"); 2996 break; 2997 #undef sd 2998 default: 2999 /* default from usbroothub */ 3000 return buflen; 3001 } 3002 break; 3003 3004 /* Hub requests */ 3005 case C(UR_CLEAR_FEATURE, UT_WRITE_CLASS_DEVICE): 3006 break; 3007 /* Clear Port Feature request */ 3008 case C(UR_CLEAR_FEATURE, UT_WRITE_CLASS_OTHER): 3009 DPRINTFN(4, "UR_CLEAR_PORT_FEATURE port=%d feature=%d", 3010 index, value, 0, 0); 3011 if (index < 1 || index > sc->sc_maxports) { 3012 return -1; 3013 } 3014 port = XHCI_PORTSC(index); 3015 v = xhci_op_read_4(sc, port); 3016 DPRINTFN(4, "portsc=0x%08x", v, 0, 0, 0); 3017 v &= ~XHCI_PS_CLEAR; 3018 switch (value) { 3019 case UHF_PORT_ENABLE: 3020 xhci_op_write_4(sc, port, v & ~XHCI_PS_PED); 3021 break; 3022 case UHF_PORT_SUSPEND: 3023 return -1; 3024 case UHF_PORT_POWER: 3025 break; 3026 case UHF_PORT_TEST: 3027 case UHF_PORT_INDICATOR: 3028 return -1; 3029 case UHF_C_PORT_CONNECTION: 3030 xhci_op_write_4(sc, port, v | XHCI_PS_CSC); 3031 break; 3032 case UHF_C_PORT_ENABLE: 3033 case UHF_C_PORT_SUSPEND: 3034 case UHF_C_PORT_OVER_CURRENT: 3035 return -1; 3036 case UHF_C_BH_PORT_RESET: 3037 xhci_op_write_4(sc, port, v | XHCI_PS_WRC); 3038 break; 3039 case UHF_C_PORT_RESET: 3040 xhci_op_write_4(sc, port, v | XHCI_PS_PRC); 3041 break; 3042 case UHF_C_PORT_LINK_STATE: 3043 xhci_op_write_4(sc, port, v | XHCI_PS_PLC); 3044 break; 3045 case UHF_C_PORT_CONFIG_ERROR: 3046 xhci_op_write_4(sc, port, v | XHCI_PS_CEC); 3047 break; 3048 default: 3049 return -1; 3050 } 3051 break; 3052 case C(UR_GET_DESCRIPTOR, UT_READ_CLASS_DEVICE): 3053 if (len == 0) 3054 break; 3055 if ((value & 0xff) != 0) { 3056 return -1; 3057 } 3058 usb_hub_descriptor_t hubd; 3059 3060 totlen = min(buflen, sizeof(hubd)); 3061 memcpy(&hubd, buf, totlen); 3062 hubd.bNbrPorts = sc->sc_maxports; 3063 USETW(hubd.wHubCharacteristics, UHD_PWR_NO_SWITCH); 3064 hubd.bPwrOn2PwrGood = 200; 3065 for (i = 0, l = sc->sc_maxports; l > 0; i++, l -= 8) 3066 hubd.DeviceRemovable[i++] = 0; /* XXX can't find out? */ 3067 hubd.bDescLength = USB_HUB_DESCRIPTOR_SIZE + i; 3068 totlen = min(totlen, hubd.bDescLength); 3069 memcpy(buf, &hubd, totlen); 3070 break; 3071 case C(UR_GET_STATUS, UT_READ_CLASS_DEVICE): 3072 if (len != 4) { 3073 return -1; 3074 } 3075 memset(buf, 0, len); /* ? XXX */ 3076 totlen = len; 3077 break; 3078 /* Get Port Status request */ 3079 case C(UR_GET_STATUS, UT_READ_CLASS_OTHER): 3080 DPRINTFN(8, "get port status i=%d", index, 0, 0, 0); 3081 if (index < 1 || index > sc->sc_maxports) { 3082 return -1; 3083 } 3084 if (len != 4) { 3085 return -1; 3086 } 3087 v = xhci_op_read_4(sc, XHCI_PORTSC(index)); 3088 DPRINTFN(4, "getrhportsc %d %08x", index, v, 0, 0); 3089 i = xhci_xspeed2psspeed(XHCI_PS_SPEED_GET(v)); 3090 if (v & XHCI_PS_CCS) i |= UPS_CURRENT_CONNECT_STATUS; 3091 if (v & XHCI_PS_PED) i |= UPS_PORT_ENABLED; 3092 if (v & XHCI_PS_OCA) i |= UPS_OVERCURRENT_INDICATOR; 3093 //if (v & XHCI_PS_SUSP) i |= UPS_SUSPEND; 3094 if (v & XHCI_PS_PR) i |= UPS_RESET; 3095 if (v & XHCI_PS_PP) { 3096 if (i & UPS_OTHER_SPEED) 3097 i |= UPS_PORT_POWER_SS; 3098 else 3099 i |= UPS_PORT_POWER; 3100 } 3101 if (i & UPS_OTHER_SPEED) 3102 i |= UPS_PORT_LS_SET(XHCI_PS_PLS_GET(v)); 3103 if (sc->sc_vendor_port_status) 3104 i = sc->sc_vendor_port_status(sc, v, i); 3105 USETW(ps.wPortStatus, i); 3106 i = 0; 3107 if (v & XHCI_PS_CSC) i |= UPS_C_CONNECT_STATUS; 3108 if (v & XHCI_PS_PEC) i |= UPS_C_PORT_ENABLED; 3109 if (v & XHCI_PS_OCC) i |= UPS_C_OVERCURRENT_INDICATOR; 3110 if (v & XHCI_PS_PRC) i |= UPS_C_PORT_RESET; 3111 if (v & XHCI_PS_WRC) i |= UPS_C_BH_PORT_RESET; 3112 if (v & XHCI_PS_PLC) i |= UPS_C_PORT_LINK_STATE; 3113 if (v & XHCI_PS_CEC) i |= UPS_C_PORT_CONFIG_ERROR; 3114 USETW(ps.wPortChange, i); 3115 totlen = min(len, sizeof(ps)); 3116 memcpy(buf, &ps, totlen); 3117 break; 3118 case C(UR_SET_DESCRIPTOR, UT_WRITE_CLASS_DEVICE): 3119 return -1; 3120 case C(UR_SET_HUB_DEPTH, UT_WRITE_CLASS_DEVICE): 3121 break; 3122 case C(UR_SET_FEATURE, UT_WRITE_CLASS_DEVICE): 3123 break; 3124 /* Set Port Feature request */ 3125 case C(UR_SET_FEATURE, UT_WRITE_CLASS_OTHER): { 3126 int optval = (index >> 8) & 0xff; 3127 index &= 0xff; 3128 if (index < 1 || index > sc->sc_maxports) { 3129 return -1; 3130 } 3131 port = XHCI_PORTSC(index); 3132 v = xhci_op_read_4(sc, port); 3133 DPRINTFN(4, "portsc=0x%08x", v, 0, 0, 0); 3134 v &= ~XHCI_PS_CLEAR; 3135 switch (value) { 3136 case UHF_PORT_ENABLE: 3137 xhci_op_write_4(sc, port, v | XHCI_PS_PED); 3138 break; 3139 case UHF_PORT_SUSPEND: 3140 /* XXX suspend */ 3141 break; 3142 case UHF_PORT_RESET: 3143 v &= ~(XHCI_PS_PED | XHCI_PS_PR); 3144 xhci_op_write_4(sc, port, v | XHCI_PS_PR); 3145 /* Wait for reset to complete. */ 3146 usb_delay_ms(&sc->sc_bus, USB_PORT_ROOT_RESET_DELAY); 3147 if (sc->sc_dying) { 3148 return -1; 3149 } 3150 v = xhci_op_read_4(sc, port); 3151 if (v & XHCI_PS_PR) { 3152 xhci_op_write_4(sc, port, v & ~XHCI_PS_PR); 3153 usb_delay_ms(&sc->sc_bus, 10); 3154 /* XXX */ 3155 } 3156 break; 3157 case UHF_PORT_POWER: 3158 /* XXX power control */ 3159 break; 3160 /* XXX more */ 3161 case UHF_C_PORT_RESET: 3162 xhci_op_write_4(sc, port, v | XHCI_PS_PRC); 3163 break; 3164 case UHF_PORT_U1_TIMEOUT: 3165 if (XHCI_PS_SPEED_GET(v) < XHCI_PS_SPEED_SS) { 3166 return -1; 3167 } 3168 port = XHCI_PORTPMSC(index); 3169 v = xhci_op_read_4(sc, port); 3170 v &= ~XHCI_PM3_U1TO_SET(0xff); 3171 v |= XHCI_PM3_U1TO_SET(optval); 3172 xhci_op_write_4(sc, port, v); 3173 break; 3174 case UHF_PORT_U2_TIMEOUT: 3175 if (XHCI_PS_SPEED_GET(v) < XHCI_PS_SPEED_SS) { 3176 return -1; 3177 } 3178 port = XHCI_PORTPMSC(index); 3179 v = xhci_op_read_4(sc, port); 3180 v &= ~XHCI_PM3_U2TO_SET(0xff); 3181 v |= XHCI_PM3_U2TO_SET(optval); 3182 xhci_op_write_4(sc, port, v); 3183 break; 3184 default: 3185 return -1; 3186 } 3187 } 3188 break; 3189 case C(UR_CLEAR_TT_BUFFER, UT_WRITE_CLASS_OTHER): 3190 case C(UR_RESET_TT, UT_WRITE_CLASS_OTHER): 3191 case C(UR_GET_TT_STATE, UT_READ_CLASS_OTHER): 3192 case C(UR_STOP_TT, UT_WRITE_CLASS_OTHER): 3193 break; 3194 default: 3195 /* default from usbroothub */ 3196 return buflen; 3197 } 3198 3199 return totlen; 3200 } 3201 3202 /* root hub interrupt */ 3203 3204 static usbd_status 3205 xhci_root_intr_transfer(struct usbd_xfer *xfer) 3206 { 3207 struct xhci_softc * const sc = XHCI_XFER2SC(xfer); 3208 usbd_status err; 3209 3210 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 3211 3212 /* Insert last in queue. */ 3213 mutex_enter(&sc->sc_lock); 3214 err = usb_insert_transfer(xfer); 3215 mutex_exit(&sc->sc_lock); 3216 if (err) 3217 return err; 3218 3219 /* Pipe isn't running, start first */ 3220 return xhci_root_intr_start(SIMPLEQ_FIRST(&xfer->ux_pipe->up_queue)); 3221 } 3222 3223 /* Wait for roothub port status/change */ 3224 static usbd_status 3225 xhci_root_intr_start(struct usbd_xfer *xfer) 3226 { 3227 struct xhci_softc * const sc = XHCI_XFER2SC(xfer); 3228 3229 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 3230 3231 if (sc->sc_dying) 3232 return USBD_IOERROR; 3233 3234 mutex_enter(&sc->sc_lock); 3235 sc->sc_intrxfer = xfer; 3236 mutex_exit(&sc->sc_lock); 3237 3238 return USBD_IN_PROGRESS; 3239 } 3240 3241 static void 3242 xhci_root_intr_abort(struct usbd_xfer *xfer) 3243 { 3244 struct xhci_softc * const sc = XHCI_XFER2SC(xfer); 3245 3246 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 3247 3248 KASSERT(mutex_owned(&sc->sc_lock)); 3249 KASSERT(xfer->ux_pipe->up_intrxfer == xfer); 3250 3251 sc->sc_intrxfer = NULL; 3252 3253 xfer->ux_status = USBD_CANCELLED; 3254 usb_transfer_complete(xfer); 3255 } 3256 3257 static void 3258 xhci_root_intr_close(struct usbd_pipe *pipe) 3259 { 3260 struct xhci_softc * const sc = XHCI_PIPE2SC(pipe); 3261 3262 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 3263 3264 KASSERT(mutex_owned(&sc->sc_lock)); 3265 3266 sc->sc_intrxfer = NULL; 3267 } 3268 3269 static void 3270 xhci_root_intr_done(struct usbd_xfer *xfer) 3271 { 3272 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 3273 3274 } 3275 3276 /* -------------- */ 3277 /* device control */ 3278 3279 static usbd_status 3280 xhci_device_ctrl_transfer(struct usbd_xfer *xfer) 3281 { 3282 struct xhci_softc * const sc = XHCI_XFER2SC(xfer); 3283 usbd_status err; 3284 3285 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 3286 3287 /* Insert last in queue. */ 3288 mutex_enter(&sc->sc_lock); 3289 err = usb_insert_transfer(xfer); 3290 mutex_exit(&sc->sc_lock); 3291 if (err) 3292 return err; 3293 3294 /* Pipe isn't running, start first */ 3295 return xhci_device_ctrl_start(SIMPLEQ_FIRST(&xfer->ux_pipe->up_queue)); 3296 } 3297 3298 static usbd_status 3299 xhci_device_ctrl_start(struct usbd_xfer *xfer) 3300 { 3301 struct xhci_softc * const sc = XHCI_XFER2SC(xfer); 3302 struct xhci_slot * const xs = xfer->ux_pipe->up_dev->ud_hcpriv; 3303 const u_int dci = xhci_ep_get_dci(xfer->ux_pipe->up_endpoint->ue_edesc); 3304 struct xhci_ring * const tr = &xs->xs_ep[dci].xe_tr; 3305 struct xhci_xfer * const xx = XHCI_XFER2XXFER(xfer); 3306 usb_device_request_t * const req = &xfer->ux_request; 3307 const int isread = usbd_xfer_isread(xfer); 3308 const uint32_t len = UGETW(req->wLength); 3309 usb_dma_t * const dma = &xfer->ux_dmabuf; 3310 uint64_t parameter; 3311 uint32_t status; 3312 uint32_t control; 3313 u_int i; 3314 3315 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 3316 DPRINTFN(12, "req: %04x %04x %04x %04x", 3317 req->bmRequestType | (req->bRequest << 8), UGETW(req->wValue), 3318 UGETW(req->wIndex), UGETW(req->wLength)); 3319 3320 /* we rely on the bottom bits for extra info */ 3321 KASSERT(((uintptr_t)xfer & 0x3) == 0x0); 3322 3323 KASSERT((xfer->ux_rqflags & URQ_REQUEST) != 0); 3324 3325 i = 0; 3326 3327 /* setup phase */ 3328 memcpy(¶meter, req, sizeof(*req)); 3329 status = XHCI_TRB_2_IRQ_SET(0) | XHCI_TRB_2_BYTES_SET(sizeof(*req)); 3330 control = ((len == 0) ? XHCI_TRB_3_TRT_NONE : 3331 (isread ? XHCI_TRB_3_TRT_IN : XHCI_TRB_3_TRT_OUT)) | 3332 XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_SETUP_STAGE) | 3333 XHCI_TRB_3_IDT_BIT; 3334 xhci_trb_put(&xx->xx_trb[i++], parameter, status, control); 3335 3336 if (len != 0) { 3337 /* data phase */ 3338 parameter = DMAADDR(dma, 0); 3339 KASSERT(len <= 0x10000); 3340 status = XHCI_TRB_2_IRQ_SET(0) | 3341 XHCI_TRB_2_TDSZ_SET(1) | 3342 XHCI_TRB_2_BYTES_SET(len); 3343 control = (isread ? XHCI_TRB_3_DIR_IN : 0) | 3344 XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_DATA_STAGE) | 3345 XHCI_TRB_3_CHAIN_BIT | XHCI_TRB_3_ENT_BIT; 3346 xhci_trb_put(&xx->xx_trb[i++], parameter, status, control); 3347 3348 parameter = (uintptr_t)xfer | 0x3; 3349 status = XHCI_TRB_2_IRQ_SET(0); 3350 control = XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_EVENT_DATA) | 3351 XHCI_TRB_3_IOC_BIT; 3352 xhci_trb_put(&xx->xx_trb[i++], parameter, status, control); 3353 } 3354 3355 parameter = 0; 3356 status = XHCI_TRB_2_IRQ_SET(0); 3357 /* the status stage has inverted direction */ 3358 control = ((isread && (len > 0)) ? 0 : XHCI_TRB_3_DIR_IN) | 3359 XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_STATUS_STAGE) | 3360 XHCI_TRB_3_CHAIN_BIT | XHCI_TRB_3_ENT_BIT; 3361 xhci_trb_put(&xx->xx_trb[i++], parameter, status, control); 3362 3363 parameter = (uintptr_t)xfer | 0x0; 3364 status = XHCI_TRB_2_IRQ_SET(0); 3365 control = XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_EVENT_DATA) | 3366 XHCI_TRB_3_IOC_BIT; 3367 xhci_trb_put(&xx->xx_trb[i++], parameter, status, control); 3368 3369 mutex_enter(&tr->xr_lock); 3370 xhci_ring_put(sc, tr, xfer, xx->xx_trb, i); 3371 mutex_exit(&tr->xr_lock); 3372 3373 xhci_db_write_4(sc, XHCI_DOORBELL(xs->xs_idx), dci); 3374 3375 if (xfer->ux_timeout && !sc->sc_bus.ub_usepolling) { 3376 callout_reset(&xfer->ux_callout, mstohz(xfer->ux_timeout), 3377 xhci_timeout, xfer); 3378 } 3379 3380 return USBD_IN_PROGRESS; 3381 } 3382 3383 static void 3384 xhci_device_ctrl_done(struct usbd_xfer *xfer) 3385 { 3386 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 3387 usb_device_request_t *req = &xfer->ux_request; 3388 int len = UGETW(req->wLength); 3389 int rd = req->bmRequestType & UT_READ; 3390 3391 if (len) 3392 usb_syncmem(&xfer->ux_dmabuf, 0, len, 3393 rd ? BUS_DMASYNC_POSTREAD : BUS_DMASYNC_POSTWRITE); 3394 } 3395 3396 static void 3397 xhci_device_ctrl_abort(struct usbd_xfer *xfer) 3398 { 3399 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 3400 3401 xhci_abort_xfer(xfer, USBD_CANCELLED); 3402 } 3403 3404 static void 3405 xhci_device_ctrl_close(struct usbd_pipe *pipe) 3406 { 3407 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 3408 3409 xhci_close_pipe(pipe); 3410 } 3411 3412 /* ------------------ */ 3413 /* device isochronous */ 3414 3415 /* ----------- */ 3416 /* device bulk */ 3417 3418 static usbd_status 3419 xhci_device_bulk_transfer(struct usbd_xfer *xfer) 3420 { 3421 struct xhci_softc * const sc = XHCI_XFER2SC(xfer); 3422 usbd_status err; 3423 3424 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 3425 3426 /* Insert last in queue. */ 3427 mutex_enter(&sc->sc_lock); 3428 err = usb_insert_transfer(xfer); 3429 mutex_exit(&sc->sc_lock); 3430 if (err) 3431 return err; 3432 3433 /* 3434 * Pipe isn't running (otherwise err would be USBD_INPROG), 3435 * so start it first. 3436 */ 3437 return xhci_device_bulk_start(SIMPLEQ_FIRST(&xfer->ux_pipe->up_queue)); 3438 } 3439 3440 static usbd_status 3441 xhci_device_bulk_start(struct usbd_xfer *xfer) 3442 { 3443 struct xhci_softc * const sc = XHCI_XFER2SC(xfer); 3444 struct xhci_slot * const xs = xfer->ux_pipe->up_dev->ud_hcpriv; 3445 const u_int dci = xhci_ep_get_dci(xfer->ux_pipe->up_endpoint->ue_edesc); 3446 struct xhci_ring * const tr = &xs->xs_ep[dci].xe_tr; 3447 struct xhci_xfer * const xx = XHCI_XFER2XXFER(xfer); 3448 const uint32_t len = xfer->ux_length; 3449 usb_dma_t * const dma = &xfer->ux_dmabuf; 3450 uint64_t parameter; 3451 uint32_t status; 3452 uint32_t control; 3453 u_int i = 0; 3454 3455 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 3456 3457 DPRINTFN(15, "%p slot %u dci %u", xfer, xs->xs_idx, dci, 0); 3458 3459 if (sc->sc_dying) 3460 return USBD_IOERROR; 3461 3462 KASSERT((xfer->ux_rqflags & URQ_REQUEST) == 0); 3463 3464 parameter = DMAADDR(dma, 0); 3465 /* 3466 * XXX: (dsl) The physical buffer must not cross a 64k boundary. 3467 * If the user supplied buffer crosses such a boundary then 2 3468 * (or more) TRB should be used. 3469 * If multiple TRB are used the td_size field must be set correctly. 3470 * For v1.0 devices (like ivy bridge) this is the number of usb data 3471 * blocks needed to complete the transfer. 3472 * Setting it to 1 in the last TRB causes an extra zero-length 3473 * data block be sent. 3474 * The earlier documentation differs, I don't know how it behaves. 3475 */ 3476 KASSERT(len <= 0x10000); 3477 status = XHCI_TRB_2_IRQ_SET(0) | 3478 XHCI_TRB_2_TDSZ_SET(1) | 3479 XHCI_TRB_2_BYTES_SET(len); 3480 control = XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_NORMAL) | 3481 XHCI_TRB_3_ISP_BIT | XHCI_TRB_3_IOC_BIT; 3482 xhci_trb_put(&xx->xx_trb[i++], parameter, status, control); 3483 3484 mutex_enter(&tr->xr_lock); 3485 xhci_ring_put(sc, tr, xfer, xx->xx_trb, i); 3486 mutex_exit(&tr->xr_lock); 3487 3488 xhci_db_write_4(sc, XHCI_DOORBELL(xs->xs_idx), dci); 3489 3490 if (xfer->ux_timeout && !sc->sc_bus.ub_usepolling) { 3491 callout_reset(&xfer->ux_callout, mstohz(xfer->ux_timeout), 3492 xhci_timeout, xfer); 3493 } 3494 3495 return USBD_IN_PROGRESS; 3496 } 3497 3498 static void 3499 xhci_device_bulk_done(struct usbd_xfer *xfer) 3500 { 3501 #ifdef USB_DEBUG 3502 struct xhci_slot * const xs = xfer->ux_pipe->up_dev->ud_hcpriv; 3503 const u_int dci = xhci_ep_get_dci(xfer->ux_pipe->up_endpoint->ue_edesc); 3504 #endif 3505 const int isread = usbd_xfer_isread(xfer); 3506 3507 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 3508 3509 DPRINTFN(15, "%p slot %u dci %u", xfer, xs->xs_idx, dci, 0); 3510 3511 usb_syncmem(&xfer->ux_dmabuf, 0, xfer->ux_length, 3512 isread ? BUS_DMASYNC_POSTREAD : BUS_DMASYNC_POSTWRITE); 3513 } 3514 3515 static void 3516 xhci_device_bulk_abort(struct usbd_xfer *xfer) 3517 { 3518 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 3519 3520 xhci_abort_xfer(xfer, USBD_CANCELLED); 3521 } 3522 3523 static void 3524 xhci_device_bulk_close(struct usbd_pipe *pipe) 3525 { 3526 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 3527 3528 xhci_close_pipe(pipe); 3529 } 3530 3531 /* ---------------- */ 3532 /* device interrupt */ 3533 3534 static usbd_status 3535 xhci_device_intr_transfer(struct usbd_xfer *xfer) 3536 { 3537 struct xhci_softc * const sc = XHCI_XFER2SC(xfer); 3538 usbd_status err; 3539 3540 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 3541 3542 /* Insert last in queue. */ 3543 mutex_enter(&sc->sc_lock); 3544 err = usb_insert_transfer(xfer); 3545 mutex_exit(&sc->sc_lock); 3546 if (err) 3547 return err; 3548 3549 /* 3550 * Pipe isn't running (otherwise err would be USBD_INPROG), 3551 * so start it first. 3552 */ 3553 return xhci_device_intr_start(SIMPLEQ_FIRST(&xfer->ux_pipe->up_queue)); 3554 } 3555 3556 static usbd_status 3557 xhci_device_intr_start(struct usbd_xfer *xfer) 3558 { 3559 struct xhci_softc * const sc = XHCI_XFER2SC(xfer); 3560 struct xhci_slot * const xs = xfer->ux_pipe->up_dev->ud_hcpriv; 3561 const u_int dci = xhci_ep_get_dci(xfer->ux_pipe->up_endpoint->ue_edesc); 3562 struct xhci_ring * const tr = &xs->xs_ep[dci].xe_tr; 3563 struct xhci_xfer * const xx = XHCI_XFER2XXFER(xfer); 3564 const uint32_t len = xfer->ux_length; 3565 usb_dma_t * const dma = &xfer->ux_dmabuf; 3566 uint64_t parameter; 3567 uint32_t status; 3568 uint32_t control; 3569 u_int i = 0; 3570 3571 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 3572 3573 DPRINTFN(15, "%p slot %u dci %u", xfer, xs->xs_idx, dci, 0); 3574 3575 if (sc->sc_dying) 3576 return USBD_IOERROR; 3577 3578 KASSERT((xfer->ux_rqflags & URQ_REQUEST) == 0); 3579 3580 parameter = DMAADDR(dma, 0); 3581 KASSERT(len <= 0x10000); 3582 status = XHCI_TRB_2_IRQ_SET(0) | 3583 XHCI_TRB_2_TDSZ_SET(1) | 3584 XHCI_TRB_2_BYTES_SET(len); 3585 control = XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_NORMAL) | 3586 XHCI_TRB_3_ISP_BIT | XHCI_TRB_3_IOC_BIT; 3587 xhci_trb_put(&xx->xx_trb[i++], parameter, status, control); 3588 3589 mutex_enter(&tr->xr_lock); 3590 xhci_ring_put(sc, tr, xfer, xx->xx_trb, i); 3591 mutex_exit(&tr->xr_lock); 3592 3593 xhci_db_write_4(sc, XHCI_DOORBELL(xs->xs_idx), dci); 3594 3595 if (xfer->ux_timeout && !sc->sc_bus.ub_usepolling) { 3596 callout_reset(&xfer->ux_callout, mstohz(xfer->ux_timeout), 3597 xhci_timeout, xfer); 3598 } 3599 3600 return USBD_IN_PROGRESS; 3601 } 3602 3603 static void 3604 xhci_device_intr_done(struct usbd_xfer *xfer) 3605 { 3606 struct xhci_softc * const sc __diagused = XHCI_XFER2SC(xfer); 3607 #ifdef USB_DEBUG 3608 struct xhci_slot * const xs = xfer->ux_pipe->up_dev->ud_hcpriv; 3609 const u_int dci = xhci_ep_get_dci(xfer->ux_pipe->up_endpoint->ue_edesc); 3610 #endif 3611 const int isread = usbd_xfer_isread(xfer); 3612 3613 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 3614 3615 DPRINTFN(15, "%p slot %u dci %u", xfer, xs->xs_idx, dci, 0); 3616 3617 KASSERT(sc->sc_bus.ub_usepolling || mutex_owned(&sc->sc_lock)); 3618 3619 usb_syncmem(&xfer->ux_dmabuf, 0, xfer->ux_length, 3620 isread ? BUS_DMASYNC_POSTREAD : BUS_DMASYNC_POSTWRITE); 3621 } 3622 3623 static void 3624 xhci_device_intr_abort(struct usbd_xfer *xfer) 3625 { 3626 struct xhci_softc * const sc __diagused = XHCI_XFER2SC(xfer); 3627 3628 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 3629 3630 KASSERT(mutex_owned(&sc->sc_lock)); 3631 DPRINTFN(15, "%p", xfer, 0, 0, 0); 3632 KASSERT(xfer->ux_pipe->up_intrxfer == xfer); 3633 xhci_abort_xfer(xfer, USBD_CANCELLED); 3634 } 3635 3636 static void 3637 xhci_device_intr_close(struct usbd_pipe *pipe) 3638 { 3639 //struct xhci_softc * const sc = XHCI_PIPE2SC(pipe); 3640 3641 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 3642 DPRINTFN(15, "%p", pipe, 0, 0, 0); 3643 3644 xhci_close_pipe(pipe); 3645 } 3646 3647 /* ------------ */ 3648 3649 static void 3650 xhci_timeout(void *addr) 3651 { 3652 struct xhci_xfer * const xx = addr; 3653 struct usbd_xfer * const xfer = &xx->xx_xfer; 3654 struct xhci_softc * const sc = XHCI_XFER2SC(xfer); 3655 3656 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 3657 3658 if (sc->sc_dying) { 3659 return; 3660 } 3661 3662 usb_init_task(&xfer->ux_aborttask, xhci_timeout_task, addr, 3663 USB_TASKQ_MPSAFE); 3664 usb_add_task(xx->xx_xfer.ux_pipe->up_dev, &xfer->ux_aborttask, 3665 USB_TASKQ_HC); 3666 } 3667 3668 static void 3669 xhci_timeout_task(void *addr) 3670 { 3671 struct usbd_xfer * const xfer = addr; 3672 struct xhci_softc * const sc = XHCI_XFER2SC(xfer); 3673 3674 XHCIHIST_FUNC(); XHCIHIST_CALLED(); 3675 3676 mutex_enter(&sc->sc_lock); 3677 #if 0 3678 xhci_abort_xfer(xfer, USBD_TIMEOUT); 3679 #else 3680 xfer->ux_status = USBD_TIMEOUT; 3681 usb_transfer_complete(xfer); 3682 #endif 3683 mutex_exit(&sc->sc_lock); 3684 } 3685
Indexes created Wed Oct 01 15:09:59 GMT 2025