xhci.c revision 1.134 1 1.134 jakllsch /* $NetBSD: xhci.c,v 1.134 2020/08/21 20:46:03 jakllsch Exp $ */
2 1.1 jakllsch
3 1.1 jakllsch /*
4 1.1 jakllsch * Copyright (c) 2013 Jonathan A. Kollasch
5 1.1 jakllsch * All rights reserved.
6 1.1 jakllsch *
7 1.1 jakllsch * Redistribution and use in source and binary forms, with or without
8 1.1 jakllsch * modification, are permitted provided that the following conditions
9 1.1 jakllsch * are met:
10 1.1 jakllsch * 1. Redistributions of source code must retain the above copyright
11 1.1 jakllsch * notice, this list of conditions and the following disclaimer.
12 1.1 jakllsch * 2. Redistributions in binary form must reproduce the above copyright
13 1.1 jakllsch * notice, this list of conditions and the following disclaimer in the
14 1.1 jakllsch * documentation and/or other materials provided with the distribution.
15 1.1 jakllsch *
16 1.1 jakllsch * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
17 1.1 jakllsch * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
18 1.1 jakllsch * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
19 1.1 jakllsch * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
20 1.1 jakllsch * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
21 1.1 jakllsch * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
22 1.1 jakllsch * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
23 1.1 jakllsch * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
24 1.1 jakllsch * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
25 1.1 jakllsch * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
26 1.1 jakllsch * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27 1.1 jakllsch */
28 1.1 jakllsch
29 1.34 skrll /*
30 1.41 skrll * USB rev 2.0 and rev 3.1 specification
31 1.41 skrll * http://www.usb.org/developers/docs/
32 1.34 skrll * xHCI rev 1.1 specification
33 1.41 skrll * http://www.intel.com/technology/usb/spec.htm
34 1.34 skrll */
35 1.34 skrll
36 1.1 jakllsch #include <sys/cdefs.h>
37 1.134 jakllsch __KERNEL_RCSID(0, "$NetBSD: xhci.c,v 1.134 2020/08/21 20:46:03 jakllsch Exp $");
38 1.27 skrll
39 1.46 pooka #ifdef _KERNEL_OPT
40 1.27 skrll #include "opt_usb.h"
41 1.46 pooka #endif
42 1.1 jakllsch
43 1.1 jakllsch #include <sys/param.h>
44 1.1 jakllsch #include <sys/systm.h>
45 1.1 jakllsch #include <sys/kernel.h>
46 1.1 jakllsch #include <sys/kmem.h>
47 1.1 jakllsch #include <sys/device.h>
48 1.1 jakllsch #include <sys/select.h>
49 1.1 jakllsch #include <sys/proc.h>
50 1.1 jakllsch #include <sys/queue.h>
51 1.1 jakllsch #include <sys/mutex.h>
52 1.1 jakllsch #include <sys/condvar.h>
53 1.1 jakllsch #include <sys/bus.h>
54 1.1 jakllsch #include <sys/cpu.h>
55 1.27 skrll #include <sys/sysctl.h>
56 1.1 jakllsch
57 1.1 jakllsch #include <machine/endian.h>
58 1.1 jakllsch
59 1.1 jakllsch #include <dev/usb/usb.h>
60 1.1 jakllsch #include <dev/usb/usbdi.h>
61 1.1 jakllsch #include <dev/usb/usbdivar.h>
62 1.34 skrll #include <dev/usb/usbdi_util.h>
63 1.27 skrll #include <dev/usb/usbhist.h>
64 1.1 jakllsch #include <dev/usb/usb_mem.h>
65 1.1 jakllsch #include <dev/usb/usb_quirks.h>
66 1.1 jakllsch
67 1.1 jakllsch #include <dev/usb/xhcireg.h>
68 1.1 jakllsch #include <dev/usb/xhcivar.h>
69 1.34 skrll #include <dev/usb/usbroothub.h>
70 1.1 jakllsch
71 1.27 skrll
72 1.27 skrll #ifdef USB_DEBUG
73 1.27 skrll #ifndef XHCI_DEBUG
74 1.27 skrll #define xhcidebug 0
75 1.34 skrll #else /* !XHCI_DEBUG */
76 1.79 christos #define HEXDUMP(a, b, c) \
77 1.79 christos do { \
78 1.79 christos if (xhcidebug > 0) \
79 1.80 christos hexdump(printf, a, b, c); \
80 1.79 christos } while (/*CONSTCOND*/0)
81 1.27 skrll static int xhcidebug = 0;
82 1.27 skrll
83 1.27 skrll SYSCTL_SETUP(sysctl_hw_xhci_setup, "sysctl hw.xhci setup")
84 1.27 skrll {
85 1.27 skrll int err;
86 1.27 skrll const struct sysctlnode *rnode;
87 1.27 skrll const struct sysctlnode *cnode;
88 1.27 skrll
89 1.27 skrll err = sysctl_createv(clog, 0, NULL, &rnode,
90 1.27 skrll CTLFLAG_PERMANENT, CTLTYPE_NODE, "xhci",
91 1.27 skrll SYSCTL_DESCR("xhci global controls"),
92 1.27 skrll NULL, 0, NULL, 0, CTL_HW, CTL_CREATE, CTL_EOL);
93 1.27 skrll
94 1.27 skrll if (err)
95 1.27 skrll goto fail;
96 1.27 skrll
97 1.27 skrll /* control debugging printfs */
98 1.27 skrll err = sysctl_createv(clog, 0, &rnode, &cnode,
99 1.27 skrll CTLFLAG_PERMANENT|CTLFLAG_READWRITE, CTLTYPE_INT,
100 1.27 skrll "debug", SYSCTL_DESCR("Enable debugging output"),
101 1.27 skrll NULL, 0, &xhcidebug, sizeof(xhcidebug), CTL_CREATE, CTL_EOL);
102 1.27 skrll if (err)
103 1.27 skrll goto fail;
104 1.27 skrll
105 1.27 skrll return;
106 1.27 skrll fail:
107 1.27 skrll aprint_error("%s: sysctl_createv failed (err = %d)\n", __func__, err);
108 1.27 skrll }
109 1.27 skrll
110 1.34 skrll #endif /* !XHCI_DEBUG */
111 1.27 skrll #endif /* USB_DEBUG */
112 1.27 skrll
113 1.79 christos #ifndef HEXDUMP
114 1.79 christos #define HEXDUMP(a, b, c)
115 1.79 christos #endif
116 1.79 christos
117 1.111 mrg #define DPRINTF(FMT,A,B,C,D) USBHIST_LOG(xhcidebug,FMT,A,B,C,D)
118 1.111 mrg #define DPRINTFN(N,FMT,A,B,C,D) USBHIST_LOGN(xhcidebug,N,FMT,A,B,C,D)
119 1.111 mrg #define XHCIHIST_FUNC() USBHIST_FUNC()
120 1.111 mrg #define XHCIHIST_CALLED(name) USBHIST_CALLED(xhcidebug)
121 1.111 mrg #define XHCIHIST_CALLARGS(FMT,A,B,C,D) \
122 1.111 mrg USBHIST_CALLARGS(xhcidebug,FMT,A,B,C,D)
123 1.1 jakllsch
124 1.1 jakllsch #define XHCI_DCI_SLOT 0
125 1.1 jakllsch #define XHCI_DCI_EP_CONTROL 1
126 1.1 jakllsch
127 1.1 jakllsch #define XHCI_ICI_INPUT_CONTROL 0
128 1.1 jakllsch
129 1.1 jakllsch struct xhci_pipe {
130 1.1 jakllsch struct usbd_pipe xp_pipe;
131 1.34 skrll struct usb_task xp_async_task;
132 1.134 jakllsch int16_t xp_isoc_next; /* next frame */
133 1.134 jakllsch uint8_t xp_maxb; /* max burst */
134 1.134 jakllsch uint8_t xp_mult;
135 1.1 jakllsch };
136 1.1 jakllsch
137 1.1 jakllsch #define XHCI_COMMAND_RING_TRBS 256
138 1.1 jakllsch #define XHCI_EVENT_RING_TRBS 256
139 1.1 jakllsch #define XHCI_EVENT_RING_SEGMENTS 1
140 1.1 jakllsch #define XHCI_TRB_3_ED_BIT XHCI_TRB_3_ISP_BIT
141 1.1 jakllsch
142 1.34 skrll static usbd_status xhci_open(struct usbd_pipe *);
143 1.34 skrll static void xhci_close_pipe(struct usbd_pipe *);
144 1.1 jakllsch static int xhci_intr1(struct xhci_softc * const);
145 1.1 jakllsch static void xhci_softintr(void *);
146 1.1 jakllsch static void xhci_poll(struct usbd_bus *);
147 1.34 skrll static struct usbd_xfer *xhci_allocx(struct usbd_bus *, unsigned int);
148 1.34 skrll static void xhci_freex(struct usbd_bus *, struct usbd_xfer *);
149 1.116 riastrad static void xhci_abortx(struct usbd_xfer *);
150 1.116 riastrad static bool xhci_dying(struct usbd_bus *);
151 1.1 jakllsch static void xhci_get_lock(struct usbd_bus *, kmutex_t **);
152 1.34 skrll static usbd_status xhci_new_device(device_t, struct usbd_bus *, int, int, int,
153 1.1 jakllsch struct usbd_port *);
154 1.34 skrll static int xhci_roothub_ctrl(struct usbd_bus *, usb_device_request_t *,
155 1.34 skrll void *, int);
156 1.1 jakllsch
157 1.34 skrll static usbd_status xhci_configure_endpoint(struct usbd_pipe *);
158 1.34 skrll //static usbd_status xhci_unconfigure_endpoint(struct usbd_pipe *);
159 1.34 skrll static usbd_status xhci_reset_endpoint(struct usbd_pipe *);
160 1.34 skrll static usbd_status xhci_stop_endpoint(struct usbd_pipe *);
161 1.1 jakllsch
162 1.55 skrll static void xhci_host_dequeue(struct xhci_ring * const);
163 1.34 skrll static usbd_status xhci_set_dequeue(struct usbd_pipe *);
164 1.1 jakllsch
165 1.1 jakllsch static usbd_status xhci_do_command(struct xhci_softc * const,
166 1.101 jakllsch struct xhci_soft_trb * const, int);
167 1.34 skrll static usbd_status xhci_do_command_locked(struct xhci_softc * const,
168 1.101 jakllsch struct xhci_soft_trb * const, int);
169 1.48 skrll static usbd_status xhci_init_slot(struct usbd_device *, uint32_t);
170 1.123 skrll static void xhci_free_slot(struct xhci_softc *, struct xhci_slot *);
171 1.51 skrll static usbd_status xhci_set_address(struct usbd_device *, uint32_t, bool);
172 1.1 jakllsch static usbd_status xhci_enable_slot(struct xhci_softc * const,
173 1.1 jakllsch uint8_t * const);
174 1.34 skrll static usbd_status xhci_disable_slot(struct xhci_softc * const, uint8_t);
175 1.1 jakllsch static usbd_status xhci_address_device(struct xhci_softc * const,
176 1.1 jakllsch uint64_t, uint8_t, bool);
177 1.34 skrll static void xhci_set_dcba(struct xhci_softc * const, uint64_t, int);
178 1.1 jakllsch static usbd_status xhci_update_ep0_mps(struct xhci_softc * const,
179 1.1 jakllsch struct xhci_slot * const, u_int);
180 1.1 jakllsch static usbd_status xhci_ring_init(struct xhci_softc * const,
181 1.123 skrll struct xhci_ring **, size_t, size_t);
182 1.123 skrll static void xhci_ring_free(struct xhci_softc * const,
183 1.123 skrll struct xhci_ring ** const);
184 1.1 jakllsch
185 1.51 skrll static void xhci_setup_ctx(struct usbd_pipe *);
186 1.51 skrll static void xhci_setup_route(struct usbd_pipe *, uint32_t *);
187 1.51 skrll static void xhci_setup_tthub(struct usbd_pipe *, uint32_t *);
188 1.51 skrll static void xhci_setup_maxburst(struct usbd_pipe *, uint32_t *);
189 1.51 skrll static uint32_t xhci_bival2ival(uint32_t, uint32_t);
190 1.51 skrll
191 1.34 skrll static void xhci_noop(struct usbd_pipe *);
192 1.1 jakllsch
193 1.34 skrll static usbd_status xhci_root_intr_transfer(struct usbd_xfer *);
194 1.34 skrll static usbd_status xhci_root_intr_start(struct usbd_xfer *);
195 1.34 skrll static void xhci_root_intr_abort(struct usbd_xfer *);
196 1.34 skrll static void xhci_root_intr_close(struct usbd_pipe *);
197 1.34 skrll static void xhci_root_intr_done(struct usbd_xfer *);
198 1.34 skrll
199 1.34 skrll static usbd_status xhci_device_ctrl_transfer(struct usbd_xfer *);
200 1.34 skrll static usbd_status xhci_device_ctrl_start(struct usbd_xfer *);
201 1.34 skrll static void xhci_device_ctrl_abort(struct usbd_xfer *);
202 1.34 skrll static void xhci_device_ctrl_close(struct usbd_pipe *);
203 1.34 skrll static void xhci_device_ctrl_done(struct usbd_xfer *);
204 1.34 skrll
205 1.134 jakllsch static usbd_status xhci_device_isoc_transfer(struct usbd_xfer *);
206 1.134 jakllsch static usbd_status xhci_device_isoc_enter(struct usbd_xfer *);
207 1.134 jakllsch static void xhci_device_isoc_abort(struct usbd_xfer *);
208 1.134 jakllsch static void xhci_device_isoc_close(struct usbd_pipe *);
209 1.134 jakllsch static void xhci_device_isoc_done(struct usbd_xfer *);
210 1.134 jakllsch
211 1.34 skrll static usbd_status xhci_device_intr_transfer(struct usbd_xfer *);
212 1.34 skrll static usbd_status xhci_device_intr_start(struct usbd_xfer *);
213 1.34 skrll static void xhci_device_intr_abort(struct usbd_xfer *);
214 1.34 skrll static void xhci_device_intr_close(struct usbd_pipe *);
215 1.34 skrll static void xhci_device_intr_done(struct usbd_xfer *);
216 1.34 skrll
217 1.34 skrll static usbd_status xhci_device_bulk_transfer(struct usbd_xfer *);
218 1.34 skrll static usbd_status xhci_device_bulk_start(struct usbd_xfer *);
219 1.34 skrll static void xhci_device_bulk_abort(struct usbd_xfer *);
220 1.34 skrll static void xhci_device_bulk_close(struct usbd_pipe *);
221 1.34 skrll static void xhci_device_bulk_done(struct usbd_xfer *);
222 1.1 jakllsch
223 1.1 jakllsch static const struct usbd_bus_methods xhci_bus_methods = {
224 1.34 skrll .ubm_open = xhci_open,
225 1.34 skrll .ubm_softint = xhci_softintr,
226 1.34 skrll .ubm_dopoll = xhci_poll,
227 1.34 skrll .ubm_allocx = xhci_allocx,
228 1.34 skrll .ubm_freex = xhci_freex,
229 1.116 riastrad .ubm_abortx = xhci_abortx,
230 1.116 riastrad .ubm_dying = xhci_dying,
231 1.34 skrll .ubm_getlock = xhci_get_lock,
232 1.34 skrll .ubm_newdev = xhci_new_device,
233 1.34 skrll .ubm_rhctrl = xhci_roothub_ctrl,
234 1.1 jakllsch };
235 1.1 jakllsch
236 1.1 jakllsch static const struct usbd_pipe_methods xhci_root_intr_methods = {
237 1.34 skrll .upm_transfer = xhci_root_intr_transfer,
238 1.34 skrll .upm_start = xhci_root_intr_start,
239 1.34 skrll .upm_abort = xhci_root_intr_abort,
240 1.34 skrll .upm_close = xhci_root_intr_close,
241 1.34 skrll .upm_cleartoggle = xhci_noop,
242 1.34 skrll .upm_done = xhci_root_intr_done,
243 1.1 jakllsch };
244 1.1 jakllsch
245 1.1 jakllsch
246 1.1 jakllsch static const struct usbd_pipe_methods xhci_device_ctrl_methods = {
247 1.34 skrll .upm_transfer = xhci_device_ctrl_transfer,
248 1.34 skrll .upm_start = xhci_device_ctrl_start,
249 1.34 skrll .upm_abort = xhci_device_ctrl_abort,
250 1.34 skrll .upm_close = xhci_device_ctrl_close,
251 1.34 skrll .upm_cleartoggle = xhci_noop,
252 1.34 skrll .upm_done = xhci_device_ctrl_done,
253 1.1 jakllsch };
254 1.1 jakllsch
255 1.1 jakllsch static const struct usbd_pipe_methods xhci_device_isoc_methods = {
256 1.134 jakllsch .upm_transfer = xhci_device_isoc_transfer,
257 1.134 jakllsch .upm_abort = xhci_device_isoc_abort,
258 1.134 jakllsch .upm_close = xhci_device_isoc_close,
259 1.34 skrll .upm_cleartoggle = xhci_noop,
260 1.134 jakllsch .upm_done = xhci_device_isoc_done,
261 1.1 jakllsch };
262 1.1 jakllsch
263 1.1 jakllsch static const struct usbd_pipe_methods xhci_device_bulk_methods = {
264 1.34 skrll .upm_transfer = xhci_device_bulk_transfer,
265 1.34 skrll .upm_start = xhci_device_bulk_start,
266 1.34 skrll .upm_abort = xhci_device_bulk_abort,
267 1.34 skrll .upm_close = xhci_device_bulk_close,
268 1.34 skrll .upm_cleartoggle = xhci_noop,
269 1.34 skrll .upm_done = xhci_device_bulk_done,
270 1.1 jakllsch };
271 1.1 jakllsch
272 1.1 jakllsch static const struct usbd_pipe_methods xhci_device_intr_methods = {
273 1.34 skrll .upm_transfer = xhci_device_intr_transfer,
274 1.34 skrll .upm_start = xhci_device_intr_start,
275 1.34 skrll .upm_abort = xhci_device_intr_abort,
276 1.34 skrll .upm_close = xhci_device_intr_close,
277 1.34 skrll .upm_cleartoggle = xhci_noop,
278 1.34 skrll .upm_done = xhci_device_intr_done,
279 1.1 jakllsch };
280 1.1 jakllsch
281 1.1 jakllsch static inline uint32_t
282 1.34 skrll xhci_read_1(const struct xhci_softc * const sc, bus_size_t offset)
283 1.34 skrll {
284 1.34 skrll return bus_space_read_1(sc->sc_iot, sc->sc_ioh, offset);
285 1.34 skrll }
286 1.34 skrll
287 1.34 skrll static inline uint32_t
288 1.130 skrll xhci_read_2(const struct xhci_softc * const sc, bus_size_t offset)
289 1.130 skrll {
290 1.130 skrll return bus_space_read_2(sc->sc_iot, sc->sc_ioh, offset);
291 1.130 skrll }
292 1.130 skrll
293 1.130 skrll static inline uint32_t
294 1.1 jakllsch xhci_read_4(const struct xhci_softc * const sc, bus_size_t offset)
295 1.1 jakllsch {
296 1.1 jakllsch return bus_space_read_4(sc->sc_iot, sc->sc_ioh, offset);
297 1.1 jakllsch }
298 1.1 jakllsch
299 1.34 skrll static inline void
300 1.34 skrll xhci_write_1(const struct xhci_softc * const sc, bus_size_t offset,
301 1.34 skrll uint32_t value)
302 1.34 skrll {
303 1.34 skrll bus_space_write_1(sc->sc_iot, sc->sc_ioh, offset, value);
304 1.34 skrll }
305 1.34 skrll
306 1.4 apb #if 0 /* unused */
307 1.1 jakllsch static inline void
308 1.1 jakllsch xhci_write_4(const struct xhci_softc * const sc, bus_size_t offset,
309 1.1 jakllsch uint32_t value)
310 1.1 jakllsch {
311 1.1 jakllsch bus_space_write_4(sc->sc_iot, sc->sc_ioh, offset, value);
312 1.1 jakllsch }
313 1.4 apb #endif /* unused */
314 1.1 jakllsch
315 1.1 jakllsch static inline uint32_t
316 1.1 jakllsch xhci_cap_read_4(const struct xhci_softc * const sc, bus_size_t offset)
317 1.1 jakllsch {
318 1.1 jakllsch return bus_space_read_4(sc->sc_iot, sc->sc_cbh, offset);
319 1.1 jakllsch }
320 1.1 jakllsch
321 1.1 jakllsch static inline uint32_t
322 1.1 jakllsch xhci_op_read_4(const struct xhci_softc * const sc, bus_size_t offset)
323 1.1 jakllsch {
324 1.1 jakllsch return bus_space_read_4(sc->sc_iot, sc->sc_obh, offset);
325 1.1 jakllsch }
326 1.1 jakllsch
327 1.1 jakllsch static inline void
328 1.1 jakllsch xhci_op_write_4(const struct xhci_softc * const sc, bus_size_t offset,
329 1.1 jakllsch uint32_t value)
330 1.1 jakllsch {
331 1.1 jakllsch bus_space_write_4(sc->sc_iot, sc->sc_obh, offset, value);
332 1.1 jakllsch }
333 1.1 jakllsch
334 1.1 jakllsch static inline uint64_t
335 1.1 jakllsch xhci_op_read_8(const struct xhci_softc * const sc, bus_size_t offset)
336 1.1 jakllsch {
337 1.1 jakllsch uint64_t value;
338 1.1 jakllsch
339 1.133 jakllsch if (XHCI_HCC_AC64(sc->sc_hcc)) {
340 1.1 jakllsch #ifdef XHCI_USE_BUS_SPACE_8
341 1.1 jakllsch value = bus_space_read_8(sc->sc_iot, sc->sc_obh, offset);
342 1.1 jakllsch #else
343 1.1 jakllsch value = bus_space_read_4(sc->sc_iot, sc->sc_obh, offset);
344 1.1 jakllsch value |= (uint64_t)bus_space_read_4(sc->sc_iot, sc->sc_obh,
345 1.1 jakllsch offset + 4) << 32;
346 1.1 jakllsch #endif
347 1.1 jakllsch } else {
348 1.1 jakllsch value = bus_space_read_4(sc->sc_iot, sc->sc_obh, offset);
349 1.1 jakllsch }
350 1.1 jakllsch
351 1.1 jakllsch return value;
352 1.1 jakllsch }
353 1.1 jakllsch
354 1.1 jakllsch static inline void
355 1.1 jakllsch xhci_op_write_8(const struct xhci_softc * const sc, bus_size_t offset,
356 1.1 jakllsch uint64_t value)
357 1.1 jakllsch {
358 1.133 jakllsch if (XHCI_HCC_AC64(sc->sc_hcc)) {
359 1.1 jakllsch #ifdef XHCI_USE_BUS_SPACE_8
360 1.1 jakllsch bus_space_write_8(sc->sc_iot, sc->sc_obh, offset, value);
361 1.1 jakllsch #else
362 1.1 jakllsch bus_space_write_4(sc->sc_iot, sc->sc_obh, offset + 0,
363 1.1 jakllsch (value >> 0) & 0xffffffff);
364 1.1 jakllsch bus_space_write_4(sc->sc_iot, sc->sc_obh, offset + 4,
365 1.1 jakllsch (value >> 32) & 0xffffffff);
366 1.1 jakllsch #endif
367 1.1 jakllsch } else {
368 1.1 jakllsch bus_space_write_4(sc->sc_iot, sc->sc_obh, offset, value);
369 1.1 jakllsch }
370 1.1 jakllsch }
371 1.1 jakllsch
372 1.103 skrll static inline void
373 1.103 skrll xhci_op_barrier(const struct xhci_softc * const sc, bus_size_t offset,
374 1.103 skrll bus_size_t len, int flags)
375 1.103 skrll {
376 1.103 skrll bus_space_barrier(sc->sc_iot, sc->sc_obh, offset, len, flags);
377 1.103 skrll }
378 1.103 skrll
379 1.1 jakllsch static inline uint32_t
380 1.1 jakllsch xhci_rt_read_4(const struct xhci_softc * const sc, bus_size_t offset)
381 1.1 jakllsch {
382 1.1 jakllsch return bus_space_read_4(sc->sc_iot, sc->sc_rbh, offset);
383 1.1 jakllsch }
384 1.1 jakllsch
385 1.1 jakllsch static inline void
386 1.1 jakllsch xhci_rt_write_4(const struct xhci_softc * const sc, bus_size_t offset,
387 1.1 jakllsch uint32_t value)
388 1.1 jakllsch {
389 1.1 jakllsch bus_space_write_4(sc->sc_iot, sc->sc_rbh, offset, value);
390 1.1 jakllsch }
391 1.1 jakllsch
392 1.4 apb #if 0 /* unused */
393 1.1 jakllsch static inline uint64_t
394 1.1 jakllsch xhci_rt_read_8(const struct xhci_softc * const sc, bus_size_t offset)
395 1.1 jakllsch {
396 1.1 jakllsch uint64_t value;
397 1.1 jakllsch
398 1.133 jakllsch if (XHCI_HCC_AC64(sc->sc_hcc)) {
399 1.1 jakllsch #ifdef XHCI_USE_BUS_SPACE_8
400 1.1 jakllsch value = bus_space_read_8(sc->sc_iot, sc->sc_rbh, offset);
401 1.1 jakllsch #else
402 1.1 jakllsch value = bus_space_read_4(sc->sc_iot, sc->sc_rbh, offset);
403 1.1 jakllsch value |= (uint64_t)bus_space_read_4(sc->sc_iot, sc->sc_rbh,
404 1.1 jakllsch offset + 4) << 32;
405 1.1 jakllsch #endif
406 1.1 jakllsch } else {
407 1.1 jakllsch value = bus_space_read_4(sc->sc_iot, sc->sc_rbh, offset);
408 1.1 jakllsch }
409 1.1 jakllsch
410 1.1 jakllsch return value;
411 1.1 jakllsch }
412 1.4 apb #endif /* unused */
413 1.1 jakllsch
414 1.1 jakllsch static inline void
415 1.1 jakllsch xhci_rt_write_8(const struct xhci_softc * const sc, bus_size_t offset,
416 1.1 jakllsch uint64_t value)
417 1.1 jakllsch {
418 1.133 jakllsch if (XHCI_HCC_AC64(sc->sc_hcc)) {
419 1.1 jakllsch #ifdef XHCI_USE_BUS_SPACE_8
420 1.1 jakllsch bus_space_write_8(sc->sc_iot, sc->sc_rbh, offset, value);
421 1.1 jakllsch #else
422 1.1 jakllsch bus_space_write_4(sc->sc_iot, sc->sc_rbh, offset + 0,
423 1.1 jakllsch (value >> 0) & 0xffffffff);
424 1.1 jakllsch bus_space_write_4(sc->sc_iot, sc->sc_rbh, offset + 4,
425 1.1 jakllsch (value >> 32) & 0xffffffff);
426 1.1 jakllsch #endif
427 1.1 jakllsch } else {
428 1.1 jakllsch bus_space_write_4(sc->sc_iot, sc->sc_rbh, offset, value);
429 1.1 jakllsch }
430 1.1 jakllsch }
431 1.1 jakllsch
432 1.4 apb #if 0 /* unused */
433 1.1 jakllsch static inline uint32_t
434 1.1 jakllsch xhci_db_read_4(const struct xhci_softc * const sc, bus_size_t offset)
435 1.1 jakllsch {
436 1.1 jakllsch return bus_space_read_4(sc->sc_iot, sc->sc_dbh, offset);
437 1.1 jakllsch }
438 1.4 apb #endif /* unused */
439 1.1 jakllsch
440 1.1 jakllsch static inline void
441 1.1 jakllsch xhci_db_write_4(const struct xhci_softc * const sc, bus_size_t offset,
442 1.1 jakllsch uint32_t value)
443 1.1 jakllsch {
444 1.1 jakllsch bus_space_write_4(sc->sc_iot, sc->sc_dbh, offset, value);
445 1.1 jakllsch }
446 1.1 jakllsch
447 1.1 jakllsch /* --- */
448 1.1 jakllsch
449 1.1 jakllsch static inline uint8_t
450 1.1 jakllsch xhci_ep_get_type(usb_endpoint_descriptor_t * const ed)
451 1.1 jakllsch {
452 1.34 skrll u_int eptype = 0;
453 1.1 jakllsch
454 1.1 jakllsch switch (UE_GET_XFERTYPE(ed->bmAttributes)) {
455 1.1 jakllsch case UE_CONTROL:
456 1.1 jakllsch eptype = 0x0;
457 1.1 jakllsch break;
458 1.1 jakllsch case UE_ISOCHRONOUS:
459 1.1 jakllsch eptype = 0x1;
460 1.1 jakllsch break;
461 1.1 jakllsch case UE_BULK:
462 1.1 jakllsch eptype = 0x2;
463 1.1 jakllsch break;
464 1.1 jakllsch case UE_INTERRUPT:
465 1.1 jakllsch eptype = 0x3;
466 1.1 jakllsch break;
467 1.1 jakllsch }
468 1.1 jakllsch
469 1.1 jakllsch if ((UE_GET_XFERTYPE(ed->bmAttributes) == UE_CONTROL) ||
470 1.1 jakllsch (UE_GET_DIR(ed->bEndpointAddress) == UE_DIR_IN))
471 1.1 jakllsch return eptype | 0x4;
472 1.1 jakllsch else
473 1.1 jakllsch return eptype;
474 1.1 jakllsch }
475 1.1 jakllsch
476 1.1 jakllsch static u_int
477 1.1 jakllsch xhci_ep_get_dci(usb_endpoint_descriptor_t * const ed)
478 1.1 jakllsch {
479 1.1 jakllsch /* xHCI 1.0 section 4.5.1 */
480 1.1 jakllsch u_int epaddr = UE_GET_ADDR(ed->bEndpointAddress);
481 1.1 jakllsch u_int in = 0;
482 1.1 jakllsch
483 1.1 jakllsch if ((UE_GET_XFERTYPE(ed->bmAttributes) == UE_CONTROL) ||
484 1.1 jakllsch (UE_GET_DIR(ed->bEndpointAddress) == UE_DIR_IN))
485 1.1 jakllsch in = 1;
486 1.1 jakllsch
487 1.1 jakllsch return epaddr * 2 + in;
488 1.1 jakllsch }
489 1.1 jakllsch
490 1.1 jakllsch static inline u_int
491 1.1 jakllsch xhci_dci_to_ici(const u_int i)
492 1.1 jakllsch {
493 1.1 jakllsch return i + 1;
494 1.1 jakllsch }
495 1.1 jakllsch
496 1.1 jakllsch static inline void *
497 1.1 jakllsch xhci_slot_get_dcv(struct xhci_softc * const sc, struct xhci_slot * const xs,
498 1.1 jakllsch const u_int dci)
499 1.1 jakllsch {
500 1.1 jakllsch return KERNADDR(&xs->xs_dc_dma, sc->sc_ctxsz * dci);
501 1.1 jakllsch }
502 1.1 jakllsch
503 1.4 apb #if 0 /* unused */
504 1.1 jakllsch static inline bus_addr_t
505 1.1 jakllsch xhci_slot_get_dcp(struct xhci_softc * const sc, struct xhci_slot * const xs,
506 1.1 jakllsch const u_int dci)
507 1.1 jakllsch {
508 1.1 jakllsch return DMAADDR(&xs->xs_dc_dma, sc->sc_ctxsz * dci);
509 1.1 jakllsch }
510 1.4 apb #endif /* unused */
511 1.1 jakllsch
512 1.1 jakllsch static inline void *
513 1.1 jakllsch xhci_slot_get_icv(struct xhci_softc * const sc, struct xhci_slot * const xs,
514 1.1 jakllsch const u_int ici)
515 1.1 jakllsch {
516 1.1 jakllsch return KERNADDR(&xs->xs_ic_dma, sc->sc_ctxsz * ici);
517 1.1 jakllsch }
518 1.1 jakllsch
519 1.1 jakllsch static inline bus_addr_t
520 1.1 jakllsch xhci_slot_get_icp(struct xhci_softc * const sc, struct xhci_slot * const xs,
521 1.1 jakllsch const u_int ici)
522 1.1 jakllsch {
523 1.1 jakllsch return DMAADDR(&xs->xs_ic_dma, sc->sc_ctxsz * ici);
524 1.1 jakllsch }
525 1.1 jakllsch
526 1.1 jakllsch static inline struct xhci_trb *
527 1.1 jakllsch xhci_ring_trbv(struct xhci_ring * const xr, u_int idx)
528 1.1 jakllsch {
529 1.1 jakllsch return KERNADDR(&xr->xr_dma, XHCI_TRB_SIZE * idx);
530 1.1 jakllsch }
531 1.1 jakllsch
532 1.1 jakllsch static inline bus_addr_t
533 1.1 jakllsch xhci_ring_trbp(struct xhci_ring * const xr, u_int idx)
534 1.1 jakllsch {
535 1.1 jakllsch return DMAADDR(&xr->xr_dma, XHCI_TRB_SIZE * idx);
536 1.1 jakllsch }
537 1.1 jakllsch
538 1.1 jakllsch static inline void
539 1.127 jakllsch xhci_xfer_put_trb(struct xhci_xfer * const xx, u_int idx,
540 1.101 jakllsch uint64_t parameter, uint32_t status, uint32_t control)
541 1.101 jakllsch {
542 1.128 jakllsch KASSERTMSG(idx < xx->xx_ntrb, "idx=%u xx_ntrb=%u", idx, xx->xx_ntrb);
543 1.127 jakllsch xx->xx_trb[idx].trb_0 = parameter;
544 1.127 jakllsch xx->xx_trb[idx].trb_2 = status;
545 1.127 jakllsch xx->xx_trb[idx].trb_3 = control;
546 1.101 jakllsch }
547 1.101 jakllsch
548 1.101 jakllsch static inline void
549 1.1 jakllsch xhci_trb_put(struct xhci_trb * const trb, uint64_t parameter, uint32_t status,
550 1.1 jakllsch uint32_t control)
551 1.1 jakllsch {
552 1.34 skrll trb->trb_0 = htole64(parameter);
553 1.34 skrll trb->trb_2 = htole32(status);
554 1.34 skrll trb->trb_3 = htole32(control);
555 1.1 jakllsch }
556 1.1 jakllsch
557 1.40 skrll static int
558 1.40 skrll xhci_trb_get_idx(struct xhci_ring *xr, uint64_t trb_0, int *idx)
559 1.40 skrll {
560 1.40 skrll /* base address of TRBs */
561 1.40 skrll bus_addr_t trbp = xhci_ring_trbp(xr, 0);
562 1.40 skrll
563 1.40 skrll /* trb_0 range sanity check */
564 1.40 skrll if (trb_0 == 0 || trb_0 < trbp ||
565 1.40 skrll (trb_0 - trbp) % sizeof(struct xhci_trb) != 0 ||
566 1.40 skrll (trb_0 - trbp) / sizeof(struct xhci_trb) >= xr->xr_ntrb) {
567 1.40 skrll return 1;
568 1.40 skrll }
569 1.40 skrll *idx = (trb_0 - trbp) / sizeof(struct xhci_trb);
570 1.40 skrll return 0;
571 1.40 skrll }
572 1.40 skrll
573 1.63 skrll static unsigned int
574 1.63 skrll xhci_get_epstate(struct xhci_softc * const sc, struct xhci_slot * const xs,
575 1.63 skrll u_int dci)
576 1.63 skrll {
577 1.63 skrll uint32_t *cp;
578 1.63 skrll
579 1.63 skrll usb_syncmem(&xs->xs_dc_dma, 0, sc->sc_pgsz, BUS_DMASYNC_POSTREAD);
580 1.63 skrll cp = xhci_slot_get_dcv(sc, xs, dci);
581 1.63 skrll return XHCI_EPCTX_0_EPSTATE_GET(le32toh(cp[0]));
582 1.63 skrll }
583 1.63 skrll
584 1.68 skrll static inline unsigned int
585 1.68 skrll xhci_ctlrport2bus(struct xhci_softc * const sc, unsigned int ctlrport)
586 1.68 skrll {
587 1.68 skrll const unsigned int port = ctlrport - 1;
588 1.68 skrll const uint8_t bit = __BIT(port % NBBY);
589 1.68 skrll
590 1.68 skrll return __SHIFTOUT(sc->sc_ctlrportbus[port / NBBY], bit);
591 1.68 skrll }
592 1.68 skrll
593 1.68 skrll /*
594 1.68 skrll * Return the roothub port for a controller port. Both are 1..n.
595 1.68 skrll */
596 1.68 skrll static inline unsigned int
597 1.68 skrll xhci_ctlrport2rhport(struct xhci_softc * const sc, unsigned int ctrlport)
598 1.68 skrll {
599 1.68 skrll
600 1.68 skrll return sc->sc_ctlrportmap[ctrlport - 1];
601 1.68 skrll }
602 1.68 skrll
603 1.68 skrll /*
604 1.68 skrll * Return the controller port for a bus roothub port. Both are 1..n.
605 1.68 skrll */
606 1.68 skrll static inline unsigned int
607 1.68 skrll xhci_rhport2ctlrport(struct xhci_softc * const sc, unsigned int bn,
608 1.68 skrll unsigned int rhport)
609 1.68 skrll {
610 1.68 skrll
611 1.68 skrll return sc->sc_rhportmap[bn][rhport - 1];
612 1.68 skrll }
613 1.68 skrll
614 1.1 jakllsch /* --- */
615 1.1 jakllsch
616 1.1 jakllsch void
617 1.1 jakllsch xhci_childdet(device_t self, device_t child)
618 1.1 jakllsch {
619 1.1 jakllsch struct xhci_softc * const sc = device_private(self);
620 1.1 jakllsch
621 1.84 msaitoh KASSERT((sc->sc_child == child) || (sc->sc_child2 == child));
622 1.84 msaitoh if (child == sc->sc_child2)
623 1.84 msaitoh sc->sc_child2 = NULL;
624 1.84 msaitoh else if (child == sc->sc_child)
625 1.1 jakllsch sc->sc_child = NULL;
626 1.1 jakllsch }
627 1.1 jakllsch
628 1.1 jakllsch int
629 1.1 jakllsch xhci_detach(struct xhci_softc *sc, int flags)
630 1.1 jakllsch {
631 1.1 jakllsch int rv = 0;
632 1.1 jakllsch
633 1.68 skrll if (sc->sc_child2 != NULL) {
634 1.68 skrll rv = config_detach(sc->sc_child2, flags);
635 1.68 skrll if (rv != 0)
636 1.68 skrll return rv;
637 1.88 jdolecek KASSERT(sc->sc_child2 == NULL);
638 1.68 skrll }
639 1.68 skrll
640 1.68 skrll if (sc->sc_child != NULL) {
641 1.1 jakllsch rv = config_detach(sc->sc_child, flags);
642 1.68 skrll if (rv != 0)
643 1.68 skrll return rv;
644 1.88 jdolecek KASSERT(sc->sc_child == NULL);
645 1.68 skrll }
646 1.1 jakllsch
647 1.1 jakllsch /* XXX unconfigure/free slots */
648 1.1 jakllsch
649 1.1 jakllsch /* verify: */
650 1.1 jakllsch xhci_rt_write_4(sc, XHCI_IMAN(0), 0);
651 1.1 jakllsch xhci_op_write_4(sc, XHCI_USBCMD, 0);
652 1.1 jakllsch /* do we need to wait for stop? */
653 1.1 jakllsch
654 1.1 jakllsch xhci_op_write_8(sc, XHCI_CRCR, 0);
655 1.1 jakllsch xhci_ring_free(sc, &sc->sc_cr);
656 1.1 jakllsch cv_destroy(&sc->sc_command_cv);
657 1.68 skrll cv_destroy(&sc->sc_cmdbusy_cv);
658 1.1 jakllsch
659 1.1 jakllsch xhci_rt_write_4(sc, XHCI_ERSTSZ(0), 0);
660 1.1 jakllsch xhci_rt_write_8(sc, XHCI_ERSTBA(0), 0);
661 1.132 skrll xhci_rt_write_8(sc, XHCI_ERDP(0), 0 | XHCI_ERDP_BUSY);
662 1.1 jakllsch xhci_ring_free(sc, &sc->sc_er);
663 1.1 jakllsch
664 1.1 jakllsch usb_freemem(&sc->sc_bus, &sc->sc_eventst_dma);
665 1.1 jakllsch
666 1.1 jakllsch xhci_op_write_8(sc, XHCI_DCBAAP, 0);
667 1.1 jakllsch usb_freemem(&sc->sc_bus, &sc->sc_dcbaa_dma);
668 1.1 jakllsch
669 1.1 jakllsch kmem_free(sc->sc_slots, sizeof(*sc->sc_slots) * sc->sc_maxslots);
670 1.1 jakllsch
671 1.82 skrll kmem_free(sc->sc_ctlrportbus,
672 1.70 skrll howmany(sc->sc_maxports * sizeof(uint8_t), NBBY));
673 1.68 skrll kmem_free(sc->sc_ctlrportmap, sc->sc_maxports * sizeof(int));
674 1.68 skrll
675 1.68 skrll for (size_t j = 0; j < __arraycount(sc->sc_rhportmap); j++) {
676 1.68 skrll kmem_free(sc->sc_rhportmap[j], sc->sc_maxports * sizeof(int));
677 1.68 skrll }
678 1.68 skrll
679 1.1 jakllsch mutex_destroy(&sc->sc_lock);
680 1.1 jakllsch mutex_destroy(&sc->sc_intr_lock);
681 1.1 jakllsch
682 1.1 jakllsch pool_cache_destroy(sc->sc_xferpool);
683 1.1 jakllsch
684 1.1 jakllsch return rv;
685 1.1 jakllsch }
686 1.1 jakllsch
687 1.1 jakllsch int
688 1.1 jakllsch xhci_activate(device_t self, enum devact act)
689 1.1 jakllsch {
690 1.1 jakllsch struct xhci_softc * const sc = device_private(self);
691 1.1 jakllsch
692 1.1 jakllsch switch (act) {
693 1.1 jakllsch case DVACT_DEACTIVATE:
694 1.1 jakllsch sc->sc_dying = true;
695 1.1 jakllsch return 0;
696 1.1 jakllsch default:
697 1.1 jakllsch return EOPNOTSUPP;
698 1.1 jakllsch }
699 1.1 jakllsch }
700 1.1 jakllsch
701 1.1 jakllsch bool
702 1.1 jakllsch xhci_suspend(device_t dv, const pmf_qual_t *qual)
703 1.1 jakllsch {
704 1.1 jakllsch return false;
705 1.1 jakllsch }
706 1.1 jakllsch
707 1.1 jakllsch bool
708 1.1 jakllsch xhci_resume(device_t dv, const pmf_qual_t *qual)
709 1.1 jakllsch {
710 1.1 jakllsch return false;
711 1.1 jakllsch }
712 1.1 jakllsch
713 1.1 jakllsch bool
714 1.1 jakllsch xhci_shutdown(device_t self, int flags)
715 1.1 jakllsch {
716 1.1 jakllsch return false;
717 1.1 jakllsch }
718 1.1 jakllsch
719 1.40 skrll static int
720 1.40 skrll xhci_hc_reset(struct xhci_softc * const sc)
721 1.40 skrll {
722 1.40 skrll uint32_t usbcmd, usbsts;
723 1.40 skrll int i;
724 1.40 skrll
725 1.40 skrll /* Check controller not ready */
726 1.42 skrll for (i = 0; i < XHCI_WAIT_CNR; i++) {
727 1.40 skrll usbsts = xhci_op_read_4(sc, XHCI_USBSTS);
728 1.40 skrll if ((usbsts & XHCI_STS_CNR) == 0)
729 1.40 skrll break;
730 1.40 skrll usb_delay_ms(&sc->sc_bus, 1);
731 1.40 skrll }
732 1.42 skrll if (i >= XHCI_WAIT_CNR) {
733 1.40 skrll aprint_error_dev(sc->sc_dev, "controller not ready timeout\n");
734 1.40 skrll return EIO;
735 1.40 skrll }
736 1.40 skrll
737 1.40 skrll /* Halt controller */
738 1.40 skrll usbcmd = 0;
739 1.40 skrll xhci_op_write_4(sc, XHCI_USBCMD, usbcmd);
740 1.40 skrll usb_delay_ms(&sc->sc_bus, 1);
741 1.40 skrll
742 1.40 skrll /* Reset controller */
743 1.40 skrll usbcmd = XHCI_CMD_HCRST;
744 1.40 skrll xhci_op_write_4(sc, XHCI_USBCMD, usbcmd);
745 1.42 skrll for (i = 0; i < XHCI_WAIT_HCRST; i++) {
746 1.76 msaitoh /*
747 1.76 msaitoh * Wait 1ms first. Existing Intel xHCI requies 1ms delay to
748 1.76 msaitoh * prevent system hang (Errata).
749 1.76 msaitoh */
750 1.76 msaitoh usb_delay_ms(&sc->sc_bus, 1);
751 1.40 skrll usbcmd = xhci_op_read_4(sc, XHCI_USBCMD);
752 1.40 skrll if ((usbcmd & XHCI_CMD_HCRST) == 0)
753 1.40 skrll break;
754 1.40 skrll }
755 1.42 skrll if (i >= XHCI_WAIT_HCRST) {
756 1.40 skrll aprint_error_dev(sc->sc_dev, "host controller reset timeout\n");
757 1.40 skrll return EIO;
758 1.40 skrll }
759 1.40 skrll
760 1.40 skrll /* Check controller not ready */
761 1.42 skrll for (i = 0; i < XHCI_WAIT_CNR; i++) {
762 1.40 skrll usbsts = xhci_op_read_4(sc, XHCI_USBSTS);
763 1.40 skrll if ((usbsts & XHCI_STS_CNR) == 0)
764 1.40 skrll break;
765 1.40 skrll usb_delay_ms(&sc->sc_bus, 1);
766 1.40 skrll }
767 1.42 skrll if (i >= XHCI_WAIT_CNR) {
768 1.40 skrll aprint_error_dev(sc->sc_dev,
769 1.40 skrll "controller not ready timeout after reset\n");
770 1.40 skrll return EIO;
771 1.40 skrll }
772 1.40 skrll
773 1.40 skrll return 0;
774 1.40 skrll }
775 1.40 skrll
776 1.1 jakllsch
777 1.68 skrll /* 7.2 xHCI Support Protocol Capability */
778 1.68 skrll static void
779 1.68 skrll xhci_id_protocols(struct xhci_softc *sc, bus_size_t ecp)
780 1.68 skrll {
781 1.109 mrg XHCIHIST_FUNC(); XHCIHIST_CALLED();
782 1.109 mrg
783 1.68 skrll /* XXX Cache this lot */
784 1.68 skrll
785 1.68 skrll const uint32_t w0 = xhci_read_4(sc, ecp);
786 1.68 skrll const uint32_t w4 = xhci_read_4(sc, ecp + 4);
787 1.68 skrll const uint32_t w8 = xhci_read_4(sc, ecp + 8);
788 1.68 skrll const uint32_t wc = xhci_read_4(sc, ecp + 0xc);
789 1.68 skrll
790 1.68 skrll aprint_debug_dev(sc->sc_dev,
791 1.121 christos " SP: 0x%08x 0x%08x 0x%08x 0x%08x\n", w0, w4, w8, wc);
792 1.68 skrll
793 1.68 skrll if (w4 != XHCI_XECP_USBID)
794 1.68 skrll return;
795 1.68 skrll
796 1.68 skrll const int major = XHCI_XECP_SP_W0_MAJOR(w0);
797 1.68 skrll const int minor = XHCI_XECP_SP_W0_MINOR(w0);
798 1.68 skrll const uint8_t cpo = XHCI_XECP_SP_W8_CPO(w8);
799 1.68 skrll const uint8_t cpc = XHCI_XECP_SP_W8_CPC(w8);
800 1.68 skrll
801 1.68 skrll const uint16_t mm = __SHIFTOUT(w0, __BITS(31, 16));
802 1.68 skrll switch (mm) {
803 1.68 skrll case 0x0200:
804 1.68 skrll case 0x0300:
805 1.68 skrll case 0x0301:
806 1.109 mrg case 0x0310:
807 1.68 skrll aprint_debug_dev(sc->sc_dev, " %s ports %d - %d\n",
808 1.68 skrll major == 3 ? "ss" : "hs", cpo, cpo + cpc -1);
809 1.68 skrll break;
810 1.68 skrll default:
811 1.110 mrg aprint_error_dev(sc->sc_dev, " unknown major/minor (%d/%d)\n",
812 1.68 skrll major, minor);
813 1.68 skrll return;
814 1.68 skrll }
815 1.68 skrll
816 1.68 skrll const size_t bus = (major == 3) ? 0 : 1;
817 1.68 skrll
818 1.68 skrll /* Index arrays with 0..n-1 where ports are numbered 1..n */
819 1.68 skrll for (size_t cp = cpo - 1; cp < cpo + cpc - 1; cp++) {
820 1.68 skrll if (sc->sc_ctlrportmap[cp] != 0) {
821 1.108 skrll aprint_error_dev(sc->sc_dev, "controller port %zu "
822 1.68 skrll "already assigned", cp);
823 1.68 skrll continue;
824 1.68 skrll }
825 1.68 skrll
826 1.68 skrll sc->sc_ctlrportbus[cp / NBBY] |=
827 1.68 skrll bus == 0 ? 0 : __BIT(cp % NBBY);
828 1.68 skrll
829 1.68 skrll const size_t rhp = sc->sc_rhportcount[bus]++;
830 1.68 skrll
831 1.68 skrll KASSERTMSG(sc->sc_rhportmap[bus][rhp] == 0,
832 1.68 skrll "bus %zu rhp %zu is %d", bus, rhp,
833 1.68 skrll sc->sc_rhportmap[bus][rhp]);
834 1.68 skrll
835 1.68 skrll sc->sc_rhportmap[bus][rhp] = cp + 1;
836 1.68 skrll sc->sc_ctlrportmap[cp] = rhp + 1;
837 1.68 skrll }
838 1.68 skrll }
839 1.68 skrll
840 1.40 skrll /* Process extended capabilities */
841 1.40 skrll static void
842 1.133 jakllsch xhci_ecp(struct xhci_softc *sc)
843 1.40 skrll {
844 1.40 skrll XHCIHIST_FUNC(); XHCIHIST_CALLED();
845 1.40 skrll
846 1.133 jakllsch bus_size_t ecp = XHCI_HCC_XECP(sc->sc_hcc) * 4;
847 1.40 skrll while (ecp != 0) {
848 1.68 skrll uint32_t ecr = xhci_read_4(sc, ecp);
849 1.121 christos aprint_debug_dev(sc->sc_dev, "ECR: 0x%08x\n", ecr);
850 1.40 skrll switch (XHCI_XECP_ID(ecr)) {
851 1.40 skrll case XHCI_ID_PROTOCOLS: {
852 1.68 skrll xhci_id_protocols(sc, ecp);
853 1.40 skrll break;
854 1.40 skrll }
855 1.40 skrll case XHCI_ID_USB_LEGACY: {
856 1.40 skrll uint8_t bios_sem;
857 1.40 skrll
858 1.40 skrll /* Take host controller ownership from BIOS */
859 1.40 skrll bios_sem = xhci_read_1(sc, ecp + XHCI_XECP_BIOS_SEM);
860 1.40 skrll if (bios_sem) {
861 1.40 skrll /* sets xHCI to be owned by OS */
862 1.40 skrll xhci_write_1(sc, ecp + XHCI_XECP_OS_SEM, 1);
863 1.40 skrll aprint_debug_dev(sc->sc_dev,
864 1.40 skrll "waiting for BIOS to give up control\n");
865 1.40 skrll for (int i = 0; i < 5000; i++) {
866 1.40 skrll bios_sem = xhci_read_1(sc, ecp +
867 1.40 skrll XHCI_XECP_BIOS_SEM);
868 1.40 skrll if (bios_sem == 0)
869 1.40 skrll break;
870 1.40 skrll DELAY(1000);
871 1.40 skrll }
872 1.40 skrll if (bios_sem) {
873 1.40 skrll aprint_error_dev(sc->sc_dev,
874 1.40 skrll "timed out waiting for BIOS\n");
875 1.40 skrll }
876 1.40 skrll }
877 1.40 skrll break;
878 1.40 skrll }
879 1.40 skrll default:
880 1.40 skrll break;
881 1.40 skrll }
882 1.40 skrll ecr = xhci_read_4(sc, ecp);
883 1.40 skrll if (XHCI_XECP_NEXT(ecr) == 0) {
884 1.40 skrll ecp = 0;
885 1.40 skrll } else {
886 1.40 skrll ecp += XHCI_XECP_NEXT(ecr) * 4;
887 1.40 skrll }
888 1.40 skrll }
889 1.40 skrll }
890 1.40 skrll
891 1.34 skrll #define XHCI_HCCPREV1_BITS \
892 1.34 skrll "\177\020" /* New bitmask */ \
893 1.34 skrll "f\020\020XECP\0" \
894 1.34 skrll "f\014\4MAXPSA\0" \
895 1.34 skrll "b\013CFC\0" \
896 1.34 skrll "b\012SEC\0" \
897 1.34 skrll "b\011SBD\0" \
898 1.34 skrll "b\010FSE\0" \
899 1.34 skrll "b\7NSS\0" \
900 1.34 skrll "b\6LTC\0" \
901 1.34 skrll "b\5LHRC\0" \
902 1.34 skrll "b\4PIND\0" \
903 1.34 skrll "b\3PPC\0" \
904 1.34 skrll "b\2CZC\0" \
905 1.34 skrll "b\1BNC\0" \
906 1.34 skrll "b\0AC64\0" \
907 1.34 skrll "\0"
908 1.34 skrll #define XHCI_HCCV1_x_BITS \
909 1.34 skrll "\177\020" /* New bitmask */ \
910 1.34 skrll "f\020\020XECP\0" \
911 1.34 skrll "f\014\4MAXPSA\0" \
912 1.34 skrll "b\013CFC\0" \
913 1.34 skrll "b\012SEC\0" \
914 1.34 skrll "b\011SPC\0" \
915 1.34 skrll "b\010PAE\0" \
916 1.34 skrll "b\7NSS\0" \
917 1.34 skrll "b\6LTC\0" \
918 1.34 skrll "b\5LHRC\0" \
919 1.34 skrll "b\4PIND\0" \
920 1.34 skrll "b\3PPC\0" \
921 1.34 skrll "b\2CSZ\0" \
922 1.34 skrll "b\1BNC\0" \
923 1.34 skrll "b\0AC64\0" \
924 1.34 skrll "\0"
925 1.1 jakllsch
926 1.95 msaitoh #define XHCI_HCC2_BITS \
927 1.95 msaitoh "\177\020" /* New bitmask */ \
928 1.95 msaitoh "b\7ETC_TSC\0" \
929 1.95 msaitoh "b\6ETC\0" \
930 1.95 msaitoh "b\5CIC\0" \
931 1.95 msaitoh "b\4LEC\0" \
932 1.95 msaitoh "b\3CTC\0" \
933 1.95 msaitoh "b\2FSC\0" \
934 1.95 msaitoh "b\1CMC\0" \
935 1.95 msaitoh "b\0U3C\0" \
936 1.95 msaitoh "\0"
937 1.95 msaitoh
938 1.74 jmcneill void
939 1.74 jmcneill xhci_start(struct xhci_softc *sc)
940 1.74 jmcneill {
941 1.74 jmcneill xhci_rt_write_4(sc, XHCI_IMAN(0), XHCI_IMAN_INTR_ENA);
942 1.74 jmcneill if ((sc->sc_quirks & XHCI_QUIRK_INTEL) != 0)
943 1.74 jmcneill /* Intel xhci needs interrupt rate moderated. */
944 1.74 jmcneill xhci_rt_write_4(sc, XHCI_IMOD(0), XHCI_IMOD_DEFAULT_LP);
945 1.74 jmcneill else
946 1.74 jmcneill xhci_rt_write_4(sc, XHCI_IMOD(0), 0);
947 1.74 jmcneill aprint_debug_dev(sc->sc_dev, "current IMOD %u\n",
948 1.74 jmcneill xhci_rt_read_4(sc, XHCI_IMOD(0)));
949 1.74 jmcneill
950 1.102 skrll /* Go! */
951 1.102 skrll xhci_op_write_4(sc, XHCI_USBCMD, XHCI_CMD_INTE|XHCI_CMD_RS);
952 1.121 christos aprint_debug_dev(sc->sc_dev, "USBCMD 0x%08"PRIx32"\n",
953 1.74 jmcneill xhci_op_read_4(sc, XHCI_USBCMD));
954 1.74 jmcneill }
955 1.74 jmcneill
956 1.15 skrll int
957 1.1 jakllsch xhci_init(struct xhci_softc *sc)
958 1.1 jakllsch {
959 1.1 jakllsch bus_size_t bsz;
960 1.133 jakllsch uint32_t hcs1, hcs2, hcs3, dboff, rtsoff;
961 1.40 skrll uint32_t pagesize, config;
962 1.40 skrll int i = 0;
963 1.1 jakllsch uint16_t hciversion;
964 1.1 jakllsch uint8_t caplength;
965 1.1 jakllsch
966 1.27 skrll XHCIHIST_FUNC(); XHCIHIST_CALLED();
967 1.1 jakllsch
968 1.68 skrll /* Set up the bus struct for the usb 3 and usb 2 buses */
969 1.68 skrll sc->sc_bus.ub_methods = &xhci_bus_methods;
970 1.68 skrll sc->sc_bus.ub_pipesize = sizeof(struct xhci_pipe);
971 1.34 skrll sc->sc_bus.ub_usedma = true;
972 1.68 skrll sc->sc_bus.ub_hcpriv = sc;
973 1.68 skrll
974 1.68 skrll sc->sc_bus2.ub_methods = &xhci_bus_methods;
975 1.68 skrll sc->sc_bus2.ub_pipesize = sizeof(struct xhci_pipe);
976 1.68 skrll sc->sc_bus2.ub_revision = USBREV_2_0;
977 1.68 skrll sc->sc_bus2.ub_usedma = true;
978 1.68 skrll sc->sc_bus2.ub_hcpriv = sc;
979 1.68 skrll sc->sc_bus2.ub_dmatag = sc->sc_bus.ub_dmatag;
980 1.1 jakllsch
981 1.130 skrll caplength = xhci_read_1(sc, XHCI_CAPLENGTH);
982 1.130 skrll hciversion = xhci_read_2(sc, XHCI_HCIVERSION);
983 1.1 jakllsch
984 1.34 skrll if (hciversion < XHCI_HCIVERSION_0_96 ||
985 1.97 jakllsch hciversion >= 0x0200) {
986 1.1 jakllsch aprint_normal_dev(sc->sc_dev,
987 1.1 jakllsch "xHCI version %x.%x not known to be supported\n",
988 1.1 jakllsch (hciversion >> 8) & 0xff, (hciversion >> 0) & 0xff);
989 1.1 jakllsch } else {
990 1.1 jakllsch aprint_verbose_dev(sc->sc_dev, "xHCI version %x.%x\n",
991 1.1 jakllsch (hciversion >> 8) & 0xff, (hciversion >> 0) & 0xff);
992 1.1 jakllsch }
993 1.1 jakllsch
994 1.1 jakllsch if (bus_space_subregion(sc->sc_iot, sc->sc_ioh, 0, caplength,
995 1.1 jakllsch &sc->sc_cbh) != 0) {
996 1.1 jakllsch aprint_error_dev(sc->sc_dev, "capability subregion failure\n");
997 1.15 skrll return ENOMEM;
998 1.1 jakllsch }
999 1.1 jakllsch
1000 1.1 jakllsch hcs1 = xhci_cap_read_4(sc, XHCI_HCSPARAMS1);
1001 1.1 jakllsch sc->sc_maxslots = XHCI_HCS1_MAXSLOTS(hcs1);
1002 1.1 jakllsch sc->sc_maxintrs = XHCI_HCS1_MAXINTRS(hcs1);
1003 1.1 jakllsch sc->sc_maxports = XHCI_HCS1_MAXPORTS(hcs1);
1004 1.1 jakllsch hcs2 = xhci_cap_read_4(sc, XHCI_HCSPARAMS2);
1005 1.34 skrll hcs3 = xhci_cap_read_4(sc, XHCI_HCSPARAMS3);
1006 1.34 skrll aprint_debug_dev(sc->sc_dev,
1007 1.34 skrll "hcs1=%"PRIx32" hcs2=%"PRIx32" hcs3=%"PRIx32"\n", hcs1, hcs2, hcs3);
1008 1.34 skrll
1009 1.133 jakllsch sc->sc_hcc = xhci_cap_read_4(sc, XHCI_HCCPARAMS);
1010 1.133 jakllsch sc->sc_ctxsz = XHCI_HCC_CSZ(sc->sc_hcc) ? 64 : 32;
1011 1.1 jakllsch
1012 1.34 skrll char sbuf[128];
1013 1.34 skrll if (hciversion < XHCI_HCIVERSION_1_0)
1014 1.133 jakllsch snprintb(sbuf, sizeof(sbuf), XHCI_HCCPREV1_BITS, sc->sc_hcc);
1015 1.34 skrll else
1016 1.133 jakllsch snprintb(sbuf, sizeof(sbuf), XHCI_HCCV1_x_BITS, sc->sc_hcc);
1017 1.34 skrll aprint_debug_dev(sc->sc_dev, "hcc=%s\n", sbuf);
1018 1.131 skrll aprint_debug_dev(sc->sc_dev, "xECP %" __PRIxBITS "\n",
1019 1.133 jakllsch XHCI_HCC_XECP(sc->sc_hcc) * 4);
1020 1.95 msaitoh if (hciversion >= XHCI_HCIVERSION_1_1) {
1021 1.133 jakllsch sc->sc_hcc2 = xhci_cap_read_4(sc, XHCI_HCCPARAMS2);
1022 1.133 jakllsch snprintb(sbuf, sizeof(sbuf), XHCI_HCC2_BITS, sc->sc_hcc2);
1023 1.95 msaitoh aprint_debug_dev(sc->sc_dev, "hcc2=%s\n", sbuf);
1024 1.95 msaitoh }
1025 1.34 skrll
1026 1.68 skrll /* default all ports to bus 0, i.e. usb 3 */
1027 1.70 skrll sc->sc_ctlrportbus = kmem_zalloc(
1028 1.70 skrll howmany(sc->sc_maxports * sizeof(uint8_t), NBBY), KM_SLEEP);
1029 1.68 skrll sc->sc_ctlrportmap = kmem_zalloc(sc->sc_maxports * sizeof(int), KM_SLEEP);
1030 1.68 skrll
1031 1.68 skrll /* controller port to bus roothub port map */
1032 1.68 skrll for (size_t j = 0; j < __arraycount(sc->sc_rhportmap); j++) {
1033 1.68 skrll sc->sc_rhportmap[j] = kmem_zalloc(sc->sc_maxports * sizeof(int), KM_SLEEP);
1034 1.68 skrll }
1035 1.68 skrll
1036 1.68 skrll /*
1037 1.68 skrll * Process all Extended Capabilities
1038 1.68 skrll */
1039 1.133 jakllsch xhci_ecp(sc);
1040 1.1 jakllsch
1041 1.68 skrll bsz = XHCI_PORTSC(sc->sc_maxports);
1042 1.1 jakllsch if (bus_space_subregion(sc->sc_iot, sc->sc_ioh, caplength, bsz,
1043 1.1 jakllsch &sc->sc_obh) != 0) {
1044 1.1 jakllsch aprint_error_dev(sc->sc_dev, "operational subregion failure\n");
1045 1.15 skrll return ENOMEM;
1046 1.1 jakllsch }
1047 1.1 jakllsch
1048 1.1 jakllsch dboff = xhci_cap_read_4(sc, XHCI_DBOFF);
1049 1.1 jakllsch if (bus_space_subregion(sc->sc_iot, sc->sc_ioh, dboff,
1050 1.1 jakllsch sc->sc_maxslots * 4, &sc->sc_dbh) != 0) {
1051 1.1 jakllsch aprint_error_dev(sc->sc_dev, "doorbell subregion failure\n");
1052 1.15 skrll return ENOMEM;
1053 1.1 jakllsch }
1054 1.1 jakllsch
1055 1.1 jakllsch rtsoff = xhci_cap_read_4(sc, XHCI_RTSOFF);
1056 1.1 jakllsch if (bus_space_subregion(sc->sc_iot, sc->sc_ioh, rtsoff,
1057 1.1 jakllsch sc->sc_maxintrs * 0x20, &sc->sc_rbh) != 0) {
1058 1.1 jakllsch aprint_error_dev(sc->sc_dev, "runtime subregion failure\n");
1059 1.15 skrll return ENOMEM;
1060 1.1 jakllsch }
1061 1.1 jakllsch
1062 1.40 skrll int rv;
1063 1.40 skrll rv = xhci_hc_reset(sc);
1064 1.40 skrll if (rv != 0) {
1065 1.40 skrll return rv;
1066 1.37 skrll }
1067 1.1 jakllsch
1068 1.34 skrll if (sc->sc_vendor_init)
1069 1.34 skrll sc->sc_vendor_init(sc);
1070 1.34 skrll
1071 1.1 jakllsch pagesize = xhci_op_read_4(sc, XHCI_PAGESIZE);
1072 1.121 christos aprint_debug_dev(sc->sc_dev, "PAGESIZE 0x%08x\n", pagesize);
1073 1.1 jakllsch pagesize = ffs(pagesize);
1074 1.37 skrll if (pagesize == 0) {
1075 1.37 skrll aprint_error_dev(sc->sc_dev, "pagesize is 0\n");
1076 1.15 skrll return EIO;
1077 1.37 skrll }
1078 1.1 jakllsch sc->sc_pgsz = 1 << (12 + (pagesize - 1));
1079 1.121 christos aprint_debug_dev(sc->sc_dev, "sc_pgsz 0x%08x\n", (uint32_t)sc->sc_pgsz);
1080 1.121 christos aprint_debug_dev(sc->sc_dev, "sc_maxslots 0x%08x\n",
1081 1.1 jakllsch (uint32_t)sc->sc_maxslots);
1082 1.34 skrll aprint_debug_dev(sc->sc_dev, "sc_maxports %d\n", sc->sc_maxports);
1083 1.1 jakllsch
1084 1.5 matt usbd_status err;
1085 1.5 matt
1086 1.5 matt sc->sc_maxspbuf = XHCI_HCS2_MAXSPBUF(hcs2);
1087 1.12 jakllsch aprint_debug_dev(sc->sc_dev, "sc_maxspbuf %d\n", sc->sc_maxspbuf);
1088 1.5 matt if (sc->sc_maxspbuf != 0) {
1089 1.5 matt err = usb_allocmem(&sc->sc_bus,
1090 1.5 matt sizeof(uint64_t) * sc->sc_maxspbuf, sizeof(uint64_t),
1091 1.124 skrll USBMALLOC_COHERENT, &sc->sc_spbufarray_dma);
1092 1.37 skrll if (err) {
1093 1.37 skrll aprint_error_dev(sc->sc_dev,
1094 1.37 skrll "spbufarray init fail, err %d\n", err);
1095 1.37 skrll return ENOMEM;
1096 1.37 skrll }
1097 1.30 skrll
1098 1.36 skrll sc->sc_spbuf_dma = kmem_zalloc(sizeof(*sc->sc_spbuf_dma) *
1099 1.36 skrll sc->sc_maxspbuf, KM_SLEEP);
1100 1.5 matt uint64_t *spbufarray = KERNADDR(&sc->sc_spbufarray_dma, 0);
1101 1.5 matt for (i = 0; i < sc->sc_maxspbuf; i++) {
1102 1.5 matt usb_dma_t * const dma = &sc->sc_spbuf_dma[i];
1103 1.5 matt /* allocate contexts */
1104 1.5 matt err = usb_allocmem(&sc->sc_bus, sc->sc_pgsz,
1105 1.124 skrll sc->sc_pgsz, USBMALLOC_COHERENT, dma);
1106 1.37 skrll if (err) {
1107 1.37 skrll aprint_error_dev(sc->sc_dev,
1108 1.37 skrll "spbufarray_dma init fail, err %d\n", err);
1109 1.37 skrll rv = ENOMEM;
1110 1.37 skrll goto bad1;
1111 1.37 skrll }
1112 1.5 matt spbufarray[i] = htole64(DMAADDR(dma, 0));
1113 1.5 matt usb_syncmem(dma, 0, sc->sc_pgsz,
1114 1.5 matt BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE);
1115 1.5 matt }
1116 1.5 matt
1117 1.30 skrll usb_syncmem(&sc->sc_spbufarray_dma, 0,
1118 1.5 matt sizeof(uint64_t) * sc->sc_maxspbuf, BUS_DMASYNC_PREWRITE);
1119 1.5 matt }
1120 1.5 matt
1121 1.1 jakllsch config = xhci_op_read_4(sc, XHCI_CONFIG);
1122 1.1 jakllsch config &= ~0xFF;
1123 1.1 jakllsch config |= sc->sc_maxslots & 0xFF;
1124 1.1 jakllsch xhci_op_write_4(sc, XHCI_CONFIG, config);
1125 1.1 jakllsch
1126 1.1 jakllsch err = xhci_ring_init(sc, &sc->sc_cr, XHCI_COMMAND_RING_TRBS,
1127 1.1 jakllsch XHCI_COMMAND_RING_SEGMENTS_ALIGN);
1128 1.1 jakllsch if (err) {
1129 1.37 skrll aprint_error_dev(sc->sc_dev, "command ring init fail, err %d\n",
1130 1.37 skrll err);
1131 1.37 skrll rv = ENOMEM;
1132 1.37 skrll goto bad1;
1133 1.1 jakllsch }
1134 1.1 jakllsch
1135 1.1 jakllsch err = xhci_ring_init(sc, &sc->sc_er, XHCI_EVENT_RING_TRBS,
1136 1.1 jakllsch XHCI_EVENT_RING_SEGMENTS_ALIGN);
1137 1.1 jakllsch if (err) {
1138 1.37 skrll aprint_error_dev(sc->sc_dev, "event ring init fail, err %d\n",
1139 1.37 skrll err);
1140 1.37 skrll rv = ENOMEM;
1141 1.37 skrll goto bad2;
1142 1.1 jakllsch }
1143 1.1 jakllsch
1144 1.16 skrll usb_dma_t *dma;
1145 1.16 skrll size_t size;
1146 1.16 skrll size_t align;
1147 1.16 skrll
1148 1.16 skrll dma = &sc->sc_eventst_dma;
1149 1.16 skrll size = roundup2(XHCI_EVENT_RING_SEGMENTS * XHCI_ERSTE_SIZE,
1150 1.16 skrll XHCI_EVENT_RING_SEGMENT_TABLE_ALIGN);
1151 1.37 skrll KASSERTMSG(size <= (512 * 1024), "eventst size %zu too large", size);
1152 1.16 skrll align = XHCI_EVENT_RING_SEGMENT_TABLE_ALIGN;
1153 1.124 skrll err = usb_allocmem(&sc->sc_bus, size, align, USBMALLOC_COHERENT, dma);
1154 1.37 skrll if (err) {
1155 1.37 skrll aprint_error_dev(sc->sc_dev, "eventst init fail, err %d\n",
1156 1.37 skrll err);
1157 1.37 skrll rv = ENOMEM;
1158 1.37 skrll goto bad3;
1159 1.37 skrll }
1160 1.16 skrll
1161 1.16 skrll memset(KERNADDR(dma, 0), 0, size);
1162 1.16 skrll usb_syncmem(dma, 0, size, BUS_DMASYNC_PREWRITE);
1163 1.121 christos aprint_debug_dev(sc->sc_dev, "eventst: 0x%016jx %p %zx\n",
1164 1.16 skrll (uintmax_t)DMAADDR(&sc->sc_eventst_dma, 0),
1165 1.16 skrll KERNADDR(&sc->sc_eventst_dma, 0),
1166 1.34 skrll sc->sc_eventst_dma.udma_block->size);
1167 1.16 skrll
1168 1.16 skrll dma = &sc->sc_dcbaa_dma;
1169 1.16 skrll size = (1 + sc->sc_maxslots) * sizeof(uint64_t);
1170 1.37 skrll KASSERTMSG(size <= 2048, "dcbaa size %zu too large", size);
1171 1.16 skrll align = XHCI_DEVICE_CONTEXT_BASE_ADDRESS_ARRAY_ALIGN;
1172 1.124 skrll err = usb_allocmem(&sc->sc_bus, size, align, USBMALLOC_COHERENT, dma);
1173 1.37 skrll if (err) {
1174 1.37 skrll aprint_error_dev(sc->sc_dev, "dcbaa init fail, err %d\n", err);
1175 1.37 skrll rv = ENOMEM;
1176 1.37 skrll goto bad4;
1177 1.37 skrll }
1178 1.121 christos aprint_debug_dev(sc->sc_dev, "dcbaa: 0x%016jx %p %zx\n",
1179 1.37 skrll (uintmax_t)DMAADDR(&sc->sc_dcbaa_dma, 0),
1180 1.37 skrll KERNADDR(&sc->sc_dcbaa_dma, 0),
1181 1.37 skrll sc->sc_dcbaa_dma.udma_block->size);
1182 1.16 skrll
1183 1.16 skrll memset(KERNADDR(dma, 0), 0, size);
1184 1.16 skrll if (sc->sc_maxspbuf != 0) {
1185 1.16 skrll /*
1186 1.16 skrll * DCBA entry 0 hold the scratchbuf array pointer.
1187 1.16 skrll */
1188 1.16 skrll *(uint64_t *)KERNADDR(dma, 0) =
1189 1.16 skrll htole64(DMAADDR(&sc->sc_spbufarray_dma, 0));
1190 1.1 jakllsch }
1191 1.16 skrll usb_syncmem(dma, 0, size, BUS_DMASYNC_PREWRITE);
1192 1.1 jakllsch
1193 1.1 jakllsch sc->sc_slots = kmem_zalloc(sizeof(*sc->sc_slots) * sc->sc_maxslots,
1194 1.1 jakllsch KM_SLEEP);
1195 1.37 skrll if (sc->sc_slots == NULL) {
1196 1.37 skrll aprint_error_dev(sc->sc_dev, "slots init fail, err %d\n", err);
1197 1.37 skrll rv = ENOMEM;
1198 1.37 skrll goto bad;
1199 1.37 skrll }
1200 1.37 skrll
1201 1.37 skrll sc->sc_xferpool = pool_cache_init(sizeof(struct xhci_xfer), 0, 0, 0,
1202 1.37 skrll "xhcixfer", NULL, IPL_USB, NULL, NULL, NULL);
1203 1.37 skrll if (sc->sc_xferpool == NULL) {
1204 1.37 skrll aprint_error_dev(sc->sc_dev, "pool_cache init fail, err %d\n",
1205 1.37 skrll err);
1206 1.37 skrll rv = ENOMEM;
1207 1.37 skrll goto bad;
1208 1.37 skrll }
1209 1.1 jakllsch
1210 1.1 jakllsch cv_init(&sc->sc_command_cv, "xhcicmd");
1211 1.68 skrll cv_init(&sc->sc_cmdbusy_cv, "xhcicmdq");
1212 1.34 skrll mutex_init(&sc->sc_lock, MUTEX_DEFAULT, IPL_SOFTUSB);
1213 1.34 skrll mutex_init(&sc->sc_intr_lock, MUTEX_DEFAULT, IPL_USB);
1214 1.34 skrll
1215 1.1 jakllsch struct xhci_erste *erst;
1216 1.1 jakllsch erst = KERNADDR(&sc->sc_eventst_dma, 0);
1217 1.123 skrll erst[0].erste_0 = htole64(xhci_ring_trbp(sc->sc_er, 0));
1218 1.123 skrll erst[0].erste_2 = htole32(sc->sc_er->xr_ntrb);
1219 1.1 jakllsch erst[0].erste_3 = htole32(0);
1220 1.1 jakllsch usb_syncmem(&sc->sc_eventst_dma, 0,
1221 1.1 jakllsch XHCI_ERSTE_SIZE * XHCI_EVENT_RING_SEGMENTS, BUS_DMASYNC_PREWRITE);
1222 1.1 jakllsch
1223 1.1 jakllsch xhci_rt_write_4(sc, XHCI_ERSTSZ(0), XHCI_EVENT_RING_SEGMENTS);
1224 1.1 jakllsch xhci_rt_write_8(sc, XHCI_ERSTBA(0), DMAADDR(&sc->sc_eventst_dma, 0));
1225 1.123 skrll xhci_rt_write_8(sc, XHCI_ERDP(0), xhci_ring_trbp(sc->sc_er, 0) |
1226 1.132 skrll XHCI_ERDP_BUSY);
1227 1.103 skrll
1228 1.1 jakllsch xhci_op_write_8(sc, XHCI_DCBAAP, DMAADDR(&sc->sc_dcbaa_dma, 0));
1229 1.123 skrll xhci_op_write_8(sc, XHCI_CRCR, xhci_ring_trbp(sc->sc_cr, 0) |
1230 1.123 skrll sc->sc_cr->xr_cs);
1231 1.1 jakllsch
1232 1.103 skrll xhci_op_barrier(sc, 0, 4, BUS_SPACE_BARRIER_WRITE);
1233 1.103 skrll
1234 1.79 christos HEXDUMP("eventst", KERNADDR(&sc->sc_eventst_dma, 0),
1235 1.1 jakllsch XHCI_ERSTE_SIZE * XHCI_EVENT_RING_SEGMENTS);
1236 1.1 jakllsch
1237 1.74 jmcneill if ((sc->sc_quirks & XHCI_DEFERRED_START) == 0)
1238 1.74 jmcneill xhci_start(sc);
1239 1.1 jakllsch
1240 1.37 skrll return 0;
1241 1.37 skrll
1242 1.37 skrll bad:
1243 1.37 skrll if (sc->sc_xferpool) {
1244 1.37 skrll pool_cache_destroy(sc->sc_xferpool);
1245 1.37 skrll sc->sc_xferpool = NULL;
1246 1.37 skrll }
1247 1.37 skrll
1248 1.37 skrll if (sc->sc_slots) {
1249 1.37 skrll kmem_free(sc->sc_slots, sizeof(*sc->sc_slots) *
1250 1.37 skrll sc->sc_maxslots);
1251 1.37 skrll sc->sc_slots = NULL;
1252 1.37 skrll }
1253 1.37 skrll
1254 1.37 skrll usb_freemem(&sc->sc_bus, &sc->sc_dcbaa_dma);
1255 1.37 skrll bad4:
1256 1.37 skrll usb_freemem(&sc->sc_bus, &sc->sc_eventst_dma);
1257 1.37 skrll bad3:
1258 1.37 skrll xhci_ring_free(sc, &sc->sc_er);
1259 1.37 skrll bad2:
1260 1.37 skrll xhci_ring_free(sc, &sc->sc_cr);
1261 1.37 skrll i = sc->sc_maxspbuf;
1262 1.37 skrll bad1:
1263 1.37 skrll for (int j = 0; j < i; j++)
1264 1.37 skrll usb_freemem(&sc->sc_bus, &sc->sc_spbuf_dma[j]);
1265 1.37 skrll usb_freemem(&sc->sc_bus, &sc->sc_spbufarray_dma);
1266 1.37 skrll
1267 1.37 skrll return rv;
1268 1.1 jakllsch }
1269 1.1 jakllsch
1270 1.73 skrll static inline bool
1271 1.73 skrll xhci_polling_p(struct xhci_softc * const sc)
1272 1.73 skrll {
1273 1.73 skrll return sc->sc_bus.ub_usepolling || sc->sc_bus2.ub_usepolling;
1274 1.73 skrll }
1275 1.73 skrll
1276 1.1 jakllsch int
1277 1.1 jakllsch xhci_intr(void *v)
1278 1.1 jakllsch {
1279 1.1 jakllsch struct xhci_softc * const sc = v;
1280 1.25 skrll int ret = 0;
1281 1.1 jakllsch
1282 1.27 skrll XHCIHIST_FUNC(); XHCIHIST_CALLED();
1283 1.27 skrll
1284 1.25 skrll if (sc == NULL)
1285 1.1 jakllsch return 0;
1286 1.1 jakllsch
1287 1.25 skrll mutex_spin_enter(&sc->sc_intr_lock);
1288 1.25 skrll
1289 1.25 skrll if (sc->sc_dying || !device_has_power(sc->sc_dev))
1290 1.25 skrll goto done;
1291 1.25 skrll
1292 1.1 jakllsch /* If we get an interrupt while polling, then just ignore it. */
1293 1.73 skrll if (xhci_polling_p(sc)) {
1294 1.1 jakllsch #ifdef DIAGNOSTIC
1295 1.27 skrll DPRINTFN(16, "ignored interrupt while polling", 0, 0, 0, 0);
1296 1.1 jakllsch #endif
1297 1.25 skrll goto done;
1298 1.1 jakllsch }
1299 1.1 jakllsch
1300 1.25 skrll ret = xhci_intr1(sc);
1301 1.73 skrll if (ret) {
1302 1.89 jdolecek KASSERT(sc->sc_child || sc->sc_child2);
1303 1.89 jdolecek
1304 1.89 jdolecek /*
1305 1.89 jdolecek * One of child busses could be already detached. It doesn't
1306 1.89 jdolecek * matter on which of the two the softintr is scheduled.
1307 1.89 jdolecek */
1308 1.89 jdolecek if (sc->sc_child)
1309 1.89 jdolecek usb_schedsoftintr(&sc->sc_bus);
1310 1.89 jdolecek else
1311 1.89 jdolecek usb_schedsoftintr(&sc->sc_bus2);
1312 1.73 skrll }
1313 1.25 skrll done:
1314 1.25 skrll mutex_spin_exit(&sc->sc_intr_lock);
1315 1.25 skrll return ret;
1316 1.1 jakllsch }
1317 1.1 jakllsch
1318 1.1 jakllsch int
1319 1.1 jakllsch xhci_intr1(struct xhci_softc * const sc)
1320 1.1 jakllsch {
1321 1.1 jakllsch uint32_t usbsts;
1322 1.1 jakllsch uint32_t iman;
1323 1.1 jakllsch
1324 1.111 mrg XHCIHIST_FUNC();
1325 1.27 skrll
1326 1.1 jakllsch usbsts = xhci_op_read_4(sc, XHCI_USBSTS);
1327 1.121 christos XHCIHIST_CALLARGS("USBSTS 0x%08jx", usbsts, 0, 0, 0);
1328 1.90 jdolecek if ((usbsts & (XHCI_STS_HSE | XHCI_STS_EINT | XHCI_STS_PCD |
1329 1.90 jdolecek XHCI_STS_HCE)) == 0) {
1330 1.120 christos DPRINTFN(16, "ignored intr not for %jd",
1331 1.122 christos device_unit(sc->sc_dev), 0, 0, 0);
1332 1.1 jakllsch return 0;
1333 1.1 jakllsch }
1334 1.90 jdolecek
1335 1.90 jdolecek /*
1336 1.90 jdolecek * Clear EINT and other transient flags, to not misenterpret
1337 1.90 jdolecek * next shared interrupt. Also, to avoid race, EINT must be cleared
1338 1.90 jdolecek * before XHCI_IMAN_INTR_PEND is cleared.
1339 1.90 jdolecek */
1340 1.90 jdolecek xhci_op_write_4(sc, XHCI_USBSTS, usbsts & XHCI_STS_RSVDP0);
1341 1.90 jdolecek
1342 1.90 jdolecek #ifdef XHCI_DEBUG
1343 1.1 jakllsch usbsts = xhci_op_read_4(sc, XHCI_USBSTS);
1344 1.121 christos DPRINTFN(16, "USBSTS 0x%08jx", usbsts, 0, 0, 0);
1345 1.90 jdolecek #endif
1346 1.1 jakllsch
1347 1.1 jakllsch iman = xhci_rt_read_4(sc, XHCI_IMAN(0));
1348 1.121 christos DPRINTFN(16, "IMAN0 0x%08jx", iman, 0, 0, 0);
1349 1.34 skrll iman |= XHCI_IMAN_INTR_PEND;
1350 1.1 jakllsch xhci_rt_write_4(sc, XHCI_IMAN(0), iman);
1351 1.90 jdolecek
1352 1.90 jdolecek #ifdef XHCI_DEBUG
1353 1.1 jakllsch iman = xhci_rt_read_4(sc, XHCI_IMAN(0));
1354 1.121 christos DPRINTFN(16, "IMAN0 0x%08jx", iman, 0, 0, 0);
1355 1.1 jakllsch usbsts = xhci_op_read_4(sc, XHCI_USBSTS);
1356 1.121 christos DPRINTFN(16, "USBSTS 0x%08jx", usbsts, 0, 0, 0);
1357 1.90 jdolecek #endif
1358 1.1 jakllsch
1359 1.1 jakllsch return 1;
1360 1.1 jakllsch }
1361 1.1 jakllsch
1362 1.34 skrll /*
1363 1.34 skrll * 3 port speed types used in USB stack
1364 1.34 skrll *
1365 1.34 skrll * usbdi speed
1366 1.34 skrll * definition: USB_SPEED_* in usb.h
1367 1.34 skrll * They are used in struct usbd_device in USB stack.
1368 1.34 skrll * ioctl interface uses these values too.
1369 1.34 skrll * port_status speed
1370 1.34 skrll * definition: UPS_*_SPEED in usb.h
1371 1.34 skrll * They are used in usb_port_status_t and valid only for USB 2.0.
1372 1.34 skrll * Speed value is always 0 for Super Speed or more, and dwExtPortStatus
1373 1.34 skrll * of usb_port_status_ext_t indicates port speed.
1374 1.34 skrll * Note that some 3.0 values overlap with 2.0 values.
1375 1.34 skrll * (e.g. 0x200 means UPS_POER_POWER_SS in SS and
1376 1.34 skrll * means UPS_LOW_SPEED in HS.)
1377 1.34 skrll * port status returned from hub also uses these values.
1378 1.34 skrll * On NetBSD UPS_OTHER_SPEED indicates port speed is super speed
1379 1.34 skrll * or more.
1380 1.34 skrll * xspeed:
1381 1.34 skrll * definition: Protocol Speed ID (PSI) (xHCI 1.1 7.2.1)
1382 1.34 skrll * They are used in only slot context and PORTSC reg of xhci.
1383 1.34 skrll * The difference between usbdi speed and xspeed is
1384 1.34 skrll * that FS and LS values are swapped.
1385 1.34 skrll */
1386 1.34 skrll
1387 1.34 skrll /* convert usbdi speed to xspeed */
1388 1.34 skrll static int
1389 1.34 skrll xhci_speed2xspeed(int speed)
1390 1.34 skrll {
1391 1.34 skrll switch (speed) {
1392 1.34 skrll case USB_SPEED_LOW: return 2;
1393 1.34 skrll case USB_SPEED_FULL: return 1;
1394 1.34 skrll default: return speed;
1395 1.34 skrll }
1396 1.34 skrll }
1397 1.34 skrll
1398 1.34 skrll #if 0
1399 1.34 skrll /* convert xspeed to usbdi speed */
1400 1.34 skrll static int
1401 1.34 skrll xhci_xspeed2speed(int xspeed)
1402 1.34 skrll {
1403 1.34 skrll switch (xspeed) {
1404 1.34 skrll case 1: return USB_SPEED_FULL;
1405 1.34 skrll case 2: return USB_SPEED_LOW;
1406 1.34 skrll default: return xspeed;
1407 1.34 skrll }
1408 1.34 skrll }
1409 1.34 skrll #endif
1410 1.34 skrll
1411 1.34 skrll /* convert xspeed to port status speed */
1412 1.34 skrll static int
1413 1.34 skrll xhci_xspeed2psspeed(int xspeed)
1414 1.34 skrll {
1415 1.34 skrll switch (xspeed) {
1416 1.34 skrll case 0: return 0;
1417 1.34 skrll case 1: return UPS_FULL_SPEED;
1418 1.34 skrll case 2: return UPS_LOW_SPEED;
1419 1.34 skrll case 3: return UPS_HIGH_SPEED;
1420 1.34 skrll default: return UPS_OTHER_SPEED;
1421 1.34 skrll }
1422 1.34 skrll }
1423 1.34 skrll
1424 1.34 skrll /*
1425 1.54 skrll * Construct input contexts and issue TRB to open pipe.
1426 1.34 skrll */
1427 1.1 jakllsch static usbd_status
1428 1.34 skrll xhci_configure_endpoint(struct usbd_pipe *pipe)
1429 1.1 jakllsch {
1430 1.34 skrll struct xhci_softc * const sc = XHCI_PIPE2SC(pipe);
1431 1.34 skrll struct xhci_slot * const xs = pipe->up_dev->ud_hcpriv;
1432 1.81 hannken #ifdef USB_DEBUG
1433 1.34 skrll const u_int dci = xhci_ep_get_dci(pipe->up_endpoint->ue_edesc);
1434 1.79 christos #endif
1435 1.101 jakllsch struct xhci_soft_trb trb;
1436 1.1 jakllsch usbd_status err;
1437 1.1 jakllsch
1438 1.111 mrg XHCIHIST_FUNC();
1439 1.121 christos XHCIHIST_CALLARGS("slot %ju dci %ju epaddr 0x%02jx attr 0x%02jx",
1440 1.34 skrll xs->xs_idx, dci, pipe->up_endpoint->ue_edesc->bEndpointAddress,
1441 1.34 skrll pipe->up_endpoint->ue_edesc->bmAttributes);
1442 1.1 jakllsch
1443 1.1 jakllsch /* XXX ensure input context is available? */
1444 1.1 jakllsch
1445 1.1 jakllsch memset(xhci_slot_get_icv(sc, xs, 0), 0, sc->sc_pgsz);
1446 1.1 jakllsch
1447 1.51 skrll /* set up context */
1448 1.51 skrll xhci_setup_ctx(pipe);
1449 1.1 jakllsch
1450 1.79 christos HEXDUMP("input control context", xhci_slot_get_icv(sc, xs, 0),
1451 1.1 jakllsch sc->sc_ctxsz * 1);
1452 1.79 christos HEXDUMP("input endpoint context", xhci_slot_get_icv(sc, xs,
1453 1.1 jakllsch xhci_dci_to_ici(dci)), sc->sc_ctxsz * 1);
1454 1.1 jakllsch
1455 1.1 jakllsch trb.trb_0 = xhci_slot_get_icp(sc, xs, 0);
1456 1.1 jakllsch trb.trb_2 = 0;
1457 1.1 jakllsch trb.trb_3 = XHCI_TRB_3_SLOT_SET(xs->xs_idx) |
1458 1.1 jakllsch XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_CONFIGURE_EP);
1459 1.1 jakllsch
1460 1.1 jakllsch err = xhci_do_command(sc, &trb, USBD_DEFAULT_TIMEOUT);
1461 1.1 jakllsch
1462 1.1 jakllsch usb_syncmem(&xs->xs_dc_dma, 0, sc->sc_pgsz, BUS_DMASYNC_POSTREAD);
1463 1.79 christos HEXDUMP("output context", xhci_slot_get_dcv(sc, xs, dci),
1464 1.1 jakllsch sc->sc_ctxsz * 1);
1465 1.1 jakllsch
1466 1.1 jakllsch return err;
1467 1.1 jakllsch }
1468 1.1 jakllsch
1469 1.34 skrll #if 0
1470 1.1 jakllsch static usbd_status
1471 1.34 skrll xhci_unconfigure_endpoint(struct usbd_pipe *pipe)
1472 1.1 jakllsch {
1473 1.27 skrll #ifdef USB_DEBUG
1474 1.34 skrll struct xhci_slot * const xs = pipe->up_dev->ud_hcpriv;
1475 1.27 skrll #endif
1476 1.27 skrll
1477 1.111 mrg XHCIHIST_FUNC();
1478 1.111 mrg XHCIHIST_CALLARGS("slot %ju", xs->xs_idx, 0, 0, 0);
1479 1.27 skrll
1480 1.1 jakllsch return USBD_NORMAL_COMPLETION;
1481 1.1 jakllsch }
1482 1.34 skrll #endif
1483 1.1 jakllsch
1484 1.34 skrll /* 4.6.8, 6.4.3.7 */
1485 1.1 jakllsch static usbd_status
1486 1.63 skrll xhci_reset_endpoint_locked(struct usbd_pipe *pipe)
1487 1.1 jakllsch {
1488 1.34 skrll struct xhci_softc * const sc = XHCI_PIPE2SC(pipe);
1489 1.34 skrll struct xhci_slot * const xs = pipe->up_dev->ud_hcpriv;
1490 1.34 skrll const u_int dci = xhci_ep_get_dci(pipe->up_endpoint->ue_edesc);
1491 1.101 jakllsch struct xhci_soft_trb trb;
1492 1.1 jakllsch usbd_status err;
1493 1.1 jakllsch
1494 1.111 mrg XHCIHIST_FUNC();
1495 1.111 mrg XHCIHIST_CALLARGS("slot %ju dci %ju", xs->xs_idx, dci, 0, 0);
1496 1.34 skrll
1497 1.63 skrll KASSERT(mutex_owned(&sc->sc_lock));
1498 1.63 skrll
1499 1.1 jakllsch trb.trb_0 = 0;
1500 1.1 jakllsch trb.trb_2 = 0;
1501 1.1 jakllsch trb.trb_3 = XHCI_TRB_3_SLOT_SET(xs->xs_idx) |
1502 1.1 jakllsch XHCI_TRB_3_EP_SET(dci) |
1503 1.1 jakllsch XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_RESET_EP);
1504 1.1 jakllsch
1505 1.63 skrll err = xhci_do_command_locked(sc, &trb, USBD_DEFAULT_TIMEOUT);
1506 1.1 jakllsch
1507 1.1 jakllsch return err;
1508 1.1 jakllsch }
1509 1.1 jakllsch
1510 1.63 skrll static usbd_status
1511 1.63 skrll xhci_reset_endpoint(struct usbd_pipe *pipe)
1512 1.63 skrll {
1513 1.63 skrll struct xhci_softc * const sc = XHCI_PIPE2SC(pipe);
1514 1.63 skrll
1515 1.63 skrll mutex_enter(&sc->sc_lock);
1516 1.63 skrll usbd_status ret = xhci_reset_endpoint_locked(pipe);
1517 1.63 skrll mutex_exit(&sc->sc_lock);
1518 1.63 skrll
1519 1.63 skrll return ret;
1520 1.63 skrll }
1521 1.63 skrll
1522 1.34 skrll /*
1523 1.34 skrll * 4.6.9, 6.4.3.8
1524 1.34 skrll * Stop execution of TDs on xfer ring.
1525 1.34 skrll * Should be called with sc_lock held.
1526 1.34 skrll */
1527 1.1 jakllsch static usbd_status
1528 1.34 skrll xhci_stop_endpoint(struct usbd_pipe *pipe)
1529 1.1 jakllsch {
1530 1.34 skrll struct xhci_softc * const sc = XHCI_PIPE2SC(pipe);
1531 1.34 skrll struct xhci_slot * const xs = pipe->up_dev->ud_hcpriv;
1532 1.101 jakllsch struct xhci_soft_trb trb;
1533 1.1 jakllsch usbd_status err;
1534 1.34 skrll const u_int dci = xhci_ep_get_dci(pipe->up_endpoint->ue_edesc);
1535 1.1 jakllsch
1536 1.111 mrg XHCIHIST_FUNC();
1537 1.111 mrg XHCIHIST_CALLARGS("slot %ju dci %ju", xs->xs_idx, dci, 0, 0);
1538 1.34 skrll
1539 1.34 skrll KASSERT(mutex_owned(&sc->sc_lock));
1540 1.1 jakllsch
1541 1.1 jakllsch trb.trb_0 = 0;
1542 1.1 jakllsch trb.trb_2 = 0;
1543 1.1 jakllsch trb.trb_3 = XHCI_TRB_3_SLOT_SET(xs->xs_idx) |
1544 1.1 jakllsch XHCI_TRB_3_EP_SET(dci) |
1545 1.1 jakllsch XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_STOP_EP);
1546 1.1 jakllsch
1547 1.34 skrll err = xhci_do_command_locked(sc, &trb, USBD_DEFAULT_TIMEOUT);
1548 1.1 jakllsch
1549 1.1 jakllsch return err;
1550 1.1 jakllsch }
1551 1.1 jakllsch
1552 1.34 skrll /*
1553 1.34 skrll * Set TR Dequeue Pointer.
1554 1.54 skrll * xHCI 1.1 4.6.10 6.4.3.9
1555 1.54 skrll * Purge all of the TRBs on ring and reinitialize ring.
1556 1.54 skrll * Set TR dequeue Pointr to 0 and Cycle State to 1.
1557 1.54 skrll * EPSTATE of endpoint must be ERROR or STOPPED, otherwise CONTEXT_STATE
1558 1.54 skrll * error will be generated.
1559 1.34 skrll */
1560 1.1 jakllsch static usbd_status
1561 1.63 skrll xhci_set_dequeue_locked(struct usbd_pipe *pipe)
1562 1.1 jakllsch {
1563 1.34 skrll struct xhci_softc * const sc = XHCI_PIPE2SC(pipe);
1564 1.34 skrll struct xhci_slot * const xs = pipe->up_dev->ud_hcpriv;
1565 1.34 skrll const u_int dci = xhci_ep_get_dci(pipe->up_endpoint->ue_edesc);
1566 1.123 skrll struct xhci_ring * const xr = xs->xs_xr[dci];
1567 1.101 jakllsch struct xhci_soft_trb trb;
1568 1.1 jakllsch usbd_status err;
1569 1.1 jakllsch
1570 1.111 mrg XHCIHIST_FUNC();
1571 1.111 mrg XHCIHIST_CALLARGS("slot %ju dci %ju", xs->xs_idx, dci, 0, 0);
1572 1.1 jakllsch
1573 1.63 skrll KASSERT(mutex_owned(&sc->sc_lock));
1574 1.123 skrll KASSERT(xr != NULL);
1575 1.63 skrll
1576 1.56 skrll xhci_host_dequeue(xr);
1577 1.1 jakllsch
1578 1.34 skrll /* set DCS */
1579 1.1 jakllsch trb.trb_0 = xhci_ring_trbp(xr, 0) | 1; /* XXX */
1580 1.1 jakllsch trb.trb_2 = 0;
1581 1.1 jakllsch trb.trb_3 = XHCI_TRB_3_SLOT_SET(xs->xs_idx) |
1582 1.1 jakllsch XHCI_TRB_3_EP_SET(dci) |
1583 1.1 jakllsch XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_SET_TR_DEQUEUE);
1584 1.1 jakllsch
1585 1.63 skrll err = xhci_do_command_locked(sc, &trb, USBD_DEFAULT_TIMEOUT);
1586 1.1 jakllsch
1587 1.1 jakllsch return err;
1588 1.1 jakllsch }
1589 1.1 jakllsch
1590 1.63 skrll static usbd_status
1591 1.63 skrll xhci_set_dequeue(struct usbd_pipe *pipe)
1592 1.63 skrll {
1593 1.63 skrll struct xhci_softc * const sc = XHCI_PIPE2SC(pipe);
1594 1.63 skrll
1595 1.63 skrll mutex_enter(&sc->sc_lock);
1596 1.63 skrll usbd_status ret = xhci_set_dequeue_locked(pipe);
1597 1.63 skrll mutex_exit(&sc->sc_lock);
1598 1.63 skrll
1599 1.63 skrll return ret;
1600 1.63 skrll }
1601 1.63 skrll
1602 1.34 skrll /*
1603 1.34 skrll * Open new pipe: called from usbd_setup_pipe_flags.
1604 1.34 skrll * Fills methods of pipe.
1605 1.34 skrll * If pipe is not for ep0, calls configure_endpoint.
1606 1.34 skrll */
1607 1.1 jakllsch static usbd_status
1608 1.34 skrll xhci_open(struct usbd_pipe *pipe)
1609 1.1 jakllsch {
1610 1.34 skrll struct usbd_device * const dev = pipe->up_dev;
1611 1.134 jakllsch struct xhci_pipe * const xpipe = (struct xhci_pipe *)pipe;
1612 1.34 skrll struct xhci_softc * const sc = XHCI_BUS2SC(dev->ud_bus);
1613 1.123 skrll struct xhci_slot * const xs = pipe->up_dev->ud_hcpriv;
1614 1.34 skrll usb_endpoint_descriptor_t * const ed = pipe->up_endpoint->ue_edesc;
1615 1.123 skrll const u_int dci = xhci_ep_get_dci(ed);
1616 1.1 jakllsch const uint8_t xfertype = UE_GET_XFERTYPE(ed->bmAttributes);
1617 1.123 skrll usbd_status err;
1618 1.1 jakllsch
1619 1.111 mrg XHCIHIST_FUNC();
1620 1.111 mrg XHCIHIST_CALLARGS("addr %jd depth %jd port %jd speed %jd", dev->ud_addr,
1621 1.53 skrll dev->ud_depth, dev->ud_powersrc->up_portno, dev->ud_speed);
1622 1.121 christos DPRINTFN(1, " dci %ju type 0x%02jx epaddr 0x%02jx attr 0x%02jx",
1623 1.53 skrll xhci_ep_get_dci(ed), ed->bDescriptorType, ed->bEndpointAddress,
1624 1.53 skrll ed->bmAttributes);
1625 1.75 pgoyette DPRINTFN(1, " mps %ju ival %ju", UGETW(ed->wMaxPacketSize),
1626 1.75 pgoyette ed->bInterval, 0, 0);
1627 1.1 jakllsch
1628 1.1 jakllsch if (sc->sc_dying)
1629 1.1 jakllsch return USBD_IOERROR;
1630 1.1 jakllsch
1631 1.1 jakllsch /* Root Hub */
1632 1.34 skrll if (dev->ud_depth == 0 && dev->ud_powersrc->up_portno == 0) {
1633 1.1 jakllsch switch (ed->bEndpointAddress) {
1634 1.1 jakllsch case USB_CONTROL_ENDPOINT:
1635 1.34 skrll pipe->up_methods = &roothub_ctrl_methods;
1636 1.1 jakllsch break;
1637 1.34 skrll case UE_DIR_IN | USBROOTHUB_INTR_ENDPT:
1638 1.34 skrll pipe->up_methods = &xhci_root_intr_methods;
1639 1.1 jakllsch break;
1640 1.1 jakllsch default:
1641 1.34 skrll pipe->up_methods = NULL;
1642 1.121 christos DPRINTFN(0, "bad bEndpointAddress 0x%02jx",
1643 1.27 skrll ed->bEndpointAddress, 0, 0, 0);
1644 1.1 jakllsch return USBD_INVAL;
1645 1.1 jakllsch }
1646 1.1 jakllsch return USBD_NORMAL_COMPLETION;
1647 1.1 jakllsch }
1648 1.1 jakllsch
1649 1.1 jakllsch switch (xfertype) {
1650 1.1 jakllsch case UE_CONTROL:
1651 1.34 skrll pipe->up_methods = &xhci_device_ctrl_methods;
1652 1.1 jakllsch break;
1653 1.1 jakllsch case UE_ISOCHRONOUS:
1654 1.34 skrll pipe->up_methods = &xhci_device_isoc_methods;
1655 1.134 jakllsch pipe->up_serialise = false;
1656 1.134 jakllsch xpipe->xp_isoc_next = -1;
1657 1.1 jakllsch break;
1658 1.1 jakllsch case UE_BULK:
1659 1.34 skrll pipe->up_methods = &xhci_device_bulk_methods;
1660 1.1 jakllsch break;
1661 1.1 jakllsch case UE_INTERRUPT:
1662 1.34 skrll pipe->up_methods = &xhci_device_intr_methods;
1663 1.1 jakllsch break;
1664 1.1 jakllsch default:
1665 1.1 jakllsch return USBD_IOERROR;
1666 1.1 jakllsch break;
1667 1.1 jakllsch }
1668 1.1 jakllsch
1669 1.123 skrll KASSERT(xs != NULL);
1670 1.123 skrll KASSERT(xs->xs_xr[dci] == NULL);
1671 1.123 skrll
1672 1.123 skrll /* allocate transfer ring */
1673 1.123 skrll err = xhci_ring_init(sc, &xs->xs_xr[dci], XHCI_TRANSFER_RING_TRBS,
1674 1.123 skrll XHCI_TRB_ALIGN);
1675 1.123 skrll if (err) {
1676 1.123 skrll DPRINTFN(1, "ring alloc failed %jd", err, 0, 0, 0);
1677 1.123 skrll return err;
1678 1.123 skrll }
1679 1.123 skrll
1680 1.1 jakllsch if (ed->bEndpointAddress != USB_CONTROL_ENDPOINT)
1681 1.34 skrll return xhci_configure_endpoint(pipe);
1682 1.1 jakllsch
1683 1.1 jakllsch return USBD_NORMAL_COMPLETION;
1684 1.1 jakllsch }
1685 1.1 jakllsch
1686 1.34 skrll /*
1687 1.34 skrll * Closes pipe, called from usbd_kill_pipe via close methods.
1688 1.34 skrll * If the endpoint to be closed is ep0, disable_slot.
1689 1.34 skrll * Should be called with sc_lock held.
1690 1.34 skrll */
1691 1.1 jakllsch static void
1692 1.34 skrll xhci_close_pipe(struct usbd_pipe *pipe)
1693 1.1 jakllsch {
1694 1.34 skrll struct xhci_softc * const sc = XHCI_PIPE2SC(pipe);
1695 1.34 skrll struct xhci_slot * const xs = pipe->up_dev->ud_hcpriv;
1696 1.34 skrll usb_endpoint_descriptor_t * const ed = pipe->up_endpoint->ue_edesc;
1697 1.34 skrll const u_int dci = xhci_ep_get_dci(ed);
1698 1.101 jakllsch struct xhci_soft_trb trb;
1699 1.34 skrll uint32_t *cp;
1700 1.1 jakllsch
1701 1.111 mrg XHCIHIST_FUNC();
1702 1.1 jakllsch
1703 1.34 skrll if (sc->sc_dying)
1704 1.1 jakllsch return;
1705 1.1 jakllsch
1706 1.41 skrll /* xs is uninitialized before xhci_init_slot */
1707 1.34 skrll if (xs == NULL || xs->xs_idx == 0)
1708 1.1 jakllsch return;
1709 1.1 jakllsch
1710 1.111 mrg XHCIHIST_CALLARGS("pipe %#jx slot %ju dci %ju",
1711 1.111 mrg (uintptr_t)pipe, xs->xs_idx, dci, 0);
1712 1.1 jakllsch
1713 1.34 skrll KASSERTMSG(!cpu_intr_p() && !cpu_softintr_p(), "called from intr ctx");
1714 1.34 skrll KASSERT(mutex_owned(&sc->sc_lock));
1715 1.1 jakllsch
1716 1.34 skrll if (pipe->up_dev->ud_depth == 0)
1717 1.34 skrll return;
1718 1.1 jakllsch
1719 1.34 skrll if (dci == XHCI_DCI_EP_CONTROL) {
1720 1.34 skrll DPRINTFN(4, "closing ep0", 0, 0, 0, 0);
1721 1.123 skrll /* This frees all rings */
1722 1.34 skrll xhci_disable_slot(sc, xs->xs_idx);
1723 1.34 skrll return;
1724 1.34 skrll }
1725 1.1 jakllsch
1726 1.66 skrll if (xhci_get_epstate(sc, xs, dci) != XHCI_EPSTATE_STOPPED)
1727 1.66 skrll (void)xhci_stop_endpoint(pipe);
1728 1.1 jakllsch
1729 1.34 skrll /*
1730 1.34 skrll * set appropriate bit to be dropped.
1731 1.34 skrll * don't set DC bit to 1, otherwise all endpoints
1732 1.34 skrll * would be deconfigured.
1733 1.34 skrll */
1734 1.34 skrll cp = xhci_slot_get_icv(sc, xs, XHCI_ICI_INPUT_CONTROL);
1735 1.34 skrll cp[0] = htole32(XHCI_INCTX_0_DROP_MASK(dci));
1736 1.34 skrll cp[1] = htole32(0);
1737 1.1 jakllsch
1738 1.34 skrll /* XXX should be most significant one, not dci? */
1739 1.34 skrll cp = xhci_slot_get_icv(sc, xs, xhci_dci_to_ici(XHCI_DCI_SLOT));
1740 1.34 skrll cp[0] = htole32(XHCI_SCTX_0_CTX_NUM_SET(dci));
1741 1.1 jakllsch
1742 1.55 skrll /* configure ep context performs an implicit dequeue */
1743 1.123 skrll xhci_host_dequeue(xs->xs_xr[dci]);
1744 1.55 skrll
1745 1.34 skrll /* sync input contexts before they are read from memory */
1746 1.34 skrll usb_syncmem(&xs->xs_ic_dma, 0, sc->sc_pgsz, BUS_DMASYNC_PREWRITE);
1747 1.1 jakllsch
1748 1.34 skrll trb.trb_0 = xhci_slot_get_icp(sc, xs, 0);
1749 1.34 skrll trb.trb_2 = 0;
1750 1.34 skrll trb.trb_3 = XHCI_TRB_3_SLOT_SET(xs->xs_idx) |
1751 1.34 skrll XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_CONFIGURE_EP);
1752 1.1 jakllsch
1753 1.34 skrll (void)xhci_do_command_locked(sc, &trb, USBD_DEFAULT_TIMEOUT);
1754 1.34 skrll usb_syncmem(&xs->xs_dc_dma, 0, sc->sc_pgsz, BUS_DMASYNC_POSTREAD);
1755 1.123 skrll
1756 1.123 skrll xhci_ring_free(sc, &xs->xs_xr[dci]);
1757 1.34 skrll }
1758 1.1 jakllsch
1759 1.34 skrll /*
1760 1.34 skrll * Abort transfer.
1761 1.63 skrll * Should be called with sc_lock held.
1762 1.34 skrll */
1763 1.34 skrll static void
1764 1.116 riastrad xhci_abortx(struct usbd_xfer *xfer)
1765 1.34 skrll {
1766 1.111 mrg XHCIHIST_FUNC();
1767 1.34 skrll struct xhci_softc * const sc = XHCI_XFER2SC(xfer);
1768 1.63 skrll struct xhci_slot * const xs = xfer->ux_pipe->up_dev->ud_hcpriv;
1769 1.63 skrll const u_int dci = xhci_ep_get_dci(xfer->ux_pipe->up_endpoint->ue_edesc);
1770 1.1 jakllsch
1771 1.116 riastrad XHCIHIST_CALLARGS("xfer %#jx pipe %#jx",
1772 1.116 riastrad (uintptr_t)xfer, (uintptr_t)xfer->ux_pipe, 0, 0);
1773 1.1 jakllsch
1774 1.34 skrll KASSERT(mutex_owned(&sc->sc_lock));
1775 1.96 mrg ASSERT_SLEEPABLE();
1776 1.1 jakllsch
1777 1.116 riastrad KASSERTMSG((xfer->ux_status == USBD_CANCELLED ||
1778 1.116 riastrad xfer->ux_status == USBD_TIMEOUT),
1779 1.116 riastrad "bad abort status: %d", xfer->ux_status);
1780 1.63 skrll
1781 1.63 skrll /*
1782 1.96 mrg * If we're dying, skip the hardware action and just notify the
1783 1.96 mrg * software that we're done.
1784 1.63 skrll */
1785 1.96 mrg if (sc->sc_dying) {
1786 1.96 mrg DPRINTFN(4, "xfer %#jx dying %ju", (uintptr_t)xfer,
1787 1.96 mrg xfer->ux_status, 0, 0);
1788 1.96 mrg goto dying;
1789 1.96 mrg }
1790 1.63 skrll
1791 1.63 skrll /*
1792 1.96 mrg * HC Step 1: Stop execution of TD on the ring.
1793 1.63 skrll */
1794 1.63 skrll switch (xhci_get_epstate(sc, xs, dci)) {
1795 1.63 skrll case XHCI_EPSTATE_HALTED:
1796 1.63 skrll (void)xhci_reset_endpoint_locked(xfer->ux_pipe);
1797 1.63 skrll break;
1798 1.63 skrll case XHCI_EPSTATE_STOPPED:
1799 1.63 skrll break;
1800 1.63 skrll default:
1801 1.63 skrll (void)xhci_stop_endpoint(xfer->ux_pipe);
1802 1.63 skrll break;
1803 1.63 skrll }
1804 1.63 skrll #ifdef DIAGNOSTIC
1805 1.63 skrll uint32_t epst = xhci_get_epstate(sc, xs, dci);
1806 1.63 skrll if (epst != XHCI_EPSTATE_STOPPED)
1807 1.75 pgoyette DPRINTFN(4, "dci %ju not stopped %ju", dci, epst, 0, 0);
1808 1.63 skrll #endif
1809 1.63 skrll
1810 1.63 skrll /*
1811 1.96 mrg * HC Step 2: Remove any vestiges of the xfer from the ring.
1812 1.63 skrll */
1813 1.63 skrll xhci_set_dequeue_locked(xfer->ux_pipe);
1814 1.63 skrll
1815 1.63 skrll /*
1816 1.96 mrg * Final Step: Notify completion to waiting xfers.
1817 1.63 skrll */
1818 1.96 mrg dying:
1819 1.34 skrll usb_transfer_complete(xfer);
1820 1.34 skrll DPRINTFN(14, "end", 0, 0, 0, 0);
1821 1.34 skrll
1822 1.34 skrll KASSERT(mutex_owned(&sc->sc_lock));
1823 1.1 jakllsch }
1824 1.1 jakllsch
1825 1.55 skrll static void
1826 1.55 skrll xhci_host_dequeue(struct xhci_ring * const xr)
1827 1.55 skrll {
1828 1.55 skrll /* When dequeueing the controller, update our struct copy too */
1829 1.55 skrll memset(xr->xr_trb, 0, xr->xr_ntrb * XHCI_TRB_SIZE);
1830 1.55 skrll usb_syncmem(&xr->xr_dma, 0, xr->xr_ntrb * XHCI_TRB_SIZE,
1831 1.55 skrll BUS_DMASYNC_PREWRITE);
1832 1.55 skrll memset(xr->xr_cookies, 0, xr->xr_ntrb * sizeof(*xr->xr_cookies));
1833 1.55 skrll
1834 1.55 skrll xr->xr_ep = 0;
1835 1.55 skrll xr->xr_cs = 1;
1836 1.55 skrll }
1837 1.55 skrll
1838 1.34 skrll /*
1839 1.34 skrll * Recover STALLed endpoint.
1840 1.34 skrll * xHCI 1.1 sect 4.10.2.1
1841 1.34 skrll * Issue RESET_EP to recover halt condition and SET_TR_DEQUEUE to remove
1842 1.34 skrll * all transfers on transfer ring.
1843 1.34 skrll * These are done in thread context asynchronously.
1844 1.34 skrll */
1845 1.1 jakllsch static void
1846 1.34 skrll xhci_clear_endpoint_stall_async_task(void *cookie)
1847 1.1 jakllsch {
1848 1.34 skrll struct usbd_xfer * const xfer = cookie;
1849 1.34 skrll struct xhci_softc * const sc = XHCI_XFER2SC(xfer);
1850 1.34 skrll struct xhci_slot * const xs = xfer->ux_pipe->up_dev->ud_hcpriv;
1851 1.34 skrll const u_int dci = xhci_ep_get_dci(xfer->ux_pipe->up_endpoint->ue_edesc);
1852 1.123 skrll struct xhci_ring * const tr = xs->xs_xr[dci];
1853 1.1 jakllsch
1854 1.111 mrg XHCIHIST_FUNC();
1855 1.111 mrg XHCIHIST_CALLARGS("xfer %#jx slot %ju dci %ju", (uintptr_t)xfer, xs->xs_idx,
1856 1.75 pgoyette dci, 0);
1857 1.1 jakllsch
1858 1.107 mrg /*
1859 1.107 mrg * XXXMRG: Stall task can run after slot is disabled when yanked.
1860 1.107 mrg * This hack notices that the xs has been memset() in
1861 1.107 mrg * xhci_disable_slot() and returns. Both xhci_reset_endpoint()
1862 1.107 mrg * and xhci_set_dequeue() rely upon a valid ring setup for correct
1863 1.107 mrg * operation, and the latter will fault, as would
1864 1.107 mrg * usb_transfer_complete() if it got that far.
1865 1.107 mrg */
1866 1.107 mrg if (xs->xs_idx == 0) {
1867 1.107 mrg DPRINTFN(4, "ends xs_idx is 0", 0, 0, 0, 0);
1868 1.107 mrg return;
1869 1.107 mrg }
1870 1.107 mrg
1871 1.123 skrll KASSERT(tr != NULL);
1872 1.123 skrll
1873 1.34 skrll xhci_reset_endpoint(xfer->ux_pipe);
1874 1.34 skrll xhci_set_dequeue(xfer->ux_pipe);
1875 1.34 skrll
1876 1.34 skrll mutex_enter(&sc->sc_lock);
1877 1.34 skrll tr->is_halted = false;
1878 1.34 skrll usb_transfer_complete(xfer);
1879 1.34 skrll mutex_exit(&sc->sc_lock);
1880 1.34 skrll DPRINTFN(4, "ends", 0, 0, 0, 0);
1881 1.34 skrll }
1882 1.34 skrll
1883 1.34 skrll static usbd_status
1884 1.34 skrll xhci_clear_endpoint_stall_async(struct usbd_xfer *xfer)
1885 1.34 skrll {
1886 1.34 skrll struct xhci_softc * const sc = XHCI_XFER2SC(xfer);
1887 1.34 skrll struct xhci_pipe * const xp = (struct xhci_pipe *)xfer->ux_pipe;
1888 1.34 skrll
1889 1.111 mrg XHCIHIST_FUNC();
1890 1.111 mrg XHCIHIST_CALLARGS("xfer %#jx", (uintptr_t)xfer, 0, 0, 0);
1891 1.34 skrll
1892 1.34 skrll if (sc->sc_dying) {
1893 1.34 skrll return USBD_IOERROR;
1894 1.34 skrll }
1895 1.34 skrll
1896 1.34 skrll usb_init_task(&xp->xp_async_task,
1897 1.34 skrll xhci_clear_endpoint_stall_async_task, xfer, USB_TASKQ_MPSAFE);
1898 1.34 skrll usb_add_task(xfer->ux_pipe->up_dev, &xp->xp_async_task, USB_TASKQ_HC);
1899 1.34 skrll DPRINTFN(4, "ends", 0, 0, 0, 0);
1900 1.34 skrll
1901 1.34 skrll return USBD_NORMAL_COMPLETION;
1902 1.34 skrll }
1903 1.34 skrll
1904 1.34 skrll /* Process roothub port status/change events and notify to uhub_intr. */
1905 1.34 skrll static void
1906 1.68 skrll xhci_rhpsc(struct xhci_softc * const sc, u_int ctlrport)
1907 1.34 skrll {
1908 1.111 mrg XHCIHIST_FUNC();
1909 1.111 mrg XHCIHIST_CALLARGS("xhci%jd: port %ju status change",
1910 1.111 mrg device_unit(sc->sc_dev), ctlrport, 0, 0);
1911 1.34 skrll
1912 1.68 skrll if (ctlrport > sc->sc_maxports)
1913 1.34 skrll return;
1914 1.34 skrll
1915 1.68 skrll const size_t bn = xhci_ctlrport2bus(sc, ctlrport);
1916 1.68 skrll const size_t rhp = xhci_ctlrport2rhport(sc, ctlrport);
1917 1.68 skrll struct usbd_xfer * const xfer = sc->sc_intrxfer[bn];
1918 1.68 skrll
1919 1.75 pgoyette DPRINTFN(4, "xhci%jd: bus %jd bp %ju xfer %#jx status change",
1920 1.75 pgoyette device_unit(sc->sc_dev), bn, rhp, (uintptr_t)xfer);
1921 1.68 skrll
1922 1.68 skrll if (xfer == NULL)
1923 1.34 skrll return;
1924 1.118 riastrad KASSERT(xfer->ux_status == USBD_IN_PROGRESS);
1925 1.34 skrll
1926 1.68 skrll uint8_t *p = xfer->ux_buf;
1927 1.34 skrll memset(p, 0, xfer->ux_length);
1928 1.68 skrll p[rhp / NBBY] |= 1 << (rhp % NBBY);
1929 1.34 skrll xfer->ux_actlen = xfer->ux_length;
1930 1.34 skrll xfer->ux_status = USBD_NORMAL_COMPLETION;
1931 1.34 skrll usb_transfer_complete(xfer);
1932 1.34 skrll }
1933 1.34 skrll
1934 1.34 skrll /* Process Transfer Events */
1935 1.34 skrll static void
1936 1.34 skrll xhci_event_transfer(struct xhci_softc * const sc,
1937 1.34 skrll const struct xhci_trb * const trb)
1938 1.34 skrll {
1939 1.34 skrll uint64_t trb_0;
1940 1.34 skrll uint32_t trb_2, trb_3;
1941 1.34 skrll uint8_t trbcode;
1942 1.34 skrll u_int slot, dci;
1943 1.34 skrll struct xhci_slot *xs;
1944 1.34 skrll struct xhci_ring *xr;
1945 1.34 skrll struct xhci_xfer *xx;
1946 1.34 skrll struct usbd_xfer *xfer;
1947 1.34 skrll usbd_status err;
1948 1.34 skrll
1949 1.34 skrll XHCIHIST_FUNC(); XHCIHIST_CALLED();
1950 1.34 skrll
1951 1.34 skrll trb_0 = le64toh(trb->trb_0);
1952 1.34 skrll trb_2 = le32toh(trb->trb_2);
1953 1.34 skrll trb_3 = le32toh(trb->trb_3);
1954 1.34 skrll trbcode = XHCI_TRB_2_ERROR_GET(trb_2);
1955 1.34 skrll slot = XHCI_TRB_3_SLOT_GET(trb_3);
1956 1.34 skrll dci = XHCI_TRB_3_EP_GET(trb_3);
1957 1.34 skrll xs = &sc->sc_slots[slot];
1958 1.123 skrll xr = xs->xs_xr[dci];
1959 1.34 skrll
1960 1.34 skrll /* sanity check */
1961 1.123 skrll KASSERT(xr != NULL);
1962 1.34 skrll KASSERTMSG(xs->xs_idx != 0 && xs->xs_idx <= sc->sc_maxslots,
1963 1.34 skrll "invalid xs_idx %u slot %u", xs->xs_idx, slot);
1964 1.34 skrll
1965 1.40 skrll int idx = 0;
1966 1.34 skrll if ((trb_3 & XHCI_TRB_3_ED_BIT) == 0) {
1967 1.40 skrll if (xhci_trb_get_idx(xr, trb_0, &idx)) {
1968 1.120 christos DPRINTFN(0, "invalid trb_0 %#jx", trb_0, 0, 0, 0);
1969 1.34 skrll return;
1970 1.34 skrll }
1971 1.34 skrll xx = xr->xr_cookies[idx];
1972 1.34 skrll
1973 1.63 skrll /* clear cookie of consumed TRB */
1974 1.63 skrll xr->xr_cookies[idx] = NULL;
1975 1.63 skrll
1976 1.34 skrll /*
1977 1.63 skrll * xx is NULL if pipe is opened but xfer is not started.
1978 1.63 skrll * It happens when stopping idle pipe.
1979 1.34 skrll */
1980 1.34 skrll if (xx == NULL || trbcode == XHCI_TRB_ERROR_LENGTH) {
1981 1.75 pgoyette DPRINTFN(1, "Ignore #%ju: cookie %#jx cc %ju dci %ju",
1982 1.75 pgoyette idx, (uintptr_t)xx, trbcode, dci);
1983 1.120 christos DPRINTFN(1, " orig TRB %#jx type %ju", trb_0,
1984 1.53 skrll XHCI_TRB_3_TYPE_GET(le32toh(xr->xr_trb[idx].trb_3)),
1985 1.53 skrll 0, 0);
1986 1.63 skrll return;
1987 1.34 skrll }
1988 1.34 skrll } else {
1989 1.54 skrll /* When ED != 0, trb_0 is virtual addr of struct xhci_xfer. */
1990 1.34 skrll xx = (void *)(uintptr_t)(trb_0 & ~0x3);
1991 1.34 skrll }
1992 1.34 skrll /* XXX this may not happen */
1993 1.34 skrll if (xx == NULL) {
1994 1.34 skrll DPRINTFN(1, "xfer done: xx is NULL", 0, 0, 0, 0);
1995 1.34 skrll return;
1996 1.34 skrll }
1997 1.34 skrll xfer = &xx->xx_xfer;
1998 1.34 skrll /* XXX this may happen when detaching */
1999 1.34 skrll if (xfer == NULL) {
2000 1.75 pgoyette DPRINTFN(1, "xx(%#jx)->xx_xfer is NULL trb_0 %#jx",
2001 1.75 pgoyette (uintptr_t)xx, trb_0, 0, 0);
2002 1.34 skrll return;
2003 1.34 skrll }
2004 1.75 pgoyette DPRINTFN(14, "xfer %#jx", (uintptr_t)xfer, 0, 0, 0);
2005 1.34 skrll /* XXX I dunno why this happens */
2006 1.34 skrll KASSERTMSG(xfer->ux_pipe != NULL, "xfer(%p)->ux_pipe is NULL", xfer);
2007 1.34 skrll
2008 1.34 skrll if (!xfer->ux_pipe->up_repeat &&
2009 1.34 skrll SIMPLEQ_EMPTY(&xfer->ux_pipe->up_queue)) {
2010 1.75 pgoyette DPRINTFN(1, "xfer(%#jx)->pipe not queued", (uintptr_t)xfer,
2011 1.75 pgoyette 0, 0, 0);
2012 1.34 skrll return;
2013 1.34 skrll }
2014 1.34 skrll
2015 1.134 jakllsch const uint8_t xfertype =
2016 1.134 jakllsch UE_GET_XFERTYPE(xfer->ux_pipe->up_endpoint->ue_edesc->bmAttributes);
2017 1.134 jakllsch
2018 1.34 skrll /* 4.11.5.2 Event Data TRB */
2019 1.34 skrll if ((trb_3 & XHCI_TRB_3_ED_BIT) != 0) {
2020 1.121 christos DPRINTFN(14, "transfer Event Data: 0x%016jx 0x%08jx"
2021 1.75 pgoyette " %02jx", trb_0, XHCI_TRB_2_REM_GET(trb_2), trbcode, 0);
2022 1.34 skrll if ((trb_0 & 0x3) == 0x3) {
2023 1.34 skrll xfer->ux_actlen = XHCI_TRB_2_REM_GET(trb_2);
2024 1.34 skrll }
2025 1.34 skrll }
2026 1.34 skrll
2027 1.34 skrll switch (trbcode) {
2028 1.34 skrll case XHCI_TRB_ERROR_SHORT_PKT:
2029 1.34 skrll case XHCI_TRB_ERROR_SUCCESS:
2030 1.54 skrll /*
2031 1.63 skrll * A ctrl transfer can generate two events if it has a Data
2032 1.63 skrll * stage. A short data stage can be OK and should not
2033 1.63 skrll * complete the transfer as the status stage needs to be
2034 1.63 skrll * performed.
2035 1.54 skrll *
2036 1.54 skrll * Note: Data and Status stage events point at same xfer.
2037 1.54 skrll * ux_actlen and ux_dmabuf will be passed to
2038 1.54 skrll * usb_transfer_complete after the Status stage event.
2039 1.54 skrll *
2040 1.54 skrll * It can be distingished which stage generates the event:
2041 1.54 skrll * + by checking least 3 bits of trb_0 if ED==1.
2042 1.54 skrll * (see xhci_device_ctrl_start).
2043 1.54 skrll * + by checking the type of original TRB if ED==0.
2044 1.54 skrll *
2045 1.54 skrll * In addition, intr, bulk, and isoc transfer currently
2046 1.54 skrll * consists of single TD, so the "skip" is not needed.
2047 1.54 skrll * ctrl xfer uses EVENT_DATA, and others do not.
2048 1.54 skrll * Thus driver can switch the flow by checking ED bit.
2049 1.54 skrll */
2050 1.134 jakllsch if (xfertype == UE_ISOCHRONOUS) {
2051 1.134 jakllsch xfer->ux_frlengths[xx->xx_isoc_done] -=
2052 1.134 jakllsch XHCI_TRB_2_REM_GET(trb_2);
2053 1.134 jakllsch xfer->ux_actlen += xfer->ux_frlengths[xx->xx_isoc_done];
2054 1.134 jakllsch if (++xx->xx_isoc_done < xfer->ux_nframes)
2055 1.134 jakllsch return;
2056 1.134 jakllsch } else
2057 1.63 skrll if ((trb_3 & XHCI_TRB_3_ED_BIT) == 0) {
2058 1.63 skrll if (xfer->ux_actlen == 0)
2059 1.63 skrll xfer->ux_actlen = xfer->ux_length -
2060 1.63 skrll XHCI_TRB_2_REM_GET(trb_2);
2061 1.63 skrll if (XHCI_TRB_3_TYPE_GET(le32toh(xr->xr_trb[idx].trb_3))
2062 1.63 skrll == XHCI_TRB_TYPE_DATA_STAGE) {
2063 1.63 skrll return;
2064 1.63 skrll }
2065 1.63 skrll } else if ((trb_0 & 0x3) == 0x3) {
2066 1.63 skrll return;
2067 1.63 skrll }
2068 1.34 skrll err = USBD_NORMAL_COMPLETION;
2069 1.34 skrll break;
2070 1.63 skrll case XHCI_TRB_ERROR_STOPPED:
2071 1.63 skrll case XHCI_TRB_ERROR_LENGTH:
2072 1.63 skrll case XHCI_TRB_ERROR_STOPPED_SHORT:
2073 1.116 riastrad err = USBD_IOERROR;
2074 1.63 skrll break;
2075 1.34 skrll case XHCI_TRB_ERROR_STALL:
2076 1.34 skrll case XHCI_TRB_ERROR_BABBLE:
2077 1.75 pgoyette DPRINTFN(1, "ERR %ju slot %ju dci %ju", trbcode, slot, dci, 0);
2078 1.34 skrll xr->is_halted = true;
2079 1.34 skrll /*
2080 1.129 jakllsch * Try to claim this xfer for completion. If it has already
2081 1.129 jakllsch * completed or aborted, drop it on the floor.
2082 1.129 jakllsch */
2083 1.129 jakllsch if (!usbd_xfer_trycomplete(xfer))
2084 1.129 jakllsch return;
2085 1.129 jakllsch
2086 1.129 jakllsch /*
2087 1.34 skrll * Stalled endpoints can be recoverd by issuing
2088 1.34 skrll * command TRB TYPE_RESET_EP on xHCI instead of
2089 1.34 skrll * issuing request CLEAR_FEATURE UF_ENDPOINT_HALT
2090 1.34 skrll * on the endpoint. However, this function may be
2091 1.34 skrll * called from softint context (e.g. from umass),
2092 1.34 skrll * in that case driver gets KASSERT in cv_timedwait
2093 1.34 skrll * in xhci_do_command.
2094 1.34 skrll * To avoid this, this runs reset_endpoint and
2095 1.34 skrll * usb_transfer_complete in usb task thread
2096 1.34 skrll * asynchronously (and then umass issues clear
2097 1.34 skrll * UF_ENDPOINT_HALT).
2098 1.34 skrll */
2099 1.96 mrg
2100 1.96 mrg /* Override the status. */
2101 1.96 mrg xfer->ux_status = USBD_STALLED;
2102 1.96 mrg
2103 1.34 skrll xhci_clear_endpoint_stall_async(xfer);
2104 1.34 skrll return;
2105 1.34 skrll default:
2106 1.75 pgoyette DPRINTFN(1, "ERR %ju slot %ju dci %ju", trbcode, slot, dci, 0);
2107 1.34 skrll err = USBD_IOERROR;
2108 1.34 skrll break;
2109 1.34 skrll }
2110 1.96 mrg
2111 1.129 jakllsch /*
2112 1.129 jakllsch * Try to claim this xfer for completion. If it has already
2113 1.129 jakllsch * completed or aborted, drop it on the floor.
2114 1.129 jakllsch */
2115 1.129 jakllsch if (!usbd_xfer_trycomplete(xfer))
2116 1.129 jakllsch return;
2117 1.129 jakllsch
2118 1.116 riastrad /* Set the status. */
2119 1.34 skrll xfer->ux_status = err;
2120 1.34 skrll
2121 1.96 mrg if ((trb_3 & XHCI_TRB_3_ED_BIT) == 0 ||
2122 1.96 mrg (trb_0 & 0x3) == 0x0) {
2123 1.34 skrll usb_transfer_complete(xfer);
2124 1.34 skrll }
2125 1.34 skrll }
2126 1.34 skrll
2127 1.34 skrll /* Process Command complete events */
2128 1.34 skrll static void
2129 1.50 skrll xhci_event_cmd(struct xhci_softc * const sc, const struct xhci_trb * const trb)
2130 1.34 skrll {
2131 1.34 skrll uint64_t trb_0;
2132 1.34 skrll uint32_t trb_2, trb_3;
2133 1.34 skrll
2134 1.34 skrll XHCIHIST_FUNC(); XHCIHIST_CALLED();
2135 1.34 skrll
2136 1.68 skrll KASSERT(mutex_owned(&sc->sc_lock));
2137 1.68 skrll
2138 1.34 skrll trb_0 = le64toh(trb->trb_0);
2139 1.34 skrll trb_2 = le32toh(trb->trb_2);
2140 1.34 skrll trb_3 = le32toh(trb->trb_3);
2141 1.34 skrll
2142 1.34 skrll if (trb_0 == sc->sc_command_addr) {
2143 1.68 skrll sc->sc_resultpending = false;
2144 1.68 skrll
2145 1.34 skrll sc->sc_result_trb.trb_0 = trb_0;
2146 1.34 skrll sc->sc_result_trb.trb_2 = trb_2;
2147 1.34 skrll sc->sc_result_trb.trb_3 = trb_3;
2148 1.34 skrll if (XHCI_TRB_2_ERROR_GET(trb_2) !=
2149 1.34 skrll XHCI_TRB_ERROR_SUCCESS) {
2150 1.34 skrll DPRINTFN(1, "command completion "
2151 1.121 christos "failure: 0x%016jx 0x%08jx 0x%08jx",
2152 1.75 pgoyette trb_0, trb_2, trb_3, 0);
2153 1.34 skrll }
2154 1.34 skrll cv_signal(&sc->sc_command_cv);
2155 1.34 skrll } else {
2156 1.121 christos DPRINTFN(1, "spurious event: %#jx 0x%016jx "
2157 1.121 christos "0x%08jx 0x%08jx", (uintptr_t)trb, trb_0, trb_2, trb_3);
2158 1.34 skrll }
2159 1.34 skrll }
2160 1.34 skrll
2161 1.34 skrll /*
2162 1.34 skrll * Process events.
2163 1.34 skrll * called from xhci_softintr
2164 1.34 skrll */
2165 1.34 skrll static void
2166 1.34 skrll xhci_handle_event(struct xhci_softc * const sc,
2167 1.34 skrll const struct xhci_trb * const trb)
2168 1.34 skrll {
2169 1.34 skrll uint64_t trb_0;
2170 1.34 skrll uint32_t trb_2, trb_3;
2171 1.34 skrll
2172 1.111 mrg XHCIHIST_FUNC();
2173 1.34 skrll
2174 1.34 skrll trb_0 = le64toh(trb->trb_0);
2175 1.34 skrll trb_2 = le32toh(trb->trb_2);
2176 1.34 skrll trb_3 = le32toh(trb->trb_3);
2177 1.34 skrll
2178 1.121 christos XHCIHIST_CALLARGS("event: %#jx 0x%016jx 0x%08jx 0x%08jx",
2179 1.75 pgoyette (uintptr_t)trb, trb_0, trb_2, trb_3);
2180 1.34 skrll
2181 1.34 skrll /*
2182 1.34 skrll * 4.11.3.1, 6.4.2.1
2183 1.34 skrll * TRB Pointer is invalid for these completion codes.
2184 1.34 skrll */
2185 1.34 skrll switch (XHCI_TRB_2_ERROR_GET(trb_2)) {
2186 1.34 skrll case XHCI_TRB_ERROR_RING_UNDERRUN:
2187 1.34 skrll case XHCI_TRB_ERROR_RING_OVERRUN:
2188 1.34 skrll case XHCI_TRB_ERROR_VF_RING_FULL:
2189 1.34 skrll return;
2190 1.34 skrll default:
2191 1.34 skrll if (trb_0 == 0) {
2192 1.34 skrll return;
2193 1.34 skrll }
2194 1.34 skrll break;
2195 1.34 skrll }
2196 1.34 skrll
2197 1.34 skrll switch (XHCI_TRB_3_TYPE_GET(trb_3)) {
2198 1.34 skrll case XHCI_TRB_EVENT_TRANSFER:
2199 1.34 skrll xhci_event_transfer(sc, trb);
2200 1.34 skrll break;
2201 1.34 skrll case XHCI_TRB_EVENT_CMD_COMPLETE:
2202 1.34 skrll xhci_event_cmd(sc, trb);
2203 1.34 skrll break;
2204 1.34 skrll case XHCI_TRB_EVENT_PORT_STS_CHANGE:
2205 1.34 skrll xhci_rhpsc(sc, (uint32_t)((trb_0 >> 24) & 0xff));
2206 1.34 skrll break;
2207 1.34 skrll default:
2208 1.34 skrll break;
2209 1.34 skrll }
2210 1.34 skrll }
2211 1.34 skrll
2212 1.34 skrll static void
2213 1.34 skrll xhci_softintr(void *v)
2214 1.34 skrll {
2215 1.34 skrll struct usbd_bus * const bus = v;
2216 1.34 skrll struct xhci_softc * const sc = XHCI_BUS2SC(bus);
2217 1.123 skrll struct xhci_ring * const er = sc->sc_er;
2218 1.34 skrll struct xhci_trb *trb;
2219 1.34 skrll int i, j, k;
2220 1.34 skrll
2221 1.111 mrg XHCIHIST_FUNC();
2222 1.34 skrll
2223 1.73 skrll KASSERT(xhci_polling_p(sc) || mutex_owned(&sc->sc_lock));
2224 1.34 skrll
2225 1.34 skrll i = er->xr_ep;
2226 1.34 skrll j = er->xr_cs;
2227 1.1 jakllsch
2228 1.111 mrg XHCIHIST_CALLARGS("er: xr_ep %jd xr_cs %jd", i, j, 0, 0);
2229 1.27 skrll
2230 1.1 jakllsch while (1) {
2231 1.1 jakllsch usb_syncmem(&er->xr_dma, XHCI_TRB_SIZE * i, XHCI_TRB_SIZE,
2232 1.1 jakllsch BUS_DMASYNC_POSTREAD);
2233 1.1 jakllsch trb = &er->xr_trb[i];
2234 1.1 jakllsch k = (le32toh(trb->trb_3) & XHCI_TRB_3_CYCLE_BIT) ? 1 : 0;
2235 1.1 jakllsch
2236 1.1 jakllsch if (j != k)
2237 1.1 jakllsch break;
2238 1.1 jakllsch
2239 1.1 jakllsch xhci_handle_event(sc, trb);
2240 1.1 jakllsch
2241 1.1 jakllsch i++;
2242 1.52 skrll if (i == er->xr_ntrb) {
2243 1.1 jakllsch i = 0;
2244 1.1 jakllsch j ^= 1;
2245 1.1 jakllsch }
2246 1.1 jakllsch }
2247 1.1 jakllsch
2248 1.1 jakllsch er->xr_ep = i;
2249 1.1 jakllsch er->xr_cs = j;
2250 1.1 jakllsch
2251 1.1 jakllsch xhci_rt_write_8(sc, XHCI_ERDP(0), xhci_ring_trbp(er, er->xr_ep) |
2252 1.132 skrll XHCI_ERDP_BUSY);
2253 1.1 jakllsch
2254 1.27 skrll DPRINTFN(16, "ends", 0, 0, 0, 0);
2255 1.1 jakllsch
2256 1.1 jakllsch return;
2257 1.1 jakllsch }
2258 1.1 jakllsch
2259 1.1 jakllsch static void
2260 1.1 jakllsch xhci_poll(struct usbd_bus *bus)
2261 1.1 jakllsch {
2262 1.34 skrll struct xhci_softc * const sc = XHCI_BUS2SC(bus);
2263 1.1 jakllsch
2264 1.27 skrll XHCIHIST_FUNC(); XHCIHIST_CALLED();
2265 1.1 jakllsch
2266 1.94 christos mutex_enter(&sc->sc_intr_lock);
2267 1.73 skrll int ret = xhci_intr1(sc);
2268 1.73 skrll if (ret) {
2269 1.73 skrll xhci_softintr(bus);
2270 1.73 skrll }
2271 1.94 christos mutex_exit(&sc->sc_intr_lock);
2272 1.1 jakllsch
2273 1.1 jakllsch return;
2274 1.1 jakllsch }
2275 1.1 jakllsch
2276 1.34 skrll static struct usbd_xfer *
2277 1.34 skrll xhci_allocx(struct usbd_bus *bus, unsigned int nframes)
2278 1.1 jakllsch {
2279 1.34 skrll struct xhci_softc * const sc = XHCI_BUS2SC(bus);
2280 1.125 jakllsch struct xhci_xfer *xx;
2281 1.128 jakllsch u_int ntrbs;
2282 1.1 jakllsch
2283 1.27 skrll XHCIHIST_FUNC(); XHCIHIST_CALLED();
2284 1.1 jakllsch
2285 1.134 jakllsch ntrbs = uimax(3, nframes);
2286 1.128 jakllsch const size_t trbsz = sizeof(*xx->xx_trb) * ntrbs;
2287 1.128 jakllsch
2288 1.125 jakllsch xx = pool_cache_get(sc->sc_xferpool, PR_WAITOK);
2289 1.125 jakllsch if (xx != NULL) {
2290 1.125 jakllsch memset(xx, 0, sizeof(*xx));
2291 1.128 jakllsch if (ntrbs > 0) {
2292 1.128 jakllsch xx->xx_trb = kmem_alloc(trbsz, KM_SLEEP);
2293 1.128 jakllsch xx->xx_ntrb = ntrbs;
2294 1.128 jakllsch }
2295 1.1 jakllsch #ifdef DIAGNOSTIC
2296 1.125 jakllsch xx->xx_xfer.ux_state = XFER_BUSY;
2297 1.1 jakllsch #endif
2298 1.1 jakllsch }
2299 1.1 jakllsch
2300 1.125 jakllsch return &xx->xx_xfer;
2301 1.1 jakllsch }
2302 1.1 jakllsch
2303 1.1 jakllsch static void
2304 1.34 skrll xhci_freex(struct usbd_bus *bus, struct usbd_xfer *xfer)
2305 1.1 jakllsch {
2306 1.34 skrll struct xhci_softc * const sc = XHCI_BUS2SC(bus);
2307 1.125 jakllsch struct xhci_xfer * const xx = XHCI_XFER2XXFER(xfer);
2308 1.1 jakllsch
2309 1.27 skrll XHCIHIST_FUNC(); XHCIHIST_CALLED();
2310 1.1 jakllsch
2311 1.1 jakllsch #ifdef DIAGNOSTIC
2312 1.106 rin if (xfer->ux_state != XFER_BUSY &&
2313 1.106 rin xfer->ux_status != USBD_NOT_STARTED) {
2314 1.121 christos DPRINTFN(0, "xfer=%#jx not busy, 0x%08jx",
2315 1.75 pgoyette (uintptr_t)xfer, xfer->ux_state, 0, 0);
2316 1.1 jakllsch }
2317 1.34 skrll xfer->ux_state = XFER_FREE;
2318 1.1 jakllsch #endif
2319 1.128 jakllsch if (xx->xx_ntrb > 0) {
2320 1.128 jakllsch kmem_free(xx->xx_trb, xx->xx_ntrb * sizeof(*xx->xx_trb));
2321 1.128 jakllsch xx->xx_trb = NULL;
2322 1.128 jakllsch xx->xx_ntrb = 0;
2323 1.128 jakllsch }
2324 1.125 jakllsch pool_cache_put(sc->sc_xferpool, xx);
2325 1.1 jakllsch }
2326 1.1 jakllsch
2327 1.116 riastrad static bool
2328 1.116 riastrad xhci_dying(struct usbd_bus *bus)
2329 1.116 riastrad {
2330 1.116 riastrad struct xhci_softc * const sc = XHCI_BUS2SC(bus);
2331 1.116 riastrad
2332 1.116 riastrad return sc->sc_dying;
2333 1.116 riastrad }
2334 1.116 riastrad
2335 1.1 jakllsch static void
2336 1.1 jakllsch xhci_get_lock(struct usbd_bus *bus, kmutex_t **lock)
2337 1.1 jakllsch {
2338 1.34 skrll struct xhci_softc * const sc = XHCI_BUS2SC(bus);
2339 1.1 jakllsch
2340 1.1 jakllsch *lock = &sc->sc_lock;
2341 1.1 jakllsch }
2342 1.1 jakllsch
2343 1.34 skrll extern uint32_t usb_cookie_no;
2344 1.1 jakllsch
2345 1.34 skrll /*
2346 1.41 skrll * xHCI 4.3
2347 1.41 skrll * Called when uhub_explore finds a new device (via usbd_new_device).
2348 1.41 skrll * Port initialization and speed detection (4.3.1) are already done in uhub.c.
2349 1.41 skrll * This function does:
2350 1.41 skrll * Allocate and construct dev structure of default endpoint (ep0).
2351 1.41 skrll * Allocate and open pipe of ep0.
2352 1.41 skrll * Enable slot and initialize slot context.
2353 1.41 skrll * Set Address.
2354 1.41 skrll * Read initial device descriptor.
2355 1.34 skrll * Determine initial MaxPacketSize (mps) by speed.
2356 1.41 skrll * Read full device descriptor.
2357 1.41 skrll * Register this device.
2358 1.54 skrll * Finally state of device transitions ADDRESSED.
2359 1.34 skrll */
2360 1.1 jakllsch static usbd_status
2361 1.34 skrll xhci_new_device(device_t parent, struct usbd_bus *bus, int depth,
2362 1.1 jakllsch int speed, int port, struct usbd_port *up)
2363 1.1 jakllsch {
2364 1.34 skrll struct xhci_softc * const sc = XHCI_BUS2SC(bus);
2365 1.34 skrll struct usbd_device *dev;
2366 1.1 jakllsch usbd_status err;
2367 1.1 jakllsch usb_device_descriptor_t *dd;
2368 1.1 jakllsch struct xhci_slot *xs;
2369 1.1 jakllsch uint32_t *cp;
2370 1.1 jakllsch
2371 1.111 mrg XHCIHIST_FUNC();
2372 1.111 mrg XHCIHIST_CALLARGS("port %ju depth %ju speed %ju up %#jx",
2373 1.75 pgoyette port, depth, speed, (uintptr_t)up);
2374 1.27 skrll
2375 1.34 skrll dev = kmem_zalloc(sizeof(*dev), KM_SLEEP);
2376 1.34 skrll dev->ud_bus = bus;
2377 1.51 skrll dev->ud_quirks = &usbd_no_quirk;
2378 1.51 skrll dev->ud_addr = 0;
2379 1.51 skrll dev->ud_ddesc.bMaxPacketSize = 0;
2380 1.51 skrll dev->ud_depth = depth;
2381 1.51 skrll dev->ud_powersrc = up;
2382 1.51 skrll dev->ud_myhub = up->up_parent;
2383 1.51 skrll dev->ud_speed = speed;
2384 1.51 skrll dev->ud_langid = USBD_NOLANG;
2385 1.51 skrll dev->ud_cookie.cookie = ++usb_cookie_no;
2386 1.1 jakllsch
2387 1.1 jakllsch /* Set up default endpoint handle. */
2388 1.34 skrll dev->ud_ep0.ue_edesc = &dev->ud_ep0desc;
2389 1.51 skrll /* doesn't matter, just don't let it uninitialized */
2390 1.51 skrll dev->ud_ep0.ue_toggle = 0;
2391 1.1 jakllsch
2392 1.1 jakllsch /* Set up default endpoint descriptor. */
2393 1.34 skrll dev->ud_ep0desc.bLength = USB_ENDPOINT_DESCRIPTOR_SIZE;
2394 1.34 skrll dev->ud_ep0desc.bDescriptorType = UDESC_ENDPOINT;
2395 1.34 skrll dev->ud_ep0desc.bEndpointAddress = USB_CONTROL_ENDPOINT;
2396 1.34 skrll dev->ud_ep0desc.bmAttributes = UE_CONTROL;
2397 1.51 skrll dev->ud_ep0desc.bInterval = 0;
2398 1.50 skrll
2399 1.34 skrll /* 4.3, 4.8.2.1 */
2400 1.34 skrll switch (speed) {
2401 1.34 skrll case USB_SPEED_SUPER:
2402 1.34 skrll case USB_SPEED_SUPER_PLUS:
2403 1.34 skrll USETW(dev->ud_ep0desc.wMaxPacketSize, USB_3_MAX_CTRL_PACKET);
2404 1.34 skrll break;
2405 1.34 skrll case USB_SPEED_FULL:
2406 1.34 skrll /* XXX using 64 as initial mps of ep0 in FS */
2407 1.34 skrll case USB_SPEED_HIGH:
2408 1.34 skrll USETW(dev->ud_ep0desc.wMaxPacketSize, USB_2_MAX_CTRL_PACKET);
2409 1.34 skrll break;
2410 1.34 skrll case USB_SPEED_LOW:
2411 1.34 skrll default:
2412 1.34 skrll USETW(dev->ud_ep0desc.wMaxPacketSize, USB_MAX_IPACKET);
2413 1.34 skrll break;
2414 1.34 skrll }
2415 1.1 jakllsch
2416 1.51 skrll up->up_dev = dev;
2417 1.51 skrll
2418 1.51 skrll dd = &dev->ud_ddesc;
2419 1.1 jakllsch
2420 1.68 skrll if (depth == 0 && port == 0) {
2421 1.68 skrll KASSERT(bus->ub_devices[USB_ROOTHUB_INDEX] == NULL);
2422 1.68 skrll bus->ub_devices[USB_ROOTHUB_INDEX] = dev;
2423 1.123 skrll
2424 1.123 skrll /* Establish the default pipe. */
2425 1.123 skrll err = usbd_setup_pipe(dev, 0, &dev->ud_ep0,
2426 1.123 skrll USBD_DEFAULT_INTERVAL, &dev->ud_pipe0);
2427 1.61 skrll if (err) {
2428 1.123 skrll DPRINTFN(1, "setup default pipe failed %jd", err,0,0,0);
2429 1.34 skrll goto bad;
2430 1.61 skrll }
2431 1.123 skrll err = usbd_get_initial_ddesc(dev, dd);
2432 1.61 skrll if (err) {
2433 1.123 skrll DPRINTFN(1, "get_initial_ddesc %ju", err, 0, 0, 0);
2434 1.34 skrll goto bad;
2435 1.61 skrll }
2436 1.1 jakllsch } else {
2437 1.49 skrll uint8_t slot = 0;
2438 1.49 skrll
2439 1.48 skrll /* 4.3.2 */
2440 1.1 jakllsch err = xhci_enable_slot(sc, &slot);
2441 1.63 skrll if (err) {
2442 1.75 pgoyette DPRINTFN(1, "enable slot %ju", err, 0, 0, 0);
2443 1.34 skrll goto bad;
2444 1.63 skrll }
2445 1.50 skrll
2446 1.1 jakllsch xs = &sc->sc_slots[slot];
2447 1.34 skrll dev->ud_hcpriv = xs;
2448 1.50 skrll
2449 1.48 skrll /* 4.3.3 initialize slot structure */
2450 1.48 skrll err = xhci_init_slot(dev, slot);
2451 1.34 skrll if (err) {
2452 1.75 pgoyette DPRINTFN(1, "init slot %ju", err, 0, 0, 0);
2453 1.34 skrll dev->ud_hcpriv = NULL;
2454 1.34 skrll /*
2455 1.34 skrll * We have to disable_slot here because
2456 1.34 skrll * xs->xs_idx == 0 when xhci_init_slot fails,
2457 1.34 skrll * in that case usbd_remove_dev won't work.
2458 1.34 skrll */
2459 1.34 skrll mutex_enter(&sc->sc_lock);
2460 1.34 skrll xhci_disable_slot(sc, slot);
2461 1.34 skrll mutex_exit(&sc->sc_lock);
2462 1.34 skrll goto bad;
2463 1.34 skrll }
2464 1.34 skrll
2465 1.123 skrll /*
2466 1.123 skrll * We have to establish the default pipe _after_ slot
2467 1.123 skrll * structure has been prepared.
2468 1.123 skrll */
2469 1.123 skrll err = usbd_setup_pipe(dev, 0, &dev->ud_ep0,
2470 1.123 skrll USBD_DEFAULT_INTERVAL, &dev->ud_pipe0);
2471 1.123 skrll if (err) {
2472 1.123 skrll DPRINTFN(1, "setup default pipe failed %jd", err, 0, 0,
2473 1.123 skrll 0);
2474 1.123 skrll goto bad;
2475 1.123 skrll }
2476 1.123 skrll
2477 1.48 skrll /* 4.3.4 Address Assignment */
2478 1.51 skrll err = xhci_set_address(dev, slot, false);
2479 1.61 skrll if (err) {
2480 1.111 mrg DPRINTFN(1, "failed! to set address: %ju", err, 0, 0, 0);
2481 1.48 skrll goto bad;
2482 1.61 skrll }
2483 1.48 skrll
2484 1.34 skrll /* Allow device time to set new address */
2485 1.34 skrll usbd_delay_ms(dev, USB_SET_ADDRESS_SETTLE);
2486 1.50 skrll
2487 1.92 jakllsch usb_syncmem(&xs->xs_dc_dma, 0, sc->sc_pgsz, BUS_DMASYNC_POSTREAD);
2488 1.1 jakllsch cp = xhci_slot_get_dcv(sc, xs, XHCI_DCI_SLOT);
2489 1.79 christos HEXDUMP("slot context", cp, sc->sc_ctxsz);
2490 1.64 skrll uint8_t addr = XHCI_SCTX_3_DEV_ADDR_GET(le32toh(cp[3]));
2491 1.75 pgoyette DPRINTFN(4, "device address %ju", addr, 0, 0, 0);
2492 1.68 skrll /*
2493 1.68 skrll * XXX ensure we know when the hardware does something
2494 1.68 skrll * we can't yet cope with
2495 1.68 skrll */
2496 1.59 maya KASSERTMSG(addr >= 1 && addr <= 127, "addr %d", addr);
2497 1.34 skrll dev->ud_addr = addr;
2498 1.68 skrll
2499 1.68 skrll KASSERTMSG(bus->ub_devices[usb_addr2dindex(dev->ud_addr)] == NULL,
2500 1.68 skrll "addr %d already allocated", dev->ud_addr);
2501 1.68 skrll /*
2502 1.68 skrll * The root hub is given its own slot
2503 1.68 skrll */
2504 1.68 skrll bus->ub_devices[usb_addr2dindex(dev->ud_addr)] = dev;
2505 1.1 jakllsch
2506 1.1 jakllsch err = usbd_get_initial_ddesc(dev, dd);
2507 1.61 skrll if (err) {
2508 1.75 pgoyette DPRINTFN(1, "get_initial_ddesc %ju", err, 0, 0, 0);
2509 1.34 skrll goto bad;
2510 1.61 skrll }
2511 1.50 skrll
2512 1.24 skrll /* 4.8.2.1 */
2513 1.34 skrll if (USB_IS_SS(speed)) {
2514 1.34 skrll if (dd->bMaxPacketSize != 9) {
2515 1.34 skrll printf("%s: invalid mps 2^%u for SS ep0,"
2516 1.34 skrll " using 512\n",
2517 1.34 skrll device_xname(sc->sc_dev),
2518 1.34 skrll dd->bMaxPacketSize);
2519 1.34 skrll dd->bMaxPacketSize = 9;
2520 1.34 skrll }
2521 1.34 skrll USETW(dev->ud_ep0desc.wMaxPacketSize,
2522 1.24 skrll (1 << dd->bMaxPacketSize));
2523 1.34 skrll } else
2524 1.34 skrll USETW(dev->ud_ep0desc.wMaxPacketSize,
2525 1.24 skrll dd->bMaxPacketSize);
2526 1.75 pgoyette DPRINTFN(4, "bMaxPacketSize %ju", dd->bMaxPacketSize, 0, 0, 0);
2527 1.62 skrll err = xhci_update_ep0_mps(sc, xs,
2528 1.34 skrll UGETW(dev->ud_ep0desc.wMaxPacketSize));
2529 1.62 skrll if (err) {
2530 1.75 pgoyette DPRINTFN(1, "update mps of ep0 %ju", err, 0, 0, 0);
2531 1.62 skrll goto bad;
2532 1.62 skrll }
2533 1.123 skrll }
2534 1.50 skrll
2535 1.123 skrll err = usbd_reload_device_desc(dev);
2536 1.123 skrll if (err) {
2537 1.123 skrll DPRINTFN(1, "reload desc %ju", err, 0, 0, 0);
2538 1.123 skrll goto bad;
2539 1.1 jakllsch }
2540 1.1 jakllsch
2541 1.75 pgoyette DPRINTFN(1, "adding unit addr=%jd, rev=%02jx,",
2542 1.34 skrll dev->ud_addr, UGETW(dd->bcdUSB), 0, 0);
2543 1.75 pgoyette DPRINTFN(1, " class=%jd, subclass=%jd, protocol=%jd,",
2544 1.27 skrll dd->bDeviceClass, dd->bDeviceSubClass,
2545 1.27 skrll dd->bDeviceProtocol, 0);
2546 1.75 pgoyette DPRINTFN(1, " mps=%jd, len=%jd, noconf=%jd, speed=%jd",
2547 1.27 skrll dd->bMaxPacketSize, dd->bLength, dd->bNumConfigurations,
2548 1.34 skrll dev->ud_speed);
2549 1.1 jakllsch
2550 1.33 skrll usbd_get_device_strings(dev);
2551 1.33 skrll
2552 1.1 jakllsch usbd_add_dev_event(USB_EVENT_DEVICE_ATTACH, dev);
2553 1.1 jakllsch
2554 1.68 skrll if (depth == 0 && port == 0) {
2555 1.1 jakllsch usbd_attach_roothub(parent, dev);
2556 1.75 pgoyette DPRINTFN(1, "root hub %#jx", (uintptr_t)dev, 0, 0, 0);
2557 1.1 jakllsch return USBD_NORMAL_COMPLETION;
2558 1.1 jakllsch }
2559 1.1 jakllsch
2560 1.34 skrll err = usbd_probe_and_attach(parent, dev, port, dev->ud_addr);
2561 1.34 skrll bad:
2562 1.34 skrll if (err != USBD_NORMAL_COMPLETION) {
2563 1.1 jakllsch usbd_remove_device(dev, up);
2564 1.1 jakllsch }
2565 1.1 jakllsch
2566 1.34 skrll return err;
2567 1.1 jakllsch }
2568 1.1 jakllsch
2569 1.1 jakllsch static usbd_status
2570 1.123 skrll xhci_ring_init(struct xhci_softc * const sc, struct xhci_ring **xrp,
2571 1.1 jakllsch size_t ntrb, size_t align)
2572 1.1 jakllsch {
2573 1.1 jakllsch usbd_status err;
2574 1.1 jakllsch size_t size = ntrb * XHCI_TRB_SIZE;
2575 1.123 skrll struct xhci_ring *xr;
2576 1.1 jakllsch
2577 1.111 mrg XHCIHIST_FUNC();
2578 1.111 mrg XHCIHIST_CALLARGS("xr %#jx ntrb %#jx align %#jx",
2579 1.123 skrll (uintptr_t)*xrp, ntrb, align, 0);
2580 1.123 skrll
2581 1.123 skrll xr = kmem_zalloc(sizeof(struct xhci_ring), KM_SLEEP);
2582 1.123 skrll DPRINTFN(1, "ring %#jx", (uintptr_t)xr, 0, 0, 0);
2583 1.27 skrll
2584 1.124 skrll err = usb_allocmem(&sc->sc_bus, size, align, USBMALLOC_COHERENT,
2585 1.124 skrll &xr->xr_dma);
2586 1.123 skrll if (err) {
2587 1.123 skrll kmem_free(xr, sizeof(struct xhci_ring));
2588 1.123 skrll DPRINTFN(1, "alloc xr_dma failed %jd", err, 0, 0, 0);
2589 1.1 jakllsch return err;
2590 1.123 skrll }
2591 1.1 jakllsch mutex_init(&xr->xr_lock, MUTEX_DEFAULT, IPL_SOFTUSB);
2592 1.1 jakllsch xr->xr_cookies = kmem_zalloc(sizeof(*xr->xr_cookies) * ntrb, KM_SLEEP);
2593 1.1 jakllsch xr->xr_trb = xhci_ring_trbv(xr, 0);
2594 1.1 jakllsch xr->xr_ntrb = ntrb;
2595 1.1 jakllsch xr->is_halted = false;
2596 1.55 skrll xhci_host_dequeue(xr);
2597 1.123 skrll *xrp = xr;
2598 1.1 jakllsch
2599 1.1 jakllsch return USBD_NORMAL_COMPLETION;
2600 1.1 jakllsch }
2601 1.1 jakllsch
2602 1.1 jakllsch static void
2603 1.123 skrll xhci_ring_free(struct xhci_softc * const sc, struct xhci_ring ** const xr)
2604 1.1 jakllsch {
2605 1.123 skrll if (*xr == NULL)
2606 1.123 skrll return;
2607 1.123 skrll
2608 1.123 skrll usb_freemem(&sc->sc_bus, &(*xr)->xr_dma);
2609 1.123 skrll mutex_destroy(&(*xr)->xr_lock);
2610 1.123 skrll kmem_free((*xr)->xr_cookies,
2611 1.123 skrll sizeof(*(*xr)->xr_cookies) * (*xr)->xr_ntrb);
2612 1.123 skrll kmem_free(*xr, sizeof(struct xhci_ring));
2613 1.123 skrll *xr = NULL;
2614 1.1 jakllsch }
2615 1.1 jakllsch
2616 1.1 jakllsch static void
2617 1.1 jakllsch xhci_ring_put(struct xhci_softc * const sc, struct xhci_ring * const xr,
2618 1.101 jakllsch void *cookie, struct xhci_soft_trb * const trbs, size_t ntrbs)
2619 1.1 jakllsch {
2620 1.1 jakllsch size_t i;
2621 1.1 jakllsch u_int ri;
2622 1.1 jakllsch u_int cs;
2623 1.1 jakllsch uint64_t parameter;
2624 1.1 jakllsch uint32_t status;
2625 1.1 jakllsch uint32_t control;
2626 1.1 jakllsch
2627 1.111 mrg XHCIHIST_FUNC();
2628 1.120 christos XHCIHIST_CALLARGS("%#jx xr_ep %#jx xr_cs %ju",
2629 1.111 mrg (uintptr_t)xr, xr->xr_ep, xr->xr_cs, 0);
2630 1.27 skrll
2631 1.127 jakllsch KASSERTMSG(ntrbs < xr->xr_ntrb, "ntrbs %zu, xr->xr_ntrb %u",
2632 1.127 jakllsch ntrbs, xr->xr_ntrb);
2633 1.1 jakllsch for (i = 0; i < ntrbs; i++) {
2634 1.75 pgoyette DPRINTFN(12, "xr %#jx trbs %#jx num %ju", (uintptr_t)xr,
2635 1.75 pgoyette (uintptr_t)trbs, i, 0);
2636 1.121 christos DPRINTFN(12, " 0x%016jx 0x%08jx 0x%08jx",
2637 1.27 skrll trbs[i].trb_0, trbs[i].trb_2, trbs[i].trb_3, 0);
2638 1.59 maya KASSERTMSG(XHCI_TRB_3_TYPE_GET(trbs[i].trb_3) !=
2639 1.63 skrll XHCI_TRB_TYPE_LINK, "trbs[%zu].trb3 %#x", i, trbs[i].trb_3);
2640 1.1 jakllsch }
2641 1.1 jakllsch
2642 1.1 jakllsch ri = xr->xr_ep;
2643 1.1 jakllsch cs = xr->xr_cs;
2644 1.1 jakllsch
2645 1.11 dsl /*
2646 1.11 dsl * Although the xhci hardware can do scatter/gather dma from
2647 1.11 dsl * arbitrary sized buffers, there is a non-obvious restriction
2648 1.11 dsl * that a LINK trb is only allowed at the end of a burst of
2649 1.11 dsl * transfers - which might be 16kB.
2650 1.11 dsl * Arbitrary aligned LINK trb definitely fail on Ivy bridge.
2651 1.11 dsl * The simple solution is not to allow a LINK trb in the middle
2652 1.11 dsl * of anything - as here.
2653 1.13 dsl * XXX: (dsl) There are xhci controllers out there (eg some made by
2654 1.13 dsl * ASMedia) that seem to lock up if they process a LINK trb but
2655 1.13 dsl * cannot process the linked-to trb yet.
2656 1.13 dsl * The code should write the 'cycle' bit on the link trb AFTER
2657 1.13 dsl * adding the other trb.
2658 1.11 dsl */
2659 1.65 skrll u_int firstep = xr->xr_ep;
2660 1.65 skrll u_int firstcs = xr->xr_cs;
2661 1.1 jakllsch
2662 1.65 skrll for (i = 0; i < ntrbs; ) {
2663 1.65 skrll u_int oldri = ri;
2664 1.65 skrll u_int oldcs = cs;
2665 1.65 skrll
2666 1.65 skrll if (ri >= (xr->xr_ntrb - 1)) {
2667 1.65 skrll /* Put Link TD at the end of ring */
2668 1.65 skrll parameter = xhci_ring_trbp(xr, 0);
2669 1.65 skrll status = 0;
2670 1.65 skrll control = XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_LINK) |
2671 1.65 skrll XHCI_TRB_3_TC_BIT;
2672 1.65 skrll xr->xr_cookies[ri] = NULL;
2673 1.65 skrll xr->xr_ep = 0;
2674 1.65 skrll xr->xr_cs ^= 1;
2675 1.65 skrll ri = xr->xr_ep;
2676 1.65 skrll cs = xr->xr_cs;
2677 1.1 jakllsch } else {
2678 1.65 skrll parameter = trbs[i].trb_0;
2679 1.65 skrll status = trbs[i].trb_2;
2680 1.65 skrll control = trbs[i].trb_3;
2681 1.65 skrll
2682 1.65 skrll xr->xr_cookies[ri] = cookie;
2683 1.65 skrll ri++;
2684 1.65 skrll i++;
2685 1.1 jakllsch }
2686 1.65 skrll /*
2687 1.65 skrll * If this is a first TRB, mark it invalid to prevent
2688 1.65 skrll * xHC from running it immediately.
2689 1.65 skrll */
2690 1.65 skrll if (oldri == firstep) {
2691 1.65 skrll if (oldcs) {
2692 1.65 skrll control &= ~XHCI_TRB_3_CYCLE_BIT;
2693 1.65 skrll } else {
2694 1.65 skrll control |= XHCI_TRB_3_CYCLE_BIT;
2695 1.65 skrll }
2696 1.65 skrll } else {
2697 1.65 skrll if (oldcs) {
2698 1.65 skrll control |= XHCI_TRB_3_CYCLE_BIT;
2699 1.65 skrll } else {
2700 1.65 skrll control &= ~XHCI_TRB_3_CYCLE_BIT;
2701 1.65 skrll }
2702 1.65 skrll }
2703 1.65 skrll xhci_trb_put(&xr->xr_trb[oldri], parameter, status, control);
2704 1.65 skrll usb_syncmem(&xr->xr_dma, XHCI_TRB_SIZE * oldri,
2705 1.65 skrll XHCI_TRB_SIZE * 1, BUS_DMASYNC_PREWRITE);
2706 1.1 jakllsch }
2707 1.1 jakllsch
2708 1.65 skrll /* Now invert cycle bit of first TRB */
2709 1.65 skrll if (firstcs) {
2710 1.65 skrll xr->xr_trb[firstep].trb_3 |= htole32(XHCI_TRB_3_CYCLE_BIT);
2711 1.34 skrll } else {
2712 1.65 skrll xr->xr_trb[firstep].trb_3 &= ~htole32(XHCI_TRB_3_CYCLE_BIT);
2713 1.34 skrll }
2714 1.65 skrll usb_syncmem(&xr->xr_dma, XHCI_TRB_SIZE * firstep,
2715 1.65 skrll XHCI_TRB_SIZE * 1, BUS_DMASYNC_PREWRITE);
2716 1.1 jakllsch
2717 1.1 jakllsch xr->xr_ep = ri;
2718 1.1 jakllsch xr->xr_cs = cs;
2719 1.1 jakllsch
2720 1.120 christos DPRINTFN(12, "%#jx xr_ep %#jx xr_cs %ju", (uintptr_t)xr, xr->xr_ep,
2721 1.75 pgoyette xr->xr_cs, 0);
2722 1.1 jakllsch }
2723 1.1 jakllsch
2724 1.127 jakllsch static inline void
2725 1.127 jakllsch xhci_ring_put_xfer(struct xhci_softc * const sc, struct xhci_ring * const tr,
2726 1.127 jakllsch struct xhci_xfer *xx, u_int ntrb)
2727 1.127 jakllsch {
2728 1.128 jakllsch KASSERT(ntrb <= xx->xx_ntrb);
2729 1.127 jakllsch xhci_ring_put(sc, tr, xx, xx->xx_trb, ntrb);
2730 1.127 jakllsch }
2731 1.127 jakllsch
2732 1.34 skrll /*
2733 1.39 skrll * Stop execution commands, purge all commands on command ring, and
2734 1.54 skrll * rewind dequeue pointer.
2735 1.39 skrll */
2736 1.39 skrll static void
2737 1.39 skrll xhci_abort_command(struct xhci_softc *sc)
2738 1.39 skrll {
2739 1.123 skrll struct xhci_ring * const cr = sc->sc_cr;
2740 1.39 skrll uint64_t crcr;
2741 1.39 skrll int i;
2742 1.39 skrll
2743 1.111 mrg XHCIHIST_FUNC();
2744 1.111 mrg XHCIHIST_CALLARGS("command %#jx timeout, aborting",
2745 1.39 skrll sc->sc_command_addr, 0, 0, 0);
2746 1.39 skrll
2747 1.39 skrll mutex_enter(&cr->xr_lock);
2748 1.39 skrll
2749 1.39 skrll /* 4.6.1.2 Aborting a Command */
2750 1.39 skrll crcr = xhci_op_read_8(sc, XHCI_CRCR);
2751 1.39 skrll xhci_op_write_8(sc, XHCI_CRCR, crcr | XHCI_CRCR_LO_CA);
2752 1.39 skrll
2753 1.39 skrll for (i = 0; i < 500; i++) {
2754 1.39 skrll crcr = xhci_op_read_8(sc, XHCI_CRCR);
2755 1.39 skrll if ((crcr & XHCI_CRCR_LO_CRR) == 0)
2756 1.39 skrll break;
2757 1.39 skrll usb_delay_ms(&sc->sc_bus, 1);
2758 1.39 skrll }
2759 1.39 skrll if ((crcr & XHCI_CRCR_LO_CRR) != 0) {
2760 1.39 skrll DPRINTFN(1, "Command Abort timeout", 0, 0, 0, 0);
2761 1.39 skrll /* reset HC here? */
2762 1.39 skrll }
2763 1.39 skrll
2764 1.39 skrll /* reset command ring dequeue pointer */
2765 1.39 skrll cr->xr_ep = 0;
2766 1.39 skrll cr->xr_cs = 1;
2767 1.39 skrll xhci_op_write_8(sc, XHCI_CRCR, xhci_ring_trbp(cr, 0) | cr->xr_cs);
2768 1.39 skrll
2769 1.39 skrll mutex_exit(&cr->xr_lock);
2770 1.39 skrll }
2771 1.39 skrll
2772 1.39 skrll /*
2773 1.34 skrll * Put a command on command ring, ring bell, set timer, and cv_timedwait.
2774 1.54 skrll * Command completion is notified by cv_signal from xhci_event_cmd()
2775 1.54 skrll * (called from xhci_softint), or timed-out.
2776 1.54 skrll * The completion code is copied to sc->sc_result_trb in xhci_event_cmd(),
2777 1.54 skrll * then do_command examines it.
2778 1.34 skrll */
2779 1.1 jakllsch static usbd_status
2780 1.50 skrll xhci_do_command_locked(struct xhci_softc * const sc,
2781 1.101 jakllsch struct xhci_soft_trb * const trb, int timeout)
2782 1.1 jakllsch {
2783 1.123 skrll struct xhci_ring * const cr = sc->sc_cr;
2784 1.1 jakllsch usbd_status err;
2785 1.1 jakllsch
2786 1.111 mrg XHCIHIST_FUNC();
2787 1.121 christos XHCIHIST_CALLARGS("input: 0x%016jx 0x%08jx 0x%08jx",
2788 1.27 skrll trb->trb_0, trb->trb_2, trb->trb_3, 0);
2789 1.1 jakllsch
2790 1.34 skrll KASSERTMSG(!cpu_intr_p() && !cpu_softintr_p(), "called from intr ctx");
2791 1.34 skrll KASSERT(mutex_owned(&sc->sc_lock));
2792 1.1 jakllsch
2793 1.68 skrll while (sc->sc_command_addr != 0)
2794 1.68 skrll cv_wait(&sc->sc_cmdbusy_cv, &sc->sc_lock);
2795 1.68 skrll
2796 1.67 skrll /*
2797 1.67 skrll * If enqueue pointer points at last of ring, it's Link TRB,
2798 1.67 skrll * command TRB will be stored in 0th TRB.
2799 1.67 skrll */
2800 1.67 skrll if (cr->xr_ep == cr->xr_ntrb - 1)
2801 1.67 skrll sc->sc_command_addr = xhci_ring_trbp(cr, 0);
2802 1.67 skrll else
2803 1.67 skrll sc->sc_command_addr = xhci_ring_trbp(cr, cr->xr_ep);
2804 1.1 jakllsch
2805 1.68 skrll sc->sc_resultpending = true;
2806 1.68 skrll
2807 1.1 jakllsch mutex_enter(&cr->xr_lock);
2808 1.1 jakllsch xhci_ring_put(sc, cr, NULL, trb, 1);
2809 1.1 jakllsch mutex_exit(&cr->xr_lock);
2810 1.1 jakllsch
2811 1.1 jakllsch xhci_db_write_4(sc, XHCI_DOORBELL(0), 0);
2812 1.1 jakllsch
2813 1.68 skrll while (sc->sc_resultpending) {
2814 1.68 skrll if (cv_timedwait(&sc->sc_command_cv, &sc->sc_lock,
2815 1.68 skrll MAX(1, mstohz(timeout))) == EWOULDBLOCK) {
2816 1.68 skrll xhci_abort_command(sc);
2817 1.68 skrll err = USBD_TIMEOUT;
2818 1.68 skrll goto timedout;
2819 1.68 skrll }
2820 1.1 jakllsch }
2821 1.1 jakllsch
2822 1.1 jakllsch trb->trb_0 = sc->sc_result_trb.trb_0;
2823 1.1 jakllsch trb->trb_2 = sc->sc_result_trb.trb_2;
2824 1.1 jakllsch trb->trb_3 = sc->sc_result_trb.trb_3;
2825 1.1 jakllsch
2826 1.121 christos DPRINTFN(12, "output: 0x%016jx 0x%08jx 0x%08jx",
2827 1.27 skrll trb->trb_0, trb->trb_2, trb->trb_3, 0);
2828 1.1 jakllsch
2829 1.1 jakllsch switch (XHCI_TRB_2_ERROR_GET(trb->trb_2)) {
2830 1.1 jakllsch case XHCI_TRB_ERROR_SUCCESS:
2831 1.1 jakllsch err = USBD_NORMAL_COMPLETION;
2832 1.1 jakllsch break;
2833 1.1 jakllsch default:
2834 1.1 jakllsch case 192 ... 223:
2835 1.120 christos DPRINTFN(5, "error %#jx",
2836 1.111 mrg XHCI_TRB_2_ERROR_GET(trb->trb_2), 0, 0, 0);
2837 1.1 jakllsch err = USBD_IOERROR;
2838 1.1 jakllsch break;
2839 1.1 jakllsch case 224 ... 255:
2840 1.1 jakllsch err = USBD_NORMAL_COMPLETION;
2841 1.1 jakllsch break;
2842 1.1 jakllsch }
2843 1.1 jakllsch
2844 1.1 jakllsch timedout:
2845 1.68 skrll sc->sc_resultpending = false;
2846 1.1 jakllsch sc->sc_command_addr = 0;
2847 1.68 skrll cv_broadcast(&sc->sc_cmdbusy_cv);
2848 1.68 skrll
2849 1.34 skrll return err;
2850 1.34 skrll }
2851 1.34 skrll
2852 1.34 skrll static usbd_status
2853 1.101 jakllsch xhci_do_command(struct xhci_softc * const sc, struct xhci_soft_trb * const trb,
2854 1.34 skrll int timeout)
2855 1.34 skrll {
2856 1.34 skrll
2857 1.34 skrll mutex_enter(&sc->sc_lock);
2858 1.38 skrll usbd_status ret = xhci_do_command_locked(sc, trb, timeout);
2859 1.1 jakllsch mutex_exit(&sc->sc_lock);
2860 1.34 skrll
2861 1.34 skrll return ret;
2862 1.1 jakllsch }
2863 1.1 jakllsch
2864 1.1 jakllsch static usbd_status
2865 1.1 jakllsch xhci_enable_slot(struct xhci_softc * const sc, uint8_t * const slotp)
2866 1.1 jakllsch {
2867 1.101 jakllsch struct xhci_soft_trb trb;
2868 1.1 jakllsch usbd_status err;
2869 1.1 jakllsch
2870 1.27 skrll XHCIHIST_FUNC(); XHCIHIST_CALLED();
2871 1.27 skrll
2872 1.1 jakllsch trb.trb_0 = 0;
2873 1.1 jakllsch trb.trb_2 = 0;
2874 1.1 jakllsch trb.trb_3 = XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_ENABLE_SLOT);
2875 1.1 jakllsch
2876 1.1 jakllsch err = xhci_do_command(sc, &trb, USBD_DEFAULT_TIMEOUT);
2877 1.1 jakllsch if (err != USBD_NORMAL_COMPLETION) {
2878 1.1 jakllsch return err;
2879 1.1 jakllsch }
2880 1.1 jakllsch
2881 1.1 jakllsch *slotp = XHCI_TRB_3_SLOT_GET(trb.trb_3);
2882 1.1 jakllsch
2883 1.1 jakllsch return err;
2884 1.1 jakllsch }
2885 1.1 jakllsch
2886 1.34 skrll /*
2887 1.41 skrll * xHCI 4.6.4
2888 1.41 skrll * Deallocate ring and device/input context DMA buffers, and disable_slot.
2889 1.41 skrll * All endpoints in the slot should be stopped.
2890 1.34 skrll * Should be called with sc_lock held.
2891 1.34 skrll */
2892 1.34 skrll static usbd_status
2893 1.34 skrll xhci_disable_slot(struct xhci_softc * const sc, uint8_t slot)
2894 1.34 skrll {
2895 1.101 jakllsch struct xhci_soft_trb trb;
2896 1.34 skrll struct xhci_slot *xs;
2897 1.34 skrll usbd_status err;
2898 1.34 skrll
2899 1.34 skrll XHCIHIST_FUNC(); XHCIHIST_CALLED();
2900 1.34 skrll
2901 1.34 skrll if (sc->sc_dying)
2902 1.34 skrll return USBD_IOERROR;
2903 1.34 skrll
2904 1.34 skrll trb.trb_0 = 0;
2905 1.34 skrll trb.trb_2 = 0;
2906 1.101 jakllsch trb.trb_3 = XHCI_TRB_3_SLOT_SET(slot) |
2907 1.101 jakllsch XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_DISABLE_SLOT);
2908 1.34 skrll
2909 1.34 skrll err = xhci_do_command_locked(sc, &trb, USBD_DEFAULT_TIMEOUT);
2910 1.34 skrll
2911 1.34 skrll if (!err) {
2912 1.34 skrll xs = &sc->sc_slots[slot];
2913 1.34 skrll if (xs->xs_idx != 0) {
2914 1.123 skrll xhci_free_slot(sc, xs);
2915 1.34 skrll xhci_set_dcba(sc, 0, slot);
2916 1.34 skrll memset(xs, 0, sizeof(*xs));
2917 1.34 skrll }
2918 1.34 skrll }
2919 1.34 skrll
2920 1.34 skrll return err;
2921 1.34 skrll }
2922 1.34 skrll
2923 1.34 skrll /*
2924 1.41 skrll * Set address of device and transition slot state from ENABLED to ADDRESSED
2925 1.41 skrll * if Block Setaddress Request (BSR) is false.
2926 1.41 skrll * If BSR==true, transition slot state from ENABLED to DEFAULT.
2927 1.34 skrll * see xHCI 1.1 4.5.3, 3.3.4
2928 1.41 skrll * Should be called without sc_lock held.
2929 1.34 skrll */
2930 1.1 jakllsch static usbd_status
2931 1.1 jakllsch xhci_address_device(struct xhci_softc * const sc,
2932 1.1 jakllsch uint64_t icp, uint8_t slot_id, bool bsr)
2933 1.1 jakllsch {
2934 1.101 jakllsch struct xhci_soft_trb trb;
2935 1.1 jakllsch usbd_status err;
2936 1.1 jakllsch
2937 1.111 mrg XHCIHIST_FUNC();
2938 1.114 mrg if (bsr) {
2939 1.120 christos XHCIHIST_CALLARGS("icp %#jx slot %#jx with bsr",
2940 1.112 mrg icp, slot_id, 0, 0);
2941 1.114 mrg } else {
2942 1.120 christos XHCIHIST_CALLARGS("icp %#jx slot %#jx nobsr",
2943 1.112 mrg icp, slot_id, 0, 0);
2944 1.114 mrg }
2945 1.27 skrll
2946 1.1 jakllsch trb.trb_0 = icp;
2947 1.1 jakllsch trb.trb_2 = 0;
2948 1.1 jakllsch trb.trb_3 = XHCI_TRB_3_SLOT_SET(slot_id) |
2949 1.1 jakllsch XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_ADDRESS_DEVICE) |
2950 1.1 jakllsch (bsr ? XHCI_TRB_3_BSR_BIT : 0);
2951 1.1 jakllsch
2952 1.1 jakllsch err = xhci_do_command(sc, &trb, USBD_DEFAULT_TIMEOUT);
2953 1.34 skrll
2954 1.34 skrll if (XHCI_TRB_2_ERROR_GET(trb.trb_2) == XHCI_TRB_ERROR_NO_SLOTS)
2955 1.34 skrll err = USBD_NO_ADDR;
2956 1.34 skrll
2957 1.1 jakllsch return err;
2958 1.1 jakllsch }
2959 1.1 jakllsch
2960 1.1 jakllsch static usbd_status
2961 1.1 jakllsch xhci_update_ep0_mps(struct xhci_softc * const sc,
2962 1.1 jakllsch struct xhci_slot * const xs, u_int mps)
2963 1.1 jakllsch {
2964 1.101 jakllsch struct xhci_soft_trb trb;
2965 1.1 jakllsch usbd_status err;
2966 1.1 jakllsch uint32_t * cp;
2967 1.1 jakllsch
2968 1.111 mrg XHCIHIST_FUNC();
2969 1.111 mrg XHCIHIST_CALLARGS("slot %ju mps %ju", xs->xs_idx, mps, 0, 0);
2970 1.1 jakllsch
2971 1.1 jakllsch cp = xhci_slot_get_icv(sc, xs, XHCI_ICI_INPUT_CONTROL);
2972 1.1 jakllsch cp[0] = htole32(0);
2973 1.1 jakllsch cp[1] = htole32(XHCI_INCTX_1_ADD_MASK(XHCI_DCI_EP_CONTROL));
2974 1.1 jakllsch
2975 1.1 jakllsch cp = xhci_slot_get_icv(sc, xs, xhci_dci_to_ici(XHCI_DCI_EP_CONTROL));
2976 1.1 jakllsch cp[1] = htole32(XHCI_EPCTX_1_MAXP_SIZE_SET(mps));
2977 1.1 jakllsch
2978 1.1 jakllsch /* sync input contexts before they are read from memory */
2979 1.1 jakllsch usb_syncmem(&xs->xs_ic_dma, 0, sc->sc_pgsz, BUS_DMASYNC_PREWRITE);
2980 1.79 christos HEXDUMP("input context", xhci_slot_get_icv(sc, xs, 0),
2981 1.1 jakllsch sc->sc_ctxsz * 4);
2982 1.1 jakllsch
2983 1.1 jakllsch trb.trb_0 = xhci_slot_get_icp(sc, xs, 0);
2984 1.1 jakllsch trb.trb_2 = 0;
2985 1.1 jakllsch trb.trb_3 = XHCI_TRB_3_SLOT_SET(xs->xs_idx) |
2986 1.1 jakllsch XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_EVALUATE_CTX);
2987 1.1 jakllsch
2988 1.1 jakllsch err = xhci_do_command(sc, &trb, USBD_DEFAULT_TIMEOUT);
2989 1.1 jakllsch return err;
2990 1.1 jakllsch }
2991 1.1 jakllsch
2992 1.1 jakllsch static void
2993 1.1 jakllsch xhci_set_dcba(struct xhci_softc * const sc, uint64_t dcba, int si)
2994 1.1 jakllsch {
2995 1.1 jakllsch uint64_t * const dcbaa = KERNADDR(&sc->sc_dcbaa_dma, 0);
2996 1.1 jakllsch
2997 1.111 mrg XHCIHIST_FUNC();
2998 1.121 christos XHCIHIST_CALLARGS("dcbaa %#jx dc 0x%016jx slot %jd",
2999 1.75 pgoyette (uintptr_t)&dcbaa[si], dcba, si, 0);
3000 1.1 jakllsch
3001 1.5 matt dcbaa[si] = htole64(dcba);
3002 1.1 jakllsch usb_syncmem(&sc->sc_dcbaa_dma, si * sizeof(uint64_t), sizeof(uint64_t),
3003 1.1 jakllsch BUS_DMASYNC_PREWRITE);
3004 1.1 jakllsch }
3005 1.1 jakllsch
3006 1.34 skrll /*
3007 1.48 skrll * Allocate device and input context DMA buffer, and
3008 1.48 skrll * TRB DMA buffer for each endpoint.
3009 1.34 skrll */
3010 1.1 jakllsch static usbd_status
3011 1.48 skrll xhci_init_slot(struct usbd_device *dev, uint32_t slot)
3012 1.1 jakllsch {
3013 1.34 skrll struct xhci_softc * const sc = XHCI_BUS2SC(dev->ud_bus);
3014 1.1 jakllsch struct xhci_slot *xs;
3015 1.1 jakllsch usbd_status err;
3016 1.1 jakllsch
3017 1.111 mrg XHCIHIST_FUNC();
3018 1.111 mrg XHCIHIST_CALLARGS("slot %ju", slot, 0, 0, 0);
3019 1.1 jakllsch
3020 1.1 jakllsch xs = &sc->sc_slots[slot];
3021 1.1 jakllsch
3022 1.1 jakllsch /* allocate contexts */
3023 1.1 jakllsch err = usb_allocmem(&sc->sc_bus, sc->sc_pgsz, sc->sc_pgsz,
3024 1.124 skrll USBMALLOC_COHERENT, &xs->xs_dc_dma);
3025 1.123 skrll if (err) {
3026 1.123 skrll DPRINTFN(1, "failed to allocmem output device context %jd",
3027 1.123 skrll err, 0, 0, 0);
3028 1.1 jakllsch return err;
3029 1.123 skrll }
3030 1.1 jakllsch memset(KERNADDR(&xs->xs_dc_dma, 0), 0, sc->sc_pgsz);
3031 1.1 jakllsch
3032 1.1 jakllsch err = usb_allocmem(&sc->sc_bus, sc->sc_pgsz, sc->sc_pgsz,
3033 1.124 skrll USBMALLOC_COHERENT, &xs->xs_ic_dma);
3034 1.123 skrll if (err) {
3035 1.123 skrll DPRINTFN(1, "failed to allocmem input device context %jd",
3036 1.123 skrll err, 0, 0, 0);
3037 1.34 skrll goto bad1;
3038 1.123 skrll }
3039 1.1 jakllsch memset(KERNADDR(&xs->xs_ic_dma, 0), 0, sc->sc_pgsz);
3040 1.1 jakllsch
3041 1.123 skrll memset(&xs->xs_xr[0], 0, sizeof(xs->xs_xr));
3042 1.123 skrll xs->xs_idx = slot;
3043 1.48 skrll
3044 1.123 skrll return USBD_NORMAL_COMPLETION;
3045 1.48 skrll
3046 1.48 skrll bad1:
3047 1.48 skrll usb_freemem(&sc->sc_bus, &xs->xs_dc_dma);
3048 1.48 skrll xs->xs_idx = 0;
3049 1.48 skrll return err;
3050 1.48 skrll }
3051 1.48 skrll
3052 1.48 skrll static void
3053 1.123 skrll xhci_free_slot(struct xhci_softc *sc, struct xhci_slot *xs)
3054 1.48 skrll {
3055 1.48 skrll u_int dci;
3056 1.48 skrll
3057 1.111 mrg XHCIHIST_FUNC();
3058 1.123 skrll XHCIHIST_CALLARGS("slot %ju", xs->xs_idx, 0, 0, 0);
3059 1.48 skrll
3060 1.123 skrll /* deallocate all allocated rings in the slot */
3061 1.123 skrll for (dci = XHCI_DCI_SLOT; dci <= XHCI_MAX_DCI; dci++) {
3062 1.123 skrll if (xs->xs_xr[dci] != NULL)
3063 1.123 skrll xhci_ring_free(sc, &xs->xs_xr[dci]);
3064 1.48 skrll }
3065 1.48 skrll usb_freemem(&sc->sc_bus, &xs->xs_ic_dma);
3066 1.48 skrll usb_freemem(&sc->sc_bus, &xs->xs_dc_dma);
3067 1.48 skrll xs->xs_idx = 0;
3068 1.48 skrll }
3069 1.48 skrll
3070 1.48 skrll /*
3071 1.48 skrll * Setup slot context, set Device Context Base Address, and issue
3072 1.48 skrll * Set Address Device command.
3073 1.48 skrll */
3074 1.48 skrll static usbd_status
3075 1.51 skrll xhci_set_address(struct usbd_device *dev, uint32_t slot, bool bsr)
3076 1.48 skrll {
3077 1.48 skrll struct xhci_softc * const sc = XHCI_BUS2SC(dev->ud_bus);
3078 1.48 skrll struct xhci_slot *xs;
3079 1.48 skrll usbd_status err;
3080 1.51 skrll
3081 1.111 mrg XHCIHIST_FUNC();
3082 1.111 mrg XHCIHIST_CALLARGS("slot %ju bsr %ju", slot, bsr, 0, 0);
3083 1.51 skrll
3084 1.51 skrll xs = &sc->sc_slots[slot];
3085 1.51 skrll
3086 1.51 skrll xhci_setup_ctx(dev->ud_pipe0);
3087 1.51 skrll
3088 1.79 christos HEXDUMP("input context", xhci_slot_get_icv(sc, xs, 0),
3089 1.51 skrll sc->sc_ctxsz * 3);
3090 1.51 skrll
3091 1.51 skrll xhci_set_dcba(sc, DMAADDR(&xs->xs_dc_dma, 0), slot);
3092 1.51 skrll
3093 1.51 skrll err = xhci_address_device(sc, xhci_slot_get_icp(sc, xs, 0), slot, bsr);
3094 1.51 skrll
3095 1.51 skrll usb_syncmem(&xs->xs_dc_dma, 0, sc->sc_pgsz, BUS_DMASYNC_POSTREAD);
3096 1.79 christos HEXDUMP("output context", xhci_slot_get_dcv(sc, xs, 0),
3097 1.51 skrll sc->sc_ctxsz * 2);
3098 1.51 skrll
3099 1.51 skrll return err;
3100 1.51 skrll }
3101 1.51 skrll
3102 1.51 skrll /*
3103 1.51 skrll * 4.8.2, 6.2.3.2
3104 1.51 skrll * construct slot/endpoint context parameters and do syncmem
3105 1.51 skrll */
3106 1.51 skrll static void
3107 1.51 skrll xhci_setup_ctx(struct usbd_pipe *pipe)
3108 1.51 skrll {
3109 1.51 skrll struct xhci_softc * const sc = XHCI_PIPE2SC(pipe);
3110 1.51 skrll struct usbd_device *dev = pipe->up_dev;
3111 1.51 skrll struct xhci_slot * const xs = dev->ud_hcpriv;
3112 1.51 skrll usb_endpoint_descriptor_t * const ed = pipe->up_endpoint->ue_edesc;
3113 1.51 skrll const u_int dci = xhci_ep_get_dci(ed);
3114 1.51 skrll const uint8_t xfertype = UE_GET_XFERTYPE(ed->bmAttributes);
3115 1.48 skrll uint32_t *cp;
3116 1.51 skrll uint16_t mps = UGETW(ed->wMaxPacketSize);
3117 1.51 skrll uint8_t speed = dev->ud_speed;
3118 1.51 skrll uint8_t ival = ed->bInterval;
3119 1.48 skrll
3120 1.111 mrg XHCIHIST_FUNC();
3121 1.111 mrg XHCIHIST_CALLARGS("pipe %#jx: slot %ju dci %ju speed %ju",
3122 1.75 pgoyette (uintptr_t)pipe, xs->xs_idx, dci, speed);
3123 1.48 skrll
3124 1.1 jakllsch /* set up initial input control context */
3125 1.1 jakllsch cp = xhci_slot_get_icv(sc, xs, XHCI_ICI_INPUT_CONTROL);
3126 1.1 jakllsch cp[0] = htole32(0);
3127 1.51 skrll cp[1] = htole32(XHCI_INCTX_1_ADD_MASK(dci));
3128 1.71 skrll cp[1] |= htole32(XHCI_INCTX_1_ADD_MASK(XHCI_DCI_SLOT));
3129 1.51 skrll cp[7] = htole32(0);
3130 1.1 jakllsch
3131 1.1 jakllsch /* set up input slot context */
3132 1.1 jakllsch cp = xhci_slot_get_icv(sc, xs, xhci_dci_to_ici(XHCI_DCI_SLOT));
3133 1.51 skrll cp[0] =
3134 1.51 skrll XHCI_SCTX_0_CTX_NUM_SET(dci) |
3135 1.51 skrll XHCI_SCTX_0_SPEED_SET(xhci_speed2xspeed(speed));
3136 1.51 skrll cp[1] = 0;
3137 1.51 skrll cp[2] = XHCI_SCTX_2_IRQ_TARGET_SET(0);
3138 1.51 skrll cp[3] = 0;
3139 1.51 skrll xhci_setup_route(pipe, cp);
3140 1.51 skrll xhci_setup_tthub(pipe, cp);
3141 1.51 skrll
3142 1.51 skrll cp[0] = htole32(cp[0]);
3143 1.51 skrll cp[1] = htole32(cp[1]);
3144 1.51 skrll cp[2] = htole32(cp[2]);
3145 1.51 skrll cp[3] = htole32(cp[3]);
3146 1.51 skrll
3147 1.51 skrll /* set up input endpoint context */
3148 1.51 skrll cp = xhci_slot_get_icv(sc, xs, xhci_dci_to_ici(dci));
3149 1.51 skrll cp[0] =
3150 1.51 skrll XHCI_EPCTX_0_EPSTATE_SET(0) |
3151 1.51 skrll XHCI_EPCTX_0_MULT_SET(0) |
3152 1.51 skrll XHCI_EPCTX_0_MAXP_STREAMS_SET(0) |
3153 1.51 skrll XHCI_EPCTX_0_LSA_SET(0) |
3154 1.51 skrll XHCI_EPCTX_0_MAX_ESIT_PAYLOAD_HI_SET(0);
3155 1.51 skrll cp[1] =
3156 1.51 skrll XHCI_EPCTX_1_EPTYPE_SET(xhci_ep_get_type(ed)) |
3157 1.51 skrll XHCI_EPCTX_1_HID_SET(0) |
3158 1.51 skrll XHCI_EPCTX_1_MAXB_SET(0);
3159 1.51 skrll
3160 1.51 skrll if (xfertype != UE_ISOCHRONOUS)
3161 1.51 skrll cp[1] |= XHCI_EPCTX_1_CERR_SET(3);
3162 1.51 skrll
3163 1.51 skrll if (xfertype == UE_CONTROL)
3164 1.51 skrll cp[4] = XHCI_EPCTX_4_AVG_TRB_LEN_SET(8); /* 6.2.3 */
3165 1.51 skrll else if (USB_IS_SS(speed))
3166 1.51 skrll cp[4] = XHCI_EPCTX_4_AVG_TRB_LEN_SET(mps);
3167 1.51 skrll else
3168 1.51 skrll cp[4] = XHCI_EPCTX_4_AVG_TRB_LEN_SET(UE_GET_SIZE(mps));
3169 1.51 skrll
3170 1.51 skrll xhci_setup_maxburst(pipe, cp);
3171 1.51 skrll
3172 1.51 skrll switch (xfertype) {
3173 1.51 skrll case UE_CONTROL:
3174 1.51 skrll break;
3175 1.51 skrll case UE_BULK:
3176 1.51 skrll /* XXX Set MaxPStreams, HID, and LSA if streams enabled */
3177 1.51 skrll break;
3178 1.51 skrll case UE_INTERRUPT:
3179 1.51 skrll if (pipe->up_interval != USBD_DEFAULT_INTERVAL)
3180 1.51 skrll ival = pipe->up_interval;
3181 1.51 skrll
3182 1.51 skrll ival = xhci_bival2ival(ival, speed);
3183 1.51 skrll cp[0] |= XHCI_EPCTX_0_IVAL_SET(ival);
3184 1.51 skrll break;
3185 1.51 skrll case UE_ISOCHRONOUS:
3186 1.51 skrll if (pipe->up_interval != USBD_DEFAULT_INTERVAL)
3187 1.51 skrll ival = pipe->up_interval;
3188 1.51 skrll
3189 1.51 skrll /* xHCI 6.2.3.6 Table 65, USB 2.0 9.6.6 */
3190 1.51 skrll if (speed == USB_SPEED_FULL)
3191 1.51 skrll ival += 3; /* 1ms -> 125us */
3192 1.51 skrll ival--;
3193 1.51 skrll cp[0] |= XHCI_EPCTX_0_IVAL_SET(ival);
3194 1.51 skrll break;
3195 1.51 skrll default:
3196 1.51 skrll break;
3197 1.51 skrll }
3198 1.75 pgoyette DPRINTFN(4, "setting ival %ju MaxBurst %#jx",
3199 1.53 skrll XHCI_EPCTX_0_IVAL_GET(cp[0]), XHCI_EPCTX_1_MAXB_GET(cp[1]), 0, 0);
3200 1.1 jakllsch
3201 1.55 skrll /* rewind TR dequeue pointer in xHC */
3202 1.1 jakllsch /* can't use xhci_ep_get_dci() yet? */
3203 1.1 jakllsch *(uint64_t *)(&cp[2]) = htole64(
3204 1.123 skrll xhci_ring_trbp(xs->xs_xr[dci], 0) |
3205 1.1 jakllsch XHCI_EPCTX_2_DCS_SET(1));
3206 1.51 skrll
3207 1.51 skrll cp[0] = htole32(cp[0]);
3208 1.51 skrll cp[1] = htole32(cp[1]);
3209 1.51 skrll cp[4] = htole32(cp[4]);
3210 1.1 jakllsch
3211 1.55 skrll /* rewind TR dequeue pointer in driver */
3212 1.123 skrll struct xhci_ring *xr = xs->xs_xr[dci];
3213 1.55 skrll mutex_enter(&xr->xr_lock);
3214 1.55 skrll xhci_host_dequeue(xr);
3215 1.55 skrll mutex_exit(&xr->xr_lock);
3216 1.55 skrll
3217 1.1 jakllsch /* sync input contexts before they are read from memory */
3218 1.1 jakllsch usb_syncmem(&xs->xs_ic_dma, 0, sc->sc_pgsz, BUS_DMASYNC_PREWRITE);
3219 1.51 skrll }
3220 1.51 skrll
3221 1.51 skrll /*
3222 1.51 skrll * Setup route string and roothub port of given device for slot context
3223 1.51 skrll */
3224 1.51 skrll static void
3225 1.51 skrll xhci_setup_route(struct usbd_pipe *pipe, uint32_t *cp)
3226 1.51 skrll {
3227 1.68 skrll struct xhci_softc * const sc = XHCI_PIPE2SC(pipe);
3228 1.51 skrll struct usbd_device *dev = pipe->up_dev;
3229 1.51 skrll struct usbd_port *up = dev->ud_powersrc;
3230 1.51 skrll struct usbd_device *hub;
3231 1.51 skrll struct usbd_device *adev;
3232 1.51 skrll uint8_t rhport = 0;
3233 1.51 skrll uint32_t route = 0;
3234 1.51 skrll
3235 1.51 skrll XHCIHIST_FUNC(); XHCIHIST_CALLED();
3236 1.51 skrll
3237 1.51 skrll /* Locate root hub port and Determine route string */
3238 1.51 skrll /* 4.3.3 route string does not include roothub port */
3239 1.51 skrll for (hub = dev; hub != NULL; hub = hub->ud_myhub) {
3240 1.51 skrll uint32_t dep;
3241 1.51 skrll
3242 1.122 christos DPRINTFN(4, "hub %#jx depth %jd upport %#jx upportno %jd",
3243 1.75 pgoyette (uintptr_t)hub, hub->ud_depth, (uintptr_t)hub->ud_powersrc,
3244 1.75 pgoyette hub->ud_powersrc ? (uintptr_t)hub->ud_powersrc->up_portno :
3245 1.75 pgoyette -1);
3246 1.51 skrll
3247 1.51 skrll if (hub->ud_powersrc == NULL)
3248 1.51 skrll break;
3249 1.51 skrll dep = hub->ud_depth;
3250 1.51 skrll if (dep == 0)
3251 1.51 skrll break;
3252 1.51 skrll rhport = hub->ud_powersrc->up_portno;
3253 1.51 skrll if (dep > USB_HUB_MAX_DEPTH)
3254 1.51 skrll continue;
3255 1.51 skrll
3256 1.51 skrll route |=
3257 1.51 skrll (rhport > UHD_SS_NPORTS_MAX ? UHD_SS_NPORTS_MAX : rhport)
3258 1.51 skrll << ((dep - 1) * 4);
3259 1.51 skrll }
3260 1.51 skrll route = route >> 4;
3261 1.68 skrll size_t bn = hub == sc->sc_bus.ub_roothub ? 0 : 1;
3262 1.51 skrll
3263 1.51 skrll /* Locate port on upstream high speed hub */
3264 1.51 skrll for (adev = dev, hub = up->up_parent;
3265 1.51 skrll hub != NULL && hub->ud_speed != USB_SPEED_HIGH;
3266 1.51 skrll adev = hub, hub = hub->ud_myhub)
3267 1.51 skrll ;
3268 1.51 skrll if (hub) {
3269 1.51 skrll int p;
3270 1.119 skrll for (p = 1; p <= hub->ud_hub->uh_hubdesc.bNbrPorts; p++) {
3271 1.119 skrll if (hub->ud_hub->uh_ports[p - 1].up_dev == adev) {
3272 1.119 skrll dev->ud_myhsport = &hub->ud_hub->uh_ports[p - 1];
3273 1.51 skrll goto found;
3274 1.51 skrll }
3275 1.51 skrll }
3276 1.68 skrll panic("%s: cannot find HS port", __func__);
3277 1.51 skrll found:
3278 1.75 pgoyette DPRINTFN(4, "high speed port %jd", p, 0, 0, 0);
3279 1.51 skrll } else {
3280 1.51 skrll dev->ud_myhsport = NULL;
3281 1.51 skrll }
3282 1.51 skrll
3283 1.68 skrll const size_t ctlrport = xhci_rhport2ctlrport(sc, bn, rhport);
3284 1.68 skrll
3285 1.75 pgoyette DPRINTFN(4, "rhport %ju ctlrport %ju Route %05jx hub %#jx", rhport,
3286 1.75 pgoyette ctlrport, route, (uintptr_t)hub);
3287 1.68 skrll
3288 1.51 skrll cp[0] |= XHCI_SCTX_0_ROUTE_SET(route);
3289 1.68 skrll cp[1] |= XHCI_SCTX_1_RH_PORT_SET(ctlrport);
3290 1.51 skrll }
3291 1.51 skrll
3292 1.51 skrll /*
3293 1.51 skrll * Setup whether device is hub, whether device uses MTT, and
3294 1.51 skrll * TT informations if it uses MTT.
3295 1.51 skrll */
3296 1.51 skrll static void
3297 1.51 skrll xhci_setup_tthub(struct usbd_pipe *pipe, uint32_t *cp)
3298 1.51 skrll {
3299 1.51 skrll struct usbd_device *dev = pipe->up_dev;
3300 1.78 christos struct usbd_port *myhsport = dev->ud_myhsport;
3301 1.51 skrll usb_device_descriptor_t * const dd = &dev->ud_ddesc;
3302 1.51 skrll uint32_t speed = dev->ud_speed;
3303 1.83 skrll uint8_t rhaddr = dev->ud_bus->ub_rhaddr;
3304 1.51 skrll uint8_t tthubslot, ttportnum;
3305 1.51 skrll bool ishub;
3306 1.51 skrll bool usemtt;
3307 1.51 skrll
3308 1.111 mrg XHCIHIST_FUNC();
3309 1.51 skrll
3310 1.51 skrll /*
3311 1.51 skrll * 6.2.2, Table 57-60, 6.2.2.1, 6.2.2.2
3312 1.51 skrll * tthubslot:
3313 1.51 skrll * This is the slot ID of parent HS hub
3314 1.51 skrll * if LS/FS device is connected && connected through HS hub.
3315 1.51 skrll * This is 0 if device is not LS/FS device ||
3316 1.51 skrll * parent hub is not HS hub ||
3317 1.51 skrll * attached to root hub.
3318 1.51 skrll * ttportnum:
3319 1.51 skrll * This is the downstream facing port of parent HS hub
3320 1.51 skrll * if LS/FS device is connected.
3321 1.51 skrll * This is 0 if device is not LS/FS device ||
3322 1.51 skrll * parent hub is not HS hub ||
3323 1.51 skrll * attached to root hub.
3324 1.51 skrll */
3325 1.83 skrll if (myhsport &&
3326 1.83 skrll myhsport->up_parent->ud_addr != rhaddr &&
3327 1.51 skrll (speed == USB_SPEED_LOW || speed == USB_SPEED_FULL)) {
3328 1.78 christos ttportnum = myhsport->up_portno;
3329 1.78 christos tthubslot = myhsport->up_parent->ud_addr;
3330 1.51 skrll } else {
3331 1.51 skrll ttportnum = 0;
3332 1.51 skrll tthubslot = 0;
3333 1.51 skrll }
3334 1.111 mrg XHCIHIST_CALLARGS("myhsport %#jx ttportnum=%jd tthubslot=%jd",
3335 1.78 christos (uintptr_t)myhsport, ttportnum, tthubslot, 0);
3336 1.51 skrll
3337 1.51 skrll /* ishub is valid after reading UDESC_DEVICE */
3338 1.51 skrll ishub = (dd->bDeviceClass == UDCLASS_HUB);
3339 1.51 skrll
3340 1.51 skrll /* dev->ud_hub is valid after reading UDESC_HUB */
3341 1.51 skrll if (ishub && dev->ud_hub) {
3342 1.51 skrll usb_hub_descriptor_t *hd = &dev->ud_hub->uh_hubdesc;
3343 1.51 skrll uint8_t ttt =
3344 1.51 skrll __SHIFTOUT(UGETW(hd->wHubCharacteristics), UHD_TT_THINK);
3345 1.51 skrll
3346 1.51 skrll cp[1] |= XHCI_SCTX_1_NUM_PORTS_SET(hd->bNbrPorts);
3347 1.51 skrll cp[2] |= XHCI_SCTX_2_TT_THINK_TIME_SET(ttt);
3348 1.75 pgoyette DPRINTFN(4, "nports=%jd ttt=%jd", hd->bNbrPorts, ttt, 0, 0);
3349 1.51 skrll }
3350 1.51 skrll
3351 1.83 skrll #define IS_MTTHUB(dd) \
3352 1.83 skrll ((dd)->bDeviceProtocol == UDPROTO_HSHUBMTT)
3353 1.51 skrll
3354 1.51 skrll /*
3355 1.51 skrll * MTT flag is set if
3356 1.83 skrll * 1. this is HS hub && MTTs are supported and enabled; or
3357 1.83 skrll * 2. this is LS or FS device && there is a parent HS hub where MTTs
3358 1.83 skrll * are supported and enabled.
3359 1.83 skrll *
3360 1.83 skrll * XXX enabled is not tested yet
3361 1.51 skrll */
3362 1.83 skrll if (ishub && speed == USB_SPEED_HIGH && IS_MTTHUB(dd))
3363 1.51 skrll usemtt = true;
3364 1.83 skrll else if ((speed == USB_SPEED_LOW || speed == USB_SPEED_FULL) &&
3365 1.83 skrll myhsport &&
3366 1.83 skrll myhsport->up_parent->ud_addr != rhaddr &&
3367 1.83 skrll IS_MTTHUB(&myhsport->up_parent->ud_ddesc))
3368 1.51 skrll usemtt = true;
3369 1.51 skrll else
3370 1.51 skrll usemtt = false;
3371 1.75 pgoyette DPRINTFN(4, "class %ju proto %ju ishub %jd usemtt %jd",
3372 1.51 skrll dd->bDeviceClass, dd->bDeviceProtocol, ishub, usemtt);
3373 1.51 skrll
3374 1.83 skrll #undef IS_MTTHUB
3375 1.51 skrll
3376 1.51 skrll cp[0] |=
3377 1.51 skrll XHCI_SCTX_0_HUB_SET(ishub ? 1 : 0) |
3378 1.51 skrll XHCI_SCTX_0_MTT_SET(usemtt ? 1 : 0);
3379 1.51 skrll cp[2] |=
3380 1.51 skrll XHCI_SCTX_2_TT_HUB_SID_SET(tthubslot) |
3381 1.51 skrll XHCI_SCTX_2_TT_PORT_NUM_SET(ttportnum);
3382 1.51 skrll }
3383 1.51 skrll
3384 1.51 skrll /* set up params for periodic endpoint */
3385 1.51 skrll static void
3386 1.51 skrll xhci_setup_maxburst(struct usbd_pipe *pipe, uint32_t *cp)
3387 1.51 skrll {
3388 1.134 jakllsch struct xhci_pipe * const xpipe = (struct xhci_pipe *)pipe;
3389 1.51 skrll struct usbd_device *dev = pipe->up_dev;
3390 1.51 skrll usb_endpoint_descriptor_t * const ed = pipe->up_endpoint->ue_edesc;
3391 1.51 skrll const uint8_t xfertype = UE_GET_XFERTYPE(ed->bmAttributes);
3392 1.51 skrll usbd_desc_iter_t iter;
3393 1.51 skrll const usb_cdc_descriptor_t *cdcd;
3394 1.51 skrll uint32_t maxb = 0;
3395 1.51 skrll uint16_t mps = UGETW(ed->wMaxPacketSize);
3396 1.51 skrll uint8_t speed = dev->ud_speed;
3397 1.134 jakllsch uint8_t mult = 0;
3398 1.51 skrll uint8_t ep;
3399 1.51 skrll
3400 1.51 skrll /* config desc is NULL when opening ep0 */
3401 1.51 skrll if (dev == NULL || dev->ud_cdesc == NULL)
3402 1.51 skrll goto no_cdcd;
3403 1.51 skrll cdcd = (const usb_cdc_descriptor_t *)usb_find_desc(dev,
3404 1.51 skrll UDESC_INTERFACE, USBD_CDCSUBTYPE_ANY);
3405 1.51 skrll if (cdcd == NULL)
3406 1.51 skrll goto no_cdcd;
3407 1.51 skrll usb_desc_iter_init(dev, &iter);
3408 1.51 skrll iter.cur = (const void *)cdcd;
3409 1.51 skrll
3410 1.51 skrll /* find endpoint_ss_comp desc for ep of this pipe */
3411 1.51 skrll for (ep = 0;;) {
3412 1.51 skrll cdcd = (const usb_cdc_descriptor_t *)usb_desc_iter_next(&iter);
3413 1.51 skrll if (cdcd == NULL)
3414 1.51 skrll break;
3415 1.51 skrll if (ep == 0 && cdcd->bDescriptorType == UDESC_ENDPOINT) {
3416 1.51 skrll ep = ((const usb_endpoint_descriptor_t *)cdcd)->
3417 1.51 skrll bEndpointAddress;
3418 1.51 skrll if (UE_GET_ADDR(ep) ==
3419 1.51 skrll UE_GET_ADDR(ed->bEndpointAddress)) {
3420 1.51 skrll cdcd = (const usb_cdc_descriptor_t *)
3421 1.51 skrll usb_desc_iter_next(&iter);
3422 1.51 skrll break;
3423 1.51 skrll }
3424 1.51 skrll ep = 0;
3425 1.51 skrll }
3426 1.51 skrll }
3427 1.51 skrll if (cdcd != NULL && cdcd->bDescriptorType == UDESC_ENDPOINT_SS_COMP) {
3428 1.51 skrll const usb_endpoint_ss_comp_descriptor_t * esscd =
3429 1.51 skrll (const usb_endpoint_ss_comp_descriptor_t *)cdcd;
3430 1.51 skrll maxb = esscd->bMaxBurst;
3431 1.134 jakllsch mult = UE_GET_SS_ISO_MULT(esscd->bmAttributes);
3432 1.51 skrll }
3433 1.51 skrll
3434 1.51 skrll no_cdcd:
3435 1.51 skrll /* 6.2.3.4, 4.8.2.4 */
3436 1.51 skrll if (USB_IS_SS(speed)) {
3437 1.60 skrll /* USB 3.1 9.6.6 */
3438 1.51 skrll cp[1] |= XHCI_EPCTX_1_MAXP_SIZE_SET(mps);
3439 1.60 skrll /* USB 3.1 9.6.7 */
3440 1.51 skrll cp[1] |= XHCI_EPCTX_1_MAXB_SET(maxb);
3441 1.51 skrll #ifdef notyet
3442 1.51 skrll if (xfertype == UE_ISOCHRONOUS) {
3443 1.51 skrll }
3444 1.51 skrll if (XHCI_HCC2_LEC(sc->sc_hcc2) != 0) {
3445 1.51 skrll /* use ESIT */
3446 1.51 skrll cp[4] |= XHCI_EPCTX_4_MAX_ESIT_PAYLOAD_SET(x);
3447 1.51 skrll cp[0] |= XHCI_EPCTX_0_MAX_ESIT_PAYLOAD_HI_SET(x);
3448 1.51 skrll
3449 1.51 skrll /* XXX if LEC = 1, set ESIT instead */
3450 1.51 skrll cp[0] |= XHCI_EPCTX_0_MULT_SET(0);
3451 1.51 skrll } else {
3452 1.51 skrll /* use ival */
3453 1.51 skrll }
3454 1.51 skrll #endif
3455 1.51 skrll } else {
3456 1.60 skrll /* USB 2.0 9.6.6 */
3457 1.51 skrll cp[1] |= XHCI_EPCTX_1_MAXP_SIZE_SET(UE_GET_SIZE(mps));
3458 1.1 jakllsch
3459 1.51 skrll /* 6.2.3.4 */
3460 1.51 skrll if (speed == USB_SPEED_HIGH &&
3461 1.51 skrll (xfertype == UE_ISOCHRONOUS || xfertype == UE_INTERRUPT)) {
3462 1.51 skrll maxb = UE_GET_TRANS(mps);
3463 1.51 skrll } else {
3464 1.51 skrll /* LS/FS or HS CTRL or HS BULK */
3465 1.51 skrll maxb = 0;
3466 1.51 skrll }
3467 1.51 skrll cp[1] |= XHCI_EPCTX_1_MAXB_SET(maxb);
3468 1.51 skrll }
3469 1.134 jakllsch xpipe->xp_maxb = maxb + 1;
3470 1.134 jakllsch xpipe->xp_mult = mult + 1;
3471 1.51 skrll }
3472 1.1 jakllsch
3473 1.51 skrll /*
3474 1.51 skrll * Convert endpoint bInterval value to endpoint context interval value
3475 1.51 skrll * for Interrupt pipe.
3476 1.51 skrll * xHCI 6.2.3.6 Table 65, USB 2.0 9.6.6
3477 1.51 skrll */
3478 1.51 skrll static uint32_t
3479 1.51 skrll xhci_bival2ival(uint32_t ival, uint32_t speed)
3480 1.51 skrll {
3481 1.51 skrll if (speed == USB_SPEED_LOW || speed == USB_SPEED_FULL) {
3482 1.51 skrll int i;
3483 1.1 jakllsch
3484 1.51 skrll /*
3485 1.51 skrll * round ival down to "the nearest base 2 multiple of
3486 1.51 skrll * bInterval * 8".
3487 1.51 skrll * bInterval is at most 255 as its type is uByte.
3488 1.51 skrll * 255(ms) = 2040(x 125us) < 2^11, so start with 10.
3489 1.51 skrll */
3490 1.51 skrll for (i = 10; i > 0; i--) {
3491 1.51 skrll if ((ival * 8) >= (1 << i))
3492 1.51 skrll break;
3493 1.51 skrll }
3494 1.51 skrll ival = i;
3495 1.51 skrll } else {
3496 1.51 skrll /* Interval = bInterval-1 for SS/HS */
3497 1.51 skrll ival--;
3498 1.51 skrll }
3499 1.1 jakllsch
3500 1.51 skrll return ival;
3501 1.1 jakllsch }
3502 1.1 jakllsch
3503 1.1 jakllsch /* ----- */
3504 1.1 jakllsch
3505 1.1 jakllsch static void
3506 1.34 skrll xhci_noop(struct usbd_pipe *pipe)
3507 1.1 jakllsch {
3508 1.27 skrll XHCIHIST_FUNC(); XHCIHIST_CALLED();
3509 1.1 jakllsch }
3510 1.1 jakllsch
3511 1.34 skrll /*
3512 1.34 skrll * Process root hub request.
3513 1.34 skrll */
3514 1.34 skrll static int
3515 1.34 skrll xhci_roothub_ctrl(struct usbd_bus *bus, usb_device_request_t *req,
3516 1.34 skrll void *buf, int buflen)
3517 1.1 jakllsch {
3518 1.34 skrll struct xhci_softc * const sc = XHCI_BUS2SC(bus);
3519 1.1 jakllsch usb_port_status_t ps;
3520 1.1 jakllsch int l, totlen = 0;
3521 1.34 skrll uint16_t len, value, index;
3522 1.1 jakllsch int port, i;
3523 1.1 jakllsch uint32_t v;
3524 1.1 jakllsch
3525 1.111 mrg XHCIHIST_FUNC();
3526 1.1 jakllsch
3527 1.1 jakllsch if (sc->sc_dying)
3528 1.34 skrll return -1;
3529 1.1 jakllsch
3530 1.68 skrll size_t bn = bus == &sc->sc_bus ? 0 : 1;
3531 1.68 skrll
3532 1.34 skrll len = UGETW(req->wLength);
3533 1.1 jakllsch value = UGETW(req->wValue);
3534 1.1 jakllsch index = UGETW(req->wIndex);
3535 1.1 jakllsch
3536 1.111 mrg XHCIHIST_CALLARGS("rhreq: %04jx %04jx %04jx %04jx",
3537 1.27 skrll req->bmRequestType | (req->bRequest << 8), value, index, len);
3538 1.1 jakllsch
3539 1.1 jakllsch #define C(x,y) ((x) | ((y) << 8))
3540 1.34 skrll switch (C(req->bRequest, req->bmRequestType)) {
3541 1.1 jakllsch case C(UR_GET_DESCRIPTOR, UT_READ_DEVICE):
3542 1.121 christos DPRINTFN(8, "getdesc: wValue=0x%04jx", value, 0, 0, 0);
3543 1.1 jakllsch if (len == 0)
3544 1.1 jakllsch break;
3545 1.34 skrll switch (value) {
3546 1.34 skrll #define sd ((usb_string_descriptor_t *)buf)
3547 1.34 skrll case C(2, UDESC_STRING):
3548 1.34 skrll /* Product */
3549 1.91 jmcneill totlen = usb_makestrdesc(sd, len, "xHCI root hub");
3550 1.1 jakllsch break;
3551 1.1 jakllsch #undef sd
3552 1.1 jakllsch default:
3553 1.34 skrll /* default from usbroothub */
3554 1.34 skrll return buflen;
3555 1.1 jakllsch }
3556 1.1 jakllsch break;
3557 1.34 skrll
3558 1.1 jakllsch /* Hub requests */
3559 1.1 jakllsch case C(UR_CLEAR_FEATURE, UT_WRITE_CLASS_DEVICE):
3560 1.1 jakllsch break;
3561 1.34 skrll /* Clear Port Feature request */
3562 1.68 skrll case C(UR_CLEAR_FEATURE, UT_WRITE_CLASS_OTHER): {
3563 1.68 skrll const size_t cp = xhci_rhport2ctlrport(sc, bn, index);
3564 1.68 skrll
3565 1.75 pgoyette DPRINTFN(4, "UR_CLEAR_PORT_FEAT bp=%jd feat=%jd bus=%jd cp=%jd",
3566 1.68 skrll index, value, bn, cp);
3567 1.68 skrll if (index < 1 || index > sc->sc_rhportcount[bn]) {
3568 1.34 skrll return -1;
3569 1.1 jakllsch }
3570 1.68 skrll port = XHCI_PORTSC(cp);
3571 1.1 jakllsch v = xhci_op_read_4(sc, port);
3572 1.121 christos DPRINTFN(4, "portsc=0x%08jx", v, 0, 0, 0);
3573 1.1 jakllsch v &= ~XHCI_PS_CLEAR;
3574 1.1 jakllsch switch (value) {
3575 1.1 jakllsch case UHF_PORT_ENABLE:
3576 1.34 skrll xhci_op_write_4(sc, port, v & ~XHCI_PS_PED);
3577 1.1 jakllsch break;
3578 1.1 jakllsch case UHF_PORT_SUSPEND:
3579 1.34 skrll return -1;
3580 1.1 jakllsch case UHF_PORT_POWER:
3581 1.1 jakllsch break;
3582 1.1 jakllsch case UHF_PORT_TEST:
3583 1.1 jakllsch case UHF_PORT_INDICATOR:
3584 1.34 skrll return -1;
3585 1.1 jakllsch case UHF_C_PORT_CONNECTION:
3586 1.1 jakllsch xhci_op_write_4(sc, port, v | XHCI_PS_CSC);
3587 1.1 jakllsch break;
3588 1.1 jakllsch case UHF_C_PORT_ENABLE:
3589 1.1 jakllsch case UHF_C_PORT_SUSPEND:
3590 1.1 jakllsch case UHF_C_PORT_OVER_CURRENT:
3591 1.34 skrll return -1;
3592 1.34 skrll case UHF_C_BH_PORT_RESET:
3593 1.34 skrll xhci_op_write_4(sc, port, v | XHCI_PS_WRC);
3594 1.34 skrll break;
3595 1.1 jakllsch case UHF_C_PORT_RESET:
3596 1.1 jakllsch xhci_op_write_4(sc, port, v | XHCI_PS_PRC);
3597 1.1 jakllsch break;
3598 1.34 skrll case UHF_C_PORT_LINK_STATE:
3599 1.34 skrll xhci_op_write_4(sc, port, v | XHCI_PS_PLC);
3600 1.34 skrll break;
3601 1.34 skrll case UHF_C_PORT_CONFIG_ERROR:
3602 1.34 skrll xhci_op_write_4(sc, port, v | XHCI_PS_CEC);
3603 1.34 skrll break;
3604 1.1 jakllsch default:
3605 1.34 skrll return -1;
3606 1.1 jakllsch }
3607 1.1 jakllsch break;
3608 1.68 skrll }
3609 1.1 jakllsch case C(UR_GET_DESCRIPTOR, UT_READ_CLASS_DEVICE):
3610 1.1 jakllsch if (len == 0)
3611 1.1 jakllsch break;
3612 1.1 jakllsch if ((value & 0xff) != 0) {
3613 1.34 skrll return -1;
3614 1.1 jakllsch }
3615 1.34 skrll usb_hub_descriptor_t hubd;
3616 1.34 skrll
3617 1.98 riastrad totlen = uimin(buflen, sizeof(hubd));
3618 1.34 skrll memcpy(&hubd, buf, totlen);
3619 1.68 skrll hubd.bNbrPorts = sc->sc_rhportcount[bn];
3620 1.1 jakllsch USETW(hubd.wHubCharacteristics, UHD_PWR_NO_SWITCH);
3621 1.1 jakllsch hubd.bPwrOn2PwrGood = 200;
3622 1.68 skrll for (i = 0, l = sc->sc_rhportcount[bn]; l > 0; i++, l -= 8) {
3623 1.68 skrll /* XXX can't find out? */
3624 1.68 skrll hubd.DeviceRemovable[i++] = 0;
3625 1.68 skrll }
3626 1.3 skrll hubd.bDescLength = USB_HUB_DESCRIPTOR_SIZE + i;
3627 1.98 riastrad totlen = uimin(totlen, hubd.bDescLength);
3628 1.34 skrll memcpy(buf, &hubd, totlen);
3629 1.1 jakllsch break;
3630 1.1 jakllsch case C(UR_GET_STATUS, UT_READ_CLASS_DEVICE):
3631 1.1 jakllsch if (len != 4) {
3632 1.34 skrll return -1;
3633 1.1 jakllsch }
3634 1.1 jakllsch memset(buf, 0, len); /* ? XXX */
3635 1.1 jakllsch totlen = len;
3636 1.1 jakllsch break;
3637 1.34 skrll /* Get Port Status request */
3638 1.68 skrll case C(UR_GET_STATUS, UT_READ_CLASS_OTHER): {
3639 1.68 skrll const size_t cp = xhci_rhport2ctlrport(sc, bn, index);
3640 1.68 skrll
3641 1.75 pgoyette DPRINTFN(8, "get port status bn=%jd i=%jd cp=%ju",
3642 1.75 pgoyette bn, index, cp, 0);
3643 1.68 skrll if (index < 1 || index > sc->sc_rhportcount[bn]) {
3644 1.111 mrg DPRINTFN(5, "bad get port status: index=%jd bn=%jd "
3645 1.111 mrg "portcount=%jd",
3646 1.111 mrg index, bn, sc->sc_rhportcount[bn], 0);
3647 1.34 skrll return -1;
3648 1.1 jakllsch }
3649 1.1 jakllsch if (len != 4) {
3650 1.120 christos DPRINTFN(5, "bad get port status: len %jd != 4",
3651 1.111 mrg len, 0, 0, 0);
3652 1.34 skrll return -1;
3653 1.1 jakllsch }
3654 1.68 skrll v = xhci_op_read_4(sc, XHCI_PORTSC(cp));
3655 1.121 christos DPRINTFN(4, "getrhportsc %jd 0x%08jx", cp, v, 0, 0);
3656 1.34 skrll i = xhci_xspeed2psspeed(XHCI_PS_SPEED_GET(v));
3657 1.1 jakllsch if (v & XHCI_PS_CCS) i |= UPS_CURRENT_CONNECT_STATUS;
3658 1.1 jakllsch if (v & XHCI_PS_PED) i |= UPS_PORT_ENABLED;
3659 1.1 jakllsch if (v & XHCI_PS_OCA) i |= UPS_OVERCURRENT_INDICATOR;
3660 1.1 jakllsch //if (v & XHCI_PS_SUSP) i |= UPS_SUSPEND;
3661 1.1 jakllsch if (v & XHCI_PS_PR) i |= UPS_RESET;
3662 1.34 skrll if (v & XHCI_PS_PP) {
3663 1.34 skrll if (i & UPS_OTHER_SPEED)
3664 1.34 skrll i |= UPS_PORT_POWER_SS;
3665 1.34 skrll else
3666 1.34 skrll i |= UPS_PORT_POWER;
3667 1.34 skrll }
3668 1.34 skrll if (i & UPS_OTHER_SPEED)
3669 1.34 skrll i |= UPS_PORT_LS_SET(XHCI_PS_PLS_GET(v));
3670 1.34 skrll if (sc->sc_vendor_port_status)
3671 1.34 skrll i = sc->sc_vendor_port_status(sc, v, i);
3672 1.1 jakllsch USETW(ps.wPortStatus, i);
3673 1.1 jakllsch i = 0;
3674 1.1 jakllsch if (v & XHCI_PS_CSC) i |= UPS_C_CONNECT_STATUS;
3675 1.1 jakllsch if (v & XHCI_PS_PEC) i |= UPS_C_PORT_ENABLED;
3676 1.1 jakllsch if (v & XHCI_PS_OCC) i |= UPS_C_OVERCURRENT_INDICATOR;
3677 1.1 jakllsch if (v & XHCI_PS_PRC) i |= UPS_C_PORT_RESET;
3678 1.34 skrll if (v & XHCI_PS_WRC) i |= UPS_C_BH_PORT_RESET;
3679 1.34 skrll if (v & XHCI_PS_PLC) i |= UPS_C_PORT_LINK_STATE;
3680 1.34 skrll if (v & XHCI_PS_CEC) i |= UPS_C_PORT_CONFIG_ERROR;
3681 1.1 jakllsch USETW(ps.wPortChange, i);
3682 1.98 riastrad totlen = uimin(len, sizeof(ps));
3683 1.34 skrll memcpy(buf, &ps, totlen);
3684 1.120 christos DPRINTFN(5, "get port status: wPortStatus %#jx wPortChange %#jx"
3685 1.120 christos " totlen %jd",
3686 1.111 mrg UGETW(ps.wPortStatus), UGETW(ps.wPortChange), totlen, 0);
3687 1.1 jakllsch break;
3688 1.68 skrll }
3689 1.1 jakllsch case C(UR_SET_DESCRIPTOR, UT_WRITE_CLASS_DEVICE):
3690 1.34 skrll return -1;
3691 1.34 skrll case C(UR_SET_HUB_DEPTH, UT_WRITE_CLASS_DEVICE):
3692 1.34 skrll break;
3693 1.1 jakllsch case C(UR_SET_FEATURE, UT_WRITE_CLASS_DEVICE):
3694 1.1 jakllsch break;
3695 1.34 skrll /* Set Port Feature request */
3696 1.34 skrll case C(UR_SET_FEATURE, UT_WRITE_CLASS_OTHER): {
3697 1.34 skrll int optval = (index >> 8) & 0xff;
3698 1.34 skrll index &= 0xff;
3699 1.68 skrll if (index < 1 || index > sc->sc_rhportcount[bn]) {
3700 1.34 skrll return -1;
3701 1.1 jakllsch }
3702 1.68 skrll
3703 1.68 skrll const size_t cp = xhci_rhport2ctlrport(sc, bn, index);
3704 1.68 skrll
3705 1.68 skrll port = XHCI_PORTSC(cp);
3706 1.1 jakllsch v = xhci_op_read_4(sc, port);
3707 1.121 christos DPRINTFN(4, "index %jd cp %jd portsc=0x%08jx", index, cp, v, 0);
3708 1.1 jakllsch v &= ~XHCI_PS_CLEAR;
3709 1.1 jakllsch switch (value) {
3710 1.1 jakllsch case UHF_PORT_ENABLE:
3711 1.1 jakllsch xhci_op_write_4(sc, port, v | XHCI_PS_PED);
3712 1.1 jakllsch break;
3713 1.1 jakllsch case UHF_PORT_SUSPEND:
3714 1.1 jakllsch /* XXX suspend */
3715 1.1 jakllsch break;
3716 1.1 jakllsch case UHF_PORT_RESET:
3717 1.34 skrll v &= ~(XHCI_PS_PED | XHCI_PS_PR);
3718 1.1 jakllsch xhci_op_write_4(sc, port, v | XHCI_PS_PR);
3719 1.1 jakllsch /* Wait for reset to complete. */
3720 1.1 jakllsch usb_delay_ms(&sc->sc_bus, USB_PORT_ROOT_RESET_DELAY);
3721 1.1 jakllsch if (sc->sc_dying) {
3722 1.34 skrll return -1;
3723 1.1 jakllsch }
3724 1.1 jakllsch v = xhci_op_read_4(sc, port);
3725 1.1 jakllsch if (v & XHCI_PS_PR) {
3726 1.1 jakllsch xhci_op_write_4(sc, port, v & ~XHCI_PS_PR);
3727 1.1 jakllsch usb_delay_ms(&sc->sc_bus, 10);
3728 1.1 jakllsch /* XXX */
3729 1.1 jakllsch }
3730 1.1 jakllsch break;
3731 1.1 jakllsch case UHF_PORT_POWER:
3732 1.1 jakllsch /* XXX power control */
3733 1.1 jakllsch break;
3734 1.1 jakllsch /* XXX more */
3735 1.1 jakllsch case UHF_C_PORT_RESET:
3736 1.1 jakllsch xhci_op_write_4(sc, port, v | XHCI_PS_PRC);
3737 1.1 jakllsch break;
3738 1.34 skrll case UHF_PORT_U1_TIMEOUT:
3739 1.34 skrll if (XHCI_PS_SPEED_GET(v) < XHCI_PS_SPEED_SS) {
3740 1.34 skrll return -1;
3741 1.34 skrll }
3742 1.68 skrll port = XHCI_PORTPMSC(cp);
3743 1.34 skrll v = xhci_op_read_4(sc, port);
3744 1.121 christos DPRINTFN(4, "index %jd cp %jd portpmsc=0x%08jx",
3745 1.75 pgoyette index, cp, v, 0);
3746 1.34 skrll v &= ~XHCI_PM3_U1TO_SET(0xff);
3747 1.34 skrll v |= XHCI_PM3_U1TO_SET(optval);
3748 1.34 skrll xhci_op_write_4(sc, port, v);
3749 1.34 skrll break;
3750 1.34 skrll case UHF_PORT_U2_TIMEOUT:
3751 1.34 skrll if (XHCI_PS_SPEED_GET(v) < XHCI_PS_SPEED_SS) {
3752 1.34 skrll return -1;
3753 1.34 skrll }
3754 1.68 skrll port = XHCI_PORTPMSC(cp);
3755 1.34 skrll v = xhci_op_read_4(sc, port);
3756 1.121 christos DPRINTFN(4, "index %jd cp %jd portpmsc=0x%08jx",
3757 1.75 pgoyette index, cp, v, 0);
3758 1.34 skrll v &= ~XHCI_PM3_U2TO_SET(0xff);
3759 1.34 skrll v |= XHCI_PM3_U2TO_SET(optval);
3760 1.34 skrll xhci_op_write_4(sc, port, v);
3761 1.34 skrll break;
3762 1.1 jakllsch default:
3763 1.34 skrll return -1;
3764 1.1 jakllsch }
3765 1.34 skrll }
3766 1.1 jakllsch break;
3767 1.1 jakllsch case C(UR_CLEAR_TT_BUFFER, UT_WRITE_CLASS_OTHER):
3768 1.1 jakllsch case C(UR_RESET_TT, UT_WRITE_CLASS_OTHER):
3769 1.1 jakllsch case C(UR_GET_TT_STATE, UT_READ_CLASS_OTHER):
3770 1.1 jakllsch case C(UR_STOP_TT, UT_WRITE_CLASS_OTHER):
3771 1.1 jakllsch break;
3772 1.1 jakllsch default:
3773 1.34 skrll /* default from usbroothub */
3774 1.34 skrll return buflen;
3775 1.1 jakllsch }
3776 1.27 skrll
3777 1.34 skrll return totlen;
3778 1.1 jakllsch }
3779 1.1 jakllsch
3780 1.28 skrll /* root hub interrupt */
3781 1.1 jakllsch
3782 1.1 jakllsch static usbd_status
3783 1.34 skrll xhci_root_intr_transfer(struct usbd_xfer *xfer)
3784 1.1 jakllsch {
3785 1.34 skrll struct xhci_softc * const sc = XHCI_XFER2SC(xfer);
3786 1.1 jakllsch usbd_status err;
3787 1.1 jakllsch
3788 1.27 skrll XHCIHIST_FUNC(); XHCIHIST_CALLED();
3789 1.27 skrll
3790 1.1 jakllsch /* Insert last in queue. */
3791 1.1 jakllsch mutex_enter(&sc->sc_lock);
3792 1.1 jakllsch err = usb_insert_transfer(xfer);
3793 1.1 jakllsch mutex_exit(&sc->sc_lock);
3794 1.1 jakllsch if (err)
3795 1.1 jakllsch return err;
3796 1.1 jakllsch
3797 1.1 jakllsch /* Pipe isn't running, start first */
3798 1.34 skrll return xhci_root_intr_start(SIMPLEQ_FIRST(&xfer->ux_pipe->up_queue));
3799 1.1 jakllsch }
3800 1.1 jakllsch
3801 1.34 skrll /* Wait for roothub port status/change */
3802 1.1 jakllsch static usbd_status
3803 1.34 skrll xhci_root_intr_start(struct usbd_xfer *xfer)
3804 1.1 jakllsch {
3805 1.34 skrll struct xhci_softc * const sc = XHCI_XFER2SC(xfer);
3806 1.68 skrll const size_t bn = XHCI_XFER2BUS(xfer) == &sc->sc_bus ? 0 : 1;
3807 1.100 mrg const bool polling = xhci_polling_p(sc);
3808 1.1 jakllsch
3809 1.27 skrll XHCIHIST_FUNC(); XHCIHIST_CALLED();
3810 1.27 skrll
3811 1.1 jakllsch if (sc->sc_dying)
3812 1.1 jakllsch return USBD_IOERROR;
3813 1.1 jakllsch
3814 1.99 mrg if (!polling)
3815 1.99 mrg mutex_enter(&sc->sc_lock);
3816 1.117 riastrad KASSERT(sc->sc_intrxfer[bn] == NULL);
3817 1.68 skrll sc->sc_intrxfer[bn] = xfer;
3818 1.118 riastrad xfer->ux_status = USBD_IN_PROGRESS;
3819 1.99 mrg if (!polling)
3820 1.99 mrg mutex_exit(&sc->sc_lock);
3821 1.1 jakllsch
3822 1.1 jakllsch return USBD_IN_PROGRESS;
3823 1.1 jakllsch }
3824 1.1 jakllsch
3825 1.1 jakllsch static void
3826 1.34 skrll xhci_root_intr_abort(struct usbd_xfer *xfer)
3827 1.1 jakllsch {
3828 1.117 riastrad struct xhci_softc * const sc = XHCI_XFER2SC(xfer);
3829 1.117 riastrad const size_t bn = XHCI_XFER2BUS(xfer) == &sc->sc_bus ? 0 : 1;
3830 1.1 jakllsch
3831 1.27 skrll XHCIHIST_FUNC(); XHCIHIST_CALLED();
3832 1.27 skrll
3833 1.1 jakllsch KASSERT(mutex_owned(&sc->sc_lock));
3834 1.34 skrll KASSERT(xfer->ux_pipe->up_intrxfer == xfer);
3835 1.21 skrll
3836 1.117 riastrad /* If xfer has already completed, nothing to do here. */
3837 1.117 riastrad if (sc->sc_intrxfer[bn] == NULL)
3838 1.117 riastrad return;
3839 1.117 riastrad
3840 1.117 riastrad /*
3841 1.117 riastrad * Otherwise, sc->sc_intrxfer[bn] had better be this transfer.
3842 1.117 riastrad * Cancel it.
3843 1.117 riastrad */
3844 1.117 riastrad KASSERT(sc->sc_intrxfer[bn] == xfer);
3845 1.34 skrll xfer->ux_status = USBD_CANCELLED;
3846 1.1 jakllsch usb_transfer_complete(xfer);
3847 1.1 jakllsch }
3848 1.1 jakllsch
3849 1.1 jakllsch static void
3850 1.34 skrll xhci_root_intr_close(struct usbd_pipe *pipe)
3851 1.1 jakllsch {
3852 1.117 riastrad struct xhci_softc * const sc __diagused = XHCI_PIPE2SC(pipe);
3853 1.117 riastrad const struct usbd_xfer *xfer __diagused = pipe->up_intrxfer;
3854 1.117 riastrad const size_t bn __diagused = XHCI_XFER2BUS(xfer) == &sc->sc_bus ? 0 : 1;
3855 1.1 jakllsch
3856 1.27 skrll XHCIHIST_FUNC(); XHCIHIST_CALLED();
3857 1.27 skrll
3858 1.1 jakllsch KASSERT(mutex_owned(&sc->sc_lock));
3859 1.1 jakllsch
3860 1.117 riastrad /*
3861 1.117 riastrad * Caller must guarantee the xfer has completed first, by
3862 1.117 riastrad * closing the pipe only after normal completion or an abort.
3863 1.117 riastrad */
3864 1.117 riastrad KASSERT(sc->sc_intrxfer[bn] == NULL);
3865 1.1 jakllsch }
3866 1.1 jakllsch
3867 1.1 jakllsch static void
3868 1.34 skrll xhci_root_intr_done(struct usbd_xfer *xfer)
3869 1.1 jakllsch {
3870 1.117 riastrad struct xhci_softc * const sc = XHCI_XFER2SC(xfer);
3871 1.117 riastrad const size_t bn = XHCI_XFER2BUS(xfer) == &sc->sc_bus ? 0 : 1;
3872 1.117 riastrad
3873 1.27 skrll XHCIHIST_FUNC(); XHCIHIST_CALLED();
3874 1.27 skrll
3875 1.117 riastrad KASSERT(mutex_owned(&sc->sc_lock));
3876 1.117 riastrad
3877 1.117 riastrad /* Claim the xfer so it doesn't get completed again. */
3878 1.117 riastrad KASSERT(sc->sc_intrxfer[bn] == xfer);
3879 1.117 riastrad KASSERT(xfer->ux_status != USBD_IN_PROGRESS);
3880 1.117 riastrad sc->sc_intrxfer[bn] = NULL;
3881 1.1 jakllsch }
3882 1.1 jakllsch
3883 1.1 jakllsch /* -------------- */
3884 1.1 jakllsch /* device control */
3885 1.1 jakllsch
3886 1.1 jakllsch static usbd_status
3887 1.34 skrll xhci_device_ctrl_transfer(struct usbd_xfer *xfer)
3888 1.1 jakllsch {
3889 1.34 skrll struct xhci_softc * const sc = XHCI_XFER2SC(xfer);
3890 1.1 jakllsch usbd_status err;
3891 1.1 jakllsch
3892 1.27 skrll XHCIHIST_FUNC(); XHCIHIST_CALLED();
3893 1.27 skrll
3894 1.1 jakllsch /* Insert last in queue. */
3895 1.1 jakllsch mutex_enter(&sc->sc_lock);
3896 1.1 jakllsch err = usb_insert_transfer(xfer);
3897 1.1 jakllsch mutex_exit(&sc->sc_lock);
3898 1.1 jakllsch if (err)
3899 1.34 skrll return err;
3900 1.1 jakllsch
3901 1.1 jakllsch /* Pipe isn't running, start first */
3902 1.34 skrll return xhci_device_ctrl_start(SIMPLEQ_FIRST(&xfer->ux_pipe->up_queue));
3903 1.1 jakllsch }
3904 1.1 jakllsch
3905 1.1 jakllsch static usbd_status
3906 1.34 skrll xhci_device_ctrl_start(struct usbd_xfer *xfer)
3907 1.1 jakllsch {
3908 1.34 skrll struct xhci_softc * const sc = XHCI_XFER2SC(xfer);
3909 1.34 skrll struct xhci_slot * const xs = xfer->ux_pipe->up_dev->ud_hcpriv;
3910 1.34 skrll const u_int dci = xhci_ep_get_dci(xfer->ux_pipe->up_endpoint->ue_edesc);
3911 1.123 skrll struct xhci_ring * const tr = xs->xs_xr[dci];
3912 1.35 skrll struct xhci_xfer * const xx = XHCI_XFER2XXFER(xfer);
3913 1.34 skrll usb_device_request_t * const req = &xfer->ux_request;
3914 1.124 skrll const bool isread = usbd_xfer_isread(xfer);
3915 1.1 jakllsch const uint32_t len = UGETW(req->wLength);
3916 1.34 skrll usb_dma_t * const dma = &xfer->ux_dmabuf;
3917 1.1 jakllsch uint64_t parameter;
3918 1.1 jakllsch uint32_t status;
3919 1.1 jakllsch uint32_t control;
3920 1.1 jakllsch u_int i;
3921 1.100 mrg const bool polling = xhci_polling_p(sc);
3922 1.1 jakllsch
3923 1.111 mrg XHCIHIST_FUNC();
3924 1.111 mrg XHCIHIST_CALLARGS("req: %04jx %04jx %04jx %04jx",
3925 1.27 skrll req->bmRequestType | (req->bRequest << 8), UGETW(req->wValue),
3926 1.27 skrll UGETW(req->wIndex), UGETW(req->wLength));
3927 1.1 jakllsch
3928 1.1 jakllsch /* we rely on the bottom bits for extra info */
3929 1.59 maya KASSERTMSG(((uintptr_t)xfer & 0x3) == 0x0, "xfer %zx",
3930 1.59 maya (uintptr_t) xfer);
3931 1.1 jakllsch
3932 1.34 skrll KASSERT((xfer->ux_rqflags & URQ_REQUEST) != 0);
3933 1.1 jakllsch
3934 1.1 jakllsch i = 0;
3935 1.1 jakllsch
3936 1.1 jakllsch /* setup phase */
3937 1.126 jakllsch parameter = le64dec(req); /* to keep USB endian after xhci_trb_put() */
3938 1.1 jakllsch status = XHCI_TRB_2_IRQ_SET(0) | XHCI_TRB_2_BYTES_SET(sizeof(*req));
3939 1.1 jakllsch control = ((len == 0) ? XHCI_TRB_3_TRT_NONE :
3940 1.1 jakllsch (isread ? XHCI_TRB_3_TRT_IN : XHCI_TRB_3_TRT_OUT)) |
3941 1.1 jakllsch XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_SETUP_STAGE) |
3942 1.1 jakllsch XHCI_TRB_3_IDT_BIT;
3943 1.127 jakllsch xhci_xfer_put_trb(xx, i++, parameter, status, control);
3944 1.1 jakllsch
3945 1.34 skrll if (len != 0) {
3946 1.34 skrll /* data phase */
3947 1.34 skrll parameter = DMAADDR(dma, 0);
3948 1.59 maya KASSERTMSG(len <= 0x10000, "len %d", len);
3949 1.34 skrll status = XHCI_TRB_2_IRQ_SET(0) |
3950 1.113 mrg XHCI_TRB_2_TDSZ_SET(0) |
3951 1.34 skrll XHCI_TRB_2_BYTES_SET(len);
3952 1.34 skrll control = (isread ? XHCI_TRB_3_DIR_IN : 0) |
3953 1.34 skrll XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_DATA_STAGE) |
3954 1.124 skrll (isread ? XHCI_TRB_3_ISP_BIT : 0) |
3955 1.34 skrll XHCI_TRB_3_IOC_BIT;
3956 1.127 jakllsch xhci_xfer_put_trb(xx, i++, parameter, status, control);
3957 1.124 skrll
3958 1.124 skrll usb_syncmem(dma, 0, len,
3959 1.124 skrll isread ? BUS_DMASYNC_PREREAD : BUS_DMASYNC_PREWRITE);
3960 1.34 skrll }
3961 1.1 jakllsch
3962 1.1 jakllsch parameter = 0;
3963 1.28 skrll status = XHCI_TRB_2_IRQ_SET(0);
3964 1.1 jakllsch /* the status stage has inverted direction */
3965 1.28 skrll control = ((isread && (len > 0)) ? 0 : XHCI_TRB_3_DIR_IN) |
3966 1.1 jakllsch XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_STATUS_STAGE) |
3967 1.1 jakllsch XHCI_TRB_3_IOC_BIT;
3968 1.127 jakllsch xhci_xfer_put_trb(xx, i++, parameter, status, control);
3969 1.1 jakllsch
3970 1.99 mrg if (!polling)
3971 1.99 mrg mutex_enter(&tr->xr_lock);
3972 1.127 jakllsch xhci_ring_put_xfer(sc, tr, xx, i);
3973 1.99 mrg if (!polling)
3974 1.99 mrg mutex_exit(&tr->xr_lock);
3975 1.1 jakllsch
3976 1.115 skrll if (!polling)
3977 1.115 skrll mutex_enter(&sc->sc_lock);
3978 1.115 skrll xfer->ux_status = USBD_IN_PROGRESS;
3979 1.1 jakllsch xhci_db_write_4(sc, XHCI_DOORBELL(xs->xs_idx), dci);
3980 1.116 riastrad usbd_xfer_schedule_timeout(xfer);
3981 1.115 skrll if (!polling)
3982 1.115 skrll mutex_exit(&sc->sc_lock);
3983 1.1 jakllsch
3984 1.1 jakllsch return USBD_IN_PROGRESS;
3985 1.1 jakllsch }
3986 1.1 jakllsch
3987 1.1 jakllsch static void
3988 1.34 skrll xhci_device_ctrl_done(struct usbd_xfer *xfer)
3989 1.1 jakllsch {
3990 1.27 skrll XHCIHIST_FUNC(); XHCIHIST_CALLED();
3991 1.34 skrll usb_device_request_t *req = &xfer->ux_request;
3992 1.34 skrll int len = UGETW(req->wLength);
3993 1.34 skrll int rd = req->bmRequestType & UT_READ;
3994 1.1 jakllsch
3995 1.34 skrll if (len)
3996 1.34 skrll usb_syncmem(&xfer->ux_dmabuf, 0, len,
3997 1.34 skrll rd ? BUS_DMASYNC_POSTREAD : BUS_DMASYNC_POSTWRITE);
3998 1.1 jakllsch }
3999 1.1 jakllsch
4000 1.1 jakllsch static void
4001 1.34 skrll xhci_device_ctrl_abort(struct usbd_xfer *xfer)
4002 1.1 jakllsch {
4003 1.27 skrll XHCIHIST_FUNC(); XHCIHIST_CALLED();
4004 1.34 skrll
4005 1.116 riastrad usbd_xfer_abort(xfer);
4006 1.1 jakllsch }
4007 1.1 jakllsch
4008 1.1 jakllsch static void
4009 1.34 skrll xhci_device_ctrl_close(struct usbd_pipe *pipe)
4010 1.1 jakllsch {
4011 1.27 skrll XHCIHIST_FUNC(); XHCIHIST_CALLED();
4012 1.34 skrll
4013 1.34 skrll xhci_close_pipe(pipe);
4014 1.1 jakllsch }
4015 1.1 jakllsch
4016 1.34 skrll /* ------------------ */
4017 1.34 skrll /* device isochronous */
4018 1.1 jakllsch
4019 1.134 jakllsch static usbd_status
4020 1.134 jakllsch xhci_device_isoc_transfer(struct usbd_xfer *xfer)
4021 1.134 jakllsch {
4022 1.134 jakllsch struct xhci_softc * const sc = XHCI_XFER2SC(xfer);
4023 1.134 jakllsch usbd_status err;
4024 1.134 jakllsch
4025 1.134 jakllsch XHCIHIST_FUNC(); XHCIHIST_CALLED();
4026 1.134 jakllsch
4027 1.134 jakllsch /* Insert last in queue. */
4028 1.134 jakllsch mutex_enter(&sc->sc_lock);
4029 1.134 jakllsch err = usb_insert_transfer(xfer);
4030 1.134 jakllsch mutex_exit(&sc->sc_lock);
4031 1.134 jakllsch if (err)
4032 1.134 jakllsch return err;
4033 1.134 jakllsch
4034 1.134 jakllsch return xhci_device_isoc_enter(xfer);
4035 1.134 jakllsch }
4036 1.134 jakllsch
4037 1.134 jakllsch static usbd_status
4038 1.134 jakllsch xhci_device_isoc_enter(struct usbd_xfer *xfer)
4039 1.134 jakllsch {
4040 1.134 jakllsch struct xhci_softc * const sc = XHCI_XFER2SC(xfer);
4041 1.134 jakllsch struct xhci_slot * const xs = xfer->ux_pipe->up_dev->ud_hcpriv;
4042 1.134 jakllsch const u_int dci = xhci_ep_get_dci(xfer->ux_pipe->up_endpoint->ue_edesc);
4043 1.134 jakllsch struct xhci_ring * const tr = xs->xs_xr[dci];
4044 1.134 jakllsch struct xhci_xfer * const xx = XHCI_XFER2XXFER(xfer);
4045 1.134 jakllsch struct xhci_pipe * const xpipe = (struct xhci_pipe *)xfer->ux_pipe;
4046 1.134 jakllsch uint32_t len = xfer->ux_length;
4047 1.134 jakllsch usb_dma_t * const dma = &xfer->ux_dmabuf;
4048 1.134 jakllsch uint64_t parameter;
4049 1.134 jakllsch uint32_t status;
4050 1.134 jakllsch uint32_t control;
4051 1.134 jakllsch uint32_t mfindex;
4052 1.134 jakllsch uint32_t offs;
4053 1.134 jakllsch int i, ival;
4054 1.134 jakllsch const bool polling = xhci_polling_p(sc);
4055 1.134 jakllsch const uint16_t MPS = UGETW(xfer->ux_pipe->up_endpoint->ue_edesc->wMaxPacketSize);
4056 1.134 jakllsch const uint16_t mps = UE_GET_SIZE(MPS);
4057 1.134 jakllsch const uint8_t maxb = xpipe->xp_maxb;
4058 1.134 jakllsch u_int tdpc, tbc, tlbpc;
4059 1.134 jakllsch
4060 1.134 jakllsch XHCIHIST_FUNC();
4061 1.134 jakllsch XHCIHIST_CALLARGS("%#jx slot %ju dci %ju",
4062 1.134 jakllsch (uintptr_t)xfer, xs->xs_idx, dci, 0);
4063 1.134 jakllsch
4064 1.134 jakllsch if (sc->sc_dying)
4065 1.134 jakllsch return USBD_IOERROR;
4066 1.134 jakllsch
4067 1.134 jakllsch KASSERT(xfer->ux_nframes != 0 && xfer->ux_frlengths);
4068 1.134 jakllsch KASSERT((xfer->ux_rqflags & URQ_REQUEST) == 0);
4069 1.134 jakllsch
4070 1.134 jakllsch const bool isread = usbd_xfer_isread(xfer);
4071 1.134 jakllsch if (xfer->ux_length)
4072 1.134 jakllsch usb_syncmem(dma, 0, xfer->ux_length,
4073 1.134 jakllsch isread ? BUS_DMASYNC_PREREAD : BUS_DMASYNC_PREWRITE);
4074 1.134 jakllsch
4075 1.134 jakllsch ival = xfer->ux_pipe->up_endpoint->ue_edesc->bInterval;
4076 1.134 jakllsch if (ival >= 1 && ival <= 16)
4077 1.134 jakllsch ival = 1 << (ival - 1);
4078 1.134 jakllsch else
4079 1.134 jakllsch ival = 1; /* fake something up */
4080 1.134 jakllsch
4081 1.134 jakllsch if (xpipe->xp_isoc_next == -1) {
4082 1.134 jakllsch mfindex = xhci_rt_read_4(sc, XHCI_MFINDEX);
4083 1.134 jakllsch DPRINTF("mfindex %jx", (uintmax_t)mfindex, 0, 0, 0);
4084 1.134 jakllsch mfindex = XHCI_MFINDEX_GET(mfindex + 1);
4085 1.134 jakllsch mfindex /= USB_UFRAMES_PER_FRAME;
4086 1.134 jakllsch mfindex += 7; /* 7 frames is max possible IST */
4087 1.134 jakllsch xpipe->xp_isoc_next = roundup2(mfindex, ival);
4088 1.134 jakllsch }
4089 1.134 jakllsch
4090 1.134 jakllsch offs = 0;
4091 1.134 jakllsch for (i = 0; i < xfer->ux_nframes; i++) {
4092 1.134 jakllsch len = xfer->ux_frlengths[i];
4093 1.134 jakllsch
4094 1.134 jakllsch tdpc = howmany(len, mps);
4095 1.134 jakllsch tbc = howmany(tdpc, maxb) - 1;
4096 1.134 jakllsch tlbpc = tdpc % maxb;
4097 1.134 jakllsch tlbpc = tlbpc ? tlbpc - 1 : maxb - 1;
4098 1.134 jakllsch
4099 1.134 jakllsch KASSERTMSG(len <= 0x10000, "len %d", len);
4100 1.134 jakllsch parameter = DMAADDR(dma, offs);
4101 1.134 jakllsch status = XHCI_TRB_2_IRQ_SET(0) |
4102 1.134 jakllsch XHCI_TRB_2_TDSZ_SET(0) |
4103 1.134 jakllsch XHCI_TRB_2_BYTES_SET(len);
4104 1.134 jakllsch control = XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_ISOCH) |
4105 1.134 jakllsch (isread ? XHCI_TRB_3_ISP_BIT : 0) |
4106 1.134 jakllsch XHCI_TRB_3_TBC_SET(tbc) |
4107 1.134 jakllsch XHCI_TRB_3_TLBPC_SET(tlbpc) |
4108 1.134 jakllsch XHCI_TRB_3_IOC_BIT;
4109 1.134 jakllsch if (XHCI_HCC_CFC(sc->sc_hcc)) {
4110 1.134 jakllsch control |= XHCI_TRB_3_FRID_SET(xpipe->xp_isoc_next);
4111 1.134 jakllsch #if 0
4112 1.134 jakllsch } else if (xpipe->xp_isoc_next == -1) {
4113 1.134 jakllsch control |= XHCI_TRB_3_FRID_SET(xpipe->xp_isoc_next);
4114 1.134 jakllsch #endif
4115 1.134 jakllsch } else {
4116 1.134 jakllsch control |= XHCI_TRB_3_ISO_SIA_BIT;
4117 1.134 jakllsch }
4118 1.134 jakllsch #if 0
4119 1.134 jakllsch if (i != xfer->ux_nframes - 1)
4120 1.134 jakllsch control |= XHCI_TRB_3_BEI_BIT;
4121 1.134 jakllsch #endif
4122 1.134 jakllsch xhci_xfer_put_trb(xx, i, parameter, status, control);
4123 1.134 jakllsch
4124 1.134 jakllsch xpipe->xp_isoc_next += ival;
4125 1.134 jakllsch offs += len;
4126 1.134 jakllsch }
4127 1.134 jakllsch
4128 1.134 jakllsch xx->xx_isoc_done = 0;
4129 1.134 jakllsch
4130 1.134 jakllsch if (!polling)
4131 1.134 jakllsch mutex_enter(&tr->xr_lock);
4132 1.134 jakllsch xhci_ring_put_xfer(sc, tr, xx, i);
4133 1.134 jakllsch if (!polling)
4134 1.134 jakllsch mutex_exit(&tr->xr_lock);
4135 1.134 jakllsch
4136 1.134 jakllsch if (!polling)
4137 1.134 jakllsch mutex_enter(&sc->sc_lock);
4138 1.134 jakllsch xfer->ux_status = USBD_IN_PROGRESS;
4139 1.134 jakllsch xhci_db_write_4(sc, XHCI_DOORBELL(xs->xs_idx), dci);
4140 1.134 jakllsch usbd_xfer_schedule_timeout(xfer);
4141 1.134 jakllsch if (!polling)
4142 1.134 jakllsch mutex_exit(&sc->sc_lock);
4143 1.134 jakllsch
4144 1.134 jakllsch return USBD_IN_PROGRESS;
4145 1.134 jakllsch }
4146 1.134 jakllsch
4147 1.134 jakllsch static void
4148 1.134 jakllsch xhci_device_isoc_abort(struct usbd_xfer *xfer)
4149 1.134 jakllsch {
4150 1.134 jakllsch XHCIHIST_FUNC(); XHCIHIST_CALLED();
4151 1.134 jakllsch
4152 1.134 jakllsch usbd_xfer_abort(xfer);
4153 1.134 jakllsch }
4154 1.134 jakllsch
4155 1.134 jakllsch static void
4156 1.134 jakllsch xhci_device_isoc_close(struct usbd_pipe *pipe)
4157 1.134 jakllsch {
4158 1.134 jakllsch XHCIHIST_FUNC(); XHCIHIST_CALLED();
4159 1.134 jakllsch
4160 1.134 jakllsch xhci_close_pipe(pipe);
4161 1.134 jakllsch }
4162 1.134 jakllsch
4163 1.134 jakllsch static void
4164 1.134 jakllsch xhci_device_isoc_done(struct usbd_xfer *xfer)
4165 1.134 jakllsch {
4166 1.134 jakllsch #ifdef USB_DEBUG
4167 1.134 jakllsch struct xhci_slot * const xs = xfer->ux_pipe->up_dev->ud_hcpriv;
4168 1.134 jakllsch const u_int dci = xhci_ep_get_dci(xfer->ux_pipe->up_endpoint->ue_edesc);
4169 1.134 jakllsch #endif
4170 1.134 jakllsch const bool isread = usbd_xfer_isread(xfer);
4171 1.134 jakllsch
4172 1.134 jakllsch XHCIHIST_FUNC();
4173 1.134 jakllsch XHCIHIST_CALLARGS("%#jx slot %ju dci %ju",
4174 1.134 jakllsch (uintptr_t)xfer, xs->xs_idx, dci, 0);
4175 1.134 jakllsch
4176 1.134 jakllsch usb_syncmem(&xfer->ux_dmabuf, 0, xfer->ux_length,
4177 1.134 jakllsch isread ? BUS_DMASYNC_POSTREAD : BUS_DMASYNC_POSTWRITE);
4178 1.134 jakllsch }
4179 1.134 jakllsch
4180 1.1 jakllsch /* ----------- */
4181 1.1 jakllsch /* device bulk */
4182 1.1 jakllsch
4183 1.1 jakllsch static usbd_status
4184 1.34 skrll xhci_device_bulk_transfer(struct usbd_xfer *xfer)
4185 1.1 jakllsch {
4186 1.34 skrll struct xhci_softc * const sc = XHCI_XFER2SC(xfer);
4187 1.1 jakllsch usbd_status err;
4188 1.1 jakllsch
4189 1.27 skrll XHCIHIST_FUNC(); XHCIHIST_CALLED();
4190 1.27 skrll
4191 1.1 jakllsch /* Insert last in queue. */
4192 1.1 jakllsch mutex_enter(&sc->sc_lock);
4193 1.1 jakllsch err = usb_insert_transfer(xfer);
4194 1.1 jakllsch mutex_exit(&sc->sc_lock);
4195 1.1 jakllsch if (err)
4196 1.1 jakllsch return err;
4197 1.1 jakllsch
4198 1.1 jakllsch /*
4199 1.1 jakllsch * Pipe isn't running (otherwise err would be USBD_INPROG),
4200 1.1 jakllsch * so start it first.
4201 1.1 jakllsch */
4202 1.34 skrll return xhci_device_bulk_start(SIMPLEQ_FIRST(&xfer->ux_pipe->up_queue));
4203 1.1 jakllsch }
4204 1.1 jakllsch
4205 1.1 jakllsch static usbd_status
4206 1.34 skrll xhci_device_bulk_start(struct usbd_xfer *xfer)
4207 1.1 jakllsch {
4208 1.34 skrll struct xhci_softc * const sc = XHCI_XFER2SC(xfer);
4209 1.34 skrll struct xhci_slot * const xs = xfer->ux_pipe->up_dev->ud_hcpriv;
4210 1.34 skrll const u_int dci = xhci_ep_get_dci(xfer->ux_pipe->up_endpoint->ue_edesc);
4211 1.123 skrll struct xhci_ring * const tr = xs->xs_xr[dci];
4212 1.35 skrll struct xhci_xfer * const xx = XHCI_XFER2XXFER(xfer);
4213 1.34 skrll const uint32_t len = xfer->ux_length;
4214 1.34 skrll usb_dma_t * const dma = &xfer->ux_dmabuf;
4215 1.1 jakllsch uint64_t parameter;
4216 1.1 jakllsch uint32_t status;
4217 1.1 jakllsch uint32_t control;
4218 1.1 jakllsch u_int i = 0;
4219 1.100 mrg const bool polling = xhci_polling_p(sc);
4220 1.1 jakllsch
4221 1.111 mrg XHCIHIST_FUNC();
4222 1.111 mrg XHCIHIST_CALLARGS("%#jx slot %ju dci %ju",
4223 1.111 mrg (uintptr_t)xfer, xs->xs_idx, dci, 0);
4224 1.1 jakllsch
4225 1.1 jakllsch if (sc->sc_dying)
4226 1.1 jakllsch return USBD_IOERROR;
4227 1.1 jakllsch
4228 1.34 skrll KASSERT((xfer->ux_rqflags & URQ_REQUEST) == 0);
4229 1.1 jakllsch
4230 1.1 jakllsch parameter = DMAADDR(dma, 0);
4231 1.124 skrll const bool isread = usbd_xfer_isread(xfer);
4232 1.124 skrll if (len)
4233 1.124 skrll usb_syncmem(dma, 0, len,
4234 1.124 skrll isread ? BUS_DMASYNC_PREREAD : BUS_DMASYNC_PREWRITE);
4235 1.124 skrll
4236 1.11 dsl /*
4237 1.13 dsl * XXX: (dsl) The physical buffer must not cross a 64k boundary.
4238 1.11 dsl * If the user supplied buffer crosses such a boundary then 2
4239 1.11 dsl * (or more) TRB should be used.
4240 1.11 dsl * If multiple TRB are used the td_size field must be set correctly.
4241 1.11 dsl * For v1.0 devices (like ivy bridge) this is the number of usb data
4242 1.11 dsl * blocks needed to complete the transfer.
4243 1.11 dsl * Setting it to 1 in the last TRB causes an extra zero-length
4244 1.11 dsl * data block be sent.
4245 1.11 dsl * The earlier documentation differs, I don't know how it behaves.
4246 1.11 dsl */
4247 1.59 maya KASSERTMSG(len <= 0x10000, "len %d", len);
4248 1.1 jakllsch status = XHCI_TRB_2_IRQ_SET(0) |
4249 1.113 mrg XHCI_TRB_2_TDSZ_SET(0) |
4250 1.1 jakllsch XHCI_TRB_2_BYTES_SET(len);
4251 1.1 jakllsch control = XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_NORMAL) |
4252 1.124 skrll (isread ? XHCI_TRB_3_ISP_BIT : 0) |
4253 1.63 skrll XHCI_TRB_3_IOC_BIT;
4254 1.127 jakllsch xhci_xfer_put_trb(xx, i++, parameter, status, control);
4255 1.1 jakllsch
4256 1.99 mrg if (!polling)
4257 1.99 mrg mutex_enter(&tr->xr_lock);
4258 1.127 jakllsch xhci_ring_put_xfer(sc, tr, xx, i);
4259 1.99 mrg if (!polling)
4260 1.99 mrg mutex_exit(&tr->xr_lock);
4261 1.1 jakllsch
4262 1.115 skrll if (!polling)
4263 1.115 skrll mutex_enter(&sc->sc_lock);
4264 1.115 skrll xfer->ux_status = USBD_IN_PROGRESS;
4265 1.1 jakllsch xhci_db_write_4(sc, XHCI_DOORBELL(xs->xs_idx), dci);
4266 1.116 riastrad usbd_xfer_schedule_timeout(xfer);
4267 1.115 skrll if (!polling)
4268 1.115 skrll mutex_exit(&sc->sc_lock);
4269 1.34 skrll
4270 1.1 jakllsch return USBD_IN_PROGRESS;
4271 1.1 jakllsch }
4272 1.1 jakllsch
4273 1.1 jakllsch static void
4274 1.34 skrll xhci_device_bulk_done(struct usbd_xfer *xfer)
4275 1.1 jakllsch {
4276 1.27 skrll #ifdef USB_DEBUG
4277 1.34 skrll struct xhci_slot * const xs = xfer->ux_pipe->up_dev->ud_hcpriv;
4278 1.34 skrll const u_int dci = xhci_ep_get_dci(xfer->ux_pipe->up_endpoint->ue_edesc);
4279 1.27 skrll #endif
4280 1.124 skrll const bool isread = usbd_xfer_isread(xfer);
4281 1.1 jakllsch
4282 1.111 mrg XHCIHIST_FUNC();
4283 1.111 mrg XHCIHIST_CALLARGS("%#jx slot %ju dci %ju",
4284 1.111 mrg (uintptr_t)xfer, xs->xs_idx, dci, 0);
4285 1.1 jakllsch
4286 1.34 skrll usb_syncmem(&xfer->ux_dmabuf, 0, xfer->ux_length,
4287 1.1 jakllsch isread ? BUS_DMASYNC_POSTREAD : BUS_DMASYNC_POSTWRITE);
4288 1.1 jakllsch }
4289 1.1 jakllsch
4290 1.1 jakllsch static void
4291 1.34 skrll xhci_device_bulk_abort(struct usbd_xfer *xfer)
4292 1.1 jakllsch {
4293 1.27 skrll XHCIHIST_FUNC(); XHCIHIST_CALLED();
4294 1.34 skrll
4295 1.116 riastrad usbd_xfer_abort(xfer);
4296 1.1 jakllsch }
4297 1.1 jakllsch
4298 1.1 jakllsch static void
4299 1.34 skrll xhci_device_bulk_close(struct usbd_pipe *pipe)
4300 1.1 jakllsch {
4301 1.27 skrll XHCIHIST_FUNC(); XHCIHIST_CALLED();
4302 1.34 skrll
4303 1.34 skrll xhci_close_pipe(pipe);
4304 1.1 jakllsch }
4305 1.1 jakllsch
4306 1.34 skrll /* ---------------- */
4307 1.34 skrll /* device interrupt */
4308 1.1 jakllsch
4309 1.1 jakllsch static usbd_status
4310 1.34 skrll xhci_device_intr_transfer(struct usbd_xfer *xfer)
4311 1.1 jakllsch {
4312 1.34 skrll struct xhci_softc * const sc = XHCI_XFER2SC(xfer);
4313 1.1 jakllsch usbd_status err;
4314 1.1 jakllsch
4315 1.27 skrll XHCIHIST_FUNC(); XHCIHIST_CALLED();
4316 1.27 skrll
4317 1.1 jakllsch /* Insert last in queue. */
4318 1.1 jakllsch mutex_enter(&sc->sc_lock);
4319 1.1 jakllsch err = usb_insert_transfer(xfer);
4320 1.1 jakllsch mutex_exit(&sc->sc_lock);
4321 1.1 jakllsch if (err)
4322 1.1 jakllsch return err;
4323 1.1 jakllsch
4324 1.1 jakllsch /*
4325 1.1 jakllsch * Pipe isn't running (otherwise err would be USBD_INPROG),
4326 1.1 jakllsch * so start it first.
4327 1.1 jakllsch */
4328 1.34 skrll return xhci_device_intr_start(SIMPLEQ_FIRST(&xfer->ux_pipe->up_queue));
4329 1.1 jakllsch }
4330 1.1 jakllsch
4331 1.1 jakllsch static usbd_status
4332 1.34 skrll xhci_device_intr_start(struct usbd_xfer *xfer)
4333 1.1 jakllsch {
4334 1.34 skrll struct xhci_softc * const sc = XHCI_XFER2SC(xfer);
4335 1.34 skrll struct xhci_slot * const xs = xfer->ux_pipe->up_dev->ud_hcpriv;
4336 1.34 skrll const u_int dci = xhci_ep_get_dci(xfer->ux_pipe->up_endpoint->ue_edesc);
4337 1.123 skrll struct xhci_ring * const tr = xs->xs_xr[dci];
4338 1.35 skrll struct xhci_xfer * const xx = XHCI_XFER2XXFER(xfer);
4339 1.34 skrll const uint32_t len = xfer->ux_length;
4340 1.94 christos const bool polling = xhci_polling_p(sc);
4341 1.34 skrll usb_dma_t * const dma = &xfer->ux_dmabuf;
4342 1.1 jakllsch uint64_t parameter;
4343 1.1 jakllsch uint32_t status;
4344 1.1 jakllsch uint32_t control;
4345 1.1 jakllsch u_int i = 0;
4346 1.1 jakllsch
4347 1.111 mrg XHCIHIST_FUNC();
4348 1.111 mrg XHCIHIST_CALLARGS("%#jx slot %ju dci %ju",
4349 1.111 mrg (uintptr_t)xfer, xs->xs_idx, dci, 0);
4350 1.1 jakllsch
4351 1.1 jakllsch if (sc->sc_dying)
4352 1.1 jakllsch return USBD_IOERROR;
4353 1.1 jakllsch
4354 1.34 skrll KASSERT((xfer->ux_rqflags & URQ_REQUEST) == 0);
4355 1.1 jakllsch
4356 1.124 skrll const bool isread = usbd_xfer_isread(xfer);
4357 1.124 skrll if (len)
4358 1.124 skrll usb_syncmem(dma, 0, len,
4359 1.124 skrll isread ? BUS_DMASYNC_PREREAD : BUS_DMASYNC_PREWRITE);
4360 1.124 skrll
4361 1.1 jakllsch parameter = DMAADDR(dma, 0);
4362 1.59 maya KASSERTMSG(len <= 0x10000, "len %d", len);
4363 1.1 jakllsch status = XHCI_TRB_2_IRQ_SET(0) |
4364 1.113 mrg XHCI_TRB_2_TDSZ_SET(0) |
4365 1.1 jakllsch XHCI_TRB_2_BYTES_SET(len);
4366 1.1 jakllsch control = XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_NORMAL) |
4367 1.124 skrll (isread ? XHCI_TRB_3_ISP_BIT : 0) | XHCI_TRB_3_IOC_BIT;
4368 1.127 jakllsch xhci_xfer_put_trb(xx, i++, parameter, status, control);
4369 1.1 jakllsch
4370 1.94 christos if (!polling)
4371 1.94 christos mutex_enter(&tr->xr_lock);
4372 1.127 jakllsch xhci_ring_put_xfer(sc, tr, xx, i);
4373 1.94 christos if (!polling)
4374 1.94 christos mutex_exit(&tr->xr_lock);
4375 1.1 jakllsch
4376 1.115 skrll if (!polling)
4377 1.115 skrll mutex_enter(&sc->sc_lock);
4378 1.115 skrll xfer->ux_status = USBD_IN_PROGRESS;
4379 1.1 jakllsch xhci_db_write_4(sc, XHCI_DOORBELL(xs->xs_idx), dci);
4380 1.116 riastrad usbd_xfer_schedule_timeout(xfer);
4381 1.115 skrll if (!polling)
4382 1.115 skrll mutex_exit(&sc->sc_lock);
4383 1.34 skrll
4384 1.1 jakllsch return USBD_IN_PROGRESS;
4385 1.1 jakllsch }
4386 1.1 jakllsch
4387 1.1 jakllsch static void
4388 1.34 skrll xhci_device_intr_done(struct usbd_xfer *xfer)
4389 1.1 jakllsch {
4390 1.34 skrll struct xhci_softc * const sc __diagused = XHCI_XFER2SC(xfer);
4391 1.27 skrll #ifdef USB_DEBUG
4392 1.34 skrll struct xhci_slot * const xs = xfer->ux_pipe->up_dev->ud_hcpriv;
4393 1.34 skrll const u_int dci = xhci_ep_get_dci(xfer->ux_pipe->up_endpoint->ue_edesc);
4394 1.19 ozaki #endif
4395 1.124 skrll const bool isread = usbd_xfer_isread(xfer);
4396 1.1 jakllsch
4397 1.111 mrg XHCIHIST_FUNC();
4398 1.111 mrg XHCIHIST_CALLARGS("%#jx slot %ju dci %ju",
4399 1.111 mrg (uintptr_t)xfer, xs->xs_idx, dci, 0);
4400 1.1 jakllsch
4401 1.73 skrll KASSERT(xhci_polling_p(sc) || mutex_owned(&sc->sc_lock));
4402 1.1 jakllsch
4403 1.34 skrll usb_syncmem(&xfer->ux_dmabuf, 0, xfer->ux_length,
4404 1.1 jakllsch isread ? BUS_DMASYNC_POSTREAD : BUS_DMASYNC_POSTWRITE);
4405 1.1 jakllsch }
4406 1.1 jakllsch
4407 1.1 jakllsch static void
4408 1.34 skrll xhci_device_intr_abort(struct usbd_xfer *xfer)
4409 1.1 jakllsch {
4410 1.34 skrll struct xhci_softc * const sc __diagused = XHCI_XFER2SC(xfer);
4411 1.27 skrll
4412 1.111 mrg XHCIHIST_FUNC();
4413 1.111 mrg XHCIHIST_CALLARGS("%#jx", (uintptr_t)xfer, 0, 0, 0);
4414 1.10 skrll
4415 1.10 skrll KASSERT(mutex_owned(&sc->sc_lock));
4416 1.34 skrll KASSERT(xfer->ux_pipe->up_intrxfer == xfer);
4417 1.116 riastrad usbd_xfer_abort(xfer);
4418 1.1 jakllsch }
4419 1.1 jakllsch
4420 1.1 jakllsch static void
4421 1.34 skrll xhci_device_intr_close(struct usbd_pipe *pipe)
4422 1.1 jakllsch {
4423 1.34 skrll //struct xhci_softc * const sc = XHCI_PIPE2SC(pipe);
4424 1.27 skrll
4425 1.111 mrg XHCIHIST_FUNC();
4426 1.111 mrg XHCIHIST_CALLARGS("%#jx", (uintptr_t)pipe, 0, 0, 0);
4427 1.27 skrll
4428 1.34 skrll xhci_close_pipe(pipe);
4429 1.1 jakllsch }
4430