pxa2x0_dmac.c revision 1.1.2.2
1 1.1.2.2 kent /* $NetBSD: pxa2x0_dmac.c,v 1.1.2.2 2005/04/29 11:28:05 kent Exp $ */ 2 1.1.2.2 kent 3 1.1.2.2 kent /* 4 1.1.2.2 kent * Copyright (c) 2003, 2005 Wasabi Systems, Inc. 5 1.1.2.2 kent * All rights reserved. 6 1.1.2.2 kent * 7 1.1.2.2 kent * Written by Steve C. Woodford for Wasabi Systems, Inc. 8 1.1.2.2 kent * 9 1.1.2.2 kent * Redistribution and use in source and binary forms, with or without 10 1.1.2.2 kent * modification, are permitted provided that the following conditions 11 1.1.2.2 kent * are met: 12 1.1.2.2 kent * 1. Redistributions of source code must retain the above copyright 13 1.1.2.2 kent * notice, this list of conditions and the following disclaimer. 14 1.1.2.2 kent * 2. Redistributions in binary form must reproduce the above copyright 15 1.1.2.2 kent * notice, this list of conditions and the following disclaimer in the 16 1.1.2.2 kent * documentation and/or other materials provided with the distribution. 17 1.1.2.2 kent * 3. All advertising materials mentioning features or use of this software 18 1.1.2.2 kent * must display the following acknowledgement: 19 1.1.2.2 kent * This product includes software developed for the NetBSD Project by 20 1.1.2.2 kent * Wasabi Systems, Inc. 21 1.1.2.2 kent * 4. The name of Wasabi Systems, Inc. may not be used to endorse 22 1.1.2.2 kent * or promote products derived from this software without specific prior 23 1.1.2.2 kent * written permission. 24 1.1.2.2 kent * 25 1.1.2.2 kent * THIS SOFTWARE IS PROVIDED BY WASABI SYSTEMS, INC. ``AS IS'' AND 26 1.1.2.2 kent * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 27 1.1.2.2 kent * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 28 1.1.2.2 kent * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL WASABI SYSTEMS, INC 29 1.1.2.2 kent * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 30 1.1.2.2 kent * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 31 1.1.2.2 kent * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 32 1.1.2.2 kent * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 33 1.1.2.2 kent * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 34 1.1.2.2 kent * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 35 1.1.2.2 kent * POSSIBILITY OF SUCH DAMAGE. 36 1.1.2.2 kent */ 37 1.1.2.2 kent 38 1.1.2.2 kent #include "opt_pxa2x0_dmac.h" 39 1.1.2.2 kent 40 1.1.2.2 kent #include <sys/param.h> 41 1.1.2.2 kent #include <sys/systm.h> 42 1.1.2.2 kent #include <sys/device.h> 43 1.1.2.2 kent #include <sys/kernel.h> 44 1.1.2.2 kent #include <sys/malloc.h> 45 1.1.2.2 kent #include <sys/queue.h> 46 1.1.2.2 kent 47 1.1.2.2 kent #include <uvm/uvm_param.h> /* For PAGE_SIZE */ 48 1.1.2.2 kent 49 1.1.2.2 kent #include <machine/intr.h> 50 1.1.2.2 kent #include <machine/bus.h> 51 1.1.2.2 kent 52 1.1.2.2 kent #include <dev/dmover/dmovervar.h> 53 1.1.2.2 kent 54 1.1.2.2 kent #include <arm/xscale/pxa2x0reg.h> 55 1.1.2.2 kent #include <arm/xscale/pxa2x0var.h> 56 1.1.2.2 kent 57 1.1.2.2 kent #include <arm/xscale/pxa2x0_dmac.h> 58 1.1.2.2 kent 59 1.1.2.2 kent #include "locators.h" 60 1.1.2.2 kent 61 1.1.2.2 kent #undef DMAC_N_PRIORITIES 62 1.1.2.2 kent #ifndef PXA2X0_DMAC_FIXED_PRIORITY 63 1.1.2.2 kent #define DMAC_N_PRIORITIES 3 64 1.1.2.2 kent #define DMAC_PRI(p) (p) 65 1.1.2.2 kent #else 66 1.1.2.2 kent #define DMAC_N_PRIORITIES 1 67 1.1.2.2 kent #define DMAC_PRI(p) (0) 68 1.1.2.2 kent #endif 69 1.1.2.2 kent 70 1.1.2.2 kent struct dmac_desc { 71 1.1.2.2 kent SLIST_ENTRY(dmac_desc) d_link; 72 1.1.2.2 kent struct pxa2x0_dma_desc *d_desc; 73 1.1.2.2 kent paddr_t d_desc_pa; 74 1.1.2.2 kent }; 75 1.1.2.2 kent 76 1.1.2.2 kent /* 77 1.1.2.2 kent * This is used to maintain state for an in-progress transfer. 78 1.1.2.2 kent * It tracks the current DMA segment, and offset within the segment 79 1.1.2.2 kent * in the case where we had to split a request into several DMA 80 1.1.2.2 kent * operations due to a shortage of DMAC descriptors. 81 1.1.2.2 kent */ 82 1.1.2.2 kent struct dmac_desc_segs { 83 1.1.2.2 kent bus_dma_segment_t *ds_curseg; /* Current segment */ 84 1.1.2.2 kent u_int ds_nsegs; /* Remaining segments */ 85 1.1.2.2 kent bus_size_t ds_offset; /* Offset within current seg */ 86 1.1.2.2 kent }; 87 1.1.2.2 kent 88 1.1.2.2 kent SIMPLEQ_HEAD(dmac_xfer_state_head, dmac_xfer_state); 89 1.1.2.2 kent 90 1.1.2.2 kent struct dmac_xfer_state { 91 1.1.2.2 kent struct dmac_xfer dxs_xfer; 92 1.1.2.2 kent #define dxs_cookie dxs_xfer.dx_cookie 93 1.1.2.2 kent #define dxs_done dxs_xfer.dx_done 94 1.1.2.2 kent #define dxs_priority dxs_xfer.dx_priority 95 1.1.2.2 kent #define dxs_peripheral dxs_xfer.dx_peripheral 96 1.1.2.2 kent #define dxs_flow dxs_xfer.dx_flow 97 1.1.2.2 kent #define dxs_dev_width dxs_xfer.dx_dev_width 98 1.1.2.2 kent #define dxs_burst_size dxs_xfer.dx_burst_size 99 1.1.2.2 kent #define dxs_loop_notify dxs_xfer.dx_loop_notify 100 1.1.2.2 kent #define dxs_desc dxs_xfer.dx_desc 101 1.1.2.2 kent SIMPLEQ_ENTRY(dmac_xfer_state) dxs_link; 102 1.1.2.2 kent SLIST_HEAD(, dmac_desc) dxs_descs; 103 1.1.2.2 kent struct dmac_xfer_state_head *dxs_queue; 104 1.1.2.2 kent u_int dxs_channel; 105 1.1.2.2 kent #define DMAC_NO_CHANNEL (~0) 106 1.1.2.2 kent u_int32_t dxs_dcmd; 107 1.1.2.2 kent struct dmac_desc_segs dxs_segs[2]; 108 1.1.2.2 kent }; 109 1.1.2.2 kent 110 1.1.2.2 kent 111 1.1.2.2 kent #if (PXA2X0_DMAC_DMOVER_CONCURRENCY > 0) 112 1.1.2.2 kent /* 113 1.1.2.2 kent * This structure is used to maintain state for the dmover(9) backend 114 1.1.2.2 kent * part of the driver. We can have a number of concurrent dmover 115 1.1.2.2 kent * requests in progress at any given time. The exact number is given 116 1.1.2.2 kent * by the PXA2X0_DMAC_DMOVER_CONCURRENCY compile-time constant. One of 117 1.1.2.2 kent * these structures is allocated for each concurrent request. 118 1.1.2.2 kent */ 119 1.1.2.2 kent struct dmac_dmover_state { 120 1.1.2.2 kent LIST_ENTRY(dmac_dmover_state) ds_link; /* List of idle dmover chans */ 121 1.1.2.2 kent struct pxadmac_softc *ds_sc; /* Uplink to pxadmac softc */ 122 1.1.2.2 kent struct dmover_request *ds_current; /* Current dmover request */ 123 1.1.2.2 kent struct dmac_xfer_state ds_xfer; 124 1.1.2.2 kent bus_dmamap_t ds_src_dmap; 125 1.1.2.2 kent bus_dmamap_t ds_dst_dmap; 126 1.1.2.2 kent /* 127 1.1.2.2 kent * There is no inherent size limit in the DMA engine. 128 1.1.2.2 kent * The following limit is somewhat arbitrary. 129 1.1.2.2 kent */ 130 1.1.2.2 kent #define DMAC_DMOVER_MAX_XFER (8*1024*1024) 131 1.1.2.2 kent #if 0 132 1.1.2.2 kent /* This would require 16KB * 2 just for segments... */ 133 1.1.2.2 kent #define DMAC_DMOVER_NSEGS ((DMAC_DMOVER_MAX_XFER / PAGE_SIZE) + 1) 134 1.1.2.2 kent #else 135 1.1.2.2 kent #define DMAC_DMOVER_NSEGS 512 /* XXX: Only enough for 2MB */ 136 1.1.2.2 kent #endif 137 1.1.2.2 kent bus_dma_segment_t ds_zero_seg; /* Used for zero-fill ops */ 138 1.1.2.2 kent caddr_t ds_zero_va; 139 1.1.2.2 kent bus_dma_segment_t ds_fill_seg; /* Used for fill8 ops */ 140 1.1.2.2 kent caddr_t ds_fill_va; 141 1.1.2.2 kent 142 1.1.2.2 kent #define ds_src_addr_hold ds_xfer.dxs_desc[DMAC_DESC_SRC].xd_addr_hold 143 1.1.2.2 kent #define ds_dst_addr_hold ds_xfer.dxs_desc[DMAC_DESC_DST].xd_addr_hold 144 1.1.2.2 kent #define ds_src_burst ds_xfer.dxs_desc[DMAC_DESC_SRC].xd_burst_size 145 1.1.2.2 kent #define ds_dst_burst ds_xfer.dxs_desc[DMAC_DESC_DST].xd_burst_size 146 1.1.2.2 kent #define ds_src_dma_segs ds_xfer.dxs_desc[DMAC_DESC_SRC].xd_dma_segs 147 1.1.2.2 kent #define ds_dst_dma_segs ds_xfer.dxs_desc[DMAC_DESC_DST].xd_dma_segs 148 1.1.2.2 kent #define ds_src_nsegs ds_xfer.dxs_desc[DMAC_DESC_SRC].xd_nsegs 149 1.1.2.2 kent #define ds_dst_nsegs ds_xfer.dxs_desc[DMAC_DESC_DST].xd_nsegs 150 1.1.2.2 kent }; 151 1.1.2.2 kent 152 1.1.2.2 kent /* 153 1.1.2.2 kent * Overall dmover(9) backend state 154 1.1.2.2 kent */ 155 1.1.2.2 kent struct dmac_dmover { 156 1.1.2.2 kent struct dmover_backend dd_backend; 157 1.1.2.2 kent int dd_busy; 158 1.1.2.2 kent LIST_HEAD(, dmac_dmover_state) dd_free; 159 1.1.2.2 kent struct dmac_dmover_state dd_state[PXA2X0_DMAC_DMOVER_CONCURRENCY]; 160 1.1.2.2 kent }; 161 1.1.2.2 kent #endif 162 1.1.2.2 kent 163 1.1.2.2 kent struct pxadmac_softc { 164 1.1.2.2 kent struct device sc_dev; 165 1.1.2.2 kent bus_space_tag_t sc_bust; 166 1.1.2.2 kent bus_dma_tag_t sc_dmat; 167 1.1.2.2 kent bus_space_handle_t sc_bush; 168 1.1.2.2 kent void *sc_irqcookie; 169 1.1.2.2 kent 170 1.1.2.2 kent /* 171 1.1.2.2 kent * Queue of pending requests, per priority 172 1.1.2.2 kent */ 173 1.1.2.2 kent struct dmac_xfer_state_head sc_queue[DMAC_N_PRIORITIES]; 174 1.1.2.2 kent 175 1.1.2.2 kent /* 176 1.1.2.2 kent * Queue of pending requests, per peripheral 177 1.1.2.2 kent */ 178 1.1.2.2 kent struct { 179 1.1.2.2 kent struct dmac_xfer_state_head sp_queue; 180 1.1.2.2 kent u_int sp_busy; 181 1.1.2.2 kent } sc_periph[DMAC_N_PERIPH]; 182 1.1.2.2 kent 183 1.1.2.2 kent /* 184 1.1.2.2 kent * Active requests, per channel. 185 1.1.2.2 kent */ 186 1.1.2.2 kent struct dmac_xfer_state *sc_active[DMAC_N_CHANNELS]; 187 1.1.2.2 kent 188 1.1.2.2 kent /* 189 1.1.2.2 kent * Channel Priority Allocation 190 1.1.2.2 kent */ 191 1.1.2.2 kent struct { 192 1.1.2.2 kent u_int8_t p_first; 193 1.1.2.2 kent u_int8_t p_pri[DMAC_N_CHANNELS]; 194 1.1.2.2 kent } sc_prio[DMAC_N_PRIORITIES]; 195 1.1.2.2 kent #define DMAC_PRIO_END (~0) 196 1.1.2.2 kent u_int8_t sc_channel_priority[DMAC_N_CHANNELS]; 197 1.1.2.2 kent 198 1.1.2.2 kent /* 199 1.1.2.2 kent * DMA descriptor management 200 1.1.2.2 kent */ 201 1.1.2.2 kent bus_dmamap_t sc_desc_map; 202 1.1.2.2 kent bus_dma_segment_t sc_segs; 203 1.1.2.2 kent #define DMAC_N_DESCS ((PAGE_SIZE * 2) / sizeof(struct pxa2x0_dma_desc)) 204 1.1.2.2 kent #define DMAC_DESCS_SIZE (DMAC_N_DESCS * sizeof(struct pxa2x0_dma_desc)) 205 1.1.2.2 kent struct dmac_desc sc_all_descs[DMAC_N_DESCS]; 206 1.1.2.2 kent u_int sc_free_descs; 207 1.1.2.2 kent SLIST_HEAD(, dmac_desc) sc_descs; 208 1.1.2.2 kent 209 1.1.2.2 kent #if (PXA2X0_DMAC_DMOVER_CONCURRENCY > 0) 210 1.1.2.2 kent /* 211 1.1.2.2 kent * dmover(9) backend state 212 1.1.2.2 kent */ 213 1.1.2.2 kent struct dmac_dmover sc_dmover; 214 1.1.2.2 kent #endif 215 1.1.2.2 kent }; 216 1.1.2.2 kent 217 1.1.2.2 kent static int pxadmac_match(struct device *, struct cfdata *, void *); 218 1.1.2.2 kent static void pxadmac_attach(struct device *, struct device *, void *); 219 1.1.2.2 kent 220 1.1.2.2 kent CFATTACH_DECL(pxadmac, sizeof(struct pxadmac_softc), 221 1.1.2.2 kent pxadmac_match, pxadmac_attach, NULL, NULL); 222 1.1.2.2 kent 223 1.1.2.2 kent static struct pxadmac_softc *pxadmac_sc; 224 1.1.2.2 kent 225 1.1.2.2 kent static void dmac_start(struct pxadmac_softc *, dmac_priority_t); 226 1.1.2.2 kent static int dmac_continue_xfer(struct pxadmac_softc *, struct dmac_xfer_state *); 227 1.1.2.2 kent static u_int dmac_channel_intr(struct pxadmac_softc *, u_int); 228 1.1.2.2 kent static int dmac_intr(void *); 229 1.1.2.2 kent 230 1.1.2.2 kent #if (PXA2X0_DMAC_DMOVER_CONCURRENCY > 0) 231 1.1.2.2 kent static void dmac_dmover_attach(struct pxadmac_softc *); 232 1.1.2.2 kent static void dmac_dmover_process(struct dmover_backend *); 233 1.1.2.2 kent static void dmac_dmover_run(struct dmover_backend *); 234 1.1.2.2 kent static void dmac_dmover_done(struct dmac_xfer *, int); 235 1.1.2.2 kent #endif 236 1.1.2.2 kent 237 1.1.2.2 kent static __inline u_int32_t 238 1.1.2.2 kent dmac_reg_read(struct pxadmac_softc *sc, int reg) 239 1.1.2.2 kent { 240 1.1.2.2 kent 241 1.1.2.2 kent return (bus_space_read_4(sc->sc_bust, sc->sc_bush, reg)); 242 1.1.2.2 kent } 243 1.1.2.2 kent 244 1.1.2.2 kent static __inline void 245 1.1.2.2 kent dmac_reg_write(struct pxadmac_softc *sc, int reg, u_int32_t val) 246 1.1.2.2 kent { 247 1.1.2.2 kent 248 1.1.2.2 kent bus_space_write_4(sc->sc_bust, sc->sc_bush, reg, val); 249 1.1.2.2 kent } 250 1.1.2.2 kent 251 1.1.2.2 kent static __inline int 252 1.1.2.2 kent dmac_allocate_channel(struct pxadmac_softc *sc, dmac_priority_t priority, 253 1.1.2.2 kent u_int *chanp) 254 1.1.2.2 kent { 255 1.1.2.2 kent u_int channel; 256 1.1.2.2 kent 257 1.1.2.2 kent KDASSERT((u_int)priority < DMAC_N_PRIORITIES); 258 1.1.2.2 kent 259 1.1.2.2 kent if ((channel = sc->sc_prio[priority].p_first) == DMAC_PRIO_END) 260 1.1.2.2 kent return (-1); 261 1.1.2.2 kent sc->sc_prio[priority].p_first = sc->sc_prio[priority].p_pri[channel]; 262 1.1.2.2 kent 263 1.1.2.2 kent *chanp = channel; 264 1.1.2.2 kent return (0); 265 1.1.2.2 kent } 266 1.1.2.2 kent 267 1.1.2.2 kent static __inline void 268 1.1.2.2 kent dmac_free_channel(struct pxadmac_softc *sc, dmac_priority_t priority, 269 1.1.2.2 kent u_int channel) 270 1.1.2.2 kent { 271 1.1.2.2 kent 272 1.1.2.2 kent KDASSERT((u_int)priority < DMAC_N_PRIORITIES); 273 1.1.2.2 kent 274 1.1.2.2 kent sc->sc_prio[priority].p_pri[channel] = sc->sc_prio[priority].p_first; 275 1.1.2.2 kent sc->sc_prio[priority].p_first = channel; 276 1.1.2.2 kent } 277 1.1.2.2 kent 278 1.1.2.2 kent static int 279 1.1.2.2 kent pxadmac_match(struct device *parent, struct cfdata *cf, void *aux) 280 1.1.2.2 kent { 281 1.1.2.2 kent struct pxaip_attach_args *pxa = aux; 282 1.1.2.2 kent 283 1.1.2.2 kent if (pxadmac_sc || pxa->pxa_addr != PXA2X0_DMAC_BASE || 284 1.1.2.2 kent pxa->pxa_intr != PXA2X0_INT_DMA) 285 1.1.2.2 kent return (0); 286 1.1.2.2 kent 287 1.1.2.2 kent pxa->pxa_size = PXA2X0_DMAC_SIZE; 288 1.1.2.2 kent 289 1.1.2.2 kent return (1); 290 1.1.2.2 kent } 291 1.1.2.2 kent 292 1.1.2.2 kent static void 293 1.1.2.2 kent pxadmac_attach(struct device *parent, struct device *self, void *aux) 294 1.1.2.2 kent { 295 1.1.2.2 kent struct pxadmac_softc *sc = (struct pxadmac_softc *)self; 296 1.1.2.2 kent struct pxaip_attach_args *pxa = aux; 297 1.1.2.2 kent struct pxa2x0_dma_desc *dd; 298 1.1.2.2 kent int i, nsegs; 299 1.1.2.2 kent 300 1.1.2.2 kent sc->sc_bust = pxa->pxa_iot; 301 1.1.2.2 kent sc->sc_dmat = pxa->pxa_dmat; 302 1.1.2.2 kent 303 1.1.2.2 kent aprint_normal(": DMA Controller\n"); 304 1.1.2.2 kent 305 1.1.2.2 kent if (bus_space_map(sc->sc_bust, pxa->pxa_addr, pxa->pxa_size, 0, 306 1.1.2.2 kent &sc->sc_bush)) { 307 1.1.2.2 kent aprint_error("%s: Can't map registers!\n", sc->sc_dev.dv_xname); 308 1.1.2.2 kent return; 309 1.1.2.2 kent } 310 1.1.2.2 kent 311 1.1.2.2 kent pxadmac_sc = sc; 312 1.1.2.2 kent 313 1.1.2.2 kent /* 314 1.1.2.2 kent * Make sure the DMAC is quiescent 315 1.1.2.2 kent */ 316 1.1.2.2 kent for (i = 0; i < DMAC_N_CHANNELS; i++) { 317 1.1.2.2 kent dmac_reg_write(sc, DMAC_DCSR(i), 0); 318 1.1.2.2 kent dmac_reg_write(sc, DMAC_DRCMR(i), 0); 319 1.1.2.2 kent sc->sc_active[i] = NULL; 320 1.1.2.2 kent } 321 1.1.2.2 kent dmac_reg_write(sc, DMAC_DINT, 322 1.1.2.2 kent dmac_reg_read(sc, DMAC_DINT) & DMAC_DINT_MASK); 323 1.1.2.2 kent 324 1.1.2.2 kent /* 325 1.1.2.2 kent * Initialise the request queues 326 1.1.2.2 kent */ 327 1.1.2.2 kent for (i = 0; i < DMAC_N_PRIORITIES; i++) 328 1.1.2.2 kent SIMPLEQ_INIT(&sc->sc_queue[i]); 329 1.1.2.2 kent 330 1.1.2.2 kent /* 331 1.1.2.2 kent * Initialise the request queues 332 1.1.2.2 kent */ 333 1.1.2.2 kent for (i = 0; i < DMAC_N_PERIPH; i++) { 334 1.1.2.2 kent sc->sc_periph[i].sp_busy = 0; 335 1.1.2.2 kent SIMPLEQ_INIT(&sc->sc_periph[i].sp_queue); 336 1.1.2.2 kent } 337 1.1.2.2 kent 338 1.1.2.2 kent /* 339 1.1.2.2 kent * Initialise the channel priority metadata 340 1.1.2.2 kent */ 341 1.1.2.2 kent memset(sc->sc_prio, DMAC_PRIO_END, sizeof(sc->sc_prio)); 342 1.1.2.2 kent for (i = 0; i < DMAC_N_CHANNELS; i++) { 343 1.1.2.2 kent #if (DMAC_N_PRIORITIES > 1) 344 1.1.2.2 kent if (i <= 3) 345 1.1.2.2 kent dmac_free_channel(sc, DMAC_PRIORITY_HIGH, i); 346 1.1.2.2 kent else 347 1.1.2.2 kent if (i <= 7) 348 1.1.2.2 kent dmac_free_channel(sc, DMAC_PRIORITY_MED, i); 349 1.1.2.2 kent else 350 1.1.2.2 kent dmac_free_channel(sc, DMAC_PRIORITY_LOW, i); 351 1.1.2.2 kent #else 352 1.1.2.2 kent dmac_free_channel(sc, DMAC_PRIORITY_NORMAL, i); 353 1.1.2.2 kent #endif 354 1.1.2.2 kent } 355 1.1.2.2 kent 356 1.1.2.2 kent /* 357 1.1.2.2 kent * Initialise DMA descriptors and associated metadata 358 1.1.2.2 kent */ 359 1.1.2.2 kent if (bus_dmamem_alloc(sc->sc_dmat, DMAC_DESCS_SIZE, DMAC_DESCS_SIZE, 0, 360 1.1.2.2 kent &sc->sc_segs, 1, &nsegs, BUS_DMA_NOWAIT)) 361 1.1.2.2 kent panic("dmac_pxaip_attach: bus_dmamem_alloc failed"); 362 1.1.2.2 kent 363 1.1.2.2 kent if (bus_dmamem_map(sc->sc_dmat, &sc->sc_segs, 1, DMAC_DESCS_SIZE, 364 1.1.2.2 kent (void *)&dd, BUS_DMA_COHERENT|BUS_DMA_NOCACHE)) 365 1.1.2.2 kent panic("dmac_pxaip_attach: bus_dmamem_map failed"); 366 1.1.2.2 kent 367 1.1.2.2 kent if (bus_dmamap_create(sc->sc_dmat, DMAC_DESCS_SIZE, 1, 368 1.1.2.2 kent DMAC_DESCS_SIZE, 0, BUS_DMA_NOWAIT, &sc->sc_desc_map)) 369 1.1.2.2 kent panic("dmac_pxaip_attach: bus_dmamap_create failed"); 370 1.1.2.2 kent 371 1.1.2.2 kent if (bus_dmamap_load(sc->sc_dmat, sc->sc_desc_map, (void *)dd, 372 1.1.2.2 kent DMAC_DESCS_SIZE, NULL, BUS_DMA_NOWAIT)) 373 1.1.2.2 kent panic("dmac_pxaip_attach: bus_dmamap_load failed"); 374 1.1.2.2 kent 375 1.1.2.2 kent SLIST_INIT(&sc->sc_descs); 376 1.1.2.2 kent sc->sc_free_descs = DMAC_N_DESCS; 377 1.1.2.2 kent for (i = 0; i < DMAC_N_DESCS; i++, dd++) { 378 1.1.2.2 kent SLIST_INSERT_HEAD(&sc->sc_descs, &sc->sc_all_descs[i], d_link); 379 1.1.2.2 kent sc->sc_all_descs[i].d_desc = dd; 380 1.1.2.2 kent sc->sc_all_descs[i].d_desc_pa = 381 1.1.2.2 kent sc->sc_segs.ds_addr + (sizeof(struct pxa2x0_dma_desc) * i); 382 1.1.2.2 kent } 383 1.1.2.2 kent 384 1.1.2.2 kent sc->sc_irqcookie = pxa2x0_intr_establish(pxa->pxa_intr, IPL_BIO, 385 1.1.2.2 kent dmac_intr, sc); 386 1.1.2.2 kent KASSERT(sc->sc_irqcookie != NULL); 387 1.1.2.2 kent 388 1.1.2.2 kent #if (PXA2X0_DMAC_DMOVER_CONCURRENCY > 0) 389 1.1.2.2 kent dmac_dmover_attach(sc); 390 1.1.2.2 kent #endif 391 1.1.2.2 kent } 392 1.1.2.2 kent 393 1.1.2.2 kent #if (PXA2X0_DMAC_DMOVER_CONCURRENCY > 0) 394 1.1.2.2 kent /* 395 1.1.2.2 kent * We support the following dmover(9) operations 396 1.1.2.2 kent */ 397 1.1.2.2 kent static const struct dmover_algdesc dmac_dmover_algdescs[] = { 398 1.1.2.2 kent {DMOVER_FUNC_ZERO, NULL, 0}, /* Zero-fill */ 399 1.1.2.2 kent {DMOVER_FUNC_FILL8, NULL, 0}, /* Fill with 8-bit immediate value */ 400 1.1.2.2 kent {DMOVER_FUNC_COPY, NULL, 1} /* Copy */ 401 1.1.2.2 kent }; 402 1.1.2.2 kent #define DMAC_DMOVER_ALGDESC_COUNT \ 403 1.1.2.2 kent (sizeof(dmac_dmover_algdescs) / sizeof(dmac_dmover_algdescs[0])) 404 1.1.2.2 kent 405 1.1.2.2 kent static void 406 1.1.2.2 kent dmac_dmover_attach(struct pxadmac_softc *sc) 407 1.1.2.2 kent { 408 1.1.2.2 kent struct dmac_dmover *dd = &sc->sc_dmover; 409 1.1.2.2 kent struct dmac_dmover_state *ds; 410 1.1.2.2 kent int i, dummy; 411 1.1.2.2 kent 412 1.1.2.2 kent /* 413 1.1.2.2 kent * Describe ourselves to the dmover(9) code 414 1.1.2.2 kent */ 415 1.1.2.2 kent dd->dd_backend.dmb_name = "pxadmac"; 416 1.1.2.2 kent dd->dd_backend.dmb_speed = 100*1024*1024; /* XXX */ 417 1.1.2.2 kent dd->dd_backend.dmb_cookie = sc; 418 1.1.2.2 kent dd->dd_backend.dmb_algdescs = dmac_dmover_algdescs; 419 1.1.2.2 kent dd->dd_backend.dmb_nalgdescs = DMAC_DMOVER_ALGDESC_COUNT; 420 1.1.2.2 kent dd->dd_backend.dmb_process = dmac_dmover_process; 421 1.1.2.2 kent dd->dd_busy = 0; 422 1.1.2.2 kent LIST_INIT(&dd->dd_free); 423 1.1.2.2 kent 424 1.1.2.2 kent for (i = 0; i < PXA2X0_DMAC_DMOVER_CONCURRENCY; i++) { 425 1.1.2.2 kent ds = &dd->dd_state[i]; 426 1.1.2.2 kent ds->ds_sc = sc; 427 1.1.2.2 kent ds->ds_current = NULL; 428 1.1.2.2 kent ds->ds_xfer.dxs_cookie = ds; 429 1.1.2.2 kent ds->ds_xfer.dxs_done = dmac_dmover_done; 430 1.1.2.2 kent ds->ds_xfer.dxs_priority = DMAC_PRIORITY_NORMAL; 431 1.1.2.2 kent ds->ds_xfer.dxs_peripheral = DMAC_PERIPH_NONE; 432 1.1.2.2 kent ds->ds_xfer.dxs_flow = DMAC_FLOW_CTRL_NONE; 433 1.1.2.2 kent ds->ds_xfer.dxs_dev_width = DMAC_DEV_WIDTH_DEFAULT; 434 1.1.2.2 kent ds->ds_xfer.dxs_burst_size = DMAC_BURST_SIZE_8; /* XXX */ 435 1.1.2.2 kent ds->ds_xfer.dxs_loop_notify = DMAC_DONT_LOOP; 436 1.1.2.2 kent ds->ds_src_addr_hold = FALSE; 437 1.1.2.2 kent ds->ds_dst_addr_hold = FALSE; 438 1.1.2.2 kent ds->ds_src_nsegs = 0; 439 1.1.2.2 kent ds->ds_dst_nsegs = 0; 440 1.1.2.2 kent LIST_INSERT_HEAD(&dd->dd_free, ds, ds_link); 441 1.1.2.2 kent 442 1.1.2.2 kent /* 443 1.1.2.2 kent * Create dma maps for both source and destination buffers. 444 1.1.2.2 kent */ 445 1.1.2.2 kent if (bus_dmamap_create(sc->sc_dmat, DMAC_DMOVER_MAX_XFER, 446 1.1.2.2 kent DMAC_DMOVER_NSEGS, DMAC_DMOVER_MAX_XFER, 447 1.1.2.2 kent 0, BUS_DMA_NOWAIT | BUS_DMA_ALLOCNOW, 448 1.1.2.2 kent &ds->ds_src_dmap) || 449 1.1.2.2 kent bus_dmamap_create(sc->sc_dmat, DMAC_DMOVER_MAX_XFER, 450 1.1.2.2 kent DMAC_DMOVER_NSEGS, DMAC_DMOVER_MAX_XFER, 451 1.1.2.2 kent 0, BUS_DMA_NOWAIT | BUS_DMA_ALLOCNOW, 452 1.1.2.2 kent &ds->ds_dst_dmap)) { 453 1.1.2.2 kent panic("dmac_dmover_attach: bus_dmamap_create failed"); 454 1.1.2.2 kent } 455 1.1.2.2 kent 456 1.1.2.2 kent /* 457 1.1.2.2 kent * Allocate some dma memory to be used as source buffers 458 1.1.2.2 kent * for the zero-fill and fill-8 operations. We only need 459 1.1.2.2 kent * small buffers here, since we set up the DMAC source 460 1.1.2.2 kent * descriptor with 'ds_addr_hold' set to TRUE. 461 1.1.2.2 kent */ 462 1.1.2.2 kent if (bus_dmamem_alloc(sc->sc_dmat, 463 1.1.2.2 kent arm_pdcache_line_size, arm_pdcache_line_size, 0, 464 1.1.2.2 kent &ds->ds_zero_seg, 1, &dummy, BUS_DMA_NOWAIT) || 465 1.1.2.2 kent bus_dmamem_alloc(sc->sc_dmat, 466 1.1.2.2 kent arm_pdcache_line_size, arm_pdcache_line_size, 0, 467 1.1.2.2 kent &ds->ds_fill_seg, 1, &dummy, BUS_DMA_NOWAIT)) { 468 1.1.2.2 kent panic("dmac_dmover_attach: bus_dmamem_alloc failed"); 469 1.1.2.2 kent } 470 1.1.2.2 kent 471 1.1.2.2 kent if (bus_dmamem_map(sc->sc_dmat, &ds->ds_zero_seg, 1, 472 1.1.2.2 kent arm_pdcache_line_size, &ds->ds_zero_va, 473 1.1.2.2 kent BUS_DMA_NOWAIT) || 474 1.1.2.2 kent bus_dmamem_map(sc->sc_dmat, &ds->ds_fill_seg, 1, 475 1.1.2.2 kent arm_pdcache_line_size, &ds->ds_fill_va, 476 1.1.2.2 kent BUS_DMA_NOWAIT)) { 477 1.1.2.2 kent panic("dmac_dmover_attach: bus_dmamem_map failed"); 478 1.1.2.2 kent } 479 1.1.2.2 kent 480 1.1.2.2 kent /* 481 1.1.2.2 kent * Make sure the zero-fill source buffer really is zero filled 482 1.1.2.2 kent */ 483 1.1.2.2 kent memset(ds->ds_zero_va, 0, arm_pdcache_line_size); 484 1.1.2.2 kent } 485 1.1.2.2 kent 486 1.1.2.2 kent dmover_backend_register(&sc->sc_dmover.dd_backend); 487 1.1.2.2 kent } 488 1.1.2.2 kent 489 1.1.2.2 kent static void 490 1.1.2.2 kent dmac_dmover_process(struct dmover_backend *dmb) 491 1.1.2.2 kent { 492 1.1.2.2 kent struct pxadmac_softc *sc = dmb->dmb_cookie; 493 1.1.2.2 kent int s = splbio(); 494 1.1.2.2 kent 495 1.1.2.2 kent /* 496 1.1.2.2 kent * If the backend is currently idle, go process the queue. 497 1.1.2.2 kent */ 498 1.1.2.2 kent if (sc->sc_dmover.dd_busy == 0) 499 1.1.2.2 kent dmac_dmover_run(&sc->sc_dmover.dd_backend); 500 1.1.2.2 kent splx(s); 501 1.1.2.2 kent } 502 1.1.2.2 kent 503 1.1.2.2 kent static void 504 1.1.2.2 kent dmac_dmover_run(struct dmover_backend *dmb) 505 1.1.2.2 kent { 506 1.1.2.2 kent struct dmover_request *dreq; 507 1.1.2.2 kent struct pxadmac_softc *sc; 508 1.1.2.2 kent struct dmac_dmover *dd; 509 1.1.2.2 kent struct dmac_dmover_state *ds; 510 1.1.2.2 kent size_t len_src, len_dst; 511 1.1.2.2 kent int rv; 512 1.1.2.2 kent 513 1.1.2.2 kent sc = dmb->dmb_cookie; 514 1.1.2.2 kent dd = &sc->sc_dmover; 515 1.1.2.2 kent sc->sc_dmover.dd_busy = 1; 516 1.1.2.2 kent 517 1.1.2.2 kent /* 518 1.1.2.2 kent * As long as we can queue up dmover requests... 519 1.1.2.2 kent */ 520 1.1.2.2 kent while ((dreq = TAILQ_FIRST(&dmb->dmb_pendreqs)) != NULL && 521 1.1.2.2 kent (ds = LIST_FIRST(&dd->dd_free)) != NULL) { 522 1.1.2.2 kent /* 523 1.1.2.2 kent * Pull the request off the queue, mark it 'running', 524 1.1.2.2 kent * and make it 'current'. 525 1.1.2.2 kent */ 526 1.1.2.2 kent dmover_backend_remque(dmb, dreq); 527 1.1.2.2 kent dreq->dreq_flags |= DMOVER_REQ_RUNNING; 528 1.1.2.2 kent LIST_REMOVE(ds, ds_link); 529 1.1.2.2 kent ds->ds_current = dreq; 530 1.1.2.2 kent 531 1.1.2.2 kent switch (dreq->dreq_outbuf_type) { 532 1.1.2.2 kent case DMOVER_BUF_LINEAR: 533 1.1.2.2 kent len_dst = dreq->dreq_outbuf.dmbuf_linear.l_len; 534 1.1.2.2 kent break; 535 1.1.2.2 kent case DMOVER_BUF_UIO: 536 1.1.2.2 kent len_dst = dreq->dreq_outbuf.dmbuf_uio->uio_resid; 537 1.1.2.2 kent break; 538 1.1.2.2 kent default: 539 1.1.2.2 kent goto error; 540 1.1.2.2 kent } 541 1.1.2.2 kent 542 1.1.2.2 kent /* 543 1.1.2.2 kent * Fix up the appropriate DMA 'source' buffer 544 1.1.2.2 kent */ 545 1.1.2.2 kent if (dreq->dreq_assignment->das_algdesc->dad_ninputs) { 546 1.1.2.2 kent struct uio *uio; 547 1.1.2.2 kent /* 548 1.1.2.2 kent * This is a 'copy' operation. 549 1.1.2.2 kent * Load up the specified source buffer 550 1.1.2.2 kent */ 551 1.1.2.2 kent switch (dreq->dreq_inbuf_type) { 552 1.1.2.2 kent case DMOVER_BUF_LINEAR: 553 1.1.2.2 kent len_src= dreq->dreq_inbuf[0].dmbuf_linear.l_len; 554 1.1.2.2 kent if (len_src != len_dst) 555 1.1.2.2 kent goto error; 556 1.1.2.2 kent if (bus_dmamap_load(sc->sc_dmat,ds->ds_src_dmap, 557 1.1.2.2 kent dreq->dreq_inbuf[0].dmbuf_linear.l_addr, 558 1.1.2.2 kent len_src, NULL, 559 1.1.2.2 kent BUS_DMA_NOWAIT | BUS_DMA_STREAMING | 560 1.1.2.2 kent BUS_DMA_READ)) 561 1.1.2.2 kent goto error; 562 1.1.2.2 kent break; 563 1.1.2.2 kent 564 1.1.2.2 kent case DMOVER_BUF_UIO: 565 1.1.2.2 kent uio = dreq->dreq_inbuf[0].dmbuf_uio; 566 1.1.2.2 kent len_src = uio->uio_resid; 567 1.1.2.2 kent if (uio->uio_rw != UIO_WRITE || 568 1.1.2.2 kent len_src != len_dst) 569 1.1.2.2 kent goto error; 570 1.1.2.2 kent if (bus_dmamap_load_uio(sc->sc_dmat, 571 1.1.2.2 kent ds->ds_src_dmap, uio, 572 1.1.2.2 kent BUS_DMA_NOWAIT | BUS_DMA_STREAMING | 573 1.1.2.2 kent BUS_DMA_READ)) 574 1.1.2.2 kent goto error; 575 1.1.2.2 kent break; 576 1.1.2.2 kent 577 1.1.2.2 kent default: 578 1.1.2.2 kent goto error; 579 1.1.2.2 kent } 580 1.1.2.2 kent 581 1.1.2.2 kent ds->ds_src_addr_hold = FALSE; 582 1.1.2.2 kent } else 583 1.1.2.2 kent if (dreq->dreq_assignment->das_algdesc->dad_name == 584 1.1.2.2 kent DMOVER_FUNC_ZERO) { 585 1.1.2.2 kent /* 586 1.1.2.2 kent * Zero-fill operation. 587 1.1.2.2 kent * Simply load up the pre-zeroed source buffer 588 1.1.2.2 kent */ 589 1.1.2.2 kent if (bus_dmamap_load(sc->sc_dmat, ds->ds_src_dmap, 590 1.1.2.2 kent ds->ds_zero_va, arm_pdcache_line_size, NULL, 591 1.1.2.2 kent BUS_DMA_NOWAIT | BUS_DMA_STREAMING | BUS_DMA_READ)) 592 1.1.2.2 kent goto error; 593 1.1.2.2 kent 594 1.1.2.2 kent ds->ds_src_addr_hold = TRUE; 595 1.1.2.2 kent } else 596 1.1.2.2 kent if (dreq->dreq_assignment->das_algdesc->dad_name == 597 1.1.2.2 kent DMOVER_FUNC_FILL8) { 598 1.1.2.2 kent /* 599 1.1.2.2 kent * Fill-8 operation. 600 1.1.2.2 kent * Initialise our fill-8 buffer, and load it up. 601 1.1.2.2 kent * 602 1.1.2.2 kent * XXX: Experiment with exactly how much of the 603 1.1.2.2 kent * source buffer needs to be filled. Particularly WRT 604 1.1.2.2 kent * burst size (which is hardcoded to 8 for dmover). 605 1.1.2.2 kent */ 606 1.1.2.2 kent memset(ds->ds_fill_va, dreq->dreq_immediate[0], 607 1.1.2.2 kent arm_pdcache_line_size); 608 1.1.2.2 kent 609 1.1.2.2 kent if (bus_dmamap_load(sc->sc_dmat, ds->ds_src_dmap, 610 1.1.2.2 kent ds->ds_fill_va, arm_pdcache_line_size, NULL, 611 1.1.2.2 kent BUS_DMA_NOWAIT | BUS_DMA_STREAMING | BUS_DMA_READ)) 612 1.1.2.2 kent goto error; 613 1.1.2.2 kent 614 1.1.2.2 kent ds->ds_src_addr_hold = TRUE; 615 1.1.2.2 kent } else { 616 1.1.2.2 kent goto error; 617 1.1.2.2 kent } 618 1.1.2.2 kent 619 1.1.2.2 kent /* 620 1.1.2.2 kent * Now do the same for the destination buffer 621 1.1.2.2 kent */ 622 1.1.2.2 kent switch (dreq->dreq_outbuf_type) { 623 1.1.2.2 kent case DMOVER_BUF_LINEAR: 624 1.1.2.2 kent if (bus_dmamap_load(sc->sc_dmat, ds->ds_dst_dmap, 625 1.1.2.2 kent dreq->dreq_outbuf.dmbuf_linear.l_addr, 626 1.1.2.2 kent len_dst, NULL, 627 1.1.2.2 kent BUS_DMA_NOWAIT | BUS_DMA_STREAMING | BUS_DMA_WRITE)) 628 1.1.2.2 kent goto error_unload_src; 629 1.1.2.2 kent break; 630 1.1.2.2 kent 631 1.1.2.2 kent case DMOVER_BUF_UIO: 632 1.1.2.2 kent if (dreq->dreq_outbuf.dmbuf_uio->uio_rw != UIO_READ) 633 1.1.2.2 kent goto error_unload_src; 634 1.1.2.2 kent if (bus_dmamap_load_uio(sc->sc_dmat, ds->ds_dst_dmap, 635 1.1.2.2 kent dreq->dreq_outbuf.dmbuf_uio, 636 1.1.2.2 kent BUS_DMA_NOWAIT | BUS_DMA_STREAMING | BUS_DMA_WRITE)) 637 1.1.2.2 kent goto error_unload_src; 638 1.1.2.2 kent break; 639 1.1.2.2 kent 640 1.1.2.2 kent default: 641 1.1.2.2 kent error_unload_src: 642 1.1.2.2 kent bus_dmamap_unload(sc->sc_dmat, ds->ds_src_dmap); 643 1.1.2.2 kent error: 644 1.1.2.2 kent dreq->dreq_error = EINVAL; 645 1.1.2.2 kent dreq->dreq_flags |= DMOVER_REQ_ERROR; 646 1.1.2.2 kent ds->ds_current = NULL; 647 1.1.2.2 kent LIST_INSERT_HEAD(&dd->dd_free, ds, ds_link); 648 1.1.2.2 kent dmover_done(dreq); 649 1.1.2.2 kent continue; 650 1.1.2.2 kent } 651 1.1.2.2 kent 652 1.1.2.2 kent /* 653 1.1.2.2 kent * The last step before shipping the request off to the 654 1.1.2.2 kent * DMAC driver is to sync the dma maps. 655 1.1.2.2 kent */ 656 1.1.2.2 kent bus_dmamap_sync(sc->sc_dmat, ds->ds_src_dmap, 0, 657 1.1.2.2 kent ds->ds_src_dmap->dm_mapsize, BUS_DMASYNC_PREWRITE); 658 1.1.2.2 kent ds->ds_src_dma_segs = ds->ds_src_dmap->dm_segs; 659 1.1.2.2 kent ds->ds_src_nsegs = ds->ds_src_dmap->dm_nsegs; 660 1.1.2.2 kent 661 1.1.2.2 kent bus_dmamap_sync(sc->sc_dmat, ds->ds_dst_dmap, 0, 662 1.1.2.2 kent ds->ds_dst_dmap->dm_mapsize, BUS_DMASYNC_PREREAD); 663 1.1.2.2 kent ds->ds_dst_dma_segs = ds->ds_dst_dmap->dm_segs; 664 1.1.2.2 kent ds->ds_dst_nsegs = ds->ds_dst_dmap->dm_nsegs; 665 1.1.2.2 kent 666 1.1.2.2 kent /* 667 1.1.2.2 kent * Hand the request over to the dmac section of the driver. 668 1.1.2.2 kent */ 669 1.1.2.2 kent if ((rv = pxa2x0_dmac_start_xfer(&ds->ds_xfer.dxs_xfer)) != 0) { 670 1.1.2.2 kent bus_dmamap_unload(sc->sc_dmat, ds->ds_src_dmap); 671 1.1.2.2 kent bus_dmamap_unload(sc->sc_dmat, ds->ds_dst_dmap); 672 1.1.2.2 kent dreq->dreq_error = rv; 673 1.1.2.2 kent dreq->dreq_flags |= DMOVER_REQ_ERROR; 674 1.1.2.2 kent ds->ds_current = NULL; 675 1.1.2.2 kent LIST_INSERT_HEAD(&dd->dd_free, ds, ds_link); 676 1.1.2.2 kent dmover_done(dreq); 677 1.1.2.2 kent } 678 1.1.2.2 kent } 679 1.1.2.2 kent 680 1.1.2.2 kent /* All done */ 681 1.1.2.2 kent sc->sc_dmover.dd_busy = 0; 682 1.1.2.2 kent } 683 1.1.2.2 kent 684 1.1.2.2 kent static void 685 1.1.2.2 kent dmac_dmover_done(struct dmac_xfer *dx, int error) 686 1.1.2.2 kent { 687 1.1.2.2 kent struct dmac_dmover_state *ds = dx->dx_cookie; 688 1.1.2.2 kent struct pxadmac_softc *sc = ds->ds_sc; 689 1.1.2.2 kent struct dmover_request *dreq = ds->ds_current; 690 1.1.2.2 kent 691 1.1.2.2 kent /* 692 1.1.2.2 kent * A dmover(9) request has just completed. 693 1.1.2.2 kent */ 694 1.1.2.2 kent 695 1.1.2.2 kent KDASSERT(dreq != NULL); 696 1.1.2.2 kent 697 1.1.2.2 kent /* 698 1.1.2.2 kent * Sync and unload the DMA maps 699 1.1.2.2 kent */ 700 1.1.2.2 kent bus_dmamap_sync(sc->sc_dmat, ds->ds_src_dmap, 0, 701 1.1.2.2 kent ds->ds_src_dmap->dm_mapsize, BUS_DMASYNC_POSTREAD); 702 1.1.2.2 kent bus_dmamap_sync(sc->sc_dmat, ds->ds_dst_dmap, 0, 703 1.1.2.2 kent ds->ds_dst_dmap->dm_mapsize, BUS_DMASYNC_POSTWRITE); 704 1.1.2.2 kent 705 1.1.2.2 kent bus_dmamap_unload(sc->sc_dmat, ds->ds_src_dmap); 706 1.1.2.2 kent bus_dmamap_unload(sc->sc_dmat, ds->ds_dst_dmap); 707 1.1.2.2 kent 708 1.1.2.2 kent ds->ds_current = NULL; 709 1.1.2.2 kent LIST_INSERT_HEAD(&sc->sc_dmover.dd_free, ds, ds_link); 710 1.1.2.2 kent 711 1.1.2.2 kent /* 712 1.1.2.2 kent * Record the completion status of the transfer 713 1.1.2.2 kent */ 714 1.1.2.2 kent if (error) { 715 1.1.2.2 kent dreq->dreq_error = error; 716 1.1.2.2 kent dreq->dreq_flags |= DMOVER_REQ_ERROR; 717 1.1.2.2 kent } else { 718 1.1.2.2 kent if (dreq->dreq_outbuf_type == DMOVER_BUF_UIO) 719 1.1.2.2 kent dreq->dreq_outbuf.dmbuf_uio->uio_resid = 0; 720 1.1.2.2 kent if (dreq->dreq_assignment->das_algdesc->dad_ninputs && 721 1.1.2.2 kent dreq->dreq_inbuf_type == DMOVER_BUF_UIO) 722 1.1.2.2 kent dreq->dreq_inbuf[0].dmbuf_uio->uio_resid = 0; 723 1.1.2.2 kent } 724 1.1.2.2 kent 725 1.1.2.2 kent /* 726 1.1.2.2 kent * Done! 727 1.1.2.2 kent */ 728 1.1.2.2 kent dmover_done(dreq); 729 1.1.2.2 kent 730 1.1.2.2 kent /* 731 1.1.2.2 kent * See if we can start some more dmover(9) requests. 732 1.1.2.2 kent * 733 1.1.2.2 kent * Note: We're already at splbio() here. 734 1.1.2.2 kent */ 735 1.1.2.2 kent if (sc->sc_dmover.dd_busy == 0) 736 1.1.2.2 kent dmac_dmover_run(&sc->sc_dmover.dd_backend); 737 1.1.2.2 kent } 738 1.1.2.2 kent #endif 739 1.1.2.2 kent 740 1.1.2.2 kent struct dmac_xfer * 741 1.1.2.2 kent pxa2x0_dmac_allocate_xfer(int flags) 742 1.1.2.2 kent { 743 1.1.2.2 kent struct dmac_xfer_state *dxs; 744 1.1.2.2 kent 745 1.1.2.2 kent dxs = malloc(sizeof(struct dmac_xfer_state), M_DEVBUF, flags); 746 1.1.2.2 kent 747 1.1.2.2 kent return ((struct dmac_xfer *)dxs); 748 1.1.2.2 kent } 749 1.1.2.2 kent 750 1.1.2.2 kent void 751 1.1.2.2 kent pxa2x0_dmac_free_xfer(struct dmac_xfer *dx) 752 1.1.2.2 kent { 753 1.1.2.2 kent 754 1.1.2.2 kent /* 755 1.1.2.2 kent * XXX: Should verify the DMAC is not actively using this 756 1.1.2.2 kent * structure before freeing... 757 1.1.2.2 kent */ 758 1.1.2.2 kent free(dx, M_DEVBUF); 759 1.1.2.2 kent } 760 1.1.2.2 kent 761 1.1.2.2 kent static __inline int 762 1.1.2.2 kent dmac_validate_desc(struct dmac_xfer_desc *xd, size_t *psize) 763 1.1.2.2 kent { 764 1.1.2.2 kent size_t size; 765 1.1.2.2 kent int i; 766 1.1.2.2 kent 767 1.1.2.2 kent /* 768 1.1.2.2 kent * Make sure the transfer parameters are acceptable. 769 1.1.2.2 kent */ 770 1.1.2.2 kent 771 1.1.2.2 kent if (xd->xd_addr_hold && 772 1.1.2.2 kent (xd->xd_nsegs != 1 || xd->xd_dma_segs[0].ds_len == 0)) 773 1.1.2.2 kent return (EINVAL); 774 1.1.2.2 kent 775 1.1.2.2 kent for (i = 0, size = 0; i < xd->xd_nsegs; i++) { 776 1.1.2.2 kent if (xd->xd_dma_segs[i].ds_addr & 0x7) 777 1.1.2.2 kent return (EFAULT); 778 1.1.2.2 kent size += xd->xd_dma_segs[i].ds_len; 779 1.1.2.2 kent } 780 1.1.2.2 kent 781 1.1.2.2 kent *psize = size; 782 1.1.2.2 kent return (0); 783 1.1.2.2 kent } 784 1.1.2.2 kent 785 1.1.2.2 kent static __inline int 786 1.1.2.2 kent dmac_init_desc(struct dmac_desc_segs *ds, struct dmac_xfer_desc *xd, 787 1.1.2.2 kent size_t *psize) 788 1.1.2.2 kent { 789 1.1.2.2 kent int err; 790 1.1.2.2 kent 791 1.1.2.2 kent if ((err = dmac_validate_desc(xd, psize))) 792 1.1.2.2 kent return (err); 793 1.1.2.2 kent 794 1.1.2.2 kent ds->ds_curseg = xd->xd_dma_segs; 795 1.1.2.2 kent ds->ds_nsegs = xd->xd_nsegs; 796 1.1.2.2 kent ds->ds_offset = 0; 797 1.1.2.2 kent return (0); 798 1.1.2.2 kent } 799 1.1.2.2 kent 800 1.1.2.2 kent int 801 1.1.2.2 kent pxa2x0_dmac_start_xfer(struct dmac_xfer *dx) 802 1.1.2.2 kent { 803 1.1.2.2 kent struct pxadmac_softc *sc = pxadmac_sc; 804 1.1.2.2 kent struct dmac_xfer_state *dxs = (struct dmac_xfer_state *)dx; 805 1.1.2.2 kent struct dmac_xfer_desc *src, *dst; 806 1.1.2.2 kent size_t size; 807 1.1.2.2 kent int err, s; 808 1.1.2.2 kent 809 1.1.2.2 kent if (dxs->dxs_peripheral != DMAC_PERIPH_NONE && 810 1.1.2.2 kent dxs->dxs_peripheral >= DMAC_N_PERIPH) 811 1.1.2.2 kent return (EINVAL); 812 1.1.2.2 kent 813 1.1.2.2 kent src = &dxs->dxs_desc[DMAC_DESC_SRC]; 814 1.1.2.2 kent dst = &dxs->dxs_desc[DMAC_DESC_DST]; 815 1.1.2.2 kent 816 1.1.2.2 kent if ((err = dmac_init_desc(&dxs->dxs_segs[DMAC_DESC_SRC], src, &size))) 817 1.1.2.2 kent return (err); 818 1.1.2.2 kent if (src->xd_addr_hold == FALSE && 819 1.1.2.2 kent dxs->dxs_loop_notify != DMAC_DONT_LOOP && 820 1.1.2.2 kent (size % dxs->dxs_loop_notify) != 0) 821 1.1.2.2 kent return (EINVAL); 822 1.1.2.2 kent 823 1.1.2.2 kent if ((err = dmac_init_desc(&dxs->dxs_segs[DMAC_DESC_DST], dst, &size))) 824 1.1.2.2 kent return (err); 825 1.1.2.2 kent if (dst->xd_addr_hold == FALSE && 826 1.1.2.2 kent dxs->dxs_loop_notify != DMAC_DONT_LOOP && 827 1.1.2.2 kent (size % dxs->dxs_loop_notify) != 0) 828 1.1.2.2 kent return (EINVAL); 829 1.1.2.2 kent 830 1.1.2.2 kent SLIST_INIT(&dxs->dxs_descs); 831 1.1.2.2 kent dxs->dxs_channel = DMAC_NO_CHANNEL; 832 1.1.2.2 kent dxs->dxs_dcmd = (((u_int32_t)dxs->dxs_dev_width) << DCMD_WIDTH_SHIFT) | 833 1.1.2.2 kent (((u_int32_t)dxs->dxs_burst_size) << DCMD_SIZE_SHIFT); 834 1.1.2.2 kent 835 1.1.2.2 kent switch (dxs->dxs_flow) { 836 1.1.2.2 kent case DMAC_FLOW_CTRL_NONE: 837 1.1.2.2 kent break; 838 1.1.2.2 kent case DMAC_FLOW_CTRL_SRC: 839 1.1.2.2 kent dxs->dxs_dcmd |= DCMD_FLOWSRC; 840 1.1.2.2 kent break; 841 1.1.2.2 kent case DMAC_FLOW_CTRL_DEST: 842 1.1.2.2 kent dxs->dxs_dcmd |= DCMD_FLOWTRG; 843 1.1.2.2 kent break; 844 1.1.2.2 kent } 845 1.1.2.2 kent 846 1.1.2.2 kent if (src->xd_addr_hold == FALSE) 847 1.1.2.2 kent dxs->dxs_dcmd |= DCMD_INCSRCADDR; 848 1.1.2.2 kent if (dst->xd_addr_hold == FALSE) 849 1.1.2.2 kent dxs->dxs_dcmd |= DCMD_INCTRGADDR; 850 1.1.2.2 kent 851 1.1.2.2 kent s = splbio(); 852 1.1.2.2 kent if (dxs->dxs_peripheral == DMAC_PERIPH_NONE || 853 1.1.2.2 kent sc->sc_periph[dxs->dxs_peripheral].sp_busy == 0) { 854 1.1.2.2 kent dxs->dxs_queue = &sc->sc_queue[DMAC_PRI(dxs->dxs_priority)]; 855 1.1.2.2 kent SIMPLEQ_INSERT_TAIL(dxs->dxs_queue, dxs, dxs_link); 856 1.1.2.2 kent if (dxs->dxs_peripheral != DMAC_PERIPH_NONE) 857 1.1.2.2 kent sc->sc_periph[dxs->dxs_peripheral].sp_busy++; 858 1.1.2.2 kent dmac_start(sc, DMAC_PRI(dxs->dxs_priority)); 859 1.1.2.2 kent } else { 860 1.1.2.2 kent dxs->dxs_queue = &sc->sc_periph[dxs->dxs_peripheral].sp_queue; 861 1.1.2.2 kent SIMPLEQ_INSERT_TAIL(dxs->dxs_queue, dxs, dxs_link); 862 1.1.2.2 kent sc->sc_periph[dxs->dxs_peripheral].sp_busy++; 863 1.1.2.2 kent } 864 1.1.2.2 kent splx(s); 865 1.1.2.2 kent 866 1.1.2.2 kent return (0); 867 1.1.2.2 kent } 868 1.1.2.2 kent 869 1.1.2.2 kent void 870 1.1.2.2 kent pxa2x0_dmac_abort_xfer(struct dmac_xfer *dx) 871 1.1.2.2 kent { 872 1.1.2.2 kent struct pxadmac_softc *sc = pxadmac_sc; 873 1.1.2.2 kent struct dmac_xfer_state *ndxs, *dxs = (struct dmac_xfer_state *)dx; 874 1.1.2.2 kent struct dmac_desc *desc, *ndesc; 875 1.1.2.2 kent struct dmac_xfer_state_head *queue; 876 1.1.2.2 kent u_int32_t rv; 877 1.1.2.2 kent int s, timeout, need_start = 0; 878 1.1.2.2 kent 879 1.1.2.2 kent s = splbio(); 880 1.1.2.2 kent 881 1.1.2.2 kent queue = dxs->dxs_queue; 882 1.1.2.2 kent 883 1.1.2.2 kent if (dxs->dxs_channel == DMAC_NO_CHANNEL) { 884 1.1.2.2 kent /* 885 1.1.2.2 kent * The request has not yet started, or it has already 886 1.1.2.2 kent * completed. If the request is not on a queue, just 887 1.1.2.2 kent * return. 888 1.1.2.2 kent */ 889 1.1.2.2 kent if (queue == NULL) { 890 1.1.2.2 kent splx(s); 891 1.1.2.2 kent return; 892 1.1.2.2 kent } 893 1.1.2.2 kent 894 1.1.2.2 kent dxs->dxs_queue = NULL; 895 1.1.2.2 kent SIMPLEQ_REMOVE(queue, dxs, dmac_xfer_state, dxs_link); 896 1.1.2.2 kent } else { 897 1.1.2.2 kent /* 898 1.1.2.2 kent * The request is in progress. This is a bit trickier. 899 1.1.2.2 kent */ 900 1.1.2.2 kent dmac_reg_write(sc, DMAC_DCSR(dxs->dxs_channel), 0); 901 1.1.2.2 kent 902 1.1.2.2 kent for (timeout = 5000; timeout; timeout--) { 903 1.1.2.2 kent rv = dmac_reg_read(sc, DMAC_DCSR(dxs->dxs_channel)); 904 1.1.2.2 kent if (rv & DCSR_STOPSTATE) 905 1.1.2.2 kent break; 906 1.1.2.2 kent delay(1); 907 1.1.2.2 kent } 908 1.1.2.2 kent 909 1.1.2.2 kent if ((rv & DCSR_STOPSTATE) == 0) 910 1.1.2.2 kent panic( 911 1.1.2.2 kent "pxa2x0_dmac_abort_xfer: channel %d failed to abort", 912 1.1.2.2 kent dxs->dxs_channel); 913 1.1.2.2 kent 914 1.1.2.2 kent /* 915 1.1.2.2 kent * Free resources allocated to the request 916 1.1.2.2 kent */ 917 1.1.2.2 kent for (desc = SLIST_FIRST(&dxs->dxs_descs); desc; desc = ndesc) { 918 1.1.2.2 kent ndesc = SLIST_NEXT(desc, d_link); 919 1.1.2.2 kent SLIST_INSERT_HEAD(&sc->sc_descs, desc, d_link); 920 1.1.2.2 kent sc->sc_free_descs++; 921 1.1.2.2 kent } 922 1.1.2.2 kent 923 1.1.2.2 kent sc->sc_active[dxs->dxs_channel] = NULL; 924 1.1.2.2 kent dmac_free_channel(sc, DMAC_PRI(dxs->dxs_priority), 925 1.1.2.2 kent dxs->dxs_channel); 926 1.1.2.2 kent 927 1.1.2.2 kent if (dxs->dxs_peripheral != DMAC_PERIPH_NONE) 928 1.1.2.2 kent dmac_reg_write(sc, DMAC_DRCMR(dxs->dxs_peripheral), 0); 929 1.1.2.2 kent 930 1.1.2.2 kent need_start = 1; 931 1.1.2.2 kent dxs->dxs_queue = NULL; 932 1.1.2.2 kent } 933 1.1.2.2 kent 934 1.1.2.2 kent if (dxs->dxs_peripheral == DMAC_PERIPH_NONE || 935 1.1.2.2 kent sc->sc_periph[dxs->dxs_peripheral].sp_busy-- == 1 || 936 1.1.2.2 kent queue == &sc->sc_periph[dxs->dxs_peripheral].sp_queue) 937 1.1.2.2 kent goto out; 938 1.1.2.2 kent 939 1.1.2.2 kent /* 940 1.1.2.2 kent * We've just removed the current item for this 941 1.1.2.2 kent * peripheral, and there is at least one more 942 1.1.2.2 kent * pending item waiting. Make it current. 943 1.1.2.2 kent */ 944 1.1.2.2 kent ndxs = SIMPLEQ_FIRST(&sc->sc_periph[dxs->dxs_peripheral].sp_queue); 945 1.1.2.2 kent dxs = ndxs; 946 1.1.2.2 kent KDASSERT(dxs != NULL); 947 1.1.2.2 kent SIMPLEQ_REMOVE_HEAD(&sc->sc_periph[dxs->dxs_peripheral].sp_queue, 948 1.1.2.2 kent dxs_link); 949 1.1.2.2 kent 950 1.1.2.2 kent dxs->dxs_queue = &sc->sc_queue[DMAC_PRI(dxs->dxs_priority)]; 951 1.1.2.2 kent SIMPLEQ_INSERT_TAIL(dxs->dxs_queue, dxs, dxs_link); 952 1.1.2.2 kent need_start = 1; 953 1.1.2.2 kent 954 1.1.2.2 kent /* 955 1.1.2.2 kent * Try to start any pending requests with the same 956 1.1.2.2 kent * priority. 957 1.1.2.2 kent */ 958 1.1.2.2 kent out: 959 1.1.2.2 kent if (need_start) 960 1.1.2.2 kent dmac_start(sc, DMAC_PRI(dxs->dxs_priority)); 961 1.1.2.2 kent splx(s); 962 1.1.2.2 kent } 963 1.1.2.2 kent 964 1.1.2.2 kent static void 965 1.1.2.2 kent dmac_start(struct pxadmac_softc *sc, dmac_priority_t priority) 966 1.1.2.2 kent { 967 1.1.2.2 kent struct dmac_xfer_state *dxs; 968 1.1.2.2 kent u_int channel; 969 1.1.2.2 kent 970 1.1.2.2 kent while (sc->sc_free_descs && 971 1.1.2.2 kent (dxs = SIMPLEQ_FIRST(&sc->sc_queue[priority])) != NULL && 972 1.1.2.2 kent dmac_allocate_channel(sc, priority, &channel) == 0) { 973 1.1.2.2 kent /* 974 1.1.2.2 kent * Yay, got some descriptors, a transfer request, and 975 1.1.2.2 kent * an available DMA channel. 976 1.1.2.2 kent */ 977 1.1.2.2 kent KDASSERT(sc->sc_active[channel] == NULL); 978 1.1.2.2 kent SIMPLEQ_REMOVE_HEAD(&sc->sc_queue[priority], dxs_link); 979 1.1.2.2 kent 980 1.1.2.2 kent dxs->dxs_channel = channel; 981 1.1.2.2 kent sc->sc_active[channel] = dxs; 982 1.1.2.2 kent (void) dmac_continue_xfer(sc, dxs); 983 1.1.2.2 kent /* 984 1.1.2.2 kent * XXX: Deal with descriptor allocation failure for loops 985 1.1.2.2 kent */ 986 1.1.2.2 kent } 987 1.1.2.2 kent } 988 1.1.2.2 kent 989 1.1.2.2 kent static int 990 1.1.2.2 kent dmac_continue_xfer(struct pxadmac_softc *sc, struct dmac_xfer_state *dxs) 991 1.1.2.2 kent { 992 1.1.2.2 kent struct dmac_desc *desc, *prev_desc; 993 1.1.2.2 kent struct pxa2x0_dma_desc *dd; 994 1.1.2.2 kent struct dmac_desc_segs *src_ds, *dst_ds; 995 1.1.2.2 kent struct dmac_xfer_desc *src_xd, *dst_xd; 996 1.1.2.2 kent bus_dma_segment_t *src_seg, *dst_seg; 997 1.1.2.2 kent bus_addr_t src_mem_addr, dst_mem_addr; 998 1.1.2.2 kent bus_size_t src_size, dst_size, this_size; 999 1.1.2.2 kent 1000 1.1.2.2 kent desc = NULL; 1001 1.1.2.2 kent prev_desc = NULL; 1002 1.1.2.2 kent dd = NULL; 1003 1.1.2.2 kent src_ds = &dxs->dxs_segs[DMAC_DESC_SRC]; 1004 1.1.2.2 kent dst_ds = &dxs->dxs_segs[DMAC_DESC_DST]; 1005 1.1.2.2 kent src_xd = &dxs->dxs_desc[DMAC_DESC_SRC]; 1006 1.1.2.2 kent dst_xd = &dxs->dxs_desc[DMAC_DESC_DST]; 1007 1.1.2.2 kent SLIST_INIT(&dxs->dxs_descs); 1008 1.1.2.2 kent 1009 1.1.2.2 kent /* 1010 1.1.2.2 kent * As long as the source/destination buffers have DMA segments, 1011 1.1.2.2 kent * and we have free descriptors, build a DMA chain. 1012 1.1.2.2 kent */ 1013 1.1.2.2 kent while (src_ds->ds_nsegs && dst_ds->ds_nsegs && sc->sc_free_descs) { 1014 1.1.2.2 kent src_seg = src_ds->ds_curseg; 1015 1.1.2.2 kent src_mem_addr = src_seg->ds_addr + src_ds->ds_offset; 1016 1.1.2.2 kent if (src_xd->xd_addr_hold == FALSE && 1017 1.1.2.2 kent dxs->dxs_loop_notify != DMAC_DONT_LOOP) 1018 1.1.2.2 kent src_size = dxs->dxs_loop_notify; 1019 1.1.2.2 kent else 1020 1.1.2.2 kent src_size = src_seg->ds_len - src_ds->ds_offset; 1021 1.1.2.2 kent 1022 1.1.2.2 kent dst_seg = dst_ds->ds_curseg; 1023 1.1.2.2 kent dst_mem_addr = dst_seg->ds_addr + dst_ds->ds_offset; 1024 1.1.2.2 kent if (dst_xd->xd_addr_hold == FALSE && 1025 1.1.2.2 kent dxs->dxs_loop_notify != DMAC_DONT_LOOP) 1026 1.1.2.2 kent dst_size = dxs->dxs_loop_notify; 1027 1.1.2.2 kent else 1028 1.1.2.2 kent dst_size = dst_seg->ds_len - dst_ds->ds_offset; 1029 1.1.2.2 kent 1030 1.1.2.2 kent /* 1031 1.1.2.2 kent * We may need to split a source or destination segment 1032 1.1.2.2 kent * across two or more DMAC descriptors. 1033 1.1.2.2 kent */ 1034 1.1.2.2 kent while (src_size && dst_size && 1035 1.1.2.2 kent (desc = SLIST_FIRST(&sc->sc_descs)) != NULL) { 1036 1.1.2.2 kent SLIST_REMOVE_HEAD(&sc->sc_descs, d_link); 1037 1.1.2.2 kent sc->sc_free_descs--; 1038 1.1.2.2 kent 1039 1.1.2.2 kent /* 1040 1.1.2.2 kent * Decide how much data we're going to transfer 1041 1.1.2.2 kent * using this DMAC descriptor. 1042 1.1.2.2 kent */ 1043 1.1.2.2 kent if (src_xd->xd_addr_hold) 1044 1.1.2.2 kent this_size = dst_size; 1045 1.1.2.2 kent else 1046 1.1.2.2 kent if (dst_xd->xd_addr_hold) 1047 1.1.2.2 kent this_size = src_size; 1048 1.1.2.2 kent else 1049 1.1.2.2 kent this_size = min(dst_size, src_size); 1050 1.1.2.2 kent 1051 1.1.2.2 kent /* 1052 1.1.2.2 kent * But clamp the transfer size to the DMAC 1053 1.1.2.2 kent * descriptor's maximum. 1054 1.1.2.2 kent */ 1055 1.1.2.2 kent this_size = min(this_size, DCMD_LENGTH_MASK & ~0x1f); 1056 1.1.2.2 kent 1057 1.1.2.2 kent /* 1058 1.1.2.2 kent * Fill in the DMAC descriptor 1059 1.1.2.2 kent */ 1060 1.1.2.2 kent dd = desc->d_desc; 1061 1.1.2.2 kent dd->dd_dsadr = src_mem_addr; 1062 1.1.2.2 kent dd->dd_dtadr = dst_mem_addr; 1063 1.1.2.2 kent dd->dd_dcmd = dxs->dxs_dcmd | this_size; 1064 1.1.2.2 kent 1065 1.1.2.2 kent /* 1066 1.1.2.2 kent * Link it into the chain 1067 1.1.2.2 kent */ 1068 1.1.2.2 kent if (prev_desc) { 1069 1.1.2.2 kent SLIST_INSERT_AFTER(prev_desc, desc, d_link); 1070 1.1.2.2 kent prev_desc->d_desc->dd_ddadr = desc->d_desc_pa; 1071 1.1.2.2 kent } else { 1072 1.1.2.2 kent SLIST_INSERT_HEAD(&dxs->dxs_descs, desc, 1073 1.1.2.2 kent d_link); 1074 1.1.2.2 kent } 1075 1.1.2.2 kent prev_desc = desc; 1076 1.1.2.2 kent 1077 1.1.2.2 kent /* 1078 1.1.2.2 kent * Update the source/destination pointers 1079 1.1.2.2 kent */ 1080 1.1.2.2 kent if (src_xd->xd_addr_hold == FALSE) { 1081 1.1.2.2 kent src_size -= this_size; 1082 1.1.2.2 kent src_ds->ds_offset += this_size; 1083 1.1.2.2 kent if (src_ds->ds_offset == src_seg->ds_len) { 1084 1.1.2.2 kent KDASSERT(src_size == 0); 1085 1.1.2.2 kent src_ds->ds_curseg = ++src_seg; 1086 1.1.2.2 kent src_ds->ds_offset = 0; 1087 1.1.2.2 kent src_ds->ds_nsegs--; 1088 1.1.2.2 kent } else 1089 1.1.2.2 kent src_mem_addr += this_size; 1090 1.1.2.2 kent } 1091 1.1.2.2 kent 1092 1.1.2.2 kent if (dst_xd->xd_addr_hold == FALSE) { 1093 1.1.2.2 kent dst_size -= this_size; 1094 1.1.2.2 kent dst_ds->ds_offset += this_size; 1095 1.1.2.2 kent if (dst_ds->ds_offset == dst_seg->ds_len) { 1096 1.1.2.2 kent KDASSERT(dst_size == 0); 1097 1.1.2.2 kent dst_ds->ds_curseg = ++dst_seg; 1098 1.1.2.2 kent dst_ds->ds_offset = 0; 1099 1.1.2.2 kent dst_ds->ds_nsegs--; 1100 1.1.2.2 kent } else 1101 1.1.2.2 kent dst_mem_addr += this_size; 1102 1.1.2.2 kent } 1103 1.1.2.2 kent } 1104 1.1.2.2 kent 1105 1.1.2.2 kent if (dxs->dxs_loop_notify != DMAC_DONT_LOOP) { 1106 1.1.2.2 kent /* 1107 1.1.2.2 kent * We must be able to allocate descriptors for the 1108 1.1.2.2 kent * entire loop. Otherwise, return them to the pool 1109 1.1.2.2 kent * and bail. 1110 1.1.2.2 kent */ 1111 1.1.2.2 kent if (desc == NULL) { 1112 1.1.2.2 kent struct dmac_desc *ndesc; 1113 1.1.2.2 kent for (desc = SLIST_FIRST(&dxs->dxs_descs); 1114 1.1.2.2 kent desc; desc = ndesc) { 1115 1.1.2.2 kent ndesc = SLIST_NEXT(desc, d_link); 1116 1.1.2.2 kent SLIST_INSERT_HEAD(&sc->sc_descs, desc, 1117 1.1.2.2 kent d_link); 1118 1.1.2.2 kent sc->sc_free_descs++; 1119 1.1.2.2 kent } 1120 1.1.2.2 kent 1121 1.1.2.2 kent return (0); 1122 1.1.2.2 kent } 1123 1.1.2.2 kent 1124 1.1.2.2 kent KASSERT(dd != NULL); 1125 1.1.2.2 kent dd->dd_dcmd |= DCMD_ENDIRQEN; 1126 1.1.2.2 kent } 1127 1.1.2.2 kent } 1128 1.1.2.2 kent 1129 1.1.2.2 kent /* 1130 1.1.2.2 kent * Did we manage to build a chain? 1131 1.1.2.2 kent * If not, just return. 1132 1.1.2.2 kent */ 1133 1.1.2.2 kent if (dd == NULL) 1134 1.1.2.2 kent return (0); 1135 1.1.2.2 kent 1136 1.1.2.2 kent if (dxs->dxs_loop_notify == DMAC_DONT_LOOP) { 1137 1.1.2.2 kent dd->dd_dcmd |= DCMD_ENDIRQEN; 1138 1.1.2.2 kent dd->dd_ddadr = DMAC_DESC_LAST; 1139 1.1.2.2 kent } else 1140 1.1.2.2 kent dd->dd_ddadr = SLIST_FIRST(&dxs->dxs_descs)->d_desc_pa; 1141 1.1.2.2 kent 1142 1.1.2.2 kent if (dxs->dxs_peripheral != DMAC_PERIPH_NONE) { 1143 1.1.2.2 kent dmac_reg_write(sc, DMAC_DRCMR(dxs->dxs_peripheral), 1144 1.1.2.2 kent dxs->dxs_channel | DRCMR_MAPVLD); 1145 1.1.2.2 kent } 1146 1.1.2.2 kent dmac_reg_write(sc, DMAC_DDADR(dxs->dxs_channel), 1147 1.1.2.2 kent SLIST_FIRST(&dxs->dxs_descs)->d_desc_pa); 1148 1.1.2.2 kent dmac_reg_write(sc, DMAC_DCSR(dxs->dxs_channel), 1149 1.1.2.2 kent DCSR_ENDINTR | DCSR_RUN); 1150 1.1.2.2 kent 1151 1.1.2.2 kent return (1); 1152 1.1.2.2 kent } 1153 1.1.2.2 kent 1154 1.1.2.2 kent static u_int 1155 1.1.2.2 kent dmac_channel_intr(struct pxadmac_softc *sc, u_int channel) 1156 1.1.2.2 kent { 1157 1.1.2.2 kent struct dmac_xfer_state *dxs; 1158 1.1.2.2 kent struct dmac_desc *desc, *ndesc; 1159 1.1.2.2 kent u_int32_t dcsr; 1160 1.1.2.2 kent u_int rv = 0; 1161 1.1.2.2 kent 1162 1.1.2.2 kent dcsr = dmac_reg_read(sc, DMAC_DCSR(channel)); 1163 1.1.2.2 kent dmac_reg_write(sc, DMAC_DCSR(channel), dcsr); 1164 1.1.2.2 kent if (dmac_reg_read(sc, DMAC_DCSR(channel)) & DCSR_STOPSTATE) 1165 1.1.2.2 kent dmac_reg_write(sc, DMAC_DCSR(channel), dcsr & ~DCSR_RUN); 1166 1.1.2.2 kent 1167 1.1.2.2 kent if ((dxs = sc->sc_active[channel]) == NULL) { 1168 1.1.2.2 kent printf("%s: Stray DMAC interrupt for unallocated channel %d\n", 1169 1.1.2.2 kent sc->sc_dev.dv_xname, channel); 1170 1.1.2.2 kent return (0); 1171 1.1.2.2 kent } 1172 1.1.2.2 kent 1173 1.1.2.2 kent /* 1174 1.1.2.2 kent * Clear down the interrupt in the DMA Interrupt Register 1175 1.1.2.2 kent */ 1176 1.1.2.2 kent dmac_reg_write(sc, DMAC_DINT, (1u << channel)); 1177 1.1.2.2 kent 1178 1.1.2.2 kent /* 1179 1.1.2.2 kent * If this is a looping request, invoke the 'done' callback and 1180 1.1.2.2 kent * return immediately. 1181 1.1.2.2 kent */ 1182 1.1.2.2 kent if (dxs->dxs_loop_notify != DMAC_DONT_LOOP && 1183 1.1.2.2 kent (dcsr & DCSR_BUSERRINTR) == 0) { 1184 1.1.2.2 kent (dxs->dxs_done)(&dxs->dxs_xfer, 0); 1185 1.1.2.2 kent return (0); 1186 1.1.2.2 kent } 1187 1.1.2.2 kent 1188 1.1.2.2 kent /* 1189 1.1.2.2 kent * Free the descriptors allocated to the completed transfer 1190 1.1.2.2 kent * 1191 1.1.2.2 kent * XXX: If there is more data to transfer in this request, 1192 1.1.2.2 kent * we could simply reuse some or all of the descriptors 1193 1.1.2.2 kent * already allocated for the transfer which just completed. 1194 1.1.2.2 kent */ 1195 1.1.2.2 kent for (desc = SLIST_FIRST(&dxs->dxs_descs); desc; desc = ndesc) { 1196 1.1.2.2 kent ndesc = SLIST_NEXT(desc, d_link); 1197 1.1.2.2 kent SLIST_INSERT_HEAD(&sc->sc_descs, desc, d_link); 1198 1.1.2.2 kent sc->sc_free_descs++; 1199 1.1.2.2 kent } 1200 1.1.2.2 kent 1201 1.1.2.2 kent if ((dcsr & DCSR_BUSERRINTR) || dmac_continue_xfer(sc, dxs) == 0) { 1202 1.1.2.2 kent /* 1203 1.1.2.2 kent * The transfer completed (possibly due to an error), 1204 1.1.2.2 kent * -OR- we were unable to continue any remaining 1205 1.1.2.2 kent * segment of the transfer due to a lack of descriptors. 1206 1.1.2.2 kent * 1207 1.1.2.2 kent * In either case, we have to free up DMAC resources 1208 1.1.2.2 kent * allocated to the request. 1209 1.1.2.2 kent */ 1210 1.1.2.2 kent sc->sc_active[channel] = NULL; 1211 1.1.2.2 kent dmac_free_channel(sc, DMAC_PRI(dxs->dxs_priority), channel); 1212 1.1.2.2 kent dxs->dxs_channel = DMAC_NO_CHANNEL; 1213 1.1.2.2 kent if (dxs->dxs_peripheral != DMAC_PERIPH_NONE) 1214 1.1.2.2 kent dmac_reg_write(sc, DMAC_DRCMR(dxs->dxs_peripheral), 0); 1215 1.1.2.2 kent 1216 1.1.2.2 kent if (dxs->dxs_segs[DMAC_DESC_SRC].ds_nsegs == 0 || 1217 1.1.2.2 kent dxs->dxs_segs[DMAC_DESC_DST].ds_nsegs == 0 || 1218 1.1.2.2 kent (dcsr & DCSR_BUSERRINTR)) { 1219 1.1.2.2 kent 1220 1.1.2.2 kent /* 1221 1.1.2.2 kent * The transfer is complete. 1222 1.1.2.2 kent */ 1223 1.1.2.2 kent dxs->dxs_queue = NULL; 1224 1.1.2.2 kent rv = 1u << DMAC_PRI(dxs->dxs_priority); 1225 1.1.2.2 kent 1226 1.1.2.2 kent if (dxs->dxs_peripheral != DMAC_PERIPH_NONE && 1227 1.1.2.2 kent --sc->sc_periph[dxs->dxs_peripheral].sp_busy != 0) { 1228 1.1.2.2 kent struct dmac_xfer_state *ndxs; 1229 1.1.2.2 kent /* 1230 1.1.2.2 kent * We've just removed the current item for this 1231 1.1.2.2 kent * peripheral, and there is at least one more 1232 1.1.2.2 kent * pending item waiting. Make it current. 1233 1.1.2.2 kent */ 1234 1.1.2.2 kent ndxs = SIMPLEQ_FIRST( 1235 1.1.2.2 kent &sc->sc_periph[dxs->dxs_peripheral].sp_queue); 1236 1.1.2.2 kent KDASSERT(ndxs != NULL); 1237 1.1.2.2 kent SIMPLEQ_REMOVE_HEAD( 1238 1.1.2.2 kent &sc->sc_periph[dxs->dxs_peripheral].sp_queue, 1239 1.1.2.2 kent dxs_link); 1240 1.1.2.2 kent 1241 1.1.2.2 kent ndxs->dxs_queue = 1242 1.1.2.2 kent &sc->sc_queue[DMAC_PRI(dxs->dxs_priority)]; 1243 1.1.2.2 kent SIMPLEQ_INSERT_TAIL(ndxs->dxs_queue, ndxs, 1244 1.1.2.2 kent dxs_link); 1245 1.1.2.2 kent } 1246 1.1.2.2 kent 1247 1.1.2.2 kent (dxs->dxs_done)(&dxs->dxs_xfer, 1248 1.1.2.2 kent (dcsr & DCSR_BUSERRINTR) ? EFAULT : 0); 1249 1.1.2.2 kent } else { 1250 1.1.2.2 kent /* 1251 1.1.2.2 kent * The request is not yet complete, but we were unable 1252 1.1.2.2 kent * to make any headway at this time because there are 1253 1.1.2.2 kent * no free descriptors. Put the request back at the 1254 1.1.2.2 kent * head of the appropriate priority queue. It'll be 1255 1.1.2.2 kent * dealt with as other in-progress transfers complete. 1256 1.1.2.2 kent */ 1257 1.1.2.2 kent SIMPLEQ_INSERT_HEAD( 1258 1.1.2.2 kent &sc->sc_queue[DMAC_PRI(dxs->dxs_priority)], dxs, 1259 1.1.2.2 kent dxs_link); 1260 1.1.2.2 kent } 1261 1.1.2.2 kent } 1262 1.1.2.2 kent 1263 1.1.2.2 kent return (rv); 1264 1.1.2.2 kent } 1265 1.1.2.2 kent 1266 1.1.2.2 kent static int 1267 1.1.2.2 kent dmac_intr(void *arg) 1268 1.1.2.2 kent { 1269 1.1.2.2 kent struct pxadmac_softc *sc = arg; 1270 1.1.2.2 kent u_int32_t rv, mask; 1271 1.1.2.2 kent u_int chan, pri; 1272 1.1.2.2 kent 1273 1.1.2.2 kent rv = dmac_reg_read(sc, DMAC_DINT); 1274 1.1.2.2 kent if ((rv & DMAC_DINT_MASK) == 0) 1275 1.1.2.2 kent return (0); 1276 1.1.2.2 kent 1277 1.1.2.2 kent /* 1278 1.1.2.2 kent * Deal with completed transfers 1279 1.1.2.2 kent */ 1280 1.1.2.2 kent for (chan = 0, mask = 1u, pri = 0; 1281 1.1.2.2 kent chan < DMAC_N_CHANNELS; chan++, mask <<= 1) { 1282 1.1.2.2 kent if (rv & mask) 1283 1.1.2.2 kent pri |= dmac_channel_intr(sc, chan); 1284 1.1.2.2 kent } 1285 1.1.2.2 kent 1286 1.1.2.2 kent /* 1287 1.1.2.2 kent * Now try to start any queued transfers 1288 1.1.2.2 kent */ 1289 1.1.2.2 kent #if (DMAC_N_PRIORITIES > 1) 1290 1.1.2.2 kent if (pri & (1u << DMAC_PRIORITY_HIGH)) 1291 1.1.2.2 kent dmac_start(sc, DMAC_PRIORITY_HIGH); 1292 1.1.2.2 kent if (pri & (1u << DMAC_PRIORITY_MED)) 1293 1.1.2.2 kent dmac_start(sc, DMAC_PRIORITY_MED); 1294 1.1.2.2 kent if (pri & (1u << DMAC_PRIORITY_LOW)) 1295 1.1.2.2 kent dmac_start(sc, DMAC_PRIORITY_LOW); 1296 1.1.2.2 kent #else 1297 1.1.2.2 kent if (pri) 1298 1.1.2.2 kent dmac_start(sc, DMAC_PRIORITY_NORMAL); 1299 1.1.2.2 kent #endif 1300 1.1.2.2 kent 1301 1.1.2.2 kent return (1); 1302 1.1.2.2 kent } 1303
Indexes created Mon Oct 27 17:09:47 GMT 2025