bus_dma.c revision 1.78
1 /* $NetBSD: bus_dma.c,v 1.78 2013/02/15 01:03:43 matt Exp $ */ 2 3 /*- 4 * Copyright (c) 1996, 1997, 1998 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility, 9 * NASA Ames Research Center. 10 * 11 * Redistribution and use in source and binary forms, with or without 12 * modification, are permitted provided that the following conditions 13 * are met: 14 * 1. Redistributions of source code must retain the above copyright 15 * notice, this list of conditions and the following disclaimer. 16 * 2. Redistributions in binary form must reproduce the above copyright 17 * notice, this list of conditions and the following disclaimer in the 18 * documentation and/or other materials provided with the distribution. 19 * 20 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 21 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 22 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 23 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 24 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 25 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 26 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 27 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 28 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 29 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 30 * POSSIBILITY OF SUCH DAMAGE. 31 */ 32 33 #define _ARM32_BUS_DMA_PRIVATE 34 35 #include <sys/cdefs.h> 36 __KERNEL_RCSID(0, "$NetBSD: bus_dma.c,v 1.78 2013/02/15 01:03:43 matt Exp $"); 37 38 #include <sys/param.h> 39 #include <sys/systm.h> 40 #include <sys/kernel.h> 41 #include <sys/proc.h> 42 #include <sys/buf.h> 43 #include <sys/reboot.h> 44 #include <sys/conf.h> 45 #include <sys/file.h> 46 #include <sys/malloc.h> 47 #include <sys/mbuf.h> 48 #include <sys/vnode.h> 49 #include <sys/device.h> 50 51 #include <uvm/uvm.h> 52 53 #include <sys/bus.h> 54 #include <machine/cpu.h> 55 56 #include <arm/cpufunc.h> 57 58 #ifdef BUSDMA_COUNTERS 59 static struct evcnt bus_dma_creates = 60 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "busdma", "creates"); 61 static struct evcnt bus_dma_bounced_creates = 62 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "busdma", "bounced creates"); 63 static struct evcnt bus_dma_loads = 64 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "busdma", "loads"); 65 static struct evcnt bus_dma_bounced_loads = 66 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "busdma", "bounced loads"); 67 static struct evcnt bus_dma_read_bounces = 68 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "busdma", "read bounces"); 69 static struct evcnt bus_dma_write_bounces = 70 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "busdma", "write bounces"); 71 static struct evcnt bus_dma_bounced_unloads = 72 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "busdma", "bounced unloads"); 73 static struct evcnt bus_dma_unloads = 74 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "busdma", "unloads"); 75 static struct evcnt bus_dma_bounced_destroys = 76 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "busdma", "bounced destroys"); 77 static struct evcnt bus_dma_destroys = 78 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "busdma", "destroys"); 79 static struct evcnt bus_dma_sync_prereadwrite = 80 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "busdma", "sync prereadwrite"); 81 static struct evcnt bus_dma_sync_preread_begin = 82 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "busdma", "sync preread begin"); 83 static struct evcnt bus_dma_sync_preread = 84 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "busdma", "sync preread"); 85 static struct evcnt bus_dma_sync_preread_tail = 86 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "busdma", "sync preread tail"); 87 static struct evcnt bus_dma_sync_prewrite = 88 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "busdma", "sync prewrite"); 89 static struct evcnt bus_dma_sync_postread = 90 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "busdma", "sync postread"); 91 static struct evcnt bus_dma_sync_postreadwrite = 92 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "busdma", "sync postreadwrite"); 93 static struct evcnt bus_dma_sync_postwrite = 94 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "busdma", "sync postwrite"); 95 96 EVCNT_ATTACH_STATIC(bus_dma_creates); 97 EVCNT_ATTACH_STATIC(bus_dma_bounced_creates); 98 EVCNT_ATTACH_STATIC(bus_dma_loads); 99 EVCNT_ATTACH_STATIC(bus_dma_bounced_loads); 100 EVCNT_ATTACH_STATIC(bus_dma_read_bounces); 101 EVCNT_ATTACH_STATIC(bus_dma_write_bounces); 102 EVCNT_ATTACH_STATIC(bus_dma_unloads); 103 EVCNT_ATTACH_STATIC(bus_dma_bounced_unloads); 104 EVCNT_ATTACH_STATIC(bus_dma_destroys); 105 EVCNT_ATTACH_STATIC(bus_dma_bounced_destroys); 106 EVCNT_ATTACH_STATIC(bus_dma_sync_prereadwrite); 107 EVCNT_ATTACH_STATIC(bus_dma_sync_preread_begin); 108 EVCNT_ATTACH_STATIC(bus_dma_sync_preread); 109 EVCNT_ATTACH_STATIC(bus_dma_sync_preread_tail); 110 EVCNT_ATTACH_STATIC(bus_dma_sync_prewrite); 111 EVCNT_ATTACH_STATIC(bus_dma_sync_postread); 112 EVCNT_ATTACH_STATIC(bus_dma_sync_postreadwrite); 113 EVCNT_ATTACH_STATIC(bus_dma_sync_postwrite); 114 115 #define STAT_INCR(x) (bus_dma_ ## x.ev_count++) 116 #else 117 #define STAT_INCR(x) /*(bus_dma_ ## x.ev_count++)*/ 118 #endif 119 120 int _bus_dmamap_load_buffer(bus_dma_tag_t, bus_dmamap_t, void *, 121 bus_size_t, struct vmspace *, int); 122 static struct arm32_dma_range * 123 _bus_dma_paddr_inrange(struct arm32_dma_range *, int, paddr_t); 124 125 /* 126 * Check to see if the specified page is in an allowed DMA range. 127 */ 128 inline struct arm32_dma_range * 129 _bus_dma_paddr_inrange(struct arm32_dma_range *ranges, int nranges, 130 bus_addr_t curaddr) 131 { 132 struct arm32_dma_range *dr; 133 int i; 134 135 for (i = 0, dr = ranges; i < nranges; i++, dr++) { 136 if (curaddr >= dr->dr_sysbase && 137 round_page(curaddr) <= (dr->dr_sysbase + dr->dr_len)) 138 return (dr); 139 } 140 141 return (NULL); 142 } 143 144 /* 145 * Check to see if the specified busaddr is in an allowed DMA range. 146 */ 147 static inline paddr_t 148 _bus_dma_busaddr_to_paddr(bus_dma_tag_t t, bus_addr_t curaddr) 149 { 150 struct arm32_dma_range *dr; 151 u_int i; 152 153 if (t->_nranges == 0) 154 return curaddr; 155 156 for (i = 0, dr = t->_ranges; i < t->_nranges; i++, dr++) { 157 if (dr->dr_busbase <= curaddr 158 && round_page(curaddr) <= dr->dr_busbase + dr->dr_len) 159 return curaddr - dr->dr_busbase + dr->dr_sysbase; 160 } 161 panic("%s: curaddr %#lx not in range", __func__, curaddr); 162 } 163 164 /* 165 * Common function to load the specified physical address into the 166 * DMA map, coalescing segments and boundary checking as necessary. 167 */ 168 static int 169 _bus_dmamap_load_paddr(bus_dma_tag_t t, bus_dmamap_t map, 170 bus_addr_t paddr, bus_size_t size, bool coherent) 171 { 172 bus_dma_segment_t * const segs = map->dm_segs; 173 int nseg = map->dm_nsegs; 174 bus_addr_t lastaddr; 175 bus_addr_t bmask = ~(map->_dm_boundary - 1); 176 bus_addr_t curaddr; 177 bus_size_t sgsize; 178 uint32_t _ds_flags = coherent ? _BUS_DMAMAP_COHERENT : 0; 179 180 if (nseg > 0) 181 lastaddr = segs[nseg-1].ds_addr + segs[nseg-1].ds_len; 182 else 183 lastaddr = 0xdead; 184 185 again: 186 sgsize = size; 187 188 /* Make sure we're in an allowed DMA range. */ 189 if (t->_ranges != NULL) { 190 /* XXX cache last result? */ 191 const struct arm32_dma_range * const dr = 192 _bus_dma_paddr_inrange(t->_ranges, t->_nranges, paddr); 193 if (dr == NULL) 194 return (EINVAL); 195 196 /* 197 * If this region is coherent, mark the segment as coherent. 198 */ 199 _ds_flags |= dr->dr_flags & _BUS_DMAMAP_COHERENT; 200 201 /* 202 * In a valid DMA range. Translate the physical 203 * memory address to an address in the DMA window. 204 */ 205 curaddr = (paddr - dr->dr_sysbase) + dr->dr_busbase; 206 #if 0 207 printf("%p: %#lx: range %#lx/%#lx/%#lx/%#x: %#x <-- %#lx\n", 208 t, paddr, dr->dr_sysbase, dr->dr_busbase, 209 dr->dr_len, dr->dr_flags, _ds_flags, curaddr); 210 #endif 211 } else 212 curaddr = paddr; 213 214 /* 215 * Make sure we don't cross any boundaries. 216 */ 217 if (map->_dm_boundary > 0) { 218 bus_addr_t baddr; /* next boundary address */ 219 220 baddr = (curaddr + map->_dm_boundary) & bmask; 221 if (sgsize > (baddr - curaddr)) 222 sgsize = (baddr - curaddr); 223 } 224 225 /* 226 * Insert chunk into a segment, coalescing with the 227 * previous segment if possible. 228 */ 229 if (nseg > 0 && curaddr == lastaddr && 230 segs[nseg-1].ds_len + sgsize <= map->dm_maxsegsz && 231 ((segs[nseg-1]._ds_flags ^ _ds_flags) & _BUS_DMAMAP_COHERENT) == 0 && 232 (map->_dm_boundary == 0 || 233 (segs[nseg-1].ds_addr & bmask) == (curaddr & bmask))) { 234 /* coalesce */ 235 segs[nseg-1].ds_len += sgsize; 236 } else if (nseg >= map->_dm_segcnt) { 237 return (EFBIG); 238 } else { 239 /* new segment */ 240 segs[nseg].ds_addr = curaddr; 241 segs[nseg].ds_len = sgsize; 242 segs[nseg]._ds_flags = _ds_flags; 243 nseg++; 244 } 245 246 lastaddr = curaddr + sgsize; 247 248 paddr += sgsize; 249 size -= sgsize; 250 if (size > 0) 251 goto again; 252 253 map->_dm_flags &= (_ds_flags & _BUS_DMAMAP_COHERENT); 254 map->dm_nsegs = nseg; 255 return (0); 256 } 257 258 #ifdef _ARM32_NEED_BUS_DMA_BOUNCE 259 static int _bus_dma_alloc_bouncebuf(bus_dma_tag_t t, bus_dmamap_t map, 260 bus_size_t size, int flags); 261 static void _bus_dma_free_bouncebuf(bus_dma_tag_t t, bus_dmamap_t map); 262 static int _bus_dma_uiomove(void *buf, struct uio *uio, size_t n, 263 int direction); 264 265 static int 266 _bus_dma_load_bouncebuf(bus_dma_tag_t t, bus_dmamap_t map, void *buf, 267 size_t buflen, int buftype, int flags) 268 { 269 struct arm32_bus_dma_cookie * const cookie = map->_dm_cookie; 270 struct vmspace * const vm = vmspace_kernel(); 271 int error; 272 273 KASSERT(cookie != NULL); 274 KASSERT(cookie->id_flags & _BUS_DMA_MIGHT_NEED_BOUNCE); 275 276 /* 277 * Allocate bounce pages, if necessary. 278 */ 279 if ((cookie->id_flags & _BUS_DMA_HAS_BOUNCE) == 0) { 280 error = _bus_dma_alloc_bouncebuf(t, map, buflen, flags); 281 if (error) 282 return (error); 283 } 284 285 /* 286 * Cache a pointer to the caller's buffer and load the DMA map 287 * with the bounce buffer. 288 */ 289 cookie->id_origbuf = buf; 290 cookie->id_origbuflen = buflen; 291 error = _bus_dmamap_load_buffer(t, map, cookie->id_bouncebuf, 292 buflen, vm, flags); 293 if (error) 294 return (error); 295 296 STAT_INCR(bounced_loads); 297 map->dm_mapsize = buflen; 298 map->_dm_vmspace = vm; 299 map->_dm_buftype = buftype; 300 301 /* ...so _bus_dmamap_sync() knows we're bouncing */ 302 map->_dm_flags |= _BUS_DMAMAP_IS_BOUNCING; 303 cookie->id_flags |= _BUS_DMA_IS_BOUNCING; 304 return 0; 305 } 306 #endif /* _ARM32_NEED_BUS_DMA_BOUNCE */ 307 308 /* 309 * Common function for DMA map creation. May be called by bus-specific 310 * DMA map creation functions. 311 */ 312 int 313 _bus_dmamap_create(bus_dma_tag_t t, bus_size_t size, int nsegments, 314 bus_size_t maxsegsz, bus_size_t boundary, int flags, bus_dmamap_t *dmamp) 315 { 316 struct arm32_bus_dmamap *map; 317 void *mapstore; 318 size_t mapsize; 319 320 #ifdef DEBUG_DMA 321 printf("dmamap_create: t=%p size=%lx nseg=%x msegsz=%lx boundary=%lx flags=%x\n", 322 t, size, nsegments, maxsegsz, boundary, flags); 323 #endif /* DEBUG_DMA */ 324 325 /* 326 * Allocate and initialize the DMA map. The end of the map 327 * is a variable-sized array of segments, so we allocate enough 328 * room for them in one shot. 329 * 330 * Note we don't preserve the WAITOK or NOWAIT flags. Preservation 331 * of ALLOCNOW notifies others that we've reserved these resources, 332 * and they are not to be freed. 333 * 334 * The bus_dmamap_t includes one bus_dma_segment_t, hence 335 * the (nsegments - 1). 336 */ 337 mapsize = sizeof(struct arm32_bus_dmamap) + 338 (sizeof(bus_dma_segment_t) * (nsegments - 1)); 339 const int mallocflags = M_ZERO|(flags & BUS_DMA_NOWAIT) ? M_NOWAIT : M_WAITOK; 340 if ((mapstore = malloc(mapsize, M_DMAMAP, mallocflags)) == NULL) 341 return (ENOMEM); 342 343 map = (struct arm32_bus_dmamap *)mapstore; 344 map->_dm_size = size; 345 map->_dm_segcnt = nsegments; 346 map->_dm_maxmaxsegsz = maxsegsz; 347 map->_dm_boundary = boundary; 348 map->_dm_flags = flags & ~(BUS_DMA_WAITOK|BUS_DMA_NOWAIT); 349 map->_dm_origbuf = NULL; 350 map->_dm_buftype = _BUS_DMA_BUFTYPE_INVALID; 351 map->_dm_vmspace = vmspace_kernel(); 352 map->_dm_cookie = NULL; 353 map->dm_maxsegsz = maxsegsz; 354 map->dm_mapsize = 0; /* no valid mappings */ 355 map->dm_nsegs = 0; 356 357 *dmamp = map; 358 359 #ifdef _ARM32_NEED_BUS_DMA_BOUNCE 360 struct arm32_bus_dma_cookie *cookie; 361 int cookieflags; 362 void *cookiestore; 363 size_t cookiesize; 364 int error; 365 366 cookieflags = 0; 367 368 if (t->_may_bounce != NULL) { 369 error = (*t->_may_bounce)(t, map, flags, &cookieflags); 370 if (error != 0) 371 goto out; 372 } 373 374 if (t->_ranges != NULL) 375 cookieflags |= _BUS_DMA_MIGHT_NEED_BOUNCE; 376 377 if ((cookieflags & _BUS_DMA_MIGHT_NEED_BOUNCE) == 0) { 378 STAT_INCR(creates); 379 return 0; 380 } 381 382 cookiesize = sizeof(struct arm32_bus_dma_cookie) + 383 (sizeof(bus_dma_segment_t) * map->_dm_segcnt); 384 385 /* 386 * Allocate our cookie. 387 */ 388 if ((cookiestore = malloc(cookiesize, M_DMAMAP, mallocflags)) == NULL) { 389 error = ENOMEM; 390 goto out; 391 } 392 cookie = (struct arm32_bus_dma_cookie *)cookiestore; 393 cookie->id_flags = cookieflags; 394 map->_dm_cookie = cookie; 395 STAT_INCR(bounced_creates); 396 397 error = _bus_dma_alloc_bouncebuf(t, map, size, flags); 398 out: 399 if (error) 400 _bus_dmamap_destroy(t, map); 401 #else 402 STAT_INCR(creates); 403 #endif /* _ARM32_NEED_BUS_DMA_BOUNCE */ 404 405 #ifdef DEBUG_DMA 406 printf("dmamap_create:map=%p\n", map); 407 #endif /* DEBUG_DMA */ 408 return (0); 409 } 410 411 /* 412 * Common function for DMA map destruction. May be called by bus-specific 413 * DMA map destruction functions. 414 */ 415 void 416 _bus_dmamap_destroy(bus_dma_tag_t t, bus_dmamap_t map) 417 { 418 419 #ifdef DEBUG_DMA 420 printf("dmamap_destroy: t=%p map=%p\n", t, map); 421 #endif /* DEBUG_DMA */ 422 #ifdef _ARM32_NEED_BUS_DMA_BOUNCE 423 struct arm32_bus_dma_cookie *cookie = map->_dm_cookie; 424 425 /* 426 * Free any bounce pages this map might hold. 427 */ 428 if (cookie != NULL) { 429 if (cookie->id_flags & _BUS_DMA_IS_BOUNCING) 430 STAT_INCR(bounced_unloads); 431 map->dm_nsegs = 0; 432 if (cookie->id_flags & _BUS_DMA_HAS_BOUNCE) 433 _bus_dma_free_bouncebuf(t, map); 434 STAT_INCR(bounced_destroys); 435 free(cookie, M_DMAMAP); 436 } else 437 #endif 438 STAT_INCR(destroys); 439 440 if (map->dm_nsegs > 0) 441 STAT_INCR(unloads); 442 443 free(map, M_DMAMAP); 444 } 445 446 /* 447 * Common function for loading a DMA map with a linear buffer. May 448 * be called by bus-specific DMA map load functions. 449 */ 450 int 451 _bus_dmamap_load(bus_dma_tag_t t, bus_dmamap_t map, void *buf, 452 bus_size_t buflen, struct proc *p, int flags) 453 { 454 struct vmspace *vm; 455 int error; 456 457 #ifdef DEBUG_DMA 458 printf("dmamap_load: t=%p map=%p buf=%p len=%lx p=%p f=%d\n", 459 t, map, buf, buflen, p, flags); 460 #endif /* DEBUG_DMA */ 461 462 if (map->dm_nsegs > 0) { 463 #ifdef _ARM32_NEED_BUS_DMA_BOUNCE 464 struct arm32_bus_dma_cookie *cookie = map->_dm_cookie; 465 if (cookie != NULL) { 466 if (cookie->id_flags & _BUS_DMA_IS_BOUNCING) { 467 STAT_INCR(bounced_unloads); 468 cookie->id_flags &= ~_BUS_DMA_IS_BOUNCING; 469 map->_dm_flags &= ~_BUS_DMAMAP_IS_BOUNCING; 470 } 471 } else 472 #endif 473 STAT_INCR(unloads); 474 } 475 476 /* 477 * Make sure that on error condition we return "no valid mappings". 478 */ 479 map->dm_mapsize = 0; 480 map->dm_nsegs = 0; 481 map->_dm_buftype = _BUS_DMA_BUFTYPE_INVALID; 482 KASSERTMSG(map->dm_maxsegsz <= map->_dm_maxmaxsegsz, 483 "dm_maxsegsz %lu _dm_maxmaxsegsz %lu", 484 map->dm_maxsegsz, map->_dm_maxmaxsegsz); 485 486 if (buflen > map->_dm_size) 487 return (EINVAL); 488 489 if (p != NULL) { 490 vm = p->p_vmspace; 491 } else { 492 vm = vmspace_kernel(); 493 } 494 495 /* _bus_dmamap_load_buffer() clears this if we're not... */ 496 map->_dm_flags |= _BUS_DMAMAP_COHERENT; 497 498 error = _bus_dmamap_load_buffer(t, map, buf, buflen, vm, flags); 499 if (error == 0) { 500 map->dm_mapsize = buflen; 501 map->_dm_vmspace = vm; 502 map->_dm_origbuf = buf; 503 map->_dm_buftype = _BUS_DMA_BUFTYPE_LINEAR; 504 return 0; 505 } 506 #ifdef _ARM32_NEED_BUS_DMA_BOUNCE 507 struct arm32_bus_dma_cookie * const cookie = map->_dm_cookie; 508 if (cookie != NULL && (cookie->id_flags & _BUS_DMA_MIGHT_NEED_BOUNCE)) { 509 error = _bus_dma_load_bouncebuf(t, map, buf, buflen, 510 _BUS_DMA_BUFTYPE_LINEAR, flags); 511 } 512 #endif 513 return (error); 514 } 515 516 /* 517 * Like _bus_dmamap_load(), but for mbufs. 518 */ 519 int 520 _bus_dmamap_load_mbuf(bus_dma_tag_t t, bus_dmamap_t map, struct mbuf *m0, 521 int flags) 522 { 523 int error; 524 struct mbuf *m; 525 526 #ifdef DEBUG_DMA 527 printf("dmamap_load_mbuf: t=%p map=%p m0=%p f=%d\n", 528 t, map, m0, flags); 529 #endif /* DEBUG_DMA */ 530 531 if (map->dm_nsegs > 0) { 532 #ifdef _ARM32_NEED_BUS_DMA_BOUNCE 533 struct arm32_bus_dma_cookie *cookie = map->_dm_cookie; 534 if (cookie != NULL) { 535 if (cookie->id_flags & _BUS_DMA_IS_BOUNCING) { 536 STAT_INCR(bounced_unloads); 537 cookie->id_flags &= ~_BUS_DMA_IS_BOUNCING; 538 map->_dm_flags &= ~_BUS_DMAMAP_IS_BOUNCING; 539 } 540 } else 541 #endif 542 STAT_INCR(unloads); 543 } 544 545 /* 546 * Make sure that on error condition we return "no valid mappings." 547 */ 548 map->dm_mapsize = 0; 549 map->dm_nsegs = 0; 550 map->_dm_buftype = _BUS_DMA_BUFTYPE_INVALID; 551 KASSERTMSG(map->dm_maxsegsz <= map->_dm_maxmaxsegsz, 552 "dm_maxsegsz %lu _dm_maxmaxsegsz %lu", 553 map->dm_maxsegsz, map->_dm_maxmaxsegsz); 554 555 #ifdef DIAGNOSTIC 556 if ((m0->m_flags & M_PKTHDR) == 0) 557 panic("_bus_dmamap_load_mbuf: no packet header"); 558 #endif /* DIAGNOSTIC */ 559 560 if (m0->m_pkthdr.len > map->_dm_size) 561 return (EINVAL); 562 563 /* _bus_dmamap_load_paddr() clears this if we're not... */ 564 map->_dm_flags |= _BUS_DMAMAP_COHERENT; 565 566 error = 0; 567 for (m = m0; m != NULL && error == 0; m = m->m_next) { 568 int offset; 569 int remainbytes; 570 const struct vm_page * const *pgs; 571 paddr_t paddr; 572 int size; 573 574 if (m->m_len == 0) 575 continue; 576 /* 577 * Don't allow reads in read-only mbufs. 578 */ 579 if (M_ROMAP(m) && (flags & BUS_DMA_READ)) { 580 error = EFAULT; 581 break; 582 } 583 switch (m->m_flags & (M_EXT|M_CLUSTER|M_EXT_PAGES)) { 584 case M_EXT|M_CLUSTER: 585 /* XXX KDASSERT */ 586 KASSERT(m->m_ext.ext_paddr != M_PADDR_INVALID); 587 paddr = m->m_ext.ext_paddr + 588 (m->m_data - m->m_ext.ext_buf); 589 size = m->m_len; 590 error = _bus_dmamap_load_paddr(t, map, paddr, size, 591 false); 592 break; 593 594 case M_EXT|M_EXT_PAGES: 595 KASSERT(m->m_ext.ext_buf <= m->m_data); 596 KASSERT(m->m_data <= 597 m->m_ext.ext_buf + m->m_ext.ext_size); 598 599 offset = (vaddr_t)m->m_data - 600 trunc_page((vaddr_t)m->m_ext.ext_buf); 601 remainbytes = m->m_len; 602 603 /* skip uninteresting pages */ 604 pgs = (const struct vm_page * const *) 605 m->m_ext.ext_pgs + (offset >> PAGE_SHIFT); 606 607 offset &= PAGE_MASK; /* offset in the first page */ 608 609 /* load each page */ 610 while (remainbytes > 0) { 611 const struct vm_page *pg; 612 613 size = MIN(remainbytes, PAGE_SIZE - offset); 614 615 pg = *pgs++; 616 KASSERT(pg); 617 paddr = VM_PAGE_TO_PHYS(pg) + offset; 618 619 error = _bus_dmamap_load_paddr(t, map, 620 paddr, size, false); 621 if (error) 622 break; 623 offset = 0; 624 remainbytes -= size; 625 } 626 break; 627 628 case 0: 629 paddr = m->m_paddr + M_BUFOFFSET(m) + 630 (m->m_data - M_BUFADDR(m)); 631 size = m->m_len; 632 error = _bus_dmamap_load_paddr(t, map, paddr, size, 633 false); 634 break; 635 636 default: 637 error = _bus_dmamap_load_buffer(t, map, m->m_data, 638 m->m_len, vmspace_kernel(), flags); 639 } 640 } 641 if (error == 0) { 642 map->dm_mapsize = m0->m_pkthdr.len; 643 map->_dm_origbuf = m0; 644 map->_dm_buftype = _BUS_DMA_BUFTYPE_MBUF; 645 map->_dm_vmspace = vmspace_kernel(); /* always kernel */ 646 return 0; 647 } 648 #ifdef _ARM32_NEED_BUS_DMA_BOUNCE 649 struct arm32_bus_dma_cookie * const cookie = map->_dm_cookie; 650 if (cookie != NULL && (cookie->id_flags & _BUS_DMA_MIGHT_NEED_BOUNCE)) { 651 error = _bus_dma_load_bouncebuf(t, map, m0, m0->m_pkthdr.len, 652 _BUS_DMA_BUFTYPE_MBUF, flags); 653 } 654 #endif 655 return (error); 656 } 657 658 /* 659 * Like _bus_dmamap_load(), but for uios. 660 */ 661 int 662 _bus_dmamap_load_uio(bus_dma_tag_t t, bus_dmamap_t map, struct uio *uio, 663 int flags) 664 { 665 int i, error; 666 bus_size_t minlen, resid; 667 struct iovec *iov; 668 void *addr; 669 670 /* 671 * Make sure that on error condition we return "no valid mappings." 672 */ 673 map->dm_mapsize = 0; 674 map->dm_nsegs = 0; 675 KASSERTMSG(map->dm_maxsegsz <= map->_dm_maxmaxsegsz, 676 "dm_maxsegsz %lu _dm_maxmaxsegsz %lu", 677 map->dm_maxsegsz, map->_dm_maxmaxsegsz); 678 679 resid = uio->uio_resid; 680 iov = uio->uio_iov; 681 682 /* _bus_dmamap_load_buffer() clears this if we're not... */ 683 map->_dm_flags |= _BUS_DMAMAP_COHERENT; 684 685 error = 0; 686 for (i = 0; i < uio->uio_iovcnt && resid != 0 && error == 0; i++) { 687 /* 688 * Now at the first iovec to load. Load each iovec 689 * until we have exhausted the residual count. 690 */ 691 minlen = resid < iov[i].iov_len ? resid : iov[i].iov_len; 692 addr = (void *)iov[i].iov_base; 693 694 error = _bus_dmamap_load_buffer(t, map, addr, minlen, 695 uio->uio_vmspace, flags); 696 697 resid -= minlen; 698 } 699 if (error == 0) { 700 map->dm_mapsize = uio->uio_resid; 701 map->_dm_origbuf = uio; 702 map->_dm_buftype = _BUS_DMA_BUFTYPE_UIO; 703 map->_dm_vmspace = uio->uio_vmspace; 704 } 705 return (error); 706 } 707 708 /* 709 * Like _bus_dmamap_load(), but for raw memory allocated with 710 * bus_dmamem_alloc(). 711 */ 712 int 713 _bus_dmamap_load_raw(bus_dma_tag_t t, bus_dmamap_t map, 714 bus_dma_segment_t *segs, int nsegs, bus_size_t size, int flags) 715 { 716 717 panic("_bus_dmamap_load_raw: not implemented"); 718 } 719 720 /* 721 * Common function for unloading a DMA map. May be called by 722 * bus-specific DMA map unload functions. 723 */ 724 void 725 _bus_dmamap_unload(bus_dma_tag_t t, bus_dmamap_t map) 726 { 727 728 #ifdef DEBUG_DMA 729 printf("dmamap_unload: t=%p map=%p\n", t, map); 730 #endif /* DEBUG_DMA */ 731 732 /* 733 * No resources to free; just mark the mappings as 734 * invalid. 735 */ 736 map->dm_mapsize = 0; 737 map->dm_nsegs = 0; 738 map->_dm_origbuf = NULL; 739 map->_dm_buftype = _BUS_DMA_BUFTYPE_INVALID; 740 map->_dm_vmspace = NULL; 741 } 742 743 static void 744 _bus_dmamap_sync_segment(vaddr_t va, paddr_t pa, vsize_t len, int ops, bool readonly_p) 745 { 746 KASSERT((va & PAGE_MASK) == (pa & PAGE_MASK)); 747 #if 0 748 printf("sync_segment: va=%#lx pa=%#lx len=%#lx ops=%#x ro=%d\n", 749 va, pa, len, ops, readonly_p); 750 #endif 751 752 switch (ops) { 753 case BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE: 754 if (!readonly_p) { 755 STAT_INCR(sync_prereadwrite); 756 cpu_dcache_wbinv_range(va, len); 757 cpu_sdcache_wbinv_range(va, pa, len); 758 break; 759 } 760 /* FALLTHROUGH */ 761 762 case BUS_DMASYNC_PREREAD: { 763 const size_t line_size = arm_dcache_align; 764 const size_t line_mask = arm_dcache_align_mask; 765 vsize_t misalignment = va & line_mask; 766 if (misalignment) { 767 va -= misalignment; 768 pa -= misalignment; 769 len += misalignment; 770 STAT_INCR(sync_preread_begin); 771 cpu_dcache_wbinv_range(va, line_size); 772 cpu_sdcache_wbinv_range(va, pa, line_size); 773 if (len <= line_size) 774 break; 775 va += line_size; 776 pa += line_size; 777 len -= line_size; 778 } 779 misalignment = len & line_mask; 780 len -= misalignment; 781 if (len > 0) { 782 STAT_INCR(sync_preread); 783 cpu_dcache_inv_range(va, len); 784 cpu_sdcache_inv_range(va, pa, len); 785 } 786 if (misalignment) { 787 va += len; 788 pa += len; 789 STAT_INCR(sync_preread_tail); 790 cpu_dcache_wbinv_range(va, line_size); 791 cpu_sdcache_wbinv_range(va, pa, line_size); 792 } 793 break; 794 } 795 796 case BUS_DMASYNC_PREWRITE: 797 STAT_INCR(sync_prewrite); 798 cpu_dcache_wb_range(va, len); 799 cpu_sdcache_wb_range(va, pa, len); 800 break; 801 802 #ifdef CPU_CORTEX 803 /* 804 * Cortex CPUs can do speculative loads so we need to clean the cache 805 * after a DMA read to deal with any speculatively loaded cache lines. 806 * Since these can't be dirty, we can just invalidate them and don't 807 * have to worry about having to write back their contents. 808 */ 809 case BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE: 810 STAT_INCR(sync_postreadwrite); 811 cpu_dcache_inv_range(va, len); 812 cpu_sdcache_inv_range(va, pa, len); 813 break; 814 case BUS_DMASYNC_POSTREAD: 815 STAT_INCR(sync_postread); 816 cpu_dcache_inv_range(va, len); 817 cpu_sdcache_inv_range(va, pa, len); 818 break; 819 #endif 820 } 821 } 822 823 static inline void 824 _bus_dmamap_sync_linear(bus_dma_tag_t t, bus_dmamap_t map, bus_addr_t offset, 825 bus_size_t len, int ops) 826 { 827 bus_dma_segment_t *ds = map->dm_segs; 828 vaddr_t va = (vaddr_t) map->_dm_origbuf; 829 #ifdef _ARM32_NEED_BUS_DMA_BOUNCE 830 if (map->_dm_flags & _BUS_DMAMAP_IS_BOUNCING) { 831 struct arm32_bus_dma_cookie * const cookie = map->_dm_cookie; 832 va = (vaddr_t) cookie->id_bouncebuf; 833 } 834 #endif 835 836 while (len > 0) { 837 while (offset >= ds->ds_len) { 838 offset -= ds->ds_len; 839 va += ds->ds_len; 840 ds++; 841 } 842 843 paddr_t pa = _bus_dma_busaddr_to_paddr(t, ds->ds_addr + offset); 844 size_t seglen = min(len, ds->ds_len - offset); 845 846 if ((ds->_ds_flags & _BUS_DMAMAP_COHERENT) == 0) 847 _bus_dmamap_sync_segment(va + offset, pa, seglen, ops, 848 false); 849 850 offset += seglen; 851 len -= seglen; 852 } 853 } 854 855 static inline void 856 _bus_dmamap_sync_mbuf(bus_dma_tag_t t, bus_dmamap_t map, bus_size_t offset, 857 bus_size_t len, int ops) 858 { 859 bus_dma_segment_t *ds = map->dm_segs; 860 struct mbuf *m = map->_dm_origbuf; 861 bus_size_t voff = offset; 862 bus_size_t ds_off = offset; 863 864 while (len > 0) { 865 /* Find the current dma segment */ 866 while (ds_off >= ds->ds_len) { 867 ds_off -= ds->ds_len; 868 ds++; 869 } 870 /* Find the current mbuf. */ 871 while (voff >= m->m_len) { 872 voff -= m->m_len; 873 m = m->m_next; 874 } 875 876 /* 877 * Now at the first mbuf to sync; nail each one until 878 * we have exhausted the length. 879 */ 880 vsize_t seglen = min(len, min(m->m_len - voff, ds->ds_len - ds_off)); 881 vaddr_t va = mtod(m, vaddr_t) + voff; 882 paddr_t pa = _bus_dma_busaddr_to_paddr(t, ds->ds_addr + ds_off); 883 884 /* 885 * We can save a lot of work here if we know the mapping 886 * is read-only at the MMU: 887 * 888 * If a mapping is read-only, no dirty cache blocks will 889 * exist for it. If a writable mapping was made read-only, 890 * we know any dirty cache lines for the range will have 891 * been cleaned for us already. Therefore, if the upper 892 * layer can tell us we have a read-only mapping, we can 893 * skip all cache cleaning. 894 * 895 * NOTE: This only works if we know the pmap cleans pages 896 * before making a read-write -> read-only transition. If 897 * this ever becomes non-true (e.g. Physically Indexed 898 * cache), this will have to be revisited. 899 */ 900 901 if ((ds->_ds_flags & _BUS_DMAMAP_COHERENT) == 0) 902 _bus_dmamap_sync_segment(va, pa, seglen, ops, 903 M_ROMAP(m)); 904 voff += seglen; 905 ds_off += seglen; 906 len -= seglen; 907 } 908 } 909 910 static inline void 911 _bus_dmamap_sync_uio(bus_dma_tag_t t, bus_dmamap_t map, bus_addr_t offset, 912 bus_size_t len, int ops) 913 { 914 bus_dma_segment_t *ds = map->dm_segs; 915 struct uio *uio = map->_dm_origbuf; 916 struct iovec *iov = uio->uio_iov; 917 bus_size_t voff = offset; 918 bus_size_t ds_off = offset; 919 920 while (len > 0) { 921 /* Find the current dma segment */ 922 while (ds_off >= ds->ds_len) { 923 ds_off -= ds->ds_len; 924 ds++; 925 } 926 927 /* Find the current iovec. */ 928 while (voff >= iov->iov_len) { 929 voff -= iov->iov_len; 930 iov++; 931 } 932 933 /* 934 * Now at the first iovec to sync; nail each one until 935 * we have exhausted the length. 936 */ 937 vsize_t seglen = min(len, min(iov->iov_len - voff, ds->ds_len - ds_off)); 938 vaddr_t va = (vaddr_t) iov->iov_base + voff; 939 paddr_t pa = _bus_dma_busaddr_to_paddr(t, ds->ds_addr + ds_off); 940 941 if ((ds->_ds_flags & _BUS_DMAMAP_COHERENT) == 0) 942 _bus_dmamap_sync_segment(va, pa, seglen, ops, false); 943 944 voff += seglen; 945 ds_off += seglen; 946 len -= seglen; 947 } 948 } 949 950 /* 951 * Common function for DMA map synchronization. May be called 952 * by bus-specific DMA map synchronization functions. 953 * 954 * This version works for the Virtually Indexed Virtually Tagged 955 * cache found on 32-bit ARM processors. 956 * 957 * XXX Should have separate versions for write-through vs. 958 * XXX write-back caches. We currently assume write-back 959 * XXX here, which is not as efficient as it could be for 960 * XXX the write-through case. 961 */ 962 void 963 _bus_dmamap_sync(bus_dma_tag_t t, bus_dmamap_t map, bus_addr_t offset, 964 bus_size_t len, int ops) 965 { 966 #ifdef DEBUG_DMA 967 printf("dmamap_sync: t=%p map=%p offset=%lx len=%lx ops=%x\n", 968 t, map, offset, len, ops); 969 #endif /* DEBUG_DMA */ 970 971 /* 972 * Mixing of PRE and POST operations is not allowed. 973 */ 974 if ((ops & (BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE)) != 0 && 975 (ops & (BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE)) != 0) 976 panic("_bus_dmamap_sync: mix PRE and POST"); 977 978 #ifdef DIAGNOSTIC 979 if (offset >= map->dm_mapsize) 980 panic("_bus_dmamap_sync: bad offset %lu (map size is %lu)", 981 offset, map->dm_mapsize); 982 if (len == 0 || (offset + len) > map->dm_mapsize) 983 panic("_bus_dmamap_sync: bad length"); 984 #endif 985 986 /* 987 * For a virtually-indexed write-back cache, we need 988 * to do the following things: 989 * 990 * PREREAD -- Invalidate the D-cache. We do this 991 * here in case a write-back is required by the back-end. 992 * 993 * PREWRITE -- Write-back the D-cache. Note that if 994 * we are doing a PREREAD|PREWRITE, we can collapse 995 * the whole thing into a single Wb-Inv. 996 * 997 * POSTREAD -- Re-invalidate the D-cache in case speculative 998 * memory accesses caused cachelines to become valid with now 999 * invalid data. 1000 * 1001 * POSTWRITE -- Nothing. 1002 */ 1003 #ifdef _ARM32_NEED_BUS_DMA_BOUNCE 1004 const bool bouncing = (map->_dm_flags & _BUS_DMAMAP_IS_BOUNCING); 1005 #else 1006 const bool bouncing = false; 1007 #endif 1008 1009 const int pre_ops = ops & (BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE); 1010 #ifdef CPU_CORTEX 1011 const int post_ops = ops & (BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE); 1012 #else 1013 const int post_ops = 0; 1014 #endif 1015 if (!bouncing && pre_ops == 0 && post_ops == BUS_DMASYNC_POSTWRITE) { 1016 STAT_INCR(sync_postwrite); 1017 return; 1018 } 1019 KASSERTMSG(bouncing || pre_ops != 0 || (post_ops & BUS_DMASYNC_POSTREAD), 1020 "pre_ops %#x post_ops %#x", pre_ops, post_ops); 1021 #ifdef _ARM32_NEED_BUS_DMA_BOUNCE 1022 if (bouncing && (ops & BUS_DMASYNC_PREWRITE)) { 1023 struct arm32_bus_dma_cookie * const cookie = map->_dm_cookie; 1024 STAT_INCR(write_bounces); 1025 char * const dataptr = (char *)cookie->id_bouncebuf + offset; 1026 /* 1027 * Copy the caller's buffer to the bounce buffer. 1028 */ 1029 switch (map->_dm_buftype) { 1030 case _BUS_DMA_BUFTYPE_LINEAR: 1031 memcpy(dataptr, cookie->id_origlinearbuf + offset, len); 1032 break; 1033 case _BUS_DMA_BUFTYPE_MBUF: 1034 m_copydata(cookie->id_origmbuf, offset, len, dataptr); 1035 break; 1036 case _BUS_DMA_BUFTYPE_UIO: 1037 _bus_dma_uiomove(dataptr, cookie->id_origuio, len, UIO_WRITE); 1038 break; 1039 #ifdef DIAGNOSTIC 1040 case _BUS_DMA_BUFTYPE_RAW: 1041 panic("_bus_dmamap_sync(pre): _BUS_DMA_BUFTYPE_RAW"); 1042 break; 1043 1044 case _BUS_DMA_BUFTYPE_INVALID: 1045 panic("_bus_dmamap_sync(pre): _BUS_DMA_BUFTYPE_INVALID"); 1046 break; 1047 1048 default: 1049 panic("_bus_dmamap_sync(pre): map %p: unknown buffer type %d\n", 1050 map, map->_dm_buftype); 1051 break; 1052 #endif /* DIAGNOSTIC */ 1053 } 1054 } 1055 #endif /* _ARM32_NEED_BUS_DMA_BOUNCE */ 1056 1057 /* Skip cache frobbing if mapping was COHERENT. */ 1058 if (!bouncing && (map->_dm_flags & _BUS_DMAMAP_COHERENT)) { 1059 /* Drain the write buffer. */ 1060 if (pre_ops & BUS_DMASYNC_PREWRITE) 1061 cpu_drain_writebuf(); 1062 return; 1063 } 1064 1065 #ifdef _ARM32_NEED_BUS_DMA_BOUNCE 1066 if (bouncing && ((map->_dm_flags & _BUS_DMAMAP_COHERENT) || pre_ops == 0)) { 1067 goto bounce_it; 1068 } 1069 #endif /* _ARM32_NEED_BUS_DMA_BOUNCE */ 1070 1071 /* 1072 * If the mapping belongs to a non-kernel vmspace, and the 1073 * vmspace has not been active since the last time a full 1074 * cache flush was performed, we don't need to do anything. 1075 */ 1076 if (__predict_false(!VMSPACE_IS_KERNEL_P(map->_dm_vmspace) && 1077 vm_map_pmap(&map->_dm_vmspace->vm_map)->pm_cstate.cs_cache_d == 0)) 1078 return; 1079 1080 int buftype = map->_dm_buftype; 1081 #ifdef _ARM32_NEED_BUS_DMA_BOUNCE 1082 if (bouncing) { 1083 buftype = _BUS_DMA_BUFTYPE_LINEAR; 1084 } 1085 #endif 1086 1087 switch (buftype) { 1088 case _BUS_DMA_BUFTYPE_LINEAR: 1089 _bus_dmamap_sync_linear(t, map, offset, len, ops); 1090 break; 1091 1092 case _BUS_DMA_BUFTYPE_MBUF: 1093 _bus_dmamap_sync_mbuf(t, map, offset, len, ops); 1094 break; 1095 1096 case _BUS_DMA_BUFTYPE_UIO: 1097 _bus_dmamap_sync_uio(t, map, offset, len, ops); 1098 break; 1099 1100 case _BUS_DMA_BUFTYPE_RAW: 1101 panic("_bus_dmamap_sync: _BUS_DMA_BUFTYPE_RAW"); 1102 break; 1103 1104 case _BUS_DMA_BUFTYPE_INVALID: 1105 panic("_bus_dmamap_sync: _BUS_DMA_BUFTYPE_INVALID"); 1106 break; 1107 1108 default: 1109 panic("_bus_dmamap_sync: map %p: unknown buffer type %d\n", 1110 map, map->_dm_buftype); 1111 } 1112 1113 /* Drain the write buffer. */ 1114 cpu_drain_writebuf(); 1115 1116 #ifdef _ARM32_NEED_BUS_DMA_BOUNCE 1117 bounce_it: 1118 if (!bouncing || (ops & BUS_DMASYNC_POSTREAD) == 0) 1119 return; 1120 1121 struct arm32_bus_dma_cookie * const cookie = map->_dm_cookie; 1122 char * const dataptr = (char *)cookie->id_bouncebuf + offset; 1123 STAT_INCR(read_bounces); 1124 /* 1125 * Copy the bounce buffer to the caller's buffer. 1126 */ 1127 switch (map->_dm_buftype) { 1128 case _BUS_DMA_BUFTYPE_LINEAR: 1129 memcpy(cookie->id_origlinearbuf + offset, dataptr, len); 1130 break; 1131 1132 case _BUS_DMA_BUFTYPE_MBUF: 1133 m_copyback(cookie->id_origmbuf, offset, len, dataptr); 1134 break; 1135 1136 case _BUS_DMA_BUFTYPE_UIO: 1137 _bus_dma_uiomove(dataptr, cookie->id_origuio, len, UIO_READ); 1138 break; 1139 #ifdef DIAGNOSTIC 1140 case _BUS_DMA_BUFTYPE_RAW: 1141 panic("_bus_dmamap_sync(post): _BUS_DMA_BUFTYPE_RAW"); 1142 break; 1143 1144 case _BUS_DMA_BUFTYPE_INVALID: 1145 panic("_bus_dmamap_sync(post): _BUS_DMA_BUFTYPE_INVALID"); 1146 break; 1147 1148 default: 1149 panic("_bus_dmamap_sync(post): map %p: unknown buffer type %d\n", 1150 map, map->_dm_buftype); 1151 break; 1152 #endif 1153 } 1154 #endif /* _ARM32_NEED_BUS_DMA_BOUNCE */ 1155 } 1156 1157 /* 1158 * Common function for DMA-safe memory allocation. May be called 1159 * by bus-specific DMA memory allocation functions. 1160 */ 1161 1162 extern paddr_t physical_start; 1163 extern paddr_t physical_end; 1164 1165 int 1166 _bus_dmamem_alloc(bus_dma_tag_t t, bus_size_t size, bus_size_t alignment, 1167 bus_size_t boundary, bus_dma_segment_t *segs, int nsegs, int *rsegs, 1168 int flags) 1169 { 1170 struct arm32_dma_range *dr; 1171 int error, i; 1172 1173 #ifdef DEBUG_DMA 1174 printf("dmamem_alloc t=%p size=%lx align=%lx boundary=%lx " 1175 "segs=%p nsegs=%x rsegs=%p flags=%x\n", t, size, alignment, 1176 boundary, segs, nsegs, rsegs, flags); 1177 #endif 1178 1179 if ((dr = t->_ranges) != NULL) { 1180 error = ENOMEM; 1181 for (i = 0; i < t->_nranges; i++, dr++) { 1182 if (dr->dr_len == 0 1183 || (dr->dr_flags & _BUS_DMAMAP_NOALLOC)) 1184 continue; 1185 error = _bus_dmamem_alloc_range(t, size, alignment, 1186 boundary, segs, nsegs, rsegs, flags, 1187 trunc_page(dr->dr_sysbase), 1188 trunc_page(dr->dr_sysbase + dr->dr_len)); 1189 if (error == 0) 1190 break; 1191 } 1192 } else { 1193 error = _bus_dmamem_alloc_range(t, size, alignment, boundary, 1194 segs, nsegs, rsegs, flags, trunc_page(physical_start), 1195 trunc_page(physical_end)); 1196 } 1197 1198 #ifdef DEBUG_DMA 1199 printf("dmamem_alloc: =%d\n", error); 1200 #endif 1201 1202 return(error); 1203 } 1204 1205 /* 1206 * Common function for freeing DMA-safe memory. May be called by 1207 * bus-specific DMA memory free functions. 1208 */ 1209 void 1210 _bus_dmamem_free(bus_dma_tag_t t, bus_dma_segment_t *segs, int nsegs) 1211 { 1212 struct vm_page *m; 1213 bus_addr_t addr; 1214 struct pglist mlist; 1215 int curseg; 1216 1217 #ifdef DEBUG_DMA 1218 printf("dmamem_free: t=%p segs=%p nsegs=%x\n", t, segs, nsegs); 1219 #endif /* DEBUG_DMA */ 1220 1221 /* 1222 * Build a list of pages to free back to the VM system. 1223 */ 1224 TAILQ_INIT(&mlist); 1225 for (curseg = 0; curseg < nsegs; curseg++) { 1226 for (addr = segs[curseg].ds_addr; 1227 addr < (segs[curseg].ds_addr + segs[curseg].ds_len); 1228 addr += PAGE_SIZE) { 1229 m = PHYS_TO_VM_PAGE(addr); 1230 TAILQ_INSERT_TAIL(&mlist, m, pageq.queue); 1231 } 1232 } 1233 uvm_pglistfree(&mlist); 1234 } 1235 1236 /* 1237 * Common function for mapping DMA-safe memory. May be called by 1238 * bus-specific DMA memory map functions. 1239 */ 1240 int 1241 _bus_dmamem_map(bus_dma_tag_t t, bus_dma_segment_t *segs, int nsegs, 1242 size_t size, void **kvap, int flags) 1243 { 1244 vaddr_t va; 1245 paddr_t pa; 1246 int curseg; 1247 pt_entry_t *ptep; 1248 const uvm_flag_t kmflags = UVM_KMF_VAONLY 1249 | ((flags & BUS_DMA_NOWAIT) != 0 ? UVM_KMF_NOWAIT : 0); 1250 vsize_t align = 0; 1251 1252 #ifdef DEBUG_DMA 1253 printf("dmamem_map: t=%p segs=%p nsegs=%x size=%lx flags=%x\n", t, 1254 segs, nsegs, (unsigned long)size, flags); 1255 #endif /* DEBUG_DMA */ 1256 1257 #ifdef PMAP_MAP_POOLPAGE 1258 /* 1259 * If all of memory is mapped, and we are mapping a single physically 1260 * contiguous area then this area is already mapped. Let's see if we 1261 * avoid having a separate mapping for it. 1262 */ 1263 if (nsegs == 1) { 1264 /* 1265 * If this is a non-COHERENT mapping, then the existing kernel 1266 * mapping is already compatible with it. 1267 */ 1268 bool direct_mapable = (flags & BUS_DMA_COHERENT) == 0; 1269 pa = segs[0].ds_addr; 1270 1271 /* 1272 * This is a COHERENT mapping which, unless this address is in 1273 * a COHERENT dma range, will not be compatible. 1274 */ 1275 if (t->_ranges != NULL) { 1276 const struct arm32_dma_range * const dr = 1277 _bus_dma_paddr_inrange(t->_ranges, t->_nranges, pa); 1278 if (dr != NULL 1279 && (dr->dr_flags & _BUS_DMAMAP_COHERENT)) { 1280 direct_mapable = true; 1281 } 1282 } 1283 1284 if (direct_mapable) { 1285 *kvap = (void *)PMAP_MAP_POOLPAGE(pa); 1286 #ifdef DEBUG_DMA 1287 printf("dmamem_map: =%p\n", *kvap); 1288 #endif /* DEBUG_DMA */ 1289 return 0; 1290 } 1291 } 1292 #endif 1293 1294 size = round_page(size); 1295 if (__predict_true(size > L2_L_SIZE)) { 1296 #if (ARM_MMU_V6 + ARM_MMU_V7) > 0 1297 if (size >= L1_SS_SIZE) 1298 align = L1_SS_SIZE; 1299 else 1300 #endif 1301 if (size >= L1_S_SIZE) 1302 align = L1_S_SIZE; 1303 else 1304 align = L2_S_SIZE; 1305 } 1306 1307 va = uvm_km_alloc(kernel_map, size, align, kmflags); 1308 if (__predict_false(va == 0 && align > 0)) { 1309 align = 0; 1310 va = uvm_km_alloc(kernel_map, size, 0, kmflags); 1311 } 1312 1313 if (va == 0) 1314 return (ENOMEM); 1315 1316 *kvap = (void *)va; 1317 1318 for (curseg = 0; curseg < nsegs; curseg++) { 1319 for (pa = segs[curseg].ds_addr; 1320 pa < (segs[curseg].ds_addr + segs[curseg].ds_len); 1321 pa += PAGE_SIZE, va += PAGE_SIZE, size -= PAGE_SIZE) { 1322 bool uncached = (flags & BUS_DMA_COHERENT); 1323 #ifdef DEBUG_DMA 1324 printf("wiring p%lx to v%lx", pa, va); 1325 #endif /* DEBUG_DMA */ 1326 if (size == 0) 1327 panic("_bus_dmamem_map: size botch"); 1328 1329 const struct arm32_dma_range * const dr = 1330 _bus_dma_paddr_inrange(t->_ranges, t->_nranges, pa); 1331 /* 1332 * If this dma region is coherent then there is 1333 * no need for an uncached mapping. 1334 */ 1335 if (dr != NULL 1336 && (dr->dr_flags & _BUS_DMAMAP_COHERENT)) { 1337 uncached = false; 1338 } 1339 1340 pmap_kenter_pa(va, pa, 1341 VM_PROT_READ | VM_PROT_WRITE, PMAP_WIRED); 1342 1343 /* 1344 * If the memory must remain coherent with the 1345 * cache then we must make the memory uncacheable 1346 * in order to maintain virtual cache coherency. 1347 * We must also guarantee the cache does not already 1348 * contain the virtal addresses we are making 1349 * uncacheable. 1350 */ 1351 if (uncached) { 1352 cpu_dcache_wbinv_range(va, PAGE_SIZE); 1353 cpu_sdcache_wbinv_range(va, pa, PAGE_SIZE); 1354 cpu_drain_writebuf(); 1355 ptep = vtopte(va); 1356 *ptep &= ~L2_S_CACHE_MASK; 1357 PTE_SYNC(ptep); 1358 tlb_flush(); 1359 } 1360 #ifdef DEBUG_DMA 1361 ptep = vtopte(va); 1362 printf(" pte=v%p *pte=%x\n", ptep, *ptep); 1363 #endif /* DEBUG_DMA */ 1364 } 1365 } 1366 pmap_update(pmap_kernel()); 1367 #ifdef DEBUG_DMA 1368 printf("dmamem_map: =%p\n", *kvap); 1369 #endif /* DEBUG_DMA */ 1370 return (0); 1371 } 1372 1373 /* 1374 * Common function for unmapping DMA-safe memory. May be called by 1375 * bus-specific DMA memory unmapping functions. 1376 */ 1377 void 1378 _bus_dmamem_unmap(bus_dma_tag_t t, void *kva, size_t size) 1379 { 1380 1381 #ifdef DEBUG_DMA 1382 printf("dmamem_unmap: t=%p kva=%p size=%zx\n", t, kva, size); 1383 #endif /* DEBUG_DMA */ 1384 #ifdef DIAGNOSTIC 1385 if ((u_long)kva & PGOFSET) 1386 panic("_bus_dmamem_unmap"); 1387 #endif /* DIAGNOSTIC */ 1388 1389 size = round_page(size); 1390 pmap_kremove((vaddr_t)kva, size); 1391 pmap_update(pmap_kernel()); 1392 uvm_km_free(kernel_map, (vaddr_t)kva, size, UVM_KMF_VAONLY); 1393 } 1394 1395 /* 1396 * Common functin for mmap(2)'ing DMA-safe memory. May be called by 1397 * bus-specific DMA mmap(2)'ing functions. 1398 */ 1399 paddr_t 1400 _bus_dmamem_mmap(bus_dma_tag_t t, bus_dma_segment_t *segs, int nsegs, 1401 off_t off, int prot, int flags) 1402 { 1403 paddr_t map_flags; 1404 int i; 1405 1406 for (i = 0; i < nsegs; i++) { 1407 #ifdef DIAGNOSTIC 1408 if (off & PGOFSET) 1409 panic("_bus_dmamem_mmap: offset unaligned"); 1410 if (segs[i].ds_addr & PGOFSET) 1411 panic("_bus_dmamem_mmap: segment unaligned"); 1412 if (segs[i].ds_len & PGOFSET) 1413 panic("_bus_dmamem_mmap: segment size not multiple" 1414 " of page size"); 1415 #endif /* DIAGNOSTIC */ 1416 if (off >= segs[i].ds_len) { 1417 off -= segs[i].ds_len; 1418 continue; 1419 } 1420 1421 map_flags = 0; 1422 if (flags & BUS_DMA_PREFETCHABLE) 1423 map_flags |= ARM32_MMAP_WRITECOMBINE; 1424 1425 return (arm_btop((u_long)segs[i].ds_addr + off) | map_flags); 1426 1427 } 1428 1429 /* Page not found. */ 1430 return (-1); 1431 } 1432 1433 /********************************************************************** 1434 * DMA utility functions 1435 **********************************************************************/ 1436 1437 /* 1438 * Utility function to load a linear buffer. lastaddrp holds state 1439 * between invocations (for multiple-buffer loads). segp contains 1440 * the starting segment on entrace, and the ending segment on exit. 1441 * first indicates if this is the first invocation of this function. 1442 */ 1443 int 1444 _bus_dmamap_load_buffer(bus_dma_tag_t t, bus_dmamap_t map, void *buf, 1445 bus_size_t buflen, struct vmspace *vm, int flags) 1446 { 1447 bus_size_t sgsize; 1448 bus_addr_t curaddr; 1449 vaddr_t vaddr = (vaddr_t)buf; 1450 int error; 1451 pmap_t pmap; 1452 1453 #ifdef DEBUG_DMA 1454 printf("_bus_dmamem_load_buffer(buf=%p, len=%lx, flags=%d)\n", 1455 buf, buflen, flags); 1456 #endif /* DEBUG_DMA */ 1457 1458 pmap = vm_map_pmap(&vm->vm_map); 1459 1460 while (buflen > 0) { 1461 /* 1462 * Get the physical address for this segment. 1463 * 1464 * XXX Doesn't support checking for coherent mappings 1465 * XXX in user address space. 1466 */ 1467 bool coherent; 1468 if (__predict_true(pmap == pmap_kernel())) { 1469 pd_entry_t *pde; 1470 pt_entry_t *ptep; 1471 (void) pmap_get_pde_pte(pmap, vaddr, &pde, &ptep); 1472 if (__predict_false(pmap_pde_section(pde))) { 1473 paddr_t s_frame = L1_S_FRAME; 1474 paddr_t s_offset = L1_S_OFFSET; 1475 #if (ARM_MMU_V6 + ARM_MMU_V7) > 0 1476 if (__predict_false(pmap_pde_supersection(pde))) { 1477 s_frame = L1_SS_FRAME; 1478 s_offset = L1_SS_OFFSET; 1479 } 1480 #endif 1481 curaddr = (*pde & s_frame) | (vaddr & s_offset); 1482 coherent = (*pde & L1_S_CACHE_MASK) == 0; 1483 } else { 1484 pt_entry_t pte = *ptep; 1485 KDASSERTMSG((pte & L2_TYPE_MASK) != L2_TYPE_INV, 1486 "va=%#"PRIxVADDR" pde=%#x ptep=%p pte=%#x", 1487 vaddr, *pde, ptep, pte); 1488 if (__predict_false((pte & L2_TYPE_MASK) 1489 == L2_TYPE_L)) { 1490 curaddr = (pte & L2_L_FRAME) | 1491 (vaddr & L2_L_OFFSET); 1492 coherent = (pte & L2_L_CACHE_MASK) == 0; 1493 } else { 1494 curaddr = (pte & L2_S_FRAME) | 1495 (vaddr & L2_S_OFFSET); 1496 coherent = (pte & L2_S_CACHE_MASK) == 0; 1497 } 1498 } 1499 } else { 1500 (void) pmap_extract(pmap, vaddr, &curaddr); 1501 coherent = false; 1502 } 1503 1504 /* 1505 * Compute the segment size, and adjust counts. 1506 */ 1507 sgsize = PAGE_SIZE - ((u_long)vaddr & PGOFSET); 1508 if (buflen < sgsize) 1509 sgsize = buflen; 1510 1511 error = _bus_dmamap_load_paddr(t, map, curaddr, sgsize, 1512 coherent); 1513 if (error) 1514 return (error); 1515 1516 vaddr += sgsize; 1517 buflen -= sgsize; 1518 } 1519 1520 return (0); 1521 } 1522 1523 /* 1524 * Allocate physical memory from the given physical address range. 1525 * Called by DMA-safe memory allocation methods. 1526 */ 1527 int 1528 _bus_dmamem_alloc_range(bus_dma_tag_t t, bus_size_t size, bus_size_t alignment, 1529 bus_size_t boundary, bus_dma_segment_t *segs, int nsegs, int *rsegs, 1530 int flags, paddr_t low, paddr_t high) 1531 { 1532 paddr_t curaddr, lastaddr; 1533 struct vm_page *m; 1534 struct pglist mlist; 1535 int curseg, error; 1536 1537 KASSERTMSG(boundary == 0 || (boundary & (boundary-1)) == 0, 1538 "invalid boundary %#lx", boundary); 1539 1540 #ifdef DEBUG_DMA 1541 printf("alloc_range: t=%p size=%lx align=%lx boundary=%lx segs=%p nsegs=%x rsegs=%p flags=%x lo=%lx hi=%lx\n", 1542 t, size, alignment, boundary, segs, nsegs, rsegs, flags, low, high); 1543 #endif /* DEBUG_DMA */ 1544 1545 /* Always round the size. */ 1546 size = round_page(size); 1547 1548 /* 1549 * We accept boundaries < size, splitting in multiple segments 1550 * if needed. uvm_pglistalloc does not, so compute an appropriate 1551 * boundary: next power of 2 >= size 1552 */ 1553 bus_size_t uboundary = boundary; 1554 if (uboundary <= PAGE_SIZE) { 1555 uboundary = 0; 1556 } else { 1557 while (uboundary < size) { 1558 uboundary <<= 1; 1559 } 1560 } 1561 1562 /* 1563 * Allocate pages from the VM system. 1564 */ 1565 error = uvm_pglistalloc(size, low, high, alignment, uboundary, 1566 &mlist, nsegs, (flags & BUS_DMA_NOWAIT) == 0); 1567 if (error) 1568 return (error); 1569 1570 /* 1571 * Compute the location, size, and number of segments actually 1572 * returned by the VM code. 1573 */ 1574 m = TAILQ_FIRST(&mlist); 1575 curseg = 0; 1576 lastaddr = segs[curseg].ds_addr = VM_PAGE_TO_PHYS(m); 1577 segs[curseg].ds_len = PAGE_SIZE; 1578 #ifdef DEBUG_DMA 1579 printf("alloc: page %lx\n", lastaddr); 1580 #endif /* DEBUG_DMA */ 1581 m = TAILQ_NEXT(m, pageq.queue); 1582 1583 for (; m != NULL; m = TAILQ_NEXT(m, pageq.queue)) { 1584 curaddr = VM_PAGE_TO_PHYS(m); 1585 KASSERTMSG(low <= curaddr && curaddr < high, 1586 "uvm_pglistalloc returned non-sensicaladdress %#lx " 1587 "(low=%#lx, high=%#lx\n", curaddr, low, high); 1588 #ifdef DEBUG_DMA 1589 printf("alloc: page %lx\n", curaddr); 1590 #endif /* DEBUG_DMA */ 1591 if (curaddr == lastaddr + PAGE_SIZE 1592 && (lastaddr & boundary) == (curaddr & boundary)) 1593 segs[curseg].ds_len += PAGE_SIZE; 1594 else { 1595 curseg++; 1596 if (curseg >= nsegs) { 1597 uvm_pglistfree(&mlist); 1598 return EFBIG; 1599 } 1600 segs[curseg].ds_addr = curaddr; 1601 segs[curseg].ds_len = PAGE_SIZE; 1602 } 1603 lastaddr = curaddr; 1604 } 1605 1606 *rsegs = curseg + 1; 1607 1608 return (0); 1609 } 1610 1611 /* 1612 * Check if a memory region intersects with a DMA range, and return the 1613 * page-rounded intersection if it does. 1614 */ 1615 int 1616 arm32_dma_range_intersect(struct arm32_dma_range *ranges, int nranges, 1617 paddr_t pa, psize_t size, paddr_t *pap, psize_t *sizep) 1618 { 1619 struct arm32_dma_range *dr; 1620 int i; 1621 1622 if (ranges == NULL) 1623 return (0); 1624 1625 for (i = 0, dr = ranges; i < nranges; i++, dr++) { 1626 if (dr->dr_sysbase <= pa && 1627 pa < (dr->dr_sysbase + dr->dr_len)) { 1628 /* 1629 * Beginning of region intersects with this range. 1630 */ 1631 *pap = trunc_page(pa); 1632 *sizep = round_page(min(pa + size, 1633 dr->dr_sysbase + dr->dr_len) - pa); 1634 return (1); 1635 } 1636 if (pa < dr->dr_sysbase && dr->dr_sysbase < (pa + size)) { 1637 /* 1638 * End of region intersects with this range. 1639 */ 1640 *pap = trunc_page(dr->dr_sysbase); 1641 *sizep = round_page(min((pa + size) - dr->dr_sysbase, 1642 dr->dr_len)); 1643 return (1); 1644 } 1645 } 1646 1647 /* No intersection found. */ 1648 return (0); 1649 } 1650 1651 #ifdef _ARM32_NEED_BUS_DMA_BOUNCE 1652 static int 1653 _bus_dma_alloc_bouncebuf(bus_dma_tag_t t, bus_dmamap_t map, 1654 bus_size_t size, int flags) 1655 { 1656 struct arm32_bus_dma_cookie *cookie = map->_dm_cookie; 1657 int error = 0; 1658 1659 #ifdef DIAGNOSTIC 1660 if (cookie == NULL) 1661 panic("_bus_dma_alloc_bouncebuf: no cookie"); 1662 #endif 1663 1664 cookie->id_bouncebuflen = round_page(size); 1665 error = _bus_dmamem_alloc(t, cookie->id_bouncebuflen, 1666 PAGE_SIZE, map->_dm_boundary, cookie->id_bouncesegs, 1667 map->_dm_segcnt, &cookie->id_nbouncesegs, flags); 1668 if (error == 0) { 1669 error = _bus_dmamem_map(t, cookie->id_bouncesegs, 1670 cookie->id_nbouncesegs, cookie->id_bouncebuflen, 1671 (void **)&cookie->id_bouncebuf, flags); 1672 if (error) { 1673 _bus_dmamem_free(t, cookie->id_bouncesegs, 1674 cookie->id_nbouncesegs); 1675 cookie->id_bouncebuflen = 0; 1676 cookie->id_nbouncesegs = 0; 1677 } else { 1678 cookie->id_flags |= _BUS_DMA_HAS_BOUNCE; 1679 } 1680 } else { 1681 cookie->id_bouncebuflen = 0; 1682 cookie->id_nbouncesegs = 0; 1683 } 1684 1685 return (error); 1686 } 1687 1688 static void 1689 _bus_dma_free_bouncebuf(bus_dma_tag_t t, bus_dmamap_t map) 1690 { 1691 struct arm32_bus_dma_cookie *cookie = map->_dm_cookie; 1692 1693 #ifdef DIAGNOSTIC 1694 if (cookie == NULL) 1695 panic("_bus_dma_alloc_bouncebuf: no cookie"); 1696 #endif 1697 1698 _bus_dmamem_unmap(t, cookie->id_bouncebuf, cookie->id_bouncebuflen); 1699 _bus_dmamem_free(t, cookie->id_bouncesegs, 1700 cookie->id_nbouncesegs); 1701 cookie->id_bouncebuflen = 0; 1702 cookie->id_nbouncesegs = 0; 1703 cookie->id_flags &= ~_BUS_DMA_HAS_BOUNCE; 1704 } 1705 1706 /* 1707 * This function does the same as uiomove, but takes an explicit 1708 * direction, and does not update the uio structure. 1709 */ 1710 static int 1711 _bus_dma_uiomove(void *buf, struct uio *uio, size_t n, int direction) 1712 { 1713 struct iovec *iov; 1714 int error; 1715 struct vmspace *vm; 1716 char *cp; 1717 size_t resid, cnt; 1718 int i; 1719 1720 iov = uio->uio_iov; 1721 vm = uio->uio_vmspace; 1722 cp = buf; 1723 resid = n; 1724 1725 for (i = 0; i < uio->uio_iovcnt && resid > 0; i++) { 1726 iov = &uio->uio_iov[i]; 1727 if (iov->iov_len == 0) 1728 continue; 1729 cnt = MIN(resid, iov->iov_len); 1730 1731 if (!VMSPACE_IS_KERNEL_P(vm) && 1732 (curlwp->l_cpu->ci_schedstate.spc_flags & SPCF_SHOULDYIELD) 1733 != 0) { 1734 preempt(); 1735 } 1736 if (direction == UIO_READ) { 1737 error = copyout_vmspace(vm, cp, iov->iov_base, cnt); 1738 } else { 1739 error = copyin_vmspace(vm, iov->iov_base, cp, cnt); 1740 } 1741 if (error) 1742 return (error); 1743 cp += cnt; 1744 resid -= cnt; 1745 } 1746 return (0); 1747 } 1748 #endif /* _ARM32_NEED_BUS_DMA_BOUNCE */ 1749 1750 int 1751 _bus_dmatag_subregion(bus_dma_tag_t tag, bus_addr_t min_addr, 1752 bus_addr_t max_addr, bus_dma_tag_t *newtag, int flags) 1753 { 1754 1755 #ifdef _ARM32_NEED_BUS_DMA_BOUNCE 1756 struct arm32_dma_range *dr; 1757 bool subset = false; 1758 size_t nranges = 0; 1759 size_t i; 1760 for (i = 0, dr = tag->_ranges; i < tag->_nranges; i++, dr++) { 1761 if (dr->dr_sysbase <= min_addr 1762 && max_addr <= dr->dr_sysbase + dr->dr_len - 1) { 1763 subset = true; 1764 } 1765 if (min_addr <= dr->dr_sysbase + dr->dr_len 1766 && max_addr >= dr->dr_sysbase) { 1767 nranges++; 1768 } 1769 } 1770 if (subset) { 1771 *newtag = tag; 1772 /* if the tag must be freed, add a reference */ 1773 if (tag->_tag_needs_free) 1774 (tag->_tag_needs_free)++; 1775 return 0; 1776 } 1777 if (nranges == 0) { 1778 nranges = 1; 1779 } 1780 1781 size_t mallocsize = sizeof(*tag) + nranges * sizeof(*dr); 1782 if ((*newtag = malloc(mallocsize, M_DMAMAP, 1783 (flags & BUS_DMA_NOWAIT) ? M_NOWAIT : M_WAITOK)) == NULL) 1784 return ENOMEM; 1785 1786 dr = (void *)(*newtag + 1); 1787 **newtag = *tag; 1788 (*newtag)->_tag_needs_free = 1; 1789 (*newtag)->_ranges = dr; 1790 (*newtag)->_nranges = nranges; 1791 1792 if (tag->_ranges == NULL) { 1793 dr->dr_sysbase = min_addr; 1794 dr->dr_busbase = min_addr; 1795 dr->dr_len = max_addr + 1 - min_addr; 1796 } else { 1797 for (i = 0; i < nranges; i++) { 1798 if (min_addr > dr->dr_sysbase + dr->dr_len 1799 || max_addr < dr->dr_sysbase) 1800 continue; 1801 dr[0] = tag->_ranges[i]; 1802 if (dr->dr_sysbase < min_addr) { 1803 psize_t diff = min_addr - dr->dr_sysbase; 1804 dr->dr_busbase += diff; 1805 dr->dr_len -= diff; 1806 dr->dr_sysbase += diff; 1807 } 1808 if (max_addr != 0xffffffff 1809 && max_addr + 1 < dr->dr_sysbase + dr->dr_len) { 1810 dr->dr_len = max_addr + 1 - dr->dr_sysbase; 1811 } 1812 dr++; 1813 } 1814 } 1815 1816 return 0; 1817 #else 1818 return EOPNOTSUPP; 1819 #endif /* _ARM32_NEED_BUS_DMA_BOUNCE */ 1820 } 1821 1822 void 1823 _bus_dmatag_destroy(bus_dma_tag_t tag) 1824 { 1825 #ifdef _ARM32_NEED_BUS_DMA_BOUNCE 1826 switch (tag->_tag_needs_free) { 1827 case 0: 1828 break; /* not allocated with malloc */ 1829 case 1: 1830 free(tag, M_DMAMAP); /* last reference to tag */ 1831 break; 1832 default: 1833 (tag->_tag_needs_free)--; /* one less reference */ 1834 } 1835 #endif 1836 } 1837
Indexes created Mon Sep 29 21:09:56 GMT 2025