bus_dma.c revision 1.149
1 /* $NetBSD: bus_dma.c,v 1.149 2025/03/08 21:00:45 jmcneill Exp $ */ 2 3 /*- 4 * Copyright (c) 1996, 1997, 1998, 2020 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility, 9 * NASA Ames Research Center. 10 * 11 * Redistribution and use in source and binary forms, with or without 12 * modification, are permitted provided that the following conditions 13 * are met: 14 * 1. Redistributions of source code must retain the above copyright 15 * notice, this list of conditions and the following disclaimer. 16 * 2. Redistributions in binary form must reproduce the above copyright 17 * notice, this list of conditions and the following disclaimer in the 18 * documentation and/or other materials provided with the distribution. 19 * 20 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 21 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 22 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 23 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 24 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 25 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 26 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 27 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 28 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 29 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 30 * POSSIBILITY OF SUCH DAMAGE. 31 */ 32 33 #define _ARM32_BUS_DMA_PRIVATE 34 35 #include "opt_arm_bus_space.h" 36 #include "opt_cputypes.h" 37 38 #include <sys/cdefs.h> 39 __KERNEL_RCSID(0, "$NetBSD: bus_dma.c,v 1.149 2025/03/08 21:00:45 jmcneill Exp $"); 40 41 #include <sys/param.h> 42 43 #include <sys/bus.h> 44 #include <sys/cpu.h> 45 #include <sys/kmem.h> 46 #include <sys/mbuf.h> 47 48 #include <uvm/uvm.h> 49 50 #include <arm/cpuconf.h> 51 #include <arm/cpufunc.h> 52 53 #ifdef __HAVE_MM_MD_DIRECT_MAPPED_PHYS 54 #include <dev/mm.h> 55 #endif 56 57 #ifdef BUSDMA_COUNTERS 58 static struct evcnt bus_dma_creates = 59 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "busdma", "creates"); 60 static struct evcnt bus_dma_bounced_creates = 61 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "busdma", "bounced creates"); 62 static struct evcnt bus_dma_loads = 63 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "busdma", "loads"); 64 static struct evcnt bus_dma_bounced_loads = 65 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "busdma", "bounced loads"); 66 static struct evcnt bus_dma_coherent_loads = 67 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "busdma", "coherent loads"); 68 static struct evcnt bus_dma_read_bounces = 69 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "busdma", "read bounces"); 70 static struct evcnt bus_dma_write_bounces = 71 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "busdma", "write bounces"); 72 static struct evcnt bus_dma_bounced_unloads = 73 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "busdma", "bounced unloads"); 74 static struct evcnt bus_dma_bounced_mbuf_loads = 75 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "busdma", "bounced mbuf loads"); 76 static struct evcnt bus_dma_unloads = 77 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "busdma", "unloads"); 78 static struct evcnt bus_dma_bounced_destroys = 79 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "busdma", "bounced destroys"); 80 static struct evcnt bus_dma_destroys = 81 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "busdma", "destroys"); 82 static struct evcnt bus_dma_sync_prereadwrite = 83 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "busdma", "sync prereadwrite"); 84 static struct evcnt bus_dma_sync_preread_begin = 85 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "busdma", "sync preread begin"); 86 static struct evcnt bus_dma_sync_preread = 87 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "busdma", "sync preread"); 88 static struct evcnt bus_dma_sync_preread_tail = 89 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "busdma", "sync preread tail"); 90 static struct evcnt bus_dma_sync_prewrite = 91 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "busdma", "sync prewrite"); 92 static struct evcnt bus_dma_sync_postread = 93 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "busdma", "sync postread"); 94 static struct evcnt bus_dma_sync_postreadwrite = 95 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "busdma", "sync postreadwrite"); 96 static struct evcnt bus_dma_sync_postwrite = 97 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "busdma", "sync postwrite"); 98 static struct evcnt bus_dma_inrange_fail = 99 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "busdma", "inrange check failed"); 100 101 static struct evcnt bus_dma_sync_coherent_prereadwrite = 102 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "busdma", "sync coherent prereadwrite"); 103 static struct evcnt bus_dma_sync_coherent_preread = 104 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "busdma", "sync coherent preread"); 105 static struct evcnt bus_dma_sync_coherent_prewrite = 106 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "busdma", "sync coherent prewrite"); 107 static struct evcnt bus_dma_sync_coherent_postread = 108 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "busdma", "sync coherent postread"); 109 static struct evcnt bus_dma_sync_coherent_postreadwrite = 110 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "busdma", "sync coherent postreadwrite"); 111 static struct evcnt bus_dma_sync_coherent_postwrite = 112 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "busdma", "sync coherent postwrite"); 113 114 EVCNT_ATTACH_STATIC(bus_dma_creates); 115 EVCNT_ATTACH_STATIC(bus_dma_bounced_creates); 116 EVCNT_ATTACH_STATIC(bus_dma_loads); 117 EVCNT_ATTACH_STATIC(bus_dma_bounced_loads); 118 EVCNT_ATTACH_STATIC(bus_dma_coherent_loads); 119 EVCNT_ATTACH_STATIC(bus_dma_read_bounces); 120 EVCNT_ATTACH_STATIC(bus_dma_write_bounces); 121 EVCNT_ATTACH_STATIC(bus_dma_unloads); 122 EVCNT_ATTACH_STATIC(bus_dma_bounced_unloads); 123 EVCNT_ATTACH_STATIC(bus_dma_destroys); 124 EVCNT_ATTACH_STATIC(bus_dma_bounced_destroys); 125 EVCNT_ATTACH_STATIC(bus_dma_bounced_mbuf_loads); 126 EVCNT_ATTACH_STATIC(bus_dma_sync_prereadwrite); 127 EVCNT_ATTACH_STATIC(bus_dma_sync_preread_begin); 128 EVCNT_ATTACH_STATIC(bus_dma_sync_preread); 129 EVCNT_ATTACH_STATIC(bus_dma_sync_preread_tail); 130 EVCNT_ATTACH_STATIC(bus_dma_sync_prewrite); 131 EVCNT_ATTACH_STATIC(bus_dma_sync_postread); 132 EVCNT_ATTACH_STATIC(bus_dma_sync_postreadwrite); 133 EVCNT_ATTACH_STATIC(bus_dma_sync_postwrite); 134 EVCNT_ATTACH_STATIC(bus_dma_inrange_fail); 135 136 EVCNT_ATTACH_STATIC(bus_dma_sync_coherent_prereadwrite); 137 EVCNT_ATTACH_STATIC(bus_dma_sync_coherent_preread); 138 EVCNT_ATTACH_STATIC(bus_dma_sync_coherent_prewrite); 139 EVCNT_ATTACH_STATIC(bus_dma_sync_coherent_postread); 140 EVCNT_ATTACH_STATIC(bus_dma_sync_coherent_postreadwrite); 141 EVCNT_ATTACH_STATIC(bus_dma_sync_coherent_postwrite); 142 143 #define STAT_INCR(x) (bus_dma_ ## x.ev_count++) 144 #else 145 #define STAT_INCR(x) __nothing 146 #endif 147 148 int _bus_dmamap_load_buffer(bus_dma_tag_t, bus_dmamap_t, void *, 149 bus_size_t, struct vmspace *, int); 150 151 /* 152 * Check to see if the specified page is in an allowed DMA range. 153 */ 154 static inline struct arm32_dma_range * 155 _bus_dma_paddr_inrange(struct arm32_dma_range *ranges, int nranges, 156 bus_addr_t curaddr) 157 { 158 struct arm32_dma_range *dr; 159 int i; 160 161 for (i = 0, dr = ranges; i < nranges; i++, dr++) { 162 if (curaddr >= dr->dr_sysbase && 163 curaddr < (dr->dr_sysbase + dr->dr_len)) 164 return dr; 165 } 166 167 return NULL; 168 } 169 170 /* 171 * Check to see if the specified busaddr is in an allowed DMA range. 172 */ 173 static inline paddr_t 174 _bus_dma_busaddr_to_paddr(bus_dma_tag_t t, bus_addr_t curaddr) 175 { 176 struct arm32_dma_range *dr; 177 u_int i; 178 179 if (t->_nranges == 0) 180 return curaddr; 181 182 for (i = 0, dr = t->_ranges; i < t->_nranges; i++, dr++) { 183 if (dr->dr_busbase <= curaddr 184 && curaddr < dr->dr_busbase + dr->dr_len) 185 return curaddr - dr->dr_busbase + dr->dr_sysbase; 186 } 187 panic("%s: curaddr %#" PRIxBUSADDR "not in range", __func__, curaddr); 188 } 189 190 /* 191 * Common function to load the specified physical address into the 192 * DMA map, coalescing segments and boundary checking as necessary. 193 */ 194 static int 195 _bus_dmamap_load_paddr(bus_dma_tag_t t, bus_dmamap_t map, 196 bus_addr_t paddr, bus_size_t size, bool coherent) 197 { 198 bus_dma_segment_t * const segs = map->dm_segs; 199 int nseg = map->dm_nsegs; 200 bus_addr_t lastaddr; 201 bus_addr_t bmask = ~(map->_dm_boundary - 1); 202 bus_addr_t curaddr; 203 bus_size_t sgsize; 204 uint32_t _ds_flags = coherent ? _BUS_DMAMAP_COHERENT : 0; 205 206 if (nseg > 0) 207 lastaddr = segs[nseg - 1].ds_addr + segs[nseg - 1].ds_len; 208 else 209 lastaddr = 0xdead; 210 211 again: 212 sgsize = size; 213 214 /* Make sure we're in an allowed DMA range. */ 215 if (t->_ranges != NULL) { 216 /* XXX cache last result? */ 217 const struct arm32_dma_range * const dr = 218 _bus_dma_paddr_inrange(t->_ranges, t->_nranges, paddr); 219 if (__predict_false(dr == NULL)) { 220 STAT_INCR(inrange_fail); 221 return EINVAL; 222 } 223 224 /* 225 * If this region is coherent, mark the segment as coherent. 226 */ 227 _ds_flags |= dr->dr_flags & _BUS_DMAMAP_COHERENT; 228 229 /* 230 * In a valid DMA range. Translate the physical 231 * memory address to an address in the DMA window. 232 */ 233 curaddr = (paddr - dr->dr_sysbase) + dr->dr_busbase; 234 #if 0 235 printf("%p: %#" PRIxPADDR 236 ": range %#" PRIxPADDR "/%#" PRIxBUSADDR 237 "/%#" PRIxBUSSIZE "/%#" PRIx32 ": %#" PRIx32 238 " <-- %#" PRIxBUSADDR "\n", 239 t, paddr, dr->dr_sysbase, dr->dr_busbase, 240 dr->dr_len, dr->dr_flags, _ds_flags, curaddr); 241 #endif 242 } else 243 curaddr = paddr; 244 245 /* 246 * Make sure we don't cross any boundaries. 247 */ 248 if (map->_dm_boundary > 0) { 249 bus_addr_t baddr; /* next boundary address */ 250 251 baddr = (curaddr + map->_dm_boundary) & bmask; 252 if (sgsize > (baddr - curaddr)) 253 sgsize = (baddr - curaddr); 254 } 255 256 /* 257 * Insert chunk into a segment, coalescing with the 258 * previous segment if possible. 259 */ 260 if (nseg > 0 && curaddr == lastaddr && 261 segs[nseg - 1].ds_len + sgsize <= map->dm_maxsegsz && 262 ((segs[nseg - 1]._ds_flags ^ _ds_flags) & _BUS_DMAMAP_COHERENT) == 0 && 263 (map->_dm_boundary == 0 || 264 (segs[nseg - 1].ds_addr & bmask) == (curaddr & bmask))) { 265 /* coalesce */ 266 segs[nseg - 1].ds_len += sgsize; 267 } else if (__predict_false(nseg >= map->_dm_segcnt)) { 268 return EFBIG; 269 } else { 270 /* new segment */ 271 segs[nseg].ds_addr = curaddr; 272 segs[nseg].ds_len = sgsize; 273 segs[nseg]._ds_paddr = curaddr; 274 segs[nseg]._ds_flags = _ds_flags; 275 nseg++; 276 } 277 278 lastaddr = curaddr + sgsize; 279 280 paddr += sgsize; 281 size -= sgsize; 282 if (size > 0) 283 goto again; 284 285 map->_dm_flags &= (_ds_flags & _BUS_DMAMAP_COHERENT); 286 map->dm_nsegs = nseg; 287 return 0; 288 } 289 290 static int _bus_dma_uiomove(void *buf, struct uio *uio, size_t n, 291 int direction); 292 293 #ifdef _ARM32_NEED_BUS_DMA_BOUNCE 294 static int _bus_dma_alloc_bouncebuf(bus_dma_tag_t t, bus_dmamap_t map, 295 bus_size_t size, int flags); 296 static void _bus_dma_free_bouncebuf(bus_dma_tag_t t, bus_dmamap_t map); 297 298 static int 299 _bus_dma_load_bouncebuf(bus_dma_tag_t t, bus_dmamap_t map, void *buf, 300 size_t buflen, int buftype, int flags) 301 { 302 struct arm32_bus_dma_cookie * const cookie = map->_dm_cookie; 303 struct vmspace * const vm = vmspace_kernel(); 304 int error; 305 306 KASSERT(cookie != NULL); 307 KASSERT(cookie->id_flags & _BUS_DMA_MIGHT_NEED_BOUNCE); 308 309 /* 310 * Allocate bounce pages, if necessary. 311 */ 312 if ((cookie->id_flags & _BUS_DMA_HAS_BOUNCE) == 0) { 313 error = _bus_dma_alloc_bouncebuf(t, map, buflen, flags); 314 if (__predict_false(error)) 315 return error; 316 } 317 318 /* 319 * Since we're trying again, clear the previous attempt. 320 */ 321 map->dm_mapsize = 0; 322 map->dm_nsegs = 0; 323 map->_dm_buftype = _BUS_DMA_BUFTYPE_INVALID; 324 /* _bus_dmamap_load_buffer() clears this if we're not... */ 325 map->_dm_flags |= _BUS_DMAMAP_COHERENT; 326 327 /* 328 * Cache a pointer to the caller's buffer and load the DMA map 329 * with the bounce buffer. 330 */ 331 cookie->id_origbuf = buf; 332 cookie->id_origbuflen = buflen; 333 error = _bus_dmamap_load_buffer(t, map, cookie->id_bouncebuf, 334 buflen, vm, flags); 335 if (__predict_false(error)) 336 return error; 337 338 STAT_INCR(bounced_loads); 339 map->dm_mapsize = buflen; 340 map->_dm_vmspace = vm; 341 map->_dm_buftype = buftype; 342 343 /* ...so _bus_dmamap_sync() knows we're bouncing */ 344 map->_dm_flags |= _BUS_DMAMAP_IS_BOUNCING; 345 cookie->id_flags |= _BUS_DMA_IS_BOUNCING; 346 return 0; 347 } 348 #endif /* _ARM32_NEED_BUS_DMA_BOUNCE */ 349 350 /* 351 * Common function for DMA map creation. May be called by bus-specific 352 * DMA map creation functions. 353 */ 354 int 355 _bus_dmamap_create(bus_dma_tag_t t, bus_size_t size, int nsegments, 356 bus_size_t maxsegsz, bus_size_t boundary, int flags, bus_dmamap_t *dmamp) 357 { 358 struct arm32_bus_dmamap *map; 359 void *mapstore; 360 int error = 0; 361 362 #ifdef DEBUG_DMA 363 printf("dmamap_create: t=%p size=%#" PRIxBUSSIZE 364 " nseg=%#x msegsz=%#" PRIxBUSSIZE 365 " boundary=%#" PRIxBUSSIZE 366 " flags=%#x\n", t, size, nsegments, maxsegsz, boundary, flags); 367 #endif /* DEBUG_DMA */ 368 369 /* 370 * Allocate and initialize the DMA map. The end of the map 371 * is a variable-sized array of segments, so we allocate enough 372 * room for them in one shot. 373 * 374 * Note we don't preserve the WAITOK or NOWAIT flags. Preservation 375 * of ALLOCNOW notifies others that we've reserved these resources, 376 * and they are not to be freed. 377 * 378 * The bus_dmamap_t includes one bus_dma_segment_t, hence 379 * the (nsegments - 1). 380 */ 381 const size_t mapsize = sizeof(struct arm32_bus_dmamap) + 382 (sizeof(bus_dma_segment_t) * (nsegments - 1)); 383 const int zallocflags = (flags & BUS_DMA_NOWAIT) ? KM_NOSLEEP : KM_SLEEP; 384 if ((mapstore = kmem_intr_zalloc(mapsize, zallocflags)) == NULL) 385 return ENOMEM; 386 387 map = (struct arm32_bus_dmamap *)mapstore; 388 map->_dm_size = size; 389 map->_dm_segcnt = nsegments; 390 map->_dm_maxmaxsegsz = maxsegsz; 391 map->_dm_boundary = boundary; 392 map->_dm_flags = flags & ~(BUS_DMA_WAITOK | BUS_DMA_NOWAIT); 393 map->_dm_origbuf = NULL; 394 map->_dm_buftype = _BUS_DMA_BUFTYPE_INVALID; 395 map->_dm_vmspace = vmspace_kernel(); 396 map->_dm_cookie = NULL; 397 map->dm_maxsegsz = maxsegsz; 398 map->dm_mapsize = 0; /* no valid mappings */ 399 map->dm_nsegs = 0; 400 401 #ifdef _ARM32_NEED_BUS_DMA_BOUNCE 402 struct arm32_bus_dma_cookie *cookie; 403 int cookieflags; 404 void *cookiestore; 405 406 cookieflags = 0; 407 408 if (t->_may_bounce != NULL) { 409 error = (*t->_may_bounce)(t, map, flags, &cookieflags); 410 if (error != 0) 411 goto out; 412 } 413 414 if (t->_ranges != NULL) { 415 /* 416 * If ranges are defined, we may have to bounce. The only 417 * exception is if there is exactly one range that covers 418 * all of physical memory. 419 */ 420 switch (t->_nranges) { 421 case 1: 422 if (t->_ranges[0].dr_sysbase == 0 && 423 t->_ranges[0].dr_len == UINTPTR_MAX) { 424 break; 425 } 426 /* FALLTHROUGH */ 427 default: 428 cookieflags |= _BUS_DMA_MIGHT_NEED_BOUNCE; 429 } 430 } 431 432 if ((cookieflags & _BUS_DMA_MIGHT_NEED_BOUNCE) == 0) { 433 STAT_INCR(creates); 434 *dmamp = map; 435 return 0; 436 } 437 438 const size_t cookiesize = sizeof(struct arm32_bus_dma_cookie) + 439 (sizeof(bus_dma_segment_t) * map->_dm_segcnt); 440 441 /* 442 * Allocate our cookie. 443 */ 444 if ((cookiestore = kmem_intr_zalloc(cookiesize, zallocflags)) == NULL) { 445 error = ENOMEM; 446 goto out; 447 } 448 cookie = (struct arm32_bus_dma_cookie *)cookiestore; 449 cookie->id_flags = cookieflags; 450 map->_dm_cookie = cookie; 451 STAT_INCR(bounced_creates); 452 453 error = _bus_dma_alloc_bouncebuf(t, map, size, flags); 454 out: 455 if (error) 456 _bus_dmamap_destroy(t, map); 457 else 458 *dmamp = map; 459 #else 460 *dmamp = map; 461 STAT_INCR(creates); 462 #endif /* _ARM32_NEED_BUS_DMA_BOUNCE */ 463 #ifdef DEBUG_DMA 464 printf("dmamap_create:map=%p\n", map); 465 #endif /* DEBUG_DMA */ 466 return error; 467 } 468 469 /* 470 * Common function for DMA map destruction. May be called by bus-specific 471 * DMA map destruction functions. 472 */ 473 void 474 _bus_dmamap_destroy(bus_dma_tag_t t, bus_dmamap_t map) 475 { 476 477 #ifdef DEBUG_DMA 478 printf("dmamap_destroy: t=%p map=%p\n", t, map); 479 #endif /* DEBUG_DMA */ 480 #ifdef _ARM32_NEED_BUS_DMA_BOUNCE 481 struct arm32_bus_dma_cookie *cookie = map->_dm_cookie; 482 483 /* 484 * Free any bounce pages this map might hold. 485 */ 486 if (cookie != NULL) { 487 const size_t cookiesize = sizeof(struct arm32_bus_dma_cookie) + 488 (sizeof(bus_dma_segment_t) * map->_dm_segcnt); 489 490 if (cookie->id_flags & _BUS_DMA_IS_BOUNCING) 491 STAT_INCR(bounced_unloads); 492 map->dm_nsegs = 0; 493 if (cookie->id_flags & _BUS_DMA_HAS_BOUNCE) 494 _bus_dma_free_bouncebuf(t, map); 495 STAT_INCR(bounced_destroys); 496 kmem_intr_free(cookie, cookiesize); 497 } else 498 #endif 499 STAT_INCR(destroys); 500 501 if (map->dm_nsegs > 0) 502 STAT_INCR(unloads); 503 504 const size_t mapsize = sizeof(struct arm32_bus_dmamap) + 505 (sizeof(bus_dma_segment_t) * (map->_dm_segcnt - 1)); 506 kmem_intr_free(map, mapsize); 507 } 508 509 /* 510 * Common function for loading a DMA map with a linear buffer. May 511 * be called by bus-specific DMA map load functions. 512 */ 513 int 514 _bus_dmamap_load(bus_dma_tag_t t, bus_dmamap_t map, void *buf, 515 bus_size_t buflen, struct proc *p, int flags) 516 { 517 struct vmspace *vm; 518 int error; 519 520 #ifdef DEBUG_DMA 521 printf("dmamap_load: t=%p map=%p buf=%p len=%#" PRIxBUSSIZE 522 " p=%p f=%#x\n", t, map, buf, buflen, p, flags); 523 #endif /* DEBUG_DMA */ 524 525 if (map->dm_nsegs > 0) { 526 #ifdef _ARM32_NEED_BUS_DMA_BOUNCE 527 struct arm32_bus_dma_cookie *cookie = map->_dm_cookie; 528 if (cookie != NULL) { 529 if (cookie->id_flags & _BUS_DMA_IS_BOUNCING) { 530 STAT_INCR(bounced_unloads); 531 cookie->id_flags &= ~_BUS_DMA_IS_BOUNCING; 532 map->_dm_flags &= ~_BUS_DMAMAP_IS_BOUNCING; 533 } 534 } else 535 #endif 536 STAT_INCR(unloads); 537 } 538 539 /* 540 * Make sure that on error condition we return "no valid mappings". 541 */ 542 map->dm_mapsize = 0; 543 map->dm_nsegs = 0; 544 map->_dm_buftype = _BUS_DMA_BUFTYPE_INVALID; 545 KASSERTMSG(map->dm_maxsegsz <= map->_dm_maxmaxsegsz, 546 "dm_maxsegsz %" PRIuBUSSIZE " _dm_maxmaxsegsz %" PRIuBUSSIZE, 547 map->dm_maxsegsz, map->_dm_maxmaxsegsz); 548 549 if (__predict_false(buflen > map->_dm_size)) 550 return EINVAL; 551 552 if (p != NULL) { 553 vm = p->p_vmspace; 554 } else { 555 vm = vmspace_kernel(); 556 } 557 558 /* _bus_dmamap_load_buffer() clears this if we're not... */ 559 map->_dm_flags |= _BUS_DMAMAP_COHERENT; 560 561 error = _bus_dmamap_load_buffer(t, map, buf, buflen, vm, flags); 562 if (__predict_true(error == 0)) { 563 map->dm_mapsize = buflen; 564 map->_dm_vmspace = vm; 565 map->_dm_origbuf = buf; 566 map->_dm_buftype = _BUS_DMA_BUFTYPE_LINEAR; 567 if (map->_dm_flags & _BUS_DMAMAP_COHERENT) { 568 STAT_INCR(coherent_loads); 569 } else { 570 STAT_INCR(loads); 571 } 572 return 0; 573 } 574 #ifdef _ARM32_NEED_BUS_DMA_BOUNCE 575 struct arm32_bus_dma_cookie * const cookie = map->_dm_cookie; 576 if (cookie != NULL && (cookie->id_flags & _BUS_DMA_MIGHT_NEED_BOUNCE)) { 577 error = _bus_dma_load_bouncebuf(t, map, buf, buflen, 578 _BUS_DMA_BUFTYPE_LINEAR, flags); 579 } 580 #endif 581 return error; 582 } 583 584 /* 585 * Like _bus_dmamap_load(), but for mbufs. 586 */ 587 int 588 _bus_dmamap_load_mbuf(bus_dma_tag_t t, bus_dmamap_t map, struct mbuf *m0, 589 int flags) 590 { 591 struct mbuf *m; 592 int error; 593 594 #ifdef DEBUG_DMA 595 printf("dmamap_load_mbuf: t=%p map=%p m0=%p f=%#x\n", 596 t, map, m0, flags); 597 #endif /* DEBUG_DMA */ 598 599 if (map->dm_nsegs > 0) { 600 #ifdef _ARM32_NEED_BUS_DMA_BOUNCE 601 struct arm32_bus_dma_cookie *cookie = map->_dm_cookie; 602 if (cookie != NULL) { 603 if (cookie->id_flags & _BUS_DMA_IS_BOUNCING) { 604 STAT_INCR(bounced_unloads); 605 cookie->id_flags &= ~_BUS_DMA_IS_BOUNCING; 606 map->_dm_flags &= ~_BUS_DMAMAP_IS_BOUNCING; 607 } 608 } else 609 #endif 610 STAT_INCR(unloads); 611 } 612 613 /* 614 * Make sure that on error condition we return "no valid mappings." 615 */ 616 map->dm_mapsize = 0; 617 map->dm_nsegs = 0; 618 map->_dm_buftype = _BUS_DMA_BUFTYPE_INVALID; 619 KASSERTMSG(map->dm_maxsegsz <= map->_dm_maxmaxsegsz, 620 "dm_maxsegsz %" PRIuBUSSIZE " _dm_maxmaxsegsz %" PRIuBUSSIZE, 621 map->dm_maxsegsz, map->_dm_maxmaxsegsz); 622 623 KASSERT(m0->m_flags & M_PKTHDR); 624 625 if (__predict_false(m0->m_pkthdr.len > map->_dm_size)) 626 return EINVAL; 627 628 /* _bus_dmamap_load_paddr() clears this if we're not... */ 629 map->_dm_flags |= _BUS_DMAMAP_COHERENT; 630 631 error = 0; 632 for (m = m0; m != NULL && error == 0; m = m->m_next) { 633 int offset; 634 int remainbytes; 635 const struct vm_page * const *pgs; 636 paddr_t paddr; 637 int size; 638 639 if (m->m_len == 0) 640 continue; 641 /* 642 * Don't allow reads in read-only mbufs. 643 */ 644 if (__predict_false(M_ROMAP(m) && (flags & BUS_DMA_READ))) { 645 error = EFAULT; 646 break; 647 } 648 switch (m->m_flags & (M_EXT | M_EXT_CLUSTER | M_EXT_PAGES)) { 649 case M_EXT | M_EXT_CLUSTER: 650 KASSERT(m->m_ext.ext_paddr != M_PADDR_INVALID); 651 paddr = m->m_ext.ext_paddr + 652 (m->m_data - m->m_ext.ext_buf); 653 size = m->m_len; 654 error = _bus_dmamap_load_paddr(t, map, paddr, size, 655 false); 656 break; 657 658 case M_EXT | M_EXT_PAGES: 659 KASSERT(m->m_ext.ext_buf <= m->m_data); 660 KASSERT(m->m_data <= 661 m->m_ext.ext_buf + m->m_ext.ext_size); 662 663 offset = (vaddr_t)m->m_data - 664 trunc_page((vaddr_t)m->m_ext.ext_buf); 665 remainbytes = m->m_len; 666 667 /* skip uninteresting pages */ 668 pgs = (const struct vm_page * const *) 669 m->m_ext.ext_pgs + (offset >> PAGE_SHIFT); 670 671 offset &= PAGE_MASK; /* offset in the first page */ 672 673 /* load each page */ 674 while (remainbytes > 0) { 675 const struct vm_page *pg; 676 677 size = MIN(remainbytes, PAGE_SIZE - offset); 678 679 pg = *pgs++; 680 KASSERT(pg); 681 paddr = VM_PAGE_TO_PHYS(pg) + offset; 682 683 error = _bus_dmamap_load_paddr(t, map, 684 paddr, size, false); 685 if (__predict_false(error)) 686 break; 687 offset = 0; 688 remainbytes -= size; 689 } 690 break; 691 692 case 0: 693 paddr = m->m_paddr + M_BUFOFFSET(m) + 694 (m->m_data - M_BUFADDR(m)); 695 size = m->m_len; 696 error = _bus_dmamap_load_paddr(t, map, paddr, size, 697 false); 698 break; 699 700 default: 701 error = _bus_dmamap_load_buffer(t, map, m->m_data, 702 m->m_len, vmspace_kernel(), flags); 703 } 704 } 705 if (__predict_true(error == 0)) { 706 map->dm_mapsize = m0->m_pkthdr.len; 707 map->_dm_origbuf = m0; 708 map->_dm_buftype = _BUS_DMA_BUFTYPE_MBUF; 709 map->_dm_vmspace = vmspace_kernel(); /* always kernel */ 710 if (map->_dm_flags & _BUS_DMAMAP_COHERENT) { 711 STAT_INCR(coherent_loads); 712 } else { 713 STAT_INCR(loads); 714 } 715 return 0; 716 } 717 #ifdef _ARM32_NEED_BUS_DMA_BOUNCE 718 struct arm32_bus_dma_cookie * const cookie = map->_dm_cookie; 719 if (cookie != NULL && (cookie->id_flags & _BUS_DMA_MIGHT_NEED_BOUNCE)) { 720 error = _bus_dma_load_bouncebuf(t, map, m0, m0->m_pkthdr.len, 721 _BUS_DMA_BUFTYPE_MBUF, flags); 722 STAT_INCR(bounced_mbuf_loads); 723 } 724 #endif 725 return error; 726 } 727 728 /* 729 * Like _bus_dmamap_load(), but for uios. 730 */ 731 int 732 _bus_dmamap_load_uio(bus_dma_tag_t t, bus_dmamap_t map, struct uio *uio, 733 int flags) 734 { 735 bus_size_t minlen, resid; 736 struct iovec *iov; 737 void *addr; 738 int i, error; 739 740 /* 741 * Make sure that on error condition we return "no valid mappings." 742 */ 743 map->dm_mapsize = 0; 744 map->dm_nsegs = 0; 745 KASSERTMSG(map->dm_maxsegsz <= map->_dm_maxmaxsegsz, 746 "dm_maxsegsz %" PRIuBUSSIZE " _dm_maxmaxsegsz %" PRIuBUSSIZE, 747 map->dm_maxsegsz, map->_dm_maxmaxsegsz); 748 749 resid = uio->uio_resid; 750 iov = uio->uio_iov; 751 752 /* _bus_dmamap_load_buffer() clears this if we're not... */ 753 map->_dm_flags |= _BUS_DMAMAP_COHERENT; 754 755 error = 0; 756 for (i = 0; i < uio->uio_iovcnt && resid != 0 && error == 0; i++) { 757 /* 758 * Now at the first iovec to load. Load each iovec 759 * until we have exhausted the residual count. 760 */ 761 minlen = resid < iov[i].iov_len ? resid : iov[i].iov_len; 762 addr = (void *)iov[i].iov_base; 763 764 error = _bus_dmamap_load_buffer(t, map, addr, minlen, 765 uio->uio_vmspace, flags); 766 767 resid -= minlen; 768 } 769 if (__predict_true(error == 0)) { 770 map->dm_mapsize = uio->uio_resid; 771 map->_dm_origbuf = uio; 772 map->_dm_buftype = _BUS_DMA_BUFTYPE_UIO; 773 map->_dm_vmspace = uio->uio_vmspace; 774 if (map->_dm_flags & _BUS_DMAMAP_COHERENT) { 775 STAT_INCR(coherent_loads); 776 } else { 777 STAT_INCR(loads); 778 } 779 } 780 return error; 781 } 782 783 /* 784 * Like _bus_dmamap_load(), but for raw memory allocated with 785 * bus_dmamem_alloc(). 786 */ 787 int 788 _bus_dmamap_load_raw(bus_dma_tag_t t, bus_dmamap_t map, 789 bus_dma_segment_t *segs, int nsegs, bus_size_t size0, int flags) 790 { 791 792 bus_size_t size; 793 int i, error = 0; 794 795 /* 796 * Make sure that on error conditions we return "no valid mappings." 797 */ 798 map->dm_mapsize = 0; 799 map->dm_nsegs = 0; 800 KASSERT(map->dm_maxsegsz <= map->_dm_maxmaxsegsz); 801 802 if (__predict_false(size0 > map->_dm_size)) 803 return EINVAL; 804 805 for (i = 0, size = size0; i < nsegs && size > 0; i++) { 806 bus_dma_segment_t *ds = &segs[i]; 807 bus_size_t sgsize; 808 809 sgsize = MIN(ds->ds_len, size); 810 if (sgsize == 0) 811 continue; 812 const bool coherent = 813 (ds->_ds_flags & _BUS_DMAMAP_COHERENT) != 0; 814 error = _bus_dmamap_load_paddr(t, map, ds->ds_addr, 815 sgsize, coherent); 816 if (__predict_false(error != 0)) 817 break; 818 size -= sgsize; 819 } 820 821 if (__predict_false(error != 0)) { 822 map->dm_mapsize = 0; 823 map->dm_nsegs = 0; 824 return error; 825 } 826 827 /* XXX TBD bounce */ 828 829 map->dm_mapsize = size0; 830 map->_dm_origbuf = NULL; 831 map->_dm_buftype = _BUS_DMA_BUFTYPE_RAW; 832 map->_dm_vmspace = NULL; 833 return 0; 834 } 835 836 /* 837 * Common function for unloading a DMA map. May be called by 838 * bus-specific DMA map unload functions. 839 */ 840 void 841 _bus_dmamap_unload(bus_dma_tag_t t, bus_dmamap_t map) 842 { 843 844 #ifdef DEBUG_DMA 845 printf("dmamap_unload: t=%p map=%p\n", t, map); 846 #endif /* DEBUG_DMA */ 847 848 /* 849 * No resources to free; just mark the mappings as 850 * invalid. 851 */ 852 map->dm_mapsize = 0; 853 map->dm_nsegs = 0; 854 map->_dm_origbuf = NULL; 855 map->_dm_buftype = _BUS_DMA_BUFTYPE_INVALID; 856 map->_dm_vmspace = NULL; 857 } 858 859 static void 860 _bus_dmamap_sync_segment(vaddr_t va, paddr_t pa, vsize_t len, int ops, 861 bool readonly_p) 862 { 863 864 #if defined(ARM_MMU_EXTENDED) 865 /* 866 * No optimisations are available for readonly mbufs on armv6+, so 867 * assume it's not readonly from here on. 868 * 869 * See the comment in _bus_dmamap_sync_mbuf 870 */ 871 readonly_p = false; 872 #endif 873 874 KASSERTMSG((va & PAGE_MASK) == (pa & PAGE_MASK), 875 "va %#" PRIxVADDR " pa %#" PRIxPADDR, va, pa); 876 #if 0 877 printf("sync_segment: va=%#" PRIxVADDR 878 " pa=%#" PRIxPADDR " len=%#" PRIxVSIZE " ops=%#x\n", 879 va, pa, len, ops); 880 #endif 881 882 switch (ops) { 883 case BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE: 884 if (!readonly_p) { 885 STAT_INCR(sync_prereadwrite); 886 cpu_dcache_wbinv_range(va, len); 887 cpu_sdcache_wbinv_range(va, pa, len); 888 break; 889 } 890 /* FALLTHROUGH */ 891 892 case BUS_DMASYNC_PREREAD: { 893 const size_t line_size = arm_dcache_align; 894 const size_t line_mask = arm_dcache_align_mask; 895 vsize_t misalignment = va & line_mask; 896 if (misalignment) { 897 va -= misalignment; 898 pa -= misalignment; 899 len += misalignment; 900 STAT_INCR(sync_preread_begin); 901 cpu_dcache_wbinv_range(va, line_size); 902 cpu_sdcache_wbinv_range(va, pa, line_size); 903 if (len <= line_size) 904 break; 905 va += line_size; 906 pa += line_size; 907 len -= line_size; 908 } 909 misalignment = len & line_mask; 910 len -= misalignment; 911 if (len > 0) { 912 STAT_INCR(sync_preread); 913 cpu_dcache_inv_range(va, len); 914 cpu_sdcache_inv_range(va, pa, len); 915 } 916 if (misalignment) { 917 va += len; 918 pa += len; 919 STAT_INCR(sync_preread_tail); 920 cpu_dcache_wbinv_range(va, line_size); 921 cpu_sdcache_wbinv_range(va, pa, line_size); 922 } 923 break; 924 } 925 926 case BUS_DMASYNC_PREWRITE: 927 STAT_INCR(sync_prewrite); 928 cpu_dcache_wb_range(va, len); 929 cpu_sdcache_wb_range(va, pa, len); 930 break; 931 932 #if defined(CPU_CORTEX) || defined(CPU_ARMV8) 933 934 /* 935 * Cortex CPUs can do speculative loads so we need to clean the cache 936 * after a DMA read to deal with any speculatively loaded cache lines. 937 * Since these can't be dirty, we can just invalidate them and don't 938 * have to worry about having to write back their contents. 939 */ 940 case BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE: 941 STAT_INCR(sync_postreadwrite); 942 cpu_dcache_inv_range(va, len); 943 cpu_sdcache_inv_range(va, pa, len); 944 break; 945 946 case BUS_DMASYNC_POSTREAD: 947 STAT_INCR(sync_postread); 948 cpu_dcache_inv_range(va, len); 949 cpu_sdcache_inv_range(va, pa, len); 950 break; 951 #endif 952 } 953 } 954 955 static inline void 956 _bus_dmamap_sync_linear(bus_dma_tag_t t, bus_dmamap_t map, bus_addr_t offset, 957 bus_size_t len, int ops) 958 { 959 bus_dma_segment_t *ds = map->dm_segs; 960 vaddr_t va = (vaddr_t) map->_dm_origbuf; 961 #ifdef _ARM32_NEED_BUS_DMA_BOUNCE 962 if (map->_dm_flags & _BUS_DMAMAP_IS_BOUNCING) { 963 struct arm32_bus_dma_cookie * const cookie = map->_dm_cookie; 964 va = (vaddr_t) cookie->id_bouncebuf; 965 } 966 #endif 967 968 while (len > 0) { 969 while (offset >= ds->ds_len) { 970 offset -= ds->ds_len; 971 va += ds->ds_len; 972 ds++; 973 } 974 975 paddr_t pa = _bus_dma_busaddr_to_paddr(t, ds->ds_addr + offset); 976 size_t seglen = uimin(len, ds->ds_len - offset); 977 978 if ((ds->_ds_flags & _BUS_DMAMAP_COHERENT) == 0) 979 _bus_dmamap_sync_segment(va + offset, pa, seglen, ops, 980 false); 981 982 offset += seglen; 983 len -= seglen; 984 } 985 } 986 987 static inline void 988 _bus_dmamap_sync_mbuf(bus_dma_tag_t t, bus_dmamap_t map, bus_size_t offset, 989 bus_size_t len, int ops) 990 { 991 bus_dma_segment_t *ds = map->dm_segs; 992 struct mbuf *m = map->_dm_origbuf; 993 bus_size_t voff = offset; 994 bus_size_t ds_off = offset; 995 996 while (len > 0) { 997 /* Find the current dma segment */ 998 while (ds_off >= ds->ds_len) { 999 ds_off -= ds->ds_len; 1000 ds++; 1001 } 1002 /* Find the current mbuf. */ 1003 while (voff >= m->m_len) { 1004 voff -= m->m_len; 1005 m = m->m_next; 1006 } 1007 1008 /* 1009 * Now at the first mbuf to sync; nail each one until 1010 * we have exhausted the length. 1011 */ 1012 vsize_t seglen = uimin(len, uimin(m->m_len - voff, ds->ds_len - ds_off)); 1013 vaddr_t va = mtod(m, vaddr_t) + voff; 1014 paddr_t pa = _bus_dma_busaddr_to_paddr(t, ds->ds_addr + ds_off); 1015 1016 /* 1017 * We can save a lot of work here if we know the mapping 1018 * is read-only at the MMU and we aren't using the armv6+ 1019 * MMU: 1020 * 1021 * If a mapping is read-only, no dirty cache blocks will 1022 * exist for it. If a writable mapping was made read-only, 1023 * we know any dirty cache lines for the range will have 1024 * been cleaned for us already. Therefore, if the upper 1025 * layer can tell us we have a read-only mapping, we can 1026 * skip all cache cleaning. 1027 * 1028 * NOTE: This only works if we know the pmap cleans pages 1029 * before making a read-write -> read-only transition. If 1030 * this ever becomes non-true (e.g. Physically Indexed 1031 * cache), this will have to be revisited. 1032 */ 1033 1034 if ((ds->_ds_flags & _BUS_DMAMAP_COHERENT) == 0) { 1035 /* 1036 * If we are doing preread (DMAing into the mbuf), 1037 * this mbuf better not be readonly, 1038 */ 1039 KASSERT(!(ops & BUS_DMASYNC_PREREAD) || !M_ROMAP(m)); 1040 _bus_dmamap_sync_segment(va, pa, seglen, ops, 1041 M_ROMAP(m)); 1042 } 1043 voff += seglen; 1044 ds_off += seglen; 1045 len -= seglen; 1046 } 1047 } 1048 1049 static inline void 1050 _bus_dmamap_sync_uio(bus_dma_tag_t t, bus_dmamap_t map, bus_addr_t offset, 1051 bus_size_t len, int ops) 1052 { 1053 bus_dma_segment_t *ds = map->dm_segs; 1054 struct uio *uio = map->_dm_origbuf; 1055 struct iovec *iov = uio->uio_iov; 1056 bus_size_t voff = offset; 1057 bus_size_t ds_off = offset; 1058 1059 while (len > 0) { 1060 /* Find the current dma segment */ 1061 while (ds_off >= ds->ds_len) { 1062 ds_off -= ds->ds_len; 1063 ds++; 1064 } 1065 1066 /* Find the current iovec. */ 1067 while (voff >= iov->iov_len) { 1068 voff -= iov->iov_len; 1069 iov++; 1070 } 1071 1072 /* 1073 * Now at the first iovec to sync; nail each one until 1074 * we have exhausted the length. 1075 */ 1076 vsize_t seglen = uimin(len, uimin(iov->iov_len - voff, ds->ds_len - ds_off)); 1077 vaddr_t va = (vaddr_t) iov->iov_base + voff; 1078 paddr_t pa = _bus_dma_busaddr_to_paddr(t, ds->ds_addr + ds_off); 1079 1080 if ((ds->_ds_flags & _BUS_DMAMAP_COHERENT) == 0) 1081 _bus_dmamap_sync_segment(va, pa, seglen, ops, false); 1082 1083 voff += seglen; 1084 ds_off += seglen; 1085 len -= seglen; 1086 } 1087 } 1088 1089 /* 1090 * Common function for DMA map synchronization. May be called 1091 * by bus-specific DMA map synchronization functions. 1092 * 1093 * XXX Should have separate versions for write-through vs. 1094 * XXX write-back caches. We currently assume write-back 1095 * XXX here, which is not as efficient as it could be for 1096 * XXX the write-through case. 1097 */ 1098 void 1099 _bus_dmamap_sync(bus_dma_tag_t t, bus_dmamap_t map, bus_addr_t offset, 1100 bus_size_t len, int ops) 1101 { 1102 #ifdef DEBUG_DMA 1103 printf("dmamap_sync: t=%p map=%p offset=%#" PRIxBUSADDR 1104 " len=%#" PRIxBUSSIZE " ops=%#x\n", t, map, offset, len, ops); 1105 #endif /* DEBUG_DMA */ 1106 1107 /* 1108 * Mixing of PRE and POST operations is not allowed. 1109 */ 1110 KASSERTMSG((((ops & (BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE)) == 0) 1111 || ((ops & (BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE)) == 0)), 1112 "%s: mix PRE and POST", __func__); 1113 1114 KASSERTMSG(offset < map->dm_mapsize, 1115 "offset %" PRIxBUSADDR " mapsize %" PRIuBUSSIZE, 1116 offset, map->dm_mapsize); 1117 KASSERTMSG(len > 0 && offset + len <= map->dm_mapsize, 1118 "len %" PRIuBUSSIZE " offset %" PRIxBUSADDR " mapsize %" PRIuBUSSIZE, 1119 len, offset, map->dm_mapsize); 1120 1121 /* 1122 * For a write-back cache, we need to do the following things: 1123 * 1124 * PREREAD -- Invalidate the D-cache. We do this 1125 * here in case a write-back is required by the back-end. 1126 * 1127 * PREWRITE -- Write-back the D-cache. Note that if 1128 * we are doing a PREREAD | PREWRITE, we can collapse 1129 * the whole thing into a single Wb-Inv. 1130 * 1131 * POSTREAD -- Re-invalidate the D-cache in case speculative 1132 * memory accesses caused cachelines to become valid with now 1133 * invalid data. 1134 * 1135 * POSTWRITE -- Nothing. 1136 */ 1137 #ifdef _ARM32_NEED_BUS_DMA_BOUNCE 1138 const bool bouncing = (map->_dm_flags & _BUS_DMAMAP_IS_BOUNCING); 1139 #else 1140 const bool bouncing = false; 1141 #endif 1142 1143 const int pre_ops = ops & (BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1144 #if defined(CPU_CORTEX) || defined(CPU_ARMV8) 1145 const int post_ops = ops & (BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 1146 #else 1147 const int post_ops = 0; 1148 #endif 1149 if (pre_ops == 0 && post_ops == 0) 1150 return; 1151 1152 if (post_ops == BUS_DMASYNC_POSTWRITE) { 1153 KASSERT(pre_ops == 0); 1154 if ((map->_dm_flags & _BUS_DMAMAP_COHERENT)) { 1155 STAT_INCR(sync_coherent_postwrite); 1156 } else { 1157 STAT_INCR(sync_postwrite); 1158 } 1159 return; 1160 } 1161 1162 KASSERTMSG(bouncing || pre_ops != 0 || (post_ops & BUS_DMASYNC_POSTREAD), 1163 "pre_ops %#x post_ops %#x", pre_ops, post_ops); 1164 1165 if (bouncing && (ops & BUS_DMASYNC_PREWRITE)) { 1166 struct arm32_bus_dma_cookie * const cookie = map->_dm_cookie; 1167 STAT_INCR(write_bounces); 1168 char * const dataptr = (char *)cookie->id_bouncebuf + offset; 1169 /* 1170 * Copy the caller's buffer to the bounce buffer. 1171 */ 1172 switch (map->_dm_buftype) { 1173 case _BUS_DMA_BUFTYPE_LINEAR: 1174 memcpy(dataptr, cookie->id_origlinearbuf + offset, len); 1175 break; 1176 1177 case _BUS_DMA_BUFTYPE_MBUF: 1178 m_copydata(cookie->id_origmbuf, offset, len, dataptr); 1179 break; 1180 1181 case _BUS_DMA_BUFTYPE_UIO: 1182 _bus_dma_uiomove(dataptr, cookie->id_origuio, len, 1183 UIO_WRITE); 1184 break; 1185 1186 #ifdef DIAGNOSTIC 1187 case _BUS_DMA_BUFTYPE_RAW: 1188 panic("%s:(pre): _BUS_DMA_BUFTYPE_RAW", __func__); 1189 break; 1190 1191 case _BUS_DMA_BUFTYPE_INVALID: 1192 panic("%s(pre): _BUS_DMA_BUFTYPE_INVALID", __func__); 1193 break; 1194 1195 default: 1196 panic("%s(pre): map %p: unknown buffer type %d\n", 1197 __func__, map, map->_dm_buftype); 1198 break; 1199 #endif /* DIAGNOSTIC */ 1200 } 1201 } 1202 1203 /* Skip cache frobbing if mapping was COHERENT */ 1204 if ((map->_dm_flags & _BUS_DMAMAP_COHERENT)) { 1205 switch (ops) { 1206 case BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE: 1207 STAT_INCR(sync_coherent_prereadwrite); 1208 break; 1209 1210 case BUS_DMASYNC_PREREAD: 1211 STAT_INCR(sync_coherent_preread); 1212 break; 1213 1214 case BUS_DMASYNC_PREWRITE: 1215 STAT_INCR(sync_coherent_prewrite); 1216 break; 1217 1218 case BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE: 1219 STAT_INCR(sync_coherent_postreadwrite); 1220 break; 1221 1222 case BUS_DMASYNC_POSTREAD: 1223 STAT_INCR(sync_coherent_postread); 1224 break; 1225 1226 /* BUS_DMASYNC_POSTWRITE was aleady handled as a fastpath */ 1227 } 1228 /* 1229 * Drain the write buffer of DMA operators. 1230 * 1) when cpu->device (prewrite) 1231 * 2) when device->cpu (postread) 1232 */ 1233 if ((pre_ops & BUS_DMASYNC_PREWRITE) || (post_ops & BUS_DMASYNC_POSTREAD)) 1234 cpu_drain_writebuf(); 1235 1236 /* 1237 * Only thing left to do for COHERENT mapping is copy from bounce 1238 * in the POSTREAD case. 1239 */ 1240 if (bouncing && (post_ops & BUS_DMASYNC_POSTREAD)) 1241 goto bounce_it; 1242 1243 return; 1244 } 1245 1246 #if !defined(ARM_MMU_EXTENDED) 1247 /* 1248 * If the mapping belongs to a non-kernel vmspace, and the 1249 * vmspace has not been active since the last time a full 1250 * cache flush was performed, we don't need to do anything. 1251 */ 1252 if (__predict_false(!VMSPACE_IS_KERNEL_P(map->_dm_vmspace) && 1253 vm_map_pmap(&map->_dm_vmspace->vm_map)->pm_cstate.cs_cache_d == 0)) 1254 return; 1255 #endif 1256 1257 int buftype = map->_dm_buftype; 1258 if (bouncing) { 1259 buftype = _BUS_DMA_BUFTYPE_LINEAR; 1260 } 1261 1262 switch (buftype) { 1263 case _BUS_DMA_BUFTYPE_LINEAR: 1264 case _BUS_DMA_BUFTYPE_RAW: 1265 _bus_dmamap_sync_linear(t, map, offset, len, ops); 1266 break; 1267 1268 case _BUS_DMA_BUFTYPE_MBUF: 1269 _bus_dmamap_sync_mbuf(t, map, offset, len, ops); 1270 break; 1271 1272 case _BUS_DMA_BUFTYPE_UIO: 1273 _bus_dmamap_sync_uio(t, map, offset, len, ops); 1274 break; 1275 1276 case _BUS_DMA_BUFTYPE_INVALID: 1277 panic("%s: _BUS_DMA_BUFTYPE_INVALID", __func__); 1278 break; 1279 1280 default: 1281 panic("%s: map %p: unknown buffer type %d\n", __func__, map, 1282 map->_dm_buftype); 1283 } 1284 1285 /* Drain the write buffer. */ 1286 cpu_drain_writebuf(); 1287 1288 if (!bouncing || (ops & BUS_DMASYNC_POSTREAD) == 0) 1289 return; 1290 1291 bounce_it: 1292 STAT_INCR(read_bounces); 1293 1294 struct arm32_bus_dma_cookie * const cookie = map->_dm_cookie; 1295 char * const dataptr = (char *)cookie->id_bouncebuf + offset; 1296 /* 1297 * Copy the bounce buffer to the caller's buffer. 1298 */ 1299 switch (map->_dm_buftype) { 1300 case _BUS_DMA_BUFTYPE_LINEAR: 1301 memcpy(cookie->id_origlinearbuf + offset, dataptr, len); 1302 break; 1303 1304 case _BUS_DMA_BUFTYPE_MBUF: 1305 m_copyback(cookie->id_origmbuf, offset, len, dataptr); 1306 break; 1307 1308 case _BUS_DMA_BUFTYPE_UIO: 1309 _bus_dma_uiomove(dataptr, cookie->id_origuio, len, UIO_READ); 1310 break; 1311 1312 #ifdef DIAGNOSTIC 1313 case _BUS_DMA_BUFTYPE_RAW: 1314 panic("%s(post): _BUS_DMA_BUFTYPE_RAW", __func__); 1315 break; 1316 1317 case _BUS_DMA_BUFTYPE_INVALID: 1318 panic("%s(post): _BUS_DMA_BUFTYPE_INVALID", __func__); 1319 break; 1320 1321 default: 1322 panic("%s(post): map %p: unknown buffer type %d\n", __func__, 1323 map, map->_dm_buftype); 1324 break; 1325 #endif 1326 } 1327 } 1328 1329 /* 1330 * Common function for DMA-safe memory allocation. May be called 1331 * by bus-specific DMA memory allocation functions. 1332 */ 1333 1334 int 1335 _bus_dmamem_alloc(bus_dma_tag_t t, bus_size_t size, bus_size_t alignment, 1336 bus_size_t boundary, bus_dma_segment_t *segs, int nsegs, int *rsegs, 1337 int flags) 1338 { 1339 struct arm32_dma_range *dr; 1340 int error, i; 1341 1342 #ifdef DEBUG_DMA 1343 printf("dmamem_alloc t=%p size=%#" PRIxBUSSIZE 1344 " align=%#" PRIxBUSSIZE 1345 " boundary=%#" PRIxBUSSIZE " " 1346 "segs=%p nsegs=%#x rsegs=%p flags=%#x\n", t, size, alignment, 1347 boundary, segs, nsegs, rsegs, flags); 1348 #endif 1349 1350 if ((dr = t->_ranges) != NULL) { 1351 error = ENOMEM; 1352 for (i = 0; i < t->_nranges; i++, dr++) { 1353 if (dr->dr_len == 0 1354 || (dr->dr_flags & _BUS_DMAMAP_NOALLOC)) 1355 continue; 1356 error = _bus_dmamem_alloc_range(t, size, alignment, 1357 boundary, segs, nsegs, rsegs, flags, 1358 trunc_page(dr->dr_sysbase), 1359 trunc_page(dr->dr_sysbase + dr->dr_len)); 1360 if (error == 0) 1361 break; 1362 } 1363 } else { 1364 error = _bus_dmamem_alloc_range(t, size, alignment, boundary, 1365 segs, nsegs, rsegs, flags, 0UL, ~0UL); 1366 } 1367 1368 #ifdef DEBUG_DMA 1369 printf("dmamem_alloc: =%d\n", error); 1370 #endif 1371 1372 return error; 1373 } 1374 1375 /* 1376 * Common function for freeing DMA-safe memory. May be called by 1377 * bus-specific DMA memory free functions. 1378 */ 1379 void 1380 _bus_dmamem_free(bus_dma_tag_t t, bus_dma_segment_t *segs, int nsegs) 1381 { 1382 struct vm_page *m; 1383 bus_addr_t addr; 1384 struct pglist mlist; 1385 int curseg; 1386 1387 #ifdef DEBUG_DMA 1388 printf("dmamem_free: t=%p segs=%p nsegs=%#x\n", t, segs, nsegs); 1389 #endif /* DEBUG_DMA */ 1390 1391 /* 1392 * Build a list of pages to free back to the VM system. 1393 */ 1394 TAILQ_INIT(&mlist); 1395 for (curseg = 0; curseg < nsegs; curseg++) { 1396 for (addr = segs[curseg].ds_addr; 1397 addr < (segs[curseg].ds_addr + segs[curseg].ds_len); 1398 addr += PAGE_SIZE) { 1399 m = PHYS_TO_VM_PAGE(addr); 1400 TAILQ_INSERT_TAIL(&mlist, m, pageq.queue); 1401 } 1402 } 1403 uvm_pglistfree(&mlist); 1404 } 1405 1406 /* 1407 * Common function for mapping DMA-safe memory. May be called by 1408 * bus-specific DMA memory map functions. 1409 */ 1410 int 1411 _bus_dmamem_map(bus_dma_tag_t t, bus_dma_segment_t *segs, int nsegs, 1412 size_t size, void **kvap, int flags) 1413 { 1414 vaddr_t va; 1415 paddr_t pa; 1416 int curseg; 1417 const uvm_flag_t kmflags = UVM_KMF_VAONLY 1418 | ((flags & BUS_DMA_NOWAIT) != 0 ? UVM_KMF_NOWAIT : 0); 1419 vsize_t align = 0; 1420 1421 #ifdef DEBUG_DMA 1422 printf("dmamem_map: t=%p segs=%p nsegs=%#x size=%#zx flags=%#x\n", t, 1423 segs, nsegs, size, flags); 1424 #endif /* DEBUG_DMA */ 1425 1426 #ifdef PMAP_MAP_POOLPAGE 1427 /* 1428 * If all of memory is mapped, and we are mapping a single physically 1429 * contiguous area then this area is already mapped. Let's see if we 1430 * avoid having a separate mapping for it. 1431 */ 1432 if (nsegs == 1 && 1433 (flags & (BUS_DMA_PREFETCHABLE|BUS_DMA_NOCACHE)) == 0) { 1434 /* 1435 * If this is a non-COHERENT mapping, then the existing kernel 1436 * mapping is already compatible with it. 1437 */ 1438 bool direct_mapable = (flags & BUS_DMA_COHERENT) == 0; 1439 pa = segs[0].ds_addr; 1440 1441 /* 1442 * This is a COHERENT mapping which, unless this address is in 1443 * a COHERENT dma range, will not be compatible. 1444 */ 1445 if (t->_ranges != NULL) { 1446 const struct arm32_dma_range * const dr = 1447 _bus_dma_paddr_inrange(t->_ranges, t->_nranges, pa); 1448 if (dr != NULL 1449 && (dr->dr_flags & _BUS_DMAMAP_COHERENT)) { 1450 direct_mapable = true; 1451 } 1452 } 1453 1454 #ifdef PMAP_NEED_ALLOC_POOLPAGE 1455 /* 1456 * The page can only be direct mapped if was allocated out 1457 * of the arm poolpage vm freelist. 1458 */ 1459 uvm_physseg_t upm = uvm_physseg_find(atop(pa), NULL); 1460 KASSERT(uvm_physseg_valid_p(upm)); 1461 if (direct_mapable) { 1462 direct_mapable = 1463 (arm_poolpage_vmfreelist == uvm_physseg_get_free_list(upm)); 1464 } 1465 #endif 1466 1467 if (direct_mapable) { 1468 *kvap = (void *)PMAP_MAP_POOLPAGE(pa); 1469 #ifdef DEBUG_DMA 1470 printf("dmamem_map: =%p\n", *kvap); 1471 #endif /* DEBUG_DMA */ 1472 return 0; 1473 } 1474 } 1475 #endif 1476 1477 size = round_page(size); 1478 1479 #ifdef PMAP_MAPSIZE1 1480 if (size >= PMAP_MAPSIZE1) 1481 align = PMAP_MAPSIZE1; 1482 1483 #ifdef PMAP_MAPSIZE2 1484 1485 #if PMAP_MAPSIZE1 > PMAP_MAPSIZE2 1486 #error PMAP_MAPSIZE1 must be smaller than PMAP_MAPSIZE2 1487 #endif 1488 1489 if (size >= PMAP_MAPSIZE2) 1490 align = PMAP_MAPSIZE2; 1491 1492 #ifdef PMAP_MAPSIZE3 1493 1494 #if PMAP_MAPSIZE2 > PMAP_MAPSIZE3 1495 #error PMAP_MAPSIZE2 must be smaller than PMAP_MAPSIZE3 1496 #endif 1497 1498 if (size >= PMAP_MAPSIZE3) 1499 align = PMAP_MAPSIZE3; 1500 #endif 1501 #endif 1502 #endif 1503 1504 va = uvm_km_alloc(kernel_map, size, align, kmflags); 1505 if (__predict_false(va == 0 && align > 0)) { 1506 align = 0; 1507 va = uvm_km_alloc(kernel_map, size, 0, kmflags); 1508 } 1509 1510 if (va == 0) 1511 return ENOMEM; 1512 1513 *kvap = (void *)va; 1514 1515 for (curseg = 0; curseg < nsegs; curseg++) { 1516 for (pa = segs[curseg].ds_addr; 1517 pa < (segs[curseg].ds_addr + segs[curseg].ds_len); 1518 pa += PAGE_SIZE, va += PAGE_SIZE, size -= PAGE_SIZE) { 1519 bool uncached = 1520 (flags & (BUS_DMA_COHERENT | BUS_DMA_NOCACHE)) != 0; 1521 bool prefetchable = (flags & BUS_DMA_PREFETCHABLE); 1522 #ifdef DEBUG_DMA 1523 printf("wiring P%#" PRIxPADDR 1524 " to V%#" PRIxVADDR "\n", pa, va); 1525 #endif /* DEBUG_DMA */ 1526 if (size == 0) 1527 panic("_bus_dmamem_map: size botch"); 1528 1529 const struct arm32_dma_range * const dr = 1530 _bus_dma_paddr_inrange(t->_ranges, t->_nranges, pa); 1531 /* 1532 * If this dma region is coherent then there is 1533 * no need for an uncached mapping. 1534 */ 1535 if ((flags & BUS_DMA_NOCACHE) == 0 && 1536 dr != NULL 1537 && (dr->dr_flags & _BUS_DMAMAP_COHERENT)) { 1538 uncached = false; 1539 } 1540 1541 u_int pmap_flags = PMAP_WIRED; 1542 if (prefetchable) 1543 pmap_flags |= PMAP_WRITE_COMBINE; 1544 else if (uncached) 1545 pmap_flags |= PMAP_NOCACHE; 1546 1547 pmap_kenter_pa(va, pa, VM_PROT_READ | VM_PROT_WRITE, 1548 pmap_flags); 1549 } 1550 } 1551 pmap_update(pmap_kernel()); 1552 #ifdef DEBUG_DMA 1553 printf("dmamem_map: =%p\n", *kvap); 1554 #endif /* DEBUG_DMA */ 1555 return 0; 1556 } 1557 1558 /* 1559 * Common function for unmapping DMA-safe memory. May be called by 1560 * bus-specific DMA memory unmapping functions. 1561 */ 1562 void 1563 _bus_dmamem_unmap(bus_dma_tag_t t, void *kva, size_t size) 1564 { 1565 1566 #ifdef DEBUG_DMA 1567 printf("dmamem_unmap: t=%p kva=%p size=%#zx\n", t, kva, size); 1568 #endif /* DEBUG_DMA */ 1569 KASSERTMSG(((uintptr_t)kva & PAGE_MASK) == 0, 1570 "kva %p (%#"PRIxPTR")", kva, ((uintptr_t)kva & PAGE_MASK)); 1571 1572 #ifdef __HAVE_MM_MD_DIRECT_MAPPED_PHYS 1573 /* 1574 * Check to see if this used direct mapped memory. Get its physical 1575 * address and try to map it. If the resultant matches the kva, then 1576 * it was and so we can just return since we have nothing to free up. 1577 */ 1578 paddr_t pa; 1579 vaddr_t va; 1580 (void)pmap_extract(pmap_kernel(), (vaddr_t)kva, &pa); 1581 if (mm_md_direct_mapped_phys(pa, &va) && va == (vaddr_t)kva) 1582 return; 1583 #endif 1584 1585 size = round_page(size); 1586 pmap_kremove((vaddr_t)kva, size); 1587 pmap_update(pmap_kernel()); 1588 uvm_km_free(kernel_map, (vaddr_t)kva, size, UVM_KMF_VAONLY); 1589 } 1590 1591 /* 1592 * Common function for mmap(2)'ing DMA-safe memory. May be called by 1593 * bus-specific DMA mmap(2)'ing functions. 1594 */ 1595 paddr_t 1596 _bus_dmamem_mmap(bus_dma_tag_t t, bus_dma_segment_t *segs, int nsegs, 1597 off_t off, int prot, int flags) 1598 { 1599 paddr_t map_flags; 1600 int i; 1601 1602 for (i = 0; i < nsegs; i++) { 1603 KASSERTMSG((off & PAGE_MASK) == 0, 1604 "off %#jx (%#x)", (uintmax_t)off, (int)off & PAGE_MASK); 1605 KASSERTMSG((segs[i].ds_addr & PAGE_MASK) == 0, 1606 "ds_addr %#lx (%#x)", segs[i].ds_addr, 1607 (int)segs[i].ds_addr & PAGE_MASK); 1608 KASSERTMSG((segs[i].ds_len & PAGE_MASK) == 0, 1609 "ds_len %#lx (%#x)", segs[i].ds_addr, 1610 (int)segs[i].ds_addr & PAGE_MASK); 1611 if (off >= segs[i].ds_len) { 1612 off -= segs[i].ds_len; 1613 continue; 1614 } 1615 1616 map_flags = 0; 1617 if (flags & BUS_DMA_PREFETCHABLE) 1618 map_flags |= ARM_MMAP_WRITECOMBINE; 1619 1620 return arm_btop((u_long)segs[i].ds_addr + off) | map_flags; 1621 1622 } 1623 1624 /* Page not found. */ 1625 return -1; 1626 } 1627 1628 /********************************************************************** 1629 * DMA utility functions 1630 **********************************************************************/ 1631 1632 /* 1633 * Utility function to load a linear buffer. lastaddrp holds state 1634 * between invocations (for multiple-buffer loads). segp contains 1635 * the starting segment on entrance, and the ending segment on exit. 1636 * first indicates if this is the first invocation of this function. 1637 */ 1638 int 1639 _bus_dmamap_load_buffer(bus_dma_tag_t t, bus_dmamap_t map, void *buf, 1640 bus_size_t buflen, struct vmspace *vm, int flags) 1641 { 1642 bus_size_t sgsize; 1643 bus_addr_t curaddr; 1644 vaddr_t vaddr = (vaddr_t)buf; 1645 int error; 1646 1647 #ifdef DEBUG_DMA 1648 printf("_bus_dmamap_load_buffer(buf=%p, len=%#" PRIxBUSSIZE 1649 ", flags=%#x)\n", buf, buflen, flags); 1650 #endif /* DEBUG_DMA */ 1651 1652 pmap_t pmap = vm_map_pmap(&vm->vm_map); 1653 1654 while (buflen > 0) { 1655 /* 1656 * Get the physical address for this segment. 1657 * 1658 */ 1659 bool coherent; 1660 bool ok __diagused; 1661 ok = pmap_extract_coherency(pmap, vaddr, &curaddr, &coherent); 1662 1663 /* 1664 * trying to bus_dmamap_load an unmapped buffer is a 1665 * programming error. 1666 */ 1667 KASSERT(ok); 1668 1669 KASSERTMSG((vaddr & PAGE_MASK) == (curaddr & PAGE_MASK), 1670 "va %#" PRIxVADDR " curaddr %#" PRIxBUSADDR, vaddr, curaddr); 1671 1672 /* 1673 * Compute the segment size, and adjust counts. 1674 */ 1675 sgsize = PAGE_SIZE - ((u_long)vaddr & PGOFSET); 1676 if (buflen < sgsize) 1677 sgsize = buflen; 1678 1679 error = _bus_dmamap_load_paddr(t, map, curaddr, sgsize, 1680 coherent); 1681 if (__predict_false(error)) 1682 return error; 1683 1684 vaddr += sgsize; 1685 buflen -= sgsize; 1686 } 1687 1688 return 0; 1689 } 1690 1691 /* 1692 * Allocate physical memory from the given physical address range. 1693 * Called by DMA-safe memory allocation methods. 1694 */ 1695 int 1696 _bus_dmamem_alloc_range(bus_dma_tag_t t, bus_size_t size, bus_size_t alignment, 1697 bus_size_t boundary, bus_dma_segment_t *segs, int nsegs, int *rsegs, 1698 int flags, paddr_t low, paddr_t high) 1699 { 1700 paddr_t curaddr, lastaddr; 1701 struct vm_page *m; 1702 struct pglist mlist; 1703 int curseg, error; 1704 1705 KASSERTMSG(boundary == 0 || (boundary & (boundary - 1)) == 0, 1706 "invalid boundary %#" PRIxBUSSIZE, boundary); 1707 1708 #ifdef DEBUG_DMA 1709 printf("alloc_range: t=%p size=%#" PRIxBUSSIZE 1710 " align=%#" PRIxBUSSIZE " boundary=%#" PRIxBUSSIZE 1711 " segs=%p nsegs=%#x rsegs=%p flags=%#x" 1712 " lo=%#" PRIxPADDR " hi=%#" PRIxPADDR "\n", 1713 t, size, alignment, boundary, segs, nsegs, rsegs, flags, low, high); 1714 #endif /* DEBUG_DMA */ 1715 1716 /* Always round the size. */ 1717 size = round_page(size); 1718 1719 /* 1720 * We accept boundaries < size, splitting in multiple segments 1721 * if needed. uvm_pglistalloc does not, so compute an appropriate 1722 * boundary: next power of 2 >= size 1723 */ 1724 bus_size_t uboundary = boundary; 1725 if (uboundary <= PAGE_SIZE) { 1726 uboundary = 0; 1727 } else { 1728 while (uboundary < size) { 1729 uboundary <<= 1; 1730 } 1731 } 1732 1733 /* 1734 * Allocate pages from the VM system. 1735 */ 1736 error = uvm_pglistalloc(size, low, high, alignment, uboundary, 1737 &mlist, nsegs, (flags & BUS_DMA_NOWAIT) == 0); 1738 if (error) 1739 return error; 1740 1741 /* 1742 * Compute the location, size, and number of segments actually 1743 * returned by the VM code. 1744 */ 1745 m = TAILQ_FIRST(&mlist); 1746 curseg = 0; 1747 lastaddr = segs[curseg].ds_addr = segs[curseg]._ds_paddr = 1748 VM_PAGE_TO_PHYS(m); 1749 segs[curseg].ds_len = PAGE_SIZE; 1750 #ifdef DEBUG_DMA 1751 printf("alloc: page %#" PRIxPADDR "\n", lastaddr); 1752 #endif /* DEBUG_DMA */ 1753 m = TAILQ_NEXT(m, pageq.queue); 1754 1755 for (; m != NULL; m = TAILQ_NEXT(m, pageq.queue)) { 1756 curaddr = VM_PAGE_TO_PHYS(m); 1757 KASSERTMSG(low <= curaddr && curaddr < high, 1758 "uvm_pglistalloc returned non-sensicaladdress %#" PRIxPADDR 1759 "(low=%#" PRIxPADDR ", high=%#" PRIxPADDR "\n", 1760 curaddr, low, high); 1761 #ifdef DEBUG_DMA 1762 printf("alloc: page %#" PRIxPADDR "\n", curaddr); 1763 #endif /* DEBUG_DMA */ 1764 if (curaddr == lastaddr + PAGE_SIZE 1765 && (lastaddr & boundary) == (curaddr & boundary)) 1766 segs[curseg].ds_len += PAGE_SIZE; 1767 else { 1768 curseg++; 1769 if (curseg >= nsegs) { 1770 uvm_pglistfree(&mlist); 1771 return EFBIG; 1772 } 1773 segs[curseg].ds_addr = curaddr; 1774 segs[curseg]._ds_paddr = curaddr; 1775 segs[curseg].ds_len = PAGE_SIZE; 1776 } 1777 lastaddr = curaddr; 1778 } 1779 1780 *rsegs = curseg + 1; 1781 1782 return 0; 1783 } 1784 1785 /* 1786 * Check if a memory region intersects with a DMA range, and return the 1787 * page-rounded intersection if it does. 1788 */ 1789 int 1790 arm32_dma_range_intersect(struct arm32_dma_range *ranges, int nranges, 1791 paddr_t pa, psize_t size, paddr_t *pap, psize_t *sizep) 1792 { 1793 struct arm32_dma_range *dr; 1794 int i; 1795 1796 if (ranges == NULL) 1797 return 0; 1798 1799 for (i = 0, dr = ranges; i < nranges; i++, dr++) { 1800 if (dr->dr_sysbase <= pa && 1801 pa < (dr->dr_sysbase + dr->dr_len)) { 1802 /* 1803 * Beginning of region intersects with this range. 1804 */ 1805 *pap = trunc_page(pa); 1806 *sizep = round_page(uimin(pa + size, 1807 dr->dr_sysbase + dr->dr_len) - pa); 1808 return 1; 1809 } 1810 if (pa < dr->dr_sysbase && dr->dr_sysbase < (pa + size)) { 1811 /* 1812 * End of region intersects with this range. 1813 */ 1814 *pap = trunc_page(dr->dr_sysbase); 1815 *sizep = round_page(uimin((pa + size) - dr->dr_sysbase, 1816 dr->dr_len)); 1817 return 1; 1818 } 1819 } 1820 1821 /* No intersection found. */ 1822 return 0; 1823 } 1824 1825 #ifdef _ARM32_NEED_BUS_DMA_BOUNCE 1826 static int 1827 _bus_dma_alloc_bouncebuf(bus_dma_tag_t t, bus_dmamap_t map, 1828 bus_size_t size, int flags) 1829 { 1830 struct arm32_bus_dma_cookie *cookie = map->_dm_cookie; 1831 int error = 0; 1832 1833 KASSERT(cookie != NULL); 1834 1835 cookie->id_bouncebuflen = round_page(size); 1836 error = _bus_dmamem_alloc(t, cookie->id_bouncebuflen, 1837 PAGE_SIZE, map->_dm_boundary, cookie->id_bouncesegs, 1838 map->_dm_segcnt, &cookie->id_nbouncesegs, flags); 1839 if (error == 0) { 1840 error = _bus_dmamem_map(t, cookie->id_bouncesegs, 1841 cookie->id_nbouncesegs, cookie->id_bouncebuflen, 1842 (void **)&cookie->id_bouncebuf, flags); 1843 if (error) { 1844 _bus_dmamem_free(t, cookie->id_bouncesegs, 1845 cookie->id_nbouncesegs); 1846 cookie->id_bouncebuflen = 0; 1847 cookie->id_nbouncesegs = 0; 1848 } else { 1849 cookie->id_flags |= _BUS_DMA_HAS_BOUNCE; 1850 } 1851 } else { 1852 cookie->id_bouncebuflen = 0; 1853 cookie->id_nbouncesegs = 0; 1854 } 1855 1856 return error; 1857 } 1858 1859 static void 1860 _bus_dma_free_bouncebuf(bus_dma_tag_t t, bus_dmamap_t map) 1861 { 1862 struct arm32_bus_dma_cookie *cookie = map->_dm_cookie; 1863 1864 KASSERT(cookie != NULL); 1865 1866 _bus_dmamem_unmap(t, cookie->id_bouncebuf, cookie->id_bouncebuflen); 1867 _bus_dmamem_free(t, cookie->id_bouncesegs, cookie->id_nbouncesegs); 1868 cookie->id_bouncebuflen = 0; 1869 cookie->id_nbouncesegs = 0; 1870 cookie->id_flags &= ~_BUS_DMA_HAS_BOUNCE; 1871 } 1872 #endif /* _ARM32_NEED_BUS_DMA_BOUNCE */ 1873 1874 /* 1875 * This function does the same as uiomove, but takes an explicit 1876 * direction, and does not update the uio structure. 1877 */ 1878 static int 1879 _bus_dma_uiomove(void *buf, struct uio *uio, size_t n, int direction) 1880 { 1881 struct iovec *iov; 1882 int error; 1883 struct vmspace *vm; 1884 char *cp; 1885 size_t resid, cnt; 1886 int i; 1887 1888 iov = uio->uio_iov; 1889 vm = uio->uio_vmspace; 1890 cp = buf; 1891 resid = n; 1892 1893 for (i = 0; i < uio->uio_iovcnt && resid > 0; i++) { 1894 iov = &uio->uio_iov[i]; 1895 if (iov->iov_len == 0) 1896 continue; 1897 cnt = MIN(resid, iov->iov_len); 1898 1899 if (!VMSPACE_IS_KERNEL_P(vm)) { 1900 preempt_point(); 1901 } 1902 if (direction == UIO_READ) { 1903 error = copyout_vmspace(vm, cp, iov->iov_base, cnt); 1904 } else { 1905 error = copyin_vmspace(vm, iov->iov_base, cp, cnt); 1906 } 1907 if (error) 1908 return error; 1909 cp += cnt; 1910 resid -= cnt; 1911 } 1912 return 0; 1913 } 1914 1915 int 1916 _bus_dmatag_subregion(bus_dma_tag_t tag, bus_addr_t min_addr, 1917 bus_addr_t max_addr, bus_dma_tag_t *newtag, int flags) 1918 { 1919 #ifdef _ARM32_NEED_BUS_DMA_BOUNCE 1920 if (min_addr >= max_addr) 1921 return EOPNOTSUPP; 1922 1923 struct arm32_dma_range *dr; 1924 bool psubset = true; 1925 size_t nranges = 0; 1926 size_t i; 1927 for (i = 0, dr = tag->_ranges; i < tag->_nranges; i++, dr++) { 1928 /* 1929 * If the new {min,max}_addr are narrower than any of the 1930 * ranges in the parent tag then we need a new tag; 1931 * otherwise the parent tag is a subset of the new 1932 * range and can continue to be used. 1933 */ 1934 if (min_addr > dr->dr_sysbase 1935 || max_addr < dr->dr_sysbase + dr->dr_len - 1) { 1936 psubset = false; 1937 } 1938 if (min_addr <= dr->dr_sysbase + dr->dr_len 1939 && max_addr >= dr->dr_sysbase) { 1940 nranges++; 1941 } 1942 } 1943 if (nranges == 0) { 1944 nranges = 1; 1945 psubset = false; 1946 } 1947 if (psubset) { 1948 *newtag = tag; 1949 /* if the tag must be freed, add a reference */ 1950 if (tag->_tag_needs_free) 1951 (tag->_tag_needs_free)++; 1952 return 0; 1953 } 1954 1955 const size_t tagsize = sizeof(*tag) + nranges * sizeof(*dr); 1956 if ((*newtag = kmem_intr_zalloc(tagsize, 1957 (flags & BUS_DMA_NOWAIT) ? KM_NOSLEEP : KM_SLEEP)) == NULL) 1958 return ENOMEM; 1959 1960 dr = (void *)(*newtag + 1); 1961 **newtag = *tag; 1962 (*newtag)->_tag_needs_free = 1; 1963 (*newtag)->_ranges = dr; 1964 (*newtag)->_nranges = nranges; 1965 1966 if (tag->_ranges == NULL) { 1967 dr->dr_sysbase = min_addr; 1968 dr->dr_busbase = min_addr; 1969 dr->dr_len = max_addr + 1 - min_addr; 1970 } else { 1971 struct arm32_dma_range *pdr; 1972 1973 for (i = 0, pdr = tag->_ranges; i < tag->_nranges; i++, pdr++) { 1974 KASSERT(nranges != 0); 1975 1976 if (min_addr > pdr->dr_sysbase + pdr->dr_len 1977 || max_addr < pdr->dr_sysbase) { 1978 /* 1979 * this range doesn't overlap with new limits, 1980 * so skip. 1981 */ 1982 continue; 1983 } 1984 /* 1985 * Copy the range and adjust to fit within the new 1986 * limits 1987 */ 1988 dr[0] = pdr[0]; 1989 if (dr->dr_sysbase < min_addr) { 1990 psize_t diff = min_addr - dr->dr_sysbase; 1991 dr->dr_busbase += diff; 1992 dr->dr_len -= diff; 1993 dr->dr_sysbase += diff; 1994 } 1995 if (max_addr <= dr->dr_sysbase + dr->dr_len - 1) { 1996 dr->dr_len = max_addr + 1 - dr->dr_sysbase; 1997 } 1998 dr++; 1999 nranges--; 2000 } 2001 } 2002 2003 return 0; 2004 #else 2005 return EOPNOTSUPP; 2006 #endif /* _ARM32_NEED_BUS_DMA_BOUNCE */ 2007 } 2008 2009 void 2010 _bus_dmatag_destroy(bus_dma_tag_t tag) 2011 { 2012 #ifdef _ARM32_NEED_BUS_DMA_BOUNCE 2013 switch (tag->_tag_needs_free) { 2014 case 0: 2015 break; /* not allocated with kmem */ 2016 case 1: { 2017 const size_t tagsize = sizeof(*tag) 2018 + tag->_nranges * sizeof(*tag->_ranges); 2019 kmem_intr_free(tag, tagsize); /* last reference to tag */ 2020 break; 2021 } 2022 default: 2023 (tag->_tag_needs_free)--; /* one less reference */ 2024 } 2025 #endif 2026 } 2027
Indexes created Tue Sep 30 17:09:57 GMT 2025