1 /* $NetBSD: bus_dma.c,v 1.56 2023/12/15 09:43:59 rin Exp $ */ 2 3 /*- 4 * Copyright (c) 1996, 1997, 1998 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility, 9 * NASA Ames Research Center. 10 * 11 * Redistribution and use in source and binary forms, with or without 12 * modification, are permitted provided that the following conditions 13 * are met: 14 * 1. Redistributions of source code must retain the above copyright 15 * notice, this list of conditions and the following disclaimer. 16 * 2. Redistributions in binary form must reproduce the above copyright 17 * notice, this list of conditions and the following disclaimer in the 18 * documentation and/or other materials provided with the distribution. 19 * 20 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 21 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 22 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 23 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 24 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 25 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 26 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 27 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 28 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 29 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 30 * POSSIBILITY OF SUCH DAMAGE. 31 */ 32 33 #define _POWERPC_BUS_DMA_PRIVATE 34 35 #include <sys/cdefs.h> 36 __KERNEL_RCSID(0, "$NetBSD: bus_dma.c,v 1.56 2023/12/15 09:43:59 rin Exp $"); 37 38 #ifdef _KERNEL_OPT 39 #include "opt_ppcarch.h" 40 #endif 41 42 #include <sys/param.h> 43 #include <sys/systm.h> 44 #include <sys/kernel.h> 45 #include <sys/device.h> 46 #include <sys/kmem.h> 47 #include <sys/proc.h> 48 #include <sys/mbuf.h> 49 #include <sys/bus.h> 50 #include <sys/intr.h> 51 52 #include <uvm/uvm.h> 53 #include <uvm/uvm_physseg.h> 54 55 #if defined(PPC_BOOKE) 56 #define EIEIO __asm volatile("mbar\t0" ::: "memory") 57 #define SYNC __asm volatile("msync" ::: "memory") 58 #elif defined(PPC_IBM4XX) && !defined(PPC_IBM440) 59 /* eieio is implemented as sync */ 60 #define EIEIO __asm volatile("eieio" ::: "memory") 61 #define SYNC /* nothing */ 62 #else 63 #define EIEIO __asm volatile("eieio" ::: "memory") 64 #define SYNC __asm volatile("sync" ::: "memory") 65 #endif 66 67 int _bus_dmamap_load_buffer (bus_dma_tag_t, bus_dmamap_t, void *, 68 bus_size_t, struct vmspace *, int, paddr_t *, int *, int); 69 70 static inline void 71 dcbst(paddr_t pa, long len, int dcache_line_size) 72 { 73 paddr_t epa; 74 for (epa = pa + len; pa < epa; pa += dcache_line_size) 75 __asm volatile("dcbst 0,%0" :: "r"(pa) : "memory"); 76 } 77 78 static inline void 79 dcbi(paddr_t pa, long len, int dcache_line_size) 80 { 81 paddr_t epa; 82 for (epa = pa + len; pa < epa; pa += dcache_line_size) 83 __asm volatile("dcbi 0,%0" :: "r"(pa) : "memory"); 84 } 85 86 static inline void 87 dcbf(paddr_t pa, long len, int dcache_line_size) 88 { 89 paddr_t epa; 90 for (epa = pa + len; pa < epa; pa += dcache_line_size) 91 __asm volatile("dcbf 0,%0" :: "r"(pa) : "memory"); 92 } 93 94 /* 95 * Common function for DMA map creation. May be called by bus-specific 96 * DMA map creation functions. 97 */ 98 int 99 _bus_dmamap_create(bus_dma_tag_t t, bus_size_t size, int nsegments, bus_size_t maxsegsz, bus_size_t boundary, int flags, bus_dmamap_t *dmamp) 100 { 101 struct powerpc_bus_dmamap *map; 102 void *mapstore; 103 size_t mapsize; 104 105 /* 106 * Allocate and initialize the DMA map. The end of the map 107 * is a variable-sized array of segments, so we allocate enough 108 * room for them in one shot. 109 * 110 * Note we don't preserve the WAITOK or NOWAIT flags. Preservation 111 * of ALLOCNOW notifies others that we've reserved these resources, 112 * and they are not to be freed. 113 * 114 * The bus_dmamap_t includes one bus_dma_segment_t, hence 115 * the (nsegments - 1). 116 */ 117 mapsize = sizeof(*map) + sizeof(bus_dma_segment_t [nsegments - 1]); 118 if ((mapstore = kmem_intr_alloc(mapsize, 119 (flags & BUS_DMA_NOWAIT) ? KM_NOSLEEP : KM_SLEEP)) == NULL) 120 return (ENOMEM); 121 122 memset(mapstore, 0, mapsize); 123 map = (struct powerpc_bus_dmamap *)mapstore; 124 map->_dm_size = size; 125 map->_dm_segcnt = nsegments; 126 map->_dm_maxmaxsegsz = maxsegsz; 127 map->_dm_boundary = boundary; 128 map->_dm_bounce_thresh = t->_bounce_thresh; 129 map->_dm_flags = flags & ~(BUS_DMA_WAITOK|BUS_DMA_NOWAIT); 130 map->dm_maxsegsz = maxsegsz; 131 map->dm_mapsize = 0; /* no valid mappings */ 132 map->dm_nsegs = 0; 133 134 *dmamp = map; 135 return (0); 136 } 137 138 /* 139 * Common function for DMA map destruction. May be called by bus-specific 140 * DMA map destruction functions. 141 */ 142 void 143 _bus_dmamap_destroy(bus_dma_tag_t t, bus_dmamap_t map) 144 { 145 146 size_t mapsize = sizeof(*map) 147 + sizeof(bus_dma_segment_t [map->_dm_segcnt - 1]); 148 kmem_intr_free(map, mapsize); 149 } 150 151 /* 152 * Utility function to load a linear buffer. lastaddrp holds state 153 * between invocations (for multiple-buffer loads). segp contains 154 * the starting segment on entrance, and the ending segment on exit. 155 * first indicates if this is the first invocation of this function. 156 */ 157 int 158 _bus_dmamap_load_buffer(bus_dma_tag_t t, bus_dmamap_t map, void *buf, bus_size_t buflen, struct vmspace *vm, int flags, paddr_t *lastaddrp, int *segp, int first) 159 { 160 bus_size_t sgsize; 161 bus_addr_t curaddr, lastaddr, baddr, bmask; 162 vaddr_t vaddr = (vaddr_t)buf; 163 int seg; 164 165 // printf("%s(%p,%p,%p,%u,%p,%#x,%p,%p,%u)\n", __func__, 166 // t, map, buf, buflen, vm, flags, lastaddrp, segp, first); 167 168 lastaddr = *lastaddrp; 169 bmask = ~(map->_dm_boundary - 1); 170 171 for (seg = *segp; buflen > 0 ; ) { 172 /* 173 * Get the physical address for this segment. 174 */ 175 if (!VMSPACE_IS_KERNEL_P(vm)) 176 (void) pmap_extract(vm_map_pmap(&vm->vm_map), 177 vaddr, (void *)&curaddr); 178 else 179 curaddr = vtophys(vaddr); 180 181 /* 182 * If we're beyond the bounce threshold, notify 183 * the caller. 184 */ 185 if (map->_dm_bounce_thresh != 0 && 186 curaddr >= map->_dm_bounce_thresh) 187 return (EINVAL); 188 189 /* 190 * Compute the segment size, and adjust counts. 191 */ 192 sgsize = PAGE_SIZE - ((u_long)vaddr & PGOFSET); 193 if (buflen < sgsize) 194 sgsize = buflen; 195 sgsize = uimin(sgsize, map->dm_maxsegsz); 196 197 /* 198 * Make sure we don't cross any boundaries. 199 */ 200 if (map->_dm_boundary > 0) { 201 baddr = (curaddr + map->_dm_boundary) & bmask; 202 if (sgsize > (baddr - curaddr)) 203 sgsize = (baddr - curaddr); 204 } 205 206 /* 207 * Insert chunk into a segment, coalescing with 208 * the previous segment if possible. 209 */ 210 if (first) { 211 map->dm_segs[seg].ds_addr = PHYS_TO_BUS_MEM(t, curaddr); 212 map->dm_segs[seg].ds_len = sgsize; 213 first = 0; 214 } else { 215 if (curaddr == lastaddr && 216 (map->dm_segs[seg].ds_len + sgsize) <= 217 map->dm_maxsegsz && 218 (map->_dm_boundary == 0 || 219 (map->dm_segs[seg].ds_addr & bmask) == 220 (PHYS_TO_BUS_MEM(t, curaddr) & bmask))) 221 map->dm_segs[seg].ds_len += sgsize; 222 else { 223 if (++seg >= map->_dm_segcnt) 224 break; 225 map->dm_segs[seg].ds_addr = 226 PHYS_TO_BUS_MEM(t, curaddr); 227 map->dm_segs[seg].ds_len = sgsize; 228 } 229 } 230 231 lastaddr = curaddr + sgsize; 232 vaddr += sgsize; 233 buflen -= sgsize; 234 } 235 236 *segp = seg; 237 *lastaddrp = lastaddr; 238 239 /* 240 * Did we fit? 241 */ 242 if (buflen != 0) 243 return (EFBIG); /* XXX better return value here? */ 244 245 return (0); 246 } 247 248 /* 249 * Common function for loading a DMA map with a linear buffer. May 250 * be called by bus-specific DMA map load functions. 251 */ 252 int 253 _bus_dmamap_load(bus_dma_tag_t t, bus_dmamap_t map, void *buf, bus_size_t buflen, struct proc *p, int flags) 254 { 255 paddr_t lastaddr = 0; 256 int seg, error; 257 struct vmspace *vm; 258 259 /* 260 * Make sure that on error condition we return "no valid mappings". 261 */ 262 map->dm_mapsize = 0; 263 map->dm_nsegs = 0; 264 KASSERT(map->dm_maxsegsz <= map->_dm_maxmaxsegsz); 265 266 if (buflen > map->_dm_size) 267 return (EINVAL); 268 269 if (p != NULL) { 270 vm = p->p_vmspace; 271 } else { 272 vm = vmspace_kernel(); 273 } 274 275 seg = 0; 276 error = _bus_dmamap_load_buffer(t, map, buf, buflen, vm, flags, 277 &lastaddr, &seg, 1); 278 if (error == 0) { 279 map->dm_mapsize = buflen; 280 map->dm_nsegs = seg + 1; 281 } 282 return (error); 283 } 284 285 /* 286 * Like _bus_dmamap_load(), but for mbufs. 287 */ 288 int 289 _bus_dmamap_load_mbuf(bus_dma_tag_t t, bus_dmamap_t map, struct mbuf *m0, int flags) 290 { 291 paddr_t lastaddr = 0; 292 int seg, error, first; 293 struct mbuf *m; 294 295 /* 296 * Make sure that on error condition we return "no valid mappings." 297 */ 298 map->dm_mapsize = 0; 299 map->dm_nsegs = 0; 300 KASSERT(map->dm_maxsegsz <= map->_dm_maxmaxsegsz); 301 302 #ifdef DIAGNOSTIC 303 if ((m0->m_flags & M_PKTHDR) == 0) 304 panic("_bus_dmamap_load_mbuf: no packet header"); 305 #endif 306 307 if (m0->m_pkthdr.len > map->_dm_size) 308 return (EINVAL); 309 310 first = 1; 311 seg = 0; 312 error = 0; 313 for (m = m0; m != NULL && error == 0; m = m->m_next, first = 0) { 314 if (m->m_len == 0) 315 continue; 316 #ifdef POOL_VTOPHYS 317 /* XXX Could be better about coalescing. */ 318 /* XXX Doesn't check boundaries. */ 319 switch (m->m_flags & (M_EXT|M_EXT_CLUSTER)) { 320 case M_EXT|M_EXT_CLUSTER: 321 /* XXX KDASSERT */ 322 KASSERT(m->m_ext.ext_paddr != M_PADDR_INVALID); 323 lastaddr = m->m_ext.ext_paddr + 324 (m->m_data - m->m_ext.ext_buf); 325 have_addr: 326 if (first == 0 && ++seg >= map->_dm_segcnt) { 327 error = EFBIG; 328 continue; 329 } 330 map->dm_segs[seg].ds_addr = 331 PHYS_TO_BUS_MEM(t, lastaddr); 332 map->dm_segs[seg].ds_len = m->m_len; 333 lastaddr += m->m_len; 334 continue; 335 336 case 0: 337 lastaddr = m->m_paddr + M_BUFOFFSET(m) + 338 (m->m_data - M_BUFADDR(m)); 339 goto have_addr; 340 341 default: 342 break; 343 } 344 #endif 345 error = _bus_dmamap_load_buffer(t, map, m->m_data, 346 m->m_len, vmspace_kernel(), flags, &lastaddr, &seg, first); 347 } 348 if (error == 0) { 349 map->dm_mapsize = m0->m_pkthdr.len; 350 map->dm_nsegs = seg + 1; 351 } 352 return (error); 353 } 354 355 /* 356 * Like _bus_dmamap_load(), but for uios. 357 */ 358 int 359 _bus_dmamap_load_uio(bus_dma_tag_t t, bus_dmamap_t map, struct uio *uio, int flags) 360 { 361 paddr_t lastaddr = 0; 362 int seg, i, error, first; 363 bus_size_t minlen, resid; 364 struct iovec *iov; 365 void *addr; 366 367 /* 368 * Make sure that on error condition we return "no valid mappings." 369 */ 370 map->dm_mapsize = 0; 371 map->dm_nsegs = 0; 372 KASSERT(map->dm_maxsegsz <= map->_dm_maxmaxsegsz); 373 374 resid = uio->uio_resid; 375 iov = uio->uio_iov; 376 377 first = 1; 378 seg = 0; 379 error = 0; 380 for (i = 0; i < uio->uio_iovcnt && resid != 0 && error == 0; i++) { 381 /* 382 * Now at the first iovec to load. Load each iovec 383 * until we have exhausted the residual count. 384 */ 385 minlen = resid < iov[i].iov_len ? resid : iov[i].iov_len; 386 addr = (void *)iov[i].iov_base; 387 388 error = _bus_dmamap_load_buffer(t, map, addr, minlen, 389 uio->uio_vmspace, flags, &lastaddr, &seg, first); 390 first = 0; 391 392 resid -= minlen; 393 } 394 if (error == 0) { 395 map->dm_mapsize = uio->uio_resid; 396 map->dm_nsegs = seg + 1; 397 } 398 return (error); 399 } 400 401 /* 402 * Like _bus_dmamap_load(), but for raw memory allocated with 403 * bus_dmamem_alloc(). 404 * 405 * XXX This is too much copypasta of _bus_dmamap_load_buffer. 406 */ 407 int 408 _bus_dmamap_load_raw(bus_dma_tag_t t, bus_dmamap_t map, 409 bus_dma_segment_t *segs, int nsegs, bus_size_t size, int flags) 410 { 411 bus_size_t sgsize, isgsize; 412 bus_size_t busaddr, curaddr, lastaddr, baddr, bmask; 413 int seg, iseg, first; 414 415 if (size == 0) 416 return 0; 417 418 lastaddr = 0; 419 bmask = ~(map->_dm_boundary - 1); 420 421 first = 0; 422 iseg = 0; 423 busaddr = segs[iseg].ds_addr; 424 isgsize = segs[iseg].ds_len; 425 for (seg = 0; size > 0;) { 426 /* 427 * Get the physical address for this segment. 428 */ 429 curaddr = BUS_MEM_TO_PHYS(t, busaddr); 430 431 /* 432 * If we're beyond the bounce threshold, notify 433 * the caller. 434 */ 435 if (map->_dm_bounce_thresh != 0 && 436 curaddr >= map->_dm_bounce_thresh) 437 return EINVAL; 438 439 /* 440 * Compute the segment size, and adjust counts. 441 */ 442 sgsize = PAGE_SIZE - ((u_long)curaddr & PGOFSET); 443 sgsize = MIN(sgsize, isgsize); 444 sgsize = MIN(sgsize, size); 445 sgsize = MIN(sgsize, map->dm_maxsegsz); 446 447 /* 448 * Make sure we don't cross any boundaries. 449 */ 450 if (map->_dm_boundary > 0) { 451 baddr = (curaddr + map->_dm_boundary) & bmask; 452 if (sgsize > (baddr - curaddr)) 453 sgsize = (baddr - curaddr); 454 } 455 456 /* 457 * Insert chunk into a segment, coalescing with 458 * the previous segment if possible. 459 */ 460 if (first) { 461 map->dm_segs[seg].ds_addr = 462 PHYS_TO_BUS_MEM(t, curaddr); 463 map->dm_segs[seg].ds_len = sgsize; 464 first = 0; 465 } else { 466 if (curaddr == lastaddr && 467 (map->dm_segs[seg].ds_len + sgsize) <= 468 map->dm_maxsegsz && 469 (map->_dm_boundary == 0 || 470 (map->dm_segs[seg].ds_addr & bmask) == 471 (PHYS_TO_BUS_MEM(t, curaddr) & bmask))) 472 map->dm_segs[seg].ds_len += sgsize; 473 else { 474 if (++seg >= map->_dm_segcnt) 475 break; 476 map->dm_segs[seg].ds_addr = 477 PHYS_TO_BUS_MEM(t, curaddr); 478 map->dm_segs[seg].ds_len = sgsize; 479 } 480 } 481 482 lastaddr = curaddr + sgsize; 483 size -= sgsize; 484 if ((isgsize -= sgsize) == 0) { 485 iseg++; 486 KASSERT(iseg < nsegs); 487 busaddr = segs[iseg].ds_addr; 488 isgsize = segs[iseg].ds_len; 489 } 490 } 491 492 if (size > 0) 493 return EFBIG; 494 495 return 0; 496 } 497 498 /* 499 * Common function for unloading a DMA map. May be called by 500 * chipset-specific DMA map unload functions. 501 */ 502 void 503 _bus_dmamap_unload(bus_dma_tag_t t, bus_dmamap_t map) 504 { 505 506 /* 507 * No resources to free; just mark the mappings as 508 * invalid. 509 */ 510 map->dm_maxsegsz = map->_dm_maxmaxsegsz; 511 map->dm_mapsize = 0; 512 map->dm_nsegs = 0; 513 } 514 515 /* 516 * Common function for DMA map synchronization. May be called 517 * by chipset-specific DMA map synchronization functions. 518 */ 519 void 520 _bus_dmamap_sync(bus_dma_tag_t t, bus_dmamap_t map, bus_addr_t offset, bus_size_t len, int ops) 521 { 522 const int dcache_line_size = curcpu()->ci_ci.dcache_line_size; 523 const bus_dma_segment_t *ds = map->dm_segs; 524 525 // printf("%s(%p,%p,%#x,%u,%#x) from %p\n", __func__, 526 // t, map, offset, len, ops, __builtin_return_address(0)); 527 528 if ((ops & (BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE)) != 0 && 529 (ops & (BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE)) != 0) 530 panic("_bus_dmamap_sync: invalid ops %#x", ops); 531 532 #ifdef DIAGNOSTIC 533 if (offset + len > map->dm_mapsize) 534 panic("%s: ops %#x mapsize %u: bad offset (%u) and/or length (%u)", __func__, ops, map->dm_mapsize, offset, len); 535 #endif 536 537 /* 538 * Skip leading amount 539 */ 540 while (offset >= ds->ds_len) { 541 offset -= ds->ds_len; 542 ds++; 543 } 544 EIEIO; 545 for (; len > 0; ds++, offset = 0) { 546 bus_size_t seglen = ds->ds_len - offset; 547 bus_addr_t addr = BUS_MEM_TO_PHYS(t, ds->ds_addr) + offset; 548 if (seglen > len) 549 seglen = len; 550 len -= seglen; 551 KASSERT(ds < &map->dm_segs[map->dm_nsegs]); 552 /* 553 * Readjust things to start on cacheline boundarys 554 */ 555 offset = (addr & (dcache_line_size-1)); 556 seglen += offset; 557 addr -= offset; 558 /* 559 * Now do the appropriate thing. 560 */ 561 switch (ops) { 562 case BUS_DMASYNC_PREWRITE: 563 /* 564 * Make sure cache contents are in memory for the DMA. 565 */ 566 dcbst(addr, seglen, dcache_line_size); 567 break; 568 case BUS_DMASYNC_PREREAD: 569 /* 570 * If the region to be invalidated doesn't fall on 571 * cacheline boundary, flush that cacheline so we 572 * preserve the leading content. 573 */ 574 if (offset) { 575 dcbf(addr, 1, 1); 576 /* 577 * If we are doing <= one cache line, stop now. 578 */ 579 if (seglen <= dcache_line_size) 580 break; 581 /* 582 * Advance one cache line since we've flushed 583 * this one. 584 */ 585 addr += dcache_line_size; 586 seglen -= dcache_line_size; 587 } 588 /* 589 * If the byte after the region to be invalidated 590 * doesn't fall on cacheline boundary, flush that 591 * cacheline so we preserve the trailing content. 592 */ 593 if (seglen & (dcache_line_size-1)) { 594 dcbf(addr + seglen, 1, 1); 595 if (seglen <= dcache_line_size) 596 break; 597 /* 598 * Truncate the length to a multiple of a 599 * dcache line size. No reason to flush 600 * the last entry again. 601 */ 602 seglen &= ~(dcache_line_size - 1); 603 } 604 SYNC; /* is this needed? */ 605 EIEIO; /* is this needed? */ 606 /* FALLTHROUGH */ 607 case BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE: 608 case BUS_DMASYNC_POSTREAD: 609 /* 610 * The contents will have changed, make sure to remove 611 * them from the cache. Note: some implementation 612 * implement dcbi identically to dcbf. Thus if the 613 * cacheline has data, it will be written to memory. 614 * If the DMA is updating the same cacheline at the 615 * time, bad things can happen. 616 */ 617 dcbi(addr, seglen, dcache_line_size); 618 break; 619 case BUS_DMASYNC_POSTWRITE: 620 /* 621 * Do nothing. 622 */ 623 break; 624 case BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE: 625 /* 626 * Force it to memory and remove from cache. 627 */ 628 dcbf(addr, seglen, dcache_line_size); 629 break; 630 } 631 } 632 __asm volatile("sync"); 633 } 634 635 /* 636 * Common function for DMA-safe memory allocation. May be called 637 * by bus-specific DMA memory allocation functions. 638 */ 639 int 640 _bus_dmamem_alloc(bus_dma_tag_t t, bus_size_t size, bus_size_t alignment, bus_size_t boundary, bus_dma_segment_t *segs, int nsegs, int *rsegs, int flags) 641 { 642 paddr_t start = 0xffffffff, end = 0; 643 uvm_physseg_t bank; 644 645 for (bank = uvm_physseg_get_first(); 646 uvm_physseg_valid_p(bank); 647 bank = uvm_physseg_get_next(bank)) { 648 if (start > ptoa(uvm_physseg_get_avail_start(bank))) 649 start = ptoa(uvm_physseg_get_avail_start(bank)); 650 if (end < ptoa(uvm_physseg_get_avail_end(bank))) 651 end = ptoa(uvm_physseg_get_avail_end(bank)); 652 } 653 654 return _bus_dmamem_alloc_range(t, size, alignment, boundary, segs, 655 nsegs, rsegs, flags, start, end - PAGE_SIZE); 656 } 657 658 /* 659 * Common function for freeing DMA-safe memory. May be called by 660 * bus-specific DMA memory free functions. 661 */ 662 void 663 _bus_dmamem_free(bus_dma_tag_t t, bus_dma_segment_t *segs, int nsegs) 664 { 665 struct vm_page *m; 666 bus_addr_t addr; 667 struct pglist mlist; 668 int curseg; 669 670 /* 671 * Build a list of pages to free back to the VM system. 672 */ 673 TAILQ_INIT(&mlist); 674 for (curseg = 0; curseg < nsegs; curseg++) { 675 for (addr = BUS_MEM_TO_PHYS(t, segs[curseg].ds_addr); 676 addr < (BUS_MEM_TO_PHYS(t, segs[curseg].ds_addr) 677 + segs[curseg].ds_len); 678 addr += PAGE_SIZE) { 679 m = PHYS_TO_VM_PAGE(addr); 680 TAILQ_INSERT_TAIL(&mlist, m, pageq.queue); 681 } 682 } 683 684 uvm_pglistfree(&mlist); 685 } 686 687 /* 688 * Common function for mapping DMA-safe memory. May be called by 689 * bus-specific DMA memory map functions. 690 */ 691 int 692 _bus_dmamem_map(bus_dma_tag_t t, bus_dma_segment_t *segs, int nsegs, size_t size, void **kvap, int flags) 693 { 694 vaddr_t va; 695 bus_addr_t addr; 696 int curseg; 697 const uvm_flag_t kmflags = 698 (flags & BUS_DMA_NOWAIT) != 0 ? UVM_KMF_NOWAIT : 0; 699 700 size = round_page(size); 701 702 #ifdef PMAP_MAP_POOLPAGE 703 /* 704 * If we are mapping a cacheable physically contiguous segment, treat 705 * it as if we are mapping a poolpage and avoid consuming any KVAs. 706 */ 707 if (nsegs == 1 && (flags & BUS_DMA_DONTCACHE) == 0) { 708 KASSERT(size == segs->ds_len); 709 addr = BUS_MEM_TO_PHYS(t, segs->ds_addr); 710 if (__predict_true(addr + size < PMAP_DIRECT_MAPPED_LEN)) { 711 *kvap = (void *)PMAP_MAP_POOLPAGE(addr); 712 return 0; 713 } 714 } 715 #endif 716 717 va = uvm_km_alloc(kernel_map, size, 0, UVM_KMF_VAONLY | kmflags); 718 719 if (va == 0) 720 return (ENOMEM); 721 722 *kvap = (void *)va; 723 724 for (curseg = 0; curseg < nsegs; curseg++) { 725 for (addr = BUS_MEM_TO_PHYS(t, segs[curseg].ds_addr); 726 addr < (BUS_MEM_TO_PHYS(t, segs[curseg].ds_addr) 727 + segs[curseg].ds_len); 728 addr += PAGE_SIZE, va += PAGE_SIZE, size -= PAGE_SIZE) { 729 if (size == 0) 730 panic("_bus_dmamem_map: size botch"); 731 /* 732 * If we are mapping nocache, flush the page from 733 * cache before we map it. 734 */ 735 if (flags & BUS_DMA_DONTCACHE) 736 dcbf(addr, PAGE_SIZE, 737 curcpu()->ci_ci.dcache_line_size); 738 pmap_kenter_pa(va, addr, 739 VM_PROT_READ | VM_PROT_WRITE, 740 PMAP_WIRED | 741 ((flags & BUS_DMA_DONTCACHE) ? PMAP_NOCACHE : 0)); 742 } 743 } 744 745 return (0); 746 } 747 748 /* 749 * Common function for unmapping DMA-safe memory. May be called by 750 * bus-specific DMA memory unmapping functions. 751 */ 752 void 753 _bus_dmamem_unmap(bus_dma_tag_t t, void *kva, size_t size) 754 { 755 vaddr_t va = (vaddr_t) kva; 756 757 #ifdef DIAGNOSTIC 758 if (va & PGOFSET) 759 panic("_bus_dmamem_unmap"); 760 #endif 761 762 if (va >= VM_MIN_KERNEL_ADDRESS && va < VM_MAX_KERNEL_ADDRESS) { 763 size = round_page(size); 764 pmap_kremove(va, size); 765 uvm_km_free(kernel_map, va, size, UVM_KMF_VAONLY); 766 } 767 } 768 769 /* 770 * Common function for mmap(2)'ing DMA-safe memory. May be called by 771 * bus-specific DMA mmap(2)'ing functions. 772 */ 773 paddr_t 774 _bus_dmamem_mmap(bus_dma_tag_t t, bus_dma_segment_t *segs, int nsegs, off_t off, int prot, int flags) 775 { 776 int i; 777 778 for (i = 0; i < nsegs; i++) { 779 #ifdef DIAGNOSTIC 780 if (off & PGOFSET) 781 panic("_bus_dmamem_mmap: offset unaligned"); 782 if (BUS_MEM_TO_PHYS(t, segs[i].ds_addr) & PGOFSET) 783 panic("_bus_dmamem_mmap: segment unaligned"); 784 if (segs[i].ds_len & PGOFSET) 785 panic("_bus_dmamem_mmap: segment size not multiple" 786 " of page size"); 787 #endif 788 if (off >= segs[i].ds_len) { 789 off -= segs[i].ds_len; 790 continue; 791 } 792 793 return (BUS_MEM_TO_PHYS(t, segs[i].ds_addr) + off); 794 } 795 796 /* Page not found. */ 797 return (-1); 798 } 799 800 /* 801 * Allocate physical memory from the given physical address range. 802 * Called by DMA-safe memory allocation methods. 803 */ 804 int 805 _bus_dmamem_alloc_range( 806 bus_dma_tag_t t, 807 bus_size_t size, 808 bus_size_t alignment, 809 bus_size_t boundary, 810 bus_dma_segment_t *segs, 811 int nsegs, 812 int *rsegs, 813 int flags, 814 paddr_t low, 815 paddr_t high) 816 { 817 paddr_t curaddr, lastaddr; 818 struct vm_page *m; 819 struct pglist mlist; 820 int curseg, error; 821 822 /* Always round the size. */ 823 size = round_page(size); 824 825 /* 826 * Allocate pages from the VM system. 827 */ 828 error = uvm_pglistalloc(size, low, high, alignment, boundary, 829 &mlist, nsegs, (flags & BUS_DMA_NOWAIT) == 0); 830 if (error) 831 return (error); 832 833 /* 834 * Compute the location, size, and number of segments actually 835 * returned by the VM code. 836 */ 837 m = mlist.tqh_first; 838 curseg = 0; 839 lastaddr = VM_PAGE_TO_PHYS(m); 840 segs[curseg].ds_addr = PHYS_TO_BUS_MEM(t, lastaddr); 841 segs[curseg].ds_len = PAGE_SIZE; 842 m = m->pageq.queue.tqe_next; 843 844 for (; m != NULL; m = m->pageq.queue.tqe_next) { 845 curaddr = VM_PAGE_TO_PHYS(m); 846 #ifdef DIAGNOSTIC 847 if (curaddr < low || curaddr >= high) { 848 printf("vm_page_alloc_memory returned non-sensical" 849 " address 0x%lx\n", curaddr); 850 panic("_bus_dmamem_alloc_range"); 851 } 852 #endif 853 if (curaddr == (lastaddr + PAGE_SIZE)) 854 segs[curseg].ds_len += PAGE_SIZE; 855 else { 856 curseg++; 857 segs[curseg].ds_addr = PHYS_TO_BUS_MEM(t, curaddr); 858 segs[curseg].ds_len = PAGE_SIZE; 859 } 860 lastaddr = curaddr; 861 } 862 863 *rsegs = curseg + 1; 864 865 return (0); 866 } 867 868 /* 869 * Generic form of PHYS_TO_BUS_MEM(). 870 */ 871 bus_addr_t 872 _bus_dma_phys_to_bus_mem_generic(bus_dma_tag_t t, bus_addr_t addr) 873 { 874 875 return (addr); 876 } 877 878 /* 879 * Generic form of BUS_MEM_TO_PHYS(). 880 */ 881 bus_addr_t 882 _bus_dma_bus_mem_to_phys_generic(bus_dma_tag_t t, bus_addr_t addr) 883 { 884 885 return (addr); 886 } 887