Home | History | Annotate | Line # | Download | only in powerpc
      1 /*	$NetBSD: bus_dma.c,v 1.56 2023/12/15 09:43:59 rin Exp $	*/
      2 
      3 /*-
      4  * Copyright (c) 1996, 1997, 1998 The NetBSD Foundation, Inc.
      5  * All rights reserved.
      6  *
      7  * This code is derived from software contributed to The NetBSD Foundation
      8  * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility,
      9  * NASA Ames Research Center.
     10  *
     11  * Redistribution and use in source and binary forms, with or without
     12  * modification, are permitted provided that the following conditions
     13  * are met:
     14  * 1. Redistributions of source code must retain the above copyright
     15  *    notice, this list of conditions and the following disclaimer.
     16  * 2. Redistributions in binary form must reproduce the above copyright
     17  *    notice, this list of conditions and the following disclaimer in the
     18  *    documentation and/or other materials provided with the distribution.
     19  *
     20  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
     21  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
     22  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
     23  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
     24  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
     25  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
     26  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
     27  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
     28  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
     29  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
     30  * POSSIBILITY OF SUCH DAMAGE.
     31  */
     32 
     33 #define _POWERPC_BUS_DMA_PRIVATE
     34 
     35 #include <sys/cdefs.h>
     36 __KERNEL_RCSID(0, "$NetBSD: bus_dma.c,v 1.56 2023/12/15 09:43:59 rin Exp $");
     37 
     38 #ifdef _KERNEL_OPT
     39 #include "opt_ppcarch.h"
     40 #endif
     41 
     42 #include <sys/param.h>
     43 #include <sys/systm.h>
     44 #include <sys/kernel.h>
     45 #include <sys/device.h>
     46 #include <sys/kmem.h>
     47 #include <sys/proc.h>
     48 #include <sys/mbuf.h>
     49 #include <sys/bus.h>
     50 #include <sys/intr.h>
     51 
     52 #include <uvm/uvm.h>
     53 #include <uvm/uvm_physseg.h>
     54 
     55 #if defined(PPC_BOOKE)
     56 #define	EIEIO	__asm volatile("mbar\t0" ::: "memory")
     57 #define	SYNC	__asm volatile("msync" ::: "memory")
     58 #elif defined(PPC_IBM4XX) && !defined(PPC_IBM440)
     59 /* eieio is implemented as sync */
     60 #define	EIEIO	__asm volatile("eieio" ::: "memory")
     61 #define	SYNC	/* nothing */
     62 #else
     63 #define	EIEIO	__asm volatile("eieio" ::: "memory")
     64 #define	SYNC	__asm volatile("sync" ::: "memory")
     65 #endif
     66 
     67 int	_bus_dmamap_load_buffer (bus_dma_tag_t, bus_dmamap_t, void *,
     68 	    bus_size_t, struct vmspace *, int, paddr_t *, int *, int);
     69 
     70 static inline void
     71 dcbst(paddr_t pa, long len, int dcache_line_size)
     72 {
     73 	paddr_t epa;
     74 	for (epa = pa + len; pa < epa; pa += dcache_line_size)
     75 		__asm volatile("dcbst 0,%0" :: "r"(pa) : "memory");
     76 }
     77 
     78 static inline void
     79 dcbi(paddr_t pa, long len, int dcache_line_size)
     80 {
     81 	paddr_t epa;
     82 	for (epa = pa + len; pa < epa; pa += dcache_line_size)
     83 		__asm volatile("dcbi 0,%0" :: "r"(pa) : "memory");
     84 }
     85 
     86 static inline void
     87 dcbf(paddr_t pa, long len, int dcache_line_size)
     88 {
     89 	paddr_t epa;
     90 	for (epa = pa + len; pa < epa; pa += dcache_line_size)
     91 		__asm volatile("dcbf 0,%0" :: "r"(pa) : "memory");
     92 }
     93 
     94 /*
     95  * Common function for DMA map creation.  May be called by bus-specific
     96  * DMA map creation functions.
     97  */
     98 int
     99 _bus_dmamap_create(bus_dma_tag_t t, bus_size_t size, int nsegments, bus_size_t maxsegsz, bus_size_t boundary, int flags, bus_dmamap_t *dmamp)
    100 {
    101 	struct powerpc_bus_dmamap *map;
    102 	void *mapstore;
    103 	size_t mapsize;
    104 
    105 	/*
    106 	 * Allocate and initialize the DMA map.  The end of the map
    107 	 * is a variable-sized array of segments, so we allocate enough
    108 	 * room for them in one shot.
    109 	 *
    110 	 * Note we don't preserve the WAITOK or NOWAIT flags.  Preservation
    111 	 * of ALLOCNOW notifies others that we've reserved these resources,
    112 	 * and they are not to be freed.
    113 	 *
    114 	 * The bus_dmamap_t includes one bus_dma_segment_t, hence
    115 	 * the (nsegments - 1).
    116 	 */
    117 	mapsize = sizeof(*map) + sizeof(bus_dma_segment_t [nsegments - 1]);
    118 	if ((mapstore = kmem_intr_alloc(mapsize,
    119 	    (flags & BUS_DMA_NOWAIT) ? KM_NOSLEEP : KM_SLEEP)) == NULL)
    120 		return (ENOMEM);
    121 
    122 	memset(mapstore, 0, mapsize);
    123 	map = (struct powerpc_bus_dmamap *)mapstore;
    124 	map->_dm_size = size;
    125 	map->_dm_segcnt = nsegments;
    126 	map->_dm_maxmaxsegsz = maxsegsz;
    127 	map->_dm_boundary = boundary;
    128 	map->_dm_bounce_thresh = t->_bounce_thresh;
    129 	map->_dm_flags = flags & ~(BUS_DMA_WAITOK|BUS_DMA_NOWAIT);
    130 	map->dm_maxsegsz = maxsegsz;
    131 	map->dm_mapsize = 0;		/* no valid mappings */
    132 	map->dm_nsegs = 0;
    133 
    134 	*dmamp = map;
    135 	return (0);
    136 }
    137 
    138 /*
    139  * Common function for DMA map destruction.  May be called by bus-specific
    140  * DMA map destruction functions.
    141  */
    142 void
    143 _bus_dmamap_destroy(bus_dma_tag_t t, bus_dmamap_t map)
    144 {
    145 
    146 	size_t mapsize = sizeof(*map)
    147 	    + sizeof(bus_dma_segment_t [map->_dm_segcnt - 1]);
    148 	kmem_intr_free(map, mapsize);
    149 }
    150 
    151 /*
    152  * Utility function to load a linear buffer.  lastaddrp holds state
    153  * between invocations (for multiple-buffer loads).  segp contains
    154  * the starting segment on entrance, and the ending segment on exit.
    155  * first indicates if this is the first invocation of this function.
    156  */
    157 int
    158 _bus_dmamap_load_buffer(bus_dma_tag_t t, bus_dmamap_t map, void *buf, bus_size_t buflen, struct vmspace *vm, int flags, paddr_t *lastaddrp, int *segp, int first)
    159 {
    160 	bus_size_t sgsize;
    161 	bus_addr_t curaddr, lastaddr, baddr, bmask;
    162 	vaddr_t vaddr = (vaddr_t)buf;
    163 	int seg;
    164 
    165 //	printf("%s(%p,%p,%p,%u,%p,%#x,%p,%p,%u)\n", __func__,
    166 //	    t, map, buf, buflen, vm, flags, lastaddrp, segp, first);
    167 
    168 	lastaddr = *lastaddrp;
    169 	bmask = ~(map->_dm_boundary - 1);
    170 
    171 	for (seg = *segp; buflen > 0 ; ) {
    172 		/*
    173 		 * Get the physical address for this segment.
    174 		 */
    175 		if (!VMSPACE_IS_KERNEL_P(vm))
    176 			(void) pmap_extract(vm_map_pmap(&vm->vm_map),
    177 			    vaddr, (void *)&curaddr);
    178 		else
    179 			curaddr = vtophys(vaddr);
    180 
    181 		/*
    182 		 * If we're beyond the bounce threshold, notify
    183 		 * the caller.
    184 		 */
    185 		if (map->_dm_bounce_thresh != 0 &&
    186 		    curaddr >= map->_dm_bounce_thresh)
    187 			return (EINVAL);
    188 
    189 		/*
    190 		 * Compute the segment size, and adjust counts.
    191 		 */
    192 		sgsize = PAGE_SIZE - ((u_long)vaddr & PGOFSET);
    193 		if (buflen < sgsize)
    194 			sgsize = buflen;
    195 		sgsize = uimin(sgsize, map->dm_maxsegsz);
    196 
    197 		/*
    198 		 * Make sure we don't cross any boundaries.
    199 		 */
    200 		if (map->_dm_boundary > 0) {
    201 			baddr = (curaddr + map->_dm_boundary) & bmask;
    202 			if (sgsize > (baddr - curaddr))
    203 				sgsize = (baddr - curaddr);
    204 		}
    205 
    206 		/*
    207 		 * Insert chunk into a segment, coalescing with
    208 		 * the previous segment if possible.
    209 		 */
    210 		if (first) {
    211 			map->dm_segs[seg].ds_addr = PHYS_TO_BUS_MEM(t, curaddr);
    212 			map->dm_segs[seg].ds_len = sgsize;
    213 			first = 0;
    214 		} else {
    215 			if (curaddr == lastaddr &&
    216 			    (map->dm_segs[seg].ds_len + sgsize) <=
    217 			     map->dm_maxsegsz &&
    218 			    (map->_dm_boundary == 0 ||
    219 			     (map->dm_segs[seg].ds_addr & bmask) ==
    220 			     (PHYS_TO_BUS_MEM(t, curaddr) & bmask)))
    221 				map->dm_segs[seg].ds_len += sgsize;
    222 			else {
    223 				if (++seg >= map->_dm_segcnt)
    224 					break;
    225 				map->dm_segs[seg].ds_addr =
    226 					PHYS_TO_BUS_MEM(t, curaddr);
    227 				map->dm_segs[seg].ds_len = sgsize;
    228 			}
    229 		}
    230 
    231 		lastaddr = curaddr + sgsize;
    232 		vaddr += sgsize;
    233 		buflen -= sgsize;
    234 	}
    235 
    236 	*segp = seg;
    237 	*lastaddrp = lastaddr;
    238 
    239 	/*
    240 	 * Did we fit?
    241 	 */
    242 	if (buflen != 0)
    243 		return (EFBIG);		/* XXX better return value here? */
    244 
    245 	return (0);
    246 }
    247 
    248 /*
    249  * Common function for loading a DMA map with a linear buffer.  May
    250  * be called by bus-specific DMA map load functions.
    251  */
    252 int
    253 _bus_dmamap_load(bus_dma_tag_t t, bus_dmamap_t map, void *buf, bus_size_t buflen, struct proc *p, int flags)
    254 {
    255 	paddr_t lastaddr = 0;
    256 	int seg, error;
    257 	struct vmspace *vm;
    258 
    259 	/*
    260 	 * Make sure that on error condition we return "no valid mappings".
    261 	 */
    262 	map->dm_mapsize = 0;
    263 	map->dm_nsegs = 0;
    264 	KASSERT(map->dm_maxsegsz <= map->_dm_maxmaxsegsz);
    265 
    266 	if (buflen > map->_dm_size)
    267 		return (EINVAL);
    268 
    269 	if (p != NULL) {
    270 		vm = p->p_vmspace;
    271 	} else {
    272 		vm = vmspace_kernel();
    273 	}
    274 
    275 	seg = 0;
    276 	error = _bus_dmamap_load_buffer(t, map, buf, buflen, vm, flags,
    277 		&lastaddr, &seg, 1);
    278 	if (error == 0) {
    279 		map->dm_mapsize = buflen;
    280 		map->dm_nsegs = seg + 1;
    281 	}
    282 	return (error);
    283 }
    284 
    285 /*
    286  * Like _bus_dmamap_load(), but for mbufs.
    287  */
    288 int
    289 _bus_dmamap_load_mbuf(bus_dma_tag_t t, bus_dmamap_t map, struct mbuf *m0, int flags)
    290 {
    291 	paddr_t lastaddr = 0;
    292 	int seg, error, first;
    293 	struct mbuf *m;
    294 
    295 	/*
    296 	 * Make sure that on error condition we return "no valid mappings."
    297 	 */
    298 	map->dm_mapsize = 0;
    299 	map->dm_nsegs = 0;
    300 	KASSERT(map->dm_maxsegsz <= map->_dm_maxmaxsegsz);
    301 
    302 #ifdef DIAGNOSTIC
    303 	if ((m0->m_flags & M_PKTHDR) == 0)
    304 		panic("_bus_dmamap_load_mbuf: no packet header");
    305 #endif
    306 
    307 	if (m0->m_pkthdr.len > map->_dm_size)
    308 		return (EINVAL);
    309 
    310 	first = 1;
    311 	seg = 0;
    312 	error = 0;
    313 	for (m = m0; m != NULL && error == 0; m = m->m_next, first = 0) {
    314 		if (m->m_len == 0)
    315 			continue;
    316 #ifdef POOL_VTOPHYS
    317 		/* XXX Could be better about coalescing. */
    318 		/* XXX Doesn't check boundaries. */
    319 		switch (m->m_flags & (M_EXT|M_EXT_CLUSTER)) {
    320 		case M_EXT|M_EXT_CLUSTER:
    321 			/* XXX KDASSERT */
    322 			KASSERT(m->m_ext.ext_paddr != M_PADDR_INVALID);
    323 			lastaddr = m->m_ext.ext_paddr +
    324 			    (m->m_data - m->m_ext.ext_buf);
    325  have_addr:
    326 			if (first == 0 && ++seg >= map->_dm_segcnt) {
    327 				error = EFBIG;
    328 				continue;
    329 			}
    330 			map->dm_segs[seg].ds_addr =
    331 			    PHYS_TO_BUS_MEM(t, lastaddr);
    332 			map->dm_segs[seg].ds_len = m->m_len;
    333 			lastaddr += m->m_len;
    334 			continue;
    335 
    336 		case 0:
    337 			lastaddr = m->m_paddr + M_BUFOFFSET(m) +
    338 			    (m->m_data - M_BUFADDR(m));
    339 			goto have_addr;
    340 
    341 		default:
    342 			break;
    343 		}
    344 #endif
    345 		error = _bus_dmamap_load_buffer(t, map, m->m_data,
    346 		    m->m_len, vmspace_kernel(), flags, &lastaddr, &seg, first);
    347 	}
    348 	if (error == 0) {
    349 		map->dm_mapsize = m0->m_pkthdr.len;
    350 		map->dm_nsegs = seg + 1;
    351 	}
    352 	return (error);
    353 }
    354 
    355 /*
    356  * Like _bus_dmamap_load(), but for uios.
    357  */
    358 int
    359 _bus_dmamap_load_uio(bus_dma_tag_t t, bus_dmamap_t map, struct uio *uio, int flags)
    360 {
    361 	paddr_t lastaddr = 0;
    362 	int seg, i, error, first;
    363 	bus_size_t minlen, resid;
    364 	struct iovec *iov;
    365 	void *addr;
    366 
    367 	/*
    368 	 * Make sure that on error condition we return "no valid mappings."
    369 	 */
    370 	map->dm_mapsize = 0;
    371 	map->dm_nsegs = 0;
    372 	KASSERT(map->dm_maxsegsz <= map->_dm_maxmaxsegsz);
    373 
    374 	resid = uio->uio_resid;
    375 	iov = uio->uio_iov;
    376 
    377 	first = 1;
    378 	seg = 0;
    379 	error = 0;
    380 	for (i = 0; i < uio->uio_iovcnt && resid != 0 && error == 0; i++) {
    381 		/*
    382 		 * Now at the first iovec to load.  Load each iovec
    383 		 * until we have exhausted the residual count.
    384 		 */
    385 		minlen = resid < iov[i].iov_len ? resid : iov[i].iov_len;
    386 		addr = (void *)iov[i].iov_base;
    387 
    388 		error = _bus_dmamap_load_buffer(t, map, addr, minlen,
    389 		    uio->uio_vmspace, flags, &lastaddr, &seg, first);
    390 		first = 0;
    391 
    392 		resid -= minlen;
    393 	}
    394 	if (error == 0) {
    395 		map->dm_mapsize = uio->uio_resid;
    396 		map->dm_nsegs = seg + 1;
    397 	}
    398 	return (error);
    399 }
    400 
    401 /*
    402  * Like _bus_dmamap_load(), but for raw memory allocated with
    403  * bus_dmamem_alloc().
    404  *
    405  * XXX This is too much copypasta of _bus_dmamap_load_buffer.
    406  */
    407 int
    408 _bus_dmamap_load_raw(bus_dma_tag_t t, bus_dmamap_t map,
    409     bus_dma_segment_t *segs, int nsegs, bus_size_t size, int flags)
    410 {
    411 	bus_size_t sgsize, isgsize;
    412 	bus_size_t busaddr, curaddr, lastaddr, baddr, bmask;
    413 	int seg, iseg, first;
    414 
    415 	if (size == 0)
    416 		return 0;
    417 
    418 	lastaddr = 0;
    419 	bmask = ~(map->_dm_boundary - 1);
    420 
    421 	first = 0;
    422 	iseg = 0;
    423 	busaddr = segs[iseg].ds_addr;
    424 	isgsize = segs[iseg].ds_len;
    425 	for (seg = 0; size > 0;) {
    426 		/*
    427 		 * Get the physical address for this segment.
    428 		 */
    429 		curaddr = BUS_MEM_TO_PHYS(t, busaddr);
    430 
    431 		/*
    432 		 * If we're beyond the bounce threshold, notify
    433 		 * the caller.
    434 		 */
    435 		if (map->_dm_bounce_thresh != 0 &&
    436 		    curaddr >= map->_dm_bounce_thresh)
    437 			return EINVAL;
    438 
    439 		/*
    440 		 * Compute the segment size, and adjust counts.
    441 		 */
    442 		sgsize = PAGE_SIZE - ((u_long)curaddr & PGOFSET);
    443 		sgsize = MIN(sgsize, isgsize);
    444 		sgsize = MIN(sgsize, size);
    445 		sgsize = MIN(sgsize, map->dm_maxsegsz);
    446 
    447 		/*
    448 		 * Make sure we don't cross any boundaries.
    449 		 */
    450 		if (map->_dm_boundary > 0) {
    451 			baddr = (curaddr + map->_dm_boundary) & bmask;
    452 			if (sgsize > (baddr - curaddr))
    453 				sgsize = (baddr - curaddr);
    454 		}
    455 
    456 		/*
    457 		 * Insert chunk into a segment, coalescing with
    458 		 * the previous segment if possible.
    459 		 */
    460 		if (first) {
    461 			map->dm_segs[seg].ds_addr =
    462 			    PHYS_TO_BUS_MEM(t, curaddr);
    463 			map->dm_segs[seg].ds_len = sgsize;
    464 			first = 0;
    465 		} else {
    466 			if (curaddr == lastaddr &&
    467 			    (map->dm_segs[seg].ds_len + sgsize) <=
    468 			     map->dm_maxsegsz &&
    469 			    (map->_dm_boundary == 0 ||
    470 			     (map->dm_segs[seg].ds_addr & bmask) ==
    471 			     (PHYS_TO_BUS_MEM(t, curaddr) & bmask)))
    472 				map->dm_segs[seg].ds_len += sgsize;
    473 			else {
    474 				if (++seg >= map->_dm_segcnt)
    475 					break;
    476 				map->dm_segs[seg].ds_addr =
    477 					PHYS_TO_BUS_MEM(t, curaddr);
    478 				map->dm_segs[seg].ds_len = sgsize;
    479 			}
    480 		}
    481 
    482 		lastaddr = curaddr + sgsize;
    483 		size -= sgsize;
    484 		if ((isgsize -= sgsize) == 0) {
    485 			iseg++;
    486 			KASSERT(iseg < nsegs);
    487 			busaddr = segs[iseg].ds_addr;
    488 			isgsize = segs[iseg].ds_len;
    489 		}
    490 	}
    491 
    492 	if (size > 0)
    493 		return EFBIG;
    494 
    495 	return 0;
    496 }
    497 
    498 /*
    499  * Common function for unloading a DMA map.  May be called by
    500  * chipset-specific DMA map unload functions.
    501  */
    502 void
    503 _bus_dmamap_unload(bus_dma_tag_t t, bus_dmamap_t map)
    504 {
    505 
    506 	/*
    507 	 * No resources to free; just mark the mappings as
    508 	 * invalid.
    509 	 */
    510 	map->dm_maxsegsz = map->_dm_maxmaxsegsz;
    511 	map->dm_mapsize = 0;
    512 	map->dm_nsegs = 0;
    513 }
    514 
    515 /*
    516  * Common function for DMA map synchronization.  May be called
    517  * by chipset-specific DMA map synchronization functions.
    518  */
    519 void
    520 _bus_dmamap_sync(bus_dma_tag_t t, bus_dmamap_t map, bus_addr_t offset, bus_size_t len, int ops)
    521 {
    522 	const int dcache_line_size = curcpu()->ci_ci.dcache_line_size;
    523 	const bus_dma_segment_t *ds = map->dm_segs;
    524 
    525 //	printf("%s(%p,%p,%#x,%u,%#x) from %p\n", __func__,
    526 //	    t, map, offset, len, ops, __builtin_return_address(0));
    527 
    528 	if ((ops & (BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE)) != 0 &&
    529 	    (ops & (BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE)) != 0)
    530 		panic("_bus_dmamap_sync: invalid ops %#x", ops);
    531 
    532 #ifdef DIAGNOSTIC
    533 	if (offset + len > map->dm_mapsize)
    534 		panic("%s: ops %#x mapsize %u: bad offset (%u) and/or length (%u)", __func__, ops, map->dm_mapsize, offset, len);
    535 #endif
    536 
    537 	/*
    538 	 * Skip leading amount
    539 	 */
    540 	while (offset >= ds->ds_len) {
    541 		offset -= ds->ds_len;
    542 		ds++;
    543 	}
    544 	EIEIO;
    545 	for (; len > 0; ds++, offset = 0) {
    546 		bus_size_t seglen = ds->ds_len - offset;
    547 		bus_addr_t addr = BUS_MEM_TO_PHYS(t, ds->ds_addr) + offset;
    548 		if (seglen > len)
    549 			seglen = len;
    550 		len -= seglen;
    551 		KASSERT(ds < &map->dm_segs[map->dm_nsegs]);
    552 		/*
    553 		 * Readjust things to start on cacheline boundarys
    554 		 */
    555 		offset = (addr & (dcache_line_size-1));
    556 		seglen += offset;
    557 		addr -= offset;
    558 		/*
    559 		 * Now do the appropriate thing.
    560 		 */
    561 		switch (ops) {
    562 		case BUS_DMASYNC_PREWRITE:
    563 			/*
    564 			 * Make sure cache contents are in memory for the DMA.
    565 			 */
    566 			dcbst(addr, seglen, dcache_line_size);
    567 			break;
    568 		case BUS_DMASYNC_PREREAD:
    569 			/*
    570 			 * If the region to be invalidated doesn't fall on
    571 			 * cacheline boundary, flush that cacheline so we
    572 			 * preserve the leading content.
    573 			 */
    574 			if (offset) {
    575 				dcbf(addr, 1, 1);
    576 				/*
    577 				 * If we are doing <= one cache line, stop now.
    578 				 */
    579 				if (seglen <= dcache_line_size)
    580 					break;
    581 				/*
    582 				 * Advance one cache line since we've flushed
    583 				 * this one.
    584 				 */
    585 				addr += dcache_line_size;
    586 				seglen -= dcache_line_size;
    587 			}
    588 			/*
    589 			 * If the byte after the region to be invalidated
    590 			 * doesn't fall on cacheline boundary, flush that
    591 			 * cacheline so we preserve the trailing content.
    592 			 */
    593 			if (seglen & (dcache_line_size-1)) {
    594 				dcbf(addr + seglen, 1, 1);
    595 				if (seglen <= dcache_line_size)
    596 					break;
    597 				/*
    598 				 * Truncate the length to a multiple of a
    599 				 * dcache line size.  No reason to flush
    600 				 * the last entry again.
    601 				 */
    602 				seglen &= ~(dcache_line_size - 1);
    603 			}
    604 			SYNC;			/* is this needed? */
    605 			EIEIO;			/* is this needed? */
    606 			/* FALLTHROUGH */
    607 		case BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE:
    608 		case BUS_DMASYNC_POSTREAD:
    609 			/*
    610 			 * The contents will have changed, make sure to remove
    611 			 * them from the cache.  Note: some implementation
    612 			 * implement dcbi identically to dcbf.  Thus if the
    613 			 * cacheline has data, it will be written to memory.
    614 			 * If the DMA is updating the same cacheline at the
    615 			 * time, bad things can happen.
    616 			 */
    617 			dcbi(addr, seglen, dcache_line_size);
    618 			break;
    619 		case BUS_DMASYNC_POSTWRITE:
    620 			/*
    621 			 * Do nothing.
    622 			 */
    623 			break;
    624 		case BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE:
    625 			/*
    626 			 * Force it to memory and remove from cache.
    627 			 */
    628 			dcbf(addr, seglen, dcache_line_size);
    629 			break;
    630 		}
    631 	}
    632 	__asm volatile("sync");
    633 }
    634 
    635 /*
    636  * Common function for DMA-safe memory allocation.  May be called
    637  * by bus-specific DMA memory allocation functions.
    638  */
    639 int
    640 _bus_dmamem_alloc(bus_dma_tag_t t, bus_size_t size, bus_size_t alignment, bus_size_t boundary, bus_dma_segment_t *segs, int nsegs, int *rsegs, int flags)
    641 {
    642 	paddr_t start = 0xffffffff, end = 0;
    643 	uvm_physseg_t bank;
    644 
    645 	for (bank = uvm_physseg_get_first();
    646 	     uvm_physseg_valid_p(bank);
    647 	     bank = uvm_physseg_get_next(bank)) {
    648 		if (start > ptoa(uvm_physseg_get_avail_start(bank)))
    649 			start = ptoa(uvm_physseg_get_avail_start(bank));
    650 		if (end < ptoa(uvm_physseg_get_avail_end(bank)))
    651 			end = ptoa(uvm_physseg_get_avail_end(bank));
    652 	}
    653 
    654 	return _bus_dmamem_alloc_range(t, size, alignment, boundary, segs,
    655 	    nsegs, rsegs, flags, start, end - PAGE_SIZE);
    656 }
    657 
    658 /*
    659  * Common function for freeing DMA-safe memory.  May be called by
    660  * bus-specific DMA memory free functions.
    661  */
    662 void
    663 _bus_dmamem_free(bus_dma_tag_t t, bus_dma_segment_t *segs, int nsegs)
    664 {
    665 	struct vm_page *m;
    666 	bus_addr_t addr;
    667 	struct pglist mlist;
    668 	int curseg;
    669 
    670 	/*
    671 	 * Build a list of pages to free back to the VM system.
    672 	 */
    673 	TAILQ_INIT(&mlist);
    674 	for (curseg = 0; curseg < nsegs; curseg++) {
    675 		for (addr = BUS_MEM_TO_PHYS(t, segs[curseg].ds_addr);
    676 		    addr < (BUS_MEM_TO_PHYS(t, segs[curseg].ds_addr)
    677 			+ segs[curseg].ds_len);
    678 		    addr += PAGE_SIZE) {
    679 			m = PHYS_TO_VM_PAGE(addr);
    680 			TAILQ_INSERT_TAIL(&mlist, m, pageq.queue);
    681 		}
    682 	}
    683 
    684 	uvm_pglistfree(&mlist);
    685 }
    686 
    687 /*
    688  * Common function for mapping DMA-safe memory.  May be called by
    689  * bus-specific DMA memory map functions.
    690  */
    691 int
    692 _bus_dmamem_map(bus_dma_tag_t t, bus_dma_segment_t *segs, int nsegs, size_t size, void **kvap, int flags)
    693 {
    694 	vaddr_t va;
    695 	bus_addr_t addr;
    696 	int curseg;
    697 	const uvm_flag_t kmflags =
    698 	    (flags & BUS_DMA_NOWAIT) != 0 ? UVM_KMF_NOWAIT : 0;
    699 
    700 	size = round_page(size);
    701 
    702 #ifdef PMAP_MAP_POOLPAGE
    703 	/*
    704 	 * If we are mapping a cacheable physically contiguous segment, treat
    705 	 * it as if we are mapping a poolpage and avoid consuming any KVAs.
    706 	 */
    707 	if (nsegs == 1 && (flags & BUS_DMA_DONTCACHE) == 0) {
    708 		KASSERT(size == segs->ds_len);
    709 		addr = BUS_MEM_TO_PHYS(t, segs->ds_addr);
    710 		if (__predict_true(addr + size < PMAP_DIRECT_MAPPED_LEN)) {
    711 			*kvap = (void *)PMAP_MAP_POOLPAGE(addr);
    712 			return 0;
    713 		}
    714 	}
    715 #endif
    716 
    717 	va = uvm_km_alloc(kernel_map, size, 0, UVM_KMF_VAONLY | kmflags);
    718 
    719 	if (va == 0)
    720 		return (ENOMEM);
    721 
    722 	*kvap = (void *)va;
    723 
    724 	for (curseg = 0; curseg < nsegs; curseg++) {
    725 		for (addr = BUS_MEM_TO_PHYS(t, segs[curseg].ds_addr);
    726 		    addr < (BUS_MEM_TO_PHYS(t, segs[curseg].ds_addr)
    727 			+ segs[curseg].ds_len);
    728 		    addr += PAGE_SIZE, va += PAGE_SIZE, size -= PAGE_SIZE) {
    729 			if (size == 0)
    730 				panic("_bus_dmamem_map: size botch");
    731 			/*
    732 			 * If we are mapping nocache, flush the page from
    733 			 * cache before we map it.
    734 			 */
    735 			if (flags & BUS_DMA_DONTCACHE)
    736 				dcbf(addr, PAGE_SIZE,
    737 				    curcpu()->ci_ci.dcache_line_size);
    738 			pmap_kenter_pa(va, addr,
    739 			    VM_PROT_READ | VM_PROT_WRITE,
    740 			    PMAP_WIRED |
    741 			    ((flags & BUS_DMA_DONTCACHE) ? PMAP_NOCACHE : 0));
    742 		}
    743 	}
    744 
    745 	return (0);
    746 }
    747 
    748 /*
    749  * Common function for unmapping DMA-safe memory.  May be called by
    750  * bus-specific DMA memory unmapping functions.
    751  */
    752 void
    753 _bus_dmamem_unmap(bus_dma_tag_t t, void *kva, size_t size)
    754 {
    755 	vaddr_t va = (vaddr_t) kva;
    756 
    757 #ifdef DIAGNOSTIC
    758 	if (va & PGOFSET)
    759 		panic("_bus_dmamem_unmap");
    760 #endif
    761 
    762 	if (va >= VM_MIN_KERNEL_ADDRESS && va < VM_MAX_KERNEL_ADDRESS) {
    763 		size = round_page(size);
    764 		pmap_kremove(va, size);
    765 		uvm_km_free(kernel_map, va, size, UVM_KMF_VAONLY);
    766 	}
    767 }
    768 
    769 /*
    770  * Common function for mmap(2)'ing DMA-safe memory.  May be called by
    771  * bus-specific DMA mmap(2)'ing functions.
    772  */
    773 paddr_t
    774 _bus_dmamem_mmap(bus_dma_tag_t t, bus_dma_segment_t *segs, int nsegs, off_t off, int prot, int flags)
    775 {
    776 	int i;
    777 
    778 	for (i = 0; i < nsegs; i++) {
    779 #ifdef DIAGNOSTIC
    780 		if (off & PGOFSET)
    781 			panic("_bus_dmamem_mmap: offset unaligned");
    782 		if (BUS_MEM_TO_PHYS(t, segs[i].ds_addr) & PGOFSET)
    783 			panic("_bus_dmamem_mmap: segment unaligned");
    784 		if (segs[i].ds_len & PGOFSET)
    785 			panic("_bus_dmamem_mmap: segment size not multiple"
    786 			    " of page size");
    787 #endif
    788 		if (off >= segs[i].ds_len) {
    789 			off -= segs[i].ds_len;
    790 			continue;
    791 		}
    792 
    793 		return (BUS_MEM_TO_PHYS(t, segs[i].ds_addr) + off);
    794 	}
    795 
    796 	/* Page not found. */
    797 	return (-1);
    798 }
    799 
    800 /*
    801  * Allocate physical memory from the given physical address range.
    802  * Called by DMA-safe memory allocation methods.
    803  */
    804 int
    805 _bus_dmamem_alloc_range(
    806 	bus_dma_tag_t t,
    807 	bus_size_t size,
    808 	bus_size_t alignment,
    809 	bus_size_t boundary,
    810 	bus_dma_segment_t *segs,
    811 	int nsegs,
    812 	int *rsegs,
    813 	int flags,
    814 	paddr_t low,
    815 	paddr_t high)
    816 {
    817 	paddr_t curaddr, lastaddr;
    818 	struct vm_page *m;
    819 	struct pglist mlist;
    820 	int curseg, error;
    821 
    822 	/* Always round the size. */
    823 	size = round_page(size);
    824 
    825 	/*
    826 	 * Allocate pages from the VM system.
    827 	 */
    828 	error = uvm_pglistalloc(size, low, high, alignment, boundary,
    829 	    &mlist, nsegs, (flags & BUS_DMA_NOWAIT) == 0);
    830 	if (error)
    831 		return (error);
    832 
    833 	/*
    834 	 * Compute the location, size, and number of segments actually
    835 	 * returned by the VM code.
    836 	 */
    837 	m = mlist.tqh_first;
    838 	curseg = 0;
    839 	lastaddr = VM_PAGE_TO_PHYS(m);
    840 	segs[curseg].ds_addr = PHYS_TO_BUS_MEM(t, lastaddr);
    841 	segs[curseg].ds_len = PAGE_SIZE;
    842 	m = m->pageq.queue.tqe_next;
    843 
    844 	for (; m != NULL; m = m->pageq.queue.tqe_next) {
    845 		curaddr = VM_PAGE_TO_PHYS(m);
    846 #ifdef DIAGNOSTIC
    847 		if (curaddr < low || curaddr >= high) {
    848 			printf("vm_page_alloc_memory returned non-sensical"
    849 			    " address 0x%lx\n", curaddr);
    850 			panic("_bus_dmamem_alloc_range");
    851 		}
    852 #endif
    853 		if (curaddr == (lastaddr + PAGE_SIZE))
    854 			segs[curseg].ds_len += PAGE_SIZE;
    855 		else {
    856 			curseg++;
    857 			segs[curseg].ds_addr = PHYS_TO_BUS_MEM(t, curaddr);
    858 			segs[curseg].ds_len = PAGE_SIZE;
    859 		}
    860 		lastaddr = curaddr;
    861 	}
    862 
    863 	*rsegs = curseg + 1;
    864 
    865 	return (0);
    866 }
    867 
    868 /*
    869  * Generic form of PHYS_TO_BUS_MEM().
    870  */
    871 bus_addr_t
    872 _bus_dma_phys_to_bus_mem_generic(bus_dma_tag_t t, bus_addr_t addr)
    873 {
    874 
    875 	return (addr);
    876 }
    877 
    878 /*
    879  * Generic form of BUS_MEM_TO_PHYS().
    880  */
    881 bus_addr_t
    882 _bus_dma_bus_mem_to_phys_generic(bus_dma_tag_t t, bus_addr_t addr)
    883 {
    884 
    885 	return (addr);
    886 }
    887