Home | History | Annotate | Line # | Download | only in dev
iommu.c revision 1.98.2.3
      1  1.98.2.2     rmind /*	$NetBSD: iommu.c,v 1.98.2.3 2011/04/21 01:41:26 rmind Exp $	*/
      2      1.82       mrg 
      3      1.82       mrg /*
      4      1.82       mrg  * Copyright (c) 1999, 2000 Matthew R. Green
      5      1.82       mrg  * All rights reserved.
      6      1.82       mrg  *
      7      1.82       mrg  * Redistribution and use in source and binary forms, with or without
      8      1.82       mrg  * modification, are permitted provided that the following conditions
      9      1.82       mrg  * are met:
     10      1.82       mrg  * 1. Redistributions of source code must retain the above copyright
     11      1.82       mrg  *    notice, this list of conditions and the following disclaimer.
     12      1.82       mrg  * 2. Redistributions in binary form must reproduce the above copyright
     13      1.82       mrg  *    notice, this list of conditions and the following disclaimer in the
     14      1.82       mrg  *    documentation and/or other materials provided with the distribution.
     15      1.82       mrg  *
     16      1.82       mrg  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
     17      1.82       mrg  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
     18      1.82       mrg  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
     19      1.82       mrg  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
     20      1.82       mrg  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
     21      1.82       mrg  * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
     22      1.82       mrg  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
     23      1.82       mrg  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
     24      1.82       mrg  * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
     25      1.82       mrg  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
     26      1.82       mrg  * SUCH DAMAGE.
     27      1.82       mrg  */
     28       1.7       mrg 
     29       1.7       mrg /*
     30      1.48       eeh  * Copyright (c) 2001, 2002 Eduardo Horvath
     31       1.7       mrg  * All rights reserved.
     32       1.7       mrg  *
     33       1.7       mrg  * Redistribution and use in source and binary forms, with or without
     34       1.7       mrg  * modification, are permitted provided that the following conditions
     35       1.7       mrg  * are met:
     36       1.7       mrg  * 1. Redistributions of source code must retain the above copyright
     37       1.7       mrg  *    notice, this list of conditions and the following disclaimer.
     38       1.7       mrg  * 2. Redistributions in binary form must reproduce the above copyright
     39       1.7       mrg  *    notice, this list of conditions and the following disclaimer in the
     40       1.7       mrg  *    documentation and/or other materials provided with the distribution.
     41       1.7       mrg  * 3. The name of the author may not be used to endorse or promote products
     42       1.7       mrg  *    derived from this software without specific prior written permission.
     43       1.7       mrg  *
     44       1.7       mrg  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
     45       1.7       mrg  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
     46       1.7       mrg  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
     47       1.7       mrg  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
     48       1.7       mrg  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
     49       1.7       mrg  * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
     50       1.7       mrg  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
     51       1.7       mrg  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
     52       1.7       mrg  * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
     53       1.7       mrg  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
     54       1.7       mrg  * SUCH DAMAGE.
     55       1.7       mrg  */
     56       1.1       mrg 
     57       1.7       mrg /*
     58       1.7       mrg  * UltraSPARC IOMMU support; used by both the sbus and pci code.
     59       1.7       mrg  */
     60      1.66     lukem 
     61      1.66     lukem #include <sys/cdefs.h>
     62  1.98.2.2     rmind __KERNEL_RCSID(0, "$NetBSD: iommu.c,v 1.98.2.3 2011/04/21 01:41:26 rmind Exp $");
     63      1.66     lukem 
     64       1.4       mrg #include "opt_ddb.h"
     65       1.4       mrg 
     66       1.1       mrg #include <sys/param.h>
     67       1.1       mrg #include <sys/extent.h>
     68       1.1       mrg #include <sys/malloc.h>
     69       1.1       mrg #include <sys/systm.h>
     70       1.1       mrg #include <sys/device.h>
     71      1.41       chs #include <sys/proc.h>
     72      1.18       mrg 
     73  1.98.2.2     rmind #include <uvm/uvm.h>
     74       1.1       mrg 
     75       1.1       mrg #include <machine/bus.h>
     76       1.1       mrg #include <sparc64/dev/iommureg.h>
     77       1.1       mrg #include <sparc64/dev/iommuvar.h>
     78       1.1       mrg 
     79       1.1       mrg #include <machine/autoconf.h>
     80       1.1       mrg #include <machine/cpu.h>
     81       1.1       mrg 
     82       1.1       mrg #ifdef DEBUG
     83      1.22       mrg #define IDB_BUSDMA	0x1
     84      1.22       mrg #define IDB_IOMMU	0x2
     85      1.22       mrg #define IDB_INFO	0x4
     86      1.36       eeh #define	IDB_SYNC	0x8
     87      1.10       mrg int iommudebug = 0x0;
     88       1.4       mrg #define DPRINTF(l, s)   do { if (iommudebug & l) printf s; } while (0)
     89      1.90  nakayama #define IOTTE_DEBUG(n)	(n)
     90       1.4       mrg #else
     91       1.4       mrg #define DPRINTF(l, s)
     92      1.90  nakayama #define IOTTE_DEBUG(n)	0
     93       1.1       mrg #endif
     94       1.1       mrg 
     95      1.55       eeh #define iommu_strbuf_flush(i, v) do {					\
     96      1.55       eeh 	if ((i)->sb_flush)						\
     97      1.55       eeh 		bus_space_write_8((i)->sb_is->is_bustag, (i)->sb_sb,	\
     98      1.50       eeh 			STRBUFREG(strbuf_pgflush), (v));		\
     99      1.42       eeh 	} while (0)
    100      1.42       eeh 
    101      1.78       cdi static	int iommu_strbuf_flush_done(struct strbuf_ctl *);
    102      1.85  nakayama static	void _iommu_dvmamap_sync(bus_dma_tag_t, bus_dmamap_t, bus_addr_t,
    103      1.85  nakayama 		bus_size_t, int);
    104      1.11       eeh 
    105       1.1       mrg /*
    106       1.1       mrg  * initialise the UltraSPARC IOMMU (SBUS or PCI):
    107       1.1       mrg  *	- allocate and setup the iotsb.
    108       1.1       mrg  *	- enable the IOMMU
    109       1.7       mrg  *	- initialise the streaming buffers (if they exist)
    110       1.1       mrg  *	- create a private DVMA map.
    111       1.1       mrg  */
    112       1.1       mrg void
    113      1.79       cdi iommu_init(char *name, struct iommu_state *is, int tsbsize, uint32_t iovabase)
    114       1.1       mrg {
    115      1.11       eeh 	psize_t size;
    116      1.11       eeh 	vaddr_t va;
    117      1.11       eeh 	paddr_t pa;
    118      1.58       chs 	struct vm_page *pg;
    119      1.58       chs 	struct pglist pglist;
    120       1.1       mrg 
    121       1.1       mrg 	/*
    122       1.1       mrg 	 * Setup the iommu.
    123       1.1       mrg 	 *
    124      1.45       eeh 	 * The sun4u iommu is part of the SBUS or PCI controller so we will
    125      1.45       eeh 	 * deal with it here..
    126       1.1       mrg 	 *
    127      1.45       eeh 	 * For sysio and psycho/psycho+ the IOMMU address space always ends at
    128      1.45       eeh 	 * 0xffffe000, but the starting address depends on the size of the
    129      1.45       eeh 	 * map.  The map size is 1024 * 2 ^ is->is_tsbsize entries, where each
    130      1.45       eeh 	 * entry is 8 bytes.  The start of the map can be calculated by
    131      1.45       eeh 	 * (0xffffe000 << (8 + is->is_tsbsize)).
    132      1.45       eeh 	 *
    133      1.45       eeh 	 * But sabre and hummingbird use a different scheme that seems to
    134      1.45       eeh 	 * be hard-wired, so we read the start and size from the PROM and
    135      1.45       eeh 	 * just use those values.
    136       1.2       eeh 	 */
    137  1.98.2.3     rmind 	is->is_cr = IOMMUCR_EN;
    138      1.11       eeh 	is->is_tsbsize = tsbsize;
    139      1.45       eeh 	if (iovabase == -1) {
    140      1.45       eeh 		is->is_dvmabase = IOTSB_VSTART(is->is_tsbsize);
    141      1.90  nakayama 		is->is_dvmaend = IOTSB_VEND - 1;
    142      1.45       eeh 	} else {
    143      1.45       eeh 		is->is_dvmabase = iovabase;
    144      1.90  nakayama 		is->is_dvmaend = iovabase + IOTSB_VSIZE(tsbsize) - 1;
    145      1.45       eeh 	}
    146      1.11       eeh 
    147      1.11       eeh 	/*
    148      1.15       eeh 	 * Allocate memory for I/O pagetables.  They need to be physically
    149      1.15       eeh 	 * contiguous.
    150      1.11       eeh 	 */
    151      1.11       eeh 
    152      1.64   thorpej 	size = PAGE_SIZE << is->is_tsbsize;
    153      1.11       eeh 	if (uvm_pglistalloc((psize_t)size, (paddr_t)0, (paddr_t)-1,
    154      1.64   thorpej 		(paddr_t)PAGE_SIZE, (paddr_t)0, &pglist, 1, 0) != 0)
    155      1.11       eeh 		panic("iommu_init: no memory");
    156      1.11       eeh 
    157      1.76      yamt 	va = uvm_km_alloc(kernel_map, size, 0, UVM_KMF_VAONLY);
    158      1.11       eeh 	if (va == 0)
    159      1.11       eeh 		panic("iommu_init: no memory");
    160      1.11       eeh 	is->is_tsb = (int64_t *)va;
    161      1.11       eeh 
    162      1.58       chs 	is->is_ptsb = VM_PAGE_TO_PHYS(TAILQ_FIRST(&pglist));
    163      1.11       eeh 
    164      1.11       eeh 	/* Map the pages */
    165      1.83        ad 	TAILQ_FOREACH(pg, &pglist, pageq.queue) {
    166      1.58       chs 		pa = VM_PAGE_TO_PHYS(pg);
    167      1.88    cegger 		pmap_kenter_pa(va, pa | PMAP_NVC,
    168      1.88    cegger 		    VM_PROT_READ | VM_PROT_WRITE, 0);
    169      1.64   thorpej 		va += PAGE_SIZE;
    170      1.11       eeh 	}
    171      1.38     chris 	pmap_update(pmap_kernel());
    172      1.58       chs 	memset(is->is_tsb, 0, size);
    173       1.1       mrg 
    174       1.1       mrg #ifdef DEBUG
    175      1.22       mrg 	if (iommudebug & IDB_INFO)
    176       1.1       mrg 	{
    177       1.1       mrg 		/* Probe the iommu */
    178       1.1       mrg 
    179      1.50       eeh 		printf("iommu cr=%llx tsb=%llx\n",
    180      1.50       eeh 			(unsigned long long)bus_space_read_8(is->is_bustag,
    181      1.50       eeh 				is->is_iommu,
    182  1.98.2.3     rmind 				offsetof(struct iommureg, iommu_cr)),
    183      1.50       eeh 			(unsigned long long)bus_space_read_8(is->is_bustag,
    184      1.50       eeh 				is->is_iommu,
    185  1.98.2.3     rmind 				offsetof(struct iommureg, iommu_tsb)));
    186      1.58       chs 		printf("TSB base %p phys %llx\n", (void *)is->is_tsb,
    187      1.50       eeh 			(unsigned long long)is->is_ptsb);
    188       1.1       mrg 		delay(1000000); /* 1 s */
    189       1.1       mrg 	}
    190       1.1       mrg #endif
    191       1.1       mrg 
    192       1.1       mrg 	/*
    193       1.1       mrg 	 * Now all the hardware's working we need to allocate a dvma map.
    194       1.1       mrg 	 */
    195      1.98       mrg 	aprint_debug("DVMA map: %x to %x\n",
    196      1.11       eeh 		(unsigned int)is->is_dvmabase,
    197      1.45       eeh 		(unsigned int)is->is_dvmaend);
    198      1.98       mrg 	aprint_debug("IOTSB: %llx to %llx\n",
    199      1.47       eeh 		(unsigned long long)is->is_ptsb,
    200      1.90  nakayama 		(unsigned long long)(is->is_ptsb + size - 1));
    201       1.1       mrg 	is->is_dvmamap = extent_create(name,
    202      1.90  nakayama 	    is->is_dvmabase, is->is_dvmaend,
    203      1.64   thorpej 	    M_DEVBUF, 0, 0, EX_NOWAIT);
    204  1.98.2.1     rmind 	/* XXXMRG Check is_dvmamap is valid. */
    205  1.98.2.3     rmind 
    206  1.98.2.3     rmind 	/*
    207  1.98.2.3     rmind 	 * Set the TSB size.  The relevant bits were moved to the TSB
    208  1.98.2.3     rmind 	 * base register in the PCIe host bridges.
    209  1.98.2.3     rmind 	 */
    210  1.98.2.3     rmind 	if (is->is_flags & IOMMU_TSBSIZE_IN_PTSB)
    211  1.98.2.3     rmind 		is->is_ptsb |= is->is_tsbsize;
    212  1.98.2.3     rmind 	else
    213  1.98.2.3     rmind 		is->is_cr |= (is->is_tsbsize << 16);
    214  1.98.2.3     rmind 
    215  1.98.2.3     rmind 	/*
    216  1.98.2.3     rmind 	 * now actually start up the IOMMU
    217  1.98.2.3     rmind 	 */
    218  1.98.2.3     rmind 	iommu_reset(is);
    219       1.1       mrg }
    220       1.1       mrg 
    221       1.8       mrg /*
    222       1.8       mrg  * Streaming buffers don't exist on the UltraSPARC IIi; we should have
    223       1.8       mrg  * detected that already and disabled them.  If not, we will notice that
    224       1.8       mrg  * they aren't there when the STRBUF_EN bit does not remain.
    225       1.8       mrg  */
    226       1.1       mrg void
    227      1.78       cdi iommu_reset(struct iommu_state *is)
    228       1.1       mrg {
    229      1.45       eeh 	int i;
    230      1.55       eeh 	struct strbuf_ctl *sb;
    231       1.1       mrg 
    232  1.98.2.3     rmind 	IOMMUREG_WRITE(is, iommu_tsb, is->is_ptsb);
    233      1.50       eeh 
    234      1.11       eeh 	/* Enable IOMMU in diagnostic mode */
    235  1.98.2.3     rmind 	IOMMUREG_WRITE(is, iommu_cr, is->is_cr|IOMMUCR_DE);
    236      1.11       eeh 
    237      1.58       chs 	for (i = 0; i < 2; i++) {
    238      1.55       eeh 		if ((sb = is->is_sb[i])) {
    239       1.5       mrg 
    240      1.45       eeh 			/* Enable diagnostics mode? */
    241      1.58       chs 			bus_space_write_8(is->is_bustag, is->is_sb[i]->sb_sb,
    242      1.50       eeh 				STRBUFREG(strbuf_ctl), STRBUF_EN);
    243      1.45       eeh 
    244  1.98.2.3     rmind 			membar_lookaside();
    245  1.98.2.3     rmind 
    246      1.45       eeh 			/* No streaming buffers? Disable them */
    247      1.58       chs 			if (bus_space_read_8(is->is_bustag,
    248      1.58       chs 				is->is_sb[i]->sb_sb,
    249      1.55       eeh 				STRBUFREG(strbuf_ctl)) == 0) {
    250      1.55       eeh 				is->is_sb[i]->sb_flush = NULL;
    251      1.55       eeh 			} else {
    252      1.58       chs 
    253      1.55       eeh 				/*
    254      1.55       eeh 				 * locate the pa of the flush buffer.
    255      1.55       eeh 				 */
    256  1.98.2.3     rmind 				if (pmap_extract(pmap_kernel(),
    257  1.98.2.3     rmind 				     (vaddr_t)is->is_sb[i]->sb_flush,
    258  1.98.2.3     rmind 				     &is->is_sb[i]->sb_flushpa) == FALSE)
    259  1.98.2.3     rmind 					is->is_sb[i]->sb_flush = NULL;
    260      1.55       eeh 			}
    261      1.45       eeh 		}
    262      1.42       eeh 	}
    263  1.98.2.3     rmind 
    264  1.98.2.3     rmind 	if (is->is_flags & IOMMU_FLUSH_CACHE)
    265  1.98.2.3     rmind 		IOMMUREG_WRITE(is, iommu_cache_invalidate, -1ULL);
    266       1.2       eeh }
    267       1.2       eeh 
    268       1.2       eeh /*
    269      1.58       chs  * Here are the iommu control routines.
    270       1.2       eeh  */
    271       1.2       eeh void
    272      1.78       cdi iommu_enter(struct strbuf_ctl *sb, vaddr_t va, int64_t pa, int flags)
    273       1.2       eeh {
    274      1.55       eeh 	struct iommu_state *is = sb->sb_is;
    275      1.55       eeh 	int strbuf = (flags & BUS_DMA_STREAMING);
    276       1.2       eeh 	int64_t tte;
    277       1.2       eeh 
    278       1.2       eeh #ifdef DIAGNOSTIC
    279      1.45       eeh 	if (va < is->is_dvmabase || va > is->is_dvmaend)
    280      1.13       mrg 		panic("iommu_enter: va %#lx not in DVMA space", va);
    281       1.2       eeh #endif
    282       1.2       eeh 
    283      1.55       eeh 	/* Is the streamcache flush really needed? */
    284      1.91  nakayama 	if (sb->sb_flush)
    285      1.55       eeh 		iommu_strbuf_flush(sb, va);
    286      1.91  nakayama 	else
    287      1.55       eeh 		/* If we can't flush the strbuf don't enable it. */
    288      1.55       eeh 		strbuf = 0;
    289      1.55       eeh 
    290      1.58       chs 	tte = MAKEIOTTE(pa, !(flags & BUS_DMA_NOWRITE),
    291      1.55       eeh 		!(flags & BUS_DMA_NOCACHE), (strbuf));
    292      1.50       eeh #ifdef DEBUG
    293      1.50       eeh 	tte |= (flags & 0xff000LL)<<(4*8);
    294      1.50       eeh #endif
    295      1.58       chs 
    296       1.2       eeh 	is->is_tsb[IOTSBSLOT(va,is->is_tsbsize)] = tte;
    297      1.58       chs 	bus_space_write_8(is->is_bustag, is->is_iommu,
    298      1.50       eeh 		IOMMUREG(iommu_flush), va);
    299  1.98.2.3     rmind 	DPRINTF(IDB_IOMMU, ("iommu_enter: slot %d va %lx pa %lx "
    300  1.98.2.3     rmind 		"TSB[%lx]@%p=%lx\n", (int)IOTSBSLOT(va,is->is_tsbsize),
    301      1.50       eeh 		va, (long)pa, (u_long)IOTSBSLOT(va,is->is_tsbsize),
    302      1.50       eeh 		(void *)(u_long)&is->is_tsb[IOTSBSLOT(va,is->is_tsbsize)],
    303      1.50       eeh 		(u_long)tte));
    304      1.39       eeh }
    305      1.39       eeh 
    306      1.39       eeh /*
    307      1.39       eeh  * Find the value of a DVMA address (debug routine).
    308      1.39       eeh  */
    309      1.39       eeh paddr_t
    310      1.78       cdi iommu_extract(struct iommu_state *is, vaddr_t dva)
    311      1.39       eeh {
    312      1.39       eeh 	int64_t tte = 0;
    313      1.58       chs 
    314      1.90  nakayama 	if (dva >= is->is_dvmabase && dva <= is->is_dvmaend)
    315      1.55       eeh 		tte = is->is_tsb[IOTSBSLOT(dva, is->is_tsbsize)];
    316      1.39       eeh 
    317      1.54       eeh 	if ((tte & IOTTE_V) == 0)
    318      1.39       eeh 		return ((paddr_t)-1L);
    319      1.54       eeh 	return (tte & IOTTE_PAMASK);
    320       1.2       eeh }
    321       1.2       eeh 
    322       1.2       eeh /*
    323       1.2       eeh  * iommu_remove: removes mappings created by iommu_enter
    324       1.2       eeh  *
    325       1.2       eeh  * Only demap from IOMMU if flag is set.
    326       1.8       mrg  *
    327       1.8       mrg  * XXX: this function needs better internal error checking.
    328       1.2       eeh  */
    329       1.2       eeh void
    330      1.78       cdi iommu_remove(struct iommu_state *is, vaddr_t va, size_t len)
    331       1.2       eeh {
    332  1.98.2.3     rmind 	int slot;
    333       1.2       eeh 
    334       1.2       eeh #ifdef DIAGNOSTIC
    335      1.45       eeh 	if (va < is->is_dvmabase || va > is->is_dvmaend)
    336      1.25       mrg 		panic("iommu_remove: va 0x%lx not in DVMA space", (u_long)va);
    337       1.2       eeh 	if ((long)(va + len) < (long)va)
    338      1.58       chs 		panic("iommu_remove: va 0x%lx + len 0x%lx wraps",
    339       1.2       eeh 		      (long) va, (long) len);
    340      1.58       chs 	if (len & ~0xfffffff)
    341      1.72       snj 		panic("iommu_remove: ridiculous len 0x%lx", (u_long)len);
    342       1.2       eeh #endif
    343       1.2       eeh 
    344       1.2       eeh 	va = trunc_page(va);
    345      1.22       mrg 	DPRINTF(IDB_IOMMU, ("iommu_remove: va %lx TSB[%lx]@%p\n",
    346      1.50       eeh 		va, (u_long)IOTSBSLOT(va, is->is_tsbsize),
    347      1.50       eeh 		&is->is_tsb[IOTSBSLOT(va, is->is_tsbsize)]));
    348       1.2       eeh 	while (len > 0) {
    349      1.50       eeh 		DPRINTF(IDB_IOMMU, ("iommu_remove: clearing TSB slot %d "
    350      1.50       eeh 			"for va %p size %lx\n",
    351      1.50       eeh 			(int)IOTSBSLOT(va,is->is_tsbsize), (void *)(u_long)va,
    352      1.50       eeh 			(u_long)len));
    353      1.64   thorpej 		if (len <= PAGE_SIZE)
    354      1.10       mrg 			len = 0;
    355      1.10       mrg 		else
    356      1.64   thorpej 			len -= PAGE_SIZE;
    357       1.8       mrg 
    358  1.98.2.1     rmind #if 0
    359      1.94  nakayama 		/*
    360      1.94  nakayama 		 * XXX Zero-ing the entry would not require RMW
    361      1.94  nakayama 		 *
    362      1.94  nakayama 		 * Disabling valid bit while a page is used by a device
    363      1.94  nakayama 		 * causes an uncorrectable DMA error.
    364      1.94  nakayama 		 * Workaround to avoid an uncorrectable DMA error is
    365      1.94  nakayama 		 * eliminating the next line, but the page is mapped
    366      1.94  nakayama 		 * until the next iommu_enter call.
    367      1.94  nakayama 		 */
    368      1.47       eeh 		is->is_tsb[IOTSBSLOT(va,is->is_tsbsize)] &= ~IOTTE_V;
    369      1.94  nakayama 		membar_storestore();
    370  1.98.2.1     rmind #endif
    371  1.98.2.3     rmind 		IOMMUREG_WRITE(is, iommu_flush, va);
    372  1.98.2.3     rmind 
    373  1.98.2.3     rmind 		/* Flush cache if necessary. */
    374  1.98.2.3     rmind 		slot = IOTSBSLOT(trunc_page(va), is->is_tsbsize);
    375  1.98.2.3     rmind 		if ((is->is_flags & IOMMU_FLUSH_CACHE) &&
    376  1.98.2.3     rmind 		    (len == 0 || (slot % 8) == 7))
    377  1.98.2.3     rmind 			IOMMUREG_WRITE(is, iommu_cache_flush,
    378  1.98.2.3     rmind 			    is->is_ptsb + slot * 8);
    379  1.98.2.3     rmind 
    380      1.64   thorpej 		va += PAGE_SIZE;
    381       1.2       eeh 	}
    382       1.2       eeh }
    383       1.2       eeh 
    384      1.58       chs static int
    385      1.78       cdi iommu_strbuf_flush_done(struct strbuf_ctl *sb)
    386       1.2       eeh {
    387      1.55       eeh 	struct iommu_state *is = sb->sb_is;
    388       1.2       eeh 	struct timeval cur, flushtimeout;
    389       1.2       eeh 
    390       1.2       eeh #define BUMPTIME(t, usec) { \
    391       1.2       eeh 	register volatile struct timeval *tp = (t); \
    392       1.2       eeh 	register long us; \
    393       1.2       eeh  \
    394       1.2       eeh 	tp->tv_usec = us = tp->tv_usec + (usec); \
    395       1.2       eeh 	if (us >= 1000000) { \
    396       1.2       eeh 		tp->tv_usec = us - 1000000; \
    397       1.2       eeh 		tp->tv_sec++; \
    398       1.2       eeh 	} \
    399       1.2       eeh }
    400       1.5       mrg 
    401      1.55       eeh 	if (!sb->sb_flush)
    402       1.5       mrg 		return (0);
    403      1.58       chs 
    404       1.7       mrg 	/*
    405       1.7       mrg 	 * Streaming buffer flushes:
    406      1.58       chs 	 *
    407       1.7       mrg 	 *   1 Tell strbuf to flush by storing va to strbuf_pgflush.  If
    408       1.7       mrg 	 *     we're not on a cache line boundary (64-bits):
    409       1.7       mrg 	 *   2 Store 0 in flag
    410       1.7       mrg 	 *   3 Store pointer to flag in flushsync
    411       1.7       mrg 	 *   4 wait till flushsync becomes 0x1
    412       1.7       mrg 	 *
    413       1.7       mrg 	 * If it takes more than .5 sec, something
    414       1.7       mrg 	 * went wrong.
    415       1.7       mrg 	 */
    416       1.2       eeh 
    417      1.55       eeh 	*sb->sb_flush = 0;
    418      1.58       chs 	bus_space_write_8(is->is_bustag, sb->sb_sb,
    419      1.55       eeh 		STRBUFREG(strbuf_flushsync), sb->sb_flushpa);
    420       1.2       eeh 
    421      1.58       chs 	microtime(&flushtimeout);
    422       1.2       eeh 	cur = flushtimeout;
    423       1.2       eeh 	BUMPTIME(&flushtimeout, 500000); /* 1/2 sec */
    424      1.58       chs 
    425  1.98.2.3     rmind 	DPRINTF(IDB_IOMMU, ("%s: flush = %lx at va = %lx pa = %lx now="
    426  1.98.2.3     rmind 		"%"PRIx64":%"PRIx32" until = %"PRIx64":%"PRIx32"\n", __func__,
    427      1.58       chs 		(long)*sb->sb_flush, (long)sb->sb_flush, (long)sb->sb_flushpa,
    428      1.42       eeh 		cur.tv_sec, cur.tv_usec,
    429      1.42       eeh 		flushtimeout.tv_sec, flushtimeout.tv_usec));
    430      1.42       eeh 
    431       1.2       eeh 	/* Bypass non-coherent D$ */
    432      1.55       eeh 	while ((!ldxa(sb->sb_flushpa, ASI_PHYS_CACHED)) &&
    433      1.98       mrg 	       timercmp(&cur, &flushtimeout, <=))
    434       1.2       eeh 		microtime(&cur);
    435       1.2       eeh 
    436       1.2       eeh #ifdef DIAGNOSTIC
    437      1.55       eeh 	if (!ldxa(sb->sb_flushpa, ASI_PHYS_CACHED)) {
    438  1.98.2.3     rmind 		printf("%s: flush timeout %p, at %p\n", __func__,
    439      1.55       eeh 			(void *)(u_long)*sb->sb_flush,
    440      1.55       eeh 			(void *)(u_long)sb->sb_flushpa); /* panic? */
    441       1.2       eeh #ifdef DDB
    442       1.2       eeh 		Debugger();
    443       1.2       eeh #endif
    444       1.2       eeh 	}
    445       1.2       eeh #endif
    446  1.98.2.3     rmind 	DPRINTF(IDB_IOMMU, ("%s: flushed\n", __func__));
    447      1.55       eeh 	return (*sb->sb_flush);
    448       1.7       mrg }
    449       1.7       mrg 
    450       1.7       mrg /*
    451       1.7       mrg  * IOMMU DVMA operations, common to SBUS and PCI.
    452       1.7       mrg  */
    453       1.7       mrg int
    454      1.85  nakayama iommu_dvmamap_load(bus_dma_tag_t t, bus_dmamap_t map, void *buf,
    455      1.85  nakayama 	bus_size_t buflen, struct proc *p, int flags)
    456       1.7       mrg {
    457      1.85  nakayama 	struct strbuf_ctl *sb = (struct strbuf_ctl *)map->_dm_cookie;
    458      1.55       eeh 	struct iommu_state *is = sb->sb_is;
    459       1.7       mrg 	int s;
    460      1.91  nakayama 	int err, needsflush;
    461       1.7       mrg 	bus_size_t sgsize;
    462       1.7       mrg 	paddr_t curaddr;
    463      1.90  nakayama 	u_long dvmaddr, sgstart, sgend, bmask;
    464      1.71   tsutsui 	bus_size_t align, boundary, len;
    465       1.7       mrg 	vaddr_t vaddr = (vaddr_t)buf;
    466      1.40       eeh 	int seg;
    467      1.58       chs 	struct pmap *pmap;
    468  1.98.2.3     rmind 	int slot;
    469       1.7       mrg 
    470       1.7       mrg 	if (map->dm_nsegs) {
    471       1.7       mrg 		/* Already in use?? */
    472       1.7       mrg #ifdef DIAGNOSTIC
    473       1.7       mrg 		printf("iommu_dvmamap_load: map still in use\n");
    474       1.7       mrg #endif
    475       1.7       mrg 		bus_dmamap_unload(t, map);
    476       1.7       mrg 	}
    477      1.58       chs 
    478       1.7       mrg 	/*
    479       1.7       mrg 	 * Make sure that on error condition we return "no valid mappings".
    480       1.7       mrg 	 */
    481       1.7       mrg 	map->dm_nsegs = 0;
    482      1.96  nakayama 	KASSERT(map->dm_maxsegsz <= map->_dm_maxmaxsegsz);
    483      1.96  nakayama 
    484       1.7       mrg 	if (buflen > map->_dm_size) {
    485      1.22       mrg 		DPRINTF(IDB_BUSDMA,
    486       1.7       mrg 		    ("iommu_dvmamap_load(): error %d > %d -- "
    487      1.25       mrg 		     "map size exceeded!\n", (int)buflen, (int)map->_dm_size));
    488       1.7       mrg 		return (EINVAL);
    489       1.7       mrg 	}
    490       1.7       mrg 
    491       1.7       mrg 	sgsize = round_page(buflen + ((int)vaddr & PGOFSET));
    492      1.20       mrg 
    493       1.7       mrg 	/*
    494      1.21       eeh 	 * A boundary presented to bus_dmamem_alloc() takes precedence
    495      1.21       eeh 	 * over boundary in the map.
    496       1.7       mrg 	 */
    497      1.21       eeh 	if ((boundary = (map->dm_segs[0]._ds_boundary)) == 0)
    498      1.21       eeh 		boundary = map->_dm_boundary;
    499      1.64   thorpej 	align = max(map->dm_segs[0]._ds_align, PAGE_SIZE);
    500      1.58       chs 
    501      1.58       chs 	/*
    502      1.58       chs 	 * If our segment size is larger than the boundary we need to
    503      1.40       eeh 	 * split the transfer up int little pieces ourselves.
    504      1.40       eeh 	 */
    505  1.98.2.3     rmind 	KASSERT(is->is_dvmamap);
    506      1.58       chs 	s = splhigh();
    507      1.58       chs 	err = extent_alloc(is->is_dvmamap, sgsize, align,
    508      1.71   tsutsui 	    (sgsize > boundary) ? 0 : boundary,
    509      1.71   tsutsui 	    EX_NOWAIT|EX_BOUNDZERO, &dvmaddr);
    510       1.7       mrg 	splx(s);
    511       1.7       mrg 
    512       1.7       mrg #ifdef DEBUG
    513      1.71   tsutsui 	if (err || (dvmaddr == (u_long)-1)) {
    514       1.7       mrg 		printf("iommu_dvmamap_load(): extent_alloc(%d, %x) failed!\n",
    515      1.25       mrg 		    (int)sgsize, flags);
    516      1.40       eeh #ifdef DDB
    517       1.7       mrg 		Debugger();
    518      1.40       eeh #endif
    519      1.58       chs 	}
    520      1.58       chs #endif
    521      1.11       eeh 	if (err != 0)
    522      1.11       eeh 		return (err);
    523      1.11       eeh 
    524      1.65  nakayama 	if (dvmaddr == (u_long)-1)
    525       1.7       mrg 		return (ENOMEM);
    526       1.7       mrg 
    527      1.40       eeh 	/* Set the active DVMA map */
    528      1.40       eeh 	map->_dm_dvmastart = dvmaddr;
    529      1.40       eeh 	map->_dm_dvmasize = sgsize;
    530      1.40       eeh 
    531      1.40       eeh 	/*
    532      1.40       eeh 	 * Now split the DVMA range into segments, not crossing
    533      1.40       eeh 	 * the boundary.
    534      1.40       eeh 	 */
    535      1.40       eeh 	seg = 0;
    536      1.40       eeh 	sgstart = dvmaddr + (vaddr & PGOFSET);
    537      1.40       eeh 	sgend = sgstart + buflen - 1;
    538      1.40       eeh 	map->dm_segs[seg].ds_addr = sgstart;
    539      1.71   tsutsui 	DPRINTF(IDB_INFO, ("iommu_dvmamap_load: boundary %lx boundary - 1 %lx "
    540      1.71   tsutsui 	    "~(boundary - 1) %lx\n", (long)boundary, (long)(boundary - 1),
    541      1.71   tsutsui 	    (long)~(boundary - 1)));
    542      1.90  nakayama 	bmask = ~(boundary - 1);
    543      1.96  nakayama 	while ((sgstart & bmask) != (sgend & bmask) ||
    544      1.96  nakayama 	       sgend - sgstart + 1 > map->dm_maxsegsz) {
    545      1.96  nakayama 		/* Oops. We crossed a boundary or large seg. Split the xfer. */
    546      1.96  nakayama 		len = map->dm_maxsegsz;
    547      1.96  nakayama 		if ((sgstart & bmask) != (sgend & bmask))
    548      1.96  nakayama 			len = min(len, boundary - (sgstart & (boundary - 1)));
    549      1.71   tsutsui 		map->dm_segs[seg].ds_len = len;
    550      1.40       eeh 		DPRINTF(IDB_INFO, ("iommu_dvmamap_load: "
    551      1.71   tsutsui 		    "seg %d start %lx size %lx\n", seg,
    552      1.71   tsutsui 		    (long)map->dm_segs[seg].ds_addr,
    553      1.71   tsutsui 		    (long)map->dm_segs[seg].ds_len));
    554      1.53       eeh 		if (++seg >= map->_dm_segcnt) {
    555      1.40       eeh 			/* Too many segments.  Fail the operation. */
    556      1.40       eeh 			DPRINTF(IDB_INFO, ("iommu_dvmamap_load: "
    557      1.71   tsutsui 			    "too many segments %d\n", seg));
    558      1.40       eeh 			s = splhigh();
    559      1.40       eeh 			err = extent_free(is->is_dvmamap,
    560      1.71   tsutsui 			    dvmaddr, sgsize, EX_NOWAIT);
    561      1.40       eeh 			map->_dm_dvmastart = 0;
    562      1.40       eeh 			map->_dm_dvmasize = 0;
    563      1.43       eeh 			splx(s);
    564      1.95  nakayama 			if (err != 0)
    565      1.95  nakayama 				printf("warning: %s: %" PRId64
    566      1.95  nakayama 				    " of DVMA space lost\n", __func__, sgsize);
    567      1.80       mrg 			return (EFBIG);
    568      1.40       eeh 		}
    569      1.71   tsutsui 		sgstart += len;
    570      1.40       eeh 		map->dm_segs[seg].ds_addr = sgstart;
    571      1.40       eeh 	}
    572      1.40       eeh 	map->dm_segs[seg].ds_len = sgend - sgstart + 1;
    573      1.40       eeh 	DPRINTF(IDB_INFO, ("iommu_dvmamap_load: "
    574      1.71   tsutsui 	    "seg %d start %lx size %lx\n", seg,
    575      1.71   tsutsui 	    (long)map->dm_segs[seg].ds_addr, (long)map->dm_segs[seg].ds_len));
    576      1.71   tsutsui 	map->dm_nsegs = seg + 1;
    577       1.7       mrg 	map->dm_mapsize = buflen;
    578       1.7       mrg 
    579       1.7       mrg 	if (p != NULL)
    580       1.7       mrg 		pmap = p->p_vmspace->vm_map.pmap;
    581       1.7       mrg 	else
    582       1.7       mrg 		pmap = pmap_kernel();
    583       1.7       mrg 
    584      1.91  nakayama 	needsflush = 0;
    585       1.7       mrg 	for (; buflen > 0; ) {
    586      1.58       chs 
    587       1.7       mrg 		/*
    588       1.7       mrg 		 * Get the physical address for this page.
    589       1.7       mrg 		 */
    590       1.7       mrg 		if (pmap_extract(pmap, (vaddr_t)vaddr, &curaddr) == FALSE) {
    591      1.74    petrov #ifdef DIAGNOSTIC
    592      1.74    petrov 			printf("iommu_dvmamap_load: pmap_extract failed %lx\n", vaddr);
    593      1.74    petrov #endif
    594       1.7       mrg 			bus_dmamap_unload(t, map);
    595       1.7       mrg 			return (-1);
    596       1.7       mrg 		}
    597       1.7       mrg 
    598       1.7       mrg 		/*
    599       1.7       mrg 		 * Compute the segment size, and adjust counts.
    600       1.7       mrg 		 */
    601      1.64   thorpej 		sgsize = PAGE_SIZE - ((u_long)vaddr & PGOFSET);
    602       1.7       mrg 		if (buflen < sgsize)
    603       1.7       mrg 			sgsize = buflen;
    604       1.7       mrg 
    605      1.22       mrg 		DPRINTF(IDB_BUSDMA,
    606      1.36       eeh 		    ("iommu_dvmamap_load: map %p loading va %p "
    607      1.71   tsutsui 		    "dva %lx at pa %lx\n",
    608      1.71   tsutsui 		    map, (void *)vaddr, (long)dvmaddr,
    609      1.87  nakayama 		    (long)trunc_page(curaddr)));
    610      1.55       eeh 		iommu_enter(sb, trunc_page(dvmaddr), trunc_page(curaddr),
    611      1.90  nakayama 		    flags | IOTTE_DEBUG(0x4000));
    612      1.91  nakayama 		needsflush = 1;
    613      1.58       chs 
    614       1.7       mrg 		vaddr += sgsize;
    615       1.7       mrg 		buflen -= sgsize;
    616  1.98.2.3     rmind 
    617  1.98.2.3     rmind 		/* Flush cache if necessary. */
    618  1.98.2.3     rmind 		slot = IOTSBSLOT(trunc_page(dvmaddr), is->is_tsbsize);
    619  1.98.2.3     rmind 		if ((is->is_flags & IOMMU_FLUSH_CACHE) &&
    620  1.98.2.3     rmind 		    (buflen <= 0 || (slot % 8) == 7))
    621  1.98.2.3     rmind 			IOMMUREG_WRITE(is, iommu_cache_flush,
    622  1.98.2.3     rmind 			    is->is_ptsb + slot * 8);
    623  1.98.2.3     rmind 
    624  1.98.2.3     rmind 		dvmaddr += PAGE_SIZE;
    625       1.7       mrg 	}
    626      1.91  nakayama 	if (needsflush)
    627      1.91  nakayama 		iommu_strbuf_flush_done(sb);
    628      1.45       eeh #ifdef DIAGNOSTIC
    629      1.45       eeh 	for (seg = 0; seg < map->dm_nsegs; seg++) {
    630      1.45       eeh 		if (map->dm_segs[seg].ds_addr < is->is_dvmabase ||
    631      1.45       eeh 			map->dm_segs[seg].ds_addr > is->is_dvmaend) {
    632      1.45       eeh 			printf("seg %d dvmaddr %lx out of range %x - %x\n",
    633      1.71   tsutsui 			    seg, (long)map->dm_segs[seg].ds_addr,
    634      1.71   tsutsui 			    is->is_dvmabase, is->is_dvmaend);
    635      1.57       chs #ifdef DDB
    636      1.45       eeh 			Debugger();
    637      1.57       chs #endif
    638      1.45       eeh 		}
    639      1.45       eeh 	}
    640      1.45       eeh #endif
    641       1.7       mrg 	return (0);
    642       1.7       mrg }
    643       1.7       mrg 
    644       1.7       mrg 
    645       1.7       mrg void
    646      1.85  nakayama iommu_dvmamap_unload(bus_dma_tag_t t, bus_dmamap_t map)
    647       1.7       mrg {
    648      1.85  nakayama 	struct strbuf_ctl *sb = (struct strbuf_ctl *)map->_dm_cookie;
    649      1.55       eeh 	struct iommu_state *is = sb->sb_is;
    650      1.40       eeh 	int error, s;
    651      1.70  christos 	bus_size_t sgsize = map->_dm_dvmasize;
    652       1.7       mrg 
    653      1.40       eeh 	/* Flush the iommu */
    654      1.40       eeh #ifdef DEBUG
    655      1.40       eeh 	if (!map->_dm_dvmastart) {
    656      1.40       eeh 		printf("iommu_dvmamap_unload: No dvmastart is zero\n");
    657      1.40       eeh #ifdef DDB
    658      1.40       eeh 		Debugger();
    659      1.40       eeh #endif
    660      1.40       eeh 	}
    661      1.40       eeh #endif
    662      1.40       eeh 	iommu_remove(is, map->_dm_dvmastart, map->_dm_dvmasize);
    663       1.7       mrg 
    664      1.23       eeh 	/* Flush the caches */
    665      1.23       eeh 	bus_dmamap_unload(t->_parent, map);
    666      1.23       eeh 
    667       1.7       mrg 	s = splhigh();
    668      1.58       chs 	error = extent_free(is->is_dvmamap, map->_dm_dvmastart,
    669      1.40       eeh 		map->_dm_dvmasize, EX_NOWAIT);
    670      1.43       eeh 	map->_dm_dvmastart = 0;
    671      1.43       eeh 	map->_dm_dvmasize = 0;
    672       1.7       mrg 	splx(s);
    673       1.7       mrg 	if (error != 0)
    674      1.95  nakayama 		printf("warning: %s: %" PRId64 " of DVMA space lost\n",
    675      1.95  nakayama 		    __func__, sgsize);
    676      1.40       eeh 
    677      1.40       eeh 	/* Clear the map */
    678       1.9       eeh }
    679       1.9       eeh 
    680       1.9       eeh 
    681       1.9       eeh int
    682      1.85  nakayama iommu_dvmamap_load_raw(bus_dma_tag_t t, bus_dmamap_t map,
    683      1.85  nakayama 	bus_dma_segment_t *segs, int nsegs, bus_size_t size, int flags)
    684       1.9       eeh {
    685      1.85  nakayama 	struct strbuf_ctl *sb = (struct strbuf_ctl *)map->_dm_cookie;
    686      1.55       eeh 	struct iommu_state *is = sb->sb_is;
    687      1.58       chs 	struct vm_page *pg;
    688      1.40       eeh 	int i, j, s;
    689      1.26    martin 	int left;
    690      1.91  nakayama 	int err, needsflush;
    691       1.9       eeh 	bus_size_t sgsize;
    692       1.9       eeh 	paddr_t pa;
    693      1.21       eeh 	bus_size_t boundary, align;
    694      1.90  nakayama 	u_long dvmaddr, sgstart, sgend, bmask;
    695      1.58       chs 	struct pglist *pglist;
    696      1.90  nakayama 	const int pagesz = PAGE_SIZE;
    697  1.98.2.3     rmind 	int slot;
    698      1.90  nakayama #ifdef DEBUG
    699      1.90  nakayama 	int npg = 0;
    700      1.90  nakayama #endif
    701       1.9       eeh 
    702       1.9       eeh 	if (map->dm_nsegs) {
    703       1.9       eeh 		/* Already in use?? */
    704       1.9       eeh #ifdef DIAGNOSTIC
    705       1.9       eeh 		printf("iommu_dvmamap_load_raw: map still in use\n");
    706       1.9       eeh #endif
    707       1.9       eeh 		bus_dmamap_unload(t, map);
    708       1.9       eeh 	}
    709      1.40       eeh 
    710      1.40       eeh 	/*
    711      1.40       eeh 	 * A boundary presented to bus_dmamem_alloc() takes precedence
    712      1.40       eeh 	 * over boundary in the map.
    713      1.40       eeh 	 */
    714      1.40       eeh 	if ((boundary = segs[0]._ds_boundary) == 0)
    715      1.40       eeh 		boundary = map->_dm_boundary;
    716      1.40       eeh 
    717      1.45       eeh 	align = max(segs[0]._ds_align, pagesz);
    718      1.40       eeh 
    719       1.9       eeh 	/*
    720       1.9       eeh 	 * Make sure that on error condition we return "no valid mappings".
    721       1.9       eeh 	 */
    722       1.9       eeh 	map->dm_nsegs = 0;
    723      1.26    martin 	/* Count up the total number of pages we need */
    724      1.93  nakayama 	pa = trunc_page(segs[0].ds_addr);
    725      1.26    martin 	sgsize = 0;
    726      1.40       eeh 	left = size;
    727      1.93  nakayama 	for (i = 0; left > 0 && i < nsegs; i++) {
    728      1.26    martin 		if (round_page(pa) != round_page(segs[i].ds_addr))
    729      1.93  nakayama 			sgsize = round_page(sgsize) +
    730      1.93  nakayama 			    (segs[i].ds_addr & PGOFSET);
    731      1.40       eeh 		sgsize += min(left, segs[i].ds_len);
    732      1.40       eeh 		left -= segs[i].ds_len;
    733      1.26    martin 		pa = segs[i].ds_addr + segs[i].ds_len;
    734      1.26    martin 	}
    735      1.93  nakayama 	sgsize = round_page(sgsize);
    736       1.9       eeh 
    737      1.40       eeh 	s = splhigh();
    738      1.58       chs 	/*
    739      1.58       chs 	 * If our segment size is larger than the boundary we need to
    740      1.45       eeh 	 * split the transfer up into little pieces ourselves.
    741       1.9       eeh 	 */
    742      1.40       eeh 	err = extent_alloc(is->is_dvmamap, sgsize, align,
    743      1.40       eeh 		(sgsize > boundary) ? 0 : boundary,
    744      1.40       eeh 		((flags & BUS_DMA_NOWAIT) == 0 ? EX_WAITOK : EX_NOWAIT) |
    745      1.54       eeh 		EX_BOUNDZERO, &dvmaddr);
    746       1.9       eeh 	splx(s);
    747       1.9       eeh 
    748       1.9       eeh 	if (err != 0)
    749       1.9       eeh 		return (err);
    750       1.9       eeh 
    751       1.9       eeh #ifdef DEBUG
    752      1.65  nakayama 	if (dvmaddr == (u_long)-1)
    753      1.58       chs 	{
    754       1.9       eeh 		printf("iommu_dvmamap_load_raw(): extent_alloc(%d, %x) failed!\n",
    755      1.25       mrg 		    (int)sgsize, flags);
    756      1.57       chs #ifdef DDB
    757       1.9       eeh 		Debugger();
    758      1.57       chs #endif
    759      1.58       chs 	}
    760      1.58       chs #endif
    761      1.65  nakayama 	if (dvmaddr == (u_long)-1)
    762       1.9       eeh 		return (ENOMEM);
    763       1.9       eeh 
    764      1.40       eeh 	/* Set the active DVMA map */
    765      1.40       eeh 	map->_dm_dvmastart = dvmaddr;
    766      1.40       eeh 	map->_dm_dvmasize = sgsize;
    767      1.40       eeh 
    768      1.90  nakayama 	bmask = ~(boundary - 1);
    769      1.58       chs 	if ((pglist = segs[0]._ds_mlist) == NULL) {
    770      1.92  nakayama 		u_long prev_va = 0UL, last_va = dvmaddr;
    771      1.45       eeh 		paddr_t prev_pa = 0;
    772      1.45       eeh 		int end = 0, offset;
    773      1.92  nakayama 		bus_size_t len = size;
    774      1.45       eeh 
    775      1.26    martin 		/*
    776      1.45       eeh 		 * This segs is made up of individual physical
    777      1.58       chs 		 *  segments, probably by _bus_dmamap_load_uio() or
    778      1.26    martin 		 * _bus_dmamap_load_mbuf().  Ignore the mlist and
    779      1.45       eeh 		 * load each one individually.
    780      1.26    martin 		 */
    781      1.45       eeh 		j = 0;
    782      1.91  nakayama 		needsflush = 0;
    783      1.45       eeh 		for (i = 0; i < nsegs ; i++) {
    784      1.40       eeh 
    785      1.45       eeh 			pa = segs[i].ds_addr;
    786      1.45       eeh 			offset = (pa & PGOFSET);
    787      1.45       eeh 			pa = trunc_page(pa);
    788      1.45       eeh 			dvmaddr = trunc_page(dvmaddr);
    789      1.92  nakayama 			left = min(len, segs[i].ds_len);
    790      1.45       eeh 
    791      1.45       eeh 			DPRINTF(IDB_INFO, ("iommu_dvmamap_load_raw: converting "
    792      1.58       chs 				"physseg %d start %lx size %lx\n", i,
    793      1.61    martin 				(long)segs[i].ds_addr, (long)segs[i].ds_len));
    794      1.26    martin 
    795      1.58       chs 			if ((pa == prev_pa) &&
    796      1.47       eeh 				((offset != 0) || (end != offset))) {
    797      1.45       eeh 				/* We can re-use this mapping */
    798      1.45       eeh 				dvmaddr = prev_va;
    799      1.45       eeh 			}
    800      1.29    martin 
    801      1.45       eeh 			sgstart = dvmaddr + offset;
    802      1.45       eeh 			sgend = sgstart + left - 1;
    803      1.26    martin 
    804      1.45       eeh 			/* Are the segments virtually adjacent? */
    805      1.58       chs 			if ((j > 0) && (end == offset) &&
    806      1.96  nakayama 			    ((offset == 0) || (pa == prev_pa)) &&
    807      1.96  nakayama 			    (map->dm_segs[j-1].ds_len + left <=
    808      1.96  nakayama 			     map->dm_maxsegsz)) {
    809      1.45       eeh 				/* Just append to the previous segment. */
    810      1.45       eeh 				map->dm_segs[--j].ds_len += left;
    811      1.93  nakayama 				/* Restore sgstart for boundary check */
    812      1.93  nakayama 				sgstart = map->dm_segs[j].ds_addr;
    813      1.45       eeh 				DPRINTF(IDB_INFO, ("iommu_dvmamap_load_raw: "
    814      1.45       eeh 					"appending seg %d start %lx size %lx\n", j,
    815      1.58       chs 					(long)map->dm_segs[j].ds_addr,
    816      1.61    martin 					(long)map->dm_segs[j].ds_len));
    817      1.45       eeh 			} else {
    818      1.53       eeh 				if (j >= map->_dm_segcnt) {
    819      1.92  nakayama 					iommu_remove(is, map->_dm_dvmastart,
    820      1.92  nakayama 					    last_va - map->_dm_dvmastart);
    821      1.92  nakayama 					goto fail;
    822      1.53       eeh 				}
    823      1.45       eeh 				map->dm_segs[j].ds_addr = sgstart;
    824      1.45       eeh 				map->dm_segs[j].ds_len = left;
    825      1.45       eeh 				DPRINTF(IDB_INFO, ("iommu_dvmamap_load_raw: "
    826      1.45       eeh 					"seg %d start %lx size %lx\n", j,
    827      1.48       eeh 					(long)map->dm_segs[j].ds_addr,
    828      1.61    martin 					(long)map->dm_segs[j].ds_len));
    829      1.40       eeh 			}
    830      1.45       eeh 			end = (offset + left) & PGOFSET;
    831      1.40       eeh 
    832      1.40       eeh 			/* Check for boundary issues */
    833      1.90  nakayama 			while ((sgstart & bmask) != (sgend & bmask)) {
    834      1.40       eeh 				/* Need a new segment. */
    835      1.40       eeh 				map->dm_segs[j].ds_len =
    836      1.53       eeh 					boundary - (sgstart & (boundary - 1));
    837      1.40       eeh 				DPRINTF(IDB_INFO, ("iommu_dvmamap_load_raw: "
    838      1.40       eeh 					"seg %d start %lx size %lx\n", j,
    839      1.58       chs 					(long)map->dm_segs[j].ds_addr,
    840      1.61    martin 					(long)map->dm_segs[j].ds_len));
    841      1.53       eeh 				if (++j >= map->_dm_segcnt) {
    842      1.92  nakayama 					iommu_remove(is, map->_dm_dvmastart,
    843      1.92  nakayama 					    last_va - map->_dm_dvmastart);
    844      1.92  nakayama 					goto fail;
    845      1.40       eeh 				}
    846      1.93  nakayama 				sgstart += map->dm_segs[j-1].ds_len;
    847      1.40       eeh 				map->dm_segs[j].ds_addr = sgstart;
    848      1.40       eeh 				map->dm_segs[j].ds_len = sgend - sgstart + 1;
    849      1.40       eeh 			}
    850      1.40       eeh 
    851      1.26    martin 			if (sgsize == 0)
    852      1.26    martin 				panic("iommu_dmamap_load_raw: size botch");
    853      1.40       eeh 
    854      1.45       eeh 			/* Now map a series of pages. */
    855      1.51       eeh 			while (dvmaddr <= sgend) {
    856      1.45       eeh 				DPRINTF(IDB_BUSDMA,
    857      1.45       eeh 					("iommu_dvmamap_load_raw: map %p "
    858      1.45       eeh 						"loading va %lx at pa %lx\n",
    859      1.45       eeh 						map, (long)dvmaddr,
    860      1.45       eeh 						(long)(pa)));
    861      1.45       eeh 				/* Enter it if we haven't before. */
    862      1.91  nakayama 				if (prev_va != dvmaddr) {
    863      1.55       eeh 					iommu_enter(sb, prev_va = dvmaddr,
    864      1.90  nakayama 					    prev_pa = pa,
    865      1.90  nakayama 					    flags | IOTTE_DEBUG(++npg << 12));
    866      1.91  nakayama 					needsflush = 1;
    867  1.98.2.3     rmind 
    868  1.98.2.3     rmind 					/* Flush cache if necessary. */
    869  1.98.2.3     rmind 					slot = IOTSBSLOT(trunc_page(dvmaddr), is->is_tsbsize);
    870  1.98.2.3     rmind 					if ((is->is_flags & IOMMU_FLUSH_CACHE) &&
    871  1.98.2.3     rmind 					    ((dvmaddr + pagesz) > sgend || (slot % 8) == 7))
    872  1.98.2.3     rmind 						IOMMUREG_WRITE(is, iommu_cache_flush,
    873  1.98.2.3     rmind 						    is->is_ptsb + slot * 8);
    874      1.91  nakayama 				}
    875  1.98.2.3     rmind 
    876      1.45       eeh 				dvmaddr += pagesz;
    877      1.45       eeh 				pa += pagesz;
    878      1.92  nakayama 				last_va = dvmaddr;
    879      1.45       eeh 			}
    880      1.45       eeh 
    881      1.92  nakayama 			len -= left;
    882      1.45       eeh 			++j;
    883      1.26    martin 		}
    884      1.91  nakayama 		if (needsflush)
    885      1.91  nakayama 			iommu_strbuf_flush_done(sb);
    886      1.45       eeh 
    887      1.92  nakayama 		map->dm_mapsize = size;
    888      1.45       eeh 		map->dm_nsegs = j;
    889      1.45       eeh #ifdef DIAGNOSTIC
    890      1.45       eeh 		{ int seg;
    891      1.45       eeh 	for (seg = 0; seg < map->dm_nsegs; seg++) {
    892      1.45       eeh 		if (map->dm_segs[seg].ds_addr < is->is_dvmabase ||
    893  1.98.2.3     rmind 		    map->dm_segs[seg].ds_addr > is->is_dvmaend) {
    894      1.45       eeh 			printf("seg %d dvmaddr %lx out of range %x - %x\n",
    895      1.58       chs 				seg, (long)map->dm_segs[seg].ds_addr,
    896      1.45       eeh 				is->is_dvmabase, is->is_dvmaend);
    897      1.57       chs #ifdef DDB
    898      1.45       eeh 			Debugger();
    899      1.57       chs #endif
    900      1.45       eeh 		}
    901      1.45       eeh 	}
    902      1.45       eeh 		}
    903      1.45       eeh #endif
    904      1.26    martin 		return (0);
    905      1.26    martin 	}
    906      1.58       chs 
    907       1.9       eeh 	/*
    908      1.40       eeh 	 * This was allocated with bus_dmamem_alloc.
    909      1.58       chs 	 * The pages are on a `pglist'.
    910       1.9       eeh 	 */
    911      1.26    martin 	i = 0;
    912      1.40       eeh 	sgstart = dvmaddr;
    913      1.40       eeh 	sgend = sgstart + size - 1;
    914      1.40       eeh 	map->dm_segs[i].ds_addr = sgstart;
    915      1.90  nakayama 	while ((sgstart & bmask) != (sgend & bmask)) {
    916      1.40       eeh 		/* Oops.  We crossed a boundary.  Split the xfer. */
    917      1.53       eeh 		map->dm_segs[i].ds_len = boundary - (sgstart & (boundary - 1));
    918      1.40       eeh 		DPRINTF(IDB_INFO, ("iommu_dvmamap_load_raw: "
    919      1.40       eeh 			"seg %d start %lx size %lx\n", i,
    920      1.48       eeh 			(long)map->dm_segs[i].ds_addr,
    921      1.61    martin 			(long)map->dm_segs[i].ds_len));
    922      1.53       eeh 		if (++i >= map->_dm_segcnt) {
    923      1.40       eeh 			/* Too many segments.  Fail the operation. */
    924      1.92  nakayama 			goto fail;
    925      1.40       eeh 		}
    926      1.93  nakayama 		sgstart += map->dm_segs[i-1].ds_len;
    927      1.40       eeh 		map->dm_segs[i].ds_addr = sgstart;
    928      1.40       eeh 	}
    929      1.40       eeh 	DPRINTF(IDB_INFO, ("iommu_dvmamap_load_raw: "
    930      1.40       eeh 			"seg %d start %lx size %lx\n", i,
    931      1.61    martin 			(long)map->dm_segs[i].ds_addr, (long)map->dm_segs[i].ds_len));
    932      1.40       eeh 	map->dm_segs[i].ds_len = sgend - sgstart + 1;
    933       1.9       eeh 
    934      1.91  nakayama 	needsflush = 0;
    935      1.83        ad 	TAILQ_FOREACH(pg, pglist, pageq.queue) {
    936       1.9       eeh 		if (sgsize == 0)
    937       1.9       eeh 			panic("iommu_dmamap_load_raw: size botch");
    938      1.58       chs 		pa = VM_PAGE_TO_PHYS(pg);
    939       1.9       eeh 
    940      1.22       mrg 		DPRINTF(IDB_BUSDMA,
    941       1.9       eeh 		    ("iommu_dvmamap_load_raw: map %p loading va %lx at pa %lx\n",
    942       1.9       eeh 		    map, (long)dvmaddr, (long)(pa)));
    943      1.90  nakayama 		iommu_enter(sb, dvmaddr, pa, flags | IOTTE_DEBUG(0x8000));
    944      1.91  nakayama 		needsflush = 1;
    945      1.58       chs 
    946       1.9       eeh 		sgsize -= pagesz;
    947  1.98.2.3     rmind 
    948  1.98.2.3     rmind 		/* Flush cache if necessary. */
    949  1.98.2.3     rmind 		slot = IOTSBSLOT(trunc_page(dvmaddr), is->is_tsbsize);
    950  1.98.2.3     rmind 		if ((is->is_flags & IOMMU_FLUSH_CACHE) &&
    951  1.98.2.3     rmind 		    (sgsize == 0 || (slot % 8) == 7))
    952  1.98.2.3     rmind 			IOMMUREG_WRITE(is, iommu_cache_flush,
    953  1.98.2.3     rmind 			    is->is_ptsb + slot * 8);
    954  1.98.2.3     rmind 
    955  1.98.2.3     rmind 		dvmaddr += pagesz;
    956       1.9       eeh 	}
    957      1.91  nakayama 	if (needsflush)
    958      1.91  nakayama 		iommu_strbuf_flush_done(sb);
    959      1.40       eeh 	map->dm_mapsize = size;
    960      1.40       eeh 	map->dm_nsegs = i+1;
    961      1.45       eeh #ifdef DIAGNOSTIC
    962      1.45       eeh 	{ int seg;
    963      1.45       eeh 	for (seg = 0; seg < map->dm_nsegs; seg++) {
    964      1.45       eeh 		if (map->dm_segs[seg].ds_addr < is->is_dvmabase ||
    965      1.45       eeh 			map->dm_segs[seg].ds_addr > is->is_dvmaend) {
    966      1.45       eeh 			printf("seg %d dvmaddr %lx out of range %x - %x\n",
    967      1.58       chs 				seg, (long)map->dm_segs[seg].ds_addr,
    968      1.45       eeh 				is->is_dvmabase, is->is_dvmaend);
    969      1.57       chs #ifdef DDB
    970      1.45       eeh 			Debugger();
    971      1.57       chs #endif
    972      1.45       eeh 		}
    973      1.45       eeh 	}
    974      1.45       eeh 	}
    975      1.45       eeh #endif
    976       1.9       eeh 	return (0);
    977      1.92  nakayama 
    978      1.92  nakayama fail:
    979      1.92  nakayama 	s = splhigh();
    980      1.92  nakayama 	err = extent_free(is->is_dvmamap, map->_dm_dvmastart, sgsize,
    981      1.92  nakayama 	    EX_NOWAIT);
    982      1.92  nakayama 	map->_dm_dvmastart = 0;
    983      1.92  nakayama 	map->_dm_dvmasize = 0;
    984      1.92  nakayama 	splx(s);
    985      1.95  nakayama 	if (err != 0)
    986      1.95  nakayama 		printf("warning: %s: %" PRId64 " of DVMA space lost\n",
    987      1.95  nakayama 		    __func__, sgsize);
    988      1.92  nakayama 	return (EFBIG);
    989       1.7       mrg }
    990       1.7       mrg 
    991      1.67    petrov 
    992      1.67    petrov /*
    993      1.67    petrov  * Flush an individual dma segment, returns non-zero if the streaming buffers
    994      1.67    petrov  * need flushing afterwards.
    995      1.67    petrov  */
    996      1.67    petrov static int
    997      1.67    petrov iommu_dvmamap_sync_range(struct strbuf_ctl *sb, vaddr_t va, bus_size_t len)
    998      1.67    petrov {
    999      1.67    petrov 	vaddr_t vaend;
   1000      1.67    petrov 	struct iommu_state *is = sb->sb_is;
   1001      1.67    petrov 
   1002      1.67    petrov #ifdef DIAGNOSTIC
   1003      1.67    petrov 	if (va < is->is_dvmabase || va > is->is_dvmaend)
   1004      1.67    petrov 		panic("invalid va: %llx", (long long)va);
   1005      1.67    petrov #endif
   1006      1.67    petrov 
   1007      1.67    petrov 	if ((is->is_tsb[IOTSBSLOT(va, is->is_tsbsize)] & IOTTE_STREAM) == 0) {
   1008  1.98.2.3     rmind 		DPRINTF(IDB_SYNC,
   1009      1.67    petrov 			("iommu_dvmamap_sync_range: attempting to flush "
   1010      1.67    petrov 			 "non-streaming entry\n"));
   1011      1.67    petrov 		return (0);
   1012      1.67    petrov 	}
   1013      1.67    petrov 
   1014      1.90  nakayama 	vaend = round_page(va + len) - 1;
   1015      1.87  nakayama 	va = trunc_page(va);
   1016      1.67    petrov 
   1017      1.67    petrov #ifdef DIAGNOSTIC
   1018      1.67    petrov 	if (va < is->is_dvmabase || vaend > is->is_dvmaend)
   1019      1.67    petrov 		panic("invalid va range: %llx to %llx (%x to %x)",
   1020      1.67    petrov 		    (long long)va, (long long)vaend,
   1021      1.67    petrov 		    is->is_dvmabase,
   1022      1.67    petrov 		    is->is_dvmaend);
   1023      1.67    petrov #endif
   1024      1.67    petrov 
   1025      1.67    petrov 	for ( ; va <= vaend; va += PAGE_SIZE) {
   1026  1.98.2.3     rmind 		DPRINTF(IDB_SYNC,
   1027      1.67    petrov 		    ("iommu_dvmamap_sync_range: flushing va %p\n",
   1028      1.67    petrov 		    (void *)(u_long)va));
   1029      1.67    petrov 		iommu_strbuf_flush(sb, va);
   1030      1.67    petrov 	}
   1031      1.67    petrov 
   1032      1.67    petrov 	return (1);
   1033      1.67    petrov }
   1034      1.67    petrov 
   1035      1.85  nakayama static void
   1036      1.85  nakayama _iommu_dvmamap_sync(bus_dma_tag_t t, bus_dmamap_t map, bus_addr_t offset,
   1037      1.85  nakayama 	bus_size_t len, int ops)
   1038       1.7       mrg {
   1039      1.85  nakayama 	struct strbuf_ctl *sb = (struct strbuf_ctl *)map->_dm_cookie;
   1040      1.67    petrov 	bus_size_t count;
   1041      1.67    petrov 	int i, needsflush = 0;
   1042      1.63    petrov 
   1043      1.63    petrov 	if (!sb->sb_flush)
   1044      1.63    petrov 		return;
   1045       1.7       mrg 
   1046      1.67    petrov 	for (i = 0; i < map->dm_nsegs; i++) {
   1047      1.67    petrov 		if (offset < map->dm_segs[i].ds_len)
   1048      1.67    petrov 			break;
   1049      1.67    petrov 		offset -= map->dm_segs[i].ds_len;
   1050      1.67    petrov 	}
   1051      1.60    petrov 
   1052      1.67    petrov 	if (i == map->dm_nsegs)
   1053  1.98.2.3     rmind 		panic("%s: segment too short %llu", __func__,
   1054      1.68    martin 		    (unsigned long long)offset);
   1055      1.60    petrov 
   1056      1.62    petrov 	if (ops & (BUS_DMASYNC_PREREAD | BUS_DMASYNC_POSTWRITE)) {
   1057      1.60    petrov 		/* Nothing to do */;
   1058      1.60    petrov 	}
   1059      1.60    petrov 
   1060      1.62    petrov 	if (ops & (BUS_DMASYNC_POSTREAD | BUS_DMASYNC_PREWRITE)) {
   1061      1.67    petrov 
   1062      1.67    petrov 		for (; len > 0 && i < map->dm_nsegs; i++) {
   1063      1.67    petrov 			count = MIN(map->dm_segs[i].ds_len - offset, len);
   1064      1.67    petrov 			if (count > 0 &&
   1065      1.67    petrov 			    iommu_dvmamap_sync_range(sb,
   1066      1.67    petrov 				map->dm_segs[i].ds_addr + offset, count))
   1067      1.67    petrov 				needsflush = 1;
   1068      1.67    petrov 			offset = 0;
   1069      1.67    petrov 			len -= count;
   1070      1.67    petrov 		}
   1071      1.60    petrov #ifdef DIAGNOSTIC
   1072      1.67    petrov 		if (i == map->dm_nsegs && len > 0)
   1073  1.98.2.3     rmind 			panic("%s: leftover %llu", __func__,
   1074      1.73  nakayama 			    (unsigned long long)len);
   1075      1.60    petrov #endif
   1076      1.55       eeh 
   1077      1.67    petrov 		if (needsflush)
   1078      1.58       chs 			iommu_strbuf_flush_done(sb);
   1079       1.7       mrg 	}
   1080       1.7       mrg }
   1081       1.7       mrg 
   1082      1.85  nakayama void
   1083      1.85  nakayama iommu_dvmamap_sync(bus_dma_tag_t t, bus_dmamap_t map, bus_addr_t offset,
   1084      1.85  nakayama 	bus_size_t len, int ops)
   1085      1.85  nakayama {
   1086      1.85  nakayama 
   1087      1.89       jdc 	/* If len is 0, then there is nothing to do */
   1088      1.89       jdc 	if (len == 0)
   1089      1.89       jdc 		return;
   1090      1.89       jdc 
   1091      1.85  nakayama 	if (ops & (BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE)) {
   1092      1.85  nakayama 		/* Flush the CPU then the IOMMU */
   1093      1.85  nakayama 		bus_dmamap_sync(t->_parent, map, offset, len, ops);
   1094      1.85  nakayama 		_iommu_dvmamap_sync(t, map, offset, len, ops);
   1095      1.85  nakayama 	}
   1096      1.85  nakayama 	if (ops & (BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE)) {
   1097      1.85  nakayama 		/* Flush the IOMMU then the CPU */
   1098      1.85  nakayama 		_iommu_dvmamap_sync(t, map, offset, len, ops);
   1099      1.85  nakayama 		bus_dmamap_sync(t->_parent, map, offset, len, ops);
   1100      1.85  nakayama 	}
   1101      1.85  nakayama }
   1102      1.85  nakayama 
   1103       1.7       mrg int
   1104      1.85  nakayama iommu_dvmamem_alloc(bus_dma_tag_t t, bus_size_t size, bus_size_t alignment,
   1105      1.85  nakayama 	bus_size_t boundary, bus_dma_segment_t *segs, int nsegs, int *rsegs,
   1106      1.85  nakayama 	int flags)
   1107       1.7       mrg {
   1108       1.7       mrg 
   1109      1.25       mrg 	DPRINTF(IDB_BUSDMA, ("iommu_dvmamem_alloc: sz %llx align %llx bound %llx "
   1110      1.25       mrg 	   "segp %p flags %d\n", (unsigned long long)size,
   1111      1.25       mrg 	   (unsigned long long)alignment, (unsigned long long)boundary,
   1112      1.25       mrg 	   segs, flags));
   1113       1.7       mrg 	return (bus_dmamem_alloc(t->_parent, size, alignment, boundary,
   1114      1.21       eeh 	    segs, nsegs, rsegs, flags|BUS_DMA_DVMA));
   1115       1.7       mrg }
   1116       1.7       mrg 
   1117       1.7       mrg void
   1118      1.85  nakayama iommu_dvmamem_free(bus_dma_tag_t t, bus_dma_segment_t *segs, int nsegs)
   1119       1.7       mrg {
   1120       1.7       mrg 
   1121      1.22       mrg 	DPRINTF(IDB_BUSDMA, ("iommu_dvmamem_free: segp %p nsegs %d\n",
   1122       1.7       mrg 	    segs, nsegs));
   1123       1.7       mrg 	bus_dmamem_free(t->_parent, segs, nsegs);
   1124       1.7       mrg }
   1125       1.7       mrg 
   1126       1.7       mrg /*
   1127       1.7       mrg  * Map the DVMA mappings into the kernel pmap.
   1128       1.7       mrg  * Check the flags to see whether we're streaming or coherent.
   1129       1.7       mrg  */
   1130       1.7       mrg int
   1131      1.85  nakayama iommu_dvmamem_map(bus_dma_tag_t t, bus_dma_segment_t *segs, int nsegs,
   1132      1.85  nakayama 	size_t size, void **kvap, int flags)
   1133       1.7       mrg {
   1134      1.58       chs 	struct vm_page *pg;
   1135       1.7       mrg 	vaddr_t va;
   1136       1.7       mrg 	bus_addr_t addr;
   1137      1.58       chs 	struct pglist *pglist;
   1138       1.8       mrg 	int cbit;
   1139      1.77      yamt 	const uvm_flag_t kmflags =
   1140      1.77      yamt 	    (flags & BUS_DMA_NOWAIT) != 0 ? UVM_KMF_NOWAIT : 0;
   1141       1.7       mrg 
   1142      1.22       mrg 	DPRINTF(IDB_BUSDMA, ("iommu_dvmamem_map: segp %p nsegs %d size %lx\n",
   1143       1.7       mrg 	    segs, nsegs, size));
   1144       1.7       mrg 
   1145       1.7       mrg 	/*
   1146       1.8       mrg 	 * Allocate some space in the kernel map, and then map these pages
   1147       1.8       mrg 	 * into this space.
   1148       1.7       mrg 	 */
   1149       1.8       mrg 	size = round_page(size);
   1150      1.77      yamt 	va = uvm_km_alloc(kernel_map, size, 0, UVM_KMF_VAONLY | kmflags);
   1151       1.8       mrg 	if (va == 0)
   1152       1.8       mrg 		return (ENOMEM);
   1153       1.7       mrg 
   1154      1.81  christos 	*kvap = (void *)va;
   1155       1.7       mrg 
   1156      1.58       chs 	/*
   1157       1.7       mrg 	 * digest flags:
   1158       1.7       mrg 	 */
   1159       1.7       mrg 	cbit = 0;
   1160       1.7       mrg 	if (flags & BUS_DMA_COHERENT)	/* Disable vcache */
   1161       1.7       mrg 		cbit |= PMAP_NVC;
   1162      1.97     skrll 	if (flags & BUS_DMA_NOCACHE)	/* side effects */
   1163       1.7       mrg 		cbit |= PMAP_NC;
   1164       1.7       mrg 
   1165       1.7       mrg 	/*
   1166       1.8       mrg 	 * Now take this and map it into the CPU.
   1167       1.7       mrg 	 */
   1168      1.58       chs 	pglist = segs[0]._ds_mlist;
   1169      1.83        ad 	TAILQ_FOREACH(pg, pglist, pageq.queue) {
   1170       1.8       mrg #ifdef DIAGNOSTIC
   1171       1.7       mrg 		if (size == 0)
   1172       1.7       mrg 			panic("iommu_dvmamem_map: size botch");
   1173       1.8       mrg #endif
   1174      1.58       chs 		addr = VM_PAGE_TO_PHYS(pg);
   1175      1.22       mrg 		DPRINTF(IDB_BUSDMA, ("iommu_dvmamem_map: "
   1176      1.25       mrg 		    "mapping va %lx at %llx\n", va, (unsigned long long)addr | cbit));
   1177      1.88    cegger 		pmap_kenter_pa(va, addr | cbit,
   1178      1.88    cegger 		    VM_PROT_READ | VM_PROT_WRITE, 0);
   1179       1.7       mrg 		va += PAGE_SIZE;
   1180       1.7       mrg 		size -= PAGE_SIZE;
   1181       1.7       mrg 	}
   1182      1.38     chris 	pmap_update(pmap_kernel());
   1183       1.7       mrg 	return (0);
   1184       1.7       mrg }
   1185       1.7       mrg 
   1186       1.7       mrg /*
   1187       1.7       mrg  * Unmap DVMA mappings from kernel
   1188       1.7       mrg  */
   1189       1.7       mrg void
   1190      1.85  nakayama iommu_dvmamem_unmap(bus_dma_tag_t t, void *kva, size_t size)
   1191       1.7       mrg {
   1192      1.58       chs 
   1193      1.22       mrg 	DPRINTF(IDB_BUSDMA, ("iommu_dvmamem_unmap: kvm %p size %lx\n",
   1194       1.7       mrg 	    kva, size));
   1195      1.58       chs 
   1196       1.7       mrg #ifdef DIAGNOSTIC
   1197       1.7       mrg 	if ((u_long)kva & PGOFSET)
   1198       1.7       mrg 		panic("iommu_dvmamem_unmap");
   1199       1.7       mrg #endif
   1200      1.58       chs 
   1201       1.7       mrg 	size = round_page(size);
   1202      1.58       chs 	pmap_kremove((vaddr_t)kva, size);
   1203      1.38     chris 	pmap_update(pmap_kernel());
   1204      1.76      yamt 	uvm_km_free(kernel_map, (vaddr_t)kva, size, UVM_KMF_VAONLY);
   1205       1.1       mrg }
   1206