Home | History | Annotate | Line # | Download | only in dev
iommu.c revision 1.117
      1  1.117   thorpej /*	$NetBSD: iommu.c,v 1.117 2023/12/01 06:47:59 thorpej Exp $	*/
      2   1.82       mrg 
      3   1.82       mrg /*
      4   1.82       mrg  * Copyright (c) 1999, 2000 Matthew R. Green
      5   1.82       mrg  * All rights reserved.
      6   1.82       mrg  *
      7   1.82       mrg  * Redistribution and use in source and binary forms, with or without
      8   1.82       mrg  * modification, are permitted provided that the following conditions
      9   1.82       mrg  * are met:
     10   1.82       mrg  * 1. Redistributions of source code must retain the above copyright
     11   1.82       mrg  *    notice, this list of conditions and the following disclaimer.
     12   1.82       mrg  * 2. Redistributions in binary form must reproduce the above copyright
     13   1.82       mrg  *    notice, this list of conditions and the following disclaimer in the
     14   1.82       mrg  *    documentation and/or other materials provided with the distribution.
     15   1.82       mrg  *
     16   1.82       mrg  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
     17   1.82       mrg  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
     18   1.82       mrg  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
     19   1.82       mrg  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
     20   1.82       mrg  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
     21   1.82       mrg  * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
     22   1.82       mrg  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
     23   1.82       mrg  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
     24   1.82       mrg  * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
     25   1.82       mrg  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
     26   1.82       mrg  * SUCH DAMAGE.
     27   1.82       mrg  */
     28    1.7       mrg 
     29    1.7       mrg /*
     30   1.48       eeh  * Copyright (c) 2001, 2002 Eduardo Horvath
     31    1.7       mrg  * All rights reserved.
     32    1.7       mrg  *
     33    1.7       mrg  * Redistribution and use in source and binary forms, with or without
     34    1.7       mrg  * modification, are permitted provided that the following conditions
     35    1.7       mrg  * are met:
     36    1.7       mrg  * 1. Redistributions of source code must retain the above copyright
     37    1.7       mrg  *    notice, this list of conditions and the following disclaimer.
     38    1.7       mrg  * 2. Redistributions in binary form must reproduce the above copyright
     39    1.7       mrg  *    notice, this list of conditions and the following disclaimer in the
     40    1.7       mrg  *    documentation and/or other materials provided with the distribution.
     41    1.7       mrg  * 3. The name of the author may not be used to endorse or promote products
     42    1.7       mrg  *    derived from this software without specific prior written permission.
     43    1.7       mrg  *
     44    1.7       mrg  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
     45    1.7       mrg  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
     46    1.7       mrg  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
     47    1.7       mrg  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
     48    1.7       mrg  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
     49    1.7       mrg  * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
     50    1.7       mrg  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
     51    1.7       mrg  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
     52    1.7       mrg  * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
     53    1.7       mrg  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
     54    1.7       mrg  * SUCH DAMAGE.
     55    1.7       mrg  */
     56    1.1       mrg 
     57    1.7       mrg /*
     58    1.7       mrg  * UltraSPARC IOMMU support; used by both the sbus and pci code.
     59    1.7       mrg  */
     60   1.66     lukem 
     61   1.66     lukem #include <sys/cdefs.h>
     62  1.117   thorpej __KERNEL_RCSID(0, "$NetBSD: iommu.c,v 1.117 2023/12/01 06:47:59 thorpej Exp $");
     63   1.66     lukem 
     64    1.4       mrg #include "opt_ddb.h"
     65    1.4       mrg 
     66    1.1       mrg #include <sys/param.h>
     67    1.1       mrg #include <sys/extent.h>
     68    1.1       mrg #include <sys/malloc.h>
     69    1.1       mrg #include <sys/systm.h>
     70    1.1       mrg #include <sys/device.h>
     71   1.41       chs #include <sys/proc.h>
     72   1.18       mrg 
     73  1.100  uebayasi #include <uvm/uvm.h>
     74    1.1       mrg 
     75  1.104    dyoung #include <sys/bus.h>
     76    1.1       mrg #include <sparc64/dev/iommureg.h>
     77    1.1       mrg #include <sparc64/dev/iommuvar.h>
     78    1.1       mrg 
     79    1.1       mrg #include <machine/autoconf.h>
     80    1.1       mrg #include <machine/cpu.h>
     81  1.110  nakayama #include <machine/hypervisor.h>
     82    1.1       mrg 
     83    1.1       mrg #ifdef DEBUG
     84   1.22       mrg #define IDB_BUSDMA	0x1
     85   1.22       mrg #define IDB_IOMMU	0x2
     86   1.22       mrg #define IDB_INFO	0x4
     87   1.36       eeh #define	IDB_SYNC	0x8
     88   1.10       mrg int iommudebug = 0x0;
     89    1.4       mrg #define DPRINTF(l, s)   do { if (iommudebug & l) printf s; } while (0)
     90   1.90  nakayama #define IOTTE_DEBUG(n)	(n)
     91    1.4       mrg #else
     92    1.4       mrg #define DPRINTF(l, s)
     93   1.90  nakayama #define IOTTE_DEBUG(n)	0
     94    1.1       mrg #endif
     95    1.1       mrg 
     96   1.55       eeh #define iommu_strbuf_flush(i, v) do {					\
     97   1.55       eeh 	if ((i)->sb_flush)						\
     98   1.55       eeh 		bus_space_write_8((i)->sb_is->is_bustag, (i)->sb_sb,	\
     99   1.50       eeh 			STRBUFREG(strbuf_pgflush), (v));		\
    100   1.42       eeh 	} while (0)
    101   1.42       eeh 
    102   1.78       cdi static	int iommu_strbuf_flush_done(struct strbuf_ctl *);
    103   1.85  nakayama static	void _iommu_dvmamap_sync(bus_dma_tag_t, bus_dmamap_t, bus_addr_t,
    104   1.85  nakayama 		bus_size_t, int);
    105  1.109     palle static void iommu_enter_sun4u(struct strbuf_ctl *sb, vaddr_t va, int64_t pa, int flags);
    106  1.109     palle static void iommu_enter_sun4v(struct strbuf_ctl *sb, vaddr_t va, int64_t pa, int flags);
    107  1.109     palle static void iommu_remove_sun4u(struct iommu_state *is, vaddr_t va, size_t len);
    108  1.109     palle static void iommu_remove_sun4v(struct iommu_state *is, vaddr_t va, size_t len);
    109   1.11       eeh 
    110    1.1       mrg /*
    111    1.1       mrg  * initialise the UltraSPARC IOMMU (SBUS or PCI):
    112    1.1       mrg  *	- allocate and setup the iotsb.
    113    1.1       mrg  *	- enable the IOMMU
    114    1.7       mrg  *	- initialise the streaming buffers (if they exist)
    115    1.1       mrg  *	- create a private DVMA map.
    116    1.1       mrg  */
    117    1.1       mrg void
    118   1.79       cdi iommu_init(char *name, struct iommu_state *is, int tsbsize, uint32_t iovabase)
    119    1.1       mrg {
    120   1.11       eeh 	psize_t size;
    121   1.11       eeh 	vaddr_t va;
    122   1.11       eeh 	paddr_t pa;
    123   1.58       chs 	struct vm_page *pg;
    124   1.58       chs 	struct pglist pglist;
    125    1.1       mrg 
    126  1.109     palle 	DPRINTF(IDB_INFO, ("iommu_init: tsbsize %x iovabase %x\n", tsbsize, iovabase));
    127  1.109     palle 
    128    1.1       mrg 	/*
    129    1.1       mrg 	 * Setup the iommu.
    130    1.1       mrg 	 *
    131   1.45       eeh 	 * The sun4u iommu is part of the SBUS or PCI controller so we will
    132   1.45       eeh 	 * deal with it here..
    133    1.1       mrg 	 *
    134   1.45       eeh 	 * For sysio and psycho/psycho+ the IOMMU address space always ends at
    135   1.45       eeh 	 * 0xffffe000, but the starting address depends on the size of the
    136   1.45       eeh 	 * map.  The map size is 1024 * 2 ^ is->is_tsbsize entries, where each
    137   1.45       eeh 	 * entry is 8 bytes.  The start of the map can be calculated by
    138   1.45       eeh 	 * (0xffffe000 << (8 + is->is_tsbsize)).
    139   1.45       eeh 	 *
    140   1.45       eeh 	 * But sabre and hummingbird use a different scheme that seems to
    141   1.45       eeh 	 * be hard-wired, so we read the start and size from the PROM and
    142   1.45       eeh 	 * just use those values.
    143    1.2       eeh 	 */
    144  1.108     palle 	if (strncmp(name, "pyro", 4) == 0) {
    145  1.108     palle 		is->is_cr = IOMMUREG_READ(is, iommu_cr);
    146  1.108     palle 		is->is_cr &= ~IOMMUCR_FIRE_BE;
    147  1.108     palle 		is->is_cr |= (IOMMUCR_FIRE_SE | IOMMUCR_FIRE_CM_EN |
    148  1.108     palle 		    IOMMUCR_FIRE_TE);
    149  1.108     palle 	} else
    150  1.108     palle 		is->is_cr = IOMMUCR_EN;
    151   1.11       eeh 	is->is_tsbsize = tsbsize;
    152   1.45       eeh 	if (iovabase == -1) {
    153   1.45       eeh 		is->is_dvmabase = IOTSB_VSTART(is->is_tsbsize);
    154   1.90  nakayama 		is->is_dvmaend = IOTSB_VEND - 1;
    155   1.45       eeh 	} else {
    156   1.45       eeh 		is->is_dvmabase = iovabase;
    157   1.90  nakayama 		is->is_dvmaend = iovabase + IOTSB_VSIZE(tsbsize) - 1;
    158   1.45       eeh 	}
    159   1.11       eeh 
    160   1.11       eeh 	/*
    161   1.15       eeh 	 * Allocate memory for I/O pagetables.  They need to be physically
    162   1.15       eeh 	 * contiguous.
    163   1.11       eeh 	 */
    164   1.11       eeh 
    165   1.64   thorpej 	size = PAGE_SIZE << is->is_tsbsize;
    166   1.11       eeh 	if (uvm_pglistalloc((psize_t)size, (paddr_t)0, (paddr_t)-1,
    167   1.64   thorpej 		(paddr_t)PAGE_SIZE, (paddr_t)0, &pglist, 1, 0) != 0)
    168   1.11       eeh 		panic("iommu_init: no memory");
    169   1.11       eeh 
    170   1.76      yamt 	va = uvm_km_alloc(kernel_map, size, 0, UVM_KMF_VAONLY);
    171   1.11       eeh 	if (va == 0)
    172   1.11       eeh 		panic("iommu_init: no memory");
    173   1.11       eeh 	is->is_tsb = (int64_t *)va;
    174   1.11       eeh 
    175   1.58       chs 	is->is_ptsb = VM_PAGE_TO_PHYS(TAILQ_FIRST(&pglist));
    176   1.11       eeh 
    177   1.11       eeh 	/* Map the pages */
    178   1.83        ad 	TAILQ_FOREACH(pg, &pglist, pageq.queue) {
    179   1.58       chs 		pa = VM_PAGE_TO_PHYS(pg);
    180   1.88    cegger 		pmap_kenter_pa(va, pa | PMAP_NVC,
    181   1.88    cegger 		    VM_PROT_READ | VM_PROT_WRITE, 0);
    182   1.64   thorpej 		va += PAGE_SIZE;
    183   1.11       eeh 	}
    184   1.38     chris 	pmap_update(pmap_kernel());
    185   1.58       chs 	memset(is->is_tsb, 0, size);
    186    1.1       mrg 
    187    1.1       mrg #ifdef DEBUG
    188  1.102       mrg 	if (iommudebug & IDB_INFO)
    189    1.1       mrg 	{
    190    1.1       mrg 		/* Probe the iommu */
    191  1.109     palle 		if (!CPU_ISSUN4V) {
    192  1.109     palle 			printf("iommu cr=%llx tsb=%llx\n",
    193  1.109     palle 			    (unsigned long long)bus_space_read_8(is->is_bustag,
    194   1.50       eeh 				is->is_iommu,
    195  1.103       mrg 				offsetof(struct iommureg, iommu_cr)),
    196  1.109     palle 			    (unsigned long long)bus_space_read_8(is->is_bustag,
    197   1.50       eeh 				is->is_iommu,
    198  1.103       mrg 				offsetof(struct iommureg, iommu_tsb)));
    199  1.109     palle 			printf("TSB base %p phys %llx\n", (void *)is->is_tsb,
    200  1.109     palle 			    (unsigned long long)is->is_ptsb);
    201  1.109     palle 			delay(1000000); /* 1 s */
    202  1.109     palle 		}
    203    1.1       mrg 	}
    204    1.1       mrg #endif
    205    1.1       mrg 
    206    1.1       mrg 	/*
    207    1.1       mrg 	 * Now all the hardware's working we need to allocate a dvma map.
    208    1.1       mrg 	 */
    209   1.98       mrg 	aprint_debug("DVMA map: %x to %x\n",
    210   1.11       eeh 		(unsigned int)is->is_dvmabase,
    211   1.45       eeh 		(unsigned int)is->is_dvmaend);
    212   1.98       mrg 	aprint_debug("IOTSB: %llx to %llx\n",
    213   1.47       eeh 		(unsigned long long)is->is_ptsb,
    214   1.90  nakayama 		(unsigned long long)(is->is_ptsb + size - 1));
    215  1.117   thorpej 	is->is_dvmamap = vmem_create(name,
    216  1.117   thorpej 				     is->is_dvmabase,
    217  1.117   thorpej 				     (is->is_dvmaend + 1) - is->is_dvmabase,
    218  1.117   thorpej 				     PAGE_SIZE,		/* quantum */
    219  1.117   thorpej 				     NULL,		/* importfn */
    220  1.117   thorpej 				     NULL,		/* releasefn */
    221  1.117   thorpej 				     NULL,		/* source */
    222  1.117   thorpej 				     0,			/* qcache_max */
    223  1.117   thorpej 				     VM_SLEEP,
    224  1.117   thorpej 				     IPL_VM);
    225  1.117   thorpej 	KASSERT(is->is_dvmamap != NULL);
    226  1.107       mrg 
    227  1.103       mrg 	/*
    228  1.103       mrg 	 * Set the TSB size.  The relevant bits were moved to the TSB
    229  1.103       mrg 	 * base register in the PCIe host bridges.
    230  1.103       mrg 	 */
    231  1.103       mrg 	if (is->is_flags & IOMMU_TSBSIZE_IN_PTSB)
    232  1.103       mrg 		is->is_ptsb |= is->is_tsbsize;
    233  1.103       mrg 	else
    234  1.103       mrg 		is->is_cr |= (is->is_tsbsize << 16);
    235  1.103       mrg 
    236  1.103       mrg 	/*
    237  1.103       mrg 	 * now actually start up the IOMMU
    238  1.103       mrg 	 */
    239  1.103       mrg 	iommu_reset(is);
    240    1.1       mrg }
    241    1.1       mrg 
    242    1.8       mrg /*
    243    1.8       mrg  * Streaming buffers don't exist on the UltraSPARC IIi; we should have
    244    1.8       mrg  * detected that already and disabled them.  If not, we will notice that
    245    1.8       mrg  * they aren't there when the STRBUF_EN bit does not remain.
    246    1.8       mrg  */
    247    1.1       mrg void
    248   1.78       cdi iommu_reset(struct iommu_state *is)
    249    1.1       mrg {
    250   1.45       eeh 	int i;
    251   1.55       eeh 	struct strbuf_ctl *sb;
    252    1.1       mrg 
    253  1.109     palle 	if (CPU_ISSUN4V)
    254  1.109     palle 		return;
    255  1.109     palle 
    256  1.103       mrg 	IOMMUREG_WRITE(is, iommu_tsb, is->is_ptsb);
    257   1.50       eeh 
    258   1.11       eeh 	/* Enable IOMMU in diagnostic mode */
    259  1.103       mrg 	IOMMUREG_WRITE(is, iommu_cr, is->is_cr|IOMMUCR_DE);
    260   1.11       eeh 
    261   1.58       chs 	for (i = 0; i < 2; i++) {
    262   1.55       eeh 		if ((sb = is->is_sb[i])) {
    263    1.5       mrg 
    264   1.45       eeh 			/* Enable diagnostics mode? */
    265   1.58       chs 			bus_space_write_8(is->is_bustag, is->is_sb[i]->sb_sb,
    266   1.50       eeh 				STRBUFREG(strbuf_ctl), STRBUF_EN);
    267   1.45       eeh 
    268  1.105  nakayama 			membar_Lookaside();
    269  1.103       mrg 
    270   1.45       eeh 			/* No streaming buffers? Disable them */
    271   1.58       chs 			if (bus_space_read_8(is->is_bustag,
    272   1.58       chs 				is->is_sb[i]->sb_sb,
    273   1.55       eeh 				STRBUFREG(strbuf_ctl)) == 0) {
    274   1.55       eeh 				is->is_sb[i]->sb_flush = NULL;
    275   1.55       eeh 			} else {
    276   1.58       chs 
    277   1.55       eeh 				/*
    278   1.55       eeh 				 * locate the pa of the flush buffer.
    279   1.55       eeh 				 */
    280  1.103       mrg 				if (pmap_extract(pmap_kernel(),
    281  1.103       mrg 				     (vaddr_t)is->is_sb[i]->sb_flush,
    282  1.103       mrg 				     &is->is_sb[i]->sb_flushpa) == FALSE)
    283  1.103       mrg 					is->is_sb[i]->sb_flush = NULL;
    284   1.55       eeh 			}
    285   1.45       eeh 		}
    286   1.42       eeh 	}
    287  1.103       mrg 
    288  1.103       mrg 	if (is->is_flags & IOMMU_FLUSH_CACHE)
    289  1.103       mrg 		IOMMUREG_WRITE(is, iommu_cache_invalidate, -1ULL);
    290    1.2       eeh }
    291    1.2       eeh 
    292    1.2       eeh /*
    293   1.58       chs  * Here are the iommu control routines.
    294    1.2       eeh  */
    295  1.109     palle 
    296  1.116       mrg static void
    297   1.78       cdi iommu_enter(struct strbuf_ctl *sb, vaddr_t va, int64_t pa, int flags)
    298    1.2       eeh {
    299  1.109     palle 	DPRINTF(IDB_IOMMU, ("iommu_enter: va %lx pa %lx flags %x\n",
    300  1.109     palle 	    va, (long)pa, flags));
    301  1.109     palle 	if (!CPU_ISSUN4V)
    302  1.109     palle 		iommu_enter_sun4u(sb, va, pa, flags);
    303  1.109     palle 	else
    304  1.109     palle 		iommu_enter_sun4v(sb, va, pa, flags);
    305  1.109     palle }
    306  1.109     palle 
    307  1.109     palle 
    308  1.109     palle void
    309  1.109     palle iommu_enter_sun4u(struct strbuf_ctl *sb, vaddr_t va, int64_t pa, int flags)
    310  1.109     palle {
    311   1.55       eeh 	struct iommu_state *is = sb->sb_is;
    312   1.55       eeh 	int strbuf = (flags & BUS_DMA_STREAMING);
    313    1.2       eeh 	int64_t tte;
    314    1.2       eeh 
    315    1.2       eeh #ifdef DIAGNOSTIC
    316   1.45       eeh 	if (va < is->is_dvmabase || va > is->is_dvmaend)
    317   1.13       mrg 		panic("iommu_enter: va %#lx not in DVMA space", va);
    318    1.2       eeh #endif
    319    1.2       eeh 
    320   1.55       eeh 	/* Is the streamcache flush really needed? */
    321   1.91  nakayama 	if (sb->sb_flush)
    322   1.55       eeh 		iommu_strbuf_flush(sb, va);
    323   1.91  nakayama 	else
    324   1.55       eeh 		/* If we can't flush the strbuf don't enable it. */
    325   1.55       eeh 		strbuf = 0;
    326   1.55       eeh 
    327   1.58       chs 	tte = MAKEIOTTE(pa, !(flags & BUS_DMA_NOWRITE),
    328   1.55       eeh 		!(flags & BUS_DMA_NOCACHE), (strbuf));
    329   1.50       eeh #ifdef DEBUG
    330   1.50       eeh 	tte |= (flags & 0xff000LL)<<(4*8);
    331   1.50       eeh #endif
    332   1.58       chs 
    333    1.2       eeh 	is->is_tsb[IOTSBSLOT(va,is->is_tsbsize)] = tte;
    334   1.58       chs 	bus_space_write_8(is->is_bustag, is->is_iommu,
    335   1.50       eeh 		IOMMUREG(iommu_flush), va);
    336  1.103       mrg 	DPRINTF(IDB_IOMMU, ("iommu_enter: slot %d va %lx pa %lx "
    337  1.103       mrg 		"TSB[%lx]@%p=%lx\n", (int)IOTSBSLOT(va,is->is_tsbsize),
    338   1.50       eeh 		va, (long)pa, (u_long)IOTSBSLOT(va,is->is_tsbsize),
    339   1.50       eeh 		(void *)(u_long)&is->is_tsb[IOTSBSLOT(va,is->is_tsbsize)],
    340   1.50       eeh 		(u_long)tte));
    341   1.39       eeh }
    342   1.39       eeh 
    343  1.109     palle void
    344  1.109     palle iommu_enter_sun4v(struct strbuf_ctl *sb, vaddr_t va, int64_t pa, int flags)
    345  1.109     palle {
    346  1.109     palle 	struct iommu_state *is = sb->sb_is;
    347  1.109     palle 	u_int64_t tsbid = IOTSBSLOT(va, is->is_tsbsize);
    348  1.109     palle 	paddr_t page_list[1], addr;
    349  1.109     palle 	u_int64_t attr, nmapped;
    350  1.109     palle 	int err;
    351  1.109     palle 
    352  1.109     palle #ifdef DIAGNOSTIC
    353  1.109     palle 	if (va < is->is_dvmabase || (va + PAGE_MASK) > is->is_dvmaend)
    354  1.109     palle 		panic("viommu_enter: va %#lx not in DVMA space", va);
    355  1.109     palle #endif
    356  1.109     palle 
    357  1.109     palle 	attr = PCI_MAP_ATTR_READ | PCI_MAP_ATTR_WRITE;
    358  1.109     palle 	if (flags & BUS_DMA_READ)
    359  1.109     palle 		attr &= ~PCI_MAP_ATTR_READ;
    360  1.109     palle 	if (flags & BUS_DMA_WRITE)
    361  1.109     palle 		attr &= ~PCI_MAP_ATTR_WRITE;
    362  1.109     palle 
    363  1.109     palle 	page_list[0] = trunc_page(pa);
    364  1.109     palle 	if (!pmap_extract(pmap_kernel(), (vaddr_t)page_list, &addr))
    365  1.109     palle 		panic("viommu_enter: pmap_extract failed");
    366  1.109     palle 	err = hv_pci_iommu_map(is->is_devhandle, tsbid, 1, attr,
    367  1.109     palle 	    addr, &nmapped);
    368  1.109     palle 	if (err != H_EOK || nmapped != 1)
    369  1.109     palle 		panic("hv_pci_iommu_map: err=%d, nmapped=%lu", err, (long unsigned int)nmapped);
    370  1.109     palle }
    371  1.109     palle 
    372   1.39       eeh /*
    373   1.39       eeh  * Find the value of a DVMA address (debug routine).
    374   1.39       eeh  */
    375   1.39       eeh paddr_t
    376   1.78       cdi iommu_extract(struct iommu_state *is, vaddr_t dva)
    377   1.39       eeh {
    378   1.39       eeh 	int64_t tte = 0;
    379   1.58       chs 
    380   1.90  nakayama 	if (dva >= is->is_dvmabase && dva <= is->is_dvmaend)
    381   1.55       eeh 		tte = is->is_tsb[IOTSBSLOT(dva, is->is_tsbsize)];
    382   1.39       eeh 
    383   1.54       eeh 	if ((tte & IOTTE_V) == 0)
    384   1.39       eeh 		return ((paddr_t)-1L);
    385   1.54       eeh 	return (tte & IOTTE_PAMASK);
    386    1.2       eeh }
    387    1.2       eeh 
    388    1.2       eeh /*
    389    1.2       eeh  * iommu_remove: removes mappings created by iommu_enter
    390    1.2       eeh  *
    391    1.2       eeh  * Only demap from IOMMU if flag is set.
    392    1.8       mrg  *
    393    1.8       mrg  * XXX: this function needs better internal error checking.
    394    1.2       eeh  */
    395  1.109     palle 
    396  1.116       mrg static void
    397   1.78       cdi iommu_remove(struct iommu_state *is, vaddr_t va, size_t len)
    398    1.2       eeh {
    399  1.109     palle 	DPRINTF(IDB_IOMMU, ("iommu_remove: va %lx len %zu\n", va, len));
    400  1.109     palle 	if (!CPU_ISSUN4V)
    401  1.109     palle 		iommu_remove_sun4u(is, va, len);
    402  1.109     palle 	else
    403  1.109     palle 		iommu_remove_sun4v(is, va, len);
    404  1.109     palle }
    405  1.109     palle 
    406  1.109     palle void
    407  1.109     palle iommu_remove_sun4u(struct iommu_state *is, vaddr_t va, size_t len)
    408  1.109     palle {
    409  1.109     palle 
    410  1.103       mrg 	int slot;
    411    1.2       eeh 
    412    1.2       eeh #ifdef DIAGNOSTIC
    413   1.45       eeh 	if (va < is->is_dvmabase || va > is->is_dvmaend)
    414   1.25       mrg 		panic("iommu_remove: va 0x%lx not in DVMA space", (u_long)va);
    415    1.2       eeh 	if ((long)(va + len) < (long)va)
    416   1.58       chs 		panic("iommu_remove: va 0x%lx + len 0x%lx wraps",
    417    1.2       eeh 		      (long) va, (long) len);
    418   1.58       chs 	if (len & ~0xfffffff)
    419   1.72       snj 		panic("iommu_remove: ridiculous len 0x%lx", (u_long)len);
    420    1.2       eeh #endif
    421    1.2       eeh 
    422    1.2       eeh 	va = trunc_page(va);
    423   1.22       mrg 	DPRINTF(IDB_IOMMU, ("iommu_remove: va %lx TSB[%lx]@%p\n",
    424   1.50       eeh 		va, (u_long)IOTSBSLOT(va, is->is_tsbsize),
    425   1.50       eeh 		&is->is_tsb[IOTSBSLOT(va, is->is_tsbsize)]));
    426    1.2       eeh 	while (len > 0) {
    427   1.50       eeh 		DPRINTF(IDB_IOMMU, ("iommu_remove: clearing TSB slot %d "
    428   1.50       eeh 			"for va %p size %lx\n",
    429   1.50       eeh 			(int)IOTSBSLOT(va,is->is_tsbsize), (void *)(u_long)va,
    430   1.50       eeh 			(u_long)len));
    431   1.64   thorpej 		if (len <= PAGE_SIZE)
    432   1.10       mrg 			len = 0;
    433   1.10       mrg 		else
    434   1.64   thorpej 			len -= PAGE_SIZE;
    435    1.8       mrg 
    436   1.99       mrg #if 0
    437   1.94  nakayama 		/*
    438   1.94  nakayama 		 * XXX Zero-ing the entry would not require RMW
    439   1.94  nakayama 		 *
    440   1.94  nakayama 		 * Disabling valid bit while a page is used by a device
    441   1.94  nakayama 		 * causes an uncorrectable DMA error.
    442   1.94  nakayama 		 * Workaround to avoid an uncorrectable DMA error is
    443   1.94  nakayama 		 * eliminating the next line, but the page is mapped
    444   1.94  nakayama 		 * until the next iommu_enter call.
    445   1.94  nakayama 		 */
    446   1.47       eeh 		is->is_tsb[IOTSBSLOT(va,is->is_tsbsize)] &= ~IOTTE_V;
    447  1.105  nakayama 		membar_StoreStore();
    448   1.99       mrg #endif
    449  1.103       mrg 		IOMMUREG_WRITE(is, iommu_flush, va);
    450  1.103       mrg 
    451  1.103       mrg 		/* Flush cache if necessary. */
    452  1.103       mrg 		slot = IOTSBSLOT(trunc_page(va), is->is_tsbsize);
    453  1.103       mrg 		if ((is->is_flags & IOMMU_FLUSH_CACHE) &&
    454  1.103       mrg 		    (len == 0 || (slot % 8) == 7))
    455  1.103       mrg 			IOMMUREG_WRITE(is, iommu_cache_flush,
    456  1.103       mrg 			    is->is_ptsb + slot * 8);
    457  1.103       mrg 
    458   1.64   thorpej 		va += PAGE_SIZE;
    459    1.2       eeh 	}
    460    1.2       eeh }
    461    1.2       eeh 
    462  1.109     palle void
    463  1.109     palle iommu_remove_sun4v(struct iommu_state *is, vaddr_t va, size_t len)
    464  1.109     palle {
    465  1.109     palle 	u_int64_t tsbid = IOTSBSLOT(va, is->is_tsbsize);
    466  1.109     palle 	u_int64_t ndemapped;
    467  1.109     palle 	int err;
    468  1.109     palle 
    469  1.109     palle #ifdef DIAGNOSTIC
    470  1.109     palle 	if (va < is->is_dvmabase || (va + PAGE_MASK) > is->is_dvmaend)
    471  1.109     palle 		panic("iommu_remove: va 0x%lx not in DVMA space", (u_long)va);
    472  1.109     palle 	if (va != trunc_page(va)) {
    473  1.109     palle 		printf("iommu_remove: unaligned va: %lx\n", va);
    474  1.109     palle 		va = trunc_page(va);
    475  1.109     palle 	}
    476  1.109     palle #endif
    477  1.109     palle 
    478  1.109     palle 	err = hv_pci_iommu_demap(is->is_devhandle, tsbid, 1, &ndemapped);
    479  1.109     palle 	if (err != H_EOK || ndemapped != 1)
    480  1.109     palle 		panic("hv_pci_iommu_unmap: err=%d", err);
    481  1.109     palle }
    482  1.109     palle 
    483   1.58       chs static int
    484   1.78       cdi iommu_strbuf_flush_done(struct strbuf_ctl *sb)
    485    1.2       eeh {
    486   1.55       eeh 	struct iommu_state *is = sb->sb_is;
    487    1.2       eeh 	struct timeval cur, flushtimeout;
    488    1.2       eeh 
    489    1.2       eeh #define BUMPTIME(t, usec) { \
    490    1.2       eeh 	register volatile struct timeval *tp = (t); \
    491    1.2       eeh 	register long us; \
    492    1.2       eeh  \
    493    1.2       eeh 	tp->tv_usec = us = tp->tv_usec + (usec); \
    494    1.2       eeh 	if (us >= 1000000) { \
    495    1.2       eeh 		tp->tv_usec = us - 1000000; \
    496    1.2       eeh 		tp->tv_sec++; \
    497    1.2       eeh 	} \
    498    1.2       eeh }
    499    1.5       mrg 
    500   1.55       eeh 	if (!sb->sb_flush)
    501    1.5       mrg 		return (0);
    502   1.58       chs 
    503    1.7       mrg 	/*
    504    1.7       mrg 	 * Streaming buffer flushes:
    505   1.58       chs 	 *
    506    1.7       mrg 	 *   1 Tell strbuf to flush by storing va to strbuf_pgflush.  If
    507    1.7       mrg 	 *     we're not on a cache line boundary (64-bits):
    508    1.7       mrg 	 *   2 Store 0 in flag
    509    1.7       mrg 	 *   3 Store pointer to flag in flushsync
    510    1.7       mrg 	 *   4 wait till flushsync becomes 0x1
    511    1.7       mrg 	 *
    512    1.7       mrg 	 * If it takes more than .5 sec, something
    513    1.7       mrg 	 * went wrong.
    514    1.7       mrg 	 */
    515    1.2       eeh 
    516   1.55       eeh 	*sb->sb_flush = 0;
    517   1.58       chs 	bus_space_write_8(is->is_bustag, sb->sb_sb,
    518   1.55       eeh 		STRBUFREG(strbuf_flushsync), sb->sb_flushpa);
    519    1.2       eeh 
    520   1.58       chs 	microtime(&flushtimeout);
    521    1.2       eeh 	cur = flushtimeout;
    522    1.2       eeh 	BUMPTIME(&flushtimeout, 500000); /* 1/2 sec */
    523   1.58       chs 
    524  1.103       mrg 	DPRINTF(IDB_IOMMU, ("%s: flush = %lx at va = %lx pa = %lx now="
    525  1.103       mrg 		"%"PRIx64":%"PRIx32" until = %"PRIx64":%"PRIx32"\n", __func__,
    526   1.58       chs 		(long)*sb->sb_flush, (long)sb->sb_flush, (long)sb->sb_flushpa,
    527   1.42       eeh 		cur.tv_sec, cur.tv_usec,
    528   1.42       eeh 		flushtimeout.tv_sec, flushtimeout.tv_usec));
    529   1.42       eeh 
    530    1.2       eeh 	/* Bypass non-coherent D$ */
    531   1.55       eeh 	while ((!ldxa(sb->sb_flushpa, ASI_PHYS_CACHED)) &&
    532   1.98       mrg 	       timercmp(&cur, &flushtimeout, <=))
    533    1.2       eeh 		microtime(&cur);
    534    1.2       eeh 
    535    1.2       eeh #ifdef DIAGNOSTIC
    536   1.55       eeh 	if (!ldxa(sb->sb_flushpa, ASI_PHYS_CACHED)) {
    537  1.103       mrg 		printf("%s: flush timeout %p, at %p\n", __func__,
    538   1.55       eeh 			(void *)(u_long)*sb->sb_flush,
    539   1.55       eeh 			(void *)(u_long)sb->sb_flushpa); /* panic? */
    540    1.2       eeh #ifdef DDB
    541    1.2       eeh 		Debugger();
    542    1.2       eeh #endif
    543    1.2       eeh 	}
    544    1.2       eeh #endif
    545  1.103       mrg 	DPRINTF(IDB_IOMMU, ("%s: flushed\n", __func__));
    546   1.55       eeh 	return (*sb->sb_flush);
    547    1.7       mrg }
    548    1.7       mrg 
    549    1.7       mrg /*
    550    1.7       mrg  * IOMMU DVMA operations, common to SBUS and PCI.
    551    1.7       mrg  */
    552    1.7       mrg int
    553   1.85  nakayama iommu_dvmamap_load(bus_dma_tag_t t, bus_dmamap_t map, void *buf,
    554   1.85  nakayama 	bus_size_t buflen, struct proc *p, int flags)
    555    1.7       mrg {
    556   1.85  nakayama 	struct strbuf_ctl *sb = (struct strbuf_ctl *)map->_dm_cookie;
    557   1.55       eeh 	struct iommu_state *is = sb->sb_is;
    558   1.91  nakayama 	int err, needsflush;
    559    1.7       mrg 	bus_size_t sgsize;
    560    1.7       mrg 	paddr_t curaddr;
    561  1.117   thorpej 	u_long sgstart, sgend, bmask;
    562  1.117   thorpej 	vmem_addr_t dvmaddr;
    563   1.71   tsutsui 	bus_size_t align, boundary, len;
    564    1.7       mrg 	vaddr_t vaddr = (vaddr_t)buf;
    565   1.40       eeh 	int seg;
    566   1.58       chs 	struct pmap *pmap;
    567  1.103       mrg 	int slot;
    568    1.7       mrg 
    569    1.7       mrg 	if (map->dm_nsegs) {
    570    1.7       mrg 		/* Already in use?? */
    571    1.7       mrg #ifdef DIAGNOSTIC
    572    1.7       mrg 		printf("iommu_dvmamap_load: map still in use\n");
    573    1.7       mrg #endif
    574    1.7       mrg 		bus_dmamap_unload(t, map);
    575    1.7       mrg 	}
    576   1.58       chs 
    577    1.7       mrg 	/*
    578    1.7       mrg 	 * Make sure that on error condition we return "no valid mappings".
    579    1.7       mrg 	 */
    580    1.7       mrg 	map->dm_nsegs = 0;
    581   1.96  nakayama 	KASSERT(map->dm_maxsegsz <= map->_dm_maxmaxsegsz);
    582   1.96  nakayama 
    583    1.7       mrg 	if (buflen > map->_dm_size) {
    584   1.22       mrg 		DPRINTF(IDB_BUSDMA,
    585    1.7       mrg 		    ("iommu_dvmamap_load(): error %d > %d -- "
    586   1.25       mrg 		     "map size exceeded!\n", (int)buflen, (int)map->_dm_size));
    587    1.7       mrg 		return (EINVAL);
    588    1.7       mrg 	}
    589    1.7       mrg 
    590    1.7       mrg 	sgsize = round_page(buflen + ((int)vaddr & PGOFSET));
    591   1.20       mrg 
    592    1.7       mrg 	/*
    593   1.21       eeh 	 * A boundary presented to bus_dmamem_alloc() takes precedence
    594   1.21       eeh 	 * over boundary in the map.
    595    1.7       mrg 	 */
    596   1.21       eeh 	if ((boundary = (map->dm_segs[0]._ds_boundary)) == 0)
    597   1.21       eeh 		boundary = map->_dm_boundary;
    598  1.114  riastrad 	align = uimax(map->dm_segs[0]._ds_align, PAGE_SIZE);
    599   1.58       chs 
    600   1.58       chs 	/*
    601   1.58       chs 	 * If our segment size is larger than the boundary we need to
    602   1.40       eeh 	 * split the transfer up int little pieces ourselves.
    603   1.40       eeh 	 */
    604  1.117   thorpej 	KASSERT(is->is_dvmamap != NULL);
    605  1.117   thorpej 	err = vmem_xalloc(is->is_dvmamap, sgsize,
    606  1.117   thorpej 			  align,		/* alignment */
    607  1.117   thorpej 			  0,			/* phase */
    608  1.117   thorpej 			  (sgsize > boundary) ? 0 : boundary,
    609  1.117   thorpej 			  VMEM_ADDR_MIN,	/* minaddr */
    610  1.117   thorpej 			  VMEM_ADDR_MAX,	/* maxaddr */
    611  1.117   thorpej 			  VM_NOSLEEP | VM_BESTFIT,
    612  1.117   thorpej 			  &dvmaddr);
    613    1.7       mrg 
    614    1.7       mrg #ifdef DEBUG
    615   1.71   tsutsui 	if (err || (dvmaddr == (u_long)-1)) {
    616    1.7       mrg 		printf("iommu_dvmamap_load(): extent_alloc(%d, %x) failed!\n",
    617   1.25       mrg 		    (int)sgsize, flags);
    618   1.40       eeh #ifdef DDB
    619    1.7       mrg 		Debugger();
    620   1.40       eeh #endif
    621   1.58       chs 	}
    622   1.58       chs #endif
    623   1.11       eeh 	if (err != 0)
    624   1.11       eeh 		return (err);
    625   1.11       eeh 
    626   1.65  nakayama 	if (dvmaddr == (u_long)-1)
    627    1.7       mrg 		return (ENOMEM);
    628    1.7       mrg 
    629   1.40       eeh 	/* Set the active DVMA map */
    630   1.40       eeh 	map->_dm_dvmastart = dvmaddr;
    631   1.40       eeh 	map->_dm_dvmasize = sgsize;
    632   1.40       eeh 
    633   1.40       eeh 	/*
    634   1.40       eeh 	 * Now split the DVMA range into segments, not crossing
    635   1.40       eeh 	 * the boundary.
    636   1.40       eeh 	 */
    637   1.40       eeh 	seg = 0;
    638   1.40       eeh 	sgstart = dvmaddr + (vaddr & PGOFSET);
    639   1.40       eeh 	sgend = sgstart + buflen - 1;
    640   1.40       eeh 	map->dm_segs[seg].ds_addr = sgstart;
    641   1.71   tsutsui 	DPRINTF(IDB_INFO, ("iommu_dvmamap_load: boundary %lx boundary - 1 %lx "
    642   1.71   tsutsui 	    "~(boundary - 1) %lx\n", (long)boundary, (long)(boundary - 1),
    643   1.71   tsutsui 	    (long)~(boundary - 1)));
    644   1.90  nakayama 	bmask = ~(boundary - 1);
    645   1.96  nakayama 	while ((sgstart & bmask) != (sgend & bmask) ||
    646   1.96  nakayama 	       sgend - sgstart + 1 > map->dm_maxsegsz) {
    647   1.96  nakayama 		/* Oops. We crossed a boundary or large seg. Split the xfer. */
    648   1.96  nakayama 		len = map->dm_maxsegsz;
    649   1.96  nakayama 		if ((sgstart & bmask) != (sgend & bmask))
    650  1.114  riastrad 			len = uimin(len, boundary - (sgstart & (boundary - 1)));
    651   1.71   tsutsui 		map->dm_segs[seg].ds_len = len;
    652   1.40       eeh 		DPRINTF(IDB_INFO, ("iommu_dvmamap_load: "
    653   1.71   tsutsui 		    "seg %d start %lx size %lx\n", seg,
    654   1.71   tsutsui 		    (long)map->dm_segs[seg].ds_addr,
    655   1.71   tsutsui 		    (long)map->dm_segs[seg].ds_len));
    656   1.53       eeh 		if (++seg >= map->_dm_segcnt) {
    657   1.40       eeh 			/* Too many segments.  Fail the operation. */
    658   1.40       eeh 			DPRINTF(IDB_INFO, ("iommu_dvmamap_load: "
    659   1.71   tsutsui 			    "too many segments %d\n", seg));
    660  1.117   thorpej 			vmem_xfree(is->is_dvmamap, dvmaddr, sgsize);
    661   1.40       eeh 			map->_dm_dvmastart = 0;
    662   1.40       eeh 			map->_dm_dvmasize = 0;
    663   1.80       mrg 			return (EFBIG);
    664   1.40       eeh 		}
    665   1.71   tsutsui 		sgstart += len;
    666   1.40       eeh 		map->dm_segs[seg].ds_addr = sgstart;
    667   1.40       eeh 	}
    668   1.40       eeh 	map->dm_segs[seg].ds_len = sgend - sgstart + 1;
    669   1.40       eeh 	DPRINTF(IDB_INFO, ("iommu_dvmamap_load: "
    670   1.71   tsutsui 	    "seg %d start %lx size %lx\n", seg,
    671   1.71   tsutsui 	    (long)map->dm_segs[seg].ds_addr, (long)map->dm_segs[seg].ds_len));
    672   1.71   tsutsui 	map->dm_nsegs = seg + 1;
    673    1.7       mrg 	map->dm_mapsize = buflen;
    674    1.7       mrg 
    675    1.7       mrg 	if (p != NULL)
    676    1.7       mrg 		pmap = p->p_vmspace->vm_map.pmap;
    677    1.7       mrg 	else
    678    1.7       mrg 		pmap = pmap_kernel();
    679    1.7       mrg 
    680   1.91  nakayama 	needsflush = 0;
    681    1.7       mrg 	for (; buflen > 0; ) {
    682   1.58       chs 
    683    1.7       mrg 		/*
    684    1.7       mrg 		 * Get the physical address for this page.
    685    1.7       mrg 		 */
    686    1.7       mrg 		if (pmap_extract(pmap, (vaddr_t)vaddr, &curaddr) == FALSE) {
    687   1.74    petrov #ifdef DIAGNOSTIC
    688   1.74    petrov 			printf("iommu_dvmamap_load: pmap_extract failed %lx\n", vaddr);
    689   1.74    petrov #endif
    690    1.7       mrg 			bus_dmamap_unload(t, map);
    691    1.7       mrg 			return (-1);
    692    1.7       mrg 		}
    693    1.7       mrg 
    694    1.7       mrg 		/*
    695    1.7       mrg 		 * Compute the segment size, and adjust counts.
    696    1.7       mrg 		 */
    697   1.64   thorpej 		sgsize = PAGE_SIZE - ((u_long)vaddr & PGOFSET);
    698    1.7       mrg 		if (buflen < sgsize)
    699    1.7       mrg 			sgsize = buflen;
    700    1.7       mrg 
    701   1.22       mrg 		DPRINTF(IDB_BUSDMA,
    702   1.36       eeh 		    ("iommu_dvmamap_load: map %p loading va %p "
    703   1.71   tsutsui 		    "dva %lx at pa %lx\n",
    704   1.71   tsutsui 		    map, (void *)vaddr, (long)dvmaddr,
    705   1.87  nakayama 		    (long)trunc_page(curaddr)));
    706   1.55       eeh 		iommu_enter(sb, trunc_page(dvmaddr), trunc_page(curaddr),
    707   1.90  nakayama 		    flags | IOTTE_DEBUG(0x4000));
    708   1.91  nakayama 		needsflush = 1;
    709   1.58       chs 
    710    1.7       mrg 		vaddr += sgsize;
    711    1.7       mrg 		buflen -= sgsize;
    712  1.103       mrg 
    713  1.103       mrg 		/* Flush cache if necessary. */
    714  1.103       mrg 		slot = IOTSBSLOT(trunc_page(dvmaddr), is->is_tsbsize);
    715  1.103       mrg 		if ((is->is_flags & IOMMU_FLUSH_CACHE) &&
    716  1.103       mrg 		    (buflen <= 0 || (slot % 8) == 7))
    717  1.103       mrg 			IOMMUREG_WRITE(is, iommu_cache_flush,
    718  1.103       mrg 			    is->is_ptsb + slot * 8);
    719  1.103       mrg 
    720  1.103       mrg 		dvmaddr += PAGE_SIZE;
    721    1.7       mrg 	}
    722   1.91  nakayama 	if (needsflush)
    723   1.91  nakayama 		iommu_strbuf_flush_done(sb);
    724   1.45       eeh #ifdef DIAGNOSTIC
    725   1.45       eeh 	for (seg = 0; seg < map->dm_nsegs; seg++) {
    726   1.45       eeh 		if (map->dm_segs[seg].ds_addr < is->is_dvmabase ||
    727   1.45       eeh 			map->dm_segs[seg].ds_addr > is->is_dvmaend) {
    728   1.45       eeh 			printf("seg %d dvmaddr %lx out of range %x - %x\n",
    729   1.71   tsutsui 			    seg, (long)map->dm_segs[seg].ds_addr,
    730   1.71   tsutsui 			    is->is_dvmabase, is->is_dvmaend);
    731   1.57       chs #ifdef DDB
    732   1.45       eeh 			Debugger();
    733   1.57       chs #endif
    734   1.45       eeh 		}
    735   1.45       eeh 	}
    736   1.45       eeh #endif
    737    1.7       mrg 	return (0);
    738    1.7       mrg }
    739    1.7       mrg 
    740    1.7       mrg 
    741    1.7       mrg void
    742   1.85  nakayama iommu_dvmamap_unload(bus_dma_tag_t t, bus_dmamap_t map)
    743    1.7       mrg {
    744   1.85  nakayama 	struct strbuf_ctl *sb = (struct strbuf_ctl *)map->_dm_cookie;
    745   1.55       eeh 	struct iommu_state *is = sb->sb_is;
    746    1.7       mrg 
    747   1.40       eeh 	/* Flush the iommu */
    748  1.113  christos 	if (!map->_dm_dvmastart)
    749  1.113  christos 		panic("%s: error dvmastart is zero!\n", __func__);
    750    1.7       mrg 
    751  1.115       mrg 	if (is->is_flags & IOMMU_SYNC_BEFORE_UNMAP) {
    752  1.115       mrg 
    753  1.115       mrg 		/* Flush the caches */
    754  1.115       mrg 		bus_dmamap_unload(t->_parent, map);
    755  1.115       mrg 
    756  1.115       mrg 		iommu_remove(is, map->_dm_dvmastart, map->_dm_dvmasize);
    757  1.115       mrg 
    758  1.115       mrg 	} else {
    759  1.115       mrg 
    760  1.115       mrg 		iommu_remove(is, map->_dm_dvmastart, map->_dm_dvmasize);
    761  1.115       mrg 
    762  1.115       mrg 		/* Flush the caches */
    763  1.115       mrg 		bus_dmamap_unload(t->_parent, map);
    764  1.115       mrg 	}
    765   1.23       eeh 
    766  1.117   thorpej 	vmem_xfree(is->is_dvmamap, map->_dm_dvmastart, map->_dm_dvmasize);
    767   1.43       eeh 	map->_dm_dvmastart = 0;
    768   1.43       eeh 	map->_dm_dvmasize = 0;
    769   1.40       eeh 
    770   1.40       eeh 	/* Clear the map */
    771    1.9       eeh }
    772    1.9       eeh 
    773    1.9       eeh 
    774    1.9       eeh int
    775   1.85  nakayama iommu_dvmamap_load_raw(bus_dma_tag_t t, bus_dmamap_t map,
    776   1.85  nakayama 	bus_dma_segment_t *segs, int nsegs, bus_size_t size, int flags)
    777    1.9       eeh {
    778   1.85  nakayama 	struct strbuf_ctl *sb = (struct strbuf_ctl *)map->_dm_cookie;
    779   1.55       eeh 	struct iommu_state *is = sb->sb_is;
    780   1.58       chs 	struct vm_page *pg;
    781  1.107       mrg 	int i, j;
    782   1.26    martin 	int left;
    783   1.91  nakayama 	int err, needsflush;
    784    1.9       eeh 	bus_size_t sgsize;
    785    1.9       eeh 	paddr_t pa;
    786   1.21       eeh 	bus_size_t boundary, align;
    787   1.90  nakayama 	u_long dvmaddr, sgstart, sgend, bmask;
    788   1.58       chs 	struct pglist *pglist;
    789   1.90  nakayama 	const int pagesz = PAGE_SIZE;
    790  1.103       mrg 	int slot;
    791   1.90  nakayama #ifdef DEBUG
    792   1.90  nakayama 	int npg = 0;
    793   1.90  nakayama #endif
    794    1.9       eeh 
    795    1.9       eeh 	if (map->dm_nsegs) {
    796    1.9       eeh 		/* Already in use?? */
    797    1.9       eeh #ifdef DIAGNOSTIC
    798    1.9       eeh 		printf("iommu_dvmamap_load_raw: map still in use\n");
    799    1.9       eeh #endif
    800    1.9       eeh 		bus_dmamap_unload(t, map);
    801    1.9       eeh 	}
    802   1.40       eeh 
    803   1.40       eeh 	/*
    804   1.40       eeh 	 * A boundary presented to bus_dmamem_alloc() takes precedence
    805   1.40       eeh 	 * over boundary in the map.
    806   1.40       eeh 	 */
    807   1.40       eeh 	if ((boundary = segs[0]._ds_boundary) == 0)
    808   1.40       eeh 		boundary = map->_dm_boundary;
    809   1.40       eeh 
    810  1.114  riastrad 	align = uimax(segs[0]._ds_align, pagesz);
    811   1.40       eeh 
    812    1.9       eeh 	/*
    813    1.9       eeh 	 * Make sure that on error condition we return "no valid mappings".
    814    1.9       eeh 	 */
    815    1.9       eeh 	map->dm_nsegs = 0;
    816   1.26    martin 	/* Count up the total number of pages we need */
    817   1.93  nakayama 	pa = trunc_page(segs[0].ds_addr);
    818   1.26    martin 	sgsize = 0;
    819   1.40       eeh 	left = size;
    820   1.93  nakayama 	for (i = 0; left > 0 && i < nsegs; i++) {
    821   1.26    martin 		if (round_page(pa) != round_page(segs[i].ds_addr))
    822   1.93  nakayama 			sgsize = round_page(sgsize) +
    823   1.93  nakayama 			    (segs[i].ds_addr & PGOFSET);
    824  1.114  riastrad 		sgsize += uimin(left, segs[i].ds_len);
    825   1.40       eeh 		left -= segs[i].ds_len;
    826   1.26    martin 		pa = segs[i].ds_addr + segs[i].ds_len;
    827   1.26    martin 	}
    828   1.93  nakayama 	sgsize = round_page(sgsize);
    829    1.9       eeh 
    830   1.58       chs 	/*
    831   1.58       chs 	 * If our segment size is larger than the boundary we need to
    832   1.45       eeh 	 * split the transfer up into little pieces ourselves.
    833    1.9       eeh 	 */
    834  1.117   thorpej 	const vm_flag_t vmflags = VM_BESTFIT |
    835  1.117   thorpej 	    ((flags & BUS_DMA_NOWAIT) ? VM_NOSLEEP : VM_SLEEP);
    836    1.9       eeh 
    837  1.117   thorpej 	err = vmem_xalloc(is->is_dvmamap, sgsize,
    838  1.117   thorpej 			  align,		/* alignment */
    839  1.117   thorpej 			  0,			/* phase */
    840  1.117   thorpej 			  (sgsize > boundary) ? 0 : boundary,
    841  1.117   thorpej 			  VMEM_ADDR_MIN,	/* minaddr */
    842  1.117   thorpej 			  VMEM_ADDR_MAX,	/* maxaddr */
    843  1.117   thorpej 			  vmflags,
    844  1.117   thorpej 			  &dvmaddr);
    845    1.9       eeh 	if (err != 0)
    846    1.9       eeh 		return (err);
    847    1.9       eeh 
    848    1.9       eeh #ifdef DEBUG
    849   1.65  nakayama 	if (dvmaddr == (u_long)-1)
    850   1.58       chs 	{
    851    1.9       eeh 		printf("iommu_dvmamap_load_raw(): extent_alloc(%d, %x) failed!\n",
    852   1.25       mrg 		    (int)sgsize, flags);
    853   1.57       chs #ifdef DDB
    854    1.9       eeh 		Debugger();
    855   1.57       chs #endif
    856   1.58       chs 	}
    857   1.58       chs #endif
    858   1.65  nakayama 	if (dvmaddr == (u_long)-1)
    859    1.9       eeh 		return (ENOMEM);
    860    1.9       eeh 
    861   1.40       eeh 	/* Set the active DVMA map */
    862   1.40       eeh 	map->_dm_dvmastart = dvmaddr;
    863   1.40       eeh 	map->_dm_dvmasize = sgsize;
    864   1.40       eeh 
    865   1.90  nakayama 	bmask = ~(boundary - 1);
    866   1.58       chs 	if ((pglist = segs[0]._ds_mlist) == NULL) {
    867   1.92  nakayama 		u_long prev_va = 0UL, last_va = dvmaddr;
    868   1.45       eeh 		paddr_t prev_pa = 0;
    869   1.45       eeh 		int end = 0, offset;
    870   1.92  nakayama 		bus_size_t len = size;
    871   1.45       eeh 
    872   1.26    martin 		/*
    873   1.45       eeh 		 * This segs is made up of individual physical
    874   1.58       chs 		 *  segments, probably by _bus_dmamap_load_uio() or
    875   1.26    martin 		 * _bus_dmamap_load_mbuf().  Ignore the mlist and
    876   1.45       eeh 		 * load each one individually.
    877   1.26    martin 		 */
    878   1.45       eeh 		j = 0;
    879   1.91  nakayama 		needsflush = 0;
    880   1.45       eeh 		for (i = 0; i < nsegs ; i++) {
    881   1.40       eeh 
    882   1.45       eeh 			pa = segs[i].ds_addr;
    883   1.45       eeh 			offset = (pa & PGOFSET);
    884   1.45       eeh 			pa = trunc_page(pa);
    885   1.45       eeh 			dvmaddr = trunc_page(dvmaddr);
    886  1.114  riastrad 			left = uimin(len, segs[i].ds_len);
    887   1.45       eeh 
    888   1.45       eeh 			DPRINTF(IDB_INFO, ("iommu_dvmamap_load_raw: converting "
    889   1.58       chs 				"physseg %d start %lx size %lx\n", i,
    890   1.61    martin 				(long)segs[i].ds_addr, (long)segs[i].ds_len));
    891   1.26    martin 
    892   1.58       chs 			if ((pa == prev_pa) &&
    893   1.47       eeh 				((offset != 0) || (end != offset))) {
    894   1.45       eeh 				/* We can re-use this mapping */
    895   1.45       eeh 				dvmaddr = prev_va;
    896   1.45       eeh 			}
    897   1.29    martin 
    898   1.45       eeh 			sgstart = dvmaddr + offset;
    899   1.45       eeh 			sgend = sgstart + left - 1;
    900   1.26    martin 
    901   1.45       eeh 			/* Are the segments virtually adjacent? */
    902   1.58       chs 			if ((j > 0) && (end == offset) &&
    903   1.96  nakayama 			    ((offset == 0) || (pa == prev_pa)) &&
    904   1.96  nakayama 			    (map->dm_segs[j-1].ds_len + left <=
    905   1.96  nakayama 			     map->dm_maxsegsz)) {
    906   1.45       eeh 				/* Just append to the previous segment. */
    907   1.45       eeh 				map->dm_segs[--j].ds_len += left;
    908   1.93  nakayama 				/* Restore sgstart for boundary check */
    909   1.93  nakayama 				sgstart = map->dm_segs[j].ds_addr;
    910   1.45       eeh 				DPRINTF(IDB_INFO, ("iommu_dvmamap_load_raw: "
    911   1.45       eeh 					"appending seg %d start %lx size %lx\n", j,
    912   1.58       chs 					(long)map->dm_segs[j].ds_addr,
    913   1.61    martin 					(long)map->dm_segs[j].ds_len));
    914   1.45       eeh 			} else {
    915   1.53       eeh 				if (j >= map->_dm_segcnt) {
    916   1.92  nakayama 					iommu_remove(is, map->_dm_dvmastart,
    917   1.92  nakayama 					    last_va - map->_dm_dvmastart);
    918   1.92  nakayama 					goto fail;
    919   1.53       eeh 				}
    920   1.45       eeh 				map->dm_segs[j].ds_addr = sgstart;
    921   1.45       eeh 				map->dm_segs[j].ds_len = left;
    922   1.45       eeh 				DPRINTF(IDB_INFO, ("iommu_dvmamap_load_raw: "
    923   1.45       eeh 					"seg %d start %lx size %lx\n", j,
    924   1.48       eeh 					(long)map->dm_segs[j].ds_addr,
    925   1.61    martin 					(long)map->dm_segs[j].ds_len));
    926   1.40       eeh 			}
    927   1.45       eeh 			end = (offset + left) & PGOFSET;
    928   1.40       eeh 
    929   1.40       eeh 			/* Check for boundary issues */
    930   1.90  nakayama 			while ((sgstart & bmask) != (sgend & bmask)) {
    931   1.40       eeh 				/* Need a new segment. */
    932   1.40       eeh 				map->dm_segs[j].ds_len =
    933   1.53       eeh 					boundary - (sgstart & (boundary - 1));
    934   1.40       eeh 				DPRINTF(IDB_INFO, ("iommu_dvmamap_load_raw: "
    935   1.40       eeh 					"seg %d start %lx size %lx\n", j,
    936   1.58       chs 					(long)map->dm_segs[j].ds_addr,
    937   1.61    martin 					(long)map->dm_segs[j].ds_len));
    938   1.53       eeh 				if (++j >= map->_dm_segcnt) {
    939   1.92  nakayama 					iommu_remove(is, map->_dm_dvmastart,
    940   1.92  nakayama 					    last_va - map->_dm_dvmastart);
    941   1.92  nakayama 					goto fail;
    942   1.40       eeh 				}
    943   1.93  nakayama 				sgstart += map->dm_segs[j-1].ds_len;
    944   1.40       eeh 				map->dm_segs[j].ds_addr = sgstart;
    945   1.40       eeh 				map->dm_segs[j].ds_len = sgend - sgstart + 1;
    946   1.40       eeh 			}
    947   1.40       eeh 
    948   1.26    martin 			if (sgsize == 0)
    949   1.26    martin 				panic("iommu_dmamap_load_raw: size botch");
    950   1.40       eeh 
    951   1.45       eeh 			/* Now map a series of pages. */
    952   1.51       eeh 			while (dvmaddr <= sgend) {
    953   1.45       eeh 				DPRINTF(IDB_BUSDMA,
    954   1.45       eeh 					("iommu_dvmamap_load_raw: map %p "
    955   1.45       eeh 						"loading va %lx at pa %lx\n",
    956   1.45       eeh 						map, (long)dvmaddr,
    957   1.45       eeh 						(long)(pa)));
    958   1.45       eeh 				/* Enter it if we haven't before. */
    959   1.91  nakayama 				if (prev_va != dvmaddr) {
    960   1.55       eeh 					iommu_enter(sb, prev_va = dvmaddr,
    961   1.90  nakayama 					    prev_pa = pa,
    962   1.90  nakayama 					    flags | IOTTE_DEBUG(++npg << 12));
    963   1.91  nakayama 					needsflush = 1;
    964  1.103       mrg 
    965  1.103       mrg 					/* Flush cache if necessary. */
    966  1.103       mrg 					slot = IOTSBSLOT(trunc_page(dvmaddr), is->is_tsbsize);
    967  1.103       mrg 					if ((is->is_flags & IOMMU_FLUSH_CACHE) &&
    968  1.103       mrg 					    ((dvmaddr + pagesz) > sgend || (slot % 8) == 7))
    969  1.103       mrg 						IOMMUREG_WRITE(is, iommu_cache_flush,
    970  1.103       mrg 						    is->is_ptsb + slot * 8);
    971   1.91  nakayama 				}
    972  1.103       mrg 
    973   1.45       eeh 				dvmaddr += pagesz;
    974   1.45       eeh 				pa += pagesz;
    975   1.92  nakayama 				last_va = dvmaddr;
    976   1.45       eeh 			}
    977   1.45       eeh 
    978   1.92  nakayama 			len -= left;
    979   1.45       eeh 			++j;
    980   1.26    martin 		}
    981   1.91  nakayama 		if (needsflush)
    982   1.91  nakayama 			iommu_strbuf_flush_done(sb);
    983   1.45       eeh 
    984   1.92  nakayama 		map->dm_mapsize = size;
    985   1.45       eeh 		map->dm_nsegs = j;
    986   1.45       eeh #ifdef DIAGNOSTIC
    987   1.45       eeh 		{ int seg;
    988   1.45       eeh 	for (seg = 0; seg < map->dm_nsegs; seg++) {
    989   1.45       eeh 		if (map->dm_segs[seg].ds_addr < is->is_dvmabase ||
    990  1.103       mrg 		    map->dm_segs[seg].ds_addr > is->is_dvmaend) {
    991   1.45       eeh 			printf("seg %d dvmaddr %lx out of range %x - %x\n",
    992   1.58       chs 				seg, (long)map->dm_segs[seg].ds_addr,
    993   1.45       eeh 				is->is_dvmabase, is->is_dvmaend);
    994   1.57       chs #ifdef DDB
    995   1.45       eeh 			Debugger();
    996   1.57       chs #endif
    997   1.45       eeh 		}
    998   1.45       eeh 	}
    999   1.45       eeh 		}
   1000   1.45       eeh #endif
   1001   1.26    martin 		return (0);
   1002   1.26    martin 	}
   1003   1.58       chs 
   1004    1.9       eeh 	/*
   1005   1.40       eeh 	 * This was allocated with bus_dmamem_alloc.
   1006   1.58       chs 	 * The pages are on a `pglist'.
   1007    1.9       eeh 	 */
   1008   1.26    martin 	i = 0;
   1009   1.40       eeh 	sgstart = dvmaddr;
   1010   1.40       eeh 	sgend = sgstart + size - 1;
   1011   1.40       eeh 	map->dm_segs[i].ds_addr = sgstart;
   1012   1.90  nakayama 	while ((sgstart & bmask) != (sgend & bmask)) {
   1013   1.40       eeh 		/* Oops.  We crossed a boundary.  Split the xfer. */
   1014   1.53       eeh 		map->dm_segs[i].ds_len = boundary - (sgstart & (boundary - 1));
   1015   1.40       eeh 		DPRINTF(IDB_INFO, ("iommu_dvmamap_load_raw: "
   1016   1.40       eeh 			"seg %d start %lx size %lx\n", i,
   1017   1.48       eeh 			(long)map->dm_segs[i].ds_addr,
   1018   1.61    martin 			(long)map->dm_segs[i].ds_len));
   1019   1.53       eeh 		if (++i >= map->_dm_segcnt) {
   1020   1.40       eeh 			/* Too many segments.  Fail the operation. */
   1021   1.92  nakayama 			goto fail;
   1022   1.40       eeh 		}
   1023   1.93  nakayama 		sgstart += map->dm_segs[i-1].ds_len;
   1024   1.40       eeh 		map->dm_segs[i].ds_addr = sgstart;
   1025   1.40       eeh 	}
   1026   1.40       eeh 	DPRINTF(IDB_INFO, ("iommu_dvmamap_load_raw: "
   1027   1.40       eeh 			"seg %d start %lx size %lx\n", i,
   1028   1.61    martin 			(long)map->dm_segs[i].ds_addr, (long)map->dm_segs[i].ds_len));
   1029   1.40       eeh 	map->dm_segs[i].ds_len = sgend - sgstart + 1;
   1030    1.9       eeh 
   1031   1.91  nakayama 	needsflush = 0;
   1032   1.83        ad 	TAILQ_FOREACH(pg, pglist, pageq.queue) {
   1033    1.9       eeh 		if (sgsize == 0)
   1034    1.9       eeh 			panic("iommu_dmamap_load_raw: size botch");
   1035   1.58       chs 		pa = VM_PAGE_TO_PHYS(pg);
   1036    1.9       eeh 
   1037   1.22       mrg 		DPRINTF(IDB_BUSDMA,
   1038    1.9       eeh 		    ("iommu_dvmamap_load_raw: map %p loading va %lx at pa %lx\n",
   1039    1.9       eeh 		    map, (long)dvmaddr, (long)(pa)));
   1040   1.90  nakayama 		iommu_enter(sb, dvmaddr, pa, flags | IOTTE_DEBUG(0x8000));
   1041   1.91  nakayama 		needsflush = 1;
   1042   1.58       chs 
   1043  1.103       mrg 		sgsize -= pagesz;
   1044  1.103       mrg 
   1045  1.103       mrg 		/* Flush cache if necessary. */
   1046  1.103       mrg 		slot = IOTSBSLOT(trunc_page(dvmaddr), is->is_tsbsize);
   1047  1.103       mrg 		if ((is->is_flags & IOMMU_FLUSH_CACHE) &&
   1048  1.103       mrg 		    (sgsize == 0 || (slot % 8) == 7))
   1049  1.103       mrg 			IOMMUREG_WRITE(is, iommu_cache_flush,
   1050  1.103       mrg 			    is->is_ptsb + slot * 8);
   1051  1.103       mrg 
   1052  1.102       mrg 		dvmaddr += pagesz;
   1053    1.9       eeh 	}
   1054   1.91  nakayama 	if (needsflush)
   1055   1.91  nakayama 		iommu_strbuf_flush_done(sb);
   1056   1.40       eeh 	map->dm_mapsize = size;
   1057   1.40       eeh 	map->dm_nsegs = i+1;
   1058   1.45       eeh #ifdef DIAGNOSTIC
   1059   1.45       eeh 	{ int seg;
   1060   1.45       eeh 	for (seg = 0; seg < map->dm_nsegs; seg++) {
   1061   1.45       eeh 		if (map->dm_segs[seg].ds_addr < is->is_dvmabase ||
   1062   1.45       eeh 			map->dm_segs[seg].ds_addr > is->is_dvmaend) {
   1063   1.45       eeh 			printf("seg %d dvmaddr %lx out of range %x - %x\n",
   1064   1.58       chs 				seg, (long)map->dm_segs[seg].ds_addr,
   1065   1.45       eeh 				is->is_dvmabase, is->is_dvmaend);
   1066   1.57       chs #ifdef DDB
   1067   1.45       eeh 			Debugger();
   1068   1.57       chs #endif
   1069   1.45       eeh 		}
   1070   1.45       eeh 	}
   1071   1.45       eeh 	}
   1072   1.45       eeh #endif
   1073    1.9       eeh 	return (0);
   1074   1.92  nakayama 
   1075   1.92  nakayama fail:
   1076  1.117   thorpej 	vmem_free(is->is_dvmamap, map->_dm_dvmastart, sgsize);
   1077   1.92  nakayama 	map->_dm_dvmastart = 0;
   1078   1.92  nakayama 	map->_dm_dvmasize = 0;
   1079   1.92  nakayama 	return (EFBIG);
   1080    1.7       mrg }
   1081    1.7       mrg 
   1082   1.67    petrov 
   1083   1.67    petrov /*
   1084   1.67    petrov  * Flush an individual dma segment, returns non-zero if the streaming buffers
   1085   1.67    petrov  * need flushing afterwards.
   1086   1.67    petrov  */
   1087   1.67    petrov static int
   1088   1.67    petrov iommu_dvmamap_sync_range(struct strbuf_ctl *sb, vaddr_t va, bus_size_t len)
   1089   1.67    petrov {
   1090   1.67    petrov 	vaddr_t vaend;
   1091   1.67    petrov 	struct iommu_state *is = sb->sb_is;
   1092   1.67    petrov 
   1093   1.67    petrov #ifdef DIAGNOSTIC
   1094   1.67    petrov 	if (va < is->is_dvmabase || va > is->is_dvmaend)
   1095   1.67    petrov 		panic("invalid va: %llx", (long long)va);
   1096   1.67    petrov #endif
   1097   1.67    petrov 
   1098   1.67    petrov 	if ((is->is_tsb[IOTSBSLOT(va, is->is_tsbsize)] & IOTTE_STREAM) == 0) {
   1099  1.103       mrg 		DPRINTF(IDB_SYNC,
   1100   1.67    petrov 			("iommu_dvmamap_sync_range: attempting to flush "
   1101   1.67    petrov 			 "non-streaming entry\n"));
   1102   1.67    petrov 		return (0);
   1103   1.67    petrov 	}
   1104   1.67    petrov 
   1105   1.90  nakayama 	vaend = round_page(va + len) - 1;
   1106   1.87  nakayama 	va = trunc_page(va);
   1107   1.67    petrov 
   1108   1.67    petrov #ifdef DIAGNOSTIC
   1109   1.67    petrov 	if (va < is->is_dvmabase || vaend > is->is_dvmaend)
   1110   1.67    petrov 		panic("invalid va range: %llx to %llx (%x to %x)",
   1111   1.67    petrov 		    (long long)va, (long long)vaend,
   1112   1.67    petrov 		    is->is_dvmabase,
   1113   1.67    petrov 		    is->is_dvmaend);
   1114   1.67    petrov #endif
   1115   1.67    petrov 
   1116   1.67    petrov 	for ( ; va <= vaend; va += PAGE_SIZE) {
   1117  1.103       mrg 		DPRINTF(IDB_SYNC,
   1118   1.67    petrov 		    ("iommu_dvmamap_sync_range: flushing va %p\n",
   1119   1.67    petrov 		    (void *)(u_long)va));
   1120   1.67    petrov 		iommu_strbuf_flush(sb, va);
   1121   1.67    petrov 	}
   1122   1.67    petrov 
   1123   1.67    petrov 	return (1);
   1124   1.67    petrov }
   1125   1.67    petrov 
   1126   1.85  nakayama static void
   1127   1.85  nakayama _iommu_dvmamap_sync(bus_dma_tag_t t, bus_dmamap_t map, bus_addr_t offset,
   1128   1.85  nakayama 	bus_size_t len, int ops)
   1129    1.7       mrg {
   1130   1.85  nakayama 	struct strbuf_ctl *sb = (struct strbuf_ctl *)map->_dm_cookie;
   1131   1.67    petrov 	bus_size_t count;
   1132   1.67    petrov 	int i, needsflush = 0;
   1133   1.63    petrov 
   1134   1.63    petrov 	if (!sb->sb_flush)
   1135   1.63    petrov 		return;
   1136    1.7       mrg 
   1137   1.67    petrov 	for (i = 0; i < map->dm_nsegs; i++) {
   1138   1.67    petrov 		if (offset < map->dm_segs[i].ds_len)
   1139   1.67    petrov 			break;
   1140   1.67    petrov 		offset -= map->dm_segs[i].ds_len;
   1141   1.67    petrov 	}
   1142   1.60    petrov 
   1143   1.67    petrov 	if (i == map->dm_nsegs)
   1144  1.103       mrg 		panic("%s: segment too short %llu", __func__,
   1145   1.68    martin 		    (unsigned long long)offset);
   1146   1.60    petrov 
   1147   1.62    petrov 	if (ops & (BUS_DMASYNC_PREREAD | BUS_DMASYNC_POSTWRITE)) {
   1148   1.60    petrov 		/* Nothing to do */;
   1149   1.60    petrov 	}
   1150   1.60    petrov 
   1151   1.62    petrov 	if (ops & (BUS_DMASYNC_POSTREAD | BUS_DMASYNC_PREWRITE)) {
   1152   1.67    petrov 
   1153   1.67    petrov 		for (; len > 0 && i < map->dm_nsegs; i++) {
   1154   1.67    petrov 			count = MIN(map->dm_segs[i].ds_len - offset, len);
   1155   1.67    petrov 			if (count > 0 &&
   1156   1.67    petrov 			    iommu_dvmamap_sync_range(sb,
   1157   1.67    petrov 				map->dm_segs[i].ds_addr + offset, count))
   1158   1.67    petrov 				needsflush = 1;
   1159   1.67    petrov 			offset = 0;
   1160   1.67    petrov 			len -= count;
   1161   1.67    petrov 		}
   1162   1.60    petrov #ifdef DIAGNOSTIC
   1163   1.67    petrov 		if (i == map->dm_nsegs && len > 0)
   1164  1.103       mrg 			panic("%s: leftover %llu", __func__,
   1165   1.73  nakayama 			    (unsigned long long)len);
   1166   1.60    petrov #endif
   1167   1.55       eeh 
   1168   1.67    petrov 		if (needsflush)
   1169   1.58       chs 			iommu_strbuf_flush_done(sb);
   1170    1.7       mrg 	}
   1171    1.7       mrg }
   1172    1.7       mrg 
   1173   1.85  nakayama void
   1174   1.85  nakayama iommu_dvmamap_sync(bus_dma_tag_t t, bus_dmamap_t map, bus_addr_t offset,
   1175   1.85  nakayama 	bus_size_t len, int ops)
   1176   1.85  nakayama {
   1177   1.85  nakayama 
   1178   1.89       jdc 	/* If len is 0, then there is nothing to do */
   1179   1.89       jdc 	if (len == 0)
   1180   1.89       jdc 		return;
   1181   1.89       jdc 
   1182   1.85  nakayama 	if (ops & (BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE)) {
   1183   1.85  nakayama 		/* Flush the CPU then the IOMMU */
   1184   1.85  nakayama 		bus_dmamap_sync(t->_parent, map, offset, len, ops);
   1185   1.85  nakayama 		_iommu_dvmamap_sync(t, map, offset, len, ops);
   1186   1.85  nakayama 	}
   1187   1.85  nakayama 	if (ops & (BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE)) {
   1188   1.85  nakayama 		/* Flush the IOMMU then the CPU */
   1189   1.85  nakayama 		_iommu_dvmamap_sync(t, map, offset, len, ops);
   1190   1.85  nakayama 		bus_dmamap_sync(t->_parent, map, offset, len, ops);
   1191   1.85  nakayama 	}
   1192   1.85  nakayama }
   1193   1.85  nakayama 
   1194    1.7       mrg int
   1195   1.85  nakayama iommu_dvmamem_alloc(bus_dma_tag_t t, bus_size_t size, bus_size_t alignment,
   1196   1.85  nakayama 	bus_size_t boundary, bus_dma_segment_t *segs, int nsegs, int *rsegs,
   1197   1.85  nakayama 	int flags)
   1198    1.7       mrg {
   1199    1.7       mrg 
   1200   1.25       mrg 	DPRINTF(IDB_BUSDMA, ("iommu_dvmamem_alloc: sz %llx align %llx bound %llx "
   1201   1.25       mrg 	   "segp %p flags %d\n", (unsigned long long)size,
   1202   1.25       mrg 	   (unsigned long long)alignment, (unsigned long long)boundary,
   1203   1.25       mrg 	   segs, flags));
   1204    1.7       mrg 	return (bus_dmamem_alloc(t->_parent, size, alignment, boundary,
   1205   1.21       eeh 	    segs, nsegs, rsegs, flags|BUS_DMA_DVMA));
   1206    1.7       mrg }
   1207    1.7       mrg 
   1208    1.7       mrg void
   1209   1.85  nakayama iommu_dvmamem_free(bus_dma_tag_t t, bus_dma_segment_t *segs, int nsegs)
   1210    1.7       mrg {
   1211    1.7       mrg 
   1212   1.22       mrg 	DPRINTF(IDB_BUSDMA, ("iommu_dvmamem_free: segp %p nsegs %d\n",
   1213    1.7       mrg 	    segs, nsegs));
   1214    1.7       mrg 	bus_dmamem_free(t->_parent, segs, nsegs);
   1215    1.7       mrg }
   1216    1.7       mrg 
   1217    1.7       mrg /*
   1218    1.7       mrg  * Map the DVMA mappings into the kernel pmap.
   1219    1.7       mrg  * Check the flags to see whether we're streaming or coherent.
   1220    1.7       mrg  */
   1221    1.7       mrg int
   1222   1.85  nakayama iommu_dvmamem_map(bus_dma_tag_t t, bus_dma_segment_t *segs, int nsegs,
   1223   1.85  nakayama 	size_t size, void **kvap, int flags)
   1224    1.7       mrg {
   1225   1.58       chs 	struct vm_page *pg;
   1226    1.7       mrg 	vaddr_t va;
   1227    1.7       mrg 	bus_addr_t addr;
   1228   1.58       chs 	struct pglist *pglist;
   1229    1.8       mrg 	int cbit;
   1230   1.77      yamt 	const uvm_flag_t kmflags =
   1231   1.77      yamt 	    (flags & BUS_DMA_NOWAIT) != 0 ? UVM_KMF_NOWAIT : 0;
   1232    1.7       mrg 
   1233   1.22       mrg 	DPRINTF(IDB_BUSDMA, ("iommu_dvmamem_map: segp %p nsegs %d size %lx\n",
   1234    1.7       mrg 	    segs, nsegs, size));
   1235    1.7       mrg 
   1236    1.7       mrg 	/*
   1237    1.8       mrg 	 * Allocate some space in the kernel map, and then map these pages
   1238    1.8       mrg 	 * into this space.
   1239    1.7       mrg 	 */
   1240    1.8       mrg 	size = round_page(size);
   1241   1.77      yamt 	va = uvm_km_alloc(kernel_map, size, 0, UVM_KMF_VAONLY | kmflags);
   1242    1.8       mrg 	if (va == 0)
   1243    1.8       mrg 		return (ENOMEM);
   1244    1.7       mrg 
   1245   1.81  christos 	*kvap = (void *)va;
   1246    1.7       mrg 
   1247   1.58       chs 	/*
   1248    1.7       mrg 	 * digest flags:
   1249    1.7       mrg 	 */
   1250    1.7       mrg 	cbit = 0;
   1251    1.7       mrg 	if (flags & BUS_DMA_COHERENT)	/* Disable vcache */
   1252    1.7       mrg 		cbit |= PMAP_NVC;
   1253   1.97     skrll 	if (flags & BUS_DMA_NOCACHE)	/* side effects */
   1254    1.7       mrg 		cbit |= PMAP_NC;
   1255    1.7       mrg 
   1256    1.7       mrg 	/*
   1257    1.8       mrg 	 * Now take this and map it into the CPU.
   1258    1.7       mrg 	 */
   1259   1.58       chs 	pglist = segs[0]._ds_mlist;
   1260   1.83        ad 	TAILQ_FOREACH(pg, pglist, pageq.queue) {
   1261    1.8       mrg #ifdef DIAGNOSTIC
   1262    1.7       mrg 		if (size == 0)
   1263    1.7       mrg 			panic("iommu_dvmamem_map: size botch");
   1264    1.8       mrg #endif
   1265   1.58       chs 		addr = VM_PAGE_TO_PHYS(pg);
   1266   1.22       mrg 		DPRINTF(IDB_BUSDMA, ("iommu_dvmamem_map: "
   1267   1.25       mrg 		    "mapping va %lx at %llx\n", va, (unsigned long long)addr | cbit));
   1268   1.88    cegger 		pmap_kenter_pa(va, addr | cbit,
   1269   1.88    cegger 		    VM_PROT_READ | VM_PROT_WRITE, 0);
   1270    1.7       mrg 		va += PAGE_SIZE;
   1271    1.7       mrg 		size -= PAGE_SIZE;
   1272    1.7       mrg 	}
   1273   1.38     chris 	pmap_update(pmap_kernel());
   1274    1.7       mrg 	return (0);
   1275    1.7       mrg }
   1276    1.7       mrg 
   1277    1.7       mrg /*
   1278    1.7       mrg  * Unmap DVMA mappings from kernel
   1279    1.7       mrg  */
   1280    1.7       mrg void
   1281   1.85  nakayama iommu_dvmamem_unmap(bus_dma_tag_t t, void *kva, size_t size)
   1282    1.7       mrg {
   1283   1.58       chs 
   1284   1.22       mrg 	DPRINTF(IDB_BUSDMA, ("iommu_dvmamem_unmap: kvm %p size %lx\n",
   1285    1.7       mrg 	    kva, size));
   1286   1.58       chs 
   1287    1.7       mrg #ifdef DIAGNOSTIC
   1288    1.7       mrg 	if ((u_long)kva & PGOFSET)
   1289    1.7       mrg 		panic("iommu_dvmamem_unmap");
   1290    1.7       mrg #endif
   1291   1.58       chs 
   1292    1.7       mrg 	size = round_page(size);
   1293   1.58       chs 	pmap_kremove((vaddr_t)kva, size);
   1294   1.38     chris 	pmap_update(pmap_kernel());
   1295   1.76      yamt 	uvm_km_free(kernel_map, (vaddr_t)kva, size, UVM_KMF_VAONLY);
   1296    1.1       mrg }
   1297