Home | History | Annotate | Line # | Download | only in dev
iommu.c revision 1.35.2.8
      1  1.35.2.8  jdolecek /*	$NetBSD: iommu.c,v 1.35.2.8 2002/10/10 18:36:32 jdolecek Exp $	*/
      2       1.7       mrg 
      3       1.7       mrg /*
      4  1.35.2.5  jdolecek  * Copyright (c) 2001, 2002 Eduardo Horvath
      5       1.7       mrg  * Copyright (c) 1999, 2000 Matthew R. Green
      6       1.7       mrg  * All rights reserved.
      7       1.7       mrg  *
      8       1.7       mrg  * Redistribution and use in source and binary forms, with or without
      9       1.7       mrg  * modification, are permitted provided that the following conditions
     10       1.7       mrg  * are met:
     11       1.7       mrg  * 1. Redistributions of source code must retain the above copyright
     12       1.7       mrg  *    notice, this list of conditions and the following disclaimer.
     13       1.7       mrg  * 2. Redistributions in binary form must reproduce the above copyright
     14       1.7       mrg  *    notice, this list of conditions and the following disclaimer in the
     15       1.7       mrg  *    documentation and/or other materials provided with the distribution.
     16       1.7       mrg  * 3. The name of the author may not be used to endorse or promote products
     17       1.7       mrg  *    derived from this software without specific prior written permission.
     18       1.7       mrg  *
     19       1.7       mrg  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
     20       1.7       mrg  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
     21       1.7       mrg  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
     22       1.7       mrg  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
     23       1.7       mrg  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
     24       1.7       mrg  * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
     25       1.7       mrg  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
     26       1.7       mrg  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
     27       1.7       mrg  * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
     28       1.7       mrg  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
     29       1.7       mrg  * SUCH DAMAGE.
     30       1.7       mrg  */
     31       1.1       mrg 
     32       1.7       mrg /*
     33       1.7       mrg  * UltraSPARC IOMMU support; used by both the sbus and pci code.
     34       1.7       mrg  */
     35       1.4       mrg #include "opt_ddb.h"
     36       1.4       mrg 
     37       1.1       mrg #include <sys/param.h>
     38       1.1       mrg #include <sys/extent.h>
     39       1.1       mrg #include <sys/malloc.h>
     40       1.1       mrg #include <sys/systm.h>
     41       1.1       mrg #include <sys/device.h>
     42  1.35.2.4   thorpej #include <sys/proc.h>
     43      1.18       mrg 
     44      1.18       mrg #include <uvm/uvm_extern.h>
     45       1.1       mrg 
     46       1.1       mrg #include <machine/bus.h>
     47       1.7       mrg #include <sparc64/sparc64/cache.h>
     48       1.1       mrg #include <sparc64/dev/iommureg.h>
     49       1.1       mrg #include <sparc64/dev/iommuvar.h>
     50       1.1       mrg 
     51       1.1       mrg #include <machine/autoconf.h>
     52       1.1       mrg #include <machine/cpu.h>
     53       1.1       mrg 
     54       1.1       mrg #ifdef DEBUG
     55      1.22       mrg #define IDB_BUSDMA	0x1
     56      1.22       mrg #define IDB_IOMMU	0x2
     57      1.22       mrg #define IDB_INFO	0x4
     58  1.35.2.1     lukem #define	IDB_SYNC	0x8
     59      1.10       mrg int iommudebug = 0x0;
     60       1.4       mrg #define DPRINTF(l, s)   do { if (iommudebug & l) printf s; } while (0)
     61       1.4       mrg #else
     62       1.4       mrg #define DPRINTF(l, s)
     63       1.1       mrg #endif
     64       1.1       mrg 
     65  1.35.2.7  jdolecek #define iommu_strbuf_flush(i, v) do {					\
     66  1.35.2.7  jdolecek 	if ((i)->sb_flush)						\
     67  1.35.2.7  jdolecek 		bus_space_write_8((i)->sb_is->is_bustag, (i)->sb_sb,	\
     68  1.35.2.6  jdolecek 			STRBUFREG(strbuf_pgflush), (v));		\
     69  1.35.2.4   thorpej 	} while (0)
     70  1.35.2.4   thorpej 
     71  1.35.2.7  jdolecek static	int iommu_strbuf_flush_done __P((struct strbuf_ctl *));
     72      1.11       eeh 
     73       1.1       mrg /*
     74       1.1       mrg  * initialise the UltraSPARC IOMMU (SBUS or PCI):
     75       1.1       mrg  *	- allocate and setup the iotsb.
     76       1.1       mrg  *	- enable the IOMMU
     77       1.7       mrg  *	- initialise the streaming buffers (if they exist)
     78       1.1       mrg  *	- create a private DVMA map.
     79       1.1       mrg  */
     80       1.1       mrg void
     81  1.35.2.1     lukem iommu_init(name, is, tsbsize, iovabase)
     82       1.1       mrg 	char *name;
     83       1.1       mrg 	struct iommu_state *is;
     84       1.1       mrg 	int tsbsize;
     85  1.35.2.1     lukem 	u_int32_t iovabase;
     86       1.1       mrg {
     87      1.11       eeh 	psize_t size;
     88      1.11       eeh 	vaddr_t va;
     89      1.11       eeh 	paddr_t pa;
     90  1.35.2.8  jdolecek 	struct vm_page *pg;
     91  1.35.2.8  jdolecek 	struct pglist pglist;
     92       1.1       mrg 
     93       1.1       mrg 	/*
     94       1.1       mrg 	 * Setup the iommu.
     95       1.1       mrg 	 *
     96  1.35.2.5  jdolecek 	 * The sun4u iommu is part of the SBUS or PCI controller so we will
     97  1.35.2.5  jdolecek 	 * deal with it here..
     98       1.1       mrg 	 *
     99  1.35.2.5  jdolecek 	 * For sysio and psycho/psycho+ the IOMMU address space always ends at
    100  1.35.2.5  jdolecek 	 * 0xffffe000, but the starting address depends on the size of the
    101  1.35.2.5  jdolecek 	 * map.  The map size is 1024 * 2 ^ is->is_tsbsize entries, where each
    102  1.35.2.5  jdolecek 	 * entry is 8 bytes.  The start of the map can be calculated by
    103  1.35.2.5  jdolecek 	 * (0xffffe000 << (8 + is->is_tsbsize)).
    104  1.35.2.5  jdolecek 	 *
    105  1.35.2.5  jdolecek 	 * But sabre and hummingbird use a different scheme that seems to
    106  1.35.2.5  jdolecek 	 * be hard-wired, so we read the start and size from the PROM and
    107  1.35.2.5  jdolecek 	 * just use those values.
    108       1.2       eeh 	 */
    109      1.11       eeh 	is->is_cr = (tsbsize << 16) | IOMMUCR_EN;
    110      1.11       eeh 	is->is_tsbsize = tsbsize;
    111  1.35.2.5  jdolecek 	if (iovabase == -1) {
    112  1.35.2.5  jdolecek 		is->is_dvmabase = IOTSB_VSTART(is->is_tsbsize);
    113  1.35.2.5  jdolecek 		is->is_dvmaend = IOTSB_VEND;
    114  1.35.2.5  jdolecek 	} else {
    115  1.35.2.5  jdolecek 		is->is_dvmabase = iovabase;
    116  1.35.2.5  jdolecek 		is->is_dvmaend = iovabase + IOTSB_VSIZE(tsbsize);
    117  1.35.2.5  jdolecek 	}
    118      1.11       eeh 
    119      1.11       eeh 	/*
    120      1.15       eeh 	 * Allocate memory for I/O pagetables.  They need to be physically
    121      1.15       eeh 	 * contiguous.
    122      1.11       eeh 	 */
    123      1.11       eeh 
    124  1.35.2.8  jdolecek 	size = NBPG << is->is_tsbsize;
    125      1.11       eeh 	if (uvm_pglistalloc((psize_t)size, (paddr_t)0, (paddr_t)-1,
    126  1.35.2.8  jdolecek 		(paddr_t)NBPG, (paddr_t)0, &pglist, 1, 0) != 0)
    127      1.11       eeh 		panic("iommu_init: no memory");
    128      1.11       eeh 
    129      1.11       eeh 	va = uvm_km_valloc(kernel_map, size);
    130      1.11       eeh 	if (va == 0)
    131      1.11       eeh 		panic("iommu_init: no memory");
    132      1.11       eeh 	is->is_tsb = (int64_t *)va;
    133      1.11       eeh 
    134  1.35.2.8  jdolecek 	is->is_ptsb = VM_PAGE_TO_PHYS(TAILQ_FIRST(&pglist));
    135      1.11       eeh 
    136      1.11       eeh 	/* Map the pages */
    137  1.35.2.8  jdolecek 	TAILQ_FOREACH(pg, &pglist, pageq) {
    138  1.35.2.8  jdolecek 		pa = VM_PAGE_TO_PHYS(pg);
    139  1.35.2.8  jdolecek 		pmap_kenter_pa(va, pa | PMAP_NVC, VM_PROT_READ | VM_PROT_WRITE);
    140      1.11       eeh 		va += NBPG;
    141      1.11       eeh 	}
    142  1.35.2.3   thorpej 	pmap_update(pmap_kernel());
    143  1.35.2.8  jdolecek 	memset(is->is_tsb, 0, size);
    144       1.1       mrg 
    145       1.1       mrg #ifdef DEBUG
    146      1.22       mrg 	if (iommudebug & IDB_INFO)
    147       1.1       mrg 	{
    148       1.1       mrg 		/* Probe the iommu */
    149       1.1       mrg 
    150      1.25       mrg 		printf("iommu regs at: cr=%lx tsb=%lx flush=%lx\n",
    151  1.35.2.6  jdolecek 			(u_long)bus_space_read_8(is->is_bustag, is->is_iommu,
    152  1.35.2.6  jdolecek 				offsetof (struct iommureg, iommu_cr)),
    153  1.35.2.6  jdolecek 			(u_long)bus_space_read_8(is->is_bustag, is->is_iommu,
    154  1.35.2.6  jdolecek 				offsetof (struct iommureg, iommu_tsb)),
    155  1.35.2.6  jdolecek 			(u_long)bus_space_read_8(is->is_bustag, is->is_iommu,
    156  1.35.2.6  jdolecek 				offsetof (struct iommureg, iommu_flush)));
    157  1.35.2.6  jdolecek 		printf("iommu cr=%llx tsb=%llx\n",
    158  1.35.2.6  jdolecek 			(unsigned long long)bus_space_read_8(is->is_bustag,
    159  1.35.2.6  jdolecek 				is->is_iommu,
    160  1.35.2.6  jdolecek 				offsetof (struct iommureg, iommu_cr)),
    161  1.35.2.6  jdolecek 			(unsigned long long)bus_space_read_8(is->is_bustag,
    162  1.35.2.6  jdolecek 				is->is_iommu,
    163  1.35.2.6  jdolecek 				offsetof (struct iommureg, iommu_tsb)));
    164  1.35.2.8  jdolecek 		printf("TSB base %p phys %llx\n", (void *)is->is_tsb,
    165  1.35.2.6  jdolecek 			(unsigned long long)is->is_ptsb);
    166       1.1       mrg 		delay(1000000); /* 1 s */
    167       1.1       mrg 	}
    168       1.1       mrg #endif
    169       1.1       mrg 
    170       1.1       mrg 	/*
    171       1.1       mrg 	 * now actually start up the IOMMU
    172       1.1       mrg 	 */
    173       1.1       mrg 	iommu_reset(is);
    174       1.1       mrg 
    175       1.1       mrg 	/*
    176       1.1       mrg 	 * Now all the hardware's working we need to allocate a dvma map.
    177       1.1       mrg 	 */
    178  1.35.2.8  jdolecek 	printf("DVMA map: %x to %x\n",
    179      1.11       eeh 		(unsigned int)is->is_dvmabase,
    180  1.35.2.5  jdolecek 		(unsigned int)is->is_dvmaend);
    181  1.35.2.8  jdolecek 	printf("IOTSB: %llx to %llx\n",
    182  1.35.2.5  jdolecek 		(unsigned long long)is->is_ptsb,
    183  1.35.2.5  jdolecek 		(unsigned long long)(is->is_ptsb + size));
    184       1.1       mrg 	is->is_dvmamap = extent_create(name,
    185  1.35.2.5  jdolecek 				       is->is_dvmabase, is->is_dvmaend - NBPG,
    186       1.1       mrg 				       M_DEVBUF, 0, 0, EX_NOWAIT);
    187       1.1       mrg }
    188       1.1       mrg 
    189       1.8       mrg /*
    190       1.8       mrg  * Streaming buffers don't exist on the UltraSPARC IIi; we should have
    191       1.8       mrg  * detected that already and disabled them.  If not, we will notice that
    192       1.8       mrg  * they aren't there when the STRBUF_EN bit does not remain.
    193       1.8       mrg  */
    194       1.1       mrg void
    195       1.1       mrg iommu_reset(is)
    196       1.1       mrg 	struct iommu_state *is;
    197       1.1       mrg {
    198  1.35.2.5  jdolecek 	int i;
    199  1.35.2.7  jdolecek 	struct strbuf_ctl *sb;
    200       1.1       mrg 
    201       1.1       mrg 	/* Need to do 64-bit stores */
    202  1.35.2.8  jdolecek 	bus_space_write_8(is->is_bustag, is->is_iommu, IOMMUREG(iommu_tsb),
    203  1.35.2.6  jdolecek 		is->is_ptsb);
    204  1.35.2.6  jdolecek 
    205      1.11       eeh 	/* Enable IOMMU in diagnostic mode */
    206  1.35.2.6  jdolecek 	bus_space_write_8(is->is_bustag, is->is_iommu, IOMMUREG(iommu_cr),
    207  1.35.2.6  jdolecek 		is->is_cr|IOMMUCR_DE);
    208       1.5       mrg 
    209  1.35.2.8  jdolecek 	for (i = 0; i < 2; i++) {
    210  1.35.2.7  jdolecek 		if ((sb = is->is_sb[i])) {
    211  1.35.2.4   thorpej 
    212  1.35.2.5  jdolecek 			/* Enable diagnostics mode? */
    213  1.35.2.8  jdolecek 			bus_space_write_8(is->is_bustag, is->is_sb[i]->sb_sb,
    214  1.35.2.6  jdolecek 				STRBUFREG(strbuf_ctl), STRBUF_EN);
    215  1.35.2.5  jdolecek 
    216  1.35.2.5  jdolecek 			/* No streaming buffers? Disable them */
    217  1.35.2.8  jdolecek 			if (bus_space_read_8(is->is_bustag,
    218  1.35.2.8  jdolecek 				is->is_sb[i]->sb_sb,
    219  1.35.2.7  jdolecek 				STRBUFREG(strbuf_ctl)) == 0) {
    220  1.35.2.7  jdolecek 				is->is_sb[i]->sb_flush = NULL;
    221  1.35.2.7  jdolecek 			} else {
    222  1.35.2.8  jdolecek 
    223  1.35.2.7  jdolecek 				/*
    224  1.35.2.7  jdolecek 				 * locate the pa of the flush buffer.
    225  1.35.2.7  jdolecek 				 */
    226  1.35.2.7  jdolecek 				(void)pmap_extract(pmap_kernel(),
    227  1.35.2.7  jdolecek 					(vaddr_t)is->is_sb[i]->sb_flush,
    228  1.35.2.7  jdolecek 					&is->is_sb[i]->sb_flushpa);
    229  1.35.2.7  jdolecek 			}
    230  1.35.2.5  jdolecek 		}
    231  1.35.2.4   thorpej 	}
    232       1.2       eeh }
    233       1.2       eeh 
    234       1.2       eeh /*
    235  1.35.2.8  jdolecek  * Here are the iommu control routines.
    236       1.2       eeh  */
    237       1.2       eeh void
    238  1.35.2.7  jdolecek iommu_enter(sb, va, pa, flags)
    239  1.35.2.7  jdolecek 	struct strbuf_ctl *sb;
    240       1.2       eeh 	vaddr_t va;
    241       1.2       eeh 	int64_t pa;
    242       1.2       eeh 	int flags;
    243       1.2       eeh {
    244  1.35.2.7  jdolecek 	struct iommu_state *is = sb->sb_is;
    245  1.35.2.7  jdolecek 	int strbuf = (flags & BUS_DMA_STREAMING);
    246       1.2       eeh 	int64_t tte;
    247       1.2       eeh 
    248       1.2       eeh #ifdef DIAGNOSTIC
    249  1.35.2.5  jdolecek 	if (va < is->is_dvmabase || va > is->is_dvmaend)
    250      1.13       mrg 		panic("iommu_enter: va %#lx not in DVMA space", va);
    251       1.2       eeh #endif
    252       1.2       eeh 
    253  1.35.2.7  jdolecek 	/* Is the streamcache flush really needed? */
    254  1.35.2.7  jdolecek 	if (sb->sb_flush) {
    255  1.35.2.7  jdolecek 		iommu_strbuf_flush(sb, va);
    256  1.35.2.7  jdolecek 		iommu_strbuf_flush_done(sb);
    257  1.35.2.7  jdolecek 	} else
    258  1.35.2.7  jdolecek 		/* If we can't flush the strbuf don't enable it. */
    259  1.35.2.7  jdolecek 		strbuf = 0;
    260  1.35.2.7  jdolecek 
    261  1.35.2.8  jdolecek 	tte = MAKEIOTTE(pa, !(flags & BUS_DMA_NOWRITE),
    262  1.35.2.7  jdolecek 		!(flags & BUS_DMA_NOCACHE), (strbuf));
    263  1.35.2.6  jdolecek #ifdef DEBUG
    264  1.35.2.6  jdolecek 	tte |= (flags & 0xff000LL)<<(4*8);
    265  1.35.2.6  jdolecek #endif
    266  1.35.2.8  jdolecek 
    267  1.35.2.8  jdolecek 	DPRINTF(IDB_IOMMU, ("Clearing TSB slot %d for va %p\n",
    268      1.25       mrg 		       (int)IOTSBSLOT(va,is->is_tsbsize), (void *)(u_long)va));
    269       1.2       eeh 	is->is_tsb[IOTSBSLOT(va,is->is_tsbsize)] = tte;
    270  1.35.2.8  jdolecek 	bus_space_write_8(is->is_bustag, is->is_iommu,
    271  1.35.2.6  jdolecek 		IOMMUREG(iommu_flush), va);
    272      1.22       mrg 	DPRINTF(IDB_IOMMU, ("iommu_enter: va %lx pa %lx TSB[%lx]@%p=%lx\n",
    273  1.35.2.6  jdolecek 		va, (long)pa, (u_long)IOTSBSLOT(va,is->is_tsbsize),
    274  1.35.2.6  jdolecek 		(void *)(u_long)&is->is_tsb[IOTSBSLOT(va,is->is_tsbsize)],
    275  1.35.2.6  jdolecek 		(u_long)tte));
    276       1.2       eeh }
    277       1.2       eeh 
    278  1.35.2.4   thorpej /*
    279  1.35.2.4   thorpej  * Find the value of a DVMA address (debug routine).
    280  1.35.2.4   thorpej  */
    281  1.35.2.4   thorpej paddr_t
    282  1.35.2.4   thorpej iommu_extract(is, dva)
    283  1.35.2.4   thorpej 	struct iommu_state *is;
    284  1.35.2.4   thorpej 	vaddr_t dva;
    285  1.35.2.4   thorpej {
    286  1.35.2.4   thorpej 	int64_t tte = 0;
    287  1.35.2.8  jdolecek 
    288  1.35.2.5  jdolecek 	if (dva >= is->is_dvmabase && dva < is->is_dvmaend)
    289  1.35.2.7  jdolecek 		tte = is->is_tsb[IOTSBSLOT(dva, is->is_tsbsize)];
    290  1.35.2.4   thorpej 
    291  1.35.2.6  jdolecek 	if ((tte & IOTTE_V) == 0)
    292  1.35.2.4   thorpej 		return ((paddr_t)-1L);
    293  1.35.2.6  jdolecek 	return (tte & IOTTE_PAMASK);
    294  1.35.2.4   thorpej }
    295  1.35.2.4   thorpej 
    296       1.2       eeh /*
    297       1.2       eeh  * iommu_remove: removes mappings created by iommu_enter
    298       1.2       eeh  *
    299       1.2       eeh  * Only demap from IOMMU if flag is set.
    300       1.8       mrg  *
    301       1.8       mrg  * XXX: this function needs better internal error checking.
    302       1.2       eeh  */
    303       1.2       eeh void
    304       1.2       eeh iommu_remove(is, va, len)
    305       1.2       eeh 	struct iommu_state *is;
    306       1.2       eeh 	vaddr_t va;
    307       1.2       eeh 	size_t len;
    308       1.2       eeh {
    309       1.2       eeh 
    310       1.2       eeh #ifdef DIAGNOSTIC
    311  1.35.2.5  jdolecek 	if (va < is->is_dvmabase || va > is->is_dvmaend)
    312      1.25       mrg 		panic("iommu_remove: va 0x%lx not in DVMA space", (u_long)va);
    313       1.2       eeh 	if ((long)(va + len) < (long)va)
    314  1.35.2.8  jdolecek 		panic("iommu_remove: va 0x%lx + len 0x%lx wraps",
    315       1.2       eeh 		      (long) va, (long) len);
    316  1.35.2.8  jdolecek 	if (len & ~0xfffffff)
    317      1.25       mrg 		panic("iommu_remove: rediculous len 0x%lx", (u_long)len);
    318       1.2       eeh #endif
    319       1.2       eeh 
    320       1.2       eeh 	va = trunc_page(va);
    321      1.22       mrg 	DPRINTF(IDB_IOMMU, ("iommu_remove: va %lx TSB[%lx]@%p\n",
    322  1.35.2.6  jdolecek 		va, (u_long)IOTSBSLOT(va, is->is_tsbsize),
    323  1.35.2.6  jdolecek 		&is->is_tsb[IOTSBSLOT(va, is->is_tsbsize)]));
    324       1.2       eeh 	while (len > 0) {
    325  1.35.2.6  jdolecek 		DPRINTF(IDB_IOMMU, ("iommu_remove: clearing TSB slot %d "
    326  1.35.2.6  jdolecek 			"for va %p size %lx\n",
    327  1.35.2.6  jdolecek 			(int)IOTSBSLOT(va,is->is_tsbsize), (void *)(u_long)va,
    328  1.35.2.6  jdolecek 			(u_long)len));
    329      1.10       mrg 		if (len <= NBPG)
    330      1.10       mrg 			len = 0;
    331      1.10       mrg 		else
    332       1.8       mrg 			len -= NBPG;
    333       1.8       mrg 
    334  1.35.2.5  jdolecek 		/* XXX Zero-ing the entry would not require RMW */
    335  1.35.2.5  jdolecek 		is->is_tsb[IOTSBSLOT(va,is->is_tsbsize)] &= ~IOTTE_V;
    336  1.35.2.8  jdolecek 		bus_space_write_8(is->is_bustag, is->is_iommu,
    337  1.35.2.6  jdolecek 			IOMMUREG(iommu_flush), va);
    338       1.2       eeh 		va += NBPG;
    339       1.2       eeh 	}
    340       1.2       eeh }
    341       1.2       eeh 
    342  1.35.2.8  jdolecek static int
    343  1.35.2.7  jdolecek iommu_strbuf_flush_done(sb)
    344  1.35.2.7  jdolecek 	struct strbuf_ctl *sb;
    345       1.2       eeh {
    346  1.35.2.7  jdolecek 	struct iommu_state *is = sb->sb_is;
    347       1.2       eeh 	struct timeval cur, flushtimeout;
    348       1.2       eeh 
    349       1.2       eeh #define BUMPTIME(t, usec) { \
    350       1.2       eeh 	register volatile struct timeval *tp = (t); \
    351       1.2       eeh 	register long us; \
    352       1.2       eeh  \
    353       1.2       eeh 	tp->tv_usec = us = tp->tv_usec + (usec); \
    354       1.2       eeh 	if (us >= 1000000) { \
    355       1.2       eeh 		tp->tv_usec = us - 1000000; \
    356       1.2       eeh 		tp->tv_sec++; \
    357       1.2       eeh 	} \
    358       1.2       eeh }
    359       1.5       mrg 
    360  1.35.2.7  jdolecek 	if (!sb->sb_flush)
    361       1.5       mrg 		return (0);
    362  1.35.2.8  jdolecek 
    363       1.7       mrg 	/*
    364       1.7       mrg 	 * Streaming buffer flushes:
    365  1.35.2.8  jdolecek 	 *
    366       1.7       mrg 	 *   1 Tell strbuf to flush by storing va to strbuf_pgflush.  If
    367       1.7       mrg 	 *     we're not on a cache line boundary (64-bits):
    368       1.7       mrg 	 *   2 Store 0 in flag
    369       1.7       mrg 	 *   3 Store pointer to flag in flushsync
    370       1.7       mrg 	 *   4 wait till flushsync becomes 0x1
    371       1.7       mrg 	 *
    372       1.7       mrg 	 * If it takes more than .5 sec, something
    373       1.7       mrg 	 * went wrong.
    374       1.7       mrg 	 */
    375       1.2       eeh 
    376  1.35.2.7  jdolecek 	*sb->sb_flush = 0;
    377  1.35.2.8  jdolecek 	bus_space_write_8(is->is_bustag, sb->sb_sb,
    378  1.35.2.7  jdolecek 		STRBUFREG(strbuf_flushsync), sb->sb_flushpa);
    379       1.2       eeh 
    380  1.35.2.8  jdolecek 	microtime(&flushtimeout);
    381       1.2       eeh 	cur = flushtimeout;
    382       1.2       eeh 	BUMPTIME(&flushtimeout, 500000); /* 1/2 sec */
    383  1.35.2.8  jdolecek 
    384  1.35.2.7  jdolecek 	DPRINTF(IDB_IOMMU, ("iommu_strbuf_flush_done: flush = %lx "
    385  1.35.2.4   thorpej 		"at va = %lx pa = %lx now=%lx:%lx until = %lx:%lx\n",
    386  1.35.2.8  jdolecek 		(long)*sb->sb_flush, (long)sb->sb_flush, (long)sb->sb_flushpa,
    387  1.35.2.4   thorpej 		cur.tv_sec, cur.tv_usec,
    388  1.35.2.4   thorpej 		flushtimeout.tv_sec, flushtimeout.tv_usec));
    389  1.35.2.4   thorpej 
    390       1.2       eeh 	/* Bypass non-coherent D$ */
    391  1.35.2.7  jdolecek 	while ((!ldxa(sb->sb_flushpa, ASI_PHYS_CACHED)) &&
    392  1.35.2.8  jdolecek 		timercmp(&cur, &flushtimeout, <=))
    393       1.2       eeh 		microtime(&cur);
    394       1.2       eeh 
    395       1.2       eeh #ifdef DIAGNOSTIC
    396  1.35.2.7  jdolecek 	if (!ldxa(sb->sb_flushpa, ASI_PHYS_CACHED)) {
    397  1.35.2.7  jdolecek 		printf("iommu_strbuf_flush_done: flush timeout %p, at %p\n",
    398  1.35.2.7  jdolecek 			(void *)(u_long)*sb->sb_flush,
    399  1.35.2.7  jdolecek 			(void *)(u_long)sb->sb_flushpa); /* panic? */
    400       1.2       eeh #ifdef DDB
    401       1.2       eeh 		Debugger();
    402       1.2       eeh #endif
    403       1.2       eeh 	}
    404       1.2       eeh #endif
    405      1.31       eeh 	DPRINTF(IDB_IOMMU, ("iommu_strbuf_flush_done: flushed\n"));
    406  1.35.2.7  jdolecek 	return (*sb->sb_flush);
    407       1.7       mrg }
    408       1.7       mrg 
    409       1.7       mrg /*
    410       1.7       mrg  * IOMMU DVMA operations, common to SBUS and PCI.
    411       1.7       mrg  */
    412       1.7       mrg int
    413  1.35.2.7  jdolecek iommu_dvmamap_load(t, sb, map, buf, buflen, p, flags)
    414       1.7       mrg 	bus_dma_tag_t t;
    415  1.35.2.7  jdolecek 	struct strbuf_ctl *sb;
    416       1.7       mrg 	bus_dmamap_t map;
    417       1.7       mrg 	void *buf;
    418       1.7       mrg 	bus_size_t buflen;
    419       1.7       mrg 	struct proc *p;
    420       1.7       mrg 	int flags;
    421       1.7       mrg {
    422  1.35.2.7  jdolecek 	struct iommu_state *is = sb->sb_is;
    423       1.7       mrg 	int s;
    424       1.7       mrg 	int err;
    425       1.7       mrg 	bus_size_t sgsize;
    426       1.7       mrg 	paddr_t curaddr;
    427  1.35.2.4   thorpej 	u_long dvmaddr, sgstart, sgend;
    428      1.21       eeh 	bus_size_t align, boundary;
    429       1.7       mrg 	vaddr_t vaddr = (vaddr_t)buf;
    430  1.35.2.4   thorpej 	int seg;
    431  1.35.2.8  jdolecek 	struct pmap *pmap;
    432       1.7       mrg 
    433       1.7       mrg 	if (map->dm_nsegs) {
    434       1.7       mrg 		/* Already in use?? */
    435       1.7       mrg #ifdef DIAGNOSTIC
    436       1.7       mrg 		printf("iommu_dvmamap_load: map still in use\n");
    437       1.7       mrg #endif
    438       1.7       mrg 		bus_dmamap_unload(t, map);
    439       1.7       mrg 	}
    440  1.35.2.8  jdolecek 
    441       1.7       mrg 	/*
    442       1.7       mrg 	 * Make sure that on error condition we return "no valid mappings".
    443       1.7       mrg 	 */
    444       1.7       mrg 	map->dm_nsegs = 0;
    445       1.7       mrg 	if (buflen > map->_dm_size) {
    446      1.22       mrg 		DPRINTF(IDB_BUSDMA,
    447       1.7       mrg 		    ("iommu_dvmamap_load(): error %d > %d -- "
    448      1.25       mrg 		     "map size exceeded!\n", (int)buflen, (int)map->_dm_size));
    449       1.7       mrg 		return (EINVAL);
    450       1.7       mrg 	}
    451       1.7       mrg 
    452       1.7       mrg 	sgsize = round_page(buflen + ((int)vaddr & PGOFSET));
    453      1.20       mrg 
    454       1.7       mrg 	/*
    455      1.21       eeh 	 * A boundary presented to bus_dmamem_alloc() takes precedence
    456      1.21       eeh 	 * over boundary in the map.
    457       1.7       mrg 	 */
    458      1.21       eeh 	if ((boundary = (map->dm_segs[0]._ds_boundary)) == 0)
    459      1.21       eeh 		boundary = map->_dm_boundary;
    460      1.21       eeh 	align = max(map->dm_segs[0]._ds_align, NBPG);
    461  1.35.2.8  jdolecek 
    462  1.35.2.8  jdolecek 	/*
    463  1.35.2.8  jdolecek 	 * If our segment size is larger than the boundary we need to
    464  1.35.2.4   thorpej 	 * split the transfer up int little pieces ourselves.
    465  1.35.2.4   thorpej 	 */
    466  1.35.2.8  jdolecek 	s = splhigh();
    467  1.35.2.8  jdolecek 	err = extent_alloc(is->is_dvmamap, sgsize, align,
    468  1.35.2.8  jdolecek 		(sgsize > boundary) ? 0 : boundary,
    469  1.35.2.6  jdolecek 		EX_NOWAIT|EX_BOUNDZERO, &dvmaddr);
    470       1.7       mrg 	splx(s);
    471       1.7       mrg 
    472       1.7       mrg #ifdef DEBUG
    473  1.35.2.8  jdolecek 	if (err || (dvmaddr == (bus_addr_t)-1))
    474  1.35.2.8  jdolecek 	{
    475       1.7       mrg 		printf("iommu_dvmamap_load(): extent_alloc(%d, %x) failed!\n",
    476      1.25       mrg 		    (int)sgsize, flags);
    477  1.35.2.4   thorpej #ifdef DDB
    478       1.7       mrg 		Debugger();
    479  1.35.2.4   thorpej #endif
    480  1.35.2.8  jdolecek 	}
    481  1.35.2.8  jdolecek #endif
    482      1.11       eeh 	if (err != 0)
    483      1.11       eeh 		return (err);
    484      1.11       eeh 
    485       1.7       mrg 	if (dvmaddr == (bus_addr_t)-1)
    486       1.7       mrg 		return (ENOMEM);
    487       1.7       mrg 
    488  1.35.2.4   thorpej 	/* Set the active DVMA map */
    489  1.35.2.4   thorpej 	map->_dm_dvmastart = dvmaddr;
    490  1.35.2.4   thorpej 	map->_dm_dvmasize = sgsize;
    491  1.35.2.4   thorpej 
    492  1.35.2.4   thorpej 	/*
    493  1.35.2.4   thorpej 	 * Now split the DVMA range into segments, not crossing
    494  1.35.2.4   thorpej 	 * the boundary.
    495  1.35.2.4   thorpej 	 */
    496  1.35.2.4   thorpej 	seg = 0;
    497  1.35.2.4   thorpej 	sgstart = dvmaddr + (vaddr & PGOFSET);
    498  1.35.2.4   thorpej 	sgend = sgstart + buflen - 1;
    499  1.35.2.4   thorpej 	map->dm_segs[seg].ds_addr = sgstart;
    500  1.35.2.4   thorpej 	DPRINTF(IDB_INFO, ("iommu_dvmamap_load: boundary %lx boundary-1 %lx "
    501  1.35.2.4   thorpej 		"~(boundary-1) %lx\n", boundary, (boundary-1), ~(boundary-1)));
    502  1.35.2.4   thorpej 	while ((sgstart & ~(boundary - 1)) != (sgend & ~(boundary - 1))) {
    503  1.35.2.4   thorpej 		/* Oops.  We crossed a boundary.  Split the xfer. */
    504  1.35.2.4   thorpej 		DPRINTF(IDB_INFO, ("iommu_dvmamap_load: "
    505  1.35.2.4   thorpej 			"seg %d start %lx size %lx\n", seg,
    506  1.35.2.8  jdolecek 			(long)map->dm_segs[seg].ds_addr,
    507  1.35.2.5  jdolecek 			map->dm_segs[seg].ds_len));
    508  1.35.2.5  jdolecek 		map->dm_segs[seg].ds_len =
    509  1.35.2.5  jdolecek 		    boundary - (sgstart & (boundary - 1));
    510  1.35.2.6  jdolecek 		if (++seg >= map->_dm_segcnt) {
    511  1.35.2.4   thorpej 			/* Too many segments.  Fail the operation. */
    512  1.35.2.4   thorpej 			DPRINTF(IDB_INFO, ("iommu_dvmamap_load: "
    513  1.35.2.4   thorpej 				"too many segments %d\n", seg));
    514  1.35.2.4   thorpej 			s = splhigh();
    515  1.35.2.4   thorpej 			/* How can this fail?  And if it does what can we do? */
    516  1.35.2.4   thorpej 			err = extent_free(is->is_dvmamap,
    517  1.35.2.4   thorpej 				dvmaddr, sgsize, EX_NOWAIT);
    518  1.35.2.4   thorpej 			map->_dm_dvmastart = 0;
    519  1.35.2.4   thorpej 			map->_dm_dvmasize = 0;
    520  1.35.2.4   thorpej 			splx(s);
    521  1.35.2.4   thorpej 			return (E2BIG);
    522  1.35.2.4   thorpej 		}
    523  1.35.2.4   thorpej 		sgstart = roundup(sgstart, boundary);
    524  1.35.2.4   thorpej 		map->dm_segs[seg].ds_addr = sgstart;
    525  1.35.2.4   thorpej 	}
    526  1.35.2.4   thorpej 	map->dm_segs[seg].ds_len = sgend - sgstart + 1;
    527  1.35.2.4   thorpej 	DPRINTF(IDB_INFO, ("iommu_dvmamap_load: "
    528  1.35.2.4   thorpej 		"seg %d start %lx size %lx\n", seg,
    529  1.35.2.5  jdolecek 		(long)map->dm_segs[seg].ds_addr, map->dm_segs[seg].ds_len));
    530  1.35.2.4   thorpej 	map->dm_nsegs = seg+1;
    531       1.7       mrg 	map->dm_mapsize = buflen;
    532       1.7       mrg 
    533       1.7       mrg 	if (p != NULL)
    534       1.7       mrg 		pmap = p->p_vmspace->vm_map.pmap;
    535       1.7       mrg 	else
    536       1.7       mrg 		pmap = pmap_kernel();
    537       1.7       mrg 
    538       1.7       mrg 	for (; buflen > 0; ) {
    539  1.35.2.8  jdolecek 
    540       1.7       mrg 		/*
    541       1.7       mrg 		 * Get the physical address for this page.
    542       1.7       mrg 		 */
    543       1.7       mrg 		if (pmap_extract(pmap, (vaddr_t)vaddr, &curaddr) == FALSE) {
    544       1.7       mrg 			bus_dmamap_unload(t, map);
    545       1.7       mrg 			return (-1);
    546       1.7       mrg 		}
    547       1.7       mrg 
    548       1.7       mrg 		/*
    549       1.7       mrg 		 * Compute the segment size, and adjust counts.
    550       1.7       mrg 		 */
    551       1.7       mrg 		sgsize = NBPG - ((u_long)vaddr & PGOFSET);
    552       1.7       mrg 		if (buflen < sgsize)
    553       1.7       mrg 			sgsize = buflen;
    554       1.7       mrg 
    555      1.22       mrg 		DPRINTF(IDB_BUSDMA,
    556  1.35.2.1     lukem 		    ("iommu_dvmamap_load: map %p loading va %p "
    557  1.35.2.1     lukem 			    "dva %lx at pa %lx\n",
    558  1.35.2.1     lukem 			    map, (void *)vaddr, (long)dvmaddr,
    559  1.35.2.6  jdolecek 			    (long)(curaddr & ~(NBPG-1))));
    560  1.35.2.7  jdolecek 		iommu_enter(sb, trunc_page(dvmaddr), trunc_page(curaddr),
    561  1.35.2.5  jdolecek 		    flags|0x4000);
    562  1.35.2.8  jdolecek 
    563       1.7       mrg 		dvmaddr += PAGE_SIZE;
    564       1.7       mrg 		vaddr += sgsize;
    565       1.7       mrg 		buflen -= sgsize;
    566       1.7       mrg 	}
    567  1.35.2.5  jdolecek #ifdef DIAGNOSTIC
    568  1.35.2.5  jdolecek 	for (seg = 0; seg < map->dm_nsegs; seg++) {
    569  1.35.2.5  jdolecek 		if (map->dm_segs[seg].ds_addr < is->is_dvmabase ||
    570  1.35.2.5  jdolecek 			map->dm_segs[seg].ds_addr > is->is_dvmaend) {
    571  1.35.2.5  jdolecek 			printf("seg %d dvmaddr %lx out of range %x - %x\n",
    572  1.35.2.8  jdolecek 				seg, (long)map->dm_segs[seg].ds_addr,
    573  1.35.2.5  jdolecek 				is->is_dvmabase, is->is_dvmaend);
    574  1.35.2.7  jdolecek #ifdef DDB
    575  1.35.2.5  jdolecek 			Debugger();
    576  1.35.2.7  jdolecek #endif
    577  1.35.2.5  jdolecek 		}
    578  1.35.2.5  jdolecek 	}
    579  1.35.2.5  jdolecek #endif
    580       1.7       mrg 	return (0);
    581       1.7       mrg }
    582       1.7       mrg 
    583       1.7       mrg 
    584       1.7       mrg void
    585  1.35.2.7  jdolecek iommu_dvmamap_unload(t, sb, map)
    586       1.7       mrg 	bus_dma_tag_t t;
    587  1.35.2.7  jdolecek 	struct strbuf_ctl *sb;
    588       1.7       mrg 	bus_dmamap_t map;
    589       1.7       mrg {
    590  1.35.2.7  jdolecek 	struct iommu_state *is = sb->sb_is;
    591  1.35.2.4   thorpej 	int error, s;
    592       1.7       mrg 	bus_size_t sgsize;
    593       1.7       mrg 
    594  1.35.2.4   thorpej 	/* Flush the iommu */
    595  1.35.2.4   thorpej #ifdef DEBUG
    596  1.35.2.4   thorpej 	if (!map->_dm_dvmastart) {
    597  1.35.2.4   thorpej 		printf("iommu_dvmamap_unload: No dvmastart is zero\n");
    598  1.35.2.4   thorpej #ifdef DDB
    599  1.35.2.4   thorpej 		Debugger();
    600  1.35.2.4   thorpej #endif
    601      1.26    martin 	}
    602  1.35.2.4   thorpej #endif
    603  1.35.2.4   thorpej 	iommu_remove(is, map->_dm_dvmastart, map->_dm_dvmasize);
    604  1.35.2.4   thorpej 
    605      1.23       eeh 	/* Flush the caches */
    606      1.23       eeh 	bus_dmamap_unload(t->_parent, map);
    607      1.23       eeh 
    608       1.7       mrg 	/* Mark the mappings as invalid. */
    609       1.7       mrg 	map->dm_mapsize = 0;
    610       1.7       mrg 	map->dm_nsegs = 0;
    611  1.35.2.8  jdolecek 
    612       1.7       mrg 	s = splhigh();
    613  1.35.2.8  jdolecek 	error = extent_free(is->is_dvmamap, map->_dm_dvmastart,
    614  1.35.2.4   thorpej 		map->_dm_dvmasize, EX_NOWAIT);
    615  1.35.2.4   thorpej 	map->_dm_dvmastart = 0;
    616  1.35.2.4   thorpej 	map->_dm_dvmasize = 0;
    617       1.7       mrg 	splx(s);
    618       1.7       mrg 	if (error != 0)
    619       1.7       mrg 		printf("warning: %qd of DVMA space lost\n", (long long)sgsize);
    620  1.35.2.4   thorpej 
    621  1.35.2.4   thorpej 	/* Clear the map */
    622       1.9       eeh }
    623       1.9       eeh 
    624       1.9       eeh 
    625       1.9       eeh int
    626  1.35.2.7  jdolecek iommu_dvmamap_load_raw(t, sb, map, segs, nsegs, flags, size)
    627       1.9       eeh 	bus_dma_tag_t t;
    628  1.35.2.7  jdolecek 	struct strbuf_ctl *sb;
    629       1.9       eeh 	bus_dmamap_t map;
    630       1.9       eeh 	bus_dma_segment_t *segs;
    631       1.9       eeh 	int nsegs;
    632      1.22       mrg 	int flags;
    633       1.9       eeh 	bus_size_t size;
    634       1.9       eeh {
    635  1.35.2.7  jdolecek 	struct iommu_state *is = sb->sb_is;
    636  1.35.2.8  jdolecek 	struct vm_page *pg;
    637  1.35.2.4   thorpej 	int i, j, s;
    638      1.26    martin 	int left;
    639       1.9       eeh 	int err;
    640       1.9       eeh 	bus_size_t sgsize;
    641       1.9       eeh 	paddr_t pa;
    642      1.21       eeh 	bus_size_t boundary, align;
    643  1.35.2.4   thorpej 	u_long dvmaddr, sgstart, sgend;
    644  1.35.2.8  jdolecek 	struct pglist *pglist;
    645       1.9       eeh 	int pagesz = PAGE_SIZE;
    646  1.35.2.5  jdolecek 	int npg = 0; /* DEBUG */
    647       1.9       eeh 
    648       1.9       eeh 	if (map->dm_nsegs) {
    649       1.9       eeh 		/* Already in use?? */
    650       1.9       eeh #ifdef DIAGNOSTIC
    651       1.9       eeh 		printf("iommu_dvmamap_load_raw: map still in use\n");
    652       1.9       eeh #endif
    653       1.9       eeh 		bus_dmamap_unload(t, map);
    654       1.9       eeh 	}
    655  1.35.2.4   thorpej 
    656  1.35.2.4   thorpej 	/*
    657  1.35.2.4   thorpej 	 * A boundary presented to bus_dmamem_alloc() takes precedence
    658  1.35.2.4   thorpej 	 * over boundary in the map.
    659  1.35.2.4   thorpej 	 */
    660  1.35.2.4   thorpej 	if ((boundary = segs[0]._ds_boundary) == 0)
    661  1.35.2.4   thorpej 		boundary = map->_dm_boundary;
    662  1.35.2.4   thorpej 
    663  1.35.2.5  jdolecek 	align = max(segs[0]._ds_align, pagesz);
    664  1.35.2.4   thorpej 
    665       1.9       eeh 	/*
    666       1.9       eeh 	 * Make sure that on error condition we return "no valid mappings".
    667       1.9       eeh 	 */
    668       1.9       eeh 	map->dm_nsegs = 0;
    669      1.26    martin 	/* Count up the total number of pages we need */
    670      1.26    martin 	pa = segs[0].ds_addr;
    671      1.26    martin 	sgsize = 0;
    672  1.35.2.4   thorpej 	left = size;
    673  1.35.2.8  jdolecek 	for (i = 0; left && i < nsegs; i++) {
    674      1.26    martin 		if (round_page(pa) != round_page(segs[i].ds_addr))
    675      1.26    martin 			sgsize = round_page(sgsize);
    676  1.35.2.4   thorpej 		sgsize += min(left, segs[i].ds_len);
    677  1.35.2.4   thorpej 		left -= segs[i].ds_len;
    678      1.26    martin 		pa = segs[i].ds_addr + segs[i].ds_len;
    679      1.26    martin 	}
    680      1.26    martin 	sgsize = round_page(sgsize);
    681       1.9       eeh 
    682       1.9       eeh 	s = splhigh();
    683  1.35.2.8  jdolecek 	/*
    684  1.35.2.8  jdolecek 	 * If our segment size is larger than the boundary we need to
    685  1.35.2.5  jdolecek 	 * split the transfer up into little pieces ourselves.
    686  1.35.2.4   thorpej 	 */
    687  1.35.2.4   thorpej 	err = extent_alloc(is->is_dvmamap, sgsize, align,
    688  1.35.2.4   thorpej 		(sgsize > boundary) ? 0 : boundary,
    689  1.35.2.4   thorpej 		((flags & BUS_DMA_NOWAIT) == 0 ? EX_WAITOK : EX_NOWAIT) |
    690  1.35.2.6  jdolecek 		EX_BOUNDZERO, &dvmaddr);
    691       1.9       eeh 	splx(s);
    692       1.9       eeh 
    693       1.9       eeh 	if (err != 0)
    694       1.9       eeh 		return (err);
    695       1.9       eeh 
    696       1.9       eeh #ifdef DEBUG
    697  1.35.2.8  jdolecek 	if (dvmaddr == (bus_addr_t)-1)
    698  1.35.2.8  jdolecek 	{
    699       1.9       eeh 		printf("iommu_dvmamap_load_raw(): extent_alloc(%d, %x) failed!\n",
    700      1.25       mrg 		    (int)sgsize, flags);
    701  1.35.2.7  jdolecek #ifdef DDB
    702       1.9       eeh 		Debugger();
    703  1.35.2.7  jdolecek #endif
    704  1.35.2.8  jdolecek 	}
    705  1.35.2.8  jdolecek #endif
    706       1.9       eeh 	if (dvmaddr == (bus_addr_t)-1)
    707       1.9       eeh 		return (ENOMEM);
    708       1.9       eeh 
    709  1.35.2.4   thorpej 	/* Set the active DVMA map */
    710  1.35.2.4   thorpej 	map->_dm_dvmastart = dvmaddr;
    711  1.35.2.4   thorpej 	map->_dm_dvmasize = sgsize;
    712  1.35.2.4   thorpej 
    713  1.35.2.8  jdolecek 	if ((pglist = segs[0]._ds_mlist) == NULL) {
    714      1.26    martin 		u_long prev_va = NULL;
    715  1.35.2.5  jdolecek 		paddr_t prev_pa = 0;
    716  1.35.2.5  jdolecek 		int end = 0, offset;
    717  1.35.2.5  jdolecek 
    718      1.26    martin 		/*
    719  1.35.2.5  jdolecek 		 * This segs is made up of individual physical
    720  1.35.2.8  jdolecek 		 *  segments, probably by _bus_dmamap_load_uio() or
    721      1.26    martin 		 * _bus_dmamap_load_mbuf().  Ignore the mlist and
    722  1.35.2.5  jdolecek 		 * load each one individually.
    723      1.26    martin 		 */
    724  1.35.2.4   thorpej 		map->dm_mapsize = size;
    725      1.26    martin 
    726  1.35.2.5  jdolecek 		j = 0;
    727  1.35.2.5  jdolecek 		for (i = 0; i < nsegs ; i++) {
    728  1.35.2.4   thorpej 
    729  1.35.2.5  jdolecek 			pa = segs[i].ds_addr;
    730  1.35.2.5  jdolecek 			offset = (pa & PGOFSET);
    731  1.35.2.5  jdolecek 			pa = trunc_page(pa);
    732  1.35.2.5  jdolecek 			dvmaddr = trunc_page(dvmaddr);
    733  1.35.2.5  jdolecek 			left = min(size, segs[i].ds_len);
    734  1.35.2.5  jdolecek 
    735  1.35.2.5  jdolecek 			DPRINTF(IDB_INFO, ("iommu_dvmamap_load_raw: converting "
    736  1.35.2.8  jdolecek 				"physseg %d start %lx size %lx\n", i,
    737  1.35.2.5  jdolecek 				(long)segs[i].ds_addr, segs[i].ds_len));
    738  1.35.2.5  jdolecek 
    739  1.35.2.8  jdolecek 			if ((pa == prev_pa) &&
    740  1.35.2.5  jdolecek 				((offset != 0) || (end != offset))) {
    741  1.35.2.5  jdolecek 				/* We can re-use this mapping */
    742  1.35.2.5  jdolecek 				dvmaddr = prev_va;
    743  1.35.2.5  jdolecek 			}
    744  1.35.2.4   thorpej 
    745  1.35.2.5  jdolecek 			sgstart = dvmaddr + offset;
    746  1.35.2.5  jdolecek 			sgend = sgstart + left - 1;
    747  1.35.2.4   thorpej 
    748  1.35.2.5  jdolecek 			/* Are the segments virtually adjacent? */
    749  1.35.2.8  jdolecek 			if ((j > 0) && (end == offset) &&
    750  1.35.2.5  jdolecek 				((offset == 0) || (pa == prev_pa))) {
    751  1.35.2.5  jdolecek 				/* Just append to the previous segment. */
    752  1.35.2.5  jdolecek 				map->dm_segs[--j].ds_len += left;
    753  1.35.2.5  jdolecek 				DPRINTF(IDB_INFO, ("iommu_dvmamap_load_raw: "
    754  1.35.2.5  jdolecek 					"appending seg %d start %lx size %lx\n", j,
    755  1.35.2.8  jdolecek 					(long)map->dm_segs[j].ds_addr,
    756  1.35.2.5  jdolecek 					map->dm_segs[j].ds_len));
    757  1.35.2.5  jdolecek 			} else {
    758  1.35.2.6  jdolecek 				if (j >= map->_dm_segcnt) {
    759  1.35.2.7  jdolecek 					iommu_dvmamap_unload(t, sb, map);
    760  1.35.2.6  jdolecek 					return (E2BIG);
    761  1.35.2.6  jdolecek 				}
    762  1.35.2.5  jdolecek 				map->dm_segs[j].ds_addr = sgstart;
    763  1.35.2.5  jdolecek 				map->dm_segs[j].ds_len = left;
    764  1.35.2.5  jdolecek 				DPRINTF(IDB_INFO, ("iommu_dvmamap_load_raw: "
    765  1.35.2.5  jdolecek 					"seg %d start %lx size %lx\n", j,
    766  1.35.2.5  jdolecek 					(long)map->dm_segs[j].ds_addr,
    767  1.35.2.5  jdolecek 					map->dm_segs[j].ds_len));
    768  1.35.2.4   thorpej 			}
    769  1.35.2.5  jdolecek 			end = (offset + left) & PGOFSET;
    770  1.35.2.4   thorpej 
    771  1.35.2.4   thorpej 			/* Check for boundary issues */
    772  1.35.2.4   thorpej 			while ((sgstart & ~(boundary - 1)) !=
    773  1.35.2.4   thorpej 				(sgend & ~(boundary - 1))) {
    774  1.35.2.4   thorpej 				/* Need a new segment. */
    775  1.35.2.4   thorpej 				map->dm_segs[j].ds_len =
    776  1.35.2.6  jdolecek 					boundary - (sgstart & (boundary - 1));
    777  1.35.2.4   thorpej 				DPRINTF(IDB_INFO, ("iommu_dvmamap_load_raw: "
    778  1.35.2.4   thorpej 					"seg %d start %lx size %lx\n", j,
    779  1.35.2.8  jdolecek 					(long)map->dm_segs[j].ds_addr,
    780  1.35.2.4   thorpej 					map->dm_segs[j].ds_len));
    781  1.35.2.6  jdolecek 				if (++j >= map->_dm_segcnt) {
    782  1.35.2.7  jdolecek 					iommu_dvmamap_unload(t, sb, map);
    783  1.35.2.4   thorpej 					return (E2BIG);
    784  1.35.2.4   thorpej 				}
    785  1.35.2.4   thorpej 				sgstart = roundup(sgstart, boundary);
    786  1.35.2.4   thorpej 				map->dm_segs[j].ds_addr = sgstart;
    787  1.35.2.4   thorpej 				map->dm_segs[j].ds_len = sgend - sgstart + 1;
    788  1.35.2.4   thorpej 			}
    789  1.35.2.4   thorpej 
    790      1.26    martin 			if (sgsize == 0)
    791      1.26    martin 				panic("iommu_dmamap_load_raw: size botch");
    792  1.35.2.4   thorpej 
    793  1.35.2.5  jdolecek 			/* Now map a series of pages. */
    794  1.35.2.6  jdolecek 			while (dvmaddr <= sgend) {
    795  1.35.2.5  jdolecek 				DPRINTF(IDB_BUSDMA,
    796  1.35.2.5  jdolecek 					("iommu_dvmamap_load_raw: map %p "
    797  1.35.2.5  jdolecek 						"loading va %lx at pa %lx\n",
    798  1.35.2.5  jdolecek 						map, (long)dvmaddr,
    799  1.35.2.5  jdolecek 						(long)(pa)));
    800  1.35.2.5  jdolecek 				/* Enter it if we haven't before. */
    801  1.35.2.5  jdolecek 				if (prev_va != dvmaddr)
    802  1.35.2.7  jdolecek 					iommu_enter(sb, prev_va = dvmaddr,
    803  1.35.2.5  jdolecek 						prev_pa = pa,
    804  1.35.2.8  jdolecek 						flags | (++npg << 12));
    805  1.35.2.5  jdolecek 				dvmaddr += pagesz;
    806  1.35.2.5  jdolecek 				pa += pagesz;
    807  1.35.2.5  jdolecek 			}
    808  1.35.2.5  jdolecek 
    809  1.35.2.5  jdolecek 			size -= left;
    810  1.35.2.5  jdolecek 			++j;
    811      1.27    martin 		}
    812  1.35.2.5  jdolecek 
    813  1.35.2.5  jdolecek 		map->dm_nsegs = j;
    814  1.35.2.5  jdolecek #ifdef DIAGNOSTIC
    815  1.35.2.5  jdolecek 		{ int seg;
    816  1.35.2.5  jdolecek 	for (seg = 0; seg < map->dm_nsegs; seg++) {
    817  1.35.2.5  jdolecek 		if (map->dm_segs[seg].ds_addr < is->is_dvmabase ||
    818  1.35.2.5  jdolecek 			map->dm_segs[seg].ds_addr > is->is_dvmaend) {
    819  1.35.2.5  jdolecek 			printf("seg %d dvmaddr %lx out of range %x - %x\n",
    820  1.35.2.8  jdolecek 				seg, (long)map->dm_segs[seg].ds_addr,
    821  1.35.2.5  jdolecek 				is->is_dvmabase, is->is_dvmaend);
    822  1.35.2.7  jdolecek #ifdef DDB
    823  1.35.2.5  jdolecek 			Debugger();
    824  1.35.2.7  jdolecek #endif
    825  1.35.2.5  jdolecek 		}
    826  1.35.2.5  jdolecek 	}
    827  1.35.2.5  jdolecek 		}
    828  1.35.2.5  jdolecek #endif
    829      1.26    martin 		return (0);
    830      1.26    martin 	}
    831  1.35.2.8  jdolecek 
    832       1.9       eeh 	/*
    833  1.35.2.4   thorpej 	 * This was allocated with bus_dmamem_alloc.
    834  1.35.2.8  jdolecek 	 * The pages are on a `pglist'.
    835       1.9       eeh 	 */
    836       1.9       eeh 	map->dm_mapsize = size;
    837      1.26    martin 	i = 0;
    838  1.35.2.4   thorpej 	sgstart = dvmaddr;
    839  1.35.2.4   thorpej 	sgend = sgstart + size - 1;
    840  1.35.2.4   thorpej 	map->dm_segs[i].ds_addr = sgstart;
    841  1.35.2.4   thorpej 	while ((sgstart & ~(boundary - 1)) != (sgend & ~(boundary - 1))) {
    842  1.35.2.4   thorpej 		/* Oops.  We crossed a boundary.  Split the xfer. */
    843  1.35.2.6  jdolecek 		map->dm_segs[i].ds_len = boundary - (sgstart & (boundary - 1));
    844  1.35.2.4   thorpej 		DPRINTF(IDB_INFO, ("iommu_dvmamap_load_raw: "
    845  1.35.2.4   thorpej 			"seg %d start %lx size %lx\n", i,
    846  1.35.2.5  jdolecek 			(long)map->dm_segs[i].ds_addr,
    847  1.35.2.4   thorpej 			map->dm_segs[i].ds_len));
    848  1.35.2.6  jdolecek 		if (++i >= map->_dm_segcnt) {
    849  1.35.2.4   thorpej 			/* Too many segments.  Fail the operation. */
    850  1.35.2.4   thorpej 			s = splhigh();
    851  1.35.2.4   thorpej 			/* How can this fail?  And if it does what can we do? */
    852  1.35.2.4   thorpej 			err = extent_free(is->is_dvmamap,
    853  1.35.2.4   thorpej 				dvmaddr, sgsize, EX_NOWAIT);
    854  1.35.2.4   thorpej 			map->_dm_dvmastart = 0;
    855  1.35.2.4   thorpej 			map->_dm_dvmasize = 0;
    856  1.35.2.4   thorpej 			splx(s);
    857  1.35.2.4   thorpej 			return (E2BIG);
    858  1.35.2.4   thorpej 		}
    859  1.35.2.4   thorpej 		sgstart = roundup(sgstart, boundary);
    860  1.35.2.4   thorpej 		map->dm_segs[i].ds_addr = sgstart;
    861  1.35.2.4   thorpej 	}
    862  1.35.2.4   thorpej 	DPRINTF(IDB_INFO, ("iommu_dvmamap_load_raw: "
    863  1.35.2.4   thorpej 			"seg %d start %lx size %lx\n", i,
    864  1.35.2.5  jdolecek 			(long)map->dm_segs[i].ds_addr, map->dm_segs[i].ds_len));
    865  1.35.2.4   thorpej 	map->dm_segs[i].ds_len = sgend - sgstart + 1;
    866       1.9       eeh 
    867  1.35.2.8  jdolecek 	TAILQ_FOREACH(pg, pglist, pageq) {
    868       1.9       eeh 		if (sgsize == 0)
    869       1.9       eeh 			panic("iommu_dmamap_load_raw: size botch");
    870  1.35.2.8  jdolecek 		pa = VM_PAGE_TO_PHYS(pg);
    871       1.9       eeh 
    872      1.22       mrg 		DPRINTF(IDB_BUSDMA,
    873       1.9       eeh 		    ("iommu_dvmamap_load_raw: map %p loading va %lx at pa %lx\n",
    874       1.9       eeh 		    map, (long)dvmaddr, (long)(pa)));
    875  1.35.2.7  jdolecek 		iommu_enter(sb, dvmaddr, pa, flags|0x8000);
    876  1.35.2.8  jdolecek 
    877       1.9       eeh 		dvmaddr += pagesz;
    878       1.9       eeh 		sgsize -= pagesz;
    879       1.9       eeh 	}
    880  1.35.2.4   thorpej 	map->dm_mapsize = size;
    881  1.35.2.4   thorpej 	map->dm_nsegs = i+1;
    882  1.35.2.5  jdolecek #ifdef DIAGNOSTIC
    883  1.35.2.5  jdolecek 	{ int seg;
    884  1.35.2.5  jdolecek 	for (seg = 0; seg < map->dm_nsegs; seg++) {
    885  1.35.2.5  jdolecek 		if (map->dm_segs[seg].ds_addr < is->is_dvmabase ||
    886  1.35.2.5  jdolecek 			map->dm_segs[seg].ds_addr > is->is_dvmaend) {
    887  1.35.2.5  jdolecek 			printf("seg %d dvmaddr %lx out of range %x - %x\n",
    888  1.35.2.8  jdolecek 				seg, (long)map->dm_segs[seg].ds_addr,
    889  1.35.2.5  jdolecek 				is->is_dvmabase, is->is_dvmaend);
    890  1.35.2.7  jdolecek #ifdef DDB
    891  1.35.2.5  jdolecek 			Debugger();
    892  1.35.2.7  jdolecek #endif
    893  1.35.2.5  jdolecek 		}
    894  1.35.2.5  jdolecek 	}
    895  1.35.2.5  jdolecek 	}
    896  1.35.2.5  jdolecek #endif
    897       1.9       eeh 	return (0);
    898       1.7       mrg }
    899       1.7       mrg 
    900       1.7       mrg void
    901  1.35.2.7  jdolecek iommu_dvmamap_sync(t, sb, map, offset, len, ops)
    902       1.7       mrg 	bus_dma_tag_t t;
    903  1.35.2.7  jdolecek 	struct strbuf_ctl *sb;
    904       1.7       mrg 	bus_dmamap_t map;
    905       1.7       mrg 	bus_addr_t offset;
    906       1.7       mrg 	bus_size_t len;
    907       1.7       mrg 	int ops;
    908       1.7       mrg {
    909  1.35.2.7  jdolecek 	struct iommu_state *is = sb->sb_is;
    910       1.7       mrg 	vaddr_t va = map->dm_segs[0].ds_addr + offset;
    911  1.35.2.7  jdolecek 	int64_t tte;
    912       1.7       mrg 
    913       1.7       mrg 	/*
    914       1.7       mrg 	 * We only support one DMA segment; supporting more makes this code
    915  1.35.2.8  jdolecek 	 * too unwieldy.
    916       1.7       mrg 	 */
    917       1.7       mrg 
    918  1.35.2.8  jdolecek 	DPRINTF(IDB_SYNC,
    919  1.35.2.8  jdolecek 	    ("iommu_dvmamap_sync: syncing va %p len %lu "
    920  1.35.2.8  jdolecek 	     "ops 0x%x\n", (void *)(u_long)va, (u_long)len, ops));
    921  1.35.2.8  jdolecek 	if (ops & (BUS_DMASYNC_PREREAD | BUS_DMASYNC_POSTWRITE)) {
    922  1.35.2.8  jdolecek 		/* Nothing to do */
    923  1.35.2.8  jdolecek 		return;
    924       1.7       mrg 	}
    925  1.35.2.7  jdolecek #ifdef DIAGNOSTIC
    926  1.35.2.8  jdolecek 	if (va < is->is_dvmabase || va >= is->is_dvmaend)
    927  1.35.2.8  jdolecek 		panic("iommu_dvmamap_sync: invalid dva %lx", va);
    928  1.35.2.7  jdolecek #endif
    929  1.35.2.8  jdolecek 	KASSERT((ops & (BUS_DMASYNC_POSTREAD | BUS_DMASYNC_PREWRITE)) != 0);
    930  1.35.2.8  jdolecek 	tte = is->is_tsb[IOTSBSLOT(va, is->is_tsbsize)];
    931  1.35.2.7  jdolecek 
    932  1.35.2.8  jdolecek 	/* if we have a streaming buffer, flush it here first */
    933  1.35.2.8  jdolecek 	if ((tte & IOTTE_STREAM) == 0 || !sb->sb_flush)
    934  1.35.2.8  jdolecek 		return;
    935  1.35.2.8  jdolecek 
    936  1.35.2.8  jdolecek 	while (len > 0) {
    937  1.35.2.8  jdolecek 		DPRINTF(IDB_BUSDMA,
    938  1.35.2.8  jdolecek 		    ("iommu_dvmamap_sync: flushing va %p, %lu "
    939  1.35.2.8  jdolecek 		     "bytes left\n", (void *)(u_long)va, (u_long)len));
    940  1.35.2.8  jdolecek 		iommu_strbuf_flush(sb, va);
    941  1.35.2.8  jdolecek 		if (len <= NBPG) {
    942  1.35.2.8  jdolecek 			iommu_strbuf_flush_done(sb);
    943  1.35.2.8  jdolecek 			len = 0;
    944  1.35.2.8  jdolecek 		} else
    945  1.35.2.8  jdolecek 			len -= NBPG;
    946  1.35.2.8  jdolecek 		va += NBPG;
    947       1.7       mrg 	}
    948       1.7       mrg }
    949       1.7       mrg 
    950       1.7       mrg int
    951  1.35.2.7  jdolecek iommu_dvmamem_alloc(t, sb, size, alignment, boundary, segs, nsegs, rsegs, flags)
    952       1.7       mrg 	bus_dma_tag_t t;
    953  1.35.2.7  jdolecek 	struct strbuf_ctl *sb;
    954       1.7       mrg 	bus_size_t size, alignment, boundary;
    955       1.7       mrg 	bus_dma_segment_t *segs;
    956       1.7       mrg 	int nsegs;
    957       1.7       mrg 	int *rsegs;
    958       1.7       mrg 	int flags;
    959       1.7       mrg {
    960       1.7       mrg 
    961      1.25       mrg 	DPRINTF(IDB_BUSDMA, ("iommu_dvmamem_alloc: sz %llx align %llx bound %llx "
    962      1.25       mrg 	   "segp %p flags %d\n", (unsigned long long)size,
    963      1.25       mrg 	   (unsigned long long)alignment, (unsigned long long)boundary,
    964      1.25       mrg 	   segs, flags));
    965       1.7       mrg 	return (bus_dmamem_alloc(t->_parent, size, alignment, boundary,
    966      1.21       eeh 	    segs, nsegs, rsegs, flags|BUS_DMA_DVMA));
    967       1.7       mrg }
    968       1.7       mrg 
    969       1.7       mrg void
    970  1.35.2.7  jdolecek iommu_dvmamem_free(t, sb, segs, nsegs)
    971       1.7       mrg 	bus_dma_tag_t t;
    972  1.35.2.7  jdolecek 	struct strbuf_ctl *sb;
    973       1.7       mrg 	bus_dma_segment_t *segs;
    974       1.7       mrg 	int nsegs;
    975       1.7       mrg {
    976       1.7       mrg 
    977      1.22       mrg 	DPRINTF(IDB_BUSDMA, ("iommu_dvmamem_free: segp %p nsegs %d\n",
    978       1.7       mrg 	    segs, nsegs));
    979       1.7       mrg 	bus_dmamem_free(t->_parent, segs, nsegs);
    980       1.7       mrg }
    981       1.7       mrg 
    982       1.7       mrg /*
    983       1.7       mrg  * Map the DVMA mappings into the kernel pmap.
    984       1.7       mrg  * Check the flags to see whether we're streaming or coherent.
    985       1.7       mrg  */
    986       1.7       mrg int
    987  1.35.2.7  jdolecek iommu_dvmamem_map(t, sb, segs, nsegs, size, kvap, flags)
    988       1.7       mrg 	bus_dma_tag_t t;
    989  1.35.2.7  jdolecek 	struct strbuf_ctl *sb;
    990       1.7       mrg 	bus_dma_segment_t *segs;
    991       1.7       mrg 	int nsegs;
    992       1.7       mrg 	size_t size;
    993       1.7       mrg 	caddr_t *kvap;
    994       1.7       mrg 	int flags;
    995       1.7       mrg {
    996  1.35.2.8  jdolecek 	struct vm_page *pg;
    997       1.7       mrg 	vaddr_t va;
    998       1.7       mrg 	bus_addr_t addr;
    999  1.35.2.8  jdolecek 	struct pglist *pglist;
   1000       1.8       mrg 	int cbit;
   1001       1.7       mrg 
   1002      1.22       mrg 	DPRINTF(IDB_BUSDMA, ("iommu_dvmamem_map: segp %p nsegs %d size %lx\n",
   1003       1.7       mrg 	    segs, nsegs, size));
   1004       1.7       mrg 
   1005       1.7       mrg 	/*
   1006       1.8       mrg 	 * Allocate some space in the kernel map, and then map these pages
   1007       1.8       mrg 	 * into this space.
   1008       1.7       mrg 	 */
   1009       1.8       mrg 	size = round_page(size);
   1010       1.8       mrg 	va = uvm_km_valloc(kernel_map, size);
   1011       1.8       mrg 	if (va == 0)
   1012       1.8       mrg 		return (ENOMEM);
   1013       1.7       mrg 
   1014       1.8       mrg 	*kvap = (caddr_t)va;
   1015       1.7       mrg 
   1016  1.35.2.8  jdolecek 	/*
   1017       1.7       mrg 	 * digest flags:
   1018       1.7       mrg 	 */
   1019       1.7       mrg 	cbit = 0;
   1020       1.7       mrg 	if (flags & BUS_DMA_COHERENT)	/* Disable vcache */
   1021       1.7       mrg 		cbit |= PMAP_NVC;
   1022       1.7       mrg 	if (flags & BUS_DMA_NOCACHE)	/* sideffects */
   1023       1.7       mrg 		cbit |= PMAP_NC;
   1024       1.7       mrg 
   1025       1.7       mrg 	/*
   1026       1.8       mrg 	 * Now take this and map it into the CPU.
   1027       1.7       mrg 	 */
   1028  1.35.2.8  jdolecek 	pglist = segs[0]._ds_mlist;
   1029  1.35.2.8  jdolecek 	TAILQ_FOREACH(pg, pglist, pageq) {
   1030       1.8       mrg #ifdef DIAGNOSTIC
   1031       1.7       mrg 		if (size == 0)
   1032       1.7       mrg 			panic("iommu_dvmamem_map: size botch");
   1033       1.8       mrg #endif
   1034  1.35.2.8  jdolecek 		addr = VM_PAGE_TO_PHYS(pg);
   1035      1.22       mrg 		DPRINTF(IDB_BUSDMA, ("iommu_dvmamem_map: "
   1036      1.25       mrg 		    "mapping va %lx at %llx\n", va, (unsigned long long)addr | cbit));
   1037  1.35.2.8  jdolecek 		pmap_kenter_pa(va, addr | cbit, VM_PROT_READ | VM_PROT_WRITE);
   1038       1.7       mrg 		va += PAGE_SIZE;
   1039       1.7       mrg 		size -= PAGE_SIZE;
   1040       1.7       mrg 	}
   1041  1.35.2.3   thorpej 	pmap_update(pmap_kernel());
   1042       1.7       mrg 	return (0);
   1043       1.7       mrg }
   1044       1.7       mrg 
   1045       1.7       mrg /*
   1046       1.7       mrg  * Unmap DVMA mappings from kernel
   1047       1.7       mrg  */
   1048       1.7       mrg void
   1049  1.35.2.7  jdolecek iommu_dvmamem_unmap(t, sb, kva, size)
   1050       1.7       mrg 	bus_dma_tag_t t;
   1051  1.35.2.7  jdolecek 	struct strbuf_ctl *sb;
   1052       1.7       mrg 	caddr_t kva;
   1053       1.7       mrg 	size_t size;
   1054       1.7       mrg {
   1055  1.35.2.8  jdolecek 
   1056      1.22       mrg 	DPRINTF(IDB_BUSDMA, ("iommu_dvmamem_unmap: kvm %p size %lx\n",
   1057       1.7       mrg 	    kva, size));
   1058  1.35.2.8  jdolecek 
   1059       1.7       mrg #ifdef DIAGNOSTIC
   1060       1.7       mrg 	if ((u_long)kva & PGOFSET)
   1061       1.7       mrg 		panic("iommu_dvmamem_unmap");
   1062       1.7       mrg #endif
   1063  1.35.2.8  jdolecek 
   1064       1.7       mrg 	size = round_page(size);
   1065  1.35.2.8  jdolecek 	pmap_kremove((vaddr_t)kva, size);
   1066  1.35.2.3   thorpej 	pmap_update(pmap_kernel());
   1067       1.8       mrg 	uvm_km_free(kernel_map, (vaddr_t)kva, size);
   1068       1.1       mrg }
   1069