iommu.c revision 1.117 1 1.117 thorpej /* $NetBSD: iommu.c,v 1.117 2023/12/01 06:47:59 thorpej Exp $ */
2 1.82 mrg
3 1.82 mrg /*
4 1.82 mrg * Copyright (c) 1999, 2000 Matthew R. Green
5 1.82 mrg * All rights reserved.
6 1.82 mrg *
7 1.82 mrg * Redistribution and use in source and binary forms, with or without
8 1.82 mrg * modification, are permitted provided that the following conditions
9 1.82 mrg * are met:
10 1.82 mrg * 1. Redistributions of source code must retain the above copyright
11 1.82 mrg * notice, this list of conditions and the following disclaimer.
12 1.82 mrg * 2. Redistributions in binary form must reproduce the above copyright
13 1.82 mrg * notice, this list of conditions and the following disclaimer in the
14 1.82 mrg * documentation and/or other materials provided with the distribution.
15 1.82 mrg *
16 1.82 mrg * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
17 1.82 mrg * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
18 1.82 mrg * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
19 1.82 mrg * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
20 1.82 mrg * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
21 1.82 mrg * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
22 1.82 mrg * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
23 1.82 mrg * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
24 1.82 mrg * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25 1.82 mrg * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
26 1.82 mrg * SUCH DAMAGE.
27 1.82 mrg */
28 1.7 mrg
29 1.7 mrg /*
30 1.48 eeh * Copyright (c) 2001, 2002 Eduardo Horvath
31 1.7 mrg * All rights reserved.
32 1.7 mrg *
33 1.7 mrg * Redistribution and use in source and binary forms, with or without
34 1.7 mrg * modification, are permitted provided that the following conditions
35 1.7 mrg * are met:
36 1.7 mrg * 1. Redistributions of source code must retain the above copyright
37 1.7 mrg * notice, this list of conditions and the following disclaimer.
38 1.7 mrg * 2. Redistributions in binary form must reproduce the above copyright
39 1.7 mrg * notice, this list of conditions and the following disclaimer in the
40 1.7 mrg * documentation and/or other materials provided with the distribution.
41 1.7 mrg * 3. The name of the author may not be used to endorse or promote products
42 1.7 mrg * derived from this software without specific prior written permission.
43 1.7 mrg *
44 1.7 mrg * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
45 1.7 mrg * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
46 1.7 mrg * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
47 1.7 mrg * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
48 1.7 mrg * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
49 1.7 mrg * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
50 1.7 mrg * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
51 1.7 mrg * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
52 1.7 mrg * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
53 1.7 mrg * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
54 1.7 mrg * SUCH DAMAGE.
55 1.7 mrg */
56 1.1 mrg
57 1.7 mrg /*
58 1.7 mrg * UltraSPARC IOMMU support; used by both the sbus and pci code.
59 1.7 mrg */
60 1.66 lukem
61 1.66 lukem #include <sys/cdefs.h>
62 1.117 thorpej __KERNEL_RCSID(0, "$NetBSD: iommu.c,v 1.117 2023/12/01 06:47:59 thorpej Exp $");
63 1.66 lukem
64 1.4 mrg #include "opt_ddb.h"
65 1.4 mrg
66 1.1 mrg #include <sys/param.h>
67 1.1 mrg #include <sys/extent.h>
68 1.1 mrg #include <sys/malloc.h>
69 1.1 mrg #include <sys/systm.h>
70 1.1 mrg #include <sys/device.h>
71 1.41 chs #include <sys/proc.h>
72 1.18 mrg
73 1.100 uebayasi #include <uvm/uvm.h>
74 1.1 mrg
75 1.104 dyoung #include <sys/bus.h>
76 1.1 mrg #include <sparc64/dev/iommureg.h>
77 1.1 mrg #include <sparc64/dev/iommuvar.h>
78 1.1 mrg
79 1.1 mrg #include <machine/autoconf.h>
80 1.1 mrg #include <machine/cpu.h>
81 1.110 nakayama #include <machine/hypervisor.h>
82 1.1 mrg
83 1.1 mrg #ifdef DEBUG
84 1.22 mrg #define IDB_BUSDMA 0x1
85 1.22 mrg #define IDB_IOMMU 0x2
86 1.22 mrg #define IDB_INFO 0x4
87 1.36 eeh #define IDB_SYNC 0x8
88 1.10 mrg int iommudebug = 0x0;
89 1.4 mrg #define DPRINTF(l, s) do { if (iommudebug & l) printf s; } while (0)
90 1.90 nakayama #define IOTTE_DEBUG(n) (n)
91 1.4 mrg #else
92 1.4 mrg #define DPRINTF(l, s)
93 1.90 nakayama #define IOTTE_DEBUG(n) 0
94 1.1 mrg #endif
95 1.1 mrg
96 1.55 eeh #define iommu_strbuf_flush(i, v) do { \
97 1.55 eeh if ((i)->sb_flush) \
98 1.55 eeh bus_space_write_8((i)->sb_is->is_bustag, (i)->sb_sb, \
99 1.50 eeh STRBUFREG(strbuf_pgflush), (v)); \
100 1.42 eeh } while (0)
101 1.42 eeh
102 1.78 cdi static int iommu_strbuf_flush_done(struct strbuf_ctl *);
103 1.85 nakayama static void _iommu_dvmamap_sync(bus_dma_tag_t, bus_dmamap_t, bus_addr_t,
104 1.85 nakayama bus_size_t, int);
105 1.109 palle static void iommu_enter_sun4u(struct strbuf_ctl *sb, vaddr_t va, int64_t pa, int flags);
106 1.109 palle static void iommu_enter_sun4v(struct strbuf_ctl *sb, vaddr_t va, int64_t pa, int flags);
107 1.109 palle static void iommu_remove_sun4u(struct iommu_state *is, vaddr_t va, size_t len);
108 1.109 palle static void iommu_remove_sun4v(struct iommu_state *is, vaddr_t va, size_t len);
109 1.11 eeh
110 1.1 mrg /*
111 1.1 mrg * initialise the UltraSPARC IOMMU (SBUS or PCI):
112 1.1 mrg * - allocate and setup the iotsb.
113 1.1 mrg * - enable the IOMMU
114 1.7 mrg * - initialise the streaming buffers (if they exist)
115 1.1 mrg * - create a private DVMA map.
116 1.1 mrg */
117 1.1 mrg void
118 1.79 cdi iommu_init(char *name, struct iommu_state *is, int tsbsize, uint32_t iovabase)
119 1.1 mrg {
120 1.11 eeh psize_t size;
121 1.11 eeh vaddr_t va;
122 1.11 eeh paddr_t pa;
123 1.58 chs struct vm_page *pg;
124 1.58 chs struct pglist pglist;
125 1.1 mrg
126 1.109 palle DPRINTF(IDB_INFO, ("iommu_init: tsbsize %x iovabase %x\n", tsbsize, iovabase));
127 1.109 palle
128 1.1 mrg /*
129 1.1 mrg * Setup the iommu.
130 1.1 mrg *
131 1.45 eeh * The sun4u iommu is part of the SBUS or PCI controller so we will
132 1.45 eeh * deal with it here..
133 1.1 mrg *
134 1.45 eeh * For sysio and psycho/psycho+ the IOMMU address space always ends at
135 1.45 eeh * 0xffffe000, but the starting address depends on the size of the
136 1.45 eeh * map. The map size is 1024 * 2 ^ is->is_tsbsize entries, where each
137 1.45 eeh * entry is 8 bytes. The start of the map can be calculated by
138 1.45 eeh * (0xffffe000 << (8 + is->is_tsbsize)).
139 1.45 eeh *
140 1.45 eeh * But sabre and hummingbird use a different scheme that seems to
141 1.45 eeh * be hard-wired, so we read the start and size from the PROM and
142 1.45 eeh * just use those values.
143 1.2 eeh */
144 1.108 palle if (strncmp(name, "pyro", 4) == 0) {
145 1.108 palle is->is_cr = IOMMUREG_READ(is, iommu_cr);
146 1.108 palle is->is_cr &= ~IOMMUCR_FIRE_BE;
147 1.108 palle is->is_cr |= (IOMMUCR_FIRE_SE | IOMMUCR_FIRE_CM_EN |
148 1.108 palle IOMMUCR_FIRE_TE);
149 1.108 palle } else
150 1.108 palle is->is_cr = IOMMUCR_EN;
151 1.11 eeh is->is_tsbsize = tsbsize;
152 1.45 eeh if (iovabase == -1) {
153 1.45 eeh is->is_dvmabase = IOTSB_VSTART(is->is_tsbsize);
154 1.90 nakayama is->is_dvmaend = IOTSB_VEND - 1;
155 1.45 eeh } else {
156 1.45 eeh is->is_dvmabase = iovabase;
157 1.90 nakayama is->is_dvmaend = iovabase + IOTSB_VSIZE(tsbsize) - 1;
158 1.45 eeh }
159 1.11 eeh
160 1.11 eeh /*
161 1.15 eeh * Allocate memory for I/O pagetables. They need to be physically
162 1.15 eeh * contiguous.
163 1.11 eeh */
164 1.11 eeh
165 1.64 thorpej size = PAGE_SIZE << is->is_tsbsize;
166 1.11 eeh if (uvm_pglistalloc((psize_t)size, (paddr_t)0, (paddr_t)-1,
167 1.64 thorpej (paddr_t)PAGE_SIZE, (paddr_t)0, &pglist, 1, 0) != 0)
168 1.11 eeh panic("iommu_init: no memory");
169 1.11 eeh
170 1.76 yamt va = uvm_km_alloc(kernel_map, size, 0, UVM_KMF_VAONLY);
171 1.11 eeh if (va == 0)
172 1.11 eeh panic("iommu_init: no memory");
173 1.11 eeh is->is_tsb = (int64_t *)va;
174 1.11 eeh
175 1.58 chs is->is_ptsb = VM_PAGE_TO_PHYS(TAILQ_FIRST(&pglist));
176 1.11 eeh
177 1.11 eeh /* Map the pages */
178 1.83 ad TAILQ_FOREACH(pg, &pglist, pageq.queue) {
179 1.58 chs pa = VM_PAGE_TO_PHYS(pg);
180 1.88 cegger pmap_kenter_pa(va, pa | PMAP_NVC,
181 1.88 cegger VM_PROT_READ | VM_PROT_WRITE, 0);
182 1.64 thorpej va += PAGE_SIZE;
183 1.11 eeh }
184 1.38 chris pmap_update(pmap_kernel());
185 1.58 chs memset(is->is_tsb, 0, size);
186 1.1 mrg
187 1.1 mrg #ifdef DEBUG
188 1.102 mrg if (iommudebug & IDB_INFO)
189 1.1 mrg {
190 1.1 mrg /* Probe the iommu */
191 1.109 palle if (!CPU_ISSUN4V) {
192 1.109 palle printf("iommu cr=%llx tsb=%llx\n",
193 1.109 palle (unsigned long long)bus_space_read_8(is->is_bustag,
194 1.50 eeh is->is_iommu,
195 1.103 mrg offsetof(struct iommureg, iommu_cr)),
196 1.109 palle (unsigned long long)bus_space_read_8(is->is_bustag,
197 1.50 eeh is->is_iommu,
198 1.103 mrg offsetof(struct iommureg, iommu_tsb)));
199 1.109 palle printf("TSB base %p phys %llx\n", (void *)is->is_tsb,
200 1.109 palle (unsigned long long)is->is_ptsb);
201 1.109 palle delay(1000000); /* 1 s */
202 1.109 palle }
203 1.1 mrg }
204 1.1 mrg #endif
205 1.1 mrg
206 1.1 mrg /*
207 1.1 mrg * Now all the hardware's working we need to allocate a dvma map.
208 1.1 mrg */
209 1.98 mrg aprint_debug("DVMA map: %x to %x\n",
210 1.11 eeh (unsigned int)is->is_dvmabase,
211 1.45 eeh (unsigned int)is->is_dvmaend);
212 1.98 mrg aprint_debug("IOTSB: %llx to %llx\n",
213 1.47 eeh (unsigned long long)is->is_ptsb,
214 1.90 nakayama (unsigned long long)(is->is_ptsb + size - 1));
215 1.117 thorpej is->is_dvmamap = vmem_create(name,
216 1.117 thorpej is->is_dvmabase,
217 1.117 thorpej (is->is_dvmaend + 1) - is->is_dvmabase,
218 1.117 thorpej PAGE_SIZE, /* quantum */
219 1.117 thorpej NULL, /* importfn */
220 1.117 thorpej NULL, /* releasefn */
221 1.117 thorpej NULL, /* source */
222 1.117 thorpej 0, /* qcache_max */
223 1.117 thorpej VM_SLEEP,
224 1.117 thorpej IPL_VM);
225 1.117 thorpej KASSERT(is->is_dvmamap != NULL);
226 1.107 mrg
227 1.103 mrg /*
228 1.103 mrg * Set the TSB size. The relevant bits were moved to the TSB
229 1.103 mrg * base register in the PCIe host bridges.
230 1.103 mrg */
231 1.103 mrg if (is->is_flags & IOMMU_TSBSIZE_IN_PTSB)
232 1.103 mrg is->is_ptsb |= is->is_tsbsize;
233 1.103 mrg else
234 1.103 mrg is->is_cr |= (is->is_tsbsize << 16);
235 1.103 mrg
236 1.103 mrg /*
237 1.103 mrg * now actually start up the IOMMU
238 1.103 mrg */
239 1.103 mrg iommu_reset(is);
240 1.1 mrg }
241 1.1 mrg
242 1.8 mrg /*
243 1.8 mrg * Streaming buffers don't exist on the UltraSPARC IIi; we should have
244 1.8 mrg * detected that already and disabled them. If not, we will notice that
245 1.8 mrg * they aren't there when the STRBUF_EN bit does not remain.
246 1.8 mrg */
247 1.1 mrg void
248 1.78 cdi iommu_reset(struct iommu_state *is)
249 1.1 mrg {
250 1.45 eeh int i;
251 1.55 eeh struct strbuf_ctl *sb;
252 1.1 mrg
253 1.109 palle if (CPU_ISSUN4V)
254 1.109 palle return;
255 1.109 palle
256 1.103 mrg IOMMUREG_WRITE(is, iommu_tsb, is->is_ptsb);
257 1.50 eeh
258 1.11 eeh /* Enable IOMMU in diagnostic mode */
259 1.103 mrg IOMMUREG_WRITE(is, iommu_cr, is->is_cr|IOMMUCR_DE);
260 1.11 eeh
261 1.58 chs for (i = 0; i < 2; i++) {
262 1.55 eeh if ((sb = is->is_sb[i])) {
263 1.5 mrg
264 1.45 eeh /* Enable diagnostics mode? */
265 1.58 chs bus_space_write_8(is->is_bustag, is->is_sb[i]->sb_sb,
266 1.50 eeh STRBUFREG(strbuf_ctl), STRBUF_EN);
267 1.45 eeh
268 1.105 nakayama membar_Lookaside();
269 1.103 mrg
270 1.45 eeh /* No streaming buffers? Disable them */
271 1.58 chs if (bus_space_read_8(is->is_bustag,
272 1.58 chs is->is_sb[i]->sb_sb,
273 1.55 eeh STRBUFREG(strbuf_ctl)) == 0) {
274 1.55 eeh is->is_sb[i]->sb_flush = NULL;
275 1.55 eeh } else {
276 1.58 chs
277 1.55 eeh /*
278 1.55 eeh * locate the pa of the flush buffer.
279 1.55 eeh */
280 1.103 mrg if (pmap_extract(pmap_kernel(),
281 1.103 mrg (vaddr_t)is->is_sb[i]->sb_flush,
282 1.103 mrg &is->is_sb[i]->sb_flushpa) == FALSE)
283 1.103 mrg is->is_sb[i]->sb_flush = NULL;
284 1.55 eeh }
285 1.45 eeh }
286 1.42 eeh }
287 1.103 mrg
288 1.103 mrg if (is->is_flags & IOMMU_FLUSH_CACHE)
289 1.103 mrg IOMMUREG_WRITE(is, iommu_cache_invalidate, -1ULL);
290 1.2 eeh }
291 1.2 eeh
292 1.2 eeh /*
293 1.58 chs * Here are the iommu control routines.
294 1.2 eeh */
295 1.109 palle
296 1.116 mrg static void
297 1.78 cdi iommu_enter(struct strbuf_ctl *sb, vaddr_t va, int64_t pa, int flags)
298 1.2 eeh {
299 1.109 palle DPRINTF(IDB_IOMMU, ("iommu_enter: va %lx pa %lx flags %x\n",
300 1.109 palle va, (long)pa, flags));
301 1.109 palle if (!CPU_ISSUN4V)
302 1.109 palle iommu_enter_sun4u(sb, va, pa, flags);
303 1.109 palle else
304 1.109 palle iommu_enter_sun4v(sb, va, pa, flags);
305 1.109 palle }
306 1.109 palle
307 1.109 palle
308 1.109 palle void
309 1.109 palle iommu_enter_sun4u(struct strbuf_ctl *sb, vaddr_t va, int64_t pa, int flags)
310 1.109 palle {
311 1.55 eeh struct iommu_state *is = sb->sb_is;
312 1.55 eeh int strbuf = (flags & BUS_DMA_STREAMING);
313 1.2 eeh int64_t tte;
314 1.2 eeh
315 1.2 eeh #ifdef DIAGNOSTIC
316 1.45 eeh if (va < is->is_dvmabase || va > is->is_dvmaend)
317 1.13 mrg panic("iommu_enter: va %#lx not in DVMA space", va);
318 1.2 eeh #endif
319 1.2 eeh
320 1.55 eeh /* Is the streamcache flush really needed? */
321 1.91 nakayama if (sb->sb_flush)
322 1.55 eeh iommu_strbuf_flush(sb, va);
323 1.91 nakayama else
324 1.55 eeh /* If we can't flush the strbuf don't enable it. */
325 1.55 eeh strbuf = 0;
326 1.55 eeh
327 1.58 chs tte = MAKEIOTTE(pa, !(flags & BUS_DMA_NOWRITE),
328 1.55 eeh !(flags & BUS_DMA_NOCACHE), (strbuf));
329 1.50 eeh #ifdef DEBUG
330 1.50 eeh tte |= (flags & 0xff000LL)<<(4*8);
331 1.50 eeh #endif
332 1.58 chs
333 1.2 eeh is->is_tsb[IOTSBSLOT(va,is->is_tsbsize)] = tte;
334 1.58 chs bus_space_write_8(is->is_bustag, is->is_iommu,
335 1.50 eeh IOMMUREG(iommu_flush), va);
336 1.103 mrg DPRINTF(IDB_IOMMU, ("iommu_enter: slot %d va %lx pa %lx "
337 1.103 mrg "TSB[%lx]@%p=%lx\n", (int)IOTSBSLOT(va,is->is_tsbsize),
338 1.50 eeh va, (long)pa, (u_long)IOTSBSLOT(va,is->is_tsbsize),
339 1.50 eeh (void *)(u_long)&is->is_tsb[IOTSBSLOT(va,is->is_tsbsize)],
340 1.50 eeh (u_long)tte));
341 1.39 eeh }
342 1.39 eeh
343 1.109 palle void
344 1.109 palle iommu_enter_sun4v(struct strbuf_ctl *sb, vaddr_t va, int64_t pa, int flags)
345 1.109 palle {
346 1.109 palle struct iommu_state *is = sb->sb_is;
347 1.109 palle u_int64_t tsbid = IOTSBSLOT(va, is->is_tsbsize);
348 1.109 palle paddr_t page_list[1], addr;
349 1.109 palle u_int64_t attr, nmapped;
350 1.109 palle int err;
351 1.109 palle
352 1.109 palle #ifdef DIAGNOSTIC
353 1.109 palle if (va < is->is_dvmabase || (va + PAGE_MASK) > is->is_dvmaend)
354 1.109 palle panic("viommu_enter: va %#lx not in DVMA space", va);
355 1.109 palle #endif
356 1.109 palle
357 1.109 palle attr = PCI_MAP_ATTR_READ | PCI_MAP_ATTR_WRITE;
358 1.109 palle if (flags & BUS_DMA_READ)
359 1.109 palle attr &= ~PCI_MAP_ATTR_READ;
360 1.109 palle if (flags & BUS_DMA_WRITE)
361 1.109 palle attr &= ~PCI_MAP_ATTR_WRITE;
362 1.109 palle
363 1.109 palle page_list[0] = trunc_page(pa);
364 1.109 palle if (!pmap_extract(pmap_kernel(), (vaddr_t)page_list, &addr))
365 1.109 palle panic("viommu_enter: pmap_extract failed");
366 1.109 palle err = hv_pci_iommu_map(is->is_devhandle, tsbid, 1, attr,
367 1.109 palle addr, &nmapped);
368 1.109 palle if (err != H_EOK || nmapped != 1)
369 1.109 palle panic("hv_pci_iommu_map: err=%d, nmapped=%lu", err, (long unsigned int)nmapped);
370 1.109 palle }
371 1.109 palle
372 1.39 eeh /*
373 1.39 eeh * Find the value of a DVMA address (debug routine).
374 1.39 eeh */
375 1.39 eeh paddr_t
376 1.78 cdi iommu_extract(struct iommu_state *is, vaddr_t dva)
377 1.39 eeh {
378 1.39 eeh int64_t tte = 0;
379 1.58 chs
380 1.90 nakayama if (dva >= is->is_dvmabase && dva <= is->is_dvmaend)
381 1.55 eeh tte = is->is_tsb[IOTSBSLOT(dva, is->is_tsbsize)];
382 1.39 eeh
383 1.54 eeh if ((tte & IOTTE_V) == 0)
384 1.39 eeh return ((paddr_t)-1L);
385 1.54 eeh return (tte & IOTTE_PAMASK);
386 1.2 eeh }
387 1.2 eeh
388 1.2 eeh /*
389 1.2 eeh * iommu_remove: removes mappings created by iommu_enter
390 1.2 eeh *
391 1.2 eeh * Only demap from IOMMU if flag is set.
392 1.8 mrg *
393 1.8 mrg * XXX: this function needs better internal error checking.
394 1.2 eeh */
395 1.109 palle
396 1.116 mrg static void
397 1.78 cdi iommu_remove(struct iommu_state *is, vaddr_t va, size_t len)
398 1.2 eeh {
399 1.109 palle DPRINTF(IDB_IOMMU, ("iommu_remove: va %lx len %zu\n", va, len));
400 1.109 palle if (!CPU_ISSUN4V)
401 1.109 palle iommu_remove_sun4u(is, va, len);
402 1.109 palle else
403 1.109 palle iommu_remove_sun4v(is, va, len);
404 1.109 palle }
405 1.109 palle
406 1.109 palle void
407 1.109 palle iommu_remove_sun4u(struct iommu_state *is, vaddr_t va, size_t len)
408 1.109 palle {
409 1.109 palle
410 1.103 mrg int slot;
411 1.2 eeh
412 1.2 eeh #ifdef DIAGNOSTIC
413 1.45 eeh if (va < is->is_dvmabase || va > is->is_dvmaend)
414 1.25 mrg panic("iommu_remove: va 0x%lx not in DVMA space", (u_long)va);
415 1.2 eeh if ((long)(va + len) < (long)va)
416 1.58 chs panic("iommu_remove: va 0x%lx + len 0x%lx wraps",
417 1.2 eeh (long) va, (long) len);
418 1.58 chs if (len & ~0xfffffff)
419 1.72 snj panic("iommu_remove: ridiculous len 0x%lx", (u_long)len);
420 1.2 eeh #endif
421 1.2 eeh
422 1.2 eeh va = trunc_page(va);
423 1.22 mrg DPRINTF(IDB_IOMMU, ("iommu_remove: va %lx TSB[%lx]@%p\n",
424 1.50 eeh va, (u_long)IOTSBSLOT(va, is->is_tsbsize),
425 1.50 eeh &is->is_tsb[IOTSBSLOT(va, is->is_tsbsize)]));
426 1.2 eeh while (len > 0) {
427 1.50 eeh DPRINTF(IDB_IOMMU, ("iommu_remove: clearing TSB slot %d "
428 1.50 eeh "for va %p size %lx\n",
429 1.50 eeh (int)IOTSBSLOT(va,is->is_tsbsize), (void *)(u_long)va,
430 1.50 eeh (u_long)len));
431 1.64 thorpej if (len <= PAGE_SIZE)
432 1.10 mrg len = 0;
433 1.10 mrg else
434 1.64 thorpej len -= PAGE_SIZE;
435 1.8 mrg
436 1.99 mrg #if 0
437 1.94 nakayama /*
438 1.94 nakayama * XXX Zero-ing the entry would not require RMW
439 1.94 nakayama *
440 1.94 nakayama * Disabling valid bit while a page is used by a device
441 1.94 nakayama * causes an uncorrectable DMA error.
442 1.94 nakayama * Workaround to avoid an uncorrectable DMA error is
443 1.94 nakayama * eliminating the next line, but the page is mapped
444 1.94 nakayama * until the next iommu_enter call.
445 1.94 nakayama */
446 1.47 eeh is->is_tsb[IOTSBSLOT(va,is->is_tsbsize)] &= ~IOTTE_V;
447 1.105 nakayama membar_StoreStore();
448 1.99 mrg #endif
449 1.103 mrg IOMMUREG_WRITE(is, iommu_flush, va);
450 1.103 mrg
451 1.103 mrg /* Flush cache if necessary. */
452 1.103 mrg slot = IOTSBSLOT(trunc_page(va), is->is_tsbsize);
453 1.103 mrg if ((is->is_flags & IOMMU_FLUSH_CACHE) &&
454 1.103 mrg (len == 0 || (slot % 8) == 7))
455 1.103 mrg IOMMUREG_WRITE(is, iommu_cache_flush,
456 1.103 mrg is->is_ptsb + slot * 8);
457 1.103 mrg
458 1.64 thorpej va += PAGE_SIZE;
459 1.2 eeh }
460 1.2 eeh }
461 1.2 eeh
462 1.109 palle void
463 1.109 palle iommu_remove_sun4v(struct iommu_state *is, vaddr_t va, size_t len)
464 1.109 palle {
465 1.109 palle u_int64_t tsbid = IOTSBSLOT(va, is->is_tsbsize);
466 1.109 palle u_int64_t ndemapped;
467 1.109 palle int err;
468 1.109 palle
469 1.109 palle #ifdef DIAGNOSTIC
470 1.109 palle if (va < is->is_dvmabase || (va + PAGE_MASK) > is->is_dvmaend)
471 1.109 palle panic("iommu_remove: va 0x%lx not in DVMA space", (u_long)va);
472 1.109 palle if (va != trunc_page(va)) {
473 1.109 palle printf("iommu_remove: unaligned va: %lx\n", va);
474 1.109 palle va = trunc_page(va);
475 1.109 palle }
476 1.109 palle #endif
477 1.109 palle
478 1.109 palle err = hv_pci_iommu_demap(is->is_devhandle, tsbid, 1, &ndemapped);
479 1.109 palle if (err != H_EOK || ndemapped != 1)
480 1.109 palle panic("hv_pci_iommu_unmap: err=%d", err);
481 1.109 palle }
482 1.109 palle
483 1.58 chs static int
484 1.78 cdi iommu_strbuf_flush_done(struct strbuf_ctl *sb)
485 1.2 eeh {
486 1.55 eeh struct iommu_state *is = sb->sb_is;
487 1.2 eeh struct timeval cur, flushtimeout;
488 1.2 eeh
489 1.2 eeh #define BUMPTIME(t, usec) { \
490 1.2 eeh register volatile struct timeval *tp = (t); \
491 1.2 eeh register long us; \
492 1.2 eeh \
493 1.2 eeh tp->tv_usec = us = tp->tv_usec + (usec); \
494 1.2 eeh if (us >= 1000000) { \
495 1.2 eeh tp->tv_usec = us - 1000000; \
496 1.2 eeh tp->tv_sec++; \
497 1.2 eeh } \
498 1.2 eeh }
499 1.5 mrg
500 1.55 eeh if (!sb->sb_flush)
501 1.5 mrg return (0);
502 1.58 chs
503 1.7 mrg /*
504 1.7 mrg * Streaming buffer flushes:
505 1.58 chs *
506 1.7 mrg * 1 Tell strbuf to flush by storing va to strbuf_pgflush. If
507 1.7 mrg * we're not on a cache line boundary (64-bits):
508 1.7 mrg * 2 Store 0 in flag
509 1.7 mrg * 3 Store pointer to flag in flushsync
510 1.7 mrg * 4 wait till flushsync becomes 0x1
511 1.7 mrg *
512 1.7 mrg * If it takes more than .5 sec, something
513 1.7 mrg * went wrong.
514 1.7 mrg */
515 1.2 eeh
516 1.55 eeh *sb->sb_flush = 0;
517 1.58 chs bus_space_write_8(is->is_bustag, sb->sb_sb,
518 1.55 eeh STRBUFREG(strbuf_flushsync), sb->sb_flushpa);
519 1.2 eeh
520 1.58 chs microtime(&flushtimeout);
521 1.2 eeh cur = flushtimeout;
522 1.2 eeh BUMPTIME(&flushtimeout, 500000); /* 1/2 sec */
523 1.58 chs
524 1.103 mrg DPRINTF(IDB_IOMMU, ("%s: flush = %lx at va = %lx pa = %lx now="
525 1.103 mrg "%"PRIx64":%"PRIx32" until = %"PRIx64":%"PRIx32"\n", __func__,
526 1.58 chs (long)*sb->sb_flush, (long)sb->sb_flush, (long)sb->sb_flushpa,
527 1.42 eeh cur.tv_sec, cur.tv_usec,
528 1.42 eeh flushtimeout.tv_sec, flushtimeout.tv_usec));
529 1.42 eeh
530 1.2 eeh /* Bypass non-coherent D$ */
531 1.55 eeh while ((!ldxa(sb->sb_flushpa, ASI_PHYS_CACHED)) &&
532 1.98 mrg timercmp(&cur, &flushtimeout, <=))
533 1.2 eeh microtime(&cur);
534 1.2 eeh
535 1.2 eeh #ifdef DIAGNOSTIC
536 1.55 eeh if (!ldxa(sb->sb_flushpa, ASI_PHYS_CACHED)) {
537 1.103 mrg printf("%s: flush timeout %p, at %p\n", __func__,
538 1.55 eeh (void *)(u_long)*sb->sb_flush,
539 1.55 eeh (void *)(u_long)sb->sb_flushpa); /* panic? */
540 1.2 eeh #ifdef DDB
541 1.2 eeh Debugger();
542 1.2 eeh #endif
543 1.2 eeh }
544 1.2 eeh #endif
545 1.103 mrg DPRINTF(IDB_IOMMU, ("%s: flushed\n", __func__));
546 1.55 eeh return (*sb->sb_flush);
547 1.7 mrg }
548 1.7 mrg
549 1.7 mrg /*
550 1.7 mrg * IOMMU DVMA operations, common to SBUS and PCI.
551 1.7 mrg */
552 1.7 mrg int
553 1.85 nakayama iommu_dvmamap_load(bus_dma_tag_t t, bus_dmamap_t map, void *buf,
554 1.85 nakayama bus_size_t buflen, struct proc *p, int flags)
555 1.7 mrg {
556 1.85 nakayama struct strbuf_ctl *sb = (struct strbuf_ctl *)map->_dm_cookie;
557 1.55 eeh struct iommu_state *is = sb->sb_is;
558 1.91 nakayama int err, needsflush;
559 1.7 mrg bus_size_t sgsize;
560 1.7 mrg paddr_t curaddr;
561 1.117 thorpej u_long sgstart, sgend, bmask;
562 1.117 thorpej vmem_addr_t dvmaddr;
563 1.71 tsutsui bus_size_t align, boundary, len;
564 1.7 mrg vaddr_t vaddr = (vaddr_t)buf;
565 1.40 eeh int seg;
566 1.58 chs struct pmap *pmap;
567 1.103 mrg int slot;
568 1.7 mrg
569 1.7 mrg if (map->dm_nsegs) {
570 1.7 mrg /* Already in use?? */
571 1.7 mrg #ifdef DIAGNOSTIC
572 1.7 mrg printf("iommu_dvmamap_load: map still in use\n");
573 1.7 mrg #endif
574 1.7 mrg bus_dmamap_unload(t, map);
575 1.7 mrg }
576 1.58 chs
577 1.7 mrg /*
578 1.7 mrg * Make sure that on error condition we return "no valid mappings".
579 1.7 mrg */
580 1.7 mrg map->dm_nsegs = 0;
581 1.96 nakayama KASSERT(map->dm_maxsegsz <= map->_dm_maxmaxsegsz);
582 1.96 nakayama
583 1.7 mrg if (buflen > map->_dm_size) {
584 1.22 mrg DPRINTF(IDB_BUSDMA,
585 1.7 mrg ("iommu_dvmamap_load(): error %d > %d -- "
586 1.25 mrg "map size exceeded!\n", (int)buflen, (int)map->_dm_size));
587 1.7 mrg return (EINVAL);
588 1.7 mrg }
589 1.7 mrg
590 1.7 mrg sgsize = round_page(buflen + ((int)vaddr & PGOFSET));
591 1.20 mrg
592 1.7 mrg /*
593 1.21 eeh * A boundary presented to bus_dmamem_alloc() takes precedence
594 1.21 eeh * over boundary in the map.
595 1.7 mrg */
596 1.21 eeh if ((boundary = (map->dm_segs[0]._ds_boundary)) == 0)
597 1.21 eeh boundary = map->_dm_boundary;
598 1.114 riastrad align = uimax(map->dm_segs[0]._ds_align, PAGE_SIZE);
599 1.58 chs
600 1.58 chs /*
601 1.58 chs * If our segment size is larger than the boundary we need to
602 1.40 eeh * split the transfer up int little pieces ourselves.
603 1.40 eeh */
604 1.117 thorpej KASSERT(is->is_dvmamap != NULL);
605 1.117 thorpej err = vmem_xalloc(is->is_dvmamap, sgsize,
606 1.117 thorpej align, /* alignment */
607 1.117 thorpej 0, /* phase */
608 1.117 thorpej (sgsize > boundary) ? 0 : boundary,
609 1.117 thorpej VMEM_ADDR_MIN, /* minaddr */
610 1.117 thorpej VMEM_ADDR_MAX, /* maxaddr */
611 1.117 thorpej VM_NOSLEEP | VM_BESTFIT,
612 1.117 thorpej &dvmaddr);
613 1.7 mrg
614 1.7 mrg #ifdef DEBUG
615 1.71 tsutsui if (err || (dvmaddr == (u_long)-1)) {
616 1.7 mrg printf("iommu_dvmamap_load(): extent_alloc(%d, %x) failed!\n",
617 1.25 mrg (int)sgsize, flags);
618 1.40 eeh #ifdef DDB
619 1.7 mrg Debugger();
620 1.40 eeh #endif
621 1.58 chs }
622 1.58 chs #endif
623 1.11 eeh if (err != 0)
624 1.11 eeh return (err);
625 1.11 eeh
626 1.65 nakayama if (dvmaddr == (u_long)-1)
627 1.7 mrg return (ENOMEM);
628 1.7 mrg
629 1.40 eeh /* Set the active DVMA map */
630 1.40 eeh map->_dm_dvmastart = dvmaddr;
631 1.40 eeh map->_dm_dvmasize = sgsize;
632 1.40 eeh
633 1.40 eeh /*
634 1.40 eeh * Now split the DVMA range into segments, not crossing
635 1.40 eeh * the boundary.
636 1.40 eeh */
637 1.40 eeh seg = 0;
638 1.40 eeh sgstart = dvmaddr + (vaddr & PGOFSET);
639 1.40 eeh sgend = sgstart + buflen - 1;
640 1.40 eeh map->dm_segs[seg].ds_addr = sgstart;
641 1.71 tsutsui DPRINTF(IDB_INFO, ("iommu_dvmamap_load: boundary %lx boundary - 1 %lx "
642 1.71 tsutsui "~(boundary - 1) %lx\n", (long)boundary, (long)(boundary - 1),
643 1.71 tsutsui (long)~(boundary - 1)));
644 1.90 nakayama bmask = ~(boundary - 1);
645 1.96 nakayama while ((sgstart & bmask) != (sgend & bmask) ||
646 1.96 nakayama sgend - sgstart + 1 > map->dm_maxsegsz) {
647 1.96 nakayama /* Oops. We crossed a boundary or large seg. Split the xfer. */
648 1.96 nakayama len = map->dm_maxsegsz;
649 1.96 nakayama if ((sgstart & bmask) != (sgend & bmask))
650 1.114 riastrad len = uimin(len, boundary - (sgstart & (boundary - 1)));
651 1.71 tsutsui map->dm_segs[seg].ds_len = len;
652 1.40 eeh DPRINTF(IDB_INFO, ("iommu_dvmamap_load: "
653 1.71 tsutsui "seg %d start %lx size %lx\n", seg,
654 1.71 tsutsui (long)map->dm_segs[seg].ds_addr,
655 1.71 tsutsui (long)map->dm_segs[seg].ds_len));
656 1.53 eeh if (++seg >= map->_dm_segcnt) {
657 1.40 eeh /* Too many segments. Fail the operation. */
658 1.40 eeh DPRINTF(IDB_INFO, ("iommu_dvmamap_load: "
659 1.71 tsutsui "too many segments %d\n", seg));
660 1.117 thorpej vmem_xfree(is->is_dvmamap, dvmaddr, sgsize);
661 1.40 eeh map->_dm_dvmastart = 0;
662 1.40 eeh map->_dm_dvmasize = 0;
663 1.80 mrg return (EFBIG);
664 1.40 eeh }
665 1.71 tsutsui sgstart += len;
666 1.40 eeh map->dm_segs[seg].ds_addr = sgstart;
667 1.40 eeh }
668 1.40 eeh map->dm_segs[seg].ds_len = sgend - sgstart + 1;
669 1.40 eeh DPRINTF(IDB_INFO, ("iommu_dvmamap_load: "
670 1.71 tsutsui "seg %d start %lx size %lx\n", seg,
671 1.71 tsutsui (long)map->dm_segs[seg].ds_addr, (long)map->dm_segs[seg].ds_len));
672 1.71 tsutsui map->dm_nsegs = seg + 1;
673 1.7 mrg map->dm_mapsize = buflen;
674 1.7 mrg
675 1.7 mrg if (p != NULL)
676 1.7 mrg pmap = p->p_vmspace->vm_map.pmap;
677 1.7 mrg else
678 1.7 mrg pmap = pmap_kernel();
679 1.7 mrg
680 1.91 nakayama needsflush = 0;
681 1.7 mrg for (; buflen > 0; ) {
682 1.58 chs
683 1.7 mrg /*
684 1.7 mrg * Get the physical address for this page.
685 1.7 mrg */
686 1.7 mrg if (pmap_extract(pmap, (vaddr_t)vaddr, &curaddr) == FALSE) {
687 1.74 petrov #ifdef DIAGNOSTIC
688 1.74 petrov printf("iommu_dvmamap_load: pmap_extract failed %lx\n", vaddr);
689 1.74 petrov #endif
690 1.7 mrg bus_dmamap_unload(t, map);
691 1.7 mrg return (-1);
692 1.7 mrg }
693 1.7 mrg
694 1.7 mrg /*
695 1.7 mrg * Compute the segment size, and adjust counts.
696 1.7 mrg */
697 1.64 thorpej sgsize = PAGE_SIZE - ((u_long)vaddr & PGOFSET);
698 1.7 mrg if (buflen < sgsize)
699 1.7 mrg sgsize = buflen;
700 1.7 mrg
701 1.22 mrg DPRINTF(IDB_BUSDMA,
702 1.36 eeh ("iommu_dvmamap_load: map %p loading va %p "
703 1.71 tsutsui "dva %lx at pa %lx\n",
704 1.71 tsutsui map, (void *)vaddr, (long)dvmaddr,
705 1.87 nakayama (long)trunc_page(curaddr)));
706 1.55 eeh iommu_enter(sb, trunc_page(dvmaddr), trunc_page(curaddr),
707 1.90 nakayama flags | IOTTE_DEBUG(0x4000));
708 1.91 nakayama needsflush = 1;
709 1.58 chs
710 1.7 mrg vaddr += sgsize;
711 1.7 mrg buflen -= sgsize;
712 1.103 mrg
713 1.103 mrg /* Flush cache if necessary. */
714 1.103 mrg slot = IOTSBSLOT(trunc_page(dvmaddr), is->is_tsbsize);
715 1.103 mrg if ((is->is_flags & IOMMU_FLUSH_CACHE) &&
716 1.103 mrg (buflen <= 0 || (slot % 8) == 7))
717 1.103 mrg IOMMUREG_WRITE(is, iommu_cache_flush,
718 1.103 mrg is->is_ptsb + slot * 8);
719 1.103 mrg
720 1.103 mrg dvmaddr += PAGE_SIZE;
721 1.7 mrg }
722 1.91 nakayama if (needsflush)
723 1.91 nakayama iommu_strbuf_flush_done(sb);
724 1.45 eeh #ifdef DIAGNOSTIC
725 1.45 eeh for (seg = 0; seg < map->dm_nsegs; seg++) {
726 1.45 eeh if (map->dm_segs[seg].ds_addr < is->is_dvmabase ||
727 1.45 eeh map->dm_segs[seg].ds_addr > is->is_dvmaend) {
728 1.45 eeh printf("seg %d dvmaddr %lx out of range %x - %x\n",
729 1.71 tsutsui seg, (long)map->dm_segs[seg].ds_addr,
730 1.71 tsutsui is->is_dvmabase, is->is_dvmaend);
731 1.57 chs #ifdef DDB
732 1.45 eeh Debugger();
733 1.57 chs #endif
734 1.45 eeh }
735 1.45 eeh }
736 1.45 eeh #endif
737 1.7 mrg return (0);
738 1.7 mrg }
739 1.7 mrg
740 1.7 mrg
741 1.7 mrg void
742 1.85 nakayama iommu_dvmamap_unload(bus_dma_tag_t t, bus_dmamap_t map)
743 1.7 mrg {
744 1.85 nakayama struct strbuf_ctl *sb = (struct strbuf_ctl *)map->_dm_cookie;
745 1.55 eeh struct iommu_state *is = sb->sb_is;
746 1.7 mrg
747 1.40 eeh /* Flush the iommu */
748 1.113 christos if (!map->_dm_dvmastart)
749 1.113 christos panic("%s: error dvmastart is zero!\n", __func__);
750 1.7 mrg
751 1.115 mrg if (is->is_flags & IOMMU_SYNC_BEFORE_UNMAP) {
752 1.115 mrg
753 1.115 mrg /* Flush the caches */
754 1.115 mrg bus_dmamap_unload(t->_parent, map);
755 1.115 mrg
756 1.115 mrg iommu_remove(is, map->_dm_dvmastart, map->_dm_dvmasize);
757 1.115 mrg
758 1.115 mrg } else {
759 1.115 mrg
760 1.115 mrg iommu_remove(is, map->_dm_dvmastart, map->_dm_dvmasize);
761 1.115 mrg
762 1.115 mrg /* Flush the caches */
763 1.115 mrg bus_dmamap_unload(t->_parent, map);
764 1.115 mrg }
765 1.23 eeh
766 1.117 thorpej vmem_xfree(is->is_dvmamap, map->_dm_dvmastart, map->_dm_dvmasize);
767 1.43 eeh map->_dm_dvmastart = 0;
768 1.43 eeh map->_dm_dvmasize = 0;
769 1.40 eeh
770 1.40 eeh /* Clear the map */
771 1.9 eeh }
772 1.9 eeh
773 1.9 eeh
774 1.9 eeh int
775 1.85 nakayama iommu_dvmamap_load_raw(bus_dma_tag_t t, bus_dmamap_t map,
776 1.85 nakayama bus_dma_segment_t *segs, int nsegs, bus_size_t size, int flags)
777 1.9 eeh {
778 1.85 nakayama struct strbuf_ctl *sb = (struct strbuf_ctl *)map->_dm_cookie;
779 1.55 eeh struct iommu_state *is = sb->sb_is;
780 1.58 chs struct vm_page *pg;
781 1.107 mrg int i, j;
782 1.26 martin int left;
783 1.91 nakayama int err, needsflush;
784 1.9 eeh bus_size_t sgsize;
785 1.9 eeh paddr_t pa;
786 1.21 eeh bus_size_t boundary, align;
787 1.90 nakayama u_long dvmaddr, sgstart, sgend, bmask;
788 1.58 chs struct pglist *pglist;
789 1.90 nakayama const int pagesz = PAGE_SIZE;
790 1.103 mrg int slot;
791 1.90 nakayama #ifdef DEBUG
792 1.90 nakayama int npg = 0;
793 1.90 nakayama #endif
794 1.9 eeh
795 1.9 eeh if (map->dm_nsegs) {
796 1.9 eeh /* Already in use?? */
797 1.9 eeh #ifdef DIAGNOSTIC
798 1.9 eeh printf("iommu_dvmamap_load_raw: map still in use\n");
799 1.9 eeh #endif
800 1.9 eeh bus_dmamap_unload(t, map);
801 1.9 eeh }
802 1.40 eeh
803 1.40 eeh /*
804 1.40 eeh * A boundary presented to bus_dmamem_alloc() takes precedence
805 1.40 eeh * over boundary in the map.
806 1.40 eeh */
807 1.40 eeh if ((boundary = segs[0]._ds_boundary) == 0)
808 1.40 eeh boundary = map->_dm_boundary;
809 1.40 eeh
810 1.114 riastrad align = uimax(segs[0]._ds_align, pagesz);
811 1.40 eeh
812 1.9 eeh /*
813 1.9 eeh * Make sure that on error condition we return "no valid mappings".
814 1.9 eeh */
815 1.9 eeh map->dm_nsegs = 0;
816 1.26 martin /* Count up the total number of pages we need */
817 1.93 nakayama pa = trunc_page(segs[0].ds_addr);
818 1.26 martin sgsize = 0;
819 1.40 eeh left = size;
820 1.93 nakayama for (i = 0; left > 0 && i < nsegs; i++) {
821 1.26 martin if (round_page(pa) != round_page(segs[i].ds_addr))
822 1.93 nakayama sgsize = round_page(sgsize) +
823 1.93 nakayama (segs[i].ds_addr & PGOFSET);
824 1.114 riastrad sgsize += uimin(left, segs[i].ds_len);
825 1.40 eeh left -= segs[i].ds_len;
826 1.26 martin pa = segs[i].ds_addr + segs[i].ds_len;
827 1.26 martin }
828 1.93 nakayama sgsize = round_page(sgsize);
829 1.9 eeh
830 1.58 chs /*
831 1.58 chs * If our segment size is larger than the boundary we need to
832 1.45 eeh * split the transfer up into little pieces ourselves.
833 1.9 eeh */
834 1.117 thorpej const vm_flag_t vmflags = VM_BESTFIT |
835 1.117 thorpej ((flags & BUS_DMA_NOWAIT) ? VM_NOSLEEP : VM_SLEEP);
836 1.9 eeh
837 1.117 thorpej err = vmem_xalloc(is->is_dvmamap, sgsize,
838 1.117 thorpej align, /* alignment */
839 1.117 thorpej 0, /* phase */
840 1.117 thorpej (sgsize > boundary) ? 0 : boundary,
841 1.117 thorpej VMEM_ADDR_MIN, /* minaddr */
842 1.117 thorpej VMEM_ADDR_MAX, /* maxaddr */
843 1.117 thorpej vmflags,
844 1.117 thorpej &dvmaddr);
845 1.9 eeh if (err != 0)
846 1.9 eeh return (err);
847 1.9 eeh
848 1.9 eeh #ifdef DEBUG
849 1.65 nakayama if (dvmaddr == (u_long)-1)
850 1.58 chs {
851 1.9 eeh printf("iommu_dvmamap_load_raw(): extent_alloc(%d, %x) failed!\n",
852 1.25 mrg (int)sgsize, flags);
853 1.57 chs #ifdef DDB
854 1.9 eeh Debugger();
855 1.57 chs #endif
856 1.58 chs }
857 1.58 chs #endif
858 1.65 nakayama if (dvmaddr == (u_long)-1)
859 1.9 eeh return (ENOMEM);
860 1.9 eeh
861 1.40 eeh /* Set the active DVMA map */
862 1.40 eeh map->_dm_dvmastart = dvmaddr;
863 1.40 eeh map->_dm_dvmasize = sgsize;
864 1.40 eeh
865 1.90 nakayama bmask = ~(boundary - 1);
866 1.58 chs if ((pglist = segs[0]._ds_mlist) == NULL) {
867 1.92 nakayama u_long prev_va = 0UL, last_va = dvmaddr;
868 1.45 eeh paddr_t prev_pa = 0;
869 1.45 eeh int end = 0, offset;
870 1.92 nakayama bus_size_t len = size;
871 1.45 eeh
872 1.26 martin /*
873 1.45 eeh * This segs is made up of individual physical
874 1.58 chs * segments, probably by _bus_dmamap_load_uio() or
875 1.26 martin * _bus_dmamap_load_mbuf(). Ignore the mlist and
876 1.45 eeh * load each one individually.
877 1.26 martin */
878 1.45 eeh j = 0;
879 1.91 nakayama needsflush = 0;
880 1.45 eeh for (i = 0; i < nsegs ; i++) {
881 1.40 eeh
882 1.45 eeh pa = segs[i].ds_addr;
883 1.45 eeh offset = (pa & PGOFSET);
884 1.45 eeh pa = trunc_page(pa);
885 1.45 eeh dvmaddr = trunc_page(dvmaddr);
886 1.114 riastrad left = uimin(len, segs[i].ds_len);
887 1.45 eeh
888 1.45 eeh DPRINTF(IDB_INFO, ("iommu_dvmamap_load_raw: converting "
889 1.58 chs "physseg %d start %lx size %lx\n", i,
890 1.61 martin (long)segs[i].ds_addr, (long)segs[i].ds_len));
891 1.26 martin
892 1.58 chs if ((pa == prev_pa) &&
893 1.47 eeh ((offset != 0) || (end != offset))) {
894 1.45 eeh /* We can re-use this mapping */
895 1.45 eeh dvmaddr = prev_va;
896 1.45 eeh }
897 1.29 martin
898 1.45 eeh sgstart = dvmaddr + offset;
899 1.45 eeh sgend = sgstart + left - 1;
900 1.26 martin
901 1.45 eeh /* Are the segments virtually adjacent? */
902 1.58 chs if ((j > 0) && (end == offset) &&
903 1.96 nakayama ((offset == 0) || (pa == prev_pa)) &&
904 1.96 nakayama (map->dm_segs[j-1].ds_len + left <=
905 1.96 nakayama map->dm_maxsegsz)) {
906 1.45 eeh /* Just append to the previous segment. */
907 1.45 eeh map->dm_segs[--j].ds_len += left;
908 1.93 nakayama /* Restore sgstart for boundary check */
909 1.93 nakayama sgstart = map->dm_segs[j].ds_addr;
910 1.45 eeh DPRINTF(IDB_INFO, ("iommu_dvmamap_load_raw: "
911 1.45 eeh "appending seg %d start %lx size %lx\n", j,
912 1.58 chs (long)map->dm_segs[j].ds_addr,
913 1.61 martin (long)map->dm_segs[j].ds_len));
914 1.45 eeh } else {
915 1.53 eeh if (j >= map->_dm_segcnt) {
916 1.92 nakayama iommu_remove(is, map->_dm_dvmastart,
917 1.92 nakayama last_va - map->_dm_dvmastart);
918 1.92 nakayama goto fail;
919 1.53 eeh }
920 1.45 eeh map->dm_segs[j].ds_addr = sgstart;
921 1.45 eeh map->dm_segs[j].ds_len = left;
922 1.45 eeh DPRINTF(IDB_INFO, ("iommu_dvmamap_load_raw: "
923 1.45 eeh "seg %d start %lx size %lx\n", j,
924 1.48 eeh (long)map->dm_segs[j].ds_addr,
925 1.61 martin (long)map->dm_segs[j].ds_len));
926 1.40 eeh }
927 1.45 eeh end = (offset + left) & PGOFSET;
928 1.40 eeh
929 1.40 eeh /* Check for boundary issues */
930 1.90 nakayama while ((sgstart & bmask) != (sgend & bmask)) {
931 1.40 eeh /* Need a new segment. */
932 1.40 eeh map->dm_segs[j].ds_len =
933 1.53 eeh boundary - (sgstart & (boundary - 1));
934 1.40 eeh DPRINTF(IDB_INFO, ("iommu_dvmamap_load_raw: "
935 1.40 eeh "seg %d start %lx size %lx\n", j,
936 1.58 chs (long)map->dm_segs[j].ds_addr,
937 1.61 martin (long)map->dm_segs[j].ds_len));
938 1.53 eeh if (++j >= map->_dm_segcnt) {
939 1.92 nakayama iommu_remove(is, map->_dm_dvmastart,
940 1.92 nakayama last_va - map->_dm_dvmastart);
941 1.92 nakayama goto fail;
942 1.40 eeh }
943 1.93 nakayama sgstart += map->dm_segs[j-1].ds_len;
944 1.40 eeh map->dm_segs[j].ds_addr = sgstart;
945 1.40 eeh map->dm_segs[j].ds_len = sgend - sgstart + 1;
946 1.40 eeh }
947 1.40 eeh
948 1.26 martin if (sgsize == 0)
949 1.26 martin panic("iommu_dmamap_load_raw: size botch");
950 1.40 eeh
951 1.45 eeh /* Now map a series of pages. */
952 1.51 eeh while (dvmaddr <= sgend) {
953 1.45 eeh DPRINTF(IDB_BUSDMA,
954 1.45 eeh ("iommu_dvmamap_load_raw: map %p "
955 1.45 eeh "loading va %lx at pa %lx\n",
956 1.45 eeh map, (long)dvmaddr,
957 1.45 eeh (long)(pa)));
958 1.45 eeh /* Enter it if we haven't before. */
959 1.91 nakayama if (prev_va != dvmaddr) {
960 1.55 eeh iommu_enter(sb, prev_va = dvmaddr,
961 1.90 nakayama prev_pa = pa,
962 1.90 nakayama flags | IOTTE_DEBUG(++npg << 12));
963 1.91 nakayama needsflush = 1;
964 1.103 mrg
965 1.103 mrg /* Flush cache if necessary. */
966 1.103 mrg slot = IOTSBSLOT(trunc_page(dvmaddr), is->is_tsbsize);
967 1.103 mrg if ((is->is_flags & IOMMU_FLUSH_CACHE) &&
968 1.103 mrg ((dvmaddr + pagesz) > sgend || (slot % 8) == 7))
969 1.103 mrg IOMMUREG_WRITE(is, iommu_cache_flush,
970 1.103 mrg is->is_ptsb + slot * 8);
971 1.91 nakayama }
972 1.103 mrg
973 1.45 eeh dvmaddr += pagesz;
974 1.45 eeh pa += pagesz;
975 1.92 nakayama last_va = dvmaddr;
976 1.45 eeh }
977 1.45 eeh
978 1.92 nakayama len -= left;
979 1.45 eeh ++j;
980 1.26 martin }
981 1.91 nakayama if (needsflush)
982 1.91 nakayama iommu_strbuf_flush_done(sb);
983 1.45 eeh
984 1.92 nakayama map->dm_mapsize = size;
985 1.45 eeh map->dm_nsegs = j;
986 1.45 eeh #ifdef DIAGNOSTIC
987 1.45 eeh { int seg;
988 1.45 eeh for (seg = 0; seg < map->dm_nsegs; seg++) {
989 1.45 eeh if (map->dm_segs[seg].ds_addr < is->is_dvmabase ||
990 1.103 mrg map->dm_segs[seg].ds_addr > is->is_dvmaend) {
991 1.45 eeh printf("seg %d dvmaddr %lx out of range %x - %x\n",
992 1.58 chs seg, (long)map->dm_segs[seg].ds_addr,
993 1.45 eeh is->is_dvmabase, is->is_dvmaend);
994 1.57 chs #ifdef DDB
995 1.45 eeh Debugger();
996 1.57 chs #endif
997 1.45 eeh }
998 1.45 eeh }
999 1.45 eeh }
1000 1.45 eeh #endif
1001 1.26 martin return (0);
1002 1.26 martin }
1003 1.58 chs
1004 1.9 eeh /*
1005 1.40 eeh * This was allocated with bus_dmamem_alloc.
1006 1.58 chs * The pages are on a `pglist'.
1007 1.9 eeh */
1008 1.26 martin i = 0;
1009 1.40 eeh sgstart = dvmaddr;
1010 1.40 eeh sgend = sgstart + size - 1;
1011 1.40 eeh map->dm_segs[i].ds_addr = sgstart;
1012 1.90 nakayama while ((sgstart & bmask) != (sgend & bmask)) {
1013 1.40 eeh /* Oops. We crossed a boundary. Split the xfer. */
1014 1.53 eeh map->dm_segs[i].ds_len = boundary - (sgstart & (boundary - 1));
1015 1.40 eeh DPRINTF(IDB_INFO, ("iommu_dvmamap_load_raw: "
1016 1.40 eeh "seg %d start %lx size %lx\n", i,
1017 1.48 eeh (long)map->dm_segs[i].ds_addr,
1018 1.61 martin (long)map->dm_segs[i].ds_len));
1019 1.53 eeh if (++i >= map->_dm_segcnt) {
1020 1.40 eeh /* Too many segments. Fail the operation. */
1021 1.92 nakayama goto fail;
1022 1.40 eeh }
1023 1.93 nakayama sgstart += map->dm_segs[i-1].ds_len;
1024 1.40 eeh map->dm_segs[i].ds_addr = sgstart;
1025 1.40 eeh }
1026 1.40 eeh DPRINTF(IDB_INFO, ("iommu_dvmamap_load_raw: "
1027 1.40 eeh "seg %d start %lx size %lx\n", i,
1028 1.61 martin (long)map->dm_segs[i].ds_addr, (long)map->dm_segs[i].ds_len));
1029 1.40 eeh map->dm_segs[i].ds_len = sgend - sgstart + 1;
1030 1.9 eeh
1031 1.91 nakayama needsflush = 0;
1032 1.83 ad TAILQ_FOREACH(pg, pglist, pageq.queue) {
1033 1.9 eeh if (sgsize == 0)
1034 1.9 eeh panic("iommu_dmamap_load_raw: size botch");
1035 1.58 chs pa = VM_PAGE_TO_PHYS(pg);
1036 1.9 eeh
1037 1.22 mrg DPRINTF(IDB_BUSDMA,
1038 1.9 eeh ("iommu_dvmamap_load_raw: map %p loading va %lx at pa %lx\n",
1039 1.9 eeh map, (long)dvmaddr, (long)(pa)));
1040 1.90 nakayama iommu_enter(sb, dvmaddr, pa, flags | IOTTE_DEBUG(0x8000));
1041 1.91 nakayama needsflush = 1;
1042 1.58 chs
1043 1.103 mrg sgsize -= pagesz;
1044 1.103 mrg
1045 1.103 mrg /* Flush cache if necessary. */
1046 1.103 mrg slot = IOTSBSLOT(trunc_page(dvmaddr), is->is_tsbsize);
1047 1.103 mrg if ((is->is_flags & IOMMU_FLUSH_CACHE) &&
1048 1.103 mrg (sgsize == 0 || (slot % 8) == 7))
1049 1.103 mrg IOMMUREG_WRITE(is, iommu_cache_flush,
1050 1.103 mrg is->is_ptsb + slot * 8);
1051 1.103 mrg
1052 1.102 mrg dvmaddr += pagesz;
1053 1.9 eeh }
1054 1.91 nakayama if (needsflush)
1055 1.91 nakayama iommu_strbuf_flush_done(sb);
1056 1.40 eeh map->dm_mapsize = size;
1057 1.40 eeh map->dm_nsegs = i+1;
1058 1.45 eeh #ifdef DIAGNOSTIC
1059 1.45 eeh { int seg;
1060 1.45 eeh for (seg = 0; seg < map->dm_nsegs; seg++) {
1061 1.45 eeh if (map->dm_segs[seg].ds_addr < is->is_dvmabase ||
1062 1.45 eeh map->dm_segs[seg].ds_addr > is->is_dvmaend) {
1063 1.45 eeh printf("seg %d dvmaddr %lx out of range %x - %x\n",
1064 1.58 chs seg, (long)map->dm_segs[seg].ds_addr,
1065 1.45 eeh is->is_dvmabase, is->is_dvmaend);
1066 1.57 chs #ifdef DDB
1067 1.45 eeh Debugger();
1068 1.57 chs #endif
1069 1.45 eeh }
1070 1.45 eeh }
1071 1.45 eeh }
1072 1.45 eeh #endif
1073 1.9 eeh return (0);
1074 1.92 nakayama
1075 1.92 nakayama fail:
1076 1.117 thorpej vmem_free(is->is_dvmamap, map->_dm_dvmastart, sgsize);
1077 1.92 nakayama map->_dm_dvmastart = 0;
1078 1.92 nakayama map->_dm_dvmasize = 0;
1079 1.92 nakayama return (EFBIG);
1080 1.7 mrg }
1081 1.7 mrg
1082 1.67 petrov
1083 1.67 petrov /*
1084 1.67 petrov * Flush an individual dma segment, returns non-zero if the streaming buffers
1085 1.67 petrov * need flushing afterwards.
1086 1.67 petrov */
1087 1.67 petrov static int
1088 1.67 petrov iommu_dvmamap_sync_range(struct strbuf_ctl *sb, vaddr_t va, bus_size_t len)
1089 1.67 petrov {
1090 1.67 petrov vaddr_t vaend;
1091 1.67 petrov struct iommu_state *is = sb->sb_is;
1092 1.67 petrov
1093 1.67 petrov #ifdef DIAGNOSTIC
1094 1.67 petrov if (va < is->is_dvmabase || va > is->is_dvmaend)
1095 1.67 petrov panic("invalid va: %llx", (long long)va);
1096 1.67 petrov #endif
1097 1.67 petrov
1098 1.67 petrov if ((is->is_tsb[IOTSBSLOT(va, is->is_tsbsize)] & IOTTE_STREAM) == 0) {
1099 1.103 mrg DPRINTF(IDB_SYNC,
1100 1.67 petrov ("iommu_dvmamap_sync_range: attempting to flush "
1101 1.67 petrov "non-streaming entry\n"));
1102 1.67 petrov return (0);
1103 1.67 petrov }
1104 1.67 petrov
1105 1.90 nakayama vaend = round_page(va + len) - 1;
1106 1.87 nakayama va = trunc_page(va);
1107 1.67 petrov
1108 1.67 petrov #ifdef DIAGNOSTIC
1109 1.67 petrov if (va < is->is_dvmabase || vaend > is->is_dvmaend)
1110 1.67 petrov panic("invalid va range: %llx to %llx (%x to %x)",
1111 1.67 petrov (long long)va, (long long)vaend,
1112 1.67 petrov is->is_dvmabase,
1113 1.67 petrov is->is_dvmaend);
1114 1.67 petrov #endif
1115 1.67 petrov
1116 1.67 petrov for ( ; va <= vaend; va += PAGE_SIZE) {
1117 1.103 mrg DPRINTF(IDB_SYNC,
1118 1.67 petrov ("iommu_dvmamap_sync_range: flushing va %p\n",
1119 1.67 petrov (void *)(u_long)va));
1120 1.67 petrov iommu_strbuf_flush(sb, va);
1121 1.67 petrov }
1122 1.67 petrov
1123 1.67 petrov return (1);
1124 1.67 petrov }
1125 1.67 petrov
1126 1.85 nakayama static void
1127 1.85 nakayama _iommu_dvmamap_sync(bus_dma_tag_t t, bus_dmamap_t map, bus_addr_t offset,
1128 1.85 nakayama bus_size_t len, int ops)
1129 1.7 mrg {
1130 1.85 nakayama struct strbuf_ctl *sb = (struct strbuf_ctl *)map->_dm_cookie;
1131 1.67 petrov bus_size_t count;
1132 1.67 petrov int i, needsflush = 0;
1133 1.63 petrov
1134 1.63 petrov if (!sb->sb_flush)
1135 1.63 petrov return;
1136 1.7 mrg
1137 1.67 petrov for (i = 0; i < map->dm_nsegs; i++) {
1138 1.67 petrov if (offset < map->dm_segs[i].ds_len)
1139 1.67 petrov break;
1140 1.67 petrov offset -= map->dm_segs[i].ds_len;
1141 1.67 petrov }
1142 1.60 petrov
1143 1.67 petrov if (i == map->dm_nsegs)
1144 1.103 mrg panic("%s: segment too short %llu", __func__,
1145 1.68 martin (unsigned long long)offset);
1146 1.60 petrov
1147 1.62 petrov if (ops & (BUS_DMASYNC_PREREAD | BUS_DMASYNC_POSTWRITE)) {
1148 1.60 petrov /* Nothing to do */;
1149 1.60 petrov }
1150 1.60 petrov
1151 1.62 petrov if (ops & (BUS_DMASYNC_POSTREAD | BUS_DMASYNC_PREWRITE)) {
1152 1.67 petrov
1153 1.67 petrov for (; len > 0 && i < map->dm_nsegs; i++) {
1154 1.67 petrov count = MIN(map->dm_segs[i].ds_len - offset, len);
1155 1.67 petrov if (count > 0 &&
1156 1.67 petrov iommu_dvmamap_sync_range(sb,
1157 1.67 petrov map->dm_segs[i].ds_addr + offset, count))
1158 1.67 petrov needsflush = 1;
1159 1.67 petrov offset = 0;
1160 1.67 petrov len -= count;
1161 1.67 petrov }
1162 1.60 petrov #ifdef DIAGNOSTIC
1163 1.67 petrov if (i == map->dm_nsegs && len > 0)
1164 1.103 mrg panic("%s: leftover %llu", __func__,
1165 1.73 nakayama (unsigned long long)len);
1166 1.60 petrov #endif
1167 1.55 eeh
1168 1.67 petrov if (needsflush)
1169 1.58 chs iommu_strbuf_flush_done(sb);
1170 1.7 mrg }
1171 1.7 mrg }
1172 1.7 mrg
1173 1.85 nakayama void
1174 1.85 nakayama iommu_dvmamap_sync(bus_dma_tag_t t, bus_dmamap_t map, bus_addr_t offset,
1175 1.85 nakayama bus_size_t len, int ops)
1176 1.85 nakayama {
1177 1.85 nakayama
1178 1.89 jdc /* If len is 0, then there is nothing to do */
1179 1.89 jdc if (len == 0)
1180 1.89 jdc return;
1181 1.89 jdc
1182 1.85 nakayama if (ops & (BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE)) {
1183 1.85 nakayama /* Flush the CPU then the IOMMU */
1184 1.85 nakayama bus_dmamap_sync(t->_parent, map, offset, len, ops);
1185 1.85 nakayama _iommu_dvmamap_sync(t, map, offset, len, ops);
1186 1.85 nakayama }
1187 1.85 nakayama if (ops & (BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE)) {
1188 1.85 nakayama /* Flush the IOMMU then the CPU */
1189 1.85 nakayama _iommu_dvmamap_sync(t, map, offset, len, ops);
1190 1.85 nakayama bus_dmamap_sync(t->_parent, map, offset, len, ops);
1191 1.85 nakayama }
1192 1.85 nakayama }
1193 1.85 nakayama
1194 1.7 mrg int
1195 1.85 nakayama iommu_dvmamem_alloc(bus_dma_tag_t t, bus_size_t size, bus_size_t alignment,
1196 1.85 nakayama bus_size_t boundary, bus_dma_segment_t *segs, int nsegs, int *rsegs,
1197 1.85 nakayama int flags)
1198 1.7 mrg {
1199 1.7 mrg
1200 1.25 mrg DPRINTF(IDB_BUSDMA, ("iommu_dvmamem_alloc: sz %llx align %llx bound %llx "
1201 1.25 mrg "segp %p flags %d\n", (unsigned long long)size,
1202 1.25 mrg (unsigned long long)alignment, (unsigned long long)boundary,
1203 1.25 mrg segs, flags));
1204 1.7 mrg return (bus_dmamem_alloc(t->_parent, size, alignment, boundary,
1205 1.21 eeh segs, nsegs, rsegs, flags|BUS_DMA_DVMA));
1206 1.7 mrg }
1207 1.7 mrg
1208 1.7 mrg void
1209 1.85 nakayama iommu_dvmamem_free(bus_dma_tag_t t, bus_dma_segment_t *segs, int nsegs)
1210 1.7 mrg {
1211 1.7 mrg
1212 1.22 mrg DPRINTF(IDB_BUSDMA, ("iommu_dvmamem_free: segp %p nsegs %d\n",
1213 1.7 mrg segs, nsegs));
1214 1.7 mrg bus_dmamem_free(t->_parent, segs, nsegs);
1215 1.7 mrg }
1216 1.7 mrg
1217 1.7 mrg /*
1218 1.7 mrg * Map the DVMA mappings into the kernel pmap.
1219 1.7 mrg * Check the flags to see whether we're streaming or coherent.
1220 1.7 mrg */
1221 1.7 mrg int
1222 1.85 nakayama iommu_dvmamem_map(bus_dma_tag_t t, bus_dma_segment_t *segs, int nsegs,
1223 1.85 nakayama size_t size, void **kvap, int flags)
1224 1.7 mrg {
1225 1.58 chs struct vm_page *pg;
1226 1.7 mrg vaddr_t va;
1227 1.7 mrg bus_addr_t addr;
1228 1.58 chs struct pglist *pglist;
1229 1.8 mrg int cbit;
1230 1.77 yamt const uvm_flag_t kmflags =
1231 1.77 yamt (flags & BUS_DMA_NOWAIT) != 0 ? UVM_KMF_NOWAIT : 0;
1232 1.7 mrg
1233 1.22 mrg DPRINTF(IDB_BUSDMA, ("iommu_dvmamem_map: segp %p nsegs %d size %lx\n",
1234 1.7 mrg segs, nsegs, size));
1235 1.7 mrg
1236 1.7 mrg /*
1237 1.8 mrg * Allocate some space in the kernel map, and then map these pages
1238 1.8 mrg * into this space.
1239 1.7 mrg */
1240 1.8 mrg size = round_page(size);
1241 1.77 yamt va = uvm_km_alloc(kernel_map, size, 0, UVM_KMF_VAONLY | kmflags);
1242 1.8 mrg if (va == 0)
1243 1.8 mrg return (ENOMEM);
1244 1.7 mrg
1245 1.81 christos *kvap = (void *)va;
1246 1.7 mrg
1247 1.58 chs /*
1248 1.7 mrg * digest flags:
1249 1.7 mrg */
1250 1.7 mrg cbit = 0;
1251 1.7 mrg if (flags & BUS_DMA_COHERENT) /* Disable vcache */
1252 1.7 mrg cbit |= PMAP_NVC;
1253 1.97 skrll if (flags & BUS_DMA_NOCACHE) /* side effects */
1254 1.7 mrg cbit |= PMAP_NC;
1255 1.7 mrg
1256 1.7 mrg /*
1257 1.8 mrg * Now take this and map it into the CPU.
1258 1.7 mrg */
1259 1.58 chs pglist = segs[0]._ds_mlist;
1260 1.83 ad TAILQ_FOREACH(pg, pglist, pageq.queue) {
1261 1.8 mrg #ifdef DIAGNOSTIC
1262 1.7 mrg if (size == 0)
1263 1.7 mrg panic("iommu_dvmamem_map: size botch");
1264 1.8 mrg #endif
1265 1.58 chs addr = VM_PAGE_TO_PHYS(pg);
1266 1.22 mrg DPRINTF(IDB_BUSDMA, ("iommu_dvmamem_map: "
1267 1.25 mrg "mapping va %lx at %llx\n", va, (unsigned long long)addr | cbit));
1268 1.88 cegger pmap_kenter_pa(va, addr | cbit,
1269 1.88 cegger VM_PROT_READ | VM_PROT_WRITE, 0);
1270 1.7 mrg va += PAGE_SIZE;
1271 1.7 mrg size -= PAGE_SIZE;
1272 1.7 mrg }
1273 1.38 chris pmap_update(pmap_kernel());
1274 1.7 mrg return (0);
1275 1.7 mrg }
1276 1.7 mrg
1277 1.7 mrg /*
1278 1.7 mrg * Unmap DVMA mappings from kernel
1279 1.7 mrg */
1280 1.7 mrg void
1281 1.85 nakayama iommu_dvmamem_unmap(bus_dma_tag_t t, void *kva, size_t size)
1282 1.7 mrg {
1283 1.58 chs
1284 1.22 mrg DPRINTF(IDB_BUSDMA, ("iommu_dvmamem_unmap: kvm %p size %lx\n",
1285 1.7 mrg kva, size));
1286 1.58 chs
1287 1.7 mrg #ifdef DIAGNOSTIC
1288 1.7 mrg if ((u_long)kva & PGOFSET)
1289 1.7 mrg panic("iommu_dvmamem_unmap");
1290 1.7 mrg #endif
1291 1.58 chs
1292 1.7 mrg size = round_page(size);
1293 1.58 chs pmap_kremove((vaddr_t)kva, size);
1294 1.38 chris pmap_update(pmap_kernel());
1295 1.76 yamt uvm_km_free(kernel_map, (vaddr_t)kva, size, UVM_KMF_VAONLY);
1296 1.1 mrg }
1297