iommu.c revision 1.35.2.8 1 1.35.2.8 jdolecek /* $NetBSD: iommu.c,v 1.35.2.8 2002/10/10 18:36:32 jdolecek Exp $ */
2 1.7 mrg
3 1.7 mrg /*
4 1.35.2.5 jdolecek * Copyright (c) 2001, 2002 Eduardo Horvath
5 1.7 mrg * Copyright (c) 1999, 2000 Matthew R. Green
6 1.7 mrg * All rights reserved.
7 1.7 mrg *
8 1.7 mrg * Redistribution and use in source and binary forms, with or without
9 1.7 mrg * modification, are permitted provided that the following conditions
10 1.7 mrg * are met:
11 1.7 mrg * 1. Redistributions of source code must retain the above copyright
12 1.7 mrg * notice, this list of conditions and the following disclaimer.
13 1.7 mrg * 2. Redistributions in binary form must reproduce the above copyright
14 1.7 mrg * notice, this list of conditions and the following disclaimer in the
15 1.7 mrg * documentation and/or other materials provided with the distribution.
16 1.7 mrg * 3. The name of the author may not be used to endorse or promote products
17 1.7 mrg * derived from this software without specific prior written permission.
18 1.7 mrg *
19 1.7 mrg * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
20 1.7 mrg * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
21 1.7 mrg * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
22 1.7 mrg * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
23 1.7 mrg * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
24 1.7 mrg * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
25 1.7 mrg * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
26 1.7 mrg * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
27 1.7 mrg * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
28 1.7 mrg * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
29 1.7 mrg * SUCH DAMAGE.
30 1.7 mrg */
31 1.1 mrg
32 1.7 mrg /*
33 1.7 mrg * UltraSPARC IOMMU support; used by both the sbus and pci code.
34 1.7 mrg */
35 1.4 mrg #include "opt_ddb.h"
36 1.4 mrg
37 1.1 mrg #include <sys/param.h>
38 1.1 mrg #include <sys/extent.h>
39 1.1 mrg #include <sys/malloc.h>
40 1.1 mrg #include <sys/systm.h>
41 1.1 mrg #include <sys/device.h>
42 1.35.2.4 thorpej #include <sys/proc.h>
43 1.18 mrg
44 1.18 mrg #include <uvm/uvm_extern.h>
45 1.1 mrg
46 1.1 mrg #include <machine/bus.h>
47 1.7 mrg #include <sparc64/sparc64/cache.h>
48 1.1 mrg #include <sparc64/dev/iommureg.h>
49 1.1 mrg #include <sparc64/dev/iommuvar.h>
50 1.1 mrg
51 1.1 mrg #include <machine/autoconf.h>
52 1.1 mrg #include <machine/cpu.h>
53 1.1 mrg
54 1.1 mrg #ifdef DEBUG
55 1.22 mrg #define IDB_BUSDMA 0x1
56 1.22 mrg #define IDB_IOMMU 0x2
57 1.22 mrg #define IDB_INFO 0x4
58 1.35.2.1 lukem #define IDB_SYNC 0x8
59 1.10 mrg int iommudebug = 0x0;
60 1.4 mrg #define DPRINTF(l, s) do { if (iommudebug & l) printf s; } while (0)
61 1.4 mrg #else
62 1.4 mrg #define DPRINTF(l, s)
63 1.1 mrg #endif
64 1.1 mrg
65 1.35.2.7 jdolecek #define iommu_strbuf_flush(i, v) do { \
66 1.35.2.7 jdolecek if ((i)->sb_flush) \
67 1.35.2.7 jdolecek bus_space_write_8((i)->sb_is->is_bustag, (i)->sb_sb, \
68 1.35.2.6 jdolecek STRBUFREG(strbuf_pgflush), (v)); \
69 1.35.2.4 thorpej } while (0)
70 1.35.2.4 thorpej
71 1.35.2.7 jdolecek static int iommu_strbuf_flush_done __P((struct strbuf_ctl *));
72 1.11 eeh
73 1.1 mrg /*
74 1.1 mrg * initialise the UltraSPARC IOMMU (SBUS or PCI):
75 1.1 mrg * - allocate and setup the iotsb.
76 1.1 mrg * - enable the IOMMU
77 1.7 mrg * - initialise the streaming buffers (if they exist)
78 1.1 mrg * - create a private DVMA map.
79 1.1 mrg */
80 1.1 mrg void
81 1.35.2.1 lukem iommu_init(name, is, tsbsize, iovabase)
82 1.1 mrg char *name;
83 1.1 mrg struct iommu_state *is;
84 1.1 mrg int tsbsize;
85 1.35.2.1 lukem u_int32_t iovabase;
86 1.1 mrg {
87 1.11 eeh psize_t size;
88 1.11 eeh vaddr_t va;
89 1.11 eeh paddr_t pa;
90 1.35.2.8 jdolecek struct vm_page *pg;
91 1.35.2.8 jdolecek struct pglist pglist;
92 1.1 mrg
93 1.1 mrg /*
94 1.1 mrg * Setup the iommu.
95 1.1 mrg *
96 1.35.2.5 jdolecek * The sun4u iommu is part of the SBUS or PCI controller so we will
97 1.35.2.5 jdolecek * deal with it here..
98 1.1 mrg *
99 1.35.2.5 jdolecek * For sysio and psycho/psycho+ the IOMMU address space always ends at
100 1.35.2.5 jdolecek * 0xffffe000, but the starting address depends on the size of the
101 1.35.2.5 jdolecek * map. The map size is 1024 * 2 ^ is->is_tsbsize entries, where each
102 1.35.2.5 jdolecek * entry is 8 bytes. The start of the map can be calculated by
103 1.35.2.5 jdolecek * (0xffffe000 << (8 + is->is_tsbsize)).
104 1.35.2.5 jdolecek *
105 1.35.2.5 jdolecek * But sabre and hummingbird use a different scheme that seems to
106 1.35.2.5 jdolecek * be hard-wired, so we read the start and size from the PROM and
107 1.35.2.5 jdolecek * just use those values.
108 1.2 eeh */
109 1.11 eeh is->is_cr = (tsbsize << 16) | IOMMUCR_EN;
110 1.11 eeh is->is_tsbsize = tsbsize;
111 1.35.2.5 jdolecek if (iovabase == -1) {
112 1.35.2.5 jdolecek is->is_dvmabase = IOTSB_VSTART(is->is_tsbsize);
113 1.35.2.5 jdolecek is->is_dvmaend = IOTSB_VEND;
114 1.35.2.5 jdolecek } else {
115 1.35.2.5 jdolecek is->is_dvmabase = iovabase;
116 1.35.2.5 jdolecek is->is_dvmaend = iovabase + IOTSB_VSIZE(tsbsize);
117 1.35.2.5 jdolecek }
118 1.11 eeh
119 1.11 eeh /*
120 1.15 eeh * Allocate memory for I/O pagetables. They need to be physically
121 1.15 eeh * contiguous.
122 1.11 eeh */
123 1.11 eeh
124 1.35.2.8 jdolecek size = NBPG << is->is_tsbsize;
125 1.11 eeh if (uvm_pglistalloc((psize_t)size, (paddr_t)0, (paddr_t)-1,
126 1.35.2.8 jdolecek (paddr_t)NBPG, (paddr_t)0, &pglist, 1, 0) != 0)
127 1.11 eeh panic("iommu_init: no memory");
128 1.11 eeh
129 1.11 eeh va = uvm_km_valloc(kernel_map, size);
130 1.11 eeh if (va == 0)
131 1.11 eeh panic("iommu_init: no memory");
132 1.11 eeh is->is_tsb = (int64_t *)va;
133 1.11 eeh
134 1.35.2.8 jdolecek is->is_ptsb = VM_PAGE_TO_PHYS(TAILQ_FIRST(&pglist));
135 1.11 eeh
136 1.11 eeh /* Map the pages */
137 1.35.2.8 jdolecek TAILQ_FOREACH(pg, &pglist, pageq) {
138 1.35.2.8 jdolecek pa = VM_PAGE_TO_PHYS(pg);
139 1.35.2.8 jdolecek pmap_kenter_pa(va, pa | PMAP_NVC, VM_PROT_READ | VM_PROT_WRITE);
140 1.11 eeh va += NBPG;
141 1.11 eeh }
142 1.35.2.3 thorpej pmap_update(pmap_kernel());
143 1.35.2.8 jdolecek memset(is->is_tsb, 0, size);
144 1.1 mrg
145 1.1 mrg #ifdef DEBUG
146 1.22 mrg if (iommudebug & IDB_INFO)
147 1.1 mrg {
148 1.1 mrg /* Probe the iommu */
149 1.1 mrg
150 1.25 mrg printf("iommu regs at: cr=%lx tsb=%lx flush=%lx\n",
151 1.35.2.6 jdolecek (u_long)bus_space_read_8(is->is_bustag, is->is_iommu,
152 1.35.2.6 jdolecek offsetof (struct iommureg, iommu_cr)),
153 1.35.2.6 jdolecek (u_long)bus_space_read_8(is->is_bustag, is->is_iommu,
154 1.35.2.6 jdolecek offsetof (struct iommureg, iommu_tsb)),
155 1.35.2.6 jdolecek (u_long)bus_space_read_8(is->is_bustag, is->is_iommu,
156 1.35.2.6 jdolecek offsetof (struct iommureg, iommu_flush)));
157 1.35.2.6 jdolecek printf("iommu cr=%llx tsb=%llx\n",
158 1.35.2.6 jdolecek (unsigned long long)bus_space_read_8(is->is_bustag,
159 1.35.2.6 jdolecek is->is_iommu,
160 1.35.2.6 jdolecek offsetof (struct iommureg, iommu_cr)),
161 1.35.2.6 jdolecek (unsigned long long)bus_space_read_8(is->is_bustag,
162 1.35.2.6 jdolecek is->is_iommu,
163 1.35.2.6 jdolecek offsetof (struct iommureg, iommu_tsb)));
164 1.35.2.8 jdolecek printf("TSB base %p phys %llx\n", (void *)is->is_tsb,
165 1.35.2.6 jdolecek (unsigned long long)is->is_ptsb);
166 1.1 mrg delay(1000000); /* 1 s */
167 1.1 mrg }
168 1.1 mrg #endif
169 1.1 mrg
170 1.1 mrg /*
171 1.1 mrg * now actually start up the IOMMU
172 1.1 mrg */
173 1.1 mrg iommu_reset(is);
174 1.1 mrg
175 1.1 mrg /*
176 1.1 mrg * Now all the hardware's working we need to allocate a dvma map.
177 1.1 mrg */
178 1.35.2.8 jdolecek printf("DVMA map: %x to %x\n",
179 1.11 eeh (unsigned int)is->is_dvmabase,
180 1.35.2.5 jdolecek (unsigned int)is->is_dvmaend);
181 1.35.2.8 jdolecek printf("IOTSB: %llx to %llx\n",
182 1.35.2.5 jdolecek (unsigned long long)is->is_ptsb,
183 1.35.2.5 jdolecek (unsigned long long)(is->is_ptsb + size));
184 1.1 mrg is->is_dvmamap = extent_create(name,
185 1.35.2.5 jdolecek is->is_dvmabase, is->is_dvmaend - NBPG,
186 1.1 mrg M_DEVBUF, 0, 0, EX_NOWAIT);
187 1.1 mrg }
188 1.1 mrg
189 1.8 mrg /*
190 1.8 mrg * Streaming buffers don't exist on the UltraSPARC IIi; we should have
191 1.8 mrg * detected that already and disabled them. If not, we will notice that
192 1.8 mrg * they aren't there when the STRBUF_EN bit does not remain.
193 1.8 mrg */
194 1.1 mrg void
195 1.1 mrg iommu_reset(is)
196 1.1 mrg struct iommu_state *is;
197 1.1 mrg {
198 1.35.2.5 jdolecek int i;
199 1.35.2.7 jdolecek struct strbuf_ctl *sb;
200 1.1 mrg
201 1.1 mrg /* Need to do 64-bit stores */
202 1.35.2.8 jdolecek bus_space_write_8(is->is_bustag, is->is_iommu, IOMMUREG(iommu_tsb),
203 1.35.2.6 jdolecek is->is_ptsb);
204 1.35.2.6 jdolecek
205 1.11 eeh /* Enable IOMMU in diagnostic mode */
206 1.35.2.6 jdolecek bus_space_write_8(is->is_bustag, is->is_iommu, IOMMUREG(iommu_cr),
207 1.35.2.6 jdolecek is->is_cr|IOMMUCR_DE);
208 1.5 mrg
209 1.35.2.8 jdolecek for (i = 0; i < 2; i++) {
210 1.35.2.7 jdolecek if ((sb = is->is_sb[i])) {
211 1.35.2.4 thorpej
212 1.35.2.5 jdolecek /* Enable diagnostics mode? */
213 1.35.2.8 jdolecek bus_space_write_8(is->is_bustag, is->is_sb[i]->sb_sb,
214 1.35.2.6 jdolecek STRBUFREG(strbuf_ctl), STRBUF_EN);
215 1.35.2.5 jdolecek
216 1.35.2.5 jdolecek /* No streaming buffers? Disable them */
217 1.35.2.8 jdolecek if (bus_space_read_8(is->is_bustag,
218 1.35.2.8 jdolecek is->is_sb[i]->sb_sb,
219 1.35.2.7 jdolecek STRBUFREG(strbuf_ctl)) == 0) {
220 1.35.2.7 jdolecek is->is_sb[i]->sb_flush = NULL;
221 1.35.2.7 jdolecek } else {
222 1.35.2.8 jdolecek
223 1.35.2.7 jdolecek /*
224 1.35.2.7 jdolecek * locate the pa of the flush buffer.
225 1.35.2.7 jdolecek */
226 1.35.2.7 jdolecek (void)pmap_extract(pmap_kernel(),
227 1.35.2.7 jdolecek (vaddr_t)is->is_sb[i]->sb_flush,
228 1.35.2.7 jdolecek &is->is_sb[i]->sb_flushpa);
229 1.35.2.7 jdolecek }
230 1.35.2.5 jdolecek }
231 1.35.2.4 thorpej }
232 1.2 eeh }
233 1.2 eeh
234 1.2 eeh /*
235 1.35.2.8 jdolecek * Here are the iommu control routines.
236 1.2 eeh */
237 1.2 eeh void
238 1.35.2.7 jdolecek iommu_enter(sb, va, pa, flags)
239 1.35.2.7 jdolecek struct strbuf_ctl *sb;
240 1.2 eeh vaddr_t va;
241 1.2 eeh int64_t pa;
242 1.2 eeh int flags;
243 1.2 eeh {
244 1.35.2.7 jdolecek struct iommu_state *is = sb->sb_is;
245 1.35.2.7 jdolecek int strbuf = (flags & BUS_DMA_STREAMING);
246 1.2 eeh int64_t tte;
247 1.2 eeh
248 1.2 eeh #ifdef DIAGNOSTIC
249 1.35.2.5 jdolecek if (va < is->is_dvmabase || va > is->is_dvmaend)
250 1.13 mrg panic("iommu_enter: va %#lx not in DVMA space", va);
251 1.2 eeh #endif
252 1.2 eeh
253 1.35.2.7 jdolecek /* Is the streamcache flush really needed? */
254 1.35.2.7 jdolecek if (sb->sb_flush) {
255 1.35.2.7 jdolecek iommu_strbuf_flush(sb, va);
256 1.35.2.7 jdolecek iommu_strbuf_flush_done(sb);
257 1.35.2.7 jdolecek } else
258 1.35.2.7 jdolecek /* If we can't flush the strbuf don't enable it. */
259 1.35.2.7 jdolecek strbuf = 0;
260 1.35.2.7 jdolecek
261 1.35.2.8 jdolecek tte = MAKEIOTTE(pa, !(flags & BUS_DMA_NOWRITE),
262 1.35.2.7 jdolecek !(flags & BUS_DMA_NOCACHE), (strbuf));
263 1.35.2.6 jdolecek #ifdef DEBUG
264 1.35.2.6 jdolecek tte |= (flags & 0xff000LL)<<(4*8);
265 1.35.2.6 jdolecek #endif
266 1.35.2.8 jdolecek
267 1.35.2.8 jdolecek DPRINTF(IDB_IOMMU, ("Clearing TSB slot %d for va %p\n",
268 1.25 mrg (int)IOTSBSLOT(va,is->is_tsbsize), (void *)(u_long)va));
269 1.2 eeh is->is_tsb[IOTSBSLOT(va,is->is_tsbsize)] = tte;
270 1.35.2.8 jdolecek bus_space_write_8(is->is_bustag, is->is_iommu,
271 1.35.2.6 jdolecek IOMMUREG(iommu_flush), va);
272 1.22 mrg DPRINTF(IDB_IOMMU, ("iommu_enter: va %lx pa %lx TSB[%lx]@%p=%lx\n",
273 1.35.2.6 jdolecek va, (long)pa, (u_long)IOTSBSLOT(va,is->is_tsbsize),
274 1.35.2.6 jdolecek (void *)(u_long)&is->is_tsb[IOTSBSLOT(va,is->is_tsbsize)],
275 1.35.2.6 jdolecek (u_long)tte));
276 1.2 eeh }
277 1.2 eeh
278 1.35.2.4 thorpej /*
279 1.35.2.4 thorpej * Find the value of a DVMA address (debug routine).
280 1.35.2.4 thorpej */
281 1.35.2.4 thorpej paddr_t
282 1.35.2.4 thorpej iommu_extract(is, dva)
283 1.35.2.4 thorpej struct iommu_state *is;
284 1.35.2.4 thorpej vaddr_t dva;
285 1.35.2.4 thorpej {
286 1.35.2.4 thorpej int64_t tte = 0;
287 1.35.2.8 jdolecek
288 1.35.2.5 jdolecek if (dva >= is->is_dvmabase && dva < is->is_dvmaend)
289 1.35.2.7 jdolecek tte = is->is_tsb[IOTSBSLOT(dva, is->is_tsbsize)];
290 1.35.2.4 thorpej
291 1.35.2.6 jdolecek if ((tte & IOTTE_V) == 0)
292 1.35.2.4 thorpej return ((paddr_t)-1L);
293 1.35.2.6 jdolecek return (tte & IOTTE_PAMASK);
294 1.35.2.4 thorpej }
295 1.35.2.4 thorpej
296 1.2 eeh /*
297 1.2 eeh * iommu_remove: removes mappings created by iommu_enter
298 1.2 eeh *
299 1.2 eeh * Only demap from IOMMU if flag is set.
300 1.8 mrg *
301 1.8 mrg * XXX: this function needs better internal error checking.
302 1.2 eeh */
303 1.2 eeh void
304 1.2 eeh iommu_remove(is, va, len)
305 1.2 eeh struct iommu_state *is;
306 1.2 eeh vaddr_t va;
307 1.2 eeh size_t len;
308 1.2 eeh {
309 1.2 eeh
310 1.2 eeh #ifdef DIAGNOSTIC
311 1.35.2.5 jdolecek if (va < is->is_dvmabase || va > is->is_dvmaend)
312 1.25 mrg panic("iommu_remove: va 0x%lx not in DVMA space", (u_long)va);
313 1.2 eeh if ((long)(va + len) < (long)va)
314 1.35.2.8 jdolecek panic("iommu_remove: va 0x%lx + len 0x%lx wraps",
315 1.2 eeh (long) va, (long) len);
316 1.35.2.8 jdolecek if (len & ~0xfffffff)
317 1.25 mrg panic("iommu_remove: rediculous len 0x%lx", (u_long)len);
318 1.2 eeh #endif
319 1.2 eeh
320 1.2 eeh va = trunc_page(va);
321 1.22 mrg DPRINTF(IDB_IOMMU, ("iommu_remove: va %lx TSB[%lx]@%p\n",
322 1.35.2.6 jdolecek va, (u_long)IOTSBSLOT(va, is->is_tsbsize),
323 1.35.2.6 jdolecek &is->is_tsb[IOTSBSLOT(va, is->is_tsbsize)]));
324 1.2 eeh while (len > 0) {
325 1.35.2.6 jdolecek DPRINTF(IDB_IOMMU, ("iommu_remove: clearing TSB slot %d "
326 1.35.2.6 jdolecek "for va %p size %lx\n",
327 1.35.2.6 jdolecek (int)IOTSBSLOT(va,is->is_tsbsize), (void *)(u_long)va,
328 1.35.2.6 jdolecek (u_long)len));
329 1.10 mrg if (len <= NBPG)
330 1.10 mrg len = 0;
331 1.10 mrg else
332 1.8 mrg len -= NBPG;
333 1.8 mrg
334 1.35.2.5 jdolecek /* XXX Zero-ing the entry would not require RMW */
335 1.35.2.5 jdolecek is->is_tsb[IOTSBSLOT(va,is->is_tsbsize)] &= ~IOTTE_V;
336 1.35.2.8 jdolecek bus_space_write_8(is->is_bustag, is->is_iommu,
337 1.35.2.6 jdolecek IOMMUREG(iommu_flush), va);
338 1.2 eeh va += NBPG;
339 1.2 eeh }
340 1.2 eeh }
341 1.2 eeh
342 1.35.2.8 jdolecek static int
343 1.35.2.7 jdolecek iommu_strbuf_flush_done(sb)
344 1.35.2.7 jdolecek struct strbuf_ctl *sb;
345 1.2 eeh {
346 1.35.2.7 jdolecek struct iommu_state *is = sb->sb_is;
347 1.2 eeh struct timeval cur, flushtimeout;
348 1.2 eeh
349 1.2 eeh #define BUMPTIME(t, usec) { \
350 1.2 eeh register volatile struct timeval *tp = (t); \
351 1.2 eeh register long us; \
352 1.2 eeh \
353 1.2 eeh tp->tv_usec = us = tp->tv_usec + (usec); \
354 1.2 eeh if (us >= 1000000) { \
355 1.2 eeh tp->tv_usec = us - 1000000; \
356 1.2 eeh tp->tv_sec++; \
357 1.2 eeh } \
358 1.2 eeh }
359 1.5 mrg
360 1.35.2.7 jdolecek if (!sb->sb_flush)
361 1.5 mrg return (0);
362 1.35.2.8 jdolecek
363 1.7 mrg /*
364 1.7 mrg * Streaming buffer flushes:
365 1.35.2.8 jdolecek *
366 1.7 mrg * 1 Tell strbuf to flush by storing va to strbuf_pgflush. If
367 1.7 mrg * we're not on a cache line boundary (64-bits):
368 1.7 mrg * 2 Store 0 in flag
369 1.7 mrg * 3 Store pointer to flag in flushsync
370 1.7 mrg * 4 wait till flushsync becomes 0x1
371 1.7 mrg *
372 1.7 mrg * If it takes more than .5 sec, something
373 1.7 mrg * went wrong.
374 1.7 mrg */
375 1.2 eeh
376 1.35.2.7 jdolecek *sb->sb_flush = 0;
377 1.35.2.8 jdolecek bus_space_write_8(is->is_bustag, sb->sb_sb,
378 1.35.2.7 jdolecek STRBUFREG(strbuf_flushsync), sb->sb_flushpa);
379 1.2 eeh
380 1.35.2.8 jdolecek microtime(&flushtimeout);
381 1.2 eeh cur = flushtimeout;
382 1.2 eeh BUMPTIME(&flushtimeout, 500000); /* 1/2 sec */
383 1.35.2.8 jdolecek
384 1.35.2.7 jdolecek DPRINTF(IDB_IOMMU, ("iommu_strbuf_flush_done: flush = %lx "
385 1.35.2.4 thorpej "at va = %lx pa = %lx now=%lx:%lx until = %lx:%lx\n",
386 1.35.2.8 jdolecek (long)*sb->sb_flush, (long)sb->sb_flush, (long)sb->sb_flushpa,
387 1.35.2.4 thorpej cur.tv_sec, cur.tv_usec,
388 1.35.2.4 thorpej flushtimeout.tv_sec, flushtimeout.tv_usec));
389 1.35.2.4 thorpej
390 1.2 eeh /* Bypass non-coherent D$ */
391 1.35.2.7 jdolecek while ((!ldxa(sb->sb_flushpa, ASI_PHYS_CACHED)) &&
392 1.35.2.8 jdolecek timercmp(&cur, &flushtimeout, <=))
393 1.2 eeh microtime(&cur);
394 1.2 eeh
395 1.2 eeh #ifdef DIAGNOSTIC
396 1.35.2.7 jdolecek if (!ldxa(sb->sb_flushpa, ASI_PHYS_CACHED)) {
397 1.35.2.7 jdolecek printf("iommu_strbuf_flush_done: flush timeout %p, at %p\n",
398 1.35.2.7 jdolecek (void *)(u_long)*sb->sb_flush,
399 1.35.2.7 jdolecek (void *)(u_long)sb->sb_flushpa); /* panic? */
400 1.2 eeh #ifdef DDB
401 1.2 eeh Debugger();
402 1.2 eeh #endif
403 1.2 eeh }
404 1.2 eeh #endif
405 1.31 eeh DPRINTF(IDB_IOMMU, ("iommu_strbuf_flush_done: flushed\n"));
406 1.35.2.7 jdolecek return (*sb->sb_flush);
407 1.7 mrg }
408 1.7 mrg
409 1.7 mrg /*
410 1.7 mrg * IOMMU DVMA operations, common to SBUS and PCI.
411 1.7 mrg */
412 1.7 mrg int
413 1.35.2.7 jdolecek iommu_dvmamap_load(t, sb, map, buf, buflen, p, flags)
414 1.7 mrg bus_dma_tag_t t;
415 1.35.2.7 jdolecek struct strbuf_ctl *sb;
416 1.7 mrg bus_dmamap_t map;
417 1.7 mrg void *buf;
418 1.7 mrg bus_size_t buflen;
419 1.7 mrg struct proc *p;
420 1.7 mrg int flags;
421 1.7 mrg {
422 1.35.2.7 jdolecek struct iommu_state *is = sb->sb_is;
423 1.7 mrg int s;
424 1.7 mrg int err;
425 1.7 mrg bus_size_t sgsize;
426 1.7 mrg paddr_t curaddr;
427 1.35.2.4 thorpej u_long dvmaddr, sgstart, sgend;
428 1.21 eeh bus_size_t align, boundary;
429 1.7 mrg vaddr_t vaddr = (vaddr_t)buf;
430 1.35.2.4 thorpej int seg;
431 1.35.2.8 jdolecek struct pmap *pmap;
432 1.7 mrg
433 1.7 mrg if (map->dm_nsegs) {
434 1.7 mrg /* Already in use?? */
435 1.7 mrg #ifdef DIAGNOSTIC
436 1.7 mrg printf("iommu_dvmamap_load: map still in use\n");
437 1.7 mrg #endif
438 1.7 mrg bus_dmamap_unload(t, map);
439 1.7 mrg }
440 1.35.2.8 jdolecek
441 1.7 mrg /*
442 1.7 mrg * Make sure that on error condition we return "no valid mappings".
443 1.7 mrg */
444 1.7 mrg map->dm_nsegs = 0;
445 1.7 mrg if (buflen > map->_dm_size) {
446 1.22 mrg DPRINTF(IDB_BUSDMA,
447 1.7 mrg ("iommu_dvmamap_load(): error %d > %d -- "
448 1.25 mrg "map size exceeded!\n", (int)buflen, (int)map->_dm_size));
449 1.7 mrg return (EINVAL);
450 1.7 mrg }
451 1.7 mrg
452 1.7 mrg sgsize = round_page(buflen + ((int)vaddr & PGOFSET));
453 1.20 mrg
454 1.7 mrg /*
455 1.21 eeh * A boundary presented to bus_dmamem_alloc() takes precedence
456 1.21 eeh * over boundary in the map.
457 1.7 mrg */
458 1.21 eeh if ((boundary = (map->dm_segs[0]._ds_boundary)) == 0)
459 1.21 eeh boundary = map->_dm_boundary;
460 1.21 eeh align = max(map->dm_segs[0]._ds_align, NBPG);
461 1.35.2.8 jdolecek
462 1.35.2.8 jdolecek /*
463 1.35.2.8 jdolecek * If our segment size is larger than the boundary we need to
464 1.35.2.4 thorpej * split the transfer up int little pieces ourselves.
465 1.35.2.4 thorpej */
466 1.35.2.8 jdolecek s = splhigh();
467 1.35.2.8 jdolecek err = extent_alloc(is->is_dvmamap, sgsize, align,
468 1.35.2.8 jdolecek (sgsize > boundary) ? 0 : boundary,
469 1.35.2.6 jdolecek EX_NOWAIT|EX_BOUNDZERO, &dvmaddr);
470 1.7 mrg splx(s);
471 1.7 mrg
472 1.7 mrg #ifdef DEBUG
473 1.35.2.8 jdolecek if (err || (dvmaddr == (bus_addr_t)-1))
474 1.35.2.8 jdolecek {
475 1.7 mrg printf("iommu_dvmamap_load(): extent_alloc(%d, %x) failed!\n",
476 1.25 mrg (int)sgsize, flags);
477 1.35.2.4 thorpej #ifdef DDB
478 1.7 mrg Debugger();
479 1.35.2.4 thorpej #endif
480 1.35.2.8 jdolecek }
481 1.35.2.8 jdolecek #endif
482 1.11 eeh if (err != 0)
483 1.11 eeh return (err);
484 1.11 eeh
485 1.7 mrg if (dvmaddr == (bus_addr_t)-1)
486 1.7 mrg return (ENOMEM);
487 1.7 mrg
488 1.35.2.4 thorpej /* Set the active DVMA map */
489 1.35.2.4 thorpej map->_dm_dvmastart = dvmaddr;
490 1.35.2.4 thorpej map->_dm_dvmasize = sgsize;
491 1.35.2.4 thorpej
492 1.35.2.4 thorpej /*
493 1.35.2.4 thorpej * Now split the DVMA range into segments, not crossing
494 1.35.2.4 thorpej * the boundary.
495 1.35.2.4 thorpej */
496 1.35.2.4 thorpej seg = 0;
497 1.35.2.4 thorpej sgstart = dvmaddr + (vaddr & PGOFSET);
498 1.35.2.4 thorpej sgend = sgstart + buflen - 1;
499 1.35.2.4 thorpej map->dm_segs[seg].ds_addr = sgstart;
500 1.35.2.4 thorpej DPRINTF(IDB_INFO, ("iommu_dvmamap_load: boundary %lx boundary-1 %lx "
501 1.35.2.4 thorpej "~(boundary-1) %lx\n", boundary, (boundary-1), ~(boundary-1)));
502 1.35.2.4 thorpej while ((sgstart & ~(boundary - 1)) != (sgend & ~(boundary - 1))) {
503 1.35.2.4 thorpej /* Oops. We crossed a boundary. Split the xfer. */
504 1.35.2.4 thorpej DPRINTF(IDB_INFO, ("iommu_dvmamap_load: "
505 1.35.2.4 thorpej "seg %d start %lx size %lx\n", seg,
506 1.35.2.8 jdolecek (long)map->dm_segs[seg].ds_addr,
507 1.35.2.5 jdolecek map->dm_segs[seg].ds_len));
508 1.35.2.5 jdolecek map->dm_segs[seg].ds_len =
509 1.35.2.5 jdolecek boundary - (sgstart & (boundary - 1));
510 1.35.2.6 jdolecek if (++seg >= map->_dm_segcnt) {
511 1.35.2.4 thorpej /* Too many segments. Fail the operation. */
512 1.35.2.4 thorpej DPRINTF(IDB_INFO, ("iommu_dvmamap_load: "
513 1.35.2.4 thorpej "too many segments %d\n", seg));
514 1.35.2.4 thorpej s = splhigh();
515 1.35.2.4 thorpej /* How can this fail? And if it does what can we do? */
516 1.35.2.4 thorpej err = extent_free(is->is_dvmamap,
517 1.35.2.4 thorpej dvmaddr, sgsize, EX_NOWAIT);
518 1.35.2.4 thorpej map->_dm_dvmastart = 0;
519 1.35.2.4 thorpej map->_dm_dvmasize = 0;
520 1.35.2.4 thorpej splx(s);
521 1.35.2.4 thorpej return (E2BIG);
522 1.35.2.4 thorpej }
523 1.35.2.4 thorpej sgstart = roundup(sgstart, boundary);
524 1.35.2.4 thorpej map->dm_segs[seg].ds_addr = sgstart;
525 1.35.2.4 thorpej }
526 1.35.2.4 thorpej map->dm_segs[seg].ds_len = sgend - sgstart + 1;
527 1.35.2.4 thorpej DPRINTF(IDB_INFO, ("iommu_dvmamap_load: "
528 1.35.2.4 thorpej "seg %d start %lx size %lx\n", seg,
529 1.35.2.5 jdolecek (long)map->dm_segs[seg].ds_addr, map->dm_segs[seg].ds_len));
530 1.35.2.4 thorpej map->dm_nsegs = seg+1;
531 1.7 mrg map->dm_mapsize = buflen;
532 1.7 mrg
533 1.7 mrg if (p != NULL)
534 1.7 mrg pmap = p->p_vmspace->vm_map.pmap;
535 1.7 mrg else
536 1.7 mrg pmap = pmap_kernel();
537 1.7 mrg
538 1.7 mrg for (; buflen > 0; ) {
539 1.35.2.8 jdolecek
540 1.7 mrg /*
541 1.7 mrg * Get the physical address for this page.
542 1.7 mrg */
543 1.7 mrg if (pmap_extract(pmap, (vaddr_t)vaddr, &curaddr) == FALSE) {
544 1.7 mrg bus_dmamap_unload(t, map);
545 1.7 mrg return (-1);
546 1.7 mrg }
547 1.7 mrg
548 1.7 mrg /*
549 1.7 mrg * Compute the segment size, and adjust counts.
550 1.7 mrg */
551 1.7 mrg sgsize = NBPG - ((u_long)vaddr & PGOFSET);
552 1.7 mrg if (buflen < sgsize)
553 1.7 mrg sgsize = buflen;
554 1.7 mrg
555 1.22 mrg DPRINTF(IDB_BUSDMA,
556 1.35.2.1 lukem ("iommu_dvmamap_load: map %p loading va %p "
557 1.35.2.1 lukem "dva %lx at pa %lx\n",
558 1.35.2.1 lukem map, (void *)vaddr, (long)dvmaddr,
559 1.35.2.6 jdolecek (long)(curaddr & ~(NBPG-1))));
560 1.35.2.7 jdolecek iommu_enter(sb, trunc_page(dvmaddr), trunc_page(curaddr),
561 1.35.2.5 jdolecek flags|0x4000);
562 1.35.2.8 jdolecek
563 1.7 mrg dvmaddr += PAGE_SIZE;
564 1.7 mrg vaddr += sgsize;
565 1.7 mrg buflen -= sgsize;
566 1.7 mrg }
567 1.35.2.5 jdolecek #ifdef DIAGNOSTIC
568 1.35.2.5 jdolecek for (seg = 0; seg < map->dm_nsegs; seg++) {
569 1.35.2.5 jdolecek if (map->dm_segs[seg].ds_addr < is->is_dvmabase ||
570 1.35.2.5 jdolecek map->dm_segs[seg].ds_addr > is->is_dvmaend) {
571 1.35.2.5 jdolecek printf("seg %d dvmaddr %lx out of range %x - %x\n",
572 1.35.2.8 jdolecek seg, (long)map->dm_segs[seg].ds_addr,
573 1.35.2.5 jdolecek is->is_dvmabase, is->is_dvmaend);
574 1.35.2.7 jdolecek #ifdef DDB
575 1.35.2.5 jdolecek Debugger();
576 1.35.2.7 jdolecek #endif
577 1.35.2.5 jdolecek }
578 1.35.2.5 jdolecek }
579 1.35.2.5 jdolecek #endif
580 1.7 mrg return (0);
581 1.7 mrg }
582 1.7 mrg
583 1.7 mrg
584 1.7 mrg void
585 1.35.2.7 jdolecek iommu_dvmamap_unload(t, sb, map)
586 1.7 mrg bus_dma_tag_t t;
587 1.35.2.7 jdolecek struct strbuf_ctl *sb;
588 1.7 mrg bus_dmamap_t map;
589 1.7 mrg {
590 1.35.2.7 jdolecek struct iommu_state *is = sb->sb_is;
591 1.35.2.4 thorpej int error, s;
592 1.7 mrg bus_size_t sgsize;
593 1.7 mrg
594 1.35.2.4 thorpej /* Flush the iommu */
595 1.35.2.4 thorpej #ifdef DEBUG
596 1.35.2.4 thorpej if (!map->_dm_dvmastart) {
597 1.35.2.4 thorpej printf("iommu_dvmamap_unload: No dvmastart is zero\n");
598 1.35.2.4 thorpej #ifdef DDB
599 1.35.2.4 thorpej Debugger();
600 1.35.2.4 thorpej #endif
601 1.26 martin }
602 1.35.2.4 thorpej #endif
603 1.35.2.4 thorpej iommu_remove(is, map->_dm_dvmastart, map->_dm_dvmasize);
604 1.35.2.4 thorpej
605 1.23 eeh /* Flush the caches */
606 1.23 eeh bus_dmamap_unload(t->_parent, map);
607 1.23 eeh
608 1.7 mrg /* Mark the mappings as invalid. */
609 1.7 mrg map->dm_mapsize = 0;
610 1.7 mrg map->dm_nsegs = 0;
611 1.35.2.8 jdolecek
612 1.7 mrg s = splhigh();
613 1.35.2.8 jdolecek error = extent_free(is->is_dvmamap, map->_dm_dvmastart,
614 1.35.2.4 thorpej map->_dm_dvmasize, EX_NOWAIT);
615 1.35.2.4 thorpej map->_dm_dvmastart = 0;
616 1.35.2.4 thorpej map->_dm_dvmasize = 0;
617 1.7 mrg splx(s);
618 1.7 mrg if (error != 0)
619 1.7 mrg printf("warning: %qd of DVMA space lost\n", (long long)sgsize);
620 1.35.2.4 thorpej
621 1.35.2.4 thorpej /* Clear the map */
622 1.9 eeh }
623 1.9 eeh
624 1.9 eeh
625 1.9 eeh int
626 1.35.2.7 jdolecek iommu_dvmamap_load_raw(t, sb, map, segs, nsegs, flags, size)
627 1.9 eeh bus_dma_tag_t t;
628 1.35.2.7 jdolecek struct strbuf_ctl *sb;
629 1.9 eeh bus_dmamap_t map;
630 1.9 eeh bus_dma_segment_t *segs;
631 1.9 eeh int nsegs;
632 1.22 mrg int flags;
633 1.9 eeh bus_size_t size;
634 1.9 eeh {
635 1.35.2.7 jdolecek struct iommu_state *is = sb->sb_is;
636 1.35.2.8 jdolecek struct vm_page *pg;
637 1.35.2.4 thorpej int i, j, s;
638 1.26 martin int left;
639 1.9 eeh int err;
640 1.9 eeh bus_size_t sgsize;
641 1.9 eeh paddr_t pa;
642 1.21 eeh bus_size_t boundary, align;
643 1.35.2.4 thorpej u_long dvmaddr, sgstart, sgend;
644 1.35.2.8 jdolecek struct pglist *pglist;
645 1.9 eeh int pagesz = PAGE_SIZE;
646 1.35.2.5 jdolecek int npg = 0; /* DEBUG */
647 1.9 eeh
648 1.9 eeh if (map->dm_nsegs) {
649 1.9 eeh /* Already in use?? */
650 1.9 eeh #ifdef DIAGNOSTIC
651 1.9 eeh printf("iommu_dvmamap_load_raw: map still in use\n");
652 1.9 eeh #endif
653 1.9 eeh bus_dmamap_unload(t, map);
654 1.9 eeh }
655 1.35.2.4 thorpej
656 1.35.2.4 thorpej /*
657 1.35.2.4 thorpej * A boundary presented to bus_dmamem_alloc() takes precedence
658 1.35.2.4 thorpej * over boundary in the map.
659 1.35.2.4 thorpej */
660 1.35.2.4 thorpej if ((boundary = segs[0]._ds_boundary) == 0)
661 1.35.2.4 thorpej boundary = map->_dm_boundary;
662 1.35.2.4 thorpej
663 1.35.2.5 jdolecek align = max(segs[0]._ds_align, pagesz);
664 1.35.2.4 thorpej
665 1.9 eeh /*
666 1.9 eeh * Make sure that on error condition we return "no valid mappings".
667 1.9 eeh */
668 1.9 eeh map->dm_nsegs = 0;
669 1.26 martin /* Count up the total number of pages we need */
670 1.26 martin pa = segs[0].ds_addr;
671 1.26 martin sgsize = 0;
672 1.35.2.4 thorpej left = size;
673 1.35.2.8 jdolecek for (i = 0; left && i < nsegs; i++) {
674 1.26 martin if (round_page(pa) != round_page(segs[i].ds_addr))
675 1.26 martin sgsize = round_page(sgsize);
676 1.35.2.4 thorpej sgsize += min(left, segs[i].ds_len);
677 1.35.2.4 thorpej left -= segs[i].ds_len;
678 1.26 martin pa = segs[i].ds_addr + segs[i].ds_len;
679 1.26 martin }
680 1.26 martin sgsize = round_page(sgsize);
681 1.9 eeh
682 1.9 eeh s = splhigh();
683 1.35.2.8 jdolecek /*
684 1.35.2.8 jdolecek * If our segment size is larger than the boundary we need to
685 1.35.2.5 jdolecek * split the transfer up into little pieces ourselves.
686 1.35.2.4 thorpej */
687 1.35.2.4 thorpej err = extent_alloc(is->is_dvmamap, sgsize, align,
688 1.35.2.4 thorpej (sgsize > boundary) ? 0 : boundary,
689 1.35.2.4 thorpej ((flags & BUS_DMA_NOWAIT) == 0 ? EX_WAITOK : EX_NOWAIT) |
690 1.35.2.6 jdolecek EX_BOUNDZERO, &dvmaddr);
691 1.9 eeh splx(s);
692 1.9 eeh
693 1.9 eeh if (err != 0)
694 1.9 eeh return (err);
695 1.9 eeh
696 1.9 eeh #ifdef DEBUG
697 1.35.2.8 jdolecek if (dvmaddr == (bus_addr_t)-1)
698 1.35.2.8 jdolecek {
699 1.9 eeh printf("iommu_dvmamap_load_raw(): extent_alloc(%d, %x) failed!\n",
700 1.25 mrg (int)sgsize, flags);
701 1.35.2.7 jdolecek #ifdef DDB
702 1.9 eeh Debugger();
703 1.35.2.7 jdolecek #endif
704 1.35.2.8 jdolecek }
705 1.35.2.8 jdolecek #endif
706 1.9 eeh if (dvmaddr == (bus_addr_t)-1)
707 1.9 eeh return (ENOMEM);
708 1.9 eeh
709 1.35.2.4 thorpej /* Set the active DVMA map */
710 1.35.2.4 thorpej map->_dm_dvmastart = dvmaddr;
711 1.35.2.4 thorpej map->_dm_dvmasize = sgsize;
712 1.35.2.4 thorpej
713 1.35.2.8 jdolecek if ((pglist = segs[0]._ds_mlist) == NULL) {
714 1.26 martin u_long prev_va = NULL;
715 1.35.2.5 jdolecek paddr_t prev_pa = 0;
716 1.35.2.5 jdolecek int end = 0, offset;
717 1.35.2.5 jdolecek
718 1.26 martin /*
719 1.35.2.5 jdolecek * This segs is made up of individual physical
720 1.35.2.8 jdolecek * segments, probably by _bus_dmamap_load_uio() or
721 1.26 martin * _bus_dmamap_load_mbuf(). Ignore the mlist and
722 1.35.2.5 jdolecek * load each one individually.
723 1.26 martin */
724 1.35.2.4 thorpej map->dm_mapsize = size;
725 1.26 martin
726 1.35.2.5 jdolecek j = 0;
727 1.35.2.5 jdolecek for (i = 0; i < nsegs ; i++) {
728 1.35.2.4 thorpej
729 1.35.2.5 jdolecek pa = segs[i].ds_addr;
730 1.35.2.5 jdolecek offset = (pa & PGOFSET);
731 1.35.2.5 jdolecek pa = trunc_page(pa);
732 1.35.2.5 jdolecek dvmaddr = trunc_page(dvmaddr);
733 1.35.2.5 jdolecek left = min(size, segs[i].ds_len);
734 1.35.2.5 jdolecek
735 1.35.2.5 jdolecek DPRINTF(IDB_INFO, ("iommu_dvmamap_load_raw: converting "
736 1.35.2.8 jdolecek "physseg %d start %lx size %lx\n", i,
737 1.35.2.5 jdolecek (long)segs[i].ds_addr, segs[i].ds_len));
738 1.35.2.5 jdolecek
739 1.35.2.8 jdolecek if ((pa == prev_pa) &&
740 1.35.2.5 jdolecek ((offset != 0) || (end != offset))) {
741 1.35.2.5 jdolecek /* We can re-use this mapping */
742 1.35.2.5 jdolecek dvmaddr = prev_va;
743 1.35.2.5 jdolecek }
744 1.35.2.4 thorpej
745 1.35.2.5 jdolecek sgstart = dvmaddr + offset;
746 1.35.2.5 jdolecek sgend = sgstart + left - 1;
747 1.35.2.4 thorpej
748 1.35.2.5 jdolecek /* Are the segments virtually adjacent? */
749 1.35.2.8 jdolecek if ((j > 0) && (end == offset) &&
750 1.35.2.5 jdolecek ((offset == 0) || (pa == prev_pa))) {
751 1.35.2.5 jdolecek /* Just append to the previous segment. */
752 1.35.2.5 jdolecek map->dm_segs[--j].ds_len += left;
753 1.35.2.5 jdolecek DPRINTF(IDB_INFO, ("iommu_dvmamap_load_raw: "
754 1.35.2.5 jdolecek "appending seg %d start %lx size %lx\n", j,
755 1.35.2.8 jdolecek (long)map->dm_segs[j].ds_addr,
756 1.35.2.5 jdolecek map->dm_segs[j].ds_len));
757 1.35.2.5 jdolecek } else {
758 1.35.2.6 jdolecek if (j >= map->_dm_segcnt) {
759 1.35.2.7 jdolecek iommu_dvmamap_unload(t, sb, map);
760 1.35.2.6 jdolecek return (E2BIG);
761 1.35.2.6 jdolecek }
762 1.35.2.5 jdolecek map->dm_segs[j].ds_addr = sgstart;
763 1.35.2.5 jdolecek map->dm_segs[j].ds_len = left;
764 1.35.2.5 jdolecek DPRINTF(IDB_INFO, ("iommu_dvmamap_load_raw: "
765 1.35.2.5 jdolecek "seg %d start %lx size %lx\n", j,
766 1.35.2.5 jdolecek (long)map->dm_segs[j].ds_addr,
767 1.35.2.5 jdolecek map->dm_segs[j].ds_len));
768 1.35.2.4 thorpej }
769 1.35.2.5 jdolecek end = (offset + left) & PGOFSET;
770 1.35.2.4 thorpej
771 1.35.2.4 thorpej /* Check for boundary issues */
772 1.35.2.4 thorpej while ((sgstart & ~(boundary - 1)) !=
773 1.35.2.4 thorpej (sgend & ~(boundary - 1))) {
774 1.35.2.4 thorpej /* Need a new segment. */
775 1.35.2.4 thorpej map->dm_segs[j].ds_len =
776 1.35.2.6 jdolecek boundary - (sgstart & (boundary - 1));
777 1.35.2.4 thorpej DPRINTF(IDB_INFO, ("iommu_dvmamap_load_raw: "
778 1.35.2.4 thorpej "seg %d start %lx size %lx\n", j,
779 1.35.2.8 jdolecek (long)map->dm_segs[j].ds_addr,
780 1.35.2.4 thorpej map->dm_segs[j].ds_len));
781 1.35.2.6 jdolecek if (++j >= map->_dm_segcnt) {
782 1.35.2.7 jdolecek iommu_dvmamap_unload(t, sb, map);
783 1.35.2.4 thorpej return (E2BIG);
784 1.35.2.4 thorpej }
785 1.35.2.4 thorpej sgstart = roundup(sgstart, boundary);
786 1.35.2.4 thorpej map->dm_segs[j].ds_addr = sgstart;
787 1.35.2.4 thorpej map->dm_segs[j].ds_len = sgend - sgstart + 1;
788 1.35.2.4 thorpej }
789 1.35.2.4 thorpej
790 1.26 martin if (sgsize == 0)
791 1.26 martin panic("iommu_dmamap_load_raw: size botch");
792 1.35.2.4 thorpej
793 1.35.2.5 jdolecek /* Now map a series of pages. */
794 1.35.2.6 jdolecek while (dvmaddr <= sgend) {
795 1.35.2.5 jdolecek DPRINTF(IDB_BUSDMA,
796 1.35.2.5 jdolecek ("iommu_dvmamap_load_raw: map %p "
797 1.35.2.5 jdolecek "loading va %lx at pa %lx\n",
798 1.35.2.5 jdolecek map, (long)dvmaddr,
799 1.35.2.5 jdolecek (long)(pa)));
800 1.35.2.5 jdolecek /* Enter it if we haven't before. */
801 1.35.2.5 jdolecek if (prev_va != dvmaddr)
802 1.35.2.7 jdolecek iommu_enter(sb, prev_va = dvmaddr,
803 1.35.2.5 jdolecek prev_pa = pa,
804 1.35.2.8 jdolecek flags | (++npg << 12));
805 1.35.2.5 jdolecek dvmaddr += pagesz;
806 1.35.2.5 jdolecek pa += pagesz;
807 1.35.2.5 jdolecek }
808 1.35.2.5 jdolecek
809 1.35.2.5 jdolecek size -= left;
810 1.35.2.5 jdolecek ++j;
811 1.27 martin }
812 1.35.2.5 jdolecek
813 1.35.2.5 jdolecek map->dm_nsegs = j;
814 1.35.2.5 jdolecek #ifdef DIAGNOSTIC
815 1.35.2.5 jdolecek { int seg;
816 1.35.2.5 jdolecek for (seg = 0; seg < map->dm_nsegs; seg++) {
817 1.35.2.5 jdolecek if (map->dm_segs[seg].ds_addr < is->is_dvmabase ||
818 1.35.2.5 jdolecek map->dm_segs[seg].ds_addr > is->is_dvmaend) {
819 1.35.2.5 jdolecek printf("seg %d dvmaddr %lx out of range %x - %x\n",
820 1.35.2.8 jdolecek seg, (long)map->dm_segs[seg].ds_addr,
821 1.35.2.5 jdolecek is->is_dvmabase, is->is_dvmaend);
822 1.35.2.7 jdolecek #ifdef DDB
823 1.35.2.5 jdolecek Debugger();
824 1.35.2.7 jdolecek #endif
825 1.35.2.5 jdolecek }
826 1.35.2.5 jdolecek }
827 1.35.2.5 jdolecek }
828 1.35.2.5 jdolecek #endif
829 1.26 martin return (0);
830 1.26 martin }
831 1.35.2.8 jdolecek
832 1.9 eeh /*
833 1.35.2.4 thorpej * This was allocated with bus_dmamem_alloc.
834 1.35.2.8 jdolecek * The pages are on a `pglist'.
835 1.9 eeh */
836 1.9 eeh map->dm_mapsize = size;
837 1.26 martin i = 0;
838 1.35.2.4 thorpej sgstart = dvmaddr;
839 1.35.2.4 thorpej sgend = sgstart + size - 1;
840 1.35.2.4 thorpej map->dm_segs[i].ds_addr = sgstart;
841 1.35.2.4 thorpej while ((sgstart & ~(boundary - 1)) != (sgend & ~(boundary - 1))) {
842 1.35.2.4 thorpej /* Oops. We crossed a boundary. Split the xfer. */
843 1.35.2.6 jdolecek map->dm_segs[i].ds_len = boundary - (sgstart & (boundary - 1));
844 1.35.2.4 thorpej DPRINTF(IDB_INFO, ("iommu_dvmamap_load_raw: "
845 1.35.2.4 thorpej "seg %d start %lx size %lx\n", i,
846 1.35.2.5 jdolecek (long)map->dm_segs[i].ds_addr,
847 1.35.2.4 thorpej map->dm_segs[i].ds_len));
848 1.35.2.6 jdolecek if (++i >= map->_dm_segcnt) {
849 1.35.2.4 thorpej /* Too many segments. Fail the operation. */
850 1.35.2.4 thorpej s = splhigh();
851 1.35.2.4 thorpej /* How can this fail? And if it does what can we do? */
852 1.35.2.4 thorpej err = extent_free(is->is_dvmamap,
853 1.35.2.4 thorpej dvmaddr, sgsize, EX_NOWAIT);
854 1.35.2.4 thorpej map->_dm_dvmastart = 0;
855 1.35.2.4 thorpej map->_dm_dvmasize = 0;
856 1.35.2.4 thorpej splx(s);
857 1.35.2.4 thorpej return (E2BIG);
858 1.35.2.4 thorpej }
859 1.35.2.4 thorpej sgstart = roundup(sgstart, boundary);
860 1.35.2.4 thorpej map->dm_segs[i].ds_addr = sgstart;
861 1.35.2.4 thorpej }
862 1.35.2.4 thorpej DPRINTF(IDB_INFO, ("iommu_dvmamap_load_raw: "
863 1.35.2.4 thorpej "seg %d start %lx size %lx\n", i,
864 1.35.2.5 jdolecek (long)map->dm_segs[i].ds_addr, map->dm_segs[i].ds_len));
865 1.35.2.4 thorpej map->dm_segs[i].ds_len = sgend - sgstart + 1;
866 1.9 eeh
867 1.35.2.8 jdolecek TAILQ_FOREACH(pg, pglist, pageq) {
868 1.9 eeh if (sgsize == 0)
869 1.9 eeh panic("iommu_dmamap_load_raw: size botch");
870 1.35.2.8 jdolecek pa = VM_PAGE_TO_PHYS(pg);
871 1.9 eeh
872 1.22 mrg DPRINTF(IDB_BUSDMA,
873 1.9 eeh ("iommu_dvmamap_load_raw: map %p loading va %lx at pa %lx\n",
874 1.9 eeh map, (long)dvmaddr, (long)(pa)));
875 1.35.2.7 jdolecek iommu_enter(sb, dvmaddr, pa, flags|0x8000);
876 1.35.2.8 jdolecek
877 1.9 eeh dvmaddr += pagesz;
878 1.9 eeh sgsize -= pagesz;
879 1.9 eeh }
880 1.35.2.4 thorpej map->dm_mapsize = size;
881 1.35.2.4 thorpej map->dm_nsegs = i+1;
882 1.35.2.5 jdolecek #ifdef DIAGNOSTIC
883 1.35.2.5 jdolecek { int seg;
884 1.35.2.5 jdolecek for (seg = 0; seg < map->dm_nsegs; seg++) {
885 1.35.2.5 jdolecek if (map->dm_segs[seg].ds_addr < is->is_dvmabase ||
886 1.35.2.5 jdolecek map->dm_segs[seg].ds_addr > is->is_dvmaend) {
887 1.35.2.5 jdolecek printf("seg %d dvmaddr %lx out of range %x - %x\n",
888 1.35.2.8 jdolecek seg, (long)map->dm_segs[seg].ds_addr,
889 1.35.2.5 jdolecek is->is_dvmabase, is->is_dvmaend);
890 1.35.2.7 jdolecek #ifdef DDB
891 1.35.2.5 jdolecek Debugger();
892 1.35.2.7 jdolecek #endif
893 1.35.2.5 jdolecek }
894 1.35.2.5 jdolecek }
895 1.35.2.5 jdolecek }
896 1.35.2.5 jdolecek #endif
897 1.9 eeh return (0);
898 1.7 mrg }
899 1.7 mrg
900 1.7 mrg void
901 1.35.2.7 jdolecek iommu_dvmamap_sync(t, sb, map, offset, len, ops)
902 1.7 mrg bus_dma_tag_t t;
903 1.35.2.7 jdolecek struct strbuf_ctl *sb;
904 1.7 mrg bus_dmamap_t map;
905 1.7 mrg bus_addr_t offset;
906 1.7 mrg bus_size_t len;
907 1.7 mrg int ops;
908 1.7 mrg {
909 1.35.2.7 jdolecek struct iommu_state *is = sb->sb_is;
910 1.7 mrg vaddr_t va = map->dm_segs[0].ds_addr + offset;
911 1.35.2.7 jdolecek int64_t tte;
912 1.7 mrg
913 1.7 mrg /*
914 1.7 mrg * We only support one DMA segment; supporting more makes this code
915 1.35.2.8 jdolecek * too unwieldy.
916 1.7 mrg */
917 1.7 mrg
918 1.35.2.8 jdolecek DPRINTF(IDB_SYNC,
919 1.35.2.8 jdolecek ("iommu_dvmamap_sync: syncing va %p len %lu "
920 1.35.2.8 jdolecek "ops 0x%x\n", (void *)(u_long)va, (u_long)len, ops));
921 1.35.2.8 jdolecek if (ops & (BUS_DMASYNC_PREREAD | BUS_DMASYNC_POSTWRITE)) {
922 1.35.2.8 jdolecek /* Nothing to do */
923 1.35.2.8 jdolecek return;
924 1.7 mrg }
925 1.35.2.7 jdolecek #ifdef DIAGNOSTIC
926 1.35.2.8 jdolecek if (va < is->is_dvmabase || va >= is->is_dvmaend)
927 1.35.2.8 jdolecek panic("iommu_dvmamap_sync: invalid dva %lx", va);
928 1.35.2.7 jdolecek #endif
929 1.35.2.8 jdolecek KASSERT((ops & (BUS_DMASYNC_POSTREAD | BUS_DMASYNC_PREWRITE)) != 0);
930 1.35.2.8 jdolecek tte = is->is_tsb[IOTSBSLOT(va, is->is_tsbsize)];
931 1.35.2.7 jdolecek
932 1.35.2.8 jdolecek /* if we have a streaming buffer, flush it here first */
933 1.35.2.8 jdolecek if ((tte & IOTTE_STREAM) == 0 || !sb->sb_flush)
934 1.35.2.8 jdolecek return;
935 1.35.2.8 jdolecek
936 1.35.2.8 jdolecek while (len > 0) {
937 1.35.2.8 jdolecek DPRINTF(IDB_BUSDMA,
938 1.35.2.8 jdolecek ("iommu_dvmamap_sync: flushing va %p, %lu "
939 1.35.2.8 jdolecek "bytes left\n", (void *)(u_long)va, (u_long)len));
940 1.35.2.8 jdolecek iommu_strbuf_flush(sb, va);
941 1.35.2.8 jdolecek if (len <= NBPG) {
942 1.35.2.8 jdolecek iommu_strbuf_flush_done(sb);
943 1.35.2.8 jdolecek len = 0;
944 1.35.2.8 jdolecek } else
945 1.35.2.8 jdolecek len -= NBPG;
946 1.35.2.8 jdolecek va += NBPG;
947 1.7 mrg }
948 1.7 mrg }
949 1.7 mrg
950 1.7 mrg int
951 1.35.2.7 jdolecek iommu_dvmamem_alloc(t, sb, size, alignment, boundary, segs, nsegs, rsegs, flags)
952 1.7 mrg bus_dma_tag_t t;
953 1.35.2.7 jdolecek struct strbuf_ctl *sb;
954 1.7 mrg bus_size_t size, alignment, boundary;
955 1.7 mrg bus_dma_segment_t *segs;
956 1.7 mrg int nsegs;
957 1.7 mrg int *rsegs;
958 1.7 mrg int flags;
959 1.7 mrg {
960 1.7 mrg
961 1.25 mrg DPRINTF(IDB_BUSDMA, ("iommu_dvmamem_alloc: sz %llx align %llx bound %llx "
962 1.25 mrg "segp %p flags %d\n", (unsigned long long)size,
963 1.25 mrg (unsigned long long)alignment, (unsigned long long)boundary,
964 1.25 mrg segs, flags));
965 1.7 mrg return (bus_dmamem_alloc(t->_parent, size, alignment, boundary,
966 1.21 eeh segs, nsegs, rsegs, flags|BUS_DMA_DVMA));
967 1.7 mrg }
968 1.7 mrg
969 1.7 mrg void
970 1.35.2.7 jdolecek iommu_dvmamem_free(t, sb, segs, nsegs)
971 1.7 mrg bus_dma_tag_t t;
972 1.35.2.7 jdolecek struct strbuf_ctl *sb;
973 1.7 mrg bus_dma_segment_t *segs;
974 1.7 mrg int nsegs;
975 1.7 mrg {
976 1.7 mrg
977 1.22 mrg DPRINTF(IDB_BUSDMA, ("iommu_dvmamem_free: segp %p nsegs %d\n",
978 1.7 mrg segs, nsegs));
979 1.7 mrg bus_dmamem_free(t->_parent, segs, nsegs);
980 1.7 mrg }
981 1.7 mrg
982 1.7 mrg /*
983 1.7 mrg * Map the DVMA mappings into the kernel pmap.
984 1.7 mrg * Check the flags to see whether we're streaming or coherent.
985 1.7 mrg */
986 1.7 mrg int
987 1.35.2.7 jdolecek iommu_dvmamem_map(t, sb, segs, nsegs, size, kvap, flags)
988 1.7 mrg bus_dma_tag_t t;
989 1.35.2.7 jdolecek struct strbuf_ctl *sb;
990 1.7 mrg bus_dma_segment_t *segs;
991 1.7 mrg int nsegs;
992 1.7 mrg size_t size;
993 1.7 mrg caddr_t *kvap;
994 1.7 mrg int flags;
995 1.7 mrg {
996 1.35.2.8 jdolecek struct vm_page *pg;
997 1.7 mrg vaddr_t va;
998 1.7 mrg bus_addr_t addr;
999 1.35.2.8 jdolecek struct pglist *pglist;
1000 1.8 mrg int cbit;
1001 1.7 mrg
1002 1.22 mrg DPRINTF(IDB_BUSDMA, ("iommu_dvmamem_map: segp %p nsegs %d size %lx\n",
1003 1.7 mrg segs, nsegs, size));
1004 1.7 mrg
1005 1.7 mrg /*
1006 1.8 mrg * Allocate some space in the kernel map, and then map these pages
1007 1.8 mrg * into this space.
1008 1.7 mrg */
1009 1.8 mrg size = round_page(size);
1010 1.8 mrg va = uvm_km_valloc(kernel_map, size);
1011 1.8 mrg if (va == 0)
1012 1.8 mrg return (ENOMEM);
1013 1.7 mrg
1014 1.8 mrg *kvap = (caddr_t)va;
1015 1.7 mrg
1016 1.35.2.8 jdolecek /*
1017 1.7 mrg * digest flags:
1018 1.7 mrg */
1019 1.7 mrg cbit = 0;
1020 1.7 mrg if (flags & BUS_DMA_COHERENT) /* Disable vcache */
1021 1.7 mrg cbit |= PMAP_NVC;
1022 1.7 mrg if (flags & BUS_DMA_NOCACHE) /* sideffects */
1023 1.7 mrg cbit |= PMAP_NC;
1024 1.7 mrg
1025 1.7 mrg /*
1026 1.8 mrg * Now take this and map it into the CPU.
1027 1.7 mrg */
1028 1.35.2.8 jdolecek pglist = segs[0]._ds_mlist;
1029 1.35.2.8 jdolecek TAILQ_FOREACH(pg, pglist, pageq) {
1030 1.8 mrg #ifdef DIAGNOSTIC
1031 1.7 mrg if (size == 0)
1032 1.7 mrg panic("iommu_dvmamem_map: size botch");
1033 1.8 mrg #endif
1034 1.35.2.8 jdolecek addr = VM_PAGE_TO_PHYS(pg);
1035 1.22 mrg DPRINTF(IDB_BUSDMA, ("iommu_dvmamem_map: "
1036 1.25 mrg "mapping va %lx at %llx\n", va, (unsigned long long)addr | cbit));
1037 1.35.2.8 jdolecek pmap_kenter_pa(va, addr | cbit, VM_PROT_READ | VM_PROT_WRITE);
1038 1.7 mrg va += PAGE_SIZE;
1039 1.7 mrg size -= PAGE_SIZE;
1040 1.7 mrg }
1041 1.35.2.3 thorpej pmap_update(pmap_kernel());
1042 1.7 mrg return (0);
1043 1.7 mrg }
1044 1.7 mrg
1045 1.7 mrg /*
1046 1.7 mrg * Unmap DVMA mappings from kernel
1047 1.7 mrg */
1048 1.7 mrg void
1049 1.35.2.7 jdolecek iommu_dvmamem_unmap(t, sb, kva, size)
1050 1.7 mrg bus_dma_tag_t t;
1051 1.35.2.7 jdolecek struct strbuf_ctl *sb;
1052 1.7 mrg caddr_t kva;
1053 1.7 mrg size_t size;
1054 1.7 mrg {
1055 1.35.2.8 jdolecek
1056 1.22 mrg DPRINTF(IDB_BUSDMA, ("iommu_dvmamem_unmap: kvm %p size %lx\n",
1057 1.7 mrg kva, size));
1058 1.35.2.8 jdolecek
1059 1.7 mrg #ifdef DIAGNOSTIC
1060 1.7 mrg if ((u_long)kva & PGOFSET)
1061 1.7 mrg panic("iommu_dvmamem_unmap");
1062 1.7 mrg #endif
1063 1.35.2.8 jdolecek
1064 1.7 mrg size = round_page(size);
1065 1.35.2.8 jdolecek pmap_kremove((vaddr_t)kva, size);
1066 1.35.2.3 thorpej pmap_update(pmap_kernel());
1067 1.8 mrg uvm_km_free(kernel_map, (vaddr_t)kva, size);
1068 1.1 mrg }
1069