iommu.c revision 1.101
1 /* $NetBSD: iommu.c,v 1.101 2011/03/16 02:41:19 mrg Exp $ */ 2 3 /* 4 * Copyright (c) 1999, 2000 Matthew R. Green 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 17 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 18 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 19 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 20 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 21 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 22 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 23 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 24 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 * SUCH DAMAGE. 27 */ 28 29 /* 30 * Copyright (c) 2001, 2002 Eduardo Horvath 31 * All rights reserved. 32 * 33 * Redistribution and use in source and binary forms, with or without 34 * modification, are permitted provided that the following conditions 35 * are met: 36 * 1. Redistributions of source code must retain the above copyright 37 * notice, this list of conditions and the following disclaimer. 38 * 2. Redistributions in binary form must reproduce the above copyright 39 * notice, this list of conditions and the following disclaimer in the 40 * documentation and/or other materials provided with the distribution. 41 * 3. The name of the author may not be used to endorse or promote products 42 * derived from this software without specific prior written permission. 43 * 44 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 45 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 46 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 47 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 48 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 49 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 50 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 51 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 52 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 53 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 54 * SUCH DAMAGE. 55 */ 56 57 /* 58 * UltraSPARC IOMMU support; used by both the sbus and pci code. 59 */ 60 61 #include <sys/cdefs.h> 62 __KERNEL_RCSID(0, "$NetBSD: iommu.c,v 1.101 2011/03/16 02:41:19 mrg Exp $"); 63 64 #include "opt_ddb.h" 65 66 #include <sys/param.h> 67 #include <sys/extent.h> 68 #include <sys/malloc.h> 69 #include <sys/systm.h> 70 #include <sys/device.h> 71 #include <sys/proc.h> 72 73 #include <uvm/uvm.h> 74 75 #include <machine/bus.h> 76 #include <sparc64/dev/iommureg.h> 77 #include <sparc64/dev/iommuvar.h> 78 79 #include <machine/autoconf.h> 80 #include <machine/cpu.h> 81 82 #ifdef DEBUG 83 #define IDB_BUSDMA 0x1 84 #define IDB_IOMMU 0x2 85 #define IDB_INFO 0x4 86 #define IDB_SYNC 0x8 87 int iommudebug = 0x0; 88 #define DPRINTF(l, s) do { if (iommudebug & l) printf s; } while (0) 89 #define IOTTE_DEBUG(n) (n) 90 #else 91 #define DPRINTF(l, s) 92 #define IOTTE_DEBUG(n) 0 93 #endif 94 95 #define iommu_strbuf_flush(i, v) do { \ 96 if ((i)->sb_flush) \ 97 bus_space_write_8((i)->sb_is->is_bustag, (i)->sb_sb, \ 98 STRBUFREG(strbuf_pgflush), (v)); \ 99 } while (0) 100 101 static int iommu_strbuf_flush_done(struct strbuf_ctl *); 102 static void _iommu_dvmamap_sync(bus_dma_tag_t, bus_dmamap_t, bus_addr_t, 103 bus_size_t, int); 104 105 /* 106 * initialise the UltraSPARC IOMMU (SBUS or PCI): 107 * - allocate and setup the iotsb. 108 * - enable the IOMMU 109 * - initialise the streaming buffers (if they exist) 110 * - create a private DVMA map. 111 */ 112 void 113 iommu_init(char *name, struct iommu_state *is, int tsbsize, uint32_t iovabase) 114 { 115 psize_t size; 116 vaddr_t va; 117 paddr_t pa; 118 struct vm_page *pg; 119 struct pglist pglist; 120 121 /* 122 * Setup the iommu. 123 * 124 * The sun4u iommu is part of the SBUS or PCI controller so we will 125 * deal with it here.. 126 * 127 * For sysio and psycho/psycho+ the IOMMU address space always ends at 128 * 0xffffe000, but the starting address depends on the size of the 129 * map. The map size is 1024 * 2 ^ is->is_tsbsize entries, where each 130 * entry is 8 bytes. The start of the map can be calculated by 131 * (0xffffe000 << (8 + is->is_tsbsize)). 132 * 133 * But sabre and hummingbird use a different scheme that seems to 134 * be hard-wired, so we read the start and size from the PROM and 135 * just use those values. 136 */ 137 is->is_cr = IOMMUCR_EN; 138 is->is_tsbsize = tsbsize; 139 if (iovabase == -1) { 140 is->is_dvmabase = IOTSB_VSTART(is->is_tsbsize); 141 is->is_dvmaend = IOTSB_VEND - 1; 142 } else { 143 is->is_dvmabase = iovabase; 144 is->is_dvmaend = iovabase + IOTSB_VSIZE(tsbsize) - 1; 145 } 146 147 /* 148 * Allocate memory for I/O pagetables. They need to be physically 149 * contiguous. 150 */ 151 152 size = PAGE_SIZE << is->is_tsbsize; 153 if (uvm_pglistalloc((psize_t)size, (paddr_t)0, (paddr_t)-1, 154 (paddr_t)PAGE_SIZE, (paddr_t)0, &pglist, 1, 0) != 0) 155 panic("iommu_init: no memory"); 156 157 va = uvm_km_alloc(kernel_map, size, 0, UVM_KMF_VAONLY); 158 if (va == 0) 159 panic("iommu_init: no memory"); 160 is->is_tsb = (int64_t *)va; 161 162 is->is_ptsb = VM_PAGE_TO_PHYS(TAILQ_FIRST(&pglist)); 163 164 /* Map the pages */ 165 TAILQ_FOREACH(pg, &pglist, pageq.queue) { 166 pa = VM_PAGE_TO_PHYS(pg); 167 pmap_kenter_pa(va, pa | PMAP_NVC, 168 VM_PROT_READ | VM_PROT_WRITE, 0); 169 va += PAGE_SIZE; 170 } 171 pmap_update(pmap_kernel()); 172 memset(is->is_tsb, 0, size); 173 174 #ifdef DEBUG 175 if ((iommudebug & IDB_INFO) && 0) 176 { 177 /* Probe the iommu */ 178 179 printf("iommu regs at: cr=%lx tsb=%lx flush=%lx\n", 180 (u_long)bus_space_read_8(is->is_bustag, is->is_iommu, 181 offsetof (struct iommureg, iommu_cr)), 182 (u_long)bus_space_read_8(is->is_bustag, is->is_iommu, 183 offsetof (struct iommureg, iommu_tsb)), 184 (u_long)bus_space_read_8(is->is_bustag, is->is_iommu, 185 offsetof (struct iommureg, iommu_flush))); 186 printf("iommu cr=%llx tsb=%llx\n", 187 (unsigned long long)bus_space_read_8(is->is_bustag, 188 is->is_iommu, 189 offsetof (struct iommureg, iommu_cr)), 190 (unsigned long long)bus_space_read_8(is->is_bustag, 191 is->is_iommu, 192 offsetof (struct iommureg, iommu_tsb))); 193 printf("TSB base %p phys %llx\n", (void *)is->is_tsb, 194 (unsigned long long)is->is_ptsb); 195 delay(1000000); /* 1 s */ 196 } 197 #endif 198 199 /* 200 * Now all the hardware's working we need to allocate a dvma map. 201 */ 202 aprint_debug("DVMA map: %x to %x\n", 203 (unsigned int)is->is_dvmabase, 204 (unsigned int)is->is_dvmaend); 205 aprint_debug("IOTSB: %llx to %llx\n", 206 (unsigned long long)is->is_ptsb, 207 (unsigned long long)(is->is_ptsb + size - 1)); 208 is->is_dvmamap = extent_create(name, 209 is->is_dvmabase, is->is_dvmaend, 210 M_DEVBUF, 0, 0, EX_NOWAIT); 211 /* XXXMRG Check is_dvmamap is valid. */ 212 213 /* 214 * Set the TSB size. The relevant bits were moved to the TSB 215 * base register in the PCIe host bridges. 216 */ 217 if (strncmp(name, "pyro", 4) == 0) 218 is->is_ptsb |= is->is_tsbsize; 219 else 220 is->is_cr |= (is->is_tsbsize << 16); 221 222 /* 223 * now actually start up the IOMMU 224 */ 225 iommu_reset(is); 226 } 227 228 /* 229 * Streaming buffers don't exist on the UltraSPARC IIi; we should have 230 * detected that already and disabled them. If not, we will notice that 231 * they aren't there when the STRBUF_EN bit does not remain. 232 */ 233 void 234 iommu_reset(struct iommu_state *is) 235 { 236 int i; 237 struct strbuf_ctl *sb; 238 239 IOMMUREG_WRITE(is, iommu_tsb, is->is_ptsb); 240 241 /* Enable IOMMU in diagnostic mode */ 242 IOMMUREG_WRITE(is, iommu_cr, is->is_cr|IOMMUCR_DE); 243 244 for (i = 0; i < 2; i++) { 245 if ((sb = is->is_sb[i])) { 246 247 /* Enable diagnostics mode? */ 248 bus_space_write_8(is->is_bustag, is->is_sb[i]->sb_sb, 249 STRBUFREG(strbuf_ctl), STRBUF_EN); 250 251 membar_lookaside(); 252 253 /* No streaming buffers? Disable them */ 254 if (bus_space_read_8(is->is_bustag, 255 is->is_sb[i]->sb_sb, 256 STRBUFREG(strbuf_ctl)) == 0) { 257 is->is_sb[i]->sb_flush = NULL; 258 } else { 259 260 /* 261 * locate the pa of the flush buffer. 262 */ 263 if (pmap_extract(pmap_kernel(), 264 (vaddr_t)is->is_sb[i]->sb_flush, 265 &is->is_sb[i]->sb_flushpa) == FALSE) 266 is->is_sb[i]->sb_flush = NULL; 267 } 268 } 269 } 270 271 if (is->is_flags & IOMMU_FLUSH_CACHE) 272 IOMMUREG_WRITE(is, iommu_cache_invalidate, -1ULL); 273 } 274 275 /* 276 * Here are the iommu control routines. 277 */ 278 void 279 iommu_enter(struct strbuf_ctl *sb, vaddr_t va, int64_t pa, int flags) 280 { 281 struct iommu_state *is = sb->sb_is; 282 int strbuf = (flags & BUS_DMA_STREAMING); 283 int64_t tte; 284 285 #ifdef DIAGNOSTIC 286 if (va < is->is_dvmabase || va > is->is_dvmaend) 287 panic("iommu_enter: va %#lx not in DVMA space", va); 288 #endif 289 290 /* Is the streamcache flush really needed? */ 291 if (sb->sb_flush) 292 iommu_strbuf_flush(sb, va); 293 else 294 /* If we can't flush the strbuf don't enable it. */ 295 strbuf = 0; 296 297 tte = MAKEIOTTE(pa, !(flags & BUS_DMA_NOWRITE), 298 !(flags & BUS_DMA_NOCACHE), (strbuf)); 299 #ifdef DEBUG 300 tte |= (flags & 0xff000LL)<<(4*8); 301 #endif 302 303 DPRINTF(IDB_IOMMU, ("Clearing TSB slot %d for va %p\n", 304 (int)IOTSBSLOT(va,is->is_tsbsize), (void *)(u_long)va)); 305 is->is_tsb[IOTSBSLOT(va,is->is_tsbsize)] = tte; 306 bus_space_write_8(is->is_bustag, is->is_iommu, 307 IOMMUREG(iommu_flush), va); 308 DPRINTF(IDB_IOMMU, ("iommu_enter: va %lx pa %lx TSB[%lx]@%p=%lx\n", 309 va, (long)pa, (u_long)IOTSBSLOT(va,is->is_tsbsize), 310 (void *)(u_long)&is->is_tsb[IOTSBSLOT(va,is->is_tsbsize)], 311 (u_long)tte)); 312 } 313 314 /* 315 * Find the value of a DVMA address (debug routine). 316 */ 317 paddr_t 318 iommu_extract(struct iommu_state *is, vaddr_t dva) 319 { 320 int64_t tte = 0; 321 322 if (dva >= is->is_dvmabase && dva <= is->is_dvmaend) 323 tte = is->is_tsb[IOTSBSLOT(dva, is->is_tsbsize)]; 324 325 if ((tte & IOTTE_V) == 0) 326 return ((paddr_t)-1L); 327 return (tte & IOTTE_PAMASK); 328 } 329 330 /* 331 * iommu_remove: removes mappings created by iommu_enter 332 * 333 * Only demap from IOMMU if flag is set. 334 * 335 * XXX: this function needs better internal error checking. 336 */ 337 void 338 iommu_remove(struct iommu_state *is, vaddr_t va, size_t len) 339 { 340 int slot; 341 342 #ifdef DIAGNOSTIC 343 if (va < is->is_dvmabase || va > is->is_dvmaend) 344 panic("iommu_remove: va 0x%lx not in DVMA space", (u_long)va); 345 if ((long)(va + len) < (long)va) 346 panic("iommu_remove: va 0x%lx + len 0x%lx wraps", 347 (long) va, (long) len); 348 if (len & ~0xfffffff) 349 panic("iommu_remove: ridiculous len 0x%lx", (u_long)len); 350 #endif 351 352 va = trunc_page(va); 353 DPRINTF(IDB_IOMMU, ("iommu_remove: va %lx TSB[%lx]@%p\n", 354 va, (u_long)IOTSBSLOT(va, is->is_tsbsize), 355 &is->is_tsb[IOTSBSLOT(va, is->is_tsbsize)])); 356 while (len > 0) { 357 DPRINTF(IDB_IOMMU, ("iommu_remove: clearing TSB slot %d " 358 "for va %p size %lx\n", 359 (int)IOTSBSLOT(va,is->is_tsbsize), (void *)(u_long)va, 360 (u_long)len)); 361 if (len <= PAGE_SIZE) 362 len = 0; 363 else 364 len -= PAGE_SIZE; 365 366 #if 0 367 /* 368 * XXX Zero-ing the entry would not require RMW 369 * 370 * Disabling valid bit while a page is used by a device 371 * causes an uncorrectable DMA error. 372 * Workaround to avoid an uncorrectable DMA error is 373 * eliminating the next line, but the page is mapped 374 * until the next iommu_enter call. 375 */ 376 is->is_tsb[IOTSBSLOT(va,is->is_tsbsize)] &= ~IOTTE_V; 377 membar_storestore(); 378 #endif 379 IOMMUREG_WRITE(is, iommu_flush, va); 380 381 /* Flush cache if necessary. */ 382 slot = IOTSBSLOT(trunc_page(va), is->is_tsbsize); 383 if ((is->is_flags & IOMMU_FLUSH_CACHE) && 384 (len == 0 || (slot % 8) == 7)) 385 IOMMUREG_WRITE(is, iommu_cache_flush, 386 is->is_ptsb + slot * 8); 387 388 va += PAGE_SIZE; 389 } 390 } 391 392 static int 393 iommu_strbuf_flush_done(struct strbuf_ctl *sb) 394 { 395 struct iommu_state *is = sb->sb_is; 396 struct timeval cur, flushtimeout; 397 398 #define BUMPTIME(t, usec) { \ 399 register volatile struct timeval *tp = (t); \ 400 register long us; \ 401 \ 402 tp->tv_usec = us = tp->tv_usec + (usec); \ 403 if (us >= 1000000) { \ 404 tp->tv_usec = us - 1000000; \ 405 tp->tv_sec++; \ 406 } \ 407 } 408 409 if (!sb->sb_flush) 410 return (0); 411 412 /* 413 * Streaming buffer flushes: 414 * 415 * 1 Tell strbuf to flush by storing va to strbuf_pgflush. If 416 * we're not on a cache line boundary (64-bits): 417 * 2 Store 0 in flag 418 * 3 Store pointer to flag in flushsync 419 * 4 wait till flushsync becomes 0x1 420 * 421 * If it takes more than .5 sec, something 422 * went wrong. 423 */ 424 425 *sb->sb_flush = 0; 426 bus_space_write_8(is->is_bustag, sb->sb_sb, 427 STRBUFREG(strbuf_flushsync), sb->sb_flushpa); 428 429 microtime(&flushtimeout); 430 cur = flushtimeout; 431 BUMPTIME(&flushtimeout, 500000); /* 1/2 sec */ 432 433 DPRINTF(IDB_IOMMU, ("iommu_strbuf_flush_done: flush = %lx " 434 "at va = %lx pa = %lx now=%"PRIx64":%"PRIx32" until = %"PRIx64":%"PRIx32"\n", 435 (long)*sb->sb_flush, (long)sb->sb_flush, (long)sb->sb_flushpa, 436 cur.tv_sec, cur.tv_usec, 437 flushtimeout.tv_sec, flushtimeout.tv_usec)); 438 439 /* Bypass non-coherent D$ */ 440 while ((!ldxa(sb->sb_flushpa, ASI_PHYS_CACHED)) && 441 timercmp(&cur, &flushtimeout, <=)) 442 microtime(&cur); 443 444 #ifdef DIAGNOSTIC 445 if (!ldxa(sb->sb_flushpa, ASI_PHYS_CACHED)) { 446 printf("iommu_strbuf_flush_done: flush timeout %p, at %p\n", 447 (void *)(u_long)*sb->sb_flush, 448 (void *)(u_long)sb->sb_flushpa); /* panic? */ 449 #ifdef DDB 450 Debugger(); 451 #endif 452 } 453 #endif 454 DPRINTF(IDB_IOMMU, ("iommu_strbuf_flush_done: flushed\n")); 455 return (*sb->sb_flush); 456 } 457 458 /* 459 * IOMMU DVMA operations, common to SBUS and PCI. 460 */ 461 int 462 iommu_dvmamap_load(bus_dma_tag_t t, bus_dmamap_t map, void *buf, 463 bus_size_t buflen, struct proc *p, int flags) 464 { 465 struct strbuf_ctl *sb = (struct strbuf_ctl *)map->_dm_cookie; 466 struct iommu_state *is = sb->sb_is; 467 int s; 468 int err, needsflush; 469 bus_size_t sgsize; 470 paddr_t curaddr; 471 u_long dvmaddr, sgstart, sgend, bmask; 472 bus_size_t align, boundary, len; 473 vaddr_t vaddr = (vaddr_t)buf; 474 int seg; 475 struct pmap *pmap; 476 int slot; 477 478 if (map->dm_nsegs) { 479 /* Already in use?? */ 480 #ifdef DIAGNOSTIC 481 printf("iommu_dvmamap_load: map still in use\n"); 482 #endif 483 bus_dmamap_unload(t, map); 484 } 485 486 /* 487 * Make sure that on error condition we return "no valid mappings". 488 */ 489 map->dm_nsegs = 0; 490 KASSERT(map->dm_maxsegsz <= map->_dm_maxmaxsegsz); 491 492 if (buflen > map->_dm_size) { 493 DPRINTF(IDB_BUSDMA, 494 ("iommu_dvmamap_load(): error %d > %d -- " 495 "map size exceeded!\n", (int)buflen, (int)map->_dm_size)); 496 return (EINVAL); 497 } 498 499 sgsize = round_page(buflen + ((int)vaddr & PGOFSET)); 500 501 /* 502 * A boundary presented to bus_dmamem_alloc() takes precedence 503 * over boundary in the map. 504 */ 505 if ((boundary = (map->dm_segs[0]._ds_boundary)) == 0) 506 boundary = map->_dm_boundary; 507 align = max(map->dm_segs[0]._ds_align, PAGE_SIZE); 508 509 /* 510 * If our segment size is larger than the boundary we need to 511 * split the transfer up int little pieces ourselves. 512 */ 513 KASSERT(is->is_dvmamap); 514 s = splhigh(); 515 err = extent_alloc(is->is_dvmamap, sgsize, align, 516 (sgsize > boundary) ? 0 : boundary, 517 EX_NOWAIT|EX_BOUNDZERO, &dvmaddr); 518 splx(s); 519 520 #ifdef DEBUG 521 if (err || (dvmaddr == (u_long)-1)) { 522 printf("iommu_dvmamap_load(): extent_alloc(%d, %x) failed!\n", 523 (int)sgsize, flags); 524 #ifdef DDB 525 Debugger(); 526 #endif 527 } 528 #endif 529 if (err != 0) 530 return (err); 531 532 if (dvmaddr == (u_long)-1) 533 return (ENOMEM); 534 535 /* Set the active DVMA map */ 536 map->_dm_dvmastart = dvmaddr; 537 map->_dm_dvmasize = sgsize; 538 539 /* 540 * Now split the DVMA range into segments, not crossing 541 * the boundary. 542 */ 543 seg = 0; 544 sgstart = dvmaddr + (vaddr & PGOFSET); 545 sgend = sgstart + buflen - 1; 546 map->dm_segs[seg].ds_addr = sgstart; 547 DPRINTF(IDB_INFO, ("iommu_dvmamap_load: boundary %lx boundary - 1 %lx " 548 "~(boundary - 1) %lx\n", (long)boundary, (long)(boundary - 1), 549 (long)~(boundary - 1))); 550 bmask = ~(boundary - 1); 551 while ((sgstart & bmask) != (sgend & bmask) || 552 sgend - sgstart + 1 > map->dm_maxsegsz) { 553 /* Oops. We crossed a boundary or large seg. Split the xfer. */ 554 len = map->dm_maxsegsz; 555 if ((sgstart & bmask) != (sgend & bmask)) 556 len = min(len, boundary - (sgstart & (boundary - 1))); 557 map->dm_segs[seg].ds_len = len; 558 DPRINTF(IDB_INFO, ("iommu_dvmamap_load: " 559 "seg %d start %lx size %lx\n", seg, 560 (long)map->dm_segs[seg].ds_addr, 561 (long)map->dm_segs[seg].ds_len)); 562 if (++seg >= map->_dm_segcnt) { 563 /* Too many segments. Fail the operation. */ 564 DPRINTF(IDB_INFO, ("iommu_dvmamap_load: " 565 "too many segments %d\n", seg)); 566 s = splhigh(); 567 err = extent_free(is->is_dvmamap, 568 dvmaddr, sgsize, EX_NOWAIT); 569 map->_dm_dvmastart = 0; 570 map->_dm_dvmasize = 0; 571 splx(s); 572 if (err != 0) 573 printf("warning: %s: %" PRId64 574 " of DVMA space lost\n", __func__, sgsize); 575 return (EFBIG); 576 } 577 sgstart += len; 578 map->dm_segs[seg].ds_addr = sgstart; 579 } 580 map->dm_segs[seg].ds_len = sgend - sgstart + 1; 581 DPRINTF(IDB_INFO, ("iommu_dvmamap_load: " 582 "seg %d start %lx size %lx\n", seg, 583 (long)map->dm_segs[seg].ds_addr, (long)map->dm_segs[seg].ds_len)); 584 map->dm_nsegs = seg + 1; 585 map->dm_mapsize = buflen; 586 587 if (p != NULL) 588 pmap = p->p_vmspace->vm_map.pmap; 589 else 590 pmap = pmap_kernel(); 591 592 needsflush = 0; 593 for (; buflen > 0; ) { 594 595 /* 596 * Get the physical address for this page. 597 */ 598 if (pmap_extract(pmap, (vaddr_t)vaddr, &curaddr) == FALSE) { 599 #ifdef DIAGNOSTIC 600 printf("iommu_dvmamap_load: pmap_extract failed %lx\n", vaddr); 601 #endif 602 bus_dmamap_unload(t, map); 603 return (-1); 604 } 605 606 /* 607 * Compute the segment size, and adjust counts. 608 */ 609 sgsize = PAGE_SIZE - ((u_long)vaddr & PGOFSET); 610 if (buflen < sgsize) 611 sgsize = buflen; 612 613 DPRINTF(IDB_BUSDMA, 614 ("iommu_dvmamap_load: map %p loading va %p " 615 "dva %lx at pa %lx\n", 616 map, (void *)vaddr, (long)dvmaddr, 617 (long)trunc_page(curaddr))); 618 iommu_enter(sb, trunc_page(dvmaddr), trunc_page(curaddr), 619 flags | IOTTE_DEBUG(0x4000)); 620 needsflush = 1; 621 622 vaddr += sgsize; 623 buflen -= sgsize; 624 625 /* Flush cache if necessary. */ 626 slot = IOTSBSLOT(trunc_page(dvmaddr), is->is_tsbsize); 627 if ((is->is_flags & IOMMU_FLUSH_CACHE) && 628 (buflen <= 0 || (slot % 8) == 7)) 629 IOMMUREG_WRITE(is, iommu_cache_flush, 630 is->is_ptsb + slot * 8); 631 632 dvmaddr += PAGE_SIZE; 633 } 634 if (needsflush) 635 iommu_strbuf_flush_done(sb); 636 #ifdef DIAGNOSTIC 637 for (seg = 0; seg < map->dm_nsegs; seg++) { 638 if (map->dm_segs[seg].ds_addr < is->is_dvmabase || 639 map->dm_segs[seg].ds_addr > is->is_dvmaend) { 640 printf("seg %d dvmaddr %lx out of range %x - %x\n", 641 seg, (long)map->dm_segs[seg].ds_addr, 642 is->is_dvmabase, is->is_dvmaend); 643 #ifdef DDB 644 Debugger(); 645 #endif 646 } 647 } 648 #endif 649 return (0); 650 } 651 652 653 void 654 iommu_dvmamap_unload(bus_dma_tag_t t, bus_dmamap_t map) 655 { 656 struct strbuf_ctl *sb = (struct strbuf_ctl *)map->_dm_cookie; 657 struct iommu_state *is = sb->sb_is; 658 int error, s; 659 bus_size_t sgsize = map->_dm_dvmasize; 660 661 /* Flush the iommu */ 662 #ifdef DEBUG 663 if (!map->_dm_dvmastart) { 664 printf("iommu_dvmamap_unload: No dvmastart is zero\n"); 665 #ifdef DDB 666 Debugger(); 667 #endif 668 } 669 #endif 670 iommu_remove(is, map->_dm_dvmastart, map->_dm_dvmasize); 671 672 /* Flush the caches */ 673 bus_dmamap_unload(t->_parent, map); 674 675 s = splhigh(); 676 error = extent_free(is->is_dvmamap, map->_dm_dvmastart, 677 map->_dm_dvmasize, EX_NOWAIT); 678 map->_dm_dvmastart = 0; 679 map->_dm_dvmasize = 0; 680 splx(s); 681 if (error != 0) 682 printf("warning: %s: %" PRId64 " of DVMA space lost\n", 683 __func__, sgsize); 684 685 /* Clear the map */ 686 } 687 688 689 int 690 iommu_dvmamap_load_raw(bus_dma_tag_t t, bus_dmamap_t map, 691 bus_dma_segment_t *segs, int nsegs, bus_size_t size, int flags) 692 { 693 struct strbuf_ctl *sb = (struct strbuf_ctl *)map->_dm_cookie; 694 struct iommu_state *is = sb->sb_is; 695 struct vm_page *pg; 696 int i, j, s; 697 int left; 698 int err, needsflush; 699 bus_size_t sgsize; 700 paddr_t pa; 701 bus_size_t boundary, align; 702 u_long dvmaddr, sgstart, sgend, bmask; 703 struct pglist *pglist; 704 const int pagesz = PAGE_SIZE; 705 int slot; 706 #ifdef DEBUG 707 int npg = 0; 708 #endif 709 710 if (map->dm_nsegs) { 711 /* Already in use?? */ 712 #ifdef DIAGNOSTIC 713 printf("iommu_dvmamap_load_raw: map still in use\n"); 714 #endif 715 bus_dmamap_unload(t, map); 716 } 717 718 /* 719 * A boundary presented to bus_dmamem_alloc() takes precedence 720 * over boundary in the map. 721 */ 722 if ((boundary = segs[0]._ds_boundary) == 0) 723 boundary = map->_dm_boundary; 724 725 align = max(segs[0]._ds_align, pagesz); 726 727 /* 728 * Make sure that on error condition we return "no valid mappings". 729 */ 730 map->dm_nsegs = 0; 731 /* Count up the total number of pages we need */ 732 pa = trunc_page(segs[0].ds_addr); 733 sgsize = 0; 734 left = size; 735 for (i = 0; left > 0 && i < nsegs; i++) { 736 if (round_page(pa) != round_page(segs[i].ds_addr)) 737 sgsize = round_page(sgsize) + 738 (segs[i].ds_addr & PGOFSET); 739 sgsize += min(left, segs[i].ds_len); 740 left -= segs[i].ds_len; 741 pa = segs[i].ds_addr + segs[i].ds_len; 742 } 743 sgsize = round_page(sgsize); 744 745 s = splhigh(); 746 /* 747 * If our segment size is larger than the boundary we need to 748 * split the transfer up into little pieces ourselves. 749 */ 750 err = extent_alloc(is->is_dvmamap, sgsize, align, 751 (sgsize > boundary) ? 0 : boundary, 752 ((flags & BUS_DMA_NOWAIT) == 0 ? EX_WAITOK : EX_NOWAIT) | 753 EX_BOUNDZERO, &dvmaddr); 754 splx(s); 755 756 if (err != 0) 757 return (err); 758 759 #ifdef DEBUG 760 if (dvmaddr == (u_long)-1) 761 { 762 printf("iommu_dvmamap_load_raw(): extent_alloc(%d, %x) failed!\n", 763 (int)sgsize, flags); 764 #ifdef DDB 765 Debugger(); 766 #endif 767 } 768 #endif 769 if (dvmaddr == (u_long)-1) 770 return (ENOMEM); 771 772 /* Set the active DVMA map */ 773 map->_dm_dvmastart = dvmaddr; 774 map->_dm_dvmasize = sgsize; 775 776 bmask = ~(boundary - 1); 777 if ((pglist = segs[0]._ds_mlist) == NULL) { 778 u_long prev_va = 0UL, last_va = dvmaddr; 779 paddr_t prev_pa = 0; 780 int end = 0, offset; 781 bus_size_t len = size; 782 783 /* 784 * This segs is made up of individual physical 785 * segments, probably by _bus_dmamap_load_uio() or 786 * _bus_dmamap_load_mbuf(). Ignore the mlist and 787 * load each one individually. 788 */ 789 j = 0; 790 needsflush = 0; 791 for (i = 0; i < nsegs ; i++) { 792 793 pa = segs[i].ds_addr; 794 offset = (pa & PGOFSET); 795 pa = trunc_page(pa); 796 dvmaddr = trunc_page(dvmaddr); 797 left = min(len, segs[i].ds_len); 798 799 DPRINTF(IDB_INFO, ("iommu_dvmamap_load_raw: converting " 800 "physseg %d start %lx size %lx\n", i, 801 (long)segs[i].ds_addr, (long)segs[i].ds_len)); 802 803 if ((pa == prev_pa) && 804 ((offset != 0) || (end != offset))) { 805 /* We can re-use this mapping */ 806 dvmaddr = prev_va; 807 } 808 809 sgstart = dvmaddr + offset; 810 sgend = sgstart + left - 1; 811 812 /* Are the segments virtually adjacent? */ 813 if ((j > 0) && (end == offset) && 814 ((offset == 0) || (pa == prev_pa)) && 815 (map->dm_segs[j-1].ds_len + left <= 816 map->dm_maxsegsz)) { 817 /* Just append to the previous segment. */ 818 map->dm_segs[--j].ds_len += left; 819 /* Restore sgstart for boundary check */ 820 sgstart = map->dm_segs[j].ds_addr; 821 DPRINTF(IDB_INFO, ("iommu_dvmamap_load_raw: " 822 "appending seg %d start %lx size %lx\n", j, 823 (long)map->dm_segs[j].ds_addr, 824 (long)map->dm_segs[j].ds_len)); 825 } else { 826 if (j >= map->_dm_segcnt) { 827 iommu_remove(is, map->_dm_dvmastart, 828 last_va - map->_dm_dvmastart); 829 goto fail; 830 } 831 map->dm_segs[j].ds_addr = sgstart; 832 map->dm_segs[j].ds_len = left; 833 DPRINTF(IDB_INFO, ("iommu_dvmamap_load_raw: " 834 "seg %d start %lx size %lx\n", j, 835 (long)map->dm_segs[j].ds_addr, 836 (long)map->dm_segs[j].ds_len)); 837 } 838 end = (offset + left) & PGOFSET; 839 840 /* Check for boundary issues */ 841 while ((sgstart & bmask) != (sgend & bmask)) { 842 /* Need a new segment. */ 843 map->dm_segs[j].ds_len = 844 boundary - (sgstart & (boundary - 1)); 845 DPRINTF(IDB_INFO, ("iommu_dvmamap_load_raw: " 846 "seg %d start %lx size %lx\n", j, 847 (long)map->dm_segs[j].ds_addr, 848 (long)map->dm_segs[j].ds_len)); 849 if (++j >= map->_dm_segcnt) { 850 iommu_remove(is, map->_dm_dvmastart, 851 last_va - map->_dm_dvmastart); 852 goto fail; 853 } 854 sgstart += map->dm_segs[j-1].ds_len; 855 map->dm_segs[j].ds_addr = sgstart; 856 map->dm_segs[j].ds_len = sgend - sgstart + 1; 857 } 858 859 if (sgsize == 0) 860 panic("iommu_dmamap_load_raw: size botch"); 861 862 /* Now map a series of pages. */ 863 while (dvmaddr <= sgend) { 864 DPRINTF(IDB_BUSDMA, 865 ("iommu_dvmamap_load_raw: map %p " 866 "loading va %lx at pa %lx\n", 867 map, (long)dvmaddr, 868 (long)(pa))); 869 /* Enter it if we haven't before. */ 870 if (prev_va != dvmaddr) { 871 iommu_enter(sb, prev_va = dvmaddr, 872 prev_pa = pa, 873 flags | IOTTE_DEBUG(++npg << 12)); 874 needsflush = 1; 875 876 /* Flush cache if necessary. */ 877 slot = IOTSBSLOT(trunc_page(dvmaddr), is->is_tsbsize); 878 if ((is->is_flags & IOMMU_FLUSH_CACHE) && 879 ((dvmaddr + pagesz) > sgend || (slot % 8) == 7)) 880 IOMMUREG_WRITE(is, iommu_cache_flush, 881 is->is_ptsb + slot * 8); 882 } 883 884 dvmaddr += pagesz; 885 pa += pagesz; 886 last_va = dvmaddr; 887 } 888 889 len -= left; 890 ++j; 891 } 892 if (needsflush) 893 iommu_strbuf_flush_done(sb); 894 895 map->dm_mapsize = size; 896 map->dm_nsegs = j; 897 #ifdef DIAGNOSTIC 898 { int seg; 899 for (seg = 0; seg < map->dm_nsegs; seg++) { 900 if (map->dm_segs[seg].ds_addr < is->is_dvmabase || 901 map->dm_segs[seg].ds_addr > is->is_dvmaend) { 902 printf("seg %d dvmaddr %lx out of range %x - %x\n", 903 seg, (long)map->dm_segs[seg].ds_addr, 904 is->is_dvmabase, is->is_dvmaend); 905 #ifdef DDB 906 Debugger(); 907 #endif 908 } 909 } 910 } 911 #endif 912 return (0); 913 } 914 915 /* 916 * This was allocated with bus_dmamem_alloc. 917 * The pages are on a `pglist'. 918 */ 919 i = 0; 920 sgstart = dvmaddr; 921 sgend = sgstart + size - 1; 922 map->dm_segs[i].ds_addr = sgstart; 923 while ((sgstart & bmask) != (sgend & bmask)) { 924 /* Oops. We crossed a boundary. Split the xfer. */ 925 map->dm_segs[i].ds_len = boundary - (sgstart & (boundary - 1)); 926 DPRINTF(IDB_INFO, ("iommu_dvmamap_load_raw: " 927 "seg %d start %lx size %lx\n", i, 928 (long)map->dm_segs[i].ds_addr, 929 (long)map->dm_segs[i].ds_len)); 930 if (++i >= map->_dm_segcnt) { 931 /* Too many segments. Fail the operation. */ 932 goto fail; 933 } 934 sgstart += map->dm_segs[i-1].ds_len; 935 map->dm_segs[i].ds_addr = sgstart; 936 } 937 DPRINTF(IDB_INFO, ("iommu_dvmamap_load_raw: " 938 "seg %d start %lx size %lx\n", i, 939 (long)map->dm_segs[i].ds_addr, (long)map->dm_segs[i].ds_len)); 940 map->dm_segs[i].ds_len = sgend - sgstart + 1; 941 942 needsflush = 0; 943 TAILQ_FOREACH(pg, pglist, pageq.queue) { 944 if (sgsize == 0) 945 panic("iommu_dmamap_load_raw: size botch"); 946 pa = VM_PAGE_TO_PHYS(pg); 947 948 DPRINTF(IDB_BUSDMA, 949 ("iommu_dvmamap_load_raw: map %p loading va %lx at pa %lx\n", 950 map, (long)dvmaddr, (long)(pa))); 951 iommu_enter(sb, dvmaddr, pa, flags | IOTTE_DEBUG(0x8000)); 952 needsflush = 1; 953 954 sgsize -= pagesz; 955 956 /* Flush cache if necessary. */ 957 slot = IOTSBSLOT(trunc_page(dvmaddr), is->is_tsbsize); 958 if ((is->is_flags & IOMMU_FLUSH_CACHE) && 959 (sgsize == 0 || (slot % 8) == 7)) 960 IOMMUREG_WRITE(is, iommu_cache_flush, 961 is->is_ptsb + slot * 8); 962 963 dvmaddr += pagesz; 964 } 965 if (needsflush) 966 iommu_strbuf_flush_done(sb); 967 map->dm_mapsize = size; 968 map->dm_nsegs = i+1; 969 #ifdef DIAGNOSTIC 970 { int seg; 971 for (seg = 0; seg < map->dm_nsegs; seg++) { 972 if (map->dm_segs[seg].ds_addr < is->is_dvmabase || 973 map->dm_segs[seg].ds_addr > is->is_dvmaend) { 974 printf("seg %d dvmaddr %lx out of range %x - %x\n", 975 seg, (long)map->dm_segs[seg].ds_addr, 976 is->is_dvmabase, is->is_dvmaend); 977 #ifdef DDB 978 Debugger(); 979 #endif 980 } 981 } 982 } 983 #endif 984 return (0); 985 986 fail: 987 s = splhigh(); 988 err = extent_free(is->is_dvmamap, map->_dm_dvmastart, sgsize, 989 EX_NOWAIT); 990 map->_dm_dvmastart = 0; 991 map->_dm_dvmasize = 0; 992 splx(s); 993 if (err != 0) 994 printf("warning: %s: %" PRId64 " of DVMA space lost\n", 995 __func__, sgsize); 996 return (EFBIG); 997 } 998 999 1000 /* 1001 * Flush an individual dma segment, returns non-zero if the streaming buffers 1002 * need flushing afterwards. 1003 */ 1004 static int 1005 iommu_dvmamap_sync_range(struct strbuf_ctl *sb, vaddr_t va, bus_size_t len) 1006 { 1007 vaddr_t vaend; 1008 struct iommu_state *is = sb->sb_is; 1009 1010 #ifdef DIAGNOSTIC 1011 if (va < is->is_dvmabase || va > is->is_dvmaend) 1012 panic("invalid va: %llx", (long long)va); 1013 #endif 1014 1015 if ((is->is_tsb[IOTSBSLOT(va, is->is_tsbsize)] & IOTTE_STREAM) == 0) { 1016 DPRINTF(IDB_BUSDMA, 1017 ("iommu_dvmamap_sync_range: attempting to flush " 1018 "non-streaming entry\n")); 1019 return (0); 1020 } 1021 1022 vaend = round_page(va + len) - 1; 1023 va = trunc_page(va); 1024 1025 #ifdef DIAGNOSTIC 1026 if (va < is->is_dvmabase || vaend > is->is_dvmaend) 1027 panic("invalid va range: %llx to %llx (%x to %x)", 1028 (long long)va, (long long)vaend, 1029 is->is_dvmabase, 1030 is->is_dvmaend); 1031 #endif 1032 1033 for ( ; va <= vaend; va += PAGE_SIZE) { 1034 DPRINTF(IDB_BUSDMA, 1035 ("iommu_dvmamap_sync_range: flushing va %p\n", 1036 (void *)(u_long)va)); 1037 iommu_strbuf_flush(sb, va); 1038 } 1039 1040 return (1); 1041 } 1042 1043 static void 1044 _iommu_dvmamap_sync(bus_dma_tag_t t, bus_dmamap_t map, bus_addr_t offset, 1045 bus_size_t len, int ops) 1046 { 1047 struct strbuf_ctl *sb = (struct strbuf_ctl *)map->_dm_cookie; 1048 bus_size_t count; 1049 int i, needsflush = 0; 1050 1051 if (!sb->sb_flush) 1052 return; 1053 1054 for (i = 0; i < map->dm_nsegs; i++) { 1055 if (offset < map->dm_segs[i].ds_len) 1056 break; 1057 offset -= map->dm_segs[i].ds_len; 1058 } 1059 1060 if (i == map->dm_nsegs) 1061 panic("iommu_dvmamap_sync: segment too short %llu", 1062 (unsigned long long)offset); 1063 1064 if (ops & (BUS_DMASYNC_PREREAD | BUS_DMASYNC_POSTWRITE)) { 1065 /* Nothing to do */; 1066 } 1067 1068 if (ops & (BUS_DMASYNC_POSTREAD | BUS_DMASYNC_PREWRITE)) { 1069 1070 for (; len > 0 && i < map->dm_nsegs; i++) { 1071 count = MIN(map->dm_segs[i].ds_len - offset, len); 1072 if (count > 0 && 1073 iommu_dvmamap_sync_range(sb, 1074 map->dm_segs[i].ds_addr + offset, count)) 1075 needsflush = 1; 1076 offset = 0; 1077 len -= count; 1078 } 1079 #ifdef DIAGNOSTIC 1080 if (i == map->dm_nsegs && len > 0) 1081 panic("iommu_dvmamap_sync: leftover %llu", 1082 (unsigned long long)len); 1083 #endif 1084 1085 if (needsflush) 1086 iommu_strbuf_flush_done(sb); 1087 } 1088 } 1089 1090 void 1091 iommu_dvmamap_sync(bus_dma_tag_t t, bus_dmamap_t map, bus_addr_t offset, 1092 bus_size_t len, int ops) 1093 { 1094 1095 /* If len is 0, then there is nothing to do */ 1096 if (len == 0) 1097 return; 1098 1099 if (ops & (BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE)) { 1100 /* Flush the CPU then the IOMMU */ 1101 bus_dmamap_sync(t->_parent, map, offset, len, ops); 1102 _iommu_dvmamap_sync(t, map, offset, len, ops); 1103 } 1104 if (ops & (BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE)) { 1105 /* Flush the IOMMU then the CPU */ 1106 _iommu_dvmamap_sync(t, map, offset, len, ops); 1107 bus_dmamap_sync(t->_parent, map, offset, len, ops); 1108 } 1109 } 1110 1111 int 1112 iommu_dvmamem_alloc(bus_dma_tag_t t, bus_size_t size, bus_size_t alignment, 1113 bus_size_t boundary, bus_dma_segment_t *segs, int nsegs, int *rsegs, 1114 int flags) 1115 { 1116 1117 DPRINTF(IDB_BUSDMA, ("iommu_dvmamem_alloc: sz %llx align %llx bound %llx " 1118 "segp %p flags %d\n", (unsigned long long)size, 1119 (unsigned long long)alignment, (unsigned long long)boundary, 1120 segs, flags)); 1121 return (bus_dmamem_alloc(t->_parent, size, alignment, boundary, 1122 segs, nsegs, rsegs, flags|BUS_DMA_DVMA)); 1123 } 1124 1125 void 1126 iommu_dvmamem_free(bus_dma_tag_t t, bus_dma_segment_t *segs, int nsegs) 1127 { 1128 1129 DPRINTF(IDB_BUSDMA, ("iommu_dvmamem_free: segp %p nsegs %d\n", 1130 segs, nsegs)); 1131 bus_dmamem_free(t->_parent, segs, nsegs); 1132 } 1133 1134 /* 1135 * Map the DVMA mappings into the kernel pmap. 1136 * Check the flags to see whether we're streaming or coherent. 1137 */ 1138 int 1139 iommu_dvmamem_map(bus_dma_tag_t t, bus_dma_segment_t *segs, int nsegs, 1140 size_t size, void **kvap, int flags) 1141 { 1142 struct vm_page *pg; 1143 vaddr_t va; 1144 bus_addr_t addr; 1145 struct pglist *pglist; 1146 int cbit; 1147 const uvm_flag_t kmflags = 1148 (flags & BUS_DMA_NOWAIT) != 0 ? UVM_KMF_NOWAIT : 0; 1149 1150 DPRINTF(IDB_BUSDMA, ("iommu_dvmamem_map: segp %p nsegs %d size %lx\n", 1151 segs, nsegs, size)); 1152 1153 /* 1154 * Allocate some space in the kernel map, and then map these pages 1155 * into this space. 1156 */ 1157 size = round_page(size); 1158 va = uvm_km_alloc(kernel_map, size, 0, UVM_KMF_VAONLY | kmflags); 1159 if (va == 0) 1160 return (ENOMEM); 1161 1162 *kvap = (void *)va; 1163 1164 /* 1165 * digest flags: 1166 */ 1167 cbit = 0; 1168 if (flags & BUS_DMA_COHERENT) /* Disable vcache */ 1169 cbit |= PMAP_NVC; 1170 if (flags & BUS_DMA_NOCACHE) /* side effects */ 1171 cbit |= PMAP_NC; 1172 1173 /* 1174 * Now take this and map it into the CPU. 1175 */ 1176 pglist = segs[0]._ds_mlist; 1177 TAILQ_FOREACH(pg, pglist, pageq.queue) { 1178 #ifdef DIAGNOSTIC 1179 if (size == 0) 1180 panic("iommu_dvmamem_map: size botch"); 1181 #endif 1182 addr = VM_PAGE_TO_PHYS(pg); 1183 DPRINTF(IDB_BUSDMA, ("iommu_dvmamem_map: " 1184 "mapping va %lx at %llx\n", va, (unsigned long long)addr | cbit)); 1185 pmap_kenter_pa(va, addr | cbit, 1186 VM_PROT_READ | VM_PROT_WRITE, 0); 1187 va += PAGE_SIZE; 1188 size -= PAGE_SIZE; 1189 } 1190 pmap_update(pmap_kernel()); 1191 return (0); 1192 } 1193 1194 /* 1195 * Unmap DVMA mappings from kernel 1196 */ 1197 void 1198 iommu_dvmamem_unmap(bus_dma_tag_t t, void *kva, size_t size) 1199 { 1200 1201 DPRINTF(IDB_BUSDMA, ("iommu_dvmamem_unmap: kvm %p size %lx\n", 1202 kva, size)); 1203 1204 #ifdef DIAGNOSTIC 1205 if ((u_long)kva & PGOFSET) 1206 panic("iommu_dvmamem_unmap"); 1207 #endif 1208 1209 size = round_page(size); 1210 pmap_kremove((vaddr_t)kva, size); 1211 pmap_update(pmap_kernel()); 1212 uvm_km_free(kernel_map, (vaddr_t)kva, size, UVM_KMF_VAONLY); 1213 } 1214
Indexes created Fri Sep 26 20:09:58 GMT 2025