iommu.c revision 1.118
1 /* $NetBSD: iommu.c,v 1.118 2023/12/08 17:19:11 thorpej Exp $ */ 2 3 /* 4 * Copyright (c) 1999, 2000 Matthew R. Green 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 17 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 18 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 19 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 20 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 21 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 22 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 23 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 24 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 * SUCH DAMAGE. 27 */ 28 29 /* 30 * Copyright (c) 2001, 2002 Eduardo Horvath 31 * All rights reserved. 32 * 33 * Redistribution and use in source and binary forms, with or without 34 * modification, are permitted provided that the following conditions 35 * are met: 36 * 1. Redistributions of source code must retain the above copyright 37 * notice, this list of conditions and the following disclaimer. 38 * 2. Redistributions in binary form must reproduce the above copyright 39 * notice, this list of conditions and the following disclaimer in the 40 * documentation and/or other materials provided with the distribution. 41 * 3. The name of the author may not be used to endorse or promote products 42 * derived from this software without specific prior written permission. 43 * 44 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 45 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 46 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 47 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 48 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 49 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 50 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 51 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 52 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 53 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 54 * SUCH DAMAGE. 55 */ 56 57 /* 58 * UltraSPARC IOMMU support; used by both the sbus and pci code. 59 */ 60 61 #include <sys/cdefs.h> 62 __KERNEL_RCSID(0, "$NetBSD: iommu.c,v 1.118 2023/12/08 17:19:11 thorpej Exp $"); 63 64 #include "opt_ddb.h" 65 66 #include <sys/param.h> 67 #include <sys/extent.h> 68 #include <sys/malloc.h> 69 #include <sys/systm.h> 70 #include <sys/device.h> 71 #include <sys/proc.h> 72 73 #include <uvm/uvm.h> 74 75 #include <sys/bus.h> 76 #include <sparc64/dev/iommureg.h> 77 #include <sparc64/dev/iommuvar.h> 78 79 #include <machine/autoconf.h> 80 #include <machine/cpu.h> 81 #include <machine/hypervisor.h> 82 83 #ifdef DEBUG 84 #define IDB_BUSDMA 0x1 85 #define IDB_IOMMU 0x2 86 #define IDB_INFO 0x4 87 #define IDB_SYNC 0x8 88 int iommudebug = 0x0; 89 #define DPRINTF(l, s) do { if (iommudebug & l) printf s; } while (0) 90 #define IOTTE_DEBUG(n) (n) 91 #else 92 #define DPRINTF(l, s) 93 #define IOTTE_DEBUG(n) 0 94 #endif 95 96 #define iommu_strbuf_flush(i, v) do { \ 97 if ((i)->sb_flush) \ 98 bus_space_write_8((i)->sb_is->is_bustag, (i)->sb_sb, \ 99 STRBUFREG(strbuf_pgflush), (v)); \ 100 } while (0) 101 102 static int iommu_strbuf_flush_done(struct strbuf_ctl *); 103 static void _iommu_dvmamap_sync(bus_dma_tag_t, bus_dmamap_t, bus_addr_t, 104 bus_size_t, int); 105 static void iommu_enter_sun4u(struct strbuf_ctl *sb, vaddr_t va, int64_t pa, int flags); 106 static void iommu_enter_sun4v(struct strbuf_ctl *sb, vaddr_t va, int64_t pa, int flags); 107 static void iommu_remove_sun4u(struct iommu_state *is, vaddr_t va, size_t len); 108 static void iommu_remove_sun4v(struct iommu_state *is, vaddr_t va, size_t len); 109 110 /* 111 * initialise the UltraSPARC IOMMU (SBUS or PCI): 112 * - allocate and setup the iotsb. 113 * - enable the IOMMU 114 * - initialise the streaming buffers (if they exist) 115 * - create a private DVMA map. 116 */ 117 void 118 iommu_init(char *name, struct iommu_state *is, int tsbsize, uint32_t iovabase) 119 { 120 psize_t size; 121 vaddr_t va; 122 paddr_t pa; 123 struct vm_page *pg; 124 struct pglist pglist; 125 126 DPRINTF(IDB_INFO, ("iommu_init: tsbsize %x iovabase %x\n", tsbsize, iovabase)); 127 128 /* 129 * Setup the iommu. 130 * 131 * The sun4u iommu is part of the SBUS or PCI controller so we will 132 * deal with it here.. 133 * 134 * For sysio and psycho/psycho+ the IOMMU address space always ends at 135 * 0xffffe000, but the starting address depends on the size of the 136 * map. The map size is 1024 * 2 ^ is->is_tsbsize entries, where each 137 * entry is 8 bytes. The start of the map can be calculated by 138 * (0xffffe000 << (8 + is->is_tsbsize)). 139 * 140 * But sabre and hummingbird use a different scheme that seems to 141 * be hard-wired, so we read the start and size from the PROM and 142 * just use those values. 143 */ 144 if (strncmp(name, "pyro", 4) == 0) { 145 is->is_cr = IOMMUREG_READ(is, iommu_cr); 146 is->is_cr &= ~IOMMUCR_FIRE_BE; 147 is->is_cr |= (IOMMUCR_FIRE_SE | IOMMUCR_FIRE_CM_EN | 148 IOMMUCR_FIRE_TE); 149 } else 150 is->is_cr = IOMMUCR_EN; 151 is->is_tsbsize = tsbsize; 152 if (iovabase == -1) { 153 is->is_dvmabase = IOTSB_VSTART(is->is_tsbsize); 154 is->is_dvmaend = IOTSB_VEND - 1; 155 } else { 156 is->is_dvmabase = iovabase; 157 is->is_dvmaend = iovabase + IOTSB_VSIZE(tsbsize) - 1; 158 } 159 160 /* 161 * Allocate memory for I/O pagetables. They need to be physically 162 * contiguous. 163 */ 164 165 size = PAGE_SIZE << is->is_tsbsize; 166 if (uvm_pglistalloc((psize_t)size, (paddr_t)0, (paddr_t)-1, 167 (paddr_t)PAGE_SIZE, (paddr_t)0, &pglist, 1, 0) != 0) 168 panic("iommu_init: no memory"); 169 170 va = uvm_km_alloc(kernel_map, size, 0, UVM_KMF_VAONLY); 171 if (va == 0) 172 panic("iommu_init: no memory"); 173 is->is_tsb = (int64_t *)va; 174 175 is->is_ptsb = VM_PAGE_TO_PHYS(TAILQ_FIRST(&pglist)); 176 177 /* Map the pages */ 178 TAILQ_FOREACH(pg, &pglist, pageq.queue) { 179 pa = VM_PAGE_TO_PHYS(pg); 180 pmap_kenter_pa(va, pa | PMAP_NVC, 181 VM_PROT_READ | VM_PROT_WRITE, 0); 182 va += PAGE_SIZE; 183 } 184 pmap_update(pmap_kernel()); 185 memset(is->is_tsb, 0, size); 186 187 #ifdef DEBUG 188 if (iommudebug & IDB_INFO) 189 { 190 /* Probe the iommu */ 191 if (!CPU_ISSUN4V) { 192 printf("iommu cr=%llx tsb=%llx\n", 193 (unsigned long long)bus_space_read_8(is->is_bustag, 194 is->is_iommu, 195 offsetof(struct iommureg, iommu_cr)), 196 (unsigned long long)bus_space_read_8(is->is_bustag, 197 is->is_iommu, 198 offsetof(struct iommureg, iommu_tsb))); 199 printf("TSB base %p phys %llx\n", (void *)is->is_tsb, 200 (unsigned long long)is->is_ptsb); 201 delay(1000000); /* 1 s */ 202 } 203 } 204 #endif 205 206 /* 207 * Now all the hardware's working we need to allocate a dvma map. 208 */ 209 aprint_debug("DVMA map: %x to %x\n", 210 (unsigned int)is->is_dvmabase, 211 (unsigned int)is->is_dvmaend); 212 aprint_debug("IOTSB: %llx to %llx\n", 213 (unsigned long long)is->is_ptsb, 214 (unsigned long long)(is->is_ptsb + size - 1)); 215 is->is_dvmamap = vmem_create(name, 216 is->is_dvmabase, 217 (is->is_dvmaend + 1) - is->is_dvmabase, 218 PAGE_SIZE, /* quantum */ 219 NULL, /* importfn */ 220 NULL, /* releasefn */ 221 NULL, /* source */ 222 0, /* qcache_max */ 223 VM_SLEEP, 224 IPL_VM); 225 KASSERT(is->is_dvmamap != NULL); 226 227 /* 228 * Set the TSB size. The relevant bits were moved to the TSB 229 * base register in the PCIe host bridges. 230 */ 231 if (is->is_flags & IOMMU_TSBSIZE_IN_PTSB) 232 is->is_ptsb |= is->is_tsbsize; 233 else 234 is->is_cr |= (is->is_tsbsize << 16); 235 236 /* 237 * now actually start up the IOMMU 238 */ 239 iommu_reset(is); 240 } 241 242 /* 243 * Streaming buffers don't exist on the UltraSPARC IIi; we should have 244 * detected that already and disabled them. If not, we will notice that 245 * they aren't there when the STRBUF_EN bit does not remain. 246 */ 247 void 248 iommu_reset(struct iommu_state *is) 249 { 250 int i; 251 struct strbuf_ctl *sb; 252 253 if (CPU_ISSUN4V) 254 return; 255 256 IOMMUREG_WRITE(is, iommu_tsb, is->is_ptsb); 257 258 /* Enable IOMMU in diagnostic mode */ 259 IOMMUREG_WRITE(is, iommu_cr, is->is_cr|IOMMUCR_DE); 260 261 for (i = 0; i < 2; i++) { 262 if ((sb = is->is_sb[i])) { 263 264 /* Enable diagnostics mode? */ 265 bus_space_write_8(is->is_bustag, is->is_sb[i]->sb_sb, 266 STRBUFREG(strbuf_ctl), STRBUF_EN); 267 268 membar_Lookaside(); 269 270 /* No streaming buffers? Disable them */ 271 if (bus_space_read_8(is->is_bustag, 272 is->is_sb[i]->sb_sb, 273 STRBUFREG(strbuf_ctl)) == 0) { 274 is->is_sb[i]->sb_flush = NULL; 275 } else { 276 277 /* 278 * locate the pa of the flush buffer. 279 */ 280 if (pmap_extract(pmap_kernel(), 281 (vaddr_t)is->is_sb[i]->sb_flush, 282 &is->is_sb[i]->sb_flushpa) == FALSE) 283 is->is_sb[i]->sb_flush = NULL; 284 } 285 } 286 } 287 288 if (is->is_flags & IOMMU_FLUSH_CACHE) 289 IOMMUREG_WRITE(is, iommu_cache_invalidate, -1ULL); 290 } 291 292 /* 293 * Here are the iommu control routines. 294 */ 295 296 static void 297 iommu_enter(struct strbuf_ctl *sb, vaddr_t va, int64_t pa, int flags) 298 { 299 DPRINTF(IDB_IOMMU, ("iommu_enter: va %lx pa %lx flags %x\n", 300 va, (long)pa, flags)); 301 if (!CPU_ISSUN4V) 302 iommu_enter_sun4u(sb, va, pa, flags); 303 else 304 iommu_enter_sun4v(sb, va, pa, flags); 305 } 306 307 308 void 309 iommu_enter_sun4u(struct strbuf_ctl *sb, vaddr_t va, int64_t pa, int flags) 310 { 311 struct iommu_state *is = sb->sb_is; 312 int strbuf = (flags & BUS_DMA_STREAMING); 313 int64_t tte; 314 315 #ifdef DIAGNOSTIC 316 if (va < is->is_dvmabase || va > is->is_dvmaend) 317 panic("iommu_enter: va %#lx not in DVMA space", va); 318 #endif 319 320 /* Is the streamcache flush really needed? */ 321 if (sb->sb_flush) 322 iommu_strbuf_flush(sb, va); 323 else 324 /* If we can't flush the strbuf don't enable it. */ 325 strbuf = 0; 326 327 tte = MAKEIOTTE(pa, !(flags & BUS_DMA_NOWRITE), 328 !(flags & BUS_DMA_NOCACHE), (strbuf)); 329 #ifdef DEBUG 330 tte |= (flags & 0xff000LL)<<(4*8); 331 #endif 332 333 is->is_tsb[IOTSBSLOT(va,is->is_tsbsize)] = tte; 334 bus_space_write_8(is->is_bustag, is->is_iommu, 335 IOMMUREG(iommu_flush), va); 336 DPRINTF(IDB_IOMMU, ("iommu_enter: slot %d va %lx pa %lx " 337 "TSB[%lx]@%p=%lx\n", (int)IOTSBSLOT(va,is->is_tsbsize), 338 va, (long)pa, (u_long)IOTSBSLOT(va,is->is_tsbsize), 339 (void *)(u_long)&is->is_tsb[IOTSBSLOT(va,is->is_tsbsize)], 340 (u_long)tte)); 341 } 342 343 void 344 iommu_enter_sun4v(struct strbuf_ctl *sb, vaddr_t va, int64_t pa, int flags) 345 { 346 struct iommu_state *is = sb->sb_is; 347 u_int64_t tsbid = IOTSBSLOT(va, is->is_tsbsize); 348 paddr_t page_list[1], addr; 349 u_int64_t attr, nmapped; 350 int err; 351 352 #ifdef DIAGNOSTIC 353 if (va < is->is_dvmabase || (va + PAGE_MASK) > is->is_dvmaend) 354 panic("viommu_enter: va %#lx not in DVMA space", va); 355 #endif 356 357 attr = PCI_MAP_ATTR_READ | PCI_MAP_ATTR_WRITE; 358 if (flags & BUS_DMA_READ) 359 attr &= ~PCI_MAP_ATTR_READ; 360 if (flags & BUS_DMA_WRITE) 361 attr &= ~PCI_MAP_ATTR_WRITE; 362 363 page_list[0] = trunc_page(pa); 364 if (!pmap_extract(pmap_kernel(), (vaddr_t)page_list, &addr)) 365 panic("viommu_enter: pmap_extract failed"); 366 err = hv_pci_iommu_map(is->is_devhandle, tsbid, 1, attr, 367 addr, &nmapped); 368 if (err != H_EOK || nmapped != 1) 369 panic("hv_pci_iommu_map: err=%d, nmapped=%lu", err, (long unsigned int)nmapped); 370 } 371 372 /* 373 * Find the value of a DVMA address (debug routine). 374 */ 375 paddr_t 376 iommu_extract(struct iommu_state *is, vaddr_t dva) 377 { 378 int64_t tte = 0; 379 380 if (dva >= is->is_dvmabase && dva <= is->is_dvmaend) 381 tte = is->is_tsb[IOTSBSLOT(dva, is->is_tsbsize)]; 382 383 if ((tte & IOTTE_V) == 0) 384 return ((paddr_t)-1L); 385 return (tte & IOTTE_PAMASK); 386 } 387 388 /* 389 * iommu_remove: removes mappings created by iommu_enter 390 * 391 * Only demap from IOMMU if flag is set. 392 * 393 * XXX: this function needs better internal error checking. 394 */ 395 396 static void 397 iommu_remove(struct iommu_state *is, vaddr_t va, size_t len) 398 { 399 DPRINTF(IDB_IOMMU, ("iommu_remove: va %lx len %zu\n", va, len)); 400 if (!CPU_ISSUN4V) 401 iommu_remove_sun4u(is, va, len); 402 else 403 iommu_remove_sun4v(is, va, len); 404 } 405 406 void 407 iommu_remove_sun4u(struct iommu_state *is, vaddr_t va, size_t len) 408 { 409 410 int slot; 411 412 #ifdef DIAGNOSTIC 413 if (va < is->is_dvmabase || va > is->is_dvmaend) 414 panic("iommu_remove: va 0x%lx not in DVMA space", (u_long)va); 415 if ((long)(va + len) < (long)va) 416 panic("iommu_remove: va 0x%lx + len 0x%lx wraps", 417 (long) va, (long) len); 418 if (len & ~0xfffffff) 419 panic("iommu_remove: ridiculous len 0x%lx", (u_long)len); 420 #endif 421 422 va = trunc_page(va); 423 DPRINTF(IDB_IOMMU, ("iommu_remove: va %lx TSB[%lx]@%p\n", 424 va, (u_long)IOTSBSLOT(va, is->is_tsbsize), 425 &is->is_tsb[IOTSBSLOT(va, is->is_tsbsize)])); 426 while (len > 0) { 427 DPRINTF(IDB_IOMMU, ("iommu_remove: clearing TSB slot %d " 428 "for va %p size %lx\n", 429 (int)IOTSBSLOT(va,is->is_tsbsize), (void *)(u_long)va, 430 (u_long)len)); 431 if (len <= PAGE_SIZE) 432 len = 0; 433 else 434 len -= PAGE_SIZE; 435 436 #if 0 437 /* 438 * XXX Zero-ing the entry would not require RMW 439 * 440 * Disabling valid bit while a page is used by a device 441 * causes an uncorrectable DMA error. 442 * Workaround to avoid an uncorrectable DMA error is 443 * eliminating the next line, but the page is mapped 444 * until the next iommu_enter call. 445 */ 446 is->is_tsb[IOTSBSLOT(va,is->is_tsbsize)] &= ~IOTTE_V; 447 membar_StoreStore(); 448 #endif 449 IOMMUREG_WRITE(is, iommu_flush, va); 450 451 /* Flush cache if necessary. */ 452 slot = IOTSBSLOT(trunc_page(va), is->is_tsbsize); 453 if ((is->is_flags & IOMMU_FLUSH_CACHE) && 454 (len == 0 || (slot % 8) == 7)) 455 IOMMUREG_WRITE(is, iommu_cache_flush, 456 is->is_ptsb + slot * 8); 457 458 va += PAGE_SIZE; 459 } 460 } 461 462 void 463 iommu_remove_sun4v(struct iommu_state *is, vaddr_t va, size_t len) 464 { 465 u_int64_t tsbid = IOTSBSLOT(va, is->is_tsbsize); 466 u_int64_t ndemapped; 467 int err; 468 469 #ifdef DIAGNOSTIC 470 if (va < is->is_dvmabase || (va + PAGE_MASK) > is->is_dvmaend) 471 panic("iommu_remove: va 0x%lx not in DVMA space", (u_long)va); 472 if (va != trunc_page(va)) { 473 printf("iommu_remove: unaligned va: %lx\n", va); 474 va = trunc_page(va); 475 } 476 #endif 477 478 err = hv_pci_iommu_demap(is->is_devhandle, tsbid, 1, &ndemapped); 479 if (err != H_EOK || ndemapped != 1) 480 panic("hv_pci_iommu_unmap: err=%d", err); 481 } 482 483 static int 484 iommu_strbuf_flush_done(struct strbuf_ctl *sb) 485 { 486 struct iommu_state *is = sb->sb_is; 487 struct timeval cur, flushtimeout; 488 489 #define BUMPTIME(t, usec) { \ 490 register volatile struct timeval *tp = (t); \ 491 register long us; \ 492 \ 493 tp->tv_usec = us = tp->tv_usec + (usec); \ 494 if (us >= 1000000) { \ 495 tp->tv_usec = us - 1000000; \ 496 tp->tv_sec++; \ 497 } \ 498 } 499 500 if (!sb->sb_flush) 501 return (0); 502 503 /* 504 * Streaming buffer flushes: 505 * 506 * 1 Tell strbuf to flush by storing va to strbuf_pgflush. If 507 * we're not on a cache line boundary (64-bits): 508 * 2 Store 0 in flag 509 * 3 Store pointer to flag in flushsync 510 * 4 wait till flushsync becomes 0x1 511 * 512 * If it takes more than .5 sec, something 513 * went wrong. 514 */ 515 516 *sb->sb_flush = 0; 517 bus_space_write_8(is->is_bustag, sb->sb_sb, 518 STRBUFREG(strbuf_flushsync), sb->sb_flushpa); 519 520 microtime(&flushtimeout); 521 cur = flushtimeout; 522 BUMPTIME(&flushtimeout, 500000); /* 1/2 sec */ 523 524 DPRINTF(IDB_IOMMU, ("%s: flush = %lx at va = %lx pa = %lx now=" 525 "%"PRIx64":%"PRIx32" until = %"PRIx64":%"PRIx32"\n", __func__, 526 (long)*sb->sb_flush, (long)sb->sb_flush, (long)sb->sb_flushpa, 527 cur.tv_sec, cur.tv_usec, 528 flushtimeout.tv_sec, flushtimeout.tv_usec)); 529 530 /* Bypass non-coherent D$ */ 531 while ((!ldxa(sb->sb_flushpa, ASI_PHYS_CACHED)) && 532 timercmp(&cur, &flushtimeout, <=)) 533 microtime(&cur); 534 535 #ifdef DIAGNOSTIC 536 if (!ldxa(sb->sb_flushpa, ASI_PHYS_CACHED)) { 537 printf("%s: flush timeout %p, at %p\n", __func__, 538 (void *)(u_long)*sb->sb_flush, 539 (void *)(u_long)sb->sb_flushpa); /* panic? */ 540 #ifdef DDB 541 Debugger(); 542 #endif 543 } 544 #endif 545 DPRINTF(IDB_IOMMU, ("%s: flushed\n", __func__)); 546 return (*sb->sb_flush); 547 } 548 549 /* 550 * IOMMU DVMA operations, common to SBUS and PCI. 551 */ 552 int 553 iommu_dvmamap_load(bus_dma_tag_t t, bus_dmamap_t map, void *buf, 554 bus_size_t buflen, struct proc *p, int flags) 555 { 556 struct strbuf_ctl *sb = (struct strbuf_ctl *)map->_dm_cookie; 557 struct iommu_state *is = sb->sb_is; 558 int err, needsflush; 559 bus_size_t sgsize; 560 paddr_t curaddr; 561 u_long sgstart, sgend, bmask; 562 vmem_addr_t dvmaddr; 563 bus_size_t align, boundary, len; 564 vaddr_t vaddr = (vaddr_t)buf; 565 int seg; 566 struct pmap *pmap; 567 int slot; 568 569 if (map->dm_nsegs) { 570 /* Already in use?? */ 571 #ifdef DIAGNOSTIC 572 printf("iommu_dvmamap_load: map still in use\n"); 573 #endif 574 bus_dmamap_unload(t, map); 575 } 576 577 /* 578 * Make sure that on error condition we return "no valid mappings". 579 */ 580 map->dm_nsegs = 0; 581 KASSERT(map->dm_maxsegsz <= map->_dm_maxmaxsegsz); 582 583 if (buflen > map->_dm_size) { 584 DPRINTF(IDB_BUSDMA, 585 ("iommu_dvmamap_load(): error %d > %d -- " 586 "map size exceeded!\n", (int)buflen, (int)map->_dm_size)); 587 return (EINVAL); 588 } 589 590 sgsize = round_page(buflen + ((int)vaddr & PGOFSET)); 591 592 /* 593 * A boundary presented to bus_dmamem_alloc() takes precedence 594 * over boundary in the map. 595 */ 596 if ((boundary = (map->dm_segs[0]._ds_boundary)) == 0) 597 boundary = map->_dm_boundary; 598 align = uimax(map->dm_segs[0]._ds_align, PAGE_SIZE); 599 600 /* 601 * If our segment size is larger than the boundary we need to 602 * split the transfer up int little pieces ourselves. 603 */ 604 KASSERT(is->is_dvmamap != NULL); 605 err = vmem_xalloc(is->is_dvmamap, sgsize, 606 align, /* alignment */ 607 0, /* phase */ 608 (sgsize > boundary) ? 0 : boundary, 609 VMEM_ADDR_MIN, /* minaddr */ 610 VMEM_ADDR_MAX, /* maxaddr */ 611 VM_NOSLEEP | VM_BESTFIT, 612 &dvmaddr); 613 614 #ifdef DEBUG 615 if (err || (dvmaddr == (u_long)-1)) { 616 printf("iommu_dvmamap_load(): extent_alloc(%d, %x) failed!\n", 617 (int)sgsize, flags); 618 #ifdef DDB 619 Debugger(); 620 #endif 621 } 622 #endif 623 if (err != 0) 624 return (err); 625 626 if (dvmaddr == (u_long)-1) 627 return (ENOMEM); 628 629 /* Set the active DVMA map */ 630 map->_dm_dvmastart = dvmaddr; 631 map->_dm_dvmasize = sgsize; 632 633 /* 634 * Now split the DVMA range into segments, not crossing 635 * the boundary. 636 */ 637 seg = 0; 638 sgstart = dvmaddr + (vaddr & PGOFSET); 639 sgend = sgstart + buflen - 1; 640 map->dm_segs[seg].ds_addr = sgstart; 641 DPRINTF(IDB_INFO, ("iommu_dvmamap_load: boundary %lx boundary - 1 %lx " 642 "~(boundary - 1) %lx\n", (long)boundary, (long)(boundary - 1), 643 (long)~(boundary - 1))); 644 bmask = ~(boundary - 1); 645 while ((sgstart & bmask) != (sgend & bmask) || 646 sgend - sgstart + 1 > map->dm_maxsegsz) { 647 /* Oops. We crossed a boundary or large seg. Split the xfer. */ 648 len = map->dm_maxsegsz; 649 if ((sgstart & bmask) != (sgend & bmask)) 650 len = uimin(len, boundary - (sgstart & (boundary - 1))); 651 map->dm_segs[seg].ds_len = len; 652 DPRINTF(IDB_INFO, ("iommu_dvmamap_load: " 653 "seg %d start %lx size %lx\n", seg, 654 (long)map->dm_segs[seg].ds_addr, 655 (long)map->dm_segs[seg].ds_len)); 656 if (++seg >= map->_dm_segcnt) { 657 /* Too many segments. Fail the operation. */ 658 DPRINTF(IDB_INFO, ("iommu_dvmamap_load: " 659 "too many segments %d\n", seg)); 660 vmem_xfree(is->is_dvmamap, dvmaddr, sgsize); 661 map->_dm_dvmastart = 0; 662 map->_dm_dvmasize = 0; 663 return (EFBIG); 664 } 665 sgstart += len; 666 map->dm_segs[seg].ds_addr = sgstart; 667 } 668 map->dm_segs[seg].ds_len = sgend - sgstart + 1; 669 DPRINTF(IDB_INFO, ("iommu_dvmamap_load: " 670 "seg %d start %lx size %lx\n", seg, 671 (long)map->dm_segs[seg].ds_addr, (long)map->dm_segs[seg].ds_len)); 672 map->dm_nsegs = seg + 1; 673 map->dm_mapsize = buflen; 674 675 if (p != NULL) 676 pmap = p->p_vmspace->vm_map.pmap; 677 else 678 pmap = pmap_kernel(); 679 680 needsflush = 0; 681 for (; buflen > 0; ) { 682 683 /* 684 * Get the physical address for this page. 685 */ 686 if (pmap_extract(pmap, (vaddr_t)vaddr, &curaddr) == FALSE) { 687 #ifdef DIAGNOSTIC 688 printf("iommu_dvmamap_load: pmap_extract failed %lx\n", vaddr); 689 #endif 690 bus_dmamap_unload(t, map); 691 return (-1); 692 } 693 694 /* 695 * Compute the segment size, and adjust counts. 696 */ 697 sgsize = PAGE_SIZE - ((u_long)vaddr & PGOFSET); 698 if (buflen < sgsize) 699 sgsize = buflen; 700 701 DPRINTF(IDB_BUSDMA, 702 ("iommu_dvmamap_load: map %p loading va %p " 703 "dva %lx at pa %lx\n", 704 map, (void *)vaddr, (long)dvmaddr, 705 (long)trunc_page(curaddr))); 706 iommu_enter(sb, trunc_page(dvmaddr), trunc_page(curaddr), 707 flags | IOTTE_DEBUG(0x4000)); 708 needsflush = 1; 709 710 vaddr += sgsize; 711 buflen -= sgsize; 712 713 /* Flush cache if necessary. */ 714 slot = IOTSBSLOT(trunc_page(dvmaddr), is->is_tsbsize); 715 if ((is->is_flags & IOMMU_FLUSH_CACHE) && 716 (buflen <= 0 || (slot % 8) == 7)) 717 IOMMUREG_WRITE(is, iommu_cache_flush, 718 is->is_ptsb + slot * 8); 719 720 dvmaddr += PAGE_SIZE; 721 } 722 if (needsflush) 723 iommu_strbuf_flush_done(sb); 724 #ifdef DIAGNOSTIC 725 for (seg = 0; seg < map->dm_nsegs; seg++) { 726 if (map->dm_segs[seg].ds_addr < is->is_dvmabase || 727 map->dm_segs[seg].ds_addr > is->is_dvmaend) { 728 printf("seg %d dvmaddr %lx out of range %x - %x\n", 729 seg, (long)map->dm_segs[seg].ds_addr, 730 is->is_dvmabase, is->is_dvmaend); 731 #ifdef DDB 732 Debugger(); 733 #endif 734 } 735 } 736 #endif 737 return (0); 738 } 739 740 741 void 742 iommu_dvmamap_unload(bus_dma_tag_t t, bus_dmamap_t map) 743 { 744 struct strbuf_ctl *sb = (struct strbuf_ctl *)map->_dm_cookie; 745 struct iommu_state *is = sb->sb_is; 746 747 /* Flush the iommu */ 748 if (!map->_dm_dvmastart) 749 panic("%s: error dvmastart is zero!\n", __func__); 750 751 if (is->is_flags & IOMMU_SYNC_BEFORE_UNMAP) { 752 753 /* Flush the caches */ 754 bus_dmamap_unload(t->_parent, map); 755 756 iommu_remove(is, map->_dm_dvmastart, map->_dm_dvmasize); 757 758 } else { 759 760 iommu_remove(is, map->_dm_dvmastart, map->_dm_dvmasize); 761 762 /* Flush the caches */ 763 bus_dmamap_unload(t->_parent, map); 764 } 765 766 vmem_xfree(is->is_dvmamap, map->_dm_dvmastart, map->_dm_dvmasize); 767 map->_dm_dvmastart = 0; 768 map->_dm_dvmasize = 0; 769 770 /* Clear the map */ 771 } 772 773 774 int 775 iommu_dvmamap_load_raw(bus_dma_tag_t t, bus_dmamap_t map, 776 bus_dma_segment_t *segs, int nsegs, bus_size_t size, int flags) 777 { 778 struct strbuf_ctl *sb = (struct strbuf_ctl *)map->_dm_cookie; 779 struct iommu_state *is = sb->sb_is; 780 struct vm_page *pg; 781 int i, j; 782 int left; 783 int err, needsflush; 784 bus_size_t sgsize; 785 paddr_t pa; 786 bus_size_t boundary, align; 787 u_long dvmaddr, sgstart, sgend, bmask; 788 struct pglist *pglist; 789 const int pagesz = PAGE_SIZE; 790 int slot; 791 #ifdef DEBUG 792 int npg = 0; 793 #endif 794 795 if (map->dm_nsegs) { 796 /* Already in use?? */ 797 #ifdef DIAGNOSTIC 798 printf("iommu_dvmamap_load_raw: map still in use\n"); 799 #endif 800 bus_dmamap_unload(t, map); 801 } 802 803 /* 804 * A boundary presented to bus_dmamem_alloc() takes precedence 805 * over boundary in the map. 806 */ 807 if ((boundary = segs[0]._ds_boundary) == 0) 808 boundary = map->_dm_boundary; 809 810 align = uimax(segs[0]._ds_align, pagesz); 811 812 /* 813 * Make sure that on error condition we return "no valid mappings". 814 */ 815 map->dm_nsegs = 0; 816 /* Count up the total number of pages we need */ 817 pa = trunc_page(segs[0].ds_addr); 818 sgsize = 0; 819 left = size; 820 for (i = 0; left > 0 && i < nsegs; i++) { 821 if (round_page(pa) != round_page(segs[i].ds_addr)) 822 sgsize = round_page(sgsize) + 823 (segs[i].ds_addr & PGOFSET); 824 sgsize += uimin(left, segs[i].ds_len); 825 left -= segs[i].ds_len; 826 pa = segs[i].ds_addr + segs[i].ds_len; 827 } 828 sgsize = round_page(sgsize); 829 830 /* 831 * If our segment size is larger than the boundary we need to 832 * split the transfer up into little pieces ourselves. 833 */ 834 const vm_flag_t vmflags = VM_BESTFIT | 835 ((flags & BUS_DMA_NOWAIT) ? VM_NOSLEEP : VM_SLEEP); 836 837 err = vmem_xalloc(is->is_dvmamap, sgsize, 838 align, /* alignment */ 839 0, /* phase */ 840 (sgsize > boundary) ? 0 : boundary, 841 VMEM_ADDR_MIN, /* minaddr */ 842 VMEM_ADDR_MAX, /* maxaddr */ 843 vmflags, 844 &dvmaddr); 845 if (err != 0) 846 return (err); 847 848 #ifdef DEBUG 849 if (dvmaddr == (u_long)-1) 850 { 851 printf("iommu_dvmamap_load_raw(): extent_alloc(%d, %x) failed!\n", 852 (int)sgsize, flags); 853 #ifdef DDB 854 Debugger(); 855 #endif 856 } 857 #endif 858 if (dvmaddr == (u_long)-1) 859 return (ENOMEM); 860 861 /* Set the active DVMA map */ 862 map->_dm_dvmastart = dvmaddr; 863 map->_dm_dvmasize = sgsize; 864 865 bmask = ~(boundary - 1); 866 if ((pglist = segs[0]._ds_mlist) == NULL) { 867 u_long prev_va = 0UL, last_va = dvmaddr; 868 paddr_t prev_pa = 0; 869 int end = 0, offset; 870 bus_size_t len = size; 871 872 /* 873 * This segs is made up of individual physical 874 * segments, probably by _bus_dmamap_load_uio() or 875 * _bus_dmamap_load_mbuf(). Ignore the mlist and 876 * load each one individually. 877 */ 878 j = 0; 879 needsflush = 0; 880 for (i = 0; i < nsegs ; i++) { 881 882 pa = segs[i].ds_addr; 883 offset = (pa & PGOFSET); 884 pa = trunc_page(pa); 885 dvmaddr = trunc_page(dvmaddr); 886 left = uimin(len, segs[i].ds_len); 887 888 DPRINTF(IDB_INFO, ("iommu_dvmamap_load_raw: converting " 889 "physseg %d start %lx size %lx\n", i, 890 (long)segs[i].ds_addr, (long)segs[i].ds_len)); 891 892 if ((pa == prev_pa) && 893 ((offset != 0) || (end != offset))) { 894 /* We can re-use this mapping */ 895 dvmaddr = prev_va; 896 } 897 898 sgstart = dvmaddr + offset; 899 sgend = sgstart + left - 1; 900 901 /* Are the segments virtually adjacent? */ 902 if ((j > 0) && (end == offset) && 903 ((offset == 0) || (pa == prev_pa)) && 904 (map->dm_segs[j-1].ds_len + left <= 905 map->dm_maxsegsz)) { 906 /* Just append to the previous segment. */ 907 map->dm_segs[--j].ds_len += left; 908 /* Restore sgstart for boundary check */ 909 sgstart = map->dm_segs[j].ds_addr; 910 DPRINTF(IDB_INFO, ("iommu_dvmamap_load_raw: " 911 "appending seg %d start %lx size %lx\n", j, 912 (long)map->dm_segs[j].ds_addr, 913 (long)map->dm_segs[j].ds_len)); 914 } else { 915 if (j >= map->_dm_segcnt) { 916 iommu_remove(is, map->_dm_dvmastart, 917 last_va - map->_dm_dvmastart); 918 goto fail; 919 } 920 map->dm_segs[j].ds_addr = sgstart; 921 map->dm_segs[j].ds_len = left; 922 DPRINTF(IDB_INFO, ("iommu_dvmamap_load_raw: " 923 "seg %d start %lx size %lx\n", j, 924 (long)map->dm_segs[j].ds_addr, 925 (long)map->dm_segs[j].ds_len)); 926 } 927 end = (offset + left) & PGOFSET; 928 929 /* Check for boundary issues */ 930 while ((sgstart & bmask) != (sgend & bmask)) { 931 /* Need a new segment. */ 932 map->dm_segs[j].ds_len = 933 boundary - (sgstart & (boundary - 1)); 934 DPRINTF(IDB_INFO, ("iommu_dvmamap_load_raw: " 935 "seg %d start %lx size %lx\n", j, 936 (long)map->dm_segs[j].ds_addr, 937 (long)map->dm_segs[j].ds_len)); 938 if (++j >= map->_dm_segcnt) { 939 iommu_remove(is, map->_dm_dvmastart, 940 last_va - map->_dm_dvmastart); 941 goto fail; 942 } 943 sgstart += map->dm_segs[j-1].ds_len; 944 map->dm_segs[j].ds_addr = sgstart; 945 map->dm_segs[j].ds_len = sgend - sgstart + 1; 946 } 947 948 if (sgsize == 0) 949 panic("iommu_dmamap_load_raw: size botch"); 950 951 /* Now map a series of pages. */ 952 while (dvmaddr <= sgend) { 953 DPRINTF(IDB_BUSDMA, 954 ("iommu_dvmamap_load_raw: map %p " 955 "loading va %lx at pa %lx\n", 956 map, (long)dvmaddr, 957 (long)(pa))); 958 /* Enter it if we haven't before. */ 959 if (prev_va != dvmaddr) { 960 iommu_enter(sb, prev_va = dvmaddr, 961 prev_pa = pa, 962 flags | IOTTE_DEBUG(++npg << 12)); 963 needsflush = 1; 964 965 /* Flush cache if necessary. */ 966 slot = IOTSBSLOT(trunc_page(dvmaddr), is->is_tsbsize); 967 if ((is->is_flags & IOMMU_FLUSH_CACHE) && 968 ((dvmaddr + pagesz) > sgend || (slot % 8) == 7)) 969 IOMMUREG_WRITE(is, iommu_cache_flush, 970 is->is_ptsb + slot * 8); 971 } 972 973 dvmaddr += pagesz; 974 pa += pagesz; 975 last_va = dvmaddr; 976 } 977 978 len -= left; 979 ++j; 980 } 981 if (needsflush) 982 iommu_strbuf_flush_done(sb); 983 984 map->dm_mapsize = size; 985 map->dm_nsegs = j; 986 #ifdef DIAGNOSTIC 987 { int seg; 988 for (seg = 0; seg < map->dm_nsegs; seg++) { 989 if (map->dm_segs[seg].ds_addr < is->is_dvmabase || 990 map->dm_segs[seg].ds_addr > is->is_dvmaend) { 991 printf("seg %d dvmaddr %lx out of range %x - %x\n", 992 seg, (long)map->dm_segs[seg].ds_addr, 993 is->is_dvmabase, is->is_dvmaend); 994 #ifdef DDB 995 Debugger(); 996 #endif 997 } 998 } 999 } 1000 #endif 1001 return (0); 1002 } 1003 1004 /* 1005 * This was allocated with bus_dmamem_alloc. 1006 * The pages are on a `pglist'. 1007 */ 1008 i = 0; 1009 sgstart = dvmaddr; 1010 sgend = sgstart + size - 1; 1011 map->dm_segs[i].ds_addr = sgstart; 1012 while ((sgstart & bmask) != (sgend & bmask)) { 1013 /* Oops. We crossed a boundary. Split the xfer. */ 1014 map->dm_segs[i].ds_len = boundary - (sgstart & (boundary - 1)); 1015 DPRINTF(IDB_INFO, ("iommu_dvmamap_load_raw: " 1016 "seg %d start %lx size %lx\n", i, 1017 (long)map->dm_segs[i].ds_addr, 1018 (long)map->dm_segs[i].ds_len)); 1019 if (++i >= map->_dm_segcnt) { 1020 /* Too many segments. Fail the operation. */ 1021 goto fail; 1022 } 1023 sgstart += map->dm_segs[i-1].ds_len; 1024 map->dm_segs[i].ds_addr = sgstart; 1025 } 1026 DPRINTF(IDB_INFO, ("iommu_dvmamap_load_raw: " 1027 "seg %d start %lx size %lx\n", i, 1028 (long)map->dm_segs[i].ds_addr, (long)map->dm_segs[i].ds_len)); 1029 map->dm_segs[i].ds_len = sgend - sgstart + 1; 1030 1031 needsflush = 0; 1032 TAILQ_FOREACH(pg, pglist, pageq.queue) { 1033 if (sgsize == 0) 1034 panic("iommu_dmamap_load_raw: size botch"); 1035 pa = VM_PAGE_TO_PHYS(pg); 1036 1037 DPRINTF(IDB_BUSDMA, 1038 ("iommu_dvmamap_load_raw: map %p loading va %lx at pa %lx\n", 1039 map, (long)dvmaddr, (long)(pa))); 1040 iommu_enter(sb, dvmaddr, pa, flags | IOTTE_DEBUG(0x8000)); 1041 needsflush = 1; 1042 1043 sgsize -= pagesz; 1044 1045 /* Flush cache if necessary. */ 1046 slot = IOTSBSLOT(trunc_page(dvmaddr), is->is_tsbsize); 1047 if ((is->is_flags & IOMMU_FLUSH_CACHE) && 1048 (sgsize == 0 || (slot % 8) == 7)) 1049 IOMMUREG_WRITE(is, iommu_cache_flush, 1050 is->is_ptsb + slot * 8); 1051 1052 dvmaddr += pagesz; 1053 } 1054 if (needsflush) 1055 iommu_strbuf_flush_done(sb); 1056 map->dm_mapsize = size; 1057 map->dm_nsegs = i+1; 1058 #ifdef DIAGNOSTIC 1059 { int seg; 1060 for (seg = 0; seg < map->dm_nsegs; seg++) { 1061 if (map->dm_segs[seg].ds_addr < is->is_dvmabase || 1062 map->dm_segs[seg].ds_addr > is->is_dvmaend) { 1063 printf("seg %d dvmaddr %lx out of range %x - %x\n", 1064 seg, (long)map->dm_segs[seg].ds_addr, 1065 is->is_dvmabase, is->is_dvmaend); 1066 #ifdef DDB 1067 Debugger(); 1068 #endif 1069 } 1070 } 1071 } 1072 #endif 1073 return (0); 1074 1075 fail: 1076 vmem_xfree(is->is_dvmamap, map->_dm_dvmastart, sgsize); 1077 map->_dm_dvmastart = 0; 1078 map->_dm_dvmasize = 0; 1079 return (EFBIG); 1080 } 1081 1082 1083 /* 1084 * Flush an individual dma segment, returns non-zero if the streaming buffers 1085 * need flushing afterwards. 1086 */ 1087 static int 1088 iommu_dvmamap_sync_range(struct strbuf_ctl *sb, vaddr_t va, bus_size_t len) 1089 { 1090 vaddr_t vaend; 1091 struct iommu_state *is = sb->sb_is; 1092 1093 #ifdef DIAGNOSTIC 1094 if (va < is->is_dvmabase || va > is->is_dvmaend) 1095 panic("invalid va: %llx", (long long)va); 1096 #endif 1097 1098 if ((is->is_tsb[IOTSBSLOT(va, is->is_tsbsize)] & IOTTE_STREAM) == 0) { 1099 DPRINTF(IDB_SYNC, 1100 ("iommu_dvmamap_sync_range: attempting to flush " 1101 "non-streaming entry\n")); 1102 return (0); 1103 } 1104 1105 vaend = round_page(va + len) - 1; 1106 va = trunc_page(va); 1107 1108 #ifdef DIAGNOSTIC 1109 if (va < is->is_dvmabase || vaend > is->is_dvmaend) 1110 panic("invalid va range: %llx to %llx (%x to %x)", 1111 (long long)va, (long long)vaend, 1112 is->is_dvmabase, 1113 is->is_dvmaend); 1114 #endif 1115 1116 for ( ; va <= vaend; va += PAGE_SIZE) { 1117 DPRINTF(IDB_SYNC, 1118 ("iommu_dvmamap_sync_range: flushing va %p\n", 1119 (void *)(u_long)va)); 1120 iommu_strbuf_flush(sb, va); 1121 } 1122 1123 return (1); 1124 } 1125 1126 static void 1127 _iommu_dvmamap_sync(bus_dma_tag_t t, bus_dmamap_t map, bus_addr_t offset, 1128 bus_size_t len, int ops) 1129 { 1130 struct strbuf_ctl *sb = (struct strbuf_ctl *)map->_dm_cookie; 1131 bus_size_t count; 1132 int i, needsflush = 0; 1133 1134 if (!sb->sb_flush) 1135 return; 1136 1137 for (i = 0; i < map->dm_nsegs; i++) { 1138 if (offset < map->dm_segs[i].ds_len) 1139 break; 1140 offset -= map->dm_segs[i].ds_len; 1141 } 1142 1143 if (i == map->dm_nsegs) 1144 panic("%s: segment too short %llu", __func__, 1145 (unsigned long long)offset); 1146 1147 if (ops & (BUS_DMASYNC_PREREAD | BUS_DMASYNC_POSTWRITE)) { 1148 /* Nothing to do */; 1149 } 1150 1151 if (ops & (BUS_DMASYNC_POSTREAD | BUS_DMASYNC_PREWRITE)) { 1152 1153 for (; len > 0 && i < map->dm_nsegs; i++) { 1154 count = MIN(map->dm_segs[i].ds_len - offset, len); 1155 if (count > 0 && 1156 iommu_dvmamap_sync_range(sb, 1157 map->dm_segs[i].ds_addr + offset, count)) 1158 needsflush = 1; 1159 offset = 0; 1160 len -= count; 1161 } 1162 #ifdef DIAGNOSTIC 1163 if (i == map->dm_nsegs && len > 0) 1164 panic("%s: leftover %llu", __func__, 1165 (unsigned long long)len); 1166 #endif 1167 1168 if (needsflush) 1169 iommu_strbuf_flush_done(sb); 1170 } 1171 } 1172 1173 void 1174 iommu_dvmamap_sync(bus_dma_tag_t t, bus_dmamap_t map, bus_addr_t offset, 1175 bus_size_t len, int ops) 1176 { 1177 1178 /* If len is 0, then there is nothing to do */ 1179 if (len == 0) 1180 return; 1181 1182 if (ops & (BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE)) { 1183 /* Flush the CPU then the IOMMU */ 1184 bus_dmamap_sync(t->_parent, map, offset, len, ops); 1185 _iommu_dvmamap_sync(t, map, offset, len, ops); 1186 } 1187 if (ops & (BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE)) { 1188 /* Flush the IOMMU then the CPU */ 1189 _iommu_dvmamap_sync(t, map, offset, len, ops); 1190 bus_dmamap_sync(t->_parent, map, offset, len, ops); 1191 } 1192 } 1193 1194 int 1195 iommu_dvmamem_alloc(bus_dma_tag_t t, bus_size_t size, bus_size_t alignment, 1196 bus_size_t boundary, bus_dma_segment_t *segs, int nsegs, int *rsegs, 1197 int flags) 1198 { 1199 1200 DPRINTF(IDB_BUSDMA, ("iommu_dvmamem_alloc: sz %llx align %llx bound %llx " 1201 "segp %p flags %d\n", (unsigned long long)size, 1202 (unsigned long long)alignment, (unsigned long long)boundary, 1203 segs, flags)); 1204 return (bus_dmamem_alloc(t->_parent, size, alignment, boundary, 1205 segs, nsegs, rsegs, flags|BUS_DMA_DVMA)); 1206 } 1207 1208 void 1209 iommu_dvmamem_free(bus_dma_tag_t t, bus_dma_segment_t *segs, int nsegs) 1210 { 1211 1212 DPRINTF(IDB_BUSDMA, ("iommu_dvmamem_free: segp %p nsegs %d\n", 1213 segs, nsegs)); 1214 bus_dmamem_free(t->_parent, segs, nsegs); 1215 } 1216 1217 /* 1218 * Map the DVMA mappings into the kernel pmap. 1219 * Check the flags to see whether we're streaming or coherent. 1220 */ 1221 int 1222 iommu_dvmamem_map(bus_dma_tag_t t, bus_dma_segment_t *segs, int nsegs, 1223 size_t size, void **kvap, int flags) 1224 { 1225 struct vm_page *pg; 1226 vaddr_t va; 1227 bus_addr_t addr; 1228 struct pglist *pglist; 1229 int cbit; 1230 const uvm_flag_t kmflags = 1231 (flags & BUS_DMA_NOWAIT) != 0 ? UVM_KMF_NOWAIT : 0; 1232 1233 DPRINTF(IDB_BUSDMA, ("iommu_dvmamem_map: segp %p nsegs %d size %lx\n", 1234 segs, nsegs, size)); 1235 1236 /* 1237 * Allocate some space in the kernel map, and then map these pages 1238 * into this space. 1239 */ 1240 size = round_page(size); 1241 va = uvm_km_alloc(kernel_map, size, 0, UVM_KMF_VAONLY | kmflags); 1242 if (va == 0) 1243 return (ENOMEM); 1244 1245 *kvap = (void *)va; 1246 1247 /* 1248 * digest flags: 1249 */ 1250 cbit = 0; 1251 if (flags & BUS_DMA_COHERENT) /* Disable vcache */ 1252 cbit |= PMAP_NVC; 1253 if (flags & BUS_DMA_NOCACHE) /* side effects */ 1254 cbit |= PMAP_NC; 1255 1256 /* 1257 * Now take this and map it into the CPU. 1258 */ 1259 pglist = segs[0]._ds_mlist; 1260 TAILQ_FOREACH(pg, pglist, pageq.queue) { 1261 #ifdef DIAGNOSTIC 1262 if (size == 0) 1263 panic("iommu_dvmamem_map: size botch"); 1264 #endif 1265 addr = VM_PAGE_TO_PHYS(pg); 1266 DPRINTF(IDB_BUSDMA, ("iommu_dvmamem_map: " 1267 "mapping va %lx at %llx\n", va, (unsigned long long)addr | cbit)); 1268 pmap_kenter_pa(va, addr | cbit, 1269 VM_PROT_READ | VM_PROT_WRITE, 0); 1270 va += PAGE_SIZE; 1271 size -= PAGE_SIZE; 1272 } 1273 pmap_update(pmap_kernel()); 1274 return (0); 1275 } 1276 1277 /* 1278 * Unmap DVMA mappings from kernel 1279 */ 1280 void 1281 iommu_dvmamem_unmap(bus_dma_tag_t t, void *kva, size_t size) 1282 { 1283 1284 DPRINTF(IDB_BUSDMA, ("iommu_dvmamem_unmap: kvm %p size %lx\n", 1285 kva, size)); 1286 1287 #ifdef DIAGNOSTIC 1288 if ((u_long)kva & PGOFSET) 1289 panic("iommu_dvmamem_unmap"); 1290 #endif 1291 1292 size = round_page(size); 1293 pmap_kremove((vaddr_t)kva, size); 1294 pmap_update(pmap_kernel()); 1295 uvm_km_free(kernel_map, (vaddr_t)kva, size, UVM_KMF_VAONLY); 1296 } 1297
Indexes created Fri Sep 26 08:10:20 GMT 2025