1 /* $NetBSD: iommu.c,v 1.103 2026/07/18 13:54:26 thorpej Exp $ */ 2 3 /* 4 * Copyright (c) 1996 5 * The President and Fellows of Harvard College. All rights reserved. 6 * Copyright (c) 1995 Paul Kranenburg 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 3. All advertising materials mentioning features or use of this software 17 * must display the following acknowledgement: 18 * This product includes software developed by Aaron Brown and 19 * Harvard University. 20 * This product includes software developed by Paul Kranenburg. 21 * 4. Neither the name of the University nor the names of its contributors 22 * may be used to endorse or promote products derived from this software 23 * without specific prior written permission. 24 * 25 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 26 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 27 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 28 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 29 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 30 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 31 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 32 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 33 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 34 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 35 * SUCH DAMAGE. 36 * 37 */ 38 39 #include <sys/cdefs.h> 40 __KERNEL_RCSID(0, "$NetBSD: iommu.c,v 1.103 2026/07/18 13:54:26 thorpej Exp $"); 41 42 #include "opt_sparc_arch.h" 43 44 #include <sys/param.h> 45 #include <sys/malloc.h> 46 #include <sys/queue.h> 47 #include <sys/systm.h> 48 #include <sys/device.h> 49 #include <sys/proc.h> 50 #include <sys/vmem.h> 51 #include <sys/mbuf.h> 52 53 #include <uvm/uvm.h> 54 55 #define _SPARC_BUS_DMA_PRIVATE 56 #include <sys/bus.h> 57 #include <machine/autoconf.h> 58 #include <machine/ctlreg.h> 59 #include <sparc/sparc/asm.h> 60 #include <sparc/sparc/vaddrs.h> 61 #include <sparc/sparc/cpuvar.h> 62 #include <sparc/sparc/iommureg.h> 63 #include <sparc/sparc/iommuvar.h> 64 65 struct iommu_softc { 66 struct iommureg *sc_reg; 67 u_int sc_pagesize; 68 u_int sc_range; 69 bus_addr_t sc_dvmabase; 70 iopte_t *sc_ptes; 71 int sc_cachecoherent; 72 /* 73 * Note: operations on the extent map are being protected with 74 * splhigh(), since we cannot predict at which interrupt priority 75 * our clients will run. 76 */ 77 struct sparc_bus_dma_tag sc_dmatag; 78 vmem_t *sc_dvmamap; 79 }; 80 81 /* autoconfiguration driver */ 82 int iommu_print(void *, const char *); 83 void iommu_attach(device_t, device_t, void *); 84 int iommu_match(device_t, cfdata_t, void *); 85 86 #if defined(SUN4M) 87 static void iommu_copy_prom_entries(struct iommu_softc *); 88 #endif 89 90 CFATTACH_DECL_NEW(iommu, sizeof(struct iommu_softc), 91 iommu_match, iommu_attach, NULL, NULL); 92 93 /* IOMMU DMA map functions */ 94 static int iommu_dmamap_load_buffer(bus_dma_tag_t, bus_dmamap_t, void *, 95 bus_size_t, struct proc *, int); 96 97 int iommu_dmamap_create(bus_dma_tag_t, bus_size_t, int, bus_size_t, 98 bus_size_t, int, bus_dmamap_t *); 99 int iommu_dmamap_load(bus_dma_tag_t, bus_dmamap_t, void *, 100 bus_size_t, struct proc *, int); 101 int iommu_dmamap_load_mbuf(bus_dma_tag_t, bus_dmamap_t, 102 struct mbuf *, int); 103 int iommu_dmamap_load_uio(bus_dma_tag_t, bus_dmamap_t, 104 struct uio *, int); 105 int iommu_dmamap_load_raw(bus_dma_tag_t, bus_dmamap_t, 106 bus_dma_segment_t *, int, bus_size_t, int); 107 void iommu_dmamap_unload(bus_dma_tag_t, bus_dmamap_t); 108 void iommu_dmamap_sync(bus_dma_tag_t, bus_dmamap_t, bus_addr_t, 109 bus_size_t, int); 110 111 int iommu_dmamem_map(bus_dma_tag_t, bus_dma_segment_t *, 112 int, size_t, void **, int); 113 void iommu_dmamem_unmap(bus_dma_tag_t, void *, size_t); 114 paddr_t iommu_dmamem_mmap(bus_dma_tag_t, bus_dma_segment_t *, 115 int, off_t, int, int); 116 int iommu_dvma_alloc(struct iommu_softc *, bus_dmamap_t, vaddr_t, 117 bus_size_t, int, bus_addr_t *, bus_size_t *); 118 119 /* 120 * Print the location of some iommu-attached device (called just 121 * before attaching that device). If `iommu' is not NULL, the 122 * device was found but not configured; print the iommu as well. 123 * Return UNCONF (config_find ignores this if the device was configured). 124 */ 125 int 126 iommu_print(void *args, const char *iommu) 127 { 128 struct iommu_attach_args *ia = args; 129 130 if (iommu) 131 aprint_normal("%s at %s", ia->iom_name, iommu); 132 return (UNCONF); 133 } 134 135 int 136 iommu_match(device_t parent, cfdata_t cf, void *aux) 137 { 138 struct mainbus_attach_args *ma = aux; 139 140 if (CPU_ISSUN4 || CPU_ISSUN4C) 141 return (0); 142 return (strcmp(cf->cf_name, ma->ma_name) == 0); 143 } 144 145 /* 146 * Attach the iommu. 147 */ 148 void 149 iommu_attach(device_t parent, device_t self, void *aux) 150 { 151 #if defined(SUN4M) 152 struct iommu_softc *sc = device_private(self); 153 struct mainbus_attach_args *ma = aux; 154 struct sparc_bus_dma_tag *dmat = &sc->sc_dmatag; 155 bus_space_handle_t bh; 156 int node; 157 int js1_implicit_iommu; 158 int i, s; 159 u_int iopte_table_pa; 160 struct pglist mlist; 161 u_int size; 162 struct vm_page *m; 163 vaddr_t va; 164 165 dmat->_cookie = sc; 166 dmat->_dmamap_create = iommu_dmamap_create; 167 dmat->_dmamap_destroy = _bus_dmamap_destroy; 168 dmat->_dmamap_load = iommu_dmamap_load; 169 dmat->_dmamap_load_mbuf = iommu_dmamap_load_mbuf; 170 dmat->_dmamap_load_uio = iommu_dmamap_load_uio; 171 dmat->_dmamap_load_raw = iommu_dmamap_load_raw; 172 dmat->_dmamap_unload = iommu_dmamap_unload; 173 dmat->_dmamap_sync = iommu_dmamap_sync; 174 175 dmat->_dmamem_alloc = _bus_dmamem_alloc; 176 dmat->_dmamem_free = _bus_dmamem_free; 177 dmat->_dmamem_map = iommu_dmamem_map; 178 dmat->_dmamem_unmap = _bus_dmamem_unmap; 179 dmat->_dmamem_mmap = iommu_dmamem_mmap; 180 181 /* 182 * JS1/OF device tree does not have an iommu node and sbus 183 * node is directly under root. mainbus_attach detects this 184 * and calls us with sbus node instead so that we can attach 185 * implicit iommu and attach that sbus node under it. 186 */ 187 node = ma->ma_node; 188 if (strcmp(prom_getpropstring(node, "name"), "sbus") == 0) 189 js1_implicit_iommu = 1; 190 else 191 js1_implicit_iommu = 0; 192 193 /* 194 * Map registers into our space. The PROM may have done this 195 * already, but I feel better if we have our own copy. Plus, the 196 * prom doesn't map the entire register set. 197 * 198 * XXX struct iommureg is bigger than ra->ra_len; what are the 199 * other fields for? 200 */ 201 if (bus_space_map(ma->ma_bustag, ma->ma_paddr, 202 sizeof(struct iommureg), 0, &bh) != 0) { 203 printf("iommu_attach: cannot map registers\n"); 204 return; 205 } 206 sc->sc_reg = (struct iommureg *)bh; 207 208 sc->sc_cachecoherent = js1_implicit_iommu ? 0 209 : node_has_property(node, "cache-coherence?"); 210 if (CACHEINFO.c_enabled == 0) /* XXX - is this correct? */ 211 sc->sc_cachecoherent = 0; 212 213 sc->sc_pagesize = js1_implicit_iommu ? PAGE_SIZE 214 : prom_getpropint(node, "page-size", PAGE_SIZE), 215 216 /* 217 * Allocate memory for I/O pagetables. 218 * This takes 64K of contiguous physical memory to map 64M of 219 * DVMA space (starting at IOMMU_DVMA_BASE). 220 * The table must be aligned on a (-IOMMU_DVMA_BASE/pagesize) 221 * boundary (i.e. 64K for 64M of DVMA space). 222 */ 223 224 size = ((0 - IOMMU_DVMA_BASE) / sc->sc_pagesize) * sizeof(iopte_t); 225 if (uvm_pglistalloc(size, vm_first_phys, vm_first_phys+vm_num_phys, 226 size, 0, &mlist, 1, 0) != 0) 227 panic("iommu_attach: no memory"); 228 229 va = uvm_km_alloc(kernel_map, size, 0, UVM_KMF_VAONLY); 230 if (va == 0) 231 panic("iommu_attach: no memory"); 232 233 sc->sc_ptes = (iopte_t *)va; 234 235 m = TAILQ_FIRST(&mlist); 236 iopte_table_pa = VM_PAGE_TO_PHYS(m); 237 238 /* Map the pages */ 239 for (; m != NULL; m = TAILQ_NEXT(m,pageq.queue)) { 240 paddr_t pa = VM_PAGE_TO_PHYS(m); 241 pmap_kenter_pa(va, pa | PMAP_NC, 242 VM_PROT_READ | VM_PROT_WRITE, 0); 243 va += PAGE_SIZE; 244 } 245 pmap_update(pmap_kernel()); 246 247 /* 248 * Copy entries from current IOMMU table. 249 * XXX - Why do we need to do this? 250 */ 251 iommu_copy_prom_entries(sc); 252 253 /* 254 * Now we can install our new pagetable into the IOMMU 255 */ 256 sc->sc_range = 0 - IOMMU_DVMA_BASE; 257 sc->sc_dvmabase = IOMMU_DVMA_BASE; 258 259 /* calculate log2(sc->sc_range/16MB) */ 260 i = ffs(sc->sc_range/(1 << 24)) - 1; 261 if ((1 << i) != (sc->sc_range/(1 << 24))) 262 panic("iommu: bad range: %d", i); 263 264 s = splhigh(); 265 IOMMU_FLUSHALL(sc); 266 267 /* Load range and physical address of PTEs */ 268 sc->sc_reg->io_cr = (sc->sc_reg->io_cr & ~IOMMU_CTL_RANGE) | 269 (i << IOMMU_CTL_RANGESHFT) | IOMMU_CTL_ME; 270 sc->sc_reg->io_bar = (iopte_table_pa >> 4) & IOMMU_BAR_IBA; 271 272 IOMMU_FLUSHALL(sc); 273 splx(s); 274 275 printf(": version 0x%x/0x%x, page-size %d, range %dMB\n", 276 (sc->sc_reg->io_cr & IOMMU_CTL_VER) >> 24, 277 (sc->sc_reg->io_cr & IOMMU_CTL_IMPL) >> 28, 278 sc->sc_pagesize, 279 sc->sc_range >> 20); 280 281 sc->sc_dvmamap = vmem_create("iommudvma", 282 IOMMU_DVMA_BASE, 283 IOMMU_DVMA_END - IOMMU_DVMA_BASE, 284 PAGE_SIZE, /* quantum */ 285 NULL, /* importfn */ 286 NULL, /* releasefn */ 287 NULL, /* source */ 288 0, /* qcache_max */ 289 VM_SLEEP, 290 IPL_VM); 291 292 devhandle_t selfh = device_handle(self); 293 294 /* 295 * If we are attaching implicit iommu on JS1/OF we do not have 296 * an iommu node to traverse, instead mainbus_attach passed us 297 * sbus node in ma.ma_node. Attach it as the only iommu child. 298 */ 299 if (js1_implicit_iommu) { 300 struct iommu_attach_args ia; 301 struct openprom_addr sbus_iommu_reg = { 0, 0x10001000, 0x28 }; 302 303 memset(&ia, 0, sizeof ia); 304 305 /* Propagate BUS & DMA tags */ 306 ia.iom_bustag = ma->ma_bustag; 307 ia.iom_dmatag = &sc->sc_dmatag; 308 309 ia.iom_name = "sbus"; 310 ia.iom_node = node; 311 ia.iom_reg = &sbus_iommu_reg; 312 ia.iom_nreg = 1; 313 314 config_found(self, (void *)&ia, iommu_print, 315 CFARGS(.devhandle = prom_node_to_devhandle(selfh, node))); 316 return; 317 } 318 319 /* 320 * Loop through ROM children (expect Sbus among them). 321 */ 322 for (node = firstchild(node); node; node = nextsibling(node)) { 323 struct iommu_attach_args ia; 324 325 memset(&ia, 0, sizeof ia); 326 ia.iom_name = prom_getpropstring(node, "name"); 327 328 /* Propagate BUS & DMA tags */ 329 ia.iom_bustag = ma->ma_bustag; 330 ia.iom_dmatag = &sc->sc_dmatag; 331 332 ia.iom_node = node; 333 334 ia.iom_reg = NULL; 335 prom_getprop(node, "reg", sizeof(struct openprom_addr), 336 &ia.iom_nreg, &ia.iom_reg); 337 338 config_found(self, (void *)&ia, iommu_print, 339 CFARGS(.devhandle = prom_node_to_devhandle(selfh, node))); 340 if (ia.iom_reg != NULL) 341 free(ia.iom_reg, M_DEVBUF); 342 } 343 #endif 344 } 345 346 #if defined(SUN4M) 347 static void 348 iommu_copy_prom_entries(struct iommu_softc *sc) 349 { 350 u_int pbase, pa; 351 u_int range; 352 iopte_t *tpte_p; 353 u_int pagesz = sc->sc_pagesize; 354 int use_ac = (cpuinfo.cpu_impl == 4 && cpuinfo.mxcc); 355 u_int mmupcr_save; 356 357 /* 358 * We read in the original table using MMU bypass and copy all 359 * of its entries to the appropriate place in our new table, 360 * even if the sizes are different. 361 * This is pretty easy since we know DVMA ends at 0xffffffff. 362 */ 363 364 range = (1 << 24) << 365 ((sc->sc_reg->io_cr & IOMMU_CTL_RANGE) >> IOMMU_CTL_RANGESHFT); 366 367 pbase = (sc->sc_reg->io_bar & IOMMU_BAR_IBA) << 368 (14 - IOMMU_BAR_IBASHFT); 369 370 if (use_ac) { 371 /* 372 * Set MMU AC bit so we'll still read from the cache 373 * in by-pass mode. 374 */ 375 mmupcr_save = lda(SRMMU_PCR, ASI_SRMMU); 376 sta(SRMMU_PCR, ASI_SRMMU, mmupcr_save | VIKING_PCR_AC); 377 } else 378 mmupcr_save = 0; /* XXX - avoid GCC `uninitialized' warning */ 379 380 /* Flush entire IOMMU TLB before messing with the in-memory tables */ 381 IOMMU_FLUSHALL(sc); 382 383 /* 384 * tpte_p = top of our PTE table 385 * pa = top of current PTE table 386 * Then work downwards and copy entries until we hit the bottom 387 * of either table. 388 */ 389 for (tpte_p = &sc->sc_ptes[((0 - IOMMU_DVMA_BASE)/pagesz) - 1], 390 pa = (u_int)pbase + (range/pagesz - 1)*sizeof(iopte_t); 391 tpte_p >= &sc->sc_ptes[0] && pa >= (u_int)pbase; 392 tpte_p--, pa -= sizeof(iopte_t)) { 393 394 *tpte_p = lda(pa, ASI_BYPASS); 395 } 396 397 if (use_ac) { 398 /* restore mmu after bug-avoidance */ 399 sta(SRMMU_PCR, ASI_SRMMU, mmupcr_save); 400 } 401 } 402 #endif 403 404 static void 405 iommu_enter(struct iommu_softc *sc, bus_addr_t dva, paddr_t pa) 406 { 407 int pte; 408 409 /* This routine relies on the fact that sc->sc_pagesize == PAGE_SIZE */ 410 411 #ifdef DIAGNOSTIC 412 if (dva < sc->sc_dvmabase) 413 panic("iommu_enter: dva 0x%lx not in DVMA space", (long)dva); 414 #endif 415 416 pte = atop(pa) << IOPTE_PPNSHFT; 417 pte &= IOPTE_PPN; 418 pte |= IOPTE_V | IOPTE_W | (sc->sc_cachecoherent ? IOPTE_C : 0); 419 sc->sc_ptes[atop(dva - sc->sc_dvmabase)] = pte; 420 IOMMU_FLUSHPAGE(sc, dva); 421 } 422 423 /* 424 * iommu_remove: removes mappings created by iommu_enter 425 */ 426 static void 427 iommu_remove(struct iommu_softc *sc, bus_addr_t dva, bus_size_t len) 428 { 429 u_int pagesz = sc->sc_pagesize; 430 bus_addr_t base = sc->sc_dvmabase; 431 432 #ifdef DEBUG 433 if (dva < base) 434 panic("iommu_remove: va 0x%lx not in DVMA space", (long)dva); 435 #endif 436 437 while ((long)len > 0) { 438 #ifdef notyet 439 #ifdef DEBUG 440 if ((sc->sc_ptes[atop(dva - base)] & IOPTE_V) == 0) 441 panic("iommu_remove: clearing invalid pte at dva 0x%lx", 442 (long)dva); 443 #endif 444 #endif 445 sc->sc_ptes[atop(dva - base)] = 0; 446 IOMMU_FLUSHPAGE(sc, dva); 447 len -= pagesz; 448 dva += pagesz; 449 } 450 } 451 452 #if 0 /* These registers aren't there??? */ 453 void 454 iommu_error(void) 455 { 456 struct iommu_softc *sc = X; 457 struct iommureg *iop = sc->sc_reg; 458 459 printf("iommu: afsr 0x%x, afar 0x%x\n", iop->io_afsr, iop->io_afar); 460 printf("iommu: mfsr 0x%x, mfar 0x%x\n", iop->io_mfsr, iop->io_mfar); 461 } 462 463 int 464 iommu_alloc(u_int va, u_int len) 465 { 466 struct iommu_softc *sc = X; 467 int off, tva, iovaddr, pte; 468 paddr_t pa; 469 470 off = (int)va & PGOFSET; 471 len = round_page(len + off); 472 va -= off; 473 474 if ((int)sc->sc_dvmacur + len > 0) 475 sc->sc_dvmacur = sc->sc_dvmabase; 476 477 iovaddr = tva = sc->sc_dvmacur; 478 sc->sc_dvmacur += len; 479 while (len) { 480 (void) pmap_extract(pmap_kernel(), va, &pa); 481 482 #define IOMMU_PPNSHIFT 8 483 #define IOMMU_V 0x00000002 484 #define IOMMU_W 0x00000004 485 486 pte = atop(pa) << IOMMU_PPNSHIFT; 487 pte |= IOMMU_V | IOMMU_W; 488 sta(sc->sc_ptes + atop(tva - sc->sc_dvmabase), ASI_BYPASS, pte); 489 sc->sc_reg->io_flushpage = tva; 490 len -= PAGE_SIZE; 491 va += PAGE_SIZE; 492 tva += PAGE_SIZE; 493 } 494 return iovaddr + off; 495 } 496 #endif 497 498 499 /* 500 * IOMMU DMA map functions. 501 */ 502 int 503 iommu_dmamap_create(bus_dma_tag_t t, bus_size_t size, int nsegments, 504 bus_size_t maxsegsz, bus_size_t boundary, int flags, 505 bus_dmamap_t *dmamp) 506 { 507 bus_dmamap_t map; 508 int error; 509 510 if ((error = _bus_dmamap_create(t, size, nsegments, maxsegsz, 511 boundary, flags, &map)) != 0) 512 return (error); 513 514 if ((flags & BUS_DMA_24BIT) != 0) { 515 /* Limit this map to the range usable by `24-bit' devices */ 516 map->_dm_ex_start = D24_DVMA_BASE; 517 map->_dm_ex_end = D24_DVMA_END - 1; 518 } else { 519 /* Enable allocations from the entire map */ 520 map->_dm_ex_start = VMEM_ADDR_MIN; 521 map->_dm_ex_end = VMEM_ADDR_MAX; 522 } 523 524 *dmamp = map; 525 return (0); 526 } 527 528 /* 529 * Internal routine to allocate space in the IOMMU map. 530 */ 531 int 532 iommu_dvma_alloc(struct iommu_softc *sc, bus_dmamap_t map, 533 vaddr_t va, bus_size_t len, int flags, 534 bus_addr_t *dvap, bus_size_t *sgsizep) 535 { 536 bus_size_t sgsize; 537 u_long align, voff; 538 vmem_addr_t dvaddr; 539 int error; 540 int pagesz = PAGE_SIZE; 541 542 /* 543 * Remember page offset, then truncate the buffer address to 544 * a page boundary. 545 */ 546 voff = va & (pagesz - 1); 547 va &= -pagesz; 548 549 if (len > map->_dm_size) 550 return (EINVAL); 551 552 sgsize = (len + voff + pagesz - 1) & -pagesz; 553 align = dvma_cachealign ? dvma_cachealign : map->_dm_align; 554 555 const vm_flag_t vmflags = VM_BESTFIT | 556 ((flags & BUS_DMA_NOWAIT) ? VM_NOSLEEP : VM_SLEEP); 557 558 error = vmem_xalloc(sc->sc_dvmamap, sgsize, 559 align, /* alignment */ 560 va & (align-1), /* phase */ 561 map->_dm_boundary, /* nocross */ 562 map->_dm_ex_start, /* minaddr */ 563 map->_dm_ex_end, /* maxaddr */ 564 vmflags, 565 &dvaddr); 566 567 *dvap = (bus_addr_t)dvaddr; 568 *sgsizep = sgsize; 569 return (error); 570 } 571 572 static int 573 iommu_dmamap_load_buffer(bus_dma_tag_t t, bus_dmamap_t map, void *buf, 574 bus_size_t buflen, struct proc *p, int flags) 575 { 576 struct iommu_softc *sc = t->_cookie; 577 bus_size_t sgsize; 578 bus_addr_t dva; 579 vaddr_t va = (vaddr_t)buf; 580 int pagesz = PAGE_SIZE; 581 pmap_t pmap; 582 int error; 583 584 if (map->dm_nsegs >= map->_dm_segcnt) 585 return SET_ERROR(EFBIG); 586 587 /* Allocate IOMMU resources */ 588 if ((error = iommu_dvma_alloc(sc, map, va, buflen, flags, 589 &dva, &sgsize)) != 0) 590 return (error); 591 592 if ((sc->sc_cachecoherent == 0) || 593 (curcpu()->cacheinfo.ec_totalsize == 0)) 594 cache_flush(buf, buflen); /* XXX - move to bus_dma_sync? */ 595 596 /* 597 * We always use just one segment. 598 */ 599 map->dm_segs[map->dm_nsegs].ds_addr = dva + (va & (pagesz - 1)); 600 map->dm_segs[map->dm_nsegs].ds_len = buflen; 601 map->dm_segs[map->dm_nsegs]._ds_sgsize = sgsize; 602 map->dm_nsegs++; 603 604 if (p != NULL) 605 pmap = p->p_vmspace->vm_map.pmap; 606 else 607 pmap = pmap_kernel(); 608 609 for (; sgsize != 0; ) { 610 paddr_t pa; 611 /* 612 * Get the physical address for this page. 613 */ 614 if (!pmap_extract(pmap, va, &pa)) 615 return SET_ERROR(EFAULT); 616 617 iommu_enter(sc, dva, pa); 618 619 dva += pagesz; 620 va += pagesz; 621 sgsize -= pagesz; 622 } 623 624 return (0); 625 } 626 627 /* 628 * Prepare buffer for DMA transfer. 629 */ 630 int 631 iommu_dmamap_load(bus_dma_tag_t t, bus_dmamap_t map, 632 void *buf, bus_size_t buflen, 633 struct proc *p, int flags) 634 { 635 int error; 636 637 /* 638 * Make sure that on error condition we return "no valid mappings". 639 */ 640 map->dm_mapsize = buflen; 641 map->dm_nsegs = 0; 642 643 error = iommu_dmamap_load_buffer(t, map, buf, buflen, p, flags); 644 if (error) 645 iommu_dmamap_unload(t, map); 646 647 return (error); 648 } 649 650 /* 651 * Like _bus_dmamap_load(), but for mbufs. 652 */ 653 int 654 iommu_dmamap_load_mbuf(bus_dma_tag_t t, bus_dmamap_t map, 655 struct mbuf *m0, int flags) 656 { 657 struct mbuf *m; 658 int error = 0; 659 660 /* 661 * Make sure that on error condition we return "no valid mappings". 662 */ 663 map->dm_mapsize = m0->m_pkthdr.len; 664 map->dm_nsegs = 0; 665 666 for (m = m0; m != NULL && error == 0; m = m->m_next) { 667 if (m->m_len == 0) 668 continue; 669 error = iommu_dmamap_load_buffer(t, map, m->m_data, m->m_len, 670 NULL, flags); 671 } 672 673 if (error) 674 iommu_dmamap_unload(t, map); 675 676 return (error); 677 } 678 679 /* 680 * Like _bus_dmamap_load(), but for uios. 681 */ 682 int 683 iommu_dmamap_load_uio(bus_dma_tag_t t, bus_dmamap_t map, 684 struct uio *uio, int flags) 685 { 686 687 panic("_bus_dmamap_load_uio: not implemented"); 688 } 689 690 /* 691 * Like _bus_dmamap_load(), but for raw memory allocated with 692 * bus_dmamem_alloc(). 693 */ 694 int 695 iommu_dmamap_load_raw(bus_dma_tag_t t, bus_dmamap_t map, 696 bus_dma_segment_t *segs, int nsegs, bus_size_t size, 697 int flags) 698 { 699 struct iommu_softc *sc = t->_cookie; 700 struct vm_page *m; 701 paddr_t pa; 702 bus_addr_t dva; 703 bus_size_t sgsize; 704 struct pglist *mlist; 705 int pagesz = PAGE_SIZE; 706 int error; 707 708 map->dm_nsegs = 0; 709 710 /* Allocate IOMMU resources */ 711 if ((error = iommu_dvma_alloc(sc, map, segs[0]._ds_va, size, 712 flags, &dva, &sgsize)) != 0) 713 return (error); 714 715 /* 716 * Note DVMA address in case bus_dmamem_map() is called later. 717 * It can then insure cache coherency by choosing a KVA that 718 * is aligned to `ds_addr'. 719 */ 720 segs[0].ds_addr = dva; 721 segs[0].ds_len = size; 722 723 map->dm_segs[0].ds_addr = dva; 724 map->dm_segs[0].ds_len = size; 725 map->dm_segs[0]._ds_sgsize = sgsize; 726 727 /* Map physical pages into IOMMU */ 728 mlist = segs[0]._ds_mlist; 729 for (m = TAILQ_FIRST(mlist); m != NULL; m = TAILQ_NEXT(m,pageq.queue)) { 730 if (sgsize == 0) 731 panic("iommu_dmamap_load_raw: size botch"); 732 pa = VM_PAGE_TO_PHYS(m); 733 iommu_enter(sc, dva, pa); 734 dva += pagesz; 735 sgsize -= pagesz; 736 } 737 738 map->dm_nsegs = 1; 739 map->dm_mapsize = size; 740 741 return (0); 742 } 743 744 /* 745 * Unload an IOMMU DMA map. 746 */ 747 void 748 iommu_dmamap_unload(bus_dma_tag_t t, bus_dmamap_t map) 749 { 750 struct iommu_softc *sc = t->_cookie; 751 bus_dma_segment_t *segs = map->dm_segs; 752 int nsegs = map->dm_nsegs; 753 bus_addr_t dva; 754 bus_size_t len; 755 int i; 756 757 for (i = 0; i < nsegs; i++) { 758 dva = segs[i].ds_addr & -PAGE_SIZE; 759 len = segs[i]._ds_sgsize; 760 761 iommu_remove(sc, dva, len); 762 vmem_xfree(sc->sc_dvmamap, dva, len); 763 } 764 765 /* Mark the mappings as invalid. */ 766 map->dm_mapsize = 0; 767 map->dm_nsegs = 0; 768 } 769 770 /* 771 * DMA map synchronization. 772 */ 773 void 774 iommu_dmamap_sync(bus_dma_tag_t t, bus_dmamap_t map, 775 bus_addr_t offset, bus_size_t len, int ops) 776 { 777 778 /* 779 * XXX Should flush CPU write buffers. 780 */ 781 } 782 783 /* 784 * Map DMA-safe memory. 785 */ 786 int 787 iommu_dmamem_map(bus_dma_tag_t t, bus_dma_segment_t *segs, int nsegs, 788 size_t size, void **kvap, int flags) 789 { 790 struct iommu_softc *sc = t->_cookie; 791 struct vm_page *m; 792 vaddr_t va; 793 bus_addr_t addr; 794 struct pglist *mlist; 795 int cbit; 796 u_long align; 797 int pagesz = PAGE_SIZE; 798 799 if (nsegs != 1) 800 panic("iommu_dmamem_map: nsegs = %d", nsegs); 801 802 cbit = sc->sc_cachecoherent ? 0 : PMAP_NC; 803 align = dvma_cachealign ? dvma_cachealign : pagesz; 804 805 size = round_page(size); 806 807 /* 808 * In case the segment has already been loaded by 809 * iommu_dmamap_load_raw(), find a region of kernel virtual 810 * addresses that can accommodate our alignment requirements. 811 */ 812 va = _bus_dma_valloc_skewed(size, 0, align, 813 segs[0].ds_addr & (align - 1)); 814 if (va == 0) 815 return (ENOMEM); 816 817 segs[0]._ds_va = va; 818 *kvap = (void *)va; 819 820 /* 821 * Map the pages allocated in _bus_dmamem_alloc() to the 822 * kernel virtual address space. 823 */ 824 mlist = segs[0]._ds_mlist; 825 for (m = TAILQ_FIRST(mlist); m != NULL; m = TAILQ_NEXT(m,pageq.queue)) { 826 827 if (size == 0) 828 panic("iommu_dmamem_map: size botch"); 829 830 addr = VM_PAGE_TO_PHYS(m); 831 pmap_kenter_pa(va, addr | cbit, 832 VM_PROT_READ | VM_PROT_WRITE, 0); 833 #if 0 834 if (flags & BUS_DMA_COHERENT) 835 /* XXX */; 836 #endif 837 va += pagesz; 838 size -= pagesz; 839 } 840 pmap_update(pmap_kernel()); 841 842 return (0); 843 } 844 845 void 846 iommu_dmamem_unmap(bus_dma_tag_t t, void *kva, size_t size) 847 { 848 849 #ifdef DIAGNOSTIC 850 if ((u_long)kva & PAGE_MASK) 851 panic("iommu_dmamem_unmap"); 852 #endif 853 854 size = round_page(size); 855 pmap_kremove((vaddr_t)kva, size); 856 pmap_update(pmap_kernel()); 857 uvm_unmap(kernel_map, (vaddr_t)kva, (vaddr_t)kva + size); 858 } 859 860 861 /* 862 * mmap(2)'ing DMA-safe memory. 863 */ 864 paddr_t 865 iommu_dmamem_mmap(bus_dma_tag_t t, bus_dma_segment_t *segs, int nsegs, 866 off_t off, int prot, int flags) 867 { 868 869 panic("_bus_dmamem_mmap: not implemented"); 870 } 871
Indexes created Sun Oct 25 00:26:45 UTC 2026