iommu.c revision 1.81
1 /* $NetBSD: iommu.c,v 1.81 2005/04/01 11:59:34 yamt Exp $ */ 2 3 /* 4 * Copyright (c) 1996 5 * The President and Fellows of Harvard College. All rights reserved. 6 * Copyright (c) 1995 Paul Kranenburg 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 3. All advertising materials mentioning features or use of this software 17 * must display the following acknowledgement: 18 * This product includes software developed by Aaron Brown and 19 * Harvard University. 20 * This product includes software developed by Paul Kranenburg. 21 * 4. Neither the name of the University nor the names of its contributors 22 * may be used to endorse or promote products derived from this software 23 * without specific prior written permission. 24 * 25 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 26 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 27 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 28 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 29 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 30 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 31 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 32 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 33 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 34 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 35 * SUCH DAMAGE. 36 * 37 */ 38 39 #include <sys/cdefs.h> 40 __KERNEL_RCSID(0, "$NetBSD: iommu.c,v 1.81 2005/04/01 11:59:34 yamt Exp $"); 41 42 #include "opt_sparc_arch.h" 43 44 #include <sys/param.h> 45 #include <sys/extent.h> 46 #include <sys/malloc.h> 47 #include <sys/queue.h> 48 #include <sys/systm.h> 49 #include <sys/device.h> 50 #include <sys/proc.h> 51 52 #include <uvm/uvm.h> 53 54 #define _SPARC_BUS_DMA_PRIVATE 55 #include <machine/bus.h> 56 #include <machine/autoconf.h> 57 #include <machine/ctlreg.h> 58 #include <sparc/sparc/asm.h> 59 #include <sparc/sparc/vaddrs.h> 60 #include <sparc/sparc/cpuvar.h> 61 #include <sparc/sparc/iommureg.h> 62 #include <sparc/sparc/iommuvar.h> 63 64 struct iommu_softc { 65 struct device sc_dev; /* base device */ 66 struct iommureg *sc_reg; 67 u_int sc_pagesize; 68 u_int sc_range; 69 bus_addr_t sc_dvmabase; 70 iopte_t *sc_ptes; 71 int sc_cachecoherent; 72 /* 73 * Note: operations on the extent map are being protected with 74 * splhigh(), since we cannot predict at which interrupt priority 75 * our clients will run. 76 */ 77 struct sparc_bus_dma_tag sc_dmatag; 78 struct extent *sc_dvmamap; 79 }; 80 81 /* autoconfiguration driver */ 82 int iommu_print __P((void *, const char *)); 83 void iommu_attach __P((struct device *, struct device *, void *)); 84 int iommu_match __P((struct device *, struct cfdata *, void *)); 85 86 #if defined(SUN4M) 87 static void iommu_copy_prom_entries __P((struct iommu_softc *)); 88 #endif 89 90 CFATTACH_DECL(iommu, sizeof(struct iommu_softc), 91 iommu_match, iommu_attach, NULL, NULL); 92 93 /* IOMMU DMA map functions */ 94 int iommu_dmamap_create __P((bus_dma_tag_t, bus_size_t, int, bus_size_t, 95 bus_size_t, int, bus_dmamap_t *)); 96 int iommu_dmamap_load __P((bus_dma_tag_t, bus_dmamap_t, void *, 97 bus_size_t, struct proc *, int)); 98 int iommu_dmamap_load_mbuf __P((bus_dma_tag_t, bus_dmamap_t, 99 struct mbuf *, int)); 100 int iommu_dmamap_load_uio __P((bus_dma_tag_t, bus_dmamap_t, 101 struct uio *, int)); 102 int iommu_dmamap_load_raw __P((bus_dma_tag_t, bus_dmamap_t, 103 bus_dma_segment_t *, int, bus_size_t, int)); 104 void iommu_dmamap_unload __P((bus_dma_tag_t, bus_dmamap_t)); 105 void iommu_dmamap_sync __P((bus_dma_tag_t, bus_dmamap_t, bus_addr_t, 106 bus_size_t, int)); 107 108 int iommu_dmamem_map __P((bus_dma_tag_t tag, bus_dma_segment_t *segs, 109 int nsegs, size_t size, caddr_t *kvap, int flags)); 110 void iommu_dmamem_unmap __P((bus_dma_tag_t t, caddr_t kva, size_t size)); 111 paddr_t iommu_dmamem_mmap __P((bus_dma_tag_t tag, bus_dma_segment_t *segs, 112 int nsegs, off_t off, int prot, int flags)); 113 int iommu_dvma_alloc(struct iommu_softc *, bus_dmamap_t, vaddr_t, 114 bus_size_t, int, bus_addr_t *, bus_size_t *); 115 116 /* 117 * Print the location of some iommu-attached device (called just 118 * before attaching that device). If `iommu' is not NULL, the 119 * device was found but not configured; print the iommu as well. 120 * Return UNCONF (config_find ignores this if the device was configured). 121 */ 122 int 123 iommu_print(args, iommu) 124 void *args; 125 const char *iommu; 126 { 127 struct iommu_attach_args *ia = args; 128 129 if (iommu) 130 aprint_normal("%s at %s", ia->iom_name, iommu); 131 return (UNCONF); 132 } 133 134 int 135 iommu_match(parent, cf, aux) 136 struct device *parent; 137 struct cfdata *cf; 138 void *aux; 139 { 140 struct mainbus_attach_args *ma = aux; 141 142 if (CPU_ISSUN4 || CPU_ISSUN4C) 143 return (0); 144 return (strcmp(cf->cf_name, ma->ma_name) == 0); 145 } 146 147 /* 148 * Attach the iommu. 149 */ 150 void 151 iommu_attach(parent, self, aux) 152 struct device *parent; 153 struct device *self; 154 void *aux; 155 { 156 #if defined(SUN4M) 157 struct iommu_softc *sc = (struct iommu_softc *)self; 158 struct mainbus_attach_args *ma = aux; 159 struct sparc_bus_dma_tag *dmat = &sc->sc_dmatag; 160 bus_space_handle_t bh; 161 int node; 162 int js1_implicit_iommu; 163 int i, s; 164 u_int iopte_table_pa; 165 struct pglist mlist; 166 u_int size; 167 struct vm_page *m; 168 vaddr_t va; 169 170 dmat->_cookie = sc; 171 dmat->_dmamap_create = iommu_dmamap_create; 172 dmat->_dmamap_destroy = _bus_dmamap_destroy; 173 dmat->_dmamap_load = iommu_dmamap_load; 174 dmat->_dmamap_load_mbuf = iommu_dmamap_load_mbuf; 175 dmat->_dmamap_load_uio = iommu_dmamap_load_uio; 176 dmat->_dmamap_load_raw = iommu_dmamap_load_raw; 177 dmat->_dmamap_unload = iommu_dmamap_unload; 178 dmat->_dmamap_sync = iommu_dmamap_sync; 179 180 dmat->_dmamem_alloc = _bus_dmamem_alloc; 181 dmat->_dmamem_free = _bus_dmamem_free; 182 dmat->_dmamem_map = iommu_dmamem_map; 183 dmat->_dmamem_unmap = _bus_dmamem_unmap; 184 dmat->_dmamem_mmap = iommu_dmamem_mmap; 185 186 /* 187 * JS1/OF device tree does not have an iommu node and sbus 188 * node is directly under root. mainbus_attach detects this 189 * and calls us with sbus node instead so that we can attach 190 * implicit iommu and attach that sbus node under it. 191 */ 192 node = ma->ma_node; 193 if (strcmp(prom_getpropstring(node, "name"), "sbus") == 0) 194 js1_implicit_iommu = 1; 195 else 196 js1_implicit_iommu = 0; 197 198 /* 199 * Map registers into our space. The PROM may have done this 200 * already, but I feel better if we have our own copy. Plus, the 201 * prom doesn't map the entire register set. 202 * 203 * XXX struct iommureg is bigger than ra->ra_len; what are the 204 * other fields for? 205 */ 206 if (bus_space_map(ma->ma_bustag, ma->ma_paddr, 207 sizeof(struct iommureg), 0, &bh) != 0) { 208 printf("iommu_attach: cannot map registers\n"); 209 return; 210 } 211 sc->sc_reg = (struct iommureg *)bh; 212 213 sc->sc_cachecoherent = js1_implicit_iommu ? 0 214 : node_has_property(node, "cache-coherence?"); 215 if (CACHEINFO.c_enabled == 0) /* XXX - is this correct? */ 216 sc->sc_cachecoherent = 0; 217 218 sc->sc_pagesize = js1_implicit_iommu ? PAGE_SIZE 219 : prom_getpropint(node, "page-size", PAGE_SIZE), 220 221 /* 222 * Allocate memory for I/O pagetables. 223 * This takes 64K of contiguous physical memory to map 64M of 224 * DVMA space (starting at IOMMU_DVMA_BASE). 225 * The table must be aligned on a (-IOMMU_DVMA_BASE/pagesize) 226 * boundary (i.e. 64K for 64M of DVMA space). 227 */ 228 229 size = ((0 - IOMMU_DVMA_BASE) / sc->sc_pagesize) * sizeof(iopte_t); 230 if (uvm_pglistalloc(size, vm_first_phys, vm_first_phys+vm_num_phys, 231 size, 0, &mlist, 1, 0) != 0) 232 panic("iommu_attach: no memory"); 233 234 va = uvm_km_alloc(kernel_map, size, 0, UVM_KMF_VAONLY); 235 if (va == 0) 236 panic("iommu_attach: no memory"); 237 238 sc->sc_ptes = (iopte_t *)va; 239 240 m = TAILQ_FIRST(&mlist); 241 iopte_table_pa = VM_PAGE_TO_PHYS(m); 242 243 /* Map the pages */ 244 for (; m != NULL; m = TAILQ_NEXT(m,pageq)) { 245 paddr_t pa = VM_PAGE_TO_PHYS(m); 246 pmap_kenter_pa(va, pa | PMAP_NC, VM_PROT_READ | VM_PROT_WRITE); 247 va += PAGE_SIZE; 248 } 249 pmap_update(pmap_kernel()); 250 251 /* 252 * Copy entries from current IOMMU table. 253 * XXX - Why do we need to do this? 254 */ 255 iommu_copy_prom_entries(sc); 256 257 /* 258 * Now we can install our new pagetable into the IOMMU 259 */ 260 sc->sc_range = 0 - IOMMU_DVMA_BASE; 261 sc->sc_dvmabase = IOMMU_DVMA_BASE; 262 263 /* calculate log2(sc->sc_range/16MB) */ 264 i = ffs(sc->sc_range/(1 << 24)) - 1; 265 if ((1 << i) != (sc->sc_range/(1 << 24))) 266 panic("iommu: bad range: %d", i); 267 268 s = splhigh(); 269 IOMMU_FLUSHALL(sc); 270 271 /* Load range and physical address of PTEs */ 272 sc->sc_reg->io_cr = (sc->sc_reg->io_cr & ~IOMMU_CTL_RANGE) | 273 (i << IOMMU_CTL_RANGESHFT) | IOMMU_CTL_ME; 274 sc->sc_reg->io_bar = (iopte_table_pa >> 4) & IOMMU_BAR_IBA; 275 276 IOMMU_FLUSHALL(sc); 277 splx(s); 278 279 printf(": version 0x%x/0x%x, page-size %d, range %dMB\n", 280 (sc->sc_reg->io_cr & IOMMU_CTL_VER) >> 24, 281 (sc->sc_reg->io_cr & IOMMU_CTL_IMPL) >> 28, 282 sc->sc_pagesize, 283 sc->sc_range >> 20); 284 285 sc->sc_dvmamap = extent_create("iommudvma", 286 IOMMU_DVMA_BASE, IOMMU_DVMA_END, 287 M_DEVBUF, 0, 0, EX_NOWAIT); 288 if (sc->sc_dvmamap == NULL) 289 panic("iommu: unable to allocate DVMA map"); 290 291 /* 292 * If we are attaching implicit iommu on JS1/OF we do not have 293 * an iommu node to traverse, instead mainbus_attach passed us 294 * sbus node in ma.ma_node. Attach it as the only iommu child. 295 */ 296 if (js1_implicit_iommu) { 297 struct iommu_attach_args ia; 298 struct openprom_addr sbus_iommu_reg = { 0, 0x10001000, 0x28 }; 299 300 bzero(&ia, sizeof ia); 301 302 /* Propagate BUS & DMA tags */ 303 ia.iom_bustag = ma->ma_bustag; 304 ia.iom_dmatag = &sc->sc_dmatag; 305 306 ia.iom_name = "sbus"; 307 ia.iom_node = node; 308 ia.iom_reg = &sbus_iommu_reg; 309 ia.iom_nreg = 1; 310 311 (void) config_found(&sc->sc_dev, (void *)&ia, iommu_print); 312 return; 313 } 314 315 /* 316 * Loop through ROM children (expect Sbus among them). 317 */ 318 for (node = firstchild(node); node; node = nextsibling(node)) { 319 struct iommu_attach_args ia; 320 321 bzero(&ia, sizeof ia); 322 ia.iom_name = prom_getpropstring(node, "name"); 323 324 /* Propagate BUS & DMA tags */ 325 ia.iom_bustag = ma->ma_bustag; 326 ia.iom_dmatag = &sc->sc_dmatag; 327 328 ia.iom_node = node; 329 330 ia.iom_reg = NULL; 331 prom_getprop(node, "reg", sizeof(struct openprom_addr), 332 &ia.iom_nreg, &ia.iom_reg); 333 334 (void) config_found(&sc->sc_dev, (void *)&ia, iommu_print); 335 if (ia.iom_reg != NULL) 336 free(ia.iom_reg, M_DEVBUF); 337 } 338 #endif 339 } 340 341 #if defined(SUN4M) 342 static void 343 iommu_copy_prom_entries(sc) 344 struct iommu_softc *sc; 345 { 346 u_int pbase, pa; 347 u_int range; 348 iopte_t *tpte_p; 349 u_int pagesz = sc->sc_pagesize; 350 int use_ac = (cpuinfo.cpu_impl == 4 && cpuinfo.mxcc); 351 u_int mmupcr_save; 352 353 /* 354 * We read in the original table using MMU bypass and copy all 355 * of its entries to the appropriate place in our new table, 356 * even if the sizes are different. 357 * This is pretty easy since we know DVMA ends at 0xffffffff. 358 */ 359 360 range = (1 << 24) << 361 ((sc->sc_reg->io_cr & IOMMU_CTL_RANGE) >> IOMMU_CTL_RANGESHFT); 362 363 pbase = (sc->sc_reg->io_bar & IOMMU_BAR_IBA) << 364 (14 - IOMMU_BAR_IBASHFT); 365 366 if (use_ac) { 367 /* 368 * Set MMU AC bit so we'll still read from the cache 369 * in by-pass mode. 370 */ 371 mmupcr_save = lda(SRMMU_PCR, ASI_SRMMU); 372 sta(SRMMU_PCR, ASI_SRMMU, mmupcr_save | VIKING_PCR_AC); 373 } else 374 mmupcr_save = 0; /* XXX - avoid GCC `unintialized' warning */ 375 376 /* Flush entire IOMMU TLB before messing with the in-memory tables */ 377 IOMMU_FLUSHALL(sc); 378 379 /* 380 * tpte_p = top of our PTE table 381 * pa = top of current PTE table 382 * Then work downwards and copy entries until we hit the bottom 383 * of either table. 384 */ 385 for (tpte_p = &sc->sc_ptes[((0 - IOMMU_DVMA_BASE)/pagesz) - 1], 386 pa = (u_int)pbase + (range/pagesz - 1)*sizeof(iopte_t); 387 tpte_p >= &sc->sc_ptes[0] && pa >= (u_int)pbase; 388 tpte_p--, pa -= sizeof(iopte_t)) { 389 390 *tpte_p = lda(pa, ASI_BYPASS); 391 } 392 393 if (use_ac) { 394 /* restore mmu after bug-avoidance */ 395 sta(SRMMU_PCR, ASI_SRMMU, mmupcr_save); 396 } 397 } 398 #endif 399 400 static void 401 iommu_enter(struct iommu_softc *sc, bus_addr_t dva, paddr_t pa) 402 { 403 int pte; 404 405 /* This routine relies on the fact that sc->sc_pagesize == PAGE_SIZE */ 406 407 #ifdef DIAGNOSTIC 408 if (dva < sc->sc_dvmabase) 409 panic("iommu_enter: dva 0x%lx not in DVMA space", (long)dva); 410 #endif 411 412 pte = atop(pa) << IOPTE_PPNSHFT; 413 pte &= IOPTE_PPN; 414 pte |= IOPTE_V | IOPTE_W | (sc->sc_cachecoherent ? IOPTE_C : 0); 415 sc->sc_ptes[atop(dva - sc->sc_dvmabase)] = pte; 416 IOMMU_FLUSHPAGE(sc, dva); 417 } 418 419 /* 420 * iommu_remove: removes mappings created by iommu_enter 421 */ 422 static void 423 iommu_remove(struct iommu_softc *sc, bus_addr_t dva, bus_size_t len) 424 { 425 u_int pagesz = sc->sc_pagesize; 426 bus_addr_t base = sc->sc_dvmabase; 427 428 #ifdef DEBUG 429 if (dva < base) 430 panic("iommu_remove: va 0x%lx not in DVMA space", (long)dva); 431 #endif 432 433 while ((long)len > 0) { 434 #ifdef notyet 435 #ifdef DEBUG 436 if ((sc->sc_ptes[atop(dva - base)] & IOPTE_V) == 0) 437 panic("iommu_remove: clearing invalid pte at dva 0x%lx", 438 (long)dva); 439 #endif 440 #endif 441 sc->sc_ptes[atop(dva - base)] = 0; 442 IOMMU_FLUSHPAGE(sc, dva); 443 len -= pagesz; 444 dva += pagesz; 445 } 446 } 447 448 #if 0 /* These registers aren't there??? */ 449 void 450 iommu_error() 451 { 452 struct iommu_softc *sc = X; 453 struct iommureg *iop = sc->sc_reg; 454 455 printf("iommu: afsr 0x%x, afar 0x%x\n", iop->io_afsr, iop->io_afar); 456 printf("iommu: mfsr 0x%x, mfar 0x%x\n", iop->io_mfsr, iop->io_mfar); 457 } 458 int 459 iommu_alloc(va, len) 460 u_int va, len; 461 { 462 struct iommu_softc *sc = X; 463 int off, tva, iovaddr, pte; 464 paddr_t pa; 465 466 off = (int)va & PGOFSET; 467 len = round_page(len + off); 468 va -= off; 469 470 if ((int)sc->sc_dvmacur + len > 0) 471 sc->sc_dvmacur = sc->sc_dvmabase; 472 473 iovaddr = tva = sc->sc_dvmacur; 474 sc->sc_dvmacur += len; 475 while (len) { 476 (void) pmap_extract(pmap_kernel(), va, &pa); 477 478 #define IOMMU_PPNSHIFT 8 479 #define IOMMU_V 0x00000002 480 #define IOMMU_W 0x00000004 481 482 pte = atop(pa) << IOMMU_PPNSHIFT; 483 pte |= IOMMU_V | IOMMU_W; 484 sta(sc->sc_ptes + atop(tva - sc->sc_dvmabase), ASI_BYPASS, pte); 485 sc->sc_reg->io_flushpage = tva; 486 len -= PAGE_SIZE; 487 va += PAGE_SIZE; 488 tva += PAGE_SIZE; 489 } 490 return iovaddr + off; 491 } 492 #endif 493 494 495 /* 496 * IOMMU DMA map functions. 497 */ 498 int 499 iommu_dmamap_create(t, size, nsegments, maxsegsz, boundary, flags, dmamp) 500 bus_dma_tag_t t; 501 bus_size_t size; 502 int nsegments; 503 bus_size_t maxsegsz; 504 bus_size_t boundary; 505 int flags; 506 bus_dmamap_t *dmamp; 507 { 508 struct iommu_softc *sc = t->_cookie; 509 bus_dmamap_t map; 510 int error; 511 512 if ((error = _bus_dmamap_create(t, size, nsegments, maxsegsz, 513 boundary, flags, &map)) != 0) 514 return (error); 515 516 if ((flags & BUS_DMA_24BIT) != 0) { 517 /* Limit this map to the range usable by `24-bit' devices */ 518 map->_dm_ex_start = D24_DVMA_BASE; 519 map->_dm_ex_end = D24_DVMA_END; 520 } else { 521 /* Enable allocations from the entire map */ 522 map->_dm_ex_start = sc->sc_dvmamap->ex_start; 523 map->_dm_ex_end = sc->sc_dvmamap->ex_end; 524 } 525 526 *dmamp = map; 527 return (0); 528 } 529 530 /* 531 * Internal routine to allocate space in the IOMMU map. 532 */ 533 int 534 iommu_dvma_alloc(sc, map, va, len, flags, dvap, sgsizep) 535 struct iommu_softc *sc; 536 bus_dmamap_t map; 537 vaddr_t va; 538 bus_size_t len; 539 int flags; 540 bus_addr_t *dvap; 541 bus_size_t *sgsizep; 542 { 543 bus_size_t sgsize; 544 u_long align, voff, dvaddr; 545 int s, error; 546 int pagesz = PAGE_SIZE; 547 548 /* 549 * Remember page offset, then truncate the buffer address to 550 * a page boundary. 551 */ 552 voff = va & (pagesz - 1); 553 va &= -pagesz; 554 555 if (len > map->_dm_size) 556 return (EINVAL); 557 558 sgsize = (len + voff + pagesz - 1) & -pagesz; 559 align = dvma_cachealign ? dvma_cachealign : map->_dm_align; 560 561 s = splhigh(); 562 error = extent_alloc_subregion1(sc->sc_dvmamap, 563 map->_dm_ex_start, map->_dm_ex_end, 564 sgsize, align, va & (align-1), 565 map->_dm_boundary, 566 (flags & BUS_DMA_NOWAIT) == 0 567 ? EX_WAITOK : EX_NOWAIT, 568 &dvaddr); 569 splx(s); 570 *dvap = (bus_addr_t)dvaddr; 571 *sgsizep = sgsize; 572 return (error); 573 } 574 575 /* 576 * Prepare buffer for DMA transfer. 577 */ 578 int 579 iommu_dmamap_load(t, map, buf, buflen, p, flags) 580 bus_dma_tag_t t; 581 bus_dmamap_t map; 582 void *buf; 583 bus_size_t buflen; 584 struct proc *p; 585 int flags; 586 { 587 struct iommu_softc *sc = t->_cookie; 588 bus_size_t sgsize; 589 bus_addr_t dva; 590 vaddr_t va = (vaddr_t)buf; 591 int pagesz = PAGE_SIZE; 592 pmap_t pmap; 593 int error; 594 595 /* 596 * Make sure that on error condition we return "no valid mappings". 597 */ 598 map->dm_nsegs = 0; 599 600 /* Allocate IOMMU resources */ 601 if ((error = iommu_dvma_alloc(sc, map, va, buflen, flags, 602 &dva, &sgsize)) != 0) 603 return (error); 604 605 if (sc->sc_cachecoherent == 0) 606 cache_flush(buf, buflen); /* XXX - move to bus_dma_sync? */ 607 608 /* 609 * We always use just one segment. 610 */ 611 map->dm_mapsize = buflen; 612 map->dm_nsegs = 1; 613 map->dm_segs[0].ds_addr = dva + (va & (pagesz - 1)); 614 map->dm_segs[0].ds_len = buflen; 615 map->dm_segs[0]._ds_sgsize = sgsize; 616 617 if (p != NULL) 618 pmap = p->p_vmspace->vm_map.pmap; 619 else 620 pmap = pmap_kernel(); 621 622 for (; sgsize != 0; ) { 623 paddr_t pa; 624 /* 625 * Get the physical address for this page. 626 */ 627 if (!pmap_extract(pmap, va, &pa)) { 628 iommu_dmamap_unload(t, map); 629 return (EFAULT); 630 } 631 632 iommu_enter(sc, dva, pa); 633 634 dva += pagesz; 635 va += pagesz; 636 sgsize -= pagesz; 637 } 638 639 return (0); 640 } 641 642 /* 643 * Like _bus_dmamap_load(), but for mbufs. 644 */ 645 int 646 iommu_dmamap_load_mbuf(t, map, m, flags) 647 bus_dma_tag_t t; 648 bus_dmamap_t map; 649 struct mbuf *m; 650 int flags; 651 { 652 653 panic("_bus_dmamap_load_mbuf: not implemented"); 654 } 655 656 /* 657 * Like _bus_dmamap_load(), but for uios. 658 */ 659 int 660 iommu_dmamap_load_uio(t, map, uio, flags) 661 bus_dma_tag_t t; 662 bus_dmamap_t map; 663 struct uio *uio; 664 int flags; 665 { 666 667 panic("_bus_dmamap_load_uio: not implemented"); 668 } 669 670 /* 671 * Like _bus_dmamap_load(), but for raw memory allocated with 672 * bus_dmamem_alloc(). 673 */ 674 int 675 iommu_dmamap_load_raw(t, map, segs, nsegs, size, flags) 676 bus_dma_tag_t t; 677 bus_dmamap_t map; 678 bus_dma_segment_t *segs; 679 int nsegs; 680 bus_size_t size; 681 int flags; 682 { 683 struct iommu_softc *sc = t->_cookie; 684 struct vm_page *m; 685 paddr_t pa; 686 bus_addr_t dva; 687 bus_size_t sgsize; 688 struct pglist *mlist; 689 int pagesz = PAGE_SIZE; 690 int error; 691 692 map->dm_nsegs = 0; 693 694 /* Allocate IOMMU resources */ 695 if ((error = iommu_dvma_alloc(sc, map, segs[0]._ds_va, size, 696 flags, &dva, &sgsize)) != 0) 697 return (error); 698 699 /* 700 * Note DVMA address in case bus_dmamem_map() is called later. 701 * It can then insure cache coherency by choosing a KVA that 702 * is aligned to `ds_addr'. 703 */ 704 segs[0].ds_addr = dva; 705 segs[0].ds_len = size; 706 707 map->dm_segs[0].ds_addr = dva; 708 map->dm_segs[0].ds_len = size; 709 map->dm_segs[0]._ds_sgsize = sgsize; 710 711 /* Map physical pages into IOMMU */ 712 mlist = segs[0]._ds_mlist; 713 for (m = TAILQ_FIRST(mlist); m != NULL; m = TAILQ_NEXT(m,pageq)) { 714 if (sgsize == 0) 715 panic("iommu_dmamap_load_raw: size botch"); 716 pa = VM_PAGE_TO_PHYS(m); 717 iommu_enter(sc, dva, pa); 718 dva += pagesz; 719 sgsize -= pagesz; 720 } 721 722 map->dm_nsegs = 1; 723 map->dm_mapsize = size; 724 725 return (0); 726 } 727 728 /* 729 * Unload an IOMMU DMA map. 730 */ 731 void 732 iommu_dmamap_unload(t, map) 733 bus_dma_tag_t t; 734 bus_dmamap_t map; 735 { 736 struct iommu_softc *sc = t->_cookie; 737 bus_dma_segment_t *segs = map->dm_segs; 738 int nsegs = map->dm_nsegs; 739 bus_addr_t dva; 740 bus_size_t len; 741 int i, s, error; 742 743 for (i = 0; i < nsegs; i++) { 744 dva = segs[i].ds_addr & -PAGE_SIZE; 745 len = segs[i]._ds_sgsize; 746 747 iommu_remove(sc, dva, len); 748 s = splhigh(); 749 error = extent_free(sc->sc_dvmamap, dva, len, EX_NOWAIT); 750 splx(s); 751 if (error != 0) 752 printf("warning: %ld of DVMA space lost\n", (long)len); 753 } 754 755 /* Mark the mappings as invalid. */ 756 map->dm_mapsize = 0; 757 map->dm_nsegs = 0; 758 } 759 760 /* 761 * DMA map synchronization. 762 */ 763 void 764 iommu_dmamap_sync(t, map, offset, len, ops) 765 bus_dma_tag_t t; 766 bus_dmamap_t map; 767 bus_addr_t offset; 768 bus_size_t len; 769 int ops; 770 { 771 772 /* 773 * XXX Should flush CPU write buffers. 774 */ 775 } 776 777 /* 778 * Map DMA-safe memory. 779 */ 780 int 781 iommu_dmamem_map(t, segs, nsegs, size, kvap, flags) 782 bus_dma_tag_t t; 783 bus_dma_segment_t *segs; 784 int nsegs; 785 size_t size; 786 caddr_t *kvap; 787 int flags; 788 { 789 struct iommu_softc *sc = t->_cookie; 790 struct vm_page *m; 791 vaddr_t va; 792 bus_addr_t addr; 793 struct pglist *mlist; 794 int cbit; 795 u_long align; 796 int pagesz = PAGE_SIZE; 797 798 if (nsegs != 1) 799 panic("iommu_dmamem_map: nsegs = %d", nsegs); 800 801 cbit = sc->sc_cachecoherent ? 0 : PMAP_NC; 802 align = dvma_cachealign ? dvma_cachealign : pagesz; 803 804 size = round_page(size); 805 806 /* 807 * In case the segment has already been loaded by 808 * iommu_dmamap_load_raw(), find a region of kernel virtual 809 * addresses that can accomodate our aligment requirements. 810 */ 811 va = _bus_dma_valloc_skewed(size, 0, align, 812 segs[0].ds_addr & (align - 1)); 813 if (va == 0) 814 return (ENOMEM); 815 816 segs[0]._ds_va = va; 817 *kvap = (caddr_t)va; 818 819 /* 820 * Map the pages allocated in _bus_dmamem_alloc() to the 821 * kernel virtual address space. 822 */ 823 mlist = segs[0]._ds_mlist; 824 for (m = TAILQ_FIRST(mlist); m != NULL; m = TAILQ_NEXT(m,pageq)) { 825 826 if (size == 0) 827 panic("iommu_dmamem_map: size botch"); 828 829 addr = VM_PAGE_TO_PHYS(m); 830 pmap_kenter_pa(va, addr | cbit, VM_PROT_READ | VM_PROT_WRITE); 831 #if 0 832 if (flags & BUS_DMA_COHERENT) 833 /* XXX */; 834 #endif 835 va += pagesz; 836 size -= pagesz; 837 } 838 pmap_update(pmap_kernel()); 839 840 return (0); 841 } 842 843 void 844 iommu_dmamem_unmap(t, kva, size) 845 bus_dma_tag_t t; 846 caddr_t kva; 847 size_t size; 848 { 849 850 #ifdef DIAGNOSTIC 851 if ((u_long)kva & PAGE_MASK) 852 panic("iommu_dmamem_unmap"); 853 #endif 854 855 size = round_page(size); 856 pmap_kremove((vaddr_t)kva, size); 857 pmap_update(pmap_kernel()); 858 uvm_unmap(kernel_map, (vaddr_t)kva, (vaddr_t)kva + size); 859 } 860 861 862 /* 863 * mmap(2)'ing DMA-safe memory. 864 */ 865 paddr_t 866 iommu_dmamem_mmap(t, segs, nsegs, off, prot, flags) 867 bus_dma_tag_t t; 868 bus_dma_segment_t *segs; 869 int nsegs; 870 off_t off; 871 int prot, flags; 872 { 873 874 panic("_bus_dmamem_mmap: not implemented"); 875 } 876
Indexes created Tue Sep 30 20:09:53 GMT 2025