iommu.c revision 1.96
1 /* $NetBSD: iommu.c,v 1.96 2009/12/10 12:29:44 nakayama Exp $ */ 2 3 /* 4 * Copyright (c) 1999, 2000 Matthew R. Green 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 17 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 18 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 19 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 20 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 21 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 22 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 23 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 24 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 * SUCH DAMAGE. 27 */ 28 29 /* 30 * Copyright (c) 2001, 2002 Eduardo Horvath 31 * All rights reserved. 32 * 33 * Redistribution and use in source and binary forms, with or without 34 * modification, are permitted provided that the following conditions 35 * are met: 36 * 1. Redistributions of source code must retain the above copyright 37 * notice, this list of conditions and the following disclaimer. 38 * 2. Redistributions in binary form must reproduce the above copyright 39 * notice, this list of conditions and the following disclaimer in the 40 * documentation and/or other materials provided with the distribution. 41 * 3. The name of the author may not be used to endorse or promote products 42 * derived from this software without specific prior written permission. 43 * 44 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 45 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 46 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 47 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 48 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 49 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 50 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 51 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 52 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 53 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 54 * SUCH DAMAGE. 55 */ 56 57 /* 58 * UltraSPARC IOMMU support; used by both the sbus and pci code. 59 */ 60 61 #include <sys/cdefs.h> 62 __KERNEL_RCSID(0, "$NetBSD: iommu.c,v 1.96 2009/12/10 12:29:44 nakayama Exp $"); 63 64 #include "opt_ddb.h" 65 66 #include <sys/param.h> 67 #include <sys/extent.h> 68 #include <sys/malloc.h> 69 #include <sys/systm.h> 70 #include <sys/device.h> 71 #include <sys/proc.h> 72 73 #include <uvm/uvm_extern.h> 74 75 #include <machine/bus.h> 76 #include <sparc64/dev/iommureg.h> 77 #include <sparc64/dev/iommuvar.h> 78 79 #include <machine/autoconf.h> 80 #include <machine/cpu.h> 81 82 #ifdef DEBUG 83 #define IDB_BUSDMA 0x1 84 #define IDB_IOMMU 0x2 85 #define IDB_INFO 0x4 86 #define IDB_SYNC 0x8 87 int iommudebug = 0x0; 88 #define DPRINTF(l, s) do { if (iommudebug & l) printf s; } while (0) 89 #define IOTTE_DEBUG(n) (n) 90 #else 91 #define DPRINTF(l, s) 92 #define IOTTE_DEBUG(n) 0 93 #endif 94 95 #define iommu_strbuf_flush(i, v) do { \ 96 if ((i)->sb_flush) \ 97 bus_space_write_8((i)->sb_is->is_bustag, (i)->sb_sb, \ 98 STRBUFREG(strbuf_pgflush), (v)); \ 99 } while (0) 100 101 static int iommu_strbuf_flush_done(struct strbuf_ctl *); 102 static void _iommu_dvmamap_sync(bus_dma_tag_t, bus_dmamap_t, bus_addr_t, 103 bus_size_t, int); 104 105 /* 106 * initialise the UltraSPARC IOMMU (SBUS or PCI): 107 * - allocate and setup the iotsb. 108 * - enable the IOMMU 109 * - initialise the streaming buffers (if they exist) 110 * - create a private DVMA map. 111 */ 112 void 113 iommu_init(char *name, struct iommu_state *is, int tsbsize, uint32_t iovabase) 114 { 115 psize_t size; 116 vaddr_t va; 117 paddr_t pa; 118 struct vm_page *pg; 119 struct pglist pglist; 120 121 /* 122 * Setup the iommu. 123 * 124 * The sun4u iommu is part of the SBUS or PCI controller so we will 125 * deal with it here.. 126 * 127 * For sysio and psycho/psycho+ the IOMMU address space always ends at 128 * 0xffffe000, but the starting address depends on the size of the 129 * map. The map size is 1024 * 2 ^ is->is_tsbsize entries, where each 130 * entry is 8 bytes. The start of the map can be calculated by 131 * (0xffffe000 << (8 + is->is_tsbsize)). 132 * 133 * But sabre and hummingbird use a different scheme that seems to 134 * be hard-wired, so we read the start and size from the PROM and 135 * just use those values. 136 */ 137 is->is_cr = (tsbsize << 16) | IOMMUCR_EN; 138 is->is_tsbsize = tsbsize; 139 if (iovabase == -1) { 140 is->is_dvmabase = IOTSB_VSTART(is->is_tsbsize); 141 is->is_dvmaend = IOTSB_VEND - 1; 142 } else { 143 is->is_dvmabase = iovabase; 144 is->is_dvmaend = iovabase + IOTSB_VSIZE(tsbsize) - 1; 145 } 146 147 /* 148 * Allocate memory for I/O pagetables. They need to be physically 149 * contiguous. 150 */ 151 152 size = PAGE_SIZE << is->is_tsbsize; 153 if (uvm_pglistalloc((psize_t)size, (paddr_t)0, (paddr_t)-1, 154 (paddr_t)PAGE_SIZE, (paddr_t)0, &pglist, 1, 0) != 0) 155 panic("iommu_init: no memory"); 156 157 va = uvm_km_alloc(kernel_map, size, 0, UVM_KMF_VAONLY); 158 if (va == 0) 159 panic("iommu_init: no memory"); 160 is->is_tsb = (int64_t *)va; 161 162 is->is_ptsb = VM_PAGE_TO_PHYS(TAILQ_FIRST(&pglist)); 163 164 /* Map the pages */ 165 TAILQ_FOREACH(pg, &pglist, pageq.queue) { 166 pa = VM_PAGE_TO_PHYS(pg); 167 pmap_kenter_pa(va, pa | PMAP_NVC, 168 VM_PROT_READ | VM_PROT_WRITE, 0); 169 va += PAGE_SIZE; 170 } 171 pmap_update(pmap_kernel()); 172 memset(is->is_tsb, 0, size); 173 174 #ifdef DEBUG 175 if (iommudebug & IDB_INFO) 176 { 177 /* Probe the iommu */ 178 179 printf("iommu regs at: cr=%lx tsb=%lx flush=%lx\n", 180 (u_long)bus_space_read_8(is->is_bustag, is->is_iommu, 181 offsetof (struct iommureg, iommu_cr)), 182 (u_long)bus_space_read_8(is->is_bustag, is->is_iommu, 183 offsetof (struct iommureg, iommu_tsb)), 184 (u_long)bus_space_read_8(is->is_bustag, is->is_iommu, 185 offsetof (struct iommureg, iommu_flush))); 186 printf("iommu cr=%llx tsb=%llx\n", 187 (unsigned long long)bus_space_read_8(is->is_bustag, 188 is->is_iommu, 189 offsetof (struct iommureg, iommu_cr)), 190 (unsigned long long)bus_space_read_8(is->is_bustag, 191 is->is_iommu, 192 offsetof (struct iommureg, iommu_tsb))); 193 printf("TSB base %p phys %llx\n", (void *)is->is_tsb, 194 (unsigned long long)is->is_ptsb); 195 delay(1000000); /* 1 s */ 196 } 197 #endif 198 199 /* 200 * now actually start up the IOMMU 201 */ 202 iommu_reset(is); 203 204 /* 205 * Now all the hardware's working we need to allocate a dvma map. 206 */ 207 printf("DVMA map: %x to %x\n", 208 (unsigned int)is->is_dvmabase, 209 (unsigned int)is->is_dvmaend); 210 printf("IOTSB: %llx to %llx\n", 211 (unsigned long long)is->is_ptsb, 212 (unsigned long long)(is->is_ptsb + size - 1)); 213 is->is_dvmamap = extent_create(name, 214 is->is_dvmabase, is->is_dvmaend, 215 M_DEVBUF, 0, 0, EX_NOWAIT); 216 } 217 218 /* 219 * Streaming buffers don't exist on the UltraSPARC IIi; we should have 220 * detected that already and disabled them. If not, we will notice that 221 * they aren't there when the STRBUF_EN bit does not remain. 222 */ 223 void 224 iommu_reset(struct iommu_state *is) 225 { 226 int i; 227 struct strbuf_ctl *sb; 228 229 /* Need to do 64-bit stores */ 230 bus_space_write_8(is->is_bustag, is->is_iommu, IOMMUREG(iommu_tsb), 231 is->is_ptsb); 232 233 /* Enable IOMMU in diagnostic mode */ 234 bus_space_write_8(is->is_bustag, is->is_iommu, IOMMUREG(iommu_cr), 235 is->is_cr|IOMMUCR_DE); 236 237 for (i = 0; i < 2; i++) { 238 if ((sb = is->is_sb[i])) { 239 240 /* Enable diagnostics mode? */ 241 bus_space_write_8(is->is_bustag, is->is_sb[i]->sb_sb, 242 STRBUFREG(strbuf_ctl), STRBUF_EN); 243 244 /* No streaming buffers? Disable them */ 245 if (bus_space_read_8(is->is_bustag, 246 is->is_sb[i]->sb_sb, 247 STRBUFREG(strbuf_ctl)) == 0) { 248 is->is_sb[i]->sb_flush = NULL; 249 } else { 250 251 /* 252 * locate the pa of the flush buffer. 253 */ 254 (void)pmap_extract(pmap_kernel(), 255 (vaddr_t)is->is_sb[i]->sb_flush, 256 &is->is_sb[i]->sb_flushpa); 257 } 258 } 259 } 260 } 261 262 /* 263 * Here are the iommu control routines. 264 */ 265 void 266 iommu_enter(struct strbuf_ctl *sb, vaddr_t va, int64_t pa, int flags) 267 { 268 struct iommu_state *is = sb->sb_is; 269 int strbuf = (flags & BUS_DMA_STREAMING); 270 int64_t tte; 271 272 #ifdef DIAGNOSTIC 273 if (va < is->is_dvmabase || va > is->is_dvmaend) 274 panic("iommu_enter: va %#lx not in DVMA space", va); 275 #endif 276 277 /* Is the streamcache flush really needed? */ 278 if (sb->sb_flush) 279 iommu_strbuf_flush(sb, va); 280 else 281 /* If we can't flush the strbuf don't enable it. */ 282 strbuf = 0; 283 284 tte = MAKEIOTTE(pa, !(flags & BUS_DMA_NOWRITE), 285 !(flags & BUS_DMA_NOCACHE), (strbuf)); 286 #ifdef DEBUG 287 tte |= (flags & 0xff000LL)<<(4*8); 288 #endif 289 290 DPRINTF(IDB_IOMMU, ("Clearing TSB slot %d for va %p\n", 291 (int)IOTSBSLOT(va,is->is_tsbsize), (void *)(u_long)va)); 292 is->is_tsb[IOTSBSLOT(va,is->is_tsbsize)] = tte; 293 bus_space_write_8(is->is_bustag, is->is_iommu, 294 IOMMUREG(iommu_flush), va); 295 DPRINTF(IDB_IOMMU, ("iommu_enter: va %lx pa %lx TSB[%lx]@%p=%lx\n", 296 va, (long)pa, (u_long)IOTSBSLOT(va,is->is_tsbsize), 297 (void *)(u_long)&is->is_tsb[IOTSBSLOT(va,is->is_tsbsize)], 298 (u_long)tte)); 299 } 300 301 /* 302 * Find the value of a DVMA address (debug routine). 303 */ 304 paddr_t 305 iommu_extract(struct iommu_state *is, vaddr_t dva) 306 { 307 int64_t tte = 0; 308 309 if (dva >= is->is_dvmabase && dva <= is->is_dvmaend) 310 tte = is->is_tsb[IOTSBSLOT(dva, is->is_tsbsize)]; 311 312 if ((tte & IOTTE_V) == 0) 313 return ((paddr_t)-1L); 314 return (tte & IOTTE_PAMASK); 315 } 316 317 /* 318 * iommu_remove: removes mappings created by iommu_enter 319 * 320 * Only demap from IOMMU if flag is set. 321 * 322 * XXX: this function needs better internal error checking. 323 */ 324 void 325 iommu_remove(struct iommu_state *is, vaddr_t va, size_t len) 326 { 327 328 #ifdef DIAGNOSTIC 329 if (va < is->is_dvmabase || va > is->is_dvmaend) 330 panic("iommu_remove: va 0x%lx not in DVMA space", (u_long)va); 331 if ((long)(va + len) < (long)va) 332 panic("iommu_remove: va 0x%lx + len 0x%lx wraps", 333 (long) va, (long) len); 334 if (len & ~0xfffffff) 335 panic("iommu_remove: ridiculous len 0x%lx", (u_long)len); 336 #endif 337 338 va = trunc_page(va); 339 DPRINTF(IDB_IOMMU, ("iommu_remove: va %lx TSB[%lx]@%p\n", 340 va, (u_long)IOTSBSLOT(va, is->is_tsbsize), 341 &is->is_tsb[IOTSBSLOT(va, is->is_tsbsize)])); 342 while (len > 0) { 343 DPRINTF(IDB_IOMMU, ("iommu_remove: clearing TSB slot %d " 344 "for va %p size %lx\n", 345 (int)IOTSBSLOT(va,is->is_tsbsize), (void *)(u_long)va, 346 (u_long)len)); 347 if (len <= PAGE_SIZE) 348 len = 0; 349 else 350 len -= PAGE_SIZE; 351 352 /* 353 * XXX Zero-ing the entry would not require RMW 354 * 355 * Disabling valid bit while a page is used by a device 356 * causes an uncorrectable DMA error. 357 * Workaround to avoid an uncorrectable DMA error is 358 * eliminating the next line, but the page is mapped 359 * until the next iommu_enter call. 360 */ 361 is->is_tsb[IOTSBSLOT(va,is->is_tsbsize)] &= ~IOTTE_V; 362 membar_storestore(); 363 bus_space_write_8(is->is_bustag, is->is_iommu, 364 IOMMUREG(iommu_flush), va); 365 va += PAGE_SIZE; 366 } 367 } 368 369 static int 370 iommu_strbuf_flush_done(struct strbuf_ctl *sb) 371 { 372 struct iommu_state *is = sb->sb_is; 373 struct timeval cur, flushtimeout; 374 375 #define BUMPTIME(t, usec) { \ 376 register volatile struct timeval *tp = (t); \ 377 register long us; \ 378 \ 379 tp->tv_usec = us = tp->tv_usec + (usec); \ 380 if (us >= 1000000) { \ 381 tp->tv_usec = us - 1000000; \ 382 tp->tv_sec++; \ 383 } \ 384 } 385 386 if (!sb->sb_flush) 387 return (0); 388 389 /* 390 * Streaming buffer flushes: 391 * 392 * 1 Tell strbuf to flush by storing va to strbuf_pgflush. If 393 * we're not on a cache line boundary (64-bits): 394 * 2 Store 0 in flag 395 * 3 Store pointer to flag in flushsync 396 * 4 wait till flushsync becomes 0x1 397 * 398 * If it takes more than .5 sec, something 399 * went wrong. 400 */ 401 402 *sb->sb_flush = 0; 403 bus_space_write_8(is->is_bustag, sb->sb_sb, 404 STRBUFREG(strbuf_flushsync), sb->sb_flushpa); 405 406 microtime(&flushtimeout); 407 cur = flushtimeout; 408 BUMPTIME(&flushtimeout, 500000); /* 1/2 sec */ 409 410 DPRINTF(IDB_IOMMU, ("iommu_strbuf_flush_done: flush = %lx " 411 "at va = %lx pa = %lx now=%"PRIx64":%"PRIx32" until = %"PRIx64":%"PRIx32"\n", 412 (long)*sb->sb_flush, (long)sb->sb_flush, (long)sb->sb_flushpa, 413 cur.tv_sec, cur.tv_usec, 414 flushtimeout.tv_sec, flushtimeout.tv_usec)); 415 416 /* Bypass non-coherent D$ */ 417 while ((!ldxa(sb->sb_flushpa, ASI_PHYS_CACHED)) && 418 timercmp(&cur, &flushtimeout, <=)) 419 microtime(&cur); 420 421 #ifdef DIAGNOSTIC 422 if (!ldxa(sb->sb_flushpa, ASI_PHYS_CACHED)) { 423 printf("iommu_strbuf_flush_done: flush timeout %p, at %p\n", 424 (void *)(u_long)*sb->sb_flush, 425 (void *)(u_long)sb->sb_flushpa); /* panic? */ 426 #ifdef DDB 427 Debugger(); 428 #endif 429 } 430 #endif 431 DPRINTF(IDB_IOMMU, ("iommu_strbuf_flush_done: flushed\n")); 432 return (*sb->sb_flush); 433 } 434 435 /* 436 * IOMMU DVMA operations, common to SBUS and PCI. 437 */ 438 int 439 iommu_dvmamap_load(bus_dma_tag_t t, bus_dmamap_t map, void *buf, 440 bus_size_t buflen, struct proc *p, int flags) 441 { 442 struct strbuf_ctl *sb = (struct strbuf_ctl *)map->_dm_cookie; 443 struct iommu_state *is = sb->sb_is; 444 int s; 445 int err, needsflush; 446 bus_size_t sgsize; 447 paddr_t curaddr; 448 u_long dvmaddr, sgstart, sgend, bmask; 449 bus_size_t align, boundary, len; 450 vaddr_t vaddr = (vaddr_t)buf; 451 int seg; 452 struct pmap *pmap; 453 454 if (map->dm_nsegs) { 455 /* Already in use?? */ 456 #ifdef DIAGNOSTIC 457 printf("iommu_dvmamap_load: map still in use\n"); 458 #endif 459 bus_dmamap_unload(t, map); 460 } 461 462 /* 463 * Make sure that on error condition we return "no valid mappings". 464 */ 465 map->dm_nsegs = 0; 466 KASSERT(map->dm_maxsegsz <= map->_dm_maxmaxsegsz); 467 468 if (buflen > map->_dm_size) { 469 DPRINTF(IDB_BUSDMA, 470 ("iommu_dvmamap_load(): error %d > %d -- " 471 "map size exceeded!\n", (int)buflen, (int)map->_dm_size)); 472 return (EINVAL); 473 } 474 475 sgsize = round_page(buflen + ((int)vaddr & PGOFSET)); 476 477 /* 478 * A boundary presented to bus_dmamem_alloc() takes precedence 479 * over boundary in the map. 480 */ 481 if ((boundary = (map->dm_segs[0]._ds_boundary)) == 0) 482 boundary = map->_dm_boundary; 483 align = max(map->dm_segs[0]._ds_align, PAGE_SIZE); 484 485 /* 486 * If our segment size is larger than the boundary we need to 487 * split the transfer up int little pieces ourselves. 488 */ 489 s = splhigh(); 490 err = extent_alloc(is->is_dvmamap, sgsize, align, 491 (sgsize > boundary) ? 0 : boundary, 492 EX_NOWAIT|EX_BOUNDZERO, &dvmaddr); 493 splx(s); 494 495 #ifdef DEBUG 496 if (err || (dvmaddr == (u_long)-1)) { 497 printf("iommu_dvmamap_load(): extent_alloc(%d, %x) failed!\n", 498 (int)sgsize, flags); 499 #ifdef DDB 500 Debugger(); 501 #endif 502 } 503 #endif 504 if (err != 0) 505 return (err); 506 507 if (dvmaddr == (u_long)-1) 508 return (ENOMEM); 509 510 /* Set the active DVMA map */ 511 map->_dm_dvmastart = dvmaddr; 512 map->_dm_dvmasize = sgsize; 513 514 /* 515 * Now split the DVMA range into segments, not crossing 516 * the boundary. 517 */ 518 seg = 0; 519 sgstart = dvmaddr + (vaddr & PGOFSET); 520 sgend = sgstart + buflen - 1; 521 map->dm_segs[seg].ds_addr = sgstart; 522 DPRINTF(IDB_INFO, ("iommu_dvmamap_load: boundary %lx boundary - 1 %lx " 523 "~(boundary - 1) %lx\n", (long)boundary, (long)(boundary - 1), 524 (long)~(boundary - 1))); 525 bmask = ~(boundary - 1); 526 while ((sgstart & bmask) != (sgend & bmask) || 527 sgend - sgstart + 1 > map->dm_maxsegsz) { 528 /* Oops. We crossed a boundary or large seg. Split the xfer. */ 529 len = map->dm_maxsegsz; 530 if ((sgstart & bmask) != (sgend & bmask)) 531 len = min(len, boundary - (sgstart & (boundary - 1))); 532 map->dm_segs[seg].ds_len = len; 533 DPRINTF(IDB_INFO, ("iommu_dvmamap_load: " 534 "seg %d start %lx size %lx\n", seg, 535 (long)map->dm_segs[seg].ds_addr, 536 (long)map->dm_segs[seg].ds_len)); 537 if (++seg >= map->_dm_segcnt) { 538 /* Too many segments. Fail the operation. */ 539 DPRINTF(IDB_INFO, ("iommu_dvmamap_load: " 540 "too many segments %d\n", seg)); 541 s = splhigh(); 542 err = extent_free(is->is_dvmamap, 543 dvmaddr, sgsize, EX_NOWAIT); 544 map->_dm_dvmastart = 0; 545 map->_dm_dvmasize = 0; 546 splx(s); 547 if (err != 0) 548 printf("warning: %s: %" PRId64 549 " of DVMA space lost\n", __func__, sgsize); 550 return (EFBIG); 551 } 552 sgstart += len; 553 map->dm_segs[seg].ds_addr = sgstart; 554 } 555 map->dm_segs[seg].ds_len = sgend - sgstart + 1; 556 DPRINTF(IDB_INFO, ("iommu_dvmamap_load: " 557 "seg %d start %lx size %lx\n", seg, 558 (long)map->dm_segs[seg].ds_addr, (long)map->dm_segs[seg].ds_len)); 559 map->dm_nsegs = seg + 1; 560 map->dm_mapsize = buflen; 561 562 if (p != NULL) 563 pmap = p->p_vmspace->vm_map.pmap; 564 else 565 pmap = pmap_kernel(); 566 567 needsflush = 0; 568 for (; buflen > 0; ) { 569 570 /* 571 * Get the physical address for this page. 572 */ 573 if (pmap_extract(pmap, (vaddr_t)vaddr, &curaddr) == FALSE) { 574 #ifdef DIAGNOSTIC 575 printf("iommu_dvmamap_load: pmap_extract failed %lx\n", vaddr); 576 #endif 577 bus_dmamap_unload(t, map); 578 return (-1); 579 } 580 581 /* 582 * Compute the segment size, and adjust counts. 583 */ 584 sgsize = PAGE_SIZE - ((u_long)vaddr & PGOFSET); 585 if (buflen < sgsize) 586 sgsize = buflen; 587 588 DPRINTF(IDB_BUSDMA, 589 ("iommu_dvmamap_load: map %p loading va %p " 590 "dva %lx at pa %lx\n", 591 map, (void *)vaddr, (long)dvmaddr, 592 (long)trunc_page(curaddr))); 593 iommu_enter(sb, trunc_page(dvmaddr), trunc_page(curaddr), 594 flags | IOTTE_DEBUG(0x4000)); 595 needsflush = 1; 596 597 dvmaddr += PAGE_SIZE; 598 vaddr += sgsize; 599 buflen -= sgsize; 600 } 601 if (needsflush) 602 iommu_strbuf_flush_done(sb); 603 #ifdef DIAGNOSTIC 604 for (seg = 0; seg < map->dm_nsegs; seg++) { 605 if (map->dm_segs[seg].ds_addr < is->is_dvmabase || 606 map->dm_segs[seg].ds_addr > is->is_dvmaend) { 607 printf("seg %d dvmaddr %lx out of range %x - %x\n", 608 seg, (long)map->dm_segs[seg].ds_addr, 609 is->is_dvmabase, is->is_dvmaend); 610 #ifdef DDB 611 Debugger(); 612 #endif 613 } 614 } 615 #endif 616 return (0); 617 } 618 619 620 void 621 iommu_dvmamap_unload(bus_dma_tag_t t, bus_dmamap_t map) 622 { 623 struct strbuf_ctl *sb = (struct strbuf_ctl *)map->_dm_cookie; 624 struct iommu_state *is = sb->sb_is; 625 int error, s; 626 bus_size_t sgsize = map->_dm_dvmasize; 627 628 /* Flush the iommu */ 629 #ifdef DEBUG 630 if (!map->_dm_dvmastart) { 631 printf("iommu_dvmamap_unload: No dvmastart is zero\n"); 632 #ifdef DDB 633 Debugger(); 634 #endif 635 } 636 #endif 637 iommu_remove(is, map->_dm_dvmastart, map->_dm_dvmasize); 638 639 /* Flush the caches */ 640 bus_dmamap_unload(t->_parent, map); 641 642 /* Mark the mappings as invalid. */ 643 map->dm_mapsize = 0; 644 map->dm_nsegs = 0; 645 646 s = splhigh(); 647 error = extent_free(is->is_dvmamap, map->_dm_dvmastart, 648 map->_dm_dvmasize, EX_NOWAIT); 649 map->_dm_dvmastart = 0; 650 map->_dm_dvmasize = 0; 651 splx(s); 652 if (error != 0) 653 printf("warning: %s: %" PRId64 " of DVMA space lost\n", 654 __func__, sgsize); 655 656 /* Clear the map */ 657 } 658 659 660 int 661 iommu_dvmamap_load_raw(bus_dma_tag_t t, bus_dmamap_t map, 662 bus_dma_segment_t *segs, int nsegs, bus_size_t size, int flags) 663 { 664 struct strbuf_ctl *sb = (struct strbuf_ctl *)map->_dm_cookie; 665 struct iommu_state *is = sb->sb_is; 666 struct vm_page *pg; 667 int i, j, s; 668 int left; 669 int err, needsflush; 670 bus_size_t sgsize; 671 paddr_t pa; 672 bus_size_t boundary, align; 673 u_long dvmaddr, sgstart, sgend, bmask; 674 struct pglist *pglist; 675 const int pagesz = PAGE_SIZE; 676 #ifdef DEBUG 677 int npg = 0; 678 #endif 679 680 if (map->dm_nsegs) { 681 /* Already in use?? */ 682 #ifdef DIAGNOSTIC 683 printf("iommu_dvmamap_load_raw: map still in use\n"); 684 #endif 685 bus_dmamap_unload(t, map); 686 } 687 688 /* 689 * A boundary presented to bus_dmamem_alloc() takes precedence 690 * over boundary in the map. 691 */ 692 if ((boundary = segs[0]._ds_boundary) == 0) 693 boundary = map->_dm_boundary; 694 695 align = max(segs[0]._ds_align, pagesz); 696 697 /* 698 * Make sure that on error condition we return "no valid mappings". 699 */ 700 map->dm_nsegs = 0; 701 /* Count up the total number of pages we need */ 702 pa = trunc_page(segs[0].ds_addr); 703 sgsize = 0; 704 left = size; 705 for (i = 0; left > 0 && i < nsegs; i++) { 706 if (round_page(pa) != round_page(segs[i].ds_addr)) 707 sgsize = round_page(sgsize) + 708 (segs[i].ds_addr & PGOFSET); 709 sgsize += min(left, segs[i].ds_len); 710 left -= segs[i].ds_len; 711 pa = segs[i].ds_addr + segs[i].ds_len; 712 } 713 sgsize = round_page(sgsize); 714 715 s = splhigh(); 716 /* 717 * If our segment size is larger than the boundary we need to 718 * split the transfer up into little pieces ourselves. 719 */ 720 err = extent_alloc(is->is_dvmamap, sgsize, align, 721 (sgsize > boundary) ? 0 : boundary, 722 ((flags & BUS_DMA_NOWAIT) == 0 ? EX_WAITOK : EX_NOWAIT) | 723 EX_BOUNDZERO, &dvmaddr); 724 splx(s); 725 726 if (err != 0) 727 return (err); 728 729 #ifdef DEBUG 730 if (dvmaddr == (u_long)-1) 731 { 732 printf("iommu_dvmamap_load_raw(): extent_alloc(%d, %x) failed!\n", 733 (int)sgsize, flags); 734 #ifdef DDB 735 Debugger(); 736 #endif 737 } 738 #endif 739 if (dvmaddr == (u_long)-1) 740 return (ENOMEM); 741 742 /* Set the active DVMA map */ 743 map->_dm_dvmastart = dvmaddr; 744 map->_dm_dvmasize = sgsize; 745 746 bmask = ~(boundary - 1); 747 if ((pglist = segs[0]._ds_mlist) == NULL) { 748 u_long prev_va = 0UL, last_va = dvmaddr; 749 paddr_t prev_pa = 0; 750 int end = 0, offset; 751 bus_size_t len = size; 752 753 /* 754 * This segs is made up of individual physical 755 * segments, probably by _bus_dmamap_load_uio() or 756 * _bus_dmamap_load_mbuf(). Ignore the mlist and 757 * load each one individually. 758 */ 759 j = 0; 760 needsflush = 0; 761 for (i = 0; i < nsegs ; i++) { 762 763 pa = segs[i].ds_addr; 764 offset = (pa & PGOFSET); 765 pa = trunc_page(pa); 766 dvmaddr = trunc_page(dvmaddr); 767 left = min(len, segs[i].ds_len); 768 769 DPRINTF(IDB_INFO, ("iommu_dvmamap_load_raw: converting " 770 "physseg %d start %lx size %lx\n", i, 771 (long)segs[i].ds_addr, (long)segs[i].ds_len)); 772 773 if ((pa == prev_pa) && 774 ((offset != 0) || (end != offset))) { 775 /* We can re-use this mapping */ 776 dvmaddr = prev_va; 777 } 778 779 sgstart = dvmaddr + offset; 780 sgend = sgstart + left - 1; 781 782 /* Are the segments virtually adjacent? */ 783 if ((j > 0) && (end == offset) && 784 ((offset == 0) || (pa == prev_pa)) && 785 (map->dm_segs[j-1].ds_len + left <= 786 map->dm_maxsegsz)) { 787 /* Just append to the previous segment. */ 788 map->dm_segs[--j].ds_len += left; 789 /* Restore sgstart for boundary check */ 790 sgstart = map->dm_segs[j].ds_addr; 791 DPRINTF(IDB_INFO, ("iommu_dvmamap_load_raw: " 792 "appending seg %d start %lx size %lx\n", j, 793 (long)map->dm_segs[j].ds_addr, 794 (long)map->dm_segs[j].ds_len)); 795 } else { 796 if (j >= map->_dm_segcnt) { 797 iommu_remove(is, map->_dm_dvmastart, 798 last_va - map->_dm_dvmastart); 799 goto fail; 800 } 801 map->dm_segs[j].ds_addr = sgstart; 802 map->dm_segs[j].ds_len = left; 803 DPRINTF(IDB_INFO, ("iommu_dvmamap_load_raw: " 804 "seg %d start %lx size %lx\n", j, 805 (long)map->dm_segs[j].ds_addr, 806 (long)map->dm_segs[j].ds_len)); 807 } 808 end = (offset + left) & PGOFSET; 809 810 /* Check for boundary issues */ 811 while ((sgstart & bmask) != (sgend & bmask)) { 812 /* Need a new segment. */ 813 map->dm_segs[j].ds_len = 814 boundary - (sgstart & (boundary - 1)); 815 DPRINTF(IDB_INFO, ("iommu_dvmamap_load_raw: " 816 "seg %d start %lx size %lx\n", j, 817 (long)map->dm_segs[j].ds_addr, 818 (long)map->dm_segs[j].ds_len)); 819 if (++j >= map->_dm_segcnt) { 820 iommu_remove(is, map->_dm_dvmastart, 821 last_va - map->_dm_dvmastart); 822 goto fail; 823 } 824 sgstart += map->dm_segs[j-1].ds_len; 825 map->dm_segs[j].ds_addr = sgstart; 826 map->dm_segs[j].ds_len = sgend - sgstart + 1; 827 } 828 829 if (sgsize == 0) 830 panic("iommu_dmamap_load_raw: size botch"); 831 832 /* Now map a series of pages. */ 833 while (dvmaddr <= sgend) { 834 DPRINTF(IDB_BUSDMA, 835 ("iommu_dvmamap_load_raw: map %p " 836 "loading va %lx at pa %lx\n", 837 map, (long)dvmaddr, 838 (long)(pa))); 839 /* Enter it if we haven't before. */ 840 if (prev_va != dvmaddr) { 841 iommu_enter(sb, prev_va = dvmaddr, 842 prev_pa = pa, 843 flags | IOTTE_DEBUG(++npg << 12)); 844 needsflush = 1; 845 } 846 dvmaddr += pagesz; 847 pa += pagesz; 848 last_va = dvmaddr; 849 } 850 851 len -= left; 852 ++j; 853 } 854 if (needsflush) 855 iommu_strbuf_flush_done(sb); 856 857 map->dm_mapsize = size; 858 map->dm_nsegs = j; 859 #ifdef DIAGNOSTIC 860 { int seg; 861 for (seg = 0; seg < map->dm_nsegs; seg++) { 862 if (map->dm_segs[seg].ds_addr < is->is_dvmabase || 863 map->dm_segs[seg].ds_addr > is->is_dvmaend) { 864 printf("seg %d dvmaddr %lx out of range %x - %x\n", 865 seg, (long)map->dm_segs[seg].ds_addr, 866 is->is_dvmabase, is->is_dvmaend); 867 #ifdef DDB 868 Debugger(); 869 #endif 870 } 871 } 872 } 873 #endif 874 return (0); 875 } 876 877 /* 878 * This was allocated with bus_dmamem_alloc. 879 * The pages are on a `pglist'. 880 */ 881 i = 0; 882 sgstart = dvmaddr; 883 sgend = sgstart + size - 1; 884 map->dm_segs[i].ds_addr = sgstart; 885 while ((sgstart & bmask) != (sgend & bmask)) { 886 /* Oops. We crossed a boundary. Split the xfer. */ 887 map->dm_segs[i].ds_len = boundary - (sgstart & (boundary - 1)); 888 DPRINTF(IDB_INFO, ("iommu_dvmamap_load_raw: " 889 "seg %d start %lx size %lx\n", i, 890 (long)map->dm_segs[i].ds_addr, 891 (long)map->dm_segs[i].ds_len)); 892 if (++i >= map->_dm_segcnt) { 893 /* Too many segments. Fail the operation. */ 894 goto fail; 895 } 896 sgstart += map->dm_segs[i-1].ds_len; 897 map->dm_segs[i].ds_addr = sgstart; 898 } 899 DPRINTF(IDB_INFO, ("iommu_dvmamap_load_raw: " 900 "seg %d start %lx size %lx\n", i, 901 (long)map->dm_segs[i].ds_addr, (long)map->dm_segs[i].ds_len)); 902 map->dm_segs[i].ds_len = sgend - sgstart + 1; 903 904 needsflush = 0; 905 TAILQ_FOREACH(pg, pglist, pageq.queue) { 906 if (sgsize == 0) 907 panic("iommu_dmamap_load_raw: size botch"); 908 pa = VM_PAGE_TO_PHYS(pg); 909 910 DPRINTF(IDB_BUSDMA, 911 ("iommu_dvmamap_load_raw: map %p loading va %lx at pa %lx\n", 912 map, (long)dvmaddr, (long)(pa))); 913 iommu_enter(sb, dvmaddr, pa, flags | IOTTE_DEBUG(0x8000)); 914 needsflush = 1; 915 916 dvmaddr += pagesz; 917 sgsize -= pagesz; 918 } 919 if (needsflush) 920 iommu_strbuf_flush_done(sb); 921 map->dm_mapsize = size; 922 map->dm_nsegs = i+1; 923 #ifdef DIAGNOSTIC 924 { int seg; 925 for (seg = 0; seg < map->dm_nsegs; seg++) { 926 if (map->dm_segs[seg].ds_addr < is->is_dvmabase || 927 map->dm_segs[seg].ds_addr > is->is_dvmaend) { 928 printf("seg %d dvmaddr %lx out of range %x - %x\n", 929 seg, (long)map->dm_segs[seg].ds_addr, 930 is->is_dvmabase, is->is_dvmaend); 931 #ifdef DDB 932 Debugger(); 933 #endif 934 } 935 } 936 } 937 #endif 938 return (0); 939 940 fail: 941 s = splhigh(); 942 err = extent_free(is->is_dvmamap, map->_dm_dvmastart, sgsize, 943 EX_NOWAIT); 944 map->_dm_dvmastart = 0; 945 map->_dm_dvmasize = 0; 946 splx(s); 947 if (err != 0) 948 printf("warning: %s: %" PRId64 " of DVMA space lost\n", 949 __func__, sgsize); 950 return (EFBIG); 951 } 952 953 954 /* 955 * Flush an individual dma segment, returns non-zero if the streaming buffers 956 * need flushing afterwards. 957 */ 958 static int 959 iommu_dvmamap_sync_range(struct strbuf_ctl *sb, vaddr_t va, bus_size_t len) 960 { 961 vaddr_t vaend; 962 struct iommu_state *is = sb->sb_is; 963 964 #ifdef DIAGNOSTIC 965 if (va < is->is_dvmabase || va > is->is_dvmaend) 966 panic("invalid va: %llx", (long long)va); 967 #endif 968 969 if ((is->is_tsb[IOTSBSLOT(va, is->is_tsbsize)] & IOTTE_STREAM) == 0) { 970 DPRINTF(IDB_BUSDMA, 971 ("iommu_dvmamap_sync_range: attempting to flush " 972 "non-streaming entry\n")); 973 return (0); 974 } 975 976 vaend = round_page(va + len) - 1; 977 va = trunc_page(va); 978 979 #ifdef DIAGNOSTIC 980 if (va < is->is_dvmabase || vaend > is->is_dvmaend) 981 panic("invalid va range: %llx to %llx (%x to %x)", 982 (long long)va, (long long)vaend, 983 is->is_dvmabase, 984 is->is_dvmaend); 985 #endif 986 987 for ( ; va <= vaend; va += PAGE_SIZE) { 988 DPRINTF(IDB_BUSDMA, 989 ("iommu_dvmamap_sync_range: flushing va %p\n", 990 (void *)(u_long)va)); 991 iommu_strbuf_flush(sb, va); 992 } 993 994 return (1); 995 } 996 997 static void 998 _iommu_dvmamap_sync(bus_dma_tag_t t, bus_dmamap_t map, bus_addr_t offset, 999 bus_size_t len, int ops) 1000 { 1001 struct strbuf_ctl *sb = (struct strbuf_ctl *)map->_dm_cookie; 1002 bus_size_t count; 1003 int i, needsflush = 0; 1004 1005 if (!sb->sb_flush) 1006 return; 1007 1008 for (i = 0; i < map->dm_nsegs; i++) { 1009 if (offset < map->dm_segs[i].ds_len) 1010 break; 1011 offset -= map->dm_segs[i].ds_len; 1012 } 1013 1014 if (i == map->dm_nsegs) 1015 panic("iommu_dvmamap_sync: segment too short %llu", 1016 (unsigned long long)offset); 1017 1018 if (ops & (BUS_DMASYNC_PREREAD | BUS_DMASYNC_POSTWRITE)) { 1019 /* Nothing to do */; 1020 } 1021 1022 if (ops & (BUS_DMASYNC_POSTREAD | BUS_DMASYNC_PREWRITE)) { 1023 1024 for (; len > 0 && i < map->dm_nsegs; i++) { 1025 count = MIN(map->dm_segs[i].ds_len - offset, len); 1026 if (count > 0 && 1027 iommu_dvmamap_sync_range(sb, 1028 map->dm_segs[i].ds_addr + offset, count)) 1029 needsflush = 1; 1030 offset = 0; 1031 len -= count; 1032 } 1033 #ifdef DIAGNOSTIC 1034 if (i == map->dm_nsegs && len > 0) 1035 panic("iommu_dvmamap_sync: leftover %llu", 1036 (unsigned long long)len); 1037 #endif 1038 1039 if (needsflush) 1040 iommu_strbuf_flush_done(sb); 1041 } 1042 } 1043 1044 void 1045 iommu_dvmamap_sync(bus_dma_tag_t t, bus_dmamap_t map, bus_addr_t offset, 1046 bus_size_t len, int ops) 1047 { 1048 1049 /* If len is 0, then there is nothing to do */ 1050 if (len == 0) 1051 return; 1052 1053 if (ops & (BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE)) { 1054 /* Flush the CPU then the IOMMU */ 1055 bus_dmamap_sync(t->_parent, map, offset, len, ops); 1056 _iommu_dvmamap_sync(t, map, offset, len, ops); 1057 } 1058 if (ops & (BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE)) { 1059 /* Flush the IOMMU then the CPU */ 1060 _iommu_dvmamap_sync(t, map, offset, len, ops); 1061 bus_dmamap_sync(t->_parent, map, offset, len, ops); 1062 } 1063 } 1064 1065 int 1066 iommu_dvmamem_alloc(bus_dma_tag_t t, bus_size_t size, bus_size_t alignment, 1067 bus_size_t boundary, bus_dma_segment_t *segs, int nsegs, int *rsegs, 1068 int flags) 1069 { 1070 1071 DPRINTF(IDB_BUSDMA, ("iommu_dvmamem_alloc: sz %llx align %llx bound %llx " 1072 "segp %p flags %d\n", (unsigned long long)size, 1073 (unsigned long long)alignment, (unsigned long long)boundary, 1074 segs, flags)); 1075 return (bus_dmamem_alloc(t->_parent, size, alignment, boundary, 1076 segs, nsegs, rsegs, flags|BUS_DMA_DVMA)); 1077 } 1078 1079 void 1080 iommu_dvmamem_free(bus_dma_tag_t t, bus_dma_segment_t *segs, int nsegs) 1081 { 1082 1083 DPRINTF(IDB_BUSDMA, ("iommu_dvmamem_free: segp %p nsegs %d\n", 1084 segs, nsegs)); 1085 bus_dmamem_free(t->_parent, segs, nsegs); 1086 } 1087 1088 /* 1089 * Map the DVMA mappings into the kernel pmap. 1090 * Check the flags to see whether we're streaming or coherent. 1091 */ 1092 int 1093 iommu_dvmamem_map(bus_dma_tag_t t, bus_dma_segment_t *segs, int nsegs, 1094 size_t size, void **kvap, int flags) 1095 { 1096 struct vm_page *pg; 1097 vaddr_t va; 1098 bus_addr_t addr; 1099 struct pglist *pglist; 1100 int cbit; 1101 const uvm_flag_t kmflags = 1102 (flags & BUS_DMA_NOWAIT) != 0 ? UVM_KMF_NOWAIT : 0; 1103 1104 DPRINTF(IDB_BUSDMA, ("iommu_dvmamem_map: segp %p nsegs %d size %lx\n", 1105 segs, nsegs, size)); 1106 1107 /* 1108 * Allocate some space in the kernel map, and then map these pages 1109 * into this space. 1110 */ 1111 size = round_page(size); 1112 va = uvm_km_alloc(kernel_map, size, 0, UVM_KMF_VAONLY | kmflags); 1113 if (va == 0) 1114 return (ENOMEM); 1115 1116 *kvap = (void *)va; 1117 1118 /* 1119 * digest flags: 1120 */ 1121 cbit = 0; 1122 if (flags & BUS_DMA_COHERENT) /* Disable vcache */ 1123 cbit |= PMAP_NVC; 1124 if (flags & BUS_DMA_NOCACHE) /* sideffects */ 1125 cbit |= PMAP_NC; 1126 1127 /* 1128 * Now take this and map it into the CPU. 1129 */ 1130 pglist = segs[0]._ds_mlist; 1131 TAILQ_FOREACH(pg, pglist, pageq.queue) { 1132 #ifdef DIAGNOSTIC 1133 if (size == 0) 1134 panic("iommu_dvmamem_map: size botch"); 1135 #endif 1136 addr = VM_PAGE_TO_PHYS(pg); 1137 DPRINTF(IDB_BUSDMA, ("iommu_dvmamem_map: " 1138 "mapping va %lx at %llx\n", va, (unsigned long long)addr | cbit)); 1139 pmap_kenter_pa(va, addr | cbit, 1140 VM_PROT_READ | VM_PROT_WRITE, 0); 1141 va += PAGE_SIZE; 1142 size -= PAGE_SIZE; 1143 } 1144 pmap_update(pmap_kernel()); 1145 return (0); 1146 } 1147 1148 /* 1149 * Unmap DVMA mappings from kernel 1150 */ 1151 void 1152 iommu_dvmamem_unmap(bus_dma_tag_t t, void *kva, size_t size) 1153 { 1154 1155 DPRINTF(IDB_BUSDMA, ("iommu_dvmamem_unmap: kvm %p size %lx\n", 1156 kva, size)); 1157 1158 #ifdef DIAGNOSTIC 1159 if ((u_long)kva & PGOFSET) 1160 panic("iommu_dvmamem_unmap"); 1161 #endif 1162 1163 size = round_page(size); 1164 pmap_kremove((vaddr_t)kva, size); 1165 pmap_update(pmap_kernel()); 1166 uvm_km_free(kernel_map, (vaddr_t)kva, size, UVM_KMF_VAONLY); 1167 } 1168
Indexes created Fri Sep 26 20:09:58 GMT 2025