iommu.c revision 1.58
1 /* $NetBSD: iommu.c,v 1.58 2002/09/22 07:19:43 chs Exp $ */ 2 3 /* 4 * Copyright (c) 2001, 2002 Eduardo Horvath 5 * Copyright (c) 1999, 2000 Matthew R. Green 6 * All rights reserved. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 3. The name of the author may not be used to endorse or promote products 17 * derived from this software without specific prior written permission. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 20 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 21 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 22 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 23 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 24 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 25 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 26 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 27 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 28 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 29 * SUCH DAMAGE. 30 */ 31 32 /* 33 * UltraSPARC IOMMU support; used by both the sbus and pci code. 34 */ 35 #include "opt_ddb.h" 36 37 #include <sys/param.h> 38 #include <sys/extent.h> 39 #include <sys/malloc.h> 40 #include <sys/systm.h> 41 #include <sys/device.h> 42 #include <sys/proc.h> 43 44 #include <uvm/uvm_extern.h> 45 46 #include <machine/bus.h> 47 #include <sparc64/sparc64/cache.h> 48 #include <sparc64/dev/iommureg.h> 49 #include <sparc64/dev/iommuvar.h> 50 51 #include <machine/autoconf.h> 52 #include <machine/cpu.h> 53 54 #ifdef DEBUG 55 #define IDB_BUSDMA 0x1 56 #define IDB_IOMMU 0x2 57 #define IDB_INFO 0x4 58 #define IDB_SYNC 0x8 59 int iommudebug = 0x0; 60 #define DPRINTF(l, s) do { if (iommudebug & l) printf s; } while (0) 61 #else 62 #define DPRINTF(l, s) 63 #endif 64 65 #define iommu_strbuf_flush(i, v) do { \ 66 if ((i)->sb_flush) \ 67 bus_space_write_8((i)->sb_is->is_bustag, (i)->sb_sb, \ 68 STRBUFREG(strbuf_pgflush), (v)); \ 69 } while (0) 70 71 static int iommu_strbuf_flush_done __P((struct strbuf_ctl *)); 72 73 /* 74 * initialise the UltraSPARC IOMMU (SBUS or PCI): 75 * - allocate and setup the iotsb. 76 * - enable the IOMMU 77 * - initialise the streaming buffers (if they exist) 78 * - create a private DVMA map. 79 */ 80 void 81 iommu_init(name, is, tsbsize, iovabase) 82 char *name; 83 struct iommu_state *is; 84 int tsbsize; 85 u_int32_t iovabase; 86 { 87 psize_t size; 88 vaddr_t va; 89 paddr_t pa; 90 struct vm_page *pg; 91 struct pglist pglist; 92 93 /* 94 * Setup the iommu. 95 * 96 * The sun4u iommu is part of the SBUS or PCI controller so we will 97 * deal with it here.. 98 * 99 * For sysio and psycho/psycho+ the IOMMU address space always ends at 100 * 0xffffe000, but the starting address depends on the size of the 101 * map. The map size is 1024 * 2 ^ is->is_tsbsize entries, where each 102 * entry is 8 bytes. The start of the map can be calculated by 103 * (0xffffe000 << (8 + is->is_tsbsize)). 104 * 105 * But sabre and hummingbird use a different scheme that seems to 106 * be hard-wired, so we read the start and size from the PROM and 107 * just use those values. 108 */ 109 is->is_cr = (tsbsize << 16) | IOMMUCR_EN; 110 is->is_tsbsize = tsbsize; 111 if (iovabase == -1) { 112 is->is_dvmabase = IOTSB_VSTART(is->is_tsbsize); 113 is->is_dvmaend = IOTSB_VEND; 114 } else { 115 is->is_dvmabase = iovabase; 116 is->is_dvmaend = iovabase + IOTSB_VSIZE(tsbsize); 117 } 118 119 /* 120 * Allocate memory for I/O pagetables. They need to be physically 121 * contiguous. 122 */ 123 124 size = NBPG << is->is_tsbsize; 125 if (uvm_pglistalloc((psize_t)size, (paddr_t)0, (paddr_t)-1, 126 (paddr_t)NBPG, (paddr_t)0, &pglist, 1, 0) != 0) 127 panic("iommu_init: no memory"); 128 129 va = uvm_km_valloc(kernel_map, size); 130 if (va == 0) 131 panic("iommu_init: no memory"); 132 is->is_tsb = (int64_t *)va; 133 134 is->is_ptsb = VM_PAGE_TO_PHYS(TAILQ_FIRST(&pglist)); 135 136 /* Map the pages */ 137 TAILQ_FOREACH(pg, &pglist, pageq) { 138 pa = VM_PAGE_TO_PHYS(pg); 139 pmap_kenter_pa(va, pa | PMAP_NVC, VM_PROT_READ | VM_PROT_WRITE); 140 va += NBPG; 141 } 142 pmap_update(pmap_kernel()); 143 memset(is->is_tsb, 0, size); 144 145 #ifdef DEBUG 146 if (iommudebug & IDB_INFO) 147 { 148 /* Probe the iommu */ 149 150 printf("iommu regs at: cr=%lx tsb=%lx flush=%lx\n", 151 (u_long)bus_space_read_8(is->is_bustag, is->is_iommu, 152 offsetof (struct iommureg, iommu_cr)), 153 (u_long)bus_space_read_8(is->is_bustag, is->is_iommu, 154 offsetof (struct iommureg, iommu_tsb)), 155 (u_long)bus_space_read_8(is->is_bustag, is->is_iommu, 156 offsetof (struct iommureg, iommu_flush))); 157 printf("iommu cr=%llx tsb=%llx\n", 158 (unsigned long long)bus_space_read_8(is->is_bustag, 159 is->is_iommu, 160 offsetof (struct iommureg, iommu_cr)), 161 (unsigned long long)bus_space_read_8(is->is_bustag, 162 is->is_iommu, 163 offsetof (struct iommureg, iommu_tsb))); 164 printf("TSB base %p phys %llx\n", (void *)is->is_tsb, 165 (unsigned long long)is->is_ptsb); 166 delay(1000000); /* 1 s */ 167 } 168 #endif 169 170 /* 171 * now actually start up the IOMMU 172 */ 173 iommu_reset(is); 174 175 /* 176 * Now all the hardware's working we need to allocate a dvma map. 177 */ 178 printf("DVMA map: %x to %x\n", 179 (unsigned int)is->is_dvmabase, 180 (unsigned int)is->is_dvmaend); 181 printf("IOTSB: %llx to %llx\n", 182 (unsigned long long)is->is_ptsb, 183 (unsigned long long)(is->is_ptsb + size)); 184 is->is_dvmamap = extent_create(name, 185 is->is_dvmabase, is->is_dvmaend - NBPG, 186 M_DEVBUF, 0, 0, EX_NOWAIT); 187 } 188 189 /* 190 * Streaming buffers don't exist on the UltraSPARC IIi; we should have 191 * detected that already and disabled them. If not, we will notice that 192 * they aren't there when the STRBUF_EN bit does not remain. 193 */ 194 void 195 iommu_reset(is) 196 struct iommu_state *is; 197 { 198 int i; 199 struct strbuf_ctl *sb; 200 201 /* Need to do 64-bit stores */ 202 bus_space_write_8(is->is_bustag, is->is_iommu, IOMMUREG(iommu_tsb), 203 is->is_ptsb); 204 205 /* Enable IOMMU in diagnostic mode */ 206 bus_space_write_8(is->is_bustag, is->is_iommu, IOMMUREG(iommu_cr), 207 is->is_cr|IOMMUCR_DE); 208 209 for (i = 0; i < 2; i++) { 210 if ((sb = is->is_sb[i])) { 211 212 /* Enable diagnostics mode? */ 213 bus_space_write_8(is->is_bustag, is->is_sb[i]->sb_sb, 214 STRBUFREG(strbuf_ctl), STRBUF_EN); 215 216 /* No streaming buffers? Disable them */ 217 if (bus_space_read_8(is->is_bustag, 218 is->is_sb[i]->sb_sb, 219 STRBUFREG(strbuf_ctl)) == 0) { 220 is->is_sb[i]->sb_flush = NULL; 221 } else { 222 223 /* 224 * locate the pa of the flush buffer. 225 */ 226 (void)pmap_extract(pmap_kernel(), 227 (vaddr_t)is->is_sb[i]->sb_flush, 228 &is->is_sb[i]->sb_flushpa); 229 } 230 } 231 } 232 } 233 234 /* 235 * Here are the iommu control routines. 236 */ 237 void 238 iommu_enter(sb, va, pa, flags) 239 struct strbuf_ctl *sb; 240 vaddr_t va; 241 int64_t pa; 242 int flags; 243 { 244 struct iommu_state *is = sb->sb_is; 245 int strbuf = (flags & BUS_DMA_STREAMING); 246 int64_t tte; 247 248 #ifdef DIAGNOSTIC 249 if (va < is->is_dvmabase || va > is->is_dvmaend) 250 panic("iommu_enter: va %#lx not in DVMA space", va); 251 #endif 252 253 /* Is the streamcache flush really needed? */ 254 if (sb->sb_flush) { 255 iommu_strbuf_flush(sb, va); 256 iommu_strbuf_flush_done(sb); 257 } else 258 /* If we can't flush the strbuf don't enable it. */ 259 strbuf = 0; 260 261 tte = MAKEIOTTE(pa, !(flags & BUS_DMA_NOWRITE), 262 !(flags & BUS_DMA_NOCACHE), (strbuf)); 263 #ifdef DEBUG 264 tte |= (flags & 0xff000LL)<<(4*8); 265 #endif 266 267 DPRINTF(IDB_IOMMU, ("Clearing TSB slot %d for va %p\n", 268 (int)IOTSBSLOT(va,is->is_tsbsize), (void *)(u_long)va)); 269 is->is_tsb[IOTSBSLOT(va,is->is_tsbsize)] = tte; 270 bus_space_write_8(is->is_bustag, is->is_iommu, 271 IOMMUREG(iommu_flush), va); 272 DPRINTF(IDB_IOMMU, ("iommu_enter: va %lx pa %lx TSB[%lx]@%p=%lx\n", 273 va, (long)pa, (u_long)IOTSBSLOT(va,is->is_tsbsize), 274 (void *)(u_long)&is->is_tsb[IOTSBSLOT(va,is->is_tsbsize)], 275 (u_long)tte)); 276 } 277 278 /* 279 * Find the value of a DVMA address (debug routine). 280 */ 281 paddr_t 282 iommu_extract(is, dva) 283 struct iommu_state *is; 284 vaddr_t dva; 285 { 286 int64_t tte = 0; 287 288 if (dva >= is->is_dvmabase && dva < is->is_dvmaend) 289 tte = is->is_tsb[IOTSBSLOT(dva, is->is_tsbsize)]; 290 291 if ((tte & IOTTE_V) == 0) 292 return ((paddr_t)-1L); 293 return (tte & IOTTE_PAMASK); 294 } 295 296 /* 297 * iommu_remove: removes mappings created by iommu_enter 298 * 299 * Only demap from IOMMU if flag is set. 300 * 301 * XXX: this function needs better internal error checking. 302 */ 303 void 304 iommu_remove(is, va, len) 305 struct iommu_state *is; 306 vaddr_t va; 307 size_t len; 308 { 309 310 #ifdef DIAGNOSTIC 311 if (va < is->is_dvmabase || va > is->is_dvmaend) 312 panic("iommu_remove: va 0x%lx not in DVMA space", (u_long)va); 313 if ((long)(va + len) < (long)va) 314 panic("iommu_remove: va 0x%lx + len 0x%lx wraps", 315 (long) va, (long) len); 316 if (len & ~0xfffffff) 317 panic("iommu_remove: rediculous len 0x%lx", (u_long)len); 318 #endif 319 320 va = trunc_page(va); 321 DPRINTF(IDB_IOMMU, ("iommu_remove: va %lx TSB[%lx]@%p\n", 322 va, (u_long)IOTSBSLOT(va, is->is_tsbsize), 323 &is->is_tsb[IOTSBSLOT(va, is->is_tsbsize)])); 324 while (len > 0) { 325 DPRINTF(IDB_IOMMU, ("iommu_remove: clearing TSB slot %d " 326 "for va %p size %lx\n", 327 (int)IOTSBSLOT(va,is->is_tsbsize), (void *)(u_long)va, 328 (u_long)len)); 329 if (len <= NBPG) 330 len = 0; 331 else 332 len -= NBPG; 333 334 /* XXX Zero-ing the entry would not require RMW */ 335 is->is_tsb[IOTSBSLOT(va,is->is_tsbsize)] &= ~IOTTE_V; 336 bus_space_write_8(is->is_bustag, is->is_iommu, 337 IOMMUREG(iommu_flush), va); 338 va += NBPG; 339 } 340 } 341 342 static int 343 iommu_strbuf_flush_done(sb) 344 struct strbuf_ctl *sb; 345 { 346 struct iommu_state *is = sb->sb_is; 347 struct timeval cur, flushtimeout; 348 349 #define BUMPTIME(t, usec) { \ 350 register volatile struct timeval *tp = (t); \ 351 register long us; \ 352 \ 353 tp->tv_usec = us = tp->tv_usec + (usec); \ 354 if (us >= 1000000) { \ 355 tp->tv_usec = us - 1000000; \ 356 tp->tv_sec++; \ 357 } \ 358 } 359 360 if (!sb->sb_flush) 361 return (0); 362 363 /* 364 * Streaming buffer flushes: 365 * 366 * 1 Tell strbuf to flush by storing va to strbuf_pgflush. If 367 * we're not on a cache line boundary (64-bits): 368 * 2 Store 0 in flag 369 * 3 Store pointer to flag in flushsync 370 * 4 wait till flushsync becomes 0x1 371 * 372 * If it takes more than .5 sec, something 373 * went wrong. 374 */ 375 376 *sb->sb_flush = 0; 377 bus_space_write_8(is->is_bustag, sb->sb_sb, 378 STRBUFREG(strbuf_flushsync), sb->sb_flushpa); 379 380 microtime(&flushtimeout); 381 cur = flushtimeout; 382 BUMPTIME(&flushtimeout, 500000); /* 1/2 sec */ 383 384 DPRINTF(IDB_IOMMU, ("iommu_strbuf_flush_done: flush = %lx " 385 "at va = %lx pa = %lx now=%lx:%lx until = %lx:%lx\n", 386 (long)*sb->sb_flush, (long)sb->sb_flush, (long)sb->sb_flushpa, 387 cur.tv_sec, cur.tv_usec, 388 flushtimeout.tv_sec, flushtimeout.tv_usec)); 389 390 /* Bypass non-coherent D$ */ 391 while ((!ldxa(sb->sb_flushpa, ASI_PHYS_CACHED)) && 392 ((cur.tv_sec <= flushtimeout.tv_sec) && 393 (cur.tv_usec <= flushtimeout.tv_usec))) 394 microtime(&cur); 395 396 #ifdef DIAGNOSTIC 397 if (!ldxa(sb->sb_flushpa, ASI_PHYS_CACHED)) { 398 printf("iommu_strbuf_flush_done: flush timeout %p, at %p\n", 399 (void *)(u_long)*sb->sb_flush, 400 (void *)(u_long)sb->sb_flushpa); /* panic? */ 401 #ifdef DDB 402 Debugger(); 403 #endif 404 } 405 #endif 406 DPRINTF(IDB_IOMMU, ("iommu_strbuf_flush_done: flushed\n")); 407 return (*sb->sb_flush); 408 } 409 410 /* 411 * IOMMU DVMA operations, common to SBUS and PCI. 412 */ 413 int 414 iommu_dvmamap_load(t, sb, map, buf, buflen, p, flags) 415 bus_dma_tag_t t; 416 struct strbuf_ctl *sb; 417 bus_dmamap_t map; 418 void *buf; 419 bus_size_t buflen; 420 struct proc *p; 421 int flags; 422 { 423 struct iommu_state *is = sb->sb_is; 424 int s; 425 int err; 426 bus_size_t sgsize; 427 paddr_t curaddr; 428 u_long dvmaddr, sgstart, sgend; 429 bus_size_t align, boundary; 430 vaddr_t vaddr = (vaddr_t)buf; 431 int seg; 432 struct pmap *pmap; 433 434 if (map->dm_nsegs) { 435 /* Already in use?? */ 436 #ifdef DIAGNOSTIC 437 printf("iommu_dvmamap_load: map still in use\n"); 438 #endif 439 bus_dmamap_unload(t, map); 440 } 441 442 /* 443 * Make sure that on error condition we return "no valid mappings". 444 */ 445 map->dm_nsegs = 0; 446 if (buflen > map->_dm_size) { 447 DPRINTF(IDB_BUSDMA, 448 ("iommu_dvmamap_load(): error %d > %d -- " 449 "map size exceeded!\n", (int)buflen, (int)map->_dm_size)); 450 return (EINVAL); 451 } 452 453 sgsize = round_page(buflen + ((int)vaddr & PGOFSET)); 454 455 /* 456 * A boundary presented to bus_dmamem_alloc() takes precedence 457 * over boundary in the map. 458 */ 459 if ((boundary = (map->dm_segs[0]._ds_boundary)) == 0) 460 boundary = map->_dm_boundary; 461 align = max(map->dm_segs[0]._ds_align, NBPG); 462 463 /* 464 * If our segment size is larger than the boundary we need to 465 * split the transfer up int little pieces ourselves. 466 */ 467 s = splhigh(); 468 err = extent_alloc(is->is_dvmamap, sgsize, align, 469 (sgsize > boundary) ? 0 : boundary, 470 EX_NOWAIT|EX_BOUNDZERO, &dvmaddr); 471 splx(s); 472 473 #ifdef DEBUG 474 if (err || (dvmaddr == (bus_addr_t)-1)) 475 { 476 printf("iommu_dvmamap_load(): extent_alloc(%d, %x) failed!\n", 477 (int)sgsize, flags); 478 #ifdef DDB 479 Debugger(); 480 #endif 481 } 482 #endif 483 if (err != 0) 484 return (err); 485 486 if (dvmaddr == (bus_addr_t)-1) 487 return (ENOMEM); 488 489 /* Set the active DVMA map */ 490 map->_dm_dvmastart = dvmaddr; 491 map->_dm_dvmasize = sgsize; 492 493 /* 494 * Now split the DVMA range into segments, not crossing 495 * the boundary. 496 */ 497 seg = 0; 498 sgstart = dvmaddr + (vaddr & PGOFSET); 499 sgend = sgstart + buflen - 1; 500 map->dm_segs[seg].ds_addr = sgstart; 501 DPRINTF(IDB_INFO, ("iommu_dvmamap_load: boundary %lx boundary-1 %lx " 502 "~(boundary-1) %lx\n", boundary, (boundary-1), ~(boundary-1))); 503 while ((sgstart & ~(boundary - 1)) != (sgend & ~(boundary - 1))) { 504 /* Oops. We crossed a boundary. Split the xfer. */ 505 DPRINTF(IDB_INFO, ("iommu_dvmamap_load: " 506 "seg %d start %lx size %lx\n", seg, 507 (long)map->dm_segs[seg].ds_addr, 508 map->dm_segs[seg].ds_len)); 509 map->dm_segs[seg].ds_len = 510 boundary - (sgstart & (boundary - 1)); 511 if (++seg >= map->_dm_segcnt) { 512 /* Too many segments. Fail the operation. */ 513 DPRINTF(IDB_INFO, ("iommu_dvmamap_load: " 514 "too many segments %d\n", seg)); 515 s = splhigh(); 516 /* How can this fail? And if it does what can we do? */ 517 err = extent_free(is->is_dvmamap, 518 dvmaddr, sgsize, EX_NOWAIT); 519 map->_dm_dvmastart = 0; 520 map->_dm_dvmasize = 0; 521 splx(s); 522 return (E2BIG); 523 } 524 sgstart = roundup(sgstart, boundary); 525 map->dm_segs[seg].ds_addr = sgstart; 526 } 527 map->dm_segs[seg].ds_len = sgend - sgstart + 1; 528 DPRINTF(IDB_INFO, ("iommu_dvmamap_load: " 529 "seg %d start %lx size %lx\n", seg, 530 (long)map->dm_segs[seg].ds_addr, map->dm_segs[seg].ds_len)); 531 map->dm_nsegs = seg+1; 532 map->dm_mapsize = buflen; 533 534 if (p != NULL) 535 pmap = p->p_vmspace->vm_map.pmap; 536 else 537 pmap = pmap_kernel(); 538 539 for (; buflen > 0; ) { 540 541 /* 542 * Get the physical address for this page. 543 */ 544 if (pmap_extract(pmap, (vaddr_t)vaddr, &curaddr) == FALSE) { 545 bus_dmamap_unload(t, map); 546 return (-1); 547 } 548 549 /* 550 * Compute the segment size, and adjust counts. 551 */ 552 sgsize = NBPG - ((u_long)vaddr & PGOFSET); 553 if (buflen < sgsize) 554 sgsize = buflen; 555 556 DPRINTF(IDB_BUSDMA, 557 ("iommu_dvmamap_load: map %p loading va %p " 558 "dva %lx at pa %lx\n", 559 map, (void *)vaddr, (long)dvmaddr, 560 (long)(curaddr & ~(NBPG-1)))); 561 iommu_enter(sb, trunc_page(dvmaddr), trunc_page(curaddr), 562 flags|0x4000); 563 564 dvmaddr += PAGE_SIZE; 565 vaddr += sgsize; 566 buflen -= sgsize; 567 } 568 #ifdef DIAGNOSTIC 569 for (seg = 0; seg < map->dm_nsegs; seg++) { 570 if (map->dm_segs[seg].ds_addr < is->is_dvmabase || 571 map->dm_segs[seg].ds_addr > is->is_dvmaend) { 572 printf("seg %d dvmaddr %lx out of range %x - %x\n", 573 seg, (long)map->dm_segs[seg].ds_addr, 574 is->is_dvmabase, is->is_dvmaend); 575 #ifdef DDB 576 Debugger(); 577 #endif 578 } 579 } 580 #endif 581 return (0); 582 } 583 584 585 void 586 iommu_dvmamap_unload(t, sb, map) 587 bus_dma_tag_t t; 588 struct strbuf_ctl *sb; 589 bus_dmamap_t map; 590 { 591 struct iommu_state *is = sb->sb_is; 592 int error, s; 593 bus_size_t sgsize; 594 595 /* Flush the iommu */ 596 #ifdef DEBUG 597 if (!map->_dm_dvmastart) { 598 printf("iommu_dvmamap_unload: No dvmastart is zero\n"); 599 #ifdef DDB 600 Debugger(); 601 #endif 602 } 603 #endif 604 iommu_remove(is, map->_dm_dvmastart, map->_dm_dvmasize); 605 606 /* Flush the caches */ 607 bus_dmamap_unload(t->_parent, map); 608 609 /* Mark the mappings as invalid. */ 610 map->dm_mapsize = 0; 611 map->dm_nsegs = 0; 612 613 s = splhigh(); 614 error = extent_free(is->is_dvmamap, map->_dm_dvmastart, 615 map->_dm_dvmasize, EX_NOWAIT); 616 map->_dm_dvmastart = 0; 617 map->_dm_dvmasize = 0; 618 splx(s); 619 if (error != 0) 620 printf("warning: %qd of DVMA space lost\n", (long long)sgsize); 621 622 /* Clear the map */ 623 } 624 625 626 int 627 iommu_dvmamap_load_raw(t, sb, map, segs, nsegs, flags, size) 628 bus_dma_tag_t t; 629 struct strbuf_ctl *sb; 630 bus_dmamap_t map; 631 bus_dma_segment_t *segs; 632 int nsegs; 633 int flags; 634 bus_size_t size; 635 { 636 struct iommu_state *is = sb->sb_is; 637 struct vm_page *pg; 638 int i, j, s; 639 int left; 640 int err; 641 bus_size_t sgsize; 642 paddr_t pa; 643 bus_size_t boundary, align; 644 u_long dvmaddr, sgstart, sgend; 645 struct pglist *pglist; 646 int pagesz = PAGE_SIZE; 647 int npg = 0; /* DEBUG */ 648 649 if (map->dm_nsegs) { 650 /* Already in use?? */ 651 #ifdef DIAGNOSTIC 652 printf("iommu_dvmamap_load_raw: map still in use\n"); 653 #endif 654 bus_dmamap_unload(t, map); 655 } 656 657 /* 658 * A boundary presented to bus_dmamem_alloc() takes precedence 659 * over boundary in the map. 660 */ 661 if ((boundary = segs[0]._ds_boundary) == 0) 662 boundary = map->_dm_boundary; 663 664 align = max(segs[0]._ds_align, pagesz); 665 666 /* 667 * Make sure that on error condition we return "no valid mappings". 668 */ 669 map->dm_nsegs = 0; 670 /* Count up the total number of pages we need */ 671 pa = segs[0].ds_addr; 672 sgsize = 0; 673 left = size; 674 for (i = 0; left && i < nsegs; i++) { 675 if (round_page(pa) != round_page(segs[i].ds_addr)) 676 sgsize = round_page(sgsize); 677 sgsize += min(left, segs[i].ds_len); 678 left -= segs[i].ds_len; 679 pa = segs[i].ds_addr + segs[i].ds_len; 680 } 681 sgsize = round_page(sgsize); 682 683 s = splhigh(); 684 /* 685 * If our segment size is larger than the boundary we need to 686 * split the transfer up into little pieces ourselves. 687 */ 688 err = extent_alloc(is->is_dvmamap, sgsize, align, 689 (sgsize > boundary) ? 0 : boundary, 690 ((flags & BUS_DMA_NOWAIT) == 0 ? EX_WAITOK : EX_NOWAIT) | 691 EX_BOUNDZERO, &dvmaddr); 692 splx(s); 693 694 if (err != 0) 695 return (err); 696 697 #ifdef DEBUG 698 if (dvmaddr == (bus_addr_t)-1) 699 { 700 printf("iommu_dvmamap_load_raw(): extent_alloc(%d, %x) failed!\n", 701 (int)sgsize, flags); 702 #ifdef DDB 703 Debugger(); 704 #endif 705 } 706 #endif 707 if (dvmaddr == (bus_addr_t)-1) 708 return (ENOMEM); 709 710 /* Set the active DVMA map */ 711 map->_dm_dvmastart = dvmaddr; 712 map->_dm_dvmasize = sgsize; 713 714 if ((pglist = segs[0]._ds_mlist) == NULL) { 715 u_long prev_va = NULL; 716 paddr_t prev_pa = 0; 717 int end = 0, offset; 718 719 /* 720 * This segs is made up of individual physical 721 * segments, probably by _bus_dmamap_load_uio() or 722 * _bus_dmamap_load_mbuf(). Ignore the mlist and 723 * load each one individually. 724 */ 725 map->dm_mapsize = size; 726 727 j = 0; 728 for (i = 0; i < nsegs ; i++) { 729 730 pa = segs[i].ds_addr; 731 offset = (pa & PGOFSET); 732 pa = trunc_page(pa); 733 dvmaddr = trunc_page(dvmaddr); 734 left = min(size, segs[i].ds_len); 735 736 DPRINTF(IDB_INFO, ("iommu_dvmamap_load_raw: converting " 737 "physseg %d start %lx size %lx\n", i, 738 (long)segs[i].ds_addr, segs[i].ds_len)); 739 740 if ((pa == prev_pa) && 741 ((offset != 0) || (end != offset))) { 742 /* We can re-use this mapping */ 743 dvmaddr = prev_va; 744 } 745 746 sgstart = dvmaddr + offset; 747 sgend = sgstart + left - 1; 748 749 /* Are the segments virtually adjacent? */ 750 if ((j > 0) && (end == offset) && 751 ((offset == 0) || (pa == prev_pa))) { 752 /* Just append to the previous segment. */ 753 map->dm_segs[--j].ds_len += left; 754 DPRINTF(IDB_INFO, ("iommu_dvmamap_load_raw: " 755 "appending seg %d start %lx size %lx\n", j, 756 (long)map->dm_segs[j].ds_addr, 757 map->dm_segs[j].ds_len)); 758 } else { 759 if (j >= map->_dm_segcnt) { 760 iommu_dvmamap_unload(t, sb, map); 761 return (E2BIG); 762 } 763 map->dm_segs[j].ds_addr = sgstart; 764 map->dm_segs[j].ds_len = left; 765 DPRINTF(IDB_INFO, ("iommu_dvmamap_load_raw: " 766 "seg %d start %lx size %lx\n", j, 767 (long)map->dm_segs[j].ds_addr, 768 map->dm_segs[j].ds_len)); 769 } 770 end = (offset + left) & PGOFSET; 771 772 /* Check for boundary issues */ 773 while ((sgstart & ~(boundary - 1)) != 774 (sgend & ~(boundary - 1))) { 775 /* Need a new segment. */ 776 map->dm_segs[j].ds_len = 777 boundary - (sgstart & (boundary - 1)); 778 DPRINTF(IDB_INFO, ("iommu_dvmamap_load_raw: " 779 "seg %d start %lx size %lx\n", j, 780 (long)map->dm_segs[j].ds_addr, 781 map->dm_segs[j].ds_len)); 782 if (++j >= map->_dm_segcnt) { 783 iommu_dvmamap_unload(t, sb, map); 784 return (E2BIG); 785 } 786 sgstart = roundup(sgstart, boundary); 787 map->dm_segs[j].ds_addr = sgstart; 788 map->dm_segs[j].ds_len = sgend - sgstart + 1; 789 } 790 791 if (sgsize == 0) 792 panic("iommu_dmamap_load_raw: size botch"); 793 794 /* Now map a series of pages. */ 795 while (dvmaddr <= sgend) { 796 DPRINTF(IDB_BUSDMA, 797 ("iommu_dvmamap_load_raw: map %p " 798 "loading va %lx at pa %lx\n", 799 map, (long)dvmaddr, 800 (long)(pa))); 801 /* Enter it if we haven't before. */ 802 if (prev_va != dvmaddr) 803 iommu_enter(sb, prev_va = dvmaddr, 804 prev_pa = pa, 805 flags | (++npg << 12)); 806 dvmaddr += pagesz; 807 pa += pagesz; 808 } 809 810 size -= left; 811 ++j; 812 } 813 814 map->dm_nsegs = j; 815 #ifdef DIAGNOSTIC 816 { int seg; 817 for (seg = 0; seg < map->dm_nsegs; seg++) { 818 if (map->dm_segs[seg].ds_addr < is->is_dvmabase || 819 map->dm_segs[seg].ds_addr > is->is_dvmaend) { 820 printf("seg %d dvmaddr %lx out of range %x - %x\n", 821 seg, (long)map->dm_segs[seg].ds_addr, 822 is->is_dvmabase, is->is_dvmaend); 823 #ifdef DDB 824 Debugger(); 825 #endif 826 } 827 } 828 } 829 #endif 830 return (0); 831 } 832 833 /* 834 * This was allocated with bus_dmamem_alloc. 835 * The pages are on a `pglist'. 836 */ 837 map->dm_mapsize = size; 838 i = 0; 839 sgstart = dvmaddr; 840 sgend = sgstart + size - 1; 841 map->dm_segs[i].ds_addr = sgstart; 842 while ((sgstart & ~(boundary - 1)) != (sgend & ~(boundary - 1))) { 843 /* Oops. We crossed a boundary. Split the xfer. */ 844 map->dm_segs[i].ds_len = boundary - (sgstart & (boundary - 1)); 845 DPRINTF(IDB_INFO, ("iommu_dvmamap_load_raw: " 846 "seg %d start %lx size %lx\n", i, 847 (long)map->dm_segs[i].ds_addr, 848 map->dm_segs[i].ds_len)); 849 if (++i >= map->_dm_segcnt) { 850 /* Too many segments. Fail the operation. */ 851 s = splhigh(); 852 /* How can this fail? And if it does what can we do? */ 853 err = extent_free(is->is_dvmamap, 854 dvmaddr, sgsize, EX_NOWAIT); 855 map->_dm_dvmastart = 0; 856 map->_dm_dvmasize = 0; 857 splx(s); 858 return (E2BIG); 859 } 860 sgstart = roundup(sgstart, boundary); 861 map->dm_segs[i].ds_addr = sgstart; 862 } 863 DPRINTF(IDB_INFO, ("iommu_dvmamap_load_raw: " 864 "seg %d start %lx size %lx\n", i, 865 (long)map->dm_segs[i].ds_addr, map->dm_segs[i].ds_len)); 866 map->dm_segs[i].ds_len = sgend - sgstart + 1; 867 868 TAILQ_FOREACH(pg, pglist, pageq) { 869 if (sgsize == 0) 870 panic("iommu_dmamap_load_raw: size botch"); 871 pa = VM_PAGE_TO_PHYS(pg); 872 873 DPRINTF(IDB_BUSDMA, 874 ("iommu_dvmamap_load_raw: map %p loading va %lx at pa %lx\n", 875 map, (long)dvmaddr, (long)(pa))); 876 iommu_enter(sb, dvmaddr, pa, flags|0x8000); 877 878 dvmaddr += pagesz; 879 sgsize -= pagesz; 880 } 881 map->dm_mapsize = size; 882 map->dm_nsegs = i+1; 883 #ifdef DIAGNOSTIC 884 { int seg; 885 for (seg = 0; seg < map->dm_nsegs; seg++) { 886 if (map->dm_segs[seg].ds_addr < is->is_dvmabase || 887 map->dm_segs[seg].ds_addr > is->is_dvmaend) { 888 printf("seg %d dvmaddr %lx out of range %x - %x\n", 889 seg, (long)map->dm_segs[seg].ds_addr, 890 is->is_dvmabase, is->is_dvmaend); 891 #ifdef DDB 892 Debugger(); 893 #endif 894 } 895 } 896 } 897 #endif 898 return (0); 899 } 900 901 void 902 iommu_dvmamap_sync(t, sb, map, offset, len, ops) 903 bus_dma_tag_t t; 904 struct strbuf_ctl *sb; 905 bus_dmamap_t map; 906 bus_addr_t offset; 907 bus_size_t len; 908 int ops; 909 { 910 struct iommu_state *is = sb->sb_is; 911 vaddr_t va = map->dm_segs[0].ds_addr + offset; 912 int64_t tte; 913 914 /* 915 * We only support one DMA segment; supporting more makes this code 916 * too unwieldy. 917 */ 918 919 DPRINTF(IDB_SYNC, 920 ("iommu_dvmamap_sync: syncing va %p len %lu " 921 "ops 0x%x\n", (void *)(u_long)va, (u_long)len, ops)); 922 if (ops & (BUS_DMASYNC_PREREAD | BUS_DMASYNC_POSTWRITE)) { 923 /* Nothing to do */ 924 return; 925 } 926 #ifdef DIAGNOSTIC 927 if (va < is->is_dvmabase || va >= is->is_dvmaend) 928 panic("iommu_dvmamap_sync: invalid dva %lx", va); 929 #endif 930 KASSERT((ops & (BUS_DMASYNC_POSTREAD | BUS_DMASYNC_PREWRITE)) != 0); 931 tte = is->is_tsb[IOTSBSLOT(va, is->is_tsbsize)]; 932 933 /* if we have a streaming buffer, flush it here first */ 934 if ((tte & IOTTE_STREAM) == 0 || !sb->sb_flush) 935 return; 936 937 while (len > 0) { 938 DPRINTF(IDB_BUSDMA, 939 ("iommu_dvmamap_sync: flushing va %p, %lu " 940 "bytes left\n", (void *)(u_long)va, (u_long)len)); 941 iommu_strbuf_flush(sb, va); 942 if (len <= NBPG) { 943 iommu_strbuf_flush_done(sb); 944 len = 0; 945 } else 946 len -= NBPG; 947 va += NBPG; 948 } 949 } 950 951 int 952 iommu_dvmamem_alloc(t, sb, size, alignment, boundary, segs, nsegs, rsegs, flags) 953 bus_dma_tag_t t; 954 struct strbuf_ctl *sb; 955 bus_size_t size, alignment, boundary; 956 bus_dma_segment_t *segs; 957 int nsegs; 958 int *rsegs; 959 int flags; 960 { 961 962 DPRINTF(IDB_BUSDMA, ("iommu_dvmamem_alloc: sz %llx align %llx bound %llx " 963 "segp %p flags %d\n", (unsigned long long)size, 964 (unsigned long long)alignment, (unsigned long long)boundary, 965 segs, flags)); 966 return (bus_dmamem_alloc(t->_parent, size, alignment, boundary, 967 segs, nsegs, rsegs, flags|BUS_DMA_DVMA)); 968 } 969 970 void 971 iommu_dvmamem_free(t, sb, segs, nsegs) 972 bus_dma_tag_t t; 973 struct strbuf_ctl *sb; 974 bus_dma_segment_t *segs; 975 int nsegs; 976 { 977 978 DPRINTF(IDB_BUSDMA, ("iommu_dvmamem_free: segp %p nsegs %d\n", 979 segs, nsegs)); 980 bus_dmamem_free(t->_parent, segs, nsegs); 981 } 982 983 /* 984 * Map the DVMA mappings into the kernel pmap. 985 * Check the flags to see whether we're streaming or coherent. 986 */ 987 int 988 iommu_dvmamem_map(t, sb, segs, nsegs, size, kvap, flags) 989 bus_dma_tag_t t; 990 struct strbuf_ctl *sb; 991 bus_dma_segment_t *segs; 992 int nsegs; 993 size_t size; 994 caddr_t *kvap; 995 int flags; 996 { 997 struct vm_page *pg; 998 vaddr_t va; 999 bus_addr_t addr; 1000 struct pglist *pglist; 1001 int cbit; 1002 1003 DPRINTF(IDB_BUSDMA, ("iommu_dvmamem_map: segp %p nsegs %d size %lx\n", 1004 segs, nsegs, size)); 1005 1006 /* 1007 * Allocate some space in the kernel map, and then map these pages 1008 * into this space. 1009 */ 1010 size = round_page(size); 1011 va = uvm_km_valloc(kernel_map, size); 1012 if (va == 0) 1013 return (ENOMEM); 1014 1015 *kvap = (caddr_t)va; 1016 1017 /* 1018 * digest flags: 1019 */ 1020 cbit = 0; 1021 if (flags & BUS_DMA_COHERENT) /* Disable vcache */ 1022 cbit |= PMAP_NVC; 1023 if (flags & BUS_DMA_NOCACHE) /* sideffects */ 1024 cbit |= PMAP_NC; 1025 1026 /* 1027 * Now take this and map it into the CPU. 1028 */ 1029 pglist = segs[0]._ds_mlist; 1030 TAILQ_FOREACH(pg, pglist, pageq) { 1031 #ifdef DIAGNOSTIC 1032 if (size == 0) 1033 panic("iommu_dvmamem_map: size botch"); 1034 #endif 1035 addr = VM_PAGE_TO_PHYS(pg); 1036 DPRINTF(IDB_BUSDMA, ("iommu_dvmamem_map: " 1037 "mapping va %lx at %llx\n", va, (unsigned long long)addr | cbit)); 1038 pmap_kenter_pa(va, addr | cbit, VM_PROT_READ | VM_PROT_WRITE); 1039 va += PAGE_SIZE; 1040 size -= PAGE_SIZE; 1041 } 1042 pmap_update(pmap_kernel()); 1043 return (0); 1044 } 1045 1046 /* 1047 * Unmap DVMA mappings from kernel 1048 */ 1049 void 1050 iommu_dvmamem_unmap(t, sb, kva, size) 1051 bus_dma_tag_t t; 1052 struct strbuf_ctl *sb; 1053 caddr_t kva; 1054 size_t size; 1055 { 1056 1057 DPRINTF(IDB_BUSDMA, ("iommu_dvmamem_unmap: kvm %p size %lx\n", 1058 kva, size)); 1059 1060 #ifdef DIAGNOSTIC 1061 if ((u_long)kva & PGOFSET) 1062 panic("iommu_dvmamem_unmap"); 1063 #endif 1064 1065 size = round_page(size); 1066 pmap_kremove((vaddr_t)kva, size); 1067 pmap_update(pmap_kernel()); 1068 uvm_km_free(kernel_map, (vaddr_t)kva, size); 1069 } 1070
Indexes created Sat Sep 27 22:09:54 GMT 2025