iommu.c revision 1.41
1 /* $NetBSD: iommu.c,v 1.41 2001/09/28 11:59:53 chs Exp $ */ 2 3 /* 4 * Copyright (c) 1999, 2000 Matthew R. Green 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 3. The name of the author may not be used to endorse or promote products 16 * derived from this software without specific prior written permission. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 19 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 20 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 21 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 22 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 23 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 24 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 25 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 26 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 28 * SUCH DAMAGE. 29 */ 30 31 /*- 32 * Copyright (c) 1998 The NetBSD Foundation, Inc. 33 * All rights reserved. 34 * 35 * This code is derived from software contributed to The NetBSD Foundation 36 * by Paul Kranenburg. 37 * 38 * Redistribution and use in source and binary forms, with or without 39 * modification, are permitted provided that the following conditions 40 * are met: 41 * 1. Redistributions of source code must retain the above copyright 42 * notice, this list of conditions and the following disclaimer. 43 * 2. Redistributions in binary form must reproduce the above copyright 44 * notice, this list of conditions and the following disclaimer in the 45 * documentation and/or other materials provided with the distribution. 46 * 3. All advertising materials mentioning features or use of this software 47 * must display the following acknowledgement: 48 * This product includes software developed by the NetBSD 49 * Foundation, Inc. and its contributors. 50 * 4. Neither the name of The NetBSD Foundation nor the names of its 51 * contributors may be used to endorse or promote products derived 52 * from this software without specific prior written permission. 53 * 54 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 55 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 56 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 57 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 58 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 59 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 60 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 61 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 62 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 63 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 64 * POSSIBILITY OF SUCH DAMAGE. 65 */ 66 67 /* 68 * Copyright (c) 1992, 1993 69 * The Regents of the University of California. All rights reserved. 70 * 71 * This software was developed by the Computer Systems Engineering group 72 * at Lawrence Berkeley Laboratory under DARPA contract BG 91-66 and 73 * contributed to Berkeley. 74 * 75 * All advertising materials mentioning features or use of this software 76 * must display the following acknowledgement: 77 * This product includes software developed by the University of 78 * California, Lawrence Berkeley Laboratory. 79 * 80 * Redistribution and use in source and binary forms, with or without 81 * modification, are permitted provided that the following conditions 82 * are met: 83 * 1. Redistributions of source code must retain the above copyright 84 * notice, this list of conditions and the following disclaimer. 85 * 2. Redistributions in binary form must reproduce the above copyright 86 * notice, this list of conditions and the following disclaimer in the 87 * documentation and/or other materials provided with the distribution. 88 * 3. All advertising materials mentioning features or use of this software 89 * must display the following acknowledgement: 90 * This product includes software developed by the University of 91 * California, Berkeley and its contributors. 92 * 4. Neither the name of the University nor the names of its contributors 93 * may be used to endorse or promote products derived from this software 94 * without specific prior written permission. 95 * 96 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 97 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 98 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 99 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 100 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 101 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 102 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 103 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 104 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 105 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 106 * SUCH DAMAGE. 107 * 108 * from: NetBSD: sbus.c,v 1.13 1999/05/23 07:24:02 mrg Exp 109 * from: @(#)sbus.c 8.1 (Berkeley) 6/11/93 110 */ 111 112 /* 113 * UltraSPARC IOMMU support; used by both the sbus and pci code. 114 */ 115 #include "opt_ddb.h" 116 117 #include <sys/param.h> 118 #include <sys/extent.h> 119 #include <sys/malloc.h> 120 #include <sys/systm.h> 121 #include <sys/device.h> 122 #include <sys/proc.h> 123 124 #include <uvm/uvm_extern.h> 125 126 #include <machine/bus.h> 127 #include <sparc64/sparc64/cache.h> 128 #include <sparc64/dev/iommureg.h> 129 #include <sparc64/dev/iommuvar.h> 130 131 #include <machine/autoconf.h> 132 #include <machine/cpu.h> 133 134 #ifdef DEBUG 135 #define IDB_BUSDMA 0x1 136 #define IDB_IOMMU 0x2 137 #define IDB_INFO 0x4 138 #define IDB_SYNC 0x8 139 int iommudebug = 0x0; 140 #define DPRINTF(l, s) do { if (iommudebug & l) printf s; } while (0) 141 #else 142 #define DPRINTF(l, s) 143 #endif 144 145 #define iommu_strbuf_flush(i,v) bus_space_write_8((i)->is_bustag, \ 146 (bus_space_handle_t)(u_long)&(i)->is_sb->strbuf_pgflush, 0, (v)) 147 static int iommu_strbuf_flush_done __P((struct iommu_state *)); 148 149 /* 150 * initialise the UltraSPARC IOMMU (SBUS or PCI): 151 * - allocate and setup the iotsb. 152 * - enable the IOMMU 153 * - initialise the streaming buffers (if they exist) 154 * - create a private DVMA map. 155 */ 156 void 157 iommu_init(name, is, tsbsize, iovabase) 158 char *name; 159 struct iommu_state *is; 160 int tsbsize; 161 u_int32_t iovabase; 162 { 163 psize_t size; 164 vaddr_t va; 165 paddr_t pa; 166 struct vm_page *m; 167 struct pglist mlist; 168 169 /* 170 * Setup the iommu. 171 * 172 * The sun4u iommu is part of the SBUS or PCI controller so we 173 * will deal with it here.. 174 * 175 * The IOMMU address space always ends at 0xffffe000, but the starting 176 * address depends on the size of the map. The map size is 1024 * 2 ^ 177 * is->is_tsbsize entries, where each entry is 8 bytes. The start of 178 * the map can be calculated by (0xffffe000 << (8 + is->is_tsbsize)). 179 */ 180 is->is_cr = (tsbsize << 16) | IOMMUCR_EN; 181 is->is_tsbsize = tsbsize; 182 is->is_dvmabase = iovabase; 183 if (iovabase == -1) is->is_dvmabase = IOTSB_VSTART(is->is_tsbsize); 184 185 /* 186 * Allocate memory for I/O pagetables. They need to be physically 187 * contiguous. 188 */ 189 190 size = NBPG<<(is->is_tsbsize); 191 TAILQ_INIT(&mlist); 192 if (uvm_pglistalloc((psize_t)size, (paddr_t)0, (paddr_t)-1, 193 (paddr_t)NBPG, (paddr_t)0, &mlist, 1, 0) != 0) 194 panic("iommu_init: no memory"); 195 196 va = uvm_km_valloc(kernel_map, size); 197 if (va == 0) 198 panic("iommu_init: no memory"); 199 is->is_tsb = (int64_t *)va; 200 201 m = TAILQ_FIRST(&mlist); 202 is->is_ptsb = VM_PAGE_TO_PHYS(m); 203 204 /* Map the pages */ 205 for (; m != NULL; m = TAILQ_NEXT(m,pageq)) { 206 pa = VM_PAGE_TO_PHYS(m); 207 pmap_enter(pmap_kernel(), va, pa | PMAP_NVC, 208 VM_PROT_READ|VM_PROT_WRITE, 209 VM_PROT_READ|VM_PROT_WRITE|PMAP_WIRED); 210 va += NBPG; 211 } 212 pmap_update(pmap_kernel()); 213 bzero(is->is_tsb, size); 214 215 #ifdef DEBUG 216 if (iommudebug & IDB_INFO) 217 { 218 /* Probe the iommu */ 219 struct iommureg *regs = is->is_iommu; 220 221 printf("iommu regs at: cr=%lx tsb=%lx flush=%lx\n", 222 (u_long)®s->iommu_cr, 223 (u_long)®s->iommu_tsb, 224 (u_long)®s->iommu_flush); 225 printf("iommu cr=%llx tsb=%llx\n", (unsigned long long)regs->iommu_cr, (unsigned long long)regs->iommu_tsb); 226 printf("TSB base %p phys %llx\n", (void *)is->is_tsb, (unsigned long long)is->is_ptsb); 227 delay(1000000); /* 1 s */ 228 } 229 #endif 230 231 /* 232 * Initialize streaming buffer, if it is there. 233 */ 234 if (is->is_sb) 235 (void)pmap_extract(pmap_kernel(), (vaddr_t)&is->is_flush, 236 (paddr_t *)&is->is_flushpa); 237 238 /* 239 * now actually start up the IOMMU 240 */ 241 iommu_reset(is); 242 243 /* 244 * Now all the hardware's working we need to allocate a dvma map. 245 */ 246 printf("DVMA map: %x to %x\n", 247 (unsigned int)is->is_dvmabase, 248 (unsigned int)(is->is_dvmabase+(size<<10))); 249 is->is_dvmamap = extent_create(name, 250 is->is_dvmabase, (u_long)IOTSB_VEND, 251 M_DEVBUF, 0, 0, EX_NOWAIT); 252 } 253 254 /* 255 * Streaming buffers don't exist on the UltraSPARC IIi; we should have 256 * detected that already and disabled them. If not, we will notice that 257 * they aren't there when the STRBUF_EN bit does not remain. 258 */ 259 void 260 iommu_reset(is) 261 struct iommu_state *is; 262 { 263 264 /* Need to do 64-bit stores */ 265 bus_space_write_8(is->is_bustag, 266 (bus_space_handle_t)(u_long)&is->is_iommu->iommu_tsb, 267 0, is->is_ptsb); 268 /* Enable IOMMU in diagnostic mode */ 269 bus_space_write_8(is->is_bustag, 270 (bus_space_handle_t)(u_long)&is->is_iommu->iommu_cr, 0, 271 is->is_cr|IOMMUCR_DE); 272 273 274 if (!is->is_sb) 275 return; 276 277 /* Enable diagnostics mode? */ 278 bus_space_write_8(is->is_bustag, 279 (bus_space_handle_t)(u_long)&is->is_sb->strbuf_ctl, 280 0, STRBUF_EN); 281 282 /* No streaming buffers? Disable them */ 283 if (bus_space_read_8(is->is_bustag, 284 (bus_space_handle_t)(u_long)&is->is_sb->strbuf_ctl, 285 0) == 0) 286 is->is_sb = 0; 287 } 288 289 /* 290 * Here are the iommu control routines. 291 */ 292 void 293 iommu_enter(is, va, pa, flags) 294 struct iommu_state *is; 295 vaddr_t va; 296 int64_t pa; 297 int flags; 298 { 299 int64_t tte; 300 301 #ifdef DIAGNOSTIC 302 if (va < is->is_dvmabase) 303 panic("iommu_enter: va %#lx not in DVMA space", va); 304 #endif 305 306 tte = MAKEIOTTE(pa, !(flags&BUS_DMA_NOWRITE), !(flags&BUS_DMA_NOCACHE), 307 (flags&BUS_DMA_STREAMING)); 308 309 /* Is the streamcache flush really needed? */ 310 if (is->is_sb) { 311 iommu_strbuf_flush(is, va); 312 iommu_strbuf_flush_done(is); 313 } 314 DPRINTF(IDB_IOMMU, ("Clearing TSB slot %d for va %p\n", 315 (int)IOTSBSLOT(va,is->is_tsbsize), (void *)(u_long)va)); 316 is->is_tsb[IOTSBSLOT(va,is->is_tsbsize)] = tte; 317 bus_space_write_8(is->is_bustag, (bus_space_handle_t)(u_long) 318 &is->is_iommu->iommu_flush, 0, va); 319 DPRINTF(IDB_IOMMU, ("iommu_enter: va %lx pa %lx TSB[%lx]@%p=%lx\n", 320 va, (long)pa, (u_long)IOTSBSLOT(va,is->is_tsbsize), 321 (void *)(u_long)&is->is_tsb[IOTSBSLOT(va,is->is_tsbsize)], 322 (u_long)tte)); 323 } 324 325 326 /* 327 * Find the value of a DVMA address (debug routine). 328 */ 329 paddr_t 330 iommu_extract(is, dva) 331 struct iommu_state *is; 332 vaddr_t dva; 333 { 334 int64_t tte = 0; 335 336 if (dva >= is->is_dvmabase) 337 tte = is->is_tsb[IOTSBSLOT(dva,is->is_tsbsize)]; 338 339 if ((tte&IOTTE_V) == 0) 340 return ((paddr_t)-1L); 341 return (tte&IOTTE_PAMASK); 342 } 343 344 /* 345 * iommu_remove: removes mappings created by iommu_enter 346 * 347 * Only demap from IOMMU if flag is set. 348 * 349 * XXX: this function needs better internal error checking. 350 */ 351 void 352 iommu_remove(is, va, len) 353 struct iommu_state *is; 354 vaddr_t va; 355 size_t len; 356 { 357 358 #ifdef DIAGNOSTIC 359 if (va < is->is_dvmabase) 360 panic("iommu_remove: va 0x%lx not in DVMA space", (u_long)va); 361 if ((long)(va + len) < (long)va) 362 panic("iommu_remove: va 0x%lx + len 0x%lx wraps", 363 (long) va, (long) len); 364 if (len & ~0xfffffff) 365 panic("iommu_remove: rediculous len 0x%lx", (u_long)len); 366 #endif 367 368 va = trunc_page(va); 369 DPRINTF(IDB_IOMMU, ("iommu_remove: va %lx TSB[%lx]@%p\n", 370 va, (u_long)IOTSBSLOT(va,is->is_tsbsize), 371 &is->is_tsb[IOTSBSLOT(va,is->is_tsbsize)])); 372 while (len > 0) { 373 DPRINTF(IDB_IOMMU, ("iommu_remove: clearing TSB slot %d for va %p size %lx\n", 374 (int)IOTSBSLOT(va,is->is_tsbsize), (void *)(u_long)va, (u_long)len)); 375 if (is->is_sb) { 376 DPRINTF(IDB_IOMMU, ("iommu_remove: flushing va %p TSB[%lx]@%p=%lx, %lu bytes left\n", 377 (void *)(u_long)va, (long)IOTSBSLOT(va,is->is_tsbsize), 378 (void *)(u_long)&is->is_tsb[IOTSBSLOT(va,is->is_tsbsize)], 379 (long)(is->is_tsb[IOTSBSLOT(va,is->is_tsbsize)]), 380 (u_long)len)); 381 iommu_strbuf_flush(is, va); 382 if (len <= NBPG) 383 iommu_strbuf_flush_done(is); 384 DPRINTF(IDB_IOMMU, ("iommu_remove: flushed va %p TSB[%lx]@%p=%lx, %lu bytes left\n", 385 (void *)(u_long)va, (long)IOTSBSLOT(va,is->is_tsbsize), 386 (void *)(u_long)&is->is_tsb[IOTSBSLOT(va,is->is_tsbsize)], 387 (long)(is->is_tsb[IOTSBSLOT(va,is->is_tsbsize)]), 388 (u_long)len)); 389 } 390 391 if (len <= NBPG) 392 len = 0; 393 else 394 len -= NBPG; 395 396 is->is_tsb[IOTSBSLOT(va,is->is_tsbsize)] = 0; 397 bus_space_write_8(is->is_bustag, (bus_space_handle_t)(u_long) 398 &is->is_iommu->iommu_flush, 0, va); 399 va += NBPG; 400 } 401 } 402 403 static int 404 iommu_strbuf_flush_done(is) 405 struct iommu_state *is; 406 { 407 struct timeval cur, flushtimeout; 408 409 #define BUMPTIME(t, usec) { \ 410 register volatile struct timeval *tp = (t); \ 411 register long us; \ 412 \ 413 tp->tv_usec = us = tp->tv_usec + (usec); \ 414 if (us >= 1000000) { \ 415 tp->tv_usec = us - 1000000; \ 416 tp->tv_sec++; \ 417 } \ 418 } 419 420 if (!is->is_sb) 421 return (0); 422 423 /* 424 * Streaming buffer flushes: 425 * 426 * 1 Tell strbuf to flush by storing va to strbuf_pgflush. If 427 * we're not on a cache line boundary (64-bits): 428 * 2 Store 0 in flag 429 * 3 Store pointer to flag in flushsync 430 * 4 wait till flushsync becomes 0x1 431 * 432 * If it takes more than .5 sec, something 433 * went wrong. 434 */ 435 436 is->is_flush = 0; 437 membar_sync(); /* #StoreStore is prolly enuf. */ 438 bus_space_write_8(is->is_bustag, (bus_space_handle_t)(u_long) 439 &is->is_sb->strbuf_flushsync, 0, is->is_flushpa); 440 membar_sync(); /* Prolly not needed at all. */ 441 442 microtime(&flushtimeout); 443 cur = flushtimeout; 444 BUMPTIME(&flushtimeout, 500000); /* 1/2 sec */ 445 446 DPRINTF(IDB_IOMMU, ("iommu_strbuf_flush_done: flush = %lx at va = %lx pa = %lx now=%lx:%lx until = %lx:%lx\n", 447 (long)is->is_flush, (long)&is->is_flush, 448 (long)is->is_flushpa, cur.tv_sec, cur.tv_usec, 449 flushtimeout.tv_sec, flushtimeout.tv_usec)); 450 /* Bypass non-coherent D$ */ 451 while (!ldxa(is->is_flushpa, ASI_PHYS_CACHED) && 452 ((cur.tv_sec <= flushtimeout.tv_sec) && 453 (cur.tv_usec <= flushtimeout.tv_usec))) 454 microtime(&cur); 455 456 #ifdef DIAGNOSTIC 457 if (!ldxa(is->is_flushpa, ASI_PHYS_CACHED)) { 458 printf("iommu_strbuf_flush_done: flush timeout %p at %p\n", 459 (void *)(u_long)is->is_flush, 460 (void *)(u_long)is->is_flushpa); /* panic? */ 461 #ifdef DDB 462 Debugger(); 463 #endif 464 } 465 #endif 466 DPRINTF(IDB_IOMMU, ("iommu_strbuf_flush_done: flushed\n")); 467 return (is->is_flush); 468 } 469 470 /* 471 * IOMMU DVMA operations, common to SBUS and PCI. 472 */ 473 int 474 iommu_dvmamap_load(t, is, map, buf, buflen, p, flags) 475 bus_dma_tag_t t; 476 struct iommu_state *is; 477 bus_dmamap_t map; 478 void *buf; 479 bus_size_t buflen; 480 struct proc *p; 481 int flags; 482 { 483 int s; 484 int err; 485 bus_size_t sgsize; 486 paddr_t curaddr; 487 u_long dvmaddr, sgstart, sgend; 488 bus_size_t align, boundary; 489 vaddr_t vaddr = (vaddr_t)buf; 490 int seg; 491 pmap_t pmap; 492 493 if (map->dm_nsegs) { 494 /* Already in use?? */ 495 #ifdef DIAGNOSTIC 496 printf("iommu_dvmamap_load: map still in use\n"); 497 #endif 498 bus_dmamap_unload(t, map); 499 } 500 /* 501 * Make sure that on error condition we return "no valid mappings". 502 */ 503 map->dm_nsegs = 0; 504 505 if (buflen > map->_dm_size) { 506 DPRINTF(IDB_BUSDMA, 507 ("iommu_dvmamap_load(): error %d > %d -- " 508 "map size exceeded!\n", (int)buflen, (int)map->_dm_size)); 509 return (EINVAL); 510 } 511 512 sgsize = round_page(buflen + ((int)vaddr & PGOFSET)); 513 514 /* 515 * A boundary presented to bus_dmamem_alloc() takes precedence 516 * over boundary in the map. 517 */ 518 if ((boundary = (map->dm_segs[0]._ds_boundary)) == 0) 519 boundary = map->_dm_boundary; 520 align = max(map->dm_segs[0]._ds_align, NBPG); 521 s = splhigh(); 522 /* 523 * If our segment size is larger than the boundary we need to 524 * split the transfer up int little pieces ourselves. 525 */ 526 err = extent_alloc(is->is_dvmamap, sgsize, align, 527 (sgsize > boundary) ? 0 : boundary, 528 EX_NOWAIT|EX_BOUNDZERO, (u_long *)&dvmaddr); 529 splx(s); 530 531 #ifdef DEBUG 532 if (err || (dvmaddr == (bus_addr_t)-1)) 533 { 534 printf("iommu_dvmamap_load(): extent_alloc(%d, %x) failed!\n", 535 (int)sgsize, flags); 536 #ifdef DDB 537 Debugger(); 538 #endif 539 } 540 #endif 541 if (err != 0) 542 return (err); 543 544 if (dvmaddr == (bus_addr_t)-1) 545 return (ENOMEM); 546 547 /* Set the active DVMA map */ 548 map->_dm_dvmastart = dvmaddr; 549 map->_dm_dvmasize = sgsize; 550 551 /* 552 * Now split the DVMA range into segments, not crossing 553 * the boundary. 554 */ 555 seg = 0; 556 sgstart = dvmaddr + (vaddr & PGOFSET); 557 sgend = sgstart + buflen - 1; 558 map->dm_segs[seg].ds_addr = sgstart; 559 DPRINTF(IDB_INFO, ("iommu_dvmamap_load: boundary %lx boundary-1 %lx " 560 "~(boundary-1) %lx\n", boundary, (boundary-1), ~(boundary-1))); 561 while ((sgstart & ~(boundary - 1)) != (sgend & ~(boundary - 1))) { 562 /* Oops. We crossed a boundary. Split the xfer. */ 563 DPRINTF(IDB_INFO, ("iommu_dvmamap_load: " 564 "seg %d start %lx size %lx\n", seg, 565 map->dm_segs[seg].ds_addr, map->dm_segs[seg].ds_len)); 566 map->dm_segs[seg].ds_len = sgstart & (boundary - 1); 567 if (++seg > map->_dm_segcnt) { 568 /* Too many segments. Fail the operation. */ 569 DPRINTF(IDB_INFO, ("iommu_dvmamap_load: " 570 "too many segments %d\n", seg)); 571 s = splhigh(); 572 /* How can this fail? And if it does what can we do? */ 573 err = extent_free(is->is_dvmamap, 574 dvmaddr, sgsize, EX_NOWAIT); 575 splx(s); 576 map->_dm_dvmastart = 0; 577 map->_dm_dvmasize = 0; 578 return (E2BIG); 579 } 580 sgstart = roundup(sgstart, boundary); 581 map->dm_segs[seg].ds_addr = sgstart; 582 } 583 map->dm_segs[seg].ds_len = sgend - sgstart + 1; 584 DPRINTF(IDB_INFO, ("iommu_dvmamap_load: " 585 "seg %d start %lx size %lx\n", seg, 586 map->dm_segs[seg].ds_addr, map->dm_segs[seg].ds_len)); 587 map->dm_nsegs = seg+1; 588 map->dm_mapsize = buflen; 589 590 if (p != NULL) 591 pmap = p->p_vmspace->vm_map.pmap; 592 else 593 pmap = pmap_kernel(); 594 595 for (; buflen > 0; ) { 596 /* 597 * Get the physical address for this page. 598 */ 599 if (pmap_extract(pmap, (vaddr_t)vaddr, &curaddr) == FALSE) { 600 bus_dmamap_unload(t, map); 601 return (-1); 602 } 603 604 /* 605 * Compute the segment size, and adjust counts. 606 */ 607 sgsize = NBPG - ((u_long)vaddr & PGOFSET); 608 if (buflen < sgsize) 609 sgsize = buflen; 610 611 DPRINTF(IDB_BUSDMA, 612 ("iommu_dvmamap_load: map %p loading va %p " 613 "dva %lx at pa %lx\n", 614 map, (void *)vaddr, (long)dvmaddr, 615 (long)(curaddr&~(NBPG-1)))); 616 iommu_enter(is, trunc_page(dvmaddr), trunc_page(curaddr), 617 flags); 618 619 dvmaddr += PAGE_SIZE; 620 vaddr += sgsize; 621 buflen -= sgsize; 622 } 623 return (0); 624 } 625 626 627 void 628 iommu_dvmamap_unload(t, is, map) 629 bus_dma_tag_t t; 630 struct iommu_state *is; 631 bus_dmamap_t map; 632 { 633 int error, s; 634 bus_size_t sgsize; 635 636 /* Flush the iommu */ 637 #ifdef DEBUG 638 if (!map->_dm_dvmastart) { 639 printf("iommu_dvmamap_unload: No dvmastart is zero\n"); 640 #ifdef DDB 641 Debugger(); 642 #endif 643 } 644 #endif 645 iommu_remove(is, map->_dm_dvmastart, map->_dm_dvmasize); 646 647 /* Flush the caches */ 648 bus_dmamap_unload(t->_parent, map); 649 650 /* Mark the mappings as invalid. */ 651 map->dm_mapsize = 0; 652 map->dm_nsegs = 0; 653 654 s = splhigh(); 655 error = extent_free(is->is_dvmamap, map->_dm_dvmastart, 656 map->_dm_dvmasize, EX_NOWAIT); 657 splx(s); 658 if (error != 0) 659 printf("warning: %qd of DVMA space lost\n", (long long)sgsize); 660 661 /* Clear the map */ 662 map->_dm_dvmastart = 0; 663 map->_dm_dvmasize = 0; 664 } 665 666 667 int 668 iommu_dvmamap_load_raw(t, is, map, segs, nsegs, flags, size) 669 bus_dma_tag_t t; 670 struct iommu_state *is; 671 bus_dmamap_t map; 672 bus_dma_segment_t *segs; 673 int nsegs; 674 int flags; 675 bus_size_t size; 676 { 677 struct vm_page *m; 678 int i, j, s; 679 int left; 680 int err; 681 bus_size_t sgsize; 682 paddr_t pa; 683 bus_size_t boundary, align; 684 u_long dvmaddr, sgstart, sgend; 685 struct pglist *mlist; 686 int pagesz = PAGE_SIZE; 687 688 if (map->dm_nsegs) { 689 /* Already in use?? */ 690 #ifdef DIAGNOSTIC 691 printf("iommu_dvmamap_load_raw: map still in use\n"); 692 #endif 693 bus_dmamap_unload(t, map); 694 } 695 696 /* 697 * A boundary presented to bus_dmamem_alloc() takes precedence 698 * over boundary in the map. 699 */ 700 if ((boundary = segs[0]._ds_boundary) == 0) 701 boundary = map->_dm_boundary; 702 703 align = max(segs[0]._ds_align, NBPG); 704 705 /* 706 * Make sure that on error condition we return "no valid mappings". 707 */ 708 map->dm_nsegs = 0; 709 /* Count up the total number of pages we need */ 710 pa = segs[0].ds_addr; 711 sgsize = 0; 712 left = size; 713 for (i=0; left && i<nsegs; i++) { 714 if (round_page(pa) != round_page(segs[i].ds_addr)) 715 sgsize = round_page(sgsize); 716 sgsize += min(left, segs[i].ds_len); 717 left -= segs[i].ds_len; 718 pa = segs[i].ds_addr + segs[i].ds_len; 719 } 720 sgsize = round_page(sgsize); 721 722 s = splhigh(); 723 /* 724 * If our segment size is larger than the boundary we need to 725 * split the transfer up int little pieces ourselves. 726 */ 727 err = extent_alloc(is->is_dvmamap, sgsize, align, 728 (sgsize > boundary) ? 0 : boundary, 729 ((flags & BUS_DMA_NOWAIT) == 0 ? EX_WAITOK : EX_NOWAIT) | 730 EX_BOUNDZERO, (u_long *)&dvmaddr); 731 splx(s); 732 733 if (err != 0) 734 return (err); 735 736 #ifdef DEBUG 737 if (dvmaddr == (bus_addr_t)-1) 738 { 739 printf("iommu_dvmamap_load_raw(): extent_alloc(%d, %x) failed!\n", 740 (int)sgsize, flags); 741 Debugger(); 742 } 743 #endif 744 if (dvmaddr == (bus_addr_t)-1) 745 return (ENOMEM); 746 747 /* Set the active DVMA map */ 748 map->_dm_dvmastart = dvmaddr; 749 map->_dm_dvmasize = sgsize; 750 751 if ((mlist = segs[0]._ds_mlist) == NULL) { 752 u_long prev_va = NULL; 753 /* 754 * This segs is made up of individual physical pages, 755 * probably by _bus_dmamap_load_uio() or 756 * _bus_dmamap_load_mbuf(). Ignore the mlist and 757 * load each segment individually. 758 */ 759 map->dm_mapsize = size; 760 761 i = j = 0; 762 pa = segs[i].ds_addr; 763 dvmaddr += (pa & PGOFSET); 764 left = min(size, segs[i].ds_len); 765 766 sgstart = dvmaddr; 767 sgend = sgstart + left - 1; 768 769 map->dm_segs[j].ds_addr = dvmaddr; 770 map->dm_segs[j].ds_len = left; 771 772 /* Set the size (which we will be destroying */ 773 map->dm_mapsize = size; 774 775 while (size > 0) { 776 int incr; 777 778 if (left <= 0) { 779 u_long offset; 780 781 /* 782 * If the two segs are on different physical 783 * pages move to a new virtual page. 784 */ 785 if (trunc_page(pa) != 786 trunc_page(segs[++i].ds_addr)) 787 dvmaddr += NBPG; 788 789 pa = segs[i].ds_addr; 790 left = min(size, segs[i].ds_len); 791 792 offset = (pa & PGOFSET); 793 if (dvmaddr == trunc_page(dvmaddr) + offset) { 794 /* We can combine segments */ 795 map->dm_segs[j].ds_len += left; 796 sgend += left; 797 } else { 798 /* Need a new segment */ 799 dvmaddr = trunc_page(dvmaddr) + offset; 800 DPRINTF(IDB_INFO, 801 ("iommu_dvmamap_load_raw: " 802 "seg %d start %lx " 803 "size %lx\n", j, 804 map->dm_segs[j].ds_addr, 805 map->dm_segs[j]. 806 ds_len)); 807 if (++j > map->_dm_segcnt) 808 goto fail; 809 map->dm_segs[j].ds_addr = dvmaddr; 810 map->dm_segs[j].ds_len = left; 811 812 sgstart = dvmaddr; 813 sgend = sgstart + left - 1; 814 } 815 816 } 817 818 /* Check for boundary issues */ 819 while ((sgstart & ~(boundary - 1)) != 820 (sgend & ~(boundary - 1))) { 821 /* Need a new segment. */ 822 map->dm_segs[j].ds_len = 823 sgstart & (boundary - 1); 824 DPRINTF(IDB_INFO, ("iommu_dvmamap_load_raw: " 825 "seg %d start %lx size %lx\n", j, 826 map->dm_segs[j].ds_addr, 827 map->dm_segs[j].ds_len)); 828 if (++j > map->_dm_segcnt) { 829 fail: 830 iommu_dvmamap_unload(t, is, map); 831 return (E2BIG); 832 } 833 sgstart = roundup(sgstart, boundary); 834 map->dm_segs[j].ds_addr = sgstart; 835 map->dm_segs[j].ds_len = sgend - sgstart + 1; 836 } 837 838 if (sgsize == 0) 839 panic("iommu_dmamap_load_raw: size botch"); 840 841 DPRINTF(IDB_BUSDMA, 842 ("iommu_dvmamap_load_raw: map %p loading va %lx at pa %lx\n", 843 map, (long)dvmaddr, (long)(pa))); 844 /* Enter it if we haven't before. */ 845 if (prev_va != trunc_page(dvmaddr)) 846 iommu_enter(is, prev_va = trunc_page(dvmaddr), 847 trunc_page(pa), flags); 848 incr = min(pagesz, left); 849 dvmaddr += incr; 850 pa += incr; 851 left -= incr; 852 size -= incr; 853 } 854 DPRINTF(IDB_INFO, ("iommu_dvmamap_load_raw: " 855 "seg %d start %lx size %lx\n", j, 856 map->dm_segs[j].ds_addr, map->dm_segs[j].ds_len)); 857 map->dm_nsegs = j+1; 858 return (0); 859 } 860 /* 861 * This was allocated with bus_dmamem_alloc. 862 * The pages are on an `mlist'. 863 */ 864 map->dm_mapsize = size; 865 i = 0; 866 sgstart = dvmaddr; 867 sgend = sgstart + size - 1; 868 map->dm_segs[i].ds_addr = sgstart; 869 while ((sgstart & ~(boundary - 1)) != (sgend & ~(boundary - 1))) { 870 /* Oops. We crossed a boundary. Split the xfer. */ 871 map->dm_segs[i].ds_len = sgstart & (boundary - 1); 872 DPRINTF(IDB_INFO, ("iommu_dvmamap_load_raw: " 873 "seg %d start %lx size %lx\n", i, 874 map->dm_segs[i].ds_addr, 875 map->dm_segs[i].ds_len)); 876 if (++i > map->_dm_segcnt) { 877 /* Too many segments. Fail the operation. */ 878 s = splhigh(); 879 /* How can this fail? And if it does what can we do? */ 880 err = extent_free(is->is_dvmamap, 881 dvmaddr, sgsize, EX_NOWAIT); 882 splx(s); 883 map->_dm_dvmastart = 0; 884 map->_dm_dvmasize = 0; 885 return (E2BIG); 886 } 887 sgstart = roundup(sgstart, boundary); 888 map->dm_segs[i].ds_addr = sgstart; 889 } 890 DPRINTF(IDB_INFO, ("iommu_dvmamap_load_raw: " 891 "seg %d start %lx size %lx\n", i, 892 map->dm_segs[i].ds_addr, map->dm_segs[i].ds_len)); 893 map->dm_segs[i].ds_len = sgend - sgstart + 1; 894 895 for (m = TAILQ_FIRST(mlist); m != NULL; m = TAILQ_NEXT(m,pageq)) { 896 if (sgsize == 0) 897 panic("iommu_dmamap_load_raw: size botch"); 898 pa = VM_PAGE_TO_PHYS(m); 899 900 DPRINTF(IDB_BUSDMA, 901 ("iommu_dvmamap_load_raw: map %p loading va %lx at pa %lx\n", 902 map, (long)dvmaddr, (long)(pa))); 903 iommu_enter(is, dvmaddr, pa, flags); 904 905 dvmaddr += pagesz; 906 sgsize -= pagesz; 907 } 908 map->dm_mapsize = size; 909 map->dm_nsegs = i+1; 910 return (0); 911 } 912 913 void 914 iommu_dvmamap_sync(t, is, map, offset, len, ops) 915 bus_dma_tag_t t; 916 struct iommu_state *is; 917 bus_dmamap_t map; 918 bus_addr_t offset; 919 bus_size_t len; 920 int ops; 921 { 922 vaddr_t va = map->dm_segs[0].ds_addr + offset; 923 924 /* 925 * We only support one DMA segment; supporting more makes this code 926 * too unweildy. 927 */ 928 929 if (ops & BUS_DMASYNC_PREREAD) { 930 DPRINTF(IDB_SYNC, 931 ("iommu_dvmamap_sync: syncing va %p len %lu " 932 "BUS_DMASYNC_PREREAD\n", (void *)(u_long)va, (u_long)len)); 933 934 /* Nothing to do */; 935 } 936 if (ops & BUS_DMASYNC_POSTREAD) { 937 DPRINTF(IDB_SYNC, 938 ("iommu_dvmamap_sync: syncing va %p len %lu " 939 "BUS_DMASYNC_POSTREAD\n", (void *)(u_long)va, (u_long)len)); 940 /* if we have a streaming buffer, flush it here first */ 941 if (is->is_sb) 942 while (len > 0) { 943 DPRINTF(IDB_BUSDMA, 944 ("iommu_dvmamap_sync: flushing va %p, %lu " 945 "bytes left\n", (void *)(u_long)va, (u_long)len)); 946 iommu_strbuf_flush(is, va); 947 if (len <= NBPG) { 948 iommu_strbuf_flush_done(is); 949 len = 0; 950 } else 951 len -= NBPG; 952 va += NBPG; 953 } 954 } 955 if (ops & BUS_DMASYNC_PREWRITE) { 956 DPRINTF(IDB_SYNC, 957 ("iommu_dvmamap_sync: syncing va %p len %lu " 958 "BUS_DMASYNC_PREWRITE\n", (void *)(u_long)va, (u_long)len)); 959 /* if we have a streaming buffer, flush it here first */ 960 if (is->is_sb) 961 while (len > 0) { 962 DPRINTF(IDB_BUSDMA, 963 ("iommu_dvmamap_sync: flushing va %p, %lu " 964 "bytes left\n", (void *)(u_long)va, (u_long)len)); 965 iommu_strbuf_flush(is, va); 966 if (len <= NBPG) { 967 iommu_strbuf_flush_done(is); 968 len = 0; 969 } else 970 len -= NBPG; 971 va += NBPG; 972 } 973 } 974 if (ops & BUS_DMASYNC_POSTWRITE) { 975 DPRINTF(IDB_SYNC, 976 ("iommu_dvmamap_sync: syncing va %p len %lu " 977 "BUS_DMASYNC_POSTWRITE\n", (void *)(u_long)va, (u_long)len)); 978 /* Nothing to do */; 979 } 980 } 981 982 int 983 iommu_dvmamem_alloc(t, is, size, alignment, boundary, segs, nsegs, rsegs, flags) 984 bus_dma_tag_t t; 985 struct iommu_state *is; 986 bus_size_t size, alignment, boundary; 987 bus_dma_segment_t *segs; 988 int nsegs; 989 int *rsegs; 990 int flags; 991 { 992 993 DPRINTF(IDB_BUSDMA, ("iommu_dvmamem_alloc: sz %llx align %llx bound %llx " 994 "segp %p flags %d\n", (unsigned long long)size, 995 (unsigned long long)alignment, (unsigned long long)boundary, 996 segs, flags)); 997 return (bus_dmamem_alloc(t->_parent, size, alignment, boundary, 998 segs, nsegs, rsegs, flags|BUS_DMA_DVMA)); 999 } 1000 1001 void 1002 iommu_dvmamem_free(t, is, segs, nsegs) 1003 bus_dma_tag_t t; 1004 struct iommu_state *is; 1005 bus_dma_segment_t *segs; 1006 int nsegs; 1007 { 1008 1009 DPRINTF(IDB_BUSDMA, ("iommu_dvmamem_free: segp %p nsegs %d\n", 1010 segs, nsegs)); 1011 bus_dmamem_free(t->_parent, segs, nsegs); 1012 } 1013 1014 /* 1015 * Map the DVMA mappings into the kernel pmap. 1016 * Check the flags to see whether we're streaming or coherent. 1017 */ 1018 int 1019 iommu_dvmamem_map(t, is, segs, nsegs, size, kvap, flags) 1020 bus_dma_tag_t t; 1021 struct iommu_state *is; 1022 bus_dma_segment_t *segs; 1023 int nsegs; 1024 size_t size; 1025 caddr_t *kvap; 1026 int flags; 1027 { 1028 struct vm_page *m; 1029 vaddr_t va; 1030 bus_addr_t addr; 1031 struct pglist *mlist; 1032 int cbit; 1033 1034 DPRINTF(IDB_BUSDMA, ("iommu_dvmamem_map: segp %p nsegs %d size %lx\n", 1035 segs, nsegs, size)); 1036 1037 /* 1038 * Allocate some space in the kernel map, and then map these pages 1039 * into this space. 1040 */ 1041 size = round_page(size); 1042 va = uvm_km_valloc(kernel_map, size); 1043 if (va == 0) 1044 return (ENOMEM); 1045 1046 *kvap = (caddr_t)va; 1047 1048 /* 1049 * digest flags: 1050 */ 1051 cbit = 0; 1052 if (flags & BUS_DMA_COHERENT) /* Disable vcache */ 1053 cbit |= PMAP_NVC; 1054 if (flags & BUS_DMA_NOCACHE) /* sideffects */ 1055 cbit |= PMAP_NC; 1056 1057 /* 1058 * Now take this and map it into the CPU. 1059 */ 1060 mlist = segs[0]._ds_mlist; 1061 for (m = mlist->tqh_first; m != NULL; m = m->pageq.tqe_next) { 1062 #ifdef DIAGNOSTIC 1063 if (size == 0) 1064 panic("iommu_dvmamem_map: size botch"); 1065 #endif 1066 addr = VM_PAGE_TO_PHYS(m); 1067 DPRINTF(IDB_BUSDMA, ("iommu_dvmamem_map: " 1068 "mapping va %lx at %llx\n", va, (unsigned long long)addr | cbit)); 1069 pmap_enter(pmap_kernel(), va, addr | cbit, 1070 VM_PROT_READ | VM_PROT_WRITE, 1071 VM_PROT_READ | VM_PROT_WRITE | PMAP_WIRED); 1072 va += PAGE_SIZE; 1073 size -= PAGE_SIZE; 1074 } 1075 pmap_update(pmap_kernel()); 1076 1077 return (0); 1078 } 1079 1080 /* 1081 * Unmap DVMA mappings from kernel 1082 */ 1083 void 1084 iommu_dvmamem_unmap(t, is, kva, size) 1085 bus_dma_tag_t t; 1086 struct iommu_state *is; 1087 caddr_t kva; 1088 size_t size; 1089 { 1090 1091 DPRINTF(IDB_BUSDMA, ("iommu_dvmamem_unmap: kvm %p size %lx\n", 1092 kva, size)); 1093 1094 #ifdef DIAGNOSTIC 1095 if ((u_long)kva & PGOFSET) 1096 panic("iommu_dvmamem_unmap"); 1097 #endif 1098 1099 size = round_page(size); 1100 pmap_remove(pmap_kernel(), (vaddr_t)kva, size); 1101 pmap_update(pmap_kernel()); 1102 #if 0 1103 /* 1104 * XXX ? is this necessary? i think so and i think other 1105 * implementations are missing it. 1106 */ 1107 uvm_km_free(kernel_map, (vaddr_t)kva, size); 1108 #endif 1109 } 1110
Indexes created Tue Sep 23 07:09:52 GMT 2025