iommu.c revision 1.40
1 /* $NetBSD: iommu.c,v 1.40 2001/09/21 03:04:09 eeh Exp $ */ 2 3 /* 4 * Copyright (c) 1999, 2000 Matthew R. Green 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 3. The name of the author may not be used to endorse or promote products 16 * derived from this software without specific prior written permission. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 19 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 20 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 21 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 22 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 23 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 24 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 25 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 26 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 28 * SUCH DAMAGE. 29 */ 30 31 /*- 32 * Copyright (c) 1998 The NetBSD Foundation, Inc. 33 * All rights reserved. 34 * 35 * This code is derived from software contributed to The NetBSD Foundation 36 * by Paul Kranenburg. 37 * 38 * Redistribution and use in source and binary forms, with or without 39 * modification, are permitted provided that the following conditions 40 * are met: 41 * 1. Redistributions of source code must retain the above copyright 42 * notice, this list of conditions and the following disclaimer. 43 * 2. Redistributions in binary form must reproduce the above copyright 44 * notice, this list of conditions and the following disclaimer in the 45 * documentation and/or other materials provided with the distribution. 46 * 3. All advertising materials mentioning features or use of this software 47 * must display the following acknowledgement: 48 * This product includes software developed by the NetBSD 49 * Foundation, Inc. and its contributors. 50 * 4. Neither the name of The NetBSD Foundation nor the names of its 51 * contributors may be used to endorse or promote products derived 52 * from this software without specific prior written permission. 53 * 54 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 55 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 56 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 57 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 58 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 59 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 60 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 61 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 62 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 63 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 64 * POSSIBILITY OF SUCH DAMAGE. 65 */ 66 67 /* 68 * Copyright (c) 1992, 1993 69 * The Regents of the University of California. All rights reserved. 70 * 71 * This software was developed by the Computer Systems Engineering group 72 * at Lawrence Berkeley Laboratory under DARPA contract BG 91-66 and 73 * contributed to Berkeley. 74 * 75 * All advertising materials mentioning features or use of this software 76 * must display the following acknowledgement: 77 * This product includes software developed by the University of 78 * California, Lawrence Berkeley Laboratory. 79 * 80 * Redistribution and use in source and binary forms, with or without 81 * modification, are permitted provided that the following conditions 82 * are met: 83 * 1. Redistributions of source code must retain the above copyright 84 * notice, this list of conditions and the following disclaimer. 85 * 2. Redistributions in binary form must reproduce the above copyright 86 * notice, this list of conditions and the following disclaimer in the 87 * documentation and/or other materials provided with the distribution. 88 * 3. All advertising materials mentioning features or use of this software 89 * must display the following acknowledgement: 90 * This product includes software developed by the University of 91 * California, Berkeley and its contributors. 92 * 4. Neither the name of the University nor the names of its contributors 93 * may be used to endorse or promote products derived from this software 94 * without specific prior written permission. 95 * 96 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 97 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 98 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 99 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 100 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 101 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 102 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 103 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 104 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 105 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 106 * SUCH DAMAGE. 107 * 108 * from: NetBSD: sbus.c,v 1.13 1999/05/23 07:24:02 mrg Exp 109 * from: @(#)sbus.c 8.1 (Berkeley) 6/11/93 110 */ 111 112 /* 113 * UltraSPARC IOMMU support; used by both the sbus and pci code. 114 */ 115 #include "opt_ddb.h" 116 117 #include <sys/param.h> 118 #include <sys/extent.h> 119 #include <sys/malloc.h> 120 #include <sys/systm.h> 121 #include <sys/device.h> 122 123 #include <uvm/uvm_extern.h> 124 125 #include <machine/bus.h> 126 #include <sparc64/sparc64/cache.h> 127 #include <sparc64/dev/iommureg.h> 128 #include <sparc64/dev/iommuvar.h> 129 130 #include <machine/autoconf.h> 131 #include <machine/cpu.h> 132 133 #ifdef DEBUG 134 #define IDB_BUSDMA 0x1 135 #define IDB_IOMMU 0x2 136 #define IDB_INFO 0x4 137 #define IDB_SYNC 0x8 138 int iommudebug = 0x0; 139 #define DPRINTF(l, s) do { if (iommudebug & l) printf s; } while (0) 140 #else 141 #define DPRINTF(l, s) 142 #endif 143 144 #define iommu_strbuf_flush(i,v) bus_space_write_8((i)->is_bustag, \ 145 (bus_space_handle_t)(u_long)&(i)->is_sb->strbuf_pgflush, 0, (v)) 146 static int iommu_strbuf_flush_done __P((struct iommu_state *)); 147 148 /* 149 * initialise the UltraSPARC IOMMU (SBUS or PCI): 150 * - allocate and setup the iotsb. 151 * - enable the IOMMU 152 * - initialise the streaming buffers (if they exist) 153 * - create a private DVMA map. 154 */ 155 void 156 iommu_init(name, is, tsbsize, iovabase) 157 char *name; 158 struct iommu_state *is; 159 int tsbsize; 160 u_int32_t iovabase; 161 { 162 psize_t size; 163 vaddr_t va; 164 paddr_t pa; 165 struct vm_page *m; 166 struct pglist mlist; 167 168 /* 169 * Setup the iommu. 170 * 171 * The sun4u iommu is part of the SBUS or PCI controller so we 172 * will deal with it here.. 173 * 174 * The IOMMU address space always ends at 0xffffe000, but the starting 175 * address depends on the size of the map. The map size is 1024 * 2 ^ 176 * is->is_tsbsize entries, where each entry is 8 bytes. The start of 177 * the map can be calculated by (0xffffe000 << (8 + is->is_tsbsize)). 178 */ 179 is->is_cr = (tsbsize << 16) | IOMMUCR_EN; 180 is->is_tsbsize = tsbsize; 181 is->is_dvmabase = iovabase; 182 if (iovabase == -1) is->is_dvmabase = IOTSB_VSTART(is->is_tsbsize); 183 184 /* 185 * Allocate memory for I/O pagetables. They need to be physically 186 * contiguous. 187 */ 188 189 size = NBPG<<(is->is_tsbsize); 190 TAILQ_INIT(&mlist); 191 if (uvm_pglistalloc((psize_t)size, (paddr_t)0, (paddr_t)-1, 192 (paddr_t)NBPG, (paddr_t)0, &mlist, 1, 0) != 0) 193 panic("iommu_init: no memory"); 194 195 va = uvm_km_valloc(kernel_map, size); 196 if (va == 0) 197 panic("iommu_init: no memory"); 198 is->is_tsb = (int64_t *)va; 199 200 m = TAILQ_FIRST(&mlist); 201 is->is_ptsb = VM_PAGE_TO_PHYS(m); 202 203 /* Map the pages */ 204 for (; m != NULL; m = TAILQ_NEXT(m,pageq)) { 205 pa = VM_PAGE_TO_PHYS(m); 206 pmap_enter(pmap_kernel(), va, pa | PMAP_NVC, 207 VM_PROT_READ|VM_PROT_WRITE, 208 VM_PROT_READ|VM_PROT_WRITE|PMAP_WIRED); 209 va += NBPG; 210 } 211 pmap_update(pmap_kernel()); 212 bzero(is->is_tsb, size); 213 214 #ifdef DEBUG 215 if (iommudebug & IDB_INFO) 216 { 217 /* Probe the iommu */ 218 struct iommureg *regs = is->is_iommu; 219 220 printf("iommu regs at: cr=%lx tsb=%lx flush=%lx\n", 221 (u_long)®s->iommu_cr, 222 (u_long)®s->iommu_tsb, 223 (u_long)®s->iommu_flush); 224 printf("iommu cr=%llx tsb=%llx\n", (unsigned long long)regs->iommu_cr, (unsigned long long)regs->iommu_tsb); 225 printf("TSB base %p phys %llx\n", (void *)is->is_tsb, (unsigned long long)is->is_ptsb); 226 delay(1000000); /* 1 s */ 227 } 228 #endif 229 230 /* 231 * Initialize streaming buffer, if it is there. 232 */ 233 if (is->is_sb) 234 (void)pmap_extract(pmap_kernel(), (vaddr_t)&is->is_flush, 235 (paddr_t *)&is->is_flushpa); 236 237 /* 238 * now actually start up the IOMMU 239 */ 240 iommu_reset(is); 241 242 /* 243 * Now all the hardware's working we need to allocate a dvma map. 244 */ 245 printf("DVMA map: %x to %x\n", 246 (unsigned int)is->is_dvmabase, 247 (unsigned int)(is->is_dvmabase+(size<<10))); 248 is->is_dvmamap = extent_create(name, 249 is->is_dvmabase, (u_long)IOTSB_VEND, 250 M_DEVBUF, 0, 0, EX_NOWAIT); 251 } 252 253 /* 254 * Streaming buffers don't exist on the UltraSPARC IIi; we should have 255 * detected that already and disabled them. If not, we will notice that 256 * they aren't there when the STRBUF_EN bit does not remain. 257 */ 258 void 259 iommu_reset(is) 260 struct iommu_state *is; 261 { 262 263 /* Need to do 64-bit stores */ 264 bus_space_write_8(is->is_bustag, 265 (bus_space_handle_t)(u_long)&is->is_iommu->iommu_tsb, 266 0, is->is_ptsb); 267 /* Enable IOMMU in diagnostic mode */ 268 bus_space_write_8(is->is_bustag, 269 (bus_space_handle_t)(u_long)&is->is_iommu->iommu_cr, 0, 270 is->is_cr|IOMMUCR_DE); 271 272 273 if (!is->is_sb) 274 return; 275 276 /* Enable diagnostics mode? */ 277 bus_space_write_8(is->is_bustag, 278 (bus_space_handle_t)(u_long)&is->is_sb->strbuf_ctl, 279 0, STRBUF_EN); 280 281 /* No streaming buffers? Disable them */ 282 if (bus_space_read_8(is->is_bustag, 283 (bus_space_handle_t)(u_long)&is->is_sb->strbuf_ctl, 284 0) == 0) 285 is->is_sb = 0; 286 } 287 288 /* 289 * Here are the iommu control routines. 290 */ 291 void 292 iommu_enter(is, va, pa, flags) 293 struct iommu_state *is; 294 vaddr_t va; 295 int64_t pa; 296 int flags; 297 { 298 int64_t tte; 299 300 #ifdef DIAGNOSTIC 301 if (va < is->is_dvmabase) 302 panic("iommu_enter: va %#lx not in DVMA space", va); 303 #endif 304 305 tte = MAKEIOTTE(pa, !(flags&BUS_DMA_NOWRITE), !(flags&BUS_DMA_NOCACHE), 306 (flags&BUS_DMA_STREAMING)); 307 308 /* Is the streamcache flush really needed? */ 309 if (is->is_sb) { 310 iommu_strbuf_flush(is, va); 311 iommu_strbuf_flush_done(is); 312 } 313 DPRINTF(IDB_IOMMU, ("Clearing TSB slot %d for va %p\n", 314 (int)IOTSBSLOT(va,is->is_tsbsize), (void *)(u_long)va)); 315 is->is_tsb[IOTSBSLOT(va,is->is_tsbsize)] = tte; 316 bus_space_write_8(is->is_bustag, (bus_space_handle_t)(u_long) 317 &is->is_iommu->iommu_flush, 0, va); 318 DPRINTF(IDB_IOMMU, ("iommu_enter: va %lx pa %lx TSB[%lx]@%p=%lx\n", 319 va, (long)pa, (u_long)IOTSBSLOT(va,is->is_tsbsize), 320 (void *)(u_long)&is->is_tsb[IOTSBSLOT(va,is->is_tsbsize)], 321 (u_long)tte)); 322 } 323 324 325 /* 326 * Find the value of a DVMA address (debug routine). 327 */ 328 paddr_t 329 iommu_extract(is, dva) 330 struct iommu_state *is; 331 vaddr_t dva; 332 { 333 int64_t tte = 0; 334 335 if (dva >= is->is_dvmabase) 336 tte = is->is_tsb[IOTSBSLOT(dva,is->is_tsbsize)]; 337 338 if ((tte&IOTTE_V) == 0) 339 return ((paddr_t)-1L); 340 return (tte&IOTTE_PAMASK); 341 } 342 343 /* 344 * iommu_remove: removes mappings created by iommu_enter 345 * 346 * Only demap from IOMMU if flag is set. 347 * 348 * XXX: this function needs better internal error checking. 349 */ 350 void 351 iommu_remove(is, va, len) 352 struct iommu_state *is; 353 vaddr_t va; 354 size_t len; 355 { 356 357 #ifdef DIAGNOSTIC 358 if (va < is->is_dvmabase) 359 panic("iommu_remove: va 0x%lx not in DVMA space", (u_long)va); 360 if ((long)(va + len) < (long)va) 361 panic("iommu_remove: va 0x%lx + len 0x%lx wraps", 362 (long) va, (long) len); 363 if (len & ~0xfffffff) 364 panic("iommu_remove: rediculous len 0x%lx", (u_long)len); 365 #endif 366 367 va = trunc_page(va); 368 DPRINTF(IDB_IOMMU, ("iommu_remove: va %lx TSB[%lx]@%p\n", 369 va, (u_long)IOTSBSLOT(va,is->is_tsbsize), 370 &is->is_tsb[IOTSBSLOT(va,is->is_tsbsize)])); 371 while (len > 0) { 372 DPRINTF(IDB_IOMMU, ("iommu_remove: clearing TSB slot %d for va %p size %lx\n", 373 (int)IOTSBSLOT(va,is->is_tsbsize), (void *)(u_long)va, (u_long)len)); 374 if (is->is_sb) { 375 DPRINTF(IDB_IOMMU, ("iommu_remove: flushing va %p TSB[%lx]@%p=%lx, %lu bytes left\n", 376 (void *)(u_long)va, (long)IOTSBSLOT(va,is->is_tsbsize), 377 (void *)(u_long)&is->is_tsb[IOTSBSLOT(va,is->is_tsbsize)], 378 (long)(is->is_tsb[IOTSBSLOT(va,is->is_tsbsize)]), 379 (u_long)len)); 380 iommu_strbuf_flush(is, va); 381 if (len <= NBPG) 382 iommu_strbuf_flush_done(is); 383 DPRINTF(IDB_IOMMU, ("iommu_remove: flushed va %p TSB[%lx]@%p=%lx, %lu bytes left\n", 384 (void *)(u_long)va, (long)IOTSBSLOT(va,is->is_tsbsize), 385 (void *)(u_long)&is->is_tsb[IOTSBSLOT(va,is->is_tsbsize)], 386 (long)(is->is_tsb[IOTSBSLOT(va,is->is_tsbsize)]), 387 (u_long)len)); 388 } 389 390 if (len <= NBPG) 391 len = 0; 392 else 393 len -= NBPG; 394 395 is->is_tsb[IOTSBSLOT(va,is->is_tsbsize)] = 0; 396 bus_space_write_8(is->is_bustag, (bus_space_handle_t)(u_long) 397 &is->is_iommu->iommu_flush, 0, va); 398 va += NBPG; 399 } 400 } 401 402 static int 403 iommu_strbuf_flush_done(is) 404 struct iommu_state *is; 405 { 406 struct timeval cur, flushtimeout; 407 408 #define BUMPTIME(t, usec) { \ 409 register volatile struct timeval *tp = (t); \ 410 register long us; \ 411 \ 412 tp->tv_usec = us = tp->tv_usec + (usec); \ 413 if (us >= 1000000) { \ 414 tp->tv_usec = us - 1000000; \ 415 tp->tv_sec++; \ 416 } \ 417 } 418 419 if (!is->is_sb) 420 return (0); 421 422 /* 423 * Streaming buffer flushes: 424 * 425 * 1 Tell strbuf to flush by storing va to strbuf_pgflush. If 426 * we're not on a cache line boundary (64-bits): 427 * 2 Store 0 in flag 428 * 3 Store pointer to flag in flushsync 429 * 4 wait till flushsync becomes 0x1 430 * 431 * If it takes more than .5 sec, something 432 * went wrong. 433 */ 434 435 is->is_flush = 0; 436 membar_sync(); /* #StoreStore is prolly enuf. */ 437 bus_space_write_8(is->is_bustag, (bus_space_handle_t)(u_long) 438 &is->is_sb->strbuf_flushsync, 0, is->is_flushpa); 439 membar_sync(); /* Prolly not needed at all. */ 440 441 microtime(&flushtimeout); 442 cur = flushtimeout; 443 BUMPTIME(&flushtimeout, 500000); /* 1/2 sec */ 444 445 DPRINTF(IDB_IOMMU, ("iommu_strbuf_flush_done: flush = %lx at va = %lx pa = %lx now=%lx:%lx until = %lx:%lx\n", 446 (long)is->is_flush, (long)&is->is_flush, 447 (long)is->is_flushpa, cur.tv_sec, cur.tv_usec, 448 flushtimeout.tv_sec, flushtimeout.tv_usec)); 449 /* Bypass non-coherent D$ */ 450 while (!ldxa(is->is_flushpa, ASI_PHYS_CACHED) && 451 ((cur.tv_sec <= flushtimeout.tv_sec) && 452 (cur.tv_usec <= flushtimeout.tv_usec))) 453 microtime(&cur); 454 455 #ifdef DIAGNOSTIC 456 if (!ldxa(is->is_flushpa, ASI_PHYS_CACHED)) { 457 printf("iommu_strbuf_flush_done: flush timeout %p at %p\n", 458 (void *)(u_long)is->is_flush, 459 (void *)(u_long)is->is_flushpa); /* panic? */ 460 #ifdef DDB 461 Debugger(); 462 #endif 463 } 464 #endif 465 DPRINTF(IDB_IOMMU, ("iommu_strbuf_flush_done: flushed\n")); 466 return (is->is_flush); 467 } 468 469 /* 470 * IOMMU DVMA operations, common to SBUS and PCI. 471 */ 472 int 473 iommu_dvmamap_load(t, is, map, buf, buflen, p, flags) 474 bus_dma_tag_t t; 475 struct iommu_state *is; 476 bus_dmamap_t map; 477 void *buf; 478 bus_size_t buflen; 479 struct proc *p; 480 int flags; 481 { 482 int s; 483 int err; 484 bus_size_t sgsize; 485 paddr_t curaddr; 486 u_long dvmaddr, sgstart, sgend; 487 bus_size_t align, boundary; 488 vaddr_t vaddr = (vaddr_t)buf; 489 int seg; 490 pmap_t pmap; 491 492 if (map->dm_nsegs) { 493 /* Already in use?? */ 494 #ifdef DIAGNOSTIC 495 printf("iommu_dvmamap_load: map still in use\n"); 496 #endif 497 bus_dmamap_unload(t, map); 498 } 499 /* 500 * Make sure that on error condition we return "no valid mappings". 501 */ 502 map->dm_nsegs = 0; 503 504 if (buflen > map->_dm_size) { 505 DPRINTF(IDB_BUSDMA, 506 ("iommu_dvmamap_load(): error %d > %d -- " 507 "map size exceeded!\n", (int)buflen, (int)map->_dm_size)); 508 return (EINVAL); 509 } 510 511 sgsize = round_page(buflen + ((int)vaddr & PGOFSET)); 512 513 /* 514 * A boundary presented to bus_dmamem_alloc() takes precedence 515 * over boundary in the map. 516 */ 517 if ((boundary = (map->dm_segs[0]._ds_boundary)) == 0) 518 boundary = map->_dm_boundary; 519 align = max(map->dm_segs[0]._ds_align, NBPG); 520 s = splhigh(); 521 /* 522 * If our segment size is larger than the boundary we need to 523 * split the transfer up int little pieces ourselves. 524 */ 525 err = extent_alloc(is->is_dvmamap, sgsize, align, 526 (sgsize > boundary) ? 0 : boundary, 527 EX_NOWAIT|EX_BOUNDZERO, (u_long *)&dvmaddr); 528 splx(s); 529 530 #ifdef DEBUG 531 if (err || (dvmaddr == (bus_addr_t)-1)) 532 { 533 printf("iommu_dvmamap_load(): extent_alloc(%d, %x) failed!\n", 534 (int)sgsize, flags); 535 #ifdef DDB 536 Debugger(); 537 #endif 538 } 539 #endif 540 if (err != 0) 541 return (err); 542 543 if (dvmaddr == (bus_addr_t)-1) 544 return (ENOMEM); 545 546 /* Set the active DVMA map */ 547 map->_dm_dvmastart = dvmaddr; 548 map->_dm_dvmasize = sgsize; 549 550 /* 551 * Now split the DVMA range into segments, not crossing 552 * the boundary. 553 */ 554 seg = 0; 555 sgstart = dvmaddr + (vaddr & PGOFSET); 556 sgend = sgstart + buflen - 1; 557 map->dm_segs[seg].ds_addr = sgstart; 558 DPRINTF(IDB_INFO, ("iommu_dvmamap_load: boundary %lx boundary-1 %lx " 559 "~(boundary-1) %lx\n", boundary, (boundary-1), ~(boundary-1))); 560 while ((sgstart & ~(boundary - 1)) != (sgend & ~(boundary - 1))) { 561 /* Oops. We crossed a boundary. Split the xfer. */ 562 DPRINTF(IDB_INFO, ("iommu_dvmamap_load: " 563 "seg %d start %lx size %lx\n", seg, 564 map->dm_segs[seg].ds_addr, map->dm_segs[seg].ds_len)); 565 map->dm_segs[seg].ds_len = sgstart & (boundary - 1); 566 if (++seg > map->_dm_segcnt) { 567 /* Too many segments. Fail the operation. */ 568 DPRINTF(IDB_INFO, ("iommu_dvmamap_load: " 569 "too many segments %d\n", seg)); 570 s = splhigh(); 571 /* How can this fail? And if it does what can we do? */ 572 err = extent_free(is->is_dvmamap, 573 dvmaddr, sgsize, EX_NOWAIT); 574 splx(s); 575 map->_dm_dvmastart = 0; 576 map->_dm_dvmasize = 0; 577 return (E2BIG); 578 } 579 sgstart = roundup(sgstart, boundary); 580 map->dm_segs[seg].ds_addr = sgstart; 581 } 582 map->dm_segs[seg].ds_len = sgend - sgstart + 1; 583 DPRINTF(IDB_INFO, ("iommu_dvmamap_load: " 584 "seg %d start %lx size %lx\n", seg, 585 map->dm_segs[seg].ds_addr, map->dm_segs[seg].ds_len)); 586 map->dm_nsegs = seg+1; 587 map->dm_mapsize = buflen; 588 589 if (p != NULL) 590 pmap = p->p_vmspace->vm_map.pmap; 591 else 592 pmap = pmap_kernel(); 593 594 for (; buflen > 0; ) { 595 /* 596 * Get the physical address for this page. 597 */ 598 if (pmap_extract(pmap, (vaddr_t)vaddr, &curaddr) == FALSE) { 599 bus_dmamap_unload(t, map); 600 return (-1); 601 } 602 603 /* 604 * Compute the segment size, and adjust counts. 605 */ 606 sgsize = NBPG - ((u_long)vaddr & PGOFSET); 607 if (buflen < sgsize) 608 sgsize = buflen; 609 610 DPRINTF(IDB_BUSDMA, 611 ("iommu_dvmamap_load: map %p loading va %p " 612 "dva %lx at pa %lx\n", 613 map, (void *)vaddr, (long)dvmaddr, 614 (long)(curaddr&~(NBPG-1)))); 615 iommu_enter(is, trunc_page(dvmaddr), trunc_page(curaddr), 616 flags); 617 618 dvmaddr += PAGE_SIZE; 619 vaddr += sgsize; 620 buflen -= sgsize; 621 } 622 return (0); 623 } 624 625 626 void 627 iommu_dvmamap_unload(t, is, map) 628 bus_dma_tag_t t; 629 struct iommu_state *is; 630 bus_dmamap_t map; 631 { 632 int error, s; 633 bus_size_t sgsize; 634 635 /* Flush the iommu */ 636 #ifdef DEBUG 637 if (!map->_dm_dvmastart) { 638 printf("iommu_dvmamap_unload: No dvmastart is zero\n"); 639 #ifdef DDB 640 Debugger(); 641 #endif 642 } 643 #endif 644 iommu_remove(is, map->_dm_dvmastart, map->_dm_dvmasize); 645 646 /* Flush the caches */ 647 bus_dmamap_unload(t->_parent, map); 648 649 /* Mark the mappings as invalid. */ 650 map->dm_mapsize = 0; 651 map->dm_nsegs = 0; 652 653 s = splhigh(); 654 error = extent_free(is->is_dvmamap, map->_dm_dvmastart, 655 map->_dm_dvmasize, EX_NOWAIT); 656 splx(s); 657 if (error != 0) 658 printf("warning: %qd of DVMA space lost\n", (long long)sgsize); 659 660 /* Clear the map */ 661 map->_dm_dvmastart = 0; 662 map->_dm_dvmasize = 0; 663 } 664 665 666 int 667 iommu_dvmamap_load_raw(t, is, map, segs, nsegs, flags, size) 668 bus_dma_tag_t t; 669 struct iommu_state *is; 670 bus_dmamap_t map; 671 bus_dma_segment_t *segs; 672 int nsegs; 673 int flags; 674 bus_size_t size; 675 { 676 struct vm_page *m; 677 int i, j, s; 678 int left; 679 int err; 680 bus_size_t sgsize; 681 paddr_t pa; 682 bus_size_t boundary, align; 683 u_long dvmaddr, sgstart, sgend; 684 struct pglist *mlist; 685 int pagesz = PAGE_SIZE; 686 687 if (map->dm_nsegs) { 688 /* Already in use?? */ 689 #ifdef DIAGNOSTIC 690 printf("iommu_dvmamap_load_raw: map still in use\n"); 691 #endif 692 bus_dmamap_unload(t, map); 693 } 694 695 /* 696 * A boundary presented to bus_dmamem_alloc() takes precedence 697 * over boundary in the map. 698 */ 699 if ((boundary = segs[0]._ds_boundary) == 0) 700 boundary = map->_dm_boundary; 701 702 align = max(segs[0]._ds_align, NBPG); 703 704 /* 705 * Make sure that on error condition we return "no valid mappings". 706 */ 707 map->dm_nsegs = 0; 708 /* Count up the total number of pages we need */ 709 pa = segs[0].ds_addr; 710 sgsize = 0; 711 left = size; 712 for (i=0; left && i<nsegs; i++) { 713 if (round_page(pa) != round_page(segs[i].ds_addr)) 714 sgsize = round_page(sgsize); 715 sgsize += min(left, segs[i].ds_len); 716 left -= segs[i].ds_len; 717 pa = segs[i].ds_addr + segs[i].ds_len; 718 } 719 sgsize = round_page(sgsize); 720 721 s = splhigh(); 722 /* 723 * If our segment size is larger than the boundary we need to 724 * split the transfer up int little pieces ourselves. 725 */ 726 err = extent_alloc(is->is_dvmamap, sgsize, align, 727 (sgsize > boundary) ? 0 : boundary, 728 ((flags & BUS_DMA_NOWAIT) == 0 ? EX_WAITOK : EX_NOWAIT) | 729 EX_BOUNDZERO, (u_long *)&dvmaddr); 730 splx(s); 731 732 if (err != 0) 733 return (err); 734 735 #ifdef DEBUG 736 if (dvmaddr == (bus_addr_t)-1) 737 { 738 printf("iommu_dvmamap_load_raw(): extent_alloc(%d, %x) failed!\n", 739 (int)sgsize, flags); 740 Debugger(); 741 } 742 #endif 743 if (dvmaddr == (bus_addr_t)-1) 744 return (ENOMEM); 745 746 /* Set the active DVMA map */ 747 map->_dm_dvmastart = dvmaddr; 748 map->_dm_dvmasize = sgsize; 749 750 if ((mlist = segs[0]._ds_mlist) == NULL) { 751 u_long prev_va = NULL; 752 /* 753 * This segs is made up of individual physical pages, 754 * probably by _bus_dmamap_load_uio() or 755 * _bus_dmamap_load_mbuf(). Ignore the mlist and 756 * load each segment individually. 757 */ 758 map->dm_mapsize = size; 759 760 i = j = 0; 761 pa = segs[i].ds_addr; 762 dvmaddr += (pa & PGOFSET); 763 left = min(size, segs[i].ds_len); 764 765 sgstart = dvmaddr; 766 sgend = sgstart + left - 1; 767 768 map->dm_segs[j].ds_addr = dvmaddr; 769 map->dm_segs[j].ds_len = left; 770 771 /* Set the size (which we will be destroying */ 772 map->dm_mapsize = size; 773 774 while (size > 0) { 775 int incr; 776 777 if (left <= 0) { 778 u_long offset; 779 780 /* 781 * If the two segs are on different physical 782 * pages move to a new virtual page. 783 */ 784 if (trunc_page(pa) != 785 trunc_page(segs[++i].ds_addr)) 786 dvmaddr += NBPG; 787 788 pa = segs[i].ds_addr; 789 left = min(size, segs[i].ds_len); 790 791 offset = (pa & PGOFSET); 792 if (dvmaddr == trunc_page(dvmaddr) + offset) { 793 /* We can combine segments */ 794 map->dm_segs[j].ds_len += left; 795 sgend += left; 796 } else { 797 /* Need a new segment */ 798 dvmaddr = trunc_page(dvmaddr) + offset; 799 DPRINTF(IDB_INFO, 800 ("iommu_dvmamap_load_raw: " 801 "seg %d start %lx " 802 "size %lx\n", j, 803 map->dm_segs[j].ds_addr, 804 map->dm_segs[j]. 805 ds_len)); 806 if (++j > map->_dm_segcnt) 807 goto fail; 808 map->dm_segs[j].ds_addr = dvmaddr; 809 map->dm_segs[j].ds_len = left; 810 811 sgstart = dvmaddr; 812 sgend = sgstart + left - 1; 813 } 814 815 } 816 817 /* Check for boundary issues */ 818 while ((sgstart & ~(boundary - 1)) != 819 (sgend & ~(boundary - 1))) { 820 /* Need a new segment. */ 821 map->dm_segs[j].ds_len = 822 sgstart & (boundary - 1); 823 DPRINTF(IDB_INFO, ("iommu_dvmamap_load_raw: " 824 "seg %d start %lx size %lx\n", j, 825 map->dm_segs[j].ds_addr, 826 map->dm_segs[j].ds_len)); 827 if (++j > map->_dm_segcnt) { 828 fail: 829 iommu_dvmamap_unload(t, is, map); 830 return (E2BIG); 831 } 832 sgstart = roundup(sgstart, boundary); 833 map->dm_segs[j].ds_addr = sgstart; 834 map->dm_segs[j].ds_len = sgend - sgstart + 1; 835 } 836 837 if (sgsize == 0) 838 panic("iommu_dmamap_load_raw: size botch"); 839 840 DPRINTF(IDB_BUSDMA, 841 ("iommu_dvmamap_load_raw: map %p loading va %lx at pa %lx\n", 842 map, (long)dvmaddr, (long)(pa))); 843 /* Enter it if we haven't before. */ 844 if (prev_va != trunc_page(dvmaddr)) 845 iommu_enter(is, prev_va = trunc_page(dvmaddr), 846 trunc_page(pa), flags); 847 incr = min(pagesz, left); 848 dvmaddr += incr; 849 pa += incr; 850 left -= incr; 851 size -= incr; 852 } 853 DPRINTF(IDB_INFO, ("iommu_dvmamap_load_raw: " 854 "seg %d start %lx size %lx\n", j, 855 map->dm_segs[j].ds_addr, map->dm_segs[j].ds_len)); 856 map->dm_nsegs = j+1; 857 return (0); 858 } 859 /* 860 * This was allocated with bus_dmamem_alloc. 861 * The pages are on an `mlist'. 862 */ 863 map->dm_mapsize = size; 864 i = 0; 865 sgstart = dvmaddr; 866 sgend = sgstart + size - 1; 867 map->dm_segs[i].ds_addr = sgstart; 868 while ((sgstart & ~(boundary - 1)) != (sgend & ~(boundary - 1))) { 869 /* Oops. We crossed a boundary. Split the xfer. */ 870 map->dm_segs[i].ds_len = sgstart & (boundary - 1); 871 DPRINTF(IDB_INFO, ("iommu_dvmamap_load_raw: " 872 "seg %d start %lx size %lx\n", i, 873 map->dm_segs[i].ds_addr, 874 map->dm_segs[i].ds_len)); 875 if (++i > map->_dm_segcnt) { 876 /* Too many segments. Fail the operation. */ 877 s = splhigh(); 878 /* How can this fail? And if it does what can we do? */ 879 err = extent_free(is->is_dvmamap, 880 dvmaddr, sgsize, EX_NOWAIT); 881 splx(s); 882 map->_dm_dvmastart = 0; 883 map->_dm_dvmasize = 0; 884 return (E2BIG); 885 } 886 sgstart = roundup(sgstart, boundary); 887 map->dm_segs[i].ds_addr = sgstart; 888 } 889 DPRINTF(IDB_INFO, ("iommu_dvmamap_load_raw: " 890 "seg %d start %lx size %lx\n", i, 891 map->dm_segs[i].ds_addr, map->dm_segs[i].ds_len)); 892 map->dm_segs[i].ds_len = sgend - sgstart + 1; 893 894 for (m = TAILQ_FIRST(mlist); m != NULL; m = TAILQ_NEXT(m,pageq)) { 895 if (sgsize == 0) 896 panic("iommu_dmamap_load_raw: size botch"); 897 pa = VM_PAGE_TO_PHYS(m); 898 899 DPRINTF(IDB_BUSDMA, 900 ("iommu_dvmamap_load_raw: map %p loading va %lx at pa %lx\n", 901 map, (long)dvmaddr, (long)(pa))); 902 iommu_enter(is, dvmaddr, pa, flags); 903 904 dvmaddr += pagesz; 905 sgsize -= pagesz; 906 } 907 map->dm_mapsize = size; 908 map->dm_nsegs = i+1; 909 return (0); 910 } 911 912 void 913 iommu_dvmamap_sync(t, is, map, offset, len, ops) 914 bus_dma_tag_t t; 915 struct iommu_state *is; 916 bus_dmamap_t map; 917 bus_addr_t offset; 918 bus_size_t len; 919 int ops; 920 { 921 vaddr_t va = map->dm_segs[0].ds_addr + offset; 922 923 /* 924 * We only support one DMA segment; supporting more makes this code 925 * too unweildy. 926 */ 927 928 if (ops & BUS_DMASYNC_PREREAD) { 929 DPRINTF(IDB_SYNC, 930 ("iommu_dvmamap_sync: syncing va %p len %lu " 931 "BUS_DMASYNC_PREREAD\n", (void *)(u_long)va, (u_long)len)); 932 933 /* Nothing to do */; 934 } 935 if (ops & BUS_DMASYNC_POSTREAD) { 936 DPRINTF(IDB_SYNC, 937 ("iommu_dvmamap_sync: syncing va %p len %lu " 938 "BUS_DMASYNC_POSTREAD\n", (void *)(u_long)va, (u_long)len)); 939 /* if we have a streaming buffer, flush it here first */ 940 if (is->is_sb) 941 while (len > 0) { 942 DPRINTF(IDB_BUSDMA, 943 ("iommu_dvmamap_sync: flushing va %p, %lu " 944 "bytes left\n", (void *)(u_long)va, (u_long)len)); 945 iommu_strbuf_flush(is, va); 946 if (len <= NBPG) { 947 iommu_strbuf_flush_done(is); 948 len = 0; 949 } else 950 len -= NBPG; 951 va += NBPG; 952 } 953 } 954 if (ops & BUS_DMASYNC_PREWRITE) { 955 DPRINTF(IDB_SYNC, 956 ("iommu_dvmamap_sync: syncing va %p len %lu " 957 "BUS_DMASYNC_PREWRITE\n", (void *)(u_long)va, (u_long)len)); 958 /* if we have a streaming buffer, flush it here first */ 959 if (is->is_sb) 960 while (len > 0) { 961 DPRINTF(IDB_BUSDMA, 962 ("iommu_dvmamap_sync: flushing va %p, %lu " 963 "bytes left\n", (void *)(u_long)va, (u_long)len)); 964 iommu_strbuf_flush(is, va); 965 if (len <= NBPG) { 966 iommu_strbuf_flush_done(is); 967 len = 0; 968 } else 969 len -= NBPG; 970 va += NBPG; 971 } 972 } 973 if (ops & BUS_DMASYNC_POSTWRITE) { 974 DPRINTF(IDB_SYNC, 975 ("iommu_dvmamap_sync: syncing va %p len %lu " 976 "BUS_DMASYNC_POSTWRITE\n", (void *)(u_long)va, (u_long)len)); 977 /* Nothing to do */; 978 } 979 } 980 981 int 982 iommu_dvmamem_alloc(t, is, size, alignment, boundary, segs, nsegs, rsegs, flags) 983 bus_dma_tag_t t; 984 struct iommu_state *is; 985 bus_size_t size, alignment, boundary; 986 bus_dma_segment_t *segs; 987 int nsegs; 988 int *rsegs; 989 int flags; 990 { 991 992 DPRINTF(IDB_BUSDMA, ("iommu_dvmamem_alloc: sz %llx align %llx bound %llx " 993 "segp %p flags %d\n", (unsigned long long)size, 994 (unsigned long long)alignment, (unsigned long long)boundary, 995 segs, flags)); 996 return (bus_dmamem_alloc(t->_parent, size, alignment, boundary, 997 segs, nsegs, rsegs, flags|BUS_DMA_DVMA)); 998 } 999 1000 void 1001 iommu_dvmamem_free(t, is, segs, nsegs) 1002 bus_dma_tag_t t; 1003 struct iommu_state *is; 1004 bus_dma_segment_t *segs; 1005 int nsegs; 1006 { 1007 1008 DPRINTF(IDB_BUSDMA, ("iommu_dvmamem_free: segp %p nsegs %d\n", 1009 segs, nsegs)); 1010 bus_dmamem_free(t->_parent, segs, nsegs); 1011 } 1012 1013 /* 1014 * Map the DVMA mappings into the kernel pmap. 1015 * Check the flags to see whether we're streaming or coherent. 1016 */ 1017 int 1018 iommu_dvmamem_map(t, is, segs, nsegs, size, kvap, flags) 1019 bus_dma_tag_t t; 1020 struct iommu_state *is; 1021 bus_dma_segment_t *segs; 1022 int nsegs; 1023 size_t size; 1024 caddr_t *kvap; 1025 int flags; 1026 { 1027 struct vm_page *m; 1028 vaddr_t va; 1029 bus_addr_t addr; 1030 struct pglist *mlist; 1031 int cbit; 1032 1033 DPRINTF(IDB_BUSDMA, ("iommu_dvmamem_map: segp %p nsegs %d size %lx\n", 1034 segs, nsegs, size)); 1035 1036 /* 1037 * Allocate some space in the kernel map, and then map these pages 1038 * into this space. 1039 */ 1040 size = round_page(size); 1041 va = uvm_km_valloc(kernel_map, size); 1042 if (va == 0) 1043 return (ENOMEM); 1044 1045 *kvap = (caddr_t)va; 1046 1047 /* 1048 * digest flags: 1049 */ 1050 cbit = 0; 1051 if (flags & BUS_DMA_COHERENT) /* Disable vcache */ 1052 cbit |= PMAP_NVC; 1053 if (flags & BUS_DMA_NOCACHE) /* sideffects */ 1054 cbit |= PMAP_NC; 1055 1056 /* 1057 * Now take this and map it into the CPU. 1058 */ 1059 mlist = segs[0]._ds_mlist; 1060 for (m = mlist->tqh_first; m != NULL; m = m->pageq.tqe_next) { 1061 #ifdef DIAGNOSTIC 1062 if (size == 0) 1063 panic("iommu_dvmamem_map: size botch"); 1064 #endif 1065 addr = VM_PAGE_TO_PHYS(m); 1066 DPRINTF(IDB_BUSDMA, ("iommu_dvmamem_map: " 1067 "mapping va %lx at %llx\n", va, (unsigned long long)addr | cbit)); 1068 pmap_enter(pmap_kernel(), va, addr | cbit, 1069 VM_PROT_READ | VM_PROT_WRITE, 1070 VM_PROT_READ | VM_PROT_WRITE | PMAP_WIRED); 1071 va += PAGE_SIZE; 1072 size -= PAGE_SIZE; 1073 } 1074 pmap_update(pmap_kernel()); 1075 1076 return (0); 1077 } 1078 1079 /* 1080 * Unmap DVMA mappings from kernel 1081 */ 1082 void 1083 iommu_dvmamem_unmap(t, is, kva, size) 1084 bus_dma_tag_t t; 1085 struct iommu_state *is; 1086 caddr_t kva; 1087 size_t size; 1088 { 1089 1090 DPRINTF(IDB_BUSDMA, ("iommu_dvmamem_unmap: kvm %p size %lx\n", 1091 kva, size)); 1092 1093 #ifdef DIAGNOSTIC 1094 if ((u_long)kva & PGOFSET) 1095 panic("iommu_dvmamem_unmap"); 1096 #endif 1097 1098 size = round_page(size); 1099 pmap_remove(pmap_kernel(), (vaddr_t)kva, size); 1100 pmap_update(pmap_kernel()); 1101 #if 0 1102 /* 1103 * XXX ? is this necessary? i think so and i think other 1104 * implementations are missing it. 1105 */ 1106 uvm_km_free(kernel_map, (vaddr_t)kva, size); 1107 #endif 1108 } 1109
Indexes created Thu Sep 25 16:09:42 GMT 2025