1 /* $NetBSD: sgmap_typedep.c,v 1.44 2021/07/19 16:25:54 thorpej Exp $ */ 2 3 /*- 4 * Copyright (c) 1997, 1998, 2001 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility, 9 * NASA Ames Research Center. 10 * 11 * Redistribution and use in source and binary forms, with or without 12 * modification, are permitted provided that the following conditions 13 * are met: 14 * 1. Redistributions of source code must retain the above copyright 15 * notice, this list of conditions and the following disclaimer. 16 * 2. Redistributions in binary form must reproduce the above copyright 17 * notice, this list of conditions and the following disclaimer in the 18 * documentation and/or other materials provided with the distribution. 19 * 20 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 21 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 22 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 23 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 24 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 25 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 26 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 27 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 28 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 29 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 30 * POSSIBILITY OF SUCH DAMAGE. 31 */ 32 33 #include <sys/cdefs.h> 34 __KERNEL_RCSID(1, "$NetBSD: sgmap_typedep.c,v 1.44 2021/07/19 16:25:54 thorpej Exp $"); 35 36 #include "opt_ddb.h" 37 38 #include <sys/evcnt.h> 39 #include <uvm/uvm_extern.h> 40 41 #define DMA_COUNT_DECL(cnt) _DMA_COUNT_DECL(dma_sgmap, cnt) 42 #define DMA_COUNT(cnt) _DMA_COUNT(dma_sgmap, cnt) 43 44 #ifdef SGMAP_DEBUG 45 int __C(SGMAP_TYPE,_debug) = 0; 46 #endif 47 48 SGMAP_PTE_TYPE __C(SGMAP_TYPE,_prefetch_spill_page_pte); 49 50 static void __C(SGMAP_TYPE,_do_unload)(bus_dma_tag_t, bus_dmamap_t, 51 struct alpha_sgmap *); 52 53 void 54 __C(SGMAP_TYPE,_init_spill_page_pte)(void) 55 { 56 57 __C(SGMAP_TYPE,_prefetch_spill_page_pte) = 58 (alpha_sgmap_prefetch_spill_page_pa >> 59 SGPTE_PGADDR_SHIFT) | SGPTE_VALID; 60 } 61 62 DMA_COUNT_DECL(spill_page); 63 DMA_COUNT_DECL(extra_segment); 64 DMA_COUNT_DECL(extra_segment_and_spill); 65 66 static int 67 __C(SGMAP_TYPE,_load_buffer)(bus_dma_tag_t t, bus_dmamap_t map, void *buf, 68 size_t buflen, struct vmspace *vm, int flags, int * const segp, 69 struct alpha_sgmap *sgmap) 70 { 71 vaddr_t endva, va = (vaddr_t)buf; 72 paddr_t pa; 73 bus_addr_t dmaoffset, sgva, extra_sgva; 74 bus_size_t sgvalen, extra_sgvalen, boundary, alignment; 75 SGMAP_PTE_TYPE *pte, *page_table = sgmap->aps_pt; 76 int pteidx, error, spill, seg = *segp; 77 bool address_is_valid __diagused; 78 79 /* Initialize the spill page PTE if it hasn't been already. */ 80 if (__C(SGMAP_TYPE,_prefetch_spill_page_pte) == 0) 81 __C(SGMAP_TYPE,_init_spill_page_pte)(); 82 83 if (seg == map->_dm_segcnt) { 84 /* Ran of segments. */ 85 return EFBIG; 86 } 87 KASSERT(seg < map->_dm_segcnt); 88 89 /* 90 * Remember the offset into the first page and the total 91 * transfer length. 92 */ 93 dmaoffset = ((u_long)buf) & PGOFSET; 94 95 #ifdef SGMAP_DEBUG 96 if (__C(SGMAP_TYPE,_debug)) { 97 printf("sgmap_load: ----- buf = %p -----\n", buf); 98 printf("sgmap_load: dmaoffset = 0x%lx, buflen = 0x%lx\n", 99 dmaoffset, buflen); 100 } 101 #endif 102 103 /* 104 * Allocate the necessary virtual address space for the 105 * mapping. Round the size, since we deal with whole pages. 106 */ 107 108 /* 109 * XXX Always allocate a spill page for now. Note 110 * the spill page is not needed for an in-bound-only 111 * transfer. 112 */ 113 if ((flags & BUS_DMA_READ) == 0) 114 spill = 1; 115 else 116 spill = 0; 117 118 boundary = map->_dm_boundary; 119 120 /* 121 * Caller's mistake if the requested length is larger than 122 * their own boundary constraint. 123 */ 124 if (__predict_false(boundary != 0 && buflen > boundary)) { 125 return EINVAL; 126 } 127 128 endva = round_page(va + buflen); 129 va = trunc_page(va); 130 131 const vm_flag_t vmflags = VM_INSTANTFIT | 132 ((flags & BUS_DMA_NOWAIT) ? VM_NOSLEEP : VM_SLEEP); 133 134 KASSERT(t->_sgmap_minalign != 0); 135 alignment = t->_sgmap_minalign; 136 sgvalen = (endva - va); 137 138 SGMAP_PTE_TYPE spill_pte_v = __C(SGMAP_TYPE,_prefetch_spill_page_pte); 139 140 /* 141 * If we have a boundary constraint, it's possible to end up in 142 * a situation where sgvalen > boundary if the caller's buffer 143 * is not page aligned. In this case, we will have to allocate 144 * an extra SG segment and split the buffer. 145 */ 146 if (__predict_false(boundary != 0 && boundary < sgvalen)) { 147 #ifdef SGMAP_DEBUG 148 if (__C(SGMAP_TYPE,_debug)) { 149 printf("sgmap_load: extra segment needed\n"); 150 } 151 #endif 152 DMA_COUNT(extra_segment); 153 154 /* This should only ever happen for unaligned buffers. */ 155 KASSERT(dmaoffset != 0); 156 157 extra_sgvalen = sgvalen - boundary; 158 KASSERT(extra_sgvalen == PAGE_SIZE); 159 160 /* 161 * Adjust the lengths of the first segment. The length 162 * of the second segment will be dmaoffset. 163 */ 164 sgvalen -= extra_sgvalen; 165 endva -= extra_sgvalen; 166 buflen -= dmaoffset; 167 168 if (spill) { 169 DMA_COUNT(extra_segment_and_spill); 170 extra_sgvalen += PAGE_SIZE; 171 } 172 173 error = vmem_xalloc(sgmap->aps_arena, extra_sgvalen, 174 alignment, /* alignment */ 175 0, /* phase */ 176 boundary, /* nocross */ 177 VMEM_ADDR_MIN, /* minaddr */ 178 VMEM_ADDR_MAX, /* maxaddr */ 179 vmflags, 180 &extra_sgva); 181 if (error) { 182 return error; 183 } 184 } else { 185 extra_sgvalen = 0; 186 extra_sgva = 0; 187 } 188 189 190 if (spill) { 191 DMA_COUNT(spill_page); 192 sgvalen += PAGE_SIZE; 193 194 /* 195 * ARGH! If the addition of the spill page bumped us 196 * over our boundary, we have to 2x the boundary limit. 197 * To compensate (and enforce the original boundary 198 * constraint), we force our alignment to be at least the 199 * previous boundary, thus ensuring that the only boundary 200 * violation is the pre-fetch that the SGMAP controller 201 * performs that necessitates the spill page in the first 202 * place. 203 */ 204 if (boundary && boundary < sgvalen) { 205 if (alignment < boundary) { 206 alignment = boundary; 207 } 208 do { 209 boundary <<= 1; 210 } while (boundary < sgvalen); 211 } 212 } 213 214 #ifdef SGMAP_DEBUG 215 if (__C(SGMAP_TYPE,_debug)) { 216 printf("sgmap_load: va:endva = 0x%lx:0x%lx\n", va, endva); 217 printf("sgmap_load: sgvalen = 0x%lx, boundary = 0x%lx\n", 218 sgvalen, boundary); 219 } 220 #endif 221 222 error = vmem_xalloc(sgmap->aps_arena, sgvalen, 223 alignment, /* alignment */ 224 0, /* phase */ 225 boundary, /* nocross */ 226 VMEM_ADDR_MIN, /* minaddr */ 227 VMEM_ADDR_MAX, /* maxaddr */ 228 vmflags, 229 &sgva); 230 if (error) { 231 if (extra_sgvalen != 0) { 232 vmem_xfree(sgmap->aps_arena, extra_sgva, extra_sgvalen); 233 } 234 return error; 235 } 236 237 pteidx = sgva >> SGMAP_ADDR_PTEIDX_SHIFT; 238 pte = &page_table[pteidx * SGMAP_PTE_SPACING]; 239 240 #ifdef SGMAP_DEBUG 241 if (__C(SGMAP_TYPE,_debug)) 242 printf("sgmap_load: sgva = 0x%lx, pteidx = %d, " 243 "pte = %p (pt = %p)\n", sgva, pteidx, pte, 244 page_table); 245 #endif 246 247 /* Generate the DMA address. */ 248 map->dm_segs[seg].ds_addr = sgmap->aps_wbase | sgva | dmaoffset; 249 map->dm_segs[seg].ds_len = buflen; 250 if (__predict_false(extra_sgvalen != 0)) { 251 if (++seg == map->_dm_segcnt) { 252 /* Boo! Ran out of segments! */ 253 vmem_xfree(sgmap->aps_arena, extra_sgva, extra_sgvalen); 254 vmem_xfree(sgmap->aps_arena, sgva, sgvalen); 255 return EFBIG; 256 } 257 map->dm_segs[seg].ds_addr = sgmap->aps_wbase | extra_sgva; 258 map->dm_segs[seg].ds_len = dmaoffset; 259 *segp = seg; 260 } 261 262 #ifdef SGMAP_DEBUG 263 if (__C(SGMAP_TYPE,_debug)) 264 printf("sgmap_load: wbase = 0x%lx, vpage = 0x%lx, " 265 "DMA addr = 0x%lx\n", sgmap->aps_wbase, (uint64_t)sgva, 266 map->dm_segs[seg].ds_addr); 267 #endif 268 269 for (; va < endva; va += PAGE_SIZE, pteidx++, 270 pte = &page_table[pteidx * SGMAP_PTE_SPACING]) { 271 /* Get the physical address for this segment. */ 272 address_is_valid = pmap_extract(vm->vm_map.pmap, va, &pa); 273 KASSERT(address_is_valid); 274 275 /* Load the current PTE with this page. */ 276 *pte = (pa >> SGPTE_PGADDR_SHIFT) | SGPTE_VALID; 277 #ifdef SGMAP_DEBUG 278 if (__C(SGMAP_TYPE,_debug)) 279 printf("sgmap_load: pa = 0x%lx, pte = %p, " 280 "*pte = 0x%lx\n", pa, pte, (u_long)(*pte)); 281 #endif 282 } 283 284 if (__predict_false(extra_sgvalen != 0)) { 285 int extra_pteidx = extra_sgva >> SGMAP_ADDR_PTEIDX_SHIFT; 286 SGMAP_PTE_TYPE *extra_pte = 287 &page_table[extra_pteidx * SGMAP_PTE_SPACING]; 288 289 /* va == endva == address of extra page */ 290 KASSERT(va == endva); 291 address_is_valid = pmap_extract(vm->vm_map.pmap, va, &pa); 292 KASSERT(address_is_valid); 293 294 /* 295 * If a spill page is needed, the previous segment will 296 * need to use this PTE value for it. 297 */ 298 spill_pte_v = (pa >> SGPTE_PGADDR_SHIFT) | SGPTE_VALID; 299 *extra_pte = spill_pte_v; 300 301 /* ...but the extra segment uses the real spill PTE. */ 302 if (spill) { 303 extra_pteidx++; 304 extra_pte = 305 &page_table[extra_pteidx * SGMAP_PTE_SPACING]; 306 *extra_pte = __C(SGMAP_TYPE,_prefetch_spill_page_pte); 307 } 308 } 309 310 if (spill) { 311 /* ...and the prefetch-spill page. */ 312 *pte = spill_pte_v; 313 #ifdef SGMAP_DEBUG 314 if (__C(SGMAP_TYPE,_debug)) { 315 printf("sgmap_load: spill page, pte = %p, " 316 "*pte = 0x%lx\n", pte, (uint64_t)*pte); 317 } 318 #endif 319 } 320 321 return (0); 322 } 323 324 DMA_COUNT_DECL(load); 325 DMA_COUNT_DECL(load_next_window); 326 327 int 328 __C(SGMAP_TYPE,_load)(bus_dma_tag_t t, bus_dmamap_t map, void *buf, 329 bus_size_t buflen, struct proc *p, int flags, struct alpha_sgmap *sgmap) 330 { 331 int seg, error; 332 struct vmspace *vm; 333 334 /* 335 * Make sure that on error condition we return "no valid mappings". 336 */ 337 map->dm_mapsize = 0; 338 map->dm_nsegs = 0; 339 340 if (buflen > map->_dm_size) 341 return (EINVAL); 342 343 KASSERT((map->_dm_flags & (BUS_DMA_READ|BUS_DMA_WRITE)) == 0); 344 KASSERT((flags & (BUS_DMA_READ|BUS_DMA_WRITE)) != 345 (BUS_DMA_READ|BUS_DMA_WRITE)); 346 347 map->_dm_flags |= flags & (BUS_DMA_READ|BUS_DMA_WRITE); 348 349 if (p != NULL) { 350 vm = p->p_vmspace; 351 } else { 352 vm = vmspace_kernel(); 353 } 354 seg = 0; 355 error = __C(SGMAP_TYPE,_load_buffer)(t, map, buf, buflen, vm, 356 flags, &seg, sgmap); 357 358 alpha_mb(); 359 360 #if defined(SGMAP_DEBUG) && defined(DDB) 361 if (__C(SGMAP_TYPE,_debug) > 1) 362 Debugger(); 363 #endif 364 365 if (error == 0) { 366 DMA_COUNT(load); 367 map->dm_mapsize = buflen; 368 map->dm_nsegs = seg + 1; 369 map->_dm_window = t; 370 } else { 371 map->_dm_flags &= ~(BUS_DMA_READ|BUS_DMA_WRITE); 372 if (t->_next_window != NULL) { 373 /* Give the next window a chance. */ 374 DMA_COUNT(load_next_window); 375 error = bus_dmamap_load(t->_next_window, map, buf, 376 buflen, p, flags); 377 } 378 } 379 return (error); 380 } 381 382 DMA_COUNT_DECL(load_mbuf); 383 DMA_COUNT_DECL(load_mbuf_next_window); 384 385 int 386 __C(SGMAP_TYPE,_load_mbuf)(bus_dma_tag_t t, bus_dmamap_t map, 387 struct mbuf *m0, int flags, struct alpha_sgmap *sgmap) 388 { 389 struct mbuf *m; 390 int seg, error; 391 392 /* 393 * Make sure that on error condition we return "no valid mappings". 394 */ 395 map->dm_mapsize = 0; 396 map->dm_nsegs = 0; 397 398 #ifdef DIAGNOSTIC 399 if ((m0->m_flags & M_PKTHDR) == 0) 400 panic(__S(__C(SGMAP_TYPE,_load_mbuf)) ": no packet header"); 401 #endif 402 403 if (m0->m_pkthdr.len > map->_dm_size) 404 return (EINVAL); 405 406 KASSERT((map->_dm_flags & (BUS_DMA_READ|BUS_DMA_WRITE)) == 0); 407 KASSERT((flags & (BUS_DMA_READ|BUS_DMA_WRITE)) != 408 (BUS_DMA_READ|BUS_DMA_WRITE)); 409 410 map->_dm_flags |= flags & (BUS_DMA_READ|BUS_DMA_WRITE); 411 412 seg = 0; 413 error = 0; 414 for (m = m0; m != NULL && error == 0; m = m->m_next) { 415 if (m->m_len == 0) 416 continue; 417 error = __C(SGMAP_TYPE,_load_buffer)(t, map, 418 m->m_data, m->m_len, vmspace_kernel(), flags, &seg, sgmap); 419 seg++; 420 } 421 422 alpha_mb(); 423 424 #if defined(SGMAP_DEBUG) && defined(DDB) 425 if (__C(SGMAP_TYPE,_debug) > 1) 426 Debugger(); 427 #endif 428 429 if (error == 0) { 430 DMA_COUNT(load_mbuf); 431 map->dm_mapsize = m0->m_pkthdr.len; 432 map->dm_nsegs = seg; 433 map->_dm_window = t; 434 } else { 435 /* Need to back out what we've done so far. */ 436 map->dm_nsegs = seg - 1; 437 __C(SGMAP_TYPE,_do_unload)(t, map, sgmap); 438 map->_dm_flags &= ~(BUS_DMA_READ|BUS_DMA_WRITE); 439 if (t->_next_window != NULL) { 440 /* Give the next window a chance. */ 441 DMA_COUNT(load_mbuf_next_window); 442 error = bus_dmamap_load_mbuf(t->_next_window, map, 443 m0, flags); 444 } 445 } 446 447 return (error); 448 } 449 450 DMA_COUNT_DECL(load_uio); 451 DMA_COUNT_DECL(load_uio_next_window); 452 453 int 454 __C(SGMAP_TYPE,_load_uio)(bus_dma_tag_t t, bus_dmamap_t map, struct uio *uio, 455 int flags, struct alpha_sgmap *sgmap) 456 { 457 bus_size_t minlen, resid; 458 struct vmspace *vm; 459 struct iovec *iov; 460 void *addr; 461 int i, seg, error; 462 463 /* 464 * Make sure that on error condition we return "no valid mappings". 465 */ 466 map->dm_mapsize = 0; 467 map->dm_nsegs = 0; 468 469 KASSERT((map->_dm_flags & (BUS_DMA_READ|BUS_DMA_WRITE)) == 0); 470 KASSERT((flags & (BUS_DMA_READ|BUS_DMA_WRITE)) != 471 (BUS_DMA_READ|BUS_DMA_WRITE)); 472 473 map->_dm_flags |= flags & (BUS_DMA_READ|BUS_DMA_WRITE); 474 475 resid = uio->uio_resid; 476 iov = uio->uio_iov; 477 478 vm = uio->uio_vmspace; 479 480 seg = 0; 481 error = 0; 482 for (i = 0; i < uio->uio_iovcnt && resid != 0 && error == 0; i++) { 483 /* 484 * Now at the first iovec to load. Load each iovec 485 * until we have exhausted the residual count. 486 */ 487 minlen = resid < iov[i].iov_len ? resid : iov[i].iov_len; 488 addr = (void *)iov[i].iov_base; 489 490 error = __C(SGMAP_TYPE,_load_buffer)(t, map, 491 addr, minlen, vm, flags, &seg, sgmap); 492 seg++; 493 494 resid -= minlen; 495 } 496 497 alpha_mb(); 498 499 #if defined(SGMAP_DEBUG) && defined(DDB) 500 if (__C(SGMAP_TYPE,_debug) > 1) 501 Debugger(); 502 #endif 503 504 if (error == 0) { 505 DMA_COUNT(load_uio); 506 map->dm_mapsize = uio->uio_resid; 507 map->dm_nsegs = seg; 508 map->_dm_window = t; 509 } else { 510 /* Need to back out what we've done so far. */ 511 map->dm_nsegs = seg - 1; 512 __C(SGMAP_TYPE,_do_unload)(t, map, sgmap); 513 map->_dm_flags &= ~(BUS_DMA_READ|BUS_DMA_WRITE); 514 if (t->_next_window != NULL) { 515 /* Give the next window a chance. */ 516 DMA_COUNT(load_uio_next_window); 517 error = bus_dmamap_load_uio(t->_next_window, map, 518 uio, flags); 519 } 520 } 521 522 return (error); 523 } 524 525 int 526 __C(SGMAP_TYPE,_load_raw)(bus_dma_tag_t t, bus_dmamap_t map, 527 bus_dma_segment_t *segs, int nsegs, bus_size_t size, int flags, 528 struct alpha_sgmap *sgmap) 529 { 530 531 KASSERT((map->_dm_flags & (BUS_DMA_READ|BUS_DMA_WRITE)) == 0); 532 KASSERT((flags & (BUS_DMA_READ|BUS_DMA_WRITE)) != 533 (BUS_DMA_READ|BUS_DMA_WRITE)); 534 535 panic(__S(__C(SGMAP_TYPE,_load_raw)) ": not implemented"); 536 } 537 538 static void 539 __C(SGMAP_TYPE,_do_unload)(bus_dma_tag_t t, bus_dmamap_t map, 540 struct alpha_sgmap *sgmap) 541 { 542 SGMAP_PTE_TYPE *pte, *page_table = sgmap->aps_pt; 543 bus_addr_t osgva, sgva, esgva; 544 int spill, seg, pteidx; 545 546 for (seg = 0; seg < map->dm_nsegs; seg++) { 547 /* 548 * XXX Always allocate a spill page for now. Note 549 * the spill page is not needed for an in-bound-only 550 * transfer. 551 */ 552 if ((map->_dm_flags & BUS_DMA_READ) == 0) 553 spill = 1; 554 else 555 spill = 0; 556 557 sgva = map->dm_segs[seg].ds_addr & ~sgmap->aps_wbase; 558 559 esgva = round_page(sgva + map->dm_segs[seg].ds_len); 560 osgva = sgva = trunc_page(sgva); 561 562 if (spill) 563 esgva += PAGE_SIZE; 564 565 /* Invalidate the PTEs for the mapping. */ 566 for (pteidx = sgva >> SGMAP_ADDR_PTEIDX_SHIFT; 567 sgva < esgva; sgva += PAGE_SIZE, pteidx++) { 568 pte = &page_table[pteidx * SGMAP_PTE_SPACING]; 569 #ifdef SGMAP_DEBUG 570 if (__C(SGMAP_TYPE,_debug)) 571 printf("sgmap_unload: pte = %p, " 572 "*pte = 0x%lx\n", pte, (u_long)(*pte)); 573 #endif 574 *pte = 0; 575 } 576 577 alpha_mb(); 578 579 /* Free the virtual address space used by the mapping. */ 580 vmem_xfree(sgmap->aps_arena, osgva, (esgva - osgva)); 581 } 582 583 map->_dm_flags &= ~(BUS_DMA_READ|BUS_DMA_WRITE); 584 585 /* Mark the mapping invalid. */ 586 map->dm_mapsize = 0; 587 map->dm_nsegs = 0; 588 map->_dm_window = NULL; 589 } 590 591 DMA_COUNT_DECL(unload); 592 593 void 594 __C(SGMAP_TYPE,_unload)(bus_dma_tag_t t, bus_dmamap_t map, 595 struct alpha_sgmap *sgmap) 596 { 597 KASSERT(map->_dm_window == t); 598 DMA_COUNT(unload); 599 __C(SGMAP_TYPE,_do_unload)(t, map, sgmap); 600 } 601
Indexes created Sat Sep 20 22:09:52 GMT 2025