1 /* $NetBSD: radeon_vm.c,v 1.8 2021/12/18 23:45:43 riastradh Exp $ */ 2 3 /* 4 * Copyright 2008 Advanced Micro Devices, Inc. 5 * Copyright 2008 Red Hat Inc. 6 * Copyright 2009 Jerome Glisse. 7 * 8 * Permission is hereby granted, free of charge, to any person obtaining a 9 * copy of this software and associated documentation files (the "Software"), 10 * to deal in the Software without restriction, including without limitation 11 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 12 * and/or sell copies of the Software, and to permit persons to whom the 13 * Software is furnished to do so, subject to the following conditions: 14 * 15 * The above copyright notice and this permission notice shall be included in 16 * all copies or substantial portions of the Software. 17 * 18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 19 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 20 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 21 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 22 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 23 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 24 * OTHER DEALINGS IN THE SOFTWARE. 25 * 26 * Authors: Dave Airlie 27 * Alex Deucher 28 * Jerome Glisse 29 */ 30 31 #include <sys/cdefs.h> 32 __KERNEL_RCSID(0, "$NetBSD: radeon_vm.c,v 1.8 2021/12/18 23:45:43 riastradh Exp $"); 33 34 #include <drm/radeon_drm.h> 35 #include "radeon.h" 36 #include "radeon_trace.h" 37 38 #include <linux/nbsd-namespace.h> 39 40 /* 41 * GPUVM 42 * GPUVM is similar to the legacy gart on older asics, however 43 * rather than there being a single global gart table 44 * for the entire GPU, there are multiple VM page tables active 45 * at any given time. The VM page tables can contain a mix 46 * vram pages and system memory pages and system memory pages 47 * can be mapped as snooped (cached system pages) or unsnooped 48 * (uncached system pages). 49 * Each VM has an ID associated with it and there is a page table 50 * associated with each VMID. When execting a command buffer, 51 * the kernel tells the the ring what VMID to use for that command 52 * buffer. VMIDs are allocated dynamically as commands are submitted. 53 * The userspace drivers maintain their own address space and the kernel 54 * sets up their pages tables accordingly when they submit their 55 * command buffers and a VMID is assigned. 56 * Cayman/Trinity support up to 8 active VMs at any given time; 57 * SI supports 16. 58 */ 59 60 /** 61 * radeon_vm_num_pde - return the number of page directory entries 62 * 63 * @rdev: radeon_device pointer 64 * 65 * Calculate the number of page directory entries (cayman+). 66 */ 67 static unsigned radeon_vm_num_pdes(struct radeon_device *rdev) 68 { 69 return rdev->vm_manager.max_pfn >> radeon_vm_block_size; 70 } 71 72 /** 73 * radeon_vm_directory_size - returns the size of the page directory in bytes 74 * 75 * @rdev: radeon_device pointer 76 * 77 * Calculate the size of the page directory in bytes (cayman+). 78 */ 79 static unsigned radeon_vm_directory_size(struct radeon_device *rdev) 80 { 81 return RADEON_GPU_PAGE_ALIGN(radeon_vm_num_pdes(rdev) * 8); 82 } 83 84 /** 85 * radeon_vm_manager_init - init the vm manager 86 * 87 * @rdev: radeon_device pointer 88 * 89 * Init the vm manager (cayman+). 90 * Returns 0 for success, error for failure. 91 */ 92 int radeon_vm_manager_init(struct radeon_device *rdev) 93 { 94 int r; 95 96 if (!rdev->vm_manager.enabled) { 97 r = radeon_asic_vm_init(rdev); 98 if (r) 99 return r; 100 101 rdev->vm_manager.enabled = true; 102 } 103 return 0; 104 } 105 106 /** 107 * radeon_vm_manager_fini - tear down the vm manager 108 * 109 * @rdev: radeon_device pointer 110 * 111 * Tear down the VM manager (cayman+). 112 */ 113 void radeon_vm_manager_fini(struct radeon_device *rdev) 114 { 115 int i; 116 117 if (!rdev->vm_manager.enabled) 118 return; 119 120 for (i = 0; i < RADEON_NUM_VM; ++i) 121 radeon_fence_unref(&rdev->vm_manager.active[i]); 122 radeon_asic_vm_fini(rdev); 123 rdev->vm_manager.enabled = false; 124 } 125 126 /** 127 * radeon_vm_get_bos - add the vm BOs to a validation list 128 * 129 * @vm: vm providing the BOs 130 * @head: head of validation list 131 * 132 * Add the page directory to the list of BOs to 133 * validate for command submission (cayman+). 134 */ 135 struct radeon_bo_list *radeon_vm_get_bos(struct radeon_device *rdev, 136 struct radeon_vm *vm, 137 struct list_head *head) 138 { 139 struct radeon_bo_list *list; 140 unsigned i, idx; 141 142 list = kvmalloc_array(vm->max_pde_used + 2, 143 sizeof(struct radeon_bo_list), GFP_KERNEL); 144 if (!list) 145 return NULL; 146 147 /* add the vm page table to the list */ 148 list[0].robj = vm->page_directory; 149 list[0].preferred_domains = RADEON_GEM_DOMAIN_VRAM; 150 list[0].allowed_domains = RADEON_GEM_DOMAIN_VRAM; 151 list[0].tv.bo = &vm->page_directory->tbo; 152 list[0].tv.num_shared = 1; 153 list[0].tiling_flags = 0; 154 list_add(&list[0].tv.head, head); 155 156 for (i = 0, idx = 1; i <= vm->max_pde_used; i++) { 157 if (!vm->page_tables[i].bo) 158 continue; 159 160 list[idx].robj = vm->page_tables[i].bo; 161 list[idx].preferred_domains = RADEON_GEM_DOMAIN_VRAM; 162 list[idx].allowed_domains = RADEON_GEM_DOMAIN_VRAM; 163 list[idx].tv.bo = &list[idx].robj->tbo; 164 list[idx].tv.num_shared = 1; 165 list[idx].tiling_flags = 0; 166 list_add(&list[idx++].tv.head, head); 167 } 168 169 return list; 170 } 171 172 /** 173 * radeon_vm_grab_id - allocate the next free VMID 174 * 175 * @rdev: radeon_device pointer 176 * @vm: vm to allocate id for 177 * @ring: ring we want to submit job to 178 * 179 * Allocate an id for the vm (cayman+). 180 * Returns the fence we need to sync to (if any). 181 * 182 * Global and local mutex must be locked! 183 */ 184 struct radeon_fence *radeon_vm_grab_id(struct radeon_device *rdev, 185 struct radeon_vm *vm, int ring) 186 { 187 struct radeon_fence *best[RADEON_NUM_RINGS] = {}; 188 struct radeon_vm_id *vm_id = &vm->ids[ring]; 189 190 unsigned choices[2] = {}; 191 unsigned i; 192 193 /* check if the id is still valid */ 194 if (vm_id->id && vm_id->last_id_use && 195 vm_id->last_id_use == rdev->vm_manager.active[vm_id->id]) 196 return NULL; 197 198 /* we definately need to flush */ 199 vm_id->pd_gpu_addr = ~0ll; 200 201 /* skip over VMID 0, since it is the system VM */ 202 for (i = 1; i < rdev->vm_manager.nvm; ++i) { 203 struct radeon_fence *fence = rdev->vm_manager.active[i]; 204 205 if (fence == NULL) { 206 /* found a free one */ 207 vm_id->id = i; 208 trace_radeon_vm_grab_id(i, ring); 209 return NULL; 210 } 211 212 if (radeon_fence_is_earlier(fence, best[fence->ring])) { 213 best[fence->ring] = fence; 214 choices[fence->ring == ring ? 0 : 1] = i; 215 } 216 } 217 218 for (i = 0; i < 2; ++i) { 219 if (choices[i]) { 220 vm_id->id = choices[i]; 221 trace_radeon_vm_grab_id(choices[i], ring); 222 return rdev->vm_manager.active[choices[i]]; 223 } 224 } 225 226 /* should never happen */ 227 BUG(); 228 return NULL; 229 } 230 231 /** 232 * radeon_vm_flush - hardware flush the vm 233 * 234 * @rdev: radeon_device pointer 235 * @vm: vm we want to flush 236 * @ring: ring to use for flush 237 * @updates: last vm update that is waited for 238 * 239 * Flush the vm (cayman+). 240 * 241 * Global and local mutex must be locked! 242 */ 243 void radeon_vm_flush(struct radeon_device *rdev, 244 struct radeon_vm *vm, 245 int ring, struct radeon_fence *updates) 246 { 247 uint64_t pd_addr = radeon_bo_gpu_offset(vm->page_directory); 248 struct radeon_vm_id *vm_id = &vm->ids[ring]; 249 250 if (pd_addr != vm_id->pd_gpu_addr || !vm_id->flushed_updates || 251 radeon_fence_is_earlier(vm_id->flushed_updates, updates)) { 252 253 trace_radeon_vm_flush(pd_addr, ring, vm->ids[ring].id); 254 radeon_fence_unref(&vm_id->flushed_updates); 255 vm_id->flushed_updates = radeon_fence_ref(updates); 256 vm_id->pd_gpu_addr = pd_addr; 257 radeon_ring_vm_flush(rdev, &rdev->ring[ring], 258 vm_id->id, vm_id->pd_gpu_addr); 259 260 } 261 } 262 263 /** 264 * radeon_vm_fence - remember fence for vm 265 * 266 * @rdev: radeon_device pointer 267 * @vm: vm we want to fence 268 * @fence: fence to remember 269 * 270 * Fence the vm (cayman+). 271 * Set the fence used to protect page table and id. 272 * 273 * Global and local mutex must be locked! 274 */ 275 void radeon_vm_fence(struct radeon_device *rdev, 276 struct radeon_vm *vm, 277 struct radeon_fence *fence) 278 { 279 unsigned vm_id = vm->ids[fence->ring].id; 280 281 radeon_fence_unref(&rdev->vm_manager.active[vm_id]); 282 rdev->vm_manager.active[vm_id] = radeon_fence_ref(fence); 283 284 radeon_fence_unref(&vm->ids[fence->ring].last_id_use); 285 vm->ids[fence->ring].last_id_use = radeon_fence_ref(fence); 286 } 287 288 /** 289 * radeon_vm_bo_find - find the bo_va for a specific vm & bo 290 * 291 * @vm: requested vm 292 * @bo: requested buffer object 293 * 294 * Find @bo inside the requested vm (cayman+). 295 * Search inside the @bos vm list for the requested vm 296 * Returns the found bo_va or NULL if none is found 297 * 298 * Object has to be reserved! 299 */ 300 struct radeon_bo_va *radeon_vm_bo_find(struct radeon_vm *vm, 301 struct radeon_bo *bo) 302 { 303 struct radeon_bo_va *bo_va; 304 305 list_for_each_entry(bo_va, &bo->va, bo_list) { 306 if (bo_va->vm == vm) 307 return bo_va; 308 309 } 310 return NULL; 311 } 312 313 /** 314 * radeon_vm_bo_add - add a bo to a specific vm 315 * 316 * @rdev: radeon_device pointer 317 * @vm: requested vm 318 * @bo: radeon buffer object 319 * 320 * Add @bo into the requested vm (cayman+). 321 * Add @bo to the list of bos associated with the vm 322 * Returns newly added bo_va or NULL for failure 323 * 324 * Object has to be reserved! 325 */ 326 struct radeon_bo_va *radeon_vm_bo_add(struct radeon_device *rdev, 327 struct radeon_vm *vm, 328 struct radeon_bo *bo) 329 { 330 struct radeon_bo_va *bo_va; 331 332 bo_va = kzalloc(sizeof(struct radeon_bo_va), GFP_KERNEL); 333 if (bo_va == NULL) 334 return NULL; 335 336 bo_va->vm = vm; 337 bo_va->bo = bo; 338 bo_va->it.start = 0; 339 bo_va->it.last = 0; 340 bo_va->flags = 0; 341 bo_va->ref_count = 1; 342 INIT_LIST_HEAD(&bo_va->bo_list); 343 INIT_LIST_HEAD(&bo_va->vm_status); 344 345 mutex_lock(&vm->mutex); 346 list_add_tail(&bo_va->bo_list, &bo->va); 347 mutex_unlock(&vm->mutex); 348 349 return bo_va; 350 } 351 352 /** 353 * radeon_vm_set_pages - helper to call the right asic function 354 * 355 * @rdev: radeon_device pointer 356 * @ib: indirect buffer to fill with commands 357 * @pe: addr of the page entry 358 * @addr: dst addr to write into pe 359 * @count: number of page entries to update 360 * @incr: increase next addr by incr bytes 361 * @flags: hw access flags 362 * 363 * Traces the parameters and calls the right asic functions 364 * to setup the page table using the DMA. 365 */ 366 static void radeon_vm_set_pages(struct radeon_device *rdev, 367 struct radeon_ib *ib, 368 uint64_t pe, 369 uint64_t addr, unsigned count, 370 uint32_t incr, uint32_t flags) 371 { 372 trace_radeon_vm_set_page(pe, addr, count, incr, flags); 373 374 if ((flags & R600_PTE_GART_MASK) == R600_PTE_GART_MASK) { 375 uint64_t src = rdev->gart.table_addr + (addr >> 12) * 8; 376 radeon_asic_vm_copy_pages(rdev, ib, pe, src, count); 377 378 } else if ((flags & R600_PTE_SYSTEM) || (count < 3)) { 379 radeon_asic_vm_write_pages(rdev, ib, pe, addr, 380 count, incr, flags); 381 382 } else { 383 radeon_asic_vm_set_pages(rdev, ib, pe, addr, 384 count, incr, flags); 385 } 386 } 387 388 /** 389 * radeon_vm_clear_bo - initially clear the page dir/table 390 * 391 * @rdev: radeon_device pointer 392 * @bo: bo to clear 393 */ 394 static int radeon_vm_clear_bo(struct radeon_device *rdev, 395 struct radeon_bo *bo) 396 { 397 struct ttm_operation_ctx ctx = { true, false }; 398 struct radeon_ib ib; 399 unsigned entries; 400 uint64_t addr; 401 int r; 402 403 r = radeon_bo_reserve(bo, false); 404 if (r) 405 return r; 406 407 r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); 408 if (r) 409 goto error_unreserve; 410 411 addr = radeon_bo_gpu_offset(bo); 412 entries = radeon_bo_size(bo) / 8; 413 414 r = radeon_ib_get(rdev, R600_RING_TYPE_DMA_INDEX, &ib, NULL, 256); 415 if (r) 416 goto error_unreserve; 417 418 ib.length_dw = 0; 419 420 radeon_vm_set_pages(rdev, &ib, addr, 0, entries, 0, 0); 421 radeon_asic_vm_pad_ib(rdev, &ib); 422 WARN_ON(ib.length_dw > 64); 423 424 r = radeon_ib_schedule(rdev, &ib, NULL, false); 425 if (r) 426 goto error_free; 427 428 ib.fence->is_vm_update = true; 429 radeon_bo_fence(bo, ib.fence, false); 430 431 error_free: 432 radeon_ib_free(rdev, &ib); 433 434 error_unreserve: 435 radeon_bo_unreserve(bo); 436 return r; 437 } 438 439 /** 440 * radeon_vm_bo_set_addr - set bos virtual address inside a vm 441 * 442 * @rdev: radeon_device pointer 443 * @bo_va: bo_va to store the address 444 * @soffset: requested offset of the buffer in the VM address space 445 * @flags: attributes of pages (read/write/valid/etc.) 446 * 447 * Set offset of @bo_va (cayman+). 448 * Validate and set the offset requested within the vm address space. 449 * Returns 0 for success, error for failure. 450 * 451 * Object has to be reserved and gets unreserved by this function! 452 */ 453 int radeon_vm_bo_set_addr(struct radeon_device *rdev, 454 struct radeon_bo_va *bo_va, 455 uint64_t soffset, 456 uint32_t flags) 457 { 458 uint64_t size = radeon_bo_size(bo_va->bo); 459 struct radeon_vm *vm = bo_va->vm; 460 unsigned last_pfn, pt_idx; 461 uint64_t eoffset; 462 int r; 463 464 if (soffset) { 465 /* make sure object fit at this offset */ 466 eoffset = soffset + size - 1; 467 if (soffset >= eoffset) { 468 r = -EINVAL; 469 goto error_unreserve; 470 } 471 472 last_pfn = eoffset / RADEON_GPU_PAGE_SIZE; 473 if (last_pfn >= rdev->vm_manager.max_pfn) { 474 dev_err(rdev->dev, "va above limit (0x%08X >= 0x%08X)\n", 475 last_pfn, rdev->vm_manager.max_pfn); 476 r = -EINVAL; 477 goto error_unreserve; 478 } 479 480 } else { 481 eoffset = last_pfn = 0; 482 } 483 484 mutex_lock(&vm->mutex); 485 soffset /= RADEON_GPU_PAGE_SIZE; 486 eoffset /= RADEON_GPU_PAGE_SIZE; 487 if (soffset || eoffset) { 488 struct interval_tree_node *it; 489 it = interval_tree_iter_first(&vm->va, soffset, eoffset); 490 if (it && it != &bo_va->it) { 491 struct radeon_bo_va *tmp; 492 tmp = container_of(it, struct radeon_bo_va, it); 493 /* bo and tmp overlap, invalid offset */ 494 dev_err(rdev->dev, "bo %p va 0x%010"PRIx64" conflict with " 495 "(bo %p 0x%010lx 0x%010lx)\n", bo_va->bo, 496 soffset, tmp->bo, tmp->it.start, tmp->it.last); 497 mutex_unlock(&vm->mutex); 498 r = -EINVAL; 499 goto error_unreserve; 500 } 501 } 502 503 if (bo_va->it.start || bo_va->it.last) { 504 /* add a clone of the bo_va to clear the old address */ 505 struct radeon_bo_va *tmp; 506 tmp = kzalloc(sizeof(struct radeon_bo_va), GFP_KERNEL); 507 if (!tmp) { 508 mutex_unlock(&vm->mutex); 509 r = -ENOMEM; 510 goto error_unreserve; 511 } 512 tmp->it.start = bo_va->it.start; 513 tmp->it.last = bo_va->it.last; 514 tmp->vm = vm; 515 tmp->bo = radeon_bo_ref(bo_va->bo); 516 517 interval_tree_remove(&bo_va->it, &vm->va); 518 spin_lock(&vm->status_lock); 519 bo_va->it.start = 0; 520 bo_va->it.last = 0; 521 list_del_init(&bo_va->vm_status); 522 list_add(&tmp->vm_status, &vm->freed); 523 spin_unlock(&vm->status_lock); 524 } 525 526 if (soffset || eoffset) { 527 spin_lock(&vm->status_lock); 528 bo_va->it.start = soffset; 529 bo_va->it.last = eoffset; 530 list_add(&bo_va->vm_status, &vm->cleared); 531 spin_unlock(&vm->status_lock); 532 interval_tree_insert(&bo_va->it, &vm->va); 533 } 534 535 bo_va->flags = flags; 536 537 soffset >>= radeon_vm_block_size; 538 eoffset >>= radeon_vm_block_size; 539 540 BUG_ON(eoffset >= radeon_vm_num_pdes(rdev)); 541 542 if (eoffset > vm->max_pde_used) 543 vm->max_pde_used = eoffset; 544 545 radeon_bo_unreserve(bo_va->bo); 546 547 /* walk over the address space and allocate the page tables */ 548 for (pt_idx = soffset; pt_idx <= eoffset; ++pt_idx) { 549 struct radeon_bo *pt; 550 551 if (vm->page_tables[pt_idx].bo) 552 continue; 553 554 /* drop mutex to allocate and clear page table */ 555 mutex_unlock(&vm->mutex); 556 557 r = radeon_bo_create(rdev, RADEON_VM_PTE_COUNT * 8, 558 RADEON_GPU_PAGE_SIZE, true, 559 RADEON_GEM_DOMAIN_VRAM, 0, 560 NULL, NULL, &pt); 561 if (r) 562 return r; 563 564 r = radeon_vm_clear_bo(rdev, pt); 565 if (r) { 566 radeon_bo_unref(&pt); 567 return r; 568 } 569 570 /* aquire mutex again */ 571 mutex_lock(&vm->mutex); 572 if (vm->page_tables[pt_idx].bo) { 573 /* someone else allocated the pt in the meantime */ 574 mutex_unlock(&vm->mutex); 575 radeon_bo_unref(&pt); 576 mutex_lock(&vm->mutex); 577 continue; 578 } 579 580 vm->page_tables[pt_idx].addr = 0; 581 vm->page_tables[pt_idx].bo = pt; 582 } 583 584 mutex_unlock(&vm->mutex); 585 return 0; 586 587 error_unreserve: 588 radeon_bo_unreserve(bo_va->bo); 589 return r; 590 } 591 592 /** 593 * radeon_vm_map_gart - get the physical address of a gart page 594 * 595 * @rdev: radeon_device pointer 596 * @addr: the unmapped addr 597 * 598 * Look up the physical address of the page that the pte resolves 599 * to (cayman+). 600 * Returns the physical address of the page. 601 */ 602 uint64_t radeon_vm_map_gart(struct radeon_device *rdev, uint64_t addr) 603 { 604 uint64_t result; 605 606 /* page table offset */ 607 result = rdev->gart.pages_entry[addr >> RADEON_GPU_PAGE_SHIFT]; 608 result &= ~RADEON_GPU_PAGE_MASK; 609 610 return result; 611 } 612 613 /** 614 * radeon_vm_page_flags - translate page flags to what the hw uses 615 * 616 * @flags: flags comming from userspace 617 * 618 * Translate the flags the userspace ABI uses to hw flags. 619 */ 620 static uint32_t radeon_vm_page_flags(uint32_t flags) 621 { 622 uint32_t hw_flags = 0; 623 624 hw_flags |= (flags & RADEON_VM_PAGE_VALID) ? R600_PTE_VALID : 0; 625 hw_flags |= (flags & RADEON_VM_PAGE_READABLE) ? R600_PTE_READABLE : 0; 626 hw_flags |= (flags & RADEON_VM_PAGE_WRITEABLE) ? R600_PTE_WRITEABLE : 0; 627 if (flags & RADEON_VM_PAGE_SYSTEM) { 628 hw_flags |= R600_PTE_SYSTEM; 629 hw_flags |= (flags & RADEON_VM_PAGE_SNOOPED) ? R600_PTE_SNOOPED : 0; 630 } 631 return hw_flags; 632 } 633 634 /** 635 * radeon_vm_update_pdes - make sure that page directory is valid 636 * 637 * @rdev: radeon_device pointer 638 * @vm: requested vm 639 * @start: start of GPU address range 640 * @end: end of GPU address range 641 * 642 * Allocates new page tables if necessary 643 * and updates the page directory (cayman+). 644 * Returns 0 for success, error for failure. 645 * 646 * Global and local mutex must be locked! 647 */ 648 int radeon_vm_update_page_directory(struct radeon_device *rdev, 649 struct radeon_vm *vm) 650 { 651 struct radeon_bo *pd = vm->page_directory; 652 uint64_t pd_addr = radeon_bo_gpu_offset(pd); 653 uint32_t incr = RADEON_VM_PTE_COUNT * 8; 654 uint64_t last_pde = ~0, last_pt = ~0; 655 unsigned count = 0, pt_idx, ndw; 656 struct radeon_ib ib; 657 int r; 658 659 /* padding, etc. */ 660 ndw = 64; 661 662 /* assume the worst case */ 663 ndw += vm->max_pde_used * 6; 664 665 /* update too big for an IB */ 666 if (ndw > 0xfffff) 667 return -ENOMEM; 668 669 r = radeon_ib_get(rdev, R600_RING_TYPE_DMA_INDEX, &ib, NULL, ndw * 4); 670 if (r) 671 return r; 672 ib.length_dw = 0; 673 674 /* walk over the address space and update the page directory */ 675 for (pt_idx = 0; pt_idx <= vm->max_pde_used; ++pt_idx) { 676 struct radeon_bo *bo = vm->page_tables[pt_idx].bo; 677 uint64_t pde, pt; 678 679 if (bo == NULL) 680 continue; 681 682 pt = radeon_bo_gpu_offset(bo); 683 if (vm->page_tables[pt_idx].addr == pt) 684 continue; 685 vm->page_tables[pt_idx].addr = pt; 686 687 pde = pd_addr + pt_idx * 8; 688 if (((last_pde + 8 * count) != pde) || 689 ((last_pt + incr * count) != pt)) { 690 691 if (count) { 692 radeon_vm_set_pages(rdev, &ib, last_pde, 693 last_pt, count, incr, 694 R600_PTE_VALID); 695 } 696 697 count = 1; 698 last_pde = pde; 699 last_pt = pt; 700 } else { 701 ++count; 702 } 703 } 704 705 if (count) 706 radeon_vm_set_pages(rdev, &ib, last_pde, last_pt, count, 707 incr, R600_PTE_VALID); 708 709 if (ib.length_dw != 0) { 710 radeon_asic_vm_pad_ib(rdev, &ib); 711 712 radeon_sync_resv(rdev, &ib.sync, pd->tbo.base.resv, true); 713 WARN_ON(ib.length_dw > ndw); 714 r = radeon_ib_schedule(rdev, &ib, NULL, false); 715 if (r) { 716 radeon_ib_free(rdev, &ib); 717 return r; 718 } 719 ib.fence->is_vm_update = true; 720 radeon_bo_fence(pd, ib.fence, false); 721 } 722 radeon_ib_free(rdev, &ib); 723 724 return 0; 725 } 726 727 /** 728 * radeon_vm_frag_ptes - add fragment information to PTEs 729 * 730 * @rdev: radeon_device pointer 731 * @ib: IB for the update 732 * @pe_start: first PTE to handle 733 * @pe_end: last PTE to handle 734 * @addr: addr those PTEs should point to 735 * @flags: hw mapping flags 736 * 737 * Global and local mutex must be locked! 738 */ 739 static void radeon_vm_frag_ptes(struct radeon_device *rdev, 740 struct radeon_ib *ib, 741 uint64_t pe_start, uint64_t pe_end, 742 uint64_t addr, uint32_t flags) 743 { 744 /** 745 * The MC L1 TLB supports variable sized pages, based on a fragment 746 * field in the PTE. When this field is set to a non-zero value, page 747 * granularity is increased from 4KB to (1 << (12 + frag)). The PTE 748 * flags are considered valid for all PTEs within the fragment range 749 * and corresponding mappings are assumed to be physically contiguous. 750 * 751 * The L1 TLB can store a single PTE for the whole fragment, 752 * significantly increasing the space available for translation 753 * caching. This leads to large improvements in throughput when the 754 * TLB is under pressure. 755 * 756 * The L2 TLB distributes small and large fragments into two 757 * asymmetric partitions. The large fragment cache is significantly 758 * larger. Thus, we try to use large fragments wherever possible. 759 * Userspace can support this by aligning virtual base address and 760 * allocation size to the fragment size. 761 */ 762 763 /* NI is optimized for 256KB fragments, SI and newer for 64KB */ 764 uint64_t frag_flags = ((rdev->family == CHIP_CAYMAN) || 765 (rdev->family == CHIP_ARUBA)) ? 766 R600_PTE_FRAG_256KB : R600_PTE_FRAG_64KB; 767 uint64_t frag_align = ((rdev->family == CHIP_CAYMAN) || 768 (rdev->family == CHIP_ARUBA)) ? 0x200 : 0x80; 769 770 uint64_t frag_start = ALIGN(pe_start, frag_align); 771 uint64_t frag_end = pe_end & ~(frag_align - 1); 772 773 unsigned count; 774 775 /* system pages are non continuously */ 776 if ((flags & R600_PTE_SYSTEM) || !(flags & R600_PTE_VALID) || 777 (frag_start >= frag_end)) { 778 779 count = (pe_end - pe_start) / 8; 780 radeon_vm_set_pages(rdev, ib, pe_start, addr, count, 781 RADEON_GPU_PAGE_SIZE, flags); 782 return; 783 } 784 785 /* handle the 4K area at the beginning */ 786 if (pe_start != frag_start) { 787 count = (frag_start - pe_start) / 8; 788 radeon_vm_set_pages(rdev, ib, pe_start, addr, count, 789 RADEON_GPU_PAGE_SIZE, flags); 790 addr += RADEON_GPU_PAGE_SIZE * count; 791 } 792 793 /* handle the area in the middle */ 794 count = (frag_end - frag_start) / 8; 795 radeon_vm_set_pages(rdev, ib, frag_start, addr, count, 796 RADEON_GPU_PAGE_SIZE, flags | frag_flags); 797 798 /* handle the 4K area at the end */ 799 if (frag_end != pe_end) { 800 addr += RADEON_GPU_PAGE_SIZE * count; 801 count = (pe_end - frag_end) / 8; 802 radeon_vm_set_pages(rdev, ib, frag_end, addr, count, 803 RADEON_GPU_PAGE_SIZE, flags); 804 } 805 } 806 807 /** 808 * radeon_vm_update_ptes - make sure that page tables are valid 809 * 810 * @rdev: radeon_device pointer 811 * @vm: requested vm 812 * @start: start of GPU address range 813 * @end: end of GPU address range 814 * @dst: destination address to map to 815 * @flags: mapping flags 816 * 817 * Update the page tables in the range @start - @end (cayman+). 818 * 819 * Global and local mutex must be locked! 820 */ 821 static int radeon_vm_update_ptes(struct radeon_device *rdev, 822 struct radeon_vm *vm, 823 struct radeon_ib *ib, 824 uint64_t start, uint64_t end, 825 uint64_t dst, uint32_t flags) 826 { 827 uint64_t mask = RADEON_VM_PTE_COUNT - 1; 828 uint64_t last_pte = ~0, last_dst = ~0; 829 unsigned count = 0; 830 uint64_t addr; 831 832 /* walk over the address space and update the page tables */ 833 for (addr = start; addr < end; ) { 834 uint64_t pt_idx = addr >> radeon_vm_block_size; 835 struct radeon_bo *pt = vm->page_tables[pt_idx].bo; 836 unsigned nptes; 837 uint64_t pte; 838 int r; 839 840 radeon_sync_resv(rdev, &ib->sync, pt->tbo.base.resv, true); 841 r = dma_resv_reserve_shared(pt->tbo.base.resv, 1); 842 if (r) 843 return r; 844 845 if ((addr & ~mask) == (end & ~mask)) 846 nptes = end - addr; 847 else 848 nptes = RADEON_VM_PTE_COUNT - (addr & mask); 849 850 pte = radeon_bo_gpu_offset(pt); 851 pte += (addr & mask) * 8; 852 853 if ((last_pte + 8 * count) != pte) { 854 855 if (count) { 856 radeon_vm_frag_ptes(rdev, ib, last_pte, 857 last_pte + 8 * count, 858 last_dst, flags); 859 } 860 861 count = nptes; 862 last_pte = pte; 863 last_dst = dst; 864 } else { 865 count += nptes; 866 } 867 868 addr += nptes; 869 dst += nptes * RADEON_GPU_PAGE_SIZE; 870 } 871 872 if (count) { 873 radeon_vm_frag_ptes(rdev, ib, last_pte, 874 last_pte + 8 * count, 875 last_dst, flags); 876 } 877 878 return 0; 879 } 880 881 /** 882 * radeon_vm_fence_pts - fence page tables after an update 883 * 884 * @vm: requested vm 885 * @start: start of GPU address range 886 * @end: end of GPU address range 887 * @fence: fence to use 888 * 889 * Fence the page tables in the range @start - @end (cayman+). 890 * 891 * Global and local mutex must be locked! 892 */ 893 static void radeon_vm_fence_pts(struct radeon_vm *vm, 894 uint64_t start, uint64_t end, 895 struct radeon_fence *fence) 896 { 897 unsigned i; 898 899 start >>= radeon_vm_block_size; 900 end = (end - 1) >> radeon_vm_block_size; 901 902 for (i = start; i <= end; ++i) 903 radeon_bo_fence(vm->page_tables[i].bo, fence, true); 904 } 905 906 /** 907 * radeon_vm_bo_update - map a bo into the vm page table 908 * 909 * @rdev: radeon_device pointer 910 * @vm: requested vm 911 * @bo: radeon buffer object 912 * @mem: ttm mem 913 * 914 * Fill in the page table entries for @bo (cayman+). 915 * Returns 0 for success, -EINVAL for failure. 916 * 917 * Object have to be reserved and mutex must be locked! 918 */ 919 int radeon_vm_bo_update(struct radeon_device *rdev, 920 struct radeon_bo_va *bo_va, 921 struct ttm_mem_reg *mem) 922 { 923 struct radeon_vm *vm = bo_va->vm; 924 struct radeon_ib ib; 925 unsigned nptes, ncmds, ndw; 926 uint64_t addr; 927 uint32_t flags; 928 int r; 929 930 if (!bo_va->it.start) { 931 dev_err(rdev->dev, "bo %p don't has a mapping in vm %p\n", 932 bo_va->bo, vm); 933 return -EINVAL; 934 } 935 936 spin_lock(&vm->status_lock); 937 if (mem) { 938 if (list_empty(&bo_va->vm_status)) { 939 spin_unlock(&vm->status_lock); 940 return 0; 941 } 942 list_del_init(&bo_va->vm_status); 943 } else { 944 list_del(&bo_va->vm_status); 945 list_add(&bo_va->vm_status, &vm->cleared); 946 } 947 spin_unlock(&vm->status_lock); 948 949 bo_va->flags &= ~RADEON_VM_PAGE_VALID; 950 bo_va->flags &= ~RADEON_VM_PAGE_SYSTEM; 951 bo_va->flags &= ~RADEON_VM_PAGE_SNOOPED; 952 if (bo_va->bo && radeon_ttm_tt_is_readonly(bo_va->bo->tbo.ttm)) 953 bo_va->flags &= ~RADEON_VM_PAGE_WRITEABLE; 954 955 if (mem) { 956 addr = (u64)mem->start << PAGE_SHIFT; 957 if (mem->mem_type != TTM_PL_SYSTEM) 958 bo_va->flags |= RADEON_VM_PAGE_VALID; 959 960 if (mem->mem_type == TTM_PL_TT) { 961 bo_va->flags |= RADEON_VM_PAGE_SYSTEM; 962 if (!(bo_va->bo->flags & (RADEON_GEM_GTT_WC | RADEON_GEM_GTT_UC))) 963 bo_va->flags |= RADEON_VM_PAGE_SNOOPED; 964 965 } else { 966 addr += rdev->vm_manager.vram_base_offset; 967 } 968 } else { 969 addr = 0; 970 } 971 972 trace_radeon_vm_bo_update(bo_va); 973 974 nptes = bo_va->it.last - bo_va->it.start + 1; 975 976 /* reserve space for one command every (1 << BLOCK_SIZE) entries 977 or 2k dwords (whatever is smaller) */ 978 ncmds = (nptes >> min(radeon_vm_block_size, 11)) + 1; 979 980 /* padding, etc. */ 981 ndw = 64; 982 983 flags = radeon_vm_page_flags(bo_va->flags); 984 if ((flags & R600_PTE_GART_MASK) == R600_PTE_GART_MASK) { 985 /* only copy commands needed */ 986 ndw += ncmds * 7; 987 988 } else if (flags & R600_PTE_SYSTEM) { 989 /* header for write data commands */ 990 ndw += ncmds * 4; 991 992 /* body of write data command */ 993 ndw += nptes * 2; 994 995 } else { 996 /* set page commands needed */ 997 ndw += ncmds * 10; 998 999 /* two extra commands for begin/end of fragment */ 1000 ndw += 2 * 10; 1001 } 1002 1003 /* update too big for an IB */ 1004 if (ndw > 0xfffff) 1005 return -ENOMEM; 1006 1007 r = radeon_ib_get(rdev, R600_RING_TYPE_DMA_INDEX, &ib, NULL, ndw * 4); 1008 if (r) 1009 return r; 1010 ib.length_dw = 0; 1011 1012 if (!(bo_va->flags & RADEON_VM_PAGE_VALID)) { 1013 unsigned i; 1014 1015 for (i = 0; i < RADEON_NUM_RINGS; ++i) 1016 radeon_sync_fence(&ib.sync, vm->ids[i].last_id_use); 1017 } 1018 1019 r = radeon_vm_update_ptes(rdev, vm, &ib, bo_va->it.start, 1020 bo_va->it.last + 1, addr, 1021 radeon_vm_page_flags(bo_va->flags)); 1022 if (r) { 1023 radeon_ib_free(rdev, &ib); 1024 return r; 1025 } 1026 1027 radeon_asic_vm_pad_ib(rdev, &ib); 1028 WARN_ON(ib.length_dw > ndw); 1029 1030 r = radeon_ib_schedule(rdev, &ib, NULL, false); 1031 if (r) { 1032 radeon_ib_free(rdev, &ib); 1033 return r; 1034 } 1035 ib.fence->is_vm_update = true; 1036 radeon_vm_fence_pts(vm, bo_va->it.start, bo_va->it.last + 1, ib.fence); 1037 radeon_fence_unref(&bo_va->last_pt_update); 1038 bo_va->last_pt_update = radeon_fence_ref(ib.fence); 1039 radeon_ib_free(rdev, &ib); 1040 1041 return 0; 1042 } 1043 1044 /** 1045 * radeon_vm_clear_freed - clear freed BOs in the PT 1046 * 1047 * @rdev: radeon_device pointer 1048 * @vm: requested vm 1049 * 1050 * Make sure all freed BOs are cleared in the PT. 1051 * Returns 0 for success. 1052 * 1053 * PTs have to be reserved and mutex must be locked! 1054 */ 1055 int radeon_vm_clear_freed(struct radeon_device *rdev, 1056 struct radeon_vm *vm) 1057 { 1058 struct radeon_bo_va *bo_va; 1059 int r = 0; 1060 1061 spin_lock(&vm->status_lock); 1062 while (!list_empty(&vm->freed)) { 1063 bo_va = list_first_entry(&vm->freed, 1064 struct radeon_bo_va, vm_status); 1065 spin_unlock(&vm->status_lock); 1066 1067 r = radeon_vm_bo_update(rdev, bo_va, NULL); 1068 radeon_bo_unref(&bo_va->bo); 1069 radeon_fence_unref(&bo_va->last_pt_update); 1070 spin_lock(&vm->status_lock); 1071 list_del(&bo_va->vm_status); 1072 kfree(bo_va); 1073 if (r) 1074 break; 1075 1076 } 1077 spin_unlock(&vm->status_lock); 1078 return r; 1079 1080 } 1081 1082 /** 1083 * radeon_vm_clear_invalids - clear invalidated BOs in the PT 1084 * 1085 * @rdev: radeon_device pointer 1086 * @vm: requested vm 1087 * 1088 * Make sure all invalidated BOs are cleared in the PT. 1089 * Returns 0 for success. 1090 * 1091 * PTs have to be reserved and mutex must be locked! 1092 */ 1093 int radeon_vm_clear_invalids(struct radeon_device *rdev, 1094 struct radeon_vm *vm) 1095 { 1096 struct radeon_bo_va *bo_va; 1097 int r; 1098 1099 spin_lock(&vm->status_lock); 1100 while (!list_empty(&vm->invalidated)) { 1101 bo_va = list_first_entry(&vm->invalidated, 1102 struct radeon_bo_va, vm_status); 1103 spin_unlock(&vm->status_lock); 1104 1105 r = radeon_vm_bo_update(rdev, bo_va, NULL); 1106 if (r) 1107 return r; 1108 1109 spin_lock(&vm->status_lock); 1110 } 1111 spin_unlock(&vm->status_lock); 1112 1113 return 0; 1114 } 1115 1116 /** 1117 * radeon_vm_bo_rmv - remove a bo to a specific vm 1118 * 1119 * @rdev: radeon_device pointer 1120 * @bo_va: requested bo_va 1121 * 1122 * Remove @bo_va->bo from the requested vm (cayman+). 1123 * 1124 * Object have to be reserved! 1125 */ 1126 void radeon_vm_bo_rmv(struct radeon_device *rdev, 1127 struct radeon_bo_va *bo_va) 1128 { 1129 struct radeon_vm *vm = bo_va->vm; 1130 1131 list_del(&bo_va->bo_list); 1132 1133 mutex_lock(&vm->mutex); 1134 if (bo_va->it.start || bo_va->it.last) 1135 interval_tree_remove(&bo_va->it, &vm->va); 1136 1137 spin_lock(&vm->status_lock); 1138 list_del(&bo_va->vm_status); 1139 if (bo_va->it.start || bo_va->it.last) { 1140 bo_va->bo = radeon_bo_ref(bo_va->bo); 1141 list_add(&bo_va->vm_status, &vm->freed); 1142 } else { 1143 radeon_fence_unref(&bo_va->last_pt_update); 1144 kfree(bo_va); 1145 } 1146 spin_unlock(&vm->status_lock); 1147 1148 mutex_unlock(&vm->mutex); 1149 } 1150 1151 /** 1152 * radeon_vm_bo_invalidate - mark the bo as invalid 1153 * 1154 * @rdev: radeon_device pointer 1155 * @vm: requested vm 1156 * @bo: radeon buffer object 1157 * 1158 * Mark @bo as invalid (cayman+). 1159 */ 1160 void radeon_vm_bo_invalidate(struct radeon_device *rdev, 1161 struct radeon_bo *bo) 1162 { 1163 struct radeon_bo_va *bo_va; 1164 1165 list_for_each_entry(bo_va, &bo->va, bo_list) { 1166 spin_lock(&bo_va->vm->status_lock); 1167 if (list_empty(&bo_va->vm_status) && 1168 (bo_va->it.start || bo_va->it.last)) 1169 list_add(&bo_va->vm_status, &bo_va->vm->invalidated); 1170 spin_unlock(&bo_va->vm->status_lock); 1171 } 1172 } 1173 1174 /** 1175 * radeon_vm_init - initialize a vm instance 1176 * 1177 * @rdev: radeon_device pointer 1178 * @vm: requested vm 1179 * 1180 * Init @vm fields (cayman+). 1181 */ 1182 int radeon_vm_init(struct radeon_device *rdev, struct radeon_vm *vm) 1183 { 1184 const unsigned align = min(RADEON_VM_PTB_ALIGN_SIZE, 1185 RADEON_VM_PTE_COUNT * 8); 1186 unsigned pd_size, pd_entries, pts_size; 1187 int i, r; 1188 1189 vm->ib_bo_va = NULL; 1190 for (i = 0; i < RADEON_NUM_RINGS; ++i) { 1191 vm->ids[i].id = 0; 1192 vm->ids[i].flushed_updates = NULL; 1193 vm->ids[i].last_id_use = NULL; 1194 } 1195 mutex_init(&vm->mutex); 1196 #ifdef __NetBSD__ 1197 interval_tree_init(&vm->va); 1198 #else 1199 vm->va = RB_ROOT_CACHED; 1200 #endif 1201 spin_lock_init(&vm->status_lock); 1202 INIT_LIST_HEAD(&vm->invalidated); 1203 INIT_LIST_HEAD(&vm->freed); 1204 INIT_LIST_HEAD(&vm->cleared); 1205 1206 pd_size = radeon_vm_directory_size(rdev); 1207 pd_entries = radeon_vm_num_pdes(rdev); 1208 1209 /* allocate page table array */ 1210 pts_size = pd_entries * sizeof(struct radeon_vm_pt); 1211 vm->page_tables = kzalloc(pts_size, GFP_KERNEL); 1212 if (vm->page_tables == NULL) { 1213 DRM_ERROR("Cannot allocate memory for page table array\n"); 1214 return -ENOMEM; 1215 } 1216 1217 r = radeon_bo_create(rdev, pd_size, align, true, 1218 RADEON_GEM_DOMAIN_VRAM, 0, NULL, 1219 NULL, &vm->page_directory); 1220 if (r) 1221 return r; 1222 1223 r = radeon_vm_clear_bo(rdev, vm->page_directory); 1224 if (r) { 1225 radeon_bo_unref(&vm->page_directory); 1226 vm->page_directory = NULL; 1227 return r; 1228 } 1229 1230 return 0; 1231 } 1232 1233 /** 1234 * radeon_vm_fini - tear down a vm instance 1235 * 1236 * @rdev: radeon_device pointer 1237 * @vm: requested vm 1238 * 1239 * Tear down @vm (cayman+). 1240 * Unbind the VM and remove all bos from the vm bo list 1241 */ 1242 void radeon_vm_fini(struct radeon_device *rdev, struct radeon_vm *vm) 1243 { 1244 struct radeon_bo_va *bo_va, *tmp; 1245 int i, r; 1246 1247 if (!RB_EMPTY_ROOT(&vm->va.rb_root)) 1248 dev_err(rdev->dev, "still active bo inside vm\n"); 1249 1250 rbtree_postorder_for_each_entry_safe(bo_va, tmp, 1251 &vm->va.rb_root, it.rb) { 1252 interval_tree_remove(&bo_va->it, &vm->va); 1253 r = radeon_bo_reserve(bo_va->bo, false); 1254 if (!r) { 1255 list_del_init(&bo_va->bo_list); 1256 radeon_bo_unreserve(bo_va->bo); 1257 radeon_fence_unref(&bo_va->last_pt_update); 1258 kfree(bo_va); 1259 } 1260 } 1261 list_for_each_entry_safe(bo_va, tmp, &vm->freed, vm_status) { 1262 radeon_bo_unref(&bo_va->bo); 1263 radeon_fence_unref(&bo_va->last_pt_update); 1264 kfree(bo_va); 1265 } 1266 1267 for (i = 0; i < radeon_vm_num_pdes(rdev); i++) 1268 radeon_bo_unref(&vm->page_tables[i].bo); 1269 kfree(vm->page_tables); 1270 1271 radeon_bo_unref(&vm->page_directory); 1272 1273 for (i = 0; i < RADEON_NUM_RINGS; ++i) { 1274 radeon_fence_unref(&vm->ids[i].flushed_updates); 1275 radeon_fence_unref(&vm->ids[i].last_id_use); 1276 } 1277 1278 spin_lock_destroy(&vm->status_lock); 1279 mutex_destroy(&vm->mutex); 1280 } 1281
Indexes created Sat Sep 20 22:09:52 GMT 2025