radeon_ttm.c revision 1.18
1 /* $NetBSD: radeon_ttm.c,v 1.18 2021/12/18 23:45:43 riastradh Exp $ */ 2 3 /* 4 * Copyright 2009 Jerome Glisse. 5 * All Rights Reserved. 6 * 7 * Permission is hereby granted, free of charge, to any person obtaining a 8 * copy of this software and associated documentation files (the 9 * "Software"), to deal in the Software without restriction, including 10 * without limitation the rights to use, copy, modify, merge, publish, 11 * distribute, sub license, and/or sell copies of the Software, and to 12 * permit persons to whom the Software is furnished to do so, subject to 13 * the following conditions: 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL 18 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, 19 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 20 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE 21 * USE OR OTHER DEALINGS IN THE SOFTWARE. 22 * 23 * The above copyright notice and this permission notice (including the 24 * next paragraph) shall be included in all copies or substantial portions 25 * of the Software. 26 * 27 */ 28 /* 29 * Authors: 30 * Jerome Glisse <glisse (at) freedesktop.org> 31 * Thomas Hellstrom <thomas-at-tungstengraphics-dot-com> 32 * Dave Airlie 33 */ 34 35 #include <sys/cdefs.h> 36 __KERNEL_RCSID(0, "$NetBSD: radeon_ttm.c,v 1.18 2021/12/18 23:45:43 riastradh Exp $"); 37 38 #include <linux/dma-mapping.h> 39 #include <linux/pagemap.h> 40 #include <linux/pci.h> 41 #include <linux/seq_file.h> 42 #include <linux/slab.h> 43 #include <linux/swap.h> 44 #include <linux/swiotlb.h> 45 46 #include <drm/drm_agpsupport.h> 47 #include <drm/drm_debugfs.h> 48 #include <drm/drm_device.h> 49 #include <drm/drm_file.h> 50 #include <drm/drm_prime.h> 51 #include <drm/radeon_drm.h> 52 #include <drm/ttm/ttm_bo_api.h> 53 #include <drm/ttm/ttm_bo_driver.h> 54 #include <drm/ttm/ttm_module.h> 55 #include <drm/ttm/ttm_page_alloc.h> 56 #include <drm/ttm/ttm_placement.h> 57 58 #include "radeon_reg.h" 59 #include "radeon.h" 60 61 #ifdef __NetBSD__ 62 #include <uvm/uvm_extern.h> 63 #include <uvm/uvm_fault.h> 64 #include <uvm/uvm_param.h> 65 #include <drm/bus_dma_hacks.h> 66 #endif 67 68 #ifdef _LP64 69 #define DRM_FILE_PAGE_OFFSET (0x100000000ULL >> PAGE_SHIFT) 70 #else 71 #define DRM_FILE_PAGE_OFFSET (0xa0000000UL >> PAGE_SHIFT) 72 #endif 73 74 static int radeon_ttm_debugfs_init(struct radeon_device *rdev); 75 static void radeon_ttm_debugfs_fini(struct radeon_device *rdev); 76 77 static struct radeon_device *radeon_get_rdev(struct ttm_bo_device *bdev) 78 { 79 struct radeon_mman *mman; 80 struct radeon_device *rdev; 81 82 mman = container_of(bdev, struct radeon_mman, bdev); 83 rdev = container_of(mman, struct radeon_device, mman); 84 return rdev; 85 } 86 87 static int radeon_invalidate_caches(struct ttm_bo_device *bdev, uint32_t flags) 88 { 89 return 0; 90 } 91 92 static int radeon_init_mem_type(struct ttm_bo_device *bdev, uint32_t type, 93 struct ttm_mem_type_manager *man) 94 { 95 struct radeon_device *rdev; 96 97 rdev = radeon_get_rdev(bdev); 98 99 switch (type) { 100 case TTM_PL_SYSTEM: 101 /* System memory */ 102 man->flags = TTM_MEMTYPE_FLAG_MAPPABLE; 103 man->available_caching = TTM_PL_MASK_CACHING; 104 man->default_caching = TTM_PL_FLAG_CACHED; 105 break; 106 case TTM_PL_TT: 107 man->func = &ttm_bo_manager_func; 108 man->gpu_offset = rdev->mc.gtt_start; 109 man->available_caching = TTM_PL_MASK_CACHING; 110 man->default_caching = TTM_PL_FLAG_CACHED; 111 man->flags = TTM_MEMTYPE_FLAG_MAPPABLE | TTM_MEMTYPE_FLAG_CMA; 112 #if IS_ENABLED(CONFIG_AGP) 113 if (rdev->flags & RADEON_IS_AGP) { 114 if (!rdev->ddev->agp) { 115 DRM_ERROR("AGP is not enabled for memory type %u\n", 116 (unsigned)type); 117 return -EINVAL; 118 } 119 if (!rdev->ddev->agp->cant_use_aperture) 120 man->flags = TTM_MEMTYPE_FLAG_MAPPABLE; 121 man->available_caching = TTM_PL_FLAG_UNCACHED | 122 TTM_PL_FLAG_WC; 123 man->default_caching = TTM_PL_FLAG_WC; 124 } 125 #endif 126 break; 127 case TTM_PL_VRAM: 128 /* "On-card" video ram */ 129 man->func = &ttm_bo_manager_func; 130 man->gpu_offset = rdev->mc.vram_start; 131 man->flags = TTM_MEMTYPE_FLAG_FIXED | 132 TTM_MEMTYPE_FLAG_MAPPABLE; 133 man->available_caching = TTM_PL_FLAG_UNCACHED | TTM_PL_FLAG_WC; 134 man->default_caching = TTM_PL_FLAG_WC; 135 break; 136 default: 137 DRM_ERROR("Unsupported memory type %u\n", (unsigned)type); 138 return -EINVAL; 139 } 140 return 0; 141 } 142 143 static void radeon_evict_flags(struct ttm_buffer_object *bo, 144 struct ttm_placement *placement) 145 { 146 static const struct ttm_place placements = { 147 .fpfn = 0, 148 .lpfn = 0, 149 .flags = TTM_PL_MASK_CACHING | TTM_PL_FLAG_SYSTEM 150 }; 151 152 struct radeon_bo *rbo; 153 154 if (!radeon_ttm_bo_is_radeon_bo(bo)) { 155 placement->placement = &placements; 156 placement->busy_placement = &placements; 157 placement->num_placement = 1; 158 placement->num_busy_placement = 1; 159 return; 160 } 161 rbo = container_of(bo, struct radeon_bo, tbo); 162 switch (bo->mem.mem_type) { 163 case TTM_PL_VRAM: 164 if (rbo->rdev->ring[radeon_copy_ring_index(rbo->rdev)].ready == false) 165 radeon_ttm_placement_from_domain(rbo, RADEON_GEM_DOMAIN_CPU); 166 else if (rbo->rdev->mc.visible_vram_size < rbo->rdev->mc.real_vram_size && 167 bo->mem.start < (rbo->rdev->mc.visible_vram_size >> PAGE_SHIFT)) { 168 unsigned fpfn = rbo->rdev->mc.visible_vram_size >> PAGE_SHIFT; 169 int i; 170 171 /* Try evicting to the CPU inaccessible part of VRAM 172 * first, but only set GTT as busy placement, so this 173 * BO will be evicted to GTT rather than causing other 174 * BOs to be evicted from VRAM 175 */ 176 radeon_ttm_placement_from_domain(rbo, RADEON_GEM_DOMAIN_VRAM | 177 RADEON_GEM_DOMAIN_GTT); 178 rbo->placement.num_busy_placement = 0; 179 for (i = 0; i < rbo->placement.num_placement; i++) { 180 if (rbo->placements[i].flags & TTM_PL_FLAG_VRAM) { 181 if (rbo->placements[i].fpfn < fpfn) 182 rbo->placements[i].fpfn = fpfn; 183 } else { 184 rbo->placement.busy_placement = 185 &rbo->placements[i]; 186 rbo->placement.num_busy_placement = 1; 187 } 188 } 189 } else 190 radeon_ttm_placement_from_domain(rbo, RADEON_GEM_DOMAIN_GTT); 191 break; 192 case TTM_PL_TT: 193 default: 194 radeon_ttm_placement_from_domain(rbo, RADEON_GEM_DOMAIN_CPU); 195 } 196 *placement = rbo->placement; 197 } 198 199 static int radeon_verify_access(struct ttm_buffer_object *bo, struct file *filp) 200 { 201 struct radeon_bo *rbo = container_of(bo, struct radeon_bo, tbo); 202 203 if (radeon_ttm_tt_has_userptr(bo->ttm)) 204 return -EPERM; 205 return drm_vma_node_verify_access(&rbo->tbo.base.vma_node, 206 filp->private_data); 207 } 208 209 static void radeon_move_null(struct ttm_buffer_object *bo, 210 struct ttm_mem_reg *new_mem) 211 { 212 struct ttm_mem_reg *old_mem = &bo->mem; 213 214 BUG_ON(old_mem->mm_node != NULL); 215 *old_mem = *new_mem; 216 new_mem->mm_node = NULL; 217 } 218 219 static int radeon_move_blit(struct ttm_buffer_object *bo, 220 bool evict, bool no_wait_gpu, 221 struct ttm_mem_reg *new_mem, 222 struct ttm_mem_reg *old_mem) 223 { 224 struct radeon_device *rdev; 225 uint64_t old_start, new_start; 226 struct radeon_fence *fence; 227 unsigned num_pages; 228 int r, ridx; 229 230 rdev = radeon_get_rdev(bo->bdev); 231 ridx = radeon_copy_ring_index(rdev); 232 old_start = (u64)old_mem->start << PAGE_SHIFT; 233 new_start = (u64)new_mem->start << PAGE_SHIFT; 234 235 switch (old_mem->mem_type) { 236 case TTM_PL_VRAM: 237 old_start += rdev->mc.vram_start; 238 break; 239 case TTM_PL_TT: 240 old_start += rdev->mc.gtt_start; 241 break; 242 default: 243 DRM_ERROR("Unknown placement %d\n", old_mem->mem_type); 244 return -EINVAL; 245 } 246 switch (new_mem->mem_type) { 247 case TTM_PL_VRAM: 248 new_start += rdev->mc.vram_start; 249 break; 250 case TTM_PL_TT: 251 new_start += rdev->mc.gtt_start; 252 break; 253 default: 254 DRM_ERROR("Unknown placement %d\n", old_mem->mem_type); 255 return -EINVAL; 256 } 257 if (!rdev->ring[ridx].ready) { 258 DRM_ERROR("Trying to move memory with ring turned off.\n"); 259 return -EINVAL; 260 } 261 262 BUILD_BUG_ON((PAGE_SIZE % RADEON_GPU_PAGE_SIZE) != 0); 263 264 num_pages = new_mem->num_pages * (PAGE_SIZE / RADEON_GPU_PAGE_SIZE); 265 fence = radeon_copy(rdev, old_start, new_start, num_pages, bo->base.resv); 266 if (IS_ERR(fence)) 267 return PTR_ERR(fence); 268 269 r = ttm_bo_move_accel_cleanup(bo, &fence->base, evict, new_mem); 270 radeon_fence_unref(&fence); 271 return r; 272 } 273 274 static int radeon_move_vram_ram(struct ttm_buffer_object *bo, 275 bool evict, bool interruptible, 276 bool no_wait_gpu, 277 struct ttm_mem_reg *new_mem) 278 { 279 struct ttm_operation_ctx ctx = { interruptible, no_wait_gpu }; 280 struct ttm_mem_reg *old_mem = &bo->mem; 281 struct ttm_mem_reg tmp_mem; 282 struct ttm_place placements; 283 struct ttm_placement placement; 284 int r; 285 286 tmp_mem = *new_mem; 287 tmp_mem.mm_node = NULL; 288 placement.num_placement = 1; 289 placement.placement = &placements; 290 placement.num_busy_placement = 1; 291 placement.busy_placement = &placements; 292 placements.fpfn = 0; 293 placements.lpfn = 0; 294 placements.flags = TTM_PL_MASK_CACHING | TTM_PL_FLAG_TT; 295 r = ttm_bo_mem_space(bo, &placement, &tmp_mem, &ctx); 296 if (unlikely(r)) { 297 return r; 298 } 299 300 r = ttm_tt_set_placement_caching(bo->ttm, tmp_mem.placement); 301 if (unlikely(r)) { 302 goto out_cleanup; 303 } 304 305 r = ttm_tt_bind(bo->ttm, &tmp_mem, &ctx); 306 if (unlikely(r)) { 307 goto out_cleanup; 308 } 309 r = radeon_move_blit(bo, true, no_wait_gpu, &tmp_mem, old_mem); 310 if (unlikely(r)) { 311 goto out_cleanup; 312 } 313 r = ttm_bo_move_ttm(bo, &ctx, new_mem); 314 out_cleanup: 315 ttm_bo_mem_put(bo, &tmp_mem); 316 return r; 317 } 318 319 static int radeon_move_ram_vram(struct ttm_buffer_object *bo, 320 bool evict, bool interruptible, 321 bool no_wait_gpu, 322 struct ttm_mem_reg *new_mem) 323 { 324 struct ttm_operation_ctx ctx = { interruptible, no_wait_gpu }; 325 struct ttm_mem_reg *old_mem = &bo->mem; 326 struct ttm_mem_reg tmp_mem; 327 struct ttm_placement placement; 328 struct ttm_place placements; 329 int r; 330 331 tmp_mem = *new_mem; 332 tmp_mem.mm_node = NULL; 333 placement.num_placement = 1; 334 placement.placement = &placements; 335 placement.num_busy_placement = 1; 336 placement.busy_placement = &placements; 337 placements.fpfn = 0; 338 placements.lpfn = 0; 339 placements.flags = TTM_PL_MASK_CACHING | TTM_PL_FLAG_TT; 340 r = ttm_bo_mem_space(bo, &placement, &tmp_mem, &ctx); 341 if (unlikely(r)) { 342 return r; 343 } 344 r = ttm_bo_move_ttm(bo, &ctx, &tmp_mem); 345 if (unlikely(r)) { 346 goto out_cleanup; 347 } 348 r = radeon_move_blit(bo, true, no_wait_gpu, new_mem, old_mem); 349 if (unlikely(r)) { 350 goto out_cleanup; 351 } 352 out_cleanup: 353 ttm_bo_mem_put(bo, &tmp_mem); 354 return r; 355 } 356 357 static int radeon_bo_move(struct ttm_buffer_object *bo, bool evict, 358 struct ttm_operation_ctx *ctx, 359 struct ttm_mem_reg *new_mem) 360 { 361 struct radeon_device *rdev; 362 struct radeon_bo *rbo; 363 struct ttm_mem_reg *old_mem = &bo->mem; 364 int r; 365 366 r = ttm_bo_wait(bo, ctx->interruptible, ctx->no_wait_gpu); 367 if (r) 368 return r; 369 370 /* Can't move a pinned BO */ 371 rbo = container_of(bo, struct radeon_bo, tbo); 372 if (WARN_ON_ONCE(rbo->pin_count > 0)) 373 return -EINVAL; 374 375 rdev = radeon_get_rdev(bo->bdev); 376 if (old_mem->mem_type == TTM_PL_SYSTEM && bo->ttm == NULL) { 377 radeon_move_null(bo, new_mem); 378 return 0; 379 } 380 if ((old_mem->mem_type == TTM_PL_TT && 381 new_mem->mem_type == TTM_PL_SYSTEM) || 382 (old_mem->mem_type == TTM_PL_SYSTEM && 383 new_mem->mem_type == TTM_PL_TT)) { 384 /* bind is enough */ 385 radeon_move_null(bo, new_mem); 386 return 0; 387 } 388 if (!rdev->ring[radeon_copy_ring_index(rdev)].ready || 389 rdev->asic->copy.copy == NULL) { 390 /* use memcpy */ 391 goto memcpy; 392 } 393 394 if (old_mem->mem_type == TTM_PL_VRAM && 395 new_mem->mem_type == TTM_PL_SYSTEM) { 396 r = radeon_move_vram_ram(bo, evict, ctx->interruptible, 397 ctx->no_wait_gpu, new_mem); 398 } else if (old_mem->mem_type == TTM_PL_SYSTEM && 399 new_mem->mem_type == TTM_PL_VRAM) { 400 r = radeon_move_ram_vram(bo, evict, ctx->interruptible, 401 ctx->no_wait_gpu, new_mem); 402 } else { 403 r = radeon_move_blit(bo, evict, ctx->no_wait_gpu, 404 new_mem, old_mem); 405 } 406 407 if (r) { 408 memcpy: 409 r = ttm_bo_move_memcpy(bo, ctx, new_mem); 410 if (r) { 411 return r; 412 } 413 } 414 415 /* update statistics */ 416 atomic64_add((u64)bo->num_pages << PAGE_SHIFT, &rdev->num_bytes_moved); 417 return 0; 418 } 419 420 static int radeon_ttm_io_mem_reserve(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem) 421 { 422 struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type]; 423 struct radeon_device *rdev = radeon_get_rdev(bdev); 424 425 mem->bus.addr = NULL; 426 mem->bus.offset = 0; 427 mem->bus.size = mem->num_pages << PAGE_SHIFT; 428 mem->bus.base = 0; 429 mem->bus.is_iomem = false; 430 if (!(man->flags & TTM_MEMTYPE_FLAG_MAPPABLE)) 431 return -EINVAL; 432 switch (mem->mem_type) { 433 case TTM_PL_SYSTEM: 434 /* system memory */ 435 return 0; 436 case TTM_PL_TT: 437 #if IS_ENABLED(CONFIG_AGP) 438 if (rdev->flags & RADEON_IS_AGP) { 439 /* RADEON_IS_AGP is set only if AGP is active */ 440 mem->bus.offset = mem->start << PAGE_SHIFT; 441 mem->bus.base = rdev->mc.agp_base; 442 mem->bus.is_iomem = !rdev->ddev->agp->cant_use_aperture; 443 KASSERTMSG((mem->bus.base & (PAGE_SIZE - 1)) == 0, 444 "agp aperture is not page-aligned: %lx", 445 mem->bus.base); 446 KASSERT((mem->bus.offset & (PAGE_SIZE - 1)) == 0); 447 } 448 #endif 449 break; 450 case TTM_PL_VRAM: 451 mem->bus.offset = mem->start << PAGE_SHIFT; 452 /* check if it's visible */ 453 if ((mem->bus.offset + mem->bus.size) > rdev->mc.visible_vram_size) 454 return -EINVAL; 455 mem->bus.base = rdev->mc.aper_base; 456 mem->bus.is_iomem = true; 457 #ifdef __alpha__ 458 /* 459 * Alpha: use bus.addr to hold the ioremap() return, 460 * so we can modify bus.base below. 461 */ 462 if (mem->placement & TTM_PL_FLAG_WC) 463 mem->bus.addr = 464 ioremap_wc(mem->bus.base + mem->bus.offset, 465 mem->bus.size); 466 else 467 mem->bus.addr = 468 ioremap(mem->bus.base + mem->bus.offset, 469 mem->bus.size); 470 if (!mem->bus.addr) 471 return -ENOMEM; 472 473 /* 474 * Alpha: Use just the bus offset plus 475 * the hose/domain memory base for bus.base. 476 * It then can be used to build PTEs for VRAM 477 * access, as done in ttm_bo_vm_fault(). 478 */ 479 mem->bus.base = (mem->bus.base & 0x0ffffffffUL) + 480 rdev->ddev->hose->dense_mem_base; 481 #endif 482 KASSERTMSG((mem->bus.base & (PAGE_SIZE - 1)) == 0, 483 "mc aperture is not page-aligned: %lx", 484 mem->bus.base); 485 KASSERT((mem->bus.offset & (PAGE_SIZE - 1)) == 0); 486 break; 487 default: 488 return -EINVAL; 489 } 490 return 0; 491 } 492 493 static void radeon_ttm_io_mem_free(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem) 494 { 495 } 496 497 /* 498 * TTM backend functions. 499 */ 500 struct radeon_ttm_tt { 501 struct ttm_dma_tt ttm; 502 struct radeon_device *rdev; 503 u64 offset; 504 505 uint64_t userptr; 506 #ifdef __NetBSD__ 507 struct vmspace *usermm; 508 #else 509 struct mm_struct *usermm; 510 #endif 511 uint32_t userflags; 512 }; 513 514 /* prepare the sg table with the user pages */ 515 static int radeon_ttm_tt_pin_userptr(struct ttm_tt *ttm) 516 { 517 struct radeon_device *rdev = radeon_get_rdev(ttm->bdev); 518 struct radeon_ttm_tt *gtt = (void *)ttm; 519 #ifndef __NetBSD__ 520 unsigned pinned = 0, nents; 521 #endif 522 int r; 523 524 int write = !(gtt->userflags & RADEON_GEM_USERPTR_READONLY); 525 #ifndef __NetBSD__ 526 enum dma_data_direction direction = write ? 527 DMA_BIDIRECTIONAL : DMA_TO_DEVICE; 528 #endif 529 530 #ifdef __NetBSD__ 531 if (curproc->p_vmspace != gtt->usermm) 532 return -EPERM; 533 #else 534 if (current->mm != gtt->usermm) 535 return -EPERM; 536 #endif 537 538 if (gtt->userflags & RADEON_GEM_USERPTR_ANONONLY) { 539 /* check that we only pin down anonymous memory 540 to prevent problems with writeback */ 541 unsigned long end = gtt->userptr + ttm->num_pages * PAGE_SIZE; 542 #ifdef __NetBSD__ 543 /* XXX ??? TOCTOU, anyone? */ 544 /* XXX should do range_test */ 545 struct vm_map_entry *entry; 546 bool ok; 547 vm_map_lock_read(>t->usermm->vm_map); 548 ok = uvm_map_lookup_entry(>t->usermm->vm_map, 549 (vaddr_t)gtt->userptr, &entry); 550 if (ok) 551 ok = !UVM_ET_ISOBJ(entry) && end <= entry->end; 552 vm_map_unlock_read(>t->usermm->vm_map); 553 if (!ok) 554 return -EPERM; 555 #else 556 struct vm_area_struct *vma; 557 vma = find_vma(gtt->usermm, gtt->userptr); 558 if (!vma || vma->vm_file || vma->vm_end < end) 559 return -EPERM; 560 #endif 561 } 562 563 #ifdef __NetBSD__ 564 struct iovec iov = { 565 .iov_base = (void *)(vaddr_t)gtt->userptr, 566 .iov_len = ttm->num_pages << PAGE_SHIFT, 567 }; 568 struct uio uio = { 569 .uio_iov = &iov, 570 .uio_iovcnt = 1, 571 .uio_offset = 0, 572 .uio_resid = ttm->num_pages << PAGE_SHIFT, 573 .uio_rw = (write ? UIO_READ : UIO_WRITE), /* XXX ??? */ 574 .uio_vmspace = gtt->usermm, 575 }; 576 unsigned long i; 577 578 /* Wire the relevant part of the user's address space. */ 579 /* XXX What happens if user does munmap? */ 580 /* XXX errno NetBSD->Linux */ 581 r = -uvm_vslock(gtt->usermm, (void *)(vaddr_t)gtt->userptr, 582 ttm->num_pages << PAGE_SHIFT, 583 (write ? VM_PROT_WRITE : VM_PROT_READ)); /* XXX ??? */ 584 if (r) 585 goto fail0; 586 587 /* Load it up for DMA. */ 588 /* XXX errno NetBSD->Linux */ 589 r = -bus_dmamap_load_uio(rdev->ddev->dmat, gtt->ttm.dma_address, &uio, 590 BUS_DMA_WAITOK); 591 if (r) 592 goto fail1; 593 594 /* Get each of the pages as ttm requests. */ 595 for (i = 0; i < ttm->num_pages; i++) { 596 vaddr_t va = (vaddr_t)gtt->userptr + (i << PAGE_SHIFT); 597 paddr_t pa; 598 struct vm_page *vmp; 599 600 if (!pmap_extract(gtt->usermm->vm_map.pmap, va, &pa)) { 601 r = -EFAULT; 602 goto fail2; 603 } 604 vmp = PHYS_TO_VM_PAGE(pa); 605 ttm->pages[i] = container_of(vmp, struct page, p_vmp); 606 } 607 608 /* Success! */ 609 return 0; 610 611 fail2: while (i --> 0) 612 ttm->pages[i] = NULL; /* paranoia */ 613 bus_dmamap_unload(rdev->ddev->dmat, gtt->ttm.dma_address); 614 fail1: uvm_vsunlock(gtt->usermm, (void *)(vaddr_t)gtt->userptr, 615 ttm->num_pages << PAGE_SHIFT); 616 fail0: return r; 617 #else 618 do { 619 unsigned num_pages = ttm->num_pages - pinned; 620 uint64_t userptr = gtt->userptr + pinned * PAGE_SIZE; 621 struct page **pages = ttm->pages + pinned; 622 623 r = get_user_pages(userptr, num_pages, write ? FOLL_WRITE : 0, 624 pages, NULL); 625 if (r < 0) 626 goto release_pages; 627 628 pinned += r; 629 630 } while (pinned < ttm->num_pages); 631 632 r = sg_alloc_table_from_pages(ttm->sg, ttm->pages, ttm->num_pages, 0, 633 ttm->num_pages << PAGE_SHIFT, 634 GFP_KERNEL); 635 if (r) 636 goto release_sg; 637 638 r = -ENOMEM; 639 nents = dma_map_sg(rdev->dev, ttm->sg->sgl, ttm->sg->nents, direction); 640 if (nents != ttm->sg->nents) 641 goto release_sg; 642 643 drm_prime_sg_to_page_addr_arrays(ttm->sg, ttm->pages, 644 gtt->ttm.dma_address, ttm->num_pages); 645 646 return 0; 647 648 release_sg: 649 kfree(ttm->sg); 650 651 release_pages: 652 release_pages(ttm->pages, pinned); 653 return r; 654 #endif 655 } 656 657 static void radeon_ttm_tt_unpin_userptr(struct ttm_tt *ttm) 658 { 659 #ifdef __NetBSD__ 660 struct radeon_device *rdev = radeon_get_rdev(ttm->bdev); 661 struct radeon_ttm_tt *gtt = (void *)ttm; 662 663 bus_dmamap_unload(rdev->ddev->dmat, gtt->ttm.dma_address); 664 uvm_vsunlock(gtt->usermm, (void *)(vaddr_t)gtt->userptr, 665 ttm->num_pages << PAGE_SHIFT); 666 #else 667 struct radeon_device *rdev = radeon_get_rdev(ttm->bdev); 668 struct radeon_ttm_tt *gtt = (void *)ttm; 669 struct sg_page_iter sg_iter; 670 671 int write = !(gtt->userflags & RADEON_GEM_USERPTR_READONLY); 672 enum dma_data_direction direction = write ? 673 DMA_BIDIRECTIONAL : DMA_TO_DEVICE; 674 675 /* double check that we don't free the table twice */ 676 if (!ttm->sg->sgl) 677 return; 678 679 /* free the sg table and pages again */ 680 dma_unmap_sg(rdev->dev, ttm->sg->sgl, ttm->sg->nents, direction); 681 682 for_each_sg_page(ttm->sg->sgl, &sg_iter, ttm->sg->nents, 0) { 683 struct page *page = sg_page_iter_page(&sg_iter); 684 if (!(gtt->userflags & RADEON_GEM_USERPTR_READONLY)) 685 set_page_dirty(page); 686 687 mark_page_accessed(page); 688 put_page(page); 689 } 690 691 sg_free_table(ttm->sg); 692 #endif 693 } 694 695 static int radeon_ttm_backend_bind(struct ttm_tt *ttm, 696 struct ttm_mem_reg *bo_mem) 697 { 698 struct radeon_ttm_tt *gtt = (void*)ttm; 699 uint32_t flags = RADEON_GART_PAGE_VALID | RADEON_GART_PAGE_READ | 700 RADEON_GART_PAGE_WRITE; 701 int r; 702 703 if (gtt->userptr) { 704 radeon_ttm_tt_pin_userptr(ttm); 705 flags &= ~RADEON_GART_PAGE_WRITE; 706 } 707 708 gtt->offset = (unsigned long)(bo_mem->start << PAGE_SHIFT); 709 if (!ttm->num_pages) { 710 WARN(1, "nothing to bind %lu pages for mreg %p back %p!\n", 711 ttm->num_pages, bo_mem, ttm); 712 } 713 if (ttm->caching_state == tt_cached) 714 flags |= RADEON_GART_PAGE_SNOOP; 715 r = radeon_gart_bind(gtt->rdev, gtt->offset, ttm->num_pages, 716 ttm->pages, gtt->ttm.dma_address, flags); 717 if (r) { 718 DRM_ERROR("failed to bind %lu pages at 0x%08X\n", 719 ttm->num_pages, (unsigned)gtt->offset); 720 return r; 721 } 722 return 0; 723 } 724 725 static int radeon_ttm_backend_unbind(struct ttm_tt *ttm) 726 { 727 struct radeon_ttm_tt *gtt = (void *)ttm; 728 729 radeon_gart_unbind(gtt->rdev, gtt->offset, ttm->num_pages); 730 731 if (gtt->userptr) 732 radeon_ttm_tt_unpin_userptr(ttm); 733 734 return 0; 735 } 736 737 static void radeon_ttm_backend_destroy(struct ttm_tt *ttm) 738 { 739 struct radeon_ttm_tt *gtt = (void *)ttm; 740 741 ttm_dma_tt_fini(>t->ttm); 742 kfree(gtt); 743 } 744 745 static struct ttm_backend_func radeon_backend_func = { 746 .bind = &radeon_ttm_backend_bind, 747 .unbind = &radeon_ttm_backend_unbind, 748 .destroy = &radeon_ttm_backend_destroy, 749 }; 750 751 static struct ttm_tt *radeon_ttm_tt_create(struct ttm_buffer_object *bo, 752 uint32_t page_flags) 753 { 754 struct radeon_device *rdev; 755 struct radeon_ttm_tt *gtt; 756 757 rdev = radeon_get_rdev(bo->bdev); 758 #if IS_ENABLED(CONFIG_AGP) 759 if (rdev->flags & RADEON_IS_AGP) { 760 return ttm_agp_tt_create(bo, rdev->ddev->agp->bridge, 761 page_flags); 762 } 763 #endif 764 765 gtt = kzalloc(sizeof(struct radeon_ttm_tt), GFP_KERNEL); 766 if (gtt == NULL) { 767 return NULL; 768 } 769 gtt->ttm.ttm.func = &radeon_backend_func; 770 gtt->rdev = rdev; 771 if (ttm_dma_tt_init(>t->ttm, bo, page_flags)) { 772 kfree(gtt); 773 return NULL; 774 } 775 return >t->ttm.ttm; 776 } 777 778 static struct radeon_ttm_tt *radeon_ttm_tt_to_gtt(struct ttm_tt *ttm) 779 { 780 if (!ttm || ttm->func != &radeon_backend_func) 781 return NULL; 782 return (struct radeon_ttm_tt *)ttm; 783 } 784 785 static int radeon_ttm_tt_populate(struct ttm_tt *ttm, 786 struct ttm_operation_ctx *ctx) 787 { 788 struct radeon_ttm_tt *gtt = radeon_ttm_tt_to_gtt(ttm); 789 #if !defined(__NetBSD__) || IS_ENABLED(CONFIG_AGP) 790 struct radeon_device *rdev; 791 #endif 792 bool slave = !!(ttm->page_flags & TTM_PAGE_FLAG_SG); 793 794 if (gtt && gtt->userptr) { 795 #ifdef __NetBSD__ 796 ttm->sg = NULL; 797 #else 798 ttm->sg = kzalloc(sizeof(struct sg_table), GFP_KERNEL); 799 if (!ttm->sg) 800 return -ENOMEM; 801 #endif 802 803 ttm->page_flags |= TTM_PAGE_FLAG_SG; 804 ttm->state = tt_unbound; 805 return 0; 806 } 807 808 if (slave && ttm->sg) { 809 #ifdef __NetBSD__ 810 r = drm_prime_bus_dmamap_load_sgt(ttm->bdev->dmat, 811 gtt->ttm.dma_address, ttm->sg); 812 if (r) 813 return r; 814 #else 815 drm_prime_sg_to_page_addr_arrays(ttm->sg, ttm->pages, 816 gtt->ttm.dma_address, ttm->num_pages); 817 #endif 818 ttm->state = tt_unbound; 819 return 0; 820 } 821 822 #if !defined(__NetBSD__) || IS_ENABLED(CONFIG_AGP) 823 rdev = radeon_get_rdev(ttm->bdev); 824 #endif 825 #if IS_ENABLED(CONFIG_AGP) 826 if (rdev->flags & RADEON_IS_AGP) { 827 return ttm_agp_tt_populate(ttm, ctx); 828 } 829 #endif 830 831 #ifdef __NetBSD__ 832 /* XXX errno NetBSD->Linux */ 833 return ttm_bus_dma_populate(>t->ttm); 834 #else 835 836 #ifdef CONFIG_SWIOTLB 837 if (rdev->need_swiotlb && swiotlb_nr_tbl()) { 838 return ttm_dma_populate(>t->ttm, rdev->dev, ctx); 839 } 840 #endif 841 842 return ttm_populate_and_map_pages(rdev->dev, >t->ttm, ctx); 843 #endif 844 } 845 846 static void radeon_ttm_tt_unpopulate(struct ttm_tt *ttm) 847 { 848 #if !defined(__NetBSD__) || IS_ENABLED(CONFIG_AGP) 849 struct radeon_device *rdev; 850 #endif 851 struct radeon_ttm_tt *gtt = radeon_ttm_tt_to_gtt(ttm); 852 bool slave = !!(ttm->page_flags & TTM_PAGE_FLAG_SG); 853 854 #ifdef __NetBSD__ 855 if (slave && ttm->sg) { 856 bus_dmamap_unload(ttm->bdev->dmat, gtt->ttm.dma_address); 857 } 858 #endif 859 if (gtt && gtt->userptr) { 860 kfree(ttm->sg); 861 ttm->page_flags &= ~TTM_PAGE_FLAG_SG; 862 return; 863 } 864 865 if (slave) 866 return; 867 868 #if !defined(__NetBSD__) || IS_ENABLED(CONFIG_AGP) 869 rdev = radeon_get_rdev(ttm->bdev); 870 #endif 871 #if IS_ENABLED(CONFIG_AGP) 872 if (rdev->flags & RADEON_IS_AGP) { 873 ttm_agp_tt_unpopulate(ttm); 874 return; 875 } 876 #endif 877 878 #ifdef __NetBSD__ 879 ttm_bus_dma_unpopulate(>t->ttm); 880 return; 881 #else 882 883 #ifdef CONFIG_SWIOTLB 884 if (rdev->need_swiotlb && swiotlb_nr_tbl()) { 885 ttm_dma_unpopulate(>t->ttm, rdev->dev); 886 return; 887 } 888 #endif 889 890 ttm_unmap_and_unpopulate_pages(rdev->dev, >t->ttm); 891 #endif 892 } 893 894 #ifdef __NetBSD__ 895 static void radeon_ttm_tt_swapout(struct ttm_tt *ttm) 896 { 897 struct radeon_ttm_tt *gtt = container_of(ttm, struct radeon_ttm_tt, 898 ttm.ttm); 899 struct ttm_dma_tt *ttm_dma = >t->ttm; 900 901 ttm_bus_dma_swapout(ttm_dma); 902 } 903 904 static int radeon_ttm_fault(struct uvm_faultinfo *, vaddr_t, 905 struct vm_page **, int, int, vm_prot_t, int); 906 907 static const struct uvm_pagerops radeon_uvm_ops = { 908 .pgo_reference = &ttm_bo_uvm_reference, 909 .pgo_detach = &ttm_bo_uvm_detach, 910 .pgo_fault = &radeon_ttm_fault, 911 }; 912 #endif 913 914 int radeon_ttm_tt_set_userptr(struct ttm_tt *ttm, uint64_t addr, 915 uint32_t flags) 916 { 917 struct radeon_ttm_tt *gtt = radeon_ttm_tt_to_gtt(ttm); 918 919 if (gtt == NULL) 920 return -EINVAL; 921 922 gtt->userptr = addr; 923 #ifdef __NetBSD__ 924 gtt->usermm = curproc->p_vmspace; 925 #else 926 gtt->usermm = current->mm; 927 #endif 928 gtt->userflags = flags; 929 return 0; 930 } 931 932 bool radeon_ttm_tt_has_userptr(struct ttm_tt *ttm) 933 { 934 struct radeon_ttm_tt *gtt = radeon_ttm_tt_to_gtt(ttm); 935 936 if (gtt == NULL) 937 return false; 938 939 return !!gtt->userptr; 940 } 941 942 bool radeon_ttm_tt_is_readonly(struct ttm_tt *ttm) 943 { 944 struct radeon_ttm_tt *gtt = radeon_ttm_tt_to_gtt(ttm); 945 946 if (gtt == NULL) 947 return false; 948 949 return !!(gtt->userflags & RADEON_GEM_USERPTR_READONLY); 950 } 951 952 static struct ttm_bo_driver radeon_bo_driver = { 953 .ttm_tt_create = &radeon_ttm_tt_create, 954 .ttm_tt_populate = &radeon_ttm_tt_populate, 955 .ttm_tt_unpopulate = &radeon_ttm_tt_unpopulate, 956 #ifdef __NetBSD__ 957 .ttm_tt_swapout = &radeon_ttm_tt_swapout, 958 .ttm_uvm_ops = &radeon_uvm_ops, 959 #endif 960 .invalidate_caches = &radeon_invalidate_caches, 961 .init_mem_type = &radeon_init_mem_type, 962 .eviction_valuable = ttm_bo_eviction_valuable, 963 .evict_flags = &radeon_evict_flags, 964 .move = &radeon_bo_move, 965 .verify_access = &radeon_verify_access, 966 .move_notify = &radeon_bo_move_notify, 967 .fault_reserve_notify = &radeon_bo_fault_reserve_notify, 968 .io_mem_reserve = &radeon_ttm_io_mem_reserve, 969 .io_mem_free = &radeon_ttm_io_mem_free, 970 }; 971 972 int radeon_ttm_init(struct radeon_device *rdev) 973 { 974 int r; 975 976 /* No others user of address space so set it to 0 */ 977 r = ttm_bo_device_init(&rdev->mman.bdev, 978 &radeon_bo_driver, 979 #ifdef __NetBSD__ 980 rdev->ddev->bst, 981 rdev->ddev->dmat, 982 #else 983 rdev->ddev->anon_inode->i_mapping, 984 #endif 985 rdev->ddev->vma_offset_manager, 986 dma_addressing_limited(&rdev->pdev->dev)); 987 if (r) { 988 DRM_ERROR("failed initializing buffer object driver(%d).\n", r); 989 return r; 990 } 991 rdev->mman.initialized = true; 992 r = ttm_bo_init_mm(&rdev->mman.bdev, TTM_PL_VRAM, 993 rdev->mc.real_vram_size >> PAGE_SHIFT); 994 if (r) { 995 DRM_ERROR("Failed initializing VRAM heap.\n"); 996 return r; 997 } 998 /* Change the size here instead of the init above so only lpfn is affected */ 999 radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size); 1000 1001 r = radeon_bo_create(rdev, 256 * 1024, PAGE_SIZE, true, 1002 RADEON_GEM_DOMAIN_VRAM, 0, NULL, 1003 NULL, &rdev->stolen_vga_memory); 1004 if (r) { 1005 return r; 1006 } 1007 r = radeon_bo_reserve(rdev->stolen_vga_memory, false); 1008 if (r) 1009 return r; 1010 r = radeon_bo_pin(rdev->stolen_vga_memory, RADEON_GEM_DOMAIN_VRAM, NULL); 1011 radeon_bo_unreserve(rdev->stolen_vga_memory); 1012 if (r) { 1013 radeon_bo_unref(&rdev->stolen_vga_memory); 1014 return r; 1015 } 1016 DRM_INFO("radeon: %uM of VRAM memory ready\n", 1017 (unsigned) (rdev->mc.real_vram_size / (1024 * 1024))); 1018 r = ttm_bo_init_mm(&rdev->mman.bdev, TTM_PL_TT, 1019 rdev->mc.gtt_size >> PAGE_SHIFT); 1020 if (r) { 1021 DRM_ERROR("Failed initializing GTT heap.\n"); 1022 return r; 1023 } 1024 DRM_INFO("radeon: %uM of GTT memory ready.\n", 1025 (unsigned)(rdev->mc.gtt_size / (1024 * 1024))); 1026 1027 r = radeon_ttm_debugfs_init(rdev); 1028 if (r) { 1029 DRM_ERROR("Failed to init debugfs\n"); 1030 return r; 1031 } 1032 return 0; 1033 } 1034 1035 void radeon_ttm_fini(struct radeon_device *rdev) 1036 { 1037 int r; 1038 1039 if (!rdev->mman.initialized) 1040 return; 1041 radeon_ttm_debugfs_fini(rdev); 1042 if (rdev->stolen_vga_memory) { 1043 r = radeon_bo_reserve(rdev->stolen_vga_memory, false); 1044 if (r == 0) { 1045 radeon_bo_unpin(rdev->stolen_vga_memory); 1046 radeon_bo_unreserve(rdev->stolen_vga_memory); 1047 } 1048 radeon_bo_unref(&rdev->stolen_vga_memory); 1049 } 1050 ttm_bo_clean_mm(&rdev->mman.bdev, TTM_PL_VRAM); 1051 ttm_bo_clean_mm(&rdev->mman.bdev, TTM_PL_TT); 1052 ttm_bo_device_release(&rdev->mman.bdev); 1053 radeon_gart_fini(rdev); 1054 rdev->mman.initialized = false; 1055 DRM_INFO("radeon: ttm finalized\n"); 1056 } 1057 1058 /* this should only be called at bootup or when userspace 1059 * isn't running */ 1060 void radeon_ttm_set_active_vram_size(struct radeon_device *rdev, u64 size) 1061 { 1062 struct ttm_mem_type_manager *man; 1063 1064 if (!rdev->mman.initialized) 1065 return; 1066 1067 man = &rdev->mman.bdev.man[TTM_PL_VRAM]; 1068 /* this just adjusts TTM size idea, which sets lpfn to the correct value */ 1069 man->size = size >> PAGE_SHIFT; 1070 } 1071 1072 #ifdef __NetBSD__ 1073 1074 static int 1075 radeon_ttm_fault(struct uvm_faultinfo *ufi, vaddr_t vaddr, 1076 struct vm_page **pps, int npages, int centeridx, vm_prot_t access_type, 1077 int flags) 1078 { 1079 struct uvm_object *const uobj = ufi->entry->object.uvm_obj; 1080 struct ttm_buffer_object *const bo = container_of(uobj, 1081 struct ttm_buffer_object, uvmobj); 1082 struct radeon_device *const rdev = radeon_get_rdev(bo->bdev); 1083 int error; 1084 1085 KASSERT(rdev != NULL); 1086 down_read(&rdev->pm.mclk_lock); 1087 error = ttm_bo_uvm_fault(ufi, vaddr, pps, npages, centeridx, 1088 access_type, flags); 1089 up_read(&rdev->pm.mclk_lock); 1090 1091 return error; 1092 } 1093 1094 int 1095 radeon_mmap_object(struct drm_device *dev, off_t offset, size_t size, 1096 vm_prot_t prot, struct uvm_object **uobjp, voff_t *uoffsetp, 1097 struct file *file) 1098 { 1099 struct radeon_device *rdev = dev->dev_private; 1100 1101 KASSERT(0 == (offset & (PAGE_SIZE - 1))); 1102 1103 if (__predict_false(rdev == NULL)) /* XXX How?? */ 1104 return -EINVAL; 1105 1106 if (__predict_false((offset >> PAGE_SHIFT) < DRM_FILE_PAGE_OFFSET)) 1107 return -EINVAL; 1108 1109 return ttm_bo_mmap_object(&rdev->mman.bdev, offset, size, prot, 1110 uobjp, uoffsetp, file); 1111 } 1112 1113 #else 1114 1115 static vm_fault_t radeon_ttm_fault(struct vm_fault *vmf) 1116 { 1117 struct ttm_buffer_object *bo; 1118 struct radeon_device *rdev; 1119 vm_fault_t ret; 1120 1121 bo = (struct ttm_buffer_object *)vmf->vma->vm_private_data; 1122 if (bo == NULL) 1123 return VM_FAULT_NOPAGE; 1124 1125 rdev = radeon_get_rdev(bo->bdev); 1126 down_read(&rdev->pm.mclk_lock); 1127 ret = ttm_bo_vm_fault(vmf); 1128 up_read(&rdev->pm.mclk_lock); 1129 return ret; 1130 } 1131 1132 static struct vm_operations_struct radeon_ttm_vm_ops = { 1133 .fault = radeon_ttm_fault, 1134 .open = ttm_bo_vm_open, 1135 .close = ttm_bo_vm_close, 1136 .access = ttm_bo_vm_access 1137 }; 1138 1139 int radeon_mmap(struct file *filp, struct vm_area_struct *vma) 1140 { 1141 int r; 1142 struct drm_file *file_priv = filp->private_data; 1143 struct radeon_device *rdev = file_priv->minor->dev->dev_private; 1144 1145 if (rdev == NULL) 1146 return -EINVAL; 1147 1148 r = ttm_bo_mmap(filp, vma, &rdev->mman.bdev); 1149 if (unlikely(r != 0)) 1150 return r; 1151 1152 vma->vm_ops = &radeon_ttm_vm_ops; 1153 return 0; 1154 } 1155 1156 #endif /* __NetBSD__ */ 1157 1158 #if defined(CONFIG_DEBUG_FS) 1159 1160 static int radeon_mm_dump_table(struct seq_file *m, void *data) 1161 { 1162 struct drm_info_node *node = (struct drm_info_node *)m->private; 1163 unsigned ttm_pl = *(int*)node->info_ent->data; 1164 struct drm_device *dev = node->minor->dev; 1165 struct radeon_device *rdev = dev->dev_private; 1166 struct ttm_mem_type_manager *man = &rdev->mman.bdev.man[ttm_pl]; 1167 struct drm_printer p = drm_seq_file_printer(m); 1168 1169 man->func->debug(man, &p); 1170 return 0; 1171 } 1172 1173 1174 static int ttm_pl_vram = TTM_PL_VRAM; 1175 static int ttm_pl_tt = TTM_PL_TT; 1176 1177 static struct drm_info_list radeon_ttm_debugfs_list[] = { 1178 {"radeon_vram_mm", radeon_mm_dump_table, 0, &ttm_pl_vram}, 1179 {"radeon_gtt_mm", radeon_mm_dump_table, 0, &ttm_pl_tt}, 1180 {"ttm_page_pool", ttm_page_alloc_debugfs, 0, NULL}, 1181 #ifdef CONFIG_SWIOTLB 1182 {"ttm_dma_page_pool", ttm_dma_page_alloc_debugfs, 0, NULL} 1183 #endif 1184 }; 1185 1186 static int radeon_ttm_vram_open(struct inode *inode, struct file *filep) 1187 { 1188 struct radeon_device *rdev = inode->i_private; 1189 i_size_write(inode, rdev->mc.mc_vram_size); 1190 filep->private_data = inode->i_private; 1191 return 0; 1192 } 1193 1194 static ssize_t radeon_ttm_vram_read(struct file *f, char __user *buf, 1195 size_t size, loff_t *pos) 1196 { 1197 struct radeon_device *rdev = f->private_data; 1198 ssize_t result = 0; 1199 int r; 1200 1201 if (size & 0x3 || *pos & 0x3) 1202 return -EINVAL; 1203 1204 while (size) { 1205 unsigned long flags; 1206 uint32_t value; 1207 1208 if (*pos >= rdev->mc.mc_vram_size) 1209 return result; 1210 1211 spin_lock_irqsave(&rdev->mmio_idx_lock, flags); 1212 WREG32(RADEON_MM_INDEX, ((uint32_t)*pos) | 0x80000000); 1213 if (rdev->family >= CHIP_CEDAR) 1214 WREG32(EVERGREEN_MM_INDEX_HI, *pos >> 31); 1215 value = RREG32(RADEON_MM_DATA); 1216 spin_unlock_irqrestore(&rdev->mmio_idx_lock, flags); 1217 1218 r = put_user(value, (uint32_t *)buf); 1219 if (r) 1220 return r; 1221 1222 result += 4; 1223 buf += 4; 1224 *pos += 4; 1225 size -= 4; 1226 } 1227 1228 return result; 1229 } 1230 1231 static const struct file_operations radeon_ttm_vram_fops = { 1232 .owner = THIS_MODULE, 1233 .open = radeon_ttm_vram_open, 1234 .read = radeon_ttm_vram_read, 1235 .llseek = default_llseek 1236 }; 1237 1238 static int radeon_ttm_gtt_open(struct inode *inode, struct file *filep) 1239 { 1240 struct radeon_device *rdev = inode->i_private; 1241 i_size_write(inode, rdev->mc.gtt_size); 1242 filep->private_data = inode->i_private; 1243 return 0; 1244 } 1245 1246 static ssize_t radeon_ttm_gtt_read(struct file *f, char __user *buf, 1247 size_t size, loff_t *pos) 1248 { 1249 struct radeon_device *rdev = f->private_data; 1250 ssize_t result = 0; 1251 int r; 1252 1253 while (size) { 1254 loff_t p = *pos / PAGE_SIZE; 1255 unsigned off = *pos & ~PAGE_MASK; 1256 size_t cur_size = min_t(size_t, size, PAGE_SIZE - off); 1257 struct page *page; 1258 void *ptr; 1259 1260 if (p >= rdev->gart.num_cpu_pages) 1261 return result; 1262 1263 page = rdev->gart.pages[p]; 1264 if (page) { 1265 ptr = kmap(page); 1266 ptr += off; 1267 1268 r = copy_to_user(buf, ptr, cur_size); 1269 kunmap(rdev->gart.pages[p]); 1270 } else 1271 r = clear_user(buf, cur_size); 1272 1273 if (r) 1274 return -EFAULT; 1275 1276 result += cur_size; 1277 buf += cur_size; 1278 *pos += cur_size; 1279 size -= cur_size; 1280 } 1281 1282 return result; 1283 } 1284 1285 static const struct file_operations radeon_ttm_gtt_fops = { 1286 .owner = THIS_MODULE, 1287 .open = radeon_ttm_gtt_open, 1288 .read = radeon_ttm_gtt_read, 1289 .llseek = default_llseek 1290 }; 1291 1292 #endif 1293 1294 static int radeon_ttm_debugfs_init(struct radeon_device *rdev) 1295 { 1296 #if defined(CONFIG_DEBUG_FS) 1297 unsigned count; 1298 1299 struct drm_minor *minor = rdev->ddev->primary; 1300 struct dentry *root = minor->debugfs_root; 1301 1302 rdev->mman.vram = debugfs_create_file("radeon_vram", S_IFREG | S_IRUGO, 1303 root, rdev, 1304 &radeon_ttm_vram_fops); 1305 1306 rdev->mman.gtt = debugfs_create_file("radeon_gtt", S_IFREG | S_IRUGO, 1307 root, rdev, &radeon_ttm_gtt_fops); 1308 1309 count = ARRAY_SIZE(radeon_ttm_debugfs_list); 1310 1311 #ifdef CONFIG_SWIOTLB 1312 if (!(rdev->need_swiotlb && swiotlb_nr_tbl())) 1313 --count; 1314 #endif 1315 1316 return radeon_debugfs_add_files(rdev, radeon_ttm_debugfs_list, count); 1317 #else 1318 1319 return 0; 1320 #endif 1321 } 1322 1323 static void radeon_ttm_debugfs_fini(struct radeon_device *rdev) 1324 { 1325 #if defined(CONFIG_DEBUG_FS) 1326 1327 debugfs_remove(rdev->mman.vram); 1328 rdev->mman.vram = NULL; 1329 1330 debugfs_remove(rdev->mman.gtt); 1331 rdev->mman.gtt = NULL; 1332 #endif 1333 } 1334
Indexes created Sat Oct 25 16:10:12 GMT 2025