radeon_ttm.c revision 1.21
1 /* $NetBSD: radeon_ttm.c,v 1.21 2021/12/19 09:56:53 riastradh Exp $ */ 2 3 /* 4 * Copyright 2009 Jerome Glisse. 5 * All Rights Reserved. 6 * 7 * Permission is hereby granted, free of charge, to any person obtaining a 8 * copy of this software and associated documentation files (the 9 * "Software"), to deal in the Software without restriction, including 10 * without limitation the rights to use, copy, modify, merge, publish, 11 * distribute, sub license, and/or sell copies of the Software, and to 12 * permit persons to whom the Software is furnished to do so, subject to 13 * the following conditions: 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL 18 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, 19 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 20 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE 21 * USE OR OTHER DEALINGS IN THE SOFTWARE. 22 * 23 * The above copyright notice and this permission notice (including the 24 * next paragraph) shall be included in all copies or substantial portions 25 * of the Software. 26 * 27 */ 28 /* 29 * Authors: 30 * Jerome Glisse <glisse (at) freedesktop.org> 31 * Thomas Hellstrom <thomas-at-tungstengraphics-dot-com> 32 * Dave Airlie 33 */ 34 35 #include <sys/cdefs.h> 36 __KERNEL_RCSID(0, "$NetBSD: radeon_ttm.c,v 1.21 2021/12/19 09:56:53 riastradh Exp $"); 37 38 #include <linux/dma-mapping.h> 39 #include <linux/pagemap.h> 40 #include <linux/pci.h> 41 #include <linux/seq_file.h> 42 #include <linux/slab.h> 43 #include <linux/swap.h> 44 #include <linux/swiotlb.h> 45 46 #include <drm/drm_agpsupport.h> 47 #include <drm/drm_debugfs.h> 48 #include <drm/drm_device.h> 49 #include <drm/drm_file.h> 50 #include <drm/drm_prime.h> 51 #include <drm/radeon_drm.h> 52 #include <drm/ttm/ttm_bo_api.h> 53 #include <drm/ttm/ttm_bo_driver.h> 54 #include <drm/ttm/ttm_module.h> 55 #include <drm/ttm/ttm_page_alloc.h> 56 #include <drm/ttm/ttm_placement.h> 57 58 #include "radeon_reg.h" 59 #include "radeon.h" 60 61 #ifdef __NetBSD__ 62 #include <uvm/uvm_extern.h> 63 #include <uvm/uvm_fault.h> 64 #include <uvm/uvm_param.h> 65 #include <drm/bus_dma_hacks.h> 66 #endif 67 68 #ifdef _LP64 69 #define DRM_FILE_PAGE_OFFSET (0x100000000ULL >> PAGE_SHIFT) 70 #else 71 #define DRM_FILE_PAGE_OFFSET (0xa0000000UL >> PAGE_SHIFT) 72 #endif 73 74 static int radeon_ttm_debugfs_init(struct radeon_device *rdev); 75 static void radeon_ttm_debugfs_fini(struct radeon_device *rdev); 76 77 static struct radeon_device *radeon_get_rdev(struct ttm_bo_device *bdev) 78 { 79 struct radeon_mman *mman; 80 struct radeon_device *rdev; 81 82 mman = container_of(bdev, struct radeon_mman, bdev); 83 rdev = container_of(mman, struct radeon_device, mman); 84 return rdev; 85 } 86 87 static int radeon_invalidate_caches(struct ttm_bo_device *bdev, uint32_t flags) 88 { 89 return 0; 90 } 91 92 static int radeon_init_mem_type(struct ttm_bo_device *bdev, uint32_t type, 93 struct ttm_mem_type_manager *man) 94 { 95 struct radeon_device *rdev; 96 97 rdev = radeon_get_rdev(bdev); 98 99 switch (type) { 100 case TTM_PL_SYSTEM: 101 /* System memory */ 102 man->flags = TTM_MEMTYPE_FLAG_MAPPABLE; 103 man->available_caching = TTM_PL_MASK_CACHING; 104 man->default_caching = TTM_PL_FLAG_CACHED; 105 break; 106 case TTM_PL_TT: 107 man->func = &ttm_bo_manager_func; 108 man->gpu_offset = rdev->mc.gtt_start; 109 man->available_caching = TTM_PL_MASK_CACHING; 110 man->default_caching = TTM_PL_FLAG_CACHED; 111 man->flags = TTM_MEMTYPE_FLAG_MAPPABLE | TTM_MEMTYPE_FLAG_CMA; 112 #if IS_ENABLED(CONFIG_AGP) 113 if (rdev->flags & RADEON_IS_AGP) { 114 if (!rdev->ddev->agp) { 115 DRM_ERROR("AGP is not enabled for memory type %u\n", 116 (unsigned)type); 117 return -EINVAL; 118 } 119 if (!rdev->ddev->agp->cant_use_aperture) 120 man->flags = TTM_MEMTYPE_FLAG_MAPPABLE; 121 man->available_caching = TTM_PL_FLAG_UNCACHED | 122 TTM_PL_FLAG_WC; 123 man->default_caching = TTM_PL_FLAG_WC; 124 } 125 #endif 126 break; 127 case TTM_PL_VRAM: 128 /* "On-card" video ram */ 129 man->func = &ttm_bo_manager_func; 130 man->gpu_offset = rdev->mc.vram_start; 131 man->flags = TTM_MEMTYPE_FLAG_FIXED | 132 TTM_MEMTYPE_FLAG_MAPPABLE; 133 man->available_caching = TTM_PL_FLAG_UNCACHED | TTM_PL_FLAG_WC; 134 man->default_caching = TTM_PL_FLAG_WC; 135 break; 136 default: 137 DRM_ERROR("Unsupported memory type %u\n", (unsigned)type); 138 return -EINVAL; 139 } 140 return 0; 141 } 142 143 static void radeon_evict_flags(struct ttm_buffer_object *bo, 144 struct ttm_placement *placement) 145 { 146 static const struct ttm_place placements = { 147 .fpfn = 0, 148 .lpfn = 0, 149 .flags = TTM_PL_MASK_CACHING | TTM_PL_FLAG_SYSTEM 150 }; 151 152 struct radeon_bo *rbo; 153 154 if (!radeon_ttm_bo_is_radeon_bo(bo)) { 155 placement->placement = &placements; 156 placement->busy_placement = &placements; 157 placement->num_placement = 1; 158 placement->num_busy_placement = 1; 159 return; 160 } 161 rbo = container_of(bo, struct radeon_bo, tbo); 162 switch (bo->mem.mem_type) { 163 case TTM_PL_VRAM: 164 if (rbo->rdev->ring[radeon_copy_ring_index(rbo->rdev)].ready == false) 165 radeon_ttm_placement_from_domain(rbo, RADEON_GEM_DOMAIN_CPU); 166 else if (rbo->rdev->mc.visible_vram_size < rbo->rdev->mc.real_vram_size && 167 bo->mem.start < (rbo->rdev->mc.visible_vram_size >> PAGE_SHIFT)) { 168 unsigned fpfn = rbo->rdev->mc.visible_vram_size >> PAGE_SHIFT; 169 int i; 170 171 /* Try evicting to the CPU inaccessible part of VRAM 172 * first, but only set GTT as busy placement, so this 173 * BO will be evicted to GTT rather than causing other 174 * BOs to be evicted from VRAM 175 */ 176 radeon_ttm_placement_from_domain(rbo, RADEON_GEM_DOMAIN_VRAM | 177 RADEON_GEM_DOMAIN_GTT); 178 rbo->placement.num_busy_placement = 0; 179 for (i = 0; i < rbo->placement.num_placement; i++) { 180 if (rbo->placements[i].flags & TTM_PL_FLAG_VRAM) { 181 if (rbo->placements[i].fpfn < fpfn) 182 rbo->placements[i].fpfn = fpfn; 183 } else { 184 rbo->placement.busy_placement = 185 &rbo->placements[i]; 186 rbo->placement.num_busy_placement = 1; 187 } 188 } 189 } else 190 radeon_ttm_placement_from_domain(rbo, RADEON_GEM_DOMAIN_GTT); 191 break; 192 case TTM_PL_TT: 193 default: 194 radeon_ttm_placement_from_domain(rbo, RADEON_GEM_DOMAIN_CPU); 195 } 196 *placement = rbo->placement; 197 } 198 199 static int radeon_verify_access(struct ttm_buffer_object *bo, struct file *filp) 200 { 201 struct radeon_bo *rbo = container_of(bo, struct radeon_bo, tbo); 202 203 if (radeon_ttm_tt_has_userptr(bo->ttm)) 204 return -EPERM; 205 #ifdef __NetBSD__ 206 struct drm_file *drm_file = filp->f_data; 207 return drm_vma_node_verify_access(&rbo->tbo.base.vma_node, drm_file); 208 #else 209 return drm_vma_node_verify_access(&rbo->tbo.base.vma_node, 210 filp->private_data); 211 #endif 212 } 213 214 static void radeon_move_null(struct ttm_buffer_object *bo, 215 struct ttm_mem_reg *new_mem) 216 { 217 struct ttm_mem_reg *old_mem = &bo->mem; 218 219 BUG_ON(old_mem->mm_node != NULL); 220 *old_mem = *new_mem; 221 new_mem->mm_node = NULL; 222 } 223 224 static int radeon_move_blit(struct ttm_buffer_object *bo, 225 bool evict, bool no_wait_gpu, 226 struct ttm_mem_reg *new_mem, 227 struct ttm_mem_reg *old_mem) 228 { 229 struct radeon_device *rdev; 230 uint64_t old_start, new_start; 231 struct radeon_fence *fence; 232 unsigned num_pages; 233 int r, ridx; 234 235 rdev = radeon_get_rdev(bo->bdev); 236 ridx = radeon_copy_ring_index(rdev); 237 old_start = (u64)old_mem->start << PAGE_SHIFT; 238 new_start = (u64)new_mem->start << PAGE_SHIFT; 239 240 switch (old_mem->mem_type) { 241 case TTM_PL_VRAM: 242 old_start += rdev->mc.vram_start; 243 break; 244 case TTM_PL_TT: 245 old_start += rdev->mc.gtt_start; 246 break; 247 default: 248 DRM_ERROR("Unknown placement %d\n", old_mem->mem_type); 249 return -EINVAL; 250 } 251 switch (new_mem->mem_type) { 252 case TTM_PL_VRAM: 253 new_start += rdev->mc.vram_start; 254 break; 255 case TTM_PL_TT: 256 new_start += rdev->mc.gtt_start; 257 break; 258 default: 259 DRM_ERROR("Unknown placement %d\n", old_mem->mem_type); 260 return -EINVAL; 261 } 262 if (!rdev->ring[ridx].ready) { 263 DRM_ERROR("Trying to move memory with ring turned off.\n"); 264 return -EINVAL; 265 } 266 267 BUILD_BUG_ON((PAGE_SIZE % RADEON_GPU_PAGE_SIZE) != 0); 268 269 num_pages = new_mem->num_pages * (PAGE_SIZE / RADEON_GPU_PAGE_SIZE); 270 fence = radeon_copy(rdev, old_start, new_start, num_pages, bo->base.resv); 271 if (IS_ERR(fence)) 272 return PTR_ERR(fence); 273 274 r = ttm_bo_move_accel_cleanup(bo, &fence->base, evict, new_mem); 275 radeon_fence_unref(&fence); 276 return r; 277 } 278 279 static int radeon_move_vram_ram(struct ttm_buffer_object *bo, 280 bool evict, bool interruptible, 281 bool no_wait_gpu, 282 struct ttm_mem_reg *new_mem) 283 { 284 struct ttm_operation_ctx ctx = { interruptible, no_wait_gpu }; 285 struct ttm_mem_reg *old_mem = &bo->mem; 286 struct ttm_mem_reg tmp_mem; 287 struct ttm_place placements; 288 struct ttm_placement placement; 289 int r; 290 291 tmp_mem = *new_mem; 292 tmp_mem.mm_node = NULL; 293 placement.num_placement = 1; 294 placement.placement = &placements; 295 placement.num_busy_placement = 1; 296 placement.busy_placement = &placements; 297 placements.fpfn = 0; 298 placements.lpfn = 0; 299 placements.flags = TTM_PL_MASK_CACHING | TTM_PL_FLAG_TT; 300 r = ttm_bo_mem_space(bo, &placement, &tmp_mem, &ctx); 301 if (unlikely(r)) { 302 return r; 303 } 304 305 r = ttm_tt_set_placement_caching(bo->ttm, tmp_mem.placement); 306 if (unlikely(r)) { 307 goto out_cleanup; 308 } 309 310 r = ttm_tt_bind(bo->ttm, &tmp_mem, &ctx); 311 if (unlikely(r)) { 312 goto out_cleanup; 313 } 314 r = radeon_move_blit(bo, true, no_wait_gpu, &tmp_mem, old_mem); 315 if (unlikely(r)) { 316 goto out_cleanup; 317 } 318 r = ttm_bo_move_ttm(bo, &ctx, new_mem); 319 out_cleanup: 320 ttm_bo_mem_put(bo, &tmp_mem); 321 return r; 322 } 323 324 static int radeon_move_ram_vram(struct ttm_buffer_object *bo, 325 bool evict, bool interruptible, 326 bool no_wait_gpu, 327 struct ttm_mem_reg *new_mem) 328 { 329 struct ttm_operation_ctx ctx = { interruptible, no_wait_gpu }; 330 struct ttm_mem_reg *old_mem = &bo->mem; 331 struct ttm_mem_reg tmp_mem; 332 struct ttm_placement placement; 333 struct ttm_place placements; 334 int r; 335 336 tmp_mem = *new_mem; 337 tmp_mem.mm_node = NULL; 338 placement.num_placement = 1; 339 placement.placement = &placements; 340 placement.num_busy_placement = 1; 341 placement.busy_placement = &placements; 342 placements.fpfn = 0; 343 placements.lpfn = 0; 344 placements.flags = TTM_PL_MASK_CACHING | TTM_PL_FLAG_TT; 345 r = ttm_bo_mem_space(bo, &placement, &tmp_mem, &ctx); 346 if (unlikely(r)) { 347 return r; 348 } 349 r = ttm_bo_move_ttm(bo, &ctx, &tmp_mem); 350 if (unlikely(r)) { 351 goto out_cleanup; 352 } 353 r = radeon_move_blit(bo, true, no_wait_gpu, new_mem, old_mem); 354 if (unlikely(r)) { 355 goto out_cleanup; 356 } 357 out_cleanup: 358 ttm_bo_mem_put(bo, &tmp_mem); 359 return r; 360 } 361 362 static int radeon_bo_move(struct ttm_buffer_object *bo, bool evict, 363 struct ttm_operation_ctx *ctx, 364 struct ttm_mem_reg *new_mem) 365 { 366 struct radeon_device *rdev; 367 struct radeon_bo *rbo; 368 struct ttm_mem_reg *old_mem = &bo->mem; 369 int r; 370 371 r = ttm_bo_wait(bo, ctx->interruptible, ctx->no_wait_gpu); 372 if (r) 373 return r; 374 375 /* Can't move a pinned BO */ 376 rbo = container_of(bo, struct radeon_bo, tbo); 377 if (WARN_ON_ONCE(rbo->pin_count > 0)) 378 return -EINVAL; 379 380 rdev = radeon_get_rdev(bo->bdev); 381 if (old_mem->mem_type == TTM_PL_SYSTEM && bo->ttm == NULL) { 382 radeon_move_null(bo, new_mem); 383 return 0; 384 } 385 if ((old_mem->mem_type == TTM_PL_TT && 386 new_mem->mem_type == TTM_PL_SYSTEM) || 387 (old_mem->mem_type == TTM_PL_SYSTEM && 388 new_mem->mem_type == TTM_PL_TT)) { 389 /* bind is enough */ 390 radeon_move_null(bo, new_mem); 391 return 0; 392 } 393 if (!rdev->ring[radeon_copy_ring_index(rdev)].ready || 394 rdev->asic->copy.copy == NULL) { 395 /* use memcpy */ 396 goto memcpy; 397 } 398 399 if (old_mem->mem_type == TTM_PL_VRAM && 400 new_mem->mem_type == TTM_PL_SYSTEM) { 401 r = radeon_move_vram_ram(bo, evict, ctx->interruptible, 402 ctx->no_wait_gpu, new_mem); 403 } else if (old_mem->mem_type == TTM_PL_SYSTEM && 404 new_mem->mem_type == TTM_PL_VRAM) { 405 r = radeon_move_ram_vram(bo, evict, ctx->interruptible, 406 ctx->no_wait_gpu, new_mem); 407 } else { 408 r = radeon_move_blit(bo, evict, ctx->no_wait_gpu, 409 new_mem, old_mem); 410 } 411 412 if (r) { 413 memcpy: 414 r = ttm_bo_move_memcpy(bo, ctx, new_mem); 415 if (r) { 416 return r; 417 } 418 } 419 420 /* update statistics */ 421 atomic64_add((u64)bo->num_pages << PAGE_SHIFT, &rdev->num_bytes_moved); 422 return 0; 423 } 424 425 static int radeon_ttm_io_mem_reserve(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem) 426 { 427 struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type]; 428 struct radeon_device *rdev = radeon_get_rdev(bdev); 429 430 mem->bus.addr = NULL; 431 mem->bus.offset = 0; 432 mem->bus.size = mem->num_pages << PAGE_SHIFT; 433 mem->bus.base = 0; 434 mem->bus.is_iomem = false; 435 if (!(man->flags & TTM_MEMTYPE_FLAG_MAPPABLE)) 436 return -EINVAL; 437 switch (mem->mem_type) { 438 case TTM_PL_SYSTEM: 439 /* system memory */ 440 return 0; 441 case TTM_PL_TT: 442 #if IS_ENABLED(CONFIG_AGP) 443 if (rdev->flags & RADEON_IS_AGP) { 444 /* RADEON_IS_AGP is set only if AGP is active */ 445 mem->bus.offset = mem->start << PAGE_SHIFT; 446 mem->bus.base = rdev->mc.agp_base; 447 mem->bus.is_iomem = !rdev->ddev->agp->cant_use_aperture; 448 KASSERTMSG((mem->bus.base & (PAGE_SIZE - 1)) == 0, 449 "agp aperture is not page-aligned: %lx", 450 mem->bus.base); 451 KASSERT((mem->bus.offset & (PAGE_SIZE - 1)) == 0); 452 } 453 #endif 454 break; 455 case TTM_PL_VRAM: 456 mem->bus.offset = mem->start << PAGE_SHIFT; 457 /* check if it's visible */ 458 if ((mem->bus.offset + mem->bus.size) > rdev->mc.visible_vram_size) 459 return -EINVAL; 460 mem->bus.base = rdev->mc.aper_base; 461 mem->bus.is_iomem = true; 462 #ifdef __alpha__ 463 /* 464 * Alpha: use bus.addr to hold the ioremap() return, 465 * so we can modify bus.base below. 466 */ 467 if (mem->placement & TTM_PL_FLAG_WC) 468 mem->bus.addr = 469 ioremap_wc(mem->bus.base + mem->bus.offset, 470 mem->bus.size); 471 else 472 mem->bus.addr = 473 ioremap(mem->bus.base + mem->bus.offset, 474 mem->bus.size); 475 if (!mem->bus.addr) 476 return -ENOMEM; 477 478 /* 479 * Alpha: Use just the bus offset plus 480 * the hose/domain memory base for bus.base. 481 * It then can be used to build PTEs for VRAM 482 * access, as done in ttm_bo_vm_fault(). 483 */ 484 mem->bus.base = (mem->bus.base & 0x0ffffffffUL) + 485 rdev->ddev->hose->dense_mem_base; 486 #endif 487 KASSERTMSG((mem->bus.base & (PAGE_SIZE - 1)) == 0, 488 "mc aperture is not page-aligned: %lx", 489 mem->bus.base); 490 KASSERT((mem->bus.offset & (PAGE_SIZE - 1)) == 0); 491 break; 492 default: 493 return -EINVAL; 494 } 495 return 0; 496 } 497 498 static void radeon_ttm_io_mem_free(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem) 499 { 500 } 501 502 /* 503 * TTM backend functions. 504 */ 505 struct radeon_ttm_tt { 506 struct ttm_dma_tt ttm; 507 struct radeon_device *rdev; 508 u64 offset; 509 510 uint64_t userptr; 511 #ifdef __NetBSD__ 512 struct vmspace *usermm; 513 #else 514 struct mm_struct *usermm; 515 #endif 516 uint32_t userflags; 517 }; 518 519 /* prepare the sg table with the user pages */ 520 static int radeon_ttm_tt_pin_userptr(struct ttm_tt *ttm) 521 { 522 struct radeon_device *rdev = radeon_get_rdev(ttm->bdev); 523 struct radeon_ttm_tt *gtt = (void *)ttm; 524 #ifndef __NetBSD__ 525 unsigned pinned = 0, nents; 526 #endif 527 int r; 528 529 int write = !(gtt->userflags & RADEON_GEM_USERPTR_READONLY); 530 #ifndef __NetBSD__ 531 enum dma_data_direction direction = write ? 532 DMA_BIDIRECTIONAL : DMA_TO_DEVICE; 533 #endif 534 535 #ifdef __NetBSD__ 536 if (curproc->p_vmspace != gtt->usermm) 537 return -EPERM; 538 #else 539 if (current->mm != gtt->usermm) 540 return -EPERM; 541 #endif 542 543 if (gtt->userflags & RADEON_GEM_USERPTR_ANONONLY) { 544 /* check that we only pin down anonymous memory 545 to prevent problems with writeback */ 546 unsigned long end = gtt->userptr + ttm->num_pages * PAGE_SIZE; 547 #ifdef __NetBSD__ 548 /* XXX ??? TOCTOU, anyone? */ 549 /* XXX should do range_test */ 550 struct vm_map_entry *entry; 551 bool ok; 552 vm_map_lock_read(>t->usermm->vm_map); 553 ok = uvm_map_lookup_entry(>t->usermm->vm_map, 554 (vaddr_t)gtt->userptr, &entry); 555 if (ok) 556 ok = !UVM_ET_ISOBJ(entry) && end <= entry->end; 557 vm_map_unlock_read(>t->usermm->vm_map); 558 if (!ok) 559 return -EPERM; 560 #else 561 struct vm_area_struct *vma; 562 vma = find_vma(gtt->usermm, gtt->userptr); 563 if (!vma || vma->vm_file || vma->vm_end < end) 564 return -EPERM; 565 #endif 566 } 567 568 #ifdef __NetBSD__ 569 struct iovec iov = { 570 .iov_base = (void *)(vaddr_t)gtt->userptr, 571 .iov_len = ttm->num_pages << PAGE_SHIFT, 572 }; 573 struct uio uio = { 574 .uio_iov = &iov, 575 .uio_iovcnt = 1, 576 .uio_offset = 0, 577 .uio_resid = ttm->num_pages << PAGE_SHIFT, 578 .uio_rw = (write ? UIO_READ : UIO_WRITE), /* XXX ??? */ 579 .uio_vmspace = gtt->usermm, 580 }; 581 unsigned long i; 582 583 /* Wire the relevant part of the user's address space. */ 584 /* XXX What happens if user does munmap? */ 585 /* XXX errno NetBSD->Linux */ 586 r = -uvm_vslock(gtt->usermm, (void *)(vaddr_t)gtt->userptr, 587 ttm->num_pages << PAGE_SHIFT, 588 (write ? VM_PROT_WRITE : VM_PROT_READ)); /* XXX ??? */ 589 if (r) 590 goto fail0; 591 592 /* Load it up for DMA. */ 593 /* XXX errno NetBSD->Linux */ 594 r = -bus_dmamap_load_uio(rdev->ddev->dmat, gtt->ttm.dma_address, &uio, 595 BUS_DMA_WAITOK); 596 if (r) 597 goto fail1; 598 599 /* Get each of the pages as ttm requests. */ 600 for (i = 0; i < ttm->num_pages; i++) { 601 vaddr_t va = (vaddr_t)gtt->userptr + (i << PAGE_SHIFT); 602 paddr_t pa; 603 struct vm_page *vmp; 604 605 if (!pmap_extract(gtt->usermm->vm_map.pmap, va, &pa)) { 606 r = -EFAULT; 607 goto fail2; 608 } 609 vmp = PHYS_TO_VM_PAGE(pa); 610 ttm->pages[i] = container_of(vmp, struct page, p_vmp); 611 } 612 613 /* Success! */ 614 return 0; 615 616 fail2: while (i --> 0) 617 ttm->pages[i] = NULL; /* paranoia */ 618 bus_dmamap_unload(rdev->ddev->dmat, gtt->ttm.dma_address); 619 fail1: uvm_vsunlock(gtt->usermm, (void *)(vaddr_t)gtt->userptr, 620 ttm->num_pages << PAGE_SHIFT); 621 fail0: return r; 622 #else 623 do { 624 unsigned num_pages = ttm->num_pages - pinned; 625 uint64_t userptr = gtt->userptr + pinned * PAGE_SIZE; 626 struct page **pages = ttm->pages + pinned; 627 628 r = get_user_pages(userptr, num_pages, write ? FOLL_WRITE : 0, 629 pages, NULL); 630 if (r < 0) 631 goto release_pages; 632 633 pinned += r; 634 635 } while (pinned < ttm->num_pages); 636 637 r = sg_alloc_table_from_pages(ttm->sg, ttm->pages, ttm->num_pages, 0, 638 ttm->num_pages << PAGE_SHIFT, 639 GFP_KERNEL); 640 if (r) 641 goto release_sg; 642 643 r = -ENOMEM; 644 nents = dma_map_sg(rdev->dev, ttm->sg->sgl, ttm->sg->nents, direction); 645 if (nents != ttm->sg->nents) 646 goto release_sg; 647 648 drm_prime_sg_to_page_addr_arrays(ttm->sg, ttm->pages, 649 gtt->ttm.dma_address, ttm->num_pages); 650 651 return 0; 652 653 release_sg: 654 kfree(ttm->sg); 655 656 release_pages: 657 release_pages(ttm->pages, pinned); 658 return r; 659 #endif 660 } 661 662 static void radeon_ttm_tt_unpin_userptr(struct ttm_tt *ttm) 663 { 664 #ifdef __NetBSD__ 665 struct radeon_device *rdev = radeon_get_rdev(ttm->bdev); 666 struct radeon_ttm_tt *gtt = (void *)ttm; 667 668 bus_dmamap_unload(rdev->ddev->dmat, gtt->ttm.dma_address); 669 uvm_vsunlock(gtt->usermm, (void *)(vaddr_t)gtt->userptr, 670 ttm->num_pages << PAGE_SHIFT); 671 #else 672 struct radeon_device *rdev = radeon_get_rdev(ttm->bdev); 673 struct radeon_ttm_tt *gtt = (void *)ttm; 674 struct sg_page_iter sg_iter; 675 676 int write = !(gtt->userflags & RADEON_GEM_USERPTR_READONLY); 677 enum dma_data_direction direction = write ? 678 DMA_BIDIRECTIONAL : DMA_TO_DEVICE; 679 680 /* double check that we don't free the table twice */ 681 if (!ttm->sg->sgl) 682 return; 683 684 /* free the sg table and pages again */ 685 dma_unmap_sg(rdev->dev, ttm->sg->sgl, ttm->sg->nents, direction); 686 687 for_each_sg_page(ttm->sg->sgl, &sg_iter, ttm->sg->nents, 0) { 688 struct page *page = sg_page_iter_page(&sg_iter); 689 if (!(gtt->userflags & RADEON_GEM_USERPTR_READONLY)) 690 set_page_dirty(page); 691 692 mark_page_accessed(page); 693 put_page(page); 694 } 695 696 sg_free_table(ttm->sg); 697 #endif 698 } 699 700 static int radeon_ttm_backend_bind(struct ttm_tt *ttm, 701 struct ttm_mem_reg *bo_mem) 702 { 703 struct radeon_ttm_tt *gtt = (void*)ttm; 704 uint32_t flags = RADEON_GART_PAGE_VALID | RADEON_GART_PAGE_READ | 705 RADEON_GART_PAGE_WRITE; 706 int r; 707 708 if (gtt->userptr) { 709 radeon_ttm_tt_pin_userptr(ttm); 710 flags &= ~RADEON_GART_PAGE_WRITE; 711 } 712 713 gtt->offset = (unsigned long)(bo_mem->start << PAGE_SHIFT); 714 if (!ttm->num_pages) { 715 WARN(1, "nothing to bind %lu pages for mreg %p back %p!\n", 716 ttm->num_pages, bo_mem, ttm); 717 } 718 if (ttm->caching_state == tt_cached) 719 flags |= RADEON_GART_PAGE_SNOOP; 720 r = radeon_gart_bind(gtt->rdev, gtt->offset, ttm->num_pages, 721 ttm->pages, gtt->ttm.dma_address, flags); 722 if (r) { 723 DRM_ERROR("failed to bind %lu pages at 0x%08X\n", 724 ttm->num_pages, (unsigned)gtt->offset); 725 return r; 726 } 727 return 0; 728 } 729 730 static int radeon_ttm_backend_unbind(struct ttm_tt *ttm) 731 { 732 struct radeon_ttm_tt *gtt = (void *)ttm; 733 734 radeon_gart_unbind(gtt->rdev, gtt->offset, ttm->num_pages); 735 736 if (gtt->userptr) 737 radeon_ttm_tt_unpin_userptr(ttm); 738 739 return 0; 740 } 741 742 static void radeon_ttm_backend_destroy(struct ttm_tt *ttm) 743 { 744 struct radeon_ttm_tt *gtt = (void *)ttm; 745 746 ttm_dma_tt_fini(>t->ttm); 747 kfree(gtt); 748 } 749 750 static struct ttm_backend_func radeon_backend_func = { 751 .bind = &radeon_ttm_backend_bind, 752 .unbind = &radeon_ttm_backend_unbind, 753 .destroy = &radeon_ttm_backend_destroy, 754 }; 755 756 static struct ttm_tt *radeon_ttm_tt_create(struct ttm_buffer_object *bo, 757 uint32_t page_flags) 758 { 759 struct radeon_device *rdev; 760 struct radeon_ttm_tt *gtt; 761 762 rdev = radeon_get_rdev(bo->bdev); 763 #if IS_ENABLED(CONFIG_AGP) 764 if (rdev->flags & RADEON_IS_AGP) { 765 return ttm_agp_tt_create(bo, rdev->ddev->agp->bridge, 766 page_flags); 767 } 768 #endif 769 770 gtt = kzalloc(sizeof(struct radeon_ttm_tt), GFP_KERNEL); 771 if (gtt == NULL) { 772 return NULL; 773 } 774 gtt->ttm.ttm.func = &radeon_backend_func; 775 gtt->rdev = rdev; 776 if (ttm_dma_tt_init(>t->ttm, bo, page_flags)) { 777 kfree(gtt); 778 return NULL; 779 } 780 return >t->ttm.ttm; 781 } 782 783 static struct radeon_ttm_tt *radeon_ttm_tt_to_gtt(struct ttm_tt *ttm) 784 { 785 if (!ttm || ttm->func != &radeon_backend_func) 786 return NULL; 787 return (struct radeon_ttm_tt *)ttm; 788 } 789 790 static int radeon_ttm_tt_populate(struct ttm_tt *ttm, 791 struct ttm_operation_ctx *ctx) 792 { 793 struct radeon_ttm_tt *gtt = radeon_ttm_tt_to_gtt(ttm); 794 #if !defined(__NetBSD__) || IS_ENABLED(CONFIG_AGP) 795 struct radeon_device *rdev; 796 #endif 797 bool slave = !!(ttm->page_flags & TTM_PAGE_FLAG_SG); 798 799 if (gtt && gtt->userptr) { 800 #ifdef __NetBSD__ 801 ttm->sg = NULL; 802 #else 803 ttm->sg = kzalloc(sizeof(struct sg_table), GFP_KERNEL); 804 if (!ttm->sg) 805 return -ENOMEM; 806 #endif 807 808 ttm->page_flags |= TTM_PAGE_FLAG_SG; 809 ttm->state = tt_unbound; 810 return 0; 811 } 812 813 if (slave && ttm->sg) { 814 #ifdef __NetBSD__ 815 int r = drm_prime_bus_dmamap_load_sgt(ttm->bdev->dmat, 816 gtt->ttm.dma_address, ttm->sg); 817 if (r) 818 return r; 819 #else 820 drm_prime_sg_to_page_addr_arrays(ttm->sg, ttm->pages, 821 gtt->ttm.dma_address, ttm->num_pages); 822 #endif 823 ttm->state = tt_unbound; 824 return 0; 825 } 826 827 #if !defined(__NetBSD__) || IS_ENABLED(CONFIG_AGP) 828 rdev = radeon_get_rdev(ttm->bdev); 829 #endif 830 #if IS_ENABLED(CONFIG_AGP) 831 if (rdev->flags & RADEON_IS_AGP) { 832 return ttm_agp_tt_populate(ttm, ctx); 833 } 834 #endif 835 836 #ifdef __NetBSD__ 837 /* XXX errno NetBSD->Linux */ 838 return ttm_bus_dma_populate(>t->ttm); 839 #else 840 841 #ifdef CONFIG_SWIOTLB 842 if (rdev->need_swiotlb && swiotlb_nr_tbl()) { 843 return ttm_dma_populate(>t->ttm, rdev->dev, ctx); 844 } 845 #endif 846 847 return ttm_populate_and_map_pages(rdev->dev, >t->ttm, ctx); 848 #endif 849 } 850 851 static void radeon_ttm_tt_unpopulate(struct ttm_tt *ttm) 852 { 853 #if !defined(__NetBSD__) || IS_ENABLED(CONFIG_AGP) 854 struct radeon_device *rdev; 855 #endif 856 struct radeon_ttm_tt *gtt = radeon_ttm_tt_to_gtt(ttm); 857 bool slave = !!(ttm->page_flags & TTM_PAGE_FLAG_SG); 858 859 #ifdef __NetBSD__ 860 if (slave && ttm->sg) { 861 bus_dmamap_unload(ttm->bdev->dmat, gtt->ttm.dma_address); 862 } 863 #endif 864 if (gtt && gtt->userptr) { 865 kfree(ttm->sg); 866 ttm->page_flags &= ~TTM_PAGE_FLAG_SG; 867 return; 868 } 869 870 if (slave) 871 return; 872 873 #if !defined(__NetBSD__) || IS_ENABLED(CONFIG_AGP) 874 rdev = radeon_get_rdev(ttm->bdev); 875 #endif 876 #if IS_ENABLED(CONFIG_AGP) 877 if (rdev->flags & RADEON_IS_AGP) { 878 ttm_agp_tt_unpopulate(ttm); 879 return; 880 } 881 #endif 882 883 #ifdef __NetBSD__ 884 ttm_bus_dma_unpopulate(>t->ttm); 885 return; 886 #else 887 888 #ifdef CONFIG_SWIOTLB 889 if (rdev->need_swiotlb && swiotlb_nr_tbl()) { 890 ttm_dma_unpopulate(>t->ttm, rdev->dev); 891 return; 892 } 893 #endif 894 895 ttm_unmap_and_unpopulate_pages(rdev->dev, >t->ttm); 896 #endif 897 } 898 899 #ifdef __NetBSD__ 900 static void radeon_ttm_tt_swapout(struct ttm_tt *ttm) 901 { 902 struct radeon_ttm_tt *gtt = container_of(ttm, struct radeon_ttm_tt, 903 ttm.ttm); 904 struct ttm_dma_tt *ttm_dma = >t->ttm; 905 906 ttm_bus_dma_swapout(ttm_dma); 907 } 908 909 static int radeon_ttm_fault(struct uvm_faultinfo *, vaddr_t, 910 struct vm_page **, int, int, vm_prot_t, int); 911 912 static const struct uvm_pagerops radeon_uvm_ops = { 913 .pgo_reference = &ttm_bo_uvm_reference, 914 .pgo_detach = &ttm_bo_uvm_detach, 915 .pgo_fault = &radeon_ttm_fault, 916 }; 917 #endif 918 919 int radeon_ttm_tt_set_userptr(struct ttm_tt *ttm, uint64_t addr, 920 uint32_t flags) 921 { 922 struct radeon_ttm_tt *gtt = radeon_ttm_tt_to_gtt(ttm); 923 924 if (gtt == NULL) 925 return -EINVAL; 926 927 gtt->userptr = addr; 928 #ifdef __NetBSD__ 929 gtt->usermm = curproc->p_vmspace; 930 #else 931 gtt->usermm = current->mm; 932 #endif 933 gtt->userflags = flags; 934 return 0; 935 } 936 937 bool radeon_ttm_tt_has_userptr(struct ttm_tt *ttm) 938 { 939 struct radeon_ttm_tt *gtt = radeon_ttm_tt_to_gtt(ttm); 940 941 if (gtt == NULL) 942 return false; 943 944 return !!gtt->userptr; 945 } 946 947 bool radeon_ttm_tt_is_readonly(struct ttm_tt *ttm) 948 { 949 struct radeon_ttm_tt *gtt = radeon_ttm_tt_to_gtt(ttm); 950 951 if (gtt == NULL) 952 return false; 953 954 return !!(gtt->userflags & RADEON_GEM_USERPTR_READONLY); 955 } 956 957 static struct ttm_bo_driver radeon_bo_driver = { 958 .ttm_tt_create = &radeon_ttm_tt_create, 959 .ttm_tt_populate = &radeon_ttm_tt_populate, 960 .ttm_tt_unpopulate = &radeon_ttm_tt_unpopulate, 961 #ifdef __NetBSD__ 962 .ttm_tt_swapout = &radeon_ttm_tt_swapout, 963 .ttm_uvm_ops = &radeon_uvm_ops, 964 #endif 965 .invalidate_caches = &radeon_invalidate_caches, 966 .init_mem_type = &radeon_init_mem_type, 967 .eviction_valuable = ttm_bo_eviction_valuable, 968 .evict_flags = &radeon_evict_flags, 969 .move = &radeon_bo_move, 970 .verify_access = &radeon_verify_access, 971 .move_notify = &radeon_bo_move_notify, 972 .fault_reserve_notify = &radeon_bo_fault_reserve_notify, 973 .io_mem_reserve = &radeon_ttm_io_mem_reserve, 974 .io_mem_free = &radeon_ttm_io_mem_free, 975 }; 976 977 int radeon_ttm_init(struct radeon_device *rdev) 978 { 979 int r; 980 981 /* No others user of address space so set it to 0 */ 982 r = ttm_bo_device_init(&rdev->mman.bdev, 983 &radeon_bo_driver, 984 #ifdef __NetBSD__ 985 rdev->ddev->bst, 986 rdev->ddev->dmat, 987 #else 988 rdev->ddev->anon_inode->i_mapping, 989 #endif 990 rdev->ddev->vma_offset_manager, 991 dma_addressing_limited(&rdev->pdev->dev)); 992 if (r) { 993 DRM_ERROR("failed initializing buffer object driver(%d).\n", r); 994 return r; 995 } 996 rdev->mman.initialized = true; 997 r = ttm_bo_init_mm(&rdev->mman.bdev, TTM_PL_VRAM, 998 rdev->mc.real_vram_size >> PAGE_SHIFT); 999 if (r) { 1000 DRM_ERROR("Failed initializing VRAM heap.\n"); 1001 return r; 1002 } 1003 /* Change the size here instead of the init above so only lpfn is affected */ 1004 radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size); 1005 1006 r = radeon_bo_create(rdev, 256 * 1024, PAGE_SIZE, true, 1007 RADEON_GEM_DOMAIN_VRAM, 0, NULL, 1008 NULL, &rdev->stolen_vga_memory); 1009 if (r) { 1010 return r; 1011 } 1012 r = radeon_bo_reserve(rdev->stolen_vga_memory, false); 1013 if (r) 1014 return r; 1015 r = radeon_bo_pin(rdev->stolen_vga_memory, RADEON_GEM_DOMAIN_VRAM, NULL); 1016 radeon_bo_unreserve(rdev->stolen_vga_memory); 1017 if (r) { 1018 radeon_bo_unref(&rdev->stolen_vga_memory); 1019 return r; 1020 } 1021 DRM_INFO("radeon: %uM of VRAM memory ready\n", 1022 (unsigned) (rdev->mc.real_vram_size / (1024 * 1024))); 1023 r = ttm_bo_init_mm(&rdev->mman.bdev, TTM_PL_TT, 1024 rdev->mc.gtt_size >> PAGE_SHIFT); 1025 if (r) { 1026 DRM_ERROR("Failed initializing GTT heap.\n"); 1027 return r; 1028 } 1029 DRM_INFO("radeon: %uM of GTT memory ready.\n", 1030 (unsigned)(rdev->mc.gtt_size / (1024 * 1024))); 1031 1032 r = radeon_ttm_debugfs_init(rdev); 1033 if (r) { 1034 DRM_ERROR("Failed to init debugfs\n"); 1035 return r; 1036 } 1037 return 0; 1038 } 1039 1040 void radeon_ttm_fini(struct radeon_device *rdev) 1041 { 1042 int r; 1043 1044 if (!rdev->mman.initialized) 1045 return; 1046 radeon_ttm_debugfs_fini(rdev); 1047 if (rdev->stolen_vga_memory) { 1048 r = radeon_bo_reserve(rdev->stolen_vga_memory, false); 1049 if (r == 0) { 1050 radeon_bo_unpin(rdev->stolen_vga_memory); 1051 radeon_bo_unreserve(rdev->stolen_vga_memory); 1052 } 1053 radeon_bo_unref(&rdev->stolen_vga_memory); 1054 } 1055 ttm_bo_clean_mm(&rdev->mman.bdev, TTM_PL_VRAM); 1056 ttm_bo_clean_mm(&rdev->mman.bdev, TTM_PL_TT); 1057 ttm_bo_device_release(&rdev->mman.bdev); 1058 radeon_gart_fini(rdev); 1059 rdev->mman.initialized = false; 1060 DRM_INFO("radeon: ttm finalized\n"); 1061 } 1062 1063 /* this should only be called at bootup or when userspace 1064 * isn't running */ 1065 void radeon_ttm_set_active_vram_size(struct radeon_device *rdev, u64 size) 1066 { 1067 struct ttm_mem_type_manager *man; 1068 1069 if (!rdev->mman.initialized) 1070 return; 1071 1072 man = &rdev->mman.bdev.man[TTM_PL_VRAM]; 1073 /* this just adjusts TTM size idea, which sets lpfn to the correct value */ 1074 man->size = size >> PAGE_SHIFT; 1075 } 1076 1077 #ifdef __NetBSD__ 1078 1079 static int 1080 radeon_ttm_fault(struct uvm_faultinfo *ufi, vaddr_t vaddr, 1081 struct vm_page **pps, int npages, int centeridx, vm_prot_t access_type, 1082 int flags) 1083 { 1084 struct uvm_object *const uobj = ufi->entry->object.uvm_obj; 1085 struct ttm_buffer_object *const bo = container_of(uobj, 1086 struct ttm_buffer_object, uvmobj); 1087 struct radeon_device *const rdev = radeon_get_rdev(bo->bdev); 1088 int error; 1089 1090 KASSERT(rdev != NULL); 1091 down_read(&rdev->pm.mclk_lock); 1092 error = ttm_bo_uvm_fault(ufi, vaddr, pps, npages, centeridx, 1093 access_type, flags); 1094 up_read(&rdev->pm.mclk_lock); 1095 1096 return error; 1097 } 1098 1099 int 1100 radeon_mmap_object(struct drm_device *dev, off_t offset, size_t size, 1101 vm_prot_t prot, struct uvm_object **uobjp, voff_t *uoffsetp, 1102 struct file *file) 1103 { 1104 struct radeon_device *rdev = dev->dev_private; 1105 1106 KASSERT(0 == (offset & (PAGE_SIZE - 1))); 1107 1108 if (__predict_false(rdev == NULL)) /* XXX How?? */ 1109 return -EINVAL; 1110 1111 if (__predict_false((offset >> PAGE_SHIFT) < DRM_FILE_PAGE_OFFSET)) 1112 return -EINVAL; 1113 1114 return ttm_bo_mmap_object(&rdev->mman.bdev, offset, size, prot, 1115 uobjp, uoffsetp, file); 1116 } 1117 1118 #else 1119 1120 static vm_fault_t radeon_ttm_fault(struct vm_fault *vmf) 1121 { 1122 struct ttm_buffer_object *bo; 1123 struct radeon_device *rdev; 1124 vm_fault_t ret; 1125 1126 bo = (struct ttm_buffer_object *)vmf->vma->vm_private_data; 1127 if (bo == NULL) 1128 return VM_FAULT_NOPAGE; 1129 1130 rdev = radeon_get_rdev(bo->bdev); 1131 down_read(&rdev->pm.mclk_lock); 1132 ret = ttm_bo_vm_fault(vmf); 1133 up_read(&rdev->pm.mclk_lock); 1134 return ret; 1135 } 1136 1137 static struct vm_operations_struct radeon_ttm_vm_ops = { 1138 .fault = radeon_ttm_fault, 1139 .open = ttm_bo_vm_open, 1140 .close = ttm_bo_vm_close, 1141 .access = ttm_bo_vm_access 1142 }; 1143 1144 int radeon_mmap(struct file *filp, struct vm_area_struct *vma) 1145 { 1146 int r; 1147 struct drm_file *file_priv = filp->private_data; 1148 struct radeon_device *rdev = file_priv->minor->dev->dev_private; 1149 1150 if (rdev == NULL) 1151 return -EINVAL; 1152 1153 r = ttm_bo_mmap(filp, vma, &rdev->mman.bdev); 1154 if (unlikely(r != 0)) 1155 return r; 1156 1157 vma->vm_ops = &radeon_ttm_vm_ops; 1158 return 0; 1159 } 1160 1161 #endif /* __NetBSD__ */ 1162 1163 #if defined(CONFIG_DEBUG_FS) 1164 1165 static int radeon_mm_dump_table(struct seq_file *m, void *data) 1166 { 1167 struct drm_info_node *node = (struct drm_info_node *)m->private; 1168 unsigned ttm_pl = *(int*)node->info_ent->data; 1169 struct drm_device *dev = node->minor->dev; 1170 struct radeon_device *rdev = dev->dev_private; 1171 struct ttm_mem_type_manager *man = &rdev->mman.bdev.man[ttm_pl]; 1172 struct drm_printer p = drm_seq_file_printer(m); 1173 1174 man->func->debug(man, &p); 1175 return 0; 1176 } 1177 1178 1179 static int ttm_pl_vram = TTM_PL_VRAM; 1180 static int ttm_pl_tt = TTM_PL_TT; 1181 1182 static struct drm_info_list radeon_ttm_debugfs_list[] = { 1183 {"radeon_vram_mm", radeon_mm_dump_table, 0, &ttm_pl_vram}, 1184 {"radeon_gtt_mm", radeon_mm_dump_table, 0, &ttm_pl_tt}, 1185 {"ttm_page_pool", ttm_page_alloc_debugfs, 0, NULL}, 1186 #ifdef CONFIG_SWIOTLB 1187 {"ttm_dma_page_pool", ttm_dma_page_alloc_debugfs, 0, NULL} 1188 #endif 1189 }; 1190 1191 static int radeon_ttm_vram_open(struct inode *inode, struct file *filep) 1192 { 1193 struct radeon_device *rdev = inode->i_private; 1194 i_size_write(inode, rdev->mc.mc_vram_size); 1195 filep->private_data = inode->i_private; 1196 return 0; 1197 } 1198 1199 static ssize_t radeon_ttm_vram_read(struct file *f, char __user *buf, 1200 size_t size, loff_t *pos) 1201 { 1202 struct radeon_device *rdev = f->private_data; 1203 ssize_t result = 0; 1204 int r; 1205 1206 if (size & 0x3 || *pos & 0x3) 1207 return -EINVAL; 1208 1209 while (size) { 1210 unsigned long flags; 1211 uint32_t value; 1212 1213 if (*pos >= rdev->mc.mc_vram_size) 1214 return result; 1215 1216 spin_lock_irqsave(&rdev->mmio_idx_lock, flags); 1217 WREG32(RADEON_MM_INDEX, ((uint32_t)*pos) | 0x80000000); 1218 if (rdev->family >= CHIP_CEDAR) 1219 WREG32(EVERGREEN_MM_INDEX_HI, *pos >> 31); 1220 value = RREG32(RADEON_MM_DATA); 1221 spin_unlock_irqrestore(&rdev->mmio_idx_lock, flags); 1222 1223 r = put_user(value, (uint32_t *)buf); 1224 if (r) 1225 return r; 1226 1227 result += 4; 1228 buf += 4; 1229 *pos += 4; 1230 size -= 4; 1231 } 1232 1233 return result; 1234 } 1235 1236 static const struct file_operations radeon_ttm_vram_fops = { 1237 .owner = THIS_MODULE, 1238 .open = radeon_ttm_vram_open, 1239 .read = radeon_ttm_vram_read, 1240 .llseek = default_llseek 1241 }; 1242 1243 static int radeon_ttm_gtt_open(struct inode *inode, struct file *filep) 1244 { 1245 struct radeon_device *rdev = inode->i_private; 1246 i_size_write(inode, rdev->mc.gtt_size); 1247 filep->private_data = inode->i_private; 1248 return 0; 1249 } 1250 1251 static ssize_t radeon_ttm_gtt_read(struct file *f, char __user *buf, 1252 size_t size, loff_t *pos) 1253 { 1254 struct radeon_device *rdev = f->private_data; 1255 ssize_t result = 0; 1256 int r; 1257 1258 while (size) { 1259 loff_t p = *pos / PAGE_SIZE; 1260 unsigned off = *pos & ~PAGE_MASK; 1261 size_t cur_size = min_t(size_t, size, PAGE_SIZE - off); 1262 struct page *page; 1263 void *ptr; 1264 1265 if (p >= rdev->gart.num_cpu_pages) 1266 return result; 1267 1268 page = rdev->gart.pages[p]; 1269 if (page) { 1270 ptr = kmap(page); 1271 ptr += off; 1272 1273 r = copy_to_user(buf, ptr, cur_size); 1274 kunmap(rdev->gart.pages[p]); 1275 } else 1276 r = clear_user(buf, cur_size); 1277 1278 if (r) 1279 return -EFAULT; 1280 1281 result += cur_size; 1282 buf += cur_size; 1283 *pos += cur_size; 1284 size -= cur_size; 1285 } 1286 1287 return result; 1288 } 1289 1290 static const struct file_operations radeon_ttm_gtt_fops = { 1291 .owner = THIS_MODULE, 1292 .open = radeon_ttm_gtt_open, 1293 .read = radeon_ttm_gtt_read, 1294 .llseek = default_llseek 1295 }; 1296 1297 #endif 1298 1299 static int radeon_ttm_debugfs_init(struct radeon_device *rdev) 1300 { 1301 #if defined(CONFIG_DEBUG_FS) 1302 unsigned count; 1303 1304 struct drm_minor *minor = rdev->ddev->primary; 1305 struct dentry *root = minor->debugfs_root; 1306 1307 rdev->mman.vram = debugfs_create_file("radeon_vram", S_IFREG | S_IRUGO, 1308 root, rdev, 1309 &radeon_ttm_vram_fops); 1310 1311 rdev->mman.gtt = debugfs_create_file("radeon_gtt", S_IFREG | S_IRUGO, 1312 root, rdev, &radeon_ttm_gtt_fops); 1313 1314 count = ARRAY_SIZE(radeon_ttm_debugfs_list); 1315 1316 #ifdef CONFIG_SWIOTLB 1317 if (!(rdev->need_swiotlb && swiotlb_nr_tbl())) 1318 --count; 1319 #endif 1320 1321 return radeon_debugfs_add_files(rdev, radeon_ttm_debugfs_list, count); 1322 #else 1323 1324 return 0; 1325 #endif 1326 } 1327 1328 static void radeon_ttm_debugfs_fini(struct radeon_device *rdev) 1329 { 1330 #if defined(CONFIG_DEBUG_FS) 1331 1332 debugfs_remove(rdev->mman.vram); 1333 rdev->mman.vram = NULL; 1334 1335 debugfs_remove(rdev->mman.gtt); 1336 rdev->mman.gtt = NULL; 1337 #endif 1338 } 1339
Indexes created Thu Oct 23 22:10:10 GMT 2025