1 /* $NetBSD: nouveau_dmem.c,v 1.3 2021/12/19 11:34:44 riastradh Exp $ */ 2 3 /* 4 * Copyright 2018 Red Hat Inc. 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining a 7 * copy of this software and associated documentation files (the "Software"), 8 * to deal in the Software without restriction, including without limitation 9 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 10 * and/or sell copies of the Software, and to permit persons to whom the 11 * Software is furnished to do so, subject to the following conditions: 12 * 13 * The above copyright notice and this permission notice shall be included in 14 * all copies or substantial portions of the Software. 15 * 16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 19 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 22 * OTHER DEALINGS IN THE SOFTWARE. 23 */ 24 #include <sys/cdefs.h> 25 __KERNEL_RCSID(0, "$NetBSD: nouveau_dmem.c,v 1.3 2021/12/19 11:34:44 riastradh Exp $"); 26 27 #include "nouveau_dmem.h" 28 #include "nouveau_drv.h" 29 #include "nouveau_chan.h" 30 #include "nouveau_dma.h" 31 #include "nouveau_mem.h" 32 #include "nouveau_bo.h" 33 34 #include <nvif/class.h> 35 #include <nvif/object.h> 36 #include <nvif/if500b.h> 37 #include <nvif/if900b.h> 38 39 #include <linux/sched/mm.h> 40 #include <linux/hmm.h> 41 42 /* 43 * FIXME: this is ugly right now we are using TTM to allocate vram and we pin 44 * it in vram while in use. We likely want to overhaul memory management for 45 * nouveau to be more page like (not necessarily with system page size but a 46 * bigger page size) at lowest level and have some shim layer on top that would 47 * provide the same functionality as TTM. 48 */ 49 #define DMEM_CHUNK_SIZE (2UL << 20) 50 #define DMEM_CHUNK_NPAGES (DMEM_CHUNK_SIZE >> PAGE_SHIFT) 51 52 enum nouveau_aper { 53 NOUVEAU_APER_VIRT, 54 NOUVEAU_APER_VRAM, 55 NOUVEAU_APER_HOST, 56 }; 57 58 typedef int (*nouveau_migrate_copy_t)(struct nouveau_drm *drm, u64 npages, 59 enum nouveau_aper, u64 dst_addr, 60 enum nouveau_aper, u64 src_addr); 61 62 struct nouveau_dmem_chunk { 63 struct list_head list; 64 struct nouveau_bo *bo; 65 struct nouveau_drm *drm; 66 unsigned long pfn_first; 67 unsigned long callocated; 68 unsigned long bitmap[BITS_TO_LONGS(DMEM_CHUNK_NPAGES)]; 69 spinlock_t lock; 70 }; 71 72 struct nouveau_dmem_migrate { 73 nouveau_migrate_copy_t copy_func; 74 struct nouveau_channel *chan; 75 }; 76 77 struct nouveau_dmem { 78 struct nouveau_drm *drm; 79 struct dev_pagemap pagemap; 80 struct nouveau_dmem_migrate migrate; 81 struct list_head chunk_free; 82 struct list_head chunk_full; 83 struct list_head chunk_empty; 84 struct mutex mutex; 85 }; 86 87 static inline struct nouveau_dmem *page_to_dmem(struct page *page) 88 { 89 return container_of(page->pgmap, struct nouveau_dmem, pagemap); 90 } 91 92 static unsigned long nouveau_dmem_page_addr(struct page *page) 93 { 94 struct nouveau_dmem_chunk *chunk = page->zone_device_data; 95 unsigned long idx = page_to_pfn(page) - chunk->pfn_first; 96 97 return (idx << PAGE_SHIFT) + chunk->bo->bo.offset; 98 } 99 100 static void nouveau_dmem_page_free(struct page *page) 101 { 102 struct nouveau_dmem_chunk *chunk = page->zone_device_data; 103 unsigned long idx = page_to_pfn(page) - chunk->pfn_first; 104 105 /* 106 * FIXME: 107 * 108 * This is really a bad example, we need to overhaul nouveau memory 109 * management to be more page focus and allow lighter locking scheme 110 * to be use in the process. 111 */ 112 spin_lock(&chunk->lock); 113 clear_bit(idx, chunk->bitmap); 114 WARN_ON(!chunk->callocated); 115 chunk->callocated--; 116 /* 117 * FIXME when chunk->callocated reach 0 we should add the chunk to 118 * a reclaim list so that it can be freed in case of memory pressure. 119 */ 120 spin_unlock(&chunk->lock); 121 } 122 123 static void nouveau_dmem_fence_done(struct nouveau_fence **fence) 124 { 125 if (fence) { 126 nouveau_fence_wait(*fence, true, false); 127 nouveau_fence_unref(fence); 128 } else { 129 /* 130 * FIXME wait for channel to be IDLE before calling finalizing 131 * the hmem object. 132 */ 133 } 134 } 135 136 static vm_fault_t nouveau_dmem_fault_copy_one(struct nouveau_drm *drm, 137 struct vm_fault *vmf, struct migrate_vma *args, 138 dma_addr_t *dma_addr) 139 { 140 struct device *dev = drm->dev->dev; 141 struct page *dpage, *spage; 142 143 spage = migrate_pfn_to_page(args->src[0]); 144 if (!spage || !(args->src[0] & MIGRATE_PFN_MIGRATE)) 145 return 0; 146 147 dpage = alloc_page_vma(GFP_HIGHUSER, vmf->vma, vmf->address); 148 if (!dpage) 149 return VM_FAULT_SIGBUS; 150 lock_page(dpage); 151 152 *dma_addr = dma_map_page(dev, dpage, 0, PAGE_SIZE, DMA_BIDIRECTIONAL); 153 if (dma_mapping_error(dev, *dma_addr)) 154 goto error_free_page; 155 156 if (drm->dmem->migrate.copy_func(drm, 1, NOUVEAU_APER_HOST, *dma_addr, 157 NOUVEAU_APER_VRAM, nouveau_dmem_page_addr(spage))) 158 goto error_dma_unmap; 159 160 args->dst[0] = migrate_pfn(page_to_pfn(dpage)) | MIGRATE_PFN_LOCKED; 161 return 0; 162 163 error_dma_unmap: 164 dma_unmap_page(dev, *dma_addr, PAGE_SIZE, DMA_BIDIRECTIONAL); 165 error_free_page: 166 __free_page(dpage); 167 return VM_FAULT_SIGBUS; 168 } 169 170 static vm_fault_t nouveau_dmem_migrate_to_ram(struct vm_fault *vmf) 171 { 172 struct nouveau_dmem *dmem = page_to_dmem(vmf->page); 173 struct nouveau_drm *drm = dmem->drm; 174 struct nouveau_fence *fence; 175 unsigned long src = 0, dst = 0; 176 dma_addr_t dma_addr = 0; 177 vm_fault_t ret; 178 struct migrate_vma args = { 179 .vma = vmf->vma, 180 .start = vmf->address, 181 .end = vmf->address + PAGE_SIZE, 182 .src = &src, 183 .dst = &dst, 184 }; 185 186 /* 187 * FIXME what we really want is to find some heuristic to migrate more 188 * than just one page on CPU fault. When such fault happens it is very 189 * likely that more surrounding page will CPU fault too. 190 */ 191 if (migrate_vma_setup(&args) < 0) 192 return VM_FAULT_SIGBUS; 193 if (!args.cpages) 194 return 0; 195 196 ret = nouveau_dmem_fault_copy_one(drm, vmf, &args, &dma_addr); 197 if (ret || dst == 0) 198 goto done; 199 200 nouveau_fence_new(dmem->migrate.chan, false, &fence); 201 migrate_vma_pages(&args); 202 nouveau_dmem_fence_done(&fence); 203 dma_unmap_page(drm->dev->dev, dma_addr, PAGE_SIZE, DMA_BIDIRECTIONAL); 204 done: 205 migrate_vma_finalize(&args); 206 return ret; 207 } 208 209 static const struct dev_pagemap_ops nouveau_dmem_pagemap_ops = { 210 .page_free = nouveau_dmem_page_free, 211 .migrate_to_ram = nouveau_dmem_migrate_to_ram, 212 }; 213 214 static int 215 nouveau_dmem_chunk_alloc(struct nouveau_drm *drm) 216 { 217 struct nouveau_dmem_chunk *chunk; 218 int ret; 219 220 if (drm->dmem == NULL) 221 return -EINVAL; 222 223 mutex_lock(&drm->dmem->mutex); 224 chunk = list_first_entry_or_null(&drm->dmem->chunk_empty, 225 struct nouveau_dmem_chunk, 226 list); 227 if (chunk == NULL) { 228 mutex_unlock(&drm->dmem->mutex); 229 return -ENOMEM; 230 } 231 232 list_del(&chunk->list); 233 mutex_unlock(&drm->dmem->mutex); 234 235 ret = nouveau_bo_new(&drm->client, DMEM_CHUNK_SIZE, 0, 236 TTM_PL_FLAG_VRAM, 0, 0, NULL, NULL, 237 &chunk->bo); 238 if (ret) 239 goto out; 240 241 ret = nouveau_bo_pin(chunk->bo, TTM_PL_FLAG_VRAM, false); 242 if (ret) { 243 nouveau_bo_ref(NULL, &chunk->bo); 244 goto out; 245 } 246 247 bitmap_zero(chunk->bitmap, DMEM_CHUNK_NPAGES); 248 spin_lock_init(&chunk->lock); 249 250 out: 251 mutex_lock(&drm->dmem->mutex); 252 if (chunk->bo) 253 list_add(&chunk->list, &drm->dmem->chunk_empty); 254 else 255 list_add_tail(&chunk->list, &drm->dmem->chunk_empty); 256 mutex_unlock(&drm->dmem->mutex); 257 258 return ret; 259 } 260 261 static struct nouveau_dmem_chunk * 262 nouveau_dmem_chunk_first_free_locked(struct nouveau_drm *drm) 263 { 264 struct nouveau_dmem_chunk *chunk; 265 266 chunk = list_first_entry_or_null(&drm->dmem->chunk_free, 267 struct nouveau_dmem_chunk, 268 list); 269 if (chunk) 270 return chunk; 271 272 chunk = list_first_entry_or_null(&drm->dmem->chunk_empty, 273 struct nouveau_dmem_chunk, 274 list); 275 if (chunk->bo) 276 return chunk; 277 278 return NULL; 279 } 280 281 static int 282 nouveau_dmem_pages_alloc(struct nouveau_drm *drm, 283 unsigned long npages, 284 unsigned long *pages) 285 { 286 struct nouveau_dmem_chunk *chunk; 287 unsigned long c; 288 int ret; 289 290 memset(pages, 0xff, npages * sizeof(*pages)); 291 292 mutex_lock(&drm->dmem->mutex); 293 for (c = 0; c < npages;) { 294 unsigned long i; 295 296 chunk = nouveau_dmem_chunk_first_free_locked(drm); 297 if (chunk == NULL) { 298 mutex_unlock(&drm->dmem->mutex); 299 ret = nouveau_dmem_chunk_alloc(drm); 300 if (ret) { 301 if (c) 302 return 0; 303 return ret; 304 } 305 mutex_lock(&drm->dmem->mutex); 306 continue; 307 } 308 309 spin_lock(&chunk->lock); 310 i = find_first_zero_bit(chunk->bitmap, DMEM_CHUNK_NPAGES); 311 while (i < DMEM_CHUNK_NPAGES && c < npages) { 312 pages[c] = chunk->pfn_first + i; 313 set_bit(i, chunk->bitmap); 314 chunk->callocated++; 315 c++; 316 317 i = find_next_zero_bit(chunk->bitmap, 318 DMEM_CHUNK_NPAGES, i); 319 } 320 spin_unlock(&chunk->lock); 321 } 322 mutex_unlock(&drm->dmem->mutex); 323 324 return 0; 325 } 326 327 static struct page * 328 nouveau_dmem_page_alloc_locked(struct nouveau_drm *drm) 329 { 330 unsigned long pfns[1]; 331 struct page *page; 332 int ret; 333 334 /* FIXME stop all the miss-match API ... */ 335 ret = nouveau_dmem_pages_alloc(drm, 1, pfns); 336 if (ret) 337 return NULL; 338 339 page = pfn_to_page(pfns[0]); 340 get_page(page); 341 lock_page(page); 342 return page; 343 } 344 345 static void 346 nouveau_dmem_page_free_locked(struct nouveau_drm *drm, struct page *page) 347 { 348 unlock_page(page); 349 put_page(page); 350 } 351 352 void 353 nouveau_dmem_resume(struct nouveau_drm *drm) 354 { 355 struct nouveau_dmem_chunk *chunk; 356 int ret; 357 358 if (drm->dmem == NULL) 359 return; 360 361 mutex_lock(&drm->dmem->mutex); 362 list_for_each_entry (chunk, &drm->dmem->chunk_free, list) { 363 ret = nouveau_bo_pin(chunk->bo, TTM_PL_FLAG_VRAM, false); 364 /* FIXME handle pin failure */ 365 WARN_ON(ret); 366 } 367 list_for_each_entry (chunk, &drm->dmem->chunk_full, list) { 368 ret = nouveau_bo_pin(chunk->bo, TTM_PL_FLAG_VRAM, false); 369 /* FIXME handle pin failure */ 370 WARN_ON(ret); 371 } 372 mutex_unlock(&drm->dmem->mutex); 373 } 374 375 void 376 nouveau_dmem_suspend(struct nouveau_drm *drm) 377 { 378 struct nouveau_dmem_chunk *chunk; 379 380 if (drm->dmem == NULL) 381 return; 382 383 mutex_lock(&drm->dmem->mutex); 384 list_for_each_entry (chunk, &drm->dmem->chunk_free, list) { 385 nouveau_bo_unpin(chunk->bo); 386 } 387 list_for_each_entry (chunk, &drm->dmem->chunk_full, list) { 388 nouveau_bo_unpin(chunk->bo); 389 } 390 mutex_unlock(&drm->dmem->mutex); 391 } 392 393 void 394 nouveau_dmem_fini(struct nouveau_drm *drm) 395 { 396 struct nouveau_dmem_chunk *chunk, *tmp; 397 398 if (drm->dmem == NULL) 399 return; 400 401 mutex_lock(&drm->dmem->mutex); 402 403 WARN_ON(!list_empty(&drm->dmem->chunk_free)); 404 WARN_ON(!list_empty(&drm->dmem->chunk_full)); 405 406 list_for_each_entry_safe (chunk, tmp, &drm->dmem->chunk_empty, list) { 407 if (chunk->bo) { 408 nouveau_bo_unpin(chunk->bo); 409 nouveau_bo_ref(NULL, &chunk->bo); 410 } 411 list_del(&chunk->list); 412 spin_lock_destroy(&chunk->lock); 413 kfree(chunk); 414 } 415 416 mutex_unlock(&drm->dmem->mutex); 417 } 418 419 static int 420 nvc0b5_migrate_copy(struct nouveau_drm *drm, u64 npages, 421 enum nouveau_aper dst_aper, u64 dst_addr, 422 enum nouveau_aper src_aper, u64 src_addr) 423 { 424 struct nouveau_channel *chan = drm->dmem->migrate.chan; 425 u32 launch_dma = (1 << 9) /* MULTI_LINE_ENABLE. */ | 426 (1 << 8) /* DST_MEMORY_LAYOUT_PITCH. */ | 427 (1 << 7) /* SRC_MEMORY_LAYOUT_PITCH. */ | 428 (1 << 2) /* FLUSH_ENABLE_TRUE. */ | 429 (2 << 0) /* DATA_TRANSFER_TYPE_NON_PIPELINED. */; 430 int ret; 431 432 ret = RING_SPACE(chan, 13); 433 if (ret) 434 return ret; 435 436 if (src_aper != NOUVEAU_APER_VIRT) { 437 switch (src_aper) { 438 case NOUVEAU_APER_VRAM: 439 BEGIN_IMC0(chan, NvSubCopy, 0x0260, 0); 440 break; 441 case NOUVEAU_APER_HOST: 442 BEGIN_IMC0(chan, NvSubCopy, 0x0260, 1); 443 break; 444 default: 445 return -EINVAL; 446 } 447 launch_dma |= 0x00001000; /* SRC_TYPE_PHYSICAL. */ 448 } 449 450 if (dst_aper != NOUVEAU_APER_VIRT) { 451 switch (dst_aper) { 452 case NOUVEAU_APER_VRAM: 453 BEGIN_IMC0(chan, NvSubCopy, 0x0264, 0); 454 break; 455 case NOUVEAU_APER_HOST: 456 BEGIN_IMC0(chan, NvSubCopy, 0x0264, 1); 457 break; 458 default: 459 return -EINVAL; 460 } 461 launch_dma |= 0x00002000; /* DST_TYPE_PHYSICAL. */ 462 } 463 464 BEGIN_NVC0(chan, NvSubCopy, 0x0400, 8); 465 OUT_RING (chan, upper_32_bits(src_addr)); 466 OUT_RING (chan, lower_32_bits(src_addr)); 467 OUT_RING (chan, upper_32_bits(dst_addr)); 468 OUT_RING (chan, lower_32_bits(dst_addr)); 469 OUT_RING (chan, PAGE_SIZE); 470 OUT_RING (chan, PAGE_SIZE); 471 OUT_RING (chan, PAGE_SIZE); 472 OUT_RING (chan, npages); 473 BEGIN_NVC0(chan, NvSubCopy, 0x0300, 1); 474 OUT_RING (chan, launch_dma); 475 return 0; 476 } 477 478 static int 479 nouveau_dmem_migrate_init(struct nouveau_drm *drm) 480 { 481 switch (drm->ttm.copy.oclass) { 482 case PASCAL_DMA_COPY_A: 483 case PASCAL_DMA_COPY_B: 484 case VOLTA_DMA_COPY_A: 485 case TURING_DMA_COPY_A: 486 drm->dmem->migrate.copy_func = nvc0b5_migrate_copy; 487 drm->dmem->migrate.chan = drm->ttm.chan; 488 return 0; 489 default: 490 break; 491 } 492 return -ENODEV; 493 } 494 495 void 496 nouveau_dmem_init(struct nouveau_drm *drm) 497 { 498 struct device *device = drm->dev->dev; 499 struct resource *res; 500 unsigned long i, size, pfn_first; 501 int ret; 502 503 /* This only make sense on PASCAL or newer */ 504 if (drm->client.device.info.family < NV_DEVICE_INFO_V0_PASCAL) 505 return; 506 507 if (!(drm->dmem = kzalloc(sizeof(*drm->dmem), GFP_KERNEL))) 508 return; 509 510 drm->dmem->drm = drm; 511 mutex_init(&drm->dmem->mutex); 512 INIT_LIST_HEAD(&drm->dmem->chunk_free); 513 INIT_LIST_HEAD(&drm->dmem->chunk_full); 514 INIT_LIST_HEAD(&drm->dmem->chunk_empty); 515 516 size = ALIGN(drm->client.device.info.ram_user, DMEM_CHUNK_SIZE); 517 518 /* Initialize migration dma helpers before registering memory */ 519 ret = nouveau_dmem_migrate_init(drm); 520 if (ret) 521 goto out_free; 522 523 /* 524 * FIXME we need some kind of policy to decide how much VRAM we 525 * want to register with HMM. For now just register everything 526 * and latter if we want to do thing like over commit then we 527 * could revisit this. 528 */ 529 res = devm_request_free_mem_region(device, &iomem_resource, size); 530 if (IS_ERR(res)) 531 goto out_free; 532 drm->dmem->pagemap.type = MEMORY_DEVICE_PRIVATE; 533 drm->dmem->pagemap.res = *res; 534 drm->dmem->pagemap.ops = &nouveau_dmem_pagemap_ops; 535 if (IS_ERR(devm_memremap_pages(device, &drm->dmem->pagemap))) 536 goto out_free; 537 538 pfn_first = res->start >> PAGE_SHIFT; 539 for (i = 0; i < (size / DMEM_CHUNK_SIZE); ++i) { 540 struct nouveau_dmem_chunk *chunk; 541 struct page *page; 542 unsigned long j; 543 544 chunk = kzalloc(sizeof(*chunk), GFP_KERNEL); 545 if (chunk == NULL) { 546 nouveau_dmem_fini(drm); 547 return; 548 } 549 550 chunk->drm = drm; 551 chunk->pfn_first = pfn_first + (i * DMEM_CHUNK_NPAGES); 552 list_add_tail(&chunk->list, &drm->dmem->chunk_empty); 553 554 page = pfn_to_page(chunk->pfn_first); 555 for (j = 0; j < DMEM_CHUNK_NPAGES; ++j, ++page) 556 page->zone_device_data = chunk; 557 } 558 559 NV_INFO(drm, "DMEM: registered %ldMB of device memory\n", size >> 20); 560 return; 561 out_free: 562 mutex_destroy(&drm->dmem->mutex); 563 kfree(drm->dmem); 564 drm->dmem = NULL; 565 } 566 567 static unsigned long nouveau_dmem_migrate_copy_one(struct nouveau_drm *drm, 568 unsigned long src, dma_addr_t *dma_addr) 569 { 570 struct device *dev = drm->dev->dev; 571 struct page *dpage, *spage; 572 573 spage = migrate_pfn_to_page(src); 574 if (!spage || !(src & MIGRATE_PFN_MIGRATE)) 575 goto out; 576 577 dpage = nouveau_dmem_page_alloc_locked(drm); 578 if (!dpage) 579 return 0; 580 581 *dma_addr = dma_map_page(dev, spage, 0, PAGE_SIZE, DMA_BIDIRECTIONAL); 582 if (dma_mapping_error(dev, *dma_addr)) 583 goto out_free_page; 584 585 if (drm->dmem->migrate.copy_func(drm, 1, NOUVEAU_APER_VRAM, 586 nouveau_dmem_page_addr(dpage), NOUVEAU_APER_HOST, 587 *dma_addr)) 588 goto out_dma_unmap; 589 590 return migrate_pfn(page_to_pfn(dpage)) | MIGRATE_PFN_LOCKED; 591 592 out_dma_unmap: 593 dma_unmap_page(dev, *dma_addr, PAGE_SIZE, DMA_BIDIRECTIONAL); 594 out_free_page: 595 nouveau_dmem_page_free_locked(drm, dpage); 596 out: 597 return 0; 598 } 599 600 static void nouveau_dmem_migrate_chunk(struct nouveau_drm *drm, 601 struct migrate_vma *args, dma_addr_t *dma_addrs) 602 { 603 struct nouveau_fence *fence; 604 unsigned long addr = args->start, nr_dma = 0, i; 605 606 for (i = 0; addr < args->end; i++) { 607 args->dst[i] = nouveau_dmem_migrate_copy_one(drm, args->src[i], 608 dma_addrs + nr_dma); 609 if (args->dst[i]) 610 nr_dma++; 611 addr += PAGE_SIZE; 612 } 613 614 nouveau_fence_new(drm->dmem->migrate.chan, false, &fence); 615 migrate_vma_pages(args); 616 nouveau_dmem_fence_done(&fence); 617 618 while (nr_dma--) { 619 dma_unmap_page(drm->dev->dev, dma_addrs[nr_dma], PAGE_SIZE, 620 DMA_BIDIRECTIONAL); 621 } 622 /* 623 * FIXME optimization: update GPU page table to point to newly migrated 624 * memory. 625 */ 626 migrate_vma_finalize(args); 627 } 628 629 int 630 nouveau_dmem_migrate_vma(struct nouveau_drm *drm, 631 struct vm_area_struct *vma, 632 unsigned long start, 633 unsigned long end) 634 { 635 unsigned long npages = (end - start) >> PAGE_SHIFT; 636 unsigned long max = min(SG_MAX_SINGLE_ALLOC, npages); 637 dma_addr_t *dma_addrs; 638 struct migrate_vma args = { 639 .vma = vma, 640 .start = start, 641 }; 642 unsigned long c, i; 643 int ret = -ENOMEM; 644 645 args.src = kcalloc(max, sizeof(*args.src), GFP_KERNEL); 646 if (!args.src) 647 goto out; 648 args.dst = kcalloc(max, sizeof(*args.dst), GFP_KERNEL); 649 if (!args.dst) 650 goto out_free_src; 651 652 dma_addrs = kmalloc_array(max, sizeof(*dma_addrs), GFP_KERNEL); 653 if (!dma_addrs) 654 goto out_free_dst; 655 656 for (i = 0; i < npages; i += c) { 657 c = min(SG_MAX_SINGLE_ALLOC, npages); 658 args.end = start + (c << PAGE_SHIFT); 659 ret = migrate_vma_setup(&args); 660 if (ret) 661 goto out_free_dma; 662 663 if (args.cpages) 664 nouveau_dmem_migrate_chunk(drm, &args, dma_addrs); 665 args.start = args.end; 666 } 667 668 ret = 0; 669 out_free_dma: 670 kfree(dma_addrs); 671 out_free_dst: 672 kfree(args.dst); 673 out_free_src: 674 kfree(args.src); 675 out: 676 return ret; 677 } 678 679 static inline bool 680 nouveau_dmem_page(struct nouveau_drm *drm, struct page *page) 681 { 682 return is_device_private_page(page) && drm->dmem == page_to_dmem(page); 683 } 684 685 void 686 nouveau_dmem_convert_pfn(struct nouveau_drm *drm, 687 struct hmm_range *range) 688 { 689 unsigned long i, npages; 690 691 npages = (range->end - range->start) >> PAGE_SHIFT; 692 for (i = 0; i < npages; ++i) { 693 struct page *page; 694 uint64_t addr; 695 696 page = hmm_device_entry_to_page(range, range->pfns[i]); 697 if (page == NULL) 698 continue; 699 700 if (!(range->pfns[i] & range->flags[HMM_PFN_DEVICE_PRIVATE])) { 701 continue; 702 } 703 704 if (!nouveau_dmem_page(drm, page)) { 705 WARN(1, "Some unknown device memory !\n"); 706 range->pfns[i] = 0; 707 continue; 708 } 709 710 addr = nouveau_dmem_page_addr(page); 711 range->pfns[i] &= ((1UL << range->pfn_shift) - 1); 712 range->pfns[i] |= (addr >> PAGE_SHIFT) << range->pfn_shift; 713 } 714 } 715
Indexes created Tue Sep 30 20:09:53 GMT 2025