1 /* $NetBSD: nouveau_gem.c,v 1.14 2022/05/31 00:17:10 riastradh Exp $ */ 2 3 /* 4 * Copyright (C) 2008 Ben Skeggs. 5 * All Rights Reserved. 6 * 7 * Permission is hereby granted, free of charge, to any person obtaining 8 * a copy of this software and associated documentation files (the 9 * "Software"), to deal in the Software without restriction, including 10 * without limitation the rights to use, copy, modify, merge, publish, 11 * distribute, sublicense, and/or sell copies of the Software, and to 12 * permit persons to whom the Software is furnished to do so, subject to 13 * the following conditions: 14 * 15 * The above copyright notice and this permission notice (including the 16 * next paragraph) shall be included in all copies or substantial 17 * portions of the Software. 18 * 19 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 20 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 21 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. 22 * IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE 23 * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 24 * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 25 * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 26 * 27 */ 28 29 #include <sys/cdefs.h> 30 __KERNEL_RCSID(0, "$NetBSD: nouveau_gem.c,v 1.14 2022/05/31 00:17:10 riastradh Exp $"); 31 32 #include <asm/uaccess.h> 33 34 #include "nouveau_drv.h" 35 #include "nouveau_dma.h" 36 #include "nouveau_fence.h" 37 #include "nouveau_abi16.h" 38 39 #include "nouveau_ttm.h" 40 #include "nouveau_gem.h" 41 #include "nouveau_mem.h" 42 #include "nouveau_vmm.h" 43 44 #include <nvif/class.h> 45 46 #include <linux/nbsd-namespace.h> 47 48 void 49 nouveau_gem_object_del(struct drm_gem_object *gem) 50 { 51 struct nouveau_bo *nvbo = nouveau_gem_object(gem); 52 struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev); 53 struct device *dev = drm->dev->dev; 54 int ret; 55 56 ret = pm_runtime_get_sync(dev); 57 if (WARN_ON(ret < 0 && ret != -EACCES)) 58 return; 59 60 if (gem->import_attach) 61 drm_prime_gem_destroy(gem, nvbo->bo.sg); 62 63 ttm_bo_put(&nvbo->bo); 64 65 pm_runtime_mark_last_busy(dev); 66 pm_runtime_put_autosuspend(dev); 67 } 68 69 int 70 nouveau_gem_object_open(struct drm_gem_object *gem, struct drm_file *file_priv) 71 { 72 struct nouveau_cli *cli = nouveau_cli(file_priv); 73 struct nouveau_bo *nvbo = nouveau_gem_object(gem); 74 struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev); 75 struct device *dev = drm->dev->dev; 76 struct nouveau_vmm *vmm = cli->svm.cli ? &cli->svm : &cli->vmm; 77 struct nouveau_vma *vma; 78 int ret; 79 80 if (vmm->vmm.object.oclass < NVIF_CLASS_VMM_NV50) 81 return 0; 82 83 ret = ttm_bo_reserve(&nvbo->bo, false, false, NULL); 84 if (ret) 85 return ret; 86 87 ret = pm_runtime_get_sync(dev); 88 if (ret < 0 && ret != -EACCES) 89 goto out; 90 91 ret = nouveau_vma_new(nvbo, vmm, &vma); 92 pm_runtime_mark_last_busy(dev); 93 pm_runtime_put_autosuspend(dev); 94 out: 95 ttm_bo_unreserve(&nvbo->bo); 96 return ret; 97 } 98 99 struct nouveau_gem_object_unmap { 100 struct nouveau_cli_work work; 101 struct nouveau_vma *vma; 102 }; 103 104 static void 105 nouveau_gem_object_delete(struct nouveau_vma *vma) 106 { 107 nouveau_fence_unref(&vma->fence); 108 nouveau_vma_del(&vma); 109 } 110 111 static void 112 nouveau_gem_object_delete_work(struct nouveau_cli_work *w) 113 { 114 struct nouveau_gem_object_unmap *work = 115 container_of(w, typeof(*work), work); 116 nouveau_gem_object_delete(work->vma); 117 kfree(work); 118 } 119 120 static void 121 nouveau_gem_object_unmap(struct nouveau_bo *nvbo, struct nouveau_vma *vma) 122 { 123 struct dma_fence *fence = vma->fence ? &vma->fence->base : NULL; 124 struct nouveau_gem_object_unmap *work; 125 126 list_del_init(&vma->head); 127 128 if (!fence) { 129 nouveau_gem_object_delete(vma); 130 return; 131 } 132 133 if (!(work = kmalloc(sizeof(*work), GFP_KERNEL))) { 134 WARN_ON(dma_fence_wait_timeout(fence, false, 2 * HZ) <= 0); 135 nouveau_gem_object_delete(vma); 136 return; 137 } 138 139 work->work.func = nouveau_gem_object_delete_work; 140 work->vma = vma; 141 nouveau_cli_work_queue(vma->vmm->cli, fence, &work->work); 142 } 143 144 void 145 nouveau_gem_object_close(struct drm_gem_object *gem, struct drm_file *file_priv) 146 { 147 struct nouveau_cli *cli = nouveau_cli(file_priv); 148 struct nouveau_bo *nvbo = nouveau_gem_object(gem); 149 struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev); 150 struct device *dev = drm->dev->dev; 151 struct nouveau_vmm *vmm = cli->svm.cli ? &cli->svm : & cli->vmm; 152 struct nouveau_vma *vma; 153 int ret; 154 155 if (vmm->vmm.object.oclass < NVIF_CLASS_VMM_NV50) 156 return; 157 158 ret = ttm_bo_reserve(&nvbo->bo, false, false, NULL); 159 if (ret) 160 return; 161 162 vma = nouveau_vma_find(nvbo, vmm); 163 if (vma) { 164 if (--vma->refs == 0) { 165 ret = pm_runtime_get_sync(dev); 166 if (!WARN_ON(ret < 0 && ret != -EACCES)) { 167 nouveau_gem_object_unmap(nvbo, vma); 168 pm_runtime_mark_last_busy(dev); 169 pm_runtime_put_autosuspend(dev); 170 } 171 } 172 } 173 ttm_bo_unreserve(&nvbo->bo); 174 } 175 176 int 177 nouveau_gem_new(struct nouveau_cli *cli, u64 size, int align, uint32_t domain, 178 uint32_t tile_mode, uint32_t tile_flags, 179 struct nouveau_bo **pnvbo) 180 { 181 struct nouveau_drm *drm = cli->drm; 182 struct nouveau_bo *nvbo; 183 u32 flags = 0; 184 int ret; 185 186 if (domain & NOUVEAU_GEM_DOMAIN_VRAM) 187 flags |= TTM_PL_FLAG_VRAM; 188 if (domain & NOUVEAU_GEM_DOMAIN_GART) 189 flags |= TTM_PL_FLAG_TT; 190 if (!flags || domain & NOUVEAU_GEM_DOMAIN_CPU) 191 flags |= TTM_PL_FLAG_SYSTEM; 192 193 if (domain & NOUVEAU_GEM_DOMAIN_COHERENT) 194 flags |= TTM_PL_FLAG_UNCACHED; 195 196 nvbo = nouveau_bo_alloc(cli, &size, &align, flags, tile_mode, 197 tile_flags); 198 if (IS_ERR(nvbo)) 199 return PTR_ERR(nvbo); 200 201 /* Initialize the embedded gem-object. We return a single gem-reference 202 * to the caller, instead of a normal nouveau_bo ttm reference. */ 203 ret = drm_gem_object_init(drm->dev, &nvbo->bo.base, size); 204 if (ret) { 205 kfree(nvbo); 206 return ret; 207 } 208 209 ret = nouveau_bo_init(nvbo, size, align, flags, NULL, NULL); 210 if (ret) { 211 /* XXX note: if this fails it kfrees nvbo */ 212 return ret; 213 } 214 215 /* we restrict allowed domains on nv50+ to only the types 216 * that were requested at creation time. not possibly on 217 * earlier chips without busting the ABI. 218 */ 219 nvbo->valid_domains = NOUVEAU_GEM_DOMAIN_VRAM | 220 NOUVEAU_GEM_DOMAIN_GART; 221 if (drm->client.device.info.family >= NV_DEVICE_INFO_V0_TESLA) 222 nvbo->valid_domains &= domain; 223 224 #ifndef __NetBSD__ /* XXX Let TTM swap; skip GEM like radeon. */ 225 nvbo->bo.persistent_swap_storage = nvbo->bo.base.filp; 226 #endif 227 *pnvbo = nvbo; 228 return 0; 229 } 230 231 static int 232 nouveau_gem_info(struct drm_file *file_priv, struct drm_gem_object *gem, 233 struct drm_nouveau_gem_info *rep) 234 { 235 struct nouveau_cli *cli = nouveau_cli(file_priv); 236 struct nouveau_bo *nvbo = nouveau_gem_object(gem); 237 struct nouveau_vmm *vmm = cli->svm.cli ? &cli->svm : &cli->vmm; 238 struct nouveau_vma *vma; 239 240 if (is_power_of_2(nvbo->valid_domains)) 241 rep->domain = nvbo->valid_domains; 242 else if (nvbo->bo.mem.mem_type == TTM_PL_TT) 243 rep->domain = NOUVEAU_GEM_DOMAIN_GART; 244 else 245 rep->domain = NOUVEAU_GEM_DOMAIN_VRAM; 246 rep->offset = nvbo->bo.offset; 247 if (vmm->vmm.object.oclass >= NVIF_CLASS_VMM_NV50) { 248 vma = nouveau_vma_find(nvbo, vmm); 249 if (!vma) 250 return -EINVAL; 251 252 rep->offset = vma->addr; 253 } 254 255 rep->size = nvbo->bo.mem.num_pages << PAGE_SHIFT; 256 rep->map_handle = drm_vma_node_offset_addr(&nvbo->bo.base.vma_node); 257 rep->tile_mode = nvbo->mode; 258 rep->tile_flags = nvbo->contig ? 0 : NOUVEAU_GEM_TILE_NONCONTIG; 259 if (cli->device.info.family >= NV_DEVICE_INFO_V0_FERMI) 260 rep->tile_flags |= nvbo->kind << 8; 261 else 262 if (cli->device.info.family >= NV_DEVICE_INFO_V0_TESLA) 263 rep->tile_flags |= nvbo->kind << 8 | nvbo->comp << 16; 264 else 265 rep->tile_flags |= nvbo->zeta; 266 return 0; 267 } 268 269 int 270 nouveau_gem_ioctl_new(struct drm_device *dev, void *data, 271 struct drm_file *file_priv) 272 { 273 struct nouveau_cli *cli = nouveau_cli(file_priv); 274 struct drm_nouveau_gem_new *req = data; 275 struct nouveau_bo *nvbo = NULL; 276 int ret = 0; 277 278 ret = nouveau_gem_new(cli, req->info.size, req->align, 279 req->info.domain, req->info.tile_mode, 280 req->info.tile_flags, &nvbo); 281 if (ret) 282 return ret; 283 284 ret = drm_gem_handle_create(file_priv, &nvbo->bo.base, 285 &req->info.handle); 286 if (ret == 0) { 287 ret = nouveau_gem_info(file_priv, &nvbo->bo.base, &req->info); 288 if (ret) 289 drm_gem_handle_delete(file_priv, req->info.handle); 290 } 291 292 /* drop reference from allocate - handle holds it now */ 293 drm_gem_object_put_unlocked(&nvbo->bo.base); 294 return ret; 295 } 296 297 static int 298 nouveau_gem_set_domain(struct drm_gem_object *gem, uint32_t read_domains, 299 uint32_t write_domains, uint32_t valid_domains) 300 { 301 struct nouveau_bo *nvbo = nouveau_gem_object(gem); 302 struct ttm_buffer_object *bo = &nvbo->bo; 303 uint32_t domains = valid_domains & nvbo->valid_domains & 304 (write_domains ? write_domains : read_domains); 305 uint32_t pref_flags = 0, valid_flags = 0; 306 307 if (!domains) 308 return -EINVAL; 309 310 if (valid_domains & NOUVEAU_GEM_DOMAIN_VRAM) 311 valid_flags |= TTM_PL_FLAG_VRAM; 312 313 if (valid_domains & NOUVEAU_GEM_DOMAIN_GART) 314 valid_flags |= TTM_PL_FLAG_TT; 315 316 if ((domains & NOUVEAU_GEM_DOMAIN_VRAM) && 317 bo->mem.mem_type == TTM_PL_VRAM) 318 pref_flags |= TTM_PL_FLAG_VRAM; 319 320 else if ((domains & NOUVEAU_GEM_DOMAIN_GART) && 321 bo->mem.mem_type == TTM_PL_TT) 322 pref_flags |= TTM_PL_FLAG_TT; 323 324 else if (domains & NOUVEAU_GEM_DOMAIN_VRAM) 325 pref_flags |= TTM_PL_FLAG_VRAM; 326 327 else 328 pref_flags |= TTM_PL_FLAG_TT; 329 330 nouveau_bo_placement_set(nvbo, pref_flags, valid_flags); 331 332 return 0; 333 } 334 335 struct validate_op { 336 struct list_head list; 337 struct ww_acquire_ctx ticket; 338 }; 339 340 static void 341 validate_fini_no_ticket(struct validate_op *op, struct nouveau_channel *chan, 342 struct nouveau_fence *fence, 343 struct drm_nouveau_gem_pushbuf_bo *pbbo) 344 { 345 struct nouveau_bo *nvbo; 346 struct drm_nouveau_gem_pushbuf_bo *b; 347 348 while (!list_empty(&op->list)) { 349 nvbo = list_entry(op->list.next, struct nouveau_bo, entry); 350 b = &pbbo[nvbo->pbbo_index]; 351 352 if (likely(fence)) { 353 nouveau_bo_fence(nvbo, fence, !!b->write_domains); 354 355 if (chan->vmm->vmm.object.oclass >= NVIF_CLASS_VMM_NV50) { 356 struct nouveau_vma *vma = 357 (void *)(unsigned long)b->user_priv; 358 nouveau_fence_unref(&vma->fence); 359 dma_fence_get(&fence->base); 360 vma->fence = fence; 361 } 362 } 363 364 if (unlikely(nvbo->validate_mapped)) { 365 ttm_bo_kunmap(&nvbo->kmap); 366 nvbo->validate_mapped = false; 367 } 368 369 list_del(&nvbo->entry); 370 nvbo->reserved_by = NULL; 371 ttm_bo_unreserve(&nvbo->bo); 372 drm_gem_object_put_unlocked(&nvbo->bo.base); 373 } 374 } 375 376 static void 377 validate_fini(struct validate_op *op, struct nouveau_channel *chan, 378 struct nouveau_fence *fence, 379 struct drm_nouveau_gem_pushbuf_bo *pbbo) 380 { 381 validate_fini_no_ticket(op, chan, fence, pbbo); 382 ww_acquire_fini(&op->ticket); 383 } 384 385 static int 386 validate_init(struct nouveau_channel *chan, struct drm_file *file_priv, 387 struct drm_nouveau_gem_pushbuf_bo *pbbo, 388 int nr_buffers, struct validate_op *op) 389 { 390 struct nouveau_cli *cli = nouveau_cli(file_priv); 391 int trycnt = 0; 392 int ret = -EINVAL, i; 393 struct nouveau_bo *res_bo = NULL; 394 LIST_HEAD(gart_list); 395 LIST_HEAD(vram_list); 396 LIST_HEAD(both_list); 397 398 ww_acquire_init(&op->ticket, &reservation_ww_class); 399 retry: 400 if (++trycnt > 100000) { 401 NV_PRINTK(err, cli, "%s failed and gave up.\n", __func__); 402 return -EINVAL; 403 } 404 405 for (i = 0; i < nr_buffers; i++) { 406 struct drm_nouveau_gem_pushbuf_bo *b = &pbbo[i]; 407 struct drm_gem_object *gem; 408 struct nouveau_bo *nvbo; 409 410 gem = drm_gem_object_lookup(file_priv, b->handle); 411 if (!gem) { 412 NV_PRINTK(err, cli, "Unknown handle 0x%08x\n", b->handle); 413 ret = -ENOENT; 414 break; 415 } 416 nvbo = nouveau_gem_object(gem); 417 if (nvbo == res_bo) { 418 res_bo = NULL; 419 drm_gem_object_put_unlocked(gem); 420 continue; 421 } 422 423 if (nvbo->reserved_by && nvbo->reserved_by == file_priv) { 424 NV_PRINTK(err, cli, "multiple instances of buffer %d on " 425 "validation list\n", b->handle); 426 drm_gem_object_put_unlocked(gem); 427 ret = -EINVAL; 428 break; 429 } 430 431 ret = ttm_bo_reserve(&nvbo->bo, true, false, &op->ticket); 432 if (ret) { 433 list_splice_tail_init(&vram_list, &op->list); 434 list_splice_tail_init(&gart_list, &op->list); 435 list_splice_tail_init(&both_list, &op->list); 436 validate_fini_no_ticket(op, chan, NULL, NULL); 437 if (unlikely(ret == -EDEADLK)) { 438 ret = ttm_bo_reserve_slowpath(&nvbo->bo, true, 439 &op->ticket); 440 if (!ret) 441 res_bo = nvbo; 442 } 443 if (unlikely(ret)) { 444 if (ret != -ERESTARTSYS) 445 NV_PRINTK(err, cli, "fail reserve\n"); 446 break; 447 } 448 } 449 450 if (chan->vmm->vmm.object.oclass >= NVIF_CLASS_VMM_NV50) { 451 struct nouveau_vmm *vmm = chan->vmm; 452 struct nouveau_vma *vma = nouveau_vma_find(nvbo, vmm); 453 if (!vma) { 454 NV_PRINTK(err, cli, "vma not found!\n"); 455 ret = -EINVAL; 456 break; 457 } 458 459 b->user_priv = (uint64_t)(unsigned long)vma; 460 } else { 461 b->user_priv = (uint64_t)(unsigned long)nvbo; 462 } 463 464 nvbo->reserved_by = file_priv; 465 nvbo->pbbo_index = i; 466 if ((b->valid_domains & NOUVEAU_GEM_DOMAIN_VRAM) && 467 (b->valid_domains & NOUVEAU_GEM_DOMAIN_GART)) 468 list_add_tail(&nvbo->entry, &both_list); 469 else 470 if (b->valid_domains & NOUVEAU_GEM_DOMAIN_VRAM) 471 list_add_tail(&nvbo->entry, &vram_list); 472 else 473 if (b->valid_domains & NOUVEAU_GEM_DOMAIN_GART) 474 list_add_tail(&nvbo->entry, &gart_list); 475 else { 476 NV_PRINTK(err, cli, "invalid valid domains: 0x%08x\n", 477 b->valid_domains); 478 list_add_tail(&nvbo->entry, &both_list); 479 ret = -EINVAL; 480 break; 481 } 482 if (nvbo == res_bo) 483 goto retry; 484 } 485 486 ww_acquire_done(&op->ticket); 487 list_splice_tail(&vram_list, &op->list); 488 list_splice_tail(&gart_list, &op->list); 489 list_splice_tail(&both_list, &op->list); 490 if (ret) 491 validate_fini(op, chan, NULL, NULL); 492 return ret; 493 494 } 495 496 #ifdef __NetBSD__ /* XXX yargleblargh */ 497 # define __force 498 #endif 499 500 static int 501 validate_list(struct nouveau_channel *chan, struct nouveau_cli *cli, 502 struct list_head *list, struct drm_nouveau_gem_pushbuf_bo *pbbo) 503 { 504 struct nouveau_drm *drm = chan->drm; 505 struct nouveau_bo *nvbo; 506 int ret, relocs = 0; 507 508 list_for_each_entry(nvbo, list, entry) { 509 struct drm_nouveau_gem_pushbuf_bo *b = &pbbo[nvbo->pbbo_index]; 510 511 ret = nouveau_gem_set_domain(&nvbo->bo.base, b->read_domains, 512 b->write_domains, 513 b->valid_domains); 514 if (unlikely(ret)) { 515 NV_PRINTK(err, cli, "fail set_domain\n"); 516 return ret; 517 } 518 519 ret = nouveau_bo_validate(nvbo, true, false); 520 if (unlikely(ret)) { 521 if (ret != -ERESTARTSYS) 522 NV_PRINTK(err, cli, "fail ttm_validate\n"); 523 return ret; 524 } 525 526 ret = nouveau_fence_sync(nvbo, chan, !!b->write_domains, true); 527 if (unlikely(ret)) { 528 if (ret != -ERESTARTSYS) 529 NV_PRINTK(err, cli, "fail post-validate sync\n"); 530 return ret; 531 } 532 533 if (drm->client.device.info.family < NV_DEVICE_INFO_V0_TESLA) { 534 if (nvbo->bo.offset == b->presumed.offset && 535 ((nvbo->bo.mem.mem_type == TTM_PL_VRAM && 536 b->presumed.domain & NOUVEAU_GEM_DOMAIN_VRAM) || 537 (nvbo->bo.mem.mem_type == TTM_PL_TT && 538 b->presumed.domain & NOUVEAU_GEM_DOMAIN_GART))) 539 continue; 540 541 if (nvbo->bo.mem.mem_type == TTM_PL_TT) 542 b->presumed.domain = NOUVEAU_GEM_DOMAIN_GART; 543 else 544 b->presumed.domain = NOUVEAU_GEM_DOMAIN_VRAM; 545 b->presumed.offset = nvbo->bo.offset; 546 b->presumed.valid = 0; 547 relocs++; 548 } 549 } 550 551 return relocs; 552 } 553 554 static int 555 nouveau_gem_pushbuf_validate(struct nouveau_channel *chan, 556 struct drm_file *file_priv, 557 struct drm_nouveau_gem_pushbuf_bo *pbbo, 558 int nr_buffers, 559 struct validate_op *op, bool *apply_relocs) 560 { 561 struct nouveau_cli *cli = nouveau_cli(file_priv); 562 int ret; 563 564 INIT_LIST_HEAD(&op->list); 565 566 if (nr_buffers == 0) 567 return 0; 568 569 ret = validate_init(chan, file_priv, pbbo, nr_buffers, op); 570 if (unlikely(ret)) { 571 if (ret != -ERESTARTSYS) 572 NV_PRINTK(err, cli, "validate_init\n"); 573 return ret; 574 } 575 576 ret = validate_list(chan, cli, &op->list, pbbo); 577 if (unlikely(ret < 0)) { 578 if (ret != -ERESTARTSYS) 579 NV_PRINTK(err, cli, "validating bo list\n"); 580 validate_fini(op, chan, NULL, NULL); 581 return ret; 582 } 583 *apply_relocs = ret; 584 return 0; 585 } 586 587 static inline void 588 u_free(void *addr) 589 { 590 kvfree(addr); 591 } 592 593 static inline void * 594 u_memcpya(uint64_t user, unsigned nmemb, unsigned size) 595 { 596 void *mem; 597 void __user *userptr = (void __force __user *)(uintptr_t)user; 598 599 size *= nmemb; 600 601 mem = kvmalloc(size, GFP_KERNEL); 602 if (!mem) 603 return ERR_PTR(-ENOMEM); 604 605 if (copy_from_user(mem, userptr, size)) { 606 u_free(mem); 607 return ERR_PTR(-EFAULT); 608 } 609 610 return mem; 611 } 612 613 static int 614 nouveau_gem_pushbuf_reloc_apply(struct nouveau_cli *cli, 615 struct drm_nouveau_gem_pushbuf *req, 616 struct drm_nouveau_gem_pushbuf_reloc *reloc, 617 struct drm_nouveau_gem_pushbuf_bo *bo) 618 { 619 int ret = 0; 620 unsigned i; 621 622 for (i = 0; i < req->nr_relocs; i++) { 623 struct drm_nouveau_gem_pushbuf_reloc *r = &reloc[i]; 624 struct drm_nouveau_gem_pushbuf_bo *b; 625 struct nouveau_bo *nvbo; 626 uint32_t data; 627 628 if (unlikely(r->bo_index >= req->nr_buffers)) { 629 NV_PRINTK(err, cli, "reloc bo index invalid\n"); 630 ret = -EINVAL; 631 break; 632 } 633 634 b = &bo[r->bo_index]; 635 if (b->presumed.valid) 636 continue; 637 638 if (unlikely(r->reloc_bo_index >= req->nr_buffers)) { 639 NV_PRINTK(err, cli, "reloc container bo index invalid\n"); 640 ret = -EINVAL; 641 break; 642 } 643 nvbo = (void *)(unsigned long)bo[r->reloc_bo_index].user_priv; 644 645 if (unlikely(r->reloc_bo_offset + 4 > 646 nvbo->bo.mem.num_pages << PAGE_SHIFT)) { 647 NV_PRINTK(err, cli, "reloc outside of bo\n"); 648 ret = -EINVAL; 649 break; 650 } 651 652 if (!nvbo->kmap.virtual) { 653 ret = ttm_bo_kmap(&nvbo->bo, 0, nvbo->bo.mem.num_pages, 654 &nvbo->kmap); 655 if (ret) { 656 NV_PRINTK(err, cli, "failed kmap for reloc\n"); 657 break; 658 } 659 nvbo->validate_mapped = true; 660 } 661 662 if (r->flags & NOUVEAU_GEM_RELOC_LOW) 663 data = b->presumed.offset + r->data; 664 else 665 if (r->flags & NOUVEAU_GEM_RELOC_HIGH) 666 data = (b->presumed.offset + r->data) >> 32; 667 else 668 data = r->data; 669 670 if (r->flags & NOUVEAU_GEM_RELOC_OR) { 671 if (b->presumed.domain == NOUVEAU_GEM_DOMAIN_GART) 672 data |= r->tor; 673 else 674 data |= r->vor; 675 } 676 677 ret = ttm_bo_wait(&nvbo->bo, false, false); 678 if (ret) { 679 NV_PRINTK(err, cli, "reloc wait_idle failed: %d\n", ret); 680 break; 681 } 682 683 nouveau_bo_wr32(nvbo, r->reloc_bo_offset >> 2, data); 684 } 685 686 u_free(reloc); 687 return ret; 688 } 689 690 int 691 nouveau_gem_ioctl_pushbuf(struct drm_device *dev, void *data, 692 struct drm_file *file_priv) 693 { 694 struct nouveau_abi16 *abi16 = nouveau_abi16_get(file_priv); 695 struct nouveau_cli *cli = nouveau_cli(file_priv); 696 struct nouveau_abi16_chan *temp; 697 struct nouveau_drm *drm = nouveau_drm(dev); 698 struct drm_nouveau_gem_pushbuf *req = data; 699 struct drm_nouveau_gem_pushbuf_push *push; 700 struct drm_nouveau_gem_pushbuf_reloc *reloc = NULL; 701 struct drm_nouveau_gem_pushbuf_bo *bo; 702 struct nouveau_channel *chan = NULL; 703 struct validate_op op; 704 struct nouveau_fence *fence = NULL; 705 int i, j, ret = 0; 706 bool do_reloc = false, sync = false; 707 708 if (unlikely(!abi16)) 709 return -ENOMEM; 710 711 list_for_each_entry(temp, &abi16->channels, head) { 712 if (temp->chan->chid == req->channel) { 713 chan = temp->chan; 714 break; 715 } 716 } 717 718 if (!chan) 719 return nouveau_abi16_put(abi16, -ENOENT); 720 if (unlikely(atomic_read(&chan->killed))) 721 return nouveau_abi16_put(abi16, -ENODEV); 722 723 sync = req->vram_available & NOUVEAU_GEM_PUSHBUF_SYNC; 724 725 req->vram_available = drm->gem.vram_available; 726 req->gart_available = drm->gem.gart_available; 727 if (unlikely(req->nr_push == 0)) 728 goto out_next; 729 730 if (unlikely(req->nr_push > NOUVEAU_GEM_MAX_PUSH)) { 731 NV_PRINTK(err, cli, "pushbuf push count exceeds limit: %d max %d\n", 732 req->nr_push, NOUVEAU_GEM_MAX_PUSH); 733 return nouveau_abi16_put(abi16, -EINVAL); 734 } 735 736 if (unlikely(req->nr_buffers > NOUVEAU_GEM_MAX_BUFFERS)) { 737 NV_PRINTK(err, cli, "pushbuf bo count exceeds limit: %d max %d\n", 738 req->nr_buffers, NOUVEAU_GEM_MAX_BUFFERS); 739 return nouveau_abi16_put(abi16, -EINVAL); 740 } 741 742 if (unlikely(req->nr_relocs > NOUVEAU_GEM_MAX_RELOCS)) { 743 NV_PRINTK(err, cli, "pushbuf reloc count exceeds limit: %d max %d\n", 744 req->nr_relocs, NOUVEAU_GEM_MAX_RELOCS); 745 return nouveau_abi16_put(abi16, -EINVAL); 746 } 747 748 push = u_memcpya(req->push, req->nr_push, sizeof(*push)); 749 if (IS_ERR(push)) 750 return nouveau_abi16_put(abi16, PTR_ERR(push)); 751 752 bo = u_memcpya(req->buffers, req->nr_buffers, sizeof(*bo)); 753 if (IS_ERR(bo)) { 754 u_free(push); 755 return nouveau_abi16_put(abi16, PTR_ERR(bo)); 756 } 757 758 /* Ensure all push buffers are on validate list */ 759 for (i = 0; i < req->nr_push; i++) { 760 if (push[i].bo_index >= req->nr_buffers) { 761 NV_PRINTK(err, cli, "push %d buffer not in list\n", i); 762 ret = -EINVAL; 763 goto out_prevalid; 764 } 765 } 766 767 /* Validate buffer list */ 768 revalidate: 769 ret = nouveau_gem_pushbuf_validate(chan, file_priv, bo, 770 req->nr_buffers, &op, &do_reloc); 771 if (ret) { 772 if (ret != -ERESTARTSYS) 773 NV_PRINTK(err, cli, "validate: %d\n", ret); 774 goto out_prevalid; 775 } 776 777 /* Apply any relocations that are required */ 778 if (do_reloc) { 779 if (!reloc) { 780 validate_fini(&op, chan, NULL, bo); 781 reloc = u_memcpya(req->relocs, req->nr_relocs, sizeof(*reloc)); 782 if (IS_ERR(reloc)) { 783 ret = PTR_ERR(reloc); 784 goto out_prevalid; 785 } 786 787 goto revalidate; 788 } 789 790 ret = nouveau_gem_pushbuf_reloc_apply(cli, req, reloc, bo); 791 if (ret) { 792 NV_PRINTK(err, cli, "reloc apply: %d\n", ret); 793 goto out; 794 } 795 } 796 797 if (chan->dma.ib_max) { 798 ret = nouveau_dma_wait(chan, req->nr_push + 1, 16); 799 if (ret) { 800 NV_PRINTK(err, cli, "nv50cal_space: %d\n", ret); 801 goto out; 802 } 803 804 for (i = 0; i < req->nr_push; i++) { 805 struct nouveau_vma *vma = (void *)(unsigned long) 806 bo[push[i].bo_index].user_priv; 807 808 nv50_dma_push(chan, vma->addr + push[i].offset, 809 push[i].length); 810 } 811 } else 812 if (drm->client.device.info.chipset >= 0x25) { 813 ret = RING_SPACE(chan, req->nr_push * 2); 814 if (ret) { 815 NV_PRINTK(err, cli, "cal_space: %d\n", ret); 816 goto out; 817 } 818 819 for (i = 0; i < req->nr_push; i++) { 820 struct nouveau_bo *nvbo = (void *)(unsigned long) 821 bo[push[i].bo_index].user_priv; 822 823 OUT_RING(chan, (nvbo->bo.offset + push[i].offset) | 2); 824 OUT_RING(chan, 0); 825 } 826 } else { 827 ret = RING_SPACE(chan, req->nr_push * (2 + NOUVEAU_DMA_SKIPS)); 828 if (ret) { 829 NV_PRINTK(err, cli, "jmp_space: %d\n", ret); 830 goto out; 831 } 832 833 for (i = 0; i < req->nr_push; i++) { 834 struct nouveau_bo *nvbo = (void *)(unsigned long) 835 bo[push[i].bo_index].user_priv; 836 uint32_t cmd; 837 838 cmd = chan->push.addr + ((chan->dma.cur + 2) << 2); 839 cmd |= 0x20000000; 840 if (unlikely(cmd != req->suffix0)) { 841 if (!nvbo->kmap.virtual) { 842 ret = ttm_bo_kmap(&nvbo->bo, 0, 843 nvbo->bo.mem. 844 num_pages, 845 &nvbo->kmap); 846 if (ret) { 847 WIND_RING(chan); 848 goto out; 849 } 850 nvbo->validate_mapped = true; 851 } 852 853 nouveau_bo_wr32(nvbo, (push[i].offset + 854 push[i].length - 8) / 4, cmd); 855 } 856 857 OUT_RING(chan, 0x20000000 | 858 (nvbo->bo.offset + push[i].offset)); 859 OUT_RING(chan, 0); 860 for (j = 0; j < NOUVEAU_DMA_SKIPS; j++) 861 OUT_RING(chan, 0); 862 } 863 } 864 865 ret = nouveau_fence_new(chan, false, &fence); 866 if (ret) { 867 NV_PRINTK(err, cli, "error fencing pushbuf: %d\n", ret); 868 WIND_RING(chan); 869 goto out; 870 } 871 872 if (sync) { 873 if (!(ret = nouveau_fence_wait(fence, false, false))) { 874 if ((ret = dma_fence_get_status(&fence->base)) == 1) 875 ret = 0; 876 } 877 } 878 879 out: 880 validate_fini(&op, chan, fence, bo); 881 nouveau_fence_unref(&fence); 882 883 if (do_reloc) { 884 struct drm_nouveau_gem_pushbuf_bo __user *upbbo = 885 u64_to_user_ptr(req->buffers); 886 887 for (i = 0; i < req->nr_buffers; i++) { 888 if (bo[i].presumed.valid) 889 continue; 890 891 if (copy_to_user(&upbbo[i].presumed, &bo[i].presumed, 892 sizeof(bo[i].presumed))) { 893 ret = -EFAULT; 894 break; 895 } 896 } 897 u_free(reloc); 898 } 899 out_prevalid: 900 u_free(bo); 901 u_free(push); 902 903 out_next: 904 if (chan->dma.ib_max) { 905 req->suffix0 = 0x00000000; 906 req->suffix1 = 0x00000000; 907 } else 908 if (drm->client.device.info.chipset >= 0x25) { 909 req->suffix0 = 0x00020000; 910 req->suffix1 = 0x00000000; 911 } else { 912 req->suffix0 = 0x20000000 | 913 (chan->push.addr + ((chan->dma.cur + 2) << 2)); 914 req->suffix1 = 0x00000000; 915 } 916 917 return nouveau_abi16_put(abi16, ret); 918 } 919 920 int 921 nouveau_gem_ioctl_cpu_prep(struct drm_device *dev, void *data, 922 struct drm_file *file_priv) 923 { 924 struct drm_nouveau_gem_cpu_prep *req = data; 925 struct drm_gem_object *gem; 926 struct nouveau_bo *nvbo; 927 bool no_wait = !!(req->flags & NOUVEAU_GEM_CPU_PREP_NOWAIT); 928 bool write = !!(req->flags & NOUVEAU_GEM_CPU_PREP_WRITE); 929 long lret; 930 int ret; 931 932 gem = drm_gem_object_lookup(file_priv, req->handle); 933 if (!gem) 934 return -ENOENT; 935 nvbo = nouveau_gem_object(gem); 936 937 lret = dma_resv_wait_timeout_rcu(nvbo->bo.base.resv, write, true, 938 no_wait ? 0 : 30 * HZ); 939 if (!lret) 940 ret = -EBUSY; 941 else if (lret > 0) 942 ret = 0; 943 else 944 ret = lret; 945 946 nouveau_bo_sync_for_cpu(nvbo); 947 drm_gem_object_put_unlocked(gem); 948 949 return ret; 950 } 951 952 int 953 nouveau_gem_ioctl_cpu_fini(struct drm_device *dev, void *data, 954 struct drm_file *file_priv) 955 { 956 struct drm_nouveau_gem_cpu_fini *req = data; 957 struct drm_gem_object *gem; 958 struct nouveau_bo *nvbo; 959 960 gem = drm_gem_object_lookup(file_priv, req->handle); 961 if (!gem) 962 return -ENOENT; 963 nvbo = nouveau_gem_object(gem); 964 965 nouveau_bo_sync_for_device(nvbo); 966 drm_gem_object_put_unlocked(gem); 967 return 0; 968 } 969 970 int 971 nouveau_gem_ioctl_info(struct drm_device *dev, void *data, 972 struct drm_file *file_priv) 973 { 974 struct drm_nouveau_gem_info *req = data; 975 struct drm_gem_object *gem; 976 int ret; 977 978 gem = drm_gem_object_lookup(file_priv, req->handle); 979 if (!gem) 980 return -ENOENT; 981 982 ret = nouveau_gem_info(file_priv, gem, req); 983 drm_gem_object_put_unlocked(gem); 984 return ret; 985 } 986 987
Indexes created Wed Sep 24 15:09:51 GMT 2025