amdgpu_cs.c revision 1.1.1.1
1 /* $NetBSD: amdgpu_cs.c,v 1.1.1.1 2018/08/27 01:34:43 riastradh Exp $ */ 2 3 /* 4 * Copyright 2008 Jerome Glisse. 5 * All Rights Reserved. 6 * 7 * Permission is hereby granted, free of charge, to any person obtaining a 8 * copy of this software and associated documentation files (the "Software"), 9 * to deal in the Software without restriction, including without limitation 10 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 11 * and/or sell copies of the Software, and to permit persons to whom the 12 * Software is furnished to do so, subject to the following conditions: 13 * 14 * The above copyright notice and this permission notice (including the next 15 * paragraph) shall be included in all copies or substantial portions of the 16 * Software. 17 * 18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 19 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 20 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 21 * PRECISION INSIGHT AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR 22 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 23 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 24 * DEALINGS IN THE SOFTWARE. 25 * 26 * Authors: 27 * Jerome Glisse <glisse (at) freedesktop.org> 28 */ 29 #include <sys/cdefs.h> 30 __KERNEL_RCSID(0, "$NetBSD: amdgpu_cs.c,v 1.1.1.1 2018/08/27 01:34:43 riastradh Exp $"); 31 32 #include <linux/list_sort.h> 33 #include <drm/drmP.h> 34 #include <drm/amdgpu_drm.h> 35 #include "amdgpu.h" 36 #include "amdgpu_trace.h" 37 38 #define AMDGPU_CS_MAX_PRIORITY 32u 39 #define AMDGPU_CS_NUM_BUCKETS (AMDGPU_CS_MAX_PRIORITY + 1) 40 41 /* This is based on the bucket sort with O(n) time complexity. 42 * An item with priority "i" is added to bucket[i]. The lists are then 43 * concatenated in descending order. 44 */ 45 struct amdgpu_cs_buckets { 46 struct list_head bucket[AMDGPU_CS_NUM_BUCKETS]; 47 }; 48 49 static void amdgpu_cs_buckets_init(struct amdgpu_cs_buckets *b) 50 { 51 unsigned i; 52 53 for (i = 0; i < AMDGPU_CS_NUM_BUCKETS; i++) 54 INIT_LIST_HEAD(&b->bucket[i]); 55 } 56 57 static void amdgpu_cs_buckets_add(struct amdgpu_cs_buckets *b, 58 struct list_head *item, unsigned priority) 59 { 60 /* Since buffers which appear sooner in the relocation list are 61 * likely to be used more often than buffers which appear later 62 * in the list, the sort mustn't change the ordering of buffers 63 * with the same priority, i.e. it must be stable. 64 */ 65 list_add_tail(item, &b->bucket[min(priority, AMDGPU_CS_MAX_PRIORITY)]); 66 } 67 68 static void amdgpu_cs_buckets_get_list(struct amdgpu_cs_buckets *b, 69 struct list_head *out_list) 70 { 71 unsigned i; 72 73 /* Connect the sorted buckets in the output list. */ 74 for (i = 0; i < AMDGPU_CS_NUM_BUCKETS; i++) { 75 list_splice(&b->bucket[i], out_list); 76 } 77 } 78 79 int amdgpu_cs_get_ring(struct amdgpu_device *adev, u32 ip_type, 80 u32 ip_instance, u32 ring, 81 struct amdgpu_ring **out_ring) 82 { 83 /* Right now all IPs have only one instance - multiple rings. */ 84 if (ip_instance != 0) { 85 DRM_ERROR("invalid ip instance: %d\n", ip_instance); 86 return -EINVAL; 87 } 88 89 switch (ip_type) { 90 default: 91 DRM_ERROR("unknown ip type: %d\n", ip_type); 92 return -EINVAL; 93 case AMDGPU_HW_IP_GFX: 94 if (ring < adev->gfx.num_gfx_rings) { 95 *out_ring = &adev->gfx.gfx_ring[ring]; 96 } else { 97 DRM_ERROR("only %d gfx rings are supported now\n", 98 adev->gfx.num_gfx_rings); 99 return -EINVAL; 100 } 101 break; 102 case AMDGPU_HW_IP_COMPUTE: 103 if (ring < adev->gfx.num_compute_rings) { 104 *out_ring = &adev->gfx.compute_ring[ring]; 105 } else { 106 DRM_ERROR("only %d compute rings are supported now\n", 107 adev->gfx.num_compute_rings); 108 return -EINVAL; 109 } 110 break; 111 case AMDGPU_HW_IP_DMA: 112 if (ring < adev->sdma.num_instances) { 113 *out_ring = &adev->sdma.instance[ring].ring; 114 } else { 115 DRM_ERROR("only %d SDMA rings are supported\n", 116 adev->sdma.num_instances); 117 return -EINVAL; 118 } 119 break; 120 case AMDGPU_HW_IP_UVD: 121 *out_ring = &adev->uvd.ring; 122 break; 123 case AMDGPU_HW_IP_VCE: 124 if (ring < 2){ 125 *out_ring = &adev->vce.ring[ring]; 126 } else { 127 DRM_ERROR("only two VCE rings are supported\n"); 128 return -EINVAL; 129 } 130 break; 131 } 132 133 if (!(*out_ring && (*out_ring)->adev)) { 134 DRM_ERROR("Ring %d is not initialized on IP %d\n", 135 ring, ip_type); 136 return -EINVAL; 137 } 138 139 return 0; 140 } 141 142 static int amdgpu_cs_user_fence_chunk(struct amdgpu_cs_parser *p, 143 struct drm_amdgpu_cs_chunk_fence *fence_data) 144 { 145 struct drm_gem_object *gobj; 146 uint32_t handle; 147 148 handle = fence_data->handle; 149 gobj = drm_gem_object_lookup(p->adev->ddev, p->filp, 150 fence_data->handle); 151 if (gobj == NULL) 152 return -EINVAL; 153 154 p->uf.bo = amdgpu_bo_ref(gem_to_amdgpu_bo(gobj)); 155 p->uf.offset = fence_data->offset; 156 157 if (amdgpu_ttm_tt_has_userptr(p->uf.bo->tbo.ttm)) { 158 drm_gem_object_unreference_unlocked(gobj); 159 return -EINVAL; 160 } 161 162 p->uf_entry.robj = amdgpu_bo_ref(p->uf.bo); 163 p->uf_entry.prefered_domains = AMDGPU_GEM_DOMAIN_GTT; 164 p->uf_entry.allowed_domains = AMDGPU_GEM_DOMAIN_GTT; 165 p->uf_entry.priority = 0; 166 p->uf_entry.tv.bo = &p->uf_entry.robj->tbo; 167 p->uf_entry.tv.shared = true; 168 169 drm_gem_object_unreference_unlocked(gobj); 170 return 0; 171 } 172 173 int amdgpu_cs_parser_init(struct amdgpu_cs_parser *p, void *data) 174 { 175 union drm_amdgpu_cs *cs = data; 176 uint64_t *chunk_array_user; 177 uint64_t *chunk_array; 178 struct amdgpu_fpriv *fpriv = p->filp->driver_priv; 179 unsigned size; 180 int i; 181 int ret; 182 183 if (cs->in.num_chunks == 0) 184 return 0; 185 186 chunk_array = kmalloc_array(cs->in.num_chunks, sizeof(uint64_t), GFP_KERNEL); 187 if (!chunk_array) 188 return -ENOMEM; 189 190 p->ctx = amdgpu_ctx_get(fpriv, cs->in.ctx_id); 191 if (!p->ctx) { 192 ret = -EINVAL; 193 goto free_chunk; 194 } 195 196 p->bo_list = amdgpu_bo_list_get(fpriv, cs->in.bo_list_handle); 197 198 /* get chunks */ 199 INIT_LIST_HEAD(&p->validated); 200 chunk_array_user = (uint64_t __user *)(unsigned long)(cs->in.chunks); 201 if (copy_from_user(chunk_array, chunk_array_user, 202 sizeof(uint64_t)*cs->in.num_chunks)) { 203 ret = -EFAULT; 204 goto put_bo_list; 205 } 206 207 p->nchunks = cs->in.num_chunks; 208 p->chunks = kmalloc_array(p->nchunks, sizeof(struct amdgpu_cs_chunk), 209 GFP_KERNEL); 210 if (!p->chunks) { 211 ret = -ENOMEM; 212 goto put_bo_list; 213 } 214 215 for (i = 0; i < p->nchunks; i++) { 216 struct drm_amdgpu_cs_chunk __user **chunk_ptr = NULL; 217 struct drm_amdgpu_cs_chunk user_chunk; 218 uint32_t __user *cdata; 219 220 chunk_ptr = (void __user *)(unsigned long)chunk_array[i]; 221 if (copy_from_user(&user_chunk, chunk_ptr, 222 sizeof(struct drm_amdgpu_cs_chunk))) { 223 ret = -EFAULT; 224 i--; 225 goto free_partial_kdata; 226 } 227 p->chunks[i].chunk_id = user_chunk.chunk_id; 228 p->chunks[i].length_dw = user_chunk.length_dw; 229 230 size = p->chunks[i].length_dw; 231 cdata = (void __user *)(unsigned long)user_chunk.chunk_data; 232 p->chunks[i].user_ptr = cdata; 233 234 p->chunks[i].kdata = drm_malloc_ab(size, sizeof(uint32_t)); 235 if (p->chunks[i].kdata == NULL) { 236 ret = -ENOMEM; 237 i--; 238 goto free_partial_kdata; 239 } 240 size *= sizeof(uint32_t); 241 if (copy_from_user(p->chunks[i].kdata, cdata, size)) { 242 ret = -EFAULT; 243 goto free_partial_kdata; 244 } 245 246 switch (p->chunks[i].chunk_id) { 247 case AMDGPU_CHUNK_ID_IB: 248 p->num_ibs++; 249 break; 250 251 case AMDGPU_CHUNK_ID_FENCE: 252 size = sizeof(struct drm_amdgpu_cs_chunk_fence); 253 if (p->chunks[i].length_dw * sizeof(uint32_t) < size) { 254 ret = -EINVAL; 255 goto free_partial_kdata; 256 } 257 258 ret = amdgpu_cs_user_fence_chunk(p, (void *)p->chunks[i].kdata); 259 if (ret) 260 goto free_partial_kdata; 261 262 break; 263 264 case AMDGPU_CHUNK_ID_DEPENDENCIES: 265 break; 266 267 default: 268 ret = -EINVAL; 269 goto free_partial_kdata; 270 } 271 } 272 273 274 p->ibs = kcalloc(p->num_ibs, sizeof(struct amdgpu_ib), GFP_KERNEL); 275 if (!p->ibs) { 276 ret = -ENOMEM; 277 goto free_all_kdata; 278 } 279 280 kfree(chunk_array); 281 return 0; 282 283 free_all_kdata: 284 i = p->nchunks - 1; 285 free_partial_kdata: 286 for (; i >= 0; i--) 287 drm_free_large(p->chunks[i].kdata); 288 kfree(p->chunks); 289 put_bo_list: 290 if (p->bo_list) 291 amdgpu_bo_list_put(p->bo_list); 292 amdgpu_ctx_put(p->ctx); 293 free_chunk: 294 kfree(chunk_array); 295 296 return ret; 297 } 298 299 /* Returns how many bytes TTM can move per IB. 300 */ 301 static u64 amdgpu_cs_get_threshold_for_moves(struct amdgpu_device *adev) 302 { 303 u64 real_vram_size = adev->mc.real_vram_size; 304 u64 vram_usage = atomic64_read(&adev->vram_usage); 305 306 /* This function is based on the current VRAM usage. 307 * 308 * - If all of VRAM is free, allow relocating the number of bytes that 309 * is equal to 1/4 of the size of VRAM for this IB. 310 311 * - If more than one half of VRAM is occupied, only allow relocating 312 * 1 MB of data for this IB. 313 * 314 * - From 0 to one half of used VRAM, the threshold decreases 315 * linearly. 316 * __________________ 317 * 1/4 of -|\ | 318 * VRAM | \ | 319 * | \ | 320 * | \ | 321 * | \ | 322 * | \ | 323 * | \ | 324 * | \________|1 MB 325 * |----------------| 326 * VRAM 0 % 100 % 327 * used used 328 * 329 * Note: It's a threshold, not a limit. The threshold must be crossed 330 * for buffer relocations to stop, so any buffer of an arbitrary size 331 * can be moved as long as the threshold isn't crossed before 332 * the relocation takes place. We don't want to disable buffer 333 * relocations completely. 334 * 335 * The idea is that buffers should be placed in VRAM at creation time 336 * and TTM should only do a minimum number of relocations during 337 * command submission. In practice, you need to submit at least 338 * a dozen IBs to move all buffers to VRAM if they are in GTT. 339 * 340 * Also, things can get pretty crazy under memory pressure and actual 341 * VRAM usage can change a lot, so playing safe even at 50% does 342 * consistently increase performance. 343 */ 344 345 u64 half_vram = real_vram_size >> 1; 346 u64 half_free_vram = vram_usage >= half_vram ? 0 : half_vram - vram_usage; 347 u64 bytes_moved_threshold = half_free_vram >> 1; 348 return max(bytes_moved_threshold, 1024*1024ull); 349 } 350 351 int amdgpu_cs_list_validate(struct amdgpu_device *adev, 352 struct amdgpu_vm *vm, 353 struct list_head *validated) 354 { 355 struct amdgpu_bo_list_entry *lobj; 356 struct amdgpu_bo *bo; 357 u64 bytes_moved = 0, initial_bytes_moved; 358 u64 bytes_moved_threshold = amdgpu_cs_get_threshold_for_moves(adev); 359 int r; 360 361 list_for_each_entry(lobj, validated, tv.head) { 362 bo = lobj->robj; 363 if (!bo->pin_count) { 364 u32 domain = lobj->prefered_domains; 365 u32 current_domain = 366 amdgpu_mem_type_to_domain(bo->tbo.mem.mem_type); 367 368 /* Check if this buffer will be moved and don't move it 369 * if we have moved too many buffers for this IB already. 370 * 371 * Note that this allows moving at least one buffer of 372 * any size, because it doesn't take the current "bo" 373 * into account. We don't want to disallow buffer moves 374 * completely. 375 */ 376 if ((lobj->allowed_domains & current_domain) != 0 && 377 (domain & current_domain) == 0 && /* will be moved */ 378 bytes_moved > bytes_moved_threshold) { 379 /* don't move it */ 380 domain = current_domain; 381 } 382 383 retry: 384 amdgpu_ttm_placement_from_domain(bo, domain); 385 initial_bytes_moved = atomic64_read(&adev->num_bytes_moved); 386 r = ttm_bo_validate(&bo->tbo, &bo->placement, true, false); 387 bytes_moved += atomic64_read(&adev->num_bytes_moved) - 388 initial_bytes_moved; 389 390 if (unlikely(r)) { 391 if (r != -ERESTARTSYS && domain != lobj->allowed_domains) { 392 domain = lobj->allowed_domains; 393 goto retry; 394 } 395 return r; 396 } 397 } 398 lobj->bo_va = amdgpu_vm_bo_find(vm, bo); 399 } 400 return 0; 401 } 402 403 static int amdgpu_cs_parser_relocs(struct amdgpu_cs_parser *p) 404 { 405 struct amdgpu_fpriv *fpriv = p->filp->driver_priv; 406 struct amdgpu_cs_buckets buckets; 407 struct list_head duplicates; 408 bool need_mmap_lock = false; 409 int i, r; 410 411 if (p->bo_list) { 412 need_mmap_lock = p->bo_list->has_userptr; 413 amdgpu_cs_buckets_init(&buckets); 414 for (i = 0; i < p->bo_list->num_entries; i++) 415 amdgpu_cs_buckets_add(&buckets, &p->bo_list->array[i].tv.head, 416 p->bo_list->array[i].priority); 417 418 amdgpu_cs_buckets_get_list(&buckets, &p->validated); 419 } 420 421 p->vm_bos = amdgpu_vm_get_bos(p->adev, &fpriv->vm, 422 &p->validated); 423 424 if (p->uf.bo) 425 list_add(&p->uf_entry.tv.head, &p->validated); 426 427 if (need_mmap_lock) 428 down_read(¤t->mm->mmap_sem); 429 430 INIT_LIST_HEAD(&duplicates); 431 r = ttm_eu_reserve_buffers(&p->ticket, &p->validated, true, &duplicates); 432 if (unlikely(r != 0)) 433 goto error_reserve; 434 435 r = amdgpu_cs_list_validate(p->adev, &fpriv->vm, &p->validated); 436 if (r) 437 goto error_validate; 438 439 r = amdgpu_cs_list_validate(p->adev, &fpriv->vm, &duplicates); 440 441 error_validate: 442 if (r) 443 ttm_eu_backoff_reservation(&p->ticket, &p->validated); 444 445 error_reserve: 446 if (need_mmap_lock) 447 up_read(¤t->mm->mmap_sem); 448 449 return r; 450 } 451 452 static int amdgpu_cs_sync_rings(struct amdgpu_cs_parser *p) 453 { 454 struct amdgpu_bo_list_entry *e; 455 int r; 456 457 list_for_each_entry(e, &p->validated, tv.head) { 458 struct reservation_object *resv = e->robj->tbo.resv; 459 r = amdgpu_sync_resv(p->adev, &p->ibs[0].sync, resv, p->filp); 460 461 if (r) 462 return r; 463 } 464 return 0; 465 } 466 467 static int cmp_size_smaller_first(void *priv, struct list_head *a, 468 struct list_head *b) 469 { 470 struct amdgpu_bo_list_entry *la = list_entry(a, struct amdgpu_bo_list_entry, tv.head); 471 struct amdgpu_bo_list_entry *lb = list_entry(b, struct amdgpu_bo_list_entry, tv.head); 472 473 /* Sort A before B if A is smaller. */ 474 return (int)la->robj->tbo.num_pages - (int)lb->robj->tbo.num_pages; 475 } 476 477 /** 478 * cs_parser_fini() - clean parser states 479 * @parser: parser structure holding parsing context. 480 * @error: error number 481 * 482 * If error is set than unvalidate buffer, otherwise just free memory 483 * used by parsing context. 484 **/ 485 static void amdgpu_cs_parser_fini(struct amdgpu_cs_parser *parser, int error, bool backoff) 486 { 487 unsigned i; 488 489 if (!error) { 490 /* Sort the buffer list from the smallest to largest buffer, 491 * which affects the order of buffers in the LRU list. 492 * This assures that the smallest buffers are added first 493 * to the LRU list, so they are likely to be later evicted 494 * first, instead of large buffers whose eviction is more 495 * expensive. 496 * 497 * This slightly lowers the number of bytes moved by TTM 498 * per frame under memory pressure. 499 */ 500 list_sort(NULL, &parser->validated, cmp_size_smaller_first); 501 502 ttm_eu_fence_buffer_objects(&parser->ticket, 503 &parser->validated, 504 parser->fence); 505 } else if (backoff) { 506 ttm_eu_backoff_reservation(&parser->ticket, 507 &parser->validated); 508 } 509 fence_put(parser->fence); 510 511 if (parser->ctx) 512 amdgpu_ctx_put(parser->ctx); 513 if (parser->bo_list) 514 amdgpu_bo_list_put(parser->bo_list); 515 516 drm_free_large(parser->vm_bos); 517 for (i = 0; i < parser->nchunks; i++) 518 drm_free_large(parser->chunks[i].kdata); 519 kfree(parser->chunks); 520 if (parser->ibs) 521 for (i = 0; i < parser->num_ibs; i++) 522 amdgpu_ib_free(parser->adev, &parser->ibs[i]); 523 kfree(parser->ibs); 524 amdgpu_bo_unref(&parser->uf.bo); 525 amdgpu_bo_unref(&parser->uf_entry.robj); 526 } 527 528 static int amdgpu_bo_vm_update_pte(struct amdgpu_cs_parser *p, 529 struct amdgpu_vm *vm) 530 { 531 struct amdgpu_device *adev = p->adev; 532 struct amdgpu_bo_va *bo_va; 533 struct amdgpu_bo *bo; 534 int i, r; 535 536 r = amdgpu_vm_update_page_directory(adev, vm); 537 if (r) 538 return r; 539 540 r = amdgpu_sync_fence(adev, &p->ibs[0].sync, vm->page_directory_fence); 541 if (r) 542 return r; 543 544 r = amdgpu_vm_clear_freed(adev, vm); 545 if (r) 546 return r; 547 548 if (p->bo_list) { 549 for (i = 0; i < p->bo_list->num_entries; i++) { 550 struct fence *f; 551 552 /* ignore duplicates */ 553 bo = p->bo_list->array[i].robj; 554 if (!bo) 555 continue; 556 557 bo_va = p->bo_list->array[i].bo_va; 558 if (bo_va == NULL) 559 continue; 560 561 r = amdgpu_vm_bo_update(adev, bo_va, &bo->tbo.mem); 562 if (r) 563 return r; 564 565 f = bo_va->last_pt_update; 566 r = amdgpu_sync_fence(adev, &p->ibs[0].sync, f); 567 if (r) 568 return r; 569 } 570 571 } 572 573 r = amdgpu_vm_clear_invalids(adev, vm, &p->ibs[0].sync); 574 575 if (amdgpu_vm_debug && p->bo_list) { 576 /* Invalidate all BOs to test for userspace bugs */ 577 for (i = 0; i < p->bo_list->num_entries; i++) { 578 /* ignore duplicates */ 579 bo = p->bo_list->array[i].robj; 580 if (!bo) 581 continue; 582 583 amdgpu_vm_bo_invalidate(adev, bo); 584 } 585 } 586 587 return r; 588 } 589 590 static int amdgpu_cs_ib_vm_chunk(struct amdgpu_device *adev, 591 struct amdgpu_cs_parser *parser) 592 { 593 struct amdgpu_fpriv *fpriv = parser->filp->driver_priv; 594 struct amdgpu_vm *vm = &fpriv->vm; 595 struct amdgpu_ring *ring; 596 int i, r; 597 598 if (parser->num_ibs == 0) 599 return 0; 600 601 /* Only for UVD/VCE VM emulation */ 602 for (i = 0; i < parser->num_ibs; i++) { 603 ring = parser->ibs[i].ring; 604 if (ring->funcs->parse_cs) { 605 r = amdgpu_ring_parse_cs(ring, parser, i); 606 if (r) 607 return r; 608 } 609 } 610 611 r = amdgpu_bo_vm_update_pte(parser, vm); 612 if (!r) 613 amdgpu_cs_sync_rings(parser); 614 615 return r; 616 } 617 618 static int amdgpu_cs_handle_lockup(struct amdgpu_device *adev, int r) 619 { 620 if (r == -EDEADLK) { 621 r = amdgpu_gpu_reset(adev); 622 if (!r) 623 r = -EAGAIN; 624 } 625 return r; 626 } 627 628 static int amdgpu_cs_ib_fill(struct amdgpu_device *adev, 629 struct amdgpu_cs_parser *parser) 630 { 631 struct amdgpu_fpriv *fpriv = parser->filp->driver_priv; 632 struct amdgpu_vm *vm = &fpriv->vm; 633 int i, j; 634 int r; 635 636 for (i = 0, j = 0; i < parser->nchunks && j < parser->num_ibs; i++) { 637 struct amdgpu_cs_chunk *chunk; 638 struct amdgpu_ib *ib; 639 struct drm_amdgpu_cs_chunk_ib *chunk_ib; 640 struct amdgpu_ring *ring; 641 642 chunk = &parser->chunks[i]; 643 ib = &parser->ibs[j]; 644 chunk_ib = (struct drm_amdgpu_cs_chunk_ib *)chunk->kdata; 645 646 if (chunk->chunk_id != AMDGPU_CHUNK_ID_IB) 647 continue; 648 649 r = amdgpu_cs_get_ring(adev, chunk_ib->ip_type, 650 chunk_ib->ip_instance, chunk_ib->ring, 651 &ring); 652 if (r) 653 return r; 654 655 if (ring->funcs->parse_cs) { 656 struct amdgpu_bo_va_mapping *m; 657 struct amdgpu_bo *aobj = NULL; 658 uint64_t offset; 659 uint8_t *kptr; 660 661 m = amdgpu_cs_find_mapping(parser, chunk_ib->va_start, 662 &aobj); 663 if (!aobj) { 664 DRM_ERROR("IB va_start is invalid\n"); 665 return -EINVAL; 666 } 667 668 if ((chunk_ib->va_start + chunk_ib->ib_bytes) > 669 (m->it.last + 1) * AMDGPU_GPU_PAGE_SIZE) { 670 DRM_ERROR("IB va_start+ib_bytes is invalid\n"); 671 return -EINVAL; 672 } 673 674 /* the IB should be reserved at this point */ 675 r = amdgpu_bo_kmap(aobj, (void **)&kptr); 676 if (r) { 677 return r; 678 } 679 680 offset = ((uint64_t)m->it.start) * AMDGPU_GPU_PAGE_SIZE; 681 kptr += chunk_ib->va_start - offset; 682 683 r = amdgpu_ib_get(ring, NULL, chunk_ib->ib_bytes, ib); 684 if (r) { 685 DRM_ERROR("Failed to get ib !\n"); 686 return r; 687 } 688 689 memcpy(ib->ptr, kptr, chunk_ib->ib_bytes); 690 amdgpu_bo_kunmap(aobj); 691 } else { 692 r = amdgpu_ib_get(ring, vm, 0, ib); 693 if (r) { 694 DRM_ERROR("Failed to get ib !\n"); 695 return r; 696 } 697 698 ib->gpu_addr = chunk_ib->va_start; 699 } 700 701 ib->length_dw = chunk_ib->ib_bytes / 4; 702 ib->flags = chunk_ib->flags; 703 ib->ctx = parser->ctx; 704 j++; 705 } 706 707 if (!parser->num_ibs) 708 return 0; 709 710 /* add GDS resources to first IB */ 711 if (parser->bo_list) { 712 struct amdgpu_bo *gds = parser->bo_list->gds_obj; 713 struct amdgpu_bo *gws = parser->bo_list->gws_obj; 714 struct amdgpu_bo *oa = parser->bo_list->oa_obj; 715 struct amdgpu_ib *ib = &parser->ibs[0]; 716 717 if (gds) { 718 ib->gds_base = amdgpu_bo_gpu_offset(gds); 719 ib->gds_size = amdgpu_bo_size(gds); 720 } 721 if (gws) { 722 ib->gws_base = amdgpu_bo_gpu_offset(gws); 723 ib->gws_size = amdgpu_bo_size(gws); 724 } 725 if (oa) { 726 ib->oa_base = amdgpu_bo_gpu_offset(oa); 727 ib->oa_size = amdgpu_bo_size(oa); 728 } 729 } 730 /* wrap the last IB with user fence */ 731 if (parser->uf.bo) { 732 struct amdgpu_ib *ib = &parser->ibs[parser->num_ibs - 1]; 733 734 /* UVD & VCE fw doesn't support user fences */ 735 if (ib->ring->type == AMDGPU_RING_TYPE_UVD || 736 ib->ring->type == AMDGPU_RING_TYPE_VCE) 737 return -EINVAL; 738 739 ib->user = &parser->uf; 740 } 741 742 return 0; 743 } 744 745 static int amdgpu_cs_dependencies(struct amdgpu_device *adev, 746 struct amdgpu_cs_parser *p) 747 { 748 struct amdgpu_fpriv *fpriv = p->filp->driver_priv; 749 struct amdgpu_ib *ib; 750 int i, j, r; 751 752 if (!p->num_ibs) 753 return 0; 754 755 /* Add dependencies to first IB */ 756 ib = &p->ibs[0]; 757 for (i = 0; i < p->nchunks; ++i) { 758 struct drm_amdgpu_cs_chunk_dep *deps; 759 struct amdgpu_cs_chunk *chunk; 760 unsigned num_deps; 761 762 chunk = &p->chunks[i]; 763 764 if (chunk->chunk_id != AMDGPU_CHUNK_ID_DEPENDENCIES) 765 continue; 766 767 deps = (struct drm_amdgpu_cs_chunk_dep *)chunk->kdata; 768 num_deps = chunk->length_dw * 4 / 769 sizeof(struct drm_amdgpu_cs_chunk_dep); 770 771 for (j = 0; j < num_deps; ++j) { 772 struct amdgpu_ring *ring; 773 struct amdgpu_ctx *ctx; 774 struct fence *fence; 775 776 r = amdgpu_cs_get_ring(adev, deps[j].ip_type, 777 deps[j].ip_instance, 778 deps[j].ring, &ring); 779 if (r) 780 return r; 781 782 ctx = amdgpu_ctx_get(fpriv, deps[j].ctx_id); 783 if (ctx == NULL) 784 return -EINVAL; 785 786 fence = amdgpu_ctx_get_fence(ctx, ring, 787 deps[j].handle); 788 if (IS_ERR(fence)) { 789 r = PTR_ERR(fence); 790 amdgpu_ctx_put(ctx); 791 return r; 792 793 } else if (fence) { 794 r = amdgpu_sync_fence(adev, &ib->sync, fence); 795 fence_put(fence); 796 amdgpu_ctx_put(ctx); 797 if (r) 798 return r; 799 } 800 } 801 } 802 803 return 0; 804 } 805 806 static int amdgpu_cs_free_job(struct amdgpu_job *job) 807 { 808 int i; 809 if (job->ibs) 810 for (i = 0; i < job->num_ibs; i++) 811 amdgpu_ib_free(job->adev, &job->ibs[i]); 812 kfree(job->ibs); 813 if (job->uf.bo) 814 amdgpu_bo_unref(&job->uf.bo); 815 return 0; 816 } 817 818 int amdgpu_cs_ioctl(struct drm_device *dev, void *data, struct drm_file *filp) 819 { 820 struct amdgpu_device *adev = dev->dev_private; 821 union drm_amdgpu_cs *cs = data; 822 struct amdgpu_cs_parser parser = {}; 823 bool reserved_buffers = false; 824 int i, r; 825 826 if (!adev->accel_working) 827 return -EBUSY; 828 829 parser.adev = adev; 830 parser.filp = filp; 831 832 r = amdgpu_cs_parser_init(&parser, data); 833 if (r) { 834 DRM_ERROR("Failed to initialize parser !\n"); 835 amdgpu_cs_parser_fini(&parser, r, false); 836 r = amdgpu_cs_handle_lockup(adev, r); 837 return r; 838 } 839 r = amdgpu_cs_parser_relocs(&parser); 840 if (r == -ENOMEM) 841 DRM_ERROR("Not enough memory for command submission!\n"); 842 else if (r && r != -ERESTARTSYS) 843 DRM_ERROR("Failed to process the buffer list %d!\n", r); 844 else if (!r) { 845 reserved_buffers = true; 846 r = amdgpu_cs_ib_fill(adev, &parser); 847 } 848 849 if (!r) { 850 r = amdgpu_cs_dependencies(adev, &parser); 851 if (r) 852 DRM_ERROR("Failed in the dependencies handling %d!\n", r); 853 } 854 855 if (r) 856 goto out; 857 858 for (i = 0; i < parser.num_ibs; i++) 859 trace_amdgpu_cs(&parser, i); 860 861 r = amdgpu_cs_ib_vm_chunk(adev, &parser); 862 if (r) 863 goto out; 864 865 if (amdgpu_enable_scheduler && parser.num_ibs) { 866 struct amdgpu_ring * ring = parser.ibs->ring; 867 struct amd_sched_fence *fence; 868 struct amdgpu_job *job; 869 870 job = kzalloc(sizeof(struct amdgpu_job), GFP_KERNEL); 871 if (!job) { 872 r = -ENOMEM; 873 goto out; 874 } 875 876 job->base.sched = &ring->sched; 877 job->base.s_entity = &parser.ctx->rings[ring->idx].entity; 878 job->adev = parser.adev; 879 job->owner = parser.filp; 880 job->free_job = amdgpu_cs_free_job; 881 882 job->ibs = parser.ibs; 883 job->num_ibs = parser.num_ibs; 884 parser.ibs = NULL; 885 parser.num_ibs = 0; 886 887 if (job->ibs[job->num_ibs - 1].user) { 888 job->uf = parser.uf; 889 job->ibs[job->num_ibs - 1].user = &job->uf; 890 parser.uf.bo = NULL; 891 } 892 893 fence = amd_sched_fence_create(job->base.s_entity, 894 parser.filp); 895 if (!fence) { 896 r = -ENOMEM; 897 amdgpu_cs_free_job(job); 898 kfree(job); 899 goto out; 900 } 901 job->base.s_fence = fence; 902 parser.fence = fence_get(&fence->base); 903 904 cs->out.handle = amdgpu_ctx_add_fence(parser.ctx, ring, 905 &fence->base); 906 job->ibs[job->num_ibs - 1].sequence = cs->out.handle; 907 908 trace_amdgpu_cs_ioctl(job); 909 amd_sched_entity_push_job(&job->base); 910 911 } else { 912 struct amdgpu_fence *fence; 913 914 r = amdgpu_ib_schedule(adev, parser.num_ibs, parser.ibs, 915 parser.filp); 916 fence = parser.ibs[parser.num_ibs - 1].fence; 917 parser.fence = fence_get(&fence->base); 918 cs->out.handle = parser.ibs[parser.num_ibs - 1].sequence; 919 } 920 921 out: 922 amdgpu_cs_parser_fini(&parser, r, reserved_buffers); 923 r = amdgpu_cs_handle_lockup(adev, r); 924 return r; 925 } 926 927 /** 928 * amdgpu_cs_wait_ioctl - wait for a command submission to finish 929 * 930 * @dev: drm device 931 * @data: data from userspace 932 * @filp: file private 933 * 934 * Wait for the command submission identified by handle to finish. 935 */ 936 int amdgpu_cs_wait_ioctl(struct drm_device *dev, void *data, 937 struct drm_file *filp) 938 { 939 union drm_amdgpu_wait_cs *wait = data; 940 struct amdgpu_device *adev = dev->dev_private; 941 unsigned long timeout = amdgpu_gem_timeout(wait->in.timeout); 942 struct amdgpu_ring *ring = NULL; 943 struct amdgpu_ctx *ctx; 944 struct fence *fence; 945 long r; 946 947 r = amdgpu_cs_get_ring(adev, wait->in.ip_type, wait->in.ip_instance, 948 wait->in.ring, &ring); 949 if (r) 950 return r; 951 952 ctx = amdgpu_ctx_get(filp->driver_priv, wait->in.ctx_id); 953 if (ctx == NULL) 954 return -EINVAL; 955 956 fence = amdgpu_ctx_get_fence(ctx, ring, wait->in.handle); 957 if (IS_ERR(fence)) 958 r = PTR_ERR(fence); 959 else if (fence) { 960 r = fence_wait_timeout(fence, true, timeout); 961 fence_put(fence); 962 } else 963 r = 1; 964 965 amdgpu_ctx_put(ctx); 966 if (r < 0) 967 return r; 968 969 memset(wait, 0, sizeof(*wait)); 970 wait->out.status = (r == 0); 971 972 return 0; 973 } 974 975 /** 976 * amdgpu_cs_find_bo_va - find bo_va for VM address 977 * 978 * @parser: command submission parser context 979 * @addr: VM address 980 * @bo: resulting BO of the mapping found 981 * 982 * Search the buffer objects in the command submission context for a certain 983 * virtual memory address. Returns allocation structure when found, NULL 984 * otherwise. 985 */ 986 struct amdgpu_bo_va_mapping * 987 amdgpu_cs_find_mapping(struct amdgpu_cs_parser *parser, 988 uint64_t addr, struct amdgpu_bo **bo) 989 { 990 struct amdgpu_bo_list_entry *reloc; 991 struct amdgpu_bo_va_mapping *mapping; 992 993 addr /= AMDGPU_GPU_PAGE_SIZE; 994 995 list_for_each_entry(reloc, &parser->validated, tv.head) { 996 if (!reloc->bo_va) 997 continue; 998 999 list_for_each_entry(mapping, &reloc->bo_va->valids, list) { 1000 if (mapping->it.start > addr || 1001 addr > mapping->it.last) 1002 continue; 1003 1004 *bo = reloc->bo_va->bo; 1005 return mapping; 1006 } 1007 1008 list_for_each_entry(mapping, &reloc->bo_va->invalids, list) { 1009 if (mapping->it.start > addr || 1010 addr > mapping->it.last) 1011 continue; 1012 1013 *bo = reloc->bo_va->bo; 1014 return mapping; 1015 } 1016 } 1017 1018 return NULL; 1019 } 1020
Indexes created Fri Sep 26 20:09:58 GMT 2025