1 /************************************************************************** 2 * 3 * Copyright 2006 Tungsten Graphics, Inc., Cedar Park, Texas. 4 * All Rights Reserved. 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining a 7 * copy of this software and associated documentation files (the 8 * "Software"), to deal in the Software without restriction, including 9 * without limitation the rights to use, copy, modify, merge, publish, 10 * distribute, sub license, and/or sell copies of the Software, and to 11 * permit persons to whom the Software is furnished to do so, subject to 12 * the following conditions: 13 * 14 * The above copyright notice and this permission notice (including the 15 * next paragraph) shall be included in all copies or substantial portions 16 * of the Software. 17 * 18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 19 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 20 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. 21 * IN NO EVENT SHALL TUNGSTEN GRAPHICS AND/OR ITS SUPPLIERS BE LIABLE FOR 22 * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, 23 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE 24 * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 25 * 26 **************************************************************************/ 27 28 /* Originally a fake version of the buffer manager so that we can 29 * prototype the changes in a driver fairly quickly, has been fleshed 30 * out to a fully functional interim solution. 31 * 32 * Basically wraps the old style memory management in the new 33 * programming interface, but is more expressive and avoids many of 34 * the bugs in the old texture manager. 35 */ 36 37 #include <stdlib.h> 38 #include <string.h> 39 #include <assert.h> 40 #include <errno.h> 41 #include <strings.h> 42 #include <xf86drm.h> 43 #include <pthread.h> 44 #include "intel_bufmgr.h" 45 #include "intel_bufmgr_priv.h" 46 #include "drm.h" 47 #include "i915_drm.h" 48 #include "mm.h" 49 #include "libdrm_macros.h" 50 #include "libdrm_lists.h" 51 52 #define DBG(...) do { \ 53 if (bufmgr_fake->bufmgr.debug) \ 54 drmMsg(__VA_ARGS__); \ 55 } while (0) 56 57 /* Internal flags: 58 */ 59 #define BM_NO_BACKING_STORE 0x00000001 60 #define BM_NO_FENCE_SUBDATA 0x00000002 61 #define BM_PINNED 0x00000004 62 63 /* Wrapper around mm.c's mem_block, which understands that you must 64 * wait for fences to expire before memory can be freed. This is 65 * specific to our use of memcpy for uploads - an upload that was 66 * processed through the command queue wouldn't need to care about 67 * fences. 68 */ 69 #define MAX_RELOCS 4096 70 71 struct fake_buffer_reloc { 72 /** Buffer object that the relocation points at. */ 73 drm_intel_bo *target_buf; 74 /** Offset of the relocation entry within reloc_buf. */ 75 uint32_t offset; 76 /** 77 * Cached value of the offset when we last performed this relocation. 78 */ 79 uint32_t last_target_offset; 80 /** Value added to target_buf's offset to get the relocation entry. */ 81 uint32_t delta; 82 /** Cache domains the target buffer is read into. */ 83 uint32_t read_domains; 84 /** Cache domain the target buffer will have dirty cachelines in. */ 85 uint32_t write_domain; 86 }; 87 88 struct block { 89 struct block *next, *prev; 90 struct mem_block *mem; /* BM_MEM_AGP */ 91 92 /** 93 * Marks that the block is currently in the aperture and has yet to be 94 * fenced. 95 */ 96 unsigned on_hardware:1; 97 /** 98 * Marks that the block is currently fenced (being used by rendering) 99 * and can't be freed until @fence is passed. 100 */ 101 unsigned fenced:1; 102 103 /** Fence cookie for the block. */ 104 unsigned fence; /* Split to read_fence, write_fence */ 105 106 drm_intel_bo *bo; 107 void *virtual; 108 }; 109 110 typedef struct _bufmgr_fake { 111 drm_intel_bufmgr bufmgr; 112 113 pthread_mutex_t lock; 114 115 unsigned long low_offset; 116 unsigned long size; 117 void *virtual; 118 119 struct mem_block *heap; 120 121 unsigned buf_nr; /* for generating ids */ 122 123 /** 124 * List of blocks which are currently in the GART but haven't been 125 * fenced yet. 126 */ 127 struct block on_hardware; 128 /** 129 * List of blocks which are in the GART and have an active fence on 130 * them. 131 */ 132 struct block fenced; 133 /** 134 * List of blocks which have an expired fence and are ready to be 135 * evicted. 136 */ 137 struct block lru; 138 139 unsigned int last_fence; 140 141 unsigned fail:1; 142 unsigned need_fence:1; 143 int thrashing; 144 145 /** 146 * Driver callback to emit a fence, returning the cookie. 147 * 148 * This allows the driver to hook in a replacement for the DRM usage in 149 * bufmgr_fake. 150 * 151 * Currently, this also requires that a write flush be emitted before 152 * emitting the fence, but this should change. 153 */ 154 unsigned int (*fence_emit) (void *private); 155 /** Driver callback to wait for a fence cookie to have passed. */ 156 void (*fence_wait) (unsigned int fence, void *private); 157 void *fence_priv; 158 159 /** 160 * Driver callback to execute a buffer. 161 * 162 * This allows the driver to hook in a replacement for the DRM usage in 163 * bufmgr_fake. 164 */ 165 int (*exec) (drm_intel_bo *bo, unsigned int used, void *priv); 166 void *exec_priv; 167 168 /** Driver-supplied argument to driver callbacks */ 169 void *driver_priv; 170 /** 171 * Pointer to kernel-updated sarea data for the last completed user irq 172 */ 173 volatile int *last_dispatch; 174 175 int fd; 176 177 int debug; 178 179 int performed_rendering; 180 } drm_intel_bufmgr_fake; 181 182 typedef struct _drm_intel_bo_fake { 183 drm_intel_bo bo; 184 185 unsigned id; /* debug only */ 186 const char *name; 187 188 unsigned dirty:1; 189 /** 190 * has the card written to this buffer - we make need to copy it back 191 */ 192 unsigned card_dirty:1; 193 unsigned int refcount; 194 /* Flags may consist of any of the DRM_BO flags, plus 195 * DRM_BO_NO_BACKING_STORE and BM_NO_FENCE_SUBDATA, which are the 196 * first two driver private flags. 197 */ 198 uint64_t flags; 199 /** Cache domains the target buffer is read into. */ 200 uint32_t read_domains; 201 /** Cache domain the target buffer will have dirty cachelines in. */ 202 uint32_t write_domain; 203 204 unsigned int alignment; 205 int is_static, validated; 206 unsigned int map_count; 207 208 /** relocation list */ 209 struct fake_buffer_reloc *relocs; 210 int nr_relocs; 211 /** 212 * Total size of the target_bos of this buffer. 213 * 214 * Used for estimation in check_aperture. 215 */ 216 unsigned int child_size; 217 218 struct block *block; 219 void *backing_store; 220 void (*invalidate_cb) (drm_intel_bo *bo, void *ptr); 221 void *invalidate_ptr; 222 } drm_intel_bo_fake; 223 224 static int clear_fenced(drm_intel_bufmgr_fake *bufmgr_fake, 225 unsigned int fence_cookie); 226 227 #define MAXFENCE 0x7fffffff 228 229 static int 230 FENCE_LTE(unsigned a, unsigned b) 231 { 232 if (a == b) 233 return 1; 234 235 if (a < b && b - a < (1 << 24)) 236 return 1; 237 238 if (a > b && MAXFENCE - a + b < (1 << 24)) 239 return 1; 240 241 return 0; 242 } 243 244 drm_public void 245 drm_intel_bufmgr_fake_set_fence_callback(drm_intel_bufmgr *bufmgr, 246 unsigned int (*emit) (void *priv), 247 void (*wait) (unsigned int fence, 248 void *priv), 249 void *priv) 250 { 251 drm_intel_bufmgr_fake *bufmgr_fake = (drm_intel_bufmgr_fake *) bufmgr; 252 253 bufmgr_fake->fence_emit = emit; 254 bufmgr_fake->fence_wait = wait; 255 bufmgr_fake->fence_priv = priv; 256 } 257 258 static unsigned int 259 _fence_emit_internal(drm_intel_bufmgr_fake *bufmgr_fake) 260 { 261 struct drm_i915_irq_emit ie; 262 int ret, seq = 1; 263 264 if (bufmgr_fake->fence_emit != NULL) { 265 seq = bufmgr_fake->fence_emit(bufmgr_fake->fence_priv); 266 return seq; 267 } 268 269 ie.irq_seq = &seq; 270 ret = drmCommandWriteRead(bufmgr_fake->fd, DRM_I915_IRQ_EMIT, 271 &ie, sizeof(ie)); 272 if (ret) { 273 drmMsg("%s: drm_i915_irq_emit: %d\n", __func__, ret); 274 abort(); 275 } 276 277 DBG("emit 0x%08x\n", seq); 278 return seq; 279 } 280 281 static void 282 _fence_wait_internal(drm_intel_bufmgr_fake *bufmgr_fake, int seq) 283 { 284 struct drm_i915_irq_wait iw; 285 int hw_seq, busy_count = 0; 286 int ret; 287 int kernel_lied; 288 289 if (bufmgr_fake->fence_wait != NULL) { 290 bufmgr_fake->fence_wait(seq, bufmgr_fake->fence_priv); 291 clear_fenced(bufmgr_fake, seq); 292 return; 293 } 294 295 iw.irq_seq = seq; 296 297 DBG("wait 0x%08x\n", iw.irq_seq); 298 299 /* The kernel IRQ_WAIT implementation is all sorts of broken. 300 * 1) It returns 1 to 0x7fffffff instead of using the full 32-bit 301 * unsigned range. 302 * 2) It returns 0 if hw_seq >= seq, not seq - hw_seq < 0 on the 32-bit 303 * signed range. 304 * 3) It waits if seq < hw_seq, not seq - hw_seq > 0 on the 32-bit 305 * signed range. 306 * 4) It returns -EBUSY in 3 seconds even if the hardware is still 307 * successfully chewing through buffers. 308 * 309 * Assume that in userland we treat sequence numbers as ints, which 310 * makes some of the comparisons convenient, since the sequence 311 * numbers are all positive signed integers. 312 * 313 * From this we get several cases we need to handle. Here's a timeline. 314 * 0x2 0x7 0x7ffffff8 0x7ffffffd 315 * | | | | 316 * ------------------------------------------------------------ 317 * 318 * A) Normal wait for hw to catch up 319 * hw_seq seq 320 * | | 321 * ------------------------------------------------------------ 322 * seq - hw_seq = 5. If we call IRQ_WAIT, it will wait for hw to 323 * catch up. 324 * 325 * B) Normal wait for a sequence number that's already passed. 326 * seq hw_seq 327 * | | 328 * ------------------------------------------------------------ 329 * seq - hw_seq = -5. If we call IRQ_WAIT, it returns 0 quickly. 330 * 331 * C) Hardware has already wrapped around ahead of us 332 * hw_seq seq 333 * | | 334 * ------------------------------------------------------------ 335 * seq - hw_seq = 0x80000000 - 5. If we called IRQ_WAIT, it would wait 336 * for hw_seq >= seq, which may never occur. Thus, we want to catch 337 * this in userland and return 0. 338 * 339 * D) We've wrapped around ahead of the hardware. 340 * seq hw_seq 341 * | | 342 * ------------------------------------------------------------ 343 * seq - hw_seq = -(0x80000000 - 5). If we called IRQ_WAIT, it would 344 * return 0 quickly because hw_seq >= seq, even though the hardware 345 * isn't caught up. Thus, we need to catch this early return in 346 * userland and bother the kernel until the hardware really does 347 * catch up. 348 * 349 * E) Hardware might wrap after we test in userland. 350 * hw_seq seq 351 * | | 352 * ------------------------------------------------------------ 353 * seq - hw_seq = 5. If we call IRQ_WAIT, it will likely see seq >= 354 * hw_seq and wait. However, suppose hw_seq wraps before we make it 355 * into the kernel. The kernel sees hw_seq >= seq and waits for 3 356 * seconds then returns -EBUSY. This is case C). We should catch 357 * this and then return successfully. 358 * 359 * F) Hardware might take a long time on a buffer. 360 * hw_seq seq 361 * | | 362 * ------------------------------------------------------------------- 363 * seq - hw_seq = 5. If we call IRQ_WAIT, if sequence 2 through 5 364 * take too long, it will return -EBUSY. Batchbuffers in the 365 * gltestperf demo were seen to take up to 7 seconds. We should 366 * catch early -EBUSY return and keep trying. 367 */ 368 369 do { 370 /* Keep a copy of last_dispatch so that if the wait -EBUSYs 371 * because the hardware didn't catch up in 3 seconds, we can 372 * see if it at least made progress and retry. 373 */ 374 hw_seq = *bufmgr_fake->last_dispatch; 375 376 /* Catch case C */ 377 if (seq - hw_seq > 0x40000000) 378 return; 379 380 ret = drmCommandWrite(bufmgr_fake->fd, DRM_I915_IRQ_WAIT, 381 &iw, sizeof(iw)); 382 /* Catch case D */ 383 kernel_lied = (ret == 0) && (seq - *bufmgr_fake->last_dispatch < 384 -0x40000000); 385 386 /* Catch case E */ 387 if (ret == -EBUSY 388 && (seq - *bufmgr_fake->last_dispatch > 0x40000000)) 389 ret = 0; 390 391 /* Catch case F: Allow up to 15 seconds chewing on one buffer. */ 392 if ((ret == -EBUSY) && (hw_seq != *bufmgr_fake->last_dispatch)) 393 busy_count = 0; 394 else 395 busy_count++; 396 } while (kernel_lied || ret == -EAGAIN || ret == -EINTR || 397 (ret == -EBUSY && busy_count < 5)); 398 399 if (ret != 0) { 400 drmMsg("%s:%d: Error waiting for fence: %s.\n", __FILE__, 401 __LINE__, strerror(-ret)); 402 abort(); 403 } 404 clear_fenced(bufmgr_fake, seq); 405 } 406 407 static int 408 _fence_test(drm_intel_bufmgr_fake *bufmgr_fake, unsigned fence) 409 { 410 /* Slight problem with wrap-around: 411 */ 412 return fence == 0 || FENCE_LTE(fence, bufmgr_fake->last_fence); 413 } 414 415 /** 416 * Allocate a memory manager block for the buffer. 417 */ 418 static int 419 alloc_block(drm_intel_bo *bo) 420 { 421 drm_intel_bo_fake *bo_fake = (drm_intel_bo_fake *) bo; 422 drm_intel_bufmgr_fake *bufmgr_fake = 423 (drm_intel_bufmgr_fake *) bo->bufmgr; 424 struct block *block = (struct block *)calloc(sizeof *block, 1); 425 unsigned int align_log2 = ffs(bo_fake->alignment) - 1; 426 unsigned int sz; 427 428 if (!block) 429 return 1; 430 431 sz = (bo->size + bo_fake->alignment - 1) & ~(bo_fake->alignment - 1); 432 433 block->mem = mmAllocMem(bufmgr_fake->heap, sz, align_log2, 0); 434 if (!block->mem) { 435 free(block); 436 return 0; 437 } 438 439 DRMINITLISTHEAD(block); 440 441 /* Insert at head or at tail??? */ 442 DRMLISTADDTAIL(block, &bufmgr_fake->lru); 443 444 block->virtual = (uint8_t *) bufmgr_fake->virtual + 445 block->mem->ofs - bufmgr_fake->low_offset; 446 block->bo = bo; 447 448 bo_fake->block = block; 449 450 return 1; 451 } 452 453 /* Release the card storage associated with buf: 454 */ 455 static void 456 free_block(drm_intel_bufmgr_fake *bufmgr_fake, struct block *block, 457 int skip_dirty_copy) 458 { 459 drm_intel_bo_fake *bo_fake; 460 DBG("free block %p %08x %d %d\n", block, block->mem->ofs, 461 block->on_hardware, block->fenced); 462 463 if (!block) 464 return; 465 466 bo_fake = (drm_intel_bo_fake *) block->bo; 467 468 if (bo_fake->flags & (BM_PINNED | BM_NO_BACKING_STORE)) 469 skip_dirty_copy = 1; 470 471 if (!skip_dirty_copy && (bo_fake->card_dirty == 1)) { 472 memcpy(bo_fake->backing_store, block->virtual, block->bo->size); 473 bo_fake->card_dirty = 0; 474 bo_fake->dirty = 1; 475 } 476 477 if (block->on_hardware) { 478 block->bo = NULL; 479 } else if (block->fenced) { 480 block->bo = NULL; 481 } else { 482 DBG(" - free immediately\n"); 483 DRMLISTDEL(block); 484 485 mmFreeMem(block->mem); 486 free(block); 487 } 488 } 489 490 static void 491 alloc_backing_store(drm_intel_bo *bo) 492 { 493 drm_intel_bufmgr_fake *bufmgr_fake = 494 (drm_intel_bufmgr_fake *) bo->bufmgr; 495 drm_intel_bo_fake *bo_fake = (drm_intel_bo_fake *) bo; 496 assert(!bo_fake->backing_store); 497 assert(!(bo_fake->flags & (BM_PINNED | BM_NO_BACKING_STORE))); 498 499 bo_fake->backing_store = malloc(bo->size); 500 501 DBG("alloc_backing - buf %d %p %lu\n", bo_fake->id, 502 bo_fake->backing_store, bo->size); 503 assert(bo_fake->backing_store); 504 } 505 506 static void 507 free_backing_store(drm_intel_bo *bo) 508 { 509 drm_intel_bo_fake *bo_fake = (drm_intel_bo_fake *) bo; 510 511 if (bo_fake->backing_store) { 512 assert(!(bo_fake->flags & (BM_PINNED | BM_NO_BACKING_STORE))); 513 free(bo_fake->backing_store); 514 bo_fake->backing_store = NULL; 515 } 516 } 517 518 static void 519 set_dirty(drm_intel_bo *bo) 520 { 521 drm_intel_bufmgr_fake *bufmgr_fake = 522 (drm_intel_bufmgr_fake *) bo->bufmgr; 523 drm_intel_bo_fake *bo_fake = (drm_intel_bo_fake *) bo; 524 525 if (bo_fake->flags & BM_NO_BACKING_STORE 526 && bo_fake->invalidate_cb != NULL) 527 bo_fake->invalidate_cb(bo, bo_fake->invalidate_ptr); 528 529 assert(!(bo_fake->flags & BM_PINNED)); 530 531 DBG("set_dirty - buf %d\n", bo_fake->id); 532 bo_fake->dirty = 1; 533 } 534 535 static int 536 evict_lru(drm_intel_bufmgr_fake *bufmgr_fake, unsigned int max_fence) 537 { 538 struct block *block, *tmp; 539 540 DBG("%s\n", __func__); 541 542 DRMLISTFOREACHSAFE(block, tmp, &bufmgr_fake->lru) { 543 drm_intel_bo_fake *bo_fake = (drm_intel_bo_fake *) block->bo; 544 545 if (bo_fake != NULL && (bo_fake->flags & BM_NO_FENCE_SUBDATA)) 546 continue; 547 548 if (block->fence && max_fence && !FENCE_LTE(block->fence, 549 max_fence)) 550 return 0; 551 552 set_dirty(&bo_fake->bo); 553 bo_fake->block = NULL; 554 555 free_block(bufmgr_fake, block, 0); 556 return 1; 557 } 558 559 return 0; 560 } 561 562 static int 563 evict_mru(drm_intel_bufmgr_fake *bufmgr_fake) 564 { 565 struct block *block, *tmp; 566 567 DBG("%s\n", __func__); 568 569 DRMLISTFOREACHSAFEREVERSE(block, tmp, &bufmgr_fake->lru) { 570 drm_intel_bo_fake *bo_fake = (drm_intel_bo_fake *) block->bo; 571 572 if (bo_fake && (bo_fake->flags & BM_NO_FENCE_SUBDATA)) 573 continue; 574 575 set_dirty(&bo_fake->bo); 576 bo_fake->block = NULL; 577 578 free_block(bufmgr_fake, block, 0); 579 return 1; 580 } 581 582 return 0; 583 } 584 585 /** 586 * Removes all objects from the fenced list older than the given fence. 587 */ 588 static int 589 clear_fenced(drm_intel_bufmgr_fake *bufmgr_fake, unsigned int fence_cookie) 590 { 591 struct block *block, *tmp; 592 int ret = 0; 593 594 bufmgr_fake->last_fence = fence_cookie; 595 DRMLISTFOREACHSAFE(block, tmp, &bufmgr_fake->fenced) { 596 assert(block->fenced); 597 598 if (_fence_test(bufmgr_fake, block->fence)) { 599 600 block->fenced = 0; 601 602 if (!block->bo) { 603 DBG("delayed free: offset %x sz %x\n", 604 block->mem->ofs, block->mem->size); 605 DRMLISTDEL(block); 606 mmFreeMem(block->mem); 607 free(block); 608 } else { 609 DBG("return to lru: offset %x sz %x\n", 610 block->mem->ofs, block->mem->size); 611 DRMLISTDEL(block); 612 DRMLISTADDTAIL(block, &bufmgr_fake->lru); 613 } 614 615 ret = 1; 616 } else { 617 /* Blocks are ordered by fence, so if one fails, all 618 * from here will fail also: 619 */ 620 DBG("fence not passed: offset %x sz %x %d %d \n", 621 block->mem->ofs, block->mem->size, block->fence, 622 bufmgr_fake->last_fence); 623 break; 624 } 625 } 626 627 DBG("%s: %d\n", __func__, ret); 628 return ret; 629 } 630 631 static void 632 fence_blocks(drm_intel_bufmgr_fake *bufmgr_fake, unsigned fence) 633 { 634 struct block *block, *tmp; 635 636 DRMLISTFOREACHSAFE(block, tmp, &bufmgr_fake->on_hardware) { 637 DBG("Fence block %p (sz 0x%x ofs %x buf %p) with fence %d\n", 638 block, block->mem->size, block->mem->ofs, block->bo, fence); 639 block->fence = fence; 640 641 block->on_hardware = 0; 642 block->fenced = 1; 643 644 /* Move to tail of pending list here 645 */ 646 DRMLISTDEL(block); 647 DRMLISTADDTAIL(block, &bufmgr_fake->fenced); 648 } 649 650 assert(DRMLISTEMPTY(&bufmgr_fake->on_hardware)); 651 } 652 653 static int 654 evict_and_alloc_block(drm_intel_bo *bo) 655 { 656 drm_intel_bufmgr_fake *bufmgr_fake = 657 (drm_intel_bufmgr_fake *) bo->bufmgr; 658 drm_intel_bo_fake *bo_fake = (drm_intel_bo_fake *) bo; 659 660 assert(bo_fake->block == NULL); 661 662 /* Search for already free memory: 663 */ 664 if (alloc_block(bo)) 665 return 1; 666 667 /* If we're not thrashing, allow lru eviction to dig deeper into 668 * recently used textures. We'll probably be thrashing soon: 669 */ 670 if (!bufmgr_fake->thrashing) { 671 while (evict_lru(bufmgr_fake, 0)) 672 if (alloc_block(bo)) 673 return 1; 674 } 675 676 /* Keep thrashing counter alive? 677 */ 678 if (bufmgr_fake->thrashing) 679 bufmgr_fake->thrashing = 20; 680 681 /* Wait on any already pending fences - here we are waiting for any 682 * freed memory that has been submitted to hardware and fenced to 683 * become available: 684 */ 685 while (!DRMLISTEMPTY(&bufmgr_fake->fenced)) { 686 uint32_t fence = bufmgr_fake->fenced.next->fence; 687 _fence_wait_internal(bufmgr_fake, fence); 688 689 if (alloc_block(bo)) 690 return 1; 691 } 692 693 if (!DRMLISTEMPTY(&bufmgr_fake->on_hardware)) { 694 while (!DRMLISTEMPTY(&bufmgr_fake->fenced)) { 695 uint32_t fence = bufmgr_fake->fenced.next->fence; 696 _fence_wait_internal(bufmgr_fake, fence); 697 } 698 699 if (!bufmgr_fake->thrashing) { 700 DBG("thrashing\n"); 701 } 702 bufmgr_fake->thrashing = 20; 703 704 if (alloc_block(bo)) 705 return 1; 706 } 707 708 while (evict_mru(bufmgr_fake)) 709 if (alloc_block(bo)) 710 return 1; 711 712 DBG("%s 0x%lx bytes failed\n", __func__, bo->size); 713 714 return 0; 715 } 716 717 /*********************************************************************** 718 * Public functions 719 */ 720 721 /** 722 * Wait for hardware idle by emitting a fence and waiting for it. 723 */ 724 static void 725 drm_intel_bufmgr_fake_wait_idle(drm_intel_bufmgr_fake *bufmgr_fake) 726 { 727 unsigned int cookie; 728 729 cookie = _fence_emit_internal(bufmgr_fake); 730 _fence_wait_internal(bufmgr_fake, cookie); 731 } 732 733 /** 734 * Wait for rendering to a buffer to complete. 735 * 736 * It is assumed that the batchbuffer which performed the rendering included 737 * the necessary flushing. 738 */ 739 static void 740 drm_intel_fake_bo_wait_rendering_locked(drm_intel_bo *bo) 741 { 742 drm_intel_bufmgr_fake *bufmgr_fake = 743 (drm_intel_bufmgr_fake *) bo->bufmgr; 744 drm_intel_bo_fake *bo_fake = (drm_intel_bo_fake *) bo; 745 746 if (bo_fake->block == NULL || !bo_fake->block->fenced) 747 return; 748 749 _fence_wait_internal(bufmgr_fake, bo_fake->block->fence); 750 } 751 752 static void 753 drm_intel_fake_bo_wait_rendering(drm_intel_bo *bo) 754 { 755 drm_intel_bufmgr_fake *bufmgr_fake = 756 (drm_intel_bufmgr_fake *) bo->bufmgr; 757 758 pthread_mutex_lock(&bufmgr_fake->lock); 759 drm_intel_fake_bo_wait_rendering_locked(bo); 760 pthread_mutex_unlock(&bufmgr_fake->lock); 761 } 762 763 /* Specifically ignore texture memory sharing. 764 * -- just evict everything 765 * -- and wait for idle 766 */ 767 drm_public void 768 drm_intel_bufmgr_fake_contended_lock_take(drm_intel_bufmgr *bufmgr) 769 { 770 drm_intel_bufmgr_fake *bufmgr_fake = (drm_intel_bufmgr_fake *) bufmgr; 771 struct block *block, *tmp; 772 773 pthread_mutex_lock(&bufmgr_fake->lock); 774 775 bufmgr_fake->need_fence = 1; 776 bufmgr_fake->fail = 0; 777 778 /* Wait for hardware idle. We don't know where acceleration has been 779 * happening, so we'll need to wait anyway before letting anything get 780 * put on the card again. 781 */ 782 drm_intel_bufmgr_fake_wait_idle(bufmgr_fake); 783 784 /* Check that we hadn't released the lock without having fenced the last 785 * set of buffers. 786 */ 787 assert(DRMLISTEMPTY(&bufmgr_fake->fenced)); 788 assert(DRMLISTEMPTY(&bufmgr_fake->on_hardware)); 789 790 DRMLISTFOREACHSAFE(block, tmp, &bufmgr_fake->lru) { 791 assert(_fence_test(bufmgr_fake, block->fence)); 792 set_dirty(block->bo); 793 } 794 795 pthread_mutex_unlock(&bufmgr_fake->lock); 796 } 797 798 static drm_intel_bo * 799 drm_intel_fake_bo_alloc(drm_intel_bufmgr *bufmgr, 800 const char *name, 801 unsigned long size, 802 unsigned int alignment) 803 { 804 drm_intel_bufmgr_fake *bufmgr_fake; 805 drm_intel_bo_fake *bo_fake; 806 807 bufmgr_fake = (drm_intel_bufmgr_fake *) bufmgr; 808 809 assert(size != 0); 810 811 bo_fake = calloc(1, sizeof(*bo_fake)); 812 if (!bo_fake) 813 return NULL; 814 815 bo_fake->bo.size = size; 816 bo_fake->bo.offset = -1; 817 bo_fake->bo.virtual = NULL; 818 bo_fake->bo.bufmgr = bufmgr; 819 bo_fake->refcount = 1; 820 821 /* Alignment must be a power of two */ 822 assert((alignment & (alignment - 1)) == 0); 823 if (alignment == 0) 824 alignment = 1; 825 bo_fake->alignment = alignment; 826 bo_fake->id = ++bufmgr_fake->buf_nr; 827 bo_fake->name = name; 828 bo_fake->flags = 0; 829 bo_fake->is_static = 0; 830 831 DBG("drm_bo_alloc: (buf %d: %s, %lu kb)\n", bo_fake->id, bo_fake->name, 832 bo_fake->bo.size / 1024); 833 834 return &bo_fake->bo; 835 } 836 837 static drm_intel_bo * 838 drm_intel_fake_bo_alloc_tiled(drm_intel_bufmgr * bufmgr, 839 const char *name, 840 int x, int y, int cpp, 841 uint32_t *tiling_mode, 842 unsigned long *pitch, 843 unsigned long flags) 844 { 845 unsigned long stride, aligned_y; 846 847 /* No runtime tiling support for fake. */ 848 *tiling_mode = I915_TILING_NONE; 849 850 /* Align it for being a render target. Shouldn't need anything else. */ 851 stride = x * cpp; 852 stride = ROUND_UP_TO(stride, 64); 853 854 /* 965 subspan loading alignment */ 855 aligned_y = ALIGN(y, 2); 856 857 *pitch = stride; 858 859 return drm_intel_fake_bo_alloc(bufmgr, name, stride * aligned_y, 860 4096); 861 } 862 863 drm_public drm_intel_bo * 864 drm_intel_bo_fake_alloc_static(drm_intel_bufmgr *bufmgr, 865 const char *name, 866 unsigned long offset, 867 unsigned long size, void *virtual) 868 { 869 drm_intel_bufmgr_fake *bufmgr_fake; 870 drm_intel_bo_fake *bo_fake; 871 872 bufmgr_fake = (drm_intel_bufmgr_fake *) bufmgr; 873 874 assert(size != 0); 875 876 bo_fake = calloc(1, sizeof(*bo_fake)); 877 if (!bo_fake) 878 return NULL; 879 880 bo_fake->bo.size = size; 881 bo_fake->bo.offset = offset; 882 bo_fake->bo.virtual = virtual; 883 bo_fake->bo.bufmgr = bufmgr; 884 bo_fake->refcount = 1; 885 bo_fake->id = ++bufmgr_fake->buf_nr; 886 bo_fake->name = name; 887 bo_fake->flags = BM_PINNED; 888 bo_fake->is_static = 1; 889 890 DBG("drm_bo_alloc_static: (buf %d: %s, %lu kb)\n", bo_fake->id, 891 bo_fake->name, bo_fake->bo.size / 1024); 892 893 return &bo_fake->bo; 894 } 895 896 static void 897 drm_intel_fake_bo_reference(drm_intel_bo *bo) 898 { 899 drm_intel_bufmgr_fake *bufmgr_fake = 900 (drm_intel_bufmgr_fake *) bo->bufmgr; 901 drm_intel_bo_fake *bo_fake = (drm_intel_bo_fake *) bo; 902 903 pthread_mutex_lock(&bufmgr_fake->lock); 904 bo_fake->refcount++; 905 pthread_mutex_unlock(&bufmgr_fake->lock); 906 } 907 908 static void 909 drm_intel_fake_bo_reference_locked(drm_intel_bo *bo) 910 { 911 drm_intel_bo_fake *bo_fake = (drm_intel_bo_fake *) bo; 912 913 bo_fake->refcount++; 914 } 915 916 static void 917 drm_intel_fake_bo_unreference_locked(drm_intel_bo *bo) 918 { 919 drm_intel_bufmgr_fake *bufmgr_fake = 920 (drm_intel_bufmgr_fake *) bo->bufmgr; 921 drm_intel_bo_fake *bo_fake = (drm_intel_bo_fake *) bo; 922 int i; 923 924 if (--bo_fake->refcount == 0) { 925 assert(bo_fake->map_count == 0); 926 /* No remaining references, so free it */ 927 if (bo_fake->block) 928 free_block(bufmgr_fake, bo_fake->block, 1); 929 free_backing_store(bo); 930 931 for (i = 0; i < bo_fake->nr_relocs; i++) 932 drm_intel_fake_bo_unreference_locked(bo_fake->relocs[i]. 933 target_buf); 934 935 DBG("drm_bo_unreference: free buf %d %s\n", bo_fake->id, 936 bo_fake->name); 937 938 free(bo_fake->relocs); 939 free(bo); 940 } 941 } 942 943 static void 944 drm_intel_fake_bo_unreference(drm_intel_bo *bo) 945 { 946 drm_intel_bufmgr_fake *bufmgr_fake = 947 (drm_intel_bufmgr_fake *) bo->bufmgr; 948 949 pthread_mutex_lock(&bufmgr_fake->lock); 950 drm_intel_fake_bo_unreference_locked(bo); 951 pthread_mutex_unlock(&bufmgr_fake->lock); 952 } 953 954 /** 955 * Set the buffer as not requiring backing store, and instead get the callback 956 * invoked whenever it would be set dirty. 957 */ 958 drm_public void 959 drm_intel_bo_fake_disable_backing_store(drm_intel_bo *bo, 960 void (*invalidate_cb) (drm_intel_bo *bo, 961 void *ptr), 962 void *ptr) 963 { 964 drm_intel_bufmgr_fake *bufmgr_fake = 965 (drm_intel_bufmgr_fake *) bo->bufmgr; 966 drm_intel_bo_fake *bo_fake = (drm_intel_bo_fake *) bo; 967 968 pthread_mutex_lock(&bufmgr_fake->lock); 969 970 if (bo_fake->backing_store) 971 free_backing_store(bo); 972 973 bo_fake->flags |= BM_NO_BACKING_STORE; 974 975 DBG("disable_backing_store set buf %d dirty\n", bo_fake->id); 976 bo_fake->dirty = 1; 977 bo_fake->invalidate_cb = invalidate_cb; 978 bo_fake->invalidate_ptr = ptr; 979 980 /* Note that it is invalid right from the start. Also note 981 * invalidate_cb is called with the bufmgr locked, so cannot 982 * itself make bufmgr calls. 983 */ 984 if (invalidate_cb != NULL) 985 invalidate_cb(bo, ptr); 986 987 pthread_mutex_unlock(&bufmgr_fake->lock); 988 } 989 990 /** 991 * Map a buffer into bo->virtual, allocating either card memory space (If 992 * BM_NO_BACKING_STORE or BM_PINNED) or backing store, as necessary. 993 */ 994 static int 995 drm_intel_fake_bo_map_locked(drm_intel_bo *bo, int write_enable) 996 { 997 drm_intel_bufmgr_fake *bufmgr_fake = 998 (drm_intel_bufmgr_fake *) bo->bufmgr; 999 drm_intel_bo_fake *bo_fake = (drm_intel_bo_fake *) bo; 1000 1001 /* Static buffers are always mapped. */ 1002 if (bo_fake->is_static) { 1003 if (bo_fake->card_dirty) { 1004 drm_intel_bufmgr_fake_wait_idle(bufmgr_fake); 1005 bo_fake->card_dirty = 0; 1006 } 1007 return 0; 1008 } 1009 1010 /* Allow recursive mapping. Mesa may recursively map buffers with 1011 * nested display loops, and it is used internally in bufmgr_fake 1012 * for relocation. 1013 */ 1014 if (bo_fake->map_count++ != 0) 1015 return 0; 1016 1017 { 1018 DBG("drm_bo_map: (buf %d: %s, %lu kb)\n", bo_fake->id, 1019 bo_fake->name, bo_fake->bo.size / 1024); 1020 1021 if (bo->virtual != NULL) { 1022 drmMsg("%s: already mapped\n", __func__); 1023 abort(); 1024 } else if (bo_fake->flags & (BM_NO_BACKING_STORE | BM_PINNED)) { 1025 1026 if (!bo_fake->block && !evict_and_alloc_block(bo)) { 1027 DBG("%s: alloc failed\n", __func__); 1028 bufmgr_fake->fail = 1; 1029 return 1; 1030 } else { 1031 assert(bo_fake->block); 1032 bo_fake->dirty = 0; 1033 1034 if (!(bo_fake->flags & BM_NO_FENCE_SUBDATA) && 1035 bo_fake->block->fenced) { 1036 drm_intel_fake_bo_wait_rendering_locked 1037 (bo); 1038 } 1039 1040 bo->virtual = bo_fake->block->virtual; 1041 } 1042 } else { 1043 if (write_enable) 1044 set_dirty(bo); 1045 1046 if (bo_fake->backing_store == 0) 1047 alloc_backing_store(bo); 1048 1049 if ((bo_fake->card_dirty == 1) && bo_fake->block) { 1050 if (bo_fake->block->fenced) 1051 drm_intel_fake_bo_wait_rendering_locked 1052 (bo); 1053 1054 memcpy(bo_fake->backing_store, 1055 bo_fake->block->virtual, 1056 bo_fake->block->bo->size); 1057 bo_fake->card_dirty = 0; 1058 } 1059 1060 bo->virtual = bo_fake->backing_store; 1061 } 1062 } 1063 1064 return 0; 1065 } 1066 1067 static int 1068 drm_intel_fake_bo_map(drm_intel_bo *bo, int write_enable) 1069 { 1070 drm_intel_bufmgr_fake *bufmgr_fake = 1071 (drm_intel_bufmgr_fake *) bo->bufmgr; 1072 int ret; 1073 1074 pthread_mutex_lock(&bufmgr_fake->lock); 1075 ret = drm_intel_fake_bo_map_locked(bo, write_enable); 1076 pthread_mutex_unlock(&bufmgr_fake->lock); 1077 1078 return ret; 1079 } 1080 1081 static int 1082 drm_intel_fake_bo_unmap_locked(drm_intel_bo *bo) 1083 { 1084 drm_intel_bufmgr_fake *bufmgr_fake = 1085 (drm_intel_bufmgr_fake *) bo->bufmgr; 1086 drm_intel_bo_fake *bo_fake = (drm_intel_bo_fake *) bo; 1087 1088 /* Static buffers are always mapped. */ 1089 if (bo_fake->is_static) 1090 return 0; 1091 1092 assert(bo_fake->map_count != 0); 1093 if (--bo_fake->map_count != 0) 1094 return 0; 1095 1096 DBG("drm_bo_unmap: (buf %d: %s, %lu kb)\n", bo_fake->id, bo_fake->name, 1097 bo_fake->bo.size / 1024); 1098 1099 bo->virtual = NULL; 1100 1101 return 0; 1102 } 1103 1104 static int drm_intel_fake_bo_unmap(drm_intel_bo *bo) 1105 { 1106 drm_intel_bufmgr_fake *bufmgr_fake = 1107 (drm_intel_bufmgr_fake *) bo->bufmgr; 1108 int ret; 1109 1110 pthread_mutex_lock(&bufmgr_fake->lock); 1111 ret = drm_intel_fake_bo_unmap_locked(bo); 1112 pthread_mutex_unlock(&bufmgr_fake->lock); 1113 1114 return ret; 1115 } 1116 1117 static int 1118 drm_intel_fake_bo_subdata(drm_intel_bo *bo, unsigned long offset, 1119 unsigned long size, const void *data) 1120 { 1121 int ret; 1122 1123 if (size == 0 || data == NULL) 1124 return 0; 1125 1126 ret = drm_intel_bo_map(bo, 1); 1127 if (ret) 1128 return ret; 1129 memcpy((unsigned char *)bo->virtual + offset, data, size); 1130 drm_intel_bo_unmap(bo); 1131 return 0; 1132 } 1133 1134 static void 1135 drm_intel_fake_kick_all_locked(drm_intel_bufmgr_fake *bufmgr_fake) 1136 { 1137 struct block *block, *tmp; 1138 1139 bufmgr_fake->performed_rendering = 0; 1140 /* okay for ever BO that is on the HW kick it off. 1141 seriously not afraid of the POLICE right now */ 1142 DRMLISTFOREACHSAFE(block, tmp, &bufmgr_fake->on_hardware) { 1143 drm_intel_bo_fake *bo_fake = (drm_intel_bo_fake *) block->bo; 1144 1145 block->on_hardware = 0; 1146 free_block(bufmgr_fake, block, 0); 1147 bo_fake->block = NULL; 1148 bo_fake->validated = 0; 1149 if (!(bo_fake->flags & BM_NO_BACKING_STORE)) 1150 bo_fake->dirty = 1; 1151 } 1152 1153 } 1154 1155 static int 1156 drm_intel_fake_bo_validate(drm_intel_bo *bo) 1157 { 1158 drm_intel_bufmgr_fake *bufmgr_fake; 1159 drm_intel_bo_fake *bo_fake = (drm_intel_bo_fake *) bo; 1160 1161 bufmgr_fake = (drm_intel_bufmgr_fake *) bo->bufmgr; 1162 1163 DBG("drm_bo_validate: (buf %d: %s, %lu kb)\n", bo_fake->id, 1164 bo_fake->name, bo_fake->bo.size / 1024); 1165 1166 /* Sanity check: Buffers should be unmapped before being validated. 1167 * This is not so much of a problem for bufmgr_fake, but TTM refuses, 1168 * and the problem is harder to debug there. 1169 */ 1170 assert(bo_fake->map_count == 0); 1171 1172 if (bo_fake->is_static) { 1173 /* Add it to the needs-fence list */ 1174 bufmgr_fake->need_fence = 1; 1175 return 0; 1176 } 1177 1178 /* Allocate the card memory */ 1179 if (!bo_fake->block && !evict_and_alloc_block(bo)) { 1180 bufmgr_fake->fail = 1; 1181 DBG("Failed to validate buf %d:%s\n", bo_fake->id, 1182 bo_fake->name); 1183 return -1; 1184 } 1185 1186 assert(bo_fake->block); 1187 assert(bo_fake->block->bo == &bo_fake->bo); 1188 1189 bo->offset = bo_fake->block->mem->ofs; 1190 1191 /* Upload the buffer contents if necessary */ 1192 if (bo_fake->dirty) { 1193 DBG("Upload dirty buf %d:%s, sz %lu offset 0x%x\n", bo_fake->id, 1194 bo_fake->name, bo->size, bo_fake->block->mem->ofs); 1195 1196 assert(!(bo_fake->flags & (BM_NO_BACKING_STORE | BM_PINNED))); 1197 1198 /* Actually, should be able to just wait for a fence on the 1199 * memory, which we would be tracking when we free it. Waiting 1200 * for idle is a sufficiently large hammer for now. 1201 */ 1202 drm_intel_bufmgr_fake_wait_idle(bufmgr_fake); 1203 1204 /* we may never have mapped this BO so it might not have any 1205 * backing store if this happens it should be rare, but 0 the 1206 * card memory in any case */ 1207 if (bo_fake->backing_store) 1208 memcpy(bo_fake->block->virtual, bo_fake->backing_store, 1209 bo->size); 1210 else 1211 memset(bo_fake->block->virtual, 0, bo->size); 1212 1213 bo_fake->dirty = 0; 1214 } 1215 1216 bo_fake->block->fenced = 0; 1217 bo_fake->block->on_hardware = 1; 1218 DRMLISTDEL(bo_fake->block); 1219 DRMLISTADDTAIL(bo_fake->block, &bufmgr_fake->on_hardware); 1220 1221 bo_fake->validated = 1; 1222 bufmgr_fake->need_fence = 1; 1223 1224 return 0; 1225 } 1226 1227 static void 1228 drm_intel_fake_fence_validated(drm_intel_bufmgr *bufmgr) 1229 { 1230 drm_intel_bufmgr_fake *bufmgr_fake = (drm_intel_bufmgr_fake *) bufmgr; 1231 unsigned int cookie; 1232 1233 cookie = _fence_emit_internal(bufmgr_fake); 1234 fence_blocks(bufmgr_fake, cookie); 1235 1236 DBG("drm_fence_validated: 0x%08x cookie\n", cookie); 1237 } 1238 1239 static void 1240 drm_intel_fake_destroy(drm_intel_bufmgr *bufmgr) 1241 { 1242 drm_intel_bufmgr_fake *bufmgr_fake = (drm_intel_bufmgr_fake *) bufmgr; 1243 1244 pthread_mutex_destroy(&bufmgr_fake->lock); 1245 mmDestroy(bufmgr_fake->heap); 1246 free(bufmgr); 1247 } 1248 1249 static int 1250 drm_intel_fake_emit_reloc(drm_intel_bo *bo, uint32_t offset, 1251 drm_intel_bo *target_bo, uint32_t target_offset, 1252 uint32_t read_domains, uint32_t write_domain) 1253 { 1254 drm_intel_bufmgr_fake *bufmgr_fake = 1255 (drm_intel_bufmgr_fake *) bo->bufmgr; 1256 struct fake_buffer_reloc *r; 1257 drm_intel_bo_fake *bo_fake = (drm_intel_bo_fake *) bo; 1258 drm_intel_bo_fake *target_fake = (drm_intel_bo_fake *) target_bo; 1259 int i; 1260 1261 pthread_mutex_lock(&bufmgr_fake->lock); 1262 1263 assert(bo); 1264 assert(target_bo); 1265 1266 if (bo_fake->relocs == NULL) { 1267 bo_fake->relocs = 1268 malloc(sizeof(struct fake_buffer_reloc) * MAX_RELOCS); 1269 } 1270 1271 r = &bo_fake->relocs[bo_fake->nr_relocs++]; 1272 1273 assert(bo_fake->nr_relocs <= MAX_RELOCS); 1274 1275 drm_intel_fake_bo_reference_locked(target_bo); 1276 1277 if (!target_fake->is_static) { 1278 bo_fake->child_size += 1279 ALIGN(target_bo->size, target_fake->alignment); 1280 bo_fake->child_size += target_fake->child_size; 1281 } 1282 r->target_buf = target_bo; 1283 r->offset = offset; 1284 r->last_target_offset = target_bo->offset; 1285 r->delta = target_offset; 1286 r->read_domains = read_domains; 1287 r->write_domain = write_domain; 1288 1289 if (bufmgr_fake->debug) { 1290 /* Check that a conflicting relocation hasn't already been 1291 * emitted. 1292 */ 1293 for (i = 0; i < bo_fake->nr_relocs - 1; i++) { 1294 struct fake_buffer_reloc *r2 = &bo_fake->relocs[i]; 1295 1296 assert(r->offset != r2->offset); 1297 } 1298 } 1299 1300 pthread_mutex_unlock(&bufmgr_fake->lock); 1301 1302 return 0; 1303 } 1304 1305 /** 1306 * Incorporates the validation flags associated with each relocation into 1307 * the combined validation flags for the buffer on this batchbuffer submission. 1308 */ 1309 static void 1310 drm_intel_fake_calculate_domains(drm_intel_bo *bo) 1311 { 1312 drm_intel_bo_fake *bo_fake = (drm_intel_bo_fake *) bo; 1313 int i; 1314 1315 for (i = 0; i < bo_fake->nr_relocs; i++) { 1316 struct fake_buffer_reloc *r = &bo_fake->relocs[i]; 1317 drm_intel_bo_fake *target_fake = 1318 (drm_intel_bo_fake *) r->target_buf; 1319 1320 /* Do the same for the tree of buffers we depend on */ 1321 drm_intel_fake_calculate_domains(r->target_buf); 1322 1323 target_fake->read_domains |= r->read_domains; 1324 target_fake->write_domain |= r->write_domain; 1325 } 1326 } 1327 1328 static int 1329 drm_intel_fake_reloc_and_validate_buffer(drm_intel_bo *bo) 1330 { 1331 drm_intel_bufmgr_fake *bufmgr_fake = 1332 (drm_intel_bufmgr_fake *) bo->bufmgr; 1333 drm_intel_bo_fake *bo_fake = (drm_intel_bo_fake *) bo; 1334 int i, ret; 1335 1336 assert(bo_fake->map_count == 0); 1337 1338 for (i = 0; i < bo_fake->nr_relocs; i++) { 1339 struct fake_buffer_reloc *r = &bo_fake->relocs[i]; 1340 drm_intel_bo_fake *target_fake = 1341 (drm_intel_bo_fake *) r->target_buf; 1342 uint32_t reloc_data; 1343 1344 /* Validate the target buffer if that hasn't been done. */ 1345 if (!target_fake->validated) { 1346 ret = 1347 drm_intel_fake_reloc_and_validate_buffer(r->target_buf); 1348 if (ret != 0) { 1349 if (bo->virtual != NULL) 1350 drm_intel_fake_bo_unmap_locked(bo); 1351 return ret; 1352 } 1353 } 1354 1355 /* Calculate the value of the relocation entry. */ 1356 if (r->target_buf->offset != r->last_target_offset) { 1357 reloc_data = r->target_buf->offset + r->delta; 1358 1359 if (bo->virtual == NULL) 1360 drm_intel_fake_bo_map_locked(bo, 1); 1361 1362 *(uint32_t *) ((uint8_t *) bo->virtual + r->offset) = 1363 reloc_data; 1364 1365 r->last_target_offset = r->target_buf->offset; 1366 } 1367 } 1368 1369 if (bo->virtual != NULL) 1370 drm_intel_fake_bo_unmap_locked(bo); 1371 1372 if (bo_fake->write_domain != 0) { 1373 if (!(bo_fake->flags & (BM_NO_BACKING_STORE | BM_PINNED))) { 1374 if (bo_fake->backing_store == 0) 1375 alloc_backing_store(bo); 1376 } 1377 bo_fake->card_dirty = 1; 1378 bufmgr_fake->performed_rendering = 1; 1379 } 1380 1381 return drm_intel_fake_bo_validate(bo); 1382 } 1383 1384 static void 1385 drm_intel_bo_fake_post_submit(drm_intel_bo *bo) 1386 { 1387 drm_intel_bufmgr_fake *bufmgr_fake = 1388 (drm_intel_bufmgr_fake *) bo->bufmgr; 1389 drm_intel_bo_fake *bo_fake = (drm_intel_bo_fake *) bo; 1390 int i; 1391 1392 for (i = 0; i < bo_fake->nr_relocs; i++) { 1393 struct fake_buffer_reloc *r = &bo_fake->relocs[i]; 1394 drm_intel_bo_fake *target_fake = 1395 (drm_intel_bo_fake *) r->target_buf; 1396 1397 if (target_fake->validated) 1398 drm_intel_bo_fake_post_submit(r->target_buf); 1399 1400 DBG("%s@0x%08x + 0x%08x -> %s@0x%08x + 0x%08x\n", 1401 bo_fake->name, (uint32_t) bo->offset, r->offset, 1402 target_fake->name, (uint32_t) r->target_buf->offset, 1403 r->delta); 1404 } 1405 1406 assert(bo_fake->map_count == 0); 1407 bo_fake->validated = 0; 1408 bo_fake->read_domains = 0; 1409 bo_fake->write_domain = 0; 1410 } 1411 1412 drm_public void 1413 drm_intel_bufmgr_fake_set_exec_callback(drm_intel_bufmgr *bufmgr, 1414 int (*exec) (drm_intel_bo *bo, 1415 unsigned int used, 1416 void *priv), 1417 void *priv) 1418 { 1419 drm_intel_bufmgr_fake *bufmgr_fake = (drm_intel_bufmgr_fake *) bufmgr; 1420 1421 bufmgr_fake->exec = exec; 1422 bufmgr_fake->exec_priv = priv; 1423 } 1424 1425 static int 1426 drm_intel_fake_bo_exec(drm_intel_bo *bo, int used, 1427 drm_clip_rect_t * cliprects, int num_cliprects, int DR4) 1428 { 1429 drm_intel_bufmgr_fake *bufmgr_fake = 1430 (drm_intel_bufmgr_fake *) bo->bufmgr; 1431 drm_intel_bo_fake *batch_fake = (drm_intel_bo_fake *) bo; 1432 struct drm_i915_batchbuffer batch; 1433 int ret; 1434 int retry_count = 0; 1435 1436 pthread_mutex_lock(&bufmgr_fake->lock); 1437 1438 bufmgr_fake->performed_rendering = 0; 1439 1440 drm_intel_fake_calculate_domains(bo); 1441 1442 batch_fake->read_domains = I915_GEM_DOMAIN_COMMAND; 1443 1444 /* we've ran out of RAM so blow the whole lot away and retry */ 1445 restart: 1446 ret = drm_intel_fake_reloc_and_validate_buffer(bo); 1447 if (bufmgr_fake->fail == 1) { 1448 if (retry_count == 0) { 1449 retry_count++; 1450 drm_intel_fake_kick_all_locked(bufmgr_fake); 1451 bufmgr_fake->fail = 0; 1452 goto restart; 1453 } else /* dump out the memory here */ 1454 mmDumpMemInfo(bufmgr_fake->heap); 1455 } 1456 1457 assert(ret == 0); 1458 1459 if (bufmgr_fake->exec != NULL) { 1460 ret = bufmgr_fake->exec(bo, used, bufmgr_fake->exec_priv); 1461 if (ret != 0) { 1462 pthread_mutex_unlock(&bufmgr_fake->lock); 1463 return ret; 1464 } 1465 } else { 1466 batch.start = bo->offset; 1467 batch.used = used; 1468 batch.cliprects = cliprects; 1469 batch.num_cliprects = num_cliprects; 1470 batch.DR1 = 0; 1471 batch.DR4 = DR4; 1472 1473 if (drmCommandWrite 1474 (bufmgr_fake->fd, DRM_I915_BATCHBUFFER, &batch, 1475 sizeof(batch))) { 1476 drmMsg("DRM_I915_BATCHBUFFER: %d\n", -errno); 1477 pthread_mutex_unlock(&bufmgr_fake->lock); 1478 return -errno; 1479 } 1480 } 1481 1482 drm_intel_fake_fence_validated(bo->bufmgr); 1483 1484 drm_intel_bo_fake_post_submit(bo); 1485 1486 pthread_mutex_unlock(&bufmgr_fake->lock); 1487 1488 return 0; 1489 } 1490 1491 /** 1492 * Return an error if the list of BOs will exceed the aperture size. 1493 * 1494 * This is a rough guess and likely to fail, as during the validate sequence we 1495 * may place a buffer in an inopportune spot early on and then fail to fit 1496 * a set smaller than the aperture. 1497 */ 1498 static int 1499 drm_intel_fake_check_aperture_space(drm_intel_bo ** bo_array, int count) 1500 { 1501 drm_intel_bufmgr_fake *bufmgr_fake = 1502 (drm_intel_bufmgr_fake *) bo_array[0]->bufmgr; 1503 unsigned int sz = 0; 1504 int i; 1505 1506 for (i = 0; i < count; i++) { 1507 drm_intel_bo_fake *bo_fake = (drm_intel_bo_fake *) bo_array[i]; 1508 1509 if (bo_fake == NULL) 1510 continue; 1511 1512 if (!bo_fake->is_static) 1513 sz += ALIGN(bo_array[i]->size, bo_fake->alignment); 1514 sz += bo_fake->child_size; 1515 } 1516 1517 if (sz > bufmgr_fake->size) { 1518 DBG("check_space: overflowed bufmgr size, %ukb vs %lukb\n", 1519 sz / 1024, bufmgr_fake->size / 1024); 1520 return -1; 1521 } 1522 1523 DBG("drm_check_space: sz %ukb vs bufgr %lukb\n", sz / 1024, 1524 bufmgr_fake->size / 1024); 1525 return 0; 1526 } 1527 1528 /** 1529 * Evicts all buffers, waiting for fences to pass and copying contents out 1530 * as necessary. 1531 * 1532 * Used by the X Server on LeaveVT, when the card memory is no longer our 1533 * own. 1534 */ 1535 drm_public void 1536 drm_intel_bufmgr_fake_evict_all(drm_intel_bufmgr *bufmgr) 1537 { 1538 drm_intel_bufmgr_fake *bufmgr_fake = (drm_intel_bufmgr_fake *) bufmgr; 1539 struct block *block, *tmp; 1540 1541 pthread_mutex_lock(&bufmgr_fake->lock); 1542 1543 bufmgr_fake->need_fence = 1; 1544 bufmgr_fake->fail = 0; 1545 1546 /* Wait for hardware idle. We don't know where acceleration has been 1547 * happening, so we'll need to wait anyway before letting anything get 1548 * put on the card again. 1549 */ 1550 drm_intel_bufmgr_fake_wait_idle(bufmgr_fake); 1551 1552 /* Check that we hadn't released the lock without having fenced the last 1553 * set of buffers. 1554 */ 1555 assert(DRMLISTEMPTY(&bufmgr_fake->fenced)); 1556 assert(DRMLISTEMPTY(&bufmgr_fake->on_hardware)); 1557 1558 DRMLISTFOREACHSAFE(block, tmp, &bufmgr_fake->lru) { 1559 drm_intel_bo_fake *bo_fake = (drm_intel_bo_fake *) block->bo; 1560 /* Releases the memory, and memcpys dirty contents out if 1561 * necessary. 1562 */ 1563 free_block(bufmgr_fake, block, 0); 1564 bo_fake->block = NULL; 1565 } 1566 1567 pthread_mutex_unlock(&bufmgr_fake->lock); 1568 } 1569 1570 drm_public void 1571 drm_intel_bufmgr_fake_set_last_dispatch(drm_intel_bufmgr *bufmgr, 1572 volatile unsigned int 1573 *last_dispatch) 1574 { 1575 drm_intel_bufmgr_fake *bufmgr_fake = (drm_intel_bufmgr_fake *) bufmgr; 1576 1577 bufmgr_fake->last_dispatch = (volatile int *)last_dispatch; 1578 } 1579 1580 drm_public drm_intel_bufmgr * 1581 drm_intel_bufmgr_fake_init(int fd, unsigned long low_offset, 1582 void *low_virtual, unsigned long size, 1583 volatile unsigned int *last_dispatch) 1584 { 1585 drm_intel_bufmgr_fake *bufmgr_fake; 1586 1587 bufmgr_fake = calloc(1, sizeof(*bufmgr_fake)); 1588 1589 if (pthread_mutex_init(&bufmgr_fake->lock, NULL) != 0) { 1590 free(bufmgr_fake); 1591 return NULL; 1592 } 1593 1594 /* Initialize allocator */ 1595 DRMINITLISTHEAD(&bufmgr_fake->fenced); 1596 DRMINITLISTHEAD(&bufmgr_fake->on_hardware); 1597 DRMINITLISTHEAD(&bufmgr_fake->lru); 1598 1599 bufmgr_fake->low_offset = low_offset; 1600 bufmgr_fake->virtual = low_virtual; 1601 bufmgr_fake->size = size; 1602 bufmgr_fake->heap = mmInit(low_offset, size); 1603 1604 /* Hook in methods */ 1605 bufmgr_fake->bufmgr.bo_alloc = drm_intel_fake_bo_alloc; 1606 bufmgr_fake->bufmgr.bo_alloc_for_render = drm_intel_fake_bo_alloc; 1607 bufmgr_fake->bufmgr.bo_alloc_tiled = drm_intel_fake_bo_alloc_tiled; 1608 bufmgr_fake->bufmgr.bo_reference = drm_intel_fake_bo_reference; 1609 bufmgr_fake->bufmgr.bo_unreference = drm_intel_fake_bo_unreference; 1610 bufmgr_fake->bufmgr.bo_map = drm_intel_fake_bo_map; 1611 bufmgr_fake->bufmgr.bo_unmap = drm_intel_fake_bo_unmap; 1612 bufmgr_fake->bufmgr.bo_subdata = drm_intel_fake_bo_subdata; 1613 bufmgr_fake->bufmgr.bo_wait_rendering = 1614 drm_intel_fake_bo_wait_rendering; 1615 bufmgr_fake->bufmgr.bo_emit_reloc = drm_intel_fake_emit_reloc; 1616 bufmgr_fake->bufmgr.destroy = drm_intel_fake_destroy; 1617 bufmgr_fake->bufmgr.bo_exec = drm_intel_fake_bo_exec; 1618 bufmgr_fake->bufmgr.check_aperture_space = 1619 drm_intel_fake_check_aperture_space; 1620 bufmgr_fake->bufmgr.debug = 0; 1621 1622 bufmgr_fake->fd = fd; 1623 bufmgr_fake->last_dispatch = (volatile int *)last_dispatch; 1624 1625 return &bufmgr_fake->bufmgr; 1626 } 1627
Indexes created Sun Jul 05 00:25:41 UTC 2026