1/* 2 * Copyright (c) 2011 Intel Corporation 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice (including the next 12 * paragraph) shall be included in all copies or substantial portions of the 13 * Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 * SOFTWARE. 22 * 23 * Authors: 24 * Chris Wilson <chris@chris-wilson.co.uk> 25 * 26 */ 27 28#ifndef KGEM_H 29#define KGEM_H 30 31#include <stdint.h> 32#include <stdbool.h> 33#include <stdarg.h> 34 35#include <i915_drm.h> 36 37#include "compiler.h" 38#include "debug.h" 39 40struct kgem_bo { 41 struct kgem_request *rq; 42#define RQ(rq) ((struct kgem_request *)((uintptr_t)(rq) & ~3)) 43#define RQ_RING(rq) ((uintptr_t)(rq) & 3) 44#define RQ_IS_BLT(rq) (RQ_RING(rq) == KGEM_BLT) 45#define MAKE_REQUEST(rq, ring) ((struct kgem_request *)((uintptr_t)(rq) | (ring))) 46 47 struct drm_i915_gem_exec_object2 *exec; 48 49 struct kgem_bo *proxy; 50 51 struct list list; 52 struct list request; 53 struct list vma; 54 55 void *map__cpu; 56 void *map__gtt; 57 void *map__wc; 58#define MAP(ptr) ((void*)((uintptr_t)(ptr) & ~3)) 59 60 struct kgem_bo_binding { 61 struct kgem_bo_binding *next; 62 uint32_t format; 63 uint16_t offset; 64 } binding; 65 66 uint64_t presumed_offset; 67 uint32_t unique_id; 68 uint32_t refcnt; 69 uint32_t handle; 70 uint32_t target_handle; 71 uint32_t delta; 72 uint32_t active_scanout; 73 union { 74 struct { 75 uint32_t count:27; 76#define PAGE_SIZE 4096 77 uint32_t bucket:5; 78#define NUM_CACHE_BUCKETS 16 79#define MAX_CACHE_SIZE (1 << (NUM_CACHE_BUCKETS+12)) 80 } pages; 81 uint32_t bytes; 82 } size; 83 uint32_t pitch : 18; /* max 128k */ 84 uint32_t tiling : 2; 85 uint32_t reusable : 1; 86 uint32_t gpu_dirty : 1; 87 uint32_t gtt_dirty : 1; 88 uint32_t domain : 2; 89 uint32_t needs_flush : 1; 90 uint32_t snoop : 1; 91 uint32_t io : 1; 92 uint32_t flush : 1; 93 uint32_t scanout : 1; 94 uint32_t prime : 1; 95 uint32_t purged : 1; 96}; 97#define DOMAIN_NONE 0 98#define DOMAIN_CPU 1 99#define DOMAIN_GTT 2 100#define DOMAIN_GPU 3 101 102struct kgem_request { 103 struct list list; 104 struct kgem_bo *bo; 105 struct list buffers; 106 int ring; 107}; 108 109enum { 110 MAP_GTT = 0, 111 MAP_CPU, 112 NUM_MAP_TYPES, 113}; 114 115struct kgem { 116 unsigned wedged; 117 int fd; 118 unsigned gen; 119 120 uint32_t unique_id; 121 122 uint16_t nbatch; 123 uint16_t surface; 124 uint16_t nexec; 125 uint16_t nreloc; 126 uint16_t nreloc__self; 127 uint16_t nfence; 128 uint16_t batch_size; 129 130 uint32_t *batch; 131 132 enum kgem_mode { 133 /* order matches I915_EXEC_RING ordering */ 134 KGEM_NONE = 0, 135 KGEM_RENDER, 136 KGEM_BSD, 137 KGEM_BLT, 138 } mode, ring; 139 140 struct list flushing; 141 struct list large; 142 struct list large_inactive; 143 struct list active[NUM_CACHE_BUCKETS][3]; 144 struct list inactive[NUM_CACHE_BUCKETS]; 145 struct list pinned_batches[2]; 146 struct list snoop; 147 struct list scanout; 148 struct list batch_buffers, active_buffers; 149 150 struct list requests[2]; 151 struct kgem_request *fence[2]; 152 struct kgem_request *next_request; 153 struct kgem_request static_request; 154 155 struct { 156 struct list inactive[NUM_CACHE_BUCKETS]; 157 int16_t count; 158 } vma[NUM_MAP_TYPES]; 159 160 uint32_t batch_flags; 161 uint32_t batch_flags_base; 162#define I915_EXEC_SECURE (1<<9) 163#define LOCAL_EXEC_OBJECT_WRITE (1<<2) 164 165 uint32_t flush:1; 166 uint32_t need_expire:1; 167 uint32_t need_purge:1; 168 uint32_t need_retire:1; 169 uint32_t need_throttle:1; 170 uint32_t needs_semaphore:1; 171 uint32_t needs_reservation:1; 172 uint32_t scanout_busy:1; 173 uint32_t busy:1; 174 175 uint32_t has_create2 :1; 176 uint32_t has_userptr :1; 177 uint32_t has_blt :1; 178 uint32_t has_relaxed_fencing :1; 179 uint32_t has_relaxed_delta :1; 180 uint32_t has_semaphores :1; 181 uint32_t has_secure_batches :1; 182 uint32_t has_pinned_batches :1; 183 uint32_t has_caching :1; 184 uint32_t has_llc :1; 185 uint32_t has_wt :1; 186 uint32_t has_no_reloc :1; 187 uint32_t has_handle_lut :1; 188 uint32_t has_wc_mmap :1; 189 190 uint32_t can_blt_cpu :1; 191 uint32_t can_render_y :1; 192 193 uint16_t fence_max; 194 uint16_t half_cpu_cache_pages; 195 uint32_t aperture_total, aperture_high, aperture_low, aperture_mappable, aperture_fenceable; 196 uint32_t aperture, aperture_fenced, aperture_max_fence; 197 uint32_t max_upload_tile_size, max_copy_tile_size; 198 uint32_t max_gpu_size, max_cpu_size; 199 uint32_t large_object_size, max_object_size; 200 uint32_t buffer_size; 201 202 void (*context_switch)(struct kgem *kgem, int new_mode); 203 void (*retire)(struct kgem *kgem); 204 void (*expire)(struct kgem *kgem); 205 206 void (*memcpy_to_tiled_x)(const void *src, void *dst, int bpp, 207 int32_t src_stride, int32_t dst_stride, 208 int16_t src_x, int16_t src_y, 209 int16_t dst_x, int16_t dst_y, 210 uint16_t width, uint16_t height); 211 void (*memcpy_from_tiled_x)(const void *src, void *dst, int bpp, 212 int32_t src_stride, int32_t dst_stride, 213 int16_t src_x, int16_t src_y, 214 int16_t dst_x, int16_t dst_y, 215 uint16_t width, uint16_t height); 216 217 struct kgem_bo *batch_bo; 218 219 uint16_t reloc__self[256]; 220 struct drm_i915_gem_exec_object2 exec[384] page_aligned; 221 struct drm_i915_gem_relocation_entry reloc[8192] page_aligned; 222 223#ifdef DEBUG_MEMORY 224 struct { 225 int bo_allocs; 226 size_t bo_bytes; 227 } debug_memory; 228#endif 229}; 230 231#define KGEM_MAX_DEFERRED_VBO 16 232 233#define KGEM_BATCH_RESERVED 1 234#define KGEM_RELOC_RESERVED (KGEM_MAX_DEFERRED_VBO) 235#define KGEM_EXEC_RESERVED (1+KGEM_MAX_DEFERRED_VBO) 236 237#ifndef ARRAY_SIZE 238#define ARRAY_SIZE(a) (sizeof(a)/sizeof((a)[0])) 239#endif 240 241#define KGEM_BATCH_SIZE(K) ((K)->batch_size-KGEM_BATCH_RESERVED) 242#define KGEM_EXEC_SIZE(K) (int)(ARRAY_SIZE((K)->exec)-KGEM_EXEC_RESERVED) 243#define KGEM_RELOC_SIZE(K) (int)(ARRAY_SIZE((K)->reloc)-KGEM_RELOC_RESERVED) 244 245void kgem_init(struct kgem *kgem, int fd, struct pci_device *dev, unsigned gen); 246void kgem_reset(struct kgem *kgem); 247 248struct kgem_bo *kgem_create_map(struct kgem *kgem, 249 void *ptr, uint32_t size, 250 bool read_only); 251 252struct kgem_bo *kgem_create_for_name(struct kgem *kgem, uint32_t name); 253struct kgem_bo *kgem_create_for_prime(struct kgem *kgem, int name, uint32_t size); 254int kgem_bo_export_to_prime(struct kgem *kgem, struct kgem_bo *bo); 255 256struct kgem_bo *kgem_create_linear(struct kgem *kgem, int size, unsigned flags); 257struct kgem_bo *kgem_create_proxy(struct kgem *kgem, 258 struct kgem_bo *target, 259 int offset, int length); 260 261struct kgem_bo *kgem_upload_source_image(struct kgem *kgem, 262 const void *data, 263 const BoxRec *box, 264 int stride, int bpp); 265void kgem_proxy_bo_attach(struct kgem_bo *bo, struct kgem_bo **ptr); 266 267int kgem_choose_tiling(struct kgem *kgem, 268 int tiling, int width, int height, int bpp); 269unsigned kgem_can_create_2d(struct kgem *kgem, int width, int height, int depth); 270#define KGEM_CAN_CREATE_GPU 0x1 271#define KGEM_CAN_CREATE_CPU 0x2 272#define KGEM_CAN_CREATE_LARGE 0x4 273#define KGEM_CAN_CREATE_GTT 0x8 274#define KGEM_CAN_CREATE_TILED 0x10 275 276bool kgem_check_surface_size(struct kgem *kgem, 277 uint32_t width, 278 uint32_t height, 279 uint32_t bpp, 280 uint32_t tiling, 281 uint32_t pitch, 282 uint32_t size); 283 284struct kgem_bo * 285kgem_replace_bo(struct kgem *kgem, 286 struct kgem_bo *src, 287 uint32_t width, 288 uint32_t height, 289 uint32_t pitch, 290 uint32_t bpp); 291enum { 292 CREATE_EXACT = 0x1, 293 CREATE_INACTIVE = 0x2, 294 CREATE_CPU_MAP = 0x4, 295 CREATE_GTT_MAP = 0x8, 296 CREATE_SCANOUT = 0x10, 297 CREATE_PRIME = 0x20, 298 CREATE_TEMPORARY = 0x40, 299 CREATE_CACHED = 0x80, 300 CREATE_UNCACHED = 0x100, 301 CREATE_NO_RETIRE = 0x200, 302 CREATE_NO_THROTTLE = 0x400, 303}; 304struct kgem_bo *kgem_create_2d(struct kgem *kgem, 305 int width, 306 int height, 307 int bpp, 308 int tiling, 309 uint32_t flags); 310struct kgem_bo *kgem_create_cpu_2d(struct kgem *kgem, 311 int width, 312 int height, 313 int bpp, 314 uint32_t flags); 315 316bool kgem_bo_convert_to_gpu(struct kgem *kgem, 317 struct kgem_bo *bo, 318 unsigned flags); 319 320uint32_t kgem_bo_get_binding(struct kgem_bo *bo, uint32_t format); 321void kgem_bo_set_binding(struct kgem_bo *bo, uint32_t format, uint16_t offset); 322 323bool kgem_retire(struct kgem *kgem); 324void kgem_retire__buffers(struct kgem *kgem); 325 326static inline bool kgem_bo_discard_cache(struct kgem_bo *bo, bool force) 327{ 328 if (bo == NULL || bo->proxy == NULL) 329 return false; 330 331 if (force) 332 return true; 333 334 if (bo->proxy->rq) 335 return false; 336 337 return bo->snoop; 338} 339 340bool __kgem_ring_is_idle(struct kgem *kgem, int ring); 341static inline bool kgem_ring_is_idle(struct kgem *kgem, int ring) 342{ 343 ring = ring == KGEM_BLT; 344 345 if (list_is_empty(&kgem->requests[ring])) 346 return true; 347 348 return __kgem_ring_is_idle(kgem, ring); 349} 350 351static inline bool kgem_is_idle(struct kgem *kgem) 352{ 353 if (!kgem->need_retire) 354 return true; 355 356 return kgem_ring_is_idle(kgem, kgem->ring); 357} 358 359static inline bool __kgem_ring_empty(struct kgem *kgem) 360{ 361 return list_is_empty(&kgem->requests[kgem->ring == KGEM_BLT]); 362} 363 364void _kgem_submit(struct kgem *kgem); 365static inline void kgem_submit(struct kgem *kgem) 366{ 367 if (kgem->nbatch) 368 _kgem_submit(kgem); 369} 370 371static inline void kgem_bo_submit(struct kgem *kgem, struct kgem_bo *bo) 372{ 373 if (bo->exec == NULL) 374 return; 375 376 assert(bo->refcnt); 377 _kgem_submit(kgem); 378} 379 380void kgem_scanout_flush(struct kgem *kgem, struct kgem_bo *bo); 381 382static inline struct kgem_bo *kgem_bo_reference(struct kgem_bo *bo) 383{ 384 assert(bo->refcnt); 385 bo->refcnt++; 386 return bo; 387} 388 389void _kgem_bo_destroy(struct kgem *kgem, struct kgem_bo *bo); 390static inline void kgem_bo_destroy(struct kgem *kgem, struct kgem_bo *bo) 391{ 392 assert(bo->refcnt); 393 if (--bo->refcnt == 0) 394 _kgem_bo_destroy(kgem, bo); 395} 396 397void kgem_clear_dirty(struct kgem *kgem); 398 399static inline void kgem_set_mode(struct kgem *kgem, 400 enum kgem_mode mode, 401 struct kgem_bo *bo) 402{ 403 assert(!kgem->wedged); 404 405#if DEBUG_FLUSH_BATCH 406 kgem_submit(kgem); 407#endif 408 409 if (kgem->nreloc && bo->exec == NULL && kgem_ring_is_idle(kgem, kgem->ring)) { 410 DBG(("%s: flushing before new bo\n", __FUNCTION__)); 411 _kgem_submit(kgem); 412 } 413 414 if (kgem->mode == mode) 415 return; 416 417 kgem->context_switch(kgem, mode); 418 kgem->mode = mode; 419} 420 421static inline void _kgem_set_mode(struct kgem *kgem, enum kgem_mode mode) 422{ 423 assert(kgem->mode == KGEM_NONE); 424 assert(kgem->nbatch == 0); 425 assert(!kgem->wedged); 426 kgem->context_switch(kgem, mode); 427 kgem->mode = mode; 428} 429 430static inline int kgem_batch_space(struct kgem *kgem) 431{ 432 int rem = kgem->surface - kgem->nbatch; 433 assert(rem > 0); 434 return rem - KGEM_BATCH_RESERVED; 435} 436 437static inline bool kgem_check_batch(struct kgem *kgem, int num_dwords) 438{ 439 assert(num_dwords > 0); 440 assert(kgem->nbatch < kgem->surface); 441 assert(kgem->surface <= kgem->batch_size); 442 return likely(kgem->nbatch + num_dwords + KGEM_BATCH_RESERVED <= kgem->surface); 443} 444 445static inline bool kgem_check_reloc(struct kgem *kgem, int n) 446{ 447 assert(kgem->nreloc <= KGEM_RELOC_SIZE(kgem)); 448 return likely(kgem->nreloc + n <= KGEM_RELOC_SIZE(kgem)); 449} 450 451static inline bool kgem_check_exec(struct kgem *kgem, int n) 452{ 453 assert(kgem->nexec <= KGEM_EXEC_SIZE(kgem)); 454 return likely(kgem->nexec + n <= KGEM_EXEC_SIZE(kgem)); 455} 456 457static inline bool kgem_check_reloc_and_exec(struct kgem *kgem, int n) 458{ 459 return kgem_check_reloc(kgem, n) && kgem_check_exec(kgem, n); 460} 461 462static inline bool kgem_check_batch_with_surfaces(struct kgem *kgem, 463 int num_dwords, 464 int num_surfaces) 465{ 466 return (int)(kgem->nbatch + num_dwords + KGEM_BATCH_RESERVED) <= (int)(kgem->surface - num_surfaces*8) && 467 kgem_check_reloc(kgem, num_surfaces) && 468 kgem_check_exec(kgem, num_surfaces); 469} 470 471static inline uint32_t *kgem_get_batch(struct kgem *kgem) 472{ 473 if (kgem->nreloc) { 474 unsigned mode = kgem->mode; 475 _kgem_submit(kgem); 476 _kgem_set_mode(kgem, mode); 477 } 478 479 return kgem->batch + kgem->nbatch; 480} 481 482bool kgem_check_bo(struct kgem *kgem, ...) __attribute__((sentinel(0))); 483bool kgem_check_bo_fenced(struct kgem *kgem, struct kgem_bo *bo); 484bool kgem_check_many_bo_fenced(struct kgem *kgem, ...) __attribute__((sentinel(0))); 485 486#define KGEM_RELOC_FENCED 0x8000 487uint32_t kgem_add_reloc(struct kgem *kgem, 488 uint32_t pos, 489 struct kgem_bo *bo, 490 uint32_t read_write_domains, 491 uint32_t delta); 492uint64_t kgem_add_reloc64(struct kgem *kgem, 493 uint32_t pos, 494 struct kgem_bo *bo, 495 uint32_t read_write_domains, 496 uint64_t delta); 497 498void *kgem_bo_map(struct kgem *kgem, struct kgem_bo *bo); 499void *kgem_bo_map__async(struct kgem *kgem, struct kgem_bo *bo); 500void *kgem_bo_map__gtt(struct kgem *kgem, struct kgem_bo *bo); 501void *kgem_bo_map__wc(struct kgem *kgem, struct kgem_bo *bo); 502void kgem_bo_sync__gtt(struct kgem *kgem, struct kgem_bo *bo); 503void *kgem_bo_map__debug(struct kgem *kgem, struct kgem_bo *bo); 504void *kgem_bo_map__cpu(struct kgem *kgem, struct kgem_bo *bo); 505void kgem_bo_sync__cpu(struct kgem *kgem, struct kgem_bo *bo); 506void kgem_bo_sync__cpu_full(struct kgem *kgem, struct kgem_bo *bo, bool write); 507uint32_t kgem_bo_flink(struct kgem *kgem, struct kgem_bo *bo); 508 509bool kgem_bo_write(struct kgem *kgem, struct kgem_bo *bo, 510 const void *data, int length); 511 512int kgem_bo_fenced_size(struct kgem *kgem, struct kgem_bo *bo); 513void kgem_get_tile_size(struct kgem *kgem, int tiling, int pitch, 514 int *tile_width, int *tile_height, int *tile_size); 515 516static inline int __kgem_buffer_size(struct kgem_bo *bo) 517{ 518 assert(bo->proxy != NULL); 519 return bo->size.bytes; 520} 521 522static inline int __kgem_bo_size(struct kgem_bo *bo) 523{ 524 assert(bo->proxy == NULL); 525 return PAGE_SIZE * bo->size.pages.count; 526} 527 528static inline int __kgem_bo_num_pages(struct kgem_bo *bo) 529{ 530 assert(bo->proxy == NULL); 531 return bo->size.pages.count; 532} 533 534static inline int kgem_bo_size(struct kgem_bo *bo) 535{ 536 if (bo->proxy) 537 return __kgem_buffer_size(bo); 538 else 539 return __kgem_bo_size(bo); 540} 541 542static inline bool kgem_bo_blt_pitch_is_ok(struct kgem *kgem, 543 struct kgem_bo *bo) 544{ 545 int pitch = bo->pitch; 546 547 if (kgem->gen >= 0100 && pitch & (1 << 4)) { /* bdw is broken */ 548 DBG(("%s: can not blt to handle=%d, pitch=%d\n", 549 __FUNCTION__, bo->handle, pitch)); 550 return false; 551 } 552 553 if (kgem->gen >= 040 && bo->tiling) 554 pitch /= 4; 555 if (pitch > MAXSHORT) { 556 DBG(("%s: can not blt to handle=%d, adjusted pitch=%d\n", 557 __FUNCTION__, bo->handle, pitch)); 558 return false; 559 } 560 561 return true; 562} 563 564static inline bool kgem_bo_can_blt(struct kgem *kgem, 565 struct kgem_bo *bo) 566{ 567 assert(bo->refcnt); 568 569 if (bo->tiling == I915_TILING_Y) { 570 DBG(("%s: can not blt to handle=%d, tiling=Y\n", 571 __FUNCTION__, bo->handle)); 572 return false; 573 } 574 575 if (kgem->gen >= 0100 && bo->proxy && bo->delta & (1 << 4)) { 576 DBG(("%s: can not blt to handle=%d, delta=%d\n", 577 __FUNCTION__, bo->handle, bo->delta)); 578 return false; 579 } 580 581 return kgem_bo_blt_pitch_is_ok(kgem, bo); 582} 583 584static inline bool kgem_bo_is_snoop(struct kgem_bo *bo) 585{ 586 assert(bo->refcnt); 587 while (bo->proxy) 588 bo = bo->proxy; 589 return bo->snoop; 590} 591 592void kgem_bo_undo(struct kgem *kgem, struct kgem_bo *bo); 593void kgem_bo_pair_undo(struct kgem *kgem, struct kgem_bo *a, struct kgem_bo *b); 594 595bool __kgem_busy(struct kgem *kgem, int handle); 596 597static inline void kgem_bo_mark_busy(struct kgem *kgem, struct kgem_bo *bo, int ring) 598{ 599 assert(bo->refcnt); 600 bo->needs_flush = true; 601 if (bo->rq) { 602 bo->rq = MAKE_REQUEST(RQ(bo->rq), ring); 603 } else { 604 bo->rq = MAKE_REQUEST(kgem, ring); 605 list_add(&bo->request, &kgem->flushing); 606 kgem->need_retire = true; 607 } 608} 609 610inline static void __kgem_bo_clear_busy(struct kgem_bo *bo) 611{ 612 DBG(("%s: handle=%d\n", __FUNCTION__, bo->handle)); 613 bo->rq = NULL; 614 list_del(&bo->request); 615 616 bo->domain = DOMAIN_NONE; 617 bo->needs_flush = false; 618 bo->gtt_dirty = false; 619} 620 621static inline bool kgem_bo_is_busy(struct kgem_bo *bo) 622{ 623 DBG(("%s: handle=%d, domain: %d exec? %d, rq? %d\n", __FUNCTION__, 624 bo->handle, bo->domain, bo->exec != NULL, bo->rq != NULL)); 625 assert(bo->refcnt); 626 return bo->rq; 627} 628 629void __kgem_retire_requests_upto(struct kgem *kgem, struct kgem_bo *bo); 630static inline bool __kgem_bo_is_busy(struct kgem *kgem, struct kgem_bo *bo) 631{ 632 DBG(("%s: handle=%d, domain: %d exec? %d, rq? %d\n", __FUNCTION__, 633 bo->handle, bo->domain, bo->exec != NULL, bo->rq != NULL)); 634 assert(bo->refcnt); 635 636 if (bo->exec) 637 return true; 638 639 if (bo->rq && !__kgem_busy(kgem, bo->handle)) { 640 __kgem_retire_requests_upto(kgem, bo); 641 assert(list_is_empty(&bo->request)); 642 assert(bo->rq == NULL); 643 assert(bo->domain == DOMAIN_NONE); 644 } 645 646 return kgem_bo_is_busy(bo); 647} 648 649static inline bool kgem_bo_is_render(struct kgem_bo *bo) 650{ 651 DBG(("%s: handle=%d, rq? %d [%d]\n", __FUNCTION__, 652 bo->handle, bo->rq != NULL, (int)RQ_RING(bo->rq))); 653 assert(bo->refcnt); 654 return bo->rq && RQ_RING(bo->rq) == I915_EXEC_RENDER; 655} 656 657static inline void kgem_bo_mark_unreusable(struct kgem_bo *bo) 658{ 659 assert(bo->refcnt); 660 while (bo->proxy) { 661 bo->flush = true; 662 bo = bo->proxy; 663 assert(bo->refcnt); 664 } 665 bo->flush = true; 666 bo->reusable = false; 667} 668 669static inline bool kgem_bo_is_dirty(struct kgem_bo *bo) 670{ 671 if (bo == NULL) 672 return false; 673 674 assert(bo->refcnt); 675 return bo->gpu_dirty; 676} 677 678static inline void kgem_bo_unclean(struct kgem *kgem, struct kgem_bo *bo) 679{ 680 /* The bo is outside of our control, so presume it is written to */ 681 bo->needs_flush = true; 682 if (bo->rq == NULL) 683 bo->rq = (void *)kgem; 684 685 if (bo->domain != DOMAIN_GPU) 686 bo->domain = DOMAIN_NONE; 687} 688 689static inline void __kgem_bo_mark_dirty(struct kgem_bo *bo) 690{ 691 DBG(("%s: handle=%d (proxy? %d)\n", __FUNCTION__, 692 bo->handle, bo->proxy != NULL)); 693 694 assert(bo->refcnt); 695 assert(bo->exec); 696 assert(bo->rq); 697 698 bo->exec->flags |= LOCAL_EXEC_OBJECT_WRITE; 699 bo->needs_flush = bo->gpu_dirty = true; 700 list_move(&bo->request, &RQ(bo->rq)->buffers); 701} 702 703static inline void kgem_bo_mark_dirty(struct kgem_bo *bo) 704{ 705 assert(bo->refcnt); 706 do { 707 assert(bo->exec); 708 assert(bo->rq); 709 710 if (bo->gpu_dirty) 711 return; 712 713 __kgem_bo_mark_dirty(bo); 714 } while ((bo = bo->proxy)); 715} 716 717static inline bool kgem_bo_mapped(struct kgem *kgem, struct kgem_bo *bo) 718{ 719 DBG(("%s: handle=%d, map=%p:%p, tiling=%d, domain=%d\n", 720 __FUNCTION__, bo->handle, bo->map__gtt, bo->map__cpu, bo->tiling, bo->domain)); 721 assert(bo->proxy == NULL); 722 723 if (bo->tiling == I915_TILING_NONE && (bo->domain == DOMAIN_CPU || kgem->has_llc)) 724 return bo->map__cpu != NULL; 725 726 if (bo->tiling == I915_TILING_NONE && bo->map__wc) 727 return true; 728 729 return bo->map__gtt != NULL; 730} 731 732static inline bool kgem_bo_can_map(struct kgem *kgem, struct kgem_bo *bo) 733{ 734 DBG(("%s: handle=%d, map=%p:%p:%p, tiling=%d, domain=%d, offset=%ld\n", 735 __FUNCTION__, bo->handle, bo->map__gtt, bo->map__wc, bo->map__cpu, bo->tiling, bo->domain, (long)bo->presumed_offset)); 736 737 if (!bo->tiling && (kgem->has_llc || bo->domain == DOMAIN_CPU)) 738 return true; 739 740 assert(bo->proxy == NULL); 741 742 if (bo->map__gtt != NULL) 743 return true; 744 745 if (kgem->gen == 021 && bo->tiling == I915_TILING_Y) 746 return false; 747 748 if (!bo->tiling && kgem->has_wc_mmap) 749 return true; 750 751 return __kgem_bo_num_pages(bo) <= kgem->aperture_mappable / 4; 752} 753 754static inline bool kgem_bo_can_map__cpu(struct kgem *kgem, 755 struct kgem_bo *bo, 756 bool write) 757{ 758 DBG(("%s: handle=%d\n", __FUNCTION__, bo->handle)); 759 assert(bo->refcnt); 760 761 if (bo->purged || (bo->scanout && write)) { 762 DBG(("%s: no, writing to scanout? %d, or is stolen [inaccessible via CPU]? %d\n", 763 __FUNCTION__, bo->scanout && write, bo->purged)); 764 return false; 765 } 766 767 if (kgem->has_llc) { 768 DBG(("%s: yes, has LLC and target is in LLC\n", __FUNCTION__)); 769 return true; 770 } 771 772 DBG(("%s: non-LLC - CPU domain? %d, clean? %d\n", 773 __FUNCTION__, bo->domain == DOMAIN_CPU, !write || bo->exec == NULL)); 774 if (bo->domain != DOMAIN_CPU) 775 return false; 776 777 return !write || bo->exec == NULL; 778} 779 780#define KGEM_BUFFER_WRITE 0x1 781#define KGEM_BUFFER_INPLACE 0x2 782#define KGEM_BUFFER_LAST 0x4 783 784#define KGEM_BUFFER_WRITE_INPLACE (KGEM_BUFFER_WRITE | KGEM_BUFFER_INPLACE) 785 786struct kgem_bo *kgem_create_buffer(struct kgem *kgem, 787 uint32_t size, uint32_t flags, 788 void **ret); 789struct kgem_bo *kgem_create_buffer_2d(struct kgem *kgem, 790 int width, int height, int bpp, 791 uint32_t flags, 792 void **ret); 793bool kgem_buffer_is_inplace(struct kgem_bo *bo); 794void kgem_buffer_read_sync(struct kgem *kgem, struct kgem_bo *bo); 795 796int kgem_is_wedged(struct kgem *kgem); 797void kgem_throttle(struct kgem *kgem); 798#define MAX_INACTIVE_TIME 10 799bool kgem_expire_cache(struct kgem *kgem); 800bool kgem_cleanup_cache(struct kgem *kgem); 801 802void kgem_clean_scanout_cache(struct kgem *kgem); 803void kgem_clean_large_cache(struct kgem *kgem); 804 805#if HAS_DEBUG_FULL 806void __kgem_batch_debug(struct kgem *kgem, uint32_t nbatch); 807#else 808static inline void __kgem_batch_debug(struct kgem *kgem, uint32_t nbatch) 809{ 810 (void)kgem; 811 (void)nbatch; 812} 813#endif 814 815static inline void 816memcpy_to_tiled_x(struct kgem *kgem, 817 const void *src, void *dst, int bpp, 818 int32_t src_stride, int32_t dst_stride, 819 int16_t src_x, int16_t src_y, 820 int16_t dst_x, int16_t dst_y, 821 uint16_t width, uint16_t height) 822{ 823 assert(kgem->memcpy_to_tiled_x); 824 assert(src_x >= 0 && src_y >= 0); 825 assert(dst_x >= 0 && dst_y >= 0); 826 assert(8*src_stride >= (src_x+width) * bpp); 827 assert(8*dst_stride >= (dst_x+width) * bpp); 828 return kgem->memcpy_to_tiled_x(src, dst, bpp, 829 src_stride, dst_stride, 830 src_x, src_y, 831 dst_x, dst_y, 832 width, height); 833} 834 835static inline void 836memcpy_from_tiled_x(struct kgem *kgem, 837 const void *src, void *dst, int bpp, 838 int32_t src_stride, int32_t dst_stride, 839 int16_t src_x, int16_t src_y, 840 int16_t dst_x, int16_t dst_y, 841 uint16_t width, uint16_t height) 842{ 843 assert(kgem->memcpy_from_tiled_x); 844 assert(src_x >= 0 && src_y >= 0); 845 assert(dst_x >= 0 && dst_y >= 0); 846 assert(8*src_stride >= (src_x+width) * bpp); 847 assert(8*dst_stride >= (dst_x+width) * bpp); 848 return kgem->memcpy_from_tiled_x(src, dst, bpp, 849 src_stride, dst_stride, 850 src_x, src_y, 851 dst_x, dst_y, 852 width, height); 853} 854 855void choose_memcpy_tiled_x(struct kgem *kgem, int swizzling); 856 857#endif /* KGEM_H */ 858