i915_gem.c revision 1.34.4.1
1 /* $NetBSD: i915_gem.c,v 1.34.4.1 2019/06/10 22:08:05 christos Exp $ */ 2 3 /* 4 * Copyright 2008-2015 Intel Corporation 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining a 7 * copy of this software and associated documentation files (the "Software"), 8 * to deal in the Software without restriction, including without limitation 9 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 10 * and/or sell copies of the Software, and to permit persons to whom the 11 * Software is furnished to do so, subject to the following conditions: 12 * 13 * The above copyright notice and this permission notice (including the next 14 * paragraph) shall be included in all copies or substantial portions of the 15 * Software. 16 * 17 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 18 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 19 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 20 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 21 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 22 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 23 * IN THE SOFTWARE. 24 * 25 * Authors: 26 * Eric Anholt <eric (at) anholt.net> 27 * 28 */ 29 30 #include <sys/cdefs.h> 31 __KERNEL_RCSID(0, "$NetBSD: i915_gem.c,v 1.34.4.1 2019/06/10 22:08:05 christos Exp $"); 32 33 #ifdef __NetBSD__ 34 #if 0 /* XXX uvmhist option? */ 35 #include "opt_uvmhist.h" 36 #endif 37 38 #include <sys/types.h> 39 #include <sys/param.h> 40 41 #include <uvm/uvm.h> 42 #include <uvm/uvm_extern.h> 43 #include <uvm/uvm_fault.h> 44 #include <uvm/uvm_page.h> 45 #include <uvm/uvm_pmap.h> 46 #include <uvm/uvm_prot.h> 47 48 #include <drm/bus_dma_hacks.h> 49 #endif 50 51 #include <drm/drmP.h> 52 #include <drm/drm_vma_manager.h> 53 #include <drm/i915_drm.h> 54 #include "i915_drv.h" 55 #include "i915_vgpu.h" 56 #include "i915_trace.h" 57 #include "intel_drv.h" 58 #include <linux/shmem_fs.h> 59 #include <linux/slab.h> 60 #include <linux/swap.h> 61 #include <linux/pci.h> 62 #include <linux/dma-buf.h> 63 #include <linux/errno.h> 64 #include <linux/time.h> 65 #include <linux/err.h> 66 #include <linux/bitops.h> 67 #include <linux/printk.h> 68 #include <asm/param.h> 69 #include <asm/page.h> 70 #include <asm/cpufeature.h> 71 72 #define RQ_BUG_ON(expr) 73 74 static void i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj); 75 static void i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj); 76 static void 77 i915_gem_object_retire__write(struct drm_i915_gem_object *obj); 78 static void 79 i915_gem_object_retire__read(struct drm_i915_gem_object *obj, int ring); 80 81 static bool cpu_cache_is_coherent(struct drm_device *dev, 82 enum i915_cache_level level) 83 { 84 return HAS_LLC(dev) || level != I915_CACHE_NONE; 85 } 86 87 static bool cpu_write_needs_clflush(struct drm_i915_gem_object *obj) 88 { 89 if (!cpu_cache_is_coherent(obj->base.dev, obj->cache_level)) 90 return true; 91 92 return obj->pin_display; 93 } 94 95 /* some bookkeeping */ 96 static void i915_gem_info_add_obj(struct drm_i915_private *dev_priv, 97 size_t size) 98 { 99 spin_lock(&dev_priv->mm.object_stat_lock); 100 dev_priv->mm.object_count++; 101 dev_priv->mm.object_memory += size; 102 spin_unlock(&dev_priv->mm.object_stat_lock); 103 } 104 105 static void i915_gem_info_remove_obj(struct drm_i915_private *dev_priv, 106 size_t size) 107 { 108 spin_lock(&dev_priv->mm.object_stat_lock); 109 dev_priv->mm.object_count--; 110 dev_priv->mm.object_memory -= size; 111 spin_unlock(&dev_priv->mm.object_stat_lock); 112 } 113 114 static int 115 i915_gem_wait_for_error(struct i915_gpu_error *error) 116 { 117 int ret; 118 119 #define EXIT_COND (!i915_reset_in_progress(error) || \ 120 i915_terminally_wedged(error)) 121 if (EXIT_COND) 122 return 0; 123 124 /* 125 * Only wait 10 seconds for the gpu reset to complete to avoid hanging 126 * userspace. If it takes that long something really bad is going on and 127 * we should simply try to bail out and fail as gracefully as possible. 128 */ 129 #ifdef __NetBSD__ 130 spin_lock(&error->reset_lock); 131 DRM_SPIN_TIMED_WAIT_UNTIL(ret, &error->reset_queue, &error->reset_lock, 132 10*HZ, EXIT_COND); 133 spin_unlock(&error->reset_lock); 134 #else 135 ret = wait_event_interruptible_timeout(error->reset_queue, 136 EXIT_COND, 137 10*HZ); 138 #endif 139 if (ret == 0) { 140 DRM_ERROR("Timed out waiting for the gpu reset to complete\n"); 141 return -EIO; 142 } else if (ret < 0) { 143 return ret; 144 } 145 #undef EXIT_COND 146 147 return 0; 148 } 149 150 int i915_mutex_lock_interruptible(struct drm_device *dev) 151 { 152 struct drm_i915_private *dev_priv = dev->dev_private; 153 int ret; 154 155 ret = i915_gem_wait_for_error(&dev_priv->gpu_error); 156 if (ret) 157 return ret; 158 159 ret = mutex_lock_interruptible(&dev->struct_mutex); 160 if (ret) 161 return ret; 162 163 WARN_ON(i915_verify_lists(dev)); 164 return 0; 165 } 166 167 int 168 i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data, 169 struct drm_file *file) 170 { 171 struct drm_i915_private *dev_priv = dev->dev_private; 172 struct drm_i915_gem_get_aperture *args = data; 173 struct i915_gtt *ggtt = &dev_priv->gtt; 174 struct i915_vma *vma; 175 size_t pinned; 176 177 pinned = 0; 178 mutex_lock(&dev->struct_mutex); 179 list_for_each_entry(vma, &ggtt->base.active_list, mm_list) 180 if (vma->pin_count) 181 pinned += vma->node.size; 182 list_for_each_entry(vma, &ggtt->base.inactive_list, mm_list) 183 if (vma->pin_count) 184 pinned += vma->node.size; 185 mutex_unlock(&dev->struct_mutex); 186 187 args->aper_size = dev_priv->gtt.base.total; 188 args->aper_available_size = args->aper_size - pinned; 189 190 return 0; 191 } 192 193 static int 194 i915_gem_object_get_pages_phys(struct drm_i915_gem_object *obj) 195 { 196 #ifndef __NetBSD__ 197 struct address_space *mapping = file_inode(obj->base.filp)->i_mapping; 198 #endif 199 char *vaddr = obj->phys_handle->vaddr; 200 #ifndef __NetBSD__ 201 struct sg_table *st; 202 struct scatterlist *sg; 203 #endif 204 int i; 205 206 if (WARN_ON(i915_gem_object_needs_bit17_swizzle(obj))) 207 return -EINVAL; 208 209 for (i = 0; i < obj->base.size / PAGE_SIZE; i++) { 210 struct page *page; 211 char *src; 212 213 #ifdef __NetBSD__ 214 struct pglist pages = TAILQ_HEAD_INITIALIZER(pages); 215 int ret; 216 /* XXX errno NetBSD->Linux */ 217 ret = -uvm_obj_wirepages(obj->base.filp, i*PAGE_SIZE, 218 (i + 1)*PAGE_SIZE, &pages); 219 if (ret) 220 return ret; 221 page = container_of(TAILQ_FIRST(&pages), struct page, p_vmp); 222 #else 223 page = shmem_read_mapping_page(mapping, i); 224 if (IS_ERR(page)) 225 return PTR_ERR(page); 226 #endif 227 228 src = kmap_atomic(page); 229 memcpy(vaddr, src, PAGE_SIZE); 230 drm_clflush_virt_range(vaddr, PAGE_SIZE); 231 kunmap_atomic(src); 232 233 #ifdef __NetBSD__ 234 uvm_obj_unwirepages(obj->base.filp, i*PAGE_SIZE, 235 (i + 1)*PAGE_SIZE); 236 #else 237 page_cache_release(page); 238 #endif 239 vaddr += PAGE_SIZE; 240 } 241 242 i915_gem_chipset_flush(obj->base.dev); 243 244 #ifdef __NetBSD__ 245 obj->pages = obj->phys_handle->dmah_map; 246 #else 247 st = kmalloc(sizeof(*st), GFP_KERNEL); 248 if (st == NULL) 249 return -ENOMEM; 250 251 if (sg_alloc_table(st, 1, GFP_KERNEL)) { 252 kfree(st); 253 return -ENOMEM; 254 } 255 256 sg = st->sgl; 257 sg->offset = 0; 258 sg->length = obj->base.size; 259 260 sg_dma_address(sg) = obj->phys_handle->busaddr; 261 sg_dma_len(sg) = obj->base.size; 262 263 obj->pages = st; 264 #endif 265 return 0; 266 } 267 268 static void 269 i915_gem_object_put_pages_phys(struct drm_i915_gem_object *obj) 270 { 271 int ret; 272 273 BUG_ON(obj->madv == __I915_MADV_PURGED); 274 275 ret = i915_gem_object_set_to_cpu_domain(obj, true); 276 if (ret) { 277 /* In the event of a disaster, abandon all caches and 278 * hope for the best. 279 */ 280 WARN_ON(ret != -EIO); 281 obj->base.read_domains = obj->base.write_domain = I915_GEM_DOMAIN_CPU; 282 } 283 284 if (obj->madv == I915_MADV_DONTNEED) 285 obj->dirty = 0; 286 287 if (obj->dirty) { 288 #ifndef __NetBSD__ 289 struct address_space *mapping = file_inode(obj->base.filp)->i_mapping; 290 #endif 291 const char *vaddr = obj->phys_handle->vaddr; 292 int i; 293 294 for (i = 0; i < obj->base.size / PAGE_SIZE; i++) { 295 struct page *page; 296 char *dst; 297 298 #ifdef __NetBSD__ 299 struct pglist pages = TAILQ_HEAD_INITIALIZER(pages); 300 /* XXX errno NetBSD->Linux */ 301 ret = -uvm_obj_wirepages(obj->base.filp, 302 i*PAGE_SIZE, (i + 1)*PAGE_SIZE, &pages); 303 if (ret) 304 continue; 305 page = container_of(TAILQ_FIRST(&pages), struct page, 306 p_vmp); 307 #endif 308 309 dst = kmap_atomic(page); 310 drm_clflush_virt_range(vaddr, PAGE_SIZE); 311 memcpy(dst, vaddr, PAGE_SIZE); 312 kunmap_atomic(dst); 313 314 set_page_dirty(page); 315 #ifdef __NetBSD__ 316 /* XXX mark page accessed */ 317 uvm_obj_unwirepages(obj->base.filp, i*PAGE_SIZE, 318 (i+1)*PAGE_SIZE); 319 #else 320 if (obj->madv == I915_MADV_WILLNEED) 321 mark_page_accessed(page); 322 page_cache_release(page); 323 #endif 324 vaddr += PAGE_SIZE; 325 } 326 obj->dirty = 0; 327 } 328 329 #ifdef __NetBSD__ 330 obj->pages = NULL; 331 #else 332 sg_free_table(obj->pages); 333 kfree(obj->pages); 334 #endif 335 } 336 337 static void 338 i915_gem_object_release_phys(struct drm_i915_gem_object *obj) 339 { 340 drm_pci_free(obj->base.dev, obj->phys_handle); 341 } 342 343 static const struct drm_i915_gem_object_ops i915_gem_phys_ops = { 344 .get_pages = i915_gem_object_get_pages_phys, 345 .put_pages = i915_gem_object_put_pages_phys, 346 .release = i915_gem_object_release_phys, 347 }; 348 349 static int 350 drop_pages(struct drm_i915_gem_object *obj) 351 { 352 struct i915_vma *vma, *next; 353 int ret; 354 355 drm_gem_object_reference(&obj->base); 356 list_for_each_entry_safe(vma, next, &obj->vma_list, vma_link) 357 if (i915_vma_unbind(vma)) 358 break; 359 360 ret = i915_gem_object_put_pages(obj); 361 drm_gem_object_unreference(&obj->base); 362 363 return ret; 364 } 365 366 int 367 i915_gem_object_attach_phys(struct drm_i915_gem_object *obj, 368 int align) 369 { 370 drm_dma_handle_t *phys; 371 int ret; 372 373 if (obj->phys_handle) { 374 if ((unsigned long)obj->phys_handle->vaddr & (align -1)) 375 return -EBUSY; 376 377 return 0; 378 } 379 380 if (obj->madv != I915_MADV_WILLNEED) 381 return -EFAULT; 382 383 if (obj->base.filp == NULL) 384 return -EINVAL; 385 386 ret = drop_pages(obj); 387 if (ret) 388 return ret; 389 390 /* create a new object */ 391 phys = drm_pci_alloc(obj->base.dev, obj->base.size, align); 392 if (!phys) 393 return -ENOMEM; 394 395 obj->phys_handle = phys; 396 obj->ops = &i915_gem_phys_ops; 397 398 return i915_gem_object_get_pages(obj); 399 } 400 401 static int 402 i915_gem_phys_pwrite(struct drm_i915_gem_object *obj, 403 struct drm_i915_gem_pwrite *args, 404 struct drm_file *file_priv) 405 { 406 struct drm_device *dev = obj->base.dev; 407 void *vaddr = (char *)obj->phys_handle->vaddr + args->offset; 408 char __user *user_data = to_user_ptr(args->data_ptr); 409 int ret = 0; 410 411 /* We manually control the domain here and pretend that it 412 * remains coherent i.e. in the GTT domain, like shmem_pwrite. 413 */ 414 ret = i915_gem_object_wait_rendering(obj, false); 415 if (ret) 416 return ret; 417 418 intel_fb_obj_invalidate(obj, ORIGIN_CPU); 419 if (__copy_from_user_inatomic_nocache(vaddr, user_data, args->size)) { 420 unsigned long unwritten; 421 422 /* The physical object once assigned is fixed for the lifetime 423 * of the obj, so we can safely drop the lock and continue 424 * to access vaddr. 425 */ 426 mutex_unlock(&dev->struct_mutex); 427 unwritten = copy_from_user(vaddr, user_data, args->size); 428 mutex_lock(&dev->struct_mutex); 429 if (unwritten) { 430 ret = -EFAULT; 431 goto out; 432 } 433 } 434 435 drm_clflush_virt_range(vaddr, args->size); 436 i915_gem_chipset_flush(dev); 437 438 out: 439 intel_fb_obj_flush(obj, false, ORIGIN_CPU); 440 return ret; 441 } 442 443 void *i915_gem_object_alloc(struct drm_device *dev) 444 { 445 struct drm_i915_private *dev_priv = dev->dev_private; 446 return kmem_cache_zalloc(dev_priv->objects, GFP_KERNEL); 447 } 448 449 void i915_gem_object_free(struct drm_i915_gem_object *obj) 450 { 451 struct drm_i915_private *dev_priv = obj->base.dev->dev_private; 452 kmem_cache_free(dev_priv->objects, obj); 453 } 454 455 static int 456 i915_gem_create(struct drm_file *file, 457 struct drm_device *dev, 458 uint64_t size, 459 uint32_t *handle_p) 460 { 461 struct drm_i915_gem_object *obj; 462 int ret; 463 u32 handle; 464 465 size = roundup(size, PAGE_SIZE); 466 if (size == 0) 467 return -EINVAL; 468 469 /* Allocate the new object */ 470 obj = i915_gem_alloc_object(dev, size); 471 if (obj == NULL) 472 return -ENOMEM; 473 474 ret = drm_gem_handle_create(file, &obj->base, &handle); 475 /* drop reference from allocate - handle holds it now */ 476 drm_gem_object_unreference_unlocked(&obj->base); 477 if (ret) 478 return ret; 479 480 *handle_p = handle; 481 return 0; 482 } 483 484 int 485 i915_gem_dumb_create(struct drm_file *file, 486 struct drm_device *dev, 487 struct drm_mode_create_dumb *args) 488 { 489 /* have to work out size/pitch and return them */ 490 #ifdef __NetBSD__ /* ALIGN means something else. */ 491 args->pitch = round_up(args->width * DIV_ROUND_UP(args->bpp, 8), 64); 492 #else 493 args->pitch = ALIGN(args->width * DIV_ROUND_UP(args->bpp, 8), 64); 494 #endif 495 args->size = args->pitch * args->height; 496 return i915_gem_create(file, dev, 497 args->size, &args->handle); 498 } 499 500 /** 501 * Creates a new mm object and returns a handle to it. 502 */ 503 int 504 i915_gem_create_ioctl(struct drm_device *dev, void *data, 505 struct drm_file *file) 506 { 507 struct drm_i915_gem_create *args = data; 508 509 return i915_gem_create(file, dev, 510 args->size, &args->handle); 511 } 512 513 static inline int 514 __copy_to_user_swizzled(char __user *cpu_vaddr, 515 const char *gpu_vaddr, int gpu_offset, 516 int length) 517 { 518 int ret, cpu_offset = 0; 519 520 while (length > 0) { 521 #ifdef __NetBSD__ /* XXX ALIGN means something else. */ 522 int cacheline_end = round_up(gpu_offset + 1, 64); 523 #else 524 int cacheline_end = ALIGN(gpu_offset + 1, 64); 525 #endif 526 int this_length = min(cacheline_end - gpu_offset, length); 527 int swizzled_gpu_offset = gpu_offset ^ 64; 528 529 ret = __copy_to_user(cpu_vaddr + cpu_offset, 530 gpu_vaddr + swizzled_gpu_offset, 531 this_length); 532 if (ret) 533 return ret + length; 534 535 cpu_offset += this_length; 536 gpu_offset += this_length; 537 length -= this_length; 538 } 539 540 return 0; 541 } 542 543 static inline int 544 __copy_from_user_swizzled(char *gpu_vaddr, int gpu_offset, 545 const char __user *cpu_vaddr, 546 int length) 547 { 548 int ret, cpu_offset = 0; 549 550 while (length > 0) { 551 #ifdef __NetBSD__ /* XXX ALIGN means something else. */ 552 int cacheline_end = round_up(gpu_offset + 1, 64); 553 #else 554 int cacheline_end = ALIGN(gpu_offset + 1, 64); 555 #endif 556 int this_length = min(cacheline_end - gpu_offset, length); 557 int swizzled_gpu_offset = gpu_offset ^ 64; 558 559 ret = __copy_from_user(gpu_vaddr + swizzled_gpu_offset, 560 cpu_vaddr + cpu_offset, 561 this_length); 562 if (ret) 563 return ret + length; 564 565 cpu_offset += this_length; 566 gpu_offset += this_length; 567 length -= this_length; 568 } 569 570 return 0; 571 } 572 573 /* 574 * Pins the specified object's pages and synchronizes the object with 575 * GPU accesses. Sets needs_clflush to non-zero if the caller should 576 * flush the object from the CPU cache. 577 */ 578 int i915_gem_obj_prepare_shmem_read(struct drm_i915_gem_object *obj, 579 int *needs_clflush) 580 { 581 int ret; 582 583 *needs_clflush = 0; 584 585 if (!obj->base.filp) 586 return -EINVAL; 587 588 if (!(obj->base.read_domains & I915_GEM_DOMAIN_CPU)) { 589 /* If we're not in the cpu read domain, set ourself into the gtt 590 * read domain and manually flush cachelines (if required). This 591 * optimizes for the case when the gpu will dirty the data 592 * anyway again before the next pread happens. */ 593 *needs_clflush = !cpu_cache_is_coherent(obj->base.dev, 594 obj->cache_level); 595 ret = i915_gem_object_wait_rendering(obj, true); 596 if (ret) 597 return ret; 598 } 599 600 ret = i915_gem_object_get_pages(obj); 601 if (ret) 602 return ret; 603 604 i915_gem_object_pin_pages(obj); 605 606 return ret; 607 } 608 609 /* Per-page copy function for the shmem pread fastpath. 610 * Flushes invalid cachelines before reading the target if 611 * needs_clflush is set. */ 612 static int 613 shmem_pread_fast(struct page *page, int shmem_page_offset, int page_length, 614 char __user *user_data, 615 bool page_do_bit17_swizzling, bool needs_clflush) 616 { 617 #ifdef __NetBSD__ /* XXX atomic shmem fast path */ 618 return -EFAULT; 619 #else 620 char *vaddr; 621 int ret; 622 623 if (unlikely(page_do_bit17_swizzling)) 624 return -EINVAL; 625 626 vaddr = kmap_atomic(page); 627 if (needs_clflush) 628 drm_clflush_virt_range(vaddr + shmem_page_offset, 629 page_length); 630 ret = __copy_to_user_inatomic(user_data, 631 vaddr + shmem_page_offset, 632 page_length); 633 kunmap_atomic(vaddr); 634 635 return ret ? -EFAULT : 0; 636 #endif 637 } 638 639 static void 640 shmem_clflush_swizzled_range(char *addr, unsigned long length, 641 bool swizzled) 642 { 643 if (unlikely(swizzled)) { 644 unsigned long start = (unsigned long) addr; 645 unsigned long end = (unsigned long) addr + length; 646 647 /* For swizzling simply ensure that we always flush both 648 * channels. Lame, but simple and it works. Swizzled 649 * pwrite/pread is far from a hotpath - current userspace 650 * doesn't use it at all. */ 651 start = round_down(start, 128); 652 end = round_up(end, 128); 653 654 drm_clflush_virt_range((void *)start, end - start); 655 } else { 656 drm_clflush_virt_range(addr, length); 657 } 658 659 } 660 661 /* Only difference to the fast-path function is that this can handle bit17 662 * and uses non-atomic copy and kmap functions. */ 663 static int 664 shmem_pread_slow(struct page *page, int shmem_page_offset, int page_length, 665 char __user *user_data, 666 bool page_do_bit17_swizzling, bool needs_clflush) 667 { 668 char *vaddr; 669 int ret; 670 671 vaddr = kmap(page); 672 if (needs_clflush) 673 shmem_clflush_swizzled_range(vaddr + shmem_page_offset, 674 page_length, 675 page_do_bit17_swizzling); 676 677 if (page_do_bit17_swizzling) 678 ret = __copy_to_user_swizzled(user_data, 679 vaddr, shmem_page_offset, 680 page_length); 681 else 682 ret = __copy_to_user(user_data, 683 vaddr + shmem_page_offset, 684 page_length); 685 kunmap(page); 686 687 return ret ? - EFAULT : 0; 688 } 689 690 static int 691 i915_gem_shmem_pread(struct drm_device *dev, 692 struct drm_i915_gem_object *obj, 693 struct drm_i915_gem_pread *args, 694 struct drm_file *file) 695 { 696 char __user *user_data; 697 ssize_t remain; 698 loff_t offset; 699 int shmem_page_offset, page_length, ret = 0; 700 int obj_do_bit17_swizzling, page_do_bit17_swizzling; 701 #ifndef __NetBSD__ /* XXX */ 702 int prefaulted = 0; 703 #endif 704 int needs_clflush = 0; 705 #ifndef __NetBSD__ 706 struct sg_page_iter sg_iter; 707 #endif 708 709 user_data = to_user_ptr(args->data_ptr); 710 remain = args->size; 711 712 obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj); 713 714 ret = i915_gem_obj_prepare_shmem_read(obj, &needs_clflush); 715 if (ret) 716 return ret; 717 718 offset = args->offset; 719 720 #ifdef __NetBSD__ 721 while (0 < remain) 722 #else 723 for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents, 724 offset >> PAGE_SHIFT) 725 #endif 726 { 727 #ifdef __NetBSD__ 728 struct page *const page = i915_gem_object_get_page(obj, 729 atop(offset)); 730 #else 731 struct page *page = sg_page_iter_page(&sg_iter); 732 733 if (remain <= 0) 734 break; 735 #endif 736 737 /* Operation in this page 738 * 739 * shmem_page_offset = offset within page in shmem file 740 * page_length = bytes to copy for this page 741 */ 742 shmem_page_offset = offset_in_page(offset); 743 page_length = remain; 744 if ((shmem_page_offset + page_length) > PAGE_SIZE) 745 page_length = PAGE_SIZE - shmem_page_offset; 746 747 page_do_bit17_swizzling = obj_do_bit17_swizzling && 748 (page_to_phys(page) & (1 << 17)) != 0; 749 750 ret = shmem_pread_fast(page, shmem_page_offset, page_length, 751 user_data, page_do_bit17_swizzling, 752 needs_clflush); 753 if (ret == 0) 754 goto next_page; 755 756 mutex_unlock(&dev->struct_mutex); 757 #ifndef __NetBSD__ 758 if (likely(!i915.prefault_disable) && !prefaulted) { 759 ret = fault_in_multipages_writeable(user_data, remain); 760 /* Userspace is tricking us, but we've already clobbered 761 * its pages with the prefault and promised to write the 762 * data up to the first fault. Hence ignore any errors 763 * and just continue. */ 764 (void)ret; 765 prefaulted = 1; 766 } 767 #endif 768 ret = shmem_pread_slow(page, shmem_page_offset, page_length, 769 user_data, page_do_bit17_swizzling, 770 needs_clflush); 771 772 mutex_lock(&dev->struct_mutex); 773 774 if (ret) 775 goto out; 776 777 next_page: 778 remain -= page_length; 779 user_data += page_length; 780 offset += page_length; 781 } 782 783 out: 784 i915_gem_object_unpin_pages(obj); 785 786 return ret; 787 } 788 789 /** 790 * Reads data from the object referenced by handle. 791 * 792 * On error, the contents of *data are undefined. 793 */ 794 int 795 i915_gem_pread_ioctl(struct drm_device *dev, void *data, 796 struct drm_file *file) 797 { 798 struct drm_i915_gem_pread *args = data; 799 struct drm_gem_object *gobj; 800 struct drm_i915_gem_object *obj; 801 int ret = 0; 802 803 if (args->size == 0) 804 return 0; 805 806 if (!access_ok(VERIFY_WRITE, 807 to_user_ptr(args->data_ptr), 808 args->size)) 809 return -EFAULT; 810 811 ret = i915_mutex_lock_interruptible(dev); 812 if (ret) 813 return ret; 814 815 gobj = drm_gem_object_lookup(dev, file, args->handle); 816 if (gobj == NULL) { 817 ret = -ENOENT; 818 goto unlock; 819 } 820 obj = to_intel_bo(gobj); 821 822 /* Bounds check source. */ 823 if (args->offset > obj->base.size || 824 args->size > obj->base.size - args->offset) { 825 ret = -EINVAL; 826 goto out; 827 } 828 829 /* prime objects have no backing filp to GEM pread/pwrite 830 * pages from. 831 */ 832 if (!obj->base.filp) { 833 ret = -EINVAL; 834 goto out; 835 } 836 837 trace_i915_gem_object_pread(obj, args->offset, args->size); 838 839 ret = i915_gem_shmem_pread(dev, obj, args, file); 840 841 out: 842 drm_gem_object_unreference(&obj->base); 843 unlock: 844 mutex_unlock(&dev->struct_mutex); 845 return ret; 846 } 847 848 /* This is the fast write path which cannot handle 849 * page faults in the source data 850 */ 851 852 static inline int 853 fast_user_write(struct io_mapping *mapping, 854 loff_t page_base, int page_offset, 855 char __user *user_data, 856 int length) 857 { 858 #ifdef __NetBSD__ /* XXX atomic shmem fast path */ 859 return -EFAULT; 860 #else 861 void __iomem *vaddr_atomic; 862 void *vaddr; 863 unsigned long unwritten; 864 865 vaddr_atomic = io_mapping_map_atomic_wc(mapping, page_base); 866 /* We can use the cpu mem copy function because this is X86. */ 867 vaddr = (void __force*)vaddr_atomic + page_offset; 868 unwritten = __copy_from_user_inatomic_nocache(vaddr, 869 user_data, length); 870 io_mapping_unmap_atomic(vaddr_atomic); 871 return unwritten; 872 #endif 873 } 874 875 /** 876 * This is the fast pwrite path, where we copy the data directly from the 877 * user into the GTT, uncached. 878 */ 879 static int 880 i915_gem_gtt_pwrite_fast(struct drm_device *dev, 881 struct drm_i915_gem_object *obj, 882 struct drm_i915_gem_pwrite *args, 883 struct drm_file *file) 884 { 885 struct drm_i915_private *dev_priv = dev->dev_private; 886 ssize_t remain; 887 loff_t offset, page_base; 888 char __user *user_data; 889 int page_offset, page_length, ret; 890 891 ret = i915_gem_obj_ggtt_pin(obj, 0, PIN_MAPPABLE | PIN_NONBLOCK); 892 if (ret) 893 goto out; 894 895 ret = i915_gem_object_set_to_gtt_domain(obj, true); 896 if (ret) 897 goto out_unpin; 898 899 ret = i915_gem_object_put_fence(obj); 900 if (ret) 901 goto out_unpin; 902 903 user_data = to_user_ptr(args->data_ptr); 904 remain = args->size; 905 906 offset = i915_gem_obj_ggtt_offset(obj) + args->offset; 907 908 intel_fb_obj_invalidate(obj, ORIGIN_GTT); 909 910 while (remain > 0) { 911 /* Operation in this page 912 * 913 * page_base = page offset within aperture 914 * page_offset = offset within page 915 * page_length = bytes to copy for this page 916 */ 917 page_base = offset & PAGE_MASK; 918 page_offset = offset_in_page(offset); 919 page_length = remain; 920 if ((page_offset + remain) > PAGE_SIZE) 921 page_length = PAGE_SIZE - page_offset; 922 923 /* If we get a fault while copying data, then (presumably) our 924 * source page isn't available. Return the error and we'll 925 * retry in the slow path. 926 */ 927 if (fast_user_write(dev_priv->gtt.mappable, page_base, 928 page_offset, user_data, page_length)) { 929 ret = -EFAULT; 930 goto out_flush; 931 } 932 933 remain -= page_length; 934 user_data += page_length; 935 offset += page_length; 936 } 937 938 out_flush: 939 intel_fb_obj_flush(obj, false, ORIGIN_GTT); 940 out_unpin: 941 i915_gem_object_ggtt_unpin(obj); 942 out: 943 return ret; 944 } 945 946 /* Per-page copy function for the shmem pwrite fastpath. 947 * Flushes invalid cachelines before writing to the target if 948 * needs_clflush_before is set and flushes out any written cachelines after 949 * writing if needs_clflush is set. */ 950 static int 951 shmem_pwrite_fast(struct page *page, int shmem_page_offset, int page_length, 952 char __user *user_data, 953 bool page_do_bit17_swizzling, 954 bool needs_clflush_before, 955 bool needs_clflush_after) 956 { 957 #ifdef __NetBSD__ 958 return -EFAULT; 959 #else 960 char *vaddr; 961 int ret; 962 963 if (unlikely(page_do_bit17_swizzling)) 964 return -EINVAL; 965 966 vaddr = kmap_atomic(page); 967 if (needs_clflush_before) 968 drm_clflush_virt_range(vaddr + shmem_page_offset, 969 page_length); 970 ret = __copy_from_user_inatomic(vaddr + shmem_page_offset, 971 user_data, page_length); 972 if (needs_clflush_after) 973 drm_clflush_virt_range(vaddr + shmem_page_offset, 974 page_length); 975 kunmap_atomic(vaddr); 976 977 return ret ? -EFAULT : 0; 978 #endif 979 } 980 981 /* Only difference to the fast-path function is that this can handle bit17 982 * and uses non-atomic copy and kmap functions. */ 983 static int 984 shmem_pwrite_slow(struct page *page, int shmem_page_offset, int page_length, 985 char __user *user_data, 986 bool page_do_bit17_swizzling, 987 bool needs_clflush_before, 988 bool needs_clflush_after) 989 { 990 char *vaddr; 991 int ret; 992 993 vaddr = kmap(page); 994 if (unlikely(needs_clflush_before || page_do_bit17_swizzling)) 995 shmem_clflush_swizzled_range(vaddr + shmem_page_offset, 996 page_length, 997 page_do_bit17_swizzling); 998 if (page_do_bit17_swizzling) 999 ret = __copy_from_user_swizzled(vaddr, shmem_page_offset, 1000 user_data, 1001 page_length); 1002 else 1003 ret = __copy_from_user(vaddr + shmem_page_offset, 1004 user_data, 1005 page_length); 1006 if (needs_clflush_after) 1007 shmem_clflush_swizzled_range(vaddr + shmem_page_offset, 1008 page_length, 1009 page_do_bit17_swizzling); 1010 kunmap(page); 1011 1012 return ret ? -EFAULT : 0; 1013 } 1014 1015 static int 1016 i915_gem_shmem_pwrite(struct drm_device *dev, 1017 struct drm_i915_gem_object *obj, 1018 struct drm_i915_gem_pwrite *args, 1019 struct drm_file *file) 1020 { 1021 ssize_t remain; 1022 loff_t offset; 1023 char __user *user_data; 1024 int shmem_page_offset, page_length, ret = 0; 1025 int obj_do_bit17_swizzling, page_do_bit17_swizzling; 1026 int hit_slowpath = 0; 1027 int needs_clflush_after = 0; 1028 int needs_clflush_before = 0; 1029 #ifndef __NetBSD__ 1030 struct sg_page_iter sg_iter; 1031 int flush_mask = boot_cpu_data.x86_clflush_size - 1; 1032 #else 1033 int flush_mask = cpu_info_primary.ci_cflush_lsize - 1; 1034 #endif 1035 1036 user_data = to_user_ptr(args->data_ptr); 1037 remain = args->size; 1038 1039 obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj); 1040 1041 if (obj->base.write_domain != I915_GEM_DOMAIN_CPU) { 1042 /* If we're not in the cpu write domain, set ourself into the gtt 1043 * write domain and manually flush cachelines (if required). This 1044 * optimizes for the case when the gpu will use the data 1045 * right away and we therefore have to clflush anyway. */ 1046 needs_clflush_after = cpu_write_needs_clflush(obj); 1047 ret = i915_gem_object_wait_rendering(obj, false); 1048 if (ret) 1049 return ret; 1050 } 1051 /* Same trick applies to invalidate partially written cachelines read 1052 * before writing. */ 1053 if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0) 1054 needs_clflush_before = 1055 !cpu_cache_is_coherent(dev, obj->cache_level); 1056 1057 ret = i915_gem_object_get_pages(obj); 1058 if (ret) 1059 return ret; 1060 1061 intel_fb_obj_invalidate(obj, ORIGIN_CPU); 1062 1063 i915_gem_object_pin_pages(obj); 1064 1065 offset = args->offset; 1066 obj->dirty = 1; 1067 1068 #ifdef __NetBSD__ 1069 while (0 < remain) 1070 #else 1071 for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents, 1072 offset >> PAGE_SHIFT) 1073 #endif 1074 { 1075 #ifdef __NetBSD__ 1076 struct page *const page = i915_gem_object_get_page(obj, 1077 atop(offset)); 1078 #else 1079 struct page *page = sg_page_iter_page(&sg_iter); 1080 1081 if (remain <= 0) 1082 break; 1083 #endif 1084 1085 /* Operation in this page 1086 * 1087 * shmem_page_offset = offset within page in shmem file 1088 * page_length = bytes to copy for this page 1089 */ 1090 shmem_page_offset = offset_in_page(offset); 1091 1092 page_length = remain; 1093 if ((shmem_page_offset + page_length) > PAGE_SIZE) 1094 page_length = PAGE_SIZE - shmem_page_offset; 1095 1096 /* If we don't overwrite a cacheline completely we need to be 1097 * careful to have up-to-date data by first clflushing. Don't 1098 * overcomplicate things and flush the entire patch. */ 1099 const int partial_cacheline_write = needs_clflush_before && 1100 ((shmem_page_offset | page_length) & flush_mask); 1101 1102 page_do_bit17_swizzling = obj_do_bit17_swizzling && 1103 (page_to_phys(page) & (1 << 17)) != 0; 1104 1105 ret = shmem_pwrite_fast(page, shmem_page_offset, page_length, 1106 user_data, page_do_bit17_swizzling, 1107 partial_cacheline_write, 1108 needs_clflush_after); 1109 if (ret == 0) 1110 goto next_page; 1111 1112 hit_slowpath = 1; 1113 mutex_unlock(&dev->struct_mutex); 1114 ret = shmem_pwrite_slow(page, shmem_page_offset, page_length, 1115 user_data, page_do_bit17_swizzling, 1116 partial_cacheline_write, 1117 needs_clflush_after); 1118 1119 mutex_lock(&dev->struct_mutex); 1120 1121 if (ret) 1122 goto out; 1123 1124 next_page: 1125 remain -= page_length; 1126 user_data += page_length; 1127 offset += page_length; 1128 } 1129 1130 out: 1131 i915_gem_object_unpin_pages(obj); 1132 1133 if (hit_slowpath) { 1134 /* 1135 * Fixup: Flush cpu caches in case we didn't flush the dirty 1136 * cachelines in-line while writing and the object moved 1137 * out of the cpu write domain while we've dropped the lock. 1138 */ 1139 if (!needs_clflush_after && 1140 obj->base.write_domain != I915_GEM_DOMAIN_CPU) { 1141 if (i915_gem_clflush_object(obj, obj->pin_display)) 1142 needs_clflush_after = true; 1143 } 1144 } 1145 1146 if (needs_clflush_after) 1147 i915_gem_chipset_flush(dev); 1148 else 1149 obj->cache_dirty = true; 1150 1151 intel_fb_obj_flush(obj, false, ORIGIN_CPU); 1152 return ret; 1153 } 1154 1155 /** 1156 * Writes data to the object referenced by handle. 1157 * 1158 * On error, the contents of the buffer that were to be modified are undefined. 1159 */ 1160 int 1161 i915_gem_pwrite_ioctl(struct drm_device *dev, void *data, 1162 struct drm_file *file) 1163 { 1164 struct drm_i915_private *dev_priv = dev->dev_private; 1165 struct drm_i915_gem_pwrite *args = data; 1166 struct drm_gem_object *gobj; 1167 struct drm_i915_gem_object *obj; 1168 int ret; 1169 1170 if (args->size == 0) 1171 return 0; 1172 1173 if (!access_ok(VERIFY_READ, 1174 to_user_ptr(args->data_ptr), 1175 args->size)) 1176 return -EFAULT; 1177 1178 #ifndef __NetBSD__ /* XXX prefault */ 1179 if (likely(!i915.prefault_disable)) { 1180 ret = fault_in_multipages_readable(to_user_ptr(args->data_ptr), 1181 args->size); 1182 if (ret) 1183 return -EFAULT; 1184 } 1185 #endif 1186 1187 intel_runtime_pm_get(dev_priv); 1188 1189 ret = i915_mutex_lock_interruptible(dev); 1190 if (ret) 1191 goto put_rpm; 1192 1193 gobj = drm_gem_object_lookup(dev, file, args->handle); 1194 if (gobj == NULL) { 1195 ret = -ENOENT; 1196 goto unlock; 1197 } 1198 obj = to_intel_bo(gobj); 1199 1200 /* Bounds check destination. */ 1201 if (args->offset > obj->base.size || 1202 args->size > obj->base.size - args->offset) { 1203 ret = -EINVAL; 1204 goto out; 1205 } 1206 1207 /* prime objects have no backing filp to GEM pread/pwrite 1208 * pages from. 1209 */ 1210 if (!obj->base.filp) { 1211 ret = -EINVAL; 1212 goto out; 1213 } 1214 1215 trace_i915_gem_object_pwrite(obj, args->offset, args->size); 1216 1217 ret = -EFAULT; 1218 /* We can only do the GTT pwrite on untiled buffers, as otherwise 1219 * it would end up going through the fenced access, and we'll get 1220 * different detiling behavior between reading and writing. 1221 * pread/pwrite currently are reading and writing from the CPU 1222 * perspective, requiring manual detiling by the client. 1223 */ 1224 if (obj->tiling_mode == I915_TILING_NONE && 1225 obj->base.write_domain != I915_GEM_DOMAIN_CPU && 1226 cpu_write_needs_clflush(obj)) { 1227 ret = i915_gem_gtt_pwrite_fast(dev, obj, args, file); 1228 /* Note that the gtt paths might fail with non-page-backed user 1229 * pointers (e.g. gtt mappings when moving data between 1230 * textures). Fallback to the shmem path in that case. */ 1231 } 1232 1233 if (ret == -EFAULT || ret == -ENOSPC) { 1234 if (obj->phys_handle) 1235 ret = i915_gem_phys_pwrite(obj, args, file); 1236 else 1237 ret = i915_gem_shmem_pwrite(dev, obj, args, file); 1238 } 1239 1240 out: 1241 drm_gem_object_unreference(&obj->base); 1242 unlock: 1243 mutex_unlock(&dev->struct_mutex); 1244 put_rpm: 1245 intel_runtime_pm_put(dev_priv); 1246 1247 return ret; 1248 } 1249 1250 int 1251 i915_gem_check_wedge(struct i915_gpu_error *error, 1252 bool interruptible) 1253 { 1254 if (i915_reset_in_progress(error)) { 1255 /* Non-interruptible callers can't handle -EAGAIN, hence return 1256 * -EIO unconditionally for these. */ 1257 if (!interruptible) 1258 return -EIO; 1259 1260 /* Recovery complete, but the reset failed ... */ 1261 if (i915_terminally_wedged(error)) 1262 return -EIO; 1263 1264 /* 1265 * Check if GPU Reset is in progress - we need intel_ring_begin 1266 * to work properly to reinit the hw state while the gpu is 1267 * still marked as reset-in-progress. Handle this with a flag. 1268 */ 1269 if (!error->reload_in_reset) 1270 return -EAGAIN; 1271 } 1272 1273 return 0; 1274 } 1275 1276 #ifndef __NetBSD__ 1277 static void fake_irq(unsigned long data) 1278 { 1279 wake_up_process((struct task_struct *)data); 1280 } 1281 #endif 1282 1283 static bool missed_irq(struct drm_i915_private *dev_priv, 1284 struct intel_engine_cs *ring) 1285 { 1286 return test_bit(ring->id, &dev_priv->gpu_error.missed_irq_rings); 1287 } 1288 1289 #ifndef __NetBSD__ 1290 static unsigned long local_clock_us(unsigned *cpu) 1291 { 1292 unsigned long t; 1293 1294 /* Cheaply and approximately convert from nanoseconds to microseconds. 1295 * The result and subsequent calculations are also defined in the same 1296 * approximate microseconds units. The principal source of timing 1297 * error here is from the simple truncation. 1298 * 1299 * Note that local_clock() is only defined wrt to the current CPU; 1300 * the comparisons are no longer valid if we switch CPUs. Instead of 1301 * blocking preemption for the entire busywait, we can detect the CPU 1302 * switch and use that as indicator of system load and a reason to 1303 * stop busywaiting, see busywait_stop(). 1304 */ 1305 *cpu = get_cpu(); 1306 t = local_clock() >> 10; 1307 put_cpu(); 1308 1309 return t; 1310 } 1311 1312 static bool busywait_stop(unsigned long timeout, unsigned cpu) 1313 { 1314 unsigned this_cpu; 1315 1316 if (time_after(local_clock_us(&this_cpu), timeout)) 1317 return true; 1318 1319 return this_cpu != cpu; 1320 } 1321 #endif 1322 1323 static int __i915_spin_request(struct drm_i915_gem_request *req, int state) 1324 { 1325 #ifndef __NetBSD__ 1326 unsigned long timeout; 1327 unsigned cpu; 1328 #endif 1329 1330 /* When waiting for high frequency requests, e.g. during synchronous 1331 * rendering split between the CPU and GPU, the finite amount of time 1332 * required to set up the irq and wait upon it limits the response 1333 * rate. By busywaiting on the request completion for a short while we 1334 * can service the high frequency waits as quick as possible. However, 1335 * if it is a slow request, we want to sleep as quickly as possible. 1336 * The tradeoff between waiting and sleeping is roughly the time it 1337 * takes to sleep on a request, on the order of a microsecond. 1338 */ 1339 1340 if (req->ring->irq_refcount) 1341 return -EBUSY; 1342 1343 /* Only spin if we know the GPU is processing this request */ 1344 if (!i915_gem_request_started(req, true)) 1345 return -EAGAIN; 1346 1347 #ifndef __NetBSD__ /* XXX No local clock in usec. */ 1348 timeout = local_clock_us(&cpu) + 5; 1349 while (!need_resched()) { 1350 if (i915_gem_request_completed(req, true)) 1351 return 0; 1352 1353 if (signal_pending_state(state, current)) 1354 break; 1355 1356 if (busywait_stop(timeout, cpu)) 1357 break; 1358 1359 cpu_relax_lowlatency(); 1360 } 1361 #endif 1362 1363 if (i915_gem_request_completed(req, false)) 1364 return 0; 1365 1366 return -EAGAIN; 1367 } 1368 1369 /** 1370 * __i915_wait_request - wait until execution of request has finished 1371 * @req: duh! 1372 * @reset_counter: reset sequence associated with the given request 1373 * @interruptible: do an interruptible wait (normally yes) 1374 * @timeout: in - how long to wait (NULL forever); out - how much time remaining 1375 * 1376 * Note: It is of utmost importance that the passed in seqno and reset_counter 1377 * values have been read by the caller in an smp safe manner. Where read-side 1378 * locks are involved, it is sufficient to read the reset_counter before 1379 * unlocking the lock that protects the seqno. For lockless tricks, the 1380 * reset_counter _must_ be read before, and an appropriate smp_rmb must be 1381 * inserted. 1382 * 1383 * Returns 0 if the request was found within the alloted time. Else returns the 1384 * errno with remaining time filled in timeout argument. 1385 */ 1386 int __i915_wait_request(struct drm_i915_gem_request *req, 1387 unsigned reset_counter, 1388 bool interruptible, 1389 s64 *timeout, 1390 struct intel_rps_client *rps) 1391 { 1392 struct intel_engine_cs *ring = i915_gem_request_get_ring(req); 1393 struct drm_device *dev = ring->dev; 1394 struct drm_i915_private *dev_priv = dev->dev_private; 1395 const bool irq_test_in_progress = 1396 ACCESS_ONCE(dev_priv->gpu_error.test_irq_rings) & intel_ring_flag(ring); 1397 #ifdef __NetBSD__ 1398 int state = 0; 1399 bool wedged; 1400 #else 1401 int state = interruptible ? TASK_INTERRUPTIBLE : TASK_UNINTERRUPTIBLE; 1402 DEFINE_WAIT(wait); 1403 unsigned long timeout_expire; 1404 #endif 1405 s64 before, now; 1406 int ret; 1407 1408 WARN(!intel_irqs_enabled(dev_priv), "IRQs disabled"); 1409 1410 if (list_empty(&req->list)) 1411 return 0; 1412 1413 if (i915_gem_request_completed(req, true)) 1414 return 0; 1415 1416 #ifndef __NetBSD__ 1417 timeout_expire = 0; 1418 if (timeout) { 1419 if (WARN_ON(*timeout < 0)) 1420 return -EINVAL; 1421 1422 if (*timeout == 0) 1423 return -ETIME; 1424 1425 timeout_expire = jiffies + nsecs_to_jiffies_timeout(*timeout); 1426 } 1427 #endif 1428 1429 if (INTEL_INFO(dev_priv)->gen >= 6) 1430 gen6_rps_boost(dev_priv, rps, req->emitted_jiffies); 1431 1432 /* Record current time in case interrupted by signal, or wedged */ 1433 trace_i915_gem_request_wait_begin(req); 1434 before = ktime_get_raw_ns(); 1435 1436 /* Optimistic spin for the next jiffie before touching IRQs */ 1437 ret = __i915_spin_request(req, state); 1438 if (ret == 0) 1439 goto out; 1440 1441 if (!irq_test_in_progress && WARN_ON(!ring->irq_get(ring))) { 1442 ret = -ENODEV; 1443 goto out; 1444 } 1445 1446 #ifdef __NetBSD__ 1447 # define EXIT_COND \ 1448 ((wedged = (reset_counter != \ 1449 atomic_read(&dev_priv->gpu_error.reset_counter))) || \ 1450 i915_gem_request_completed(req, false)) 1451 spin_lock(&dev_priv->irq_lock); 1452 if (timeout) { 1453 int ticks = missed_irq(dev_priv, ring) ? 1 : 1454 nsecs_to_jiffies_timeout(*timeout); 1455 if (interruptible) { 1456 DRM_SPIN_TIMED_WAIT_UNTIL(ret, &ring->irq_queue, 1457 &dev_priv->irq_lock, ticks, EXIT_COND); 1458 } else { 1459 DRM_SPIN_TIMED_WAIT_NOINTR_UNTIL(ret, &ring->irq_queue, 1460 &dev_priv->irq_lock, ticks, EXIT_COND); 1461 } 1462 if (ret < 0) /* Failure: return negative error as is. */ 1463 ; 1464 else if (ret == 0) /* Timed out: return -ETIME. */ 1465 ret = -ETIME; 1466 else /* Succeeded (ret > 0): return 0. */ 1467 ret = 0; 1468 } else { 1469 if (interruptible) { 1470 DRM_SPIN_WAIT_UNTIL(ret, &ring->irq_queue, 1471 &dev_priv->irq_lock, EXIT_COND); 1472 } else { 1473 DRM_SPIN_WAIT_NOINTR_UNTIL(ret, &ring->irq_queue, 1474 &dev_priv->irq_lock, EXIT_COND); 1475 } 1476 /* ret is negative on failure or zero on success. */ 1477 } 1478 spin_unlock(&dev_priv->irq_lock); 1479 if (wedged) { 1480 ret = i915_gem_check_wedge(&dev_priv->gpu_error, interruptible); 1481 if (ret == 0) 1482 ret = -EAGAIN; 1483 } 1484 #else 1485 for (;;) { 1486 struct timer_list timer; 1487 1488 prepare_to_wait(&ring->irq_queue, &wait, state); 1489 1490 /* We need to check whether any gpu reset happened in between 1491 * the caller grabbing the seqno and now ... */ 1492 if (reset_counter != atomic_read(&dev_priv->gpu_error.reset_counter)) { 1493 /* ... but upgrade the -EAGAIN to an -EIO if the gpu 1494 * is truely gone. */ 1495 ret = i915_gem_check_wedge(&dev_priv->gpu_error, interruptible); 1496 if (ret == 0) 1497 ret = -EAGAIN; 1498 break; 1499 } 1500 1501 if (i915_gem_request_completed(req, false)) { 1502 ret = 0; 1503 break; 1504 } 1505 1506 if (signal_pending_state(state, current)) { 1507 ret = -ERESTARTSYS; 1508 break; 1509 } 1510 1511 if (timeout && time_after_eq(jiffies, timeout_expire)) { 1512 ret = -ETIME; 1513 break; 1514 } 1515 1516 timer.function = NULL; 1517 if (timeout || missed_irq(dev_priv, ring)) { 1518 unsigned long expire; 1519 1520 setup_timer_on_stack(&timer, fake_irq, (unsigned long)current); 1521 expire = missed_irq(dev_priv, ring) ? jiffies + 1 : timeout_expire; 1522 mod_timer(&timer, expire); 1523 } 1524 1525 io_schedule(); 1526 1527 if (timer.function) { 1528 del_singleshot_timer_sync(&timer); 1529 destroy_timer_on_stack(&timer); 1530 } 1531 } 1532 #endif 1533 if (!irq_test_in_progress) 1534 ring->irq_put(ring); 1535 1536 #ifndef __NetBSD__ 1537 finish_wait(&ring->irq_queue, &wait); 1538 #endif 1539 1540 out: 1541 now = ktime_get_raw_ns(); 1542 trace_i915_gem_request_wait_end(req); 1543 1544 if (timeout) { 1545 s64 tres = *timeout - (now - before); 1546 1547 *timeout = tres < 0 ? 0 : tres; 1548 1549 /* 1550 * Apparently ktime isn't accurate enough and occasionally has a 1551 * bit of mismatch in the jiffies<->nsecs<->ktime loop. So patch 1552 * things up to make the test happy. We allow up to 1 jiffy. 1553 * 1554 * This is a regrssion from the timespec->ktime conversion. 1555 */ 1556 if (ret == -ETIME && *timeout < jiffies_to_usecs(1)*1000) 1557 *timeout = 0; 1558 } 1559 1560 return ret; 1561 } 1562 1563 int i915_gem_request_add_to_client(struct drm_i915_gem_request *req, 1564 struct drm_file *file) 1565 { 1566 struct drm_i915_private *dev_private __unused; 1567 struct drm_i915_file_private *file_priv; 1568 1569 WARN_ON(!req || !file || req->file_priv); 1570 1571 if (!req || !file) 1572 return -EINVAL; 1573 1574 if (req->file_priv) 1575 return -EINVAL; 1576 1577 dev_private = req->ring->dev->dev_private; 1578 file_priv = file->driver_priv; 1579 1580 spin_lock(&file_priv->mm.lock); 1581 req->file_priv = file_priv; 1582 list_add_tail(&req->client_list, &file_priv->mm.request_list); 1583 spin_unlock(&file_priv->mm.lock); 1584 1585 #ifndef __NetBSD__ 1586 req->pid = get_pid(task_pid(current)); 1587 #endif 1588 1589 return 0; 1590 } 1591 1592 static inline void 1593 i915_gem_request_remove_from_client(struct drm_i915_gem_request *request) 1594 { 1595 struct drm_i915_file_private *file_priv = request->file_priv; 1596 1597 if (!file_priv) 1598 return; 1599 1600 spin_lock(&file_priv->mm.lock); 1601 list_del(&request->client_list); 1602 request->file_priv = NULL; 1603 spin_unlock(&file_priv->mm.lock); 1604 1605 #ifndef __NetBSD__ 1606 put_pid(request->pid); 1607 request->pid = NULL; 1608 #endif 1609 } 1610 1611 static void i915_gem_request_retire(struct drm_i915_gem_request *request) 1612 { 1613 trace_i915_gem_request_retire(request); 1614 1615 /* We know the GPU must have read the request to have 1616 * sent us the seqno + interrupt, so use the position 1617 * of tail of the request to update the last known position 1618 * of the GPU head. 1619 * 1620 * Note this requires that we are always called in request 1621 * completion order. 1622 */ 1623 request->ringbuf->last_retired_head = request->postfix; 1624 1625 list_del_init(&request->list); 1626 i915_gem_request_remove_from_client(request); 1627 1628 i915_gem_request_unreference(request); 1629 } 1630 1631 static void 1632 __i915_gem_request_retire__upto(struct drm_i915_gem_request *req) 1633 { 1634 struct intel_engine_cs *engine = req->ring; 1635 struct drm_i915_gem_request *tmp; 1636 1637 lockdep_assert_held(&engine->dev->struct_mutex); 1638 1639 if (list_empty(&req->list)) 1640 return; 1641 1642 do { 1643 tmp = list_first_entry(&engine->request_list, 1644 typeof(*tmp), list); 1645 1646 i915_gem_request_retire(tmp); 1647 } while (tmp != req); 1648 1649 WARN_ON(i915_verify_lists(engine->dev)); 1650 } 1651 1652 /** 1653 * Waits for a request to be signaled, and cleans up the 1654 * request and object lists appropriately for that event. 1655 */ 1656 int 1657 i915_wait_request(struct drm_i915_gem_request *req) 1658 { 1659 struct drm_device *dev; 1660 struct drm_i915_private *dev_priv; 1661 bool interruptible; 1662 int ret; 1663 1664 BUG_ON(req == NULL); 1665 1666 dev = req->ring->dev; 1667 dev_priv = dev->dev_private; 1668 interruptible = dev_priv->mm.interruptible; 1669 1670 BUG_ON(!mutex_is_locked(&dev->struct_mutex)); 1671 1672 ret = i915_gem_check_wedge(&dev_priv->gpu_error, interruptible); 1673 if (ret) 1674 return ret; 1675 1676 ret = __i915_wait_request(req, 1677 atomic_read(&dev_priv->gpu_error.reset_counter), 1678 interruptible, NULL, NULL); 1679 if (ret) 1680 return ret; 1681 1682 __i915_gem_request_retire__upto(req); 1683 return 0; 1684 } 1685 1686 /** 1687 * Ensures that all rendering to the object has completed and the object is 1688 * safe to unbind from the GTT or access from the CPU. 1689 */ 1690 int 1691 i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj, 1692 bool readonly) 1693 { 1694 int ret, i; 1695 1696 if (!obj->active) 1697 return 0; 1698 1699 if (readonly) { 1700 if (obj->last_write_req != NULL) { 1701 ret = i915_wait_request(obj->last_write_req); 1702 if (ret) 1703 return ret; 1704 1705 i = obj->last_write_req->ring->id; 1706 if (obj->last_read_req[i] == obj->last_write_req) 1707 i915_gem_object_retire__read(obj, i); 1708 else 1709 i915_gem_object_retire__write(obj); 1710 } 1711 } else { 1712 for (i = 0; i < I915_NUM_RINGS; i++) { 1713 if (obj->last_read_req[i] == NULL) 1714 continue; 1715 1716 ret = i915_wait_request(obj->last_read_req[i]); 1717 if (ret) 1718 return ret; 1719 1720 i915_gem_object_retire__read(obj, i); 1721 } 1722 RQ_BUG_ON(obj->active); 1723 } 1724 1725 return 0; 1726 } 1727 1728 static void 1729 i915_gem_object_retire_request(struct drm_i915_gem_object *obj, 1730 struct drm_i915_gem_request *req) 1731 { 1732 int ring = req->ring->id; 1733 1734 if (obj->last_read_req[ring] == req) 1735 i915_gem_object_retire__read(obj, ring); 1736 else if (obj->last_write_req == req) 1737 i915_gem_object_retire__write(obj); 1738 1739 __i915_gem_request_retire__upto(req); 1740 } 1741 1742 /* A nonblocking variant of the above wait. This is a highly dangerous routine 1743 * as the object state may change during this call. 1744 */ 1745 static __must_check int 1746 i915_gem_object_wait_rendering__nonblocking(struct drm_i915_gem_object *obj, 1747 struct intel_rps_client *rps, 1748 bool readonly) 1749 { 1750 struct drm_device *dev = obj->base.dev; 1751 struct drm_i915_private *dev_priv = dev->dev_private; 1752 struct drm_i915_gem_request *requests[I915_NUM_RINGS]; 1753 unsigned reset_counter; 1754 int ret, i, n = 0; 1755 1756 BUG_ON(!mutex_is_locked(&dev->struct_mutex)); 1757 BUG_ON(!dev_priv->mm.interruptible); 1758 1759 if (!obj->active) 1760 return 0; 1761 1762 ret = i915_gem_check_wedge(&dev_priv->gpu_error, true); 1763 if (ret) 1764 return ret; 1765 1766 reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter); 1767 1768 if (readonly) { 1769 struct drm_i915_gem_request *req; 1770 1771 req = obj->last_write_req; 1772 if (req == NULL) 1773 return 0; 1774 1775 requests[n++] = i915_gem_request_reference(req); 1776 } else { 1777 for (i = 0; i < I915_NUM_RINGS; i++) { 1778 struct drm_i915_gem_request *req; 1779 1780 req = obj->last_read_req[i]; 1781 if (req == NULL) 1782 continue; 1783 1784 requests[n++] = i915_gem_request_reference(req); 1785 } 1786 } 1787 1788 mutex_unlock(&dev->struct_mutex); 1789 for (i = 0; ret == 0 && i < n; i++) 1790 ret = __i915_wait_request(requests[i], reset_counter, true, 1791 NULL, rps); 1792 mutex_lock(&dev->struct_mutex); 1793 1794 for (i = 0; i < n; i++) { 1795 if (ret == 0) 1796 i915_gem_object_retire_request(obj, requests[i]); 1797 i915_gem_request_unreference(requests[i]); 1798 } 1799 1800 return ret; 1801 } 1802 1803 static struct intel_rps_client *to_rps_client(struct drm_file *file) 1804 { 1805 struct drm_i915_file_private *fpriv = file->driver_priv; 1806 return &fpriv->rps; 1807 } 1808 1809 /** 1810 * Called when user space prepares to use an object with the CPU, either 1811 * through the mmap ioctl's mapping or a GTT mapping. 1812 */ 1813 int 1814 i915_gem_set_domain_ioctl(struct drm_device *dev, void *data, 1815 struct drm_file *file) 1816 { 1817 struct drm_i915_gem_set_domain *args = data; 1818 struct drm_gem_object *gobj; 1819 struct drm_i915_gem_object *obj; 1820 uint32_t read_domains = args->read_domains; 1821 uint32_t write_domain = args->write_domain; 1822 int ret; 1823 1824 /* Only handle setting domains to types used by the CPU. */ 1825 if (write_domain & I915_GEM_GPU_DOMAINS) 1826 return -EINVAL; 1827 1828 if (read_domains & I915_GEM_GPU_DOMAINS) 1829 return -EINVAL; 1830 1831 /* Having something in the write domain implies it's in the read 1832 * domain, and only that read domain. Enforce that in the request. 1833 */ 1834 if (write_domain != 0 && read_domains != write_domain) 1835 return -EINVAL; 1836 1837 ret = i915_mutex_lock_interruptible(dev); 1838 if (ret) 1839 return ret; 1840 1841 gobj = drm_gem_object_lookup(dev, file, args->handle); 1842 if (gobj == NULL) { 1843 ret = -ENOENT; 1844 goto unlock; 1845 } 1846 obj = to_intel_bo(gobj); 1847 1848 /* Try to flush the object off the GPU without holding the lock. 1849 * We will repeat the flush holding the lock in the normal manner 1850 * to catch cases where we are gazumped. 1851 */ 1852 ret = i915_gem_object_wait_rendering__nonblocking(obj, 1853 to_rps_client(file), 1854 !write_domain); 1855 if (ret) 1856 goto unref; 1857 1858 if (read_domains & I915_GEM_DOMAIN_GTT) 1859 ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0); 1860 else 1861 ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0); 1862 1863 if (write_domain != 0) 1864 intel_fb_obj_invalidate(obj, 1865 write_domain == I915_GEM_DOMAIN_GTT ? 1866 ORIGIN_GTT : ORIGIN_CPU); 1867 1868 unref: 1869 drm_gem_object_unreference(&obj->base); 1870 unlock: 1871 mutex_unlock(&dev->struct_mutex); 1872 return ret; 1873 } 1874 1875 /** 1876 * Called when user space has done writes to this buffer 1877 */ 1878 int 1879 i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data, 1880 struct drm_file *file) 1881 { 1882 struct drm_i915_gem_sw_finish *args = data; 1883 struct drm_gem_object *gobj; 1884 struct drm_i915_gem_object *obj; 1885 int ret = 0; 1886 1887 ret = i915_mutex_lock_interruptible(dev); 1888 if (ret) 1889 return ret; 1890 1891 gobj = drm_gem_object_lookup(dev, file, args->handle); 1892 if (gobj == NULL) { 1893 ret = -ENOENT; 1894 goto unlock; 1895 } 1896 obj = to_intel_bo(gobj); 1897 1898 /* Pinned buffers may be scanout, so flush the cache */ 1899 if (obj->pin_display) 1900 i915_gem_object_flush_cpu_write_domain(obj); 1901 1902 drm_gem_object_unreference(&obj->base); 1903 unlock: 1904 mutex_unlock(&dev->struct_mutex); 1905 return ret; 1906 } 1907 1908 /** 1909 * Maps the contents of an object, returning the address it is mapped 1910 * into. 1911 * 1912 * While the mapping holds a reference on the contents of the object, it doesn't 1913 * imply a ref on the object itself. 1914 * 1915 * IMPORTANT: 1916 * 1917 * DRM driver writers who look a this function as an example for how to do GEM 1918 * mmap support, please don't implement mmap support like here. The modern way 1919 * to implement DRM mmap support is with an mmap offset ioctl (like 1920 * i915_gem_mmap_gtt) and then using the mmap syscall on the DRM fd directly. 1921 * That way debug tooling like valgrind will understand what's going on, hiding 1922 * the mmap call in a driver private ioctl will break that. The i915 driver only 1923 * does cpu mmaps this way because we didn't know better. 1924 */ 1925 int 1926 i915_gem_mmap_ioctl(struct drm_device *dev, void *data, 1927 struct drm_file *file) 1928 { 1929 struct drm_i915_gem_mmap *args = data; 1930 struct drm_gem_object *obj; 1931 unsigned long addr; 1932 #ifdef __NetBSD__ 1933 struct drm_i915_private *dev_priv = dev->dev_private; 1934 int ret; 1935 1936 if ((dev_priv->quirks & QUIRK_NETBSD_VERSION_CALLED) == 0) 1937 args->flags = 0; 1938 #endif 1939 1940 if (args->flags & ~(I915_MMAP_WC)) 1941 return -EINVAL; 1942 1943 if (args->flags & I915_MMAP_WC && !cpu_has_pat) 1944 return -ENODEV; 1945 1946 obj = drm_gem_object_lookup(dev, file, args->handle); 1947 if (obj == NULL) 1948 return -ENOENT; 1949 1950 /* prime objects have no backing filp to GEM mmap 1951 * pages from. 1952 */ 1953 if (!obj->filp) { 1954 drm_gem_object_unreference_unlocked(obj); 1955 return -EINVAL; 1956 } 1957 1958 #ifdef __NetBSD__ 1959 /* Acquire a reference for uvm_map to consume. */ 1960 uao_reference(obj->filp); 1961 addr = (*curproc->p_emul->e_vm_default_addr)(curproc, 1962 (vaddr_t)curproc->p_vmspace->vm_daddr, args->size, 1963 curproc->p_vmspace->vm_map.flags & VM_MAP_TOPDOWN); 1964 /* XXX errno NetBSD->Linux */ 1965 ret = -uvm_map(&curproc->p_vmspace->vm_map, &addr, args->size, 1966 obj->filp, args->offset, 0, 1967 UVM_MAPFLAG((VM_PROT_READ | VM_PROT_WRITE), 1968 (VM_PROT_READ | VM_PROT_WRITE), UVM_INH_COPY, UVM_ADV_NORMAL, 1969 0)); 1970 if (ret) { 1971 uao_detach(obj->filp); 1972 drm_gem_object_unreference_unlocked(obj); 1973 return ret; 1974 } 1975 drm_gem_object_unreference_unlocked(obj); 1976 #else 1977 addr = vm_mmap(obj->filp, 0, args->size, 1978 PROT_READ | PROT_WRITE, MAP_SHARED, 1979 args->offset); 1980 if (args->flags & I915_MMAP_WC) { 1981 struct mm_struct *mm = current->mm; 1982 struct vm_area_struct *vma; 1983 1984 down_write(&mm->mmap_sem); 1985 vma = find_vma(mm, addr); 1986 if (vma) 1987 vma->vm_page_prot = 1988 pgprot_writecombine(vm_get_page_prot(vma->vm_flags)); 1989 else 1990 addr = -ENOMEM; 1991 up_write(&mm->mmap_sem); 1992 } 1993 drm_gem_object_unreference_unlocked(obj); 1994 if (IS_ERR((void *)addr)) 1995 return addr; 1996 #endif 1997 1998 args->addr_ptr = (uint64_t) addr; 1999 2000 return 0; 2001 } 2002 2003 #ifdef __NetBSD__ /* XXX gem gtt fault */ 2004 static int i915_udv_fault(struct uvm_faultinfo *, vaddr_t, 2005 struct vm_page **, int, int, vm_prot_t, int, paddr_t); 2006 2007 int 2008 i915_gem_fault(struct uvm_faultinfo *ufi, vaddr_t vaddr, struct vm_page **pps, 2009 int npages, int centeridx, vm_prot_t access_type, int flags) 2010 { 2011 struct uvm_object *uobj = ufi->entry->object.uvm_obj; 2012 struct drm_gem_object *gem_obj = 2013 container_of(uobj, struct drm_gem_object, gemo_uvmobj); 2014 struct drm_i915_gem_object *obj = to_intel_bo(gem_obj); 2015 struct drm_device *dev = obj->base.dev; 2016 struct drm_i915_private *dev_priv = dev->dev_private; 2017 voff_t byte_offset; 2018 pgoff_t page_offset; 2019 int ret = 0; 2020 bool write = ISSET(access_type, VM_PROT_WRITE)? 1 : 0; 2021 2022 byte_offset = (ufi->entry->offset + (vaddr - ufi->entry->start)); 2023 KASSERT(byte_offset <= obj->base.size); 2024 page_offset = (byte_offset >> PAGE_SHIFT); 2025 2026 intel_runtime_pm_get(dev_priv); 2027 2028 /* Thanks, uvm, but we don't need this lock. */ 2029 mutex_exit(uobj->vmobjlock); 2030 2031 ret = i915_mutex_lock_interruptible(dev); 2032 if (ret) 2033 goto out; 2034 2035 trace_i915_gem_object_fault(obj, page_offset, true, write); 2036 2037 ret = i915_gem_object_wait_rendering__nonblocking(obj, NULL, !write); 2038 if (ret) 2039 goto unlock; 2040 2041 if ((obj->cache_level != I915_CACHE_NONE) && !HAS_LLC(dev)) { 2042 ret = -EINVAL; 2043 goto unlock; 2044 } 2045 2046 ret = i915_gem_obj_ggtt_pin(obj, 0, PIN_MAPPABLE); 2047 if (ret) 2048 goto unlock; 2049 2050 ret = i915_gem_object_set_to_gtt_domain(obj, write); 2051 if (ret) 2052 goto unpin; 2053 2054 ret = i915_gem_object_get_fence(obj); 2055 if (ret) 2056 goto unpin; 2057 2058 obj->fault_mappable = true; 2059 2060 /* XXX errno NetBSD->Linux */ 2061 ret = -i915_udv_fault(ufi, vaddr, pps, npages, centeridx, access_type, 2062 flags, 2063 (dev_priv->gtt.mappable_base + i915_gem_obj_ggtt_offset(obj))); 2064 unpin: 2065 i915_gem_object_ggtt_unpin(obj); 2066 unlock: 2067 mutex_unlock(&dev->struct_mutex); 2068 out: 2069 mutex_enter(uobj->vmobjlock); 2070 uvmfault_unlockall(ufi, ufi->entry->aref.ar_amap, uobj); 2071 if (ret == -ERESTART) 2072 uvm_wait("i915flt"); 2073 2074 /* 2075 * Remap EINTR to success, so that we return to userland. 2076 * On the way out, we'll deliver the signal, and if the signal 2077 * is not fatal then the user code which faulted will most likely 2078 * fault again, and we'll come back here for another try. 2079 */ 2080 if (ret == -EINTR) 2081 ret = 0; 2082 /* XXX Deal with GPU hangs here... */ 2083 intel_runtime_pm_put(dev_priv); 2084 /* XXX errno Linux->NetBSD */ 2085 return -ret; 2086 } 2087 2088 /* 2089 * XXX i915_udv_fault is copypasta of udv_fault from uvm_device.c. 2090 * 2091 * XXX pmap_enter_default instead of pmap_enter because of a problem 2092 * with using weak aliases in kernel modules or something. 2093 */ 2094 int pmap_enter_default(pmap_t, vaddr_t, paddr_t, vm_prot_t, unsigned); 2095 2096 static int 2097 i915_udv_fault(struct uvm_faultinfo *ufi, vaddr_t vaddr, struct vm_page **pps, 2098 int npages, int centeridx, vm_prot_t access_type, int flags, 2099 paddr_t gtt_paddr) 2100 { 2101 struct vm_map_entry *entry = ufi->entry; 2102 vaddr_t curr_va; 2103 off_t curr_offset; 2104 paddr_t paddr; 2105 u_int mmapflags; 2106 int lcv, retval; 2107 vm_prot_t mapprot; 2108 UVMHIST_FUNC("i915_udv_fault"); UVMHIST_CALLED(maphist); 2109 UVMHIST_LOG(maphist," flags=%jd", flags,0,0,0); 2110 2111 /* 2112 * we do not allow device mappings to be mapped copy-on-write 2113 * so we kill any attempt to do so here. 2114 */ 2115 2116 if (UVM_ET_ISCOPYONWRITE(entry)) { 2117 UVMHIST_LOG(maphist, "<- failed -- COW entry (etype=0x%jx)", 2118 entry->etype, 0,0,0); 2119 return(EIO); 2120 } 2121 2122 /* 2123 * now we must determine the offset in udv to use and the VA to 2124 * use for pmap_enter. note that we always use orig_map's pmap 2125 * for pmap_enter (even if we have a submap). since virtual 2126 * addresses in a submap must match the main map, this is ok. 2127 */ 2128 2129 /* udv offset = (offset from start of entry) + entry's offset */ 2130 curr_offset = entry->offset + (vaddr - entry->start); 2131 /* pmap va = vaddr (virtual address of pps[0]) */ 2132 curr_va = vaddr; 2133 2134 /* 2135 * loop over the page range entering in as needed 2136 */ 2137 2138 retval = 0; 2139 for (lcv = 0 ; lcv < npages ; lcv++, curr_offset += PAGE_SIZE, 2140 curr_va += PAGE_SIZE) { 2141 if ((flags & PGO_ALLPAGES) == 0 && lcv != centeridx) 2142 continue; 2143 2144 if (pps[lcv] == PGO_DONTCARE) 2145 continue; 2146 2147 paddr = (gtt_paddr + curr_offset); 2148 mmapflags = 0; 2149 mapprot = ufi->entry->protection; 2150 UVMHIST_LOG(maphist, 2151 " MAPPING: device: pm=0x%#jx, va=0x%jx, pa=0x%jx, at=%jd", 2152 (uintptr_t)ufi->orig_map->pmap, curr_va, paddr, mapprot); 2153 if (pmap_enter_default(ufi->orig_map->pmap, curr_va, paddr, mapprot, 2154 PMAP_CANFAIL | mapprot | mmapflags) != 0) { 2155 /* 2156 * pmap_enter() didn't have the resource to 2157 * enter this mapping. Unlock everything, 2158 * wait for the pagedaemon to free up some 2159 * pages, and then tell uvm_fault() to start 2160 * the fault again. 2161 * 2162 * XXX Needs some rethinking for the PGO_ALLPAGES 2163 * XXX case. 2164 */ 2165 pmap_update(ufi->orig_map->pmap); /* sync what we have so far */ 2166 return (ERESTART); 2167 } 2168 } 2169 2170 pmap_update(ufi->orig_map->pmap); 2171 return (retval); 2172 } 2173 #else 2174 /** 2175 * i915_gem_fault - fault a page into the GTT 2176 * @vma: VMA in question 2177 * @vmf: fault info 2178 * 2179 * The fault handler is set up by drm_gem_mmap() when a object is GTT mapped 2180 * from userspace. The fault handler takes care of binding the object to 2181 * the GTT (if needed), allocating and programming a fence register (again, 2182 * only if needed based on whether the old reg is still valid or the object 2183 * is tiled) and inserting a new PTE into the faulting process. 2184 * 2185 * Note that the faulting process may involve evicting existing objects 2186 * from the GTT and/or fence registers to make room. So performance may 2187 * suffer if the GTT working set is large or there are few fence registers 2188 * left. 2189 */ 2190 int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf) 2191 { 2192 struct drm_i915_gem_object *obj = to_intel_bo(vma->vm_private_data); 2193 struct drm_device *dev = obj->base.dev; 2194 struct drm_i915_private *dev_priv = dev->dev_private; 2195 struct i915_ggtt_view view = i915_ggtt_view_normal; 2196 pgoff_t page_offset; 2197 unsigned long pfn; 2198 int ret = 0; 2199 bool write = !!(vmf->flags & FAULT_FLAG_WRITE); 2200 2201 intel_runtime_pm_get(dev_priv); 2202 2203 /* We don't use vmf->pgoff since that has the fake offset */ 2204 page_offset = ((unsigned long)vmf->virtual_address - vma->vm_start) >> 2205 PAGE_SHIFT; 2206 2207 ret = i915_mutex_lock_interruptible(dev); 2208 if (ret) 2209 goto out; 2210 2211 trace_i915_gem_object_fault(obj, page_offset, true, write); 2212 2213 /* Try to flush the object off the GPU first without holding the lock. 2214 * Upon reacquiring the lock, we will perform our sanity checks and then 2215 * repeat the flush holding the lock in the normal manner to catch cases 2216 * where we are gazumped. 2217 */ 2218 ret = i915_gem_object_wait_rendering__nonblocking(obj, NULL, !write); 2219 if (ret) 2220 goto unlock; 2221 2222 /* Access to snoopable pages through the GTT is incoherent. */ 2223 if (obj->cache_level != I915_CACHE_NONE && !HAS_LLC(dev)) { 2224 ret = -EFAULT; 2225 goto unlock; 2226 } 2227 2228 /* Use a partial view if the object is bigger than the aperture. */ 2229 if (obj->base.size >= dev_priv->gtt.mappable_end && 2230 obj->tiling_mode == I915_TILING_NONE) { 2231 static const unsigned int chunk_size = 256; // 1 MiB 2232 2233 memset(&view, 0, sizeof(view)); 2234 view.type = I915_GGTT_VIEW_PARTIAL; 2235 view.params.partial.offset = rounddown(page_offset, chunk_size); 2236 view.params.partial.size = 2237 min_t(unsigned int, 2238 chunk_size, 2239 (vma->vm_end - vma->vm_start)/PAGE_SIZE - 2240 view.params.partial.offset); 2241 } 2242 2243 /* Now pin it into the GTT if needed */ 2244 ret = i915_gem_object_ggtt_pin(obj, &view, 0, PIN_MAPPABLE); 2245 if (ret) 2246 goto unlock; 2247 2248 ret = i915_gem_object_set_to_gtt_domain(obj, write); 2249 if (ret) 2250 goto unpin; 2251 2252 ret = i915_gem_object_get_fence(obj); 2253 if (ret) 2254 goto unpin; 2255 2256 /* Finally, remap it using the new GTT offset */ 2257 pfn = dev_priv->gtt.mappable_base + 2258 i915_gem_obj_ggtt_offset_view(obj, &view); 2259 pfn >>= PAGE_SHIFT; 2260 2261 if (unlikely(view.type == I915_GGTT_VIEW_PARTIAL)) { 2262 /* Overriding existing pages in partial view does not cause 2263 * us any trouble as TLBs are still valid because the fault 2264 * is due to userspace losing part of the mapping or never 2265 * having accessed it before (at this partials' range). 2266 */ 2267 unsigned long base = vma->vm_start + 2268 (view.params.partial.offset << PAGE_SHIFT); 2269 unsigned int i; 2270 2271 for (i = 0; i < view.params.partial.size; i++) { 2272 ret = vm_insert_pfn(vma, base + i * PAGE_SIZE, pfn + i); 2273 if (ret) 2274 break; 2275 } 2276 2277 obj->fault_mappable = true; 2278 } else { 2279 if (!obj->fault_mappable) { 2280 unsigned long size = min_t(unsigned long, 2281 vma->vm_end - vma->vm_start, 2282 obj->base.size); 2283 int i; 2284 2285 for (i = 0; i < size >> PAGE_SHIFT; i++) { 2286 ret = vm_insert_pfn(vma, 2287 (unsigned long)vma->vm_start + i * PAGE_SIZE, 2288 pfn + i); 2289 if (ret) 2290 break; 2291 } 2292 2293 obj->fault_mappable = true; 2294 } else 2295 ret = vm_insert_pfn(vma, 2296 (unsigned long)vmf->virtual_address, 2297 pfn + page_offset); 2298 } 2299 unpin: 2300 i915_gem_object_ggtt_unpin_view(obj, &view); 2301 unlock: 2302 mutex_unlock(&dev->struct_mutex); 2303 out: 2304 switch (ret) { 2305 case -EIO: 2306 /* 2307 * We eat errors when the gpu is terminally wedged to avoid 2308 * userspace unduly crashing (gl has no provisions for mmaps to 2309 * fail). But any other -EIO isn't ours (e.g. swap in failure) 2310 * and so needs to be reported. 2311 */ 2312 if (!i915_terminally_wedged(&dev_priv->gpu_error)) { 2313 ret = VM_FAULT_SIGBUS; 2314 break; 2315 } 2316 case -EAGAIN: 2317 /* 2318 * EAGAIN means the gpu is hung and we'll wait for the error 2319 * handler to reset everything when re-faulting in 2320 * i915_mutex_lock_interruptible. 2321 */ 2322 case 0: 2323 case -ERESTARTSYS: 2324 case -EINTR: 2325 case -EBUSY: 2326 /* 2327 * EBUSY is ok: this just means that another thread 2328 * already did the job. 2329 */ 2330 ret = VM_FAULT_NOPAGE; 2331 break; 2332 case -ENOMEM: 2333 ret = VM_FAULT_OOM; 2334 break; 2335 case -ENOSPC: 2336 case -EFAULT: 2337 ret = VM_FAULT_SIGBUS; 2338 break; 2339 default: 2340 WARN_ONCE(ret, "unhandled error in i915_gem_fault: %i\n", ret); 2341 ret = VM_FAULT_SIGBUS; 2342 break; 2343 } 2344 2345 intel_runtime_pm_put(dev_priv); 2346 return ret; 2347 } 2348 #endif 2349 2350 /** 2351 * i915_gem_release_mmap - remove physical page mappings 2352 * @obj: obj in question 2353 * 2354 * Preserve the reservation of the mmapping with the DRM core code, but 2355 * relinquish ownership of the pages back to the system. 2356 * 2357 * It is vital that we remove the page mapping if we have mapped a tiled 2358 * object through the GTT and then lose the fence register due to 2359 * resource pressure. Similarly if the object has been moved out of the 2360 * aperture, than pages mapped into userspace must be revoked. Removing the 2361 * mapping will then trigger a page fault on the next user access, allowing 2362 * fixup by i915_gem_fault(). 2363 */ 2364 void 2365 i915_gem_release_mmap(struct drm_i915_gem_object *obj) 2366 { 2367 if (!obj->fault_mappable) 2368 return; 2369 2370 #ifdef __NetBSD__ /* XXX gem gtt fault */ 2371 { 2372 struct drm_device *const dev = obj->base.dev; 2373 struct drm_i915_private *const dev_priv = dev->dev_private; 2374 const paddr_t start = dev_priv->gtt.mappable_base + 2375 i915_gem_obj_ggtt_offset(obj); 2376 const size_t size = obj->base.size; 2377 const paddr_t end = start + size; 2378 paddr_t pa; 2379 2380 KASSERT((start & (PAGE_SIZE - 1)) == 0); 2381 KASSERT((size & (PAGE_SIZE - 1)) == 0); 2382 2383 for (pa = start; pa < end; pa += PAGE_SIZE) 2384 pmap_pv_protect(pa, VM_PROT_NONE); 2385 } 2386 #else 2387 drm_vma_node_unmap(&obj->base.vma_node, 2388 obj->base.dev->anon_inode->i_mapping); 2389 #endif 2390 obj->fault_mappable = false; 2391 } 2392 2393 void 2394 i915_gem_release_all_mmaps(struct drm_i915_private *dev_priv) 2395 { 2396 struct drm_i915_gem_object *obj; 2397 2398 list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) 2399 i915_gem_release_mmap(obj); 2400 } 2401 2402 uint32_t 2403 i915_gem_get_gtt_size(struct drm_device *dev, uint32_t size, int tiling_mode) 2404 { 2405 uint32_t gtt_size; 2406 2407 if (INTEL_INFO(dev)->gen >= 4 || 2408 tiling_mode == I915_TILING_NONE) 2409 return size; 2410 2411 /* Previous chips need a power-of-two fence region when tiling */ 2412 if (INTEL_INFO(dev)->gen == 3) 2413 gtt_size = 1024*1024; 2414 else 2415 gtt_size = 512*1024; 2416 2417 while (gtt_size < size) 2418 gtt_size <<= 1; 2419 2420 return gtt_size; 2421 } 2422 2423 /** 2424 * i915_gem_get_gtt_alignment - return required GTT alignment for an object 2425 * @obj: object to check 2426 * 2427 * Return the required GTT alignment for an object, taking into account 2428 * potential fence register mapping. 2429 */ 2430 uint32_t 2431 i915_gem_get_gtt_alignment(struct drm_device *dev, uint32_t size, 2432 int tiling_mode, bool fenced) 2433 { 2434 /* 2435 * Minimum alignment is 4k (GTT page size), but might be greater 2436 * if a fence register is needed for the object. 2437 */ 2438 if (INTEL_INFO(dev)->gen >= 4 || (!fenced && IS_G33(dev)) || 2439 tiling_mode == I915_TILING_NONE) 2440 return 4096; 2441 2442 /* 2443 * Previous chips need to be aligned to the size of the smallest 2444 * fence register that can contain the object. 2445 */ 2446 return i915_gem_get_gtt_size(dev, size, tiling_mode); 2447 } 2448 2449 static int i915_gem_object_create_mmap_offset(struct drm_i915_gem_object *obj) 2450 { 2451 struct drm_i915_private *dev_priv = obj->base.dev->dev_private; 2452 int ret; 2453 2454 if (drm_vma_node_has_offset(&obj->base.vma_node)) 2455 return 0; 2456 2457 dev_priv->mm.shrinker_no_lock_stealing = true; 2458 2459 ret = drm_gem_create_mmap_offset(&obj->base); 2460 if (ret != -ENOSPC) 2461 goto out; 2462 2463 /* Badly fragmented mmap space? The only way we can recover 2464 * space is by destroying unwanted objects. We can't randomly release 2465 * mmap_offsets as userspace expects them to be persistent for the 2466 * lifetime of the objects. The closest we can is to release the 2467 * offsets on purgeable objects by truncating it and marking it purged, 2468 * which prevents userspace from ever using that object again. 2469 */ 2470 i915_gem_shrink(dev_priv, 2471 obj->base.size >> PAGE_SHIFT, 2472 I915_SHRINK_BOUND | 2473 I915_SHRINK_UNBOUND | 2474 I915_SHRINK_PURGEABLE); 2475 ret = drm_gem_create_mmap_offset(&obj->base); 2476 if (ret != -ENOSPC) 2477 goto out; 2478 2479 i915_gem_shrink_all(dev_priv); 2480 ret = drm_gem_create_mmap_offset(&obj->base); 2481 out: 2482 dev_priv->mm.shrinker_no_lock_stealing = false; 2483 2484 return ret; 2485 } 2486 2487 static void i915_gem_object_free_mmap_offset(struct drm_i915_gem_object *obj) 2488 { 2489 drm_gem_free_mmap_offset(&obj->base); 2490 } 2491 2492 int 2493 i915_gem_mmap_gtt(struct drm_file *file, 2494 struct drm_device *dev, 2495 uint32_t handle, 2496 uint64_t *offset) 2497 { 2498 struct drm_gem_object *gobj; 2499 struct drm_i915_gem_object *obj; 2500 int ret; 2501 2502 ret = i915_mutex_lock_interruptible(dev); 2503 if (ret) 2504 return ret; 2505 2506 gobj = drm_gem_object_lookup(dev, file, handle); 2507 if (gobj == NULL) { 2508 ret = -ENOENT; 2509 goto unlock; 2510 } 2511 obj = to_intel_bo(gobj); 2512 2513 if (obj->madv != I915_MADV_WILLNEED) { 2514 DRM_DEBUG("Attempting to mmap a purgeable buffer\n"); 2515 ret = -EFAULT; 2516 goto out; 2517 } 2518 2519 ret = i915_gem_object_create_mmap_offset(obj); 2520 if (ret) 2521 goto out; 2522 2523 *offset = drm_vma_node_offset_addr(&obj->base.vma_node); 2524 2525 out: 2526 drm_gem_object_unreference(&obj->base); 2527 unlock: 2528 mutex_unlock(&dev->struct_mutex); 2529 return ret; 2530 } 2531 2532 /** 2533 * i915_gem_mmap_gtt_ioctl - prepare an object for GTT mmap'ing 2534 * @dev: DRM device 2535 * @data: GTT mapping ioctl data 2536 * @file: GEM object info 2537 * 2538 * Simply returns the fake offset to userspace so it can mmap it. 2539 * The mmap call will end up in drm_gem_mmap(), which will set things 2540 * up so we can get faults in the handler above. 2541 * 2542 * The fault handler will take care of binding the object into the GTT 2543 * (since it may have been evicted to make room for something), allocating 2544 * a fence register, and mapping the appropriate aperture address into 2545 * userspace. 2546 */ 2547 int 2548 i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data, 2549 struct drm_file *file) 2550 { 2551 struct drm_i915_gem_mmap_gtt *args = data; 2552 2553 return i915_gem_mmap_gtt(file, dev, args->handle, &args->offset); 2554 } 2555 2556 /* Immediately discard the backing storage */ 2557 static void 2558 i915_gem_object_truncate(struct drm_i915_gem_object *obj) 2559 { 2560 i915_gem_object_free_mmap_offset(obj); 2561 2562 if (obj->base.filp == NULL) 2563 return; 2564 2565 #ifdef __NetBSD__ 2566 { 2567 struct uvm_object *const uobj = obj->base.filp; 2568 2569 if (uobj != NULL) { 2570 /* XXX Calling pgo_put like this is bogus. */ 2571 mutex_enter(uobj->vmobjlock); 2572 (*uobj->pgops->pgo_put)(uobj, 0, obj->base.size, 2573 (PGO_ALLPAGES | PGO_FREE)); 2574 } 2575 } 2576 #else 2577 /* Our goal here is to return as much of the memory as 2578 * is possible back to the system as we are called from OOM. 2579 * To do this we must instruct the shmfs to drop all of its 2580 * backing pages, *now*. 2581 */ 2582 shmem_truncate_range(file_inode(obj->base.filp), 0, (loff_t)-1); 2583 #endif 2584 obj->madv = __I915_MADV_PURGED; 2585 } 2586 2587 /* Try to discard unwanted pages */ 2588 static void 2589 i915_gem_object_invalidate(struct drm_i915_gem_object *obj) 2590 { 2591 #ifdef __NetBSD__ 2592 struct uvm_object *uobj; 2593 #else 2594 struct address_space *mapping; 2595 #endif 2596 2597 switch (obj->madv) { 2598 case I915_MADV_DONTNEED: 2599 i915_gem_object_truncate(obj); 2600 case __I915_MADV_PURGED: 2601 return; 2602 } 2603 2604 if (obj->base.filp == NULL) 2605 return; 2606 2607 #ifdef __NetBSD__ 2608 uobj = obj->base.filp; 2609 mutex_enter(uobj->vmobjlock); 2610 (*uobj->pgops->pgo_put)(uobj, 0, obj->base.size, 2611 PGO_ALLPAGES|PGO_DEACTIVATE|PGO_CLEANIT); 2612 #else 2613 mapping = file_inode(obj->base.filp)->i_mapping, 2614 invalidate_mapping_pages(mapping, 0, (loff_t)-1); 2615 #endif 2616 } 2617 2618 static void 2619 i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj) 2620 { 2621 #ifdef __NetBSD__ 2622 struct drm_device *const dev = obj->base.dev; 2623 struct vm_page *page; 2624 int ret; 2625 2626 /* XXX Cargo-culted from the Linux code. */ 2627 BUG_ON(obj->madv == __I915_MADV_PURGED); 2628 2629 ret = i915_gem_object_set_to_cpu_domain(obj, true); 2630 if (ret) { 2631 WARN_ON(ret != -EIO); 2632 i915_gem_clflush_object(obj, true); 2633 obj->base.read_domains = obj->base.write_domain = 2634 I915_GEM_DOMAIN_CPU; 2635 } 2636 2637 i915_gem_gtt_finish_object(obj); 2638 2639 if (i915_gem_object_needs_bit17_swizzle(obj)) 2640 i915_gem_object_save_bit_17_swizzle(obj); 2641 2642 if (obj->madv == I915_MADV_DONTNEED) 2643 obj->dirty = 0; 2644 2645 if (obj->dirty) { 2646 TAILQ_FOREACH(page, &obj->pageq, pageq.queue) { 2647 page->flags &= ~PG_CLEAN; 2648 /* XXX mark page accessed */ 2649 } 2650 } 2651 obj->dirty = 0; 2652 2653 uvm_obj_unwirepages(obj->base.filp, 0, obj->base.size); 2654 bus_dmamap_destroy(dev->dmat, obj->pages); 2655 #else 2656 struct sg_page_iter sg_iter; 2657 int ret; 2658 2659 BUG_ON(obj->madv == __I915_MADV_PURGED); 2660 2661 ret = i915_gem_object_set_to_cpu_domain(obj, true); 2662 if (ret) { 2663 /* In the event of a disaster, abandon all caches and 2664 * hope for the best. 2665 */ 2666 WARN_ON(ret != -EIO); 2667 i915_gem_clflush_object(obj, true); 2668 obj->base.read_domains = obj->base.write_domain = I915_GEM_DOMAIN_CPU; 2669 } 2670 2671 i915_gem_gtt_finish_object(obj); 2672 2673 if (i915_gem_object_needs_bit17_swizzle(obj)) 2674 i915_gem_object_save_bit_17_swizzle(obj); 2675 2676 if (obj->madv == I915_MADV_DONTNEED) 2677 obj->dirty = 0; 2678 2679 for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents, 0) { 2680 struct page *page = sg_page_iter_page(&sg_iter); 2681 2682 if (obj->dirty) 2683 set_page_dirty(page); 2684 2685 if (obj->madv == I915_MADV_WILLNEED) 2686 mark_page_accessed(page); 2687 2688 page_cache_release(page); 2689 } 2690 obj->dirty = 0; 2691 2692 sg_free_table(obj->pages); 2693 kfree(obj->pages); 2694 #endif 2695 } 2696 2697 int 2698 i915_gem_object_put_pages(struct drm_i915_gem_object *obj) 2699 { 2700 const struct drm_i915_gem_object_ops *ops = obj->ops; 2701 2702 if (obj->pages == NULL) 2703 return 0; 2704 2705 if (obj->pages_pin_count) 2706 return -EBUSY; 2707 2708 BUG_ON(i915_gem_obj_bound_any(obj)); 2709 2710 /* ->put_pages might need to allocate memory for the bit17 swizzle 2711 * array, hence protect them from being reaped by removing them from gtt 2712 * lists early. */ 2713 list_del(&obj->global_list); 2714 2715 ops->put_pages(obj); 2716 obj->pages = NULL; 2717 2718 i915_gem_object_invalidate(obj); 2719 2720 return 0; 2721 } 2722 2723 static int 2724 i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj) 2725 { 2726 #ifdef __NetBSD__ 2727 struct drm_device *const dev = obj->base.dev; 2728 struct drm_i915_private *dev_priv = dev->dev_private; 2729 struct vm_page *page; 2730 int ret; 2731 2732 BUG_ON(obj->base.read_domains & I915_GEM_GPU_DOMAINS); 2733 BUG_ON(obj->base.write_domain & I915_GEM_GPU_DOMAINS); 2734 2735 KASSERT(obj->pages == NULL); 2736 TAILQ_INIT(&obj->pageq); 2737 2738 /* XXX errno NetBSD->Linux */ 2739 ret = -bus_dmamap_create(dev->dmat, obj->base.size, 2740 obj->base.size/PAGE_SIZE, PAGE_SIZE, 0, BUS_DMA_NOWAIT, 2741 &obj->pages); 2742 if (ret) 2743 goto fail0; 2744 2745 /* XXX errno NetBSD->Linux */ 2746 ret = -uvm_obj_wirepages(obj->base.filp, 0, obj->base.size, 2747 &obj->pageq); 2748 if (ret) /* XXX Try purge, shrink. */ 2749 goto fail1; 2750 2751 /* 2752 * Check that the paddrs will fit in 40 bits, or 32 bits on i965. 2753 * 2754 * XXX This should be unnecessary: the uao should guarantee 2755 * this constraint after uao_set_pgfl. 2756 * 2757 * XXX This should also be expanded for newer devices. 2758 */ 2759 TAILQ_FOREACH(page, &obj->pageq, pageq.queue) { 2760 const uint64_t mask = 2761 (IS_BROADWATER(dev) || IS_CRESTLINE(dev)? 2762 0xffffffffULL : 0xffffffffffULL); 2763 if (VM_PAGE_TO_PHYS(page) & ~mask) { 2764 DRM_ERROR("GEM physical address exceeds %u bits" 2765 ": %"PRIxMAX"\n", 2766 popcount64(mask), 2767 (uintmax_t)VM_PAGE_TO_PHYS(page)); 2768 ret = -EIO; 2769 goto fail2; 2770 } 2771 } 2772 2773 ret = i915_gem_gtt_prepare_object(obj); 2774 if (ret) 2775 goto fail2; 2776 2777 if (i915_gem_object_needs_bit17_swizzle(obj)) 2778 i915_gem_object_do_bit_17_swizzle(obj); 2779 2780 if (obj->tiling_mode != I915_TILING_NONE && 2781 dev_priv->quirks & QUIRK_PIN_SWIZZLED_PAGES) 2782 i915_gem_object_pin_pages(obj); 2783 2784 /* Success! */ 2785 return 0; 2786 2787 fail3: __unused 2788 i915_gem_gtt_finish_object(obj); 2789 fail2: uvm_obj_unwirepages(obj->base.filp, 0, obj->base.size); 2790 fail1: bus_dmamap_destroy(dev->dmat, obj->pages); 2791 obj->pages = NULL; 2792 fail0: KASSERT(ret); 2793 return ret; 2794 #else 2795 struct drm_i915_private *dev_priv = obj->base.dev->dev_private; 2796 int page_count, i; 2797 struct address_space *mapping; 2798 struct sg_table *st; 2799 struct scatterlist *sg; 2800 struct sg_page_iter sg_iter; 2801 struct page *page; 2802 unsigned long last_pfn = 0; /* suppress gcc warning */ 2803 int ret; 2804 gfp_t gfp; 2805 2806 /* Assert that the object is not currently in any GPU domain. As it 2807 * wasn't in the GTT, there shouldn't be any way it could have been in 2808 * a GPU cache 2809 */ 2810 BUG_ON(obj->base.read_domains & I915_GEM_GPU_DOMAINS); 2811 BUG_ON(obj->base.write_domain & I915_GEM_GPU_DOMAINS); 2812 2813 st = kmalloc(sizeof(*st), GFP_KERNEL); 2814 if (st == NULL) 2815 return -ENOMEM; 2816 2817 page_count = obj->base.size / PAGE_SIZE; 2818 if (sg_alloc_table(st, page_count, GFP_KERNEL)) { 2819 kfree(st); 2820 return -ENOMEM; 2821 } 2822 2823 /* Get the list of pages out of our struct file. They'll be pinned 2824 * at this point until we release them. 2825 * 2826 * Fail silently without starting the shrinker 2827 */ 2828 mapping = file_inode(obj->base.filp)->i_mapping; 2829 gfp = mapping_gfp_constraint(mapping, ~(__GFP_IO | __GFP_RECLAIM)); 2830 gfp |= __GFP_NORETRY | __GFP_NOWARN; 2831 sg = st->sgl; 2832 st->nents = 0; 2833 for (i = 0; i < page_count; i++) { 2834 page = shmem_read_mapping_page_gfp(mapping, i, gfp); 2835 if (IS_ERR(page)) { 2836 i915_gem_shrink(dev_priv, 2837 page_count, 2838 I915_SHRINK_BOUND | 2839 I915_SHRINK_UNBOUND | 2840 I915_SHRINK_PURGEABLE); 2841 page = shmem_read_mapping_page_gfp(mapping, i, gfp); 2842 } 2843 if (IS_ERR(page)) { 2844 /* We've tried hard to allocate the memory by reaping 2845 * our own buffer, now let the real VM do its job and 2846 * go down in flames if truly OOM. 2847 */ 2848 i915_gem_shrink_all(dev_priv); 2849 page = shmem_read_mapping_page(mapping, i); 2850 if (IS_ERR(page)) { 2851 ret = PTR_ERR(page); 2852 goto err_pages; 2853 } 2854 } 2855 #ifdef CONFIG_SWIOTLB 2856 if (swiotlb_nr_tbl()) { 2857 st->nents++; 2858 sg_set_page(sg, page, PAGE_SIZE, 0); 2859 sg = sg_next(sg); 2860 continue; 2861 } 2862 #endif 2863 if (!i || page_to_pfn(page) != last_pfn + 1) { 2864 if (i) 2865 sg = sg_next(sg); 2866 st->nents++; 2867 sg_set_page(sg, page, PAGE_SIZE, 0); 2868 } else { 2869 sg->length += PAGE_SIZE; 2870 } 2871 last_pfn = page_to_pfn(page); 2872 2873 /* Check that the i965g/gm workaround works. */ 2874 WARN_ON((gfp & __GFP_DMA32) && (last_pfn >= 0x00100000UL)); 2875 } 2876 #ifdef CONFIG_SWIOTLB 2877 if (!swiotlb_nr_tbl()) 2878 #endif 2879 sg_mark_end(sg); 2880 obj->pages = st; 2881 2882 ret = i915_gem_gtt_prepare_object(obj); 2883 if (ret) 2884 goto err_pages; 2885 2886 if (i915_gem_object_needs_bit17_swizzle(obj)) 2887 i915_gem_object_do_bit_17_swizzle(obj); 2888 2889 if (obj->tiling_mode != I915_TILING_NONE && 2890 dev_priv->quirks & QUIRK_PIN_SWIZZLED_PAGES) 2891 i915_gem_object_pin_pages(obj); 2892 2893 return 0; 2894 2895 err_pages: 2896 sg_mark_end(sg); 2897 for_each_sg_page(st->sgl, &sg_iter, st->nents, 0) 2898 page_cache_release(sg_page_iter_page(&sg_iter)); 2899 sg_free_table(st); 2900 kfree(st); 2901 2902 /* shmemfs first checks if there is enough memory to allocate the page 2903 * and reports ENOSPC should there be insufficient, along with the usual 2904 * ENOMEM for a genuine allocation failure. 2905 * 2906 * We use ENOSPC in our driver to mean that we have run out of aperture 2907 * space and so want to translate the error from shmemfs back to our 2908 * usual understanding of ENOMEM. 2909 */ 2910 if (ret == -ENOSPC) 2911 ret = -ENOMEM; 2912 2913 return ret; 2914 #endif 2915 } 2916 2917 /* Ensure that the associated pages are gathered from the backing storage 2918 * and pinned into our object. i915_gem_object_get_pages() may be called 2919 * multiple times before they are released by a single call to 2920 * i915_gem_object_put_pages() - once the pages are no longer referenced 2921 * either as a result of memory pressure (reaping pages under the shrinker) 2922 * or as the object is itself released. 2923 */ 2924 int 2925 i915_gem_object_get_pages(struct drm_i915_gem_object *obj) 2926 { 2927 struct drm_i915_private *dev_priv = obj->base.dev->dev_private; 2928 const struct drm_i915_gem_object_ops *ops = obj->ops; 2929 int ret; 2930 2931 if (obj->pages) 2932 return 0; 2933 2934 if (obj->madv != I915_MADV_WILLNEED) { 2935 DRM_DEBUG("Attempting to obtain a purgeable object\n"); 2936 return -EFAULT; 2937 } 2938 2939 BUG_ON(obj->pages_pin_count); 2940 2941 ret = ops->get_pages(obj); 2942 if (ret) 2943 return ret; 2944 2945 list_add_tail(&obj->global_list, &dev_priv->mm.unbound_list); 2946 2947 #ifndef __NetBSD__ 2948 obj->get_page.sg = obj->pages->sgl; 2949 obj->get_page.last = 0; 2950 #endif 2951 2952 return 0; 2953 } 2954 2955 void i915_vma_move_to_active(struct i915_vma *vma, 2956 struct drm_i915_gem_request *req) 2957 { 2958 struct drm_i915_gem_object *obj = vma->obj; 2959 struct intel_engine_cs *ring; 2960 2961 ring = i915_gem_request_get_ring(req); 2962 2963 /* Add a reference if we're newly entering the active list. */ 2964 if (obj->active == 0) 2965 drm_gem_object_reference(&obj->base); 2966 obj->active |= intel_ring_flag(ring); 2967 2968 list_move_tail(&obj->ring_list[ring->id], &ring->active_list); 2969 i915_gem_request_assign(&obj->last_read_req[ring->id], req); 2970 2971 list_move_tail(&vma->mm_list, &vma->vm->active_list); 2972 } 2973 2974 static void 2975 i915_gem_object_retire__write(struct drm_i915_gem_object *obj) 2976 { 2977 RQ_BUG_ON(obj->last_write_req == NULL); 2978 RQ_BUG_ON(!(obj->active & intel_ring_flag(obj->last_write_req->ring))); 2979 2980 i915_gem_request_assign(&obj->last_write_req, NULL); 2981 intel_fb_obj_flush(obj, true, ORIGIN_CS); 2982 } 2983 2984 static void 2985 i915_gem_object_retire__read(struct drm_i915_gem_object *obj, int ring) 2986 { 2987 struct i915_vma *vma; 2988 2989 RQ_BUG_ON(obj->last_read_req[ring] == NULL); 2990 RQ_BUG_ON(!(obj->active & (1 << ring))); 2991 2992 list_del_init(&obj->ring_list[ring]); 2993 i915_gem_request_assign(&obj->last_read_req[ring], NULL); 2994 2995 if (obj->last_write_req && obj->last_write_req->ring->id == ring) 2996 i915_gem_object_retire__write(obj); 2997 2998 obj->active &= ~(1 << ring); 2999 if (obj->active) 3000 return; 3001 3002 /* Bump our place on the bound list to keep it roughly in LRU order 3003 * so that we don't steal from recently used but inactive objects 3004 * (unless we are forced to ofc!) 3005 */ 3006 list_move_tail(&obj->global_list, 3007 &to_i915(obj->base.dev)->mm.bound_list); 3008 3009 list_for_each_entry(vma, &obj->vma_list, vma_link) { 3010 if (!list_empty(&vma->mm_list)) 3011 list_move_tail(&vma->mm_list, &vma->vm->inactive_list); 3012 } 3013 3014 i915_gem_request_assign(&obj->last_fenced_req, NULL); 3015 drm_gem_object_unreference(&obj->base); 3016 } 3017 3018 static int 3019 i915_gem_init_seqno(struct drm_device *dev, u32 seqno) 3020 { 3021 struct drm_i915_private *dev_priv = dev->dev_private; 3022 struct intel_engine_cs *ring; 3023 int ret, i, j; 3024 3025 /* Carefully retire all requests without writing to the rings */ 3026 for_each_ring(ring, dev_priv, i) { 3027 ret = intel_ring_idle(ring); 3028 if (ret) 3029 return ret; 3030 } 3031 i915_gem_retire_requests(dev); 3032 3033 /* Finally reset hw state */ 3034 for_each_ring(ring, dev_priv, i) { 3035 intel_ring_init_seqno(ring, seqno); 3036 3037 for (j = 0; j < ARRAY_SIZE(ring->semaphore.sync_seqno); j++) 3038 ring->semaphore.sync_seqno[j] = 0; 3039 } 3040 3041 return 0; 3042 } 3043 3044 int i915_gem_set_seqno(struct drm_device *dev, u32 seqno) 3045 { 3046 struct drm_i915_private *dev_priv = dev->dev_private; 3047 int ret; 3048 3049 if (seqno == 0) 3050 return -EINVAL; 3051 3052 /* HWS page needs to be set less than what we 3053 * will inject to ring 3054 */ 3055 ret = i915_gem_init_seqno(dev, seqno - 1); 3056 if (ret) 3057 return ret; 3058 3059 /* Carefully set the last_seqno value so that wrap 3060 * detection still works 3061 */ 3062 dev_priv->next_seqno = seqno; 3063 dev_priv->last_seqno = seqno - 1; 3064 if (dev_priv->last_seqno == 0) 3065 dev_priv->last_seqno--; 3066 3067 return 0; 3068 } 3069 3070 int 3071 i915_gem_get_seqno(struct drm_device *dev, u32 *seqno) 3072 { 3073 struct drm_i915_private *dev_priv = dev->dev_private; 3074 3075 /* reserve 0 for non-seqno */ 3076 if (dev_priv->next_seqno == 0) { 3077 int ret = i915_gem_init_seqno(dev, 0); 3078 if (ret) 3079 return ret; 3080 3081 dev_priv->next_seqno = 1; 3082 } 3083 3084 *seqno = dev_priv->last_seqno = dev_priv->next_seqno++; 3085 return 0; 3086 } 3087 3088 /* 3089 * NB: This function is not allowed to fail. Doing so would mean the the 3090 * request is not being tracked for completion but the work itself is 3091 * going to happen on the hardware. This would be a Bad Thing(tm). 3092 */ 3093 void __i915_add_request(struct drm_i915_gem_request *request, 3094 struct drm_i915_gem_object *obj, 3095 bool flush_caches) 3096 { 3097 struct intel_engine_cs *ring; 3098 struct drm_i915_private *dev_priv; 3099 struct intel_ringbuffer *ringbuf; 3100 u32 request_start; 3101 int ret; 3102 3103 if (WARN_ON(request == NULL)) 3104 return; 3105 3106 ring = request->ring; 3107 dev_priv = ring->dev->dev_private; 3108 ringbuf = request->ringbuf; 3109 3110 /* 3111 * To ensure that this call will not fail, space for its emissions 3112 * should already have been reserved in the ring buffer. Let the ring 3113 * know that it is time to use that space up. 3114 */ 3115 intel_ring_reserved_space_use(ringbuf); 3116 3117 request_start = intel_ring_get_tail(ringbuf); 3118 /* 3119 * Emit any outstanding flushes - execbuf can fail to emit the flush 3120 * after having emitted the batchbuffer command. Hence we need to fix 3121 * things up similar to emitting the lazy request. The difference here 3122 * is that the flush _must_ happen before the next request, no matter 3123 * what. 3124 */ 3125 if (flush_caches) { 3126 if (i915.enable_execlists) 3127 ret = logical_ring_flush_all_caches(request); 3128 else 3129 ret = intel_ring_flush_all_caches(request); 3130 /* Not allowed to fail! */ 3131 WARN(ret, "*_ring_flush_all_caches failed: %d!\n", ret); 3132 } 3133 3134 /* Record the position of the start of the request so that 3135 * should we detect the updated seqno part-way through the 3136 * GPU processing the request, we never over-estimate the 3137 * position of the head. 3138 */ 3139 request->postfix = intel_ring_get_tail(ringbuf); 3140 3141 if (i915.enable_execlists) 3142 ret = ring->emit_request(request); 3143 else { 3144 ret = ring->add_request(request); 3145 3146 request->tail = intel_ring_get_tail(ringbuf); 3147 } 3148 /* Not allowed to fail! */ 3149 WARN(ret, "emit|add_request failed: %d!\n", ret); 3150 3151 request->head = request_start; 3152 3153 /* Whilst this request exists, batch_obj will be on the 3154 * active_list, and so will hold the active reference. Only when this 3155 * request is retired will the the batch_obj be moved onto the 3156 * inactive_list and lose its active reference. Hence we do not need 3157 * to explicitly hold another reference here. 3158 */ 3159 request->batch_obj = obj; 3160 3161 request->emitted_jiffies = jiffies; 3162 request->previous_seqno = ring->last_submitted_seqno; 3163 ring->last_submitted_seqno = request->seqno; 3164 list_add_tail(&request->list, &ring->request_list); 3165 3166 trace_i915_gem_request_add(request); 3167 3168 i915_queue_hangcheck(ring->dev); 3169 3170 queue_delayed_work(dev_priv->wq, 3171 &dev_priv->mm.retire_work, 3172 round_jiffies_up_relative(HZ)); 3173 intel_mark_busy(dev_priv->dev); 3174 3175 /* Sanity check that the reserved size was large enough. */ 3176 intel_ring_reserved_space_end(ringbuf); 3177 } 3178 3179 static bool i915_context_is_banned(struct drm_i915_private *dev_priv, 3180 const struct intel_context *ctx) 3181 { 3182 unsigned long elapsed; 3183 3184 elapsed = get_seconds() - ctx->hang_stats.guilty_ts; 3185 3186 if (ctx->hang_stats.banned) 3187 return true; 3188 3189 if (ctx->hang_stats.ban_period_seconds && 3190 elapsed <= ctx->hang_stats.ban_period_seconds) { 3191 if (!i915_gem_context_is_default(ctx)) { 3192 DRM_DEBUG("context hanging too fast, banning!\n"); 3193 return true; 3194 } else if (i915_stop_ring_allow_ban(dev_priv)) { 3195 if (i915_stop_ring_allow_warn(dev_priv)) 3196 DRM_ERROR("gpu hanging too fast, banning!\n"); 3197 return true; 3198 } 3199 } 3200 3201 return false; 3202 } 3203 3204 static void i915_set_reset_status(struct drm_i915_private *dev_priv, 3205 struct intel_context *ctx, 3206 const bool guilty) 3207 { 3208 struct i915_ctx_hang_stats *hs; 3209 3210 if (WARN_ON(!ctx)) 3211 return; 3212 3213 hs = &ctx->hang_stats; 3214 3215 if (guilty) { 3216 hs->banned = i915_context_is_banned(dev_priv, ctx); 3217 hs->batch_active++; 3218 hs->guilty_ts = get_seconds(); 3219 } else { 3220 hs->batch_pending++; 3221 } 3222 } 3223 3224 void i915_gem_request_free(struct kref *req_ref) 3225 { 3226 struct drm_i915_gem_request *req = container_of(req_ref, 3227 typeof(*req), ref); 3228 struct intel_context *ctx = req->ctx; 3229 3230 if (req->file_priv) 3231 i915_gem_request_remove_from_client(req); 3232 3233 if (ctx) { 3234 if (i915.enable_execlists) { 3235 if (ctx != req->ring->default_context) 3236 intel_lr_context_unpin(req); 3237 } 3238 3239 i915_gem_context_unreference(ctx); 3240 } 3241 3242 kmem_cache_free(req->i915->requests, req); 3243 } 3244 3245 int i915_gem_request_alloc(struct intel_engine_cs *ring, 3246 struct intel_context *ctx, 3247 struct drm_i915_gem_request **req_out) 3248 { 3249 struct drm_i915_private *dev_priv = to_i915(ring->dev); 3250 struct drm_i915_gem_request *req; 3251 int ret; 3252 3253 if (!req_out) 3254 return -EINVAL; 3255 3256 *req_out = NULL; 3257 3258 req = kmem_cache_zalloc(dev_priv->requests, GFP_KERNEL); 3259 if (req == NULL) 3260 return -ENOMEM; 3261 3262 ret = i915_gem_get_seqno(ring->dev, &req->seqno); 3263 if (ret) 3264 goto err; 3265 3266 kref_init(&req->ref); 3267 req->i915 = dev_priv; 3268 req->ring = ring; 3269 req->ctx = ctx; 3270 i915_gem_context_reference(req->ctx); 3271 3272 if (i915.enable_execlists) 3273 ret = intel_logical_ring_alloc_request_extras(req); 3274 else 3275 ret = intel_ring_alloc_request_extras(req); 3276 if (ret) { 3277 i915_gem_context_unreference(req->ctx); 3278 goto err; 3279 } 3280 3281 /* 3282 * Reserve space in the ring buffer for all the commands required to 3283 * eventually emit this request. This is to guarantee that the 3284 * i915_add_request() call can't fail. Note that the reserve may need 3285 * to be redone if the request is not actually submitted straight 3286 * away, e.g. because a GPU scheduler has deferred it. 3287 */ 3288 if (i915.enable_execlists) 3289 ret = intel_logical_ring_reserve_space(req); 3290 else 3291 ret = intel_ring_reserve_space(req); 3292 if (ret) { 3293 /* 3294 * At this point, the request is fully allocated even if not 3295 * fully prepared. Thus it can be cleaned up using the proper 3296 * free code. 3297 */ 3298 i915_gem_request_cancel(req); 3299 return ret; 3300 } 3301 3302 *req_out = req; 3303 return 0; 3304 3305 err: 3306 kmem_cache_free(dev_priv->requests, req); 3307 return ret; 3308 } 3309 3310 void i915_gem_request_cancel(struct drm_i915_gem_request *req) 3311 { 3312 intel_ring_reserved_space_cancel(req->ringbuf); 3313 3314 i915_gem_request_unreference(req); 3315 } 3316 3317 struct drm_i915_gem_request * 3318 i915_gem_find_active_request(struct intel_engine_cs *ring) 3319 { 3320 struct drm_i915_gem_request *request; 3321 3322 list_for_each_entry(request, &ring->request_list, list) { 3323 if (i915_gem_request_completed(request, false)) 3324 continue; 3325 3326 return request; 3327 } 3328 3329 return NULL; 3330 } 3331 3332 static void i915_gem_reset_ring_status(struct drm_i915_private *dev_priv, 3333 struct intel_engine_cs *ring) 3334 { 3335 struct drm_i915_gem_request *request; 3336 bool ring_hung; 3337 3338 request = i915_gem_find_active_request(ring); 3339 3340 if (request == NULL) 3341 return; 3342 3343 ring_hung = ring->hangcheck.score >= HANGCHECK_SCORE_RING_HUNG; 3344 3345 i915_set_reset_status(dev_priv, request->ctx, ring_hung); 3346 3347 list_for_each_entry_continue(request, &ring->request_list, list) 3348 i915_set_reset_status(dev_priv, request->ctx, false); 3349 } 3350 3351 static void i915_gem_reset_ring_cleanup(struct drm_i915_private *dev_priv, 3352 struct intel_engine_cs *ring) 3353 { 3354 while (!list_empty(&ring->active_list)) { 3355 struct drm_i915_gem_object *obj; 3356 3357 obj = list_first_entry(&ring->active_list, 3358 struct drm_i915_gem_object, 3359 ring_list[ring->id]); 3360 3361 i915_gem_object_retire__read(obj, ring->id); 3362 } 3363 3364 /* 3365 * Clear the execlists queue up before freeing the requests, as those 3366 * are the ones that keep the context and ringbuffer backing objects 3367 * pinned in place. 3368 */ 3369 while (!list_empty(&ring->execlist_queue)) { 3370 struct drm_i915_gem_request *submit_req; 3371 3372 submit_req = list_first_entry(&ring->execlist_queue, 3373 struct drm_i915_gem_request, 3374 execlist_link); 3375 list_del(&submit_req->execlist_link); 3376 3377 if (submit_req->ctx != ring->default_context) 3378 intel_lr_context_unpin(submit_req); 3379 3380 i915_gem_request_unreference(submit_req); 3381 } 3382 3383 /* 3384 * We must free the requests after all the corresponding objects have 3385 * been moved off active lists. Which is the same order as the normal 3386 * retire_requests function does. This is important if object hold 3387 * implicit references on things like e.g. ppgtt address spaces through 3388 * the request. 3389 */ 3390 while (!list_empty(&ring->request_list)) { 3391 struct drm_i915_gem_request *request; 3392 3393 request = list_first_entry(&ring->request_list, 3394 struct drm_i915_gem_request, 3395 list); 3396 3397 i915_gem_request_retire(request); 3398 } 3399 } 3400 3401 void i915_gem_reset(struct drm_device *dev) 3402 { 3403 struct drm_i915_private *dev_priv = dev->dev_private; 3404 struct intel_engine_cs *ring; 3405 int i; 3406 3407 /* 3408 * Before we free the objects from the requests, we need to inspect 3409 * them for finding the guilty party. As the requests only borrow 3410 * their reference to the objects, the inspection must be done first. 3411 */ 3412 for_each_ring(ring, dev_priv, i) 3413 i915_gem_reset_ring_status(dev_priv, ring); 3414 3415 for_each_ring(ring, dev_priv, i) 3416 i915_gem_reset_ring_cleanup(dev_priv, ring); 3417 3418 i915_gem_context_reset(dev); 3419 3420 i915_gem_restore_fences(dev); 3421 3422 WARN_ON(i915_verify_lists(dev)); 3423 } 3424 3425 /** 3426 * This function clears the request list as sequence numbers are passed. 3427 */ 3428 void 3429 i915_gem_retire_requests_ring(struct intel_engine_cs *ring) 3430 { 3431 WARN_ON(i915_verify_lists(ring->dev)); 3432 3433 /* Retire requests first as we use it above for the early return. 3434 * If we retire requests last, we may use a later seqno and so clear 3435 * the requests lists without clearing the active list, leading to 3436 * confusion. 3437 */ 3438 while (!list_empty(&ring->request_list)) { 3439 struct drm_i915_gem_request *request; 3440 3441 request = list_first_entry(&ring->request_list, 3442 struct drm_i915_gem_request, 3443 list); 3444 3445 if (!i915_gem_request_completed(request, true)) 3446 break; 3447 3448 i915_gem_request_retire(request); 3449 } 3450 3451 /* Move any buffers on the active list that are no longer referenced 3452 * by the ringbuffer to the flushing/inactive lists as appropriate, 3453 * before we free the context associated with the requests. 3454 */ 3455 while (!list_empty(&ring->active_list)) { 3456 struct drm_i915_gem_object *obj; 3457 3458 obj = list_first_entry(&ring->active_list, 3459 struct drm_i915_gem_object, 3460 ring_list[ring->id]); 3461 3462 if (!list_empty(&obj->last_read_req[ring->id]->list)) 3463 break; 3464 3465 i915_gem_object_retire__read(obj, ring->id); 3466 } 3467 3468 if (unlikely(ring->trace_irq_req && 3469 i915_gem_request_completed(ring->trace_irq_req, true))) { 3470 ring->irq_put(ring); 3471 i915_gem_request_assign(&ring->trace_irq_req, NULL); 3472 } 3473 3474 WARN_ON(i915_verify_lists(ring->dev)); 3475 } 3476 3477 bool 3478 i915_gem_retire_requests(struct drm_device *dev) 3479 { 3480 struct drm_i915_private *dev_priv = dev->dev_private; 3481 struct intel_engine_cs *ring; 3482 bool idle = true; 3483 int i; 3484 3485 for_each_ring(ring, dev_priv, i) { 3486 i915_gem_retire_requests_ring(ring); 3487 idle &= list_empty(&ring->request_list); 3488 if (i915.enable_execlists) { 3489 unsigned long flags; 3490 3491 spin_lock_irqsave(&ring->execlist_lock, flags); 3492 idle &= list_empty(&ring->execlist_queue); 3493 spin_unlock_irqrestore(&ring->execlist_lock, flags); 3494 3495 intel_execlists_retire_requests(ring); 3496 } 3497 } 3498 3499 if (idle) 3500 mod_delayed_work(dev_priv->wq, 3501 &dev_priv->mm.idle_work, 3502 msecs_to_jiffies(100)); 3503 3504 return idle; 3505 } 3506 3507 static void 3508 i915_gem_retire_work_handler(struct work_struct *work) 3509 { 3510 struct drm_i915_private *dev_priv = 3511 container_of(work, typeof(*dev_priv), mm.retire_work.work); 3512 struct drm_device *dev = dev_priv->dev; 3513 bool idle; 3514 3515 /* Come back later if the device is busy... */ 3516 idle = false; 3517 if (mutex_trylock(&dev->struct_mutex)) { 3518 idle = i915_gem_retire_requests(dev); 3519 mutex_unlock(&dev->struct_mutex); 3520 } 3521 if (!idle) 3522 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, 3523 round_jiffies_up_relative(HZ)); 3524 } 3525 3526 static void 3527 i915_gem_idle_work_handler(struct work_struct *work) 3528 { 3529 struct drm_i915_private *dev_priv = 3530 container_of(work, typeof(*dev_priv), mm.idle_work.work); 3531 struct drm_device *dev = dev_priv->dev; 3532 struct intel_engine_cs *ring; 3533 int i; 3534 3535 for_each_ring(ring, dev_priv, i) 3536 if (!list_empty(&ring->request_list)) 3537 return; 3538 3539 intel_mark_idle(dev); 3540 3541 if (mutex_trylock(&dev->struct_mutex)) { 3542 struct intel_engine_cs *ring; 3543 int i; 3544 3545 for_each_ring(ring, dev_priv, i) 3546 i915_gem_batch_pool_fini(&ring->batch_pool); 3547 3548 mutex_unlock(&dev->struct_mutex); 3549 } 3550 } 3551 3552 /** 3553 * Ensures that an object will eventually get non-busy by flushing any required 3554 * write domains, emitting any outstanding lazy request and retiring and 3555 * completed requests. 3556 */ 3557 static int 3558 i915_gem_object_flush_active(struct drm_i915_gem_object *obj) 3559 { 3560 int i; 3561 3562 if (!obj->active) 3563 return 0; 3564 3565 for (i = 0; i < I915_NUM_RINGS; i++) { 3566 struct drm_i915_gem_request *req; 3567 3568 req = obj->last_read_req[i]; 3569 if (req == NULL) 3570 continue; 3571 3572 if (list_empty(&req->list)) 3573 goto retire; 3574 3575 if (i915_gem_request_completed(req, true)) { 3576 __i915_gem_request_retire__upto(req); 3577 retire: 3578 i915_gem_object_retire__read(obj, i); 3579 } 3580 } 3581 3582 return 0; 3583 } 3584 3585 /** 3586 * i915_gem_wait_ioctl - implements DRM_IOCTL_I915_GEM_WAIT 3587 * @DRM_IOCTL_ARGS: standard ioctl arguments 3588 * 3589 * Returns 0 if successful, else an error is returned with the remaining time in 3590 * the timeout parameter. 3591 * -ETIME: object is still busy after timeout 3592 * -ERESTARTSYS: signal interrupted the wait 3593 * -ENONENT: object doesn't exist 3594 * Also possible, but rare: 3595 * -EAGAIN: GPU wedged 3596 * -ENOMEM: damn 3597 * -ENODEV: Internal IRQ fail 3598 * -E?: The add request failed 3599 * 3600 * The wait ioctl with a timeout of 0 reimplements the busy ioctl. With any 3601 * non-zero timeout parameter the wait ioctl will wait for the given number of 3602 * nanoseconds on an object becoming unbusy. Since the wait itself does so 3603 * without holding struct_mutex the object may become re-busied before this 3604 * function completes. A similar but shorter * race condition exists in the busy 3605 * ioctl 3606 */ 3607 int 3608 i915_gem_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *file) 3609 { 3610 struct drm_i915_private *dev_priv = dev->dev_private; 3611 struct drm_i915_gem_wait *args = data; 3612 struct drm_gem_object *gobj; 3613 struct drm_i915_gem_object *obj; 3614 struct drm_i915_gem_request *req[I915_NUM_RINGS]; 3615 unsigned reset_counter; 3616 int i, n = 0; 3617 int ret; 3618 3619 if (args->flags != 0) 3620 return -EINVAL; 3621 3622 ret = i915_mutex_lock_interruptible(dev); 3623 if (ret) 3624 return ret; 3625 3626 gobj = drm_gem_object_lookup(dev, file, args->bo_handle); 3627 if (gobj == NULL) { 3628 mutex_unlock(&dev->struct_mutex); 3629 return -ENOENT; 3630 } 3631 obj = to_intel_bo(gobj); 3632 3633 /* Need to make sure the object gets inactive eventually. */ 3634 ret = i915_gem_object_flush_active(obj); 3635 if (ret) 3636 goto out; 3637 3638 if (!obj->active) 3639 goto out; 3640 3641 /* Do this after OLR check to make sure we make forward progress polling 3642 * on this IOCTL with a timeout == 0 (like busy ioctl) 3643 */ 3644 if (args->timeout_ns == 0) { 3645 ret = -ETIME; 3646 goto out; 3647 } 3648 3649 drm_gem_object_unreference(&obj->base); 3650 reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter); 3651 3652 for (i = 0; i < I915_NUM_RINGS; i++) { 3653 if (obj->last_read_req[i] == NULL) 3654 continue; 3655 3656 req[n++] = i915_gem_request_reference(obj->last_read_req[i]); 3657 } 3658 3659 mutex_unlock(&dev->struct_mutex); 3660 3661 for (i = 0; i < n; i++) { 3662 if (ret == 0) 3663 ret = __i915_wait_request(req[i], reset_counter, true, 3664 args->timeout_ns > 0 ? &args->timeout_ns : NULL, 3665 file->driver_priv); 3666 i915_gem_request_unreference__unlocked(req[i]); 3667 } 3668 return ret; 3669 3670 out: 3671 drm_gem_object_unreference(&obj->base); 3672 mutex_unlock(&dev->struct_mutex); 3673 return ret; 3674 } 3675 3676 static int 3677 __i915_gem_object_sync(struct drm_i915_gem_object *obj, 3678 struct intel_engine_cs *to, 3679 struct drm_i915_gem_request *from_req, 3680 struct drm_i915_gem_request **to_req) 3681 { 3682 struct intel_engine_cs *from; 3683 int ret; 3684 3685 from = i915_gem_request_get_ring(from_req); 3686 if (to == from) 3687 return 0; 3688 3689 if (i915_gem_request_completed(from_req, true)) 3690 return 0; 3691 3692 if (!i915_semaphore_is_enabled(obj->base.dev)) { 3693 struct drm_i915_private *i915 = to_i915(obj->base.dev); 3694 ret = __i915_wait_request(from_req, 3695 atomic_read(&i915->gpu_error.reset_counter), 3696 i915->mm.interruptible, 3697 NULL, 3698 &i915->rps.semaphores); 3699 if (ret) 3700 return ret; 3701 3702 i915_gem_object_retire_request(obj, from_req); 3703 } else { 3704 int idx = intel_ring_sync_index(from, to); 3705 u32 seqno = i915_gem_request_get_seqno(from_req); 3706 3707 WARN_ON(!to_req); 3708 3709 if (seqno <= from->semaphore.sync_seqno[idx]) 3710 return 0; 3711 3712 if (*to_req == NULL) { 3713 ret = i915_gem_request_alloc(to, to->default_context, to_req); 3714 if (ret) 3715 return ret; 3716 } 3717 3718 trace_i915_gem_ring_sync_to(*to_req, from, from_req); 3719 ret = to->semaphore.sync_to(*to_req, from, seqno); 3720 if (ret) 3721 return ret; 3722 3723 /* We use last_read_req because sync_to() 3724 * might have just caused seqno wrap under 3725 * the radar. 3726 */ 3727 from->semaphore.sync_seqno[idx] = 3728 i915_gem_request_get_seqno(obj->last_read_req[from->id]); 3729 } 3730 3731 return 0; 3732 } 3733 3734 /** 3735 * i915_gem_object_sync - sync an object to a ring. 3736 * 3737 * @obj: object which may be in use on another ring. 3738 * @to: ring we wish to use the object on. May be NULL. 3739 * @to_req: request we wish to use the object for. See below. 3740 * This will be allocated and returned if a request is 3741 * required but not passed in. 3742 * 3743 * This code is meant to abstract object synchronization with the GPU. 3744 * Calling with NULL implies synchronizing the object with the CPU 3745 * rather than a particular GPU ring. Conceptually we serialise writes 3746 * between engines inside the GPU. We only allow one engine to write 3747 * into a buffer at any time, but multiple readers. To ensure each has 3748 * a coherent view of memory, we must: 3749 * 3750 * - If there is an outstanding write request to the object, the new 3751 * request must wait for it to complete (either CPU or in hw, requests 3752 * on the same ring will be naturally ordered). 3753 * 3754 * - If we are a write request (pending_write_domain is set), the new 3755 * request must wait for outstanding read requests to complete. 3756 * 3757 * For CPU synchronisation (NULL to) no request is required. For syncing with 3758 * rings to_req must be non-NULL. However, a request does not have to be 3759 * pre-allocated. If *to_req is NULL and sync commands will be emitted then a 3760 * request will be allocated automatically and returned through *to_req. Note 3761 * that it is not guaranteed that commands will be emitted (because the system 3762 * might already be idle). Hence there is no need to create a request that 3763 * might never have any work submitted. Note further that if a request is 3764 * returned in *to_req, it is the responsibility of the caller to submit 3765 * that request (after potentially adding more work to it). 3766 * 3767 * Returns 0 if successful, else propagates up the lower layer error. 3768 */ 3769 int 3770 i915_gem_object_sync(struct drm_i915_gem_object *obj, 3771 struct intel_engine_cs *to, 3772 struct drm_i915_gem_request **to_req) 3773 { 3774 const bool readonly = obj->base.pending_write_domain == 0; 3775 struct drm_i915_gem_request *req[I915_NUM_RINGS]; 3776 int ret, i, n; 3777 3778 if (!obj->active) 3779 return 0; 3780 3781 if (to == NULL) 3782 return i915_gem_object_wait_rendering(obj, readonly); 3783 3784 n = 0; 3785 if (readonly) { 3786 if (obj->last_write_req) 3787 req[n++] = obj->last_write_req; 3788 } else { 3789 for (i = 0; i < I915_NUM_RINGS; i++) 3790 if (obj->last_read_req[i]) 3791 req[n++] = obj->last_read_req[i]; 3792 } 3793 for (i = 0; i < n; i++) { 3794 ret = __i915_gem_object_sync(obj, to, req[i], to_req); 3795 if (ret) 3796 return ret; 3797 } 3798 3799 return 0; 3800 } 3801 3802 static void i915_gem_object_finish_gtt(struct drm_i915_gem_object *obj) 3803 { 3804 u32 old_write_domain, old_read_domains; 3805 3806 /* Force a pagefault for domain tracking on next user access */ 3807 i915_gem_release_mmap(obj); 3808 3809 if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0) 3810 return; 3811 3812 /* Wait for any direct GTT access to complete */ 3813 mb(); 3814 3815 old_read_domains = obj->base.read_domains; 3816 old_write_domain = obj->base.write_domain; 3817 3818 obj->base.read_domains &= ~I915_GEM_DOMAIN_GTT; 3819 obj->base.write_domain &= ~I915_GEM_DOMAIN_GTT; 3820 3821 trace_i915_gem_object_change_domain(obj, 3822 old_read_domains, 3823 old_write_domain); 3824 } 3825 3826 static int __i915_vma_unbind(struct i915_vma *vma, bool wait) 3827 { 3828 struct drm_i915_gem_object *obj = vma->obj; 3829 struct drm_i915_private *dev_priv = obj->base.dev->dev_private; 3830 int ret; 3831 3832 if (list_empty(&vma->vma_link)) 3833 return 0; 3834 3835 if (!drm_mm_node_allocated(&vma->node)) { 3836 i915_gem_vma_destroy(vma); 3837 return 0; 3838 } 3839 3840 if (vma->pin_count) 3841 return -EBUSY; 3842 3843 BUG_ON(obj->pages == NULL); 3844 3845 if (wait) { 3846 ret = i915_gem_object_wait_rendering(obj, false); 3847 if (ret) 3848 return ret; 3849 } 3850 3851 if (i915_is_ggtt(vma->vm) && 3852 vma->ggtt_view.type == I915_GGTT_VIEW_NORMAL) { 3853 i915_gem_object_finish_gtt(obj); 3854 3855 /* release the fence reg _after_ flushing */ 3856 ret = i915_gem_object_put_fence(obj); 3857 if (ret) 3858 return ret; 3859 } 3860 3861 trace_i915_vma_unbind(vma); 3862 3863 vma->vm->unbind_vma(vma); 3864 vma->bound = 0; 3865 3866 list_del_init(&vma->mm_list); 3867 if (i915_is_ggtt(vma->vm)) { 3868 if (vma->ggtt_view.type == I915_GGTT_VIEW_NORMAL) { 3869 obj->map_and_fenceable = false; 3870 } else if (vma->ggtt_view.pages) { 3871 #ifdef __NetBSD__ 3872 panic("rotated/partial views can't happen"); 3873 #else 3874 sg_free_table(vma->ggtt_view.pages); 3875 kfree(vma->ggtt_view.pages); 3876 #endif 3877 } 3878 vma->ggtt_view.pages = NULL; 3879 } 3880 3881 drm_mm_remove_node(&vma->node); 3882 i915_gem_vma_destroy(vma); 3883 3884 /* Since the unbound list is global, only move to that list if 3885 * no more VMAs exist. */ 3886 if (list_empty(&obj->vma_list)) 3887 list_move_tail(&obj->global_list, &dev_priv->mm.unbound_list); 3888 3889 /* And finally now the object is completely decoupled from this vma, 3890 * we can drop its hold on the backing storage and allow it to be 3891 * reaped by the shrinker. 3892 */ 3893 i915_gem_object_unpin_pages(obj); 3894 3895 return 0; 3896 } 3897 3898 int i915_vma_unbind(struct i915_vma *vma) 3899 { 3900 return __i915_vma_unbind(vma, true); 3901 } 3902 3903 int __i915_vma_unbind_no_wait(struct i915_vma *vma) 3904 { 3905 return __i915_vma_unbind(vma, false); 3906 } 3907 3908 int i915_gpu_idle(struct drm_device *dev) 3909 { 3910 struct drm_i915_private *dev_priv = dev->dev_private; 3911 struct intel_engine_cs *ring; 3912 int ret, i; 3913 3914 /* Flush everything onto the inactive list. */ 3915 for_each_ring(ring, dev_priv, i) { 3916 if (!i915.enable_execlists) { 3917 struct drm_i915_gem_request *req; 3918 3919 ret = i915_gem_request_alloc(ring, ring->default_context, &req); 3920 if (ret) 3921 return ret; 3922 3923 ret = i915_switch_context(req); 3924 if (ret) { 3925 i915_gem_request_cancel(req); 3926 return ret; 3927 } 3928 3929 i915_add_request_no_flush(req); 3930 } 3931 3932 ret = intel_ring_idle(ring); 3933 if (ret) 3934 return ret; 3935 } 3936 3937 WARN_ON(i915_verify_lists(dev)); 3938 return 0; 3939 } 3940 3941 static bool i915_gem_valid_gtt_space(struct i915_vma *vma, 3942 unsigned long cache_level) 3943 { 3944 struct drm_mm_node *gtt_space = &vma->node; 3945 struct drm_mm_node *other; 3946 3947 /* 3948 * On some machines we have to be careful when putting differing types 3949 * of snoopable memory together to avoid the prefetcher crossing memory 3950 * domains and dying. During vm initialisation, we decide whether or not 3951 * these constraints apply and set the drm_mm.color_adjust 3952 * appropriately. 3953 */ 3954 if (vma->vm->mm.color_adjust == NULL) 3955 return true; 3956 3957 if (!drm_mm_node_allocated(gtt_space)) 3958 return true; 3959 3960 if (list_empty(>t_space->node_list)) 3961 return true; 3962 3963 other = list_entry(gtt_space->node_list.prev, struct drm_mm_node, node_list); 3964 if (other->allocated && !other->hole_follows && other->color != cache_level) 3965 return false; 3966 3967 other = list_entry(gtt_space->node_list.next, struct drm_mm_node, node_list); 3968 if (other->allocated && !gtt_space->hole_follows && other->color != cache_level) 3969 return false; 3970 3971 return true; 3972 } 3973 3974 /** 3975 * Finds free space in the GTT aperture and binds the object or a view of it 3976 * there. 3977 */ 3978 static struct i915_vma * 3979 i915_gem_object_bind_to_vm(struct drm_i915_gem_object *obj, 3980 struct i915_address_space *vm, 3981 const struct i915_ggtt_view *ggtt_view, 3982 unsigned alignment, 3983 uint64_t flags) 3984 { 3985 struct drm_device *dev = obj->base.dev; 3986 struct drm_i915_private *dev_priv = dev->dev_private; 3987 u32 fence_alignment, unfenced_alignment; 3988 u32 search_flag, alloc_flag; 3989 u64 start, end; 3990 u64 size, fence_size; 3991 struct i915_vma *vma; 3992 int ret; 3993 3994 if (i915_is_ggtt(vm)) { 3995 u32 view_size; 3996 3997 if (WARN_ON(!ggtt_view)) 3998 return ERR_PTR(-EINVAL); 3999 4000 view_size = i915_ggtt_view_size(obj, ggtt_view); 4001 4002 fence_size = i915_gem_get_gtt_size(dev, 4003 view_size, 4004 obj->tiling_mode); 4005 fence_alignment = i915_gem_get_gtt_alignment(dev, 4006 view_size, 4007 obj->tiling_mode, 4008 true); 4009 unfenced_alignment = i915_gem_get_gtt_alignment(dev, 4010 view_size, 4011 obj->tiling_mode, 4012 false); 4013 size = flags & PIN_MAPPABLE ? fence_size : view_size; 4014 } else { 4015 fence_size = i915_gem_get_gtt_size(dev, 4016 obj->base.size, 4017 obj->tiling_mode); 4018 fence_alignment = i915_gem_get_gtt_alignment(dev, 4019 obj->base.size, 4020 obj->tiling_mode, 4021 true); 4022 unfenced_alignment = 4023 i915_gem_get_gtt_alignment(dev, 4024 obj->base.size, 4025 obj->tiling_mode, 4026 false); 4027 size = flags & PIN_MAPPABLE ? fence_size : obj->base.size; 4028 } 4029 4030 start = flags & PIN_OFFSET_BIAS ? flags & PIN_OFFSET_MASK : 0; 4031 end = vm->total; 4032 if (flags & PIN_MAPPABLE) 4033 end = min_t(u64, end, dev_priv->gtt.mappable_end); 4034 if (flags & PIN_ZONE_4G) 4035 end = min_t(u64, end, (1ULL << 32)); 4036 4037 if (alignment == 0) 4038 alignment = flags & PIN_MAPPABLE ? fence_alignment : 4039 unfenced_alignment; 4040 if (flags & PIN_MAPPABLE && alignment & (fence_alignment - 1)) { 4041 DRM_DEBUG("Invalid object (view type=%u) alignment requested %u\n", 4042 ggtt_view ? ggtt_view->type : 0, 4043 alignment); 4044 return ERR_PTR(-EINVAL); 4045 } 4046 4047 /* If binding the object/GGTT view requires more space than the entire 4048 * aperture has, reject it early before evicting everything in a vain 4049 * attempt to find space. 4050 */ 4051 if (size > end) { 4052 DRM_DEBUG("Attempting to bind an object (view type=%u) larger than the aperture: size=%"PRIx64" > %s aperture=%"PRIx64"\n", 4053 ggtt_view ? ggtt_view->type : 0, 4054 size, 4055 flags & PIN_MAPPABLE ? "mappable" : "total", 4056 end); 4057 return ERR_PTR(-E2BIG); 4058 } 4059 4060 ret = i915_gem_object_get_pages(obj); 4061 if (ret) 4062 return ERR_PTR(ret); 4063 4064 i915_gem_object_pin_pages(obj); 4065 4066 vma = ggtt_view ? i915_gem_obj_lookup_or_create_ggtt_vma(obj, ggtt_view) : 4067 i915_gem_obj_lookup_or_create_vma(obj, vm); 4068 4069 if (IS_ERR(vma)) 4070 goto err_unpin; 4071 4072 if (flags & PIN_HIGH) { 4073 search_flag = DRM_MM_SEARCH_BELOW; 4074 alloc_flag = DRM_MM_CREATE_TOP; 4075 } else { 4076 search_flag = DRM_MM_SEARCH_DEFAULT; 4077 alloc_flag = DRM_MM_CREATE_DEFAULT; 4078 } 4079 4080 search_free: 4081 ret = drm_mm_insert_node_in_range_generic(&vm->mm, &vma->node, 4082 size, alignment, 4083 obj->cache_level, 4084 start, end, 4085 search_flag, 4086 alloc_flag); 4087 if (ret) { 4088 ret = i915_gem_evict_something(dev, vm, size, alignment, 4089 obj->cache_level, 4090 start, end, 4091 flags); 4092 if (ret == 0) 4093 goto search_free; 4094 4095 goto err_free_vma; 4096 } 4097 if (WARN_ON(!i915_gem_valid_gtt_space(vma, obj->cache_level))) { 4098 ret = -EINVAL; 4099 goto err_remove_node; 4100 } 4101 4102 trace_i915_vma_bind(vma, flags); 4103 ret = i915_vma_bind(vma, obj->cache_level, flags); 4104 if (ret) 4105 goto err_remove_node; 4106 4107 list_move_tail(&obj->global_list, &dev_priv->mm.bound_list); 4108 list_add_tail(&vma->mm_list, &vm->inactive_list); 4109 4110 return vma; 4111 4112 err_remove_node: 4113 drm_mm_remove_node(&vma->node); 4114 err_free_vma: 4115 i915_gem_vma_destroy(vma); 4116 vma = ERR_PTR(ret); 4117 err_unpin: 4118 i915_gem_object_unpin_pages(obj); 4119 return vma; 4120 } 4121 4122 bool 4123 i915_gem_clflush_object(struct drm_i915_gem_object *obj, 4124 bool force) 4125 { 4126 /* If we don't have a page list set up, then we're not pinned 4127 * to GPU, and we can ignore the cache flush because it'll happen 4128 * again at bind time. 4129 */ 4130 if (obj->pages == NULL) 4131 return false; 4132 4133 /* 4134 * Stolen memory is always coherent with the GPU as it is explicitly 4135 * marked as wc by the system, or the system is cache-coherent. 4136 */ 4137 if (obj->stolen || obj->phys_handle) 4138 return false; 4139 4140 /* If the GPU is snooping the contents of the CPU cache, 4141 * we do not need to manually clear the CPU cache lines. However, 4142 * the caches are only snooped when the render cache is 4143 * flushed/invalidated. As we always have to emit invalidations 4144 * and flushes when moving into and out of the RENDER domain, correct 4145 * snooping behaviour occurs naturally as the result of our domain 4146 * tracking. 4147 */ 4148 if (!force && cpu_cache_is_coherent(obj->base.dev, obj->cache_level)) { 4149 obj->cache_dirty = true; 4150 return false; 4151 } 4152 4153 trace_i915_gem_object_clflush(obj); 4154 #ifdef __NetBSD__ 4155 drm_clflush_pglist(&obj->pageq); 4156 #else 4157 drm_clflush_sg(obj->pages); 4158 #endif 4159 obj->cache_dirty = false; 4160 4161 return true; 4162 } 4163 4164 /** Flushes the GTT write domain for the object if it's dirty. */ 4165 static void 4166 i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj) 4167 { 4168 uint32_t old_write_domain; 4169 4170 if (obj->base.write_domain != I915_GEM_DOMAIN_GTT) 4171 return; 4172 4173 /* No actual flushing is required for the GTT write domain. Writes 4174 * to it immediately go to main memory as far as we know, so there's 4175 * no chipset flush. It also doesn't land in render cache. 4176 * 4177 * However, we do have to enforce the order so that all writes through 4178 * the GTT land before any writes to the device, such as updates to 4179 * the GATT itself. 4180 */ 4181 wmb(); 4182 4183 old_write_domain = obj->base.write_domain; 4184 obj->base.write_domain = 0; 4185 4186 intel_fb_obj_flush(obj, false, ORIGIN_GTT); 4187 4188 trace_i915_gem_object_change_domain(obj, 4189 obj->base.read_domains, 4190 old_write_domain); 4191 } 4192 4193 /** Flushes the CPU write domain for the object if it's dirty. */ 4194 static void 4195 i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj) 4196 { 4197 uint32_t old_write_domain; 4198 4199 if (obj->base.write_domain != I915_GEM_DOMAIN_CPU) 4200 return; 4201 4202 if (i915_gem_clflush_object(obj, obj->pin_display)) 4203 i915_gem_chipset_flush(obj->base.dev); 4204 4205 old_write_domain = obj->base.write_domain; 4206 obj->base.write_domain = 0; 4207 4208 intel_fb_obj_flush(obj, false, ORIGIN_CPU); 4209 4210 trace_i915_gem_object_change_domain(obj, 4211 obj->base.read_domains, 4212 old_write_domain); 4213 } 4214 4215 /** 4216 * Moves a single object to the GTT read, and possibly write domain. 4217 * 4218 * This function returns when the move is complete, including waiting on 4219 * flushes to occur. 4220 */ 4221 int 4222 i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write) 4223 { 4224 uint32_t old_write_domain, old_read_domains; 4225 struct i915_vma *vma; 4226 int ret; 4227 4228 if (obj->base.write_domain == I915_GEM_DOMAIN_GTT) 4229 return 0; 4230 4231 ret = i915_gem_object_wait_rendering(obj, !write); 4232 if (ret) 4233 return ret; 4234 4235 /* Flush and acquire obj->pages so that we are coherent through 4236 * direct access in memory with previous cached writes through 4237 * shmemfs and that our cache domain tracking remains valid. 4238 * For example, if the obj->filp was moved to swap without us 4239 * being notified and releasing the pages, we would mistakenly 4240 * continue to assume that the obj remained out of the CPU cached 4241 * domain. 4242 */ 4243 ret = i915_gem_object_get_pages(obj); 4244 if (ret) 4245 return ret; 4246 4247 i915_gem_object_flush_cpu_write_domain(obj); 4248 4249 /* Serialise direct access to this object with the barriers for 4250 * coherent writes from the GPU, by effectively invalidating the 4251 * GTT domain upon first access. 4252 */ 4253 if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0) 4254 mb(); 4255 4256 old_write_domain = obj->base.write_domain; 4257 old_read_domains = obj->base.read_domains; 4258 4259 /* It should now be out of any other write domains, and we can update 4260 * the domain values for our changes. 4261 */ 4262 BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) != 0); 4263 obj->base.read_domains |= I915_GEM_DOMAIN_GTT; 4264 if (write) { 4265 obj->base.read_domains = I915_GEM_DOMAIN_GTT; 4266 obj->base.write_domain = I915_GEM_DOMAIN_GTT; 4267 obj->dirty = 1; 4268 } 4269 4270 trace_i915_gem_object_change_domain(obj, 4271 old_read_domains, 4272 old_write_domain); 4273 4274 /* And bump the LRU for this access */ 4275 vma = i915_gem_obj_to_ggtt(obj); 4276 if (vma && drm_mm_node_allocated(&vma->node) && !obj->active) 4277 list_move_tail(&vma->mm_list, 4278 &to_i915(obj->base.dev)->gtt.base.inactive_list); 4279 4280 return 0; 4281 } 4282 4283 /** 4284 * Changes the cache-level of an object across all VMA. 4285 * 4286 * After this function returns, the object will be in the new cache-level 4287 * across all GTT and the contents of the backing storage will be coherent, 4288 * with respect to the new cache-level. In order to keep the backing storage 4289 * coherent for all users, we only allow a single cache level to be set 4290 * globally on the object and prevent it from being changed whilst the 4291 * hardware is reading from the object. That is if the object is currently 4292 * on the scanout it will be set to uncached (or equivalent display 4293 * cache coherency) and all non-MOCS GPU access will also be uncached so 4294 * that all direct access to the scanout remains coherent. 4295 */ 4296 int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj, 4297 enum i915_cache_level cache_level) 4298 { 4299 struct drm_device *dev = obj->base.dev; 4300 struct i915_vma *vma, *next; 4301 bool bound = false; 4302 int ret = 0; 4303 4304 if (obj->cache_level == cache_level) 4305 goto out; 4306 4307 /* Inspect the list of currently bound VMA and unbind any that would 4308 * be invalid given the new cache-level. This is principally to 4309 * catch the issue of the CS prefetch crossing page boundaries and 4310 * reading an invalid PTE on older architectures. 4311 */ 4312 list_for_each_entry_safe(vma, next, &obj->vma_list, vma_link) { 4313 if (!drm_mm_node_allocated(&vma->node)) 4314 continue; 4315 4316 if (vma->pin_count) { 4317 DRM_DEBUG("can not change the cache level of pinned objects\n"); 4318 return -EBUSY; 4319 } 4320 4321 if (!i915_gem_valid_gtt_space(vma, cache_level)) { 4322 ret = i915_vma_unbind(vma); 4323 if (ret) 4324 return ret; 4325 } else 4326 bound = true; 4327 } 4328 4329 /* We can reuse the existing drm_mm nodes but need to change the 4330 * cache-level on the PTE. We could simply unbind them all and 4331 * rebind with the correct cache-level on next use. However since 4332 * we already have a valid slot, dma mapping, pages etc, we may as 4333 * rewrite the PTE in the belief that doing so tramples upon less 4334 * state and so involves less work. 4335 */ 4336 if (bound) { 4337 /* Before we change the PTE, the GPU must not be accessing it. 4338 * If we wait upon the object, we know that all the bound 4339 * VMA are no longer active. 4340 */ 4341 ret = i915_gem_object_wait_rendering(obj, false); 4342 if (ret) 4343 return ret; 4344 4345 if (!HAS_LLC(dev) && cache_level != I915_CACHE_NONE) { 4346 /* Access to snoopable pages through the GTT is 4347 * incoherent and on some machines causes a hard 4348 * lockup. Relinquish the CPU mmaping to force 4349 * userspace to refault in the pages and we can 4350 * then double check if the GTT mapping is still 4351 * valid for that pointer access. 4352 */ 4353 i915_gem_release_mmap(obj); 4354 4355 /* As we no longer need a fence for GTT access, 4356 * we can relinquish it now (and so prevent having 4357 * to steal a fence from someone else on the next 4358 * fence request). Note GPU activity would have 4359 * dropped the fence as all snoopable access is 4360 * supposed to be linear. 4361 */ 4362 ret = i915_gem_object_put_fence(obj); 4363 if (ret) 4364 return ret; 4365 } else { 4366 /* We either have incoherent backing store and 4367 * so no GTT access or the architecture is fully 4368 * coherent. In such cases, existing GTT mmaps 4369 * ignore the cache bit in the PTE and we can 4370 * rewrite it without confusing the GPU or having 4371 * to force userspace to fault back in its mmaps. 4372 */ 4373 } 4374 4375 list_for_each_entry(vma, &obj->vma_list, vma_link) { 4376 if (!drm_mm_node_allocated(&vma->node)) 4377 continue; 4378 4379 ret = i915_vma_bind(vma, cache_level, PIN_UPDATE); 4380 if (ret) 4381 return ret; 4382 } 4383 } 4384 4385 list_for_each_entry(vma, &obj->vma_list, vma_link) 4386 vma->node.color = cache_level; 4387 obj->cache_level = cache_level; 4388 4389 out: 4390 /* Flush the dirty CPU caches to the backing storage so that the 4391 * object is now coherent at its new cache level (with respect 4392 * to the access domain). 4393 */ 4394 if (obj->cache_dirty && 4395 obj->base.write_domain != I915_GEM_DOMAIN_CPU && 4396 cpu_write_needs_clflush(obj)) { 4397 if (i915_gem_clflush_object(obj, true)) 4398 i915_gem_chipset_flush(obj->base.dev); 4399 } 4400 4401 return 0; 4402 } 4403 4404 int i915_gem_get_caching_ioctl(struct drm_device *dev, void *data, 4405 struct drm_file *file) 4406 { 4407 struct drm_i915_gem_caching *args = data; 4408 struct drm_gem_object *gobj; 4409 struct drm_i915_gem_object *obj; 4410 4411 gobj = drm_gem_object_lookup(dev, file, args->handle); 4412 if (gobj == NULL) 4413 return -ENOENT; 4414 obj = to_intel_bo(gobj); 4415 4416 switch (obj->cache_level) { 4417 case I915_CACHE_LLC: 4418 case I915_CACHE_L3_LLC: 4419 args->caching = I915_CACHING_CACHED; 4420 break; 4421 4422 case I915_CACHE_WT: 4423 args->caching = I915_CACHING_DISPLAY; 4424 break; 4425 4426 default: 4427 args->caching = I915_CACHING_NONE; 4428 break; 4429 } 4430 4431 drm_gem_object_unreference_unlocked(&obj->base); 4432 return 0; 4433 } 4434 4435 int i915_gem_set_caching_ioctl(struct drm_device *dev, void *data, 4436 struct drm_file *file) 4437 { 4438 struct drm_i915_private *dev_priv = dev->dev_private; 4439 struct drm_i915_gem_caching *args = data; 4440 struct drm_gem_object *gobj; 4441 struct drm_i915_gem_object *obj; 4442 enum i915_cache_level level; 4443 int ret; 4444 4445 switch (args->caching) { 4446 case I915_CACHING_NONE: 4447 level = I915_CACHE_NONE; 4448 break; 4449 case I915_CACHING_CACHED: 4450 /* 4451 * Due to a HW issue on BXT A stepping, GPU stores via a 4452 * snooped mapping may leave stale data in a corresponding CPU 4453 * cacheline, whereas normally such cachelines would get 4454 * invalidated. 4455 */ 4456 if (IS_BROXTON(dev) && INTEL_REVID(dev) < BXT_REVID_B0) 4457 return -ENODEV; 4458 4459 level = I915_CACHE_LLC; 4460 break; 4461 case I915_CACHING_DISPLAY: 4462 level = HAS_WT(dev) ? I915_CACHE_WT : I915_CACHE_NONE; 4463 break; 4464 default: 4465 return -EINVAL; 4466 } 4467 4468 intel_runtime_pm_get(dev_priv); 4469 4470 ret = i915_mutex_lock_interruptible(dev); 4471 if (ret) 4472 goto rpm_put; 4473 4474 gobj = drm_gem_object_lookup(dev, file, args->handle); 4475 if (gobj == NULL) { 4476 ret = -ENOENT; 4477 goto unlock; 4478 } 4479 obj = to_intel_bo(gobj); 4480 4481 ret = i915_gem_object_set_cache_level(obj, level); 4482 4483 drm_gem_object_unreference(&obj->base); 4484 unlock: 4485 mutex_unlock(&dev->struct_mutex); 4486 rpm_put: 4487 intel_runtime_pm_put(dev_priv); 4488 4489 return ret; 4490 } 4491 4492 /* 4493 * Prepare buffer for display plane (scanout, cursors, etc). 4494 * Can be called from an uninterruptible phase (modesetting) and allows 4495 * any flushes to be pipelined (for pageflips). 4496 */ 4497 int 4498 i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj, 4499 u32 alignment, 4500 struct intel_engine_cs *pipelined, 4501 struct drm_i915_gem_request **pipelined_request, 4502 const struct i915_ggtt_view *view) 4503 { 4504 u32 old_read_domains, old_write_domain; 4505 int ret; 4506 4507 ret = i915_gem_object_sync(obj, pipelined, pipelined_request); 4508 if (ret) 4509 return ret; 4510 4511 /* Mark the pin_display early so that we account for the 4512 * display coherency whilst setting up the cache domains. 4513 */ 4514 obj->pin_display++; 4515 4516 /* The display engine is not coherent with the LLC cache on gen6. As 4517 * a result, we make sure that the pinning that is about to occur is 4518 * done with uncached PTEs. This is lowest common denominator for all 4519 * chipsets. 4520 * 4521 * However for gen6+, we could do better by using the GFDT bit instead 4522 * of uncaching, which would allow us to flush all the LLC-cached data 4523 * with that bit in the PTE to main memory with just one PIPE_CONTROL. 4524 */ 4525 ret = i915_gem_object_set_cache_level(obj, 4526 HAS_WT(obj->base.dev) ? I915_CACHE_WT : I915_CACHE_NONE); 4527 if (ret) 4528 goto err_unpin_display; 4529 4530 /* As the user may map the buffer once pinned in the display plane 4531 * (e.g. libkms for the bootup splash), we have to ensure that we 4532 * always use map_and_fenceable for all scanout buffers. 4533 */ 4534 ret = i915_gem_object_ggtt_pin(obj, view, alignment, 4535 view->type == I915_GGTT_VIEW_NORMAL ? 4536 PIN_MAPPABLE : 0); 4537 if (ret) 4538 goto err_unpin_display; 4539 4540 i915_gem_object_flush_cpu_write_domain(obj); 4541 4542 old_write_domain = obj->base.write_domain; 4543 old_read_domains = obj->base.read_domains; 4544 4545 /* It should now be out of any other write domains, and we can update 4546 * the domain values for our changes. 4547 */ 4548 obj->base.write_domain = 0; 4549 obj->base.read_domains |= I915_GEM_DOMAIN_GTT; 4550 4551 trace_i915_gem_object_change_domain(obj, 4552 old_read_domains, 4553 old_write_domain); 4554 4555 return 0; 4556 4557 err_unpin_display: 4558 obj->pin_display--; 4559 return ret; 4560 } 4561 4562 void 4563 i915_gem_object_unpin_from_display_plane(struct drm_i915_gem_object *obj, 4564 const struct i915_ggtt_view *view) 4565 { 4566 if (WARN_ON(obj->pin_display == 0)) 4567 return; 4568 4569 i915_gem_object_ggtt_unpin_view(obj, view); 4570 4571 obj->pin_display--; 4572 } 4573 4574 /** 4575 * Moves a single object to the CPU read, and possibly write domain. 4576 * 4577 * This function returns when the move is complete, including waiting on 4578 * flushes to occur. 4579 */ 4580 int 4581 i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write) 4582 { 4583 uint32_t old_write_domain, old_read_domains; 4584 int ret; 4585 4586 if (obj->base.write_domain == I915_GEM_DOMAIN_CPU) 4587 return 0; 4588 4589 ret = i915_gem_object_wait_rendering(obj, !write); 4590 if (ret) 4591 return ret; 4592 4593 i915_gem_object_flush_gtt_write_domain(obj); 4594 4595 old_write_domain = obj->base.write_domain; 4596 old_read_domains = obj->base.read_domains; 4597 4598 /* Flush the CPU cache if it's still invalid. */ 4599 if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0) { 4600 i915_gem_clflush_object(obj, false); 4601 4602 obj->base.read_domains |= I915_GEM_DOMAIN_CPU; 4603 } 4604 4605 /* It should now be out of any other write domains, and we can update 4606 * the domain values for our changes. 4607 */ 4608 BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) != 0); 4609 4610 /* If we're writing through the CPU, then the GPU read domains will 4611 * need to be invalidated at next use. 4612 */ 4613 if (write) { 4614 obj->base.read_domains = I915_GEM_DOMAIN_CPU; 4615 obj->base.write_domain = I915_GEM_DOMAIN_CPU; 4616 } 4617 4618 trace_i915_gem_object_change_domain(obj, 4619 old_read_domains, 4620 old_write_domain); 4621 4622 return 0; 4623 } 4624 4625 /* Throttle our rendering by waiting until the ring has completed our requests 4626 * emitted over 20 msec ago. 4627 * 4628 * Note that if we were to use the current jiffies each time around the loop, 4629 * we wouldn't escape the function with any frames outstanding if the time to 4630 * render a frame was over 20ms. 4631 * 4632 * This should get us reasonable parallelism between CPU and GPU but also 4633 * relatively low latency when blocking on a particular request to finish. 4634 */ 4635 static int 4636 i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file) 4637 { 4638 struct drm_i915_private *dev_priv = dev->dev_private; 4639 struct drm_i915_file_private *file_priv = file->driver_priv; 4640 unsigned long recent_enough = jiffies - DRM_I915_THROTTLE_JIFFIES; 4641 struct drm_i915_gem_request *request, *target = NULL; 4642 unsigned reset_counter; 4643 int ret; 4644 4645 ret = i915_gem_wait_for_error(&dev_priv->gpu_error); 4646 if (ret) 4647 return ret; 4648 4649 ret = i915_gem_check_wedge(&dev_priv->gpu_error, false); 4650 if (ret) 4651 return ret; 4652 4653 spin_lock(&file_priv->mm.lock); 4654 list_for_each_entry(request, &file_priv->mm.request_list, client_list) { 4655 if (time_after_eq(request->emitted_jiffies, recent_enough)) 4656 break; 4657 4658 /* 4659 * Note that the request might not have been submitted yet. 4660 * In which case emitted_jiffies will be zero. 4661 */ 4662 if (!request->emitted_jiffies) 4663 continue; 4664 4665 target = request; 4666 } 4667 reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter); 4668 if (target) 4669 i915_gem_request_reference(target); 4670 spin_unlock(&file_priv->mm.lock); 4671 4672 if (target == NULL) 4673 return 0; 4674 4675 ret = __i915_wait_request(target, reset_counter, true, NULL, NULL); 4676 if (ret == 0) 4677 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, 0); 4678 4679 i915_gem_request_unreference__unlocked(target); 4680 4681 return ret; 4682 } 4683 4684 static bool 4685 i915_vma_misplaced(struct i915_vma *vma, uint32_t alignment, uint64_t flags) 4686 { 4687 struct drm_i915_gem_object *obj = vma->obj; 4688 4689 if (alignment && 4690 vma->node.start & (alignment - 1)) 4691 return true; 4692 4693 if (flags & PIN_MAPPABLE && !obj->map_and_fenceable) 4694 return true; 4695 4696 if (flags & PIN_OFFSET_BIAS && 4697 vma->node.start < (flags & PIN_OFFSET_MASK)) 4698 return true; 4699 4700 return false; 4701 } 4702 4703 void __i915_vma_set_map_and_fenceable(struct i915_vma *vma) 4704 { 4705 struct drm_i915_gem_object *obj = vma->obj; 4706 bool mappable, fenceable; 4707 u32 fence_size, fence_alignment; 4708 4709 fence_size = i915_gem_get_gtt_size(obj->base.dev, 4710 obj->base.size, 4711 obj->tiling_mode); 4712 fence_alignment = i915_gem_get_gtt_alignment(obj->base.dev, 4713 obj->base.size, 4714 obj->tiling_mode, 4715 true); 4716 4717 fenceable = (vma->node.size == fence_size && 4718 (vma->node.start & (fence_alignment - 1)) == 0); 4719 4720 mappable = (vma->node.start + fence_size <= 4721 to_i915(obj->base.dev)->gtt.mappable_end); 4722 4723 obj->map_and_fenceable = mappable && fenceable; 4724 } 4725 4726 static int 4727 i915_gem_object_do_pin(struct drm_i915_gem_object *obj, 4728 struct i915_address_space *vm, 4729 const struct i915_ggtt_view *ggtt_view, 4730 uint32_t alignment, 4731 uint64_t flags) 4732 { 4733 struct drm_i915_private *dev_priv = obj->base.dev->dev_private; 4734 struct i915_vma *vma; 4735 unsigned bound; 4736 int ret; 4737 4738 if (WARN_ON(vm == &dev_priv->mm.aliasing_ppgtt->base)) 4739 return -ENODEV; 4740 4741 if (WARN_ON(flags & (PIN_GLOBAL | PIN_MAPPABLE) && !i915_is_ggtt(vm))) 4742 return -EINVAL; 4743 4744 if (WARN_ON((flags & (PIN_MAPPABLE | PIN_GLOBAL)) == PIN_MAPPABLE)) 4745 return -EINVAL; 4746 4747 if (WARN_ON(i915_is_ggtt(vm) != !!ggtt_view)) 4748 return -EINVAL; 4749 4750 vma = ggtt_view ? i915_gem_obj_to_ggtt_view(obj, ggtt_view) : 4751 i915_gem_obj_to_vma(obj, vm); 4752 4753 if (IS_ERR(vma)) 4754 return PTR_ERR(vma); 4755 4756 if (vma) { 4757 if (WARN_ON(vma->pin_count == DRM_I915_GEM_OBJECT_MAX_PIN_COUNT)) 4758 return -EBUSY; 4759 4760 if (i915_vma_misplaced(vma, alignment, flags)) { 4761 WARN(vma->pin_count, 4762 "bo is already pinned in %s with incorrect alignment:" 4763 " offset=%08x %08x, req.alignment=%x, req.map_and_fenceable=%d," 4764 " obj->map_and_fenceable=%d\n", 4765 ggtt_view ? "ggtt" : "ppgtt", 4766 upper_32_bits(vma->node.start), 4767 lower_32_bits(vma->node.start), 4768 alignment, 4769 !!(flags & PIN_MAPPABLE), 4770 obj->map_and_fenceable); 4771 ret = i915_vma_unbind(vma); 4772 if (ret) 4773 return ret; 4774 4775 vma = NULL; 4776 } 4777 } 4778 4779 bound = vma ? vma->bound : 0; 4780 if (vma == NULL || !drm_mm_node_allocated(&vma->node)) { 4781 vma = i915_gem_object_bind_to_vm(obj, vm, ggtt_view, alignment, 4782 flags); 4783 if (IS_ERR(vma)) 4784 return PTR_ERR(vma); 4785 } else { 4786 ret = i915_vma_bind(vma, obj->cache_level, flags); 4787 if (ret) 4788 return ret; 4789 } 4790 4791 if (ggtt_view && ggtt_view->type == I915_GGTT_VIEW_NORMAL && 4792 (bound ^ vma->bound) & GLOBAL_BIND) { 4793 __i915_vma_set_map_and_fenceable(vma); 4794 WARN_ON(flags & PIN_MAPPABLE && !obj->map_and_fenceable); 4795 } 4796 4797 vma->pin_count++; 4798 return 0; 4799 } 4800 4801 int 4802 i915_gem_object_pin(struct drm_i915_gem_object *obj, 4803 struct i915_address_space *vm, 4804 uint32_t alignment, 4805 uint64_t flags) 4806 { 4807 return i915_gem_object_do_pin(obj, vm, 4808 i915_is_ggtt(vm) ? &i915_ggtt_view_normal : NULL, 4809 alignment, flags); 4810 } 4811 4812 int 4813 i915_gem_object_ggtt_pin(struct drm_i915_gem_object *obj, 4814 const struct i915_ggtt_view *view, 4815 uint32_t alignment, 4816 uint64_t flags) 4817 { 4818 if (WARN_ONCE(!view, "no view specified")) 4819 return -EINVAL; 4820 4821 return i915_gem_object_do_pin(obj, i915_obj_to_ggtt(obj), view, 4822 alignment, flags | PIN_GLOBAL); 4823 } 4824 4825 void 4826 i915_gem_object_ggtt_unpin_view(struct drm_i915_gem_object *obj, 4827 const struct i915_ggtt_view *view) 4828 { 4829 struct i915_vma *vma = i915_gem_obj_to_ggtt_view(obj, view); 4830 4831 BUG_ON(!vma); 4832 WARN_ON(vma->pin_count == 0); 4833 WARN_ON(!i915_gem_obj_ggtt_bound_view(obj, view)); 4834 4835 --vma->pin_count; 4836 } 4837 4838 int 4839 i915_gem_busy_ioctl(struct drm_device *dev, void *data, 4840 struct drm_file *file) 4841 { 4842 struct drm_i915_gem_busy *args = data; 4843 struct drm_gem_object *gobj; 4844 struct drm_i915_gem_object *obj; 4845 int ret; 4846 4847 ret = i915_mutex_lock_interruptible(dev); 4848 if (ret) 4849 return ret; 4850 4851 gobj = drm_gem_object_lookup(dev, file, args->handle); 4852 if (gobj == NULL) { 4853 ret = -ENOENT; 4854 goto unlock; 4855 } 4856 obj = to_intel_bo(gobj); 4857 4858 /* Count all active objects as busy, even if they are currently not used 4859 * by the gpu. Users of this interface expect objects to eventually 4860 * become non-busy without any further actions, therefore emit any 4861 * necessary flushes here. 4862 */ 4863 ret = i915_gem_object_flush_active(obj); 4864 if (ret) 4865 goto unref; 4866 4867 BUILD_BUG_ON(I915_NUM_RINGS > 16); 4868 args->busy = obj->active << 16; 4869 if (obj->last_write_req) 4870 args->busy |= obj->last_write_req->ring->id; 4871 4872 unref: 4873 drm_gem_object_unreference(&obj->base); 4874 unlock: 4875 mutex_unlock(&dev->struct_mutex); 4876 return ret; 4877 } 4878 4879 int 4880 i915_gem_throttle_ioctl(struct drm_device *dev, void *data, 4881 struct drm_file *file_priv) 4882 { 4883 return i915_gem_ring_throttle(dev, file_priv); 4884 } 4885 4886 int 4887 i915_gem_madvise_ioctl(struct drm_device *dev, void *data, 4888 struct drm_file *file_priv) 4889 { 4890 struct drm_i915_private *dev_priv = dev->dev_private; 4891 struct drm_i915_gem_madvise *args = data; 4892 struct drm_gem_object *gobj; 4893 struct drm_i915_gem_object *obj; 4894 int ret; 4895 4896 switch (args->madv) { 4897 case I915_MADV_DONTNEED: 4898 case I915_MADV_WILLNEED: 4899 break; 4900 default: 4901 return -EINVAL; 4902 } 4903 4904 ret = i915_mutex_lock_interruptible(dev); 4905 if (ret) 4906 return ret; 4907 4908 gobj = drm_gem_object_lookup(dev, file_priv, args->handle); 4909 if (gobj == NULL) { 4910 ret = -ENOENT; 4911 goto unlock; 4912 } 4913 obj = to_intel_bo(gobj); 4914 4915 if (i915_gem_obj_is_pinned(obj)) { 4916 ret = -EINVAL; 4917 goto out; 4918 } 4919 4920 if (obj->pages && 4921 obj->tiling_mode != I915_TILING_NONE && 4922 dev_priv->quirks & QUIRK_PIN_SWIZZLED_PAGES) { 4923 if (obj->madv == I915_MADV_WILLNEED) 4924 i915_gem_object_unpin_pages(obj); 4925 if (args->madv == I915_MADV_WILLNEED) 4926 i915_gem_object_pin_pages(obj); 4927 } 4928 4929 if (obj->madv != __I915_MADV_PURGED) 4930 obj->madv = args->madv; 4931 4932 /* if the object is no longer attached, discard its backing storage */ 4933 if (obj->madv == I915_MADV_DONTNEED && obj->pages == NULL) 4934 i915_gem_object_truncate(obj); 4935 4936 args->retained = obj->madv != __I915_MADV_PURGED; 4937 4938 out: 4939 drm_gem_object_unreference(&obj->base); 4940 unlock: 4941 mutex_unlock(&dev->struct_mutex); 4942 return ret; 4943 } 4944 4945 void i915_gem_object_init(struct drm_i915_gem_object *obj, 4946 const struct drm_i915_gem_object_ops *ops) 4947 { 4948 int i; 4949 4950 INIT_LIST_HEAD(&obj->global_list); 4951 for (i = 0; i < I915_NUM_RINGS; i++) 4952 INIT_LIST_HEAD(&obj->ring_list[i]); 4953 INIT_LIST_HEAD(&obj->obj_exec_link); 4954 INIT_LIST_HEAD(&obj->vma_list); 4955 INIT_LIST_HEAD(&obj->batch_pool_link); 4956 4957 obj->ops = ops; 4958 4959 obj->fence_reg = I915_FENCE_REG_NONE; 4960 obj->madv = I915_MADV_WILLNEED; 4961 4962 i915_gem_info_add_obj(obj->base.dev->dev_private, obj->base.size); 4963 } 4964 4965 static const struct drm_i915_gem_object_ops i915_gem_object_ops = { 4966 .get_pages = i915_gem_object_get_pages_gtt, 4967 .put_pages = i915_gem_object_put_pages_gtt, 4968 }; 4969 4970 struct drm_i915_gem_object *i915_gem_alloc_object(struct drm_device *dev, 4971 size_t size) 4972 { 4973 #ifdef __NetBSD__ 4974 struct drm_i915_private *const dev_priv = dev->dev_private; 4975 #endif 4976 struct drm_i915_gem_object *obj; 4977 #ifndef __NetBSD__ 4978 struct address_space *mapping; 4979 gfp_t mask; 4980 #endif 4981 4982 obj = i915_gem_object_alloc(dev); 4983 if (obj == NULL) 4984 return NULL; 4985 4986 if (drm_gem_object_init(dev, &obj->base, size) != 0) { 4987 i915_gem_object_free(obj); 4988 return NULL; 4989 } 4990 4991 #ifdef __NetBSD__ 4992 uao_set_pgfl(obj->base.filp, dev_priv->gtt.pgfl); 4993 #else 4994 mask = GFP_HIGHUSER | __GFP_RECLAIMABLE; 4995 if (IS_CRESTLINE(dev) || IS_BROADWATER(dev)) { 4996 /* 965gm cannot relocate objects above 4GiB. */ 4997 mask &= ~__GFP_HIGHMEM; 4998 mask |= __GFP_DMA32; 4999 } 5000 5001 mapping = file_inode(obj->base.filp)->i_mapping; 5002 mapping_set_gfp_mask(mapping, mask); 5003 #endif 5004 5005 i915_gem_object_init(obj, &i915_gem_object_ops); 5006 5007 obj->base.write_domain = I915_GEM_DOMAIN_CPU; 5008 obj->base.read_domains = I915_GEM_DOMAIN_CPU; 5009 5010 if (HAS_LLC(dev)) { 5011 /* On some devices, we can have the GPU use the LLC (the CPU 5012 * cache) for about a 10% performance improvement 5013 * compared to uncached. Graphics requests other than 5014 * display scanout are coherent with the CPU in 5015 * accessing this cache. This means in this mode we 5016 * don't need to clflush on the CPU side, and on the 5017 * GPU side we only need to flush internal caches to 5018 * get data visible to the CPU. 5019 * 5020 * However, we maintain the display planes as UC, and so 5021 * need to rebind when first used as such. 5022 */ 5023 obj->cache_level = I915_CACHE_LLC; 5024 } else 5025 obj->cache_level = I915_CACHE_NONE; 5026 5027 trace_i915_gem_object_create(obj); 5028 5029 return obj; 5030 } 5031 5032 static bool discard_backing_storage(struct drm_i915_gem_object *obj) 5033 { 5034 /* If we are the last user of the backing storage (be it shmemfs 5035 * pages or stolen etc), we know that the pages are going to be 5036 * immediately released. In this case, we can then skip copying 5037 * back the contents from the GPU. 5038 */ 5039 5040 if (obj->madv != I915_MADV_WILLNEED) 5041 return false; 5042 5043 if (obj->base.filp == NULL) 5044 return true; 5045 5046 /* At first glance, this looks racy, but then again so would be 5047 * userspace racing mmap against close. However, the first external 5048 * reference to the filp can only be obtained through the 5049 * i915_gem_mmap_ioctl() which safeguards us against the user 5050 * acquiring such a reference whilst we are in the middle of 5051 * freeing the object. 5052 */ 5053 #ifdef __NetBSD__ 5054 /* XXX This number might be a fencepost. */ 5055 return obj->base.filp->uo_refs == 1; 5056 #else 5057 return atomic_long_read(&obj->base.filp->f_count) == 1; 5058 #endif 5059 } 5060 5061 void i915_gem_free_object(struct drm_gem_object *gem_obj) 5062 { 5063 struct drm_i915_gem_object *obj = to_intel_bo(gem_obj); 5064 struct drm_device *dev = obj->base.dev; 5065 struct drm_i915_private *dev_priv = dev->dev_private; 5066 struct i915_vma *vma, *next; 5067 5068 intel_runtime_pm_get(dev_priv); 5069 5070 trace_i915_gem_object_destroy(obj); 5071 5072 list_for_each_entry_safe(vma, next, &obj->vma_list, vma_link) { 5073 int ret; 5074 5075 vma->pin_count = 0; 5076 ret = i915_vma_unbind(vma); 5077 if (WARN_ON(ret == -ERESTARTSYS)) { 5078 bool was_interruptible; 5079 5080 was_interruptible = dev_priv->mm.interruptible; 5081 dev_priv->mm.interruptible = false; 5082 5083 WARN_ON(i915_vma_unbind(vma)); 5084 5085 dev_priv->mm.interruptible = was_interruptible; 5086 } 5087 } 5088 5089 /* Stolen objects don't hold a ref, but do hold pin count. Fix that up 5090 * before progressing. */ 5091 if (obj->stolen) 5092 i915_gem_object_unpin_pages(obj); 5093 5094 WARN_ON(obj->frontbuffer_bits); 5095 5096 if (obj->pages && obj->madv == I915_MADV_WILLNEED && 5097 dev_priv->quirks & QUIRK_PIN_SWIZZLED_PAGES && 5098 obj->tiling_mode != I915_TILING_NONE) 5099 i915_gem_object_unpin_pages(obj); 5100 5101 if (WARN_ON(obj->pages_pin_count)) 5102 obj->pages_pin_count = 0; 5103 if (discard_backing_storage(obj)) 5104 obj->madv = I915_MADV_DONTNEED; 5105 i915_gem_object_put_pages(obj); 5106 i915_gem_object_free_mmap_offset(obj); 5107 5108 BUG_ON(obj->pages); 5109 5110 if (obj->base.import_attach) 5111 drm_prime_gem_destroy(&obj->base, NULL); 5112 5113 if (obj->ops->release) 5114 obj->ops->release(obj); 5115 5116 drm_gem_object_release(&obj->base); 5117 i915_gem_info_remove_obj(dev_priv, obj->base.size); 5118 5119 kfree(obj->bit_17); 5120 i915_gem_object_free(obj); 5121 5122 intel_runtime_pm_put(dev_priv); 5123 } 5124 5125 struct i915_vma *i915_gem_obj_to_vma(struct drm_i915_gem_object *obj, 5126 struct i915_address_space *vm) 5127 { 5128 struct i915_vma *vma; 5129 list_for_each_entry(vma, &obj->vma_list, vma_link) { 5130 if (i915_is_ggtt(vma->vm) && 5131 vma->ggtt_view.type != I915_GGTT_VIEW_NORMAL) 5132 continue; 5133 if (vma->vm == vm) 5134 return vma; 5135 } 5136 return NULL; 5137 } 5138 5139 struct i915_vma *i915_gem_obj_to_ggtt_view(struct drm_i915_gem_object *obj, 5140 const struct i915_ggtt_view *view) 5141 { 5142 struct i915_address_space *ggtt = i915_obj_to_ggtt(obj); 5143 struct i915_vma *vma; 5144 5145 if (WARN_ONCE(!view, "no view specified")) 5146 return ERR_PTR(-EINVAL); 5147 5148 list_for_each_entry(vma, &obj->vma_list, vma_link) 5149 if (vma->vm == ggtt && 5150 i915_ggtt_view_equal(&vma->ggtt_view, view)) 5151 return vma; 5152 return NULL; 5153 } 5154 5155 void i915_gem_vma_destroy(struct i915_vma *vma) 5156 { 5157 struct i915_address_space *vm = NULL; 5158 WARN_ON(vma->node.allocated); 5159 5160 /* Keep the vma as a placeholder in the execbuffer reservation lists */ 5161 if (!list_empty(&vma->exec_list)) 5162 return; 5163 5164 vm = vma->vm; 5165 5166 if (!i915_is_ggtt(vm)) 5167 i915_ppgtt_put(i915_vm_to_ppgtt(vm)); 5168 5169 list_del(&vma->vma_link); 5170 5171 kmem_cache_free(to_i915(vma->obj->base.dev)->vmas, vma); 5172 } 5173 5174 static void 5175 i915_gem_stop_ringbuffers(struct drm_device *dev) 5176 { 5177 struct drm_i915_private *dev_priv = dev->dev_private; 5178 struct intel_engine_cs *ring; 5179 int i; 5180 5181 for_each_ring(ring, dev_priv, i) 5182 dev_priv->gt.stop_ring(ring); 5183 } 5184 5185 int 5186 i915_gem_suspend(struct drm_device *dev) 5187 { 5188 struct drm_i915_private *dev_priv = dev->dev_private; 5189 int ret = 0; 5190 5191 mutex_lock(&dev->struct_mutex); 5192 ret = i915_gpu_idle(dev); 5193 if (ret) 5194 goto err; 5195 5196 i915_gem_retire_requests(dev); 5197 5198 i915_gem_stop_ringbuffers(dev); 5199 mutex_unlock(&dev->struct_mutex); 5200 5201 cancel_delayed_work_sync(&dev_priv->gpu_error.hangcheck_work); 5202 cancel_delayed_work_sync(&dev_priv->mm.retire_work); 5203 flush_delayed_work(&dev_priv->mm.idle_work); 5204 5205 /* Assert that we sucessfully flushed all the work and 5206 * reset the GPU back to its idle, low power state. 5207 */ 5208 WARN_ON(dev_priv->mm.busy); 5209 5210 return 0; 5211 5212 err: 5213 mutex_unlock(&dev->struct_mutex); 5214 return ret; 5215 } 5216 5217 int i915_gem_l3_remap(struct drm_i915_gem_request *req, int slice) 5218 { 5219 struct intel_engine_cs *ring = req->ring; 5220 struct drm_device *dev = ring->dev; 5221 struct drm_i915_private *dev_priv = dev->dev_private; 5222 u32 reg_base = GEN7_L3LOG_BASE + (slice * 0x200); 5223 u32 *remap_info = dev_priv->l3_parity.remap_info[slice]; 5224 int i, ret; 5225 5226 if (!HAS_L3_DPF(dev) || !remap_info) 5227 return 0; 5228 5229 ret = intel_ring_begin(req, GEN7_L3LOG_SIZE / 4 * 3); 5230 if (ret) 5231 return ret; 5232 5233 /* 5234 * Note: We do not worry about the concurrent register cacheline hang 5235 * here because no other code should access these registers other than 5236 * at initialization time. 5237 */ 5238 for (i = 0; i < GEN7_L3LOG_SIZE; i += 4) { 5239 intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1)); 5240 intel_ring_emit(ring, reg_base + i); 5241 intel_ring_emit(ring, remap_info[i/4]); 5242 } 5243 5244 intel_ring_advance(ring); 5245 5246 return ret; 5247 } 5248 5249 void i915_gem_init_swizzling(struct drm_device *dev) 5250 { 5251 struct drm_i915_private *dev_priv = dev->dev_private; 5252 5253 if (INTEL_INFO(dev)->gen < 5 || 5254 dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_NONE) 5255 return; 5256 5257 I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) | 5258 DISP_TILE_SURFACE_SWIZZLING); 5259 5260 if (IS_GEN5(dev)) 5261 return; 5262 5263 I915_WRITE(TILECTL, I915_READ(TILECTL) | TILECTL_SWZCTL); 5264 if (IS_GEN6(dev)) 5265 I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_SNB)); 5266 else if (IS_GEN7(dev)) 5267 I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_IVB)); 5268 else if (IS_GEN8(dev)) 5269 I915_WRITE(GAMTARBMODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_BDW)); 5270 else 5271 BUG(); 5272 } 5273 5274 static void init_unused_ring(struct drm_device *dev, u32 base) 5275 { 5276 struct drm_i915_private *dev_priv = dev->dev_private; 5277 5278 I915_WRITE(RING_CTL(base), 0); 5279 I915_WRITE(RING_HEAD(base), 0); 5280 I915_WRITE(RING_TAIL(base), 0); 5281 I915_WRITE(RING_START(base), 0); 5282 } 5283 5284 static void init_unused_rings(struct drm_device *dev) 5285 { 5286 if (IS_I830(dev)) { 5287 init_unused_ring(dev, PRB1_BASE); 5288 init_unused_ring(dev, SRB0_BASE); 5289 init_unused_ring(dev, SRB1_BASE); 5290 init_unused_ring(dev, SRB2_BASE); 5291 init_unused_ring(dev, SRB3_BASE); 5292 } else if (IS_GEN2(dev)) { 5293 init_unused_ring(dev, SRB0_BASE); 5294 init_unused_ring(dev, SRB1_BASE); 5295 } else if (IS_GEN3(dev)) { 5296 init_unused_ring(dev, PRB1_BASE); 5297 init_unused_ring(dev, PRB2_BASE); 5298 } 5299 } 5300 5301 int i915_gem_init_rings(struct drm_device *dev) 5302 { 5303 struct drm_i915_private *dev_priv = dev->dev_private; 5304 int ret; 5305 5306 ret = intel_init_render_ring_buffer(dev); 5307 if (ret) 5308 return ret; 5309 5310 if (HAS_BSD(dev)) { 5311 ret = intel_init_bsd_ring_buffer(dev); 5312 if (ret) 5313 goto cleanup_render_ring; 5314 } 5315 5316 if (HAS_BLT(dev)) { 5317 ret = intel_init_blt_ring_buffer(dev); 5318 if (ret) 5319 goto cleanup_bsd_ring; 5320 } 5321 5322 if (HAS_VEBOX(dev)) { 5323 ret = intel_init_vebox_ring_buffer(dev); 5324 if (ret) 5325 goto cleanup_blt_ring; 5326 } 5327 5328 if (HAS_BSD2(dev)) { 5329 ret = intel_init_bsd2_ring_buffer(dev); 5330 if (ret) 5331 goto cleanup_vebox_ring; 5332 } 5333 5334 return 0; 5335 5336 cleanup_vebox_ring: 5337 intel_cleanup_ring_buffer(&dev_priv->ring[VECS]); 5338 cleanup_blt_ring: 5339 intel_cleanup_ring_buffer(&dev_priv->ring[BCS]); 5340 cleanup_bsd_ring: 5341 intel_cleanup_ring_buffer(&dev_priv->ring[VCS]); 5342 cleanup_render_ring: 5343 intel_cleanup_ring_buffer(&dev_priv->ring[RCS]); 5344 5345 return ret; 5346 } 5347 5348 int 5349 i915_gem_init_hw(struct drm_device *dev) 5350 { 5351 struct drm_i915_private *dev_priv = dev->dev_private; 5352 struct intel_engine_cs *ring; 5353 int ret, i, j; 5354 5355 if (INTEL_INFO(dev)->gen < 6 && !intel_enable_gtt()) 5356 return -EIO; 5357 5358 /* Double layer security blanket, see i915_gem_init() */ 5359 intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL); 5360 5361 if (dev_priv->ellc_size) 5362 I915_WRITE(HSW_IDICR, I915_READ(HSW_IDICR) | IDIHASHMSK(0xf)); 5363 5364 if (IS_HASWELL(dev)) 5365 I915_WRITE(MI_PREDICATE_RESULT_2, IS_HSW_GT3(dev) ? 5366 LOWER_SLICE_ENABLED : LOWER_SLICE_DISABLED); 5367 5368 if (HAS_PCH_NOP(dev)) { 5369 if (IS_IVYBRIDGE(dev)) { 5370 u32 temp = I915_READ(GEN7_MSG_CTL); 5371 temp &= ~(WAIT_FOR_PCH_FLR_ACK | WAIT_FOR_PCH_RESET_ACK); 5372 I915_WRITE(GEN7_MSG_CTL, temp); 5373 } else if (INTEL_INFO(dev)->gen >= 7) { 5374 u32 temp = I915_READ(HSW_NDE_RSTWRN_OPT); 5375 temp &= ~RESET_PCH_HANDSHAKE_ENABLE; 5376 I915_WRITE(HSW_NDE_RSTWRN_OPT, temp); 5377 } 5378 } 5379 5380 i915_gem_init_swizzling(dev); 5381 5382 /* 5383 * At least 830 can leave some of the unused rings 5384 * "active" (ie. head != tail) after resume which 5385 * will prevent c3 entry. Makes sure all unused rings 5386 * are totally idle. 5387 */ 5388 init_unused_rings(dev); 5389 5390 BUG_ON(!dev_priv->ring[RCS].default_context); 5391 5392 ret = i915_ppgtt_init_hw(dev); 5393 if (ret) { 5394 DRM_ERROR("PPGTT enable HW failed %d\n", ret); 5395 goto out; 5396 } 5397 5398 /* Need to do basic initialisation of all rings first: */ 5399 for_each_ring(ring, dev_priv, i) { 5400 ret = ring->init_hw(ring); 5401 if (ret) 5402 goto out; 5403 } 5404 5405 /* We can't enable contexts until all firmware is loaded */ 5406 if (HAS_GUC_UCODE(dev)) { 5407 ret = intel_guc_ucode_load(dev); 5408 if (ret) { 5409 /* 5410 * If we got an error and GuC submission is enabled, map 5411 * the error to -EIO so the GPU will be declared wedged. 5412 * OTOH, if we didn't intend to use the GuC anyway, just 5413 * discard the error and carry on. 5414 */ 5415 DRM_ERROR("Failed to initialize GuC, error %d%s\n", ret, 5416 i915.enable_guc_submission ? "" : 5417 " (ignored)"); 5418 ret = i915.enable_guc_submission ? -EIO : 0; 5419 if (ret) 5420 goto out; 5421 } 5422 } 5423 5424 /* 5425 * Increment the next seqno by 0x100 so we have a visible break 5426 * on re-initialisation 5427 */ 5428 ret = i915_gem_set_seqno(dev, dev_priv->next_seqno+0x100); 5429 if (ret) 5430 goto out; 5431 5432 /* Now it is safe to go back round and do everything else: */ 5433 for_each_ring(ring, dev_priv, i) { 5434 struct drm_i915_gem_request *req; 5435 5436 WARN_ON(!ring->default_context); 5437 5438 ret = i915_gem_request_alloc(ring, ring->default_context, &req); 5439 if (ret) { 5440 i915_gem_cleanup_ringbuffer(dev); 5441 goto out; 5442 } 5443 5444 if (ring->id == RCS) { 5445 for (j = 0; j < NUM_L3_SLICES(dev); j++) 5446 i915_gem_l3_remap(req, j); 5447 } 5448 5449 ret = i915_ppgtt_init_ring(req); 5450 if (ret && ret != -EIO) { 5451 DRM_ERROR("PPGTT enable ring #%d failed %d\n", i, ret); 5452 i915_gem_request_cancel(req); 5453 i915_gem_cleanup_ringbuffer(dev); 5454 goto out; 5455 } 5456 5457 ret = i915_gem_context_enable(req); 5458 if (ret && ret != -EIO) { 5459 DRM_ERROR("Context enable ring #%d failed %d\n", i, ret); 5460 i915_gem_request_cancel(req); 5461 i915_gem_cleanup_ringbuffer(dev); 5462 goto out; 5463 } 5464 5465 i915_add_request_no_flush(req); 5466 } 5467 5468 out: 5469 intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL); 5470 return ret; 5471 } 5472 5473 int i915_gem_init(struct drm_device *dev) 5474 { 5475 struct drm_i915_private *dev_priv = dev->dev_private; 5476 int ret; 5477 5478 i915.enable_execlists = intel_sanitize_enable_execlists(dev, 5479 i915.enable_execlists); 5480 5481 idr_preload(GFP_KERNEL); /* gem context */ 5482 mutex_lock(&dev->struct_mutex); 5483 5484 if (IS_VALLEYVIEW(dev)) { 5485 /* VLVA0 (potential hack), BIOS isn't actually waking us */ 5486 I915_WRITE(VLV_GTLC_WAKE_CTRL, VLV_GTLC_ALLOWWAKEREQ); 5487 if (wait_for((I915_READ(VLV_GTLC_PW_STATUS) & 5488 VLV_GTLC_ALLOWWAKEACK), 10)) 5489 DRM_DEBUG_DRIVER("allow wake ack timed out\n"); 5490 } 5491 5492 if (!i915.enable_execlists) { 5493 dev_priv->gt.execbuf_submit = i915_gem_ringbuffer_submission; 5494 dev_priv->gt.init_rings = i915_gem_init_rings; 5495 dev_priv->gt.cleanup_ring = intel_cleanup_ring_buffer; 5496 dev_priv->gt.stop_ring = intel_stop_ring_buffer; 5497 } else { 5498 dev_priv->gt.execbuf_submit = intel_execlists_submission; 5499 dev_priv->gt.init_rings = intel_logical_rings_init; 5500 dev_priv->gt.cleanup_ring = intel_logical_ring_cleanup; 5501 dev_priv->gt.stop_ring = intel_logical_ring_stop; 5502 } 5503 5504 /* This is just a security blanket to placate dragons. 5505 * On some systems, we very sporadically observe that the first TLBs 5506 * used by the CS may be stale, despite us poking the TLB reset. If 5507 * we hold the forcewake during initialisation these problems 5508 * just magically go away. 5509 */ 5510 intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL); 5511 5512 ret = i915_gem_init_userptr(dev); 5513 if (ret) 5514 goto out_unlock; 5515 5516 i915_gem_init_global_gtt(dev); 5517 5518 ret = i915_gem_context_init(dev); 5519 if (ret) 5520 goto out_unlock; 5521 5522 ret = dev_priv->gt.init_rings(dev); 5523 if (ret) 5524 goto out_unlock; 5525 5526 ret = i915_gem_init_hw(dev); 5527 if (ret == -EIO) { 5528 /* Allow ring initialisation to fail by marking the GPU as 5529 * wedged. But we only want to do this where the GPU is angry, 5530 * for all other failure, such as an allocation failure, bail. 5531 */ 5532 DRM_ERROR("Failed to initialize GPU, declaring it wedged\n"); 5533 atomic_or(I915_WEDGED, &dev_priv->gpu_error.reset_counter); 5534 ret = 0; 5535 } 5536 5537 out_unlock: 5538 intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL); 5539 mutex_unlock(&dev->struct_mutex); 5540 idr_preload_end(); 5541 5542 return ret; 5543 } 5544 5545 void 5546 i915_gem_cleanup_ringbuffer(struct drm_device *dev) 5547 { 5548 struct drm_i915_private *dev_priv = dev->dev_private; 5549 struct intel_engine_cs *ring; 5550 int i; 5551 5552 for_each_ring(ring, dev_priv, i) 5553 dev_priv->gt.cleanup_ring(ring); 5554 5555 if (i915.enable_execlists) 5556 /* 5557 * Neither the BIOS, ourselves or any other kernel 5558 * expects the system to be in execlists mode on startup, 5559 * so we need to reset the GPU back to legacy mode. 5560 */ 5561 intel_gpu_reset(dev); 5562 } 5563 5564 static void 5565 init_ring_lists(struct intel_engine_cs *ring) 5566 { 5567 INIT_LIST_HEAD(&ring->active_list); 5568 INIT_LIST_HEAD(&ring->request_list); 5569 } 5570 5571 void 5572 i915_gem_load(struct drm_device *dev) 5573 { 5574 struct drm_i915_private *dev_priv = dev->dev_private; 5575 int i; 5576 5577 dev_priv->objects = 5578 kmem_cache_create("i915_gem_object", 5579 sizeof(struct drm_i915_gem_object), 0, 5580 SLAB_HWCACHE_ALIGN, 5581 NULL); 5582 dev_priv->vmas = 5583 kmem_cache_create("i915_gem_vma", 5584 sizeof(struct i915_vma), 0, 5585 SLAB_HWCACHE_ALIGN, 5586 NULL); 5587 dev_priv->requests = 5588 kmem_cache_create("i915_gem_request", 5589 sizeof(struct drm_i915_gem_request), 0, 5590 SLAB_HWCACHE_ALIGN, 5591 NULL); 5592 5593 INIT_LIST_HEAD(&dev_priv->vm_list); 5594 INIT_LIST_HEAD(&dev_priv->context_list); 5595 INIT_LIST_HEAD(&dev_priv->mm.unbound_list); 5596 INIT_LIST_HEAD(&dev_priv->mm.bound_list); 5597 INIT_LIST_HEAD(&dev_priv->mm.fence_list); 5598 for (i = 0; i < I915_NUM_RINGS; i++) 5599 init_ring_lists(&dev_priv->ring[i]); 5600 for (i = 0; i < I915_MAX_NUM_FENCES; i++) 5601 INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list); 5602 INIT_DELAYED_WORK(&dev_priv->mm.retire_work, 5603 i915_gem_retire_work_handler); 5604 INIT_DELAYED_WORK(&dev_priv->mm.idle_work, 5605 i915_gem_idle_work_handler); 5606 #ifdef __NetBSD__ 5607 spin_lock_init(&dev_priv->gpu_error.reset_lock); 5608 DRM_INIT_WAITQUEUE(&dev_priv->gpu_error.reset_queue, "i915errst"); 5609 #else 5610 init_waitqueue_head(&dev_priv->gpu_error.reset_queue); 5611 #endif 5612 5613 dev_priv->relative_constants_mode = I915_EXEC_CONSTANTS_REL_GENERAL; 5614 5615 if (INTEL_INFO(dev)->gen >= 7 && !IS_VALLEYVIEW(dev)) 5616 dev_priv->num_fence_regs = 32; 5617 else if (INTEL_INFO(dev)->gen >= 4 || IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev)) 5618 dev_priv->num_fence_regs = 16; 5619 else 5620 dev_priv->num_fence_regs = 8; 5621 5622 if (intel_vgpu_active(dev)) 5623 dev_priv->num_fence_regs = 5624 I915_READ(vgtif_reg(avail_rs.fence_num)); 5625 5626 /* 5627 * Set initial sequence number for requests. 5628 * Using this number allows the wraparound to happen early, 5629 * catching any obvious problems. 5630 */ 5631 dev_priv->next_seqno = ((u32)~0 - 0x1100); 5632 dev_priv->last_seqno = ((u32)~0 - 0x1101); 5633 5634 /* Initialize fence registers to zero */ 5635 INIT_LIST_HEAD(&dev_priv->mm.fence_list); 5636 i915_gem_restore_fences(dev); 5637 5638 i915_gem_detect_bit_6_swizzle(dev); 5639 #ifdef __NetBSD__ 5640 DRM_INIT_WAITQUEUE(&dev_priv->pending_flip_queue, "i915flip"); 5641 spin_lock_init(&dev_priv->pending_flip_lock); 5642 #else 5643 init_waitqueue_head(&dev_priv->pending_flip_queue); 5644 #endif 5645 5646 dev_priv->mm.interruptible = true; 5647 5648 i915_gem_shrinker_init(dev_priv); 5649 #ifdef __NetBSD__ 5650 linux_mutex_init(&dev_priv->fb_tracking.lock); 5651 #else 5652 mutex_init(&dev_priv->fb_tracking.lock); 5653 #endif 5654 } 5655 5656 void i915_gem_release(struct drm_device *dev, struct drm_file *file) 5657 { 5658 struct drm_i915_file_private *file_priv = file->driver_priv; 5659 5660 /* Clean up our request list when the client is going away, so that 5661 * later retire_requests won't dereference our soon-to-be-gone 5662 * file_priv. 5663 */ 5664 spin_lock(&file_priv->mm.lock); 5665 while (!list_empty(&file_priv->mm.request_list)) { 5666 struct drm_i915_gem_request *request; 5667 5668 request = list_first_entry(&file_priv->mm.request_list, 5669 struct drm_i915_gem_request, 5670 client_list); 5671 list_del(&request->client_list); 5672 request->file_priv = NULL; 5673 } 5674 spin_unlock(&file_priv->mm.lock); 5675 5676 if (!list_empty(&file_priv->rps.link)) { 5677 spin_lock(&to_i915(dev)->rps.client_lock); 5678 list_del(&file_priv->rps.link); 5679 spin_unlock(&to_i915(dev)->rps.client_lock); 5680 } 5681 } 5682 5683 int i915_gem_open(struct drm_device *dev, struct drm_file *file) 5684 { 5685 struct drm_i915_file_private *file_priv; 5686 int ret; 5687 5688 DRM_DEBUG_DRIVER("\n"); 5689 5690 file_priv = kzalloc(sizeof(*file_priv), GFP_KERNEL); 5691 if (!file_priv) 5692 return -ENOMEM; 5693 5694 file->driver_priv = file_priv; 5695 file_priv->dev_priv = dev->dev_private; 5696 file_priv->file = file; 5697 INIT_LIST_HEAD(&file_priv->rps.link); 5698 5699 spin_lock_init(&file_priv->mm.lock); 5700 INIT_LIST_HEAD(&file_priv->mm.request_list); 5701 5702 ret = i915_gem_context_open(dev, file); 5703 if (ret) 5704 kfree(file_priv); 5705 5706 return ret; 5707 } 5708 5709 /** 5710 * i915_gem_track_fb - update frontbuffer tracking 5711 * @old: current GEM buffer for the frontbuffer slots 5712 * @new: new GEM buffer for the frontbuffer slots 5713 * @frontbuffer_bits: bitmask of frontbuffer slots 5714 * 5715 * This updates the frontbuffer tracking bits @frontbuffer_bits by clearing them 5716 * from @old and setting them in @new. Both @old and @new can be NULL. 5717 */ 5718 void i915_gem_track_fb(struct drm_i915_gem_object *old, 5719 struct drm_i915_gem_object *new, 5720 unsigned frontbuffer_bits) 5721 { 5722 if (old) { 5723 WARN_ON(!mutex_is_locked(&old->base.dev->struct_mutex)); 5724 WARN_ON(!(old->frontbuffer_bits & frontbuffer_bits)); 5725 old->frontbuffer_bits &= ~frontbuffer_bits; 5726 } 5727 5728 if (new) { 5729 WARN_ON(!mutex_is_locked(&new->base.dev->struct_mutex)); 5730 WARN_ON(new->frontbuffer_bits & frontbuffer_bits); 5731 new->frontbuffer_bits |= frontbuffer_bits; 5732 } 5733 } 5734 5735 /* All the new VM stuff */ 5736 u64 i915_gem_obj_offset(struct drm_i915_gem_object *o, 5737 struct i915_address_space *vm) 5738 { 5739 struct drm_i915_private *dev_priv = o->base.dev->dev_private; 5740 struct i915_vma *vma; 5741 5742 WARN_ON(vm == &dev_priv->mm.aliasing_ppgtt->base); 5743 5744 list_for_each_entry(vma, &o->vma_list, vma_link) { 5745 if (i915_is_ggtt(vma->vm) && 5746 vma->ggtt_view.type != I915_GGTT_VIEW_NORMAL) 5747 continue; 5748 if (vma->vm == vm) 5749 return vma->node.start; 5750 } 5751 5752 WARN(1, "%s vma for this object not found.\n", 5753 i915_is_ggtt(vm) ? "global" : "ppgtt"); 5754 return -1; 5755 } 5756 5757 u64 i915_gem_obj_ggtt_offset_view(struct drm_i915_gem_object *o, 5758 const struct i915_ggtt_view *view) 5759 { 5760 struct i915_address_space *ggtt = i915_obj_to_ggtt(o); 5761 struct i915_vma *vma; 5762 5763 list_for_each_entry(vma, &o->vma_list, vma_link) 5764 if (vma->vm == ggtt && 5765 i915_ggtt_view_equal(&vma->ggtt_view, view)) 5766 return vma->node.start; 5767 5768 WARN(1, "global vma for this object not found. (view=%u)\n", view->type); 5769 return -1; 5770 } 5771 5772 bool i915_gem_obj_bound(struct drm_i915_gem_object *o, 5773 struct i915_address_space *vm) 5774 { 5775 struct i915_vma *vma; 5776 5777 list_for_each_entry(vma, &o->vma_list, vma_link) { 5778 if (i915_is_ggtt(vma->vm) && 5779 vma->ggtt_view.type != I915_GGTT_VIEW_NORMAL) 5780 continue; 5781 if (vma->vm == vm && drm_mm_node_allocated(&vma->node)) 5782 return true; 5783 } 5784 5785 return false; 5786 } 5787 5788 bool i915_gem_obj_ggtt_bound_view(struct drm_i915_gem_object *o, 5789 const struct i915_ggtt_view *view) 5790 { 5791 struct i915_address_space *ggtt = i915_obj_to_ggtt(o); 5792 struct i915_vma *vma; 5793 5794 list_for_each_entry(vma, &o->vma_list, vma_link) 5795 if (vma->vm == ggtt && 5796 i915_ggtt_view_equal(&vma->ggtt_view, view) && 5797 drm_mm_node_allocated(&vma->node)) 5798 return true; 5799 5800 return false; 5801 } 5802 5803 bool i915_gem_obj_bound_any(struct drm_i915_gem_object *o) 5804 { 5805 struct i915_vma *vma; 5806 5807 list_for_each_entry(vma, &o->vma_list, vma_link) 5808 if (drm_mm_node_allocated(&vma->node)) 5809 return true; 5810 5811 return false; 5812 } 5813 5814 unsigned long i915_gem_obj_size(struct drm_i915_gem_object *o, 5815 struct i915_address_space *vm) 5816 { 5817 struct drm_i915_private *dev_priv = o->base.dev->dev_private; 5818 struct i915_vma *vma; 5819 5820 WARN_ON(vm == &dev_priv->mm.aliasing_ppgtt->base); 5821 5822 BUG_ON(list_empty(&o->vma_list)); 5823 5824 list_for_each_entry(vma, &o->vma_list, vma_link) { 5825 if (i915_is_ggtt(vma->vm) && 5826 vma->ggtt_view.type != I915_GGTT_VIEW_NORMAL) 5827 continue; 5828 if (vma->vm == vm) 5829 return vma->node.size; 5830 } 5831 return 0; 5832 } 5833 5834 bool i915_gem_obj_is_pinned(struct drm_i915_gem_object *obj) 5835 { 5836 struct i915_vma *vma; 5837 list_for_each_entry(vma, &obj->vma_list, vma_link) 5838 if (vma->pin_count > 0) 5839 return true; 5840 5841 return false; 5842 } 5843 5844 /* Allocate a new GEM object and fill it with the supplied data */ 5845 struct drm_i915_gem_object * 5846 i915_gem_object_create_from_data(struct drm_device *dev, 5847 const void *data, size_t size) 5848 { 5849 struct drm_i915_gem_object *obj; 5850 #ifdef __NetBSD__ 5851 struct iovec iov = { .iov_base = __UNCONST(data), .iov_len = size }; 5852 struct uio uio = { 5853 .uio_iov = &iov, 5854 .uio_iovcnt = 1, 5855 .uio_offset = 0, 5856 .uio_resid = size, 5857 .uio_rw = UIO_WRITE, 5858 }; 5859 #else 5860 struct sg_table *sg; 5861 #endif 5862 size_t bytes; 5863 int ret; 5864 5865 obj = i915_gem_alloc_object(dev, round_up(size, PAGE_SIZE)); 5866 if (IS_ERR_OR_NULL(obj)) 5867 return obj; 5868 5869 ret = i915_gem_object_set_to_cpu_domain(obj, true); 5870 if (ret) 5871 goto fail; 5872 5873 ret = i915_gem_object_get_pages(obj); 5874 if (ret) 5875 goto fail; 5876 5877 i915_gem_object_pin_pages(obj); 5878 #ifdef __NetBSD__ 5879 UIO_SETUP_SYSSPACE(&uio); 5880 /* XXX errno NetBSD->Linux */ 5881 ret = -ubc_uiomove(obj->base.filp, &uio, size, UVM_ADV_NORMAL, 5882 UBC_WRITE); 5883 if (ret) 5884 goto fail; 5885 bytes = size - uio.uio_resid; 5886 #else 5887 sg = obj->pages; 5888 bytes = sg_copy_from_buffer(sg->sgl, sg->nents, (void *)data, size); 5889 #endif 5890 i915_gem_object_unpin_pages(obj); 5891 5892 if (WARN_ON(bytes != size)) { 5893 DRM_ERROR("Incomplete copy, wrote %zu of %zu", bytes, size); 5894 ret = -EFAULT; 5895 goto fail; 5896 } 5897 5898 return obj; 5899 5900 fail: 5901 drm_gem_object_unreference(&obj->base); 5902 return ERR_PTR(ret); 5903 } 5904
Indexes created Wed Oct 22 06:10:02 GMT 2025