i915_gem.c revision 1.41 1 /* $NetBSD: i915_gem.c,v 1.41 2018/08/27 07:17:35 riastradh Exp $ */
2
3 /*
4 * Copyright 2008-2015 Intel Corporation
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 * and/or sell copies of the Software, and to permit persons to whom the
11 * Software is furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice (including the next
14 * paragraph) shall be included in all copies or substantial portions of the
15 * Software.
16 *
17 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
18 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
19 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
20 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
21 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
22 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
23 * IN THE SOFTWARE.
24 *
25 * Authors:
26 * Eric Anholt <eric (at) anholt.net>
27 *
28 */
29
30 #include <sys/cdefs.h>
31 __KERNEL_RCSID(0, "$NetBSD: i915_gem.c,v 1.41 2018/08/27 07:17:35 riastradh Exp $");
32
33 #ifdef __NetBSD__
34 #if 0 /* XXX uvmhist option? */
35 #include "opt_uvmhist.h"
36 #endif
37
38 #include <sys/types.h>
39 #include <sys/param.h>
40
41 #include <uvm/uvm.h>
42 #include <uvm/uvm_extern.h>
43 #include <uvm/uvm_fault.h>
44 #include <uvm/uvm_page.h>
45 #include <uvm/uvm_pmap.h>
46 #include <uvm/uvm_prot.h>
47
48 #include <drm/bus_dma_hacks.h>
49 #endif
50
51 #include <drm/drmP.h>
52 #include <drm/drm_vma_manager.h>
53 #include <drm/i915_drm.h>
54 #include "i915_drv.h"
55 #include "i915_vgpu.h"
56 #include "i915_trace.h"
57 #include "intel_drv.h"
58 #include <linux/shmem_fs.h>
59 #include <linux/slab.h>
60 #include <linux/swap.h>
61 #include <linux/pci.h>
62 #include <linux/dma-buf.h>
63 #include <linux/errno.h>
64 #include <linux/time.h>
65 #include <linux/err.h>
66 #include <linux/bitops.h>
67 #include <linux/printk.h>
68 #include <asm/param.h>
69 #include <asm/page.h>
70
71 #define RQ_BUG_ON(expr)
72
73 static void i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj);
74 static void i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj);
75 static void
76 i915_gem_object_retire__write(struct drm_i915_gem_object *obj);
77 static void
78 i915_gem_object_retire__read(struct drm_i915_gem_object *obj, int ring);
79
80 static bool cpu_cache_is_coherent(struct drm_device *dev,
81 enum i915_cache_level level)
82 {
83 return HAS_LLC(dev) || level != I915_CACHE_NONE;
84 }
85
86 static bool cpu_write_needs_clflush(struct drm_i915_gem_object *obj)
87 {
88 if (!cpu_cache_is_coherent(obj->base.dev, obj->cache_level))
89 return true;
90
91 return obj->pin_display;
92 }
93
94 /* some bookkeeping */
95 static void i915_gem_info_add_obj(struct drm_i915_private *dev_priv,
96 size_t size)
97 {
98 spin_lock(&dev_priv->mm.object_stat_lock);
99 dev_priv->mm.object_count++;
100 dev_priv->mm.object_memory += size;
101 spin_unlock(&dev_priv->mm.object_stat_lock);
102 }
103
104 static void i915_gem_info_remove_obj(struct drm_i915_private *dev_priv,
105 size_t size)
106 {
107 spin_lock(&dev_priv->mm.object_stat_lock);
108 dev_priv->mm.object_count--;
109 dev_priv->mm.object_memory -= size;
110 spin_unlock(&dev_priv->mm.object_stat_lock);
111 }
112
113 static int
114 i915_gem_wait_for_error(struct i915_gpu_error *error)
115 {
116 int ret;
117
118 #define EXIT_COND (!i915_reset_in_progress(error) || \
119 i915_terminally_wedged(error))
120 if (EXIT_COND)
121 return 0;
122
123 /*
124 * Only wait 10 seconds for the gpu reset to complete to avoid hanging
125 * userspace. If it takes that long something really bad is going on and
126 * we should simply try to bail out and fail as gracefully as possible.
127 */
128 #ifdef __NetBSD__
129 spin_lock(&error->reset_lock);
130 DRM_SPIN_TIMED_WAIT_UNTIL(ret, &error->reset_queue, &error->reset_lock,
131 10*HZ, EXIT_COND);
132 spin_unlock(&error->reset_lock);
133 #else
134 ret = wait_event_interruptible_timeout(error->reset_queue,
135 EXIT_COND,
136 10*HZ);
137 #endif
138 if (ret == 0) {
139 DRM_ERROR("Timed out waiting for the gpu reset to complete\n");
140 return -EIO;
141 } else if (ret < 0) {
142 return ret;
143 }
144 #undef EXIT_COND
145
146 return 0;
147 }
148
149 int i915_mutex_lock_interruptible(struct drm_device *dev)
150 {
151 struct drm_i915_private *dev_priv = dev->dev_private;
152 int ret;
153
154 ret = i915_gem_wait_for_error(&dev_priv->gpu_error);
155 if (ret)
156 return ret;
157
158 ret = mutex_lock_interruptible(&dev->struct_mutex);
159 if (ret)
160 return ret;
161
162 WARN_ON(i915_verify_lists(dev));
163 return 0;
164 }
165
166 int
167 i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
168 struct drm_file *file)
169 {
170 struct drm_i915_private *dev_priv = dev->dev_private;
171 struct drm_i915_gem_get_aperture *args = data;
172 struct i915_gtt *ggtt = &dev_priv->gtt;
173 struct i915_vma *vma;
174 size_t pinned;
175
176 pinned = 0;
177 mutex_lock(&dev->struct_mutex);
178 list_for_each_entry(vma, &ggtt->base.active_list, mm_list)
179 if (vma->pin_count)
180 pinned += vma->node.size;
181 list_for_each_entry(vma, &ggtt->base.inactive_list, mm_list)
182 if (vma->pin_count)
183 pinned += vma->node.size;
184 mutex_unlock(&dev->struct_mutex);
185
186 args->aper_size = dev_priv->gtt.base.total;
187 args->aper_available_size = args->aper_size - pinned;
188
189 return 0;
190 }
191
192 static int
193 i915_gem_object_get_pages_phys(struct drm_i915_gem_object *obj)
194 {
195 #ifndef __NetBSD__
196 struct address_space *mapping = file_inode(obj->base.filp)->i_mapping;
197 #endif
198 char *vaddr = obj->phys_handle->vaddr;
199 #ifndef __NetBSD__
200 struct sg_table *st;
201 struct scatterlist *sg;
202 #endif
203 int i;
204
205 if (WARN_ON(i915_gem_object_needs_bit17_swizzle(obj)))
206 return -EINVAL;
207
208 for (i = 0; i < obj->base.size / PAGE_SIZE; i++) {
209 struct page *page;
210 char *src;
211
212 #ifdef __NetBSD__
213 struct pglist pages = TAILQ_HEAD_INITIALIZER(pages);
214 int ret;
215 /* XXX errno NetBSD->Linux */
216 ret = -uvm_obj_wirepages(obj->base.gemo_shm_uao,
217 i*PAGE_SIZE, (i + 1)*PAGE_SIZE, &pages);
218 if (ret)
219 return ret;
220 page = container_of(TAILQ_FIRST(&pages), struct page, p_vmp);
221 #else
222 page = shmem_read_mapping_page(mapping, i);
223 if (IS_ERR(page))
224 return PTR_ERR(page);
225 #endif
226
227 src = kmap_atomic(page);
228 memcpy(vaddr, src, PAGE_SIZE);
229 drm_clflush_virt_range(vaddr, PAGE_SIZE);
230 kunmap_atomic(src);
231
232 #ifdef __NetBSD__
233 uvm_obj_unwirepages(obj->base.gemo_shm_uao,
234 i*PAGE_SIZE, (i + 1)*PAGE_SIZE);
235 #else
236 page_cache_release(page);
237 #endif
238 vaddr += PAGE_SIZE;
239 }
240
241 i915_gem_chipset_flush(obj->base.dev);
242
243 #ifdef __NetBSD__
244 obj->pages = obj->phys_handle->dmah_map;
245 #else
246 st = kmalloc(sizeof(*st), GFP_KERNEL);
247 if (st == NULL)
248 return -ENOMEM;
249
250 if (sg_alloc_table(st, 1, GFP_KERNEL)) {
251 kfree(st);
252 return -ENOMEM;
253 }
254
255 sg = st->sgl;
256 sg->offset = 0;
257 sg->length = obj->base.size;
258
259 sg_dma_address(sg) = obj->phys_handle->busaddr;
260 sg_dma_len(sg) = obj->base.size;
261
262 obj->pages = st;
263 #endif
264 return 0;
265 }
266
267 static void
268 i915_gem_object_put_pages_phys(struct drm_i915_gem_object *obj)
269 {
270 int ret;
271
272 BUG_ON(obj->madv == __I915_MADV_PURGED);
273
274 ret = i915_gem_object_set_to_cpu_domain(obj, true);
275 if (ret) {
276 /* In the event of a disaster, abandon all caches and
277 * hope for the best.
278 */
279 WARN_ON(ret != -EIO);
280 obj->base.read_domains = obj->base.write_domain = I915_GEM_DOMAIN_CPU;
281 }
282
283 if (obj->madv == I915_MADV_DONTNEED)
284 obj->dirty = 0;
285
286 if (obj->dirty) {
287 #ifndef __NetBSD__
288 struct address_space *mapping = file_inode(obj->base.filp)->i_mapping;
289 #endif
290 const char *vaddr = obj->phys_handle->vaddr;
291 int i;
292
293 for (i = 0; i < obj->base.size / PAGE_SIZE; i++) {
294 struct page *page;
295 char *dst;
296
297 #ifdef __NetBSD__
298 struct pglist pages = TAILQ_HEAD_INITIALIZER(pages);
299 /* XXX errno NetBSD->Linux */
300 ret = -uvm_obj_wirepages(obj->base.gemo_shm_uao,
301 i*PAGE_SIZE, (i + 1)*PAGE_SIZE, &pages);
302 if (ret)
303 continue;
304 page = container_of(TAILQ_FIRST(&pages), struct page,
305 p_vmp);
306 #endif
307
308 dst = kmap_atomic(page);
309 drm_clflush_virt_range(vaddr, PAGE_SIZE);
310 memcpy(dst, vaddr, PAGE_SIZE);
311 kunmap_atomic(dst);
312
313 #ifdef __NetBSD__
314 page->p_vmp.flags &= ~PG_CLEAN;
315 /* XXX mark page accessed */
316 uvm_obj_unwirepages(obj->base.gemo_shm_uao,
317 i*PAGE_SIZE, (i+1)*PAGE_SIZE);
318 #else
319 set_page_dirty(page);
320 if (obj->madv == I915_MADV_WILLNEED)
321 mark_page_accessed(page);
322 page_cache_release(page);
323 #endif
324 vaddr += PAGE_SIZE;
325 }
326 obj->dirty = 0;
327 }
328
329 #ifdef __NetBSD__
330 obj->pages = NULL;
331 #else
332 sg_free_table(obj->pages);
333 kfree(obj->pages);
334 #endif
335 }
336
337 static void
338 i915_gem_object_release_phys(struct drm_i915_gem_object *obj)
339 {
340 drm_pci_free(obj->base.dev, obj->phys_handle);
341 }
342
343 static const struct drm_i915_gem_object_ops i915_gem_phys_ops = {
344 .get_pages = i915_gem_object_get_pages_phys,
345 .put_pages = i915_gem_object_put_pages_phys,
346 .release = i915_gem_object_release_phys,
347 };
348
349 static int
350 drop_pages(struct drm_i915_gem_object *obj)
351 {
352 struct i915_vma *vma, *next;
353 int ret;
354
355 drm_gem_object_reference(&obj->base);
356 list_for_each_entry_safe(vma, next, &obj->vma_list, vma_link)
357 if (i915_vma_unbind(vma))
358 break;
359
360 ret = i915_gem_object_put_pages(obj);
361 drm_gem_object_unreference(&obj->base);
362
363 return ret;
364 }
365
366 int
367 i915_gem_object_attach_phys(struct drm_i915_gem_object *obj,
368 int align)
369 {
370 drm_dma_handle_t *phys;
371 int ret;
372
373 if (obj->phys_handle) {
374 if ((unsigned long)obj->phys_handle->vaddr & (align -1))
375 return -EBUSY;
376
377 return 0;
378 }
379
380 if (obj->madv != I915_MADV_WILLNEED)
381 return -EFAULT;
382
383 #ifdef __NetBSD__
384 if (obj->base.gemo_shm_uao == NULL)
385 return -EINVAL;
386 #else
387 if (obj->base.filp == NULL)
388 return -EINVAL;
389 #endif
390
391 ret = drop_pages(obj);
392 if (ret)
393 return ret;
394
395 /* create a new object */
396 phys = drm_pci_alloc(obj->base.dev, obj->base.size, align);
397 if (!phys)
398 return -ENOMEM;
399
400 obj->phys_handle = phys;
401 obj->ops = &i915_gem_phys_ops;
402
403 return i915_gem_object_get_pages(obj);
404 }
405
406 static int
407 i915_gem_phys_pwrite(struct drm_i915_gem_object *obj,
408 struct drm_i915_gem_pwrite *args,
409 struct drm_file *file_priv)
410 {
411 struct drm_device *dev = obj->base.dev;
412 void *vaddr = (char *)obj->phys_handle->vaddr + args->offset;
413 char __user *user_data = to_user_ptr(args->data_ptr);
414 int ret = 0;
415
416 /* We manually control the domain here and pretend that it
417 * remains coherent i.e. in the GTT domain, like shmem_pwrite.
418 */
419 ret = i915_gem_object_wait_rendering(obj, false);
420 if (ret)
421 return ret;
422
423 intel_fb_obj_invalidate(obj, ORIGIN_CPU);
424 if (__copy_from_user_inatomic_nocache(vaddr, user_data, args->size)) {
425 unsigned long unwritten;
426
427 /* The physical object once assigned is fixed for the lifetime
428 * of the obj, so we can safely drop the lock and continue
429 * to access vaddr.
430 */
431 mutex_unlock(&dev->struct_mutex);
432 unwritten = copy_from_user(vaddr, user_data, args->size);
433 mutex_lock(&dev->struct_mutex);
434 if (unwritten) {
435 ret = -EFAULT;
436 goto out;
437 }
438 }
439
440 drm_clflush_virt_range(vaddr, args->size);
441 i915_gem_chipset_flush(dev);
442
443 out:
444 intel_fb_obj_flush(obj, false, ORIGIN_CPU);
445 return ret;
446 }
447
448 void *i915_gem_object_alloc(struct drm_device *dev)
449 {
450 struct drm_i915_private *dev_priv = dev->dev_private;
451 return kmem_cache_zalloc(dev_priv->objects, GFP_KERNEL);
452 }
453
454 void i915_gem_object_free(struct drm_i915_gem_object *obj)
455 {
456 struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
457 kmem_cache_free(dev_priv->objects, obj);
458 }
459
460 static int
461 i915_gem_create(struct drm_file *file,
462 struct drm_device *dev,
463 uint64_t size,
464 uint32_t *handle_p)
465 {
466 struct drm_i915_gem_object *obj;
467 int ret;
468 u32 handle;
469
470 size = roundup(size, PAGE_SIZE);
471 if (size == 0)
472 return -EINVAL;
473
474 /* Allocate the new object */
475 obj = i915_gem_alloc_object(dev, size);
476 if (obj == NULL)
477 return -ENOMEM;
478
479 ret = drm_gem_handle_create(file, &obj->base, &handle);
480 /* drop reference from allocate - handle holds it now */
481 drm_gem_object_unreference_unlocked(&obj->base);
482 if (ret)
483 return ret;
484
485 *handle_p = handle;
486 return 0;
487 }
488
489 int
490 i915_gem_dumb_create(struct drm_file *file,
491 struct drm_device *dev,
492 struct drm_mode_create_dumb *args)
493 {
494 /* have to work out size/pitch and return them */
495 #ifdef __NetBSD__ /* ALIGN means something else. */
496 args->pitch = round_up(args->width * DIV_ROUND_UP(args->bpp, 8), 64);
497 #else
498 args->pitch = ALIGN(args->width * DIV_ROUND_UP(args->bpp, 8), 64);
499 #endif
500 args->size = args->pitch * args->height;
501 return i915_gem_create(file, dev,
502 args->size, &args->handle);
503 }
504
505 /**
506 * Creates a new mm object and returns a handle to it.
507 */
508 int
509 i915_gem_create_ioctl(struct drm_device *dev, void *data,
510 struct drm_file *file)
511 {
512 struct drm_i915_gem_create *args = data;
513
514 return i915_gem_create(file, dev,
515 args->size, &args->handle);
516 }
517
518 static inline int
519 __copy_to_user_swizzled(char __user *cpu_vaddr,
520 const char *gpu_vaddr, int gpu_offset,
521 int length)
522 {
523 int ret, cpu_offset = 0;
524
525 while (length > 0) {
526 #ifdef __NetBSD__ /* XXX ALIGN means something else. */
527 int cacheline_end = round_up(gpu_offset + 1, 64);
528 #else
529 int cacheline_end = ALIGN(gpu_offset + 1, 64);
530 #endif
531 int this_length = min(cacheline_end - gpu_offset, length);
532 int swizzled_gpu_offset = gpu_offset ^ 64;
533
534 ret = __copy_to_user(cpu_vaddr + cpu_offset,
535 gpu_vaddr + swizzled_gpu_offset,
536 this_length);
537 if (ret)
538 return ret + length;
539
540 cpu_offset += this_length;
541 gpu_offset += this_length;
542 length -= this_length;
543 }
544
545 return 0;
546 }
547
548 static inline int
549 __copy_from_user_swizzled(char *gpu_vaddr, int gpu_offset,
550 const char __user *cpu_vaddr,
551 int length)
552 {
553 int ret, cpu_offset = 0;
554
555 while (length > 0) {
556 #ifdef __NetBSD__ /* XXX ALIGN means something else. */
557 int cacheline_end = round_up(gpu_offset + 1, 64);
558 #else
559 int cacheline_end = ALIGN(gpu_offset + 1, 64);
560 #endif
561 int this_length = min(cacheline_end - gpu_offset, length);
562 int swizzled_gpu_offset = gpu_offset ^ 64;
563
564 ret = __copy_from_user(gpu_vaddr + swizzled_gpu_offset,
565 cpu_vaddr + cpu_offset,
566 this_length);
567 if (ret)
568 return ret + length;
569
570 cpu_offset += this_length;
571 gpu_offset += this_length;
572 length -= this_length;
573 }
574
575 return 0;
576 }
577
578 /*
579 * Pins the specified object's pages and synchronizes the object with
580 * GPU accesses. Sets needs_clflush to non-zero if the caller should
581 * flush the object from the CPU cache.
582 */
583 int i915_gem_obj_prepare_shmem_read(struct drm_i915_gem_object *obj,
584 int *needs_clflush)
585 {
586 int ret;
587
588 *needs_clflush = 0;
589
590 #ifdef __NetBSD__
591 if (obj->base.gemo_shm_uao == NULL)
592 return -EINVAL;
593 #else
594 if (!obj->base.filp)
595 return -EINVAL;
596 #endif
597
598 if (!(obj->base.read_domains & I915_GEM_DOMAIN_CPU)) {
599 /* If we're not in the cpu read domain, set ourself into the gtt
600 * read domain and manually flush cachelines (if required). This
601 * optimizes for the case when the gpu will dirty the data
602 * anyway again before the next pread happens. */
603 *needs_clflush = !cpu_cache_is_coherent(obj->base.dev,
604 obj->cache_level);
605 ret = i915_gem_object_wait_rendering(obj, true);
606 if (ret)
607 return ret;
608 }
609
610 ret = i915_gem_object_get_pages(obj);
611 if (ret)
612 return ret;
613
614 i915_gem_object_pin_pages(obj);
615
616 return ret;
617 }
618
619 /* Per-page copy function for the shmem pread fastpath.
620 * Flushes invalid cachelines before reading the target if
621 * needs_clflush is set. */
622 static int
623 shmem_pread_fast(struct page *page, int shmem_page_offset, int page_length,
624 char __user *user_data,
625 bool page_do_bit17_swizzling, bool needs_clflush)
626 {
627 #ifdef __NetBSD__ /* XXX atomic shmem fast path */
628 return -EFAULT;
629 #else
630 char *vaddr;
631 int ret;
632
633 if (unlikely(page_do_bit17_swizzling))
634 return -EINVAL;
635
636 vaddr = kmap_atomic(page);
637 if (needs_clflush)
638 drm_clflush_virt_range(vaddr + shmem_page_offset,
639 page_length);
640 ret = __copy_to_user_inatomic(user_data,
641 vaddr + shmem_page_offset,
642 page_length);
643 kunmap_atomic(vaddr);
644
645 return ret ? -EFAULT : 0;
646 #endif
647 }
648
649 static void
650 shmem_clflush_swizzled_range(char *addr, unsigned long length,
651 bool swizzled)
652 {
653 if (unlikely(swizzled)) {
654 unsigned long start = (unsigned long) addr;
655 unsigned long end = (unsigned long) addr + length;
656
657 /* For swizzling simply ensure that we always flush both
658 * channels. Lame, but simple and it works. Swizzled
659 * pwrite/pread is far from a hotpath - current userspace
660 * doesn't use it at all. */
661 start = round_down(start, 128);
662 end = round_up(end, 128);
663
664 drm_clflush_virt_range((void *)start, end - start);
665 } else {
666 drm_clflush_virt_range(addr, length);
667 }
668
669 }
670
671 /* Only difference to the fast-path function is that this can handle bit17
672 * and uses non-atomic copy and kmap functions. */
673 static int
674 shmem_pread_slow(struct page *page, int shmem_page_offset, int page_length,
675 char __user *user_data,
676 bool page_do_bit17_swizzling, bool needs_clflush)
677 {
678 char *vaddr;
679 int ret;
680
681 vaddr = kmap(page);
682 if (needs_clflush)
683 shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
684 page_length,
685 page_do_bit17_swizzling);
686
687 if (page_do_bit17_swizzling)
688 ret = __copy_to_user_swizzled(user_data,
689 vaddr, shmem_page_offset,
690 page_length);
691 else
692 ret = __copy_to_user(user_data,
693 vaddr + shmem_page_offset,
694 page_length);
695 kunmap(page);
696
697 return ret ? - EFAULT : 0;
698 }
699
700 static int
701 i915_gem_shmem_pread(struct drm_device *dev,
702 struct drm_i915_gem_object *obj,
703 struct drm_i915_gem_pread *args,
704 struct drm_file *file)
705 {
706 char __user *user_data;
707 ssize_t remain;
708 loff_t offset;
709 int shmem_page_offset, page_length, ret = 0;
710 int obj_do_bit17_swizzling, page_do_bit17_swizzling;
711 #ifndef __NetBSD__ /* XXX */
712 int prefaulted = 0;
713 #endif
714 int needs_clflush = 0;
715 #ifndef __NetBSD__
716 struct sg_page_iter sg_iter;
717 #endif
718
719 user_data = to_user_ptr(args->data_ptr);
720 remain = args->size;
721
722 obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
723
724 ret = i915_gem_obj_prepare_shmem_read(obj, &needs_clflush);
725 if (ret)
726 return ret;
727
728 offset = args->offset;
729
730 #ifdef __NetBSD__
731 while (0 < remain)
732 #else
733 for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents,
734 offset >> PAGE_SHIFT)
735 #endif
736 {
737 #ifdef __NetBSD__
738 struct page *const page = i915_gem_object_get_page(obj,
739 atop(offset));
740 #else
741 struct page *page = sg_page_iter_page(&sg_iter);
742
743 if (remain <= 0)
744 break;
745 #endif
746
747 /* Operation in this page
748 *
749 * shmem_page_offset = offset within page in shmem file
750 * page_length = bytes to copy for this page
751 */
752 shmem_page_offset = offset_in_page(offset);
753 page_length = remain;
754 if ((shmem_page_offset + page_length) > PAGE_SIZE)
755 page_length = PAGE_SIZE - shmem_page_offset;
756
757 page_do_bit17_swizzling = obj_do_bit17_swizzling &&
758 (page_to_phys(page) & (1 << 17)) != 0;
759
760 ret = shmem_pread_fast(page, shmem_page_offset, page_length,
761 user_data, page_do_bit17_swizzling,
762 needs_clflush);
763 if (ret == 0)
764 goto next_page;
765
766 mutex_unlock(&dev->struct_mutex);
767 #ifndef __NetBSD__
768 if (likely(!i915.prefault_disable) && !prefaulted) {
769 ret = fault_in_multipages_writeable(user_data, remain);
770 /* Userspace is tricking us, but we've already clobbered
771 * its pages with the prefault and promised to write the
772 * data up to the first fault. Hence ignore any errors
773 * and just continue. */
774 (void)ret;
775 prefaulted = 1;
776 }
777 #endif
778 ret = shmem_pread_slow(page, shmem_page_offset, page_length,
779 user_data, page_do_bit17_swizzling,
780 needs_clflush);
781
782 mutex_lock(&dev->struct_mutex);
783
784 if (ret)
785 goto out;
786
787 next_page:
788 remain -= page_length;
789 user_data += page_length;
790 offset += page_length;
791 }
792
793 out:
794 i915_gem_object_unpin_pages(obj);
795
796 return ret;
797 }
798
799 /**
800 * Reads data from the object referenced by handle.
801 *
802 * On error, the contents of *data are undefined.
803 */
804 int
805 i915_gem_pread_ioctl(struct drm_device *dev, void *data,
806 struct drm_file *file)
807 {
808 struct drm_i915_gem_pread *args = data;
809 struct drm_gem_object *gobj;
810 struct drm_i915_gem_object *obj;
811 int ret = 0;
812
813 if (args->size == 0)
814 return 0;
815
816 if (!access_ok(VERIFY_WRITE,
817 to_user_ptr(args->data_ptr),
818 args->size))
819 return -EFAULT;
820
821 ret = i915_mutex_lock_interruptible(dev);
822 if (ret)
823 return ret;
824
825 gobj = drm_gem_object_lookup(dev, file, args->handle);
826 if (gobj == NULL) {
827 ret = -ENOENT;
828 goto unlock;
829 }
830 obj = to_intel_bo(gobj);
831
832 /* Bounds check source. */
833 if (args->offset > obj->base.size ||
834 args->size > obj->base.size - args->offset) {
835 ret = -EINVAL;
836 goto out;
837 }
838
839 /* prime objects have no backing filp to GEM pread/pwrite
840 * pages from.
841 */
842 #ifdef __NetBSD__
843 /* Also stolen objects. */
844 if (obj->base.gemo_shm_uao == NULL) {
845 ret = -EINVAL;
846 goto out;
847 }
848 #else
849 if (!obj->base.filp) {
850 ret = -EINVAL;
851 goto out;
852 }
853 #endif
854
855 trace_i915_gem_object_pread(obj, args->offset, args->size);
856
857 ret = i915_gem_shmem_pread(dev, obj, args, file);
858
859 out:
860 drm_gem_object_unreference(&obj->base);
861 unlock:
862 mutex_unlock(&dev->struct_mutex);
863 return ret;
864 }
865
866 /* This is the fast write path which cannot handle
867 * page faults in the source data
868 */
869
870 static inline int
871 fast_user_write(struct io_mapping *mapping,
872 loff_t page_base, int page_offset,
873 char __user *user_data,
874 int length)
875 {
876 #ifdef __NetBSD__ /* XXX atomic shmem fast path */
877 return -EFAULT;
878 #else
879 void __iomem *vaddr_atomic;
880 void *vaddr;
881 unsigned long unwritten;
882
883 vaddr_atomic = io_mapping_map_atomic_wc(mapping, page_base);
884 /* We can use the cpu mem copy function because this is X86. */
885 vaddr = (void __force*)vaddr_atomic + page_offset;
886 unwritten = __copy_from_user_inatomic_nocache(vaddr,
887 user_data, length);
888 io_mapping_unmap_atomic(vaddr_atomic);
889 return unwritten;
890 #endif
891 }
892
893 /**
894 * This is the fast pwrite path, where we copy the data directly from the
895 * user into the GTT, uncached.
896 */
897 static int
898 i915_gem_gtt_pwrite_fast(struct drm_device *dev,
899 struct drm_i915_gem_object *obj,
900 struct drm_i915_gem_pwrite *args,
901 struct drm_file *file)
902 {
903 struct drm_i915_private *dev_priv = dev->dev_private;
904 ssize_t remain;
905 loff_t offset, page_base;
906 char __user *user_data;
907 int page_offset, page_length, ret;
908
909 ret = i915_gem_obj_ggtt_pin(obj, 0, PIN_MAPPABLE | PIN_NONBLOCK);
910 if (ret)
911 goto out;
912
913 ret = i915_gem_object_set_to_gtt_domain(obj, true);
914 if (ret)
915 goto out_unpin;
916
917 ret = i915_gem_object_put_fence(obj);
918 if (ret)
919 goto out_unpin;
920
921 user_data = to_user_ptr(args->data_ptr);
922 remain = args->size;
923
924 offset = i915_gem_obj_ggtt_offset(obj) + args->offset;
925
926 intel_fb_obj_invalidate(obj, ORIGIN_GTT);
927
928 while (remain > 0) {
929 /* Operation in this page
930 *
931 * page_base = page offset within aperture
932 * page_offset = offset within page
933 * page_length = bytes to copy for this page
934 */
935 page_base = offset & PAGE_MASK;
936 page_offset = offset_in_page(offset);
937 page_length = remain;
938 if ((page_offset + remain) > PAGE_SIZE)
939 page_length = PAGE_SIZE - page_offset;
940
941 /* If we get a fault while copying data, then (presumably) our
942 * source page isn't available. Return the error and we'll
943 * retry in the slow path.
944 */
945 if (fast_user_write(dev_priv->gtt.mappable, page_base,
946 page_offset, user_data, page_length)) {
947 ret = -EFAULT;
948 goto out_flush;
949 }
950
951 remain -= page_length;
952 user_data += page_length;
953 offset += page_length;
954 }
955
956 out_flush:
957 intel_fb_obj_flush(obj, false, ORIGIN_GTT);
958 out_unpin:
959 i915_gem_object_ggtt_unpin(obj);
960 out:
961 return ret;
962 }
963
964 /* Per-page copy function for the shmem pwrite fastpath.
965 * Flushes invalid cachelines before writing to the target if
966 * needs_clflush_before is set and flushes out any written cachelines after
967 * writing if needs_clflush is set. */
968 static int
969 shmem_pwrite_fast(struct page *page, int shmem_page_offset, int page_length,
970 char __user *user_data,
971 bool page_do_bit17_swizzling,
972 bool needs_clflush_before,
973 bool needs_clflush_after)
974 {
975 #ifdef __NetBSD__
976 return -EFAULT;
977 #else
978 char *vaddr;
979 int ret;
980
981 if (unlikely(page_do_bit17_swizzling))
982 return -EINVAL;
983
984 vaddr = kmap_atomic(page);
985 if (needs_clflush_before)
986 drm_clflush_virt_range(vaddr + shmem_page_offset,
987 page_length);
988 ret = __copy_from_user_inatomic(vaddr + shmem_page_offset,
989 user_data, page_length);
990 if (needs_clflush_after)
991 drm_clflush_virt_range(vaddr + shmem_page_offset,
992 page_length);
993 kunmap_atomic(vaddr);
994
995 return ret ? -EFAULT : 0;
996 #endif
997 }
998
999 /* Only difference to the fast-path function is that this can handle bit17
1000 * and uses non-atomic copy and kmap functions. */
1001 static int
1002 shmem_pwrite_slow(struct page *page, int shmem_page_offset, int page_length,
1003 char __user *user_data,
1004 bool page_do_bit17_swizzling,
1005 bool needs_clflush_before,
1006 bool needs_clflush_after)
1007 {
1008 char *vaddr;
1009 int ret;
1010
1011 vaddr = kmap(page);
1012 if (unlikely(needs_clflush_before || page_do_bit17_swizzling))
1013 shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
1014 page_length,
1015 page_do_bit17_swizzling);
1016 if (page_do_bit17_swizzling)
1017 ret = __copy_from_user_swizzled(vaddr, shmem_page_offset,
1018 user_data,
1019 page_length);
1020 else
1021 ret = __copy_from_user(vaddr + shmem_page_offset,
1022 user_data,
1023 page_length);
1024 if (needs_clflush_after)
1025 shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
1026 page_length,
1027 page_do_bit17_swizzling);
1028 kunmap(page);
1029
1030 return ret ? -EFAULT : 0;
1031 }
1032
1033 static int
1034 i915_gem_shmem_pwrite(struct drm_device *dev,
1035 struct drm_i915_gem_object *obj,
1036 struct drm_i915_gem_pwrite *args,
1037 struct drm_file *file)
1038 {
1039 ssize_t remain;
1040 loff_t offset;
1041 char __user *user_data;
1042 int shmem_page_offset, page_length, ret = 0;
1043 int obj_do_bit17_swizzling, page_do_bit17_swizzling;
1044 int hit_slowpath = 0;
1045 int needs_clflush_after = 0;
1046 int needs_clflush_before = 0;
1047 #ifndef __NetBSD__
1048 struct sg_page_iter sg_iter;
1049 int flush_mask = boot_cpu_data.x86_clflush_size - 1;
1050 #else
1051 int flush_mask = cpu_info_primary.ci_cflush_lsize - 1;
1052 #endif
1053
1054 user_data = to_user_ptr(args->data_ptr);
1055 remain = args->size;
1056
1057 obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
1058
1059 if (obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
1060 /* If we're not in the cpu write domain, set ourself into the gtt
1061 * write domain and manually flush cachelines (if required). This
1062 * optimizes for the case when the gpu will use the data
1063 * right away and we therefore have to clflush anyway. */
1064 needs_clflush_after = cpu_write_needs_clflush(obj);
1065 ret = i915_gem_object_wait_rendering(obj, false);
1066 if (ret)
1067 return ret;
1068 }
1069 /* Same trick applies to invalidate partially written cachelines read
1070 * before writing. */
1071 if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0)
1072 needs_clflush_before =
1073 !cpu_cache_is_coherent(dev, obj->cache_level);
1074
1075 ret = i915_gem_object_get_pages(obj);
1076 if (ret)
1077 return ret;
1078
1079 intel_fb_obj_invalidate(obj, ORIGIN_CPU);
1080
1081 i915_gem_object_pin_pages(obj);
1082
1083 offset = args->offset;
1084 obj->dirty = 1;
1085
1086 #ifdef __NetBSD__
1087 while (0 < remain)
1088 #else
1089 for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents,
1090 offset >> PAGE_SHIFT)
1091 #endif
1092 {
1093 #ifdef __NetBSD__
1094 struct page *const page = i915_gem_object_get_page(obj,
1095 atop(offset));
1096 #else
1097 struct page *page = sg_page_iter_page(&sg_iter);
1098
1099 if (remain <= 0)
1100 break;
1101 #endif
1102
1103 /* Operation in this page
1104 *
1105 * shmem_page_offset = offset within page in shmem file
1106 * page_length = bytes to copy for this page
1107 */
1108 shmem_page_offset = offset_in_page(offset);
1109
1110 page_length = remain;
1111 if ((shmem_page_offset + page_length) > PAGE_SIZE)
1112 page_length = PAGE_SIZE - shmem_page_offset;
1113
1114 /* If we don't overwrite a cacheline completely we need to be
1115 * careful to have up-to-date data by first clflushing. Don't
1116 * overcomplicate things and flush the entire patch. */
1117 const int partial_cacheline_write = needs_clflush_before &&
1118 ((shmem_page_offset | page_length) & flush_mask);
1119
1120 page_do_bit17_swizzling = obj_do_bit17_swizzling &&
1121 (page_to_phys(page) & (1 << 17)) != 0;
1122
1123 ret = shmem_pwrite_fast(page, shmem_page_offset, page_length,
1124 user_data, page_do_bit17_swizzling,
1125 partial_cacheline_write,
1126 needs_clflush_after);
1127 if (ret == 0)
1128 goto next_page;
1129
1130 hit_slowpath = 1;
1131 mutex_unlock(&dev->struct_mutex);
1132 ret = shmem_pwrite_slow(page, shmem_page_offset, page_length,
1133 user_data, page_do_bit17_swizzling,
1134 partial_cacheline_write,
1135 needs_clflush_after);
1136
1137 mutex_lock(&dev->struct_mutex);
1138
1139 if (ret)
1140 goto out;
1141
1142 next_page:
1143 remain -= page_length;
1144 user_data += page_length;
1145 offset += page_length;
1146 }
1147
1148 out:
1149 i915_gem_object_unpin_pages(obj);
1150
1151 if (hit_slowpath) {
1152 /*
1153 * Fixup: Flush cpu caches in case we didn't flush the dirty
1154 * cachelines in-line while writing and the object moved
1155 * out of the cpu write domain while we've dropped the lock.
1156 */
1157 if (!needs_clflush_after &&
1158 obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
1159 if (i915_gem_clflush_object(obj, obj->pin_display))
1160 needs_clflush_after = true;
1161 }
1162 }
1163
1164 if (needs_clflush_after)
1165 i915_gem_chipset_flush(dev);
1166 else
1167 obj->cache_dirty = true;
1168
1169 intel_fb_obj_flush(obj, false, ORIGIN_CPU);
1170 return ret;
1171 }
1172
1173 /**
1174 * Writes data to the object referenced by handle.
1175 *
1176 * On error, the contents of the buffer that were to be modified are undefined.
1177 */
1178 int
1179 i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
1180 struct drm_file *file)
1181 {
1182 struct drm_i915_private *dev_priv = dev->dev_private;
1183 struct drm_i915_gem_pwrite *args = data;
1184 struct drm_gem_object *gobj;
1185 struct drm_i915_gem_object *obj;
1186 int ret;
1187
1188 if (args->size == 0)
1189 return 0;
1190
1191 if (!access_ok(VERIFY_READ,
1192 to_user_ptr(args->data_ptr),
1193 args->size))
1194 return -EFAULT;
1195
1196 #ifndef __NetBSD__ /* XXX prefault */
1197 if (likely(!i915.prefault_disable)) {
1198 ret = fault_in_multipages_readable(to_user_ptr(args->data_ptr),
1199 args->size);
1200 if (ret)
1201 return -EFAULT;
1202 }
1203 #endif
1204
1205 intel_runtime_pm_get(dev_priv);
1206
1207 ret = i915_mutex_lock_interruptible(dev);
1208 if (ret)
1209 goto put_rpm;
1210
1211 gobj = drm_gem_object_lookup(dev, file, args->handle);
1212 if (gobj == NULL) {
1213 ret = -ENOENT;
1214 goto unlock;
1215 }
1216 obj = to_intel_bo(gobj);
1217
1218 /* Bounds check destination. */
1219 if (args->offset > obj->base.size ||
1220 args->size > obj->base.size - args->offset) {
1221 ret = -EINVAL;
1222 goto out;
1223 }
1224
1225 /* prime objects have no backing filp to GEM pread/pwrite
1226 * pages from.
1227 */
1228 #ifdef __NetBSD__
1229 /* Also stolen objects. */
1230 if (obj->base.gemo_shm_uao == NULL) {
1231 ret = -EINVAL;
1232 goto out;
1233 }
1234 #else
1235 if (!obj->base.filp) {
1236 ret = -EINVAL;
1237 goto out;
1238 }
1239 #endif
1240
1241 trace_i915_gem_object_pwrite(obj, args->offset, args->size);
1242
1243 ret = -EFAULT;
1244 /* We can only do the GTT pwrite on untiled buffers, as otherwise
1245 * it would end up going through the fenced access, and we'll get
1246 * different detiling behavior between reading and writing.
1247 * pread/pwrite currently are reading and writing from the CPU
1248 * perspective, requiring manual detiling by the client.
1249 */
1250 if (obj->tiling_mode == I915_TILING_NONE &&
1251 obj->base.write_domain != I915_GEM_DOMAIN_CPU &&
1252 cpu_write_needs_clflush(obj)) {
1253 ret = i915_gem_gtt_pwrite_fast(dev, obj, args, file);
1254 /* Note that the gtt paths might fail with non-page-backed user
1255 * pointers (e.g. gtt mappings when moving data between
1256 * textures). Fallback to the shmem path in that case. */
1257 }
1258
1259 if (ret == -EFAULT || ret == -ENOSPC) {
1260 if (obj->phys_handle)
1261 ret = i915_gem_phys_pwrite(obj, args, file);
1262 else
1263 ret = i915_gem_shmem_pwrite(dev, obj, args, file);
1264 }
1265
1266 out:
1267 drm_gem_object_unreference(&obj->base);
1268 unlock:
1269 mutex_unlock(&dev->struct_mutex);
1270 put_rpm:
1271 intel_runtime_pm_put(dev_priv);
1272
1273 return ret;
1274 }
1275
1276 int
1277 i915_gem_check_wedge(struct i915_gpu_error *error,
1278 bool interruptible)
1279 {
1280 if (i915_reset_in_progress(error)) {
1281 /* Non-interruptible callers can't handle -EAGAIN, hence return
1282 * -EIO unconditionally for these. */
1283 if (!interruptible)
1284 return -EIO;
1285
1286 /* Recovery complete, but the reset failed ... */
1287 if (i915_terminally_wedged(error))
1288 return -EIO;
1289
1290 /*
1291 * Check if GPU Reset is in progress - we need intel_ring_begin
1292 * to work properly to reinit the hw state while the gpu is
1293 * still marked as reset-in-progress. Handle this with a flag.
1294 */
1295 if (!error->reload_in_reset)
1296 return -EAGAIN;
1297 }
1298
1299 return 0;
1300 }
1301
1302 #ifndef __NetBSD__
1303 static void fake_irq(unsigned long data)
1304 {
1305 wake_up_process((struct task_struct *)data);
1306 }
1307 #endif
1308
1309 static bool missed_irq(struct drm_i915_private *dev_priv,
1310 struct intel_engine_cs *ring)
1311 {
1312 return test_bit(ring->id, &dev_priv->gpu_error.missed_irq_rings);
1313 }
1314
1315 #ifndef __NetBSD__
1316 static unsigned long local_clock_us(unsigned *cpu)
1317 {
1318 unsigned long t;
1319
1320 /* Cheaply and approximately convert from nanoseconds to microseconds.
1321 * The result and subsequent calculations are also defined in the same
1322 * approximate microseconds units. The principal source of timing
1323 * error here is from the simple truncation.
1324 *
1325 * Note that local_clock() is only defined wrt to the current CPU;
1326 * the comparisons are no longer valid if we switch CPUs. Instead of
1327 * blocking preemption for the entire busywait, we can detect the CPU
1328 * switch and use that as indicator of system load and a reason to
1329 * stop busywaiting, see busywait_stop().
1330 */
1331 *cpu = get_cpu();
1332 t = local_clock() >> 10;
1333 put_cpu();
1334
1335 return t;
1336 }
1337
1338 static bool busywait_stop(unsigned long timeout, unsigned cpu)
1339 {
1340 unsigned this_cpu;
1341
1342 if (time_after(local_clock_us(&this_cpu), timeout))
1343 return true;
1344
1345 return this_cpu != cpu;
1346 }
1347 #endif
1348
1349 static int __i915_spin_request(struct drm_i915_gem_request *req, int state)
1350 {
1351 #ifndef __NetBSD__
1352 unsigned long timeout;
1353 unsigned cpu;
1354 #endif
1355
1356 /* When waiting for high frequency requests, e.g. during synchronous
1357 * rendering split between the CPU and GPU, the finite amount of time
1358 * required to set up the irq and wait upon it limits the response
1359 * rate. By busywaiting on the request completion for a short while we
1360 * can service the high frequency waits as quick as possible. However,
1361 * if it is a slow request, we want to sleep as quickly as possible.
1362 * The tradeoff between waiting and sleeping is roughly the time it
1363 * takes to sleep on a request, on the order of a microsecond.
1364 */
1365
1366 if (req->ring->irq_refcount)
1367 return -EBUSY;
1368
1369 /* Only spin if we know the GPU is processing this request */
1370 if (!i915_gem_request_started(req, true))
1371 return -EAGAIN;
1372
1373 #ifndef __NetBSD__ /* XXX No local clock in usec. */
1374 timeout = local_clock_us(&cpu) + 5;
1375 while (!need_resched()) {
1376 if (i915_gem_request_completed(req, true))
1377 return 0;
1378
1379 if (signal_pending_state(state, current))
1380 break;
1381
1382 if (busywait_stop(timeout, cpu))
1383 break;
1384
1385 cpu_relax_lowlatency();
1386 }
1387 #endif
1388
1389 if (i915_gem_request_completed(req, false))
1390 return 0;
1391
1392 return -EAGAIN;
1393 }
1394
1395 /**
1396 * __i915_wait_request - wait until execution of request has finished
1397 * @req: duh!
1398 * @reset_counter: reset sequence associated with the given request
1399 * @interruptible: do an interruptible wait (normally yes)
1400 * @timeout: in - how long to wait (NULL forever); out - how much time remaining
1401 *
1402 * Note: It is of utmost importance that the passed in seqno and reset_counter
1403 * values have been read by the caller in an smp safe manner. Where read-side
1404 * locks are involved, it is sufficient to read the reset_counter before
1405 * unlocking the lock that protects the seqno. For lockless tricks, the
1406 * reset_counter _must_ be read before, and an appropriate smp_rmb must be
1407 * inserted.
1408 *
1409 * Returns 0 if the request was found within the alloted time. Else returns the
1410 * errno with remaining time filled in timeout argument.
1411 */
1412 int __i915_wait_request(struct drm_i915_gem_request *req,
1413 unsigned reset_counter,
1414 bool interruptible,
1415 s64 *timeout,
1416 struct intel_rps_client *rps)
1417 {
1418 struct intel_engine_cs *ring = i915_gem_request_get_ring(req);
1419 struct drm_device *dev = ring->dev;
1420 struct drm_i915_private *dev_priv = dev->dev_private;
1421 const bool irq_test_in_progress =
1422 dev_priv->gpu_error.test_irq_rings & intel_ring_flag(ring);
1423 #ifdef __NetBSD__
1424 int state = 0;
1425 bool wedged;
1426 #else
1427 int state = interruptible ? TASK_INTERRUPTIBLE : TASK_UNINTERRUPTIBLE;
1428 DEFINE_WAIT(wait);
1429 unsigned long timeout_expire;
1430 #endif
1431 s64 before, now;
1432 int ret;
1433
1434 __insn_barrier(); /* ACCESS_ONCE for irq_test_in_progress */
1435
1436 WARN(!intel_irqs_enabled(dev_priv), "IRQs disabled");
1437
1438 if (list_empty(&req->list))
1439 return 0;
1440
1441 if (i915_gem_request_completed(req, true))
1442 return 0;
1443
1444 #ifndef __NetBSD__
1445 timeout_expire = 0;
1446 if (timeout) {
1447 if (WARN_ON(*timeout < 0))
1448 return -EINVAL;
1449
1450 if (*timeout == 0)
1451 return -ETIME;
1452
1453 timeout_expire = jiffies + nsecs_to_jiffies_timeout(*timeout);
1454 }
1455 #endif
1456
1457 if (INTEL_INFO(dev_priv)->gen >= 6)
1458 gen6_rps_boost(dev_priv, rps, req->emitted_jiffies);
1459
1460 /* Record current time in case interrupted by signal, or wedged */
1461 trace_i915_gem_request_wait_begin(req);
1462 before = ktime_get_raw_ns();
1463
1464 /* Optimistic spin for the next jiffie before touching IRQs */
1465 ret = __i915_spin_request(req, state);
1466 if (ret == 0)
1467 goto out;
1468
1469 if (!irq_test_in_progress && WARN_ON(!ring->irq_get(ring))) {
1470 ret = -ENODEV;
1471 goto out;
1472 }
1473
1474 #ifdef __NetBSD__
1475 # define EXIT_COND \
1476 ((wedged = (reset_counter != \
1477 atomic_read(&dev_priv->gpu_error.reset_counter))) || \
1478 i915_gem_request_completed(req, false))
1479 if (timeout) {
1480 int ticks = missed_irq(dev_priv, ring) ? 1 :
1481 nsecs_to_jiffies_timeout(*timeout);
1482 if (interruptible) {
1483 DRM_SPIN_TIMED_WAIT_UNTIL(ret, &ring->irq_queue,
1484 &dev_priv->irq_lock, ticks, EXIT_COND);
1485 } else {
1486 DRM_SPIN_TIMED_WAIT_NOINTR_UNTIL(ret, &ring->irq_queue,
1487 &dev_priv->irq_lock, ticks, EXIT_COND);
1488 }
1489 if (ret < 0) /* Failure: return negative error as is. */
1490 ;
1491 else if (ret == 0) /* Timed out: return -ETIME. */
1492 ret = -ETIME;
1493 else /* Succeeded (ret > 0): return 0. */
1494 ret = 0;
1495 } else {
1496 if (interruptible) {
1497 DRM_SPIN_WAIT_UNTIL(ret, &ring->irq_queue,
1498 &dev_priv->irq_lock, EXIT_COND);
1499 } else {
1500 DRM_SPIN_WAIT_NOINTR_UNTIL(ret, &ring->irq_queue,
1501 &dev_priv->irq_lock, EXIT_COND);
1502 }
1503 /* ret is negative on failure or zero on success. */
1504 }
1505 if (wedged) {
1506 ret = i915_gem_check_wedge(&dev_priv->gpu_error, interruptible);
1507 if (ret == 0)
1508 ret = -EAGAIN;
1509 }
1510 #else
1511 for (;;) {
1512 struct timer_list timer;
1513
1514 prepare_to_wait(&ring->irq_queue, &wait, state);
1515
1516 /* We need to check whether any gpu reset happened in between
1517 * the caller grabbing the seqno and now ... */
1518 if (reset_counter != atomic_read(&dev_priv->gpu_error.reset_counter)) {
1519 /* ... but upgrade the -EAGAIN to an -EIO if the gpu
1520 * is truely gone. */
1521 ret = i915_gem_check_wedge(&dev_priv->gpu_error, interruptible);
1522 if (ret == 0)
1523 ret = -EAGAIN;
1524 break;
1525 }
1526
1527 if (i915_gem_request_completed(req, false)) {
1528 ret = 0;
1529 break;
1530 }
1531
1532 if (signal_pending_state(state, current)) {
1533 ret = -ERESTARTSYS;
1534 break;
1535 }
1536
1537 if (timeout && time_after_eq(jiffies, timeout_expire)) {
1538 ret = -ETIME;
1539 break;
1540 }
1541
1542 timer.function = NULL;
1543 if (timeout || missed_irq(dev_priv, ring)) {
1544 unsigned long expire;
1545
1546 setup_timer_on_stack(&timer, fake_irq, (unsigned long)current);
1547 expire = missed_irq(dev_priv, ring) ? jiffies + 1 : timeout_expire;
1548 mod_timer(&timer, expire);
1549 }
1550
1551 io_schedule();
1552
1553 if (timer.function) {
1554 del_singleshot_timer_sync(&timer);
1555 destroy_timer_on_stack(&timer);
1556 }
1557 }
1558 #endif
1559 if (!irq_test_in_progress)
1560 ring->irq_put(ring);
1561
1562 #ifndef __NetBSD__
1563 finish_wait(&ring->irq_queue, &wait);
1564 #endif
1565
1566 out:
1567 now = ktime_get_raw_ns();
1568 trace_i915_gem_request_wait_end(req);
1569
1570 if (timeout) {
1571 s64 tres = *timeout - (now - before);
1572
1573 *timeout = tres < 0 ? 0 : tres;
1574
1575 /*
1576 * Apparently ktime isn't accurate enough and occasionally has a
1577 * bit of mismatch in the jiffies<->nsecs<->ktime loop. So patch
1578 * things up to make the test happy. We allow up to 1 jiffy.
1579 *
1580 * This is a regrssion from the timespec->ktime conversion.
1581 */
1582 if (ret == -ETIME && *timeout < jiffies_to_usecs(1)*1000)
1583 *timeout = 0;
1584 }
1585
1586 return ret;
1587 }
1588
1589 int i915_gem_request_add_to_client(struct drm_i915_gem_request *req,
1590 struct drm_file *file)
1591 {
1592 struct drm_i915_private *dev_private __unused;
1593 struct drm_i915_file_private *file_priv;
1594
1595 WARN_ON(!req || !file || req->file_priv);
1596
1597 if (!req || !file)
1598 return -EINVAL;
1599
1600 if (req->file_priv)
1601 return -EINVAL;
1602
1603 dev_private = req->ring->dev->dev_private;
1604 file_priv = file->driver_priv;
1605
1606 spin_lock(&file_priv->mm.lock);
1607 req->file_priv = file_priv;
1608 list_add_tail(&req->client_list, &file_priv->mm.request_list);
1609 spin_unlock(&file_priv->mm.lock);
1610
1611 #ifndef __NetBSD__
1612 req->pid = get_pid(task_pid(current));
1613 #endif
1614
1615 return 0;
1616 }
1617
1618 static inline void
1619 i915_gem_request_remove_from_client(struct drm_i915_gem_request *request)
1620 {
1621 struct drm_i915_file_private *file_priv = request->file_priv;
1622
1623 if (!file_priv)
1624 return;
1625
1626 spin_lock(&file_priv->mm.lock);
1627 list_del(&request->client_list);
1628 request->file_priv = NULL;
1629 spin_unlock(&file_priv->mm.lock);
1630
1631 #ifndef __NetBSD__
1632 put_pid(request->pid);
1633 request->pid = NULL;
1634 #endif
1635 }
1636
1637 static void i915_gem_request_retire(struct drm_i915_gem_request *request)
1638 {
1639 trace_i915_gem_request_retire(request);
1640
1641 /* We know the GPU must have read the request to have
1642 * sent us the seqno + interrupt, so use the position
1643 * of tail of the request to update the last known position
1644 * of the GPU head.
1645 *
1646 * Note this requires that we are always called in request
1647 * completion order.
1648 */
1649 request->ringbuf->last_retired_head = request->postfix;
1650
1651 list_del_init(&request->list);
1652 i915_gem_request_remove_from_client(request);
1653
1654 i915_gem_request_unreference(request);
1655 }
1656
1657 static void
1658 __i915_gem_request_retire__upto(struct drm_i915_gem_request *req)
1659 {
1660 struct intel_engine_cs *engine = req->ring;
1661 struct drm_i915_gem_request *tmp;
1662
1663 lockdep_assert_held(&engine->dev->struct_mutex);
1664
1665 if (list_empty(&req->list))
1666 return;
1667
1668 do {
1669 tmp = list_first_entry(&engine->request_list,
1670 typeof(*tmp), list);
1671
1672 i915_gem_request_retire(tmp);
1673 } while (tmp != req);
1674
1675 WARN_ON(i915_verify_lists(engine->dev));
1676 }
1677
1678 /**
1679 * Waits for a request to be signaled, and cleans up the
1680 * request and object lists appropriately for that event.
1681 */
1682 int
1683 i915_wait_request(struct drm_i915_gem_request *req)
1684 {
1685 struct drm_device *dev;
1686 struct drm_i915_private *dev_priv;
1687 bool interruptible;
1688 int ret;
1689
1690 BUG_ON(req == NULL);
1691
1692 dev = req->ring->dev;
1693 dev_priv = dev->dev_private;
1694 interruptible = dev_priv->mm.interruptible;
1695
1696 BUG_ON(!mutex_is_locked(&dev->struct_mutex));
1697
1698 ret = i915_gem_check_wedge(&dev_priv->gpu_error, interruptible);
1699 if (ret)
1700 return ret;
1701
1702 ret = __i915_wait_request(req,
1703 atomic_read(&dev_priv->gpu_error.reset_counter),
1704 interruptible, NULL, NULL);
1705 if (ret)
1706 return ret;
1707
1708 __i915_gem_request_retire__upto(req);
1709 return 0;
1710 }
1711
1712 /**
1713 * Ensures that all rendering to the object has completed and the object is
1714 * safe to unbind from the GTT or access from the CPU.
1715 */
1716 int
1717 i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj,
1718 bool readonly)
1719 {
1720 int ret, i;
1721
1722 if (!obj->active)
1723 return 0;
1724
1725 if (readonly) {
1726 if (obj->last_write_req != NULL) {
1727 ret = i915_wait_request(obj->last_write_req);
1728 if (ret)
1729 return ret;
1730
1731 i = obj->last_write_req->ring->id;
1732 if (obj->last_read_req[i] == obj->last_write_req)
1733 i915_gem_object_retire__read(obj, i);
1734 else
1735 i915_gem_object_retire__write(obj);
1736 }
1737 } else {
1738 for (i = 0; i < I915_NUM_RINGS; i++) {
1739 if (obj->last_read_req[i] == NULL)
1740 continue;
1741
1742 ret = i915_wait_request(obj->last_read_req[i]);
1743 if (ret)
1744 return ret;
1745
1746 i915_gem_object_retire__read(obj, i);
1747 }
1748 RQ_BUG_ON(obj->active);
1749 }
1750
1751 return 0;
1752 }
1753
1754 static void
1755 i915_gem_object_retire_request(struct drm_i915_gem_object *obj,
1756 struct drm_i915_gem_request *req)
1757 {
1758 int ring = req->ring->id;
1759
1760 if (obj->last_read_req[ring] == req)
1761 i915_gem_object_retire__read(obj, ring);
1762 else if (obj->last_write_req == req)
1763 i915_gem_object_retire__write(obj);
1764
1765 __i915_gem_request_retire__upto(req);
1766 }
1767
1768 /* A nonblocking variant of the above wait. This is a highly dangerous routine
1769 * as the object state may change during this call.
1770 */
1771 static __must_check int
1772 i915_gem_object_wait_rendering__nonblocking(struct drm_i915_gem_object *obj,
1773 struct intel_rps_client *rps,
1774 bool readonly)
1775 {
1776 struct drm_device *dev = obj->base.dev;
1777 struct drm_i915_private *dev_priv = dev->dev_private;
1778 struct drm_i915_gem_request *requests[I915_NUM_RINGS];
1779 unsigned reset_counter;
1780 int ret, i, n = 0;
1781
1782 BUG_ON(!mutex_is_locked(&dev->struct_mutex));
1783 BUG_ON(!dev_priv->mm.interruptible);
1784
1785 if (!obj->active)
1786 return 0;
1787
1788 ret = i915_gem_check_wedge(&dev_priv->gpu_error, true);
1789 if (ret)
1790 return ret;
1791
1792 reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter);
1793
1794 if (readonly) {
1795 struct drm_i915_gem_request *req;
1796
1797 req = obj->last_write_req;
1798 if (req == NULL)
1799 return 0;
1800
1801 requests[n++] = i915_gem_request_reference(req);
1802 } else {
1803 for (i = 0; i < I915_NUM_RINGS; i++) {
1804 struct drm_i915_gem_request *req;
1805
1806 req = obj->last_read_req[i];
1807 if (req == NULL)
1808 continue;
1809
1810 requests[n++] = i915_gem_request_reference(req);
1811 }
1812 }
1813
1814 mutex_unlock(&dev->struct_mutex);
1815 for (i = 0; ret == 0 && i < n; i++)
1816 ret = __i915_wait_request(requests[i], reset_counter, true,
1817 NULL, rps);
1818 mutex_lock(&dev->struct_mutex);
1819
1820 for (i = 0; i < n; i++) {
1821 if (ret == 0)
1822 i915_gem_object_retire_request(obj, requests[i]);
1823 i915_gem_request_unreference(requests[i]);
1824 }
1825
1826 return ret;
1827 }
1828
1829 static struct intel_rps_client *to_rps_client(struct drm_file *file)
1830 {
1831 struct drm_i915_file_private *fpriv = file->driver_priv;
1832 return &fpriv->rps;
1833 }
1834
1835 /**
1836 * Called when user space prepares to use an object with the CPU, either
1837 * through the mmap ioctl's mapping or a GTT mapping.
1838 */
1839 int
1840 i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
1841 struct drm_file *file)
1842 {
1843 struct drm_i915_gem_set_domain *args = data;
1844 struct drm_gem_object *gobj;
1845 struct drm_i915_gem_object *obj;
1846 uint32_t read_domains = args->read_domains;
1847 uint32_t write_domain = args->write_domain;
1848 int ret;
1849
1850 /* Only handle setting domains to types used by the CPU. */
1851 if (write_domain & I915_GEM_GPU_DOMAINS)
1852 return -EINVAL;
1853
1854 if (read_domains & I915_GEM_GPU_DOMAINS)
1855 return -EINVAL;
1856
1857 /* Having something in the write domain implies it's in the read
1858 * domain, and only that read domain. Enforce that in the request.
1859 */
1860 if (write_domain != 0 && read_domains != write_domain)
1861 return -EINVAL;
1862
1863 ret = i915_mutex_lock_interruptible(dev);
1864 if (ret)
1865 return ret;
1866
1867 gobj = drm_gem_object_lookup(dev, file, args->handle);
1868 if (gobj == NULL) {
1869 ret = -ENOENT;
1870 goto unlock;
1871 }
1872 obj = to_intel_bo(gobj);
1873
1874 /* Try to flush the object off the GPU without holding the lock.
1875 * We will repeat the flush holding the lock in the normal manner
1876 * to catch cases where we are gazumped.
1877 */
1878 ret = i915_gem_object_wait_rendering__nonblocking(obj,
1879 to_rps_client(file),
1880 !write_domain);
1881 if (ret)
1882 goto unref;
1883
1884 if (read_domains & I915_GEM_DOMAIN_GTT)
1885 ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0);
1886 else
1887 ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0);
1888
1889 if (write_domain != 0)
1890 intel_fb_obj_invalidate(obj,
1891 write_domain == I915_GEM_DOMAIN_GTT ?
1892 ORIGIN_GTT : ORIGIN_CPU);
1893
1894 unref:
1895 drm_gem_object_unreference(&obj->base);
1896 unlock:
1897 mutex_unlock(&dev->struct_mutex);
1898 return ret;
1899 }
1900
1901 /**
1902 * Called when user space has done writes to this buffer
1903 */
1904 int
1905 i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data,
1906 struct drm_file *file)
1907 {
1908 struct drm_i915_gem_sw_finish *args = data;
1909 struct drm_gem_object *gobj;
1910 struct drm_i915_gem_object *obj;
1911 int ret = 0;
1912
1913 ret = i915_mutex_lock_interruptible(dev);
1914 if (ret)
1915 return ret;
1916
1917 gobj = drm_gem_object_lookup(dev, file, args->handle);
1918 if (gobj == NULL) {
1919 ret = -ENOENT;
1920 goto unlock;
1921 }
1922 obj = to_intel_bo(gobj);
1923
1924 /* Pinned buffers may be scanout, so flush the cache */
1925 if (obj->pin_display)
1926 i915_gem_object_flush_cpu_write_domain(obj);
1927
1928 drm_gem_object_unreference(&obj->base);
1929 unlock:
1930 mutex_unlock(&dev->struct_mutex);
1931 return ret;
1932 }
1933
1934 /**
1935 * Maps the contents of an object, returning the address it is mapped
1936 * into.
1937 *
1938 * While the mapping holds a reference on the contents of the object, it doesn't
1939 * imply a ref on the object itself.
1940 *
1941 * IMPORTANT:
1942 *
1943 * DRM driver writers who look a this function as an example for how to do GEM
1944 * mmap support, please don't implement mmap support like here. The modern way
1945 * to implement DRM mmap support is with an mmap offset ioctl (like
1946 * i915_gem_mmap_gtt) and then using the mmap syscall on the DRM fd directly.
1947 * That way debug tooling like valgrind will understand what's going on, hiding
1948 * the mmap call in a driver private ioctl will break that. The i915 driver only
1949 * does cpu mmaps this way because we didn't know better.
1950 */
1951 int
1952 i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
1953 struct drm_file *file)
1954 {
1955 struct drm_i915_gem_mmap *args = data;
1956 struct drm_gem_object *obj;
1957 unsigned long addr;
1958 #ifdef __NetBSD__
1959 int ret;
1960 #endif
1961
1962 if (args->flags & ~(I915_MMAP_WC))
1963 return -EINVAL;
1964
1965 #if 0
1966 /* XXX cpu_has_pat == CPUID_PAT, do we care to do this check */
1967 if (args->flags & I915_MMAP_WC && !cpu_has_pat)
1968 return -ENODEV;
1969 #endif
1970
1971 obj = drm_gem_object_lookup(dev, file, args->handle);
1972 if (obj == NULL)
1973 return -ENOENT;
1974
1975 /* prime objects have no backing filp to GEM mmap
1976 * pages from.
1977 */
1978 #ifdef __NetBSD__
1979 /* Also stolen objects (XXX can we get them here?) */
1980 if (obj->gemo_shm_uao == NULL) {
1981 drm_gem_object_unreference_unlocked(obj);
1982 return -EINVAL;
1983 }
1984 #else
1985 if (!obj->filp) {
1986 drm_gem_object_unreference_unlocked(obj);
1987 return -EINVAL;
1988 }
1989 #endif
1990
1991 #ifdef __NetBSD__
1992 addr = (*curproc->p_emul->e_vm_default_addr)(curproc,
1993 (vaddr_t)curproc->p_vmspace->vm_daddr, args->size,
1994 curproc->p_vmspace->vm_map.flags & VM_MAP_TOPDOWN);
1995 /* XXX errno NetBSD->Linux */
1996 ret = -uvm_map(&curproc->p_vmspace->vm_map, &addr, args->size,
1997 obj->gemo_shm_uao, args->offset, 0,
1998 UVM_MAPFLAG((VM_PROT_READ | VM_PROT_WRITE),
1999 (VM_PROT_READ | VM_PROT_WRITE), UVM_INH_COPY, UVM_ADV_NORMAL,
2000 0));
2001 if (ret) {
2002 drm_gem_object_unreference_unlocked(obj);
2003 return ret;
2004 }
2005 uao_reference(obj->gemo_shm_uao);
2006 drm_gem_object_unreference_unlocked(obj);
2007 #else
2008 addr = vm_mmap(obj->filp, 0, args->size,
2009 PROT_READ | PROT_WRITE, MAP_SHARED,
2010 args->offset);
2011 if (args->flags & I915_MMAP_WC) {
2012 struct mm_struct *mm = current->mm;
2013 struct vm_area_struct *vma;
2014
2015 down_write(&mm->mmap_sem);
2016 vma = find_vma(mm, addr);
2017 if (vma)
2018 vma->vm_page_prot =
2019 pgprot_writecombine(vm_get_page_prot(vma->vm_flags));
2020 else
2021 addr = -ENOMEM;
2022 up_write(&mm->mmap_sem);
2023 }
2024 drm_gem_object_unreference_unlocked(obj);
2025 if (IS_ERR((void *)addr))
2026 return addr;
2027 #endif
2028
2029 args->addr_ptr = (uint64_t) addr;
2030
2031 return 0;
2032 }
2033
2034 #ifdef __NetBSD__ /* XXX gem gtt fault */
2035 static int i915_udv_fault(struct uvm_faultinfo *, vaddr_t,
2036 struct vm_page **, int, int, vm_prot_t, int, paddr_t);
2037
2038 int
2039 i915_gem_fault(struct uvm_faultinfo *ufi, vaddr_t vaddr, struct vm_page **pps,
2040 int npages, int centeridx, vm_prot_t access_type, int flags)
2041 {
2042 struct uvm_object *uobj = ufi->entry->object.uvm_obj;
2043 struct drm_gem_object *gem_obj =
2044 container_of(uobj, struct drm_gem_object, gemo_uvmobj);
2045 struct drm_i915_gem_object *obj = to_intel_bo(gem_obj);
2046 struct drm_device *dev = obj->base.dev;
2047 struct drm_i915_private *dev_priv = dev->dev_private;
2048 voff_t byte_offset;
2049 pgoff_t page_offset;
2050 int ret = 0;
2051 bool write = ISSET(access_type, VM_PROT_WRITE)? 1 : 0;
2052
2053 byte_offset = (ufi->entry->offset + (vaddr - ufi->entry->start));
2054 KASSERT(byte_offset <= obj->base.size);
2055 page_offset = (byte_offset >> PAGE_SHIFT);
2056
2057 intel_runtime_pm_get(dev_priv);
2058
2059 /* Thanks, uvm, but we don't need this lock. */
2060 mutex_exit(uobj->vmobjlock);
2061
2062 ret = i915_mutex_lock_interruptible(dev);
2063 if (ret)
2064 goto out;
2065
2066 trace_i915_gem_object_fault(obj, page_offset, true, write);
2067
2068 ret = i915_gem_object_wait_rendering__nonblocking(obj, NULL, !write);
2069 if (ret)
2070 goto unlock;
2071
2072 if ((obj->cache_level != I915_CACHE_NONE) && !HAS_LLC(dev)) {
2073 ret = -EINVAL;
2074 goto unlock;
2075 }
2076
2077 ret = i915_gem_obj_ggtt_pin(obj, 0, PIN_MAPPABLE);
2078 if (ret)
2079 goto unlock;
2080
2081 ret = i915_gem_object_set_to_gtt_domain(obj, write);
2082 if (ret)
2083 goto unpin;
2084
2085 ret = i915_gem_object_get_fence(obj);
2086 if (ret)
2087 goto unpin;
2088
2089 obj->fault_mappable = true;
2090
2091 /* XXX errno NetBSD->Linux */
2092 ret = -i915_udv_fault(ufi, vaddr, pps, npages, centeridx, access_type,
2093 flags,
2094 (dev_priv->gtt.mappable_base + i915_gem_obj_ggtt_offset(obj)));
2095 unpin:
2096 i915_gem_object_ggtt_unpin(obj);
2097 unlock:
2098 mutex_unlock(&dev->struct_mutex);
2099 out:
2100 mutex_enter(uobj->vmobjlock);
2101 uvmfault_unlockall(ufi, ufi->entry->aref.ar_amap, uobj);
2102 if (ret == -ERESTART)
2103 uvm_wait("i915flt");
2104
2105 /*
2106 * Remap EINTR to success, so that we return to userland.
2107 * On the way out, we'll deliver the signal, and if the signal
2108 * is not fatal then the user code which faulted will most likely
2109 * fault again, and we'll come back here for another try.
2110 */
2111 if (ret == -EINTR)
2112 ret = 0;
2113 /* XXX Deal with GPU hangs here... */
2114 intel_runtime_pm_put(dev_priv);
2115 /* XXX errno Linux->NetBSD */
2116 return -ret;
2117 }
2118
2119 /*
2120 * XXX i915_udv_fault is copypasta of udv_fault from uvm_device.c.
2121 *
2122 * XXX pmap_enter_default instead of pmap_enter because of a problem
2123 * with using weak aliases in kernel modules or something.
2124 */
2125 int pmap_enter_default(pmap_t, vaddr_t, paddr_t, vm_prot_t, unsigned);
2126
2127 static int
2128 i915_udv_fault(struct uvm_faultinfo *ufi, vaddr_t vaddr, struct vm_page **pps,
2129 int npages, int centeridx, vm_prot_t access_type, int flags,
2130 paddr_t gtt_paddr)
2131 {
2132 struct vm_map_entry *entry = ufi->entry;
2133 vaddr_t curr_va;
2134 off_t curr_offset;
2135 paddr_t paddr;
2136 u_int mmapflags;
2137 int lcv, retval;
2138 vm_prot_t mapprot;
2139 UVMHIST_FUNC("i915_udv_fault"); UVMHIST_CALLED(maphist);
2140 UVMHIST_LOG(maphist," flags=%jd", flags,0,0,0);
2141
2142 /*
2143 * we do not allow device mappings to be mapped copy-on-write
2144 * so we kill any attempt to do so here.
2145 */
2146
2147 if (UVM_ET_ISCOPYONWRITE(entry)) {
2148 UVMHIST_LOG(maphist, "<- failed -- COW entry (etype=0x%jx)",
2149 entry->etype, 0,0,0);
2150 return(EIO);
2151 }
2152
2153 /*
2154 * now we must determine the offset in udv to use and the VA to
2155 * use for pmap_enter. note that we always use orig_map's pmap
2156 * for pmap_enter (even if we have a submap). since virtual
2157 * addresses in a submap must match the main map, this is ok.
2158 */
2159
2160 /* udv offset = (offset from start of entry) + entry's offset */
2161 curr_offset = entry->offset + (vaddr - entry->start);
2162 /* pmap va = vaddr (virtual address of pps[0]) */
2163 curr_va = vaddr;
2164
2165 /*
2166 * loop over the page range entering in as needed
2167 */
2168
2169 retval = 0;
2170 for (lcv = 0 ; lcv < npages ; lcv++, curr_offset += PAGE_SIZE,
2171 curr_va += PAGE_SIZE) {
2172 if ((flags & PGO_ALLPAGES) == 0 && lcv != centeridx)
2173 continue;
2174
2175 if (pps[lcv] == PGO_DONTCARE)
2176 continue;
2177
2178 paddr = (gtt_paddr + curr_offset);
2179 mmapflags = 0;
2180 mapprot = ufi->entry->protection;
2181 UVMHIST_LOG(maphist,
2182 " MAPPING: device: pm=0x%#jx, va=0x%jx, pa=0x%jx, at=%jd",
2183 (uintptr_t)ufi->orig_map->pmap, curr_va, paddr, mapprot);
2184 if (pmap_enter_default(ufi->orig_map->pmap, curr_va, paddr, mapprot,
2185 PMAP_CANFAIL | mapprot | mmapflags) != 0) {
2186 /*
2187 * pmap_enter() didn't have the resource to
2188 * enter this mapping. Unlock everything,
2189 * wait for the pagedaemon to free up some
2190 * pages, and then tell uvm_fault() to start
2191 * the fault again.
2192 *
2193 * XXX Needs some rethinking for the PGO_ALLPAGES
2194 * XXX case.
2195 */
2196 pmap_update(ufi->orig_map->pmap); /* sync what we have so far */
2197 return (ERESTART);
2198 }
2199 }
2200
2201 pmap_update(ufi->orig_map->pmap);
2202 return (retval);
2203 }
2204 #else
2205 /**
2206 * i915_gem_fault - fault a page into the GTT
2207 * @vma: VMA in question
2208 * @vmf: fault info
2209 *
2210 * The fault handler is set up by drm_gem_mmap() when a object is GTT mapped
2211 * from userspace. The fault handler takes care of binding the object to
2212 * the GTT (if needed), allocating and programming a fence register (again,
2213 * only if needed based on whether the old reg is still valid or the object
2214 * is tiled) and inserting a new PTE into the faulting process.
2215 *
2216 * Note that the faulting process may involve evicting existing objects
2217 * from the GTT and/or fence registers to make room. So performance may
2218 * suffer if the GTT working set is large or there are few fence registers
2219 * left.
2220 */
2221 int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
2222 {
2223 struct drm_i915_gem_object *obj = to_intel_bo(vma->vm_private_data);
2224 struct drm_device *dev = obj->base.dev;
2225 struct drm_i915_private *dev_priv = dev->dev_private;
2226 struct i915_ggtt_view view = i915_ggtt_view_normal;
2227 pgoff_t page_offset;
2228 unsigned long pfn;
2229 int ret = 0;
2230 bool write = !!(vmf->flags & FAULT_FLAG_WRITE);
2231
2232 intel_runtime_pm_get(dev_priv);
2233
2234 /* We don't use vmf->pgoff since that has the fake offset */
2235 page_offset = ((unsigned long)vmf->virtual_address - vma->vm_start) >>
2236 PAGE_SHIFT;
2237
2238 ret = i915_mutex_lock_interruptible(dev);
2239 if (ret)
2240 goto out;
2241
2242 trace_i915_gem_object_fault(obj, page_offset, true, write);
2243
2244 /* Try to flush the object off the GPU first without holding the lock.
2245 * Upon reacquiring the lock, we will perform our sanity checks and then
2246 * repeat the flush holding the lock in the normal manner to catch cases
2247 * where we are gazumped.
2248 */
2249 ret = i915_gem_object_wait_rendering__nonblocking(obj, NULL, !write);
2250 if (ret)
2251 goto unlock;
2252
2253 /* Access to snoopable pages through the GTT is incoherent. */
2254 if (obj->cache_level != I915_CACHE_NONE && !HAS_LLC(dev)) {
2255 ret = -EFAULT;
2256 goto unlock;
2257 }
2258
2259 /* Use a partial view if the object is bigger than the aperture. */
2260 if (obj->base.size >= dev_priv->gtt.mappable_end &&
2261 obj->tiling_mode == I915_TILING_NONE) {
2262 static const unsigned int chunk_size = 256; // 1 MiB
2263
2264 memset(&view, 0, sizeof(view));
2265 view.type = I915_GGTT_VIEW_PARTIAL;
2266 view.params.partial.offset = rounddown(page_offset, chunk_size);
2267 view.params.partial.size =
2268 min_t(unsigned int,
2269 chunk_size,
2270 (vma->vm_end - vma->vm_start)/PAGE_SIZE -
2271 view.params.partial.offset);
2272 }
2273
2274 /* Now pin it into the GTT if needed */
2275 ret = i915_gem_object_ggtt_pin(obj, &view, 0, PIN_MAPPABLE);
2276 if (ret)
2277 goto unlock;
2278
2279 ret = i915_gem_object_set_to_gtt_domain(obj, write);
2280 if (ret)
2281 goto unpin;
2282
2283 ret = i915_gem_object_get_fence(obj);
2284 if (ret)
2285 goto unpin;
2286
2287 /* Finally, remap it using the new GTT offset */
2288 pfn = dev_priv->gtt.mappable_base +
2289 i915_gem_obj_ggtt_offset_view(obj, &view);
2290 pfn >>= PAGE_SHIFT;
2291
2292 if (unlikely(view.type == I915_GGTT_VIEW_PARTIAL)) {
2293 /* Overriding existing pages in partial view does not cause
2294 * us any trouble as TLBs are still valid because the fault
2295 * is due to userspace losing part of the mapping or never
2296 * having accessed it before (at this partials' range).
2297 */
2298 unsigned long base = vma->vm_start +
2299 (view.params.partial.offset << PAGE_SHIFT);
2300 unsigned int i;
2301
2302 for (i = 0; i < view.params.partial.size; i++) {
2303 ret = vm_insert_pfn(vma, base + i * PAGE_SIZE, pfn + i);
2304 if (ret)
2305 break;
2306 }
2307
2308 obj->fault_mappable = true;
2309 } else {
2310 if (!obj->fault_mappable) {
2311 unsigned long size = min_t(unsigned long,
2312 vma->vm_end - vma->vm_start,
2313 obj->base.size);
2314 int i;
2315
2316 for (i = 0; i < size >> PAGE_SHIFT; i++) {
2317 ret = vm_insert_pfn(vma,
2318 (unsigned long)vma->vm_start + i * PAGE_SIZE,
2319 pfn + i);
2320 if (ret)
2321 break;
2322 }
2323
2324 obj->fault_mappable = true;
2325 } else
2326 ret = vm_insert_pfn(vma,
2327 (unsigned long)vmf->virtual_address,
2328 pfn + page_offset);
2329 }
2330 unpin:
2331 i915_gem_object_ggtt_unpin_view(obj, &view);
2332 unlock:
2333 mutex_unlock(&dev->struct_mutex);
2334 out:
2335 switch (ret) {
2336 case -EIO:
2337 /*
2338 * We eat errors when the gpu is terminally wedged to avoid
2339 * userspace unduly crashing (gl has no provisions for mmaps to
2340 * fail). But any other -EIO isn't ours (e.g. swap in failure)
2341 * and so needs to be reported.
2342 */
2343 if (!i915_terminally_wedged(&dev_priv->gpu_error)) {
2344 ret = VM_FAULT_SIGBUS;
2345 break;
2346 }
2347 case -EAGAIN:
2348 /*
2349 * EAGAIN means the gpu is hung and we'll wait for the error
2350 * handler to reset everything when re-faulting in
2351 * i915_mutex_lock_interruptible.
2352 */
2353 case 0:
2354 case -ERESTARTSYS:
2355 case -EINTR:
2356 case -EBUSY:
2357 /*
2358 * EBUSY is ok: this just means that another thread
2359 * already did the job.
2360 */
2361 ret = VM_FAULT_NOPAGE;
2362 break;
2363 case -ENOMEM:
2364 ret = VM_FAULT_OOM;
2365 break;
2366 case -ENOSPC:
2367 case -EFAULT:
2368 ret = VM_FAULT_SIGBUS;
2369 break;
2370 default:
2371 WARN_ONCE(ret, "unhandled error in i915_gem_fault: %i\n", ret);
2372 ret = VM_FAULT_SIGBUS;
2373 break;
2374 }
2375
2376 intel_runtime_pm_put(dev_priv);
2377 return ret;
2378 }
2379 #endif
2380
2381 /**
2382 * i915_gem_release_mmap - remove physical page mappings
2383 * @obj: obj in question
2384 *
2385 * Preserve the reservation of the mmapping with the DRM core code, but
2386 * relinquish ownership of the pages back to the system.
2387 *
2388 * It is vital that we remove the page mapping if we have mapped a tiled
2389 * object through the GTT and then lose the fence register due to
2390 * resource pressure. Similarly if the object has been moved out of the
2391 * aperture, than pages mapped into userspace must be revoked. Removing the
2392 * mapping will then trigger a page fault on the next user access, allowing
2393 * fixup by i915_gem_fault().
2394 */
2395 void
2396 i915_gem_release_mmap(struct drm_i915_gem_object *obj)
2397 {
2398 if (!obj->fault_mappable)
2399 return;
2400
2401 #ifdef __NetBSD__ /* XXX gem gtt fault */
2402 {
2403 struct drm_device *const dev = obj->base.dev;
2404 struct drm_i915_private *const dev_priv = dev->dev_private;
2405 const paddr_t start = dev_priv->gtt.mappable_base +
2406 i915_gem_obj_ggtt_offset(obj);
2407 const size_t size = obj->base.size;
2408 const paddr_t end = start + size;
2409 paddr_t pa;
2410
2411 KASSERT((start & (PAGE_SIZE - 1)) == 0);
2412 KASSERT((size & (PAGE_SIZE - 1)) == 0);
2413
2414 for (pa = start; pa < end; pa += PAGE_SIZE)
2415 pmap_pv_protect(pa, VM_PROT_NONE);
2416 }
2417 #else
2418 drm_vma_node_unmap(&obj->base.vma_node,
2419 obj->base.dev->anon_inode->i_mapping);
2420 #endif
2421 obj->fault_mappable = false;
2422 }
2423
2424 void
2425 i915_gem_release_all_mmaps(struct drm_i915_private *dev_priv)
2426 {
2427 struct drm_i915_gem_object *obj;
2428
2429 list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list)
2430 i915_gem_release_mmap(obj);
2431 }
2432
2433 uint32_t
2434 i915_gem_get_gtt_size(struct drm_device *dev, uint32_t size, int tiling_mode)
2435 {
2436 uint32_t gtt_size;
2437
2438 if (INTEL_INFO(dev)->gen >= 4 ||
2439 tiling_mode == I915_TILING_NONE)
2440 return size;
2441
2442 /* Previous chips need a power-of-two fence region when tiling */
2443 if (INTEL_INFO(dev)->gen == 3)
2444 gtt_size = 1024*1024;
2445 else
2446 gtt_size = 512*1024;
2447
2448 while (gtt_size < size)
2449 gtt_size <<= 1;
2450
2451 return gtt_size;
2452 }
2453
2454 /**
2455 * i915_gem_get_gtt_alignment - return required GTT alignment for an object
2456 * @obj: object to check
2457 *
2458 * Return the required GTT alignment for an object, taking into account
2459 * potential fence register mapping.
2460 */
2461 uint32_t
2462 i915_gem_get_gtt_alignment(struct drm_device *dev, uint32_t size,
2463 int tiling_mode, bool fenced)
2464 {
2465 /*
2466 * Minimum alignment is 4k (GTT page size), but might be greater
2467 * if a fence register is needed for the object.
2468 */
2469 if (INTEL_INFO(dev)->gen >= 4 || (!fenced && IS_G33(dev)) ||
2470 tiling_mode == I915_TILING_NONE)
2471 return 4096;
2472
2473 /*
2474 * Previous chips need to be aligned to the size of the smallest
2475 * fence register that can contain the object.
2476 */
2477 return i915_gem_get_gtt_size(dev, size, tiling_mode);
2478 }
2479
2480 static int i915_gem_object_create_mmap_offset(struct drm_i915_gem_object *obj)
2481 {
2482 struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
2483 int ret;
2484
2485 if (drm_vma_node_has_offset(&obj->base.vma_node))
2486 return 0;
2487
2488 dev_priv->mm.shrinker_no_lock_stealing = true;
2489
2490 ret = drm_gem_create_mmap_offset(&obj->base);
2491 if (ret != -ENOSPC)
2492 goto out;
2493
2494 /* Badly fragmented mmap space? The only way we can recover
2495 * space is by destroying unwanted objects. We can't randomly release
2496 * mmap_offsets as userspace expects them to be persistent for the
2497 * lifetime of the objects. The closest we can is to release the
2498 * offsets on purgeable objects by truncating it and marking it purged,
2499 * which prevents userspace from ever using that object again.
2500 */
2501 i915_gem_shrink(dev_priv,
2502 obj->base.size >> PAGE_SHIFT,
2503 I915_SHRINK_BOUND |
2504 I915_SHRINK_UNBOUND |
2505 I915_SHRINK_PURGEABLE);
2506 ret = drm_gem_create_mmap_offset(&obj->base);
2507 if (ret != -ENOSPC)
2508 goto out;
2509
2510 i915_gem_shrink_all(dev_priv);
2511 ret = drm_gem_create_mmap_offset(&obj->base);
2512 out:
2513 dev_priv->mm.shrinker_no_lock_stealing = false;
2514
2515 return ret;
2516 }
2517
2518 static void i915_gem_object_free_mmap_offset(struct drm_i915_gem_object *obj)
2519 {
2520 drm_gem_free_mmap_offset(&obj->base);
2521 }
2522
2523 int
2524 i915_gem_mmap_gtt(struct drm_file *file,
2525 struct drm_device *dev,
2526 uint32_t handle,
2527 uint64_t *offset)
2528 {
2529 struct drm_gem_object *gobj;
2530 struct drm_i915_gem_object *obj;
2531 int ret;
2532
2533 ret = i915_mutex_lock_interruptible(dev);
2534 if (ret)
2535 return ret;
2536
2537 gobj = drm_gem_object_lookup(dev, file, handle);
2538 if (gobj == NULL) {
2539 ret = -ENOENT;
2540 goto unlock;
2541 }
2542 obj = to_intel_bo(gobj);
2543
2544 if (obj->madv != I915_MADV_WILLNEED) {
2545 DRM_DEBUG("Attempting to mmap a purgeable buffer\n");
2546 ret = -EFAULT;
2547 goto out;
2548 }
2549
2550 ret = i915_gem_object_create_mmap_offset(obj);
2551 if (ret)
2552 goto out;
2553
2554 *offset = drm_vma_node_offset_addr(&obj->base.vma_node);
2555
2556 out:
2557 drm_gem_object_unreference(&obj->base);
2558 unlock:
2559 mutex_unlock(&dev->struct_mutex);
2560 return ret;
2561 }
2562
2563 /**
2564 * i915_gem_mmap_gtt_ioctl - prepare an object for GTT mmap'ing
2565 * @dev: DRM device
2566 * @data: GTT mapping ioctl data
2567 * @file: GEM object info
2568 *
2569 * Simply returns the fake offset to userspace so it can mmap it.
2570 * The mmap call will end up in drm_gem_mmap(), which will set things
2571 * up so we can get faults in the handler above.
2572 *
2573 * The fault handler will take care of binding the object into the GTT
2574 * (since it may have been evicted to make room for something), allocating
2575 * a fence register, and mapping the appropriate aperture address into
2576 * userspace.
2577 */
2578 int
2579 i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data,
2580 struct drm_file *file)
2581 {
2582 struct drm_i915_gem_mmap_gtt *args = data;
2583
2584 return i915_gem_mmap_gtt(file, dev, args->handle, &args->offset);
2585 }
2586
2587 /* Immediately discard the backing storage */
2588 static void
2589 i915_gem_object_truncate(struct drm_i915_gem_object *obj)
2590 {
2591 i915_gem_object_free_mmap_offset(obj);
2592
2593 #ifdef __NetBSD__
2594 if (obj->base.gemo_shm_uao == NULL)
2595 return;
2596
2597 {
2598 struct uvm_object *const uobj = obj->base.gemo_shm_uao;
2599
2600 if (uobj != NULL) {
2601 /* XXX Calling pgo_put like this is bogus. */
2602 mutex_enter(uobj->vmobjlock);
2603 (*uobj->pgops->pgo_put)(uobj, 0, obj->base.size,
2604 (PGO_ALLPAGES | PGO_FREE));
2605 }
2606 }
2607 #else
2608 if (obj->base.filp == NULL)
2609 return;
2610
2611 /* Our goal here is to return as much of the memory as
2612 * is possible back to the system as we are called from OOM.
2613 * To do this we must instruct the shmfs to drop all of its
2614 * backing pages, *now*.
2615 */
2616 shmem_truncate_range(file_inode(obj->base.filp), 0, (loff_t)-1);
2617 #endif
2618 obj->madv = __I915_MADV_PURGED;
2619 }
2620
2621 /* Try to discard unwanted pages */
2622 static void
2623 i915_gem_object_invalidate(struct drm_i915_gem_object *obj)
2624 {
2625 #ifdef __NetBSD__
2626 panic("XXX");
2627 #else
2628 struct address_space *mapping;
2629
2630 switch (obj->madv) {
2631 case I915_MADV_DONTNEED:
2632 i915_gem_object_truncate(obj);
2633 case __I915_MADV_PURGED:
2634 return;
2635 }
2636
2637 if (obj->base.filp == NULL)
2638 return;
2639
2640 mapping = file_inode(obj->base.filp)->i_mapping,
2641 invalidate_mapping_pages(mapping, 0, (loff_t)-1);
2642 #endif
2643 }
2644
2645 static void
2646 i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj)
2647 {
2648 #ifdef __NetBSD__
2649 struct drm_device *const dev = obj->base.dev;
2650 struct vm_page *page;
2651 int ret;
2652
2653 /* XXX Cargo-culted from the Linux code. */
2654 BUG_ON(obj->madv == __I915_MADV_PURGED);
2655
2656 ret = i915_gem_object_set_to_cpu_domain(obj, true);
2657 if (ret) {
2658 WARN_ON(ret != -EIO);
2659 i915_gem_clflush_object(obj, true);
2660 obj->base.read_domains = obj->base.write_domain =
2661 I915_GEM_DOMAIN_CPU;
2662 }
2663
2664 i915_gem_gtt_finish_object(obj);
2665
2666 if (i915_gem_object_needs_bit17_swizzle(obj))
2667 i915_gem_object_save_bit_17_swizzle(obj);
2668
2669 if (obj->madv == I915_MADV_DONTNEED)
2670 obj->dirty = 0;
2671
2672 if (obj->dirty) {
2673 TAILQ_FOREACH(page, &obj->pageq, pageq.queue) {
2674 page->flags &= ~PG_CLEAN;
2675 /* XXX mark page accessed */
2676 }
2677 }
2678 obj->dirty = 0;
2679
2680 uvm_obj_unwirepages(obj->base.gemo_shm_uao, 0, obj->base.size);
2681 bus_dmamap_destroy(dev->dmat, obj->pages);
2682 #else
2683 struct sg_page_iter sg_iter;
2684 int ret;
2685
2686 BUG_ON(obj->madv == __I915_MADV_PURGED);
2687
2688 ret = i915_gem_object_set_to_cpu_domain(obj, true);
2689 if (ret) {
2690 /* In the event of a disaster, abandon all caches and
2691 * hope for the best.
2692 */
2693 WARN_ON(ret != -EIO);
2694 i915_gem_clflush_object(obj, true);
2695 obj->base.read_domains = obj->base.write_domain = I915_GEM_DOMAIN_CPU;
2696 }
2697
2698 i915_gem_gtt_finish_object(obj);
2699
2700 if (i915_gem_object_needs_bit17_swizzle(obj))
2701 i915_gem_object_save_bit_17_swizzle(obj);
2702
2703 if (obj->madv == I915_MADV_DONTNEED)
2704 obj->dirty = 0;
2705
2706 for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents, 0) {
2707 struct page *page = sg_page_iter_page(&sg_iter);
2708
2709 if (obj->dirty)
2710 set_page_dirty(page);
2711
2712 if (obj->madv == I915_MADV_WILLNEED)
2713 mark_page_accessed(page);
2714
2715 page_cache_release(page);
2716 }
2717 obj->dirty = 0;
2718
2719 sg_free_table(obj->pages);
2720 kfree(obj->pages);
2721 #endif
2722 }
2723
2724 int
2725 i915_gem_object_put_pages(struct drm_i915_gem_object *obj)
2726 {
2727 const struct drm_i915_gem_object_ops *ops = obj->ops;
2728
2729 if (obj->pages == NULL)
2730 return 0;
2731
2732 if (obj->pages_pin_count)
2733 return -EBUSY;
2734
2735 BUG_ON(i915_gem_obj_bound_any(obj));
2736
2737 /* ->put_pages might need to allocate memory for the bit17 swizzle
2738 * array, hence protect them from being reaped by removing them from gtt
2739 * lists early. */
2740 list_del(&obj->global_list);
2741
2742 ops->put_pages(obj);
2743 obj->pages = NULL;
2744
2745 i915_gem_object_invalidate(obj);
2746
2747 return 0;
2748 }
2749
2750 static int
2751 i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj)
2752 {
2753 #ifdef __NetBSD__
2754 struct drm_device *const dev = obj->base.dev;
2755 struct drm_i915_private *dev_priv = dev->dev_private;
2756 struct vm_page *page;
2757 int ret;
2758
2759 BUG_ON(obj->base.read_domains & I915_GEM_GPU_DOMAINS);
2760 BUG_ON(obj->base.write_domain & I915_GEM_GPU_DOMAINS);
2761
2762 KASSERT(obj->pages == NULL);
2763 TAILQ_INIT(&obj->pageq);
2764
2765 /* XXX errno NetBSD->Linux */
2766 ret = -bus_dmamap_create(dev->dmat, obj->base.size,
2767 obj->base.size/PAGE_SIZE, PAGE_SIZE, 0, BUS_DMA_NOWAIT,
2768 &obj->pages);
2769 if (ret)
2770 goto fail0;
2771
2772 /* XXX errno NetBSD->Linux */
2773 ret = -uvm_obj_wirepages(obj->base.gemo_shm_uao, 0, obj->base.size,
2774 &obj->pageq);
2775 if (ret) /* XXX Try purge, shrink. */
2776 goto fail1;
2777
2778 /*
2779 * Check that the paddrs will fit in 40 bits, or 32 bits on i965.
2780 *
2781 * XXX This should be unnecessary: the uao should guarantee
2782 * this constraint after uao_set_pgfl.
2783 *
2784 * XXX This should also be expanded for newer devices.
2785 */
2786 TAILQ_FOREACH(page, &obj->pageq, pageq.queue) {
2787 const uint64_t mask =
2788 (IS_BROADWATER(dev) || IS_CRESTLINE(dev)?
2789 0xffffffffULL : 0xffffffffffULL);
2790 if (VM_PAGE_TO_PHYS(page) & ~mask) {
2791 DRM_ERROR("GEM physical address exceeds %u bits"
2792 ": %"PRIxMAX"\n",
2793 popcount64(mask),
2794 (uintmax_t)VM_PAGE_TO_PHYS(page));
2795 ret = -EIO;
2796 goto fail2;
2797 }
2798 }
2799
2800 ret = i915_gem_gtt_prepare_object(obj);
2801 if (ret)
2802 goto fail2;
2803
2804 if (i915_gem_object_needs_bit17_swizzle(obj))
2805 i915_gem_object_do_bit_17_swizzle(obj);
2806
2807 if (obj->tiling_mode != I915_TILING_NONE &&
2808 dev_priv->quirks & QUIRK_PIN_SWIZZLED_PAGES)
2809 i915_gem_object_pin_pages(obj);
2810
2811 /* Success! */
2812 return 0;
2813
2814 fail3: __unused
2815 i915_gem_gtt_finish_object(obj);
2816 fail2: uvm_obj_unwirepages(obj->base.gemo_shm_uao, 0, obj->base.size);
2817 fail1: bus_dmamap_destroy(dev->dmat, obj->pages);
2818 obj->pages = NULL;
2819 fail0: KASSERT(ret);
2820 return ret;
2821 #else
2822 struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
2823 int page_count, i;
2824 struct address_space *mapping;
2825 struct sg_table *st;
2826 struct scatterlist *sg;
2827 struct sg_page_iter sg_iter;
2828 struct page *page;
2829 unsigned long last_pfn = 0; /* suppress gcc warning */
2830 int ret;
2831 gfp_t gfp;
2832
2833 /* Assert that the object is not currently in any GPU domain. As it
2834 * wasn't in the GTT, there shouldn't be any way it could have been in
2835 * a GPU cache
2836 */
2837 BUG_ON(obj->base.read_domains & I915_GEM_GPU_DOMAINS);
2838 BUG_ON(obj->base.write_domain & I915_GEM_GPU_DOMAINS);
2839
2840 st = kmalloc(sizeof(*st), GFP_KERNEL);
2841 if (st == NULL)
2842 return -ENOMEM;
2843
2844 page_count = obj->base.size / PAGE_SIZE;
2845 if (sg_alloc_table(st, page_count, GFP_KERNEL)) {
2846 kfree(st);
2847 return -ENOMEM;
2848 }
2849
2850 /* Get the list of pages out of our struct file. They'll be pinned
2851 * at this point until we release them.
2852 *
2853 * Fail silently without starting the shrinker
2854 */
2855 mapping = file_inode(obj->base.filp)->i_mapping;
2856 gfp = mapping_gfp_constraint(mapping, ~(__GFP_IO | __GFP_RECLAIM));
2857 gfp |= __GFP_NORETRY | __GFP_NOWARN;
2858 sg = st->sgl;
2859 st->nents = 0;
2860 for (i = 0; i < page_count; i++) {
2861 page = shmem_read_mapping_page_gfp(mapping, i, gfp);
2862 if (IS_ERR(page)) {
2863 i915_gem_shrink(dev_priv,
2864 page_count,
2865 I915_SHRINK_BOUND |
2866 I915_SHRINK_UNBOUND |
2867 I915_SHRINK_PURGEABLE);
2868 page = shmem_read_mapping_page_gfp(mapping, i, gfp);
2869 }
2870 if (IS_ERR(page)) {
2871 /* We've tried hard to allocate the memory by reaping
2872 * our own buffer, now let the real VM do its job and
2873 * go down in flames if truly OOM.
2874 */
2875 i915_gem_shrink_all(dev_priv);
2876 page = shmem_read_mapping_page(mapping, i);
2877 if (IS_ERR(page)) {
2878 ret = PTR_ERR(page);
2879 goto err_pages;
2880 }
2881 }
2882 #ifdef CONFIG_SWIOTLB
2883 if (swiotlb_nr_tbl()) {
2884 st->nents++;
2885 sg_set_page(sg, page, PAGE_SIZE, 0);
2886 sg = sg_next(sg);
2887 continue;
2888 }
2889 #endif
2890 if (!i || page_to_pfn(page) != last_pfn + 1) {
2891 if (i)
2892 sg = sg_next(sg);
2893 st->nents++;
2894 sg_set_page(sg, page, PAGE_SIZE, 0);
2895 } else {
2896 sg->length += PAGE_SIZE;
2897 }
2898 last_pfn = page_to_pfn(page);
2899
2900 /* Check that the i965g/gm workaround works. */
2901 WARN_ON((gfp & __GFP_DMA32) && (last_pfn >= 0x00100000UL));
2902 }
2903 #ifdef CONFIG_SWIOTLB
2904 if (!swiotlb_nr_tbl())
2905 #endif
2906 sg_mark_end(sg);
2907 obj->pages = st;
2908
2909 ret = i915_gem_gtt_prepare_object(obj);
2910 if (ret)
2911 goto err_pages;
2912
2913 if (i915_gem_object_needs_bit17_swizzle(obj))
2914 i915_gem_object_do_bit_17_swizzle(obj);
2915
2916 if (obj->tiling_mode != I915_TILING_NONE &&
2917 dev_priv->quirks & QUIRK_PIN_SWIZZLED_PAGES)
2918 i915_gem_object_pin_pages(obj);
2919
2920 return 0;
2921
2922 err_pages:
2923 sg_mark_end(sg);
2924 for_each_sg_page(st->sgl, &sg_iter, st->nents, 0)
2925 page_cache_release(sg_page_iter_page(&sg_iter));
2926 sg_free_table(st);
2927 kfree(st);
2928
2929 /* shmemfs first checks if there is enough memory to allocate the page
2930 * and reports ENOSPC should there be insufficient, along with the usual
2931 * ENOMEM for a genuine allocation failure.
2932 *
2933 * We use ENOSPC in our driver to mean that we have run out of aperture
2934 * space and so want to translate the error from shmemfs back to our
2935 * usual understanding of ENOMEM.
2936 */
2937 if (ret == -ENOSPC)
2938 ret = -ENOMEM;
2939
2940 return ret;
2941 #endif
2942 }
2943
2944 /* Ensure that the associated pages are gathered from the backing storage
2945 * and pinned into our object. i915_gem_object_get_pages() may be called
2946 * multiple times before they are released by a single call to
2947 * i915_gem_object_put_pages() - once the pages are no longer referenced
2948 * either as a result of memory pressure (reaping pages under the shrinker)
2949 * or as the object is itself released.
2950 */
2951 int
2952 i915_gem_object_get_pages(struct drm_i915_gem_object *obj)
2953 {
2954 struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
2955 const struct drm_i915_gem_object_ops *ops = obj->ops;
2956 int ret;
2957
2958 if (obj->pages)
2959 return 0;
2960
2961 if (obj->madv != I915_MADV_WILLNEED) {
2962 DRM_DEBUG("Attempting to obtain a purgeable object\n");
2963 return -EFAULT;
2964 }
2965
2966 BUG_ON(obj->pages_pin_count);
2967
2968 ret = ops->get_pages(obj);
2969 if (ret)
2970 return ret;
2971
2972 list_add_tail(&obj->global_list, &dev_priv->mm.unbound_list);
2973
2974 #ifndef __NetBSD__
2975 obj->get_page.sg = obj->pages->sgl;
2976 obj->get_page.last = 0;
2977 #endif
2978
2979 return 0;
2980 }
2981
2982 void i915_vma_move_to_active(struct i915_vma *vma,
2983 struct drm_i915_gem_request *req)
2984 {
2985 struct drm_i915_gem_object *obj = vma->obj;
2986 struct intel_engine_cs *ring;
2987
2988 ring = i915_gem_request_get_ring(req);
2989
2990 /* Add a reference if we're newly entering the active list. */
2991 if (obj->active == 0)
2992 drm_gem_object_reference(&obj->base);
2993 obj->active |= intel_ring_flag(ring);
2994
2995 list_move_tail(&obj->ring_list[ring->id], &ring->active_list);
2996 i915_gem_request_assign(&obj->last_read_req[ring->id], req);
2997
2998 list_move_tail(&vma->mm_list, &vma->vm->active_list);
2999 }
3000
3001 static void
3002 i915_gem_object_retire__write(struct drm_i915_gem_object *obj)
3003 {
3004 RQ_BUG_ON(obj->last_write_req == NULL);
3005 RQ_BUG_ON(!(obj->active & intel_ring_flag(obj->last_write_req->ring)));
3006
3007 i915_gem_request_assign(&obj->last_write_req, NULL);
3008 intel_fb_obj_flush(obj, true, ORIGIN_CS);
3009 }
3010
3011 static void
3012 i915_gem_object_retire__read(struct drm_i915_gem_object *obj, int ring)
3013 {
3014 struct i915_vma *vma;
3015
3016 RQ_BUG_ON(obj->last_read_req[ring] == NULL);
3017 RQ_BUG_ON(!(obj->active & (1 << ring)));
3018
3019 list_del_init(&obj->ring_list[ring]);
3020 i915_gem_request_assign(&obj->last_read_req[ring], NULL);
3021
3022 if (obj->last_write_req && obj->last_write_req->ring->id == ring)
3023 i915_gem_object_retire__write(obj);
3024
3025 obj->active &= ~(1 << ring);
3026 if (obj->active)
3027 return;
3028
3029 /* Bump our place on the bound list to keep it roughly in LRU order
3030 * so that we don't steal from recently used but inactive objects
3031 * (unless we are forced to ofc!)
3032 */
3033 list_move_tail(&obj->global_list,
3034 &to_i915(obj->base.dev)->mm.bound_list);
3035
3036 list_for_each_entry(vma, &obj->vma_list, vma_link) {
3037 if (!list_empty(&vma->mm_list))
3038 list_move_tail(&vma->mm_list, &vma->vm->inactive_list);
3039 }
3040
3041 i915_gem_request_assign(&obj->last_fenced_req, NULL);
3042 drm_gem_object_unreference(&obj->base);
3043 }
3044
3045 static int
3046 i915_gem_init_seqno(struct drm_device *dev, u32 seqno)
3047 {
3048 struct drm_i915_private *dev_priv = dev->dev_private;
3049 struct intel_engine_cs *ring;
3050 int ret, i, j;
3051
3052 /* Carefully retire all requests without writing to the rings */
3053 for_each_ring(ring, dev_priv, i) {
3054 ret = intel_ring_idle(ring);
3055 if (ret)
3056 return ret;
3057 }
3058 i915_gem_retire_requests(dev);
3059
3060 /* Finally reset hw state */
3061 for_each_ring(ring, dev_priv, i) {
3062 intel_ring_init_seqno(ring, seqno);
3063
3064 for (j = 0; j < ARRAY_SIZE(ring->semaphore.sync_seqno); j++)
3065 ring->semaphore.sync_seqno[j] = 0;
3066 }
3067
3068 return 0;
3069 }
3070
3071 int i915_gem_set_seqno(struct drm_device *dev, u32 seqno)
3072 {
3073 struct drm_i915_private *dev_priv = dev->dev_private;
3074 int ret;
3075
3076 if (seqno == 0)
3077 return -EINVAL;
3078
3079 /* HWS page needs to be set less than what we
3080 * will inject to ring
3081 */
3082 ret = i915_gem_init_seqno(dev, seqno - 1);
3083 if (ret)
3084 return ret;
3085
3086 /* Carefully set the last_seqno value so that wrap
3087 * detection still works
3088 */
3089 dev_priv->next_seqno = seqno;
3090 dev_priv->last_seqno = seqno - 1;
3091 if (dev_priv->last_seqno == 0)
3092 dev_priv->last_seqno--;
3093
3094 return 0;
3095 }
3096
3097 int
3098 i915_gem_get_seqno(struct drm_device *dev, u32 *seqno)
3099 {
3100 struct drm_i915_private *dev_priv = dev->dev_private;
3101
3102 /* reserve 0 for non-seqno */
3103 if (dev_priv->next_seqno == 0) {
3104 int ret = i915_gem_init_seqno(dev, 0);
3105 if (ret)
3106 return ret;
3107
3108 dev_priv->next_seqno = 1;
3109 }
3110
3111 *seqno = dev_priv->last_seqno = dev_priv->next_seqno++;
3112 return 0;
3113 }
3114
3115 /*
3116 * NB: This function is not allowed to fail. Doing so would mean the the
3117 * request is not being tracked for completion but the work itself is
3118 * going to happen on the hardware. This would be a Bad Thing(tm).
3119 */
3120 void __i915_add_request(struct drm_i915_gem_request *request,
3121 struct drm_i915_gem_object *obj,
3122 bool flush_caches)
3123 {
3124 struct intel_engine_cs *ring;
3125 struct drm_i915_private *dev_priv;
3126 struct intel_ringbuffer *ringbuf;
3127 u32 request_start;
3128 int ret;
3129
3130 if (WARN_ON(request == NULL))
3131 return;
3132
3133 ring = request->ring;
3134 dev_priv = ring->dev->dev_private;
3135 ringbuf = request->ringbuf;
3136
3137 /*
3138 * To ensure that this call will not fail, space for its emissions
3139 * should already have been reserved in the ring buffer. Let the ring
3140 * know that it is time to use that space up.
3141 */
3142 intel_ring_reserved_space_use(ringbuf);
3143
3144 request_start = intel_ring_get_tail(ringbuf);
3145 /*
3146 * Emit any outstanding flushes - execbuf can fail to emit the flush
3147 * after having emitted the batchbuffer command. Hence we need to fix
3148 * things up similar to emitting the lazy request. The difference here
3149 * is that the flush _must_ happen before the next request, no matter
3150 * what.
3151 */
3152 if (flush_caches) {
3153 if (i915.enable_execlists)
3154 ret = logical_ring_flush_all_caches(request);
3155 else
3156 ret = intel_ring_flush_all_caches(request);
3157 /* Not allowed to fail! */
3158 WARN(ret, "*_ring_flush_all_caches failed: %d!\n", ret);
3159 }
3160
3161 /* Record the position of the start of the request so that
3162 * should we detect the updated seqno part-way through the
3163 * GPU processing the request, we never over-estimate the
3164 * position of the head.
3165 */
3166 request->postfix = intel_ring_get_tail(ringbuf);
3167
3168 if (i915.enable_execlists)
3169 ret = ring->emit_request(request);
3170 else {
3171 ret = ring->add_request(request);
3172
3173 request->tail = intel_ring_get_tail(ringbuf);
3174 }
3175 /* Not allowed to fail! */
3176 WARN(ret, "emit|add_request failed: %d!\n", ret);
3177
3178 request->head = request_start;
3179
3180 /* Whilst this request exists, batch_obj will be on the
3181 * active_list, and so will hold the active reference. Only when this
3182 * request is retired will the the batch_obj be moved onto the
3183 * inactive_list and lose its active reference. Hence we do not need
3184 * to explicitly hold another reference here.
3185 */
3186 request->batch_obj = obj;
3187
3188 request->emitted_jiffies = jiffies;
3189 request->previous_seqno = ring->last_submitted_seqno;
3190 ring->last_submitted_seqno = request->seqno;
3191 list_add_tail(&request->list, &ring->request_list);
3192
3193 trace_i915_gem_request_add(request);
3194
3195 i915_queue_hangcheck(ring->dev);
3196
3197 queue_delayed_work(dev_priv->wq,
3198 &dev_priv->mm.retire_work,
3199 round_jiffies_up_relative(HZ));
3200 intel_mark_busy(dev_priv->dev);
3201
3202 /* Sanity check that the reserved size was large enough. */
3203 intel_ring_reserved_space_end(ringbuf);
3204 }
3205
3206 static bool i915_context_is_banned(struct drm_i915_private *dev_priv,
3207 const struct intel_context *ctx)
3208 {
3209 unsigned long elapsed;
3210
3211 elapsed = get_seconds() - ctx->hang_stats.guilty_ts;
3212
3213 if (ctx->hang_stats.banned)
3214 return true;
3215
3216 if (ctx->hang_stats.ban_period_seconds &&
3217 elapsed <= ctx->hang_stats.ban_period_seconds) {
3218 if (!i915_gem_context_is_default(ctx)) {
3219 DRM_DEBUG("context hanging too fast, banning!\n");
3220 return true;
3221 } else if (i915_stop_ring_allow_ban(dev_priv)) {
3222 if (i915_stop_ring_allow_warn(dev_priv))
3223 DRM_ERROR("gpu hanging too fast, banning!\n");
3224 return true;
3225 }
3226 }
3227
3228 return false;
3229 }
3230
3231 static void i915_set_reset_status(struct drm_i915_private *dev_priv,
3232 struct intel_context *ctx,
3233 const bool guilty)
3234 {
3235 struct i915_ctx_hang_stats *hs;
3236
3237 if (WARN_ON(!ctx))
3238 return;
3239
3240 hs = &ctx->hang_stats;
3241
3242 if (guilty) {
3243 hs->banned = i915_context_is_banned(dev_priv, ctx);
3244 hs->batch_active++;
3245 hs->guilty_ts = get_seconds();
3246 } else {
3247 hs->batch_pending++;
3248 }
3249 }
3250
3251 void i915_gem_request_free(struct kref *req_ref)
3252 {
3253 struct drm_i915_gem_request *req = container_of(req_ref,
3254 typeof(*req), ref);
3255 struct intel_context *ctx = req->ctx;
3256
3257 if (req->file_priv)
3258 i915_gem_request_remove_from_client(req);
3259
3260 if (ctx) {
3261 if (i915.enable_execlists) {
3262 if (ctx != req->ring->default_context)
3263 intel_lr_context_unpin(req);
3264 }
3265
3266 i915_gem_context_unreference(ctx);
3267 }
3268
3269 kmem_cache_free(req->i915->requests, req);
3270 }
3271
3272 int i915_gem_request_alloc(struct intel_engine_cs *ring,
3273 struct intel_context *ctx,
3274 struct drm_i915_gem_request **req_out)
3275 {
3276 struct drm_i915_private *dev_priv = to_i915(ring->dev);
3277 struct drm_i915_gem_request *req;
3278 int ret;
3279
3280 if (!req_out)
3281 return -EINVAL;
3282
3283 *req_out = NULL;
3284
3285 req = kmem_cache_zalloc(dev_priv->requests, GFP_KERNEL);
3286 if (req == NULL)
3287 return -ENOMEM;
3288
3289 ret = i915_gem_get_seqno(ring->dev, &req->seqno);
3290 if (ret)
3291 goto err;
3292
3293 kref_init(&req->ref);
3294 req->i915 = dev_priv;
3295 req->ring = ring;
3296 req->ctx = ctx;
3297 i915_gem_context_reference(req->ctx);
3298
3299 if (i915.enable_execlists)
3300 ret = intel_logical_ring_alloc_request_extras(req);
3301 else
3302 ret = intel_ring_alloc_request_extras(req);
3303 if (ret) {
3304 i915_gem_context_unreference(req->ctx);
3305 goto err;
3306 }
3307
3308 /*
3309 * Reserve space in the ring buffer for all the commands required to
3310 * eventually emit this request. This is to guarantee that the
3311 * i915_add_request() call can't fail. Note that the reserve may need
3312 * to be redone if the request is not actually submitted straight
3313 * away, e.g. because a GPU scheduler has deferred it.
3314 */
3315 if (i915.enable_execlists)
3316 ret = intel_logical_ring_reserve_space(req);
3317 else
3318 ret = intel_ring_reserve_space(req);
3319 if (ret) {
3320 /*
3321 * At this point, the request is fully allocated even if not
3322 * fully prepared. Thus it can be cleaned up using the proper
3323 * free code.
3324 */
3325 i915_gem_request_cancel(req);
3326 return ret;
3327 }
3328
3329 *req_out = req;
3330 return 0;
3331
3332 err:
3333 kmem_cache_free(dev_priv->requests, req);
3334 return ret;
3335 }
3336
3337 void i915_gem_request_cancel(struct drm_i915_gem_request *req)
3338 {
3339 intel_ring_reserved_space_cancel(req->ringbuf);
3340
3341 i915_gem_request_unreference(req);
3342 }
3343
3344 struct drm_i915_gem_request *
3345 i915_gem_find_active_request(struct intel_engine_cs *ring)
3346 {
3347 struct drm_i915_gem_request *request;
3348
3349 list_for_each_entry(request, &ring->request_list, list) {
3350 if (i915_gem_request_completed(request, false))
3351 continue;
3352
3353 return request;
3354 }
3355
3356 return NULL;
3357 }
3358
3359 static void i915_gem_reset_ring_status(struct drm_i915_private *dev_priv,
3360 struct intel_engine_cs *ring)
3361 {
3362 struct drm_i915_gem_request *request;
3363 bool ring_hung;
3364
3365 request = i915_gem_find_active_request(ring);
3366
3367 if (request == NULL)
3368 return;
3369
3370 ring_hung = ring->hangcheck.score >= HANGCHECK_SCORE_RING_HUNG;
3371
3372 i915_set_reset_status(dev_priv, request->ctx, ring_hung);
3373
3374 list_for_each_entry_continue(request, &ring->request_list, list)
3375 i915_set_reset_status(dev_priv, request->ctx, false);
3376 }
3377
3378 static void i915_gem_reset_ring_cleanup(struct drm_i915_private *dev_priv,
3379 struct intel_engine_cs *ring)
3380 {
3381 while (!list_empty(&ring->active_list)) {
3382 struct drm_i915_gem_object *obj;
3383
3384 obj = list_first_entry(&ring->active_list,
3385 struct drm_i915_gem_object,
3386 ring_list[ring->id]);
3387
3388 i915_gem_object_retire__read(obj, ring->id);
3389 }
3390
3391 /*
3392 * Clear the execlists queue up before freeing the requests, as those
3393 * are the ones that keep the context and ringbuffer backing objects
3394 * pinned in place.
3395 */
3396 while (!list_empty(&ring->execlist_queue)) {
3397 struct drm_i915_gem_request *submit_req;
3398
3399 submit_req = list_first_entry(&ring->execlist_queue,
3400 struct drm_i915_gem_request,
3401 execlist_link);
3402 list_del(&submit_req->execlist_link);
3403
3404 if (submit_req->ctx != ring->default_context)
3405 intel_lr_context_unpin(submit_req);
3406
3407 i915_gem_request_unreference(submit_req);
3408 }
3409
3410 /*
3411 * We must free the requests after all the corresponding objects have
3412 * been moved off active lists. Which is the same order as the normal
3413 * retire_requests function does. This is important if object hold
3414 * implicit references on things like e.g. ppgtt address spaces through
3415 * the request.
3416 */
3417 while (!list_empty(&ring->request_list)) {
3418 struct drm_i915_gem_request *request;
3419
3420 request = list_first_entry(&ring->request_list,
3421 struct drm_i915_gem_request,
3422 list);
3423
3424 i915_gem_request_retire(request);
3425 }
3426 }
3427
3428 void i915_gem_reset(struct drm_device *dev)
3429 {
3430 struct drm_i915_private *dev_priv = dev->dev_private;
3431 struct intel_engine_cs *ring;
3432 int i;
3433
3434 /*
3435 * Before we free the objects from the requests, we need to inspect
3436 * them for finding the guilty party. As the requests only borrow
3437 * their reference to the objects, the inspection must be done first.
3438 */
3439 for_each_ring(ring, dev_priv, i)
3440 i915_gem_reset_ring_status(dev_priv, ring);
3441
3442 for_each_ring(ring, dev_priv, i)
3443 i915_gem_reset_ring_cleanup(dev_priv, ring);
3444
3445 i915_gem_context_reset(dev);
3446
3447 i915_gem_restore_fences(dev);
3448
3449 WARN_ON(i915_verify_lists(dev));
3450 }
3451
3452 /**
3453 * This function clears the request list as sequence numbers are passed.
3454 */
3455 void
3456 i915_gem_retire_requests_ring(struct intel_engine_cs *ring)
3457 {
3458 WARN_ON(i915_verify_lists(ring->dev));
3459
3460 /* Retire requests first as we use it above for the early return.
3461 * If we retire requests last, we may use a later seqno and so clear
3462 * the requests lists without clearing the active list, leading to
3463 * confusion.
3464 */
3465 while (!list_empty(&ring->request_list)) {
3466 struct drm_i915_gem_request *request;
3467
3468 request = list_first_entry(&ring->request_list,
3469 struct drm_i915_gem_request,
3470 list);
3471
3472 if (!i915_gem_request_completed(request, true))
3473 break;
3474
3475 i915_gem_request_retire(request);
3476 }
3477
3478 /* Move any buffers on the active list that are no longer referenced
3479 * by the ringbuffer to the flushing/inactive lists as appropriate,
3480 * before we free the context associated with the requests.
3481 */
3482 while (!list_empty(&ring->active_list)) {
3483 struct drm_i915_gem_object *obj;
3484
3485 obj = list_first_entry(&ring->active_list,
3486 struct drm_i915_gem_object,
3487 ring_list[ring->id]);
3488
3489 if (!list_empty(&obj->last_read_req[ring->id]->list))
3490 break;
3491
3492 i915_gem_object_retire__read(obj, ring->id);
3493 }
3494
3495 if (unlikely(ring->trace_irq_req &&
3496 i915_gem_request_completed(ring->trace_irq_req, true))) {
3497 ring->irq_put(ring);
3498 i915_gem_request_assign(&ring->trace_irq_req, NULL);
3499 }
3500
3501 WARN_ON(i915_verify_lists(ring->dev));
3502 }
3503
3504 bool
3505 i915_gem_retire_requests(struct drm_device *dev)
3506 {
3507 struct drm_i915_private *dev_priv = dev->dev_private;
3508 struct intel_engine_cs *ring;
3509 bool idle = true;
3510 int i;
3511
3512 for_each_ring(ring, dev_priv, i) {
3513 i915_gem_retire_requests_ring(ring);
3514 idle &= list_empty(&ring->request_list);
3515 if (i915.enable_execlists) {
3516 unsigned long flags;
3517
3518 spin_lock_irqsave(&ring->execlist_lock, flags);
3519 idle &= list_empty(&ring->execlist_queue);
3520 spin_unlock_irqrestore(&ring->execlist_lock, flags);
3521
3522 intel_execlists_retire_requests(ring);
3523 }
3524 }
3525
3526 if (idle)
3527 mod_delayed_work(dev_priv->wq,
3528 &dev_priv->mm.idle_work,
3529 msecs_to_jiffies(100));
3530
3531 return idle;
3532 }
3533
3534 static void
3535 i915_gem_retire_work_handler(struct work_struct *work)
3536 {
3537 struct drm_i915_private *dev_priv =
3538 container_of(work, typeof(*dev_priv), mm.retire_work.work);
3539 struct drm_device *dev = dev_priv->dev;
3540 bool idle;
3541
3542 /* Come back later if the device is busy... */
3543 idle = false;
3544 if (mutex_trylock(&dev->struct_mutex)) {
3545 idle = i915_gem_retire_requests(dev);
3546 mutex_unlock(&dev->struct_mutex);
3547 }
3548 if (!idle)
3549 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work,
3550 round_jiffies_up_relative(HZ));
3551 }
3552
3553 static void
3554 i915_gem_idle_work_handler(struct work_struct *work)
3555 {
3556 struct drm_i915_private *dev_priv =
3557 container_of(work, typeof(*dev_priv), mm.idle_work.work);
3558 struct drm_device *dev = dev_priv->dev;
3559 struct intel_engine_cs *ring;
3560 int i;
3561
3562 for_each_ring(ring, dev_priv, i)
3563 if (!list_empty(&ring->request_list))
3564 return;
3565
3566 intel_mark_idle(dev);
3567
3568 if (mutex_trylock(&dev->struct_mutex)) {
3569 struct intel_engine_cs *ring;
3570 int i;
3571
3572 for_each_ring(ring, dev_priv, i)
3573 i915_gem_batch_pool_fini(&ring->batch_pool);
3574
3575 mutex_unlock(&dev->struct_mutex);
3576 }
3577 }
3578
3579 /**
3580 * Ensures that an object will eventually get non-busy by flushing any required
3581 * write domains, emitting any outstanding lazy request and retiring and
3582 * completed requests.
3583 */
3584 static int
3585 i915_gem_object_flush_active(struct drm_i915_gem_object *obj)
3586 {
3587 int i;
3588
3589 if (!obj->active)
3590 return 0;
3591
3592 for (i = 0; i < I915_NUM_RINGS; i++) {
3593 struct drm_i915_gem_request *req;
3594
3595 req = obj->last_read_req[i];
3596 if (req == NULL)
3597 continue;
3598
3599 if (list_empty(&req->list))
3600 goto retire;
3601
3602 if (i915_gem_request_completed(req, true)) {
3603 __i915_gem_request_retire__upto(req);
3604 retire:
3605 i915_gem_object_retire__read(obj, i);
3606 }
3607 }
3608
3609 return 0;
3610 }
3611
3612 /**
3613 * i915_gem_wait_ioctl - implements DRM_IOCTL_I915_GEM_WAIT
3614 * @DRM_IOCTL_ARGS: standard ioctl arguments
3615 *
3616 * Returns 0 if successful, else an error is returned with the remaining time in
3617 * the timeout parameter.
3618 * -ETIME: object is still busy after timeout
3619 * -ERESTARTSYS: signal interrupted the wait
3620 * -ENONENT: object doesn't exist
3621 * Also possible, but rare:
3622 * -EAGAIN: GPU wedged
3623 * -ENOMEM: damn
3624 * -ENODEV: Internal IRQ fail
3625 * -E?: The add request failed
3626 *
3627 * The wait ioctl with a timeout of 0 reimplements the busy ioctl. With any
3628 * non-zero timeout parameter the wait ioctl will wait for the given number of
3629 * nanoseconds on an object becoming unbusy. Since the wait itself does so
3630 * without holding struct_mutex the object may become re-busied before this
3631 * function completes. A similar but shorter * race condition exists in the busy
3632 * ioctl
3633 */
3634 int
3635 i915_gem_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
3636 {
3637 struct drm_i915_private *dev_priv = dev->dev_private;
3638 struct drm_i915_gem_wait *args = data;
3639 struct drm_gem_object *gobj;
3640 struct drm_i915_gem_object *obj;
3641 struct drm_i915_gem_request *req[I915_NUM_RINGS];
3642 unsigned reset_counter;
3643 int i, n = 0;
3644 int ret;
3645
3646 if (args->flags != 0)
3647 return -EINVAL;
3648
3649 ret = i915_mutex_lock_interruptible(dev);
3650 if (ret)
3651 return ret;
3652
3653 gobj = drm_gem_object_lookup(dev, file, args->bo_handle);
3654 if (gobj == NULL) {
3655 mutex_unlock(&dev->struct_mutex);
3656 return -ENOENT;
3657 }
3658 obj = to_intel_bo(gobj);
3659
3660 /* Need to make sure the object gets inactive eventually. */
3661 ret = i915_gem_object_flush_active(obj);
3662 if (ret)
3663 goto out;
3664
3665 if (!obj->active)
3666 goto out;
3667
3668 /* Do this after OLR check to make sure we make forward progress polling
3669 * on this IOCTL with a timeout == 0 (like busy ioctl)
3670 */
3671 if (args->timeout_ns == 0) {
3672 ret = -ETIME;
3673 goto out;
3674 }
3675
3676 drm_gem_object_unreference(&obj->base);
3677 reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter);
3678
3679 for (i = 0; i < I915_NUM_RINGS; i++) {
3680 if (obj->last_read_req[i] == NULL)
3681 continue;
3682
3683 req[n++] = i915_gem_request_reference(obj->last_read_req[i]);
3684 }
3685
3686 mutex_unlock(&dev->struct_mutex);
3687
3688 for (i = 0; i < n; i++) {
3689 if (ret == 0)
3690 ret = __i915_wait_request(req[i], reset_counter, true,
3691 args->timeout_ns > 0 ? &args->timeout_ns : NULL,
3692 file->driver_priv);
3693 i915_gem_request_unreference__unlocked(req[i]);
3694 }
3695 return ret;
3696
3697 out:
3698 drm_gem_object_unreference(&obj->base);
3699 mutex_unlock(&dev->struct_mutex);
3700 return ret;
3701 }
3702
3703 static int
3704 __i915_gem_object_sync(struct drm_i915_gem_object *obj,
3705 struct intel_engine_cs *to,
3706 struct drm_i915_gem_request *from_req,
3707 struct drm_i915_gem_request **to_req)
3708 {
3709 struct intel_engine_cs *from;
3710 int ret;
3711
3712 from = i915_gem_request_get_ring(from_req);
3713 if (to == from)
3714 return 0;
3715
3716 if (i915_gem_request_completed(from_req, true))
3717 return 0;
3718
3719 if (!i915_semaphore_is_enabled(obj->base.dev)) {
3720 struct drm_i915_private *i915 = to_i915(obj->base.dev);
3721 ret = __i915_wait_request(from_req,
3722 atomic_read(&i915->gpu_error.reset_counter),
3723 i915->mm.interruptible,
3724 NULL,
3725 &i915->rps.semaphores);
3726 if (ret)
3727 return ret;
3728
3729 i915_gem_object_retire_request(obj, from_req);
3730 } else {
3731 int idx = intel_ring_sync_index(from, to);
3732 u32 seqno = i915_gem_request_get_seqno(from_req);
3733
3734 WARN_ON(!to_req);
3735
3736 if (seqno <= from->semaphore.sync_seqno[idx])
3737 return 0;
3738
3739 if (*to_req == NULL) {
3740 ret = i915_gem_request_alloc(to, to->default_context, to_req);
3741 if (ret)
3742 return ret;
3743 }
3744
3745 trace_i915_gem_ring_sync_to(*to_req, from, from_req);
3746 ret = to->semaphore.sync_to(*to_req, from, seqno);
3747 if (ret)
3748 return ret;
3749
3750 /* We use last_read_req because sync_to()
3751 * might have just caused seqno wrap under
3752 * the radar.
3753 */
3754 from->semaphore.sync_seqno[idx] =
3755 i915_gem_request_get_seqno(obj->last_read_req[from->id]);
3756 }
3757
3758 return 0;
3759 }
3760
3761 /**
3762 * i915_gem_object_sync - sync an object to a ring.
3763 *
3764 * @obj: object which may be in use on another ring.
3765 * @to: ring we wish to use the object on. May be NULL.
3766 * @to_req: request we wish to use the object for. See below.
3767 * This will be allocated and returned if a request is
3768 * required but not passed in.
3769 *
3770 * This code is meant to abstract object synchronization with the GPU.
3771 * Calling with NULL implies synchronizing the object with the CPU
3772 * rather than a particular GPU ring. Conceptually we serialise writes
3773 * between engines inside the GPU. We only allow one engine to write
3774 * into a buffer at any time, but multiple readers. To ensure each has
3775 * a coherent view of memory, we must:
3776 *
3777 * - If there is an outstanding write request to the object, the new
3778 * request must wait for it to complete (either CPU or in hw, requests
3779 * on the same ring will be naturally ordered).
3780 *
3781 * - If we are a write request (pending_write_domain is set), the new
3782 * request must wait for outstanding read requests to complete.
3783 *
3784 * For CPU synchronisation (NULL to) no request is required. For syncing with
3785 * rings to_req must be non-NULL. However, a request does not have to be
3786 * pre-allocated. If *to_req is NULL and sync commands will be emitted then a
3787 * request will be allocated automatically and returned through *to_req. Note
3788 * that it is not guaranteed that commands will be emitted (because the system
3789 * might already be idle). Hence there is no need to create a request that
3790 * might never have any work submitted. Note further that if a request is
3791 * returned in *to_req, it is the responsibility of the caller to submit
3792 * that request (after potentially adding more work to it).
3793 *
3794 * Returns 0 if successful, else propagates up the lower layer error.
3795 */
3796 int
3797 i915_gem_object_sync(struct drm_i915_gem_object *obj,
3798 struct intel_engine_cs *to,
3799 struct drm_i915_gem_request **to_req)
3800 {
3801 const bool readonly = obj->base.pending_write_domain == 0;
3802 struct drm_i915_gem_request *req[I915_NUM_RINGS];
3803 int ret, i, n;
3804
3805 if (!obj->active)
3806 return 0;
3807
3808 if (to == NULL)
3809 return i915_gem_object_wait_rendering(obj, readonly);
3810
3811 n = 0;
3812 if (readonly) {
3813 if (obj->last_write_req)
3814 req[n++] = obj->last_write_req;
3815 } else {
3816 for (i = 0; i < I915_NUM_RINGS; i++)
3817 if (obj->last_read_req[i])
3818 req[n++] = obj->last_read_req[i];
3819 }
3820 for (i = 0; i < n; i++) {
3821 ret = __i915_gem_object_sync(obj, to, req[i], to_req);
3822 if (ret)
3823 return ret;
3824 }
3825
3826 return 0;
3827 }
3828
3829 static void i915_gem_object_finish_gtt(struct drm_i915_gem_object *obj)
3830 {
3831 u32 old_write_domain, old_read_domains;
3832
3833 /* Force a pagefault for domain tracking on next user access */
3834 i915_gem_release_mmap(obj);
3835
3836 if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0)
3837 return;
3838
3839 /* Wait for any direct GTT access to complete */
3840 mb();
3841
3842 old_read_domains = obj->base.read_domains;
3843 old_write_domain = obj->base.write_domain;
3844
3845 obj->base.read_domains &= ~I915_GEM_DOMAIN_GTT;
3846 obj->base.write_domain &= ~I915_GEM_DOMAIN_GTT;
3847
3848 trace_i915_gem_object_change_domain(obj,
3849 old_read_domains,
3850 old_write_domain);
3851 }
3852
3853 static int __i915_vma_unbind(struct i915_vma *vma, bool wait)
3854 {
3855 struct drm_i915_gem_object *obj = vma->obj;
3856 struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
3857 int ret;
3858
3859 if (list_empty(&vma->vma_link))
3860 return 0;
3861
3862 if (!drm_mm_node_allocated(&vma->node)) {
3863 i915_gem_vma_destroy(vma);
3864 return 0;
3865 }
3866
3867 if (vma->pin_count)
3868 return -EBUSY;
3869
3870 BUG_ON(obj->pages == NULL);
3871
3872 if (wait) {
3873 ret = i915_gem_object_wait_rendering(obj, false);
3874 if (ret)
3875 return ret;
3876 }
3877
3878 if (i915_is_ggtt(vma->vm) &&
3879 vma->ggtt_view.type == I915_GGTT_VIEW_NORMAL) {
3880 i915_gem_object_finish_gtt(obj);
3881
3882 /* release the fence reg _after_ flushing */
3883 ret = i915_gem_object_put_fence(obj);
3884 if (ret)
3885 return ret;
3886 }
3887
3888 trace_i915_vma_unbind(vma);
3889
3890 vma->vm->unbind_vma(vma);
3891 vma->bound = 0;
3892
3893 list_del_init(&vma->mm_list);
3894 if (i915_is_ggtt(vma->vm)) {
3895 if (vma->ggtt_view.type == I915_GGTT_VIEW_NORMAL) {
3896 obj->map_and_fenceable = false;
3897 } else if (vma->ggtt_view.pages) {
3898 #ifdef __NetBSD__
3899 panic("rotated/partial views can't happen");
3900 #else
3901 sg_free_table(vma->ggtt_view.pages);
3902 kfree(vma->ggtt_view.pages);
3903 #endif
3904 }
3905 vma->ggtt_view.pages = NULL;
3906 }
3907
3908 drm_mm_remove_node(&vma->node);
3909 i915_gem_vma_destroy(vma);
3910
3911 /* Since the unbound list is global, only move to that list if
3912 * no more VMAs exist. */
3913 if (list_empty(&obj->vma_list))
3914 list_move_tail(&obj->global_list, &dev_priv->mm.unbound_list);
3915
3916 /* And finally now the object is completely decoupled from this vma,
3917 * we can drop its hold on the backing storage and allow it to be
3918 * reaped by the shrinker.
3919 */
3920 i915_gem_object_unpin_pages(obj);
3921
3922 return 0;
3923 }
3924
3925 int i915_vma_unbind(struct i915_vma *vma)
3926 {
3927 return __i915_vma_unbind(vma, true);
3928 }
3929
3930 int __i915_vma_unbind_no_wait(struct i915_vma *vma)
3931 {
3932 return __i915_vma_unbind(vma, false);
3933 }
3934
3935 int i915_gpu_idle(struct drm_device *dev)
3936 {
3937 struct drm_i915_private *dev_priv = dev->dev_private;
3938 struct intel_engine_cs *ring;
3939 int ret, i;
3940
3941 /* Flush everything onto the inactive list. */
3942 for_each_ring(ring, dev_priv, i) {
3943 if (!i915.enable_execlists) {
3944 struct drm_i915_gem_request *req;
3945
3946 ret = i915_gem_request_alloc(ring, ring->default_context, &req);
3947 if (ret)
3948 return ret;
3949
3950 ret = i915_switch_context(req);
3951 if (ret) {
3952 i915_gem_request_cancel(req);
3953 return ret;
3954 }
3955
3956 i915_add_request_no_flush(req);
3957 }
3958
3959 ret = intel_ring_idle(ring);
3960 if (ret)
3961 return ret;
3962 }
3963
3964 WARN_ON(i915_verify_lists(dev));
3965 return 0;
3966 }
3967
3968 static bool i915_gem_valid_gtt_space(struct i915_vma *vma,
3969 unsigned long cache_level)
3970 {
3971 struct drm_mm_node *gtt_space = &vma->node;
3972 struct drm_mm_node *other;
3973
3974 /*
3975 * On some machines we have to be careful when putting differing types
3976 * of snoopable memory together to avoid the prefetcher crossing memory
3977 * domains and dying. During vm initialisation, we decide whether or not
3978 * these constraints apply and set the drm_mm.color_adjust
3979 * appropriately.
3980 */
3981 if (vma->vm->mm.color_adjust == NULL)
3982 return true;
3983
3984 if (!drm_mm_node_allocated(gtt_space))
3985 return true;
3986
3987 if (list_empty(>t_space->node_list))
3988 return true;
3989
3990 other = list_entry(gtt_space->node_list.prev, struct drm_mm_node, node_list);
3991 if (other->allocated && !other->hole_follows && other->color != cache_level)
3992 return false;
3993
3994 other = list_entry(gtt_space->node_list.next, struct drm_mm_node, node_list);
3995 if (other->allocated && !gtt_space->hole_follows && other->color != cache_level)
3996 return false;
3997
3998 return true;
3999 }
4000
4001 /**
4002 * Finds free space in the GTT aperture and binds the object or a view of it
4003 * there.
4004 */
4005 static struct i915_vma *
4006 i915_gem_object_bind_to_vm(struct drm_i915_gem_object *obj,
4007 struct i915_address_space *vm,
4008 const struct i915_ggtt_view *ggtt_view,
4009 unsigned alignment,
4010 uint64_t flags)
4011 {
4012 struct drm_device *dev = obj->base.dev;
4013 struct drm_i915_private *dev_priv = dev->dev_private;
4014 u32 fence_alignment, unfenced_alignment;
4015 u32 search_flag, alloc_flag;
4016 u64 start, end;
4017 u64 size, fence_size;
4018 struct i915_vma *vma;
4019 int ret;
4020
4021 if (i915_is_ggtt(vm)) {
4022 u32 view_size;
4023
4024 if (WARN_ON(!ggtt_view))
4025 return ERR_PTR(-EINVAL);
4026
4027 view_size = i915_ggtt_view_size(obj, ggtt_view);
4028
4029 fence_size = i915_gem_get_gtt_size(dev,
4030 view_size,
4031 obj->tiling_mode);
4032 fence_alignment = i915_gem_get_gtt_alignment(dev,
4033 view_size,
4034 obj->tiling_mode,
4035 true);
4036 unfenced_alignment = i915_gem_get_gtt_alignment(dev,
4037 view_size,
4038 obj->tiling_mode,
4039 false);
4040 size = flags & PIN_MAPPABLE ? fence_size : view_size;
4041 } else {
4042 fence_size = i915_gem_get_gtt_size(dev,
4043 obj->base.size,
4044 obj->tiling_mode);
4045 fence_alignment = i915_gem_get_gtt_alignment(dev,
4046 obj->base.size,
4047 obj->tiling_mode,
4048 true);
4049 unfenced_alignment =
4050 i915_gem_get_gtt_alignment(dev,
4051 obj->base.size,
4052 obj->tiling_mode,
4053 false);
4054 size = flags & PIN_MAPPABLE ? fence_size : obj->base.size;
4055 }
4056
4057 start = flags & PIN_OFFSET_BIAS ? flags & PIN_OFFSET_MASK : 0;
4058 end = vm->total;
4059 if (flags & PIN_MAPPABLE)
4060 end = min_t(u64, end, dev_priv->gtt.mappable_end);
4061 if (flags & PIN_ZONE_4G)
4062 end = min_t(u64, end, (1ULL << 32));
4063
4064 if (alignment == 0)
4065 alignment = flags & PIN_MAPPABLE ? fence_alignment :
4066 unfenced_alignment;
4067 if (flags & PIN_MAPPABLE && alignment & (fence_alignment - 1)) {
4068 DRM_DEBUG("Invalid object (view type=%u) alignment requested %u\n",
4069 ggtt_view ? ggtt_view->type : 0,
4070 alignment);
4071 return ERR_PTR(-EINVAL);
4072 }
4073
4074 /* If binding the object/GGTT view requires more space than the entire
4075 * aperture has, reject it early before evicting everything in a vain
4076 * attempt to find space.
4077 */
4078 if (size > end) {
4079 DRM_DEBUG("Attempting to bind an object (view type=%u) larger than the aperture: size=%"PRIx64" > %s aperture=%"PRIx64"\n",
4080 ggtt_view ? ggtt_view->type : 0,
4081 size,
4082 flags & PIN_MAPPABLE ? "mappable" : "total",
4083 end);
4084 return ERR_PTR(-E2BIG);
4085 }
4086
4087 ret = i915_gem_object_get_pages(obj);
4088 if (ret)
4089 return ERR_PTR(ret);
4090
4091 i915_gem_object_pin_pages(obj);
4092
4093 vma = ggtt_view ? i915_gem_obj_lookup_or_create_ggtt_vma(obj, ggtt_view) :
4094 i915_gem_obj_lookup_or_create_vma(obj, vm);
4095
4096 if (IS_ERR(vma))
4097 goto err_unpin;
4098
4099 if (flags & PIN_HIGH) {
4100 search_flag = DRM_MM_SEARCH_BELOW;
4101 alloc_flag = DRM_MM_CREATE_TOP;
4102 } else {
4103 search_flag = DRM_MM_SEARCH_DEFAULT;
4104 alloc_flag = DRM_MM_CREATE_DEFAULT;
4105 }
4106
4107 search_free:
4108 ret = drm_mm_insert_node_in_range_generic(&vm->mm, &vma->node,
4109 size, alignment,
4110 obj->cache_level,
4111 start, end,
4112 search_flag,
4113 alloc_flag);
4114 if (ret) {
4115 ret = i915_gem_evict_something(dev, vm, size, alignment,
4116 obj->cache_level,
4117 start, end,
4118 flags);
4119 if (ret == 0)
4120 goto search_free;
4121
4122 goto err_free_vma;
4123 }
4124 if (WARN_ON(!i915_gem_valid_gtt_space(vma, obj->cache_level))) {
4125 ret = -EINVAL;
4126 goto err_remove_node;
4127 }
4128
4129 trace_i915_vma_bind(vma, flags);
4130 ret = i915_vma_bind(vma, obj->cache_level, flags);
4131 if (ret)
4132 goto err_remove_node;
4133
4134 list_move_tail(&obj->global_list, &dev_priv->mm.bound_list);
4135 list_add_tail(&vma->mm_list, &vm->inactive_list);
4136
4137 return vma;
4138
4139 err_remove_node:
4140 drm_mm_remove_node(&vma->node);
4141 err_free_vma:
4142 i915_gem_vma_destroy(vma);
4143 vma = ERR_PTR(ret);
4144 err_unpin:
4145 i915_gem_object_unpin_pages(obj);
4146 return vma;
4147 }
4148
4149 bool
4150 i915_gem_clflush_object(struct drm_i915_gem_object *obj,
4151 bool force)
4152 {
4153 /* If we don't have a page list set up, then we're not pinned
4154 * to GPU, and we can ignore the cache flush because it'll happen
4155 * again at bind time.
4156 */
4157 if (obj->pages == NULL)
4158 return false;
4159
4160 /*
4161 * Stolen memory is always coherent with the GPU as it is explicitly
4162 * marked as wc by the system, or the system is cache-coherent.
4163 */
4164 if (obj->stolen || obj->phys_handle)
4165 return false;
4166
4167 /* If the GPU is snooping the contents of the CPU cache,
4168 * we do not need to manually clear the CPU cache lines. However,
4169 * the caches are only snooped when the render cache is
4170 * flushed/invalidated. As we always have to emit invalidations
4171 * and flushes when moving into and out of the RENDER domain, correct
4172 * snooping behaviour occurs naturally as the result of our domain
4173 * tracking.
4174 */
4175 if (!force && cpu_cache_is_coherent(obj->base.dev, obj->cache_level)) {
4176 obj->cache_dirty = true;
4177 return false;
4178 }
4179
4180 trace_i915_gem_object_clflush(obj);
4181 #ifdef __NetBSD__
4182 drm_clflush_pglist(&obj->pageq);
4183 #else
4184 drm_clflush_sg(obj->pages);
4185 #endif
4186 obj->cache_dirty = false;
4187
4188 return true;
4189 }
4190
4191 /** Flushes the GTT write domain for the object if it's dirty. */
4192 static void
4193 i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj)
4194 {
4195 uint32_t old_write_domain;
4196
4197 if (obj->base.write_domain != I915_GEM_DOMAIN_GTT)
4198 return;
4199
4200 /* No actual flushing is required for the GTT write domain. Writes
4201 * to it immediately go to main memory as far as we know, so there's
4202 * no chipset flush. It also doesn't land in render cache.
4203 *
4204 * However, we do have to enforce the order so that all writes through
4205 * the GTT land before any writes to the device, such as updates to
4206 * the GATT itself.
4207 */
4208 wmb();
4209
4210 old_write_domain = obj->base.write_domain;
4211 obj->base.write_domain = 0;
4212
4213 intel_fb_obj_flush(obj, false, ORIGIN_GTT);
4214
4215 trace_i915_gem_object_change_domain(obj,
4216 obj->base.read_domains,
4217 old_write_domain);
4218 }
4219
4220 /** Flushes the CPU write domain for the object if it's dirty. */
4221 static void
4222 i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj)
4223 {
4224 uint32_t old_write_domain;
4225
4226 if (obj->base.write_domain != I915_GEM_DOMAIN_CPU)
4227 return;
4228
4229 if (i915_gem_clflush_object(obj, obj->pin_display))
4230 i915_gem_chipset_flush(obj->base.dev);
4231
4232 old_write_domain = obj->base.write_domain;
4233 obj->base.write_domain = 0;
4234
4235 intel_fb_obj_flush(obj, false, ORIGIN_CPU);
4236
4237 trace_i915_gem_object_change_domain(obj,
4238 obj->base.read_domains,
4239 old_write_domain);
4240 }
4241
4242 /**
4243 * Moves a single object to the GTT read, and possibly write domain.
4244 *
4245 * This function returns when the move is complete, including waiting on
4246 * flushes to occur.
4247 */
4248 int
4249 i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write)
4250 {
4251 uint32_t old_write_domain, old_read_domains;
4252 struct i915_vma *vma;
4253 int ret;
4254
4255 if (obj->base.write_domain == I915_GEM_DOMAIN_GTT)
4256 return 0;
4257
4258 ret = i915_gem_object_wait_rendering(obj, !write);
4259 if (ret)
4260 return ret;
4261
4262 /* Flush and acquire obj->pages so that we are coherent through
4263 * direct access in memory with previous cached writes through
4264 * shmemfs and that our cache domain tracking remains valid.
4265 * For example, if the obj->filp was moved to swap without us
4266 * being notified and releasing the pages, we would mistakenly
4267 * continue to assume that the obj remained out of the CPU cached
4268 * domain.
4269 */
4270 ret = i915_gem_object_get_pages(obj);
4271 if (ret)
4272 return ret;
4273
4274 i915_gem_object_flush_cpu_write_domain(obj);
4275
4276 /* Serialise direct access to this object with the barriers for
4277 * coherent writes from the GPU, by effectively invalidating the
4278 * GTT domain upon first access.
4279 */
4280 if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0)
4281 mb();
4282
4283 old_write_domain = obj->base.write_domain;
4284 old_read_domains = obj->base.read_domains;
4285
4286 /* It should now be out of any other write domains, and we can update
4287 * the domain values for our changes.
4288 */
4289 BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
4290 obj->base.read_domains |= I915_GEM_DOMAIN_GTT;
4291 if (write) {
4292 obj->base.read_domains = I915_GEM_DOMAIN_GTT;
4293 obj->base.write_domain = I915_GEM_DOMAIN_GTT;
4294 obj->dirty = 1;
4295 }
4296
4297 trace_i915_gem_object_change_domain(obj,
4298 old_read_domains,
4299 old_write_domain);
4300
4301 /* And bump the LRU for this access */
4302 vma = i915_gem_obj_to_ggtt(obj);
4303 if (vma && drm_mm_node_allocated(&vma->node) && !obj->active)
4304 list_move_tail(&vma->mm_list,
4305 &to_i915(obj->base.dev)->gtt.base.inactive_list);
4306
4307 return 0;
4308 }
4309
4310 /**
4311 * Changes the cache-level of an object across all VMA.
4312 *
4313 * After this function returns, the object will be in the new cache-level
4314 * across all GTT and the contents of the backing storage will be coherent,
4315 * with respect to the new cache-level. In order to keep the backing storage
4316 * coherent for all users, we only allow a single cache level to be set
4317 * globally on the object and prevent it from being changed whilst the
4318 * hardware is reading from the object. That is if the object is currently
4319 * on the scanout it will be set to uncached (or equivalent display
4320 * cache coherency) and all non-MOCS GPU access will also be uncached so
4321 * that all direct access to the scanout remains coherent.
4322 */
4323 int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj,
4324 enum i915_cache_level cache_level)
4325 {
4326 struct drm_device *dev = obj->base.dev;
4327 struct i915_vma *vma, *next;
4328 bool bound = false;
4329 int ret = 0;
4330
4331 if (obj->cache_level == cache_level)
4332 goto out;
4333
4334 /* Inspect the list of currently bound VMA and unbind any that would
4335 * be invalid given the new cache-level. This is principally to
4336 * catch the issue of the CS prefetch crossing page boundaries and
4337 * reading an invalid PTE on older architectures.
4338 */
4339 list_for_each_entry_safe(vma, next, &obj->vma_list, vma_link) {
4340 if (!drm_mm_node_allocated(&vma->node))
4341 continue;
4342
4343 if (vma->pin_count) {
4344 DRM_DEBUG("can not change the cache level of pinned objects\n");
4345 return -EBUSY;
4346 }
4347
4348 if (!i915_gem_valid_gtt_space(vma, cache_level)) {
4349 ret = i915_vma_unbind(vma);
4350 if (ret)
4351 return ret;
4352 } else
4353 bound = true;
4354 }
4355
4356 /* We can reuse the existing drm_mm nodes but need to change the
4357 * cache-level on the PTE. We could simply unbind them all and
4358 * rebind with the correct cache-level on next use. However since
4359 * we already have a valid slot, dma mapping, pages etc, we may as
4360 * rewrite the PTE in the belief that doing so tramples upon less
4361 * state and so involves less work.
4362 */
4363 if (bound) {
4364 /* Before we change the PTE, the GPU must not be accessing it.
4365 * If we wait upon the object, we know that all the bound
4366 * VMA are no longer active.
4367 */
4368 ret = i915_gem_object_wait_rendering(obj, false);
4369 if (ret)
4370 return ret;
4371
4372 if (!HAS_LLC(dev) && cache_level != I915_CACHE_NONE) {
4373 /* Access to snoopable pages through the GTT is
4374 * incoherent and on some machines causes a hard
4375 * lockup. Relinquish the CPU mmaping to force
4376 * userspace to refault in the pages and we can
4377 * then double check if the GTT mapping is still
4378 * valid for that pointer access.
4379 */
4380 i915_gem_release_mmap(obj);
4381
4382 /* As we no longer need a fence for GTT access,
4383 * we can relinquish it now (and so prevent having
4384 * to steal a fence from someone else on the next
4385 * fence request). Note GPU activity would have
4386 * dropped the fence as all snoopable access is
4387 * supposed to be linear.
4388 */
4389 ret = i915_gem_object_put_fence(obj);
4390 if (ret)
4391 return ret;
4392 } else {
4393 /* We either have incoherent backing store and
4394 * so no GTT access or the architecture is fully
4395 * coherent. In such cases, existing GTT mmaps
4396 * ignore the cache bit in the PTE and we can
4397 * rewrite it without confusing the GPU or having
4398 * to force userspace to fault back in its mmaps.
4399 */
4400 }
4401
4402 list_for_each_entry(vma, &obj->vma_list, vma_link) {
4403 if (!drm_mm_node_allocated(&vma->node))
4404 continue;
4405
4406 ret = i915_vma_bind(vma, cache_level, PIN_UPDATE);
4407 if (ret)
4408 return ret;
4409 }
4410 }
4411
4412 list_for_each_entry(vma, &obj->vma_list, vma_link)
4413 vma->node.color = cache_level;
4414 obj->cache_level = cache_level;
4415
4416 out:
4417 /* Flush the dirty CPU caches to the backing storage so that the
4418 * object is now coherent at its new cache level (with respect
4419 * to the access domain).
4420 */
4421 if (obj->cache_dirty &&
4422 obj->base.write_domain != I915_GEM_DOMAIN_CPU &&
4423 cpu_write_needs_clflush(obj)) {
4424 if (i915_gem_clflush_object(obj, true))
4425 i915_gem_chipset_flush(obj->base.dev);
4426 }
4427
4428 return 0;
4429 }
4430
4431 int i915_gem_get_caching_ioctl(struct drm_device *dev, void *data,
4432 struct drm_file *file)
4433 {
4434 struct drm_i915_gem_caching *args = data;
4435 struct drm_gem_object *gobj;
4436 struct drm_i915_gem_object *obj;
4437
4438 gobj = drm_gem_object_lookup(dev, file, args->handle);
4439 if (gobj == NULL)
4440 return -ENOENT;
4441 obj = to_intel_bo(gobj);
4442
4443 switch (obj->cache_level) {
4444 case I915_CACHE_LLC:
4445 case I915_CACHE_L3_LLC:
4446 args->caching = I915_CACHING_CACHED;
4447 break;
4448
4449 case I915_CACHE_WT:
4450 args->caching = I915_CACHING_DISPLAY;
4451 break;
4452
4453 default:
4454 args->caching = I915_CACHING_NONE;
4455 break;
4456 }
4457
4458 drm_gem_object_unreference_unlocked(&obj->base);
4459 return 0;
4460 }
4461
4462 int i915_gem_set_caching_ioctl(struct drm_device *dev, void *data,
4463 struct drm_file *file)
4464 {
4465 struct drm_i915_private *dev_priv = dev->dev_private;
4466 struct drm_i915_gem_caching *args = data;
4467 struct drm_gem_object *gobj;
4468 struct drm_i915_gem_object *obj;
4469 enum i915_cache_level level;
4470 int ret;
4471
4472 switch (args->caching) {
4473 case I915_CACHING_NONE:
4474 level = I915_CACHE_NONE;
4475 break;
4476 case I915_CACHING_CACHED:
4477 /*
4478 * Due to a HW issue on BXT A stepping, GPU stores via a
4479 * snooped mapping may leave stale data in a corresponding CPU
4480 * cacheline, whereas normally such cachelines would get
4481 * invalidated.
4482 */
4483 if (IS_BROXTON(dev) && INTEL_REVID(dev) < BXT_REVID_B0)
4484 return -ENODEV;
4485
4486 level = I915_CACHE_LLC;
4487 break;
4488 case I915_CACHING_DISPLAY:
4489 level = HAS_WT(dev) ? I915_CACHE_WT : I915_CACHE_NONE;
4490 break;
4491 default:
4492 return -EINVAL;
4493 }
4494
4495 intel_runtime_pm_get(dev_priv);
4496
4497 ret = i915_mutex_lock_interruptible(dev);
4498 if (ret)
4499 goto rpm_put;
4500
4501 gobj = drm_gem_object_lookup(dev, file, args->handle);
4502 if (gobj == NULL) {
4503 ret = -ENOENT;
4504 goto unlock;
4505 }
4506 obj = to_intel_bo(gobj);
4507
4508 ret = i915_gem_object_set_cache_level(obj, level);
4509
4510 drm_gem_object_unreference(&obj->base);
4511 unlock:
4512 mutex_unlock(&dev->struct_mutex);
4513 rpm_put:
4514 intel_runtime_pm_put(dev_priv);
4515
4516 return ret;
4517 }
4518
4519 /*
4520 * Prepare buffer for display plane (scanout, cursors, etc).
4521 * Can be called from an uninterruptible phase (modesetting) and allows
4522 * any flushes to be pipelined (for pageflips).
4523 */
4524 int
4525 i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj,
4526 u32 alignment,
4527 struct intel_engine_cs *pipelined,
4528 struct drm_i915_gem_request **pipelined_request,
4529 const struct i915_ggtt_view *view)
4530 {
4531 u32 old_read_domains, old_write_domain;
4532 int ret;
4533
4534 ret = i915_gem_object_sync(obj, pipelined, pipelined_request);
4535 if (ret)
4536 return ret;
4537
4538 /* Mark the pin_display early so that we account for the
4539 * display coherency whilst setting up the cache domains.
4540 */
4541 obj->pin_display++;
4542
4543 /* The display engine is not coherent with the LLC cache on gen6. As
4544 * a result, we make sure that the pinning that is about to occur is
4545 * done with uncached PTEs. This is lowest common denominator for all
4546 * chipsets.
4547 *
4548 * However for gen6+, we could do better by using the GFDT bit instead
4549 * of uncaching, which would allow us to flush all the LLC-cached data
4550 * with that bit in the PTE to main memory with just one PIPE_CONTROL.
4551 */
4552 ret = i915_gem_object_set_cache_level(obj,
4553 HAS_WT(obj->base.dev) ? I915_CACHE_WT : I915_CACHE_NONE);
4554 if (ret)
4555 goto err_unpin_display;
4556
4557 /* As the user may map the buffer once pinned in the display plane
4558 * (e.g. libkms for the bootup splash), we have to ensure that we
4559 * always use map_and_fenceable for all scanout buffers.
4560 */
4561 ret = i915_gem_object_ggtt_pin(obj, view, alignment,
4562 view->type == I915_GGTT_VIEW_NORMAL ?
4563 PIN_MAPPABLE : 0);
4564 if (ret)
4565 goto err_unpin_display;
4566
4567 i915_gem_object_flush_cpu_write_domain(obj);
4568
4569 old_write_domain = obj->base.write_domain;
4570 old_read_domains = obj->base.read_domains;
4571
4572 /* It should now be out of any other write domains, and we can update
4573 * the domain values for our changes.
4574 */
4575 obj->base.write_domain = 0;
4576 obj->base.read_domains |= I915_GEM_DOMAIN_GTT;
4577
4578 trace_i915_gem_object_change_domain(obj,
4579 old_read_domains,
4580 old_write_domain);
4581
4582 return 0;
4583
4584 err_unpin_display:
4585 obj->pin_display--;
4586 return ret;
4587 }
4588
4589 void
4590 i915_gem_object_unpin_from_display_plane(struct drm_i915_gem_object *obj,
4591 const struct i915_ggtt_view *view)
4592 {
4593 if (WARN_ON(obj->pin_display == 0))
4594 return;
4595
4596 i915_gem_object_ggtt_unpin_view(obj, view);
4597
4598 obj->pin_display--;
4599 }
4600
4601 /**
4602 * Moves a single object to the CPU read, and possibly write domain.
4603 *
4604 * This function returns when the move is complete, including waiting on
4605 * flushes to occur.
4606 */
4607 int
4608 i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write)
4609 {
4610 uint32_t old_write_domain, old_read_domains;
4611 int ret;
4612
4613 if (obj->base.write_domain == I915_GEM_DOMAIN_CPU)
4614 return 0;
4615
4616 ret = i915_gem_object_wait_rendering(obj, !write);
4617 if (ret)
4618 return ret;
4619
4620 i915_gem_object_flush_gtt_write_domain(obj);
4621
4622 old_write_domain = obj->base.write_domain;
4623 old_read_domains = obj->base.read_domains;
4624
4625 /* Flush the CPU cache if it's still invalid. */
4626 if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0) {
4627 i915_gem_clflush_object(obj, false);
4628
4629 obj->base.read_domains |= I915_GEM_DOMAIN_CPU;
4630 }
4631
4632 /* It should now be out of any other write domains, and we can update
4633 * the domain values for our changes.
4634 */
4635 BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
4636
4637 /* If we're writing through the CPU, then the GPU read domains will
4638 * need to be invalidated at next use.
4639 */
4640 if (write) {
4641 obj->base.read_domains = I915_GEM_DOMAIN_CPU;
4642 obj->base.write_domain = I915_GEM_DOMAIN_CPU;
4643 }
4644
4645 trace_i915_gem_object_change_domain(obj,
4646 old_read_domains,
4647 old_write_domain);
4648
4649 return 0;
4650 }
4651
4652 /* Throttle our rendering by waiting until the ring has completed our requests
4653 * emitted over 20 msec ago.
4654 *
4655 * Note that if we were to use the current jiffies each time around the loop,
4656 * we wouldn't escape the function with any frames outstanding if the time to
4657 * render a frame was over 20ms.
4658 *
4659 * This should get us reasonable parallelism between CPU and GPU but also
4660 * relatively low latency when blocking on a particular request to finish.
4661 */
4662 static int
4663 i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file)
4664 {
4665 struct drm_i915_private *dev_priv = dev->dev_private;
4666 struct drm_i915_file_private *file_priv = file->driver_priv;
4667 unsigned long recent_enough = jiffies - DRM_I915_THROTTLE_JIFFIES;
4668 struct drm_i915_gem_request *request, *target = NULL;
4669 unsigned reset_counter;
4670 int ret;
4671
4672 ret = i915_gem_wait_for_error(&dev_priv->gpu_error);
4673 if (ret)
4674 return ret;
4675
4676 ret = i915_gem_check_wedge(&dev_priv->gpu_error, false);
4677 if (ret)
4678 return ret;
4679
4680 spin_lock(&file_priv->mm.lock);
4681 list_for_each_entry(request, &file_priv->mm.request_list, client_list) {
4682 if (time_after_eq(request->emitted_jiffies, recent_enough))
4683 break;
4684
4685 /*
4686 * Note that the request might not have been submitted yet.
4687 * In which case emitted_jiffies will be zero.
4688 */
4689 if (!request->emitted_jiffies)
4690 continue;
4691
4692 target = request;
4693 }
4694 reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter);
4695 if (target)
4696 i915_gem_request_reference(target);
4697 spin_unlock(&file_priv->mm.lock);
4698
4699 if (target == NULL)
4700 return 0;
4701
4702 ret = __i915_wait_request(target, reset_counter, true, NULL, NULL);
4703 if (ret == 0)
4704 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, 0);
4705
4706 i915_gem_request_unreference__unlocked(target);
4707
4708 return ret;
4709 }
4710
4711 static bool
4712 i915_vma_misplaced(struct i915_vma *vma, uint32_t alignment, uint64_t flags)
4713 {
4714 struct drm_i915_gem_object *obj = vma->obj;
4715
4716 if (alignment &&
4717 vma->node.start & (alignment - 1))
4718 return true;
4719
4720 if (flags & PIN_MAPPABLE && !obj->map_and_fenceable)
4721 return true;
4722
4723 if (flags & PIN_OFFSET_BIAS &&
4724 vma->node.start < (flags & PIN_OFFSET_MASK))
4725 return true;
4726
4727 return false;
4728 }
4729
4730 void __i915_vma_set_map_and_fenceable(struct i915_vma *vma)
4731 {
4732 struct drm_i915_gem_object *obj = vma->obj;
4733 bool mappable, fenceable;
4734 u32 fence_size, fence_alignment;
4735
4736 fence_size = i915_gem_get_gtt_size(obj->base.dev,
4737 obj->base.size,
4738 obj->tiling_mode);
4739 fence_alignment = i915_gem_get_gtt_alignment(obj->base.dev,
4740 obj->base.size,
4741 obj->tiling_mode,
4742 true);
4743
4744 fenceable = (vma->node.size == fence_size &&
4745 (vma->node.start & (fence_alignment - 1)) == 0);
4746
4747 mappable = (vma->node.start + fence_size <=
4748 to_i915(obj->base.dev)->gtt.mappable_end);
4749
4750 obj->map_and_fenceable = mappable && fenceable;
4751 }
4752
4753 static int
4754 i915_gem_object_do_pin(struct drm_i915_gem_object *obj,
4755 struct i915_address_space *vm,
4756 const struct i915_ggtt_view *ggtt_view,
4757 uint32_t alignment,
4758 uint64_t flags)
4759 {
4760 struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
4761 struct i915_vma *vma;
4762 unsigned bound;
4763 int ret;
4764
4765 if (WARN_ON(vm == &dev_priv->mm.aliasing_ppgtt->base))
4766 return -ENODEV;
4767
4768 if (WARN_ON(flags & (PIN_GLOBAL | PIN_MAPPABLE) && !i915_is_ggtt(vm)))
4769 return -EINVAL;
4770
4771 if (WARN_ON((flags & (PIN_MAPPABLE | PIN_GLOBAL)) == PIN_MAPPABLE))
4772 return -EINVAL;
4773
4774 if (WARN_ON(i915_is_ggtt(vm) != !!ggtt_view))
4775 return -EINVAL;
4776
4777 vma = ggtt_view ? i915_gem_obj_to_ggtt_view(obj, ggtt_view) :
4778 i915_gem_obj_to_vma(obj, vm);
4779
4780 if (IS_ERR(vma))
4781 return PTR_ERR(vma);
4782
4783 if (vma) {
4784 if (WARN_ON(vma->pin_count == DRM_I915_GEM_OBJECT_MAX_PIN_COUNT))
4785 return -EBUSY;
4786
4787 if (i915_vma_misplaced(vma, alignment, flags)) {
4788 WARN(vma->pin_count,
4789 "bo is already pinned in %s with incorrect alignment:"
4790 " offset=%08x %08x, req.alignment=%x, req.map_and_fenceable=%d,"
4791 " obj->map_and_fenceable=%d\n",
4792 ggtt_view ? "ggtt" : "ppgtt",
4793 upper_32_bits(vma->node.start),
4794 lower_32_bits(vma->node.start),
4795 alignment,
4796 !!(flags & PIN_MAPPABLE),
4797 obj->map_and_fenceable);
4798 ret = i915_vma_unbind(vma);
4799 if (ret)
4800 return ret;
4801
4802 vma = NULL;
4803 }
4804 }
4805
4806 bound = vma ? vma->bound : 0;
4807 if (vma == NULL || !drm_mm_node_allocated(&vma->node)) {
4808 vma = i915_gem_object_bind_to_vm(obj, vm, ggtt_view, alignment,
4809 flags);
4810 if (IS_ERR(vma))
4811 return PTR_ERR(vma);
4812 } else {
4813 ret = i915_vma_bind(vma, obj->cache_level, flags);
4814 if (ret)
4815 return ret;
4816 }
4817
4818 if (ggtt_view && ggtt_view->type == I915_GGTT_VIEW_NORMAL &&
4819 (bound ^ vma->bound) & GLOBAL_BIND) {
4820 __i915_vma_set_map_and_fenceable(vma);
4821 WARN_ON(flags & PIN_MAPPABLE && !obj->map_and_fenceable);
4822 }
4823
4824 vma->pin_count++;
4825 return 0;
4826 }
4827
4828 int
4829 i915_gem_object_pin(struct drm_i915_gem_object *obj,
4830 struct i915_address_space *vm,
4831 uint32_t alignment,
4832 uint64_t flags)
4833 {
4834 return i915_gem_object_do_pin(obj, vm,
4835 i915_is_ggtt(vm) ? &i915_ggtt_view_normal : NULL,
4836 alignment, flags);
4837 }
4838
4839 int
4840 i915_gem_object_ggtt_pin(struct drm_i915_gem_object *obj,
4841 const struct i915_ggtt_view *view,
4842 uint32_t alignment,
4843 uint64_t flags)
4844 {
4845 if (WARN_ONCE(!view, "no view specified"))
4846 return -EINVAL;
4847
4848 return i915_gem_object_do_pin(obj, i915_obj_to_ggtt(obj), view,
4849 alignment, flags | PIN_GLOBAL);
4850 }
4851
4852 void
4853 i915_gem_object_ggtt_unpin_view(struct drm_i915_gem_object *obj,
4854 const struct i915_ggtt_view *view)
4855 {
4856 struct i915_vma *vma = i915_gem_obj_to_ggtt_view(obj, view);
4857
4858 BUG_ON(!vma);
4859 WARN_ON(vma->pin_count == 0);
4860 WARN_ON(!i915_gem_obj_ggtt_bound_view(obj, view));
4861
4862 --vma->pin_count;
4863 }
4864
4865 int
4866 i915_gem_busy_ioctl(struct drm_device *dev, void *data,
4867 struct drm_file *file)
4868 {
4869 struct drm_i915_gem_busy *args = data;
4870 struct drm_gem_object *gobj;
4871 struct drm_i915_gem_object *obj;
4872 int ret;
4873
4874 ret = i915_mutex_lock_interruptible(dev);
4875 if (ret)
4876 return ret;
4877
4878 gobj = drm_gem_object_lookup(dev, file, args->handle);
4879 if (gobj == NULL) {
4880 ret = -ENOENT;
4881 goto unlock;
4882 }
4883 obj = to_intel_bo(gobj);
4884
4885 /* Count all active objects as busy, even if they are currently not used
4886 * by the gpu. Users of this interface expect objects to eventually
4887 * become non-busy without any further actions, therefore emit any
4888 * necessary flushes here.
4889 */
4890 ret = i915_gem_object_flush_active(obj);
4891 if (ret)
4892 goto unref;
4893
4894 BUILD_BUG_ON(I915_NUM_RINGS > 16);
4895 args->busy = obj->active << 16;
4896 if (obj->last_write_req)
4897 args->busy |= obj->last_write_req->ring->id;
4898
4899 unref:
4900 drm_gem_object_unreference(&obj->base);
4901 unlock:
4902 mutex_unlock(&dev->struct_mutex);
4903 return ret;
4904 }
4905
4906 int
4907 i915_gem_throttle_ioctl(struct drm_device *dev, void *data,
4908 struct drm_file *file_priv)
4909 {
4910 return i915_gem_ring_throttle(dev, file_priv);
4911 }
4912
4913 int
4914 i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
4915 struct drm_file *file_priv)
4916 {
4917 struct drm_i915_private *dev_priv = dev->dev_private;
4918 struct drm_i915_gem_madvise *args = data;
4919 struct drm_gem_object *gobj;
4920 struct drm_i915_gem_object *obj;
4921 int ret;
4922
4923 switch (args->madv) {
4924 case I915_MADV_DONTNEED:
4925 case I915_MADV_WILLNEED:
4926 break;
4927 default:
4928 return -EINVAL;
4929 }
4930
4931 ret = i915_mutex_lock_interruptible(dev);
4932 if (ret)
4933 return ret;
4934
4935 gobj = drm_gem_object_lookup(dev, file_priv, args->handle);
4936 if (gobj == NULL) {
4937 ret = -ENOENT;
4938 goto unlock;
4939 }
4940 obj = to_intel_bo(gobj);
4941
4942 if (i915_gem_obj_is_pinned(obj)) {
4943 ret = -EINVAL;
4944 goto out;
4945 }
4946
4947 if (obj->pages &&
4948 obj->tiling_mode != I915_TILING_NONE &&
4949 dev_priv->quirks & QUIRK_PIN_SWIZZLED_PAGES) {
4950 if (obj->madv == I915_MADV_WILLNEED)
4951 i915_gem_object_unpin_pages(obj);
4952 if (args->madv == I915_MADV_WILLNEED)
4953 i915_gem_object_pin_pages(obj);
4954 }
4955
4956 if (obj->madv != __I915_MADV_PURGED)
4957 obj->madv = args->madv;
4958
4959 /* if the object is no longer attached, discard its backing storage */
4960 if (obj->madv == I915_MADV_DONTNEED && obj->pages == NULL)
4961 i915_gem_object_truncate(obj);
4962
4963 args->retained = obj->madv != __I915_MADV_PURGED;
4964
4965 out:
4966 drm_gem_object_unreference(&obj->base);
4967 unlock:
4968 mutex_unlock(&dev->struct_mutex);
4969 return ret;
4970 }
4971
4972 void i915_gem_object_init(struct drm_i915_gem_object *obj,
4973 const struct drm_i915_gem_object_ops *ops)
4974 {
4975 int i;
4976
4977 INIT_LIST_HEAD(&obj->global_list);
4978 for (i = 0; i < I915_NUM_RINGS; i++)
4979 INIT_LIST_HEAD(&obj->ring_list[i]);
4980 INIT_LIST_HEAD(&obj->obj_exec_link);
4981 INIT_LIST_HEAD(&obj->vma_list);
4982 INIT_LIST_HEAD(&obj->batch_pool_link);
4983
4984 obj->ops = ops;
4985
4986 obj->fence_reg = I915_FENCE_REG_NONE;
4987 obj->madv = I915_MADV_WILLNEED;
4988
4989 i915_gem_info_add_obj(obj->base.dev->dev_private, obj->base.size);
4990 }
4991
4992 static const struct drm_i915_gem_object_ops i915_gem_object_ops = {
4993 .get_pages = i915_gem_object_get_pages_gtt,
4994 .put_pages = i915_gem_object_put_pages_gtt,
4995 };
4996
4997 struct drm_i915_gem_object *i915_gem_alloc_object(struct drm_device *dev,
4998 size_t size)
4999 {
5000 #ifdef __NetBSD__
5001 struct drm_i915_private *const dev_priv = dev->dev_private;
5002 #endif
5003 struct drm_i915_gem_object *obj;
5004 #ifndef __NetBSD__
5005 struct address_space *mapping;
5006 gfp_t mask;
5007 #endif
5008
5009 obj = i915_gem_object_alloc(dev);
5010 if (obj == NULL)
5011 return NULL;
5012
5013 if (drm_gem_object_init(dev, &obj->base, size) != 0) {
5014 i915_gem_object_free(obj);
5015 return NULL;
5016 }
5017
5018 #ifdef __NetBSD__
5019 uao_set_pgfl(obj->base.gemo_shm_uao, dev_priv->gtt.pgfl);
5020 #else
5021 mask = GFP_HIGHUSER | __GFP_RECLAIMABLE;
5022 if (IS_CRESTLINE(dev) || IS_BROADWATER(dev)) {
5023 /* 965gm cannot relocate objects above 4GiB. */
5024 mask &= ~__GFP_HIGHMEM;
5025 mask |= __GFP_DMA32;
5026 }
5027
5028 mapping = file_inode(obj->base.filp)->i_mapping;
5029 mapping_set_gfp_mask(mapping, mask);
5030 #endif
5031
5032 i915_gem_object_init(obj, &i915_gem_object_ops);
5033
5034 obj->base.write_domain = I915_GEM_DOMAIN_CPU;
5035 obj->base.read_domains = I915_GEM_DOMAIN_CPU;
5036
5037 if (HAS_LLC(dev)) {
5038 /* On some devices, we can have the GPU use the LLC (the CPU
5039 * cache) for about a 10% performance improvement
5040 * compared to uncached. Graphics requests other than
5041 * display scanout are coherent with the CPU in
5042 * accessing this cache. This means in this mode we
5043 * don't need to clflush on the CPU side, and on the
5044 * GPU side we only need to flush internal caches to
5045 * get data visible to the CPU.
5046 *
5047 * However, we maintain the display planes as UC, and so
5048 * need to rebind when first used as such.
5049 */
5050 obj->cache_level = I915_CACHE_LLC;
5051 } else
5052 obj->cache_level = I915_CACHE_NONE;
5053
5054 trace_i915_gem_object_create(obj);
5055
5056 return obj;
5057 }
5058
5059 static bool discard_backing_storage(struct drm_i915_gem_object *obj)
5060 {
5061 #ifdef __NetBSD__
5062 panic("XXX");
5063 #else
5064 /* If we are the last user of the backing storage (be it shmemfs
5065 * pages or stolen etc), we know that the pages are going to be
5066 * immediately released. In this case, we can then skip copying
5067 * back the contents from the GPU.
5068 */
5069
5070 if (obj->madv != I915_MADV_WILLNEED)
5071 return false;
5072
5073 if (obj->base.filp == NULL)
5074 return true;
5075
5076 /* At first glance, this looks racy, but then again so would be
5077 * userspace racing mmap against close. However, the first external
5078 * reference to the filp can only be obtained through the
5079 * i915_gem_mmap_ioctl() which safeguards us against the user
5080 * acquiring such a reference whilst we are in the middle of
5081 * freeing the object.
5082 */
5083 return atomic_long_read(&obj->base.filp->f_count) == 1;
5084 #endif
5085 }
5086
5087 void i915_gem_free_object(struct drm_gem_object *gem_obj)
5088 {
5089 struct drm_i915_gem_object *obj = to_intel_bo(gem_obj);
5090 struct drm_device *dev = obj->base.dev;
5091 struct drm_i915_private *dev_priv = dev->dev_private;
5092 struct i915_vma *vma, *next;
5093
5094 intel_runtime_pm_get(dev_priv);
5095
5096 trace_i915_gem_object_destroy(obj);
5097
5098 list_for_each_entry_safe(vma, next, &obj->vma_list, vma_link) {
5099 int ret;
5100
5101 vma->pin_count = 0;
5102 ret = i915_vma_unbind(vma);
5103 if (WARN_ON(ret == -ERESTARTSYS)) {
5104 bool was_interruptible;
5105
5106 was_interruptible = dev_priv->mm.interruptible;
5107 dev_priv->mm.interruptible = false;
5108
5109 WARN_ON(i915_vma_unbind(vma));
5110
5111 dev_priv->mm.interruptible = was_interruptible;
5112 }
5113 }
5114
5115 /* Stolen objects don't hold a ref, but do hold pin count. Fix that up
5116 * before progressing. */
5117 if (obj->stolen)
5118 i915_gem_object_unpin_pages(obj);
5119
5120 WARN_ON(obj->frontbuffer_bits);
5121
5122 if (obj->pages && obj->madv == I915_MADV_WILLNEED &&
5123 dev_priv->quirks & QUIRK_PIN_SWIZZLED_PAGES &&
5124 obj->tiling_mode != I915_TILING_NONE)
5125 i915_gem_object_unpin_pages(obj);
5126
5127 if (WARN_ON(obj->pages_pin_count))
5128 obj->pages_pin_count = 0;
5129 if (discard_backing_storage(obj))
5130 obj->madv = I915_MADV_DONTNEED;
5131 i915_gem_object_put_pages(obj);
5132 i915_gem_object_free_mmap_offset(obj);
5133
5134 BUG_ON(obj->pages);
5135
5136 #ifndef __NetBSD__ /* XXX drm prime */
5137 if (obj->base.import_attach)
5138 drm_prime_gem_destroy(&obj->base, NULL);
5139 #endif
5140
5141 if (obj->ops->release)
5142 obj->ops->release(obj);
5143
5144 drm_gem_object_release(&obj->base);
5145 i915_gem_info_remove_obj(dev_priv, obj->base.size);
5146
5147 kfree(obj->bit_17);
5148 i915_gem_object_free(obj);
5149
5150 intel_runtime_pm_put(dev_priv);
5151 }
5152
5153 struct i915_vma *i915_gem_obj_to_vma(struct drm_i915_gem_object *obj,
5154 struct i915_address_space *vm)
5155 {
5156 struct i915_vma *vma;
5157 list_for_each_entry(vma, &obj->vma_list, vma_link) {
5158 if (i915_is_ggtt(vma->vm) &&
5159 vma->ggtt_view.type != I915_GGTT_VIEW_NORMAL)
5160 continue;
5161 if (vma->vm == vm)
5162 return vma;
5163 }
5164 return NULL;
5165 }
5166
5167 struct i915_vma *i915_gem_obj_to_ggtt_view(struct drm_i915_gem_object *obj,
5168 const struct i915_ggtt_view *view)
5169 {
5170 struct i915_address_space *ggtt = i915_obj_to_ggtt(obj);
5171 struct i915_vma *vma;
5172
5173 if (WARN_ONCE(!view, "no view specified"))
5174 return ERR_PTR(-EINVAL);
5175
5176 list_for_each_entry(vma, &obj->vma_list, vma_link)
5177 if (vma->vm == ggtt &&
5178 i915_ggtt_view_equal(&vma->ggtt_view, view))
5179 return vma;
5180 return NULL;
5181 }
5182
5183 void i915_gem_vma_destroy(struct i915_vma *vma)
5184 {
5185 struct i915_address_space *vm = NULL;
5186 WARN_ON(vma->node.allocated);
5187
5188 /* Keep the vma as a placeholder in the execbuffer reservation lists */
5189 if (!list_empty(&vma->exec_list))
5190 return;
5191
5192 vm = vma->vm;
5193
5194 if (!i915_is_ggtt(vm))
5195 i915_ppgtt_put(i915_vm_to_ppgtt(vm));
5196
5197 list_del(&vma->vma_link);
5198
5199 kmem_cache_free(to_i915(vma->obj->base.dev)->vmas, vma);
5200 }
5201
5202 static void
5203 i915_gem_stop_ringbuffers(struct drm_device *dev)
5204 {
5205 struct drm_i915_private *dev_priv = dev->dev_private;
5206 struct intel_engine_cs *ring;
5207 int i;
5208
5209 for_each_ring(ring, dev_priv, i)
5210 dev_priv->gt.stop_ring(ring);
5211 }
5212
5213 int
5214 i915_gem_suspend(struct drm_device *dev)
5215 {
5216 struct drm_i915_private *dev_priv = dev->dev_private;
5217 int ret = 0;
5218
5219 mutex_lock(&dev->struct_mutex);
5220 ret = i915_gpu_idle(dev);
5221 if (ret)
5222 goto err;
5223
5224 i915_gem_retire_requests(dev);
5225
5226 i915_gem_stop_ringbuffers(dev);
5227 mutex_unlock(&dev->struct_mutex);
5228
5229 cancel_delayed_work_sync(&dev_priv->gpu_error.hangcheck_work);
5230 cancel_delayed_work_sync(&dev_priv->mm.retire_work);
5231 flush_delayed_work(&dev_priv->mm.idle_work);
5232
5233 /* Assert that we sucessfully flushed all the work and
5234 * reset the GPU back to its idle, low power state.
5235 */
5236 WARN_ON(dev_priv->mm.busy);
5237
5238 return 0;
5239
5240 err:
5241 mutex_unlock(&dev->struct_mutex);
5242 return ret;
5243 }
5244
5245 int i915_gem_l3_remap(struct drm_i915_gem_request *req, int slice)
5246 {
5247 struct intel_engine_cs *ring = req->ring;
5248 struct drm_device *dev = ring->dev;
5249 struct drm_i915_private *dev_priv = dev->dev_private;
5250 u32 reg_base = GEN7_L3LOG_BASE + (slice * 0x200);
5251 u32 *remap_info = dev_priv->l3_parity.remap_info[slice];
5252 int i, ret;
5253
5254 if (!HAS_L3_DPF(dev) || !remap_info)
5255 return 0;
5256
5257 ret = intel_ring_begin(req, GEN7_L3LOG_SIZE / 4 * 3);
5258 if (ret)
5259 return ret;
5260
5261 /*
5262 * Note: We do not worry about the concurrent register cacheline hang
5263 * here because no other code should access these registers other than
5264 * at initialization time.
5265 */
5266 for (i = 0; i < GEN7_L3LOG_SIZE; i += 4) {
5267 intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1));
5268 intel_ring_emit(ring, reg_base + i);
5269 intel_ring_emit(ring, remap_info[i/4]);
5270 }
5271
5272 intel_ring_advance(ring);
5273
5274 return ret;
5275 }
5276
5277 void i915_gem_init_swizzling(struct drm_device *dev)
5278 {
5279 struct drm_i915_private *dev_priv = dev->dev_private;
5280
5281 if (INTEL_INFO(dev)->gen < 5 ||
5282 dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_NONE)
5283 return;
5284
5285 I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) |
5286 DISP_TILE_SURFACE_SWIZZLING);
5287
5288 if (IS_GEN5(dev))
5289 return;
5290
5291 I915_WRITE(TILECTL, I915_READ(TILECTL) | TILECTL_SWZCTL);
5292 if (IS_GEN6(dev))
5293 I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_SNB));
5294 else if (IS_GEN7(dev))
5295 I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_IVB));
5296 else if (IS_GEN8(dev))
5297 I915_WRITE(GAMTARBMODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_BDW));
5298 else
5299 BUG();
5300 }
5301
5302 static void init_unused_ring(struct drm_device *dev, u32 base)
5303 {
5304 struct drm_i915_private *dev_priv = dev->dev_private;
5305
5306 I915_WRITE(RING_CTL(base), 0);
5307 I915_WRITE(RING_HEAD(base), 0);
5308 I915_WRITE(RING_TAIL(base), 0);
5309 I915_WRITE(RING_START(base), 0);
5310 }
5311
5312 static void init_unused_rings(struct drm_device *dev)
5313 {
5314 if (IS_I830(dev)) {
5315 init_unused_ring(dev, PRB1_BASE);
5316 init_unused_ring(dev, SRB0_BASE);
5317 init_unused_ring(dev, SRB1_BASE);
5318 init_unused_ring(dev, SRB2_BASE);
5319 init_unused_ring(dev, SRB3_BASE);
5320 } else if (IS_GEN2(dev)) {
5321 init_unused_ring(dev, SRB0_BASE);
5322 init_unused_ring(dev, SRB1_BASE);
5323 } else if (IS_GEN3(dev)) {
5324 init_unused_ring(dev, PRB1_BASE);
5325 init_unused_ring(dev, PRB2_BASE);
5326 }
5327 }
5328
5329 int i915_gem_init_rings(struct drm_device *dev)
5330 {
5331 struct drm_i915_private *dev_priv = dev->dev_private;
5332 int ret;
5333
5334 ret = intel_init_render_ring_buffer(dev);
5335 if (ret)
5336 return ret;
5337
5338 if (HAS_BSD(dev)) {
5339 ret = intel_init_bsd_ring_buffer(dev);
5340 if (ret)
5341 goto cleanup_render_ring;
5342 }
5343
5344 if (HAS_BLT(dev)) {
5345 ret = intel_init_blt_ring_buffer(dev);
5346 if (ret)
5347 goto cleanup_bsd_ring;
5348 }
5349
5350 if (HAS_VEBOX(dev)) {
5351 ret = intel_init_vebox_ring_buffer(dev);
5352 if (ret)
5353 goto cleanup_blt_ring;
5354 }
5355
5356 if (HAS_BSD2(dev)) {
5357 ret = intel_init_bsd2_ring_buffer(dev);
5358 if (ret)
5359 goto cleanup_vebox_ring;
5360 }
5361
5362 return 0;
5363
5364 cleanup_vebox_ring:
5365 intel_cleanup_ring_buffer(&dev_priv->ring[VECS]);
5366 cleanup_blt_ring:
5367 intel_cleanup_ring_buffer(&dev_priv->ring[BCS]);
5368 cleanup_bsd_ring:
5369 intel_cleanup_ring_buffer(&dev_priv->ring[VCS]);
5370 cleanup_render_ring:
5371 intel_cleanup_ring_buffer(&dev_priv->ring[RCS]);
5372
5373 return ret;
5374 }
5375
5376 int
5377 i915_gem_init_hw(struct drm_device *dev)
5378 {
5379 struct drm_i915_private *dev_priv = dev->dev_private;
5380 struct intel_engine_cs *ring;
5381 int ret, i, j;
5382
5383 if (INTEL_INFO(dev)->gen < 6 && !intel_enable_gtt())
5384 return -EIO;
5385
5386 /* Double layer security blanket, see i915_gem_init() */
5387 intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
5388
5389 if (dev_priv->ellc_size)
5390 I915_WRITE(HSW_IDICR, I915_READ(HSW_IDICR) | IDIHASHMSK(0xf));
5391
5392 if (IS_HASWELL(dev))
5393 I915_WRITE(MI_PREDICATE_RESULT_2, IS_HSW_GT3(dev) ?
5394 LOWER_SLICE_ENABLED : LOWER_SLICE_DISABLED);
5395
5396 if (HAS_PCH_NOP(dev)) {
5397 if (IS_IVYBRIDGE(dev)) {
5398 u32 temp = I915_READ(GEN7_MSG_CTL);
5399 temp &= ~(WAIT_FOR_PCH_FLR_ACK | WAIT_FOR_PCH_RESET_ACK);
5400 I915_WRITE(GEN7_MSG_CTL, temp);
5401 } else if (INTEL_INFO(dev)->gen >= 7) {
5402 u32 temp = I915_READ(HSW_NDE_RSTWRN_OPT);
5403 temp &= ~RESET_PCH_HANDSHAKE_ENABLE;
5404 I915_WRITE(HSW_NDE_RSTWRN_OPT, temp);
5405 }
5406 }
5407
5408 i915_gem_init_swizzling(dev);
5409
5410 /*
5411 * At least 830 can leave some of the unused rings
5412 * "active" (ie. head != tail) after resume which
5413 * will prevent c3 entry. Makes sure all unused rings
5414 * are totally idle.
5415 */
5416 init_unused_rings(dev);
5417
5418 BUG_ON(!dev_priv->ring[RCS].default_context);
5419
5420 ret = i915_ppgtt_init_hw(dev);
5421 if (ret) {
5422 DRM_ERROR("PPGTT enable HW failed %d\n", ret);
5423 goto out;
5424 }
5425
5426 /* Need to do basic initialisation of all rings first: */
5427 for_each_ring(ring, dev_priv, i) {
5428 ret = ring->init_hw(ring);
5429 if (ret)
5430 goto out;
5431 }
5432
5433 /* We can't enable contexts until all firmware is loaded */
5434 if (HAS_GUC_UCODE(dev)) {
5435 ret = intel_guc_ucode_load(dev);
5436 if (ret) {
5437 /*
5438 * If we got an error and GuC submission is enabled, map
5439 * the error to -EIO so the GPU will be declared wedged.
5440 * OTOH, if we didn't intend to use the GuC anyway, just
5441 * discard the error and carry on.
5442 */
5443 DRM_ERROR("Failed to initialize GuC, error %d%s\n", ret,
5444 i915.enable_guc_submission ? "" :
5445 " (ignored)");
5446 ret = i915.enable_guc_submission ? -EIO : 0;
5447 if (ret)
5448 goto out;
5449 }
5450 }
5451
5452 /*
5453 * Increment the next seqno by 0x100 so we have a visible break
5454 * on re-initialisation
5455 */
5456 ret = i915_gem_set_seqno(dev, dev_priv->next_seqno+0x100);
5457 if (ret)
5458 goto out;
5459
5460 /* Now it is safe to go back round and do everything else: */
5461 for_each_ring(ring, dev_priv, i) {
5462 struct drm_i915_gem_request *req;
5463
5464 WARN_ON(!ring->default_context);
5465
5466 ret = i915_gem_request_alloc(ring, ring->default_context, &req);
5467 if (ret) {
5468 i915_gem_cleanup_ringbuffer(dev);
5469 goto out;
5470 }
5471
5472 if (ring->id == RCS) {
5473 for (j = 0; j < NUM_L3_SLICES(dev); j++)
5474 i915_gem_l3_remap(req, j);
5475 }
5476
5477 ret = i915_ppgtt_init_ring(req);
5478 if (ret && ret != -EIO) {
5479 DRM_ERROR("PPGTT enable ring #%d failed %d\n", i, ret);
5480 i915_gem_request_cancel(req);
5481 i915_gem_cleanup_ringbuffer(dev);
5482 goto out;
5483 }
5484
5485 ret = i915_gem_context_enable(req);
5486 if (ret && ret != -EIO) {
5487 DRM_ERROR("Context enable ring #%d failed %d\n", i, ret);
5488 i915_gem_request_cancel(req);
5489 i915_gem_cleanup_ringbuffer(dev);
5490 goto out;
5491 }
5492
5493 i915_add_request_no_flush(req);
5494 }
5495
5496 out:
5497 intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
5498 return ret;
5499 }
5500
5501 int i915_gem_init(struct drm_device *dev)
5502 {
5503 struct drm_i915_private *dev_priv = dev->dev_private;
5504 int ret;
5505
5506 i915.enable_execlists = intel_sanitize_enable_execlists(dev,
5507 i915.enable_execlists);
5508
5509 mutex_lock(&dev->struct_mutex);
5510
5511 if (IS_VALLEYVIEW(dev)) {
5512 /* VLVA0 (potential hack), BIOS isn't actually waking us */
5513 I915_WRITE(VLV_GTLC_WAKE_CTRL, VLV_GTLC_ALLOWWAKEREQ);
5514 if (wait_for((I915_READ(VLV_GTLC_PW_STATUS) &
5515 VLV_GTLC_ALLOWWAKEACK), 10))
5516 DRM_DEBUG_DRIVER("allow wake ack timed out\n");
5517 }
5518
5519 if (!i915.enable_execlists) {
5520 dev_priv->gt.execbuf_submit = i915_gem_ringbuffer_submission;
5521 dev_priv->gt.init_rings = i915_gem_init_rings;
5522 dev_priv->gt.cleanup_ring = intel_cleanup_ring_buffer;
5523 dev_priv->gt.stop_ring = intel_stop_ring_buffer;
5524 } else {
5525 dev_priv->gt.execbuf_submit = intel_execlists_submission;
5526 dev_priv->gt.init_rings = intel_logical_rings_init;
5527 dev_priv->gt.cleanup_ring = intel_logical_ring_cleanup;
5528 dev_priv->gt.stop_ring = intel_logical_ring_stop;
5529 }
5530
5531 /* This is just a security blanket to placate dragons.
5532 * On some systems, we very sporadically observe that the first TLBs
5533 * used by the CS may be stale, despite us poking the TLB reset. If
5534 * we hold the forcewake during initialisation these problems
5535 * just magically go away.
5536 */
5537 intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
5538
5539 ret = i915_gem_init_userptr(dev);
5540 if (ret)
5541 goto out_unlock;
5542
5543 i915_gem_init_global_gtt(dev);
5544
5545 ret = i915_gem_context_init(dev);
5546 if (ret)
5547 goto out_unlock;
5548
5549 ret = dev_priv->gt.init_rings(dev);
5550 if (ret)
5551 goto out_unlock;
5552
5553 ret = i915_gem_init_hw(dev);
5554 if (ret == -EIO) {
5555 /* Allow ring initialisation to fail by marking the GPU as
5556 * wedged. But we only want to do this where the GPU is angry,
5557 * for all other failure, such as an allocation failure, bail.
5558 */
5559 DRM_ERROR("Failed to initialize GPU, declaring it wedged\n");
5560 atomic_or(I915_WEDGED, &dev_priv->gpu_error.reset_counter);
5561 ret = 0;
5562 }
5563
5564 out_unlock:
5565 intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
5566 mutex_unlock(&dev->struct_mutex);
5567
5568 return ret;
5569 }
5570
5571 void
5572 i915_gem_cleanup_ringbuffer(struct drm_device *dev)
5573 {
5574 struct drm_i915_private *dev_priv = dev->dev_private;
5575 struct intel_engine_cs *ring;
5576 int i;
5577
5578 for_each_ring(ring, dev_priv, i)
5579 dev_priv->gt.cleanup_ring(ring);
5580
5581 if (i915.enable_execlists)
5582 /*
5583 * Neither the BIOS, ourselves or any other kernel
5584 * expects the system to be in execlists mode on startup,
5585 * so we need to reset the GPU back to legacy mode.
5586 */
5587 intel_gpu_reset(dev);
5588 }
5589
5590 static void
5591 init_ring_lists(struct intel_engine_cs *ring)
5592 {
5593 INIT_LIST_HEAD(&ring->active_list);
5594 INIT_LIST_HEAD(&ring->request_list);
5595 }
5596
5597 void
5598 i915_gem_load(struct drm_device *dev)
5599 {
5600 struct drm_i915_private *dev_priv = dev->dev_private;
5601 int i;
5602
5603 dev_priv->objects =
5604 kmem_cache_create("i915_gem_object",
5605 sizeof(struct drm_i915_gem_object), 0,
5606 SLAB_HWCACHE_ALIGN,
5607 NULL);
5608 dev_priv->vmas =
5609 kmem_cache_create("i915_gem_vma",
5610 sizeof(struct i915_vma), 0,
5611 SLAB_HWCACHE_ALIGN,
5612 NULL);
5613 dev_priv->requests =
5614 kmem_cache_create("i915_gem_request",
5615 sizeof(struct drm_i915_gem_request), 0,
5616 SLAB_HWCACHE_ALIGN,
5617 NULL);
5618
5619 INIT_LIST_HEAD(&dev_priv->vm_list);
5620 INIT_LIST_HEAD(&dev_priv->context_list);
5621 INIT_LIST_HEAD(&dev_priv->mm.unbound_list);
5622 INIT_LIST_HEAD(&dev_priv->mm.bound_list);
5623 INIT_LIST_HEAD(&dev_priv->mm.fence_list);
5624 for (i = 0; i < I915_NUM_RINGS; i++)
5625 init_ring_lists(&dev_priv->ring[i]);
5626 for (i = 0; i < I915_MAX_NUM_FENCES; i++)
5627 INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list);
5628 INIT_DELAYED_WORK(&dev_priv->mm.retire_work,
5629 i915_gem_retire_work_handler);
5630 INIT_DELAYED_WORK(&dev_priv->mm.idle_work,
5631 i915_gem_idle_work_handler);
5632 #ifdef __NetBSD__
5633 spin_lock_init(&dev_priv->gpu_error.reset_lock);
5634 DRM_INIT_WAITQUEUE(&dev_priv->gpu_error.reset_queue, "i915errst");
5635 #else
5636 init_waitqueue_head(&dev_priv->gpu_error.reset_queue);
5637 #endif
5638
5639 dev_priv->relative_constants_mode = I915_EXEC_CONSTANTS_REL_GENERAL;
5640
5641 if (INTEL_INFO(dev)->gen >= 7 && !IS_VALLEYVIEW(dev))
5642 dev_priv->num_fence_regs = 32;
5643 else if (INTEL_INFO(dev)->gen >= 4 || IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
5644 dev_priv->num_fence_regs = 16;
5645 else
5646 dev_priv->num_fence_regs = 8;
5647
5648 if (intel_vgpu_active(dev))
5649 dev_priv->num_fence_regs =
5650 I915_READ(vgtif_reg(avail_rs.fence_num));
5651
5652 /*
5653 * Set initial sequence number for requests.
5654 * Using this number allows the wraparound to happen early,
5655 * catching any obvious problems.
5656 */
5657 dev_priv->next_seqno = ((u32)~0 - 0x1100);
5658 dev_priv->last_seqno = ((u32)~0 - 0x1101);
5659
5660 /* Initialize fence registers to zero */
5661 INIT_LIST_HEAD(&dev_priv->mm.fence_list);
5662 i915_gem_restore_fences(dev);
5663
5664 i915_gem_detect_bit_6_swizzle(dev);
5665 #ifdef __NetBSD__
5666 DRM_INIT_WAITQUEUE(&dev_priv->pending_flip_queue, "i915flip");
5667 spin_lock_init(&dev_priv->pending_flip_lock);
5668 #else
5669 init_waitqueue_head(&dev_priv->pending_flip_queue);
5670 #endif
5671
5672 dev_priv->mm.interruptible = true;
5673
5674 i915_gem_shrinker_init(dev_priv);
5675 #ifdef __NetBSD__
5676 linux_mutex_init(&dev_priv->fb_tracking.lock);
5677 #else
5678 mutex_init(&dev_priv->fb_tracking.lock);
5679 #endif
5680 }
5681
5682 void i915_gem_release(struct drm_device *dev, struct drm_file *file)
5683 {
5684 struct drm_i915_file_private *file_priv = file->driver_priv;
5685
5686 /* Clean up our request list when the client is going away, so that
5687 * later retire_requests won't dereference our soon-to-be-gone
5688 * file_priv.
5689 */
5690 spin_lock(&file_priv->mm.lock);
5691 while (!list_empty(&file_priv->mm.request_list)) {
5692 struct drm_i915_gem_request *request;
5693
5694 request = list_first_entry(&file_priv->mm.request_list,
5695 struct drm_i915_gem_request,
5696 client_list);
5697 list_del(&request->client_list);
5698 request->file_priv = NULL;
5699 }
5700 spin_unlock(&file_priv->mm.lock);
5701
5702 if (!list_empty(&file_priv->rps.link)) {
5703 spin_lock(&to_i915(dev)->rps.client_lock);
5704 list_del(&file_priv->rps.link);
5705 spin_unlock(&to_i915(dev)->rps.client_lock);
5706 }
5707 }
5708
5709 int i915_gem_open(struct drm_device *dev, struct drm_file *file)
5710 {
5711 struct drm_i915_file_private *file_priv;
5712 int ret;
5713
5714 DRM_DEBUG_DRIVER("\n");
5715
5716 file_priv = kzalloc(sizeof(*file_priv), GFP_KERNEL);
5717 if (!file_priv)
5718 return -ENOMEM;
5719
5720 file->driver_priv = file_priv;
5721 file_priv->dev_priv = dev->dev_private;
5722 file_priv->file = file;
5723 INIT_LIST_HEAD(&file_priv->rps.link);
5724
5725 spin_lock_init(&file_priv->mm.lock);
5726 INIT_LIST_HEAD(&file_priv->mm.request_list);
5727
5728 ret = i915_gem_context_open(dev, file);
5729 if (ret)
5730 kfree(file_priv);
5731
5732 return ret;
5733 }
5734
5735 /**
5736 * i915_gem_track_fb - update frontbuffer tracking
5737 * @old: current GEM buffer for the frontbuffer slots
5738 * @new: new GEM buffer for the frontbuffer slots
5739 * @frontbuffer_bits: bitmask of frontbuffer slots
5740 *
5741 * This updates the frontbuffer tracking bits @frontbuffer_bits by clearing them
5742 * from @old and setting them in @new. Both @old and @new can be NULL.
5743 */
5744 void i915_gem_track_fb(struct drm_i915_gem_object *old,
5745 struct drm_i915_gem_object *new,
5746 unsigned frontbuffer_bits)
5747 {
5748 if (old) {
5749 WARN_ON(!mutex_is_locked(&old->base.dev->struct_mutex));
5750 WARN_ON(!(old->frontbuffer_bits & frontbuffer_bits));
5751 old->frontbuffer_bits &= ~frontbuffer_bits;
5752 }
5753
5754 if (new) {
5755 WARN_ON(!mutex_is_locked(&new->base.dev->struct_mutex));
5756 WARN_ON(new->frontbuffer_bits & frontbuffer_bits);
5757 new->frontbuffer_bits |= frontbuffer_bits;
5758 }
5759 }
5760
5761 /* All the new VM stuff */
5762 u64 i915_gem_obj_offset(struct drm_i915_gem_object *o,
5763 struct i915_address_space *vm)
5764 {
5765 struct drm_i915_private *dev_priv = o->base.dev->dev_private;
5766 struct i915_vma *vma;
5767
5768 WARN_ON(vm == &dev_priv->mm.aliasing_ppgtt->base);
5769
5770 list_for_each_entry(vma, &o->vma_list, vma_link) {
5771 if (i915_is_ggtt(vma->vm) &&
5772 vma->ggtt_view.type != I915_GGTT_VIEW_NORMAL)
5773 continue;
5774 if (vma->vm == vm)
5775 return vma->node.start;
5776 }
5777
5778 WARN(1, "%s vma for this object not found.\n",
5779 i915_is_ggtt(vm) ? "global" : "ppgtt");
5780 return -1;
5781 }
5782
5783 u64 i915_gem_obj_ggtt_offset_view(struct drm_i915_gem_object *o,
5784 const struct i915_ggtt_view *view)
5785 {
5786 struct i915_address_space *ggtt = i915_obj_to_ggtt(o);
5787 struct i915_vma *vma;
5788
5789 list_for_each_entry(vma, &o->vma_list, vma_link)
5790 if (vma->vm == ggtt &&
5791 i915_ggtt_view_equal(&vma->ggtt_view, view))
5792 return vma->node.start;
5793
5794 WARN(1, "global vma for this object not found. (view=%u)\n", view->type);
5795 return -1;
5796 }
5797
5798 bool i915_gem_obj_bound(struct drm_i915_gem_object *o,
5799 struct i915_address_space *vm)
5800 {
5801 struct i915_vma *vma;
5802
5803 list_for_each_entry(vma, &o->vma_list, vma_link) {
5804 if (i915_is_ggtt(vma->vm) &&
5805 vma->ggtt_view.type != I915_GGTT_VIEW_NORMAL)
5806 continue;
5807 if (vma->vm == vm && drm_mm_node_allocated(&vma->node))
5808 return true;
5809 }
5810
5811 return false;
5812 }
5813
5814 bool i915_gem_obj_ggtt_bound_view(struct drm_i915_gem_object *o,
5815 const struct i915_ggtt_view *view)
5816 {
5817 struct i915_address_space *ggtt = i915_obj_to_ggtt(o);
5818 struct i915_vma *vma;
5819
5820 list_for_each_entry(vma, &o->vma_list, vma_link)
5821 if (vma->vm == ggtt &&
5822 i915_ggtt_view_equal(&vma->ggtt_view, view) &&
5823 drm_mm_node_allocated(&vma->node))
5824 return true;
5825
5826 return false;
5827 }
5828
5829 bool i915_gem_obj_bound_any(struct drm_i915_gem_object *o)
5830 {
5831 struct i915_vma *vma;
5832
5833 list_for_each_entry(vma, &o->vma_list, vma_link)
5834 if (drm_mm_node_allocated(&vma->node))
5835 return true;
5836
5837 return false;
5838 }
5839
5840 unsigned long i915_gem_obj_size(struct drm_i915_gem_object *o,
5841 struct i915_address_space *vm)
5842 {
5843 struct drm_i915_private *dev_priv = o->base.dev->dev_private;
5844 struct i915_vma *vma;
5845
5846 WARN_ON(vm == &dev_priv->mm.aliasing_ppgtt->base);
5847
5848 BUG_ON(list_empty(&o->vma_list));
5849
5850 list_for_each_entry(vma, &o->vma_list, vma_link) {
5851 if (i915_is_ggtt(vma->vm) &&
5852 vma->ggtt_view.type != I915_GGTT_VIEW_NORMAL)
5853 continue;
5854 if (vma->vm == vm)
5855 return vma->node.size;
5856 }
5857 return 0;
5858 }
5859
5860 bool i915_gem_obj_is_pinned(struct drm_i915_gem_object *obj)
5861 {
5862 struct i915_vma *vma;
5863 list_for_each_entry(vma, &obj->vma_list, vma_link)
5864 if (vma->pin_count > 0)
5865 return true;
5866
5867 return false;
5868 }
5869
5870 /* Allocate a new GEM object and fill it with the supplied data */
5871 struct drm_i915_gem_object *
5872 i915_gem_object_create_from_data(struct drm_device *dev,
5873 const void *data, size_t size)
5874 {
5875 struct drm_i915_gem_object *obj;
5876 #ifdef __NetBSD__
5877 struct iovec iov = { .iov_base = __UNCONST(data), .iov_len = size };
5878 struct uio uio = {
5879 .uio_iov = &iov,
5880 .uio_iovcnt = 1,
5881 .uio_resid = size,
5882 .uio_rw = UIO_WRITE,
5883 };
5884 #else
5885 struct sg_table *sg;
5886 #endif
5887 size_t bytes;
5888 int ret;
5889
5890 obj = i915_gem_alloc_object(dev, round_up(size, PAGE_SIZE));
5891 if (IS_ERR_OR_NULL(obj))
5892 return obj;
5893
5894 ret = i915_gem_object_set_to_cpu_domain(obj, true);
5895 if (ret)
5896 goto fail;
5897
5898 ret = i915_gem_object_get_pages(obj);
5899 if (ret)
5900 goto fail;
5901
5902 i915_gem_object_pin_pages(obj);
5903 #ifdef __NetBSD__
5904 /* XXX errno NetBSD->Linux */
5905 ret = -ubc_uiomove(obj->base.gemo_shm_uao, &uio, size, UVM_ADV_NORMAL,
5906 UBC_WRITE);
5907 if (ret)
5908 goto fail;
5909 #else
5910 sg = obj->pages;
5911 bytes = sg_copy_from_buffer(sg->sgl, sg->nents, (void *)data, size);
5912 #endif
5913 i915_gem_object_unpin_pages(obj);
5914
5915 if (WARN_ON(bytes != size)) {
5916 DRM_ERROR("Incomplete copy, wrote %zu of %zu", bytes, size);
5917 ret = -EFAULT;
5918 goto fail;
5919 }
5920
5921 return obj;
5922
5923 fail:
5924 drm_gem_object_unreference(&obj->base);
5925 return ERR_PTR(ret);
5926 }
5927