Home | History | Annotate | Line # | Download | only in i915
i915_gem.c revision 1.41
      1 /*	$NetBSD: i915_gem.c,v 1.41 2018/08/27 07:17:35 riastradh Exp $	*/
      2 
      3 /*
      4  * Copyright  2008-2015 Intel Corporation
      5  *
      6  * Permission is hereby granted, free of charge, to any person obtaining a
      7  * copy of this software and associated documentation files (the "Software"),
      8  * to deal in the Software without restriction, including without limitation
      9  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
     10  * and/or sell copies of the Software, and to permit persons to whom the
     11  * Software is furnished to do so, subject to the following conditions:
     12  *
     13  * The above copyright notice and this permission notice (including the next
     14  * paragraph) shall be included in all copies or substantial portions of the
     15  * Software.
     16  *
     17  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
     18  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
     19  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
     20  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
     21  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
     22  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
     23  * IN THE SOFTWARE.
     24  *
     25  * Authors:
     26  *    Eric Anholt <eric (at) anholt.net>
     27  *
     28  */
     29 
     30 #include <sys/cdefs.h>
     31 __KERNEL_RCSID(0, "$NetBSD: i915_gem.c,v 1.41 2018/08/27 07:17:35 riastradh Exp $");
     32 
     33 #ifdef __NetBSD__
     34 #if 0				/* XXX uvmhist option?  */
     35 #include "opt_uvmhist.h"
     36 #endif
     37 
     38 #include <sys/types.h>
     39 #include <sys/param.h>
     40 
     41 #include <uvm/uvm.h>
     42 #include <uvm/uvm_extern.h>
     43 #include <uvm/uvm_fault.h>
     44 #include <uvm/uvm_page.h>
     45 #include <uvm/uvm_pmap.h>
     46 #include <uvm/uvm_prot.h>
     47 
     48 #include <drm/bus_dma_hacks.h>
     49 #endif
     50 
     51 #include <drm/drmP.h>
     52 #include <drm/drm_vma_manager.h>
     53 #include <drm/i915_drm.h>
     54 #include "i915_drv.h"
     55 #include "i915_vgpu.h"
     56 #include "i915_trace.h"
     57 #include "intel_drv.h"
     58 #include <linux/shmem_fs.h>
     59 #include <linux/slab.h>
     60 #include <linux/swap.h>
     61 #include <linux/pci.h>
     62 #include <linux/dma-buf.h>
     63 #include <linux/errno.h>
     64 #include <linux/time.h>
     65 #include <linux/err.h>
     66 #include <linux/bitops.h>
     67 #include <linux/printk.h>
     68 #include <asm/param.h>
     69 #include <asm/page.h>
     70 
     71 #define RQ_BUG_ON(expr)
     72 
     73 static void i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj);
     74 static void i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj);
     75 static void
     76 i915_gem_object_retire__write(struct drm_i915_gem_object *obj);
     77 static void
     78 i915_gem_object_retire__read(struct drm_i915_gem_object *obj, int ring);
     79 
     80 static bool cpu_cache_is_coherent(struct drm_device *dev,
     81 				  enum i915_cache_level level)
     82 {
     83 	return HAS_LLC(dev) || level != I915_CACHE_NONE;
     84 }
     85 
     86 static bool cpu_write_needs_clflush(struct drm_i915_gem_object *obj)
     87 {
     88 	if (!cpu_cache_is_coherent(obj->base.dev, obj->cache_level))
     89 		return true;
     90 
     91 	return obj->pin_display;
     92 }
     93 
     94 /* some bookkeeping */
     95 static void i915_gem_info_add_obj(struct drm_i915_private *dev_priv,
     96 				  size_t size)
     97 {
     98 	spin_lock(&dev_priv->mm.object_stat_lock);
     99 	dev_priv->mm.object_count++;
    100 	dev_priv->mm.object_memory += size;
    101 	spin_unlock(&dev_priv->mm.object_stat_lock);
    102 }
    103 
    104 static void i915_gem_info_remove_obj(struct drm_i915_private *dev_priv,
    105 				     size_t size)
    106 {
    107 	spin_lock(&dev_priv->mm.object_stat_lock);
    108 	dev_priv->mm.object_count--;
    109 	dev_priv->mm.object_memory -= size;
    110 	spin_unlock(&dev_priv->mm.object_stat_lock);
    111 }
    112 
    113 static int
    114 i915_gem_wait_for_error(struct i915_gpu_error *error)
    115 {
    116 	int ret;
    117 
    118 #define EXIT_COND (!i915_reset_in_progress(error) || \
    119 		   i915_terminally_wedged(error))
    120 	if (EXIT_COND)
    121 		return 0;
    122 
    123 	/*
    124 	 * Only wait 10 seconds for the gpu reset to complete to avoid hanging
    125 	 * userspace. If it takes that long something really bad is going on and
    126 	 * we should simply try to bail out and fail as gracefully as possible.
    127 	 */
    128 #ifdef __NetBSD__
    129 	spin_lock(&error->reset_lock);
    130 	DRM_SPIN_TIMED_WAIT_UNTIL(ret, &error->reset_queue, &error->reset_lock,
    131 	    10*HZ, EXIT_COND);
    132 	spin_unlock(&error->reset_lock);
    133 #else
    134 	ret = wait_event_interruptible_timeout(error->reset_queue,
    135 					       EXIT_COND,
    136 					       10*HZ);
    137 #endif
    138 	if (ret == 0) {
    139 		DRM_ERROR("Timed out waiting for the gpu reset to complete\n");
    140 		return -EIO;
    141 	} else if (ret < 0) {
    142 		return ret;
    143 	}
    144 #undef EXIT_COND
    145 
    146 	return 0;
    147 }
    148 
    149 int i915_mutex_lock_interruptible(struct drm_device *dev)
    150 {
    151 	struct drm_i915_private *dev_priv = dev->dev_private;
    152 	int ret;
    153 
    154 	ret = i915_gem_wait_for_error(&dev_priv->gpu_error);
    155 	if (ret)
    156 		return ret;
    157 
    158 	ret = mutex_lock_interruptible(&dev->struct_mutex);
    159 	if (ret)
    160 		return ret;
    161 
    162 	WARN_ON(i915_verify_lists(dev));
    163 	return 0;
    164 }
    165 
    166 int
    167 i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
    168 			    struct drm_file *file)
    169 {
    170 	struct drm_i915_private *dev_priv = dev->dev_private;
    171 	struct drm_i915_gem_get_aperture *args = data;
    172 	struct i915_gtt *ggtt = &dev_priv->gtt;
    173 	struct i915_vma *vma;
    174 	size_t pinned;
    175 
    176 	pinned = 0;
    177 	mutex_lock(&dev->struct_mutex);
    178 	list_for_each_entry(vma, &ggtt->base.active_list, mm_list)
    179 		if (vma->pin_count)
    180 			pinned += vma->node.size;
    181 	list_for_each_entry(vma, &ggtt->base.inactive_list, mm_list)
    182 		if (vma->pin_count)
    183 			pinned += vma->node.size;
    184 	mutex_unlock(&dev->struct_mutex);
    185 
    186 	args->aper_size = dev_priv->gtt.base.total;
    187 	args->aper_available_size = args->aper_size - pinned;
    188 
    189 	return 0;
    190 }
    191 
    192 static int
    193 i915_gem_object_get_pages_phys(struct drm_i915_gem_object *obj)
    194 {
    195 #ifndef __NetBSD__
    196 	struct address_space *mapping = file_inode(obj->base.filp)->i_mapping;
    197 #endif
    198 	char *vaddr = obj->phys_handle->vaddr;
    199 #ifndef __NetBSD__
    200 	struct sg_table *st;
    201 	struct scatterlist *sg;
    202 #endif
    203 	int i;
    204 
    205 	if (WARN_ON(i915_gem_object_needs_bit17_swizzle(obj)))
    206 		return -EINVAL;
    207 
    208 	for (i = 0; i < obj->base.size / PAGE_SIZE; i++) {
    209 		struct page *page;
    210 		char *src;
    211 
    212 #ifdef __NetBSD__
    213 		struct pglist pages = TAILQ_HEAD_INITIALIZER(pages);
    214 		int ret;
    215 		/* XXX errno NetBSD->Linux */
    216 		ret = -uvm_obj_wirepages(obj->base.gemo_shm_uao,
    217 		    i*PAGE_SIZE, (i + 1)*PAGE_SIZE, &pages);
    218 		if (ret)
    219 			return ret;
    220 		page = container_of(TAILQ_FIRST(&pages), struct page, p_vmp);
    221 #else
    222 		page = shmem_read_mapping_page(mapping, i);
    223 		if (IS_ERR(page))
    224 			return PTR_ERR(page);
    225 #endif
    226 
    227 		src = kmap_atomic(page);
    228 		memcpy(vaddr, src, PAGE_SIZE);
    229 		drm_clflush_virt_range(vaddr, PAGE_SIZE);
    230 		kunmap_atomic(src);
    231 
    232 #ifdef __NetBSD__
    233 		uvm_obj_unwirepages(obj->base.gemo_shm_uao,
    234 		    i*PAGE_SIZE, (i + 1)*PAGE_SIZE);
    235 #else
    236 		page_cache_release(page);
    237 #endif
    238 		vaddr += PAGE_SIZE;
    239 	}
    240 
    241 	i915_gem_chipset_flush(obj->base.dev);
    242 
    243 #ifdef __NetBSD__
    244 	obj->pages = obj->phys_handle->dmah_map;
    245 #else
    246 	st = kmalloc(sizeof(*st), GFP_KERNEL);
    247 	if (st == NULL)
    248 		return -ENOMEM;
    249 
    250 	if (sg_alloc_table(st, 1, GFP_KERNEL)) {
    251 		kfree(st);
    252 		return -ENOMEM;
    253 	}
    254 
    255 	sg = st->sgl;
    256 	sg->offset = 0;
    257 	sg->length = obj->base.size;
    258 
    259 	sg_dma_address(sg) = obj->phys_handle->busaddr;
    260 	sg_dma_len(sg) = obj->base.size;
    261 
    262 	obj->pages = st;
    263 #endif
    264 	return 0;
    265 }
    266 
    267 static void
    268 i915_gem_object_put_pages_phys(struct drm_i915_gem_object *obj)
    269 {
    270 	int ret;
    271 
    272 	BUG_ON(obj->madv == __I915_MADV_PURGED);
    273 
    274 	ret = i915_gem_object_set_to_cpu_domain(obj, true);
    275 	if (ret) {
    276 		/* In the event of a disaster, abandon all caches and
    277 		 * hope for the best.
    278 		 */
    279 		WARN_ON(ret != -EIO);
    280 		obj->base.read_domains = obj->base.write_domain = I915_GEM_DOMAIN_CPU;
    281 	}
    282 
    283 	if (obj->madv == I915_MADV_DONTNEED)
    284 		obj->dirty = 0;
    285 
    286 	if (obj->dirty) {
    287 #ifndef __NetBSD__
    288 		struct address_space *mapping = file_inode(obj->base.filp)->i_mapping;
    289 #endif
    290 		const char *vaddr = obj->phys_handle->vaddr;
    291 		int i;
    292 
    293 		for (i = 0; i < obj->base.size / PAGE_SIZE; i++) {
    294 			struct page *page;
    295 			char *dst;
    296 
    297 #ifdef __NetBSD__
    298 			struct pglist pages = TAILQ_HEAD_INITIALIZER(pages);
    299 			/* XXX errno NetBSD->Linux */
    300 			ret = -uvm_obj_wirepages(obj->base.gemo_shm_uao,
    301 			    i*PAGE_SIZE, (i + 1)*PAGE_SIZE, &pages);
    302 			if (ret)
    303 				continue;
    304 			page = container_of(TAILQ_FIRST(&pages), struct page,
    305 			    p_vmp);
    306 #endif
    307 
    308 			dst = kmap_atomic(page);
    309 			drm_clflush_virt_range(vaddr, PAGE_SIZE);
    310 			memcpy(dst, vaddr, PAGE_SIZE);
    311 			kunmap_atomic(dst);
    312 
    313 #ifdef __NetBSD__
    314 			page->p_vmp.flags &= ~PG_CLEAN;
    315 			/* XXX mark page accessed */
    316 			uvm_obj_unwirepages(obj->base.gemo_shm_uao,
    317 			    i*PAGE_SIZE, (i+1)*PAGE_SIZE);
    318 #else
    319 			set_page_dirty(page);
    320 			if (obj->madv == I915_MADV_WILLNEED)
    321 				mark_page_accessed(page);
    322 			page_cache_release(page);
    323 #endif
    324 			vaddr += PAGE_SIZE;
    325 		}
    326 		obj->dirty = 0;
    327 	}
    328 
    329 #ifdef __NetBSD__
    330 	obj->pages = NULL;
    331 #else
    332 	sg_free_table(obj->pages);
    333 	kfree(obj->pages);
    334 #endif
    335 }
    336 
    337 static void
    338 i915_gem_object_release_phys(struct drm_i915_gem_object *obj)
    339 {
    340 	drm_pci_free(obj->base.dev, obj->phys_handle);
    341 }
    342 
    343 static const struct drm_i915_gem_object_ops i915_gem_phys_ops = {
    344 	.get_pages = i915_gem_object_get_pages_phys,
    345 	.put_pages = i915_gem_object_put_pages_phys,
    346 	.release = i915_gem_object_release_phys,
    347 };
    348 
    349 static int
    350 drop_pages(struct drm_i915_gem_object *obj)
    351 {
    352 	struct i915_vma *vma, *next;
    353 	int ret;
    354 
    355 	drm_gem_object_reference(&obj->base);
    356 	list_for_each_entry_safe(vma, next, &obj->vma_list, vma_link)
    357 		if (i915_vma_unbind(vma))
    358 			break;
    359 
    360 	ret = i915_gem_object_put_pages(obj);
    361 	drm_gem_object_unreference(&obj->base);
    362 
    363 	return ret;
    364 }
    365 
    366 int
    367 i915_gem_object_attach_phys(struct drm_i915_gem_object *obj,
    368 			    int align)
    369 {
    370 	drm_dma_handle_t *phys;
    371 	int ret;
    372 
    373 	if (obj->phys_handle) {
    374 		if ((unsigned long)obj->phys_handle->vaddr & (align -1))
    375 			return -EBUSY;
    376 
    377 		return 0;
    378 	}
    379 
    380 	if (obj->madv != I915_MADV_WILLNEED)
    381 		return -EFAULT;
    382 
    383 #ifdef __NetBSD__
    384 	if (obj->base.gemo_shm_uao == NULL)
    385 		return -EINVAL;
    386 #else
    387 	if (obj->base.filp == NULL)
    388 		return -EINVAL;
    389 #endif
    390 
    391 	ret = drop_pages(obj);
    392 	if (ret)
    393 		return ret;
    394 
    395 	/* create a new object */
    396 	phys = drm_pci_alloc(obj->base.dev, obj->base.size, align);
    397 	if (!phys)
    398 		return -ENOMEM;
    399 
    400 	obj->phys_handle = phys;
    401 	obj->ops = &i915_gem_phys_ops;
    402 
    403 	return i915_gem_object_get_pages(obj);
    404 }
    405 
    406 static int
    407 i915_gem_phys_pwrite(struct drm_i915_gem_object *obj,
    408 		     struct drm_i915_gem_pwrite *args,
    409 		     struct drm_file *file_priv)
    410 {
    411 	struct drm_device *dev = obj->base.dev;
    412 	void *vaddr = (char *)obj->phys_handle->vaddr + args->offset;
    413 	char __user *user_data = to_user_ptr(args->data_ptr);
    414 	int ret = 0;
    415 
    416 	/* We manually control the domain here and pretend that it
    417 	 * remains coherent i.e. in the GTT domain, like shmem_pwrite.
    418 	 */
    419 	ret = i915_gem_object_wait_rendering(obj, false);
    420 	if (ret)
    421 		return ret;
    422 
    423 	intel_fb_obj_invalidate(obj, ORIGIN_CPU);
    424 	if (__copy_from_user_inatomic_nocache(vaddr, user_data, args->size)) {
    425 		unsigned long unwritten;
    426 
    427 		/* The physical object once assigned is fixed for the lifetime
    428 		 * of the obj, so we can safely drop the lock and continue
    429 		 * to access vaddr.
    430 		 */
    431 		mutex_unlock(&dev->struct_mutex);
    432 		unwritten = copy_from_user(vaddr, user_data, args->size);
    433 		mutex_lock(&dev->struct_mutex);
    434 		if (unwritten) {
    435 			ret = -EFAULT;
    436 			goto out;
    437 		}
    438 	}
    439 
    440 	drm_clflush_virt_range(vaddr, args->size);
    441 	i915_gem_chipset_flush(dev);
    442 
    443 out:
    444 	intel_fb_obj_flush(obj, false, ORIGIN_CPU);
    445 	return ret;
    446 }
    447 
    448 void *i915_gem_object_alloc(struct drm_device *dev)
    449 {
    450 	struct drm_i915_private *dev_priv = dev->dev_private;
    451 	return kmem_cache_zalloc(dev_priv->objects, GFP_KERNEL);
    452 }
    453 
    454 void i915_gem_object_free(struct drm_i915_gem_object *obj)
    455 {
    456 	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
    457 	kmem_cache_free(dev_priv->objects, obj);
    458 }
    459 
    460 static int
    461 i915_gem_create(struct drm_file *file,
    462 		struct drm_device *dev,
    463 		uint64_t size,
    464 		uint32_t *handle_p)
    465 {
    466 	struct drm_i915_gem_object *obj;
    467 	int ret;
    468 	u32 handle;
    469 
    470 	size = roundup(size, PAGE_SIZE);
    471 	if (size == 0)
    472 		return -EINVAL;
    473 
    474 	/* Allocate the new object */
    475 	obj = i915_gem_alloc_object(dev, size);
    476 	if (obj == NULL)
    477 		return -ENOMEM;
    478 
    479 	ret = drm_gem_handle_create(file, &obj->base, &handle);
    480 	/* drop reference from allocate - handle holds it now */
    481 	drm_gem_object_unreference_unlocked(&obj->base);
    482 	if (ret)
    483 		return ret;
    484 
    485 	*handle_p = handle;
    486 	return 0;
    487 }
    488 
    489 int
    490 i915_gem_dumb_create(struct drm_file *file,
    491 		     struct drm_device *dev,
    492 		     struct drm_mode_create_dumb *args)
    493 {
    494 	/* have to work out size/pitch and return them */
    495 #ifdef __NetBSD__		/* ALIGN means something else.  */
    496 	args->pitch = round_up(args->width * DIV_ROUND_UP(args->bpp, 8), 64);
    497 #else
    498 	args->pitch = ALIGN(args->width * DIV_ROUND_UP(args->bpp, 8), 64);
    499 #endif
    500 	args->size = args->pitch * args->height;
    501 	return i915_gem_create(file, dev,
    502 			       args->size, &args->handle);
    503 }
    504 
    505 /**
    506  * Creates a new mm object and returns a handle to it.
    507  */
    508 int
    509 i915_gem_create_ioctl(struct drm_device *dev, void *data,
    510 		      struct drm_file *file)
    511 {
    512 	struct drm_i915_gem_create *args = data;
    513 
    514 	return i915_gem_create(file, dev,
    515 			       args->size, &args->handle);
    516 }
    517 
    518 static inline int
    519 __copy_to_user_swizzled(char __user *cpu_vaddr,
    520 			const char *gpu_vaddr, int gpu_offset,
    521 			int length)
    522 {
    523 	int ret, cpu_offset = 0;
    524 
    525 	while (length > 0) {
    526 #ifdef __NetBSD__		/* XXX ALIGN means something else.  */
    527 		int cacheline_end = round_up(gpu_offset + 1, 64);
    528 #else
    529 		int cacheline_end = ALIGN(gpu_offset + 1, 64);
    530 #endif
    531 		int this_length = min(cacheline_end - gpu_offset, length);
    532 		int swizzled_gpu_offset = gpu_offset ^ 64;
    533 
    534 		ret = __copy_to_user(cpu_vaddr + cpu_offset,
    535 				     gpu_vaddr + swizzled_gpu_offset,
    536 				     this_length);
    537 		if (ret)
    538 			return ret + length;
    539 
    540 		cpu_offset += this_length;
    541 		gpu_offset += this_length;
    542 		length -= this_length;
    543 	}
    544 
    545 	return 0;
    546 }
    547 
    548 static inline int
    549 __copy_from_user_swizzled(char *gpu_vaddr, int gpu_offset,
    550 			  const char __user *cpu_vaddr,
    551 			  int length)
    552 {
    553 	int ret, cpu_offset = 0;
    554 
    555 	while (length > 0) {
    556 #ifdef __NetBSD__		/* XXX ALIGN means something else.  */
    557 		int cacheline_end = round_up(gpu_offset + 1, 64);
    558 #else
    559 		int cacheline_end = ALIGN(gpu_offset + 1, 64);
    560 #endif
    561 		int this_length = min(cacheline_end - gpu_offset, length);
    562 		int swizzled_gpu_offset = gpu_offset ^ 64;
    563 
    564 		ret = __copy_from_user(gpu_vaddr + swizzled_gpu_offset,
    565 				       cpu_vaddr + cpu_offset,
    566 				       this_length);
    567 		if (ret)
    568 			return ret + length;
    569 
    570 		cpu_offset += this_length;
    571 		gpu_offset += this_length;
    572 		length -= this_length;
    573 	}
    574 
    575 	return 0;
    576 }
    577 
    578 /*
    579  * Pins the specified object's pages and synchronizes the object with
    580  * GPU accesses. Sets needs_clflush to non-zero if the caller should
    581  * flush the object from the CPU cache.
    582  */
    583 int i915_gem_obj_prepare_shmem_read(struct drm_i915_gem_object *obj,
    584 				    int *needs_clflush)
    585 {
    586 	int ret;
    587 
    588 	*needs_clflush = 0;
    589 
    590 #ifdef __NetBSD__
    591 	if (obj->base.gemo_shm_uao == NULL)
    592 		return -EINVAL;
    593 #else
    594 	if (!obj->base.filp)
    595 		return -EINVAL;
    596 #endif
    597 
    598 	if (!(obj->base.read_domains & I915_GEM_DOMAIN_CPU)) {
    599 		/* If we're not in the cpu read domain, set ourself into the gtt
    600 		 * read domain and manually flush cachelines (if required). This
    601 		 * optimizes for the case when the gpu will dirty the data
    602 		 * anyway again before the next pread happens. */
    603 		*needs_clflush = !cpu_cache_is_coherent(obj->base.dev,
    604 							obj->cache_level);
    605 		ret = i915_gem_object_wait_rendering(obj, true);
    606 		if (ret)
    607 			return ret;
    608 	}
    609 
    610 	ret = i915_gem_object_get_pages(obj);
    611 	if (ret)
    612 		return ret;
    613 
    614 	i915_gem_object_pin_pages(obj);
    615 
    616 	return ret;
    617 }
    618 
    619 /* Per-page copy function for the shmem pread fastpath.
    620  * Flushes invalid cachelines before reading the target if
    621  * needs_clflush is set. */
    622 static int
    623 shmem_pread_fast(struct page *page, int shmem_page_offset, int page_length,
    624 		 char __user *user_data,
    625 		 bool page_do_bit17_swizzling, bool needs_clflush)
    626 {
    627 #ifdef __NetBSD__		/* XXX atomic shmem fast path */
    628 	return -EFAULT;
    629 #else
    630 	char *vaddr;
    631 	int ret;
    632 
    633 	if (unlikely(page_do_bit17_swizzling))
    634 		return -EINVAL;
    635 
    636 	vaddr = kmap_atomic(page);
    637 	if (needs_clflush)
    638 		drm_clflush_virt_range(vaddr + shmem_page_offset,
    639 				       page_length);
    640 	ret = __copy_to_user_inatomic(user_data,
    641 				      vaddr + shmem_page_offset,
    642 				      page_length);
    643 	kunmap_atomic(vaddr);
    644 
    645 	return ret ? -EFAULT : 0;
    646 #endif
    647 }
    648 
    649 static void
    650 shmem_clflush_swizzled_range(char *addr, unsigned long length,
    651 			     bool swizzled)
    652 {
    653 	if (unlikely(swizzled)) {
    654 		unsigned long start = (unsigned long) addr;
    655 		unsigned long end = (unsigned long) addr + length;
    656 
    657 		/* For swizzling simply ensure that we always flush both
    658 		 * channels. Lame, but simple and it works. Swizzled
    659 		 * pwrite/pread is far from a hotpath - current userspace
    660 		 * doesn't use it at all. */
    661 		start = round_down(start, 128);
    662 		end = round_up(end, 128);
    663 
    664 		drm_clflush_virt_range((void *)start, end - start);
    665 	} else {
    666 		drm_clflush_virt_range(addr, length);
    667 	}
    668 
    669 }
    670 
    671 /* Only difference to the fast-path function is that this can handle bit17
    672  * and uses non-atomic copy and kmap functions. */
    673 static int
    674 shmem_pread_slow(struct page *page, int shmem_page_offset, int page_length,
    675 		 char __user *user_data,
    676 		 bool page_do_bit17_swizzling, bool needs_clflush)
    677 {
    678 	char *vaddr;
    679 	int ret;
    680 
    681 	vaddr = kmap(page);
    682 	if (needs_clflush)
    683 		shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
    684 					     page_length,
    685 					     page_do_bit17_swizzling);
    686 
    687 	if (page_do_bit17_swizzling)
    688 		ret = __copy_to_user_swizzled(user_data,
    689 					      vaddr, shmem_page_offset,
    690 					      page_length);
    691 	else
    692 		ret = __copy_to_user(user_data,
    693 				     vaddr + shmem_page_offset,
    694 				     page_length);
    695 	kunmap(page);
    696 
    697 	return ret ? - EFAULT : 0;
    698 }
    699 
    700 static int
    701 i915_gem_shmem_pread(struct drm_device *dev,
    702 		     struct drm_i915_gem_object *obj,
    703 		     struct drm_i915_gem_pread *args,
    704 		     struct drm_file *file)
    705 {
    706 	char __user *user_data;
    707 	ssize_t remain;
    708 	loff_t offset;
    709 	int shmem_page_offset, page_length, ret = 0;
    710 	int obj_do_bit17_swizzling, page_do_bit17_swizzling;
    711 #ifndef __NetBSD__		/* XXX */
    712 	int prefaulted = 0;
    713 #endif
    714 	int needs_clflush = 0;
    715 #ifndef __NetBSD__
    716 	struct sg_page_iter sg_iter;
    717 #endif
    718 
    719 	user_data = to_user_ptr(args->data_ptr);
    720 	remain = args->size;
    721 
    722 	obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
    723 
    724 	ret = i915_gem_obj_prepare_shmem_read(obj, &needs_clflush);
    725 	if (ret)
    726 		return ret;
    727 
    728 	offset = args->offset;
    729 
    730 #ifdef __NetBSD__
    731 	while (0 < remain)
    732 #else
    733 	for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents,
    734 			 offset >> PAGE_SHIFT)
    735 #endif
    736 	{
    737 #ifdef __NetBSD__
    738 		struct page *const page = i915_gem_object_get_page(obj,
    739 		    atop(offset));
    740 #else
    741 		struct page *page = sg_page_iter_page(&sg_iter);
    742 
    743 		if (remain <= 0)
    744 			break;
    745 #endif
    746 
    747 		/* Operation in this page
    748 		 *
    749 		 * shmem_page_offset = offset within page in shmem file
    750 		 * page_length = bytes to copy for this page
    751 		 */
    752 		shmem_page_offset = offset_in_page(offset);
    753 		page_length = remain;
    754 		if ((shmem_page_offset + page_length) > PAGE_SIZE)
    755 			page_length = PAGE_SIZE - shmem_page_offset;
    756 
    757 		page_do_bit17_swizzling = obj_do_bit17_swizzling &&
    758 			(page_to_phys(page) & (1 << 17)) != 0;
    759 
    760 		ret = shmem_pread_fast(page, shmem_page_offset, page_length,
    761 				       user_data, page_do_bit17_swizzling,
    762 				       needs_clflush);
    763 		if (ret == 0)
    764 			goto next_page;
    765 
    766 		mutex_unlock(&dev->struct_mutex);
    767 #ifndef __NetBSD__
    768 		if (likely(!i915.prefault_disable) && !prefaulted) {
    769 			ret = fault_in_multipages_writeable(user_data, remain);
    770 			/* Userspace is tricking us, but we've already clobbered
    771 			 * its pages with the prefault and promised to write the
    772 			 * data up to the first fault. Hence ignore any errors
    773 			 * and just continue. */
    774 			(void)ret;
    775 			prefaulted = 1;
    776 		}
    777 #endif
    778 		ret = shmem_pread_slow(page, shmem_page_offset, page_length,
    779 				       user_data, page_do_bit17_swizzling,
    780 				       needs_clflush);
    781 
    782 		mutex_lock(&dev->struct_mutex);
    783 
    784 		if (ret)
    785 			goto out;
    786 
    787 next_page:
    788 		remain -= page_length;
    789 		user_data += page_length;
    790 		offset += page_length;
    791 	}
    792 
    793 out:
    794 	i915_gem_object_unpin_pages(obj);
    795 
    796 	return ret;
    797 }
    798 
    799 /**
    800  * Reads data from the object referenced by handle.
    801  *
    802  * On error, the contents of *data are undefined.
    803  */
    804 int
    805 i915_gem_pread_ioctl(struct drm_device *dev, void *data,
    806 		     struct drm_file *file)
    807 {
    808 	struct drm_i915_gem_pread *args = data;
    809 	struct drm_gem_object *gobj;
    810 	struct drm_i915_gem_object *obj;
    811 	int ret = 0;
    812 
    813 	if (args->size == 0)
    814 		return 0;
    815 
    816 	if (!access_ok(VERIFY_WRITE,
    817 		       to_user_ptr(args->data_ptr),
    818 		       args->size))
    819 		return -EFAULT;
    820 
    821 	ret = i915_mutex_lock_interruptible(dev);
    822 	if (ret)
    823 		return ret;
    824 
    825 	gobj = drm_gem_object_lookup(dev, file, args->handle);
    826 	if (gobj == NULL) {
    827 		ret = -ENOENT;
    828 		goto unlock;
    829 	}
    830 	obj = to_intel_bo(gobj);
    831 
    832 	/* Bounds check source.  */
    833 	if (args->offset > obj->base.size ||
    834 	    args->size > obj->base.size - args->offset) {
    835 		ret = -EINVAL;
    836 		goto out;
    837 	}
    838 
    839 	/* prime objects have no backing filp to GEM pread/pwrite
    840 	 * pages from.
    841 	 */
    842 #ifdef __NetBSD__
    843 	/* Also stolen objects.  */
    844 	if (obj->base.gemo_shm_uao == NULL) {
    845 		ret = -EINVAL;
    846 		goto out;
    847 	}
    848 #else
    849 	if (!obj->base.filp) {
    850 		ret = -EINVAL;
    851 		goto out;
    852 	}
    853 #endif
    854 
    855 	trace_i915_gem_object_pread(obj, args->offset, args->size);
    856 
    857 	ret = i915_gem_shmem_pread(dev, obj, args, file);
    858 
    859 out:
    860 	drm_gem_object_unreference(&obj->base);
    861 unlock:
    862 	mutex_unlock(&dev->struct_mutex);
    863 	return ret;
    864 }
    865 
    866 /* This is the fast write path which cannot handle
    867  * page faults in the source data
    868  */
    869 
    870 static inline int
    871 fast_user_write(struct io_mapping *mapping,
    872 		loff_t page_base, int page_offset,
    873 		char __user *user_data,
    874 		int length)
    875 {
    876 #ifdef __NetBSD__		/* XXX atomic shmem fast path */
    877 	return -EFAULT;
    878 #else
    879 	void __iomem *vaddr_atomic;
    880 	void *vaddr;
    881 	unsigned long unwritten;
    882 
    883 	vaddr_atomic = io_mapping_map_atomic_wc(mapping, page_base);
    884 	/* We can use the cpu mem copy function because this is X86. */
    885 	vaddr = (void __force*)vaddr_atomic + page_offset;
    886 	unwritten = __copy_from_user_inatomic_nocache(vaddr,
    887 						      user_data, length);
    888 	io_mapping_unmap_atomic(vaddr_atomic);
    889 	return unwritten;
    890 #endif
    891 }
    892 
    893 /**
    894  * This is the fast pwrite path, where we copy the data directly from the
    895  * user into the GTT, uncached.
    896  */
    897 static int
    898 i915_gem_gtt_pwrite_fast(struct drm_device *dev,
    899 			 struct drm_i915_gem_object *obj,
    900 			 struct drm_i915_gem_pwrite *args,
    901 			 struct drm_file *file)
    902 {
    903 	struct drm_i915_private *dev_priv = dev->dev_private;
    904 	ssize_t remain;
    905 	loff_t offset, page_base;
    906 	char __user *user_data;
    907 	int page_offset, page_length, ret;
    908 
    909 	ret = i915_gem_obj_ggtt_pin(obj, 0, PIN_MAPPABLE | PIN_NONBLOCK);
    910 	if (ret)
    911 		goto out;
    912 
    913 	ret = i915_gem_object_set_to_gtt_domain(obj, true);
    914 	if (ret)
    915 		goto out_unpin;
    916 
    917 	ret = i915_gem_object_put_fence(obj);
    918 	if (ret)
    919 		goto out_unpin;
    920 
    921 	user_data = to_user_ptr(args->data_ptr);
    922 	remain = args->size;
    923 
    924 	offset = i915_gem_obj_ggtt_offset(obj) + args->offset;
    925 
    926 	intel_fb_obj_invalidate(obj, ORIGIN_GTT);
    927 
    928 	while (remain > 0) {
    929 		/* Operation in this page
    930 		 *
    931 		 * page_base = page offset within aperture
    932 		 * page_offset = offset within page
    933 		 * page_length = bytes to copy for this page
    934 		 */
    935 		page_base = offset & PAGE_MASK;
    936 		page_offset = offset_in_page(offset);
    937 		page_length = remain;
    938 		if ((page_offset + remain) > PAGE_SIZE)
    939 			page_length = PAGE_SIZE - page_offset;
    940 
    941 		/* If we get a fault while copying data, then (presumably) our
    942 		 * source page isn't available.  Return the error and we'll
    943 		 * retry in the slow path.
    944 		 */
    945 		if (fast_user_write(dev_priv->gtt.mappable, page_base,
    946 				    page_offset, user_data, page_length)) {
    947 			ret = -EFAULT;
    948 			goto out_flush;
    949 		}
    950 
    951 		remain -= page_length;
    952 		user_data += page_length;
    953 		offset += page_length;
    954 	}
    955 
    956 out_flush:
    957 	intel_fb_obj_flush(obj, false, ORIGIN_GTT);
    958 out_unpin:
    959 	i915_gem_object_ggtt_unpin(obj);
    960 out:
    961 	return ret;
    962 }
    963 
    964 /* Per-page copy function for the shmem pwrite fastpath.
    965  * Flushes invalid cachelines before writing to the target if
    966  * needs_clflush_before is set and flushes out any written cachelines after
    967  * writing if needs_clflush is set. */
    968 static int
    969 shmem_pwrite_fast(struct page *page, int shmem_page_offset, int page_length,
    970 		  char __user *user_data,
    971 		  bool page_do_bit17_swizzling,
    972 		  bool needs_clflush_before,
    973 		  bool needs_clflush_after)
    974 {
    975 #ifdef __NetBSD__
    976 	return -EFAULT;
    977 #else
    978 	char *vaddr;
    979 	int ret;
    980 
    981 	if (unlikely(page_do_bit17_swizzling))
    982 		return -EINVAL;
    983 
    984 	vaddr = kmap_atomic(page);
    985 	if (needs_clflush_before)
    986 		drm_clflush_virt_range(vaddr + shmem_page_offset,
    987 				       page_length);
    988 	ret = __copy_from_user_inatomic(vaddr + shmem_page_offset,
    989 					user_data, page_length);
    990 	if (needs_clflush_after)
    991 		drm_clflush_virt_range(vaddr + shmem_page_offset,
    992 				       page_length);
    993 	kunmap_atomic(vaddr);
    994 
    995 	return ret ? -EFAULT : 0;
    996 #endif
    997 }
    998 
    999 /* Only difference to the fast-path function is that this can handle bit17
   1000  * and uses non-atomic copy and kmap functions. */
   1001 static int
   1002 shmem_pwrite_slow(struct page *page, int shmem_page_offset, int page_length,
   1003 		  char __user *user_data,
   1004 		  bool page_do_bit17_swizzling,
   1005 		  bool needs_clflush_before,
   1006 		  bool needs_clflush_after)
   1007 {
   1008 	char *vaddr;
   1009 	int ret;
   1010 
   1011 	vaddr = kmap(page);
   1012 	if (unlikely(needs_clflush_before || page_do_bit17_swizzling))
   1013 		shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
   1014 					     page_length,
   1015 					     page_do_bit17_swizzling);
   1016 	if (page_do_bit17_swizzling)
   1017 		ret = __copy_from_user_swizzled(vaddr, shmem_page_offset,
   1018 						user_data,
   1019 						page_length);
   1020 	else
   1021 		ret = __copy_from_user(vaddr + shmem_page_offset,
   1022 				       user_data,
   1023 				       page_length);
   1024 	if (needs_clflush_after)
   1025 		shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
   1026 					     page_length,
   1027 					     page_do_bit17_swizzling);
   1028 	kunmap(page);
   1029 
   1030 	return ret ? -EFAULT : 0;
   1031 }
   1032 
   1033 static int
   1034 i915_gem_shmem_pwrite(struct drm_device *dev,
   1035 		      struct drm_i915_gem_object *obj,
   1036 		      struct drm_i915_gem_pwrite *args,
   1037 		      struct drm_file *file)
   1038 {
   1039 	ssize_t remain;
   1040 	loff_t offset;
   1041 	char __user *user_data;
   1042 	int shmem_page_offset, page_length, ret = 0;
   1043 	int obj_do_bit17_swizzling, page_do_bit17_swizzling;
   1044 	int hit_slowpath = 0;
   1045 	int needs_clflush_after = 0;
   1046 	int needs_clflush_before = 0;
   1047 #ifndef __NetBSD__
   1048 	struct sg_page_iter sg_iter;
   1049 	int flush_mask = boot_cpu_data.x86_clflush_size - 1;
   1050 #else
   1051 	int flush_mask = cpu_info_primary.ci_cflush_lsize - 1;
   1052 #endif
   1053 
   1054 	user_data = to_user_ptr(args->data_ptr);
   1055 	remain = args->size;
   1056 
   1057 	obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
   1058 
   1059 	if (obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
   1060 		/* If we're not in the cpu write domain, set ourself into the gtt
   1061 		 * write domain and manually flush cachelines (if required). This
   1062 		 * optimizes for the case when the gpu will use the data
   1063 		 * right away and we therefore have to clflush anyway. */
   1064 		needs_clflush_after = cpu_write_needs_clflush(obj);
   1065 		ret = i915_gem_object_wait_rendering(obj, false);
   1066 		if (ret)
   1067 			return ret;
   1068 	}
   1069 	/* Same trick applies to invalidate partially written cachelines read
   1070 	 * before writing. */
   1071 	if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0)
   1072 		needs_clflush_before =
   1073 			!cpu_cache_is_coherent(dev, obj->cache_level);
   1074 
   1075 	ret = i915_gem_object_get_pages(obj);
   1076 	if (ret)
   1077 		return ret;
   1078 
   1079 	intel_fb_obj_invalidate(obj, ORIGIN_CPU);
   1080 
   1081 	i915_gem_object_pin_pages(obj);
   1082 
   1083 	offset = args->offset;
   1084 	obj->dirty = 1;
   1085 
   1086 #ifdef __NetBSD__
   1087 	while (0 < remain)
   1088 #else
   1089 	for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents,
   1090 			 offset >> PAGE_SHIFT)
   1091 #endif
   1092 	{
   1093 #ifdef __NetBSD__
   1094 		struct page *const page = i915_gem_object_get_page(obj,
   1095 		    atop(offset));
   1096 #else
   1097 		struct page *page = sg_page_iter_page(&sg_iter);
   1098 
   1099 		if (remain <= 0)
   1100 			break;
   1101 #endif
   1102 
   1103 		/* Operation in this page
   1104 		 *
   1105 		 * shmem_page_offset = offset within page in shmem file
   1106 		 * page_length = bytes to copy for this page
   1107 		 */
   1108 		shmem_page_offset = offset_in_page(offset);
   1109 
   1110 		page_length = remain;
   1111 		if ((shmem_page_offset + page_length) > PAGE_SIZE)
   1112 			page_length = PAGE_SIZE - shmem_page_offset;
   1113 
   1114 		/* If we don't overwrite a cacheline completely we need to be
   1115 		 * careful to have up-to-date data by first clflushing. Don't
   1116 		 * overcomplicate things and flush the entire patch. */
   1117 		const int partial_cacheline_write = needs_clflush_before &&
   1118 			((shmem_page_offset | page_length) & flush_mask);
   1119 
   1120 		page_do_bit17_swizzling = obj_do_bit17_swizzling &&
   1121 			(page_to_phys(page) & (1 << 17)) != 0;
   1122 
   1123 		ret = shmem_pwrite_fast(page, shmem_page_offset, page_length,
   1124 					user_data, page_do_bit17_swizzling,
   1125 					partial_cacheline_write,
   1126 					needs_clflush_after);
   1127 		if (ret == 0)
   1128 			goto next_page;
   1129 
   1130 		hit_slowpath = 1;
   1131 		mutex_unlock(&dev->struct_mutex);
   1132 		ret = shmem_pwrite_slow(page, shmem_page_offset, page_length,
   1133 					user_data, page_do_bit17_swizzling,
   1134 					partial_cacheline_write,
   1135 					needs_clflush_after);
   1136 
   1137 		mutex_lock(&dev->struct_mutex);
   1138 
   1139 		if (ret)
   1140 			goto out;
   1141 
   1142 next_page:
   1143 		remain -= page_length;
   1144 		user_data += page_length;
   1145 		offset += page_length;
   1146 	}
   1147 
   1148 out:
   1149 	i915_gem_object_unpin_pages(obj);
   1150 
   1151 	if (hit_slowpath) {
   1152 		/*
   1153 		 * Fixup: Flush cpu caches in case we didn't flush the dirty
   1154 		 * cachelines in-line while writing and the object moved
   1155 		 * out of the cpu write domain while we've dropped the lock.
   1156 		 */
   1157 		if (!needs_clflush_after &&
   1158 		    obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
   1159 			if (i915_gem_clflush_object(obj, obj->pin_display))
   1160 				needs_clflush_after = true;
   1161 		}
   1162 	}
   1163 
   1164 	if (needs_clflush_after)
   1165 		i915_gem_chipset_flush(dev);
   1166 	else
   1167 		obj->cache_dirty = true;
   1168 
   1169 	intel_fb_obj_flush(obj, false, ORIGIN_CPU);
   1170 	return ret;
   1171 }
   1172 
   1173 /**
   1174  * Writes data to the object referenced by handle.
   1175  *
   1176  * On error, the contents of the buffer that were to be modified are undefined.
   1177  */
   1178 int
   1179 i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
   1180 		      struct drm_file *file)
   1181 {
   1182 	struct drm_i915_private *dev_priv = dev->dev_private;
   1183 	struct drm_i915_gem_pwrite *args = data;
   1184 	struct drm_gem_object *gobj;
   1185 	struct drm_i915_gem_object *obj;
   1186 	int ret;
   1187 
   1188 	if (args->size == 0)
   1189 		return 0;
   1190 
   1191 	if (!access_ok(VERIFY_READ,
   1192 		       to_user_ptr(args->data_ptr),
   1193 		       args->size))
   1194 		return -EFAULT;
   1195 
   1196 #ifndef __NetBSD__		/* XXX prefault */
   1197 	if (likely(!i915.prefault_disable)) {
   1198 		ret = fault_in_multipages_readable(to_user_ptr(args->data_ptr),
   1199 						   args->size);
   1200 		if (ret)
   1201 			return -EFAULT;
   1202 	}
   1203 #endif
   1204 
   1205 	intel_runtime_pm_get(dev_priv);
   1206 
   1207 	ret = i915_mutex_lock_interruptible(dev);
   1208 	if (ret)
   1209 		goto put_rpm;
   1210 
   1211 	gobj = drm_gem_object_lookup(dev, file, args->handle);
   1212 	if (gobj == NULL) {
   1213 		ret = -ENOENT;
   1214 		goto unlock;
   1215 	}
   1216 	obj = to_intel_bo(gobj);
   1217 
   1218 	/* Bounds check destination. */
   1219 	if (args->offset > obj->base.size ||
   1220 	    args->size > obj->base.size - args->offset) {
   1221 		ret = -EINVAL;
   1222 		goto out;
   1223 	}
   1224 
   1225 	/* prime objects have no backing filp to GEM pread/pwrite
   1226 	 * pages from.
   1227 	 */
   1228 #ifdef __NetBSD__
   1229 	/* Also stolen objects.  */
   1230 	if (obj->base.gemo_shm_uao == NULL) {
   1231 		ret = -EINVAL;
   1232 		goto out;
   1233 	}
   1234 #else
   1235 	if (!obj->base.filp) {
   1236 		ret = -EINVAL;
   1237 		goto out;
   1238 	}
   1239 #endif
   1240 
   1241 	trace_i915_gem_object_pwrite(obj, args->offset, args->size);
   1242 
   1243 	ret = -EFAULT;
   1244 	/* We can only do the GTT pwrite on untiled buffers, as otherwise
   1245 	 * it would end up going through the fenced access, and we'll get
   1246 	 * different detiling behavior between reading and writing.
   1247 	 * pread/pwrite currently are reading and writing from the CPU
   1248 	 * perspective, requiring manual detiling by the client.
   1249 	 */
   1250 	if (obj->tiling_mode == I915_TILING_NONE &&
   1251 	    obj->base.write_domain != I915_GEM_DOMAIN_CPU &&
   1252 	    cpu_write_needs_clflush(obj)) {
   1253 		ret = i915_gem_gtt_pwrite_fast(dev, obj, args, file);
   1254 		/* Note that the gtt paths might fail with non-page-backed user
   1255 		 * pointers (e.g. gtt mappings when moving data between
   1256 		 * textures). Fallback to the shmem path in that case. */
   1257 	}
   1258 
   1259 	if (ret == -EFAULT || ret == -ENOSPC) {
   1260 		if (obj->phys_handle)
   1261 			ret = i915_gem_phys_pwrite(obj, args, file);
   1262 		else
   1263 			ret = i915_gem_shmem_pwrite(dev, obj, args, file);
   1264 	}
   1265 
   1266 out:
   1267 	drm_gem_object_unreference(&obj->base);
   1268 unlock:
   1269 	mutex_unlock(&dev->struct_mutex);
   1270 put_rpm:
   1271 	intel_runtime_pm_put(dev_priv);
   1272 
   1273 	return ret;
   1274 }
   1275 
   1276 int
   1277 i915_gem_check_wedge(struct i915_gpu_error *error,
   1278 		     bool interruptible)
   1279 {
   1280 	if (i915_reset_in_progress(error)) {
   1281 		/* Non-interruptible callers can't handle -EAGAIN, hence return
   1282 		 * -EIO unconditionally for these. */
   1283 		if (!interruptible)
   1284 			return -EIO;
   1285 
   1286 		/* Recovery complete, but the reset failed ... */
   1287 		if (i915_terminally_wedged(error))
   1288 			return -EIO;
   1289 
   1290 		/*
   1291 		 * Check if GPU Reset is in progress - we need intel_ring_begin
   1292 		 * to work properly to reinit the hw state while the gpu is
   1293 		 * still marked as reset-in-progress. Handle this with a flag.
   1294 		 */
   1295 		if (!error->reload_in_reset)
   1296 			return -EAGAIN;
   1297 	}
   1298 
   1299 	return 0;
   1300 }
   1301 
   1302 #ifndef __NetBSD__
   1303 static void fake_irq(unsigned long data)
   1304 {
   1305 	wake_up_process((struct task_struct *)data);
   1306 }
   1307 #endif
   1308 
   1309 static bool missed_irq(struct drm_i915_private *dev_priv,
   1310 		       struct intel_engine_cs *ring)
   1311 {
   1312 	return test_bit(ring->id, &dev_priv->gpu_error.missed_irq_rings);
   1313 }
   1314 
   1315 #ifndef __NetBSD__
   1316 static unsigned long local_clock_us(unsigned *cpu)
   1317 {
   1318 	unsigned long t;
   1319 
   1320 	/* Cheaply and approximately convert from nanoseconds to microseconds.
   1321 	 * The result and subsequent calculations are also defined in the same
   1322 	 * approximate microseconds units. The principal source of timing
   1323 	 * error here is from the simple truncation.
   1324 	 *
   1325 	 * Note that local_clock() is only defined wrt to the current CPU;
   1326 	 * the comparisons are no longer valid if we switch CPUs. Instead of
   1327 	 * blocking preemption for the entire busywait, we can detect the CPU
   1328 	 * switch and use that as indicator of system load and a reason to
   1329 	 * stop busywaiting, see busywait_stop().
   1330 	 */
   1331 	*cpu = get_cpu();
   1332 	t = local_clock() >> 10;
   1333 	put_cpu();
   1334 
   1335 	return t;
   1336 }
   1337 
   1338 static bool busywait_stop(unsigned long timeout, unsigned cpu)
   1339 {
   1340 	unsigned this_cpu;
   1341 
   1342 	if (time_after(local_clock_us(&this_cpu), timeout))
   1343 		return true;
   1344 
   1345 	return this_cpu != cpu;
   1346 }
   1347 #endif
   1348 
   1349 static int __i915_spin_request(struct drm_i915_gem_request *req, int state)
   1350 {
   1351 #ifndef __NetBSD__
   1352 	unsigned long timeout;
   1353 	unsigned cpu;
   1354 #endif
   1355 
   1356 	/* When waiting for high frequency requests, e.g. during synchronous
   1357 	 * rendering split between the CPU and GPU, the finite amount of time
   1358 	 * required to set up the irq and wait upon it limits the response
   1359 	 * rate. By busywaiting on the request completion for a short while we
   1360 	 * can service the high frequency waits as quick as possible. However,
   1361 	 * if it is a slow request, we want to sleep as quickly as possible.
   1362 	 * The tradeoff between waiting and sleeping is roughly the time it
   1363 	 * takes to sleep on a request, on the order of a microsecond.
   1364 	 */
   1365 
   1366 	if (req->ring->irq_refcount)
   1367 		return -EBUSY;
   1368 
   1369 	/* Only spin if we know the GPU is processing this request */
   1370 	if (!i915_gem_request_started(req, true))
   1371 		return -EAGAIN;
   1372 
   1373 #ifndef __NetBSD__		/* XXX No local clock in usec.  */
   1374 	timeout = local_clock_us(&cpu) + 5;
   1375 	while (!need_resched()) {
   1376 		if (i915_gem_request_completed(req, true))
   1377 			return 0;
   1378 
   1379 		if (signal_pending_state(state, current))
   1380 			break;
   1381 
   1382 		if (busywait_stop(timeout, cpu))
   1383 			break;
   1384 
   1385 		cpu_relax_lowlatency();
   1386 	}
   1387 #endif
   1388 
   1389 	if (i915_gem_request_completed(req, false))
   1390 		return 0;
   1391 
   1392 	return -EAGAIN;
   1393 }
   1394 
   1395 /**
   1396  * __i915_wait_request - wait until execution of request has finished
   1397  * @req: duh!
   1398  * @reset_counter: reset sequence associated with the given request
   1399  * @interruptible: do an interruptible wait (normally yes)
   1400  * @timeout: in - how long to wait (NULL forever); out - how much time remaining
   1401  *
   1402  * Note: It is of utmost importance that the passed in seqno and reset_counter
   1403  * values have been read by the caller in an smp safe manner. Where read-side
   1404  * locks are involved, it is sufficient to read the reset_counter before
   1405  * unlocking the lock that protects the seqno. For lockless tricks, the
   1406  * reset_counter _must_ be read before, and an appropriate smp_rmb must be
   1407  * inserted.
   1408  *
   1409  * Returns 0 if the request was found within the alloted time. Else returns the
   1410  * errno with remaining time filled in timeout argument.
   1411  */
   1412 int __i915_wait_request(struct drm_i915_gem_request *req,
   1413 			unsigned reset_counter,
   1414 			bool interruptible,
   1415 			s64 *timeout,
   1416 			struct intel_rps_client *rps)
   1417 {
   1418 	struct intel_engine_cs *ring = i915_gem_request_get_ring(req);
   1419 	struct drm_device *dev = ring->dev;
   1420 	struct drm_i915_private *dev_priv = dev->dev_private;
   1421 	const bool irq_test_in_progress =
   1422 	    dev_priv->gpu_error.test_irq_rings & intel_ring_flag(ring);
   1423 #ifdef __NetBSD__
   1424 	int state = 0;
   1425 	bool wedged;
   1426 #else
   1427 	int state = interruptible ? TASK_INTERRUPTIBLE : TASK_UNINTERRUPTIBLE;
   1428 	DEFINE_WAIT(wait);
   1429 	unsigned long timeout_expire;
   1430 #endif
   1431 	s64 before, now;
   1432 	int ret;
   1433 
   1434 	__insn_barrier();	/* ACCESS_ONCE for irq_test_in_progress */
   1435 
   1436 	WARN(!intel_irqs_enabled(dev_priv), "IRQs disabled");
   1437 
   1438 	if (list_empty(&req->list))
   1439 		return 0;
   1440 
   1441 	if (i915_gem_request_completed(req, true))
   1442 		return 0;
   1443 
   1444 #ifndef __NetBSD__
   1445 	timeout_expire = 0;
   1446 	if (timeout) {
   1447 		if (WARN_ON(*timeout < 0))
   1448 			return -EINVAL;
   1449 
   1450 		if (*timeout == 0)
   1451 			return -ETIME;
   1452 
   1453 		timeout_expire = jiffies + nsecs_to_jiffies_timeout(*timeout);
   1454 	}
   1455 #endif
   1456 
   1457 	if (INTEL_INFO(dev_priv)->gen >= 6)
   1458 		gen6_rps_boost(dev_priv, rps, req->emitted_jiffies);
   1459 
   1460 	/* Record current time in case interrupted by signal, or wedged */
   1461 	trace_i915_gem_request_wait_begin(req);
   1462 	before = ktime_get_raw_ns();
   1463 
   1464 	/* Optimistic spin for the next jiffie before touching IRQs */
   1465 	ret = __i915_spin_request(req, state);
   1466 	if (ret == 0)
   1467 		goto out;
   1468 
   1469 	if (!irq_test_in_progress && WARN_ON(!ring->irq_get(ring))) {
   1470 		ret = -ENODEV;
   1471 		goto out;
   1472 	}
   1473 
   1474 #ifdef __NetBSD__
   1475 #  define	EXIT_COND						      \
   1476 	((wedged = (reset_counter !=					      \
   1477 		atomic_read(&dev_priv->gpu_error.reset_counter))) ||	      \
   1478 	    i915_gem_request_completed(req, false))
   1479 	if (timeout) {
   1480 		int ticks = missed_irq(dev_priv, ring) ? 1 :
   1481 		    nsecs_to_jiffies_timeout(*timeout);
   1482 		if (interruptible) {
   1483 			DRM_SPIN_TIMED_WAIT_UNTIL(ret, &ring->irq_queue,
   1484 			    &dev_priv->irq_lock, ticks, EXIT_COND);
   1485 		} else {
   1486 			DRM_SPIN_TIMED_WAIT_NOINTR_UNTIL(ret, &ring->irq_queue,
   1487 			    &dev_priv->irq_lock, ticks, EXIT_COND);
   1488 		}
   1489 		if (ret < 0)	/* Failure: return negative error as is.  */
   1490 			;
   1491 		else if (ret == 0) /* Timed out: return -ETIME.  */
   1492 			ret = -ETIME;
   1493 		else		/* Succeeded (ret > 0): return 0.  */
   1494 			ret = 0;
   1495 	} else {
   1496 		if (interruptible) {
   1497 			DRM_SPIN_WAIT_UNTIL(ret, &ring->irq_queue,
   1498 			    &dev_priv->irq_lock, EXIT_COND);
   1499 		} else {
   1500 			DRM_SPIN_WAIT_NOINTR_UNTIL(ret, &ring->irq_queue,
   1501 			    &dev_priv->irq_lock, EXIT_COND);
   1502 		}
   1503 		/* ret is negative on failure or zero on success.  */
   1504 	}
   1505 	if (wedged) {
   1506 		ret = i915_gem_check_wedge(&dev_priv->gpu_error, interruptible);
   1507 		if (ret == 0)
   1508 			ret = -EAGAIN;
   1509 	}
   1510 #else
   1511 	for (;;) {
   1512 		struct timer_list timer;
   1513 
   1514 		prepare_to_wait(&ring->irq_queue, &wait, state);
   1515 
   1516 		/* We need to check whether any gpu reset happened in between
   1517 		 * the caller grabbing the seqno and now ... */
   1518 		if (reset_counter != atomic_read(&dev_priv->gpu_error.reset_counter)) {
   1519 			/* ... but upgrade the -EAGAIN to an -EIO if the gpu
   1520 			 * is truely gone. */
   1521 			ret = i915_gem_check_wedge(&dev_priv->gpu_error, interruptible);
   1522 			if (ret == 0)
   1523 				ret = -EAGAIN;
   1524 			break;
   1525 		}
   1526 
   1527 		if (i915_gem_request_completed(req, false)) {
   1528 			ret = 0;
   1529 			break;
   1530 		}
   1531 
   1532 		if (signal_pending_state(state, current)) {
   1533 			ret = -ERESTARTSYS;
   1534 			break;
   1535 		}
   1536 
   1537 		if (timeout && time_after_eq(jiffies, timeout_expire)) {
   1538 			ret = -ETIME;
   1539 			break;
   1540 		}
   1541 
   1542 		timer.function = NULL;
   1543 		if (timeout || missed_irq(dev_priv, ring)) {
   1544 			unsigned long expire;
   1545 
   1546 			setup_timer_on_stack(&timer, fake_irq, (unsigned long)current);
   1547 			expire = missed_irq(dev_priv, ring) ? jiffies + 1 : timeout_expire;
   1548 			mod_timer(&timer, expire);
   1549 		}
   1550 
   1551 		io_schedule();
   1552 
   1553 		if (timer.function) {
   1554 			del_singleshot_timer_sync(&timer);
   1555 			destroy_timer_on_stack(&timer);
   1556 		}
   1557 	}
   1558 #endif
   1559 	if (!irq_test_in_progress)
   1560 		ring->irq_put(ring);
   1561 
   1562 #ifndef __NetBSD__
   1563 	finish_wait(&ring->irq_queue, &wait);
   1564 #endif
   1565 
   1566 out:
   1567 	now = ktime_get_raw_ns();
   1568 	trace_i915_gem_request_wait_end(req);
   1569 
   1570 	if (timeout) {
   1571 		s64 tres = *timeout - (now - before);
   1572 
   1573 		*timeout = tres < 0 ? 0 : tres;
   1574 
   1575 		/*
   1576 		 * Apparently ktime isn't accurate enough and occasionally has a
   1577 		 * bit of mismatch in the jiffies<->nsecs<->ktime loop. So patch
   1578 		 * things up to make the test happy. We allow up to 1 jiffy.
   1579 		 *
   1580 		 * This is a regrssion from the timespec->ktime conversion.
   1581 		 */
   1582 		if (ret == -ETIME && *timeout < jiffies_to_usecs(1)*1000)
   1583 			*timeout = 0;
   1584 	}
   1585 
   1586 	return ret;
   1587 }
   1588 
   1589 int i915_gem_request_add_to_client(struct drm_i915_gem_request *req,
   1590 				   struct drm_file *file)
   1591 {
   1592 	struct drm_i915_private *dev_private __unused;
   1593 	struct drm_i915_file_private *file_priv;
   1594 
   1595 	WARN_ON(!req || !file || req->file_priv);
   1596 
   1597 	if (!req || !file)
   1598 		return -EINVAL;
   1599 
   1600 	if (req->file_priv)
   1601 		return -EINVAL;
   1602 
   1603 	dev_private = req->ring->dev->dev_private;
   1604 	file_priv = file->driver_priv;
   1605 
   1606 	spin_lock(&file_priv->mm.lock);
   1607 	req->file_priv = file_priv;
   1608 	list_add_tail(&req->client_list, &file_priv->mm.request_list);
   1609 	spin_unlock(&file_priv->mm.lock);
   1610 
   1611 #ifndef __NetBSD__
   1612 	req->pid = get_pid(task_pid(current));
   1613 #endif
   1614 
   1615 	return 0;
   1616 }
   1617 
   1618 static inline void
   1619 i915_gem_request_remove_from_client(struct drm_i915_gem_request *request)
   1620 {
   1621 	struct drm_i915_file_private *file_priv = request->file_priv;
   1622 
   1623 	if (!file_priv)
   1624 		return;
   1625 
   1626 	spin_lock(&file_priv->mm.lock);
   1627 	list_del(&request->client_list);
   1628 	request->file_priv = NULL;
   1629 	spin_unlock(&file_priv->mm.lock);
   1630 
   1631 #ifndef __NetBSD__
   1632 	put_pid(request->pid);
   1633 	request->pid = NULL;
   1634 #endif
   1635 }
   1636 
   1637 static void i915_gem_request_retire(struct drm_i915_gem_request *request)
   1638 {
   1639 	trace_i915_gem_request_retire(request);
   1640 
   1641 	/* We know the GPU must have read the request to have
   1642 	 * sent us the seqno + interrupt, so use the position
   1643 	 * of tail of the request to update the last known position
   1644 	 * of the GPU head.
   1645 	 *
   1646 	 * Note this requires that we are always called in request
   1647 	 * completion order.
   1648 	 */
   1649 	request->ringbuf->last_retired_head = request->postfix;
   1650 
   1651 	list_del_init(&request->list);
   1652 	i915_gem_request_remove_from_client(request);
   1653 
   1654 	i915_gem_request_unreference(request);
   1655 }
   1656 
   1657 static void
   1658 __i915_gem_request_retire__upto(struct drm_i915_gem_request *req)
   1659 {
   1660 	struct intel_engine_cs *engine = req->ring;
   1661 	struct drm_i915_gem_request *tmp;
   1662 
   1663 	lockdep_assert_held(&engine->dev->struct_mutex);
   1664 
   1665 	if (list_empty(&req->list))
   1666 		return;
   1667 
   1668 	do {
   1669 		tmp = list_first_entry(&engine->request_list,
   1670 				       typeof(*tmp), list);
   1671 
   1672 		i915_gem_request_retire(tmp);
   1673 	} while (tmp != req);
   1674 
   1675 	WARN_ON(i915_verify_lists(engine->dev));
   1676 }
   1677 
   1678 /**
   1679  * Waits for a request to be signaled, and cleans up the
   1680  * request and object lists appropriately for that event.
   1681  */
   1682 int
   1683 i915_wait_request(struct drm_i915_gem_request *req)
   1684 {
   1685 	struct drm_device *dev;
   1686 	struct drm_i915_private *dev_priv;
   1687 	bool interruptible;
   1688 	int ret;
   1689 
   1690 	BUG_ON(req == NULL);
   1691 
   1692 	dev = req->ring->dev;
   1693 	dev_priv = dev->dev_private;
   1694 	interruptible = dev_priv->mm.interruptible;
   1695 
   1696 	BUG_ON(!mutex_is_locked(&dev->struct_mutex));
   1697 
   1698 	ret = i915_gem_check_wedge(&dev_priv->gpu_error, interruptible);
   1699 	if (ret)
   1700 		return ret;
   1701 
   1702 	ret = __i915_wait_request(req,
   1703 				  atomic_read(&dev_priv->gpu_error.reset_counter),
   1704 				  interruptible, NULL, NULL);
   1705 	if (ret)
   1706 		return ret;
   1707 
   1708 	__i915_gem_request_retire__upto(req);
   1709 	return 0;
   1710 }
   1711 
   1712 /**
   1713  * Ensures that all rendering to the object has completed and the object is
   1714  * safe to unbind from the GTT or access from the CPU.
   1715  */
   1716 int
   1717 i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj,
   1718 			       bool readonly)
   1719 {
   1720 	int ret, i;
   1721 
   1722 	if (!obj->active)
   1723 		return 0;
   1724 
   1725 	if (readonly) {
   1726 		if (obj->last_write_req != NULL) {
   1727 			ret = i915_wait_request(obj->last_write_req);
   1728 			if (ret)
   1729 				return ret;
   1730 
   1731 			i = obj->last_write_req->ring->id;
   1732 			if (obj->last_read_req[i] == obj->last_write_req)
   1733 				i915_gem_object_retire__read(obj, i);
   1734 			else
   1735 				i915_gem_object_retire__write(obj);
   1736 		}
   1737 	} else {
   1738 		for (i = 0; i < I915_NUM_RINGS; i++) {
   1739 			if (obj->last_read_req[i] == NULL)
   1740 				continue;
   1741 
   1742 			ret = i915_wait_request(obj->last_read_req[i]);
   1743 			if (ret)
   1744 				return ret;
   1745 
   1746 			i915_gem_object_retire__read(obj, i);
   1747 		}
   1748 		RQ_BUG_ON(obj->active);
   1749 	}
   1750 
   1751 	return 0;
   1752 }
   1753 
   1754 static void
   1755 i915_gem_object_retire_request(struct drm_i915_gem_object *obj,
   1756 			       struct drm_i915_gem_request *req)
   1757 {
   1758 	int ring = req->ring->id;
   1759 
   1760 	if (obj->last_read_req[ring] == req)
   1761 		i915_gem_object_retire__read(obj, ring);
   1762 	else if (obj->last_write_req == req)
   1763 		i915_gem_object_retire__write(obj);
   1764 
   1765 	__i915_gem_request_retire__upto(req);
   1766 }
   1767 
   1768 /* A nonblocking variant of the above wait. This is a highly dangerous routine
   1769  * as the object state may change during this call.
   1770  */
   1771 static __must_check int
   1772 i915_gem_object_wait_rendering__nonblocking(struct drm_i915_gem_object *obj,
   1773 					    struct intel_rps_client *rps,
   1774 					    bool readonly)
   1775 {
   1776 	struct drm_device *dev = obj->base.dev;
   1777 	struct drm_i915_private *dev_priv = dev->dev_private;
   1778 	struct drm_i915_gem_request *requests[I915_NUM_RINGS];
   1779 	unsigned reset_counter;
   1780 	int ret, i, n = 0;
   1781 
   1782 	BUG_ON(!mutex_is_locked(&dev->struct_mutex));
   1783 	BUG_ON(!dev_priv->mm.interruptible);
   1784 
   1785 	if (!obj->active)
   1786 		return 0;
   1787 
   1788 	ret = i915_gem_check_wedge(&dev_priv->gpu_error, true);
   1789 	if (ret)
   1790 		return ret;
   1791 
   1792 	reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter);
   1793 
   1794 	if (readonly) {
   1795 		struct drm_i915_gem_request *req;
   1796 
   1797 		req = obj->last_write_req;
   1798 		if (req == NULL)
   1799 			return 0;
   1800 
   1801 		requests[n++] = i915_gem_request_reference(req);
   1802 	} else {
   1803 		for (i = 0; i < I915_NUM_RINGS; i++) {
   1804 			struct drm_i915_gem_request *req;
   1805 
   1806 			req = obj->last_read_req[i];
   1807 			if (req == NULL)
   1808 				continue;
   1809 
   1810 			requests[n++] = i915_gem_request_reference(req);
   1811 		}
   1812 	}
   1813 
   1814 	mutex_unlock(&dev->struct_mutex);
   1815 	for (i = 0; ret == 0 && i < n; i++)
   1816 		ret = __i915_wait_request(requests[i], reset_counter, true,
   1817 					  NULL, rps);
   1818 	mutex_lock(&dev->struct_mutex);
   1819 
   1820 	for (i = 0; i < n; i++) {
   1821 		if (ret == 0)
   1822 			i915_gem_object_retire_request(obj, requests[i]);
   1823 		i915_gem_request_unreference(requests[i]);
   1824 	}
   1825 
   1826 	return ret;
   1827 }
   1828 
   1829 static struct intel_rps_client *to_rps_client(struct drm_file *file)
   1830 {
   1831 	struct drm_i915_file_private *fpriv = file->driver_priv;
   1832 	return &fpriv->rps;
   1833 }
   1834 
   1835 /**
   1836  * Called when user space prepares to use an object with the CPU, either
   1837  * through the mmap ioctl's mapping or a GTT mapping.
   1838  */
   1839 int
   1840 i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
   1841 			  struct drm_file *file)
   1842 {
   1843 	struct drm_i915_gem_set_domain *args = data;
   1844 	struct drm_gem_object *gobj;
   1845 	struct drm_i915_gem_object *obj;
   1846 	uint32_t read_domains = args->read_domains;
   1847 	uint32_t write_domain = args->write_domain;
   1848 	int ret;
   1849 
   1850 	/* Only handle setting domains to types used by the CPU. */
   1851 	if (write_domain & I915_GEM_GPU_DOMAINS)
   1852 		return -EINVAL;
   1853 
   1854 	if (read_domains & I915_GEM_GPU_DOMAINS)
   1855 		return -EINVAL;
   1856 
   1857 	/* Having something in the write domain implies it's in the read
   1858 	 * domain, and only that read domain.  Enforce that in the request.
   1859 	 */
   1860 	if (write_domain != 0 && read_domains != write_domain)
   1861 		return -EINVAL;
   1862 
   1863 	ret = i915_mutex_lock_interruptible(dev);
   1864 	if (ret)
   1865 		return ret;
   1866 
   1867 	gobj = drm_gem_object_lookup(dev, file, args->handle);
   1868 	if (gobj == NULL) {
   1869 		ret = -ENOENT;
   1870 		goto unlock;
   1871 	}
   1872 	obj = to_intel_bo(gobj);
   1873 
   1874 	/* Try to flush the object off the GPU without holding the lock.
   1875 	 * We will repeat the flush holding the lock in the normal manner
   1876 	 * to catch cases where we are gazumped.
   1877 	 */
   1878 	ret = i915_gem_object_wait_rendering__nonblocking(obj,
   1879 							  to_rps_client(file),
   1880 							  !write_domain);
   1881 	if (ret)
   1882 		goto unref;
   1883 
   1884 	if (read_domains & I915_GEM_DOMAIN_GTT)
   1885 		ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0);
   1886 	else
   1887 		ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0);
   1888 
   1889 	if (write_domain != 0)
   1890 		intel_fb_obj_invalidate(obj,
   1891 					write_domain == I915_GEM_DOMAIN_GTT ?
   1892 					ORIGIN_GTT : ORIGIN_CPU);
   1893 
   1894 unref:
   1895 	drm_gem_object_unreference(&obj->base);
   1896 unlock:
   1897 	mutex_unlock(&dev->struct_mutex);
   1898 	return ret;
   1899 }
   1900 
   1901 /**
   1902  * Called when user space has done writes to this buffer
   1903  */
   1904 int
   1905 i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data,
   1906 			 struct drm_file *file)
   1907 {
   1908 	struct drm_i915_gem_sw_finish *args = data;
   1909 	struct drm_gem_object *gobj;
   1910 	struct drm_i915_gem_object *obj;
   1911 	int ret = 0;
   1912 
   1913 	ret = i915_mutex_lock_interruptible(dev);
   1914 	if (ret)
   1915 		return ret;
   1916 
   1917 	gobj = drm_gem_object_lookup(dev, file, args->handle);
   1918 	if (gobj == NULL) {
   1919 		ret = -ENOENT;
   1920 		goto unlock;
   1921 	}
   1922 	obj = to_intel_bo(gobj);
   1923 
   1924 	/* Pinned buffers may be scanout, so flush the cache */
   1925 	if (obj->pin_display)
   1926 		i915_gem_object_flush_cpu_write_domain(obj);
   1927 
   1928 	drm_gem_object_unreference(&obj->base);
   1929 unlock:
   1930 	mutex_unlock(&dev->struct_mutex);
   1931 	return ret;
   1932 }
   1933 
   1934 /**
   1935  * Maps the contents of an object, returning the address it is mapped
   1936  * into.
   1937  *
   1938  * While the mapping holds a reference on the contents of the object, it doesn't
   1939  * imply a ref on the object itself.
   1940  *
   1941  * IMPORTANT:
   1942  *
   1943  * DRM driver writers who look a this function as an example for how to do GEM
   1944  * mmap support, please don't implement mmap support like here. The modern way
   1945  * to implement DRM mmap support is with an mmap offset ioctl (like
   1946  * i915_gem_mmap_gtt) and then using the mmap syscall on the DRM fd directly.
   1947  * That way debug tooling like valgrind will understand what's going on, hiding
   1948  * the mmap call in a driver private ioctl will break that. The i915 driver only
   1949  * does cpu mmaps this way because we didn't know better.
   1950  */
   1951 int
   1952 i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
   1953 		    struct drm_file *file)
   1954 {
   1955 	struct drm_i915_gem_mmap *args = data;
   1956 	struct drm_gem_object *obj;
   1957 	unsigned long addr;
   1958 #ifdef __NetBSD__
   1959 	int ret;
   1960 #endif
   1961 
   1962 	if (args->flags & ~(I915_MMAP_WC))
   1963 		return -EINVAL;
   1964 
   1965 #if 0
   1966 	/* XXX cpu_has_pat == CPUID_PAT, do we care to do this check */
   1967 	if (args->flags & I915_MMAP_WC && !cpu_has_pat)
   1968 		return -ENODEV;
   1969 #endif
   1970 
   1971 	obj = drm_gem_object_lookup(dev, file, args->handle);
   1972 	if (obj == NULL)
   1973 		return -ENOENT;
   1974 
   1975 	/* prime objects have no backing filp to GEM mmap
   1976 	 * pages from.
   1977 	 */
   1978 #ifdef __NetBSD__
   1979 	/* Also stolen objects (XXX can we get them here?)  */
   1980 	if (obj->gemo_shm_uao == NULL) {
   1981 		drm_gem_object_unreference_unlocked(obj);
   1982 		return -EINVAL;
   1983 	}
   1984 #else
   1985 	if (!obj->filp) {
   1986 		drm_gem_object_unreference_unlocked(obj);
   1987 		return -EINVAL;
   1988 	}
   1989 #endif
   1990 
   1991 #ifdef __NetBSD__
   1992 	addr = (*curproc->p_emul->e_vm_default_addr)(curproc,
   1993 	    (vaddr_t)curproc->p_vmspace->vm_daddr, args->size,
   1994 	    curproc->p_vmspace->vm_map.flags & VM_MAP_TOPDOWN);
   1995 	/* XXX errno NetBSD->Linux */
   1996 	ret = -uvm_map(&curproc->p_vmspace->vm_map, &addr, args->size,
   1997 	    obj->gemo_shm_uao, args->offset, 0,
   1998 	    UVM_MAPFLAG((VM_PROT_READ | VM_PROT_WRITE),
   1999 		(VM_PROT_READ | VM_PROT_WRITE), UVM_INH_COPY, UVM_ADV_NORMAL,
   2000 		0));
   2001 	if (ret) {
   2002 		drm_gem_object_unreference_unlocked(obj);
   2003 		return ret;
   2004 	}
   2005 	uao_reference(obj->gemo_shm_uao);
   2006 	drm_gem_object_unreference_unlocked(obj);
   2007 #else
   2008 	addr = vm_mmap(obj->filp, 0, args->size,
   2009 		       PROT_READ | PROT_WRITE, MAP_SHARED,
   2010 		       args->offset);
   2011 	if (args->flags & I915_MMAP_WC) {
   2012 		struct mm_struct *mm = current->mm;
   2013 		struct vm_area_struct *vma;
   2014 
   2015 		down_write(&mm->mmap_sem);
   2016 		vma = find_vma(mm, addr);
   2017 		if (vma)
   2018 			vma->vm_page_prot =
   2019 				pgprot_writecombine(vm_get_page_prot(vma->vm_flags));
   2020 		else
   2021 			addr = -ENOMEM;
   2022 		up_write(&mm->mmap_sem);
   2023 	}
   2024 	drm_gem_object_unreference_unlocked(obj);
   2025 	if (IS_ERR((void *)addr))
   2026 		return addr;
   2027 #endif
   2028 
   2029 	args->addr_ptr = (uint64_t) addr;
   2030 
   2031 	return 0;
   2032 }
   2033 
   2034 #ifdef __NetBSD__		/* XXX gem gtt fault */
   2035 static int	i915_udv_fault(struct uvm_faultinfo *, vaddr_t,
   2036 		    struct vm_page **, int, int, vm_prot_t, int, paddr_t);
   2037 
   2038 int
   2039 i915_gem_fault(struct uvm_faultinfo *ufi, vaddr_t vaddr, struct vm_page **pps,
   2040     int npages, int centeridx, vm_prot_t access_type, int flags)
   2041 {
   2042 	struct uvm_object *uobj = ufi->entry->object.uvm_obj;
   2043 	struct drm_gem_object *gem_obj =
   2044 	    container_of(uobj, struct drm_gem_object, gemo_uvmobj);
   2045 	struct drm_i915_gem_object *obj = to_intel_bo(gem_obj);
   2046 	struct drm_device *dev = obj->base.dev;
   2047 	struct drm_i915_private *dev_priv = dev->dev_private;
   2048 	voff_t byte_offset;
   2049 	pgoff_t page_offset;
   2050 	int ret = 0;
   2051 	bool write = ISSET(access_type, VM_PROT_WRITE)? 1 : 0;
   2052 
   2053 	byte_offset = (ufi->entry->offset + (vaddr - ufi->entry->start));
   2054 	KASSERT(byte_offset <= obj->base.size);
   2055 	page_offset = (byte_offset >> PAGE_SHIFT);
   2056 
   2057 	intel_runtime_pm_get(dev_priv);
   2058 
   2059 	/* Thanks, uvm, but we don't need this lock.  */
   2060 	mutex_exit(uobj->vmobjlock);
   2061 
   2062 	ret = i915_mutex_lock_interruptible(dev);
   2063 	if (ret)
   2064 		goto out;
   2065 
   2066 	trace_i915_gem_object_fault(obj, page_offset, true, write);
   2067 
   2068 	ret = i915_gem_object_wait_rendering__nonblocking(obj, NULL, !write);
   2069 	if (ret)
   2070 		goto unlock;
   2071 
   2072 	if ((obj->cache_level != I915_CACHE_NONE) && !HAS_LLC(dev)) {
   2073 		ret = -EINVAL;
   2074 		goto unlock;
   2075 	}
   2076 
   2077 	ret = i915_gem_obj_ggtt_pin(obj, 0, PIN_MAPPABLE);
   2078 	if (ret)
   2079 		goto unlock;
   2080 
   2081 	ret = i915_gem_object_set_to_gtt_domain(obj, write);
   2082 	if (ret)
   2083 		goto unpin;
   2084 
   2085 	ret = i915_gem_object_get_fence(obj);
   2086 	if (ret)
   2087 		goto unpin;
   2088 
   2089 	obj->fault_mappable = true;
   2090 
   2091 	/* XXX errno NetBSD->Linux */
   2092 	ret = -i915_udv_fault(ufi, vaddr, pps, npages, centeridx, access_type,
   2093 	    flags,
   2094 	    (dev_priv->gtt.mappable_base + i915_gem_obj_ggtt_offset(obj)));
   2095 unpin:
   2096 	i915_gem_object_ggtt_unpin(obj);
   2097 unlock:
   2098 	mutex_unlock(&dev->struct_mutex);
   2099 out:
   2100 	mutex_enter(uobj->vmobjlock);
   2101 	uvmfault_unlockall(ufi, ufi->entry->aref.ar_amap, uobj);
   2102 	if (ret == -ERESTART)
   2103 		uvm_wait("i915flt");
   2104 
   2105 	/*
   2106 	 * Remap EINTR to success, so that we return to userland.
   2107 	 * On the way out, we'll deliver the signal, and if the signal
   2108 	 * is not fatal then the user code which faulted will most likely
   2109 	 * fault again, and we'll come back here for another try.
   2110 	 */
   2111 	if (ret == -EINTR)
   2112 		ret = 0;
   2113 	/* XXX Deal with GPU hangs here...  */
   2114 	intel_runtime_pm_put(dev_priv);
   2115 	/* XXX errno Linux->NetBSD */
   2116 	return -ret;
   2117 }
   2118 
   2119 /*
   2120  * XXX i915_udv_fault is copypasta of udv_fault from uvm_device.c.
   2121  *
   2122  * XXX pmap_enter_default instead of pmap_enter because of a problem
   2123  * with using weak aliases in kernel modules or something.
   2124  */
   2125 int	pmap_enter_default(pmap_t, vaddr_t, paddr_t, vm_prot_t, unsigned);
   2126 
   2127 static int
   2128 i915_udv_fault(struct uvm_faultinfo *ufi, vaddr_t vaddr, struct vm_page **pps,
   2129     int npages, int centeridx, vm_prot_t access_type, int flags,
   2130     paddr_t gtt_paddr)
   2131 {
   2132 	struct vm_map_entry *entry = ufi->entry;
   2133 	vaddr_t curr_va;
   2134 	off_t curr_offset;
   2135 	paddr_t paddr;
   2136 	u_int mmapflags;
   2137 	int lcv, retval;
   2138 	vm_prot_t mapprot;
   2139 	UVMHIST_FUNC("i915_udv_fault"); UVMHIST_CALLED(maphist);
   2140 	UVMHIST_LOG(maphist,"  flags=%jd", flags,0,0,0);
   2141 
   2142 	/*
   2143 	 * we do not allow device mappings to be mapped copy-on-write
   2144 	 * so we kill any attempt to do so here.
   2145 	 */
   2146 
   2147 	if (UVM_ET_ISCOPYONWRITE(entry)) {
   2148 		UVMHIST_LOG(maphist, "<- failed -- COW entry (etype=0x%jx)",
   2149 		entry->etype, 0,0,0);
   2150 		return(EIO);
   2151 	}
   2152 
   2153 	/*
   2154 	 * now we must determine the offset in udv to use and the VA to
   2155 	 * use for pmap_enter.  note that we always use orig_map's pmap
   2156 	 * for pmap_enter (even if we have a submap).   since virtual
   2157 	 * addresses in a submap must match the main map, this is ok.
   2158 	 */
   2159 
   2160 	/* udv offset = (offset from start of entry) + entry's offset */
   2161 	curr_offset = entry->offset + (vaddr - entry->start);
   2162 	/* pmap va = vaddr (virtual address of pps[0]) */
   2163 	curr_va = vaddr;
   2164 
   2165 	/*
   2166 	 * loop over the page range entering in as needed
   2167 	 */
   2168 
   2169 	retval = 0;
   2170 	for (lcv = 0 ; lcv < npages ; lcv++, curr_offset += PAGE_SIZE,
   2171 	    curr_va += PAGE_SIZE) {
   2172 		if ((flags & PGO_ALLPAGES) == 0 && lcv != centeridx)
   2173 			continue;
   2174 
   2175 		if (pps[lcv] == PGO_DONTCARE)
   2176 			continue;
   2177 
   2178 		paddr = (gtt_paddr + curr_offset);
   2179 		mmapflags = 0;
   2180 		mapprot = ufi->entry->protection;
   2181 		UVMHIST_LOG(maphist,
   2182 		    "  MAPPING: device: pm=0x%#jx, va=0x%jx, pa=0x%jx, at=%jd",
   2183 		    (uintptr_t)ufi->orig_map->pmap, curr_va, paddr, mapprot);
   2184 		if (pmap_enter_default(ufi->orig_map->pmap, curr_va, paddr, mapprot,
   2185 		    PMAP_CANFAIL | mapprot | mmapflags) != 0) {
   2186 			/*
   2187 			 * pmap_enter() didn't have the resource to
   2188 			 * enter this mapping.  Unlock everything,
   2189 			 * wait for the pagedaemon to free up some
   2190 			 * pages, and then tell uvm_fault() to start
   2191 			 * the fault again.
   2192 			 *
   2193 			 * XXX Needs some rethinking for the PGO_ALLPAGES
   2194 			 * XXX case.
   2195 			 */
   2196 			pmap_update(ufi->orig_map->pmap);	/* sync what we have so far */
   2197 			return (ERESTART);
   2198 		}
   2199 	}
   2200 
   2201 	pmap_update(ufi->orig_map->pmap);
   2202 	return (retval);
   2203 }
   2204 #else
   2205 /**
   2206  * i915_gem_fault - fault a page into the GTT
   2207  * @vma: VMA in question
   2208  * @vmf: fault info
   2209  *
   2210  * The fault handler is set up by drm_gem_mmap() when a object is GTT mapped
   2211  * from userspace.  The fault handler takes care of binding the object to
   2212  * the GTT (if needed), allocating and programming a fence register (again,
   2213  * only if needed based on whether the old reg is still valid or the object
   2214  * is tiled) and inserting a new PTE into the faulting process.
   2215  *
   2216  * Note that the faulting process may involve evicting existing objects
   2217  * from the GTT and/or fence registers to make room.  So performance may
   2218  * suffer if the GTT working set is large or there are few fence registers
   2219  * left.
   2220  */
   2221 int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
   2222 {
   2223 	struct drm_i915_gem_object *obj = to_intel_bo(vma->vm_private_data);
   2224 	struct drm_device *dev = obj->base.dev;
   2225 	struct drm_i915_private *dev_priv = dev->dev_private;
   2226 	struct i915_ggtt_view view = i915_ggtt_view_normal;
   2227 	pgoff_t page_offset;
   2228 	unsigned long pfn;
   2229 	int ret = 0;
   2230 	bool write = !!(vmf->flags & FAULT_FLAG_WRITE);
   2231 
   2232 	intel_runtime_pm_get(dev_priv);
   2233 
   2234 	/* We don't use vmf->pgoff since that has the fake offset */
   2235 	page_offset = ((unsigned long)vmf->virtual_address - vma->vm_start) >>
   2236 		PAGE_SHIFT;
   2237 
   2238 	ret = i915_mutex_lock_interruptible(dev);
   2239 	if (ret)
   2240 		goto out;
   2241 
   2242 	trace_i915_gem_object_fault(obj, page_offset, true, write);
   2243 
   2244 	/* Try to flush the object off the GPU first without holding the lock.
   2245 	 * Upon reacquiring the lock, we will perform our sanity checks and then
   2246 	 * repeat the flush holding the lock in the normal manner to catch cases
   2247 	 * where we are gazumped.
   2248 	 */
   2249 	ret = i915_gem_object_wait_rendering__nonblocking(obj, NULL, !write);
   2250 	if (ret)
   2251 		goto unlock;
   2252 
   2253 	/* Access to snoopable pages through the GTT is incoherent. */
   2254 	if (obj->cache_level != I915_CACHE_NONE && !HAS_LLC(dev)) {
   2255 		ret = -EFAULT;
   2256 		goto unlock;
   2257 	}
   2258 
   2259 	/* Use a partial view if the object is bigger than the aperture. */
   2260 	if (obj->base.size >= dev_priv->gtt.mappable_end &&
   2261 	    obj->tiling_mode == I915_TILING_NONE) {
   2262 		static const unsigned int chunk_size = 256; // 1 MiB
   2263 
   2264 		memset(&view, 0, sizeof(view));
   2265 		view.type = I915_GGTT_VIEW_PARTIAL;
   2266 		view.params.partial.offset = rounddown(page_offset, chunk_size);
   2267 		view.params.partial.size =
   2268 			min_t(unsigned int,
   2269 			      chunk_size,
   2270 			      (vma->vm_end - vma->vm_start)/PAGE_SIZE -
   2271 			      view.params.partial.offset);
   2272 	}
   2273 
   2274 	/* Now pin it into the GTT if needed */
   2275 	ret = i915_gem_object_ggtt_pin(obj, &view, 0, PIN_MAPPABLE);
   2276 	if (ret)
   2277 		goto unlock;
   2278 
   2279 	ret = i915_gem_object_set_to_gtt_domain(obj, write);
   2280 	if (ret)
   2281 		goto unpin;
   2282 
   2283 	ret = i915_gem_object_get_fence(obj);
   2284 	if (ret)
   2285 		goto unpin;
   2286 
   2287 	/* Finally, remap it using the new GTT offset */
   2288 	pfn = dev_priv->gtt.mappable_base +
   2289 		i915_gem_obj_ggtt_offset_view(obj, &view);
   2290 	pfn >>= PAGE_SHIFT;
   2291 
   2292 	if (unlikely(view.type == I915_GGTT_VIEW_PARTIAL)) {
   2293 		/* Overriding existing pages in partial view does not cause
   2294 		 * us any trouble as TLBs are still valid because the fault
   2295 		 * is due to userspace losing part of the mapping or never
   2296 		 * having accessed it before (at this partials' range).
   2297 		 */
   2298 		unsigned long base = vma->vm_start +
   2299 				     (view.params.partial.offset << PAGE_SHIFT);
   2300 		unsigned int i;
   2301 
   2302 		for (i = 0; i < view.params.partial.size; i++) {
   2303 			ret = vm_insert_pfn(vma, base + i * PAGE_SIZE, pfn + i);
   2304 			if (ret)
   2305 				break;
   2306 		}
   2307 
   2308 		obj->fault_mappable = true;
   2309 	} else {
   2310 		if (!obj->fault_mappable) {
   2311 			unsigned long size = min_t(unsigned long,
   2312 						   vma->vm_end - vma->vm_start,
   2313 						   obj->base.size);
   2314 			int i;
   2315 
   2316 			for (i = 0; i < size >> PAGE_SHIFT; i++) {
   2317 				ret = vm_insert_pfn(vma,
   2318 						    (unsigned long)vma->vm_start + i * PAGE_SIZE,
   2319 						    pfn + i);
   2320 				if (ret)
   2321 					break;
   2322 			}
   2323 
   2324 			obj->fault_mappable = true;
   2325 		} else
   2326 			ret = vm_insert_pfn(vma,
   2327 					    (unsigned long)vmf->virtual_address,
   2328 					    pfn + page_offset);
   2329 	}
   2330 unpin:
   2331 	i915_gem_object_ggtt_unpin_view(obj, &view);
   2332 unlock:
   2333 	mutex_unlock(&dev->struct_mutex);
   2334 out:
   2335 	switch (ret) {
   2336 	case -EIO:
   2337 		/*
   2338 		 * We eat errors when the gpu is terminally wedged to avoid
   2339 		 * userspace unduly crashing (gl has no provisions for mmaps to
   2340 		 * fail). But any other -EIO isn't ours (e.g. swap in failure)
   2341 		 * and so needs to be reported.
   2342 		 */
   2343 		if (!i915_terminally_wedged(&dev_priv->gpu_error)) {
   2344 			ret = VM_FAULT_SIGBUS;
   2345 			break;
   2346 		}
   2347 	case -EAGAIN:
   2348 		/*
   2349 		 * EAGAIN means the gpu is hung and we'll wait for the error
   2350 		 * handler to reset everything when re-faulting in
   2351 		 * i915_mutex_lock_interruptible.
   2352 		 */
   2353 	case 0:
   2354 	case -ERESTARTSYS:
   2355 	case -EINTR:
   2356 	case -EBUSY:
   2357 		/*
   2358 		 * EBUSY is ok: this just means that another thread
   2359 		 * already did the job.
   2360 		 */
   2361 		ret = VM_FAULT_NOPAGE;
   2362 		break;
   2363 	case -ENOMEM:
   2364 		ret = VM_FAULT_OOM;
   2365 		break;
   2366 	case -ENOSPC:
   2367 	case -EFAULT:
   2368 		ret = VM_FAULT_SIGBUS;
   2369 		break;
   2370 	default:
   2371 		WARN_ONCE(ret, "unhandled error in i915_gem_fault: %i\n", ret);
   2372 		ret = VM_FAULT_SIGBUS;
   2373 		break;
   2374 	}
   2375 
   2376 	intel_runtime_pm_put(dev_priv);
   2377 	return ret;
   2378 }
   2379 #endif
   2380 
   2381 /**
   2382  * i915_gem_release_mmap - remove physical page mappings
   2383  * @obj: obj in question
   2384  *
   2385  * Preserve the reservation of the mmapping with the DRM core code, but
   2386  * relinquish ownership of the pages back to the system.
   2387  *
   2388  * It is vital that we remove the page mapping if we have mapped a tiled
   2389  * object through the GTT and then lose the fence register due to
   2390  * resource pressure. Similarly if the object has been moved out of the
   2391  * aperture, than pages mapped into userspace must be revoked. Removing the
   2392  * mapping will then trigger a page fault on the next user access, allowing
   2393  * fixup by i915_gem_fault().
   2394  */
   2395 void
   2396 i915_gem_release_mmap(struct drm_i915_gem_object *obj)
   2397 {
   2398 	if (!obj->fault_mappable)
   2399 		return;
   2400 
   2401 #ifdef __NetBSD__		/* XXX gem gtt fault */
   2402 	{
   2403 		struct drm_device *const dev = obj->base.dev;
   2404 		struct drm_i915_private *const dev_priv = dev->dev_private;
   2405 		const paddr_t start = dev_priv->gtt.mappable_base +
   2406 		    i915_gem_obj_ggtt_offset(obj);
   2407 		const size_t size = obj->base.size;
   2408 		const paddr_t end = start + size;
   2409 		paddr_t pa;
   2410 
   2411 		KASSERT((start & (PAGE_SIZE - 1)) == 0);
   2412 		KASSERT((size & (PAGE_SIZE - 1)) == 0);
   2413 
   2414 		for (pa = start; pa < end; pa += PAGE_SIZE)
   2415 			pmap_pv_protect(pa, VM_PROT_NONE);
   2416 	}
   2417 #else
   2418 	drm_vma_node_unmap(&obj->base.vma_node,
   2419 			   obj->base.dev->anon_inode->i_mapping);
   2420 #endif
   2421 	obj->fault_mappable = false;
   2422 }
   2423 
   2424 void
   2425 i915_gem_release_all_mmaps(struct drm_i915_private *dev_priv)
   2426 {
   2427 	struct drm_i915_gem_object *obj;
   2428 
   2429 	list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list)
   2430 		i915_gem_release_mmap(obj);
   2431 }
   2432 
   2433 uint32_t
   2434 i915_gem_get_gtt_size(struct drm_device *dev, uint32_t size, int tiling_mode)
   2435 {
   2436 	uint32_t gtt_size;
   2437 
   2438 	if (INTEL_INFO(dev)->gen >= 4 ||
   2439 	    tiling_mode == I915_TILING_NONE)
   2440 		return size;
   2441 
   2442 	/* Previous chips need a power-of-two fence region when tiling */
   2443 	if (INTEL_INFO(dev)->gen == 3)
   2444 		gtt_size = 1024*1024;
   2445 	else
   2446 		gtt_size = 512*1024;
   2447 
   2448 	while (gtt_size < size)
   2449 		gtt_size <<= 1;
   2450 
   2451 	return gtt_size;
   2452 }
   2453 
   2454 /**
   2455  * i915_gem_get_gtt_alignment - return required GTT alignment for an object
   2456  * @obj: object to check
   2457  *
   2458  * Return the required GTT alignment for an object, taking into account
   2459  * potential fence register mapping.
   2460  */
   2461 uint32_t
   2462 i915_gem_get_gtt_alignment(struct drm_device *dev, uint32_t size,
   2463 			   int tiling_mode, bool fenced)
   2464 {
   2465 	/*
   2466 	 * Minimum alignment is 4k (GTT page size), but might be greater
   2467 	 * if a fence register is needed for the object.
   2468 	 */
   2469 	if (INTEL_INFO(dev)->gen >= 4 || (!fenced && IS_G33(dev)) ||
   2470 	    tiling_mode == I915_TILING_NONE)
   2471 		return 4096;
   2472 
   2473 	/*
   2474 	 * Previous chips need to be aligned to the size of the smallest
   2475 	 * fence register that can contain the object.
   2476 	 */
   2477 	return i915_gem_get_gtt_size(dev, size, tiling_mode);
   2478 }
   2479 
   2480 static int i915_gem_object_create_mmap_offset(struct drm_i915_gem_object *obj)
   2481 {
   2482 	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
   2483 	int ret;
   2484 
   2485 	if (drm_vma_node_has_offset(&obj->base.vma_node))
   2486 		return 0;
   2487 
   2488 	dev_priv->mm.shrinker_no_lock_stealing = true;
   2489 
   2490 	ret = drm_gem_create_mmap_offset(&obj->base);
   2491 	if (ret != -ENOSPC)
   2492 		goto out;
   2493 
   2494 	/* Badly fragmented mmap space? The only way we can recover
   2495 	 * space is by destroying unwanted objects. We can't randomly release
   2496 	 * mmap_offsets as userspace expects them to be persistent for the
   2497 	 * lifetime of the objects. The closest we can is to release the
   2498 	 * offsets on purgeable objects by truncating it and marking it purged,
   2499 	 * which prevents userspace from ever using that object again.
   2500 	 */
   2501 	i915_gem_shrink(dev_priv,
   2502 			obj->base.size >> PAGE_SHIFT,
   2503 			I915_SHRINK_BOUND |
   2504 			I915_SHRINK_UNBOUND |
   2505 			I915_SHRINK_PURGEABLE);
   2506 	ret = drm_gem_create_mmap_offset(&obj->base);
   2507 	if (ret != -ENOSPC)
   2508 		goto out;
   2509 
   2510 	i915_gem_shrink_all(dev_priv);
   2511 	ret = drm_gem_create_mmap_offset(&obj->base);
   2512 out:
   2513 	dev_priv->mm.shrinker_no_lock_stealing = false;
   2514 
   2515 	return ret;
   2516 }
   2517 
   2518 static void i915_gem_object_free_mmap_offset(struct drm_i915_gem_object *obj)
   2519 {
   2520 	drm_gem_free_mmap_offset(&obj->base);
   2521 }
   2522 
   2523 int
   2524 i915_gem_mmap_gtt(struct drm_file *file,
   2525 		  struct drm_device *dev,
   2526 		  uint32_t handle,
   2527 		  uint64_t *offset)
   2528 {
   2529 	struct drm_gem_object *gobj;
   2530 	struct drm_i915_gem_object *obj;
   2531 	int ret;
   2532 
   2533 	ret = i915_mutex_lock_interruptible(dev);
   2534 	if (ret)
   2535 		return ret;
   2536 
   2537 	gobj = drm_gem_object_lookup(dev, file, handle);
   2538 	if (gobj == NULL) {
   2539 		ret = -ENOENT;
   2540 		goto unlock;
   2541 	}
   2542 	obj = to_intel_bo(gobj);
   2543 
   2544 	if (obj->madv != I915_MADV_WILLNEED) {
   2545 		DRM_DEBUG("Attempting to mmap a purgeable buffer\n");
   2546 		ret = -EFAULT;
   2547 		goto out;
   2548 	}
   2549 
   2550 	ret = i915_gem_object_create_mmap_offset(obj);
   2551 	if (ret)
   2552 		goto out;
   2553 
   2554 	*offset = drm_vma_node_offset_addr(&obj->base.vma_node);
   2555 
   2556 out:
   2557 	drm_gem_object_unreference(&obj->base);
   2558 unlock:
   2559 	mutex_unlock(&dev->struct_mutex);
   2560 	return ret;
   2561 }
   2562 
   2563 /**
   2564  * i915_gem_mmap_gtt_ioctl - prepare an object for GTT mmap'ing
   2565  * @dev: DRM device
   2566  * @data: GTT mapping ioctl data
   2567  * @file: GEM object info
   2568  *
   2569  * Simply returns the fake offset to userspace so it can mmap it.
   2570  * The mmap call will end up in drm_gem_mmap(), which will set things
   2571  * up so we can get faults in the handler above.
   2572  *
   2573  * The fault handler will take care of binding the object into the GTT
   2574  * (since it may have been evicted to make room for something), allocating
   2575  * a fence register, and mapping the appropriate aperture address into
   2576  * userspace.
   2577  */
   2578 int
   2579 i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data,
   2580 			struct drm_file *file)
   2581 {
   2582 	struct drm_i915_gem_mmap_gtt *args = data;
   2583 
   2584 	return i915_gem_mmap_gtt(file, dev, args->handle, &args->offset);
   2585 }
   2586 
   2587 /* Immediately discard the backing storage */
   2588 static void
   2589 i915_gem_object_truncate(struct drm_i915_gem_object *obj)
   2590 {
   2591 	i915_gem_object_free_mmap_offset(obj);
   2592 
   2593 #ifdef __NetBSD__
   2594 	if (obj->base.gemo_shm_uao == NULL)
   2595 		return;
   2596 
   2597 	{
   2598 		struct uvm_object *const uobj = obj->base.gemo_shm_uao;
   2599 
   2600 		if (uobj != NULL) {
   2601 			/* XXX Calling pgo_put like this is bogus.  */
   2602 			mutex_enter(uobj->vmobjlock);
   2603 			(*uobj->pgops->pgo_put)(uobj, 0, obj->base.size,
   2604 			    (PGO_ALLPAGES | PGO_FREE));
   2605 		}
   2606 	}
   2607 #else
   2608 	if (obj->base.filp == NULL)
   2609 		return;
   2610 
   2611 	/* Our goal here is to return as much of the memory as
   2612 	 * is possible back to the system as we are called from OOM.
   2613 	 * To do this we must instruct the shmfs to drop all of its
   2614 	 * backing pages, *now*.
   2615 	 */
   2616 	shmem_truncate_range(file_inode(obj->base.filp), 0, (loff_t)-1);
   2617 #endif
   2618 	obj->madv = __I915_MADV_PURGED;
   2619 }
   2620 
   2621 /* Try to discard unwanted pages */
   2622 static void
   2623 i915_gem_object_invalidate(struct drm_i915_gem_object *obj)
   2624 {
   2625 #ifdef __NetBSD__
   2626 	panic("XXX");
   2627 #else
   2628 	struct address_space *mapping;
   2629 
   2630 	switch (obj->madv) {
   2631 	case I915_MADV_DONTNEED:
   2632 		i915_gem_object_truncate(obj);
   2633 	case __I915_MADV_PURGED:
   2634 		return;
   2635 	}
   2636 
   2637 	if (obj->base.filp == NULL)
   2638 		return;
   2639 
   2640 	mapping = file_inode(obj->base.filp)->i_mapping,
   2641 	invalidate_mapping_pages(mapping, 0, (loff_t)-1);
   2642 #endif
   2643 }
   2644 
   2645 static void
   2646 i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj)
   2647 {
   2648 #ifdef __NetBSD__
   2649 	struct drm_device *const dev = obj->base.dev;
   2650 	struct vm_page *page;
   2651 	int ret;
   2652 
   2653 	/* XXX Cargo-culted from the Linux code.  */
   2654 	BUG_ON(obj->madv == __I915_MADV_PURGED);
   2655 
   2656 	ret = i915_gem_object_set_to_cpu_domain(obj, true);
   2657 	if (ret) {
   2658 		WARN_ON(ret != -EIO);
   2659 		i915_gem_clflush_object(obj, true);
   2660 		obj->base.read_domains = obj->base.write_domain =
   2661 		    I915_GEM_DOMAIN_CPU;
   2662 	}
   2663 
   2664 	i915_gem_gtt_finish_object(obj);
   2665 
   2666 	if (i915_gem_object_needs_bit17_swizzle(obj))
   2667 		i915_gem_object_save_bit_17_swizzle(obj);
   2668 
   2669 	if (obj->madv == I915_MADV_DONTNEED)
   2670 		obj->dirty = 0;
   2671 
   2672 	if (obj->dirty) {
   2673 		TAILQ_FOREACH(page, &obj->pageq, pageq.queue) {
   2674 			page->flags &= ~PG_CLEAN;
   2675 			/* XXX mark page accessed */
   2676 		}
   2677 	}
   2678 	obj->dirty = 0;
   2679 
   2680 	uvm_obj_unwirepages(obj->base.gemo_shm_uao, 0, obj->base.size);
   2681 	bus_dmamap_destroy(dev->dmat, obj->pages);
   2682 #else
   2683 	struct sg_page_iter sg_iter;
   2684 	int ret;
   2685 
   2686 	BUG_ON(obj->madv == __I915_MADV_PURGED);
   2687 
   2688 	ret = i915_gem_object_set_to_cpu_domain(obj, true);
   2689 	if (ret) {
   2690 		/* In the event of a disaster, abandon all caches and
   2691 		 * hope for the best.
   2692 		 */
   2693 		WARN_ON(ret != -EIO);
   2694 		i915_gem_clflush_object(obj, true);
   2695 		obj->base.read_domains = obj->base.write_domain = I915_GEM_DOMAIN_CPU;
   2696 	}
   2697 
   2698 	i915_gem_gtt_finish_object(obj);
   2699 
   2700 	if (i915_gem_object_needs_bit17_swizzle(obj))
   2701 		i915_gem_object_save_bit_17_swizzle(obj);
   2702 
   2703 	if (obj->madv == I915_MADV_DONTNEED)
   2704 		obj->dirty = 0;
   2705 
   2706 	for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents, 0) {
   2707 		struct page *page = sg_page_iter_page(&sg_iter);
   2708 
   2709 		if (obj->dirty)
   2710 			set_page_dirty(page);
   2711 
   2712 		if (obj->madv == I915_MADV_WILLNEED)
   2713 			mark_page_accessed(page);
   2714 
   2715 		page_cache_release(page);
   2716 	}
   2717 	obj->dirty = 0;
   2718 
   2719 	sg_free_table(obj->pages);
   2720 	kfree(obj->pages);
   2721 #endif
   2722 }
   2723 
   2724 int
   2725 i915_gem_object_put_pages(struct drm_i915_gem_object *obj)
   2726 {
   2727 	const struct drm_i915_gem_object_ops *ops = obj->ops;
   2728 
   2729 	if (obj->pages == NULL)
   2730 		return 0;
   2731 
   2732 	if (obj->pages_pin_count)
   2733 		return -EBUSY;
   2734 
   2735 	BUG_ON(i915_gem_obj_bound_any(obj));
   2736 
   2737 	/* ->put_pages might need to allocate memory for the bit17 swizzle
   2738 	 * array, hence protect them from being reaped by removing them from gtt
   2739 	 * lists early. */
   2740 	list_del(&obj->global_list);
   2741 
   2742 	ops->put_pages(obj);
   2743 	obj->pages = NULL;
   2744 
   2745 	i915_gem_object_invalidate(obj);
   2746 
   2747 	return 0;
   2748 }
   2749 
   2750 static int
   2751 i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj)
   2752 {
   2753 #ifdef __NetBSD__
   2754 	struct drm_device *const dev = obj->base.dev;
   2755 	struct drm_i915_private *dev_priv = dev->dev_private;
   2756 	struct vm_page *page;
   2757 	int ret;
   2758 
   2759 	BUG_ON(obj->base.read_domains & I915_GEM_GPU_DOMAINS);
   2760 	BUG_ON(obj->base.write_domain & I915_GEM_GPU_DOMAINS);
   2761 
   2762 	KASSERT(obj->pages == NULL);
   2763 	TAILQ_INIT(&obj->pageq);
   2764 
   2765 	/* XXX errno NetBSD->Linux */
   2766 	ret = -bus_dmamap_create(dev->dmat, obj->base.size,
   2767 	    obj->base.size/PAGE_SIZE, PAGE_SIZE, 0, BUS_DMA_NOWAIT,
   2768 	    &obj->pages);
   2769 	if (ret)
   2770 		goto fail0;
   2771 
   2772 	/* XXX errno NetBSD->Linux */
   2773 	ret = -uvm_obj_wirepages(obj->base.gemo_shm_uao, 0, obj->base.size,
   2774 	    &obj->pageq);
   2775 	if (ret)		/* XXX Try purge, shrink.  */
   2776 		goto fail1;
   2777 
   2778 	/*
   2779 	 * Check that the paddrs will fit in 40 bits, or 32 bits on i965.
   2780 	 *
   2781 	 * XXX This should be unnecessary: the uao should guarantee
   2782 	 * this constraint after uao_set_pgfl.
   2783 	 *
   2784 	 * XXX This should also be expanded for newer devices.
   2785 	 */
   2786 	TAILQ_FOREACH(page, &obj->pageq, pageq.queue) {
   2787 		const uint64_t mask =
   2788 		    (IS_BROADWATER(dev) || IS_CRESTLINE(dev)?
   2789 			0xffffffffULL : 0xffffffffffULL);
   2790 		if (VM_PAGE_TO_PHYS(page) & ~mask) {
   2791 			DRM_ERROR("GEM physical address exceeds %u bits"
   2792 			    ": %"PRIxMAX"\n",
   2793 			    popcount64(mask),
   2794 			    (uintmax_t)VM_PAGE_TO_PHYS(page));
   2795 			ret = -EIO;
   2796 			goto fail2;
   2797 		}
   2798 	}
   2799 
   2800 	ret = i915_gem_gtt_prepare_object(obj);
   2801 	if (ret)
   2802 		goto fail2;
   2803 
   2804 	if (i915_gem_object_needs_bit17_swizzle(obj))
   2805 		i915_gem_object_do_bit_17_swizzle(obj);
   2806 
   2807 	if (obj->tiling_mode != I915_TILING_NONE &&
   2808 	    dev_priv->quirks & QUIRK_PIN_SWIZZLED_PAGES)
   2809 		i915_gem_object_pin_pages(obj);
   2810 
   2811 	/* Success!  */
   2812 	return 0;
   2813 
   2814 fail3: __unused
   2815 	i915_gem_gtt_finish_object(obj);
   2816 fail2:	uvm_obj_unwirepages(obj->base.gemo_shm_uao, 0, obj->base.size);
   2817 fail1:	bus_dmamap_destroy(dev->dmat, obj->pages);
   2818 	obj->pages = NULL;
   2819 fail0:	KASSERT(ret);
   2820 	return ret;
   2821 #else
   2822 	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
   2823 	int page_count, i;
   2824 	struct address_space *mapping;
   2825 	struct sg_table *st;
   2826 	struct scatterlist *sg;
   2827 	struct sg_page_iter sg_iter;
   2828 	struct page *page;
   2829 	unsigned long last_pfn = 0;	/* suppress gcc warning */
   2830 	int ret;
   2831 	gfp_t gfp;
   2832 
   2833 	/* Assert that the object is not currently in any GPU domain. As it
   2834 	 * wasn't in the GTT, there shouldn't be any way it could have been in
   2835 	 * a GPU cache
   2836 	 */
   2837 	BUG_ON(obj->base.read_domains & I915_GEM_GPU_DOMAINS);
   2838 	BUG_ON(obj->base.write_domain & I915_GEM_GPU_DOMAINS);
   2839 
   2840 	st = kmalloc(sizeof(*st), GFP_KERNEL);
   2841 	if (st == NULL)
   2842 		return -ENOMEM;
   2843 
   2844 	page_count = obj->base.size / PAGE_SIZE;
   2845 	if (sg_alloc_table(st, page_count, GFP_KERNEL)) {
   2846 		kfree(st);
   2847 		return -ENOMEM;
   2848 	}
   2849 
   2850 	/* Get the list of pages out of our struct file.  They'll be pinned
   2851 	 * at this point until we release them.
   2852 	 *
   2853 	 * Fail silently without starting the shrinker
   2854 	 */
   2855 	mapping = file_inode(obj->base.filp)->i_mapping;
   2856 	gfp = mapping_gfp_constraint(mapping, ~(__GFP_IO | __GFP_RECLAIM));
   2857 	gfp |= __GFP_NORETRY | __GFP_NOWARN;
   2858 	sg = st->sgl;
   2859 	st->nents = 0;
   2860 	for (i = 0; i < page_count; i++) {
   2861 		page = shmem_read_mapping_page_gfp(mapping, i, gfp);
   2862 		if (IS_ERR(page)) {
   2863 			i915_gem_shrink(dev_priv,
   2864 					page_count,
   2865 					I915_SHRINK_BOUND |
   2866 					I915_SHRINK_UNBOUND |
   2867 					I915_SHRINK_PURGEABLE);
   2868 			page = shmem_read_mapping_page_gfp(mapping, i, gfp);
   2869 		}
   2870 		if (IS_ERR(page)) {
   2871 			/* We've tried hard to allocate the memory by reaping
   2872 			 * our own buffer, now let the real VM do its job and
   2873 			 * go down in flames if truly OOM.
   2874 			 */
   2875 			i915_gem_shrink_all(dev_priv);
   2876 			page = shmem_read_mapping_page(mapping, i);
   2877 			if (IS_ERR(page)) {
   2878 				ret = PTR_ERR(page);
   2879 				goto err_pages;
   2880 			}
   2881 		}
   2882 #ifdef CONFIG_SWIOTLB
   2883 		if (swiotlb_nr_tbl()) {
   2884 			st->nents++;
   2885 			sg_set_page(sg, page, PAGE_SIZE, 0);
   2886 			sg = sg_next(sg);
   2887 			continue;
   2888 		}
   2889 #endif
   2890 		if (!i || page_to_pfn(page) != last_pfn + 1) {
   2891 			if (i)
   2892 				sg = sg_next(sg);
   2893 			st->nents++;
   2894 			sg_set_page(sg, page, PAGE_SIZE, 0);
   2895 		} else {
   2896 			sg->length += PAGE_SIZE;
   2897 		}
   2898 		last_pfn = page_to_pfn(page);
   2899 
   2900 		/* Check that the i965g/gm workaround works. */
   2901 		WARN_ON((gfp & __GFP_DMA32) && (last_pfn >= 0x00100000UL));
   2902 	}
   2903 #ifdef CONFIG_SWIOTLB
   2904 	if (!swiotlb_nr_tbl())
   2905 #endif
   2906 		sg_mark_end(sg);
   2907 	obj->pages = st;
   2908 
   2909 	ret = i915_gem_gtt_prepare_object(obj);
   2910 	if (ret)
   2911 		goto err_pages;
   2912 
   2913 	if (i915_gem_object_needs_bit17_swizzle(obj))
   2914 		i915_gem_object_do_bit_17_swizzle(obj);
   2915 
   2916 	if (obj->tiling_mode != I915_TILING_NONE &&
   2917 	    dev_priv->quirks & QUIRK_PIN_SWIZZLED_PAGES)
   2918 		i915_gem_object_pin_pages(obj);
   2919 
   2920 	return 0;
   2921 
   2922 err_pages:
   2923 	sg_mark_end(sg);
   2924 	for_each_sg_page(st->sgl, &sg_iter, st->nents, 0)
   2925 		page_cache_release(sg_page_iter_page(&sg_iter));
   2926 	sg_free_table(st);
   2927 	kfree(st);
   2928 
   2929 	/* shmemfs first checks if there is enough memory to allocate the page
   2930 	 * and reports ENOSPC should there be insufficient, along with the usual
   2931 	 * ENOMEM for a genuine allocation failure.
   2932 	 *
   2933 	 * We use ENOSPC in our driver to mean that we have run out of aperture
   2934 	 * space and so want to translate the error from shmemfs back to our
   2935 	 * usual understanding of ENOMEM.
   2936 	 */
   2937 	if (ret == -ENOSPC)
   2938 		ret = -ENOMEM;
   2939 
   2940 	return ret;
   2941 #endif
   2942 }
   2943 
   2944 /* Ensure that the associated pages are gathered from the backing storage
   2945  * and pinned into our object. i915_gem_object_get_pages() may be called
   2946  * multiple times before they are released by a single call to
   2947  * i915_gem_object_put_pages() - once the pages are no longer referenced
   2948  * either as a result of memory pressure (reaping pages under the shrinker)
   2949  * or as the object is itself released.
   2950  */
   2951 int
   2952 i915_gem_object_get_pages(struct drm_i915_gem_object *obj)
   2953 {
   2954 	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
   2955 	const struct drm_i915_gem_object_ops *ops = obj->ops;
   2956 	int ret;
   2957 
   2958 	if (obj->pages)
   2959 		return 0;
   2960 
   2961 	if (obj->madv != I915_MADV_WILLNEED) {
   2962 		DRM_DEBUG("Attempting to obtain a purgeable object\n");
   2963 		return -EFAULT;
   2964 	}
   2965 
   2966 	BUG_ON(obj->pages_pin_count);
   2967 
   2968 	ret = ops->get_pages(obj);
   2969 	if (ret)
   2970 		return ret;
   2971 
   2972 	list_add_tail(&obj->global_list, &dev_priv->mm.unbound_list);
   2973 
   2974 #ifndef __NetBSD__
   2975 	obj->get_page.sg = obj->pages->sgl;
   2976 	obj->get_page.last = 0;
   2977 #endif
   2978 
   2979 	return 0;
   2980 }
   2981 
   2982 void i915_vma_move_to_active(struct i915_vma *vma,
   2983 			     struct drm_i915_gem_request *req)
   2984 {
   2985 	struct drm_i915_gem_object *obj = vma->obj;
   2986 	struct intel_engine_cs *ring;
   2987 
   2988 	ring = i915_gem_request_get_ring(req);
   2989 
   2990 	/* Add a reference if we're newly entering the active list. */
   2991 	if (obj->active == 0)
   2992 		drm_gem_object_reference(&obj->base);
   2993 	obj->active |= intel_ring_flag(ring);
   2994 
   2995 	list_move_tail(&obj->ring_list[ring->id], &ring->active_list);
   2996 	i915_gem_request_assign(&obj->last_read_req[ring->id], req);
   2997 
   2998 	list_move_tail(&vma->mm_list, &vma->vm->active_list);
   2999 }
   3000 
   3001 static void
   3002 i915_gem_object_retire__write(struct drm_i915_gem_object *obj)
   3003 {
   3004 	RQ_BUG_ON(obj->last_write_req == NULL);
   3005 	RQ_BUG_ON(!(obj->active & intel_ring_flag(obj->last_write_req->ring)));
   3006 
   3007 	i915_gem_request_assign(&obj->last_write_req, NULL);
   3008 	intel_fb_obj_flush(obj, true, ORIGIN_CS);
   3009 }
   3010 
   3011 static void
   3012 i915_gem_object_retire__read(struct drm_i915_gem_object *obj, int ring)
   3013 {
   3014 	struct i915_vma *vma;
   3015 
   3016 	RQ_BUG_ON(obj->last_read_req[ring] == NULL);
   3017 	RQ_BUG_ON(!(obj->active & (1 << ring)));
   3018 
   3019 	list_del_init(&obj->ring_list[ring]);
   3020 	i915_gem_request_assign(&obj->last_read_req[ring], NULL);
   3021 
   3022 	if (obj->last_write_req && obj->last_write_req->ring->id == ring)
   3023 		i915_gem_object_retire__write(obj);
   3024 
   3025 	obj->active &= ~(1 << ring);
   3026 	if (obj->active)
   3027 		return;
   3028 
   3029 	/* Bump our place on the bound list to keep it roughly in LRU order
   3030 	 * so that we don't steal from recently used but inactive objects
   3031 	 * (unless we are forced to ofc!)
   3032 	 */
   3033 	list_move_tail(&obj->global_list,
   3034 		       &to_i915(obj->base.dev)->mm.bound_list);
   3035 
   3036 	list_for_each_entry(vma, &obj->vma_list, vma_link) {
   3037 		if (!list_empty(&vma->mm_list))
   3038 			list_move_tail(&vma->mm_list, &vma->vm->inactive_list);
   3039 	}
   3040 
   3041 	i915_gem_request_assign(&obj->last_fenced_req, NULL);
   3042 	drm_gem_object_unreference(&obj->base);
   3043 }
   3044 
   3045 static int
   3046 i915_gem_init_seqno(struct drm_device *dev, u32 seqno)
   3047 {
   3048 	struct drm_i915_private *dev_priv = dev->dev_private;
   3049 	struct intel_engine_cs *ring;
   3050 	int ret, i, j;
   3051 
   3052 	/* Carefully retire all requests without writing to the rings */
   3053 	for_each_ring(ring, dev_priv, i) {
   3054 		ret = intel_ring_idle(ring);
   3055 		if (ret)
   3056 			return ret;
   3057 	}
   3058 	i915_gem_retire_requests(dev);
   3059 
   3060 	/* Finally reset hw state */
   3061 	for_each_ring(ring, dev_priv, i) {
   3062 		intel_ring_init_seqno(ring, seqno);
   3063 
   3064 		for (j = 0; j < ARRAY_SIZE(ring->semaphore.sync_seqno); j++)
   3065 			ring->semaphore.sync_seqno[j] = 0;
   3066 	}
   3067 
   3068 	return 0;
   3069 }
   3070 
   3071 int i915_gem_set_seqno(struct drm_device *dev, u32 seqno)
   3072 {
   3073 	struct drm_i915_private *dev_priv = dev->dev_private;
   3074 	int ret;
   3075 
   3076 	if (seqno == 0)
   3077 		return -EINVAL;
   3078 
   3079 	/* HWS page needs to be set less than what we
   3080 	 * will inject to ring
   3081 	 */
   3082 	ret = i915_gem_init_seqno(dev, seqno - 1);
   3083 	if (ret)
   3084 		return ret;
   3085 
   3086 	/* Carefully set the last_seqno value so that wrap
   3087 	 * detection still works
   3088 	 */
   3089 	dev_priv->next_seqno = seqno;
   3090 	dev_priv->last_seqno = seqno - 1;
   3091 	if (dev_priv->last_seqno == 0)
   3092 		dev_priv->last_seqno--;
   3093 
   3094 	return 0;
   3095 }
   3096 
   3097 int
   3098 i915_gem_get_seqno(struct drm_device *dev, u32 *seqno)
   3099 {
   3100 	struct drm_i915_private *dev_priv = dev->dev_private;
   3101 
   3102 	/* reserve 0 for non-seqno */
   3103 	if (dev_priv->next_seqno == 0) {
   3104 		int ret = i915_gem_init_seqno(dev, 0);
   3105 		if (ret)
   3106 			return ret;
   3107 
   3108 		dev_priv->next_seqno = 1;
   3109 	}
   3110 
   3111 	*seqno = dev_priv->last_seqno = dev_priv->next_seqno++;
   3112 	return 0;
   3113 }
   3114 
   3115 /*
   3116  * NB: This function is not allowed to fail. Doing so would mean the the
   3117  * request is not being tracked for completion but the work itself is
   3118  * going to happen on the hardware. This would be a Bad Thing(tm).
   3119  */
   3120 void __i915_add_request(struct drm_i915_gem_request *request,
   3121 			struct drm_i915_gem_object *obj,
   3122 			bool flush_caches)
   3123 {
   3124 	struct intel_engine_cs *ring;
   3125 	struct drm_i915_private *dev_priv;
   3126 	struct intel_ringbuffer *ringbuf;
   3127 	u32 request_start;
   3128 	int ret;
   3129 
   3130 	if (WARN_ON(request == NULL))
   3131 		return;
   3132 
   3133 	ring = request->ring;
   3134 	dev_priv = ring->dev->dev_private;
   3135 	ringbuf = request->ringbuf;
   3136 
   3137 	/*
   3138 	 * To ensure that this call will not fail, space for its emissions
   3139 	 * should already have been reserved in the ring buffer. Let the ring
   3140 	 * know that it is time to use that space up.
   3141 	 */
   3142 	intel_ring_reserved_space_use(ringbuf);
   3143 
   3144 	request_start = intel_ring_get_tail(ringbuf);
   3145 	/*
   3146 	 * Emit any outstanding flushes - execbuf can fail to emit the flush
   3147 	 * after having emitted the batchbuffer command. Hence we need to fix
   3148 	 * things up similar to emitting the lazy request. The difference here
   3149 	 * is that the flush _must_ happen before the next request, no matter
   3150 	 * what.
   3151 	 */
   3152 	if (flush_caches) {
   3153 		if (i915.enable_execlists)
   3154 			ret = logical_ring_flush_all_caches(request);
   3155 		else
   3156 			ret = intel_ring_flush_all_caches(request);
   3157 		/* Not allowed to fail! */
   3158 		WARN(ret, "*_ring_flush_all_caches failed: %d!\n", ret);
   3159 	}
   3160 
   3161 	/* Record the position of the start of the request so that
   3162 	 * should we detect the updated seqno part-way through the
   3163 	 * GPU processing the request, we never over-estimate the
   3164 	 * position of the head.
   3165 	 */
   3166 	request->postfix = intel_ring_get_tail(ringbuf);
   3167 
   3168 	if (i915.enable_execlists)
   3169 		ret = ring->emit_request(request);
   3170 	else {
   3171 		ret = ring->add_request(request);
   3172 
   3173 		request->tail = intel_ring_get_tail(ringbuf);
   3174 	}
   3175 	/* Not allowed to fail! */
   3176 	WARN(ret, "emit|add_request failed: %d!\n", ret);
   3177 
   3178 	request->head = request_start;
   3179 
   3180 	/* Whilst this request exists, batch_obj will be on the
   3181 	 * active_list, and so will hold the active reference. Only when this
   3182 	 * request is retired will the the batch_obj be moved onto the
   3183 	 * inactive_list and lose its active reference. Hence we do not need
   3184 	 * to explicitly hold another reference here.
   3185 	 */
   3186 	request->batch_obj = obj;
   3187 
   3188 	request->emitted_jiffies = jiffies;
   3189 	request->previous_seqno = ring->last_submitted_seqno;
   3190 	ring->last_submitted_seqno = request->seqno;
   3191 	list_add_tail(&request->list, &ring->request_list);
   3192 
   3193 	trace_i915_gem_request_add(request);
   3194 
   3195 	i915_queue_hangcheck(ring->dev);
   3196 
   3197 	queue_delayed_work(dev_priv->wq,
   3198 			   &dev_priv->mm.retire_work,
   3199 			   round_jiffies_up_relative(HZ));
   3200 	intel_mark_busy(dev_priv->dev);
   3201 
   3202 	/* Sanity check that the reserved size was large enough. */
   3203 	intel_ring_reserved_space_end(ringbuf);
   3204 }
   3205 
   3206 static bool i915_context_is_banned(struct drm_i915_private *dev_priv,
   3207 				   const struct intel_context *ctx)
   3208 {
   3209 	unsigned long elapsed;
   3210 
   3211 	elapsed = get_seconds() - ctx->hang_stats.guilty_ts;
   3212 
   3213 	if (ctx->hang_stats.banned)
   3214 		return true;
   3215 
   3216 	if (ctx->hang_stats.ban_period_seconds &&
   3217 	    elapsed <= ctx->hang_stats.ban_period_seconds) {
   3218 		if (!i915_gem_context_is_default(ctx)) {
   3219 			DRM_DEBUG("context hanging too fast, banning!\n");
   3220 			return true;
   3221 		} else if (i915_stop_ring_allow_ban(dev_priv)) {
   3222 			if (i915_stop_ring_allow_warn(dev_priv))
   3223 				DRM_ERROR("gpu hanging too fast, banning!\n");
   3224 			return true;
   3225 		}
   3226 	}
   3227 
   3228 	return false;
   3229 }
   3230 
   3231 static void i915_set_reset_status(struct drm_i915_private *dev_priv,
   3232 				  struct intel_context *ctx,
   3233 				  const bool guilty)
   3234 {
   3235 	struct i915_ctx_hang_stats *hs;
   3236 
   3237 	if (WARN_ON(!ctx))
   3238 		return;
   3239 
   3240 	hs = &ctx->hang_stats;
   3241 
   3242 	if (guilty) {
   3243 		hs->banned = i915_context_is_banned(dev_priv, ctx);
   3244 		hs->batch_active++;
   3245 		hs->guilty_ts = get_seconds();
   3246 	} else {
   3247 		hs->batch_pending++;
   3248 	}
   3249 }
   3250 
   3251 void i915_gem_request_free(struct kref *req_ref)
   3252 {
   3253 	struct drm_i915_gem_request *req = container_of(req_ref,
   3254 						 typeof(*req), ref);
   3255 	struct intel_context *ctx = req->ctx;
   3256 
   3257 	if (req->file_priv)
   3258 		i915_gem_request_remove_from_client(req);
   3259 
   3260 	if (ctx) {
   3261 		if (i915.enable_execlists) {
   3262 			if (ctx != req->ring->default_context)
   3263 				intel_lr_context_unpin(req);
   3264 		}
   3265 
   3266 		i915_gem_context_unreference(ctx);
   3267 	}
   3268 
   3269 	kmem_cache_free(req->i915->requests, req);
   3270 }
   3271 
   3272 int i915_gem_request_alloc(struct intel_engine_cs *ring,
   3273 			   struct intel_context *ctx,
   3274 			   struct drm_i915_gem_request **req_out)
   3275 {
   3276 	struct drm_i915_private *dev_priv = to_i915(ring->dev);
   3277 	struct drm_i915_gem_request *req;
   3278 	int ret;
   3279 
   3280 	if (!req_out)
   3281 		return -EINVAL;
   3282 
   3283 	*req_out = NULL;
   3284 
   3285 	req = kmem_cache_zalloc(dev_priv->requests, GFP_KERNEL);
   3286 	if (req == NULL)
   3287 		return -ENOMEM;
   3288 
   3289 	ret = i915_gem_get_seqno(ring->dev, &req->seqno);
   3290 	if (ret)
   3291 		goto err;
   3292 
   3293 	kref_init(&req->ref);
   3294 	req->i915 = dev_priv;
   3295 	req->ring = ring;
   3296 	req->ctx  = ctx;
   3297 	i915_gem_context_reference(req->ctx);
   3298 
   3299 	if (i915.enable_execlists)
   3300 		ret = intel_logical_ring_alloc_request_extras(req);
   3301 	else
   3302 		ret = intel_ring_alloc_request_extras(req);
   3303 	if (ret) {
   3304 		i915_gem_context_unreference(req->ctx);
   3305 		goto err;
   3306 	}
   3307 
   3308 	/*
   3309 	 * Reserve space in the ring buffer for all the commands required to
   3310 	 * eventually emit this request. This is to guarantee that the
   3311 	 * i915_add_request() call can't fail. Note that the reserve may need
   3312 	 * to be redone if the request is not actually submitted straight
   3313 	 * away, e.g. because a GPU scheduler has deferred it.
   3314 	 */
   3315 	if (i915.enable_execlists)
   3316 		ret = intel_logical_ring_reserve_space(req);
   3317 	else
   3318 		ret = intel_ring_reserve_space(req);
   3319 	if (ret) {
   3320 		/*
   3321 		 * At this point, the request is fully allocated even if not
   3322 		 * fully prepared. Thus it can be cleaned up using the proper
   3323 		 * free code.
   3324 		 */
   3325 		i915_gem_request_cancel(req);
   3326 		return ret;
   3327 	}
   3328 
   3329 	*req_out = req;
   3330 	return 0;
   3331 
   3332 err:
   3333 	kmem_cache_free(dev_priv->requests, req);
   3334 	return ret;
   3335 }
   3336 
   3337 void i915_gem_request_cancel(struct drm_i915_gem_request *req)
   3338 {
   3339 	intel_ring_reserved_space_cancel(req->ringbuf);
   3340 
   3341 	i915_gem_request_unreference(req);
   3342 }
   3343 
   3344 struct drm_i915_gem_request *
   3345 i915_gem_find_active_request(struct intel_engine_cs *ring)
   3346 {
   3347 	struct drm_i915_gem_request *request;
   3348 
   3349 	list_for_each_entry(request, &ring->request_list, list) {
   3350 		if (i915_gem_request_completed(request, false))
   3351 			continue;
   3352 
   3353 		return request;
   3354 	}
   3355 
   3356 	return NULL;
   3357 }
   3358 
   3359 static void i915_gem_reset_ring_status(struct drm_i915_private *dev_priv,
   3360 				       struct intel_engine_cs *ring)
   3361 {
   3362 	struct drm_i915_gem_request *request;
   3363 	bool ring_hung;
   3364 
   3365 	request = i915_gem_find_active_request(ring);
   3366 
   3367 	if (request == NULL)
   3368 		return;
   3369 
   3370 	ring_hung = ring->hangcheck.score >= HANGCHECK_SCORE_RING_HUNG;
   3371 
   3372 	i915_set_reset_status(dev_priv, request->ctx, ring_hung);
   3373 
   3374 	list_for_each_entry_continue(request, &ring->request_list, list)
   3375 		i915_set_reset_status(dev_priv, request->ctx, false);
   3376 }
   3377 
   3378 static void i915_gem_reset_ring_cleanup(struct drm_i915_private *dev_priv,
   3379 					struct intel_engine_cs *ring)
   3380 {
   3381 	while (!list_empty(&ring->active_list)) {
   3382 		struct drm_i915_gem_object *obj;
   3383 
   3384 		obj = list_first_entry(&ring->active_list,
   3385 				       struct drm_i915_gem_object,
   3386 				       ring_list[ring->id]);
   3387 
   3388 		i915_gem_object_retire__read(obj, ring->id);
   3389 	}
   3390 
   3391 	/*
   3392 	 * Clear the execlists queue up before freeing the requests, as those
   3393 	 * are the ones that keep the context and ringbuffer backing objects
   3394 	 * pinned in place.
   3395 	 */
   3396 	while (!list_empty(&ring->execlist_queue)) {
   3397 		struct drm_i915_gem_request *submit_req;
   3398 
   3399 		submit_req = list_first_entry(&ring->execlist_queue,
   3400 				struct drm_i915_gem_request,
   3401 				execlist_link);
   3402 		list_del(&submit_req->execlist_link);
   3403 
   3404 		if (submit_req->ctx != ring->default_context)
   3405 			intel_lr_context_unpin(submit_req);
   3406 
   3407 		i915_gem_request_unreference(submit_req);
   3408 	}
   3409 
   3410 	/*
   3411 	 * We must free the requests after all the corresponding objects have
   3412 	 * been moved off active lists. Which is the same order as the normal
   3413 	 * retire_requests function does. This is important if object hold
   3414 	 * implicit references on things like e.g. ppgtt address spaces through
   3415 	 * the request.
   3416 	 */
   3417 	while (!list_empty(&ring->request_list)) {
   3418 		struct drm_i915_gem_request *request;
   3419 
   3420 		request = list_first_entry(&ring->request_list,
   3421 					   struct drm_i915_gem_request,
   3422 					   list);
   3423 
   3424 		i915_gem_request_retire(request);
   3425 	}
   3426 }
   3427 
   3428 void i915_gem_reset(struct drm_device *dev)
   3429 {
   3430 	struct drm_i915_private *dev_priv = dev->dev_private;
   3431 	struct intel_engine_cs *ring;
   3432 	int i;
   3433 
   3434 	/*
   3435 	 * Before we free the objects from the requests, we need to inspect
   3436 	 * them for finding the guilty party. As the requests only borrow
   3437 	 * their reference to the objects, the inspection must be done first.
   3438 	 */
   3439 	for_each_ring(ring, dev_priv, i)
   3440 		i915_gem_reset_ring_status(dev_priv, ring);
   3441 
   3442 	for_each_ring(ring, dev_priv, i)
   3443 		i915_gem_reset_ring_cleanup(dev_priv, ring);
   3444 
   3445 	i915_gem_context_reset(dev);
   3446 
   3447 	i915_gem_restore_fences(dev);
   3448 
   3449 	WARN_ON(i915_verify_lists(dev));
   3450 }
   3451 
   3452 /**
   3453  * This function clears the request list as sequence numbers are passed.
   3454  */
   3455 void
   3456 i915_gem_retire_requests_ring(struct intel_engine_cs *ring)
   3457 {
   3458 	WARN_ON(i915_verify_lists(ring->dev));
   3459 
   3460 	/* Retire requests first as we use it above for the early return.
   3461 	 * If we retire requests last, we may use a later seqno and so clear
   3462 	 * the requests lists without clearing the active list, leading to
   3463 	 * confusion.
   3464 	 */
   3465 	while (!list_empty(&ring->request_list)) {
   3466 		struct drm_i915_gem_request *request;
   3467 
   3468 		request = list_first_entry(&ring->request_list,
   3469 					   struct drm_i915_gem_request,
   3470 					   list);
   3471 
   3472 		if (!i915_gem_request_completed(request, true))
   3473 			break;
   3474 
   3475 		i915_gem_request_retire(request);
   3476 	}
   3477 
   3478 	/* Move any buffers on the active list that are no longer referenced
   3479 	 * by the ringbuffer to the flushing/inactive lists as appropriate,
   3480 	 * before we free the context associated with the requests.
   3481 	 */
   3482 	while (!list_empty(&ring->active_list)) {
   3483 		struct drm_i915_gem_object *obj;
   3484 
   3485 		obj = list_first_entry(&ring->active_list,
   3486 				      struct drm_i915_gem_object,
   3487 				      ring_list[ring->id]);
   3488 
   3489 		if (!list_empty(&obj->last_read_req[ring->id]->list))
   3490 			break;
   3491 
   3492 		i915_gem_object_retire__read(obj, ring->id);
   3493 	}
   3494 
   3495 	if (unlikely(ring->trace_irq_req &&
   3496 		     i915_gem_request_completed(ring->trace_irq_req, true))) {
   3497 		ring->irq_put(ring);
   3498 		i915_gem_request_assign(&ring->trace_irq_req, NULL);
   3499 	}
   3500 
   3501 	WARN_ON(i915_verify_lists(ring->dev));
   3502 }
   3503 
   3504 bool
   3505 i915_gem_retire_requests(struct drm_device *dev)
   3506 {
   3507 	struct drm_i915_private *dev_priv = dev->dev_private;
   3508 	struct intel_engine_cs *ring;
   3509 	bool idle = true;
   3510 	int i;
   3511 
   3512 	for_each_ring(ring, dev_priv, i) {
   3513 		i915_gem_retire_requests_ring(ring);
   3514 		idle &= list_empty(&ring->request_list);
   3515 		if (i915.enable_execlists) {
   3516 			unsigned long flags;
   3517 
   3518 			spin_lock_irqsave(&ring->execlist_lock, flags);
   3519 			idle &= list_empty(&ring->execlist_queue);
   3520 			spin_unlock_irqrestore(&ring->execlist_lock, flags);
   3521 
   3522 			intel_execlists_retire_requests(ring);
   3523 		}
   3524 	}
   3525 
   3526 	if (idle)
   3527 		mod_delayed_work(dev_priv->wq,
   3528 				   &dev_priv->mm.idle_work,
   3529 				   msecs_to_jiffies(100));
   3530 
   3531 	return idle;
   3532 }
   3533 
   3534 static void
   3535 i915_gem_retire_work_handler(struct work_struct *work)
   3536 {
   3537 	struct drm_i915_private *dev_priv =
   3538 		container_of(work, typeof(*dev_priv), mm.retire_work.work);
   3539 	struct drm_device *dev = dev_priv->dev;
   3540 	bool idle;
   3541 
   3542 	/* Come back later if the device is busy... */
   3543 	idle = false;
   3544 	if (mutex_trylock(&dev->struct_mutex)) {
   3545 		idle = i915_gem_retire_requests(dev);
   3546 		mutex_unlock(&dev->struct_mutex);
   3547 	}
   3548 	if (!idle)
   3549 		queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work,
   3550 				   round_jiffies_up_relative(HZ));
   3551 }
   3552 
   3553 static void
   3554 i915_gem_idle_work_handler(struct work_struct *work)
   3555 {
   3556 	struct drm_i915_private *dev_priv =
   3557 		container_of(work, typeof(*dev_priv), mm.idle_work.work);
   3558 	struct drm_device *dev = dev_priv->dev;
   3559 	struct intel_engine_cs *ring;
   3560 	int i;
   3561 
   3562 	for_each_ring(ring, dev_priv, i)
   3563 		if (!list_empty(&ring->request_list))
   3564 			return;
   3565 
   3566 	intel_mark_idle(dev);
   3567 
   3568 	if (mutex_trylock(&dev->struct_mutex)) {
   3569 		struct intel_engine_cs *ring;
   3570 		int i;
   3571 
   3572 		for_each_ring(ring, dev_priv, i)
   3573 			i915_gem_batch_pool_fini(&ring->batch_pool);
   3574 
   3575 		mutex_unlock(&dev->struct_mutex);
   3576 	}
   3577 }
   3578 
   3579 /**
   3580  * Ensures that an object will eventually get non-busy by flushing any required
   3581  * write domains, emitting any outstanding lazy request and retiring and
   3582  * completed requests.
   3583  */
   3584 static int
   3585 i915_gem_object_flush_active(struct drm_i915_gem_object *obj)
   3586 {
   3587 	int i;
   3588 
   3589 	if (!obj->active)
   3590 		return 0;
   3591 
   3592 	for (i = 0; i < I915_NUM_RINGS; i++) {
   3593 		struct drm_i915_gem_request *req;
   3594 
   3595 		req = obj->last_read_req[i];
   3596 		if (req == NULL)
   3597 			continue;
   3598 
   3599 		if (list_empty(&req->list))
   3600 			goto retire;
   3601 
   3602 		if (i915_gem_request_completed(req, true)) {
   3603 			__i915_gem_request_retire__upto(req);
   3604 retire:
   3605 			i915_gem_object_retire__read(obj, i);
   3606 		}
   3607 	}
   3608 
   3609 	return 0;
   3610 }
   3611 
   3612 /**
   3613  * i915_gem_wait_ioctl - implements DRM_IOCTL_I915_GEM_WAIT
   3614  * @DRM_IOCTL_ARGS: standard ioctl arguments
   3615  *
   3616  * Returns 0 if successful, else an error is returned with the remaining time in
   3617  * the timeout parameter.
   3618  *  -ETIME: object is still busy after timeout
   3619  *  -ERESTARTSYS: signal interrupted the wait
   3620  *  -ENONENT: object doesn't exist
   3621  * Also possible, but rare:
   3622  *  -EAGAIN: GPU wedged
   3623  *  -ENOMEM: damn
   3624  *  -ENODEV: Internal IRQ fail
   3625  *  -E?: The add request failed
   3626  *
   3627  * The wait ioctl with a timeout of 0 reimplements the busy ioctl. With any
   3628  * non-zero timeout parameter the wait ioctl will wait for the given number of
   3629  * nanoseconds on an object becoming unbusy. Since the wait itself does so
   3630  * without holding struct_mutex the object may become re-busied before this
   3631  * function completes. A similar but shorter * race condition exists in the busy
   3632  * ioctl
   3633  */
   3634 int
   3635 i915_gem_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
   3636 {
   3637 	struct drm_i915_private *dev_priv = dev->dev_private;
   3638 	struct drm_i915_gem_wait *args = data;
   3639 	struct drm_gem_object *gobj;
   3640 	struct drm_i915_gem_object *obj;
   3641 	struct drm_i915_gem_request *req[I915_NUM_RINGS];
   3642 	unsigned reset_counter;
   3643 	int i, n = 0;
   3644 	int ret;
   3645 
   3646 	if (args->flags != 0)
   3647 		return -EINVAL;
   3648 
   3649 	ret = i915_mutex_lock_interruptible(dev);
   3650 	if (ret)
   3651 		return ret;
   3652 
   3653 	gobj = drm_gem_object_lookup(dev, file, args->bo_handle);
   3654 	if (gobj == NULL) {
   3655 		mutex_unlock(&dev->struct_mutex);
   3656 		return -ENOENT;
   3657 	}
   3658 	obj = to_intel_bo(gobj);
   3659 
   3660 	/* Need to make sure the object gets inactive eventually. */
   3661 	ret = i915_gem_object_flush_active(obj);
   3662 	if (ret)
   3663 		goto out;
   3664 
   3665 	if (!obj->active)
   3666 		goto out;
   3667 
   3668 	/* Do this after OLR check to make sure we make forward progress polling
   3669 	 * on this IOCTL with a timeout == 0 (like busy ioctl)
   3670 	 */
   3671 	if (args->timeout_ns == 0) {
   3672 		ret = -ETIME;
   3673 		goto out;
   3674 	}
   3675 
   3676 	drm_gem_object_unreference(&obj->base);
   3677 	reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter);
   3678 
   3679 	for (i = 0; i < I915_NUM_RINGS; i++) {
   3680 		if (obj->last_read_req[i] == NULL)
   3681 			continue;
   3682 
   3683 		req[n++] = i915_gem_request_reference(obj->last_read_req[i]);
   3684 	}
   3685 
   3686 	mutex_unlock(&dev->struct_mutex);
   3687 
   3688 	for (i = 0; i < n; i++) {
   3689 		if (ret == 0)
   3690 			ret = __i915_wait_request(req[i], reset_counter, true,
   3691 						  args->timeout_ns > 0 ? &args->timeout_ns : NULL,
   3692 						  file->driver_priv);
   3693 		i915_gem_request_unreference__unlocked(req[i]);
   3694 	}
   3695 	return ret;
   3696 
   3697 out:
   3698 	drm_gem_object_unreference(&obj->base);
   3699 	mutex_unlock(&dev->struct_mutex);
   3700 	return ret;
   3701 }
   3702 
   3703 static int
   3704 __i915_gem_object_sync(struct drm_i915_gem_object *obj,
   3705 		       struct intel_engine_cs *to,
   3706 		       struct drm_i915_gem_request *from_req,
   3707 		       struct drm_i915_gem_request **to_req)
   3708 {
   3709 	struct intel_engine_cs *from;
   3710 	int ret;
   3711 
   3712 	from = i915_gem_request_get_ring(from_req);
   3713 	if (to == from)
   3714 		return 0;
   3715 
   3716 	if (i915_gem_request_completed(from_req, true))
   3717 		return 0;
   3718 
   3719 	if (!i915_semaphore_is_enabled(obj->base.dev)) {
   3720 		struct drm_i915_private *i915 = to_i915(obj->base.dev);
   3721 		ret = __i915_wait_request(from_req,
   3722 					  atomic_read(&i915->gpu_error.reset_counter),
   3723 					  i915->mm.interruptible,
   3724 					  NULL,
   3725 					  &i915->rps.semaphores);
   3726 		if (ret)
   3727 			return ret;
   3728 
   3729 		i915_gem_object_retire_request(obj, from_req);
   3730 	} else {
   3731 		int idx = intel_ring_sync_index(from, to);
   3732 		u32 seqno = i915_gem_request_get_seqno(from_req);
   3733 
   3734 		WARN_ON(!to_req);
   3735 
   3736 		if (seqno <= from->semaphore.sync_seqno[idx])
   3737 			return 0;
   3738 
   3739 		if (*to_req == NULL) {
   3740 			ret = i915_gem_request_alloc(to, to->default_context, to_req);
   3741 			if (ret)
   3742 				return ret;
   3743 		}
   3744 
   3745 		trace_i915_gem_ring_sync_to(*to_req, from, from_req);
   3746 		ret = to->semaphore.sync_to(*to_req, from, seqno);
   3747 		if (ret)
   3748 			return ret;
   3749 
   3750 		/* We use last_read_req because sync_to()
   3751 		 * might have just caused seqno wrap under
   3752 		 * the radar.
   3753 		 */
   3754 		from->semaphore.sync_seqno[idx] =
   3755 			i915_gem_request_get_seqno(obj->last_read_req[from->id]);
   3756 	}
   3757 
   3758 	return 0;
   3759 }
   3760 
   3761 /**
   3762  * i915_gem_object_sync - sync an object to a ring.
   3763  *
   3764  * @obj: object which may be in use on another ring.
   3765  * @to: ring we wish to use the object on. May be NULL.
   3766  * @to_req: request we wish to use the object for. See below.
   3767  *          This will be allocated and returned if a request is
   3768  *          required but not passed in.
   3769  *
   3770  * This code is meant to abstract object synchronization with the GPU.
   3771  * Calling with NULL implies synchronizing the object with the CPU
   3772  * rather than a particular GPU ring. Conceptually we serialise writes
   3773  * between engines inside the GPU. We only allow one engine to write
   3774  * into a buffer at any time, but multiple readers. To ensure each has
   3775  * a coherent view of memory, we must:
   3776  *
   3777  * - If there is an outstanding write request to the object, the new
   3778  *   request must wait for it to complete (either CPU or in hw, requests
   3779  *   on the same ring will be naturally ordered).
   3780  *
   3781  * - If we are a write request (pending_write_domain is set), the new
   3782  *   request must wait for outstanding read requests to complete.
   3783  *
   3784  * For CPU synchronisation (NULL to) no request is required. For syncing with
   3785  * rings to_req must be non-NULL. However, a request does not have to be
   3786  * pre-allocated. If *to_req is NULL and sync commands will be emitted then a
   3787  * request will be allocated automatically and returned through *to_req. Note
   3788  * that it is not guaranteed that commands will be emitted (because the system
   3789  * might already be idle). Hence there is no need to create a request that
   3790  * might never have any work submitted. Note further that if a request is
   3791  * returned in *to_req, it is the responsibility of the caller to submit
   3792  * that request (after potentially adding more work to it).
   3793  *
   3794  * Returns 0 if successful, else propagates up the lower layer error.
   3795  */
   3796 int
   3797 i915_gem_object_sync(struct drm_i915_gem_object *obj,
   3798 		     struct intel_engine_cs *to,
   3799 		     struct drm_i915_gem_request **to_req)
   3800 {
   3801 	const bool readonly = obj->base.pending_write_domain == 0;
   3802 	struct drm_i915_gem_request *req[I915_NUM_RINGS];
   3803 	int ret, i, n;
   3804 
   3805 	if (!obj->active)
   3806 		return 0;
   3807 
   3808 	if (to == NULL)
   3809 		return i915_gem_object_wait_rendering(obj, readonly);
   3810 
   3811 	n = 0;
   3812 	if (readonly) {
   3813 		if (obj->last_write_req)
   3814 			req[n++] = obj->last_write_req;
   3815 	} else {
   3816 		for (i = 0; i < I915_NUM_RINGS; i++)
   3817 			if (obj->last_read_req[i])
   3818 				req[n++] = obj->last_read_req[i];
   3819 	}
   3820 	for (i = 0; i < n; i++) {
   3821 		ret = __i915_gem_object_sync(obj, to, req[i], to_req);
   3822 		if (ret)
   3823 			return ret;
   3824 	}
   3825 
   3826 	return 0;
   3827 }
   3828 
   3829 static void i915_gem_object_finish_gtt(struct drm_i915_gem_object *obj)
   3830 {
   3831 	u32 old_write_domain, old_read_domains;
   3832 
   3833 	/* Force a pagefault for domain tracking on next user access */
   3834 	i915_gem_release_mmap(obj);
   3835 
   3836 	if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0)
   3837 		return;
   3838 
   3839 	/* Wait for any direct GTT access to complete */
   3840 	mb();
   3841 
   3842 	old_read_domains = obj->base.read_domains;
   3843 	old_write_domain = obj->base.write_domain;
   3844 
   3845 	obj->base.read_domains &= ~I915_GEM_DOMAIN_GTT;
   3846 	obj->base.write_domain &= ~I915_GEM_DOMAIN_GTT;
   3847 
   3848 	trace_i915_gem_object_change_domain(obj,
   3849 					    old_read_domains,
   3850 					    old_write_domain);
   3851 }
   3852 
   3853 static int __i915_vma_unbind(struct i915_vma *vma, bool wait)
   3854 {
   3855 	struct drm_i915_gem_object *obj = vma->obj;
   3856 	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
   3857 	int ret;
   3858 
   3859 	if (list_empty(&vma->vma_link))
   3860 		return 0;
   3861 
   3862 	if (!drm_mm_node_allocated(&vma->node)) {
   3863 		i915_gem_vma_destroy(vma);
   3864 		return 0;
   3865 	}
   3866 
   3867 	if (vma->pin_count)
   3868 		return -EBUSY;
   3869 
   3870 	BUG_ON(obj->pages == NULL);
   3871 
   3872 	if (wait) {
   3873 		ret = i915_gem_object_wait_rendering(obj, false);
   3874 		if (ret)
   3875 			return ret;
   3876 	}
   3877 
   3878 	if (i915_is_ggtt(vma->vm) &&
   3879 	    vma->ggtt_view.type == I915_GGTT_VIEW_NORMAL) {
   3880 		i915_gem_object_finish_gtt(obj);
   3881 
   3882 		/* release the fence reg _after_ flushing */
   3883 		ret = i915_gem_object_put_fence(obj);
   3884 		if (ret)
   3885 			return ret;
   3886 	}
   3887 
   3888 	trace_i915_vma_unbind(vma);
   3889 
   3890 	vma->vm->unbind_vma(vma);
   3891 	vma->bound = 0;
   3892 
   3893 	list_del_init(&vma->mm_list);
   3894 	if (i915_is_ggtt(vma->vm)) {
   3895 		if (vma->ggtt_view.type == I915_GGTT_VIEW_NORMAL) {
   3896 			obj->map_and_fenceable = false;
   3897 		} else if (vma->ggtt_view.pages) {
   3898 #ifdef __NetBSD__
   3899 			panic("rotated/partial views can't happen");
   3900 #else
   3901 			sg_free_table(vma->ggtt_view.pages);
   3902 			kfree(vma->ggtt_view.pages);
   3903 #endif
   3904 		}
   3905 		vma->ggtt_view.pages = NULL;
   3906 	}
   3907 
   3908 	drm_mm_remove_node(&vma->node);
   3909 	i915_gem_vma_destroy(vma);
   3910 
   3911 	/* Since the unbound list is global, only move to that list if
   3912 	 * no more VMAs exist. */
   3913 	if (list_empty(&obj->vma_list))
   3914 		list_move_tail(&obj->global_list, &dev_priv->mm.unbound_list);
   3915 
   3916 	/* And finally now the object is completely decoupled from this vma,
   3917 	 * we can drop its hold on the backing storage and allow it to be
   3918 	 * reaped by the shrinker.
   3919 	 */
   3920 	i915_gem_object_unpin_pages(obj);
   3921 
   3922 	return 0;
   3923 }
   3924 
   3925 int i915_vma_unbind(struct i915_vma *vma)
   3926 {
   3927 	return __i915_vma_unbind(vma, true);
   3928 }
   3929 
   3930 int __i915_vma_unbind_no_wait(struct i915_vma *vma)
   3931 {
   3932 	return __i915_vma_unbind(vma, false);
   3933 }
   3934 
   3935 int i915_gpu_idle(struct drm_device *dev)
   3936 {
   3937 	struct drm_i915_private *dev_priv = dev->dev_private;
   3938 	struct intel_engine_cs *ring;
   3939 	int ret, i;
   3940 
   3941 	/* Flush everything onto the inactive list. */
   3942 	for_each_ring(ring, dev_priv, i) {
   3943 		if (!i915.enable_execlists) {
   3944 			struct drm_i915_gem_request *req;
   3945 
   3946 			ret = i915_gem_request_alloc(ring, ring->default_context, &req);
   3947 			if (ret)
   3948 				return ret;
   3949 
   3950 			ret = i915_switch_context(req);
   3951 			if (ret) {
   3952 				i915_gem_request_cancel(req);
   3953 				return ret;
   3954 			}
   3955 
   3956 			i915_add_request_no_flush(req);
   3957 		}
   3958 
   3959 		ret = intel_ring_idle(ring);
   3960 		if (ret)
   3961 			return ret;
   3962 	}
   3963 
   3964 	WARN_ON(i915_verify_lists(dev));
   3965 	return 0;
   3966 }
   3967 
   3968 static bool i915_gem_valid_gtt_space(struct i915_vma *vma,
   3969 				     unsigned long cache_level)
   3970 {
   3971 	struct drm_mm_node *gtt_space = &vma->node;
   3972 	struct drm_mm_node *other;
   3973 
   3974 	/*
   3975 	 * On some machines we have to be careful when putting differing types
   3976 	 * of snoopable memory together to avoid the prefetcher crossing memory
   3977 	 * domains and dying. During vm initialisation, we decide whether or not
   3978 	 * these constraints apply and set the drm_mm.color_adjust
   3979 	 * appropriately.
   3980 	 */
   3981 	if (vma->vm->mm.color_adjust == NULL)
   3982 		return true;
   3983 
   3984 	if (!drm_mm_node_allocated(gtt_space))
   3985 		return true;
   3986 
   3987 	if (list_empty(&gtt_space->node_list))
   3988 		return true;
   3989 
   3990 	other = list_entry(gtt_space->node_list.prev, struct drm_mm_node, node_list);
   3991 	if (other->allocated && !other->hole_follows && other->color != cache_level)
   3992 		return false;
   3993 
   3994 	other = list_entry(gtt_space->node_list.next, struct drm_mm_node, node_list);
   3995 	if (other->allocated && !gtt_space->hole_follows && other->color != cache_level)
   3996 		return false;
   3997 
   3998 	return true;
   3999 }
   4000 
   4001 /**
   4002  * Finds free space in the GTT aperture and binds the object or a view of it
   4003  * there.
   4004  */
   4005 static struct i915_vma *
   4006 i915_gem_object_bind_to_vm(struct drm_i915_gem_object *obj,
   4007 			   struct i915_address_space *vm,
   4008 			   const struct i915_ggtt_view *ggtt_view,
   4009 			   unsigned alignment,
   4010 			   uint64_t flags)
   4011 {
   4012 	struct drm_device *dev = obj->base.dev;
   4013 	struct drm_i915_private *dev_priv = dev->dev_private;
   4014 	u32 fence_alignment, unfenced_alignment;
   4015 	u32 search_flag, alloc_flag;
   4016 	u64 start, end;
   4017 	u64 size, fence_size;
   4018 	struct i915_vma *vma;
   4019 	int ret;
   4020 
   4021 	if (i915_is_ggtt(vm)) {
   4022 		u32 view_size;
   4023 
   4024 		if (WARN_ON(!ggtt_view))
   4025 			return ERR_PTR(-EINVAL);
   4026 
   4027 		view_size = i915_ggtt_view_size(obj, ggtt_view);
   4028 
   4029 		fence_size = i915_gem_get_gtt_size(dev,
   4030 						   view_size,
   4031 						   obj->tiling_mode);
   4032 		fence_alignment = i915_gem_get_gtt_alignment(dev,
   4033 							     view_size,
   4034 							     obj->tiling_mode,
   4035 							     true);
   4036 		unfenced_alignment = i915_gem_get_gtt_alignment(dev,
   4037 								view_size,
   4038 								obj->tiling_mode,
   4039 								false);
   4040 		size = flags & PIN_MAPPABLE ? fence_size : view_size;
   4041 	} else {
   4042 		fence_size = i915_gem_get_gtt_size(dev,
   4043 						   obj->base.size,
   4044 						   obj->tiling_mode);
   4045 		fence_alignment = i915_gem_get_gtt_alignment(dev,
   4046 							     obj->base.size,
   4047 							     obj->tiling_mode,
   4048 							     true);
   4049 		unfenced_alignment =
   4050 			i915_gem_get_gtt_alignment(dev,
   4051 						   obj->base.size,
   4052 						   obj->tiling_mode,
   4053 						   false);
   4054 		size = flags & PIN_MAPPABLE ? fence_size : obj->base.size;
   4055 	}
   4056 
   4057 	start = flags & PIN_OFFSET_BIAS ? flags & PIN_OFFSET_MASK : 0;
   4058 	end = vm->total;
   4059 	if (flags & PIN_MAPPABLE)
   4060 		end = min_t(u64, end, dev_priv->gtt.mappable_end);
   4061 	if (flags & PIN_ZONE_4G)
   4062 		end = min_t(u64, end, (1ULL << 32));
   4063 
   4064 	if (alignment == 0)
   4065 		alignment = flags & PIN_MAPPABLE ? fence_alignment :
   4066 						unfenced_alignment;
   4067 	if (flags & PIN_MAPPABLE && alignment & (fence_alignment - 1)) {
   4068 		DRM_DEBUG("Invalid object (view type=%u) alignment requested %u\n",
   4069 			  ggtt_view ? ggtt_view->type : 0,
   4070 			  alignment);
   4071 		return ERR_PTR(-EINVAL);
   4072 	}
   4073 
   4074 	/* If binding the object/GGTT view requires more space than the entire
   4075 	 * aperture has, reject it early before evicting everything in a vain
   4076 	 * attempt to find space.
   4077 	 */
   4078 	if (size > end) {
   4079 		DRM_DEBUG("Attempting to bind an object (view type=%u) larger than the aperture: size=%"PRIx64" > %s aperture=%"PRIx64"\n",
   4080 			  ggtt_view ? ggtt_view->type : 0,
   4081 			  size,
   4082 			  flags & PIN_MAPPABLE ? "mappable" : "total",
   4083 			  end);
   4084 		return ERR_PTR(-E2BIG);
   4085 	}
   4086 
   4087 	ret = i915_gem_object_get_pages(obj);
   4088 	if (ret)
   4089 		return ERR_PTR(ret);
   4090 
   4091 	i915_gem_object_pin_pages(obj);
   4092 
   4093 	vma = ggtt_view ? i915_gem_obj_lookup_or_create_ggtt_vma(obj, ggtt_view) :
   4094 			  i915_gem_obj_lookup_or_create_vma(obj, vm);
   4095 
   4096 	if (IS_ERR(vma))
   4097 		goto err_unpin;
   4098 
   4099 	if (flags & PIN_HIGH) {
   4100 		search_flag = DRM_MM_SEARCH_BELOW;
   4101 		alloc_flag = DRM_MM_CREATE_TOP;
   4102 	} else {
   4103 		search_flag = DRM_MM_SEARCH_DEFAULT;
   4104 		alloc_flag = DRM_MM_CREATE_DEFAULT;
   4105 	}
   4106 
   4107 search_free:
   4108 	ret = drm_mm_insert_node_in_range_generic(&vm->mm, &vma->node,
   4109 						  size, alignment,
   4110 						  obj->cache_level,
   4111 						  start, end,
   4112 						  search_flag,
   4113 						  alloc_flag);
   4114 	if (ret) {
   4115 		ret = i915_gem_evict_something(dev, vm, size, alignment,
   4116 					       obj->cache_level,
   4117 					       start, end,
   4118 					       flags);
   4119 		if (ret == 0)
   4120 			goto search_free;
   4121 
   4122 		goto err_free_vma;
   4123 	}
   4124 	if (WARN_ON(!i915_gem_valid_gtt_space(vma, obj->cache_level))) {
   4125 		ret = -EINVAL;
   4126 		goto err_remove_node;
   4127 	}
   4128 
   4129 	trace_i915_vma_bind(vma, flags);
   4130 	ret = i915_vma_bind(vma, obj->cache_level, flags);
   4131 	if (ret)
   4132 		goto err_remove_node;
   4133 
   4134 	list_move_tail(&obj->global_list, &dev_priv->mm.bound_list);
   4135 	list_add_tail(&vma->mm_list, &vm->inactive_list);
   4136 
   4137 	return vma;
   4138 
   4139 err_remove_node:
   4140 	drm_mm_remove_node(&vma->node);
   4141 err_free_vma:
   4142 	i915_gem_vma_destroy(vma);
   4143 	vma = ERR_PTR(ret);
   4144 err_unpin:
   4145 	i915_gem_object_unpin_pages(obj);
   4146 	return vma;
   4147 }
   4148 
   4149 bool
   4150 i915_gem_clflush_object(struct drm_i915_gem_object *obj,
   4151 			bool force)
   4152 {
   4153 	/* If we don't have a page list set up, then we're not pinned
   4154 	 * to GPU, and we can ignore the cache flush because it'll happen
   4155 	 * again at bind time.
   4156 	 */
   4157 	if (obj->pages == NULL)
   4158 		return false;
   4159 
   4160 	/*
   4161 	 * Stolen memory is always coherent with the GPU as it is explicitly
   4162 	 * marked as wc by the system, or the system is cache-coherent.
   4163 	 */
   4164 	if (obj->stolen || obj->phys_handle)
   4165 		return false;
   4166 
   4167 	/* If the GPU is snooping the contents of the CPU cache,
   4168 	 * we do not need to manually clear the CPU cache lines.  However,
   4169 	 * the caches are only snooped when the render cache is
   4170 	 * flushed/invalidated.  As we always have to emit invalidations
   4171 	 * and flushes when moving into and out of the RENDER domain, correct
   4172 	 * snooping behaviour occurs naturally as the result of our domain
   4173 	 * tracking.
   4174 	 */
   4175 	if (!force && cpu_cache_is_coherent(obj->base.dev, obj->cache_level)) {
   4176 		obj->cache_dirty = true;
   4177 		return false;
   4178 	}
   4179 
   4180 	trace_i915_gem_object_clflush(obj);
   4181 #ifdef __NetBSD__
   4182 	drm_clflush_pglist(&obj->pageq);
   4183 #else
   4184 	drm_clflush_sg(obj->pages);
   4185 #endif
   4186 	obj->cache_dirty = false;
   4187 
   4188 	return true;
   4189 }
   4190 
   4191 /** Flushes the GTT write domain for the object if it's dirty. */
   4192 static void
   4193 i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj)
   4194 {
   4195 	uint32_t old_write_domain;
   4196 
   4197 	if (obj->base.write_domain != I915_GEM_DOMAIN_GTT)
   4198 		return;
   4199 
   4200 	/* No actual flushing is required for the GTT write domain.  Writes
   4201 	 * to it immediately go to main memory as far as we know, so there's
   4202 	 * no chipset flush.  It also doesn't land in render cache.
   4203 	 *
   4204 	 * However, we do have to enforce the order so that all writes through
   4205 	 * the GTT land before any writes to the device, such as updates to
   4206 	 * the GATT itself.
   4207 	 */
   4208 	wmb();
   4209 
   4210 	old_write_domain = obj->base.write_domain;
   4211 	obj->base.write_domain = 0;
   4212 
   4213 	intel_fb_obj_flush(obj, false, ORIGIN_GTT);
   4214 
   4215 	trace_i915_gem_object_change_domain(obj,
   4216 					    obj->base.read_domains,
   4217 					    old_write_domain);
   4218 }
   4219 
   4220 /** Flushes the CPU write domain for the object if it's dirty. */
   4221 static void
   4222 i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj)
   4223 {
   4224 	uint32_t old_write_domain;
   4225 
   4226 	if (obj->base.write_domain != I915_GEM_DOMAIN_CPU)
   4227 		return;
   4228 
   4229 	if (i915_gem_clflush_object(obj, obj->pin_display))
   4230 		i915_gem_chipset_flush(obj->base.dev);
   4231 
   4232 	old_write_domain = obj->base.write_domain;
   4233 	obj->base.write_domain = 0;
   4234 
   4235 	intel_fb_obj_flush(obj, false, ORIGIN_CPU);
   4236 
   4237 	trace_i915_gem_object_change_domain(obj,
   4238 					    obj->base.read_domains,
   4239 					    old_write_domain);
   4240 }
   4241 
   4242 /**
   4243  * Moves a single object to the GTT read, and possibly write domain.
   4244  *
   4245  * This function returns when the move is complete, including waiting on
   4246  * flushes to occur.
   4247  */
   4248 int
   4249 i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write)
   4250 {
   4251 	uint32_t old_write_domain, old_read_domains;
   4252 	struct i915_vma *vma;
   4253 	int ret;
   4254 
   4255 	if (obj->base.write_domain == I915_GEM_DOMAIN_GTT)
   4256 		return 0;
   4257 
   4258 	ret = i915_gem_object_wait_rendering(obj, !write);
   4259 	if (ret)
   4260 		return ret;
   4261 
   4262 	/* Flush and acquire obj->pages so that we are coherent through
   4263 	 * direct access in memory with previous cached writes through
   4264 	 * shmemfs and that our cache domain tracking remains valid.
   4265 	 * For example, if the obj->filp was moved to swap without us
   4266 	 * being notified and releasing the pages, we would mistakenly
   4267 	 * continue to assume that the obj remained out of the CPU cached
   4268 	 * domain.
   4269 	 */
   4270 	ret = i915_gem_object_get_pages(obj);
   4271 	if (ret)
   4272 		return ret;
   4273 
   4274 	i915_gem_object_flush_cpu_write_domain(obj);
   4275 
   4276 	/* Serialise direct access to this object with the barriers for
   4277 	 * coherent writes from the GPU, by effectively invalidating the
   4278 	 * GTT domain upon first access.
   4279 	 */
   4280 	if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0)
   4281 		mb();
   4282 
   4283 	old_write_domain = obj->base.write_domain;
   4284 	old_read_domains = obj->base.read_domains;
   4285 
   4286 	/* It should now be out of any other write domains, and we can update
   4287 	 * the domain values for our changes.
   4288 	 */
   4289 	BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
   4290 	obj->base.read_domains |= I915_GEM_DOMAIN_GTT;
   4291 	if (write) {
   4292 		obj->base.read_domains = I915_GEM_DOMAIN_GTT;
   4293 		obj->base.write_domain = I915_GEM_DOMAIN_GTT;
   4294 		obj->dirty = 1;
   4295 	}
   4296 
   4297 	trace_i915_gem_object_change_domain(obj,
   4298 					    old_read_domains,
   4299 					    old_write_domain);
   4300 
   4301 	/* And bump the LRU for this access */
   4302 	vma = i915_gem_obj_to_ggtt(obj);
   4303 	if (vma && drm_mm_node_allocated(&vma->node) && !obj->active)
   4304 		list_move_tail(&vma->mm_list,
   4305 			       &to_i915(obj->base.dev)->gtt.base.inactive_list);
   4306 
   4307 	return 0;
   4308 }
   4309 
   4310 /**
   4311  * Changes the cache-level of an object across all VMA.
   4312  *
   4313  * After this function returns, the object will be in the new cache-level
   4314  * across all GTT and the contents of the backing storage will be coherent,
   4315  * with respect to the new cache-level. In order to keep the backing storage
   4316  * coherent for all users, we only allow a single cache level to be set
   4317  * globally on the object and prevent it from being changed whilst the
   4318  * hardware is reading from the object. That is if the object is currently
   4319  * on the scanout it will be set to uncached (or equivalent display
   4320  * cache coherency) and all non-MOCS GPU access will also be uncached so
   4321  * that all direct access to the scanout remains coherent.
   4322  */
   4323 int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj,
   4324 				    enum i915_cache_level cache_level)
   4325 {
   4326 	struct drm_device *dev = obj->base.dev;
   4327 	struct i915_vma *vma, *next;
   4328 	bool bound = false;
   4329 	int ret = 0;
   4330 
   4331 	if (obj->cache_level == cache_level)
   4332 		goto out;
   4333 
   4334 	/* Inspect the list of currently bound VMA and unbind any that would
   4335 	 * be invalid given the new cache-level. This is principally to
   4336 	 * catch the issue of the CS prefetch crossing page boundaries and
   4337 	 * reading an invalid PTE on older architectures.
   4338 	 */
   4339 	list_for_each_entry_safe(vma, next, &obj->vma_list, vma_link) {
   4340 		if (!drm_mm_node_allocated(&vma->node))
   4341 			continue;
   4342 
   4343 		if (vma->pin_count) {
   4344 			DRM_DEBUG("can not change the cache level of pinned objects\n");
   4345 			return -EBUSY;
   4346 		}
   4347 
   4348 		if (!i915_gem_valid_gtt_space(vma, cache_level)) {
   4349 			ret = i915_vma_unbind(vma);
   4350 			if (ret)
   4351 				return ret;
   4352 		} else
   4353 			bound = true;
   4354 	}
   4355 
   4356 	/* We can reuse the existing drm_mm nodes but need to change the
   4357 	 * cache-level on the PTE. We could simply unbind them all and
   4358 	 * rebind with the correct cache-level on next use. However since
   4359 	 * we already have a valid slot, dma mapping, pages etc, we may as
   4360 	 * rewrite the PTE in the belief that doing so tramples upon less
   4361 	 * state and so involves less work.
   4362 	 */
   4363 	if (bound) {
   4364 		/* Before we change the PTE, the GPU must not be accessing it.
   4365 		 * If we wait upon the object, we know that all the bound
   4366 		 * VMA are no longer active.
   4367 		 */
   4368 		ret = i915_gem_object_wait_rendering(obj, false);
   4369 		if (ret)
   4370 			return ret;
   4371 
   4372 		if (!HAS_LLC(dev) && cache_level != I915_CACHE_NONE) {
   4373 			/* Access to snoopable pages through the GTT is
   4374 			 * incoherent and on some machines causes a hard
   4375 			 * lockup. Relinquish the CPU mmaping to force
   4376 			 * userspace to refault in the pages and we can
   4377 			 * then double check if the GTT mapping is still
   4378 			 * valid for that pointer access.
   4379 			 */
   4380 			i915_gem_release_mmap(obj);
   4381 
   4382 			/* As we no longer need a fence for GTT access,
   4383 			 * we can relinquish it now (and so prevent having
   4384 			 * to steal a fence from someone else on the next
   4385 			 * fence request). Note GPU activity would have
   4386 			 * dropped the fence as all snoopable access is
   4387 			 * supposed to be linear.
   4388 			 */
   4389 			ret = i915_gem_object_put_fence(obj);
   4390 			if (ret)
   4391 				return ret;
   4392 		} else {
   4393 			/* We either have incoherent backing store and
   4394 			 * so no GTT access or the architecture is fully
   4395 			 * coherent. In such cases, existing GTT mmaps
   4396 			 * ignore the cache bit in the PTE and we can
   4397 			 * rewrite it without confusing the GPU or having
   4398 			 * to force userspace to fault back in its mmaps.
   4399 			 */
   4400 		}
   4401 
   4402 		list_for_each_entry(vma, &obj->vma_list, vma_link) {
   4403 			if (!drm_mm_node_allocated(&vma->node))
   4404 				continue;
   4405 
   4406 			ret = i915_vma_bind(vma, cache_level, PIN_UPDATE);
   4407 			if (ret)
   4408 				return ret;
   4409 		}
   4410 	}
   4411 
   4412 	list_for_each_entry(vma, &obj->vma_list, vma_link)
   4413 		vma->node.color = cache_level;
   4414 	obj->cache_level = cache_level;
   4415 
   4416 out:
   4417 	/* Flush the dirty CPU caches to the backing storage so that the
   4418 	 * object is now coherent at its new cache level (with respect
   4419 	 * to the access domain).
   4420 	 */
   4421 	if (obj->cache_dirty &&
   4422 	    obj->base.write_domain != I915_GEM_DOMAIN_CPU &&
   4423 	    cpu_write_needs_clflush(obj)) {
   4424 		if (i915_gem_clflush_object(obj, true))
   4425 			i915_gem_chipset_flush(obj->base.dev);
   4426 	}
   4427 
   4428 	return 0;
   4429 }
   4430 
   4431 int i915_gem_get_caching_ioctl(struct drm_device *dev, void *data,
   4432 			       struct drm_file *file)
   4433 {
   4434 	struct drm_i915_gem_caching *args = data;
   4435 	struct drm_gem_object *gobj;
   4436 	struct drm_i915_gem_object *obj;
   4437 
   4438 	gobj = drm_gem_object_lookup(dev, file, args->handle);
   4439 	if (gobj == NULL)
   4440 		return -ENOENT;
   4441 	obj = to_intel_bo(gobj);
   4442 
   4443 	switch (obj->cache_level) {
   4444 	case I915_CACHE_LLC:
   4445 	case I915_CACHE_L3_LLC:
   4446 		args->caching = I915_CACHING_CACHED;
   4447 		break;
   4448 
   4449 	case I915_CACHE_WT:
   4450 		args->caching = I915_CACHING_DISPLAY;
   4451 		break;
   4452 
   4453 	default:
   4454 		args->caching = I915_CACHING_NONE;
   4455 		break;
   4456 	}
   4457 
   4458 	drm_gem_object_unreference_unlocked(&obj->base);
   4459 	return 0;
   4460 }
   4461 
   4462 int i915_gem_set_caching_ioctl(struct drm_device *dev, void *data,
   4463 			       struct drm_file *file)
   4464 {
   4465 	struct drm_i915_private *dev_priv = dev->dev_private;
   4466 	struct drm_i915_gem_caching *args = data;
   4467 	struct drm_gem_object *gobj;
   4468 	struct drm_i915_gem_object *obj;
   4469 	enum i915_cache_level level;
   4470 	int ret;
   4471 
   4472 	switch (args->caching) {
   4473 	case I915_CACHING_NONE:
   4474 		level = I915_CACHE_NONE;
   4475 		break;
   4476 	case I915_CACHING_CACHED:
   4477 		/*
   4478 		 * Due to a HW issue on BXT A stepping, GPU stores via a
   4479 		 * snooped mapping may leave stale data in a corresponding CPU
   4480 		 * cacheline, whereas normally such cachelines would get
   4481 		 * invalidated.
   4482 		 */
   4483 		if (IS_BROXTON(dev) && INTEL_REVID(dev) < BXT_REVID_B0)
   4484 			return -ENODEV;
   4485 
   4486 		level = I915_CACHE_LLC;
   4487 		break;
   4488 	case I915_CACHING_DISPLAY:
   4489 		level = HAS_WT(dev) ? I915_CACHE_WT : I915_CACHE_NONE;
   4490 		break;
   4491 	default:
   4492 		return -EINVAL;
   4493 	}
   4494 
   4495 	intel_runtime_pm_get(dev_priv);
   4496 
   4497 	ret = i915_mutex_lock_interruptible(dev);
   4498 	if (ret)
   4499 		goto rpm_put;
   4500 
   4501 	gobj = drm_gem_object_lookup(dev, file, args->handle);
   4502 	if (gobj == NULL) {
   4503 		ret = -ENOENT;
   4504 		goto unlock;
   4505 	}
   4506 	obj = to_intel_bo(gobj);
   4507 
   4508 	ret = i915_gem_object_set_cache_level(obj, level);
   4509 
   4510 	drm_gem_object_unreference(&obj->base);
   4511 unlock:
   4512 	mutex_unlock(&dev->struct_mutex);
   4513 rpm_put:
   4514 	intel_runtime_pm_put(dev_priv);
   4515 
   4516 	return ret;
   4517 }
   4518 
   4519 /*
   4520  * Prepare buffer for display plane (scanout, cursors, etc).
   4521  * Can be called from an uninterruptible phase (modesetting) and allows
   4522  * any flushes to be pipelined (for pageflips).
   4523  */
   4524 int
   4525 i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj,
   4526 				     u32 alignment,
   4527 				     struct intel_engine_cs *pipelined,
   4528 				     struct drm_i915_gem_request **pipelined_request,
   4529 				     const struct i915_ggtt_view *view)
   4530 {
   4531 	u32 old_read_domains, old_write_domain;
   4532 	int ret;
   4533 
   4534 	ret = i915_gem_object_sync(obj, pipelined, pipelined_request);
   4535 	if (ret)
   4536 		return ret;
   4537 
   4538 	/* Mark the pin_display early so that we account for the
   4539 	 * display coherency whilst setting up the cache domains.
   4540 	 */
   4541 	obj->pin_display++;
   4542 
   4543 	/* The display engine is not coherent with the LLC cache on gen6.  As
   4544 	 * a result, we make sure that the pinning that is about to occur is
   4545 	 * done with uncached PTEs. This is lowest common denominator for all
   4546 	 * chipsets.
   4547 	 *
   4548 	 * However for gen6+, we could do better by using the GFDT bit instead
   4549 	 * of uncaching, which would allow us to flush all the LLC-cached data
   4550 	 * with that bit in the PTE to main memory with just one PIPE_CONTROL.
   4551 	 */
   4552 	ret = i915_gem_object_set_cache_level(obj,
   4553 					      HAS_WT(obj->base.dev) ? I915_CACHE_WT : I915_CACHE_NONE);
   4554 	if (ret)
   4555 		goto err_unpin_display;
   4556 
   4557 	/* As the user may map the buffer once pinned in the display plane
   4558 	 * (e.g. libkms for the bootup splash), we have to ensure that we
   4559 	 * always use map_and_fenceable for all scanout buffers.
   4560 	 */
   4561 	ret = i915_gem_object_ggtt_pin(obj, view, alignment,
   4562 				       view->type == I915_GGTT_VIEW_NORMAL ?
   4563 				       PIN_MAPPABLE : 0);
   4564 	if (ret)
   4565 		goto err_unpin_display;
   4566 
   4567 	i915_gem_object_flush_cpu_write_domain(obj);
   4568 
   4569 	old_write_domain = obj->base.write_domain;
   4570 	old_read_domains = obj->base.read_domains;
   4571 
   4572 	/* It should now be out of any other write domains, and we can update
   4573 	 * the domain values for our changes.
   4574 	 */
   4575 	obj->base.write_domain = 0;
   4576 	obj->base.read_domains |= I915_GEM_DOMAIN_GTT;
   4577 
   4578 	trace_i915_gem_object_change_domain(obj,
   4579 					    old_read_domains,
   4580 					    old_write_domain);
   4581 
   4582 	return 0;
   4583 
   4584 err_unpin_display:
   4585 	obj->pin_display--;
   4586 	return ret;
   4587 }
   4588 
   4589 void
   4590 i915_gem_object_unpin_from_display_plane(struct drm_i915_gem_object *obj,
   4591 					 const struct i915_ggtt_view *view)
   4592 {
   4593 	if (WARN_ON(obj->pin_display == 0))
   4594 		return;
   4595 
   4596 	i915_gem_object_ggtt_unpin_view(obj, view);
   4597 
   4598 	obj->pin_display--;
   4599 }
   4600 
   4601 /**
   4602  * Moves a single object to the CPU read, and possibly write domain.
   4603  *
   4604  * This function returns when the move is complete, including waiting on
   4605  * flushes to occur.
   4606  */
   4607 int
   4608 i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write)
   4609 {
   4610 	uint32_t old_write_domain, old_read_domains;
   4611 	int ret;
   4612 
   4613 	if (obj->base.write_domain == I915_GEM_DOMAIN_CPU)
   4614 		return 0;
   4615 
   4616 	ret = i915_gem_object_wait_rendering(obj, !write);
   4617 	if (ret)
   4618 		return ret;
   4619 
   4620 	i915_gem_object_flush_gtt_write_domain(obj);
   4621 
   4622 	old_write_domain = obj->base.write_domain;
   4623 	old_read_domains = obj->base.read_domains;
   4624 
   4625 	/* Flush the CPU cache if it's still invalid. */
   4626 	if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0) {
   4627 		i915_gem_clflush_object(obj, false);
   4628 
   4629 		obj->base.read_domains |= I915_GEM_DOMAIN_CPU;
   4630 	}
   4631 
   4632 	/* It should now be out of any other write domains, and we can update
   4633 	 * the domain values for our changes.
   4634 	 */
   4635 	BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
   4636 
   4637 	/* If we're writing through the CPU, then the GPU read domains will
   4638 	 * need to be invalidated at next use.
   4639 	 */
   4640 	if (write) {
   4641 		obj->base.read_domains = I915_GEM_DOMAIN_CPU;
   4642 		obj->base.write_domain = I915_GEM_DOMAIN_CPU;
   4643 	}
   4644 
   4645 	trace_i915_gem_object_change_domain(obj,
   4646 					    old_read_domains,
   4647 					    old_write_domain);
   4648 
   4649 	return 0;
   4650 }
   4651 
   4652 /* Throttle our rendering by waiting until the ring has completed our requests
   4653  * emitted over 20 msec ago.
   4654  *
   4655  * Note that if we were to use the current jiffies each time around the loop,
   4656  * we wouldn't escape the function with any frames outstanding if the time to
   4657  * render a frame was over 20ms.
   4658  *
   4659  * This should get us reasonable parallelism between CPU and GPU but also
   4660  * relatively low latency when blocking on a particular request to finish.
   4661  */
   4662 static int
   4663 i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file)
   4664 {
   4665 	struct drm_i915_private *dev_priv = dev->dev_private;
   4666 	struct drm_i915_file_private *file_priv = file->driver_priv;
   4667 	unsigned long recent_enough = jiffies - DRM_I915_THROTTLE_JIFFIES;
   4668 	struct drm_i915_gem_request *request, *target = NULL;
   4669 	unsigned reset_counter;
   4670 	int ret;
   4671 
   4672 	ret = i915_gem_wait_for_error(&dev_priv->gpu_error);
   4673 	if (ret)
   4674 		return ret;
   4675 
   4676 	ret = i915_gem_check_wedge(&dev_priv->gpu_error, false);
   4677 	if (ret)
   4678 		return ret;
   4679 
   4680 	spin_lock(&file_priv->mm.lock);
   4681 	list_for_each_entry(request, &file_priv->mm.request_list, client_list) {
   4682 		if (time_after_eq(request->emitted_jiffies, recent_enough))
   4683 			break;
   4684 
   4685 		/*
   4686 		 * Note that the request might not have been submitted yet.
   4687 		 * In which case emitted_jiffies will be zero.
   4688 		 */
   4689 		if (!request->emitted_jiffies)
   4690 			continue;
   4691 
   4692 		target = request;
   4693 	}
   4694 	reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter);
   4695 	if (target)
   4696 		i915_gem_request_reference(target);
   4697 	spin_unlock(&file_priv->mm.lock);
   4698 
   4699 	if (target == NULL)
   4700 		return 0;
   4701 
   4702 	ret = __i915_wait_request(target, reset_counter, true, NULL, NULL);
   4703 	if (ret == 0)
   4704 		queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, 0);
   4705 
   4706 	i915_gem_request_unreference__unlocked(target);
   4707 
   4708 	return ret;
   4709 }
   4710 
   4711 static bool
   4712 i915_vma_misplaced(struct i915_vma *vma, uint32_t alignment, uint64_t flags)
   4713 {
   4714 	struct drm_i915_gem_object *obj = vma->obj;
   4715 
   4716 	if (alignment &&
   4717 	    vma->node.start & (alignment - 1))
   4718 		return true;
   4719 
   4720 	if (flags & PIN_MAPPABLE && !obj->map_and_fenceable)
   4721 		return true;
   4722 
   4723 	if (flags & PIN_OFFSET_BIAS &&
   4724 	    vma->node.start < (flags & PIN_OFFSET_MASK))
   4725 		return true;
   4726 
   4727 	return false;
   4728 }
   4729 
   4730 void __i915_vma_set_map_and_fenceable(struct i915_vma *vma)
   4731 {
   4732 	struct drm_i915_gem_object *obj = vma->obj;
   4733 	bool mappable, fenceable;
   4734 	u32 fence_size, fence_alignment;
   4735 
   4736 	fence_size = i915_gem_get_gtt_size(obj->base.dev,
   4737 					   obj->base.size,
   4738 					   obj->tiling_mode);
   4739 	fence_alignment = i915_gem_get_gtt_alignment(obj->base.dev,
   4740 						     obj->base.size,
   4741 						     obj->tiling_mode,
   4742 						     true);
   4743 
   4744 	fenceable = (vma->node.size == fence_size &&
   4745 		     (vma->node.start & (fence_alignment - 1)) == 0);
   4746 
   4747 	mappable = (vma->node.start + fence_size <=
   4748 		    to_i915(obj->base.dev)->gtt.mappable_end);
   4749 
   4750 	obj->map_and_fenceable = mappable && fenceable;
   4751 }
   4752 
   4753 static int
   4754 i915_gem_object_do_pin(struct drm_i915_gem_object *obj,
   4755 		       struct i915_address_space *vm,
   4756 		       const struct i915_ggtt_view *ggtt_view,
   4757 		       uint32_t alignment,
   4758 		       uint64_t flags)
   4759 {
   4760 	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
   4761 	struct i915_vma *vma;
   4762 	unsigned bound;
   4763 	int ret;
   4764 
   4765 	if (WARN_ON(vm == &dev_priv->mm.aliasing_ppgtt->base))
   4766 		return -ENODEV;
   4767 
   4768 	if (WARN_ON(flags & (PIN_GLOBAL | PIN_MAPPABLE) && !i915_is_ggtt(vm)))
   4769 		return -EINVAL;
   4770 
   4771 	if (WARN_ON((flags & (PIN_MAPPABLE | PIN_GLOBAL)) == PIN_MAPPABLE))
   4772 		return -EINVAL;
   4773 
   4774 	if (WARN_ON(i915_is_ggtt(vm) != !!ggtt_view))
   4775 		return -EINVAL;
   4776 
   4777 	vma = ggtt_view ? i915_gem_obj_to_ggtt_view(obj, ggtt_view) :
   4778 			  i915_gem_obj_to_vma(obj, vm);
   4779 
   4780 	if (IS_ERR(vma))
   4781 		return PTR_ERR(vma);
   4782 
   4783 	if (vma) {
   4784 		if (WARN_ON(vma->pin_count == DRM_I915_GEM_OBJECT_MAX_PIN_COUNT))
   4785 			return -EBUSY;
   4786 
   4787 		if (i915_vma_misplaced(vma, alignment, flags)) {
   4788 			WARN(vma->pin_count,
   4789 			     "bo is already pinned in %s with incorrect alignment:"
   4790 			     " offset=%08x %08x, req.alignment=%x, req.map_and_fenceable=%d,"
   4791 			     " obj->map_and_fenceable=%d\n",
   4792 			     ggtt_view ? "ggtt" : "ppgtt",
   4793 			     upper_32_bits(vma->node.start),
   4794 			     lower_32_bits(vma->node.start),
   4795 			     alignment,
   4796 			     !!(flags & PIN_MAPPABLE),
   4797 			     obj->map_and_fenceable);
   4798 			ret = i915_vma_unbind(vma);
   4799 			if (ret)
   4800 				return ret;
   4801 
   4802 			vma = NULL;
   4803 		}
   4804 	}
   4805 
   4806 	bound = vma ? vma->bound : 0;
   4807 	if (vma == NULL || !drm_mm_node_allocated(&vma->node)) {
   4808 		vma = i915_gem_object_bind_to_vm(obj, vm, ggtt_view, alignment,
   4809 						 flags);
   4810 		if (IS_ERR(vma))
   4811 			return PTR_ERR(vma);
   4812 	} else {
   4813 		ret = i915_vma_bind(vma, obj->cache_level, flags);
   4814 		if (ret)
   4815 			return ret;
   4816 	}
   4817 
   4818 	if (ggtt_view && ggtt_view->type == I915_GGTT_VIEW_NORMAL &&
   4819 	    (bound ^ vma->bound) & GLOBAL_BIND) {
   4820 		__i915_vma_set_map_and_fenceable(vma);
   4821 		WARN_ON(flags & PIN_MAPPABLE && !obj->map_and_fenceable);
   4822 	}
   4823 
   4824 	vma->pin_count++;
   4825 	return 0;
   4826 }
   4827 
   4828 int
   4829 i915_gem_object_pin(struct drm_i915_gem_object *obj,
   4830 		    struct i915_address_space *vm,
   4831 		    uint32_t alignment,
   4832 		    uint64_t flags)
   4833 {
   4834 	return i915_gem_object_do_pin(obj, vm,
   4835 				      i915_is_ggtt(vm) ? &i915_ggtt_view_normal : NULL,
   4836 				      alignment, flags);
   4837 }
   4838 
   4839 int
   4840 i915_gem_object_ggtt_pin(struct drm_i915_gem_object *obj,
   4841 			 const struct i915_ggtt_view *view,
   4842 			 uint32_t alignment,
   4843 			 uint64_t flags)
   4844 {
   4845 	if (WARN_ONCE(!view, "no view specified"))
   4846 		return -EINVAL;
   4847 
   4848 	return i915_gem_object_do_pin(obj, i915_obj_to_ggtt(obj), view,
   4849 				      alignment, flags | PIN_GLOBAL);
   4850 }
   4851 
   4852 void
   4853 i915_gem_object_ggtt_unpin_view(struct drm_i915_gem_object *obj,
   4854 				const struct i915_ggtt_view *view)
   4855 {
   4856 	struct i915_vma *vma = i915_gem_obj_to_ggtt_view(obj, view);
   4857 
   4858 	BUG_ON(!vma);
   4859 	WARN_ON(vma->pin_count == 0);
   4860 	WARN_ON(!i915_gem_obj_ggtt_bound_view(obj, view));
   4861 
   4862 	--vma->pin_count;
   4863 }
   4864 
   4865 int
   4866 i915_gem_busy_ioctl(struct drm_device *dev, void *data,
   4867 		    struct drm_file *file)
   4868 {
   4869 	struct drm_i915_gem_busy *args = data;
   4870 	struct drm_gem_object *gobj;
   4871 	struct drm_i915_gem_object *obj;
   4872 	int ret;
   4873 
   4874 	ret = i915_mutex_lock_interruptible(dev);
   4875 	if (ret)
   4876 		return ret;
   4877 
   4878 	gobj = drm_gem_object_lookup(dev, file, args->handle);
   4879 	if (gobj == NULL) {
   4880 		ret = -ENOENT;
   4881 		goto unlock;
   4882 	}
   4883 	obj = to_intel_bo(gobj);
   4884 
   4885 	/* Count all active objects as busy, even if they are currently not used
   4886 	 * by the gpu. Users of this interface expect objects to eventually
   4887 	 * become non-busy without any further actions, therefore emit any
   4888 	 * necessary flushes here.
   4889 	 */
   4890 	ret = i915_gem_object_flush_active(obj);
   4891 	if (ret)
   4892 		goto unref;
   4893 
   4894 	BUILD_BUG_ON(I915_NUM_RINGS > 16);
   4895 	args->busy = obj->active << 16;
   4896 	if (obj->last_write_req)
   4897 		args->busy |= obj->last_write_req->ring->id;
   4898 
   4899 unref:
   4900 	drm_gem_object_unreference(&obj->base);
   4901 unlock:
   4902 	mutex_unlock(&dev->struct_mutex);
   4903 	return ret;
   4904 }
   4905 
   4906 int
   4907 i915_gem_throttle_ioctl(struct drm_device *dev, void *data,
   4908 			struct drm_file *file_priv)
   4909 {
   4910 	return i915_gem_ring_throttle(dev, file_priv);
   4911 }
   4912 
   4913 int
   4914 i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
   4915 		       struct drm_file *file_priv)
   4916 {
   4917 	struct drm_i915_private *dev_priv = dev->dev_private;
   4918 	struct drm_i915_gem_madvise *args = data;
   4919 	struct drm_gem_object *gobj;
   4920 	struct drm_i915_gem_object *obj;
   4921 	int ret;
   4922 
   4923 	switch (args->madv) {
   4924 	case I915_MADV_DONTNEED:
   4925 	case I915_MADV_WILLNEED:
   4926 	    break;
   4927 	default:
   4928 	    return -EINVAL;
   4929 	}
   4930 
   4931 	ret = i915_mutex_lock_interruptible(dev);
   4932 	if (ret)
   4933 		return ret;
   4934 
   4935 	gobj = drm_gem_object_lookup(dev, file_priv, args->handle);
   4936 	if (gobj == NULL) {
   4937 		ret = -ENOENT;
   4938 		goto unlock;
   4939 	}
   4940 	obj = to_intel_bo(gobj);
   4941 
   4942 	if (i915_gem_obj_is_pinned(obj)) {
   4943 		ret = -EINVAL;
   4944 		goto out;
   4945 	}
   4946 
   4947 	if (obj->pages &&
   4948 	    obj->tiling_mode != I915_TILING_NONE &&
   4949 	    dev_priv->quirks & QUIRK_PIN_SWIZZLED_PAGES) {
   4950 		if (obj->madv == I915_MADV_WILLNEED)
   4951 			i915_gem_object_unpin_pages(obj);
   4952 		if (args->madv == I915_MADV_WILLNEED)
   4953 			i915_gem_object_pin_pages(obj);
   4954 	}
   4955 
   4956 	if (obj->madv != __I915_MADV_PURGED)
   4957 		obj->madv = args->madv;
   4958 
   4959 	/* if the object is no longer attached, discard its backing storage */
   4960 	if (obj->madv == I915_MADV_DONTNEED && obj->pages == NULL)
   4961 		i915_gem_object_truncate(obj);
   4962 
   4963 	args->retained = obj->madv != __I915_MADV_PURGED;
   4964 
   4965 out:
   4966 	drm_gem_object_unreference(&obj->base);
   4967 unlock:
   4968 	mutex_unlock(&dev->struct_mutex);
   4969 	return ret;
   4970 }
   4971 
   4972 void i915_gem_object_init(struct drm_i915_gem_object *obj,
   4973 			  const struct drm_i915_gem_object_ops *ops)
   4974 {
   4975 	int i;
   4976 
   4977 	INIT_LIST_HEAD(&obj->global_list);
   4978 	for (i = 0; i < I915_NUM_RINGS; i++)
   4979 		INIT_LIST_HEAD(&obj->ring_list[i]);
   4980 	INIT_LIST_HEAD(&obj->obj_exec_link);
   4981 	INIT_LIST_HEAD(&obj->vma_list);
   4982 	INIT_LIST_HEAD(&obj->batch_pool_link);
   4983 
   4984 	obj->ops = ops;
   4985 
   4986 	obj->fence_reg = I915_FENCE_REG_NONE;
   4987 	obj->madv = I915_MADV_WILLNEED;
   4988 
   4989 	i915_gem_info_add_obj(obj->base.dev->dev_private, obj->base.size);
   4990 }
   4991 
   4992 static const struct drm_i915_gem_object_ops i915_gem_object_ops = {
   4993 	.get_pages = i915_gem_object_get_pages_gtt,
   4994 	.put_pages = i915_gem_object_put_pages_gtt,
   4995 };
   4996 
   4997 struct drm_i915_gem_object *i915_gem_alloc_object(struct drm_device *dev,
   4998 						  size_t size)
   4999 {
   5000 #ifdef __NetBSD__
   5001 	struct drm_i915_private *const dev_priv = dev->dev_private;
   5002 #endif
   5003 	struct drm_i915_gem_object *obj;
   5004 #ifndef __NetBSD__
   5005 	struct address_space *mapping;
   5006 	gfp_t mask;
   5007 #endif
   5008 
   5009 	obj = i915_gem_object_alloc(dev);
   5010 	if (obj == NULL)
   5011 		return NULL;
   5012 
   5013 	if (drm_gem_object_init(dev, &obj->base, size) != 0) {
   5014 		i915_gem_object_free(obj);
   5015 		return NULL;
   5016 	}
   5017 
   5018 #ifdef __NetBSD__
   5019 	uao_set_pgfl(obj->base.gemo_shm_uao, dev_priv->gtt.pgfl);
   5020 #else
   5021 	mask = GFP_HIGHUSER | __GFP_RECLAIMABLE;
   5022 	if (IS_CRESTLINE(dev) || IS_BROADWATER(dev)) {
   5023 		/* 965gm cannot relocate objects above 4GiB. */
   5024 		mask &= ~__GFP_HIGHMEM;
   5025 		mask |= __GFP_DMA32;
   5026 	}
   5027 
   5028 	mapping = file_inode(obj->base.filp)->i_mapping;
   5029 	mapping_set_gfp_mask(mapping, mask);
   5030 #endif
   5031 
   5032 	i915_gem_object_init(obj, &i915_gem_object_ops);
   5033 
   5034 	obj->base.write_domain = I915_GEM_DOMAIN_CPU;
   5035 	obj->base.read_domains = I915_GEM_DOMAIN_CPU;
   5036 
   5037 	if (HAS_LLC(dev)) {
   5038 		/* On some devices, we can have the GPU use the LLC (the CPU
   5039 		 * cache) for about a 10% performance improvement
   5040 		 * compared to uncached.  Graphics requests other than
   5041 		 * display scanout are coherent with the CPU in
   5042 		 * accessing this cache.  This means in this mode we
   5043 		 * don't need to clflush on the CPU side, and on the
   5044 		 * GPU side we only need to flush internal caches to
   5045 		 * get data visible to the CPU.
   5046 		 *
   5047 		 * However, we maintain the display planes as UC, and so
   5048 		 * need to rebind when first used as such.
   5049 		 */
   5050 		obj->cache_level = I915_CACHE_LLC;
   5051 	} else
   5052 		obj->cache_level = I915_CACHE_NONE;
   5053 
   5054 	trace_i915_gem_object_create(obj);
   5055 
   5056 	return obj;
   5057 }
   5058 
   5059 static bool discard_backing_storage(struct drm_i915_gem_object *obj)
   5060 {
   5061 #ifdef __NetBSD__
   5062 	panic("XXX");
   5063 #else
   5064 	/* If we are the last user of the backing storage (be it shmemfs
   5065 	 * pages or stolen etc), we know that the pages are going to be
   5066 	 * immediately released. In this case, we can then skip copying
   5067 	 * back the contents from the GPU.
   5068 	 */
   5069 
   5070 	if (obj->madv != I915_MADV_WILLNEED)
   5071 		return false;
   5072 
   5073 	if (obj->base.filp == NULL)
   5074 		return true;
   5075 
   5076 	/* At first glance, this looks racy, but then again so would be
   5077 	 * userspace racing mmap against close. However, the first external
   5078 	 * reference to the filp can only be obtained through the
   5079 	 * i915_gem_mmap_ioctl() which safeguards us against the user
   5080 	 * acquiring such a reference whilst we are in the middle of
   5081 	 * freeing the object.
   5082 	 */
   5083 	return atomic_long_read(&obj->base.filp->f_count) == 1;
   5084 #endif
   5085 }
   5086 
   5087 void i915_gem_free_object(struct drm_gem_object *gem_obj)
   5088 {
   5089 	struct drm_i915_gem_object *obj = to_intel_bo(gem_obj);
   5090 	struct drm_device *dev = obj->base.dev;
   5091 	struct drm_i915_private *dev_priv = dev->dev_private;
   5092 	struct i915_vma *vma, *next;
   5093 
   5094 	intel_runtime_pm_get(dev_priv);
   5095 
   5096 	trace_i915_gem_object_destroy(obj);
   5097 
   5098 	list_for_each_entry_safe(vma, next, &obj->vma_list, vma_link) {
   5099 		int ret;
   5100 
   5101 		vma->pin_count = 0;
   5102 		ret = i915_vma_unbind(vma);
   5103 		if (WARN_ON(ret == -ERESTARTSYS)) {
   5104 			bool was_interruptible;
   5105 
   5106 			was_interruptible = dev_priv->mm.interruptible;
   5107 			dev_priv->mm.interruptible = false;
   5108 
   5109 			WARN_ON(i915_vma_unbind(vma));
   5110 
   5111 			dev_priv->mm.interruptible = was_interruptible;
   5112 		}
   5113 	}
   5114 
   5115 	/* Stolen objects don't hold a ref, but do hold pin count. Fix that up
   5116 	 * before progressing. */
   5117 	if (obj->stolen)
   5118 		i915_gem_object_unpin_pages(obj);
   5119 
   5120 	WARN_ON(obj->frontbuffer_bits);
   5121 
   5122 	if (obj->pages && obj->madv == I915_MADV_WILLNEED &&
   5123 	    dev_priv->quirks & QUIRK_PIN_SWIZZLED_PAGES &&
   5124 	    obj->tiling_mode != I915_TILING_NONE)
   5125 		i915_gem_object_unpin_pages(obj);
   5126 
   5127 	if (WARN_ON(obj->pages_pin_count))
   5128 		obj->pages_pin_count = 0;
   5129 	if (discard_backing_storage(obj))
   5130 		obj->madv = I915_MADV_DONTNEED;
   5131 	i915_gem_object_put_pages(obj);
   5132 	i915_gem_object_free_mmap_offset(obj);
   5133 
   5134 	BUG_ON(obj->pages);
   5135 
   5136 #ifndef __NetBSD__		/* XXX drm prime */
   5137 	if (obj->base.import_attach)
   5138 		drm_prime_gem_destroy(&obj->base, NULL);
   5139 #endif
   5140 
   5141 	if (obj->ops->release)
   5142 		obj->ops->release(obj);
   5143 
   5144 	drm_gem_object_release(&obj->base);
   5145 	i915_gem_info_remove_obj(dev_priv, obj->base.size);
   5146 
   5147 	kfree(obj->bit_17);
   5148 	i915_gem_object_free(obj);
   5149 
   5150 	intel_runtime_pm_put(dev_priv);
   5151 }
   5152 
   5153 struct i915_vma *i915_gem_obj_to_vma(struct drm_i915_gem_object *obj,
   5154 				     struct i915_address_space *vm)
   5155 {
   5156 	struct i915_vma *vma;
   5157 	list_for_each_entry(vma, &obj->vma_list, vma_link) {
   5158 		if (i915_is_ggtt(vma->vm) &&
   5159 		    vma->ggtt_view.type != I915_GGTT_VIEW_NORMAL)
   5160 			continue;
   5161 		if (vma->vm == vm)
   5162 			return vma;
   5163 	}
   5164 	return NULL;
   5165 }
   5166 
   5167 struct i915_vma *i915_gem_obj_to_ggtt_view(struct drm_i915_gem_object *obj,
   5168 					   const struct i915_ggtt_view *view)
   5169 {
   5170 	struct i915_address_space *ggtt = i915_obj_to_ggtt(obj);
   5171 	struct i915_vma *vma;
   5172 
   5173 	if (WARN_ONCE(!view, "no view specified"))
   5174 		return ERR_PTR(-EINVAL);
   5175 
   5176 	list_for_each_entry(vma, &obj->vma_list, vma_link)
   5177 		if (vma->vm == ggtt &&
   5178 		    i915_ggtt_view_equal(&vma->ggtt_view, view))
   5179 			return vma;
   5180 	return NULL;
   5181 }
   5182 
   5183 void i915_gem_vma_destroy(struct i915_vma *vma)
   5184 {
   5185 	struct i915_address_space *vm = NULL;
   5186 	WARN_ON(vma->node.allocated);
   5187 
   5188 	/* Keep the vma as a placeholder in the execbuffer reservation lists */
   5189 	if (!list_empty(&vma->exec_list))
   5190 		return;
   5191 
   5192 	vm = vma->vm;
   5193 
   5194 	if (!i915_is_ggtt(vm))
   5195 		i915_ppgtt_put(i915_vm_to_ppgtt(vm));
   5196 
   5197 	list_del(&vma->vma_link);
   5198 
   5199 	kmem_cache_free(to_i915(vma->obj->base.dev)->vmas, vma);
   5200 }
   5201 
   5202 static void
   5203 i915_gem_stop_ringbuffers(struct drm_device *dev)
   5204 {
   5205 	struct drm_i915_private *dev_priv = dev->dev_private;
   5206 	struct intel_engine_cs *ring;
   5207 	int i;
   5208 
   5209 	for_each_ring(ring, dev_priv, i)
   5210 		dev_priv->gt.stop_ring(ring);
   5211 }
   5212 
   5213 int
   5214 i915_gem_suspend(struct drm_device *dev)
   5215 {
   5216 	struct drm_i915_private *dev_priv = dev->dev_private;
   5217 	int ret = 0;
   5218 
   5219 	mutex_lock(&dev->struct_mutex);
   5220 	ret = i915_gpu_idle(dev);
   5221 	if (ret)
   5222 		goto err;
   5223 
   5224 	i915_gem_retire_requests(dev);
   5225 
   5226 	i915_gem_stop_ringbuffers(dev);
   5227 	mutex_unlock(&dev->struct_mutex);
   5228 
   5229 	cancel_delayed_work_sync(&dev_priv->gpu_error.hangcheck_work);
   5230 	cancel_delayed_work_sync(&dev_priv->mm.retire_work);
   5231 	flush_delayed_work(&dev_priv->mm.idle_work);
   5232 
   5233 	/* Assert that we sucessfully flushed all the work and
   5234 	 * reset the GPU back to its idle, low power state.
   5235 	 */
   5236 	WARN_ON(dev_priv->mm.busy);
   5237 
   5238 	return 0;
   5239 
   5240 err:
   5241 	mutex_unlock(&dev->struct_mutex);
   5242 	return ret;
   5243 }
   5244 
   5245 int i915_gem_l3_remap(struct drm_i915_gem_request *req, int slice)
   5246 {
   5247 	struct intel_engine_cs *ring = req->ring;
   5248 	struct drm_device *dev = ring->dev;
   5249 	struct drm_i915_private *dev_priv = dev->dev_private;
   5250 	u32 reg_base = GEN7_L3LOG_BASE + (slice * 0x200);
   5251 	u32 *remap_info = dev_priv->l3_parity.remap_info[slice];
   5252 	int i, ret;
   5253 
   5254 	if (!HAS_L3_DPF(dev) || !remap_info)
   5255 		return 0;
   5256 
   5257 	ret = intel_ring_begin(req, GEN7_L3LOG_SIZE / 4 * 3);
   5258 	if (ret)
   5259 		return ret;
   5260 
   5261 	/*
   5262 	 * Note: We do not worry about the concurrent register cacheline hang
   5263 	 * here because no other code should access these registers other than
   5264 	 * at initialization time.
   5265 	 */
   5266 	for (i = 0; i < GEN7_L3LOG_SIZE; i += 4) {
   5267 		intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1));
   5268 		intel_ring_emit(ring, reg_base + i);
   5269 		intel_ring_emit(ring, remap_info[i/4]);
   5270 	}
   5271 
   5272 	intel_ring_advance(ring);
   5273 
   5274 	return ret;
   5275 }
   5276 
   5277 void i915_gem_init_swizzling(struct drm_device *dev)
   5278 {
   5279 	struct drm_i915_private *dev_priv = dev->dev_private;
   5280 
   5281 	if (INTEL_INFO(dev)->gen < 5 ||
   5282 	    dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_NONE)
   5283 		return;
   5284 
   5285 	I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) |
   5286 				 DISP_TILE_SURFACE_SWIZZLING);
   5287 
   5288 	if (IS_GEN5(dev))
   5289 		return;
   5290 
   5291 	I915_WRITE(TILECTL, I915_READ(TILECTL) | TILECTL_SWZCTL);
   5292 	if (IS_GEN6(dev))
   5293 		I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_SNB));
   5294 	else if (IS_GEN7(dev))
   5295 		I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_IVB));
   5296 	else if (IS_GEN8(dev))
   5297 		I915_WRITE(GAMTARBMODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_BDW));
   5298 	else
   5299 		BUG();
   5300 }
   5301 
   5302 static void init_unused_ring(struct drm_device *dev, u32 base)
   5303 {
   5304 	struct drm_i915_private *dev_priv = dev->dev_private;
   5305 
   5306 	I915_WRITE(RING_CTL(base), 0);
   5307 	I915_WRITE(RING_HEAD(base), 0);
   5308 	I915_WRITE(RING_TAIL(base), 0);
   5309 	I915_WRITE(RING_START(base), 0);
   5310 }
   5311 
   5312 static void init_unused_rings(struct drm_device *dev)
   5313 {
   5314 	if (IS_I830(dev)) {
   5315 		init_unused_ring(dev, PRB1_BASE);
   5316 		init_unused_ring(dev, SRB0_BASE);
   5317 		init_unused_ring(dev, SRB1_BASE);
   5318 		init_unused_ring(dev, SRB2_BASE);
   5319 		init_unused_ring(dev, SRB3_BASE);
   5320 	} else if (IS_GEN2(dev)) {
   5321 		init_unused_ring(dev, SRB0_BASE);
   5322 		init_unused_ring(dev, SRB1_BASE);
   5323 	} else if (IS_GEN3(dev)) {
   5324 		init_unused_ring(dev, PRB1_BASE);
   5325 		init_unused_ring(dev, PRB2_BASE);
   5326 	}
   5327 }
   5328 
   5329 int i915_gem_init_rings(struct drm_device *dev)
   5330 {
   5331 	struct drm_i915_private *dev_priv = dev->dev_private;
   5332 	int ret;
   5333 
   5334 	ret = intel_init_render_ring_buffer(dev);
   5335 	if (ret)
   5336 		return ret;
   5337 
   5338 	if (HAS_BSD(dev)) {
   5339 		ret = intel_init_bsd_ring_buffer(dev);
   5340 		if (ret)
   5341 			goto cleanup_render_ring;
   5342 	}
   5343 
   5344 	if (HAS_BLT(dev)) {
   5345 		ret = intel_init_blt_ring_buffer(dev);
   5346 		if (ret)
   5347 			goto cleanup_bsd_ring;
   5348 	}
   5349 
   5350 	if (HAS_VEBOX(dev)) {
   5351 		ret = intel_init_vebox_ring_buffer(dev);
   5352 		if (ret)
   5353 			goto cleanup_blt_ring;
   5354 	}
   5355 
   5356 	if (HAS_BSD2(dev)) {
   5357 		ret = intel_init_bsd2_ring_buffer(dev);
   5358 		if (ret)
   5359 			goto cleanup_vebox_ring;
   5360 	}
   5361 
   5362 	return 0;
   5363 
   5364 cleanup_vebox_ring:
   5365 	intel_cleanup_ring_buffer(&dev_priv->ring[VECS]);
   5366 cleanup_blt_ring:
   5367 	intel_cleanup_ring_buffer(&dev_priv->ring[BCS]);
   5368 cleanup_bsd_ring:
   5369 	intel_cleanup_ring_buffer(&dev_priv->ring[VCS]);
   5370 cleanup_render_ring:
   5371 	intel_cleanup_ring_buffer(&dev_priv->ring[RCS]);
   5372 
   5373 	return ret;
   5374 }
   5375 
   5376 int
   5377 i915_gem_init_hw(struct drm_device *dev)
   5378 {
   5379 	struct drm_i915_private *dev_priv = dev->dev_private;
   5380 	struct intel_engine_cs *ring;
   5381 	int ret, i, j;
   5382 
   5383 	if (INTEL_INFO(dev)->gen < 6 && !intel_enable_gtt())
   5384 		return -EIO;
   5385 
   5386 	/* Double layer security blanket, see i915_gem_init() */
   5387 	intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
   5388 
   5389 	if (dev_priv->ellc_size)
   5390 		I915_WRITE(HSW_IDICR, I915_READ(HSW_IDICR) | IDIHASHMSK(0xf));
   5391 
   5392 	if (IS_HASWELL(dev))
   5393 		I915_WRITE(MI_PREDICATE_RESULT_2, IS_HSW_GT3(dev) ?
   5394 			   LOWER_SLICE_ENABLED : LOWER_SLICE_DISABLED);
   5395 
   5396 	if (HAS_PCH_NOP(dev)) {
   5397 		if (IS_IVYBRIDGE(dev)) {
   5398 			u32 temp = I915_READ(GEN7_MSG_CTL);
   5399 			temp &= ~(WAIT_FOR_PCH_FLR_ACK | WAIT_FOR_PCH_RESET_ACK);
   5400 			I915_WRITE(GEN7_MSG_CTL, temp);
   5401 		} else if (INTEL_INFO(dev)->gen >= 7) {
   5402 			u32 temp = I915_READ(HSW_NDE_RSTWRN_OPT);
   5403 			temp &= ~RESET_PCH_HANDSHAKE_ENABLE;
   5404 			I915_WRITE(HSW_NDE_RSTWRN_OPT, temp);
   5405 		}
   5406 	}
   5407 
   5408 	i915_gem_init_swizzling(dev);
   5409 
   5410 	/*
   5411 	 * At least 830 can leave some of the unused rings
   5412 	 * "active" (ie. head != tail) after resume which
   5413 	 * will prevent c3 entry. Makes sure all unused rings
   5414 	 * are totally idle.
   5415 	 */
   5416 	init_unused_rings(dev);
   5417 
   5418 	BUG_ON(!dev_priv->ring[RCS].default_context);
   5419 
   5420 	ret = i915_ppgtt_init_hw(dev);
   5421 	if (ret) {
   5422 		DRM_ERROR("PPGTT enable HW failed %d\n", ret);
   5423 		goto out;
   5424 	}
   5425 
   5426 	/* Need to do basic initialisation of all rings first: */
   5427 	for_each_ring(ring, dev_priv, i) {
   5428 		ret = ring->init_hw(ring);
   5429 		if (ret)
   5430 			goto out;
   5431 	}
   5432 
   5433 	/* We can't enable contexts until all firmware is loaded */
   5434 	if (HAS_GUC_UCODE(dev)) {
   5435 		ret = intel_guc_ucode_load(dev);
   5436 		if (ret) {
   5437 			/*
   5438 			 * If we got an error and GuC submission is enabled, map
   5439 			 * the error to -EIO so the GPU will be declared wedged.
   5440 			 * OTOH, if we didn't intend to use the GuC anyway, just
   5441 			 * discard the error and carry on.
   5442 			 */
   5443 			DRM_ERROR("Failed to initialize GuC, error %d%s\n", ret,
   5444 				  i915.enable_guc_submission ? "" :
   5445 				  " (ignored)");
   5446 			ret = i915.enable_guc_submission ? -EIO : 0;
   5447 			if (ret)
   5448 				goto out;
   5449 		}
   5450 	}
   5451 
   5452 	/*
   5453 	 * Increment the next seqno by 0x100 so we have a visible break
   5454 	 * on re-initialisation
   5455 	 */
   5456 	ret = i915_gem_set_seqno(dev, dev_priv->next_seqno+0x100);
   5457 	if (ret)
   5458 		goto out;
   5459 
   5460 	/* Now it is safe to go back round and do everything else: */
   5461 	for_each_ring(ring, dev_priv, i) {
   5462 		struct drm_i915_gem_request *req;
   5463 
   5464 		WARN_ON(!ring->default_context);
   5465 
   5466 		ret = i915_gem_request_alloc(ring, ring->default_context, &req);
   5467 		if (ret) {
   5468 			i915_gem_cleanup_ringbuffer(dev);
   5469 			goto out;
   5470 		}
   5471 
   5472 		if (ring->id == RCS) {
   5473 			for (j = 0; j < NUM_L3_SLICES(dev); j++)
   5474 				i915_gem_l3_remap(req, j);
   5475 		}
   5476 
   5477 		ret = i915_ppgtt_init_ring(req);
   5478 		if (ret && ret != -EIO) {
   5479 			DRM_ERROR("PPGTT enable ring #%d failed %d\n", i, ret);
   5480 			i915_gem_request_cancel(req);
   5481 			i915_gem_cleanup_ringbuffer(dev);
   5482 			goto out;
   5483 		}
   5484 
   5485 		ret = i915_gem_context_enable(req);
   5486 		if (ret && ret != -EIO) {
   5487 			DRM_ERROR("Context enable ring #%d failed %d\n", i, ret);
   5488 			i915_gem_request_cancel(req);
   5489 			i915_gem_cleanup_ringbuffer(dev);
   5490 			goto out;
   5491 		}
   5492 
   5493 		i915_add_request_no_flush(req);
   5494 	}
   5495 
   5496 out:
   5497 	intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
   5498 	return ret;
   5499 }
   5500 
   5501 int i915_gem_init(struct drm_device *dev)
   5502 {
   5503 	struct drm_i915_private *dev_priv = dev->dev_private;
   5504 	int ret;
   5505 
   5506 	i915.enable_execlists = intel_sanitize_enable_execlists(dev,
   5507 			i915.enable_execlists);
   5508 
   5509 	mutex_lock(&dev->struct_mutex);
   5510 
   5511 	if (IS_VALLEYVIEW(dev)) {
   5512 		/* VLVA0 (potential hack), BIOS isn't actually waking us */
   5513 		I915_WRITE(VLV_GTLC_WAKE_CTRL, VLV_GTLC_ALLOWWAKEREQ);
   5514 		if (wait_for((I915_READ(VLV_GTLC_PW_STATUS) &
   5515 			      VLV_GTLC_ALLOWWAKEACK), 10))
   5516 			DRM_DEBUG_DRIVER("allow wake ack timed out\n");
   5517 	}
   5518 
   5519 	if (!i915.enable_execlists) {
   5520 		dev_priv->gt.execbuf_submit = i915_gem_ringbuffer_submission;
   5521 		dev_priv->gt.init_rings = i915_gem_init_rings;
   5522 		dev_priv->gt.cleanup_ring = intel_cleanup_ring_buffer;
   5523 		dev_priv->gt.stop_ring = intel_stop_ring_buffer;
   5524 	} else {
   5525 		dev_priv->gt.execbuf_submit = intel_execlists_submission;
   5526 		dev_priv->gt.init_rings = intel_logical_rings_init;
   5527 		dev_priv->gt.cleanup_ring = intel_logical_ring_cleanup;
   5528 		dev_priv->gt.stop_ring = intel_logical_ring_stop;
   5529 	}
   5530 
   5531 	/* This is just a security blanket to placate dragons.
   5532 	 * On some systems, we very sporadically observe that the first TLBs
   5533 	 * used by the CS may be stale, despite us poking the TLB reset. If
   5534 	 * we hold the forcewake during initialisation these problems
   5535 	 * just magically go away.
   5536 	 */
   5537 	intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
   5538 
   5539 	ret = i915_gem_init_userptr(dev);
   5540 	if (ret)
   5541 		goto out_unlock;
   5542 
   5543 	i915_gem_init_global_gtt(dev);
   5544 
   5545 	ret = i915_gem_context_init(dev);
   5546 	if (ret)
   5547 		goto out_unlock;
   5548 
   5549 	ret = dev_priv->gt.init_rings(dev);
   5550 	if (ret)
   5551 		goto out_unlock;
   5552 
   5553 	ret = i915_gem_init_hw(dev);
   5554 	if (ret == -EIO) {
   5555 		/* Allow ring initialisation to fail by marking the GPU as
   5556 		 * wedged. But we only want to do this where the GPU is angry,
   5557 		 * for all other failure, such as an allocation failure, bail.
   5558 		 */
   5559 		DRM_ERROR("Failed to initialize GPU, declaring it wedged\n");
   5560 		atomic_or(I915_WEDGED, &dev_priv->gpu_error.reset_counter);
   5561 		ret = 0;
   5562 	}
   5563 
   5564 out_unlock:
   5565 	intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
   5566 	mutex_unlock(&dev->struct_mutex);
   5567 
   5568 	return ret;
   5569 }
   5570 
   5571 void
   5572 i915_gem_cleanup_ringbuffer(struct drm_device *dev)
   5573 {
   5574 	struct drm_i915_private *dev_priv = dev->dev_private;
   5575 	struct intel_engine_cs *ring;
   5576 	int i;
   5577 
   5578 	for_each_ring(ring, dev_priv, i)
   5579 		dev_priv->gt.cleanup_ring(ring);
   5580 
   5581     if (i915.enable_execlists)
   5582             /*
   5583              * Neither the BIOS, ourselves or any other kernel
   5584              * expects the system to be in execlists mode on startup,
   5585              * so we need to reset the GPU back to legacy mode.
   5586              */
   5587             intel_gpu_reset(dev);
   5588 }
   5589 
   5590 static void
   5591 init_ring_lists(struct intel_engine_cs *ring)
   5592 {
   5593 	INIT_LIST_HEAD(&ring->active_list);
   5594 	INIT_LIST_HEAD(&ring->request_list);
   5595 }
   5596 
   5597 void
   5598 i915_gem_load(struct drm_device *dev)
   5599 {
   5600 	struct drm_i915_private *dev_priv = dev->dev_private;
   5601 	int i;
   5602 
   5603 	dev_priv->objects =
   5604 		kmem_cache_create("i915_gem_object",
   5605 				  sizeof(struct drm_i915_gem_object), 0,
   5606 				  SLAB_HWCACHE_ALIGN,
   5607 				  NULL);
   5608 	dev_priv->vmas =
   5609 		kmem_cache_create("i915_gem_vma",
   5610 				  sizeof(struct i915_vma), 0,
   5611 				  SLAB_HWCACHE_ALIGN,
   5612 				  NULL);
   5613 	dev_priv->requests =
   5614 		kmem_cache_create("i915_gem_request",
   5615 				  sizeof(struct drm_i915_gem_request), 0,
   5616 				  SLAB_HWCACHE_ALIGN,
   5617 				  NULL);
   5618 
   5619 	INIT_LIST_HEAD(&dev_priv->vm_list);
   5620 	INIT_LIST_HEAD(&dev_priv->context_list);
   5621 	INIT_LIST_HEAD(&dev_priv->mm.unbound_list);
   5622 	INIT_LIST_HEAD(&dev_priv->mm.bound_list);
   5623 	INIT_LIST_HEAD(&dev_priv->mm.fence_list);
   5624 	for (i = 0; i < I915_NUM_RINGS; i++)
   5625 		init_ring_lists(&dev_priv->ring[i]);
   5626 	for (i = 0; i < I915_MAX_NUM_FENCES; i++)
   5627 		INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list);
   5628 	INIT_DELAYED_WORK(&dev_priv->mm.retire_work,
   5629 			  i915_gem_retire_work_handler);
   5630 	INIT_DELAYED_WORK(&dev_priv->mm.idle_work,
   5631 			  i915_gem_idle_work_handler);
   5632 #ifdef __NetBSD__
   5633 	spin_lock_init(&dev_priv->gpu_error.reset_lock);
   5634 	DRM_INIT_WAITQUEUE(&dev_priv->gpu_error.reset_queue, "i915errst");
   5635 #else
   5636 	init_waitqueue_head(&dev_priv->gpu_error.reset_queue);
   5637 #endif
   5638 
   5639 	dev_priv->relative_constants_mode = I915_EXEC_CONSTANTS_REL_GENERAL;
   5640 
   5641 	if (INTEL_INFO(dev)->gen >= 7 && !IS_VALLEYVIEW(dev))
   5642 		dev_priv->num_fence_regs = 32;
   5643 	else if (INTEL_INFO(dev)->gen >= 4 || IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
   5644 		dev_priv->num_fence_regs = 16;
   5645 	else
   5646 		dev_priv->num_fence_regs = 8;
   5647 
   5648 	if (intel_vgpu_active(dev))
   5649 		dev_priv->num_fence_regs =
   5650 				I915_READ(vgtif_reg(avail_rs.fence_num));
   5651 
   5652 	/*
   5653 	 * Set initial sequence number for requests.
   5654 	 * Using this number allows the wraparound to happen early,
   5655 	 * catching any obvious problems.
   5656 	 */
   5657 	dev_priv->next_seqno = ((u32)~0 - 0x1100);
   5658 	dev_priv->last_seqno = ((u32)~0 - 0x1101);
   5659 
   5660 	/* Initialize fence registers to zero */
   5661 	INIT_LIST_HEAD(&dev_priv->mm.fence_list);
   5662 	i915_gem_restore_fences(dev);
   5663 
   5664 	i915_gem_detect_bit_6_swizzle(dev);
   5665 #ifdef __NetBSD__
   5666 	DRM_INIT_WAITQUEUE(&dev_priv->pending_flip_queue, "i915flip");
   5667 	spin_lock_init(&dev_priv->pending_flip_lock);
   5668 #else
   5669 	init_waitqueue_head(&dev_priv->pending_flip_queue);
   5670 #endif
   5671 
   5672 	dev_priv->mm.interruptible = true;
   5673 
   5674 	i915_gem_shrinker_init(dev_priv);
   5675 #ifdef __NetBSD__
   5676 	linux_mutex_init(&dev_priv->fb_tracking.lock);
   5677 #else
   5678 	mutex_init(&dev_priv->fb_tracking.lock);
   5679 #endif
   5680 }
   5681 
   5682 void i915_gem_release(struct drm_device *dev, struct drm_file *file)
   5683 {
   5684 	struct drm_i915_file_private *file_priv = file->driver_priv;
   5685 
   5686 	/* Clean up our request list when the client is going away, so that
   5687 	 * later retire_requests won't dereference our soon-to-be-gone
   5688 	 * file_priv.
   5689 	 */
   5690 	spin_lock(&file_priv->mm.lock);
   5691 	while (!list_empty(&file_priv->mm.request_list)) {
   5692 		struct drm_i915_gem_request *request;
   5693 
   5694 		request = list_first_entry(&file_priv->mm.request_list,
   5695 					   struct drm_i915_gem_request,
   5696 					   client_list);
   5697 		list_del(&request->client_list);
   5698 		request->file_priv = NULL;
   5699 	}
   5700 	spin_unlock(&file_priv->mm.lock);
   5701 
   5702 	if (!list_empty(&file_priv->rps.link)) {
   5703 		spin_lock(&to_i915(dev)->rps.client_lock);
   5704 		list_del(&file_priv->rps.link);
   5705 		spin_unlock(&to_i915(dev)->rps.client_lock);
   5706 	}
   5707 }
   5708 
   5709 int i915_gem_open(struct drm_device *dev, struct drm_file *file)
   5710 {
   5711 	struct drm_i915_file_private *file_priv;
   5712 	int ret;
   5713 
   5714 	DRM_DEBUG_DRIVER("\n");
   5715 
   5716 	file_priv = kzalloc(sizeof(*file_priv), GFP_KERNEL);
   5717 	if (!file_priv)
   5718 		return -ENOMEM;
   5719 
   5720 	file->driver_priv = file_priv;
   5721 	file_priv->dev_priv = dev->dev_private;
   5722 	file_priv->file = file;
   5723 	INIT_LIST_HEAD(&file_priv->rps.link);
   5724 
   5725 	spin_lock_init(&file_priv->mm.lock);
   5726 	INIT_LIST_HEAD(&file_priv->mm.request_list);
   5727 
   5728 	ret = i915_gem_context_open(dev, file);
   5729 	if (ret)
   5730 		kfree(file_priv);
   5731 
   5732 	return ret;
   5733 }
   5734 
   5735 /**
   5736  * i915_gem_track_fb - update frontbuffer tracking
   5737  * @old: current GEM buffer for the frontbuffer slots
   5738  * @new: new GEM buffer for the frontbuffer slots
   5739  * @frontbuffer_bits: bitmask of frontbuffer slots
   5740  *
   5741  * This updates the frontbuffer tracking bits @frontbuffer_bits by clearing them
   5742  * from @old and setting them in @new. Both @old and @new can be NULL.
   5743  */
   5744 void i915_gem_track_fb(struct drm_i915_gem_object *old,
   5745 		       struct drm_i915_gem_object *new,
   5746 		       unsigned frontbuffer_bits)
   5747 {
   5748 	if (old) {
   5749 		WARN_ON(!mutex_is_locked(&old->base.dev->struct_mutex));
   5750 		WARN_ON(!(old->frontbuffer_bits & frontbuffer_bits));
   5751 		old->frontbuffer_bits &= ~frontbuffer_bits;
   5752 	}
   5753 
   5754 	if (new) {
   5755 		WARN_ON(!mutex_is_locked(&new->base.dev->struct_mutex));
   5756 		WARN_ON(new->frontbuffer_bits & frontbuffer_bits);
   5757 		new->frontbuffer_bits |= frontbuffer_bits;
   5758 	}
   5759 }
   5760 
   5761 /* All the new VM stuff */
   5762 u64 i915_gem_obj_offset(struct drm_i915_gem_object *o,
   5763 			struct i915_address_space *vm)
   5764 {
   5765 	struct drm_i915_private *dev_priv = o->base.dev->dev_private;
   5766 	struct i915_vma *vma;
   5767 
   5768 	WARN_ON(vm == &dev_priv->mm.aliasing_ppgtt->base);
   5769 
   5770 	list_for_each_entry(vma, &o->vma_list, vma_link) {
   5771 		if (i915_is_ggtt(vma->vm) &&
   5772 		    vma->ggtt_view.type != I915_GGTT_VIEW_NORMAL)
   5773 			continue;
   5774 		if (vma->vm == vm)
   5775 			return vma->node.start;
   5776 	}
   5777 
   5778 	WARN(1, "%s vma for this object not found.\n",
   5779 	     i915_is_ggtt(vm) ? "global" : "ppgtt");
   5780 	return -1;
   5781 }
   5782 
   5783 u64 i915_gem_obj_ggtt_offset_view(struct drm_i915_gem_object *o,
   5784 				  const struct i915_ggtt_view *view)
   5785 {
   5786 	struct i915_address_space *ggtt = i915_obj_to_ggtt(o);
   5787 	struct i915_vma *vma;
   5788 
   5789 	list_for_each_entry(vma, &o->vma_list, vma_link)
   5790 		if (vma->vm == ggtt &&
   5791 		    i915_ggtt_view_equal(&vma->ggtt_view, view))
   5792 			return vma->node.start;
   5793 
   5794 	WARN(1, "global vma for this object not found. (view=%u)\n", view->type);
   5795 	return -1;
   5796 }
   5797 
   5798 bool i915_gem_obj_bound(struct drm_i915_gem_object *o,
   5799 			struct i915_address_space *vm)
   5800 {
   5801 	struct i915_vma *vma;
   5802 
   5803 	list_for_each_entry(vma, &o->vma_list, vma_link) {
   5804 		if (i915_is_ggtt(vma->vm) &&
   5805 		    vma->ggtt_view.type != I915_GGTT_VIEW_NORMAL)
   5806 			continue;
   5807 		if (vma->vm == vm && drm_mm_node_allocated(&vma->node))
   5808 			return true;
   5809 	}
   5810 
   5811 	return false;
   5812 }
   5813 
   5814 bool i915_gem_obj_ggtt_bound_view(struct drm_i915_gem_object *o,
   5815 				  const struct i915_ggtt_view *view)
   5816 {
   5817 	struct i915_address_space *ggtt = i915_obj_to_ggtt(o);
   5818 	struct i915_vma *vma;
   5819 
   5820 	list_for_each_entry(vma, &o->vma_list, vma_link)
   5821 		if (vma->vm == ggtt &&
   5822 		    i915_ggtt_view_equal(&vma->ggtt_view, view) &&
   5823 		    drm_mm_node_allocated(&vma->node))
   5824 			return true;
   5825 
   5826 	return false;
   5827 }
   5828 
   5829 bool i915_gem_obj_bound_any(struct drm_i915_gem_object *o)
   5830 {
   5831 	struct i915_vma *vma;
   5832 
   5833 	list_for_each_entry(vma, &o->vma_list, vma_link)
   5834 		if (drm_mm_node_allocated(&vma->node))
   5835 			return true;
   5836 
   5837 	return false;
   5838 }
   5839 
   5840 unsigned long i915_gem_obj_size(struct drm_i915_gem_object *o,
   5841 				struct i915_address_space *vm)
   5842 {
   5843 	struct drm_i915_private *dev_priv = o->base.dev->dev_private;
   5844 	struct i915_vma *vma;
   5845 
   5846 	WARN_ON(vm == &dev_priv->mm.aliasing_ppgtt->base);
   5847 
   5848 	BUG_ON(list_empty(&o->vma_list));
   5849 
   5850 	list_for_each_entry(vma, &o->vma_list, vma_link) {
   5851 		if (i915_is_ggtt(vma->vm) &&
   5852 		    vma->ggtt_view.type != I915_GGTT_VIEW_NORMAL)
   5853 			continue;
   5854 		if (vma->vm == vm)
   5855 			return vma->node.size;
   5856 	}
   5857 	return 0;
   5858 }
   5859 
   5860 bool i915_gem_obj_is_pinned(struct drm_i915_gem_object *obj)
   5861 {
   5862 	struct i915_vma *vma;
   5863 	list_for_each_entry(vma, &obj->vma_list, vma_link)
   5864 		if (vma->pin_count > 0)
   5865 			return true;
   5866 
   5867 	return false;
   5868 }
   5869 
   5870 /* Allocate a new GEM object and fill it with the supplied data */
   5871 struct drm_i915_gem_object *
   5872 i915_gem_object_create_from_data(struct drm_device *dev,
   5873 			         const void *data, size_t size)
   5874 {
   5875 	struct drm_i915_gem_object *obj;
   5876 #ifdef __NetBSD__
   5877 	struct iovec iov = { .iov_base = __UNCONST(data), .iov_len = size };
   5878 	struct uio uio = {
   5879 	    .uio_iov = &iov,
   5880 	    .uio_iovcnt = 1,
   5881 	    .uio_resid = size,
   5882 	    .uio_rw = UIO_WRITE,
   5883 	};
   5884 #else
   5885 	struct sg_table *sg;
   5886 #endif
   5887 	size_t bytes;
   5888 	int ret;
   5889 
   5890 	obj = i915_gem_alloc_object(dev, round_up(size, PAGE_SIZE));
   5891 	if (IS_ERR_OR_NULL(obj))
   5892 		return obj;
   5893 
   5894 	ret = i915_gem_object_set_to_cpu_domain(obj, true);
   5895 	if (ret)
   5896 		goto fail;
   5897 
   5898 	ret = i915_gem_object_get_pages(obj);
   5899 	if (ret)
   5900 		goto fail;
   5901 
   5902 	i915_gem_object_pin_pages(obj);
   5903 #ifdef __NetBSD__
   5904 	/* XXX errno NetBSD->Linux */
   5905 	ret = -ubc_uiomove(obj->base.gemo_shm_uao, &uio, size, UVM_ADV_NORMAL,
   5906 	    UBC_WRITE);
   5907 	if (ret)
   5908 		goto fail;
   5909 #else
   5910 	sg = obj->pages;
   5911 	bytes = sg_copy_from_buffer(sg->sgl, sg->nents, (void *)data, size);
   5912 #endif
   5913 	i915_gem_object_unpin_pages(obj);
   5914 
   5915 	if (WARN_ON(bytes != size)) {
   5916 		DRM_ERROR("Incomplete copy, wrote %zu of %zu", bytes, size);
   5917 		ret = -EFAULT;
   5918 		goto fail;
   5919 	}
   5920 
   5921 	return obj;
   5922 
   5923 fail:
   5924 	drm_gem_object_unreference(&obj->base);
   5925 	return ERR_PTR(ret);
   5926 }
   5927